1//===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the scalar evolution analysis
10// engine, which is used primarily to analyze expressions involving induction
11// variables in loops.
12//
13// There are several aspects to this library. First is the representation of
14// scalar expressions, which are represented as subclasses of the SCEV class.
15// These classes are used to represent certain types of subexpressions that we
16// can handle. We only create one SCEV of a particular shape, so
17// pointer-comparisons for equality are legal.
18//
19// One important aspect of the SCEV objects is that they are never cyclic, even
20// if there is a cycle in the dataflow for an expression (ie, a PHI node). If
21// the PHI node is one of the idioms that we can represent (e.g., a polynomial
22// recurrence) then we represent it directly as a recurrence node, otherwise we
23// represent it as a SCEVUnknown node.
24//
25// In addition to being able to represent expressions of various types, we also
26// have folders that are used to build the *canonical* representation for a
27// particular expression. These folders are capable of using a variety of
28// rewrite rules to simplify the expressions.
29//
30// Once the folders are defined, we can implement the more interesting
31// higher-level code, such as the code that recognizes PHI nodes of various
32// types, computes the execution count of a loop, etc.
33//
34// TODO: We should use these routines and value representations to implement
35// dependence analysis!
36//
37//===----------------------------------------------------------------------===//
38//
39// There are several good references for the techniques used in this analysis.
40//
41// Chains of recurrences -- a method to expedite the evaluation
42// of closed-form functions
43// Olaf Bachmann, Paul S. Wang, Eugene V. Zima
44//
45// On computational properties of chains of recurrences
46// Eugene V. Zima
47//
48// Symbolic Evaluation of Chains of Recurrences for Loop Optimization
49// Robert A. van Engelen
50//
51// Efficient Symbolic Analysis for Optimizing Compilers
52// Robert A. van Engelen
53//
54// Using the chains of recurrences algebra for data dependence testing and
55// induction variable substitution
56// MS Thesis, Johnie Birch
57//
58//===----------------------------------------------------------------------===//
59
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/ADT/APInt.h"
62#include "llvm/ADT/ArrayRef.h"
63#include "llvm/ADT/DenseMap.h"
64#include "llvm/ADT/DepthFirstIterator.h"
65#include "llvm/ADT/FoldingSet.h"
66#include "llvm/ADT/STLExtras.h"
67#include "llvm/ADT/ScopeExit.h"
68#include "llvm/ADT/Sequence.h"
69#include "llvm/ADT/SmallPtrSet.h"
70#include "llvm/ADT/SmallVector.h"
71#include "llvm/ADT/Statistic.h"
72#include "llvm/ADT/StringExtras.h"
73#include "llvm/ADT/StringRef.h"
74#include "llvm/Analysis/AssumptionCache.h"
75#include "llvm/Analysis/ConstantFolding.h"
76#include "llvm/Analysis/InstructionSimplify.h"
77#include "llvm/Analysis/LoopInfo.h"
78#include "llvm/Analysis/MemoryBuiltins.h"
79#include "llvm/Analysis/ScalarEvolutionExpressions.h"
80#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
81#include "llvm/Analysis/TargetLibraryInfo.h"
82#include "llvm/Analysis/ValueTracking.h"
83#include "llvm/Config/llvm-config.h"
84#include "llvm/IR/Argument.h"
85#include "llvm/IR/BasicBlock.h"
86#include "llvm/IR/CFG.h"
87#include "llvm/IR/Constant.h"
88#include "llvm/IR/ConstantRange.h"
89#include "llvm/IR/Constants.h"
90#include "llvm/IR/DataLayout.h"
91#include "llvm/IR/DerivedTypes.h"
92#include "llvm/IR/Dominators.h"
93#include "llvm/IR/Function.h"
94#include "llvm/IR/GlobalAlias.h"
95#include "llvm/IR/GlobalValue.h"
96#include "llvm/IR/InstIterator.h"
97#include "llvm/IR/InstrTypes.h"
98#include "llvm/IR/Instruction.h"
99#include "llvm/IR/Instructions.h"
100#include "llvm/IR/IntrinsicInst.h"
101#include "llvm/IR/Intrinsics.h"
102#include "llvm/IR/LLVMContext.h"
103#include "llvm/IR/Operator.h"
104#include "llvm/IR/PatternMatch.h"
105#include "llvm/IR/Type.h"
106#include "llvm/IR/Use.h"
107#include "llvm/IR/User.h"
108#include "llvm/IR/Value.h"
109#include "llvm/IR/Verifier.h"
110#include "llvm/InitializePasses.h"
111#include "llvm/Pass.h"
112#include "llvm/Support/Casting.h"
113#include "llvm/Support/CommandLine.h"
114#include "llvm/Support/Compiler.h"
115#include "llvm/Support/Debug.h"
116#include "llvm/Support/ErrorHandling.h"
117#include "llvm/Support/InterleavedRange.h"
118#include "llvm/Support/KnownBits.h"
119#include "llvm/Support/SaveAndRestore.h"
120#include "llvm/Support/raw_ostream.h"
121#include <algorithm>
122#include <cassert>
123#include <climits>
124#include <cstdint>
125#include <cstdlib>
126#include <map>
127#include <memory>
128#include <numeric>
129#include <optional>
130#include <tuple>
131#include <utility>
132#include <vector>
133
134using namespace llvm;
135using namespace PatternMatch;
136using namespace SCEVPatternMatch;
137
138#define DEBUG_TYPE "scalar-evolution"
139
140STATISTIC(NumExitCountsComputed,
141 "Number of loop exits with predictable exit counts");
142STATISTIC(NumExitCountsNotComputed,
143 "Number of loop exits without predictable exit counts");
144STATISTIC(NumBruteForceTripCountsComputed,
145 "Number of loops with trip counts computed by force");
146
147#ifdef EXPENSIVE_CHECKS
148bool llvm::VerifySCEV = true;
149#else
150bool llvm::VerifySCEV = false;
151#endif
152
153static cl::opt<unsigned>
154 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
155 cl::desc("Maximum number of iterations SCEV will "
156 "symbolically execute a constant "
157 "derived loop"),
158 cl::init(Val: 100));
159
160static cl::opt<bool, true> VerifySCEVOpt(
161 "verify-scev", cl::Hidden, cl::location(L&: VerifySCEV),
162 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
163static cl::opt<bool> VerifySCEVStrict(
164 "verify-scev-strict", cl::Hidden,
165 cl::desc("Enable stricter verification with -verify-scev is passed"));
166
167static cl::opt<bool> VerifyIR(
168 "scev-verify-ir", cl::Hidden,
169 cl::desc("Verify IR correctness when making sensitive SCEV queries (slow)"),
170 cl::init(Val: false));
171
172static cl::opt<unsigned> MulOpsInlineThreshold(
173 "scev-mulops-inline-threshold", cl::Hidden,
174 cl::desc("Threshold for inlining multiplication operands into a SCEV"),
175 cl::init(Val: 32));
176
177static cl::opt<unsigned> AddOpsInlineThreshold(
178 "scev-addops-inline-threshold", cl::Hidden,
179 cl::desc("Threshold for inlining addition operands into a SCEV"),
180 cl::init(Val: 500));
181
182static cl::opt<unsigned> MaxSCEVCompareDepth(
183 "scalar-evolution-max-scev-compare-depth", cl::Hidden,
184 cl::desc("Maximum depth of recursive SCEV complexity comparisons"),
185 cl::init(Val: 32));
186
187static cl::opt<unsigned> MaxSCEVOperationsImplicationDepth(
188 "scalar-evolution-max-scev-operations-implication-depth", cl::Hidden,
189 cl::desc("Maximum depth of recursive SCEV operations implication analysis"),
190 cl::init(Val: 2));
191
192static cl::opt<unsigned> MaxValueCompareDepth(
193 "scalar-evolution-max-value-compare-depth", cl::Hidden,
194 cl::desc("Maximum depth of recursive value complexity comparisons"),
195 cl::init(Val: 2));
196
197static cl::opt<unsigned>
198 MaxArithDepth("scalar-evolution-max-arith-depth", cl::Hidden,
199 cl::desc("Maximum depth of recursive arithmetics"),
200 cl::init(Val: 32));
201
202static cl::opt<unsigned> MaxConstantEvolvingDepth(
203 "scalar-evolution-max-constant-evolving-depth", cl::Hidden,
204 cl::desc("Maximum depth of recursive constant evolving"), cl::init(Val: 32));
205
206static cl::opt<unsigned>
207 MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden,
208 cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"),
209 cl::init(Val: 8));
210
211static cl::opt<unsigned>
212 MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden,
213 cl::desc("Max coefficients in AddRec during evolving"),
214 cl::init(Val: 8));
215
216static cl::opt<unsigned>
217 HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden,
218 cl::desc("Size of the expression which is considered huge"),
219 cl::init(Val: 4096));
220
221static cl::opt<unsigned> RangeIterThreshold(
222 "scev-range-iter-threshold", cl::Hidden,
223 cl::desc("Threshold for switching to iteratively computing SCEV ranges"),
224 cl::init(Val: 32));
225
226static cl::opt<unsigned> MaxLoopGuardCollectionDepth(
227 "scalar-evolution-max-loop-guard-collection-depth", cl::Hidden,
228 cl::desc("Maximum depth for recursive loop guard collection"), cl::init(Val: 1));
229
230static cl::opt<bool>
231ClassifyExpressions("scalar-evolution-classify-expressions",
232 cl::Hidden, cl::init(Val: true),
233 cl::desc("When printing analysis, include information on every instruction"));
234
235static cl::opt<bool> UseExpensiveRangeSharpening(
236 "scalar-evolution-use-expensive-range-sharpening", cl::Hidden,
237 cl::init(Val: false),
238 cl::desc("Use more powerful methods of sharpening expression ranges. May "
239 "be costly in terms of compile time"));
240
241static cl::opt<unsigned> MaxPhiSCCAnalysisSize(
242 "scalar-evolution-max-scc-analysis-depth", cl::Hidden,
243 cl::desc("Maximum amount of nodes to process while searching SCEVUnknown "
244 "Phi strongly connected components"),
245 cl::init(Val: 8));
246
247static cl::opt<bool>
248 EnableFiniteLoopControl("scalar-evolution-finite-loop", cl::Hidden,
249 cl::desc("Handle <= and >= in finite loops"),
250 cl::init(Val: true));
251
252static cl::opt<bool> UseContextForNoWrapFlagInference(
253 "scalar-evolution-use-context-for-no-wrap-flag-strenghening", cl::Hidden,
254 cl::desc("Infer nuw/nsw flags using context where suitable"),
255 cl::init(Val: true));
256
257//===----------------------------------------------------------------------===//
258// SCEV class definitions
259//===----------------------------------------------------------------------===//
260
261void SCEV::computeAndSetCanonical(ScalarEvolution &SE) {
262 // Leaf nodes are always their own canonical.
263 switch (getSCEVType()) {
264 case scConstant:
265 case scVScale:
266 case scUnknown:
267 CanonicalSCEV = this;
268 return;
269 default:
270 break;
271 }
272
273 // For all other expressions, check whether any immediate operand has a
274 // different canonical. Since operands are always created before their parent,
275 // their canonical pointers are already set — no recursion needed.
276 bool Changed = false;
277 SmallVector<SCEVUse, 4> CanonOps;
278 for (SCEVUse Op : operands()) {
279 CanonOps.push_back(Elt: Op->getCanonical());
280 Changed |= CanonOps.back() != Op.getPointer();
281 }
282
283 if (!Changed) {
284 CanonicalSCEV = this;
285 return;
286 }
287
288 auto *NAry = dyn_cast<SCEVNAryExpr>(Val: this);
289 SCEV::NoWrapFlags Flags = NAry ? NAry->getNoWrapFlags() : SCEV::FlagAnyWrap;
290 switch (getSCEVType()) {
291 case scPtrToAddr:
292 CanonicalSCEV = SE.getPtrToAddrExpr(Op: CanonOps[0]);
293 return;
294 case scPtrToInt:
295 CanonicalSCEV = SE.getPtrToIntExpr(Op: CanonOps[0], Ty: getType());
296 return;
297 case scTruncate:
298 CanonicalSCEV = SE.getTruncateExpr(Op: CanonOps[0], Ty: getType());
299 return;
300 case scZeroExtend:
301 CanonicalSCEV = SE.getZeroExtendExpr(Op: CanonOps[0], Ty: getType());
302 return;
303 case scSignExtend:
304 CanonicalSCEV = SE.getSignExtendExpr(Op: CanonOps[0], Ty: getType());
305 return;
306 case scUDivExpr:
307 CanonicalSCEV = SE.getUDivExpr(LHS: CanonOps[0], RHS: CanonOps[1]);
308 return;
309 case scAddExpr:
310 CanonicalSCEV = SE.getAddExpr(Ops&: CanonOps, Flags);
311 return;
312 case scMulExpr:
313 CanonicalSCEV = SE.getMulExpr(Ops&: CanonOps, Flags);
314 return;
315 case scAddRecExpr:
316 CanonicalSCEV = SE.getAddRecExpr(
317 Operands&: CanonOps, L: cast<SCEVAddRecExpr>(Val: this)->getLoop(), Flags);
318 return;
319 case scSMaxExpr:
320 CanonicalSCEV = SE.getSMaxExpr(Operands&: CanonOps);
321 return;
322 case scUMaxExpr:
323 CanonicalSCEV = SE.getUMaxExpr(Operands&: CanonOps);
324 return;
325 case scSMinExpr:
326 CanonicalSCEV = SE.getSMinExpr(Operands&: CanonOps);
327 return;
328 case scUMinExpr:
329 CanonicalSCEV = SE.getUMinExpr(Operands&: CanonOps);
330 return;
331 case scSequentialUMinExpr:
332 CanonicalSCEV = SE.getUMinExpr(Operands&: CanonOps, /*Sequential=*/true);
333 return;
334 default:
335 llvm_unreachable("Unknown SCEV type");
336 }
337}
338
339//===----------------------------------------------------------------------===//
340// Implementation of the SCEV class.
341//
342
343#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
344LLVM_DUMP_METHOD void SCEV::dump() const {
345 print(dbgs());
346 dbgs() << '\n';
347}
348#endif
349
350void SCEV::print(raw_ostream &OS) const {
351 switch (getSCEVType()) {
352 case scConstant:
353 cast<SCEVConstant>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
354 return;
355 case scVScale:
356 OS << "vscale";
357 return;
358 case scPtrToAddr:
359 case scPtrToInt: {
360 const SCEVCastExpr *PtrCast = cast<SCEVCastExpr>(Val: this);
361 const SCEV *Op = PtrCast->getOperand();
362 StringRef OpS = getSCEVType() == scPtrToAddr ? "addr" : "int";
363 OS << "(ptrto" << OpS << " " << *Op->getType() << " " << *Op << " to "
364 << *PtrCast->getType() << ")";
365 return;
366 }
367 case scTruncate: {
368 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: this);
369 const SCEV *Op = Trunc->getOperand();
370 OS << "(trunc " << *Op->getType() << " " << *Op << " to "
371 << *Trunc->getType() << ")";
372 return;
373 }
374 case scZeroExtend: {
375 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: this);
376 const SCEV *Op = ZExt->getOperand();
377 OS << "(zext " << *Op->getType() << " " << *Op << " to "
378 << *ZExt->getType() << ")";
379 return;
380 }
381 case scSignExtend: {
382 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: this);
383 const SCEV *Op = SExt->getOperand();
384 OS << "(sext " << *Op->getType() << " " << *Op << " to "
385 << *SExt->getType() << ")";
386 return;
387 }
388 case scAddRecExpr: {
389 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: this);
390 OS << "{" << *AR->getOperand(i: 0);
391 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
392 OS << ",+," << *AR->getOperand(i);
393 OS << "}<";
394 if (AR->hasNoUnsignedWrap())
395 OS << "nuw><";
396 if (AR->hasNoSignedWrap())
397 OS << "nsw><";
398 if (AR->hasNoSelfWrap() && !AR->hasNoUnsignedWrap() &&
399 !AR->hasNoSignedWrap())
400 OS << "nw><";
401 AR->getLoop()->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
402 OS << ">";
403 return;
404 }
405 case scAddExpr:
406 case scMulExpr:
407 case scUMaxExpr:
408 case scSMaxExpr:
409 case scUMinExpr:
410 case scSMinExpr:
411 case scSequentialUMinExpr: {
412 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(Val: this);
413 const char *OpStr = nullptr;
414 switch (NAry->getSCEVType()) {
415 case scAddExpr: OpStr = " + "; break;
416 case scMulExpr: OpStr = " * "; break;
417 case scUMaxExpr: OpStr = " umax "; break;
418 case scSMaxExpr: OpStr = " smax "; break;
419 case scUMinExpr:
420 OpStr = " umin ";
421 break;
422 case scSMinExpr:
423 OpStr = " smin ";
424 break;
425 case scSequentialUMinExpr:
426 OpStr = " umin_seq ";
427 break;
428 default:
429 llvm_unreachable("There are no other nary expression types.");
430 }
431 OS << "("
432 << llvm::interleaved(R: llvm::make_pointee_range(Range: NAry->operands()), Separator: OpStr)
433 << ")";
434 switch (NAry->getSCEVType()) {
435 case scAddExpr:
436 case scMulExpr:
437 if (NAry->hasNoUnsignedWrap())
438 OS << "<nuw>";
439 if (NAry->hasNoSignedWrap())
440 OS << "<nsw>";
441 break;
442 default:
443 // Nothing to print for other nary expressions.
444 break;
445 }
446 return;
447 }
448 case scUDivExpr: {
449 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: this);
450 OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")";
451 return;
452 }
453 case scUnknown:
454 cast<SCEVUnknown>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
455 return;
456 case scCouldNotCompute:
457 OS << "***COULDNOTCOMPUTE***";
458 return;
459 }
460 llvm_unreachable("Unknown SCEV kind!");
461}
462
463Type *SCEV::getType() const {
464 switch (getSCEVType()) {
465 case scConstant:
466 return cast<SCEVConstant>(Val: this)->getType();
467 case scVScale:
468 return cast<SCEVVScale>(Val: this)->getType();
469 case scPtrToAddr:
470 case scPtrToInt:
471 case scTruncate:
472 case scZeroExtend:
473 case scSignExtend:
474 return cast<SCEVCastExpr>(Val: this)->getType();
475 case scAddRecExpr:
476 return cast<SCEVAddRecExpr>(Val: this)->getType();
477 case scMulExpr:
478 return cast<SCEVMulExpr>(Val: this)->getType();
479 case scUMaxExpr:
480 case scSMaxExpr:
481 case scUMinExpr:
482 case scSMinExpr:
483 return cast<SCEVMinMaxExpr>(Val: this)->getType();
484 case scSequentialUMinExpr:
485 return cast<SCEVSequentialMinMaxExpr>(Val: this)->getType();
486 case scAddExpr:
487 return cast<SCEVAddExpr>(Val: this)->getType();
488 case scUDivExpr:
489 return cast<SCEVUDivExpr>(Val: this)->getType();
490 case scUnknown:
491 return cast<SCEVUnknown>(Val: this)->getType();
492 case scCouldNotCompute:
493 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
494 }
495 llvm_unreachable("Unknown SCEV kind!");
496}
497
498ArrayRef<SCEVUse> SCEV::operands() const {
499 switch (getSCEVType()) {
500 case scConstant:
501 case scVScale:
502 case scUnknown:
503 return {};
504 case scPtrToAddr:
505 case scPtrToInt:
506 case scTruncate:
507 case scZeroExtend:
508 case scSignExtend:
509 return cast<SCEVCastExpr>(Val: this)->operands();
510 case scAddRecExpr:
511 case scAddExpr:
512 case scMulExpr:
513 case scUMaxExpr:
514 case scSMaxExpr:
515 case scUMinExpr:
516 case scSMinExpr:
517 case scSequentialUMinExpr:
518 return cast<SCEVNAryExpr>(Val: this)->operands();
519 case scUDivExpr:
520 return cast<SCEVUDivExpr>(Val: this)->operands();
521 case scCouldNotCompute:
522 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
523 }
524 llvm_unreachable("Unknown SCEV kind!");
525}
526
527bool SCEV::isZero() const { return match(S: this, P: m_scev_Zero()); }
528
529bool SCEV::isOne() const { return match(S: this, P: m_scev_One()); }
530
531bool SCEV::isAllOnesValue() const { return match(S: this, P: m_scev_AllOnes()); }
532
533bool SCEV::isNonConstantNegative() const {
534 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: this);
535 if (!Mul) return false;
536
537 // If there is a constant factor, it will be first.
538 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Mul->getOperand(i: 0));
539 if (!SC) return false;
540
541 // Return true if the value is negative, this matches things like (-42 * V).
542 return SC->getAPInt().isNegative();
543}
544
545SCEVCouldNotCompute::SCEVCouldNotCompute() :
546 SCEV(FoldingSetNodeIDRef(), scCouldNotCompute, 0) {}
547
548bool SCEVCouldNotCompute::classof(const SCEV *S) {
549 return S->getSCEVType() == scCouldNotCompute;
550}
551
552const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
553 FoldingSetNodeID ID;
554 ID.AddInteger(I: scConstant);
555 ID.AddPointer(Ptr: V);
556 void *IP = nullptr;
557 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
558 SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(Allocator&: SCEVAllocator), V);
559 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
560 S->computeAndSetCanonical(SE&: *this);
561 return S;
562}
563
564const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
565 return getConstant(V: ConstantInt::get(Context&: getContext(), V: Val));
566}
567
568const SCEV *
569ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
570 IntegerType *ITy = cast<IntegerType>(Val: getEffectiveSCEVType(Ty));
571 // TODO: Avoid implicit trunc?
572 // See https://github.com/llvm/llvm-project/issues/112510.
573 return getConstant(
574 V: ConstantInt::get(Ty: ITy, V, IsSigned: isSigned, /*ImplicitTrunc=*/true));
575}
576
577const SCEV *ScalarEvolution::getVScale(Type *Ty) {
578 FoldingSetNodeID ID;
579 ID.AddInteger(I: scVScale);
580 ID.AddPointer(Ptr: Ty);
581 void *IP = nullptr;
582 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
583 return S;
584 SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(Allocator&: SCEVAllocator), Ty);
585 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
586 S->computeAndSetCanonical(SE&: *this);
587 return S;
588}
589
590const SCEV *ScalarEvolution::getElementCount(Type *Ty, ElementCount EC,
591 SCEV::NoWrapFlags Flags) {
592 const SCEV *Res = getConstant(Ty, V: EC.getKnownMinValue());
593 if (EC.isScalable())
594 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty), Flags);
595 return Res;
596}
597
598SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
599 SCEVUse op, Type *ty)
600 : SCEV(ID, SCEVTy, computeExpressionSize(Args: op)), Op(op), Ty(ty) {}
601
602SCEVPtrToAddrExpr::SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID,
603 const SCEV *Op, Type *ITy)
604 : SCEVCastExpr(ID, scPtrToAddr, Op, ITy) {
605 assert(getOperand()->getType()->isPointerTy() && Ty->isIntegerTy() &&
606 "Must be a non-bit-width-changing pointer-to-integer cast!");
607}
608
609SCEVPtrToIntExpr::SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, SCEVUse Op,
610 Type *ITy)
611 : SCEVCastExpr(ID, scPtrToInt, Op, ITy) {
612 assert(getOperand()->getType()->isPointerTy() && Ty->isIntegerTy() &&
613 "Must be a non-bit-width-changing pointer-to-integer cast!");
614}
615
616SCEVIntegralCastExpr::SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID,
617 SCEVTypes SCEVTy, SCEVUse op,
618 Type *ty)
619 : SCEVCastExpr(ID, SCEVTy, op, ty) {}
620
621SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
622 Type *ty)
623 : SCEVIntegralCastExpr(ID, scTruncate, op, ty) {
624 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
625 "Cannot truncate non-integer value!");
626}
627
628SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
629 Type *ty)
630 : SCEVIntegralCastExpr(ID, scZeroExtend, op, ty) {
631 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
632 "Cannot zero extend non-integer value!");
633}
634
635SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
636 Type *ty)
637 : SCEVIntegralCastExpr(ID, scSignExtend, op, ty) {
638 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
639 "Cannot sign extend non-integer value!");
640}
641
642void SCEVUnknown::deleted() {
643 // Clear this SCEVUnknown from various maps.
644 SE->forgetMemoizedResults(SCEVs: {this});
645
646 // Remove this SCEVUnknown from the uniquing map.
647 SE->UniqueSCEVs.RemoveNode(N: this);
648
649 // Release the value.
650 setValPtr(nullptr);
651}
652
653void SCEVUnknown::allUsesReplacedWith(Value *New) {
654 // Clear this SCEVUnknown from various maps.
655 SE->forgetMemoizedResults(SCEVs: {this});
656
657 // Remove this SCEVUnknown from the uniquing map.
658 SE->UniqueSCEVs.RemoveNode(N: this);
659
660 // Replace the value pointer in case someone is still using this SCEVUnknown.
661 setValPtr(New);
662}
663
664//===----------------------------------------------------------------------===//
665// SCEV Utilities
666//===----------------------------------------------------------------------===//
667
668/// Compare the two values \p LV and \p RV in terms of their "complexity" where
669/// "complexity" is a partial (and somewhat ad-hoc) relation used to order
670/// operands in SCEV expressions.
671static int CompareValueComplexity(const LoopInfo *const LI, Value *LV,
672 Value *RV, unsigned Depth) {
673 if (Depth > MaxValueCompareDepth)
674 return 0;
675
676 // Order pointer values after integer values. This helps SCEVExpander form
677 // GEPs.
678 bool LIsPointer = LV->getType()->isPointerTy(),
679 RIsPointer = RV->getType()->isPointerTy();
680 if (LIsPointer != RIsPointer)
681 return (int)LIsPointer - (int)RIsPointer;
682
683 // Compare getValueID values.
684 unsigned LID = LV->getValueID(), RID = RV->getValueID();
685 if (LID != RID)
686 return (int)LID - (int)RID;
687
688 // Sort arguments by their position.
689 if (const auto *LA = dyn_cast<Argument>(Val: LV)) {
690 const auto *RA = cast<Argument>(Val: RV);
691 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
692 return (int)LArgNo - (int)RArgNo;
693 }
694
695 if (const auto *LGV = dyn_cast<GlobalValue>(Val: LV)) {
696 const auto *RGV = cast<GlobalValue>(Val: RV);
697
698 if (auto L = LGV->getLinkage() - RGV->getLinkage())
699 return L;
700
701 const auto IsGVNameSemantic = [&](const GlobalValue *GV) {
702 auto LT = GV->getLinkage();
703 return !(GlobalValue::isPrivateLinkage(Linkage: LT) ||
704 GlobalValue::isInternalLinkage(Linkage: LT));
705 };
706
707 // Use the names to distinguish the two values, but only if the
708 // names are semantically important.
709 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
710 return LGV->getName().compare(RHS: RGV->getName());
711 }
712
713 // For instructions, compare their loop depth, and their operand count. This
714 // is pretty loose.
715 if (const auto *LInst = dyn_cast<Instruction>(Val: LV)) {
716 const auto *RInst = cast<Instruction>(Val: RV);
717
718 // Compare loop depths.
719 const BasicBlock *LParent = LInst->getParent(),
720 *RParent = RInst->getParent();
721 if (LParent != RParent) {
722 unsigned LDepth = LI->getLoopDepth(BB: LParent),
723 RDepth = LI->getLoopDepth(BB: RParent);
724 if (LDepth != RDepth)
725 return (int)LDepth - (int)RDepth;
726 }
727
728 // Compare the number of operands.
729 unsigned LNumOps = LInst->getNumOperands(),
730 RNumOps = RInst->getNumOperands();
731 if (LNumOps != RNumOps)
732 return (int)LNumOps - (int)RNumOps;
733
734 for (unsigned Idx : seq(Size: LNumOps)) {
735 int Result = CompareValueComplexity(LI, LV: LInst->getOperand(i: Idx),
736 RV: RInst->getOperand(i: Idx), Depth: Depth + 1);
737 if (Result != 0)
738 return Result;
739 }
740 }
741
742 return 0;
743}
744
745// Return negative, zero, or positive, if LHS is less than, equal to, or greater
746// than RHS, respectively. A three-way result allows recursive comparisons to be
747// more efficient.
748// If the max analysis depth was reached, return std::nullopt, assuming we do
749// not know if they are equivalent for sure.
750static std::optional<int>
751CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS,
752 const SCEV *RHS, DominatorTree &DT, unsigned Depth = 0) {
753 // Fast-path: SCEVs are uniqued so we can do a quick equality check.
754 if (LHS == RHS)
755 return 0;
756
757 // Primarily, sort the SCEVs by their getSCEVType().
758 SCEVTypes LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
759 if (LType != RType)
760 return (int)LType - (int)RType;
761
762 if (Depth > MaxSCEVCompareDepth)
763 return std::nullopt;
764
765 // Aside from the getSCEVType() ordering, the particular ordering
766 // isn't very important except that it's beneficial to be consistent,
767 // so that (a + b) and (b + a) don't end up as different expressions.
768 switch (LType) {
769 case scUnknown: {
770 const SCEVUnknown *LU = cast<SCEVUnknown>(Val: LHS);
771 const SCEVUnknown *RU = cast<SCEVUnknown>(Val: RHS);
772
773 int X =
774 CompareValueComplexity(LI, LV: LU->getValue(), RV: RU->getValue(), Depth: Depth + 1);
775 return X;
776 }
777
778 case scConstant: {
779 const SCEVConstant *LC = cast<SCEVConstant>(Val: LHS);
780 const SCEVConstant *RC = cast<SCEVConstant>(Val: RHS);
781
782 // Compare constant values.
783 const APInt &LA = LC->getAPInt();
784 const APInt &RA = RC->getAPInt();
785 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
786 if (LBitWidth != RBitWidth)
787 return (int)LBitWidth - (int)RBitWidth;
788 return LA.ult(RHS: RA) ? -1 : 1;
789 }
790
791 case scVScale: {
792 const auto *LTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: LHS)->getType());
793 const auto *RTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: RHS)->getType());
794 return LTy->getBitWidth() - RTy->getBitWidth();
795 }
796
797 case scAddRecExpr: {
798 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(Val: LHS);
799 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(Val: RHS);
800
801 // There is always a dominance between two recs that are used by one SCEV,
802 // so we can safely sort recs by loop header dominance. We require such
803 // order in getAddExpr.
804 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
805 if (LLoop != RLoop) {
806 const BasicBlock *LHead = LLoop->getHeader(), *RHead = RLoop->getHeader();
807 assert(LHead != RHead && "Two loops share the same header?");
808 if (DT.dominates(A: LHead, B: RHead))
809 return 1;
810 assert(DT.dominates(RHead, LHead) &&
811 "No dominance between recurrences used by one SCEV?");
812 return -1;
813 }
814
815 [[fallthrough]];
816 }
817
818 case scTruncate:
819 case scZeroExtend:
820 case scSignExtend:
821 case scPtrToAddr:
822 case scPtrToInt:
823 case scAddExpr:
824 case scMulExpr:
825 case scUDivExpr:
826 case scSMaxExpr:
827 case scUMaxExpr:
828 case scSMinExpr:
829 case scUMinExpr:
830 case scSequentialUMinExpr: {
831 ArrayRef<SCEVUse> LOps = LHS->operands();
832 ArrayRef<SCEVUse> ROps = RHS->operands();
833
834 // Lexicographically compare n-ary-like expressions.
835 unsigned LNumOps = LOps.size(), RNumOps = ROps.size();
836 if (LNumOps != RNumOps)
837 return (int)LNumOps - (int)RNumOps;
838
839 for (unsigned i = 0; i != LNumOps; ++i) {
840 auto X = CompareSCEVComplexity(LI, LHS: LOps[i].getPointer(),
841 RHS: ROps[i].getPointer(), DT, Depth: Depth + 1);
842 if (X != 0)
843 return X;
844 }
845 return 0;
846 }
847
848 case scCouldNotCompute:
849 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
850 }
851 llvm_unreachable("Unknown SCEV kind!");
852}
853
854/// Given a list of SCEV objects, order them by their complexity, and group
855/// objects of the same complexity together by value. When this routine is
856/// finished, we know that any duplicates in the vector are consecutive and that
857/// complexity is monotonically increasing.
858///
859/// Note that we go take special precautions to ensure that we get deterministic
860/// results from this routine. In other words, we don't want the results of
861/// this to depend on where the addresses of various SCEV objects happened to
862/// land in memory.
863static void GroupByComplexity(SmallVectorImpl<SCEVUse> &Ops, LoopInfo *LI,
864 DominatorTree &DT) {
865 if (Ops.size() < 2) return; // Noop
866
867 // Whether LHS has provably less complexity than RHS.
868 auto IsLessComplex = [&](SCEVUse LHS, SCEVUse RHS) {
869 auto Complexity = CompareSCEVComplexity(LI, LHS, RHS, DT);
870 return Complexity && *Complexity < 0;
871 };
872 if (Ops.size() == 2) {
873 // This is the common case, which also happens to be trivially simple.
874 // Special case it.
875 SCEVUse &LHS = Ops[0], &RHS = Ops[1];
876 if (IsLessComplex(RHS, LHS))
877 std::swap(a&: LHS, b&: RHS);
878 return;
879 }
880
881 // Do the rough sort by complexity.
882 llvm::stable_sort(
883 Range&: Ops, C: [&](SCEVUse LHS, SCEVUse RHS) { return IsLessComplex(LHS, RHS); });
884
885 // Now that we are sorted by complexity, group elements of the same
886 // complexity. Note that this is, at worst, N^2, but the vector is likely to
887 // be extremely short in practice. Note that we take this approach because we
888 // do not want to depend on the addresses of the objects we are grouping.
889 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
890 const SCEV *S = Ops[i];
891 unsigned Complexity = S->getSCEVType();
892
893 // If there are any objects of the same complexity and same value as this
894 // one, group them.
895 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
896 if (Ops[j] == S) { // Found a duplicate.
897 // Move it to immediately after i'th element.
898 std::swap(a&: Ops[i+1], b&: Ops[j]);
899 ++i; // no need to rescan it.
900 if (i == e-2) return; // Done!
901 }
902 }
903 }
904}
905
906/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
907/// least HugeExprThreshold nodes).
908static bool hasHugeExpression(ArrayRef<SCEVUse> Ops) {
909 return any_of(Range&: Ops, P: [](const SCEV *S) {
910 return S->getExpressionSize() >= HugeExprThreshold;
911 });
912}
913
914/// Performs a number of common optimizations on the passed \p Ops. If the
915/// whole expression reduces down to a single operand, it will be returned.
916///
917/// The following optimizations are performed:
918/// * Fold constants using the \p Fold function.
919/// * Remove identity constants satisfying \p IsIdentity.
920/// * If a constant satisfies \p IsAbsorber, return it.
921/// * Sort operands by complexity.
922template <typename FoldT, typename IsIdentityT, typename IsAbsorberT>
923static const SCEV *
924constantFoldAndGroupOps(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT,
925 SmallVectorImpl<SCEVUse> &Ops, FoldT Fold,
926 IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
927 const SCEVConstant *Folded = nullptr;
928 for (unsigned Idx = 0; Idx < Ops.size();) {
929 const SCEV *Op = Ops[Idx];
930 if (const auto *C = dyn_cast<SCEVConstant>(Val: Op)) {
931 if (!Folded)
932 Folded = C;
933 else
934 Folded = cast<SCEVConstant>(
935 SE.getConstant(Fold(Folded->getAPInt(), C->getAPInt())));
936 Ops.erase(CI: Ops.begin() + Idx);
937 continue;
938 }
939 ++Idx;
940 }
941
942 if (Ops.empty()) {
943 assert(Folded && "Must have folded value");
944 return Folded;
945 }
946
947 if (Folded && IsAbsorber(Folded->getAPInt()))
948 return Folded;
949
950 GroupByComplexity(Ops, LI: &LI, DT);
951 if (Folded && !IsIdentity(Folded->getAPInt()))
952 Ops.insert(I: Ops.begin(), Elt: Folded);
953
954 return Ops.size() == 1 ? Ops[0] : nullptr;
955}
956
957//===----------------------------------------------------------------------===//
958// Simple SCEV method implementations
959//===----------------------------------------------------------------------===//
960
961/// Compute BC(It, K). The result has width W. Assume, K > 0.
962static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
963 ScalarEvolution &SE,
964 Type *ResultTy) {
965 // Handle the simplest case efficiently.
966 if (K == 1)
967 return SE.getTruncateOrZeroExtend(V: It, Ty: ResultTy);
968
969 // We are using the following formula for BC(It, K):
970 //
971 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
972 //
973 // Suppose, W is the bitwidth of the return value. We must be prepared for
974 // overflow. Hence, we must assure that the result of our computation is
975 // equal to the accurate one modulo 2^W. Unfortunately, division isn't
976 // safe in modular arithmetic.
977 //
978 // However, this code doesn't use exactly that formula; the formula it uses
979 // is something like the following, where T is the number of factors of 2 in
980 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
981 // exponentiation:
982 //
983 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
984 //
985 // This formula is trivially equivalent to the previous formula. However,
986 // this formula can be implemented much more efficiently. The trick is that
987 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
988 // arithmetic. To do exact division in modular arithmetic, all we have
989 // to do is multiply by the inverse. Therefore, this step can be done at
990 // width W.
991 //
992 // The next issue is how to safely do the division by 2^T. The way this
993 // is done is by doing the multiplication step at a width of at least W + T
994 // bits. This way, the bottom W+T bits of the product are accurate. Then,
995 // when we perform the division by 2^T (which is equivalent to a right shift
996 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get
997 // truncated out after the division by 2^T.
998 //
999 // In comparison to just directly using the first formula, this technique
1000 // is much more efficient; using the first formula requires W * K bits,
1001 // but this formula less than W + K bits. Also, the first formula requires
1002 // a division step, whereas this formula only requires multiplies and shifts.
1003 //
1004 // It doesn't matter whether the subtraction step is done in the calculation
1005 // width or the input iteration count's width; if the subtraction overflows,
1006 // the result must be zero anyway. We prefer here to do it in the width of
1007 // the induction variable because it helps a lot for certain cases; CodeGen
1008 // isn't smart enough to ignore the overflow, which leads to much less
1009 // efficient code if the width of the subtraction is wider than the native
1010 // register width.
1011 //
1012 // (It's possible to not widen at all by pulling out factors of 2 before
1013 // the multiplication; for example, K=2 can be calculated as
1014 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
1015 // extra arithmetic, so it's not an obvious win, and it gets
1016 // much more complicated for K > 3.)
1017
1018 // Protection from insane SCEVs; this bound is conservative,
1019 // but it probably doesn't matter.
1020 if (K > 1000)
1021 return SE.getCouldNotCompute();
1022
1023 unsigned W = SE.getTypeSizeInBits(Ty: ResultTy);
1024
1025 // Calculate K! / 2^T and T; we divide out the factors of two before
1026 // multiplying for calculating K! / 2^T to avoid overflow.
1027 // Other overflow doesn't matter because we only care about the bottom
1028 // W bits of the result.
1029 APInt OddFactorial(W, 1);
1030 unsigned T = 1;
1031 for (unsigned i = 3; i <= K; ++i) {
1032 unsigned TwoFactors = countr_zero(Val: i);
1033 T += TwoFactors;
1034 OddFactorial *= (i >> TwoFactors);
1035 }
1036
1037 // We need at least W + T bits for the multiplication step
1038 unsigned CalculationBits = W + T;
1039
1040 // Calculate 2^T, at width T+W.
1041 APInt DivFactor = APInt::getOneBitSet(numBits: CalculationBits, BitNo: T);
1042
1043 // Calculate the multiplicative inverse of K! / 2^T;
1044 // this multiplication factor will perform the exact division by
1045 // K! / 2^T.
1046 APInt MultiplyFactor = OddFactorial.multiplicativeInverse();
1047
1048 // Calculate the product, at width T+W
1049 IntegerType *CalculationTy = IntegerType::get(C&: SE.getContext(),
1050 NumBits: CalculationBits);
1051 const SCEV *Dividend = SE.getTruncateOrZeroExtend(V: It, Ty: CalculationTy);
1052 for (unsigned i = 1; i != K; ++i) {
1053 const SCEV *S = SE.getMinusSCEV(LHS: It, RHS: SE.getConstant(Ty: It->getType(), V: i));
1054 Dividend = SE.getMulExpr(LHS: Dividend,
1055 RHS: SE.getTruncateOrZeroExtend(V: S, Ty: CalculationTy));
1056 }
1057
1058 // Divide by 2^T
1059 const SCEV *DivResult = SE.getUDivExpr(LHS: Dividend, RHS: SE.getConstant(Val: DivFactor));
1060
1061 // Truncate the result, and divide by K! / 2^T.
1062
1063 return SE.getMulExpr(LHS: SE.getConstant(Val: MultiplyFactor),
1064 RHS: SE.getTruncateOrZeroExtend(V: DivResult, Ty: ResultTy));
1065}
1066
1067/// Return the value of this chain of recurrences at the specified iteration
1068/// number. We can evaluate this recurrence by multiplying each element in the
1069/// chain by the binomial coefficient corresponding to it. In other words, we
1070/// can evaluate {A,+,B,+,C,+,D} as:
1071///
1072/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
1073///
1074/// where BC(It, k) stands for binomial coefficient.
1075const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
1076 ScalarEvolution &SE) const {
1077 return evaluateAtIteration(Operands: operands(), It, SE);
1078}
1079
1080const SCEV *SCEVAddRecExpr::evaluateAtIteration(ArrayRef<SCEVUse> Operands,
1081 const SCEV *It,
1082 ScalarEvolution &SE) {
1083 assert(Operands.size() > 0);
1084 const SCEV *Result = Operands[0].getPointer();
1085 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
1086 // The computation is correct in the face of overflow provided that the
1087 // multiplication is performed _after_ the evaluation of the binomial
1088 // coefficient.
1089 const SCEV *Coeff = BinomialCoefficient(It, K: i, SE, ResultTy: Result->getType());
1090 if (isa<SCEVCouldNotCompute>(Val: Coeff))
1091 return Coeff;
1092
1093 Result =
1094 SE.getAddExpr(LHS: Result, RHS: SE.getMulExpr(LHS: Operands[i].getPointer(), RHS: Coeff));
1095 }
1096 return Result;
1097}
1098
1099//===----------------------------------------------------------------------===//
1100// SCEV Expression folder implementations
1101//===----------------------------------------------------------------------===//
1102
1103/// The SCEVCastSinkingRewriter takes a scalar evolution expression,
1104/// which computes a pointer-typed value, and rewrites the whole expression
1105/// tree so that *all* the computations are done on integers, and the only
1106/// pointer-typed operands in the expression are SCEVUnknown.
1107/// The CreatePtrCast callback is invoked to create the actual conversion
1108/// (ptrtoint or ptrtoaddr) at the SCEVUnknown leaves.
1109class SCEVCastSinkingRewriter
1110 : public SCEVRewriteVisitor<SCEVCastSinkingRewriter> {
1111 using Base = SCEVRewriteVisitor<SCEVCastSinkingRewriter>;
1112 using ConversionFn = function_ref<const SCEV *(const SCEVUnknown *)>;
1113 Type *TargetTy;
1114 ConversionFn CreatePtrCast;
1115
1116public:
1117 SCEVCastSinkingRewriter(ScalarEvolution &SE, Type *TargetTy,
1118 ConversionFn CreatePtrCast)
1119 : Base(SE), TargetTy(TargetTy), CreatePtrCast(std::move(CreatePtrCast)) {}
1120
1121 static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE,
1122 Type *TargetTy, ConversionFn CreatePtrCast) {
1123 SCEVCastSinkingRewriter Rewriter(SE, TargetTy, std::move(CreatePtrCast));
1124 return Rewriter.visit(S: Scev);
1125 }
1126
1127 const SCEV *visit(const SCEV *S) {
1128 Type *STy = S->getType();
1129 // If the expression is not pointer-typed, just keep it as-is.
1130 if (!STy->isPointerTy())
1131 return S;
1132 // Else, recursively sink the cast down into it.
1133 return Base::visit(S);
1134 }
1135
1136 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
1137 // Preserve wrap flags on rewritten SCEVAddExpr, which the default
1138 // implementation drops.
1139 SmallVector<SCEVUse, 2> Operands;
1140 bool Changed = false;
1141 for (SCEVUse Op : Expr->operands()) {
1142 Operands.push_back(Elt: visit(S: Op.getPointer()));
1143 Changed |= Op.getPointer() != Operands.back();
1144 }
1145 return !Changed ? Expr : SE.getAddExpr(Ops&: Operands, Flags: Expr->getNoWrapFlags());
1146 }
1147
1148 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
1149 SmallVector<SCEVUse, 2> Operands;
1150 bool Changed = false;
1151 for (SCEVUse Op : Expr->operands()) {
1152 Operands.push_back(Elt: visit(S: Op.getPointer()));
1153 Changed |= Op.getPointer() != Operands.back();
1154 }
1155 return !Changed ? Expr : SE.getMulExpr(Ops&: Operands, Flags: Expr->getNoWrapFlags());
1156 }
1157
1158 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
1159 assert(Expr->getType()->isPointerTy() &&
1160 "Should only reach pointer-typed SCEVUnknown's.");
1161 // Perform some basic constant folding. If the operand of the cast is a
1162 // null pointer, don't create a cast SCEV expression (that will be left
1163 // as-is), but produce a zero constant.
1164 if (isa<ConstantPointerNull>(Val: Expr->getValue()))
1165 return SE.getZero(Ty: TargetTy);
1166 return CreatePtrCast(Expr);
1167 }
1168};
1169
1170const SCEV *ScalarEvolution::getLosslessPtrToIntExpr(const SCEV *Op) {
1171 assert(Op->getType()->isPointerTy() && "Op must be a pointer");
1172
1173 // It isn't legal for optimizations to construct new ptrtoint expressions
1174 // for non-integral pointers.
1175 if (getDataLayout().isNonIntegralPointerType(Ty: Op->getType()))
1176 return getCouldNotCompute();
1177
1178 Type *IntPtrTy = getDataLayout().getIntPtrType(Op->getType());
1179
1180 // We can only trivially model ptrtoint if SCEV's effective (integer) type
1181 // is sufficiently wide to represent all possible pointer values.
1182 // We could theoretically teach SCEV to truncate wider pointers, but
1183 // that isn't implemented for now.
1184 if (getDataLayout().getTypeSizeInBits(Ty: getEffectiveSCEVType(Ty: Op->getType())) !=
1185 getDataLayout().getTypeSizeInBits(Ty: IntPtrTy))
1186 return getCouldNotCompute();
1187
1188 // Use the rewriter to sink the cast down to SCEVUnknown leaves.
1189 const SCEV *IntOp = SCEVCastSinkingRewriter::rewrite(
1190 Scev: Op, SE&: *this, TargetTy: IntPtrTy, CreatePtrCast: [this, IntPtrTy](const SCEVUnknown *U) {
1191 FoldingSetNodeID ID;
1192 ID.AddInteger(I: scPtrToInt);
1193 ID.AddPointer(Ptr: U);
1194 void *IP = nullptr;
1195 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
1196 return S;
1197 SCEV *S = new (SCEVAllocator)
1198 SCEVPtrToIntExpr(ID.Intern(Allocator&: SCEVAllocator), U, IntPtrTy);
1199 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1200 S->computeAndSetCanonical(SE&: *this);
1201 registerUser(User: S, Ops: U);
1202 return static_cast<const SCEV *>(S);
1203 });
1204 assert(IntOp->getType()->isIntegerTy() &&
1205 "We must have succeeded in sinking the cast, "
1206 "and ending up with an integer-typed expression!");
1207 return IntOp;
1208}
1209
1210const SCEV *ScalarEvolution::getPtrToAddrExpr(const SCEV *Op) {
1211 assert(Op->getType()->isPointerTy() && "Op must be a pointer");
1212
1213 // Treat pointers with unstable representation conservatively, since the
1214 // address bits may change.
1215 if (DL.hasUnstableRepresentation(Ty: Op->getType()))
1216 return getCouldNotCompute();
1217
1218 Type *Ty = DL.getAddressType(PtrTy: Op->getType());
1219
1220 // Use the rewriter to sink the cast down to SCEVUnknown leaves.
1221 // The rewriter handles null pointer constant folding.
1222 const SCEV *IntOp = SCEVCastSinkingRewriter::rewrite(
1223 Scev: Op, SE&: *this, TargetTy: Ty, CreatePtrCast: [this, Ty](const SCEVUnknown *U) {
1224 FoldingSetNodeID ID;
1225 ID.AddInteger(I: scPtrToAddr);
1226 ID.AddPointer(Ptr: U);
1227 ID.AddPointer(Ptr: Ty);
1228 void *IP = nullptr;
1229 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
1230 return S;
1231 SCEV *S = new (SCEVAllocator)
1232 SCEVPtrToAddrExpr(ID.Intern(Allocator&: SCEVAllocator), U, Ty);
1233 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1234 S->computeAndSetCanonical(SE&: *this);
1235 registerUser(User: S, Ops: U);
1236 return static_cast<const SCEV *>(S);
1237 });
1238 assert(IntOp->getType()->isIntegerTy() &&
1239 "We must have succeeded in sinking the cast, "
1240 "and ending up with an integer-typed expression!");
1241 return IntOp;
1242}
1243
1244const SCEV *ScalarEvolution::getPtrToIntExpr(const SCEV *Op, Type *Ty) {
1245 assert(Ty->isIntegerTy() && "Target type must be an integer type!");
1246
1247 const SCEV *IntOp = getLosslessPtrToIntExpr(Op);
1248 if (isa<SCEVCouldNotCompute>(Val: IntOp))
1249 return IntOp;
1250
1251 return getTruncateOrZeroExtend(V: IntOp, Ty);
1252}
1253
1254const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, Type *Ty,
1255 unsigned Depth) {
1256 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
1257 "This is not a truncating conversion!");
1258 assert(isSCEVable(Ty) &&
1259 "This is not a conversion to a SCEVable type!");
1260 assert(!Op->getType()->isPointerTy() && "Can't truncate pointer!");
1261 Ty = getEffectiveSCEVType(Ty);
1262
1263 FoldingSetNodeID ID;
1264 ID.AddInteger(I: scTruncate);
1265 ID.AddPointer(Ptr: Op);
1266 ID.AddPointer(Ptr: Ty);
1267 void *IP = nullptr;
1268 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
1269
1270 // Fold if the operand is constant.
1271 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
1272 return getConstant(
1273 V: cast<ConstantInt>(Val: ConstantExpr::getTrunc(C: SC->getValue(), Ty)));
1274
1275 // trunc(trunc(x)) --> trunc(x)
1276 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val: Op))
1277 return getTruncateExpr(Op: ST->getOperand(), Ty, Depth: Depth + 1);
1278
1279 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
1280 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val: Op))
1281 return getTruncateOrSignExtend(V: SS->getOperand(), Ty, Depth: Depth + 1);
1282
1283 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
1284 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val: Op))
1285 return getTruncateOrZeroExtend(V: SZ->getOperand(), Ty, Depth: Depth + 1);
1286
1287 if (Depth > MaxCastDepth) {
1288 SCEV *S =
1289 new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator), Op, Ty);
1290 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1291 S->computeAndSetCanonical(SE&: *this);
1292 registerUser(User: S, Ops: Op);
1293 return S;
1294 }
1295
1296 // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and
1297 // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN),
1298 // if after transforming we have at most one truncate, not counting truncates
1299 // that replace other casts.
1300 if (isa<SCEVAddExpr>(Val: Op) || isa<SCEVMulExpr>(Val: Op)) {
1301 auto *CommOp = cast<SCEVCommutativeExpr>(Val: Op);
1302 SmallVector<SCEVUse, 4> Operands;
1303 unsigned numTruncs = 0;
1304 for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
1305 ++i) {
1306 const SCEV *S = getTruncateExpr(Op: CommOp->getOperand(i), Ty, Depth: Depth + 1);
1307 if (!isa<SCEVIntegralCastExpr>(Val: CommOp->getOperand(i)) &&
1308 isa<SCEVTruncateExpr>(Val: S))
1309 numTruncs++;
1310 Operands.push_back(Elt: S);
1311 }
1312 if (numTruncs < 2) {
1313 if (isa<SCEVAddExpr>(Val: Op))
1314 return getAddExpr(Ops&: Operands);
1315 if (isa<SCEVMulExpr>(Val: Op))
1316 return getMulExpr(Ops&: Operands);
1317 llvm_unreachable("Unexpected SCEV type for Op.");
1318 }
1319 // Although we checked in the beginning that ID is not in the cache, it is
1320 // possible that during recursion and different modification ID was inserted
1321 // into the cache. So if we find it, just return it.
1322 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
1323 return S;
1324 }
1325
1326 // If the input value is a chrec scev, truncate the chrec's operands.
1327 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val: Op)) {
1328 SmallVector<SCEVUse, 4> Operands;
1329 for (const SCEV *Op : AddRec->operands())
1330 Operands.push_back(Elt: getTruncateExpr(Op, Ty, Depth: Depth + 1));
1331 return getAddRecExpr(Operands, L: AddRec->getLoop(), Flags: SCEV::FlagAnyWrap);
1332 }
1333
1334 // Return zero if truncating to known zeros.
1335 uint32_t MinTrailingZeros = getMinTrailingZeros(S: Op);
1336 if (MinTrailingZeros >= getTypeSizeInBits(Ty))
1337 return getZero(Ty);
1338
1339 // The cast wasn't folded; create an explicit cast node. We can reuse
1340 // the existing insert position since if we get here, we won't have
1341 // made any changes which would invalidate it.
1342 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator),
1343 Op, Ty);
1344 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1345 S->computeAndSetCanonical(SE&: *this);
1346 registerUser(User: S, Ops: Op);
1347 return S;
1348}
1349
1350// Get the limit of a recurrence such that incrementing by Step cannot cause
1351// signed overflow as long as the value of the recurrence within the
1352// loop does not exceed this limit before incrementing.
1353static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
1354 ICmpInst::Predicate *Pred,
1355 ScalarEvolution *SE) {
1356 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1357 if (SE->isKnownPositive(S: Step)) {
1358 *Pred = ICmpInst::ICMP_SLT;
1359 return SE->getConstant(Val: APInt::getSignedMinValue(numBits: BitWidth) -
1360 SE->getSignedRangeMax(S: Step));
1361 }
1362 if (SE->isKnownNegative(S: Step)) {
1363 *Pred = ICmpInst::ICMP_SGT;
1364 return SE->getConstant(Val: APInt::getSignedMaxValue(numBits: BitWidth) -
1365 SE->getSignedRangeMin(S: Step));
1366 }
1367 return nullptr;
1368}
1369
1370// Get the limit of a recurrence such that incrementing by Step cannot cause
1371// unsigned overflow as long as the value of the recurrence within the loop does
1372// not exceed this limit before incrementing.
1373static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step,
1374 ICmpInst::Predicate *Pred,
1375 ScalarEvolution *SE) {
1376 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1377 *Pred = ICmpInst::ICMP_ULT;
1378
1379 return SE->getConstant(Val: APInt::getMinValue(numBits: BitWidth) -
1380 SE->getUnsignedRangeMax(S: Step));
1381}
1382
1383namespace {
1384
1385struct ExtendOpTraitsBase {
1386 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(const SCEV *, Type *,
1387 unsigned);
1388};
1389
1390// Used to make code generic over signed and unsigned overflow.
1391template <typename ExtendOp> struct ExtendOpTraits {
1392 // Members present:
1393 //
1394 // static const SCEV::NoWrapFlags WrapType;
1395 //
1396 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
1397 //
1398 // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1399 // ICmpInst::Predicate *Pred,
1400 // ScalarEvolution *SE);
1401};
1402
1403template <>
1404struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
1405 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNSW;
1406
1407 static const GetExtendExprTy GetExtendExpr;
1408
1409 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1410 ICmpInst::Predicate *Pred,
1411 ScalarEvolution *SE) {
1412 return getSignedOverflowLimitForStep(Step, Pred, SE);
1413 }
1414};
1415
1416const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1417 SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr;
1418
1419template <>
1420struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
1421 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNUW;
1422
1423 static const GetExtendExprTy GetExtendExpr;
1424
1425 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1426 ICmpInst::Predicate *Pred,
1427 ScalarEvolution *SE) {
1428 return getUnsignedOverflowLimitForStep(Step, Pred, SE);
1429 }
1430};
1431
1432const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1433 SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr;
1434
1435} // end anonymous namespace
1436
1437// The recurrence AR has been shown to have no signed/unsigned wrap or something
1438// close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
1439// easily prove NSW/NUW for its preincrement or postincrement sibling. This
1440// allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
1441// Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
1442// expression "Step + sext/zext(PreIncAR)" is congruent with
1443// "sext/zext(PostIncAR)"
1444template <typename ExtendOpTy>
1445static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty,
1446 ScalarEvolution *SE, unsigned Depth) {
1447 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1448 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1449
1450 const Loop *L = AR->getLoop();
1451 const SCEV *Start = AR->getStart();
1452 const SCEV *Step = AR->getStepRecurrence(SE&: *SE);
1453
1454 // Check for a simple looking step prior to loop entry.
1455 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Val: Start);
1456 if (!SA)
1457 return nullptr;
1458
1459 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
1460 // subtraction is expensive. For this purpose, perform a quick and dirty
1461 // difference, by checking for Step in the operand list. Note, that
1462 // SA might have repeated ops, like %a + %a + ..., so only remove one.
1463 SmallVector<SCEVUse, 4> DiffOps(SA->operands());
1464 for (auto It = DiffOps.begin(); It != DiffOps.end(); ++It)
1465 if (*It == Step) {
1466 DiffOps.erase(CI: It);
1467 break;
1468 }
1469
1470 if (DiffOps.size() == SA->getNumOperands())
1471 return nullptr;
1472
1473 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
1474 // `Step`:
1475
1476 // 1. NSW/NUW flags on the step increment.
1477 auto PreStartFlags =
1478 ScalarEvolution::maskFlags(Flags: SA->getNoWrapFlags(), Mask: SCEV::FlagNUW);
1479 const SCEV *PreStart = SE->getAddExpr(Ops&: DiffOps, Flags: PreStartFlags);
1480 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
1481 Val: SE->getAddRecExpr(Start: PreStart, Step, L, Flags: SCEV::FlagAnyWrap));
1482
1483 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
1484 // "S+X does not sign/unsign-overflow".
1485 //
1486
1487 const SCEV *BECount = SE->getBackedgeTakenCount(L);
1488 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType)) &&
1489 !isa<SCEVCouldNotCompute>(Val: BECount) && SE->isKnownPositive(S: BECount))
1490 return PreStart;
1491
1492 // 2. Direct overflow check on the step operation's expression.
1493 unsigned BitWidth = SE->getTypeSizeInBits(Ty: AR->getType());
1494 Type *WideTy = IntegerType::get(C&: SE->getContext(), NumBits: BitWidth * 2);
1495 const SCEV *OperandExtendedStart =
1496 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
1497 (SE->*GetExtendExpr)(Step, WideTy, Depth));
1498 if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
1499 if (PreAR && any(AR->getNoWrapFlags(Mask: WrapType))) {
1500 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
1501 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
1502 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact.
1503 SE->setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(PreAR), Flags: WrapType);
1504 }
1505 return PreStart;
1506 }
1507
1508 // 3. Loop precondition.
1509 ICmpInst::Predicate Pred;
1510 const SCEV *OverflowLimit =
1511 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1512
1513 if (OverflowLimit &&
1514 SE->isLoopEntryGuardedByCond(L, Pred, LHS: PreStart, RHS: OverflowLimit))
1515 return PreStart;
1516
1517 return nullptr;
1518}
1519
1520// Get the normalized zero or sign extended expression for this AddRec's Start.
1521template <typename ExtendOpTy>
1522static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
1523 ScalarEvolution *SE,
1524 unsigned Depth) {
1525 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1526
1527 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, Ty, SE, Depth);
1528 if (!PreStart)
1529 return (SE->*GetExtendExpr)(AR->getStart(), Ty, Depth);
1530
1531 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(SE&: *SE), Ty,
1532 Depth),
1533 (SE->*GetExtendExpr)(PreStart, Ty, Depth));
1534}
1535
1536// Try to prove away overflow by looking at "nearby" add recurrences. A
1537// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
1538// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
1539//
1540// Formally:
1541//
1542// {S,+,X} == {S-T,+,X} + T
1543// => Ext({S,+,X}) == Ext({S-T,+,X} + T)
1544//
1545// If ({S-T,+,X} + T) does not overflow ... (1)
1546//
1547// RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
1548//
1549// If {S-T,+,X} does not overflow ... (2)
1550//
1551// RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
1552// == {Ext(S-T)+Ext(T),+,Ext(X)}
1553//
1554// If (S-T)+T does not overflow ... (3)
1555//
1556// RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
1557// == {Ext(S),+,Ext(X)} == LHS
1558//
1559// Thus, if (1), (2) and (3) are true for some T, then
1560// Ext({S,+,X}) == {Ext(S),+,Ext(X)}
1561//
1562// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
1563// does not overflow" restricted to the 0th iteration. Therefore we only need
1564// to check for (1) and (2).
1565//
1566// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
1567// is `Delta` (defined below).
1568template <typename ExtendOpTy>
1569bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
1570 const SCEV *Step,
1571 const Loop *L) {
1572 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1573
1574 // We restrict `Start` to a constant to prevent SCEV from spending too much
1575 // time here. It is correct (but more expensive) to continue with a
1576 // non-constant `Start` and do a general SCEV subtraction to compute
1577 // `PreStart` below.
1578 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Val: Start);
1579 if (!StartC)
1580 return false;
1581
1582 APInt StartAI = StartC->getAPInt();
1583
1584 for (unsigned Delta : {-2, -1, 1, 2}) {
1585 const SCEV *PreStart = getConstant(Val: StartAI - Delta);
1586
1587 FoldingSetNodeID ID;
1588 ID.AddInteger(I: scAddRecExpr);
1589 ID.AddPointer(Ptr: PreStart);
1590 ID.AddPointer(Ptr: Step);
1591 ID.AddPointer(Ptr: L);
1592 void *IP = nullptr;
1593 const auto *PreAR =
1594 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
1595
1596 // Give up if we don't already have the add recurrence we need because
1597 // actually constructing an add recurrence is relatively expensive.
1598 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType))) { // proves (2)
1599 const SCEV *DeltaS = getConstant(Ty: StartC->getType(), V: Delta);
1600 ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
1601 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1602 DeltaS, &Pred, this);
1603 if (Limit && isKnownPredicate(Pred, LHS: PreAR, RHS: Limit)) // proves (1)
1604 return true;
1605 }
1606 }
1607
1608 return false;
1609}
1610
1611// Finds an integer D for an expression (C + x + y + ...) such that the top
1612// level addition in (D + (C - D + x + y + ...)) would not wrap (signed or
1613// unsigned) and the number of trailing zeros of (C - D + x + y + ...) is
1614// maximized, where C is the \p ConstantTerm, x, y, ... are arbitrary SCEVs, and
1615// the (C + x + y + ...) expression is \p WholeAddExpr.
1616static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1617 const SCEVConstant *ConstantTerm,
1618 const SCEVAddExpr *WholeAddExpr) {
1619 const APInt &C = ConstantTerm->getAPInt();
1620 const unsigned BitWidth = C.getBitWidth();
1621 // Find number of trailing zeros of (x + y + ...) w/o the C first:
1622 uint32_t TZ = BitWidth;
1623 for (unsigned I = 1, E = WholeAddExpr->getNumOperands(); I < E && TZ; ++I)
1624 TZ = std::min(a: TZ, b: SE.getMinTrailingZeros(S: WholeAddExpr->getOperand(i: I)));
1625 if (TZ) {
1626 // Set D to be as many least significant bits of C as possible while still
1627 // guaranteeing that adding D to (C - D + x + y + ...) won't cause a wrap:
1628 return TZ < BitWidth ? C.trunc(width: TZ).zext(width: BitWidth) : C;
1629 }
1630 return APInt(BitWidth, 0);
1631}
1632
1633// Finds an integer D for an affine AddRec expression {C,+,x} such that the top
1634// level addition in (D + {C-D,+,x}) would not wrap (signed or unsigned) and the
1635// number of trailing zeros of (C - D + x * n) is maximized, where C is the \p
1636// ConstantStart, x is an arbitrary \p Step, and n is the loop trip count.
1637static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1638 const APInt &ConstantStart,
1639 const SCEV *Step) {
1640 const unsigned BitWidth = ConstantStart.getBitWidth();
1641 const uint32_t TZ = SE.getMinTrailingZeros(S: Step);
1642 if (TZ)
1643 return TZ < BitWidth ? ConstantStart.trunc(width: TZ).zext(width: BitWidth)
1644 : ConstantStart;
1645 return APInt(BitWidth, 0);
1646}
1647
1648static void insertFoldCacheEntry(
1649 const ScalarEvolution::FoldID &ID, const SCEV *S,
1650 DenseMap<ScalarEvolution::FoldID, const SCEV *> &FoldCache,
1651 DenseMap<const SCEV *, SmallVector<ScalarEvolution::FoldID, 2>>
1652 &FoldCacheUser) {
1653 auto I = FoldCache.insert(KV: {ID, S});
1654 if (!I.second) {
1655 // Remove FoldCacheUser entry for ID when replacing an existing FoldCache
1656 // entry.
1657 auto &UserIDs = FoldCacheUser[I.first->second];
1658 assert(count(UserIDs, ID) == 1 && "unexpected duplicates in UserIDs");
1659 for (unsigned I = 0; I != UserIDs.size(); ++I)
1660 if (UserIDs[I] == ID) {
1661 std::swap(a&: UserIDs[I], b&: UserIDs.back());
1662 break;
1663 }
1664 UserIDs.pop_back();
1665 I.first->second = S;
1666 }
1667 FoldCacheUser[S].push_back(Elt: ID);
1668}
1669
1670const SCEV *
1671ScalarEvolution::getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) {
1672 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1673 "This is not an extending conversion!");
1674 assert(isSCEVable(Ty) &&
1675 "This is not a conversion to a SCEVable type!");
1676 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1677 Ty = getEffectiveSCEVType(Ty);
1678
1679 FoldID ID(scZeroExtend, Op, Ty);
1680 if (const SCEV *S = FoldCache.lookup(Val: ID))
1681 return S;
1682
1683 const SCEV *S = getZeroExtendExprImpl(Op, Ty, Depth);
1684 if (!isa<SCEVZeroExtendExpr>(Val: S))
1685 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1686 return S;
1687}
1688
1689const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
1690 unsigned Depth) {
1691 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1692 "This is not an extending conversion!");
1693 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1694 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1695
1696 // Fold if the operand is constant.
1697 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
1698 return getConstant(Val: SC->getAPInt().zext(width: getTypeSizeInBits(Ty)));
1699
1700 // zext(zext(x)) --> zext(x)
1701 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val: Op))
1702 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1703
1704 // Before doing any expensive analysis, check to see if we've already
1705 // computed a SCEV for this Op and Ty.
1706 FoldingSetNodeID ID;
1707 ID.AddInteger(I: scZeroExtend);
1708 ID.AddPointer(Ptr: Op);
1709 ID.AddPointer(Ptr: Ty);
1710 void *IP = nullptr;
1711 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
1712 if (Depth > MaxCastDepth) {
1713 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1714 Op, Ty);
1715 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1716 S->computeAndSetCanonical(SE&: *this);
1717 registerUser(User: S, Ops: Op);
1718 return S;
1719 }
1720
1721 // zext(trunc(x)) --> zext(x) or x or trunc(x)
1722 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
1723 // It's possible the bits taken off by the truncate were all zero bits. If
1724 // so, we should be able to simplify this further.
1725 const SCEV *X = ST->getOperand();
1726 ConstantRange CR = getUnsignedRange(S: X);
1727 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1728 unsigned NewBits = getTypeSizeInBits(Ty);
1729 if (CR.truncate(BitWidth: TruncBits).zeroExtend(BitWidth: NewBits).contains(
1730 CR: CR.zextOrTrunc(BitWidth: NewBits)))
1731 return getTruncateOrZeroExtend(V: X, Ty, Depth);
1732 }
1733
1734 // If the input value is a chrec scev, and we can prove that the value
1735 // did not overflow the old, smaller, value, we can zero extend all of the
1736 // operands (often constants). This allows analysis of something like
1737 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
1738 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Op))
1739 if (AR->isAffine()) {
1740 const SCEV *Start = AR->getStart();
1741 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
1742 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
1743 const Loop *L = AR->getLoop();
1744
1745 // If we have special knowledge that this addrec won't overflow,
1746 // we don't need to do any further analysis.
1747 if (AR->hasNoUnsignedWrap()) {
1748 Start =
1749 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1750 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1751 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1752 }
1753
1754 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1755 // Note that this serves two purposes: It filters out loops that are
1756 // simply not analyzable, and it covers the case where this code is
1757 // being called from within backedge-taken count analysis, such that
1758 // attempting to ask for the backedge-taken count would likely result
1759 // in infinite recursion. In the later case, the analysis code will
1760 // cope with a conservative value, and it will take care to purge
1761 // that value once it has finished.
1762 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1763 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
1764 // Manually compute the final value for AR, checking for overflow.
1765
1766 // Check whether the backedge-taken count can be losslessly casted to
1767 // the addrec's type. The count is always unsigned.
1768 const SCEV *CastedMaxBECount =
1769 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
1770 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1771 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
1772 if (MaxBECount == RecastedMaxBECount) {
1773 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
1774 // Check whether Start+Step*MaxBECount has no unsigned overflow.
1775 const SCEV *ZMul = getMulExpr(LHS: CastedMaxBECount, RHS: Step,
1776 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1777 const SCEV *ZAdd = getZeroExtendExpr(Op: getAddExpr(LHS: Start, RHS: ZMul,
1778 Flags: SCEV::FlagAnyWrap,
1779 Depth: Depth + 1),
1780 Ty: WideTy, Depth: Depth + 1);
1781 const SCEV *WideStart = getZeroExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
1782 const SCEV *WideMaxBECount =
1783 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
1784 const SCEV *OperandExtendedAdd =
1785 getAddExpr(LHS: WideStart,
1786 RHS: getMulExpr(LHS: WideMaxBECount,
1787 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1788 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
1789 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1790 if (ZAdd == OperandExtendedAdd) {
1791 // Cache knowledge of AR NUW, which is propagated to this AddRec.
1792 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1793 // Return the expression with the addrec on the outside.
1794 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this,
1795 Depth: Depth + 1);
1796 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1797 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1798 }
1799 // Similar to above, only this time treat the step value as signed.
1800 // This covers loops that count down.
1801 OperandExtendedAdd =
1802 getAddExpr(LHS: WideStart,
1803 RHS: getMulExpr(LHS: WideMaxBECount,
1804 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1805 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
1806 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1807 if (ZAdd == OperandExtendedAdd) {
1808 // Cache knowledge of AR NW, which is propagated to this AddRec.
1809 // Negative step causes unsigned wrap, but it still can't self-wrap.
1810 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1811 // Return the expression with the addrec on the outside.
1812 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this,
1813 Depth: Depth + 1);
1814 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1815 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1816 }
1817 }
1818 }
1819
1820 // Normally, in the cases we can prove no-overflow via a
1821 // backedge guarding condition, we can also compute a backedge
1822 // taken count for the loop. The exceptions are assumptions and
1823 // guards present in the loop -- SCEV is not great at exploiting
1824 // these to compute max backedge taken counts, but can still use
1825 // these to prove lack of overflow. Use this fact to avoid
1826 // doing extra work that may not pay off.
1827 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount) || HasGuards ||
1828 !AC.assumptions().empty()) {
1829
1830 auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
1831 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
1832 if (AR->hasNoUnsignedWrap()) {
1833 // Same as nuw case above - duplicated here to avoid a compile time
1834 // issue. It's not clear that the order of checks does matter, but
1835 // it's one of two issue possible causes for a change which was
1836 // reverted. Be conservative for the moment.
1837 Start =
1838 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1839 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1840 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1841 }
1842
1843 // For a negative step, we can extend the operands iff doing so only
1844 // traverses values in the range zext([0,UINT_MAX]).
1845 if (isKnownNegative(S: Step)) {
1846 const SCEV *N = getConstant(Val: APInt::getMaxValue(numBits: BitWidth) -
1847 getSignedRangeMin(S: Step));
1848 if (isLoopBackedgeGuardedByCond(L, Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N) ||
1849 isKnownOnEveryIteration(Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N)) {
1850 // Cache knowledge of AR NW, which is propagated to this
1851 // AddRec. Negative step causes unsigned wrap, but it
1852 // still can't self-wrap.
1853 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1854 // Return the expression with the addrec on the outside.
1855 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this,
1856 Depth: Depth + 1);
1857 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1858 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1859 }
1860 }
1861 }
1862
1863 // zext({C,+,Step}) --> (zext(D) + zext({C-D,+,Step}))<nuw><nsw>
1864 // if D + (C - D + Step * n) could be proven to not unsigned wrap
1865 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1866 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
1867 const APInt &C = SC->getAPInt();
1868 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
1869 if (D != 0) {
1870 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1871 const SCEV *SResidual =
1872 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
1873 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1874 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: SCEV::FlagNSW | SCEV::FlagNUW,
1875 Depth: Depth + 1);
1876 }
1877 }
1878
1879 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1880 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1881 Start =
1882 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1883 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1884 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1885 }
1886 }
1887
1888 // zext(A % B) --> zext(A) % zext(B)
1889 {
1890 const SCEV *LHS;
1891 const SCEV *RHS;
1892 if (match(S: Op, P: m_scev_URem(LHS: m_SCEV(V&: LHS), RHS: m_SCEV(V&: RHS), SE&: *this)))
1893 return getURemExpr(LHS: getZeroExtendExpr(Op: LHS, Ty, Depth: Depth + 1),
1894 RHS: getZeroExtendExpr(Op: RHS, Ty, Depth: Depth + 1));
1895 }
1896
1897 // zext(A / B) --> zext(A) / zext(B).
1898 if (auto *Div = dyn_cast<SCEVUDivExpr>(Val: Op))
1899 return getUDivExpr(LHS: getZeroExtendExpr(Op: Div->getLHS(), Ty, Depth: Depth + 1),
1900 RHS: getZeroExtendExpr(Op: Div->getRHS(), Ty, Depth: Depth + 1));
1901
1902 if (auto *SA = dyn_cast<SCEVAddExpr>(Val: Op)) {
1903 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw>
1904 if (SA->hasNoUnsignedWrap()) {
1905 // If the addition does not unsign overflow then we can, by definition,
1906 // commute the zero extension with the addition operation.
1907 SmallVector<SCEVUse, 4> Ops;
1908 for (SCEVUse Op : SA->operands())
1909 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1910 return getAddExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1911 }
1912
1913 const APInt *C, *C2;
1914 // zext (C + A)<nsw> -> (sext(C) + sext(A))<nsw> if zext (C + A)<nsw> >=s 0.
1915 // Currently the non-negative check is done manually, as isKnownNonNegative
1916 // is too expensive.
1917 if (SA->hasNoSignedWrap() &&
1918 match(V: SA, P: m_scev_Add(Op0: m_scev_APInt(C),
1919 Op1: m_scev_SMax(Op0: m_scev_APInt(C&: C2), Op1: m_SCEV()))) &&
1920 C->isNegative() && !C->isMinSignedValue() && C2->sge(RHS: C->abs())) {
1921 assert(isKnownNonNegative(SA) && "incorrectly determined non-negative");
1922 return getAddExpr(LHS: getSignExtendExpr(Op: SA->getOperand(i: 0), Ty, Depth: Depth + 1),
1923 RHS: getSignExtendExpr(Op: SA->getOperand(i: 1), Ty, Depth: Depth + 1),
1924 Flags: SCEV::FlagNSW, Depth: Depth + 1);
1925 }
1926
1927 // zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
1928 // if D + (C - D + x + y + ...) could be proven to not unsigned wrap
1929 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1930 //
1931 // Often address arithmetics contain expressions like
1932 // (zext (add (shl X, C1), C2)), for instance, (zext (5 + (4 * X))).
1933 // This transformation is useful while proving that such expressions are
1934 // equal or differ by a small constant amount, see LoadStoreVectorizer pass.
1935 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1936 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1937 if (D != 0) {
1938 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1939 const SCEV *SResidual =
1940 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagAnyWrap, Depth);
1941 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1942 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
1943 Depth: Depth + 1);
1944 }
1945 }
1946 }
1947
1948 if (auto *SM = dyn_cast<SCEVMulExpr>(Val: Op)) {
1949 // zext((A * B * ...)<nuw>) --> (zext(A) * zext(B) * ...)<nuw>
1950 if (SM->hasNoUnsignedWrap()) {
1951 // If the multiply does not unsign overflow then we can, by definition,
1952 // commute the zero extension with the multiply operation.
1953 SmallVector<SCEVUse, 4> Ops;
1954 for (SCEVUse Op : SM->operands())
1955 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1956 return getMulExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1957 }
1958
1959 // zext(2^K * (trunc X to iN)) to iM ->
1960 // 2^K * (zext(trunc X to i{N-K}) to iM)<nuw>
1961 //
1962 // Proof:
1963 //
1964 // zext(2^K * (trunc X to iN)) to iM
1965 // = zext((trunc X to iN) << K) to iM
1966 // = zext((trunc X to i{N-K}) << K)<nuw> to iM
1967 // (because shl removes the top K bits)
1968 // = zext((2^K * (trunc X to i{N-K}))<nuw>) to iM
1969 // = (2^K * (zext(trunc X to i{N-K}) to iM))<nuw>.
1970 //
1971 const APInt *C;
1972 const SCEV *TruncRHS;
1973 if (match(V: SM,
1974 P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_scev_Trunc(Op0: m_SCEV(V&: TruncRHS)))) &&
1975 C->isPowerOf2()) {
1976 int NewTruncBits =
1977 getTypeSizeInBits(Ty: SM->getOperand(i: 1)->getType()) - C->logBase2();
1978 Type *NewTruncTy = IntegerType::get(C&: getContext(), NumBits: NewTruncBits);
1979 return getMulExpr(
1980 LHS: getZeroExtendExpr(Op: SM->getOperand(i: 0), Ty),
1981 RHS: getZeroExtendExpr(Op: getTruncateExpr(Op: TruncRHS, Ty: NewTruncTy), Ty),
1982 Flags: SCEV::FlagNUW, Depth: Depth + 1);
1983 }
1984 }
1985
1986 // zext(umin(x, y)) -> umin(zext(x), zext(y))
1987 // zext(umax(x, y)) -> umax(zext(x), zext(y))
1988 if (isa<SCEVUMinExpr>(Val: Op) || isa<SCEVUMaxExpr>(Val: Op)) {
1989 auto *MinMax = cast<SCEVMinMaxExpr>(Val: Op);
1990 SmallVector<SCEVUse, 4> Operands;
1991 for (SCEVUse Operand : MinMax->operands())
1992 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1993 if (isa<SCEVUMinExpr>(Val: MinMax))
1994 return getUMinExpr(Operands);
1995 return getUMaxExpr(Operands);
1996 }
1997
1998 // zext(umin_seq(x, y)) -> umin_seq(zext(x), zext(y))
1999 if (auto *MinMax = dyn_cast<SCEVSequentialMinMaxExpr>(Val: Op)) {
2000 assert(isa<SCEVSequentialUMinExpr>(MinMax) && "Not supported!");
2001 SmallVector<SCEVUse, 4> Operands;
2002 for (SCEVUse Operand : MinMax->operands())
2003 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
2004 return getUMinExpr(Operands, /*Sequential*/ true);
2005 }
2006
2007 // The cast wasn't folded; create an explicit cast node.
2008 // Recompute the insert position, as it may have been invalidated.
2009 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
2010 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
2011 Op, Ty);
2012 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
2013 S->computeAndSetCanonical(SE&: *this);
2014 registerUser(User: S, Ops: Op);
2015 return S;
2016}
2017
2018const SCEV *
2019ScalarEvolution::getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) {
2020 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2021 "This is not an extending conversion!");
2022 assert(isSCEVable(Ty) &&
2023 "This is not a conversion to a SCEVable type!");
2024 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
2025 Ty = getEffectiveSCEVType(Ty);
2026
2027 FoldID ID(scSignExtend, Op, Ty);
2028 if (const SCEV *S = FoldCache.lookup(Val: ID))
2029 return S;
2030
2031 const SCEV *S = getSignExtendExprImpl(Op, Ty, Depth);
2032 if (!isa<SCEVSignExtendExpr>(Val: S))
2033 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
2034 return S;
2035}
2036
2037const SCEV *ScalarEvolution::getSignExtendExprImpl(const SCEV *Op, Type *Ty,
2038 unsigned Depth) {
2039 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2040 "This is not an extending conversion!");
2041 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
2042 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
2043 Ty = getEffectiveSCEVType(Ty);
2044
2045 // Fold if the operand is constant.
2046 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
2047 return getConstant(Val: SC->getAPInt().sext(width: getTypeSizeInBits(Ty)));
2048
2049 // sext(sext(x)) --> sext(x)
2050 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val: Op))
2051 return getSignExtendExpr(Op: SS->getOperand(), Ty, Depth: Depth + 1);
2052
2053 // sext(zext(x)) --> zext(x)
2054 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val: Op))
2055 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
2056
2057 // Before doing any expensive analysis, check to see if we've already
2058 // computed a SCEV for this Op and Ty.
2059 FoldingSetNodeID ID;
2060 ID.AddInteger(I: scSignExtend);
2061 ID.AddPointer(Ptr: Op);
2062 ID.AddPointer(Ptr: Ty);
2063 void *IP = nullptr;
2064 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
2065 // Limit recursion depth.
2066 if (Depth > MaxCastDepth) {
2067 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
2068 Op, Ty);
2069 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
2070 S->computeAndSetCanonical(SE&: *this);
2071 registerUser(User: S, Ops: Op);
2072 return S;
2073 }
2074
2075 // sext(trunc(x)) --> sext(x) or x or trunc(x)
2076 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2077 // It's possible the bits taken off by the truncate were all sign bits. If
2078 // so, we should be able to simplify this further.
2079 const SCEV *X = ST->getOperand();
2080 ConstantRange CR = getSignedRange(S: X);
2081 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
2082 unsigned NewBits = getTypeSizeInBits(Ty);
2083 if (CR.truncate(BitWidth: TruncBits).signExtend(BitWidth: NewBits).contains(
2084 CR: CR.sextOrTrunc(BitWidth: NewBits)))
2085 return getTruncateOrSignExtend(V: X, Ty, Depth);
2086 }
2087
2088 if (auto *SA = dyn_cast<SCEVAddExpr>(Val: Op)) {
2089 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
2090 if (SA->hasNoSignedWrap()) {
2091 // If the addition does not sign overflow then we can, by definition,
2092 // commute the sign extension with the addition operation.
2093 SmallVector<SCEVUse, 4> Ops;
2094 for (SCEVUse Op : SA->operands())
2095 Ops.push_back(Elt: getSignExtendExpr(Op, Ty, Depth: Depth + 1));
2096 return getAddExpr(Ops, Flags: SCEV::FlagNSW, Depth: Depth + 1);
2097 }
2098
2099 // sext(C + x + y + ...) --> (sext(D) + sext((C - D) + x + y + ...))
2100 // if D + (C - D + x + y + ...) could be proven to not signed wrap
2101 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
2102 //
2103 // For instance, this will bring two seemingly different expressions:
2104 // 1 + sext(5 + 20 * %x + 24 * %y) and
2105 // sext(6 + 20 * %x + 24 * %y)
2106 // to the same form:
2107 // 2 + sext(4 + 20 * %x + 24 * %y)
2108 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
2109 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
2110 if (D != 0) {
2111 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
2112 const SCEV *SResidual =
2113 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagAnyWrap, Depth);
2114 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
2115 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
2116 Depth: Depth + 1);
2117 }
2118 }
2119 }
2120 // If the input value is a chrec scev, and we can prove that the value
2121 // did not overflow the old, smaller, value, we can sign extend all of the
2122 // operands (often constants). This allows analysis of something like
2123 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; }
2124 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Op))
2125 if (AR->isAffine()) {
2126 const SCEV *Start = AR->getStart();
2127 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
2128 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
2129 const Loop *L = AR->getLoop();
2130
2131 // If we have special knowledge that this addrec won't overflow,
2132 // we don't need to do any further analysis.
2133 if (AR->hasNoSignedWrap()) {
2134 Start =
2135 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2136 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2137 return getAddRecExpr(Start, Step, L, Flags: SCEV::FlagNSW);
2138 }
2139
2140 // Check whether the backedge-taken count is SCEVCouldNotCompute.
2141 // Note that this serves two purposes: It filters out loops that are
2142 // simply not analyzable, and it covers the case where this code is
2143 // being called from within backedge-taken count analysis, such that
2144 // attempting to ask for the backedge-taken count would likely result
2145 // in infinite recursion. In the later case, the analysis code will
2146 // cope with a conservative value, and it will take care to purge
2147 // that value once it has finished.
2148 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
2149 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
2150 // Manually compute the final value for AR, checking for
2151 // overflow.
2152
2153 // Check whether the backedge-taken count can be losslessly casted to
2154 // the addrec's type. The count is always unsigned.
2155 const SCEV *CastedMaxBECount =
2156 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
2157 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
2158 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
2159 if (MaxBECount == RecastedMaxBECount) {
2160 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
2161 // Check whether Start+Step*MaxBECount has no signed overflow.
2162 const SCEV *SMul = getMulExpr(LHS: CastedMaxBECount, RHS: Step,
2163 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2164 const SCEV *SAdd = getSignExtendExpr(Op: getAddExpr(LHS: Start, RHS: SMul,
2165 Flags: SCEV::FlagAnyWrap,
2166 Depth: Depth + 1),
2167 Ty: WideTy, Depth: Depth + 1);
2168 const SCEV *WideStart = getSignExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
2169 const SCEV *WideMaxBECount =
2170 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
2171 const SCEV *OperandExtendedAdd =
2172 getAddExpr(LHS: WideStart,
2173 RHS: getMulExpr(LHS: WideMaxBECount,
2174 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2175 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2176 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2177 if (SAdd == OperandExtendedAdd) {
2178 // Cache knowledge of AR NSW, which is propagated to this AddRec.
2179 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2180 // Return the expression with the addrec on the outside.
2181 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this,
2182 Depth: Depth + 1);
2183 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2184 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2185 }
2186 // Similar to above, only this time treat the step value as unsigned.
2187 // This covers loops that count up with an unsigned step.
2188 OperandExtendedAdd =
2189 getAddExpr(LHS: WideStart,
2190 RHS: getMulExpr(LHS: WideMaxBECount,
2191 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2192 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2193 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2194 if (SAdd == OperandExtendedAdd) {
2195 // If AR wraps around then
2196 //
2197 // abs(Step) * MaxBECount > unsigned-max(AR->getType())
2198 // => SAdd != OperandExtendedAdd
2199 //
2200 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
2201 // (SAdd == OperandExtendedAdd => AR is NW)
2202
2203 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
2204
2205 // Return the expression with the addrec on the outside.
2206 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this,
2207 Depth: Depth + 1);
2208 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2209 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2210 }
2211 }
2212 }
2213
2214 auto NewFlags = proveNoSignedWrapViaInduction(AR);
2215 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
2216 if (AR->hasNoSignedWrap()) {
2217 // Same as nsw case above - duplicated here to avoid a compile time
2218 // issue. It's not clear that the order of checks does matter, but
2219 // it's one of two issue possible causes for a change which was
2220 // reverted. Be conservative for the moment.
2221 Start =
2222 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2223 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2224 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2225 }
2226
2227 // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw>
2228 // if D + (C - D + Step * n) could be proven to not signed wrap
2229 // where D maximizes the number of trailing zeros of (C - D + Step * n)
2230 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
2231 const APInt &C = SC->getAPInt();
2232 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
2233 if (D != 0) {
2234 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
2235 const SCEV *SResidual =
2236 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
2237 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
2238 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
2239 Depth: Depth + 1);
2240 }
2241 }
2242
2243 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
2244 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2245 Start =
2246 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2247 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2248 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2249 }
2250 }
2251
2252 // If the input value is provably positive and we could not simplify
2253 // away the sext build a zext instead.
2254 if (isKnownNonNegative(S: Op))
2255 return getZeroExtendExpr(Op, Ty, Depth: Depth + 1);
2256
2257 // sext(smin(x, y)) -> smin(sext(x), sext(y))
2258 // sext(smax(x, y)) -> smax(sext(x), sext(y))
2259 if (isa<SCEVSMinExpr>(Val: Op) || isa<SCEVSMaxExpr>(Val: Op)) {
2260 auto *MinMax = cast<SCEVMinMaxExpr>(Val: Op);
2261 SmallVector<SCEVUse, 4> Operands;
2262 for (SCEVUse Operand : MinMax->operands())
2263 Operands.push_back(Elt: getSignExtendExpr(Op: Operand, Ty));
2264 if (isa<SCEVSMinExpr>(Val: MinMax))
2265 return getSMinExpr(Operands);
2266 return getSMaxExpr(Operands);
2267 }
2268
2269 // The cast wasn't folded; create an explicit cast node.
2270 // Recompute the insert position, as it may have been invalidated.
2271 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
2272 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
2273 Op, Ty);
2274 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
2275 S->computeAndSetCanonical(SE&: *this);
2276 registerUser(User: S, Ops: Op);
2277 return S;
2278}
2279
2280const SCEV *ScalarEvolution::getCastExpr(SCEVTypes Kind, const SCEV *Op,
2281 Type *Ty) {
2282 switch (Kind) {
2283 case scTruncate:
2284 return getTruncateExpr(Op, Ty);
2285 case scZeroExtend:
2286 return getZeroExtendExpr(Op, Ty);
2287 case scSignExtend:
2288 return getSignExtendExpr(Op, Ty);
2289 case scPtrToInt:
2290 return getPtrToIntExpr(Op, Ty);
2291 default:
2292 llvm_unreachable("Not a SCEV cast expression!");
2293 }
2294}
2295
2296/// getAnyExtendExpr - Return a SCEV for the given operand extended with
2297/// unspecified bits out to the given type.
2298const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op,
2299 Type *Ty) {
2300 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2301 "This is not an extending conversion!");
2302 assert(isSCEVable(Ty) &&
2303 "This is not a conversion to a SCEVable type!");
2304 Ty = getEffectiveSCEVType(Ty);
2305
2306 // Sign-extend negative constants.
2307 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
2308 if (SC->getAPInt().isNegative())
2309 return getSignExtendExpr(Op, Ty);
2310
2311 // Peel off a truncate cast.
2312 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2313 const SCEV *NewOp = T->getOperand();
2314 if (getTypeSizeInBits(Ty: NewOp->getType()) < getTypeSizeInBits(Ty))
2315 return getAnyExtendExpr(Op: NewOp, Ty);
2316 return getTruncateOrNoop(V: NewOp, Ty);
2317 }
2318
2319 // Next try a zext cast. If the cast is folded, use it.
2320 const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
2321 if (!isa<SCEVZeroExtendExpr>(Val: ZExt))
2322 return ZExt;
2323
2324 // Next try a sext cast. If the cast is folded, use it.
2325 const SCEV *SExt = getSignExtendExpr(Op, Ty);
2326 if (!isa<SCEVSignExtendExpr>(Val: SExt))
2327 return SExt;
2328
2329 // Force the cast to be folded into the operands of an addrec.
2330 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Op)) {
2331 SmallVector<SCEVUse, 4> Ops;
2332 for (const SCEV *Op : AR->operands())
2333 Ops.push_back(Elt: getAnyExtendExpr(Op, Ty));
2334 return getAddRecExpr(Operands&: Ops, L: AR->getLoop(), Flags: SCEV::FlagNW);
2335 }
2336
2337 // If the expression is obviously signed, use the sext cast value.
2338 if (isa<SCEVSMaxExpr>(Val: Op))
2339 return SExt;
2340
2341 // Absent any other information, use the zext cast value.
2342 return ZExt;
2343}
2344
2345/// Process the given Ops list, which is a list of operands to be added under
2346/// the given scale, update the given map. This is a helper function for
2347/// getAddRecExpr. As an example of what it does, given a sequence of operands
2348/// that would form an add expression like this:
2349///
2350/// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
2351///
2352/// where A and B are constants, update the map with these values:
2353///
2354/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
2355///
2356/// and add 13 + A*B*29 to AccumulatedConstant.
2357/// This will allow getAddRecExpr to produce this:
2358///
2359/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
2360///
2361/// This form often exposes folding opportunities that are hidden in
2362/// the original operand list.
2363///
2364/// Return true iff it appears that any interesting folding opportunities
2365/// may be exposed. This helps getAddRecExpr short-circuit extra work in
2366/// the common case where no interesting opportunities are present, and
2367/// is also used as a check to avoid infinite recursion.
2368static bool CollectAddOperandsWithScales(SmallDenseMap<SCEVUse, APInt, 16> &M,
2369 SmallVectorImpl<SCEVUse> &NewOps,
2370 APInt &AccumulatedConstant,
2371 ArrayRef<SCEVUse> Ops,
2372 const APInt &Scale,
2373 ScalarEvolution &SE) {
2374 bool Interesting = false;
2375
2376 // Iterate over the add operands. They are sorted, with constants first.
2377 unsigned i = 0;
2378 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Ops[i])) {
2379 ++i;
2380 // Pull a buried constant out to the outside.
2381 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
2382 Interesting = true;
2383 AccumulatedConstant += Scale * C->getAPInt();
2384 }
2385
2386 // Next comes everything else. We're especially interested in multiplies
2387 // here, but they're in the middle, so just visit the rest with one loop.
2388 for (; i != Ops.size(); ++i) {
2389 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: Ops[i]);
2390 if (Mul && isa<SCEVConstant>(Val: Mul->getOperand(i: 0))) {
2391 APInt NewScale =
2392 Scale * cast<SCEVConstant>(Val: Mul->getOperand(i: 0))->getAPInt();
2393 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Val: Mul->getOperand(i: 1))) {
2394 // A multiplication of a constant with another add; recurse.
2395 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: Mul->getOperand(i: 1));
2396 Interesting |= CollectAddOperandsWithScales(
2397 M, NewOps, AccumulatedConstant, Ops: Add->operands(), Scale: NewScale, SE);
2398 } else {
2399 // A multiplication of a constant with some other value. Update
2400 // the map.
2401 SmallVector<SCEVUse, 4> MulOps(drop_begin(RangeOrContainer: Mul->operands()));
2402 const SCEV *Key = SE.getMulExpr(Ops&: MulOps);
2403 auto Pair = M.insert(KV: {Key, NewScale});
2404 if (Pair.second) {
2405 NewOps.push_back(Elt: Pair.first->first);
2406 } else {
2407 Pair.first->second += NewScale;
2408 // The map already had an entry for this value, which may indicate
2409 // a folding opportunity.
2410 Interesting = true;
2411 }
2412 }
2413 } else {
2414 // An ordinary operand. Update the map.
2415 auto Pair = M.insert(KV: {Ops[i], Scale});
2416 if (Pair.second) {
2417 NewOps.push_back(Elt: Pair.first->first);
2418 } else {
2419 Pair.first->second += Scale;
2420 // The map already had an entry for this value, which may indicate
2421 // a folding opportunity.
2422 Interesting = true;
2423 }
2424 }
2425 }
2426
2427 return Interesting;
2428}
2429
2430bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed,
2431 const SCEV *LHS, const SCEV *RHS,
2432 const Instruction *CtxI) {
2433 const SCEV *(ScalarEvolution::*Operation)(SCEVUse, SCEVUse, SCEV::NoWrapFlags,
2434 unsigned);
2435 switch (BinOp) {
2436 default:
2437 llvm_unreachable("Unsupported binary op");
2438 case Instruction::Add:
2439 Operation = &ScalarEvolution::getAddExpr;
2440 break;
2441 case Instruction::Sub:
2442 Operation = &ScalarEvolution::getMinusSCEV;
2443 break;
2444 case Instruction::Mul:
2445 Operation = &ScalarEvolution::getMulExpr;
2446 break;
2447 }
2448
2449 const SCEV *(ScalarEvolution::*Extension)(const SCEV *, Type *, unsigned) =
2450 Signed ? &ScalarEvolution::getSignExtendExpr
2451 : &ScalarEvolution::getZeroExtendExpr;
2452
2453 // Check ext(LHS op RHS) == ext(LHS) op ext(RHS)
2454 auto *NarrowTy = cast<IntegerType>(Val: LHS->getType());
2455 auto *WideTy =
2456 IntegerType::get(C&: NarrowTy->getContext(), NumBits: NarrowTy->getBitWidth() * 2);
2457
2458 const SCEV *A = (this->*Extension)(
2459 (this->*Operation)(LHS, RHS, SCEV::FlagAnyWrap, 0), WideTy, 0);
2460 const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0);
2461 const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0);
2462 const SCEV *B = (this->*Operation)(LHSB, RHSB, SCEV::FlagAnyWrap, 0);
2463 if (A == B)
2464 return true;
2465 // Can we use context to prove the fact we need?
2466 if (!CtxI)
2467 return false;
2468 // TODO: Support mul.
2469 if (BinOp == Instruction::Mul)
2470 return false;
2471 auto *RHSC = dyn_cast<SCEVConstant>(Val: RHS);
2472 // TODO: Lift this limitation.
2473 if (!RHSC)
2474 return false;
2475 APInt C = RHSC->getAPInt();
2476 unsigned NumBits = C.getBitWidth();
2477 bool IsSub = (BinOp == Instruction::Sub);
2478 bool IsNegativeConst = (Signed && C.isNegative());
2479 // Compute the direction and magnitude by which we need to check overflow.
2480 bool OverflowDown = IsSub ^ IsNegativeConst;
2481 APInt Magnitude = C;
2482 if (IsNegativeConst) {
2483 if (C == APInt::getSignedMinValue(numBits: NumBits))
2484 // TODO: SINT_MIN on inversion gives the same negative value, we don't
2485 // want to deal with that.
2486 return false;
2487 Magnitude = -C;
2488 }
2489
2490 ICmpInst::Predicate Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2491 if (OverflowDown) {
2492 // To avoid overflow down, we need to make sure that MIN + Magnitude <= LHS.
2493 APInt Min = Signed ? APInt::getSignedMinValue(numBits: NumBits)
2494 : APInt::getMinValue(numBits: NumBits);
2495 APInt Limit = Min + Magnitude;
2496 return isKnownPredicateAt(Pred, LHS: getConstant(Val: Limit), RHS: LHS, CtxI);
2497 } else {
2498 // To avoid overflow up, we need to make sure that LHS <= MAX - Magnitude.
2499 APInt Max = Signed ? APInt::getSignedMaxValue(numBits: NumBits)
2500 : APInt::getMaxValue(numBits: NumBits);
2501 APInt Limit = Max - Magnitude;
2502 return isKnownPredicateAt(Pred, LHS, RHS: getConstant(Val: Limit), CtxI);
2503 }
2504}
2505
2506std::optional<SCEV::NoWrapFlags>
2507ScalarEvolution::getStrengthenedNoWrapFlagsFromBinOp(
2508 const OverflowingBinaryOperator *OBO) {
2509 // It cannot be done any better.
2510 if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap())
2511 return std::nullopt;
2512
2513 SCEV::NoWrapFlags Flags = SCEV::NoWrapFlags::FlagAnyWrap;
2514
2515 if (OBO->hasNoUnsignedWrap())
2516 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2517 if (OBO->hasNoSignedWrap())
2518 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2519
2520 bool Deduced = false;
2521
2522 if (OBO->getOpcode() != Instruction::Add &&
2523 OBO->getOpcode() != Instruction::Sub &&
2524 OBO->getOpcode() != Instruction::Mul)
2525 return std::nullopt;
2526
2527 const SCEV *LHS = getSCEV(V: OBO->getOperand(i_nocapture: 0));
2528 const SCEV *RHS = getSCEV(V: OBO->getOperand(i_nocapture: 1));
2529
2530 const Instruction *CtxI =
2531 UseContextForNoWrapFlagInference ? dyn_cast<Instruction>(Val: OBO) : nullptr;
2532 if (!OBO->hasNoUnsignedWrap() &&
2533 willNotOverflow(BinOp: (Instruction::BinaryOps)OBO->getOpcode(),
2534 /* Signed */ false, LHS, RHS, CtxI)) {
2535 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2536 Deduced = true;
2537 }
2538
2539 if (!OBO->hasNoSignedWrap() &&
2540 willNotOverflow(BinOp: (Instruction::BinaryOps)OBO->getOpcode(),
2541 /* Signed */ true, LHS, RHS, CtxI)) {
2542 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2543 Deduced = true;
2544 }
2545
2546 if (Deduced)
2547 return Flags;
2548 return std::nullopt;
2549}
2550
2551// We're trying to construct a SCEV of type `Type' with `Ops' as operands and
2552// `OldFlags' as can't-wrap behavior. Infer a more aggressive set of
2553// can't-overflow flags for the operation if possible.
2554static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE,
2555 SCEVTypes Type,
2556 ArrayRef<SCEVUse> Ops,
2557 SCEV::NoWrapFlags Flags) {
2558 using namespace std::placeholders;
2559
2560 using OBO = OverflowingBinaryOperator;
2561
2562 bool CanAnalyze =
2563 Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr;
2564 (void)CanAnalyze;
2565 assert(CanAnalyze && "don't call from other places!");
2566
2567 SCEV::NoWrapFlags SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
2568 SCEV::NoWrapFlags SignOrUnsignWrap =
2569 ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2570
2571 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
2572 auto IsKnownNonNegative = [&](SCEVUse U) {
2573 return SE->isKnownNonNegative(S: U);
2574 };
2575
2576 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Range&: Ops, P: IsKnownNonNegative))
2577 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SignOrUnsignMask);
2578
2579 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2580
2581 if (SignOrUnsignWrap != SignOrUnsignMask &&
2582 (Type == scAddExpr || Type == scMulExpr) && Ops.size() == 2 &&
2583 isa<SCEVConstant>(Val: Ops[0])) {
2584
2585 auto Opcode = [&] {
2586 switch (Type) {
2587 case scAddExpr:
2588 return Instruction::Add;
2589 case scMulExpr:
2590 return Instruction::Mul;
2591 default:
2592 llvm_unreachable("Unexpected SCEV op.");
2593 }
2594 }();
2595
2596 const APInt &C = cast<SCEVConstant>(Val: Ops[0])->getAPInt();
2597
2598 // (A <opcode> C) --> (A <opcode> C)<nsw> if the op doesn't sign overflow.
2599 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) {
2600 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2601 BinOp: Opcode, Other: C, NoWrapKind: OBO::NoSignedWrap);
2602 if (NSWRegion.contains(CR: SE->getSignedRange(S: Ops[1])))
2603 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2604 }
2605
2606 // (A <opcode> C) --> (A <opcode> C)<nuw> if the op doesn't unsign overflow.
2607 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) {
2608 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2609 BinOp: Opcode, Other: C, NoWrapKind: OBO::NoUnsignedWrap);
2610 if (NUWRegion.contains(CR: SE->getUnsignedRange(S: Ops[1])))
2611 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2612 }
2613 }
2614
2615 // <0,+,nonnegative><nw> is also nuw
2616 // TODO: Add corresponding nsw case
2617 if (Type == scAddRecExpr && ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNW) &&
2618 !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) && Ops.size() == 2 &&
2619 Ops[0]->isZero() && IsKnownNonNegative(Ops[1]))
2620 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2621
2622 // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW
2623 if (Type == scMulExpr && !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) &&
2624 Ops.size() == 2) {
2625 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[0]))
2626 if (UDiv->getOperand(i: 1) == Ops[1])
2627 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2628 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[1]))
2629 if (UDiv->getOperand(i: 1) == Ops[0])
2630 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2631 }
2632
2633 return Flags;
2634}
2635
2636bool ScalarEvolution::isAvailableAtLoopEntry(const SCEV *S, const Loop *L) {
2637 return isLoopInvariant(S, L) && properlyDominates(S, BB: L->getHeader());
2638}
2639
2640/// Get a canonical add expression, or something simpler if possible.
2641const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<SCEVUse> &Ops,
2642 SCEV::NoWrapFlags OrigFlags,
2643 unsigned Depth) {
2644 assert(!(OrigFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2645 "only nuw or nsw allowed");
2646 assert(!Ops.empty() && "Cannot get empty add!");
2647 if (Ops.size() == 1) return Ops[0];
2648#ifndef NDEBUG
2649 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
2650 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
2651 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
2652 "SCEVAddExpr operand types don't match!");
2653 unsigned NumPtrs = count_if(
2654 Ops, [](const SCEV *Op) { return Op->getType()->isPointerTy(); });
2655 assert(NumPtrs <= 1 && "add has at most one pointer operand");
2656#endif
2657
2658 const SCEV *Folded = constantFoldAndGroupOps(
2659 SE&: *this, LI, DT, Ops,
2660 Fold: [](const APInt &C1, const APInt &C2) { return C1 + C2; },
2661 IsIdentity: [](const APInt &C) { return C.isZero(); }, // identity
2662 IsAbsorber: [](const APInt &C) { return false; }); // absorber
2663 if (Folded)
2664 return Folded;
2665
2666 unsigned Idx = isa<SCEVConstant>(Val: Ops[0]) ? 1 : 0;
2667
2668 // Delay expensive flag strengthening until necessary.
2669 auto ComputeFlags = [this, OrigFlags](ArrayRef<SCEVUse> Ops) {
2670 return StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops, Flags: OrigFlags);
2671 };
2672
2673 // Limit recursion calls depth.
2674 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
2675 return getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops));
2676
2677 if (SCEV *S = findExistingSCEVInCache(SCEVType: scAddExpr, Ops)) {
2678 // Don't strengthen flags if we have no new information.
2679 SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S);
2680 if (Add->getNoWrapFlags(Mask: OrigFlags) != OrigFlags)
2681 Add->setNoWrapFlags(ComputeFlags(Ops));
2682 return S;
2683 }
2684
2685 // Okay, check to see if the same value occurs in the operand list more than
2686 // once. If so, merge them together into an multiply expression. Since we
2687 // sorted the list, these values are required to be adjacent.
2688 Type *Ty = Ops[0]->getType();
2689 bool FoundMatch = false;
2690 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
2691 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2
2692 // Scan ahead to count how many equal operands there are.
2693 unsigned Count = 2;
2694 while (i+Count != e && Ops[i+Count] == Ops[i])
2695 ++Count;
2696 // Merge the values into a multiply.
2697 SCEVUse Scale = getConstant(Ty, V: Count);
2698 const SCEV *Mul = getMulExpr(LHS: Scale, RHS: Ops[i], Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2699 if (Ops.size() == Count)
2700 return Mul;
2701 Ops[i] = Mul;
2702 Ops.erase(CS: Ops.begin()+i+1, CE: Ops.begin()+i+Count);
2703 --i; e -= Count - 1;
2704 FoundMatch = true;
2705 }
2706 if (FoundMatch)
2707 return getAddExpr(Ops, OrigFlags, Depth: Depth + 1);
2708
2709 // Check for truncates. If all the operands are truncated from the same
2710 // type, see if factoring out the truncate would permit the result to be
2711 // folded. eg., n*trunc(x) + m*trunc(y) --> trunc(trunc(m)*x + trunc(n)*y)
2712 // if the contents of the resulting outer trunc fold to something simple.
2713 auto FindTruncSrcType = [&]() -> Type * {
2714 // We're ultimately looking to fold an addrec of truncs and muls of only
2715 // constants and truncs, so if we find any other types of SCEV
2716 // as operands of the addrec then we bail and return nullptr here.
2717 // Otherwise, we return the type of the operand of a trunc that we find.
2718 if (auto *T = dyn_cast<SCEVTruncateExpr>(Val&: Ops[Idx]))
2719 return T->getOperand()->getType();
2720 if (const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
2721 SCEVUse LastOp = Mul->getOperand(i: Mul->getNumOperands() - 1);
2722 if (const auto *T = dyn_cast<SCEVTruncateExpr>(Val&: LastOp))
2723 return T->getOperand()->getType();
2724 }
2725 return nullptr;
2726 };
2727 if (auto *SrcType = FindTruncSrcType()) {
2728 SmallVector<SCEVUse, 8> LargeOps;
2729 bool Ok = true;
2730 // Check all the operands to see if they can be represented in the
2731 // source type of the truncate.
2732 for (const SCEV *Op : Ops) {
2733 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2734 if (T->getOperand()->getType() != SrcType) {
2735 Ok = false;
2736 break;
2737 }
2738 LargeOps.push_back(Elt: T->getOperand());
2739 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Op)) {
2740 LargeOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2741 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val: Op)) {
2742 SmallVector<SCEVUse, 8> LargeMulOps;
2743 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2744 if (const SCEVTruncateExpr *T =
2745 dyn_cast<SCEVTruncateExpr>(Val: M->getOperand(i: j))) {
2746 if (T->getOperand()->getType() != SrcType) {
2747 Ok = false;
2748 break;
2749 }
2750 LargeMulOps.push_back(Elt: T->getOperand());
2751 } else if (const auto *C = dyn_cast<SCEVConstant>(Val: M->getOperand(i: j))) {
2752 LargeMulOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2753 } else {
2754 Ok = false;
2755 break;
2756 }
2757 }
2758 if (Ok)
2759 LargeOps.push_back(Elt: getMulExpr(Ops&: LargeMulOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2760 } else {
2761 Ok = false;
2762 break;
2763 }
2764 }
2765 if (Ok) {
2766 // Evaluate the expression in the larger type.
2767 const SCEV *Fold = getAddExpr(Ops&: LargeOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2768 // If it folds to something simple, use it. Otherwise, don't.
2769 if (isa<SCEVConstant>(Val: Fold) || isa<SCEVUnknown>(Val: Fold))
2770 return getTruncateExpr(Op: Fold, Ty);
2771 }
2772 }
2773
2774 if (Ops.size() == 2) {
2775 // Check if we have an expression of the form ((X + C1) - C2), where C1 and
2776 // C2 can be folded in a way that allows retaining wrapping flags of (X +
2777 // C1).
2778 const SCEV *A = Ops[0];
2779 const SCEV *B = Ops[1];
2780 auto *AddExpr = dyn_cast<SCEVAddExpr>(Val: B);
2781 auto *C = dyn_cast<SCEVConstant>(Val: A);
2782 if (AddExpr && C && isa<SCEVConstant>(Val: AddExpr->getOperand(i: 0))) {
2783 auto C1 = cast<SCEVConstant>(Val: AddExpr->getOperand(i: 0))->getAPInt();
2784 auto C2 = C->getAPInt();
2785 SCEV::NoWrapFlags PreservedFlags = SCEV::FlagAnyWrap;
2786
2787 APInt ConstAdd = C1 + C2;
2788 auto AddFlags = AddExpr->getNoWrapFlags();
2789 // Adding a smaller constant is NUW if the original AddExpr was NUW.
2790 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNUW) &&
2791 ConstAdd.ule(RHS: C1)) {
2792 PreservedFlags =
2793 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNUW);
2794 }
2795
2796 // Adding a constant with the same sign and small magnitude is NSW, if the
2797 // original AddExpr was NSW.
2798 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNSW) &&
2799 C1.isSignBitSet() == ConstAdd.isSignBitSet() &&
2800 ConstAdd.abs().ule(RHS: C1.abs())) {
2801 PreservedFlags =
2802 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNSW);
2803 }
2804
2805 if (PreservedFlags != SCEV::FlagAnyWrap) {
2806 SmallVector<SCEVUse, 4> NewOps(AddExpr->operands());
2807 NewOps[0] = getConstant(Val: ConstAdd);
2808 return getAddExpr(Ops&: NewOps, OrigFlags: PreservedFlags);
2809 }
2810 }
2811
2812 // Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext
2813 // (B), if trunc (A) + -A + B does not unsigned-wrap.
2814 const SCEVAddExpr *InnerAdd;
2815 if (match(S: B, P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
2816 const SCEV *NarrowA = getTruncateExpr(Op: A, Ty: InnerAdd->getType());
2817 if (NarrowA == getNegativeSCEV(V: InnerAdd->getOperand(i: 0)) &&
2818 getZeroExtendExpr(Op: NarrowA, Ty: B->getType()) == A &&
2819 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops: {NarrowA, InnerAdd},
2820 Flags: SCEV::FlagAnyWrap),
2821 TestFlags: SCEV::FlagNUW)) {
2822 return getZeroExtendExpr(Op: getAddExpr(LHS: NarrowA, RHS: InnerAdd), Ty: B->getType());
2823 }
2824 }
2825 }
2826
2827 // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
2828 const SCEV *Y;
2829 if (Ops.size() == 2 &&
2830 match(U: Ops[0],
2831 P: m_scev_Mul(Op0: m_scev_AllOnes(),
2832 Op1: m_scev_URem(LHS: m_scev_Specific(S: Ops[1]), RHS: m_SCEV(V&: Y), SE&: *this))))
2833 return getMulExpr(LHS: Y, RHS: getUDivExpr(LHS: Ops[1], RHS: Y));
2834
2835 // Skip past any other cast SCEVs.
2836 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
2837 ++Idx;
2838
2839 // If there are add operands they would be next.
2840 if (Idx < Ops.size()) {
2841 bool DeletedAdd = false;
2842 // If the original flags and all inlined SCEVAddExprs are NUW, use the
2843 // common NUW flag for expression after inlining. Other flags cannot be
2844 // preserved, because they may depend on the original order of operations.
2845 SCEV::NoWrapFlags CommonFlags = maskFlags(Flags: OrigFlags, Mask: SCEV::FlagNUW);
2846 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[Idx])) {
2847 if (Ops.size() > AddOpsInlineThreshold ||
2848 Add->getNumOperands() > AddOpsInlineThreshold)
2849 break;
2850 // If we have an add, expand the add operands onto the end of the operands
2851 // list.
2852 Ops.erase(CI: Ops.begin()+Idx);
2853 append_range(C&: Ops, R: Add->operands());
2854 DeletedAdd = true;
2855 CommonFlags = maskFlags(Flags: CommonFlags, Mask: Add->getNoWrapFlags());
2856 }
2857
2858 // If we deleted at least one add, we added operands to the end of the list,
2859 // and they are not necessarily sorted. Recurse to resort and resimplify
2860 // any operands we just acquired.
2861 if (DeletedAdd)
2862 return getAddExpr(Ops, OrigFlags: CommonFlags, Depth: Depth + 1);
2863 }
2864
2865 // Skip over the add expression until we get to a multiply.
2866 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
2867 ++Idx;
2868
2869 // Check to see if there are any folding opportunities present with
2870 // operands multiplied by constant values.
2871 if (Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx])) {
2872 uint64_t BitWidth = getTypeSizeInBits(Ty);
2873 SmallDenseMap<SCEVUse, APInt, 16> M;
2874 SmallVector<SCEVUse, 8> NewOps;
2875 APInt AccumulatedConstant(BitWidth, 0);
2876 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2877 Ops, Scale: APInt(BitWidth, 1), SE&: *this)) {
2878 struct APIntCompare {
2879 bool operator()(const APInt &LHS, const APInt &RHS) const {
2880 return LHS.ult(RHS);
2881 }
2882 };
2883
2884 // Some interesting folding opportunity is present, so its worthwhile to
2885 // re-generate the operands list. Group the operands by constant scale,
2886 // to avoid multiplying by the same constant scale multiple times.
2887 std::map<APInt, SmallVector<SCEVUse, 4>, APIntCompare> MulOpLists;
2888 for (const SCEV *NewOp : NewOps)
2889 MulOpLists[M.find(Val: NewOp)->second].push_back(Elt: NewOp);
2890 // Re-generate the operands list.
2891 Ops.clear();
2892 if (AccumulatedConstant != 0)
2893 Ops.push_back(Elt: getConstant(Val: AccumulatedConstant));
2894 for (auto &MulOp : MulOpLists) {
2895 if (MulOp.first == 1) {
2896 Ops.push_back(Elt: getAddExpr(Ops&: MulOp.second, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2897 } else if (MulOp.first != 0) {
2898 Ops.push_back(Elt: getMulExpr(
2899 LHS: getConstant(Val: MulOp.first),
2900 RHS: getAddExpr(Ops&: MulOp.second, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2901 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2902 }
2903 }
2904 if (Ops.empty())
2905 return getZero(Ty);
2906 if (Ops.size() == 1)
2907 return Ops[0];
2908 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2909 }
2910 }
2911
2912 // Given a SCEVMulExpr and an operand index, return the product of all
2913 // operands except the one at OpIdx.
2914 auto StripFactor = [&](const SCEVMulExpr *M, unsigned OpIdx) -> SCEVUse {
2915 if (M->getNumOperands() == 2)
2916 return M->getOperand(i: OpIdx == 0);
2917 SmallVector<SCEVUse, 4> Remaining(M->operands().take_front(N: OpIdx));
2918 append_range(C&: Remaining, R: M->operands().drop_front(N: OpIdx + 1));
2919 return getMulExpr(Ops&: Remaining, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2920 };
2921
2922 // If we are adding something to a multiply expression, make sure the
2923 // something is not already an operand of the multiply. If so, merge it into
2924 // the multiply.
2925 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx]); ++Idx) {
2926 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val&: Ops[Idx]);
2927 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
2928 // Scan all terms to find every occurrence of common factor MulOpSCEV
2929 // and fold them in one shot:
2930 // A1*X + A2*X + ... + An*X --> X * (A1 + A2 + ... + An)
2931 const SCEV *MulOpSCEV = Mul->getOperand(i: MulOp);
2932 if (isa<SCEVConstant>(Val: MulOpSCEV))
2933 continue;
2934
2935 // Cofactors: 1 for bare addends matching MulOpSCEV, or the
2936 // remaining product for multiply terms containing MulOpSCEV.
2937 SmallVector<SCEVUse, 4> Cofactors;
2938 SmallVector<unsigned, 4> DeadIndices;
2939 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) {
2940 if (MulOpSCEV == Ops[AddOp]) {
2941 // W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1))
2942 Cofactors.push_back(Elt: getOne(Ty));
2943 DeadIndices.push_back(Elt: AddOp);
2944 continue;
2945 }
2946
2947 if (AddOp <= Idx || !isa<SCEVMulExpr>(Val: Ops[AddOp]))
2948 continue;
2949
2950 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Val&: Ops[AddOp]);
2951 for (unsigned OMulOp = 0, OE = OtherMul->getNumOperands(); OMulOp != OE;
2952 ++OMulOp) {
2953 if (OtherMul->getOperand(i: OMulOp) == MulOpSCEV) {
2954 // (A*B*C) + (A*D*E) --> A * (B*C + D*E)
2955 Cofactors.push_back(Elt: StripFactor(OtherMul, OMulOp));
2956 DeadIndices.push_back(Elt: AddOp);
2957 break;
2958 }
2959 }
2960 }
2961
2962 // Fold all collected cofactors with the anchor multiply's cofactor:
2963 // MulOpSCEV * (Cofactor_1 + ... + Cofactor_n + AnchorCofactor)
2964 if (!Cofactors.empty()) {
2965 Cofactors.push_back(Elt: StripFactor(Mul, MulOp));
2966
2967 SCEVUse InnerSum = getAddExpr(Ops&: Cofactors, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2968 SCEVUse OuterMul =
2969 getMulExpr(LHS: MulOpSCEV, RHS: InnerSum, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2970
2971 // DeadIndices does not include Idx (the anchor), hence +1.
2972 if (Ops.size() == DeadIndices.size() + 1)
2973 return OuterMul;
2974
2975 // Erase Ops[Idx] first, then erase DeadIndices in reverse order.
2976 // The -1 adjustment accounts for the shift from removing Idx;
2977 // reverse order means each erasure only shifts later positions,
2978 // which have already been processed.
2979 Ops.erase(CI: Ops.begin() + Idx);
2980 for (unsigned Dead : reverse(C&: DeadIndices))
2981 Ops.erase(CI: Ops.begin() + (Dead > Idx ? Dead - 1 : Dead));
2982
2983 Ops.push_back(Elt: OuterMul);
2984 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2985 }
2986 }
2987 }
2988
2989 // If there are any add recurrences in the operands list, see if any other
2990 // added values are loop invariant. If so, we can fold them into the
2991 // recurrence.
2992 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
2993 ++Idx;
2994
2995 // Scan over all recurrences, trying to fold loop invariants into them.
2996 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
2997 // Scan all of the other operands to this add and add them to the vector if
2998 // they are loop invariant w.r.t. the recurrence.
2999 SmallVector<SCEVUse, 8> LIOps;
3000 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
3001 const Loop *AddRecLoop = AddRec->getLoop();
3002 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3003 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRecLoop)) {
3004 LIOps.push_back(Elt: Ops[i]);
3005 Ops.erase(CI: Ops.begin()+i);
3006 --i; --e;
3007 }
3008
3009 // If we found some loop invariants, fold them into the recurrence.
3010 if (!LIOps.empty()) {
3011 // Compute nowrap flags for the addition of the loop-invariant ops and
3012 // the addrec. Temporarily push it as an operand for that purpose. These
3013 // flags are valid in the scope of the addrec only.
3014 LIOps.push_back(Elt: AddRec);
3015 SCEV::NoWrapFlags Flags = ComputeFlags(LIOps);
3016 LIOps.pop_back();
3017
3018 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step}
3019 LIOps.push_back(Elt: AddRec->getStart());
3020
3021 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
3022
3023 // It is not in general safe to propagate flags valid on an add within
3024 // the addrec scope to one outside it. We must prove that the inner
3025 // scope is guaranteed to execute if the outer one does to be able to
3026 // safely propagate. We know the program is undefined if poison is
3027 // produced on the inner scoped addrec. We also know that *for this use*
3028 // the outer scoped add can't overflow (because of the flags we just
3029 // computed for the inner scoped add) without the program being undefined.
3030 // Proving that entry to the outer scope neccesitates entry to the inner
3031 // scope, thus proves the program undefined if the flags would be violated
3032 // in the outer scope.
3033 SCEV::NoWrapFlags AddFlags = Flags;
3034 if (AddFlags != SCEV::FlagAnyWrap) {
3035 auto *DefI = getDefiningScopeBound(Ops: LIOps);
3036 auto *ReachI = &*AddRecLoop->getHeader()->begin();
3037 if (!isGuaranteedToTransferExecutionTo(A: DefI, B: ReachI))
3038 AddFlags = SCEV::FlagAnyWrap;
3039 }
3040 AddRecOps[0] = getAddExpr(Ops&: LIOps, OrigFlags: AddFlags, Depth: Depth + 1);
3041
3042 // Build the new addrec. Propagate the NUW and NSW flags if both the
3043 // outer add and the inner addrec are guaranteed to have no overflow.
3044 // Always propagate NW.
3045 Flags = AddRec->getNoWrapFlags(Mask: setFlags(Flags, OnFlags: SCEV::FlagNW));
3046 const SCEV *NewRec = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags);
3047
3048 // If all of the other operands were loop invariant, we are done.
3049 if (Ops.size() == 1) return NewRec;
3050
3051 // Otherwise, add the folded AddRec by the non-invariant parts.
3052 for (unsigned i = 0;; ++i)
3053 if (Ops[i] == AddRec) {
3054 Ops[i] = NewRec;
3055 break;
3056 }
3057 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3058 }
3059
3060 // Okay, if there weren't any loop invariants to be folded, check to see if
3061 // there are multiple AddRec's with the same loop induction variable being
3062 // added together. If so, we can fold them.
3063 for (unsigned OtherIdx = Idx+1;
3064 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3065 ++OtherIdx) {
3066 // We expect the AddRecExpr's to be sorted in reverse dominance order,
3067 // so that the 1st found AddRecExpr is dominated by all others.
3068 assert(DT.dominates(
3069 cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(),
3070 AddRec->getLoop()->getHeader()) &&
3071 "AddRecExprs are not sorted in reverse dominance order?");
3072 if (AddRecLoop == cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx])->getLoop()) {
3073 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L>
3074 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
3075 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3076 ++OtherIdx) {
3077 const auto *OtherAddRec = cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
3078 if (OtherAddRec->getLoop() == AddRecLoop) {
3079 for (unsigned i = 0, e = OtherAddRec->getNumOperands();
3080 i != e; ++i) {
3081 if (i >= AddRecOps.size()) {
3082 append_range(C&: AddRecOps, R: OtherAddRec->operands().drop_front(N: i));
3083 break;
3084 }
3085 AddRecOps[i] =
3086 getAddExpr(LHS: AddRecOps[i], RHS: OtherAddRec->getOperand(i),
3087 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3088 }
3089 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
3090 }
3091 }
3092 // Step size has changed, so we cannot guarantee no self-wraparound.
3093 Ops[Idx] = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags: SCEV::FlagAnyWrap);
3094 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3095 }
3096 }
3097
3098 // Otherwise couldn't fold anything into this recurrence. Move onto the
3099 // next one.
3100 }
3101
3102 // Okay, it looks like we really DO need an add expr. Check to see if we
3103 // already have one, otherwise create a new one.
3104 return getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops));
3105}
3106
3107const SCEV *ScalarEvolution::getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
3108 SCEV::NoWrapFlags Flags) {
3109 FoldingSetNodeID ID;
3110 ID.AddInteger(I: scAddExpr);
3111 for (const SCEV *Op : Ops)
3112 ID.AddPointer(Ptr: Op);
3113 void *IP = nullptr;
3114 SCEVAddExpr *S =
3115 static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
3116 if (!S) {
3117 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3118 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3119 S = new (SCEVAllocator)
3120 SCEVAddExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size());
3121 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3122 S->computeAndSetCanonical(SE&: *this);
3123 registerUser(User: S, Ops);
3124 }
3125 S->setNoWrapFlags(Flags);
3126 return S;
3127}
3128
3129const SCEV *ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<SCEVUse> Ops,
3130 const Loop *L,
3131 SCEV::NoWrapFlags Flags) {
3132 FoldingSetNodeID ID;
3133 ID.AddInteger(I: scAddRecExpr);
3134 for (const SCEV *Op : Ops)
3135 ID.AddPointer(Ptr: Op);
3136 ID.AddPointer(Ptr: L);
3137 void *IP = nullptr;
3138 SCEVAddRecExpr *S =
3139 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
3140 if (!S) {
3141 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3142 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3143 S = new (SCEVAllocator)
3144 SCEVAddRecExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size(), L);
3145 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3146 S->computeAndSetCanonical(SE&: *this);
3147 LoopUsers[L].push_back(Elt: S);
3148 registerUser(User: S, Ops);
3149 }
3150 setNoWrapFlags(AddRec: S, Flags);
3151 return S;
3152}
3153
3154const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
3155 SCEV::NoWrapFlags Flags) {
3156 FoldingSetNodeID ID;
3157 ID.AddInteger(I: scMulExpr);
3158 for (const SCEV *Op : Ops)
3159 ID.AddPointer(Ptr: Op);
3160 void *IP = nullptr;
3161 SCEVMulExpr *S =
3162 static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
3163 if (!S) {
3164 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3165 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3166 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(Allocator&: SCEVAllocator),
3167 O, Ops.size());
3168 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3169 S->computeAndSetCanonical(SE&: *this);
3170 registerUser(User: S, Ops);
3171 }
3172 S->setNoWrapFlags(Flags);
3173 return S;
3174}
3175
3176static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
3177 uint64_t k = i*j;
3178 if (j > 1 && k / j != i) Overflow = true;
3179 return k;
3180}
3181
3182/// Compute the result of "n choose k", the binomial coefficient. If an
3183/// intermediate computation overflows, Overflow will be set and the return will
3184/// be garbage. Overflow is not cleared on absence of overflow.
3185static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
3186 // We use the multiplicative formula:
3187 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
3188 // At each iteration, we take the n-th term of the numeral and divide by the
3189 // (k-n)th term of the denominator. This division will always produce an
3190 // integral result, and helps reduce the chance of overflow in the
3191 // intermediate computations. However, we can still overflow even when the
3192 // final result would fit.
3193
3194 if (n == 0 || n == k) return 1;
3195 if (k > n) return 0;
3196
3197 if (k > n/2)
3198 k = n-k;
3199
3200 uint64_t r = 1;
3201 for (uint64_t i = 1; i <= k; ++i) {
3202 r = umul_ov(i: r, j: n-(i-1), Overflow);
3203 r /= i;
3204 }
3205 return r;
3206}
3207
3208/// Determine if any of the operands in this SCEV are a constant or if
3209/// any of the add or multiply expressions in this SCEV contain a constant.
3210static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
3211 struct FindConstantInAddMulChain {
3212 bool FoundConstant = false;
3213
3214 bool follow(const SCEV *S) {
3215 FoundConstant |= isa<SCEVConstant>(Val: S);
3216 return isa<SCEVAddExpr>(Val: S) || isa<SCEVMulExpr>(Val: S);
3217 }
3218
3219 bool isDone() const {
3220 return FoundConstant;
3221 }
3222 };
3223
3224 FindConstantInAddMulChain F;
3225 SCEVTraversal<FindConstantInAddMulChain> ST(F);
3226 ST.visitAll(Root: StartExpr);
3227 return F.FoundConstant;
3228}
3229
3230/// Get a canonical multiply expression, or something simpler if possible.
3231const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
3232 SCEV::NoWrapFlags OrigFlags,
3233 unsigned Depth) {
3234 assert(OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3235 "only nuw or nsw allowed");
3236 assert(!Ops.empty() && "Cannot get empty mul!");
3237 if (Ops.size() == 1) return Ops[0];
3238#ifndef NDEBUG
3239 Type *ETy = Ops[0]->getType();
3240 assert(!ETy->isPointerTy());
3241 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
3242 assert(Ops[i]->getType() == ETy &&
3243 "SCEVMulExpr operand types don't match!");
3244#endif
3245
3246 const SCEV *Folded = constantFoldAndGroupOps(
3247 SE&: *this, LI, DT, Ops,
3248 Fold: [](const APInt &C1, const APInt &C2) { return C1 * C2; },
3249 IsIdentity: [](const APInt &C) { return C.isOne(); }, // identity
3250 IsAbsorber: [](const APInt &C) { return C.isZero(); }); // absorber
3251 if (Folded)
3252 return Folded;
3253
3254 // Delay expensive flag strengthening until necessary.
3255 auto ComputeFlags = [this, OrigFlags](const ArrayRef<SCEVUse> Ops) {
3256 return StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops, Flags: OrigFlags);
3257 };
3258
3259 // Limit recursion calls depth.
3260 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
3261 return getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops));
3262
3263 if (SCEV *S = findExistingSCEVInCache(SCEVType: scMulExpr, Ops)) {
3264 // Don't strengthen flags if we have no new information.
3265 SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
3266 if (Mul->getNoWrapFlags(Mask: OrigFlags) != OrigFlags)
3267 Mul->setNoWrapFlags(ComputeFlags(Ops));
3268 return S;
3269 }
3270
3271 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: Ops[0])) {
3272 if (Ops.size() == 2) {
3273 // C1*(C2+V) -> C1*C2 + C1*V
3274 // If any of Add's ops are Adds or Muls with a constant, apply this
3275 // transformation as well.
3276 //
3277 // TODO: There are some cases where this transformation is not
3278 // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of
3279 // this transformation should be narrowed down.
3280 const SCEV *Op0, *Op1;
3281 if (match(U: Ops[1], P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))) &&
3282 containsConstantInAddMulChain(StartExpr: Ops[1])) {
3283 const SCEV *LHS = getMulExpr(LHS: LHSC, RHS: Op0, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3284 const SCEV *RHS = getMulExpr(LHS: LHSC, RHS: Op1, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3285 return getAddExpr(LHS, RHS, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3286 }
3287
3288 if (Ops[0]->isAllOnesValue()) {
3289 // If we have a mul by -1 of an add, try distributing the -1 among the
3290 // add operands.
3291 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[1])) {
3292 SmallVector<SCEVUse, 4> NewOps;
3293 bool AnyFolded = false;
3294 for (const SCEV *AddOp : Add->operands()) {
3295 const SCEV *Mul = getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddOp),
3296 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3297 if (!isa<SCEVMulExpr>(Val: Mul)) AnyFolded = true;
3298 NewOps.push_back(Elt: Mul);
3299 }
3300 if (AnyFolded)
3301 return getAddExpr(Ops&: NewOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3302 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: Ops[1])) {
3303 // Negation preserves a recurrence's no self-wrap property.
3304 SmallVector<SCEVUse, 4> Operands;
3305 for (const SCEV *AddRecOp : AddRec->operands())
3306 Operands.push_back(Elt: getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddRecOp),
3307 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3308 // Let M be the minimum representable signed value. AddRec with nsw
3309 // multiplied by -1 can have signed overflow if and only if it takes a
3310 // value of M: M * (-1) would stay M and (M + 1) * (-1) would be the
3311 // maximum signed value. In all other cases signed overflow is
3312 // impossible.
3313 auto FlagsMask = SCEV::FlagNW;
3314 if (AddRec->hasNoSignedWrap()) {
3315 auto MinInt =
3316 APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: AddRec->getType()));
3317 if (getSignedRangeMin(S: AddRec) != MinInt)
3318 FlagsMask = setFlags(Flags: FlagsMask, OnFlags: SCEV::FlagNSW);
3319 }
3320 return getAddRecExpr(Operands, L: AddRec->getLoop(),
3321 Flags: AddRec->getNoWrapFlags(Mask: FlagsMask));
3322 }
3323 }
3324
3325 // Try to push the constant operand into a ZExt: C * zext (A + B) ->
3326 // zext (C*A + C*B) if trunc (C) * (A + B) does not unsigned-wrap.
3327 const SCEVAddExpr *InnerAdd;
3328 if (match(U: Ops[1], P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
3329 const SCEV *NarrowC = getTruncateExpr(Op: LHSC, Ty: InnerAdd->getType());
3330 if (isa<SCEVConstant>(Val: InnerAdd->getOperand(i: 0)) &&
3331 getZeroExtendExpr(Op: NarrowC, Ty: Ops[1]->getType()) == LHSC &&
3332 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops: {NarrowC, InnerAdd},
3333 Flags: SCEV::FlagAnyWrap),
3334 TestFlags: SCEV::FlagNUW)) {
3335 auto *Res = getMulExpr(LHS: NarrowC, RHS: InnerAdd, Flags: SCEV::FlagNUW, Depth: Depth + 1);
3336 return getZeroExtendExpr(Op: Res, Ty: Ops[1]->getType(), Depth: Depth + 1);
3337 };
3338 }
3339
3340 // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2,
3341 // D is a multiple of C2, and C1 is a multiple of C2. If C2 is a multiple
3342 // of C1, fold to (D /u (C2 /u C1)).
3343 const SCEV *D;
3344 APInt C1V = LHSC->getAPInt();
3345 // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. Don't treat -1
3346 // as -1 * 1, as it won't enable additional folds.
3347 if (C1V.isNegative() && !C1V.isMinSignedValue() && !C1V.isAllOnes())
3348 C1V = C1V.abs();
3349 const SCEVConstant *C2;
3350 if (C1V.isPowerOf2() &&
3351 match(U: Ops[1], P: m_scev_UDiv(Op0: m_SCEV(V&: D), Op1: m_SCEVConstant(V&: C2))) &&
3352 C2->getAPInt().isPowerOf2() &&
3353 C1V.logBase2() <= getMinTrailingZeros(S: D)) {
3354 const SCEV *NewMul = nullptr;
3355 if (C1V.uge(RHS: C2->getAPInt())) {
3356 NewMul = getMulExpr(LHS: getUDivExpr(LHS: getConstant(Val: C1V), RHS: C2), RHS: D);
3357 } else if (C2->getAPInt().logBase2() <= getMinTrailingZeros(S: D)) {
3358 assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier");
3359 NewMul = getUDivExpr(LHS: D, RHS: getUDivExpr(LHS: C2, RHS: getConstant(Val: C1V)));
3360 }
3361 if (NewMul)
3362 return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(V: NewMul);
3363 }
3364 }
3365 }
3366
3367 // Skip over the add expression until we get to a multiply.
3368 unsigned Idx = 0;
3369 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
3370 ++Idx;
3371
3372 // If there are mul operands inline them all into this expression.
3373 if (Idx < Ops.size()) {
3374 bool DeletedMul = false;
3375 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
3376 if (Ops.size() > MulOpsInlineThreshold)
3377 break;
3378 // If we have an mul, expand the mul operands onto the end of the
3379 // operands list.
3380 Ops.erase(CI: Ops.begin()+Idx);
3381 append_range(C&: Ops, R: Mul->operands());
3382 DeletedMul = true;
3383 }
3384
3385 // If we deleted at least one mul, we added operands to the end of the
3386 // list, and they are not necessarily sorted. Recurse to resort and
3387 // resimplify any operands we just acquired.
3388 if (DeletedMul)
3389 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3390 }
3391
3392 // If there are any add recurrences in the operands list, see if any other
3393 // added values are loop invariant. If so, we can fold them into the
3394 // recurrence.
3395 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
3396 ++Idx;
3397
3398 // Scan over all recurrences, trying to fold loop invariants into them.
3399 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
3400 // Scan all of the other operands to this mul and add them to the vector
3401 // if they are loop invariant w.r.t. the recurrence.
3402 SmallVector<SCEVUse, 8> LIOps;
3403 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
3404 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3405 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRec->getLoop())) {
3406 LIOps.push_back(Elt: Ops[i]);
3407 Ops.erase(CI: Ops.begin()+i);
3408 --i; --e;
3409 }
3410
3411 // If we found some loop invariants, fold them into the recurrence.
3412 if (!LIOps.empty()) {
3413 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step}
3414 SmallVector<SCEVUse, 4> NewOps;
3415 NewOps.reserve(N: AddRec->getNumOperands());
3416 const SCEV *Scale = getMulExpr(Ops&: LIOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3417
3418 // If both the mul and addrec are nuw, we can preserve nuw.
3419 // If both the mul and addrec are nsw, we can only preserve nsw if either
3420 // a) they are also nuw, or
3421 // b) all multiplications of addrec operands with scale are nsw.
3422 SCEV::NoWrapFlags Flags =
3423 AddRec->getNoWrapFlags(Mask: ComputeFlags({Scale, AddRec}));
3424
3425 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
3426 NewOps.push_back(Elt: getMulExpr(LHS: Scale, RHS: AddRec->getOperand(i),
3427 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3428
3429 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW) && !hasFlags(Flags, TestFlags: SCEV::FlagNUW)) {
3430 ConstantRange NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
3431 BinOp: Instruction::Mul, Other: getSignedRange(S: Scale),
3432 NoWrapKind: OverflowingBinaryOperator::NoSignedWrap);
3433 if (!NSWRegion.contains(CR: getSignedRange(S: AddRec->getOperand(i))))
3434 Flags = clearFlags(Flags, OffFlags: SCEV::FlagNSW);
3435 }
3436 }
3437
3438 const SCEV *NewRec = getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), Flags);
3439
3440 // If all of the other operands were loop invariant, we are done.
3441 if (Ops.size() == 1) return NewRec;
3442
3443 // Otherwise, multiply the folded AddRec by the non-invariant parts.
3444 for (unsigned i = 0;; ++i)
3445 if (Ops[i] == AddRec) {
3446 Ops[i] = NewRec;
3447 break;
3448 }
3449 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3450 }
3451
3452 // Okay, if there weren't any loop invariants to be folded, check to see
3453 // if there are multiple AddRec's with the same loop induction variable
3454 // being multiplied together. If so, we can fold them.
3455
3456 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
3457 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
3458 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
3459 // ]]],+,...up to x=2n}.
3460 // Note that the arguments to choose() are always integers with values
3461 // known at compile time, never SCEV objects.
3462 //
3463 // The implementation avoids pointless extra computations when the two
3464 // addrec's are of different length (mathematically, it's equivalent to
3465 // an infinite stream of zeros on the right).
3466 bool OpsModified = false;
3467 for (unsigned OtherIdx = Idx+1;
3468 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3469 ++OtherIdx) {
3470 const SCEVAddRecExpr *OtherAddRec =
3471 dyn_cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
3472 if (!OtherAddRec || OtherAddRec->getLoop() != AddRec->getLoop())
3473 continue;
3474
3475 // Limit max number of arguments to avoid creation of unreasonably big
3476 // SCEVAddRecs with very complex operands.
3477 if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
3478 MaxAddRecSize || hasHugeExpression(Ops: {AddRec, OtherAddRec}))
3479 continue;
3480
3481 bool Overflow = false;
3482 Type *Ty = AddRec->getType();
3483 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
3484 SmallVector<SCEVUse, 7> AddRecOps;
3485 for (int x = 0, xe = AddRec->getNumOperands() +
3486 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
3487 SmallVector<SCEVUse, 7> SumOps;
3488 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
3489 uint64_t Coeff1 = Choose(n: x, k: 2*x - y, Overflow);
3490 for (int z = std::max(a: y-x, b: y-(int)AddRec->getNumOperands()+1),
3491 ze = std::min(a: x+1, b: (int)OtherAddRec->getNumOperands());
3492 z < ze && !Overflow; ++z) {
3493 uint64_t Coeff2 = Choose(n: 2*x - y, k: x-z, Overflow);
3494 uint64_t Coeff;
3495 if (LargerThan64Bits)
3496 Coeff = umul_ov(i: Coeff1, j: Coeff2, Overflow);
3497 else
3498 Coeff = Coeff1*Coeff2;
3499 const SCEV *CoeffTerm = getConstant(Ty, V: Coeff);
3500 const SCEV *Term1 = AddRec->getOperand(i: y-z);
3501 const SCEV *Term2 = OtherAddRec->getOperand(i: z);
3502 SumOps.push_back(Elt: getMulExpr(Op0: CoeffTerm, Op1: Term1, Op2: Term2,
3503 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3504 }
3505 }
3506 if (SumOps.empty())
3507 SumOps.push_back(Elt: getZero(Ty));
3508 AddRecOps.push_back(Elt: getAddExpr(Ops&: SumOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3509 }
3510 if (!Overflow) {
3511 const SCEV *NewAddRec = getAddRecExpr(Operands&: AddRecOps, L: AddRec->getLoop(),
3512 Flags: SCEV::FlagAnyWrap);
3513 if (Ops.size() == 2) return NewAddRec;
3514 Ops[Idx] = NewAddRec;
3515 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
3516 OpsModified = true;
3517 AddRec = dyn_cast<SCEVAddRecExpr>(Val: NewAddRec);
3518 if (!AddRec)
3519 break;
3520 }
3521 }
3522 if (OpsModified)
3523 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3524
3525 // Otherwise couldn't fold anything into this recurrence. Move onto the
3526 // next one.
3527 }
3528
3529 // Okay, it looks like we really DO need an mul expr. Check to see if we
3530 // already have one, otherwise create a new one.
3531 return getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops));
3532}
3533
3534/// Represents an unsigned remainder expression based on unsigned division.
3535const SCEV *ScalarEvolution::getURemExpr(SCEVUse LHS, SCEVUse RHS) {
3536 assert(getEffectiveSCEVType(LHS->getType()) ==
3537 getEffectiveSCEVType(RHS->getType()) &&
3538 "SCEVURemExpr operand types don't match!");
3539
3540 // Short-circuit easy cases
3541 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3542 // If constant is one, the result is trivial
3543 if (RHSC->getValue()->isOne())
3544 return getZero(Ty: LHS->getType()); // X urem 1 --> 0
3545
3546 // If constant is a power of two, fold into a zext(trunc(LHS)).
3547 if (RHSC->getAPInt().isPowerOf2()) {
3548 Type *FullTy = LHS->getType();
3549 Type *TruncTy =
3550 IntegerType::get(C&: getContext(), NumBits: RHSC->getAPInt().logBase2());
3551 return getZeroExtendExpr(Op: getTruncateExpr(Op: LHS, Ty: TruncTy), Ty: FullTy);
3552 }
3553 }
3554
3555 // Fallback to %a == %x urem %y == %x -<nuw> ((%x udiv %y) *<nuw> %y)
3556 const SCEV *UDiv = getUDivExpr(LHS, RHS);
3557 const SCEV *Mult = getMulExpr(LHS: UDiv, RHS, Flags: SCEV::FlagNUW);
3558 return getMinusSCEV(LHS, RHS: Mult, Flags: SCEV::FlagNUW);
3559}
3560
3561/// Get a canonical unsigned division expression, or something simpler if
3562/// possible.
3563const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
3564 assert(!LHS->getType()->isPointerTy() &&
3565 "SCEVUDivExpr operand can't be pointer!");
3566 assert(LHS->getType() == RHS->getType() &&
3567 "SCEVUDivExpr operand types don't match!");
3568
3569 FoldingSetNodeID ID;
3570 ID.AddInteger(I: scUDivExpr);
3571 ID.AddPointer(Ptr: LHS);
3572 ID.AddPointer(Ptr: RHS);
3573 void *IP = nullptr;
3574 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
3575 return S;
3576
3577 // 0 udiv Y == 0
3578 if (match(U: LHS, P: m_scev_Zero()))
3579 return LHS;
3580
3581 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3582 if (RHSC->getValue()->isOne())
3583 return LHS; // X udiv 1 --> x
3584 // If the denominator is zero, the result of the udiv is undefined. Don't
3585 // try to analyze it, because the resolution chosen here may differ from
3586 // the resolution chosen in other parts of the compiler.
3587 if (!RHSC->getValue()->isZero()) {
3588 // Determine if the division can be folded into the operands of
3589 // its operands.
3590 // TODO: Generalize this to non-constants by using known-bits information.
3591 Type *Ty = LHS->getType();
3592 unsigned LZ = RHSC->getAPInt().countl_zero();
3593 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
3594 // For non-power-of-two values, effectively round the value up to the
3595 // nearest power of two.
3596 if (!RHSC->getAPInt().isPowerOf2())
3597 ++MaxShiftAmt;
3598 IntegerType *ExtTy =
3599 IntegerType::get(C&: getContext(), NumBits: getTypeSizeInBits(Ty) + MaxShiftAmt);
3600 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
3601 if (const SCEVConstant *Step =
3602 dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE&: *this))) {
3603 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
3604 const APInt &StepInt = Step->getAPInt();
3605 const APInt &DivInt = RHSC->getAPInt();
3606 if (!StepInt.urem(RHS: DivInt) &&
3607 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3608 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3609 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy),
3610 L: AR->getLoop(), Flags: SCEV::FlagAnyWrap)) {
3611 SmallVector<SCEVUse, 4> Operands;
3612 for (const SCEV *Op : AR->operands())
3613 Operands.push_back(Elt: getUDivExpr(LHS: Op, RHS));
3614 return getAddRecExpr(Operands, L: AR->getLoop(), Flags: SCEV::FlagNW);
3615 }
3616 /// Get a canonical UDivExpr for a recurrence.
3617 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
3618 const APInt *StartRem;
3619 if (!DivInt.urem(RHS: StepInt) && match(S: getURemExpr(LHS: AR->getStart(), RHS: Step),
3620 P: m_scev_APInt(C&: StartRem))) {
3621 bool NoWrap =
3622 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3623 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3624 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy), L: AR->getLoop(),
3625 Flags: SCEV::FlagAnyWrap);
3626
3627 // With N <= C and both N, C as powers-of-2, the transformation
3628 // {X,+,N}/C => {(X - X%N),+,N}/C preserves division results even
3629 // if wrapping occurs, as the division results remain equivalent for
3630 // all offsets in [[(X - X%N), X).
3631 bool CanFoldWithWrap = StepInt.ule(RHS: DivInt) && // N <= C
3632 StepInt.isPowerOf2() && DivInt.isPowerOf2();
3633 // Only fold if the subtraction can be folded in the start
3634 // expression.
3635 const SCEV *NewStart =
3636 getMinusSCEV(LHS: AR->getStart(), RHS: getConstant(Val: *StartRem));
3637 if (*StartRem != 0 && (NoWrap || CanFoldWithWrap) &&
3638 !isa<SCEVAddExpr>(Val: NewStart)) {
3639 const SCEV *NewLHS =
3640 getAddRecExpr(Start: NewStart, Step, L: AR->getLoop(),
3641 Flags: NoWrap ? SCEV::FlagNW : SCEV::FlagAnyWrap);
3642 if (LHS != NewLHS) {
3643 LHS = NewLHS;
3644
3645 // Reset the ID to include the new LHS, and check if it is
3646 // already cached.
3647 ID.clear();
3648 ID.AddInteger(I: scUDivExpr);
3649 ID.AddPointer(Ptr: LHS);
3650 ID.AddPointer(Ptr: RHS);
3651 IP = nullptr;
3652 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
3653 return S;
3654 }
3655 }
3656 }
3657 }
3658 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
3659 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val&: LHS)) {
3660 SmallVector<SCEVUse, 4> Operands;
3661 for (const SCEV *Op : M->operands())
3662 Operands.push_back(Elt: getZeroExtendExpr(Op, Ty: ExtTy));
3663 if (getZeroExtendExpr(Op: M, Ty: ExtTy) == getMulExpr(Ops&: Operands)) {
3664 // Find an operand that's safely divisible.
3665 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
3666 const SCEV *Op = M->getOperand(i);
3667 const SCEV *Div = getUDivExpr(LHS: Op, RHS: RHSC);
3668 if (!isa<SCEVUDivExpr>(Val: Div) && getMulExpr(LHS: Div, RHS: RHSC) == Op) {
3669 Operands = SmallVector<SCEVUse, 4>(M->operands());
3670 Operands[i] = Div;
3671 return getMulExpr(Ops&: Operands);
3672 }
3673 }
3674
3675 // Even if it's not divisible, try to remove a common factor.
3676 if (const auto *LHSC = dyn_cast<SCEVConstant>(Val: M->getOperand(i: 0))) {
3677 APInt Factor = APIntOps::GreatestCommonDivisor(A: LHSC->getAPInt(),
3678 B: RHSC->getAPInt());
3679 if (!Factor.isIntN(N: 1)) {
3680 SmallVector<SCEVUse, 2> NewOperands;
3681 NewOperands.push_back(Elt: getConstant(Val: LHSC->getAPInt().udiv(RHS: Factor)));
3682 append_range(C&: NewOperands, R: M->operands().drop_front());
3683 const SCEV *NewMul = getMulExpr(Ops&: NewOperands);
3684 return getUDivExpr(LHS: NewMul,
3685 RHS: getConstant(Val: RHSC->getAPInt().udiv(RHS: Factor)));
3686 }
3687 }
3688 }
3689 }
3690
3691 // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
3692 if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(Val&: LHS)) {
3693 if (auto *DivisorConstant =
3694 dyn_cast<SCEVConstant>(Val: OtherDiv->getRHS())) {
3695 bool Overflow = false;
3696 APInt NewRHS =
3697 DivisorConstant->getAPInt().umul_ov(RHS: RHSC->getAPInt(), Overflow);
3698 if (Overflow) {
3699 return getConstant(Ty: RHSC->getType(), V: 0, isSigned: false);
3700 }
3701 return getUDivExpr(LHS: OtherDiv->getLHS(), RHS: getConstant(Val: NewRHS));
3702 }
3703 }
3704
3705 // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded.
3706 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(Val&: LHS)) {
3707 SmallVector<SCEVUse, 4> Operands;
3708 for (const SCEV *Op : A->operands())
3709 Operands.push_back(Elt: getZeroExtendExpr(Op, Ty: ExtTy));
3710 if (getZeroExtendExpr(Op: A, Ty: ExtTy) == getAddExpr(Ops&: Operands)) {
3711 Operands.clear();
3712 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
3713 const SCEV *Op = getUDivExpr(LHS: A->getOperand(i), RHS);
3714 if (isa<SCEVUDivExpr>(Val: Op) ||
3715 getMulExpr(LHS: Op, RHS) != A->getOperand(i))
3716 break;
3717 Operands.push_back(Elt: Op);
3718 }
3719 if (Operands.size() == A->getNumOperands())
3720 return getAddExpr(Ops&: Operands);
3721 }
3722 }
3723
3724 // ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
3725 // This is an idiom for rounding A up to the next multiple of N, where A
3726 // is aready known to be a multiple of M. In this case, instcombine can
3727 // see that some low bits of the added constant are unused, so can clear
3728 // them, but we want to canonicalise to set the low bits. This makes the
3729 // pattern easier to match, without needing to check for known bits in
3730 // A*M.
3731 const APInt &N = RHSC->getAPInt();
3732 const APInt *NMinusM, *M;
3733 const SCEV *A;
3734 if (match(U: LHS, P: m_scev_Add(Op0: m_scev_APInt(C&: NMinusM),
3735 Op1: m_scev_Mul(Op0: m_scev_APInt(C&: M), Op1: m_SCEV(V&: A))))) {
3736 if (N.isPowerOf2() && M->isPowerOf2() && M->ult(RHS: N) &&
3737 *NMinusM == N - *M) {
3738 return getUDivExpr(
3739 LHS: getAddExpr(LHS: getConstant(Val: N - 1), RHS: getMulExpr(LHS: getConstant(Val: *M), RHS: A)),
3740 RHS);
3741 }
3742 }
3743
3744 // Fold if both operands are constant.
3745 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS))
3746 return getConstant(Val: LHSC->getAPInt().udiv(RHS: RHSC->getAPInt()));
3747 }
3748 }
3749
3750 // ((-C + (C smax %x)) /u %x) evaluates to zero, for any positive constant C.
3751 const APInt *NegC, *C;
3752 if (match(U: LHS,
3753 P: m_scev_Add(Op0: m_scev_APInt(C&: NegC),
3754 Op1: m_scev_SMax(Op0: m_scev_APInt(C), Op1: m_scev_Specific(S: RHS)))) &&
3755 NegC->isNegative() && !NegC->isMinSignedValue() && *C == -*NegC)
3756 return getZero(Ty: LHS->getType());
3757
3758 // (%a * %b)<nuw> / %b -> %a
3759 const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: LHS);
3760 if (Mul && Mul->hasNoUnsignedWrap()) {
3761 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
3762 if (Mul->getOperand(i) == RHS) {
3763 SmallVector<SCEVUse, 2> Operands;
3764 append_range(C&: Operands, R: Mul->operands().take_front(N: i));
3765 append_range(C&: Operands, R: Mul->operands().drop_front(N: i + 1));
3766 return getMulExpr(Ops&: Operands);
3767 }
3768 }
3769 }
3770
3771 // TODO: Generalize to handle any common factors.
3772 // udiv (mul nuw a, vscale), (mul nuw b, vscale) --> udiv a, b
3773 const SCEV *NewLHS, *NewRHS;
3774 if (match(U: LHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
3775 match(U: RHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale())))
3776 return getUDivExpr(LHS: NewLHS, RHS: NewRHS);
3777
3778 // The Insertion Point (IP) might be invalid by now (due to UniqueSCEVs
3779 // changes). Make sure we get a new one.
3780 IP = nullptr;
3781 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
3782 SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(Allocator&: SCEVAllocator),
3783 LHS, RHS);
3784 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3785 S->computeAndSetCanonical(SE&: *this);
3786 registerUser(User: S, Ops: ArrayRef<SCEVUse>({LHS, RHS}));
3787 return S;
3788}
3789
3790APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) {
3791 APInt A = C1->getAPInt().abs();
3792 APInt B = C2->getAPInt().abs();
3793 uint32_t ABW = A.getBitWidth();
3794 uint32_t BBW = B.getBitWidth();
3795
3796 if (ABW > BBW)
3797 B = B.zext(width: ABW);
3798 else if (ABW < BBW)
3799 A = A.zext(width: BBW);
3800
3801 return APIntOps::GreatestCommonDivisor(A: std::move(A), B: std::move(B));
3802}
3803
3804/// Get a canonical unsigned division expression, or something simpler if
3805/// possible. There is no representation for an exact udiv in SCEV IR, but we
3806/// can attempt to optimize it prior to construction.
3807const SCEV *ScalarEvolution::getUDivExactExpr(SCEVUse LHS, SCEVUse RHS) {
3808 // Currently there is no exact specific logic.
3809
3810 return getUDivExpr(LHS, RHS);
3811}
3812
3813/// Get an add recurrence expression for the specified loop. Simplify the
3814/// expression as much as possible.
3815const SCEV *ScalarEvolution::getAddRecExpr(SCEVUse Start, SCEVUse Step,
3816 const Loop *L,
3817 SCEV::NoWrapFlags Flags) {
3818 SmallVector<SCEVUse, 4> Operands;
3819 Operands.push_back(Elt: Start);
3820 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Val&: Step))
3821 if (StepChrec->getLoop() == L) {
3822 append_range(C&: Operands, R: StepChrec->operands());
3823 return getAddRecExpr(Operands, L, Flags: maskFlags(Flags, Mask: SCEV::FlagNW));
3824 }
3825
3826 Operands.push_back(Elt: Step);
3827 return getAddRecExpr(Operands, L, Flags);
3828}
3829
3830/// Get an add recurrence expression for the specified loop. Simplify the
3831/// expression as much as possible.
3832const SCEV *ScalarEvolution::getAddRecExpr(SmallVectorImpl<SCEVUse> &Operands,
3833 const Loop *L,
3834 SCEV::NoWrapFlags Flags) {
3835 if (Operands.size() == 1) return Operands[0];
3836#ifndef NDEBUG
3837 Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
3838 for (const SCEV *Op : llvm::drop_begin(Operands)) {
3839 assert(getEffectiveSCEVType(Op->getType()) == ETy &&
3840 "SCEVAddRecExpr operand types don't match!");
3841 assert(!Op->getType()->isPointerTy() && "Step must be integer");
3842 }
3843 for (const SCEV *Op : Operands)
3844 assert(isAvailableAtLoopEntry(Op, L) &&
3845 "SCEVAddRecExpr operand is not available at loop entry!");
3846#endif
3847
3848 if (Operands.back()->isZero()) {
3849 Operands.pop_back();
3850 return getAddRecExpr(Operands, L, Flags: SCEV::FlagAnyWrap); // {X,+,0} --> X
3851 }
3852
3853 // It's tempting to want to call getConstantMaxBackedgeTakenCount count here and
3854 // use that information to infer NUW and NSW flags. However, computing a
3855 // BE count requires calling getAddRecExpr, so we may not yet have a
3856 // meaningful BE count at this point (and if we don't, we'd be stuck
3857 // with a SCEVCouldNotCompute as the cached BE count).
3858
3859 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
3860
3861 // Canonicalize nested AddRecs in by nesting them in order of loop depth.
3862 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Val&: Operands[0])) {
3863 const Loop *NestedLoop = NestedAR->getLoop();
3864 if (L->contains(L: NestedLoop)
3865 ? (L->getLoopDepth() < NestedLoop->getLoopDepth())
3866 : (!NestedLoop->contains(L) &&
3867 DT.dominates(A: L->getHeader(), B: NestedLoop->getHeader()))) {
3868 SmallVector<SCEVUse, 4> NestedOperands(NestedAR->operands());
3869 Operands[0] = NestedAR->getStart();
3870 // AddRecs require their operands be loop-invariant with respect to their
3871 // loops. Don't perform this transformation if it would break this
3872 // requirement.
3873 bool AllInvariant = all_of(
3874 Range&: Operands, P: [&](const SCEV *Op) { return isLoopInvariant(S: Op, L); });
3875
3876 if (AllInvariant) {
3877 // Create a recurrence for the outer loop with the same step size.
3878 //
3879 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
3880 // inner recurrence has the same property.
3881 SCEV::NoWrapFlags OuterFlags =
3882 maskFlags(Flags, Mask: SCEV::FlagNW | NestedAR->getNoWrapFlags());
3883
3884 NestedOperands[0] = getAddRecExpr(Operands, L, Flags: OuterFlags);
3885 AllInvariant = all_of(Range&: NestedOperands, P: [&](const SCEV *Op) {
3886 return isLoopInvariant(S: Op, L: NestedLoop);
3887 });
3888
3889 if (AllInvariant) {
3890 // Ok, both add recurrences are valid after the transformation.
3891 //
3892 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
3893 // the outer recurrence has the same property.
3894 SCEV::NoWrapFlags InnerFlags =
3895 maskFlags(Flags: NestedAR->getNoWrapFlags(), Mask: SCEV::FlagNW | Flags);
3896 return getAddRecExpr(Operands&: NestedOperands, L: NestedLoop, Flags: InnerFlags);
3897 }
3898 }
3899 // Reset Operands to its original state.
3900 Operands[0] = NestedAR;
3901 }
3902 }
3903
3904 // Okay, it looks like we really DO need an addrec expr. Check to see if we
3905 // already have one, otherwise create a new one.
3906 return getOrCreateAddRecExpr(Ops: Operands, L, Flags);
3907}
3908
3909const SCEV *ScalarEvolution::getGEPExpr(GEPOperator *GEP,
3910 ArrayRef<SCEVUse> IndexExprs) {
3911 const SCEV *BaseExpr = getSCEV(V: GEP->getPointerOperand());
3912 // getSCEV(Base)->getType() has the same address space as Base->getType()
3913 // because SCEV::getType() preserves the address space.
3914 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
3915 if (NW != GEPNoWrapFlags::none()) {
3916 // We'd like to propagate flags from the IR to the corresponding SCEV nodes,
3917 // but to do that, we have to ensure that said flag is valid in the entire
3918 // defined scope of the SCEV.
3919 // TODO: non-instructions have global scope. We might be able to prove
3920 // some global scope cases
3921 auto *GEPI = dyn_cast<Instruction>(Val: GEP);
3922 if (!GEPI || !isSCEVExprNeverPoison(I: GEPI))
3923 NW = GEPNoWrapFlags::none();
3924 }
3925
3926 return getGEPExpr(BaseExpr, IndexExprs, SrcElementTy: GEP->getSourceElementType(), NW);
3927}
3928
3929const SCEV *ScalarEvolution::getGEPExpr(SCEVUse BaseExpr,
3930 ArrayRef<SCEVUse> IndexExprs,
3931 Type *SrcElementTy, GEPNoWrapFlags NW) {
3932 SCEV::NoWrapFlags OffsetWrap = SCEV::FlagAnyWrap;
3933 if (NW.hasNoUnsignedSignedWrap())
3934 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNSW);
3935 if (NW.hasNoUnsignedWrap())
3936 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNUW);
3937
3938 Type *CurTy = BaseExpr->getType();
3939 Type *IntIdxTy = getEffectiveSCEVType(Ty: BaseExpr->getType());
3940 bool FirstIter = true;
3941 SmallVector<SCEVUse, 4> Offsets;
3942 for (SCEVUse IndexExpr : IndexExprs) {
3943 // Compute the (potentially symbolic) offset in bytes for this index.
3944 if (StructType *STy = dyn_cast<StructType>(Val: CurTy)) {
3945 // For a struct, add the member offset.
3946 ConstantInt *Index = cast<SCEVConstant>(Val&: IndexExpr)->getValue();
3947 unsigned FieldNo = Index->getZExtValue();
3948 const SCEV *FieldOffset = getOffsetOfExpr(IntTy: IntIdxTy, STy, FieldNo);
3949 Offsets.push_back(Elt: FieldOffset);
3950
3951 // Update CurTy to the type of the field at Index.
3952 CurTy = STy->getTypeAtIndex(V: Index);
3953 } else {
3954 // Update CurTy to its element type.
3955 if (FirstIter) {
3956 assert(isa<PointerType>(CurTy) &&
3957 "The first index of a GEP indexes a pointer");
3958 CurTy = SrcElementTy;
3959 FirstIter = false;
3960 } else {
3961 CurTy = GetElementPtrInst::getTypeAtIndex(Ty: CurTy, Idx: (uint64_t)0);
3962 }
3963 // For an array, add the element offset, explicitly scaled.
3964 const SCEV *ElementSize = getSizeOfExpr(IntTy: IntIdxTy, AllocTy: CurTy);
3965 // Getelementptr indices are signed.
3966 IndexExpr = getTruncateOrSignExtend(V: IndexExpr, Ty: IntIdxTy);
3967
3968 // Multiply the index by the element size to compute the element offset.
3969 const SCEV *LocalOffset = getMulExpr(LHS: IndexExpr, RHS: ElementSize, Flags: OffsetWrap);
3970 Offsets.push_back(Elt: LocalOffset);
3971 }
3972 }
3973
3974 // Handle degenerate case of GEP without offsets.
3975 if (Offsets.empty())
3976 return BaseExpr;
3977
3978 // Add the offsets together, assuming nsw if inbounds.
3979 const SCEV *Offset = getAddExpr(Ops&: Offsets, OrigFlags: OffsetWrap);
3980 // Add the base address and the offset. We cannot use the nsw flag, as the
3981 // base address is unsigned. However, if we know that the offset is
3982 // non-negative, we can use nuw.
3983 bool NUW = NW.hasNoUnsignedWrap() ||
3984 (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(S: Offset));
3985 SCEV::NoWrapFlags BaseWrap = NUW ? SCEV::FlagNUW : SCEV::FlagAnyWrap;
3986 auto *GEPExpr = getAddExpr(LHS: BaseExpr, RHS: Offset, Flags: BaseWrap);
3987 assert(BaseExpr->getType() == GEPExpr->getType() &&
3988 "GEP should not change type mid-flight.");
3989 return GEPExpr;
3990}
3991
3992SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
3993 ArrayRef<const SCEV *> Ops) {
3994 FoldingSetNodeID ID;
3995 ID.AddInteger(I: SCEVType);
3996 for (const SCEV *Op : Ops)
3997 ID.AddPointer(Ptr: Op);
3998 void *IP = nullptr;
3999 return UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP);
4000}
4001
4002SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
4003 ArrayRef<SCEVUse> Ops) {
4004 FoldingSetNodeID ID;
4005 ID.AddInteger(I: SCEVType);
4006 for (const SCEV *Op : Ops)
4007 ID.AddPointer(Ptr: Op);
4008 void *IP = nullptr;
4009 return UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP);
4010}
4011
4012const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
4013 SCEV::NoWrapFlags Flags = IsNSW ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
4014 return getSMaxExpr(LHS: Op, RHS: getNegativeSCEV(V: Op, Flags));
4015}
4016
4017const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
4018 SmallVectorImpl<SCEVUse> &Ops) {
4019 assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!");
4020 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
4021 if (Ops.size() == 1) return Ops[0];
4022#ifndef NDEBUG
4023 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4024 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4025 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
4026 "Operand types don't match!");
4027 assert(Ops[0]->getType()->isPointerTy() ==
4028 Ops[i]->getType()->isPointerTy() &&
4029 "min/max should be consistently pointerish");
4030 }
4031#endif
4032
4033 bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr;
4034 bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr;
4035
4036 const SCEV *Folded = constantFoldAndGroupOps(
4037 SE&: *this, LI, DT, Ops,
4038 Fold: [&](const APInt &C1, const APInt &C2) {
4039 switch (Kind) {
4040 case scSMaxExpr:
4041 return APIntOps::smax(A: C1, B: C2);
4042 case scSMinExpr:
4043 return APIntOps::smin(A: C1, B: C2);
4044 case scUMaxExpr:
4045 return APIntOps::umax(A: C1, B: C2);
4046 case scUMinExpr:
4047 return APIntOps::umin(A: C1, B: C2);
4048 default:
4049 llvm_unreachable("Unknown SCEV min/max opcode");
4050 }
4051 },
4052 IsIdentity: [&](const APInt &C) {
4053 // identity
4054 if (IsMax)
4055 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
4056 else
4057 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
4058 },
4059 IsAbsorber: [&](const APInt &C) {
4060 // absorber
4061 if (IsMax)
4062 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
4063 else
4064 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
4065 });
4066 if (Folded)
4067 return Folded;
4068
4069 // Check if we have created the same expression before.
4070 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops)) {
4071 return S;
4072 }
4073
4074 // Find the first operation of the same kind
4075 unsigned Idx = 0;
4076 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind)
4077 ++Idx;
4078
4079 // Check to see if one of the operands is of the same kind. If so, expand its
4080 // operands onto our operand list, and recurse to simplify.
4081 if (Idx < Ops.size()) {
4082 bool DeletedAny = false;
4083 while (Ops[Idx]->getSCEVType() == Kind) {
4084 const SCEVMinMaxExpr *SMME = cast<SCEVMinMaxExpr>(Val&: Ops[Idx]);
4085 Ops.erase(CI: Ops.begin()+Idx);
4086 append_range(C&: Ops, R: SMME->operands());
4087 DeletedAny = true;
4088 }
4089
4090 if (DeletedAny)
4091 return getMinMaxExpr(Kind, Ops);
4092 }
4093
4094 // Okay, check to see if the same value occurs in the operand list twice. If
4095 // so, delete one. Since we sorted the list, these values are required to
4096 // be adjacent.
4097 llvm::CmpInst::Predicate GEPred =
4098 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
4099 llvm::CmpInst::Predicate LEPred =
4100 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
4101 llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred;
4102 llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred;
4103 for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) {
4104 if (Ops[i] == Ops[i + 1] ||
4105 isKnownViaNonRecursiveReasoning(Pred: FirstPred, LHS: Ops[i], RHS: Ops[i + 1])) {
4106 // X op Y op Y --> X op Y
4107 // X op Y --> X, if we know X, Y are ordered appropriately
4108 Ops.erase(CS: Ops.begin() + i + 1, CE: Ops.begin() + i + 2);
4109 --i;
4110 --e;
4111 } else if (isKnownViaNonRecursiveReasoning(Pred: SecondPred, LHS: Ops[i],
4112 RHS: Ops[i + 1])) {
4113 // X op Y --> Y, if we know X, Y are ordered appropriately
4114 Ops.erase(CS: Ops.begin() + i, CE: Ops.begin() + i + 1);
4115 --i;
4116 --e;
4117 }
4118 }
4119
4120 if (Ops.size() == 1) return Ops[0];
4121
4122 assert(!Ops.empty() && "Reduced smax down to nothing!");
4123
4124 // Okay, it looks like we really DO need an expr. Check to see if we
4125 // already have one, otherwise create a new one.
4126 FoldingSetNodeID ID;
4127 ID.AddInteger(I: Kind);
4128 for (const SCEV *Op : Ops)
4129 ID.AddPointer(Ptr: Op);
4130 void *IP = nullptr;
4131 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP);
4132 if (ExistingSCEV)
4133 return ExistingSCEV;
4134 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
4135 llvm::uninitialized_copy(Src&: Ops, Dst: O);
4136 SCEV *S = new (SCEVAllocator)
4137 SCEVMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
4138
4139 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
4140 S->computeAndSetCanonical(SE&: *this);
4141 registerUser(User: S, Ops);
4142 return S;
4143}
4144
4145namespace {
4146
4147class SCEVSequentialMinMaxDeduplicatingVisitor final
4148 : public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
4149 std::optional<const SCEV *>> {
4150 using RetVal = std::optional<const SCEV *>;
4151 using Base = SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor, RetVal>;
4152
4153 ScalarEvolution &SE;
4154 const SCEVTypes RootKind; // Must be a sequential min/max expression.
4155 const SCEVTypes NonSequentialRootKind; // Non-sequential variant of RootKind.
4156 SmallPtrSet<const SCEV *, 16> SeenOps;
4157
4158 bool canRecurseInto(SCEVTypes Kind) const {
4159 // We can only recurse into the SCEV expression of the same effective type
4160 // as the type of our root SCEV expression.
4161 return RootKind == Kind || NonSequentialRootKind == Kind;
4162 };
4163
4164 RetVal visitAnyMinMaxExpr(const SCEV *S) {
4165 assert((isa<SCEVMinMaxExpr>(S) || isa<SCEVSequentialMinMaxExpr>(S)) &&
4166 "Only for min/max expressions.");
4167 SCEVTypes Kind = S->getSCEVType();
4168
4169 if (!canRecurseInto(Kind))
4170 return S;
4171
4172 auto *NAry = cast<SCEVNAryExpr>(Val: S);
4173 SmallVector<SCEVUse> NewOps;
4174 bool Changed = visit(Kind, OrigOps: NAry->operands(), NewOps);
4175
4176 if (!Changed)
4177 return S;
4178 if (NewOps.empty())
4179 return std::nullopt;
4180
4181 return isa<SCEVSequentialMinMaxExpr>(Val: S)
4182 ? SE.getSequentialMinMaxExpr(Kind, Operands&: NewOps)
4183 : SE.getMinMaxExpr(Kind, Ops&: NewOps);
4184 }
4185
4186 RetVal visit(const SCEV *S) {
4187 // Has the whole operand been seen already?
4188 if (!SeenOps.insert(Ptr: S).second)
4189 return std::nullopt;
4190 return Base::visit(S);
4191 }
4192
4193public:
4194 SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
4195 SCEVTypes RootKind)
4196 : SE(SE), RootKind(RootKind),
4197 NonSequentialRootKind(
4198 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
4199 Ty: RootKind)) {}
4200
4201 bool /*Changed*/ visit(SCEVTypes Kind, ArrayRef<SCEVUse> OrigOps,
4202 SmallVectorImpl<SCEVUse> &NewOps) {
4203 bool Changed = false;
4204 SmallVector<SCEVUse> Ops;
4205 Ops.reserve(N: OrigOps.size());
4206
4207 for (const SCEV *Op : OrigOps) {
4208 RetVal NewOp = visit(S: Op);
4209 if (NewOp != Op)
4210 Changed = true;
4211 if (NewOp)
4212 Ops.emplace_back(Args&: *NewOp);
4213 }
4214
4215 if (Changed)
4216 NewOps = std::move(Ops);
4217 return Changed;
4218 }
4219
4220 RetVal visitConstant(const SCEVConstant *Constant) { return Constant; }
4221
4222 RetVal visitVScale(const SCEVVScale *VScale) { return VScale; }
4223
4224 RetVal visitPtrToAddrExpr(const SCEVPtrToAddrExpr *Expr) { return Expr; }
4225
4226 RetVal visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { return Expr; }
4227
4228 RetVal visitTruncateExpr(const SCEVTruncateExpr *Expr) { return Expr; }
4229
4230 RetVal visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { return Expr; }
4231
4232 RetVal visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { return Expr; }
4233
4234 RetVal visitAddExpr(const SCEVAddExpr *Expr) { return Expr; }
4235
4236 RetVal visitMulExpr(const SCEVMulExpr *Expr) { return Expr; }
4237
4238 RetVal visitUDivExpr(const SCEVUDivExpr *Expr) { return Expr; }
4239
4240 RetVal visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
4241
4242 RetVal visitSMaxExpr(const SCEVSMaxExpr *Expr) {
4243 return visitAnyMinMaxExpr(S: Expr);
4244 }
4245
4246 RetVal visitUMaxExpr(const SCEVUMaxExpr *Expr) {
4247 return visitAnyMinMaxExpr(S: Expr);
4248 }
4249
4250 RetVal visitSMinExpr(const SCEVSMinExpr *Expr) {
4251 return visitAnyMinMaxExpr(S: Expr);
4252 }
4253
4254 RetVal visitUMinExpr(const SCEVUMinExpr *Expr) {
4255 return visitAnyMinMaxExpr(S: Expr);
4256 }
4257
4258 RetVal visitSequentialUMinExpr(const SCEVSequentialUMinExpr *Expr) {
4259 return visitAnyMinMaxExpr(S: Expr);
4260 }
4261
4262 RetVal visitUnknown(const SCEVUnknown *Expr) { return Expr; }
4263
4264 RetVal visitCouldNotCompute(const SCEVCouldNotCompute *Expr) { return Expr; }
4265};
4266
4267} // namespace
4268
4269static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind) {
4270 switch (Kind) {
4271 case scConstant:
4272 case scVScale:
4273 case scTruncate:
4274 case scZeroExtend:
4275 case scSignExtend:
4276 case scPtrToAddr:
4277 case scPtrToInt:
4278 case scAddExpr:
4279 case scMulExpr:
4280 case scUDivExpr:
4281 case scAddRecExpr:
4282 case scUMaxExpr:
4283 case scSMaxExpr:
4284 case scUMinExpr:
4285 case scSMinExpr:
4286 case scUnknown:
4287 // If any operand is poison, the whole expression is poison.
4288 return true;
4289 case scSequentialUMinExpr:
4290 // FIXME: if the *first* operand is poison, the whole expression is poison.
4291 return false; // Pessimistically, say that it does not propagate poison.
4292 case scCouldNotCompute:
4293 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
4294 }
4295 llvm_unreachable("Unknown SCEV kind!");
4296}
4297
4298namespace {
4299// The only way poison may be introduced in a SCEV expression is from a
4300// poison SCEVUnknown (ConstantExprs are also represented as SCEVUnknown,
4301// not SCEVConstant). Notably, nowrap flags in SCEV nodes can *not*
4302// introduce poison -- they encode guaranteed, non-speculated knowledge.
4303//
4304// Additionally, all SCEV nodes propagate poison from inputs to outputs,
4305// with the notable exception of umin_seq, where only poison from the first
4306// operand is (unconditionally) propagated.
4307struct SCEVPoisonCollector {
4308 bool LookThroughMaybePoisonBlocking;
4309 SmallPtrSet<const SCEVUnknown *, 4> MaybePoison;
4310 SCEVPoisonCollector(bool LookThroughMaybePoisonBlocking)
4311 : LookThroughMaybePoisonBlocking(LookThroughMaybePoisonBlocking) {}
4312
4313 bool follow(const SCEV *S) {
4314 if (!LookThroughMaybePoisonBlocking &&
4315 !scevUnconditionallyPropagatesPoisonFromOperands(Kind: S->getSCEVType()))
4316 return false;
4317
4318 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
4319 if (!isGuaranteedNotToBePoison(V: SU->getValue()))
4320 MaybePoison.insert(Ptr: SU);
4321 }
4322 return true;
4323 }
4324 bool isDone() const { return false; }
4325};
4326} // namespace
4327
4328/// Return true if V is poison given that AssumedPoison is already poison.
4329static bool impliesPoison(const SCEV *AssumedPoison, const SCEV *S) {
4330 // First collect all SCEVs that might result in AssumedPoison to be poison.
4331 // We need to look through potentially poison-blocking operations here,
4332 // because we want to find all SCEVs that *might* result in poison, not only
4333 // those that are *required* to.
4334 SCEVPoisonCollector PC1(/* LookThroughMaybePoisonBlocking */ true);
4335 visitAll(Root: AssumedPoison, Visitor&: PC1);
4336
4337 // AssumedPoison is never poison. As the assumption is false, the implication
4338 // is true. Don't bother walking the other SCEV in this case.
4339 if (PC1.MaybePoison.empty())
4340 return true;
4341
4342 // Collect all SCEVs in S that, if poison, *will* result in S being poison
4343 // as well. We cannot look through potentially poison-blocking operations
4344 // here, as their arguments only *may* make the result poison.
4345 SCEVPoisonCollector PC2(/* LookThroughMaybePoisonBlocking */ false);
4346 visitAll(Root: S, Visitor&: PC2);
4347
4348 // Make sure that no matter which SCEV in PC1.MaybePoison is actually poison,
4349 // it will also make S poison by being part of PC2.MaybePoison.
4350 return llvm::set_is_subset(S1: PC1.MaybePoison, S2: PC2.MaybePoison);
4351}
4352
4353void ScalarEvolution::getPoisonGeneratingValues(
4354 SmallPtrSetImpl<const Value *> &Result, const SCEV *S) {
4355 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ false);
4356 visitAll(Root: S, Visitor&: PC);
4357 for (const SCEVUnknown *SU : PC.MaybePoison)
4358 Result.insert(Ptr: SU->getValue());
4359}
4360
4361bool ScalarEvolution::canReuseInstruction(
4362 const SCEV *S, Instruction *I,
4363 SmallVectorImpl<Instruction *> &DropPoisonGeneratingInsts) {
4364 // If the instruction cannot be poison, it's always safe to reuse.
4365 if (programUndefinedIfPoison(Inst: I))
4366 return true;
4367
4368 // Otherwise, it is possible that I is more poisonous that S. Collect the
4369 // poison-contributors of S, and then check whether I has any additional
4370 // poison-contributors. Poison that is contributed through poison-generating
4371 // flags is handled by dropping those flags instead.
4372 SmallPtrSet<const Value *, 8> PoisonVals;
4373 getPoisonGeneratingValues(Result&: PoisonVals, S);
4374
4375 SmallVector<Value *> Worklist;
4376 SmallPtrSet<Value *, 8> Visited;
4377 Worklist.push_back(Elt: I);
4378 while (!Worklist.empty()) {
4379 Value *V = Worklist.pop_back_val();
4380 if (!Visited.insert(Ptr: V).second)
4381 continue;
4382
4383 // Avoid walking large instruction graphs.
4384 if (Visited.size() > 16)
4385 return false;
4386
4387 // Either the value can't be poison, or the S would also be poison if it
4388 // is.
4389 if (PoisonVals.contains(Ptr: V) || ::isGuaranteedNotToBePoison(V))
4390 continue;
4391
4392 auto *I = dyn_cast<Instruction>(Val: V);
4393 if (!I)
4394 return false;
4395
4396 // Disjoint or instructions are interpreted as adds by SCEV. However, we
4397 // can't replace an arbitrary add with disjoint or, even if we drop the
4398 // flag. We would need to convert the or into an add.
4399 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: I))
4400 if (PDI->isDisjoint())
4401 return false;
4402
4403 // FIXME: Ignore vscale, even though it technically could be poison. Do this
4404 // because SCEV currently assumes it can't be poison. Remove this special
4405 // case once we proper model when vscale can be poison.
4406 if (auto *II = dyn_cast<IntrinsicInst>(Val: I);
4407 II && II->getIntrinsicID() == Intrinsic::vscale)
4408 continue;
4409
4410 if (canCreatePoison(Op: cast<Operator>(Val: I), /*ConsiderFlagsAndMetadata*/ false))
4411 return false;
4412
4413 // If the instruction can't create poison, we can recurse to its operands.
4414 if (I->hasPoisonGeneratingAnnotations())
4415 DropPoisonGeneratingInsts.push_back(Elt: I);
4416
4417 llvm::append_range(C&: Worklist, R: I->operands());
4418 }
4419 return true;
4420}
4421
4422const SCEV *
4423ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
4424 SmallVectorImpl<SCEVUse> &Ops) {
4425 assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&
4426 "Not a SCEVSequentialMinMaxExpr!");
4427 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
4428 if (Ops.size() == 1)
4429 return Ops[0];
4430#ifndef NDEBUG
4431 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4432 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4433 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
4434 "Operand types don't match!");
4435 assert(Ops[0]->getType()->isPointerTy() ==
4436 Ops[i]->getType()->isPointerTy() &&
4437 "min/max should be consistently pointerish");
4438 }
4439#endif
4440
4441 // Note that SCEVSequentialMinMaxExpr is *NOT* commutative,
4442 // so we can *NOT* do any kind of sorting of the expressions!
4443
4444 // Check if we have created the same expression before.
4445 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops))
4446 return S;
4447
4448 // FIXME: there are *some* simplifications that we can do here.
4449
4450 // Keep only the first instance of an operand.
4451 {
4452 SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*this, Kind);
4453 bool Changed = Deduplicator.visit(Kind, OrigOps: Ops, NewOps&: Ops);
4454 if (Changed)
4455 return getSequentialMinMaxExpr(Kind, Ops);
4456 }
4457
4458 // Check to see if one of the operands is of the same kind. If so, expand its
4459 // operands onto our operand list, and recurse to simplify.
4460 {
4461 unsigned Idx = 0;
4462 bool DeletedAny = false;
4463 while (Idx < Ops.size()) {
4464 if (Ops[Idx]->getSCEVType() != Kind) {
4465 ++Idx;
4466 continue;
4467 }
4468 const auto *SMME = cast<SCEVSequentialMinMaxExpr>(Val&: Ops[Idx]);
4469 Ops.erase(CI: Ops.begin() + Idx);
4470 Ops.insert(I: Ops.begin() + Idx, From: SMME->operands().begin(),
4471 To: SMME->operands().end());
4472 DeletedAny = true;
4473 }
4474
4475 if (DeletedAny)
4476 return getSequentialMinMaxExpr(Kind, Ops);
4477 }
4478
4479 const SCEV *SaturationPoint;
4480 ICmpInst::Predicate Pred;
4481 switch (Kind) {
4482 case scSequentialUMinExpr:
4483 SaturationPoint = getZero(Ty: Ops[0]->getType());
4484 Pred = ICmpInst::ICMP_ULE;
4485 break;
4486 default:
4487 llvm_unreachable("Not a sequential min/max type.");
4488 }
4489
4490 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4491 if (!isGuaranteedNotToCauseUB(Op: Ops[i]))
4492 continue;
4493 // We can replace %x umin_seq %y with %x umin %y if either:
4494 // * %y being poison implies %x is also poison.
4495 // * %x cannot be the saturating value (e.g. zero for umin).
4496 if (::impliesPoison(AssumedPoison: Ops[i], S: Ops[i - 1]) ||
4497 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_NE, LHS: Ops[i - 1],
4498 RHS: SaturationPoint)) {
4499 SmallVector<SCEVUse, 2> SeqOps = {Ops[i - 1], Ops[i]};
4500 Ops[i - 1] = getMinMaxExpr(
4501 Kind: SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Ty: Kind),
4502 Ops&: SeqOps);
4503 Ops.erase(CI: Ops.begin() + i);
4504 return getSequentialMinMaxExpr(Kind, Ops);
4505 }
4506 // Fold %x umin_seq %y to %x if %x ule %y.
4507 // TODO: We might be able to prove the predicate for a later operand.
4508 if (isKnownViaNonRecursiveReasoning(Pred, LHS: Ops[i - 1], RHS: Ops[i])) {
4509 Ops.erase(CI: Ops.begin() + i);
4510 return getSequentialMinMaxExpr(Kind, Ops);
4511 }
4512 }
4513
4514 // Okay, it looks like we really DO need an expr. Check to see if we
4515 // already have one, otherwise create a new one.
4516 FoldingSetNodeID ID;
4517 ID.AddInteger(I: Kind);
4518 for (const SCEV *Op : Ops)
4519 ID.AddPointer(Ptr: Op);
4520 void *IP = nullptr;
4521 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP);
4522 if (ExistingSCEV)
4523 return ExistingSCEV;
4524
4525 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
4526 llvm::uninitialized_copy(Src&: Ops, Dst: O);
4527 SCEV *S = new (SCEVAllocator)
4528 SCEVSequentialMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
4529
4530 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
4531 S->computeAndSetCanonical(SE&: *this);
4532 registerUser(User: S, Ops);
4533 return S;
4534}
4535
4536const SCEV *ScalarEvolution::getSMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4537 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4538 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4539}
4540
4541const SCEV *ScalarEvolution::getSMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4542 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4543}
4544
4545const SCEV *ScalarEvolution::getUMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4546 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4547 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4548}
4549
4550const SCEV *ScalarEvolution::getUMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4551 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4552}
4553
4554const SCEV *ScalarEvolution::getSMinExpr(SCEVUse LHS, SCEVUse RHS) {
4555 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4556 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4557}
4558
4559const SCEV *ScalarEvolution::getSMinExpr(SmallVectorImpl<SCEVUse> &Ops) {
4560 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4561}
4562
4563const SCEV *ScalarEvolution::getUMinExpr(SCEVUse LHS, SCEVUse RHS,
4564 bool Sequential) {
4565 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4566 return getUMinExpr(Operands&: Ops, Sequential);
4567}
4568
4569const SCEV *ScalarEvolution::getUMinExpr(SmallVectorImpl<SCEVUse> &Ops,
4570 bool Sequential) {
4571 return Sequential ? getSequentialMinMaxExpr(Kind: scSequentialUMinExpr, Ops)
4572 : getMinMaxExpr(Kind: scUMinExpr, Ops);
4573}
4574
4575const SCEV *
4576ScalarEvolution::getSizeOfExpr(Type *IntTy, TypeSize Size) {
4577 const SCEV *Res = getConstant(Ty: IntTy, V: Size.getKnownMinValue());
4578 if (Size.isScalable())
4579 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty: IntTy));
4580 return Res;
4581}
4582
4583const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
4584 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeAllocSize(Ty: AllocTy));
4585}
4586
4587const SCEV *ScalarEvolution::getStoreSizeOfExpr(Type *IntTy, Type *StoreTy) {
4588 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeStoreSize(Ty: StoreTy));
4589}
4590
4591const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
4592 StructType *STy,
4593 unsigned FieldNo) {
4594 // We can bypass creating a target-independent constant expression and then
4595 // folding it back into a ConstantInt. This is just a compile-time
4596 // optimization.
4597 const StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
4598 assert(!SL->getSizeInBits().isScalable() &&
4599 "Cannot get offset for structure containing scalable vector types");
4600 return getConstant(Ty: IntTy, V: SL->getElementOffset(Idx: FieldNo));
4601}
4602
4603const SCEV *ScalarEvolution::getUnknown(Value *V) {
4604 // Don't attempt to do anything other than create a SCEVUnknown object
4605 // here. createSCEV only calls getUnknown after checking for all other
4606 // interesting possibilities, and any other code that calls getUnknown
4607 // is doing so in order to hide a value from SCEV canonicalization.
4608
4609 FoldingSetNodeID ID;
4610 ID.AddInteger(I: scUnknown);
4611 ID.AddPointer(Ptr: V);
4612 void *IP = nullptr;
4613 if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) {
4614 assert(cast<SCEVUnknown>(S)->getValue() == V &&
4615 "Stale SCEVUnknown in uniquing map!");
4616 return S;
4617 }
4618 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(Allocator&: SCEVAllocator), V, this,
4619 FirstUnknown);
4620 FirstUnknown = cast<SCEVUnknown>(Val: S);
4621 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
4622 S->computeAndSetCanonical(SE&: *this);
4623 return S;
4624}
4625
4626//===----------------------------------------------------------------------===//
4627// Basic SCEV Analysis and PHI Idiom Recognition Code
4628//
4629
4630/// Test if values of the given type are analyzable within the SCEV
4631/// framework. This primarily includes integer types, and it can optionally
4632/// include pointer types if the ScalarEvolution class has access to
4633/// target-specific information.
4634bool ScalarEvolution::isSCEVable(Type *Ty) const {
4635 // Integers and pointers are always SCEVable.
4636 return Ty->isIntOrPtrTy();
4637}
4638
4639/// Return the size in bits of the specified type, for which isSCEVable must
4640/// return true.
4641uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
4642 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4643 if (Ty->isPointerTy())
4644 return getDataLayout().getIndexTypeSizeInBits(Ty);
4645 return getDataLayout().getTypeSizeInBits(Ty);
4646}
4647
4648/// Return a type with the same bitwidth as the given type and which represents
4649/// how SCEV will treat the given type, for which isSCEVable must return
4650/// true. For pointer types, this is the pointer index sized integer type.
4651Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
4652 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4653
4654 if (Ty->isIntegerTy())
4655 return Ty;
4656
4657 // The only other support type is pointer.
4658 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
4659 return getDataLayout().getIndexType(PtrTy: Ty);
4660}
4661
4662Type *ScalarEvolution::getWiderType(Type *T1, Type *T2) const {
4663 return getTypeSizeInBits(Ty: T1) >= getTypeSizeInBits(Ty: T2) ? T1 : T2;
4664}
4665
4666bool ScalarEvolution::instructionCouldExistWithOperands(const SCEV *A,
4667 const SCEV *B) {
4668 /// For a valid use point to exist, the defining scope of one operand
4669 /// must dominate the other.
4670 bool PreciseA, PreciseB;
4671 auto *ScopeA = getDefiningScopeBound(Ops: {A}, Precise&: PreciseA);
4672 auto *ScopeB = getDefiningScopeBound(Ops: {B}, Precise&: PreciseB);
4673 if (!PreciseA || !PreciseB)
4674 // Can't tell.
4675 return false;
4676 return (ScopeA == ScopeB) || DT.dominates(Def: ScopeA, User: ScopeB) ||
4677 DT.dominates(Def: ScopeB, User: ScopeA);
4678}
4679
4680const SCEV *ScalarEvolution::getCouldNotCompute() {
4681 return CouldNotCompute.get();
4682}
4683
4684bool ScalarEvolution::checkValidity(const SCEV *S) const {
4685 bool ContainsNulls = SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
4686 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
4687 return SU && SU->getValue() == nullptr;
4688 });
4689
4690 return !ContainsNulls;
4691}
4692
4693bool ScalarEvolution::containsAddRecurrence(const SCEV *S) {
4694 HasRecMapType::iterator I = HasRecMap.find(Val: S);
4695 if (I != HasRecMap.end())
4696 return I->second;
4697
4698 bool FoundAddRec =
4699 SCEVExprContains(Root: S, Pred: [](const SCEV *S) { return isa<SCEVAddRecExpr>(Val: S); });
4700 HasRecMap.insert(KV: {S, FoundAddRec});
4701 return FoundAddRec;
4702}
4703
4704/// Return the ValueOffsetPair set for \p S. \p S can be represented
4705/// by the value and offset from any ValueOffsetPair in the set.
4706ArrayRef<Value *> ScalarEvolution::getSCEVValues(const SCEV *S) {
4707 ExprValueMapType::iterator SI = ExprValueMap.find_as(Val: S);
4708 if (SI == ExprValueMap.end())
4709 return {};
4710 return SI->second.getArrayRef();
4711}
4712
4713/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V)
4714/// cannot be used separately. eraseValueFromMap should be used to remove
4715/// V from ValueExprMap and ExprValueMap at the same time.
4716void ScalarEvolution::eraseValueFromMap(Value *V) {
4717 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4718 if (I != ValueExprMap.end()) {
4719 auto EVIt = ExprValueMap.find(Val: I->second);
4720 bool Removed = EVIt->second.remove(X: V);
4721 (void) Removed;
4722 assert(Removed && "Value not in ExprValueMap?");
4723 ValueExprMap.erase(I);
4724 }
4725}
4726
4727void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
4728 // A recursive query may have already computed the SCEV. It should be
4729 // equivalent, but may not necessarily be exactly the same, e.g. due to lazily
4730 // inferred nowrap flags.
4731 auto It = ValueExprMap.find_as(Val: V);
4732 if (It == ValueExprMap.end()) {
4733 ValueExprMap.insert(KV: {SCEVCallbackVH(V, this), S});
4734 ExprValueMap[S].insert(X: V);
4735 }
4736}
4737
4738/// Return an existing SCEV if it exists, otherwise analyze the expression and
4739/// create a new one.
4740const SCEV *ScalarEvolution::getSCEV(Value *V) {
4741 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4742
4743 if (const SCEV *S = getExistingSCEV(V))
4744 return S;
4745 return createSCEVIter(V);
4746}
4747
4748const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
4749 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4750
4751 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4752 if (I != ValueExprMap.end()) {
4753 const SCEV *S = I->second;
4754 assert(checkValidity(S) &&
4755 "existing SCEV has not been properly invalidated");
4756 return S;
4757 }
4758 return nullptr;
4759}
4760
4761/// Return a SCEV corresponding to -V = -1*V
4762const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V,
4763 SCEV::NoWrapFlags Flags) {
4764 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4765 return getConstant(
4766 V: cast<ConstantInt>(Val: ConstantExpr::getNeg(C: VC->getValue())));
4767
4768 Type *Ty = V->getType();
4769 Ty = getEffectiveSCEVType(Ty);
4770 return getMulExpr(LHS: V, RHS: getMinusOne(Ty), Flags);
4771}
4772
4773/// If Expr computes ~A, return A else return nullptr
4774static const SCEV *MatchNotExpr(const SCEV *Expr) {
4775 const SCEV *MulOp;
4776 if (match(S: Expr, P: m_scev_Add(Op0: m_scev_AllOnes(),
4777 Op1: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: MulOp)))))
4778 return MulOp;
4779 return nullptr;
4780}
4781
4782/// Return a SCEV corresponding to ~V = -1-V
4783const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
4784 assert(!V->getType()->isPointerTy() && "Can't negate pointer");
4785
4786 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4787 return getConstant(
4788 V: cast<ConstantInt>(Val: ConstantExpr::getNot(C: VC->getValue())));
4789
4790 // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
4791 if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(Val: V)) {
4792 auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
4793 SmallVector<SCEVUse, 2> MatchedOperands;
4794 for (const SCEV *Operand : MME->operands()) {
4795 const SCEV *Matched = MatchNotExpr(Expr: Operand);
4796 if (!Matched)
4797 return (const SCEV *)nullptr;
4798 MatchedOperands.push_back(Elt: Matched);
4799 }
4800 return getMinMaxExpr(Kind: SCEVMinMaxExpr::negate(T: MME->getSCEVType()),
4801 Ops&: MatchedOperands);
4802 };
4803 if (const SCEV *Replaced = MatchMinMaxNegation(MME))
4804 return Replaced;
4805 }
4806
4807 Type *Ty = V->getType();
4808 Ty = getEffectiveSCEVType(Ty);
4809 return getMinusSCEV(LHS: getMinusOne(Ty), RHS: V);
4810}
4811
4812const SCEV *ScalarEvolution::removePointerBase(const SCEV *P) {
4813 assert(P->getType()->isPointerTy());
4814
4815 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: P)) {
4816 // The base of an AddRec is the first operand.
4817 SmallVector<SCEVUse> Ops{AddRec->operands()};
4818 Ops[0] = removePointerBase(P: Ops[0]);
4819 // Don't try to transfer nowrap flags for now. We could in some cases
4820 // (for example, if pointer operand of the AddRec is a SCEVUnknown).
4821 return getAddRecExpr(Operands&: Ops, L: AddRec->getLoop(), Flags: SCEV::FlagAnyWrap);
4822 }
4823 if (auto *Add = dyn_cast<SCEVAddExpr>(Val: P)) {
4824 // The base of an Add is the pointer operand.
4825 SmallVector<SCEVUse> Ops{Add->operands()};
4826 SCEVUse *PtrOp = nullptr;
4827 for (SCEVUse &AddOp : Ops) {
4828 if (AddOp->getType()->isPointerTy()) {
4829 assert(!PtrOp && "Cannot have multiple pointer ops");
4830 PtrOp = &AddOp;
4831 }
4832 }
4833 *PtrOp = removePointerBase(P: *PtrOp);
4834 // Don't try to transfer nowrap flags for now. We could in some cases
4835 // (for example, if the pointer operand of the Add is a SCEVUnknown).
4836 return getAddExpr(Ops);
4837 }
4838 // Any other expression must be a pointer base.
4839 return getZero(Ty: P->getType());
4840}
4841
4842const SCEV *ScalarEvolution::getMinusSCEV(SCEVUse LHS, SCEVUse RHS,
4843 SCEV::NoWrapFlags Flags,
4844 unsigned Depth) {
4845 // Fast path: X - X --> 0.
4846 if (LHS == RHS)
4847 return getZero(Ty: LHS->getType());
4848
4849 // If we subtract two pointers with different pointer bases, bail.
4850 // Eventually, we're going to add an assertion to getMulExpr that we
4851 // can't multiply by a pointer.
4852 if (RHS->getType()->isPointerTy()) {
4853 if (!LHS->getType()->isPointerTy() ||
4854 getPointerBase(V: LHS) != getPointerBase(V: RHS))
4855 return getCouldNotCompute();
4856 LHS = removePointerBase(P: LHS);
4857 RHS = removePointerBase(P: RHS);
4858 }
4859
4860 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation
4861 // makes it so that we cannot make much use of NUW.
4862 auto AddFlags = SCEV::FlagAnyWrap;
4863 const bool RHSIsNotMinSigned =
4864 !getSignedRangeMin(S: RHS).isMinSignedValue();
4865 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW)) {
4866 // Let M be the minimum representable signed value. Then (-1)*RHS
4867 // signed-wraps if and only if RHS is M. That can happen even for
4868 // a NSW subtraction because e.g. (-1)*M signed-wraps even though
4869 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS +
4870 // (-1)*RHS, we need to prove that RHS != M.
4871 //
4872 // If LHS is non-negative and we know that LHS - RHS does not
4873 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap
4874 // either by proving that RHS > M or that LHS >= 0.
4875 if (RHSIsNotMinSigned || isKnownNonNegative(S: LHS)) {
4876 AddFlags = SCEV::FlagNSW;
4877 }
4878 }
4879
4880 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS -
4881 // RHS is NSW and LHS >= 0.
4882 //
4883 // The difficulty here is that the NSW flag may have been proven
4884 // relative to a loop that is to be found in a recurrence in LHS and
4885 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a
4886 // larger scope than intended.
4887 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
4888
4889 return getAddExpr(LHS, RHS: getNegativeSCEV(V: RHS, Flags: NegFlags), Flags: AddFlags, Depth);
4890}
4891
4892const SCEV *ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty,
4893 unsigned Depth) {
4894 Type *SrcTy = V->getType();
4895 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4896 "Cannot truncate or zero extend with non-integer arguments!");
4897 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4898 return V; // No conversion
4899 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4900 return getTruncateExpr(Op: V, Ty, Depth);
4901 return getZeroExtendExpr(Op: V, Ty, Depth);
4902}
4903
4904const SCEV *ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, Type *Ty,
4905 unsigned Depth) {
4906 Type *SrcTy = V->getType();
4907 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4908 "Cannot truncate or zero extend with non-integer arguments!");
4909 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4910 return V; // No conversion
4911 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4912 return getTruncateExpr(Op: V, Ty, Depth);
4913 return getSignExtendExpr(Op: V, Ty, Depth);
4914}
4915
4916const SCEV *
4917ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
4918 Type *SrcTy = V->getType();
4919 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4920 "Cannot noop or zero extend with non-integer arguments!");
4921 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4922 "getNoopOrZeroExtend cannot truncate!");
4923 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4924 return V; // No conversion
4925 return getZeroExtendExpr(Op: V, Ty);
4926}
4927
4928const SCEV *
4929ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
4930 Type *SrcTy = V->getType();
4931 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4932 "Cannot noop or sign extend with non-integer arguments!");
4933 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4934 "getNoopOrSignExtend cannot truncate!");
4935 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4936 return V; // No conversion
4937 return getSignExtendExpr(Op: V, Ty);
4938}
4939
4940const SCEV *
4941ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
4942 Type *SrcTy = V->getType();
4943 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4944 "Cannot noop or any extend with non-integer arguments!");
4945 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4946 "getNoopOrAnyExtend cannot truncate!");
4947 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4948 return V; // No conversion
4949 return getAnyExtendExpr(Op: V, Ty);
4950}
4951
4952const SCEV *
4953ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
4954 Type *SrcTy = V->getType();
4955 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4956 "Cannot truncate or noop with non-integer arguments!");
4957 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
4958 "getTruncateOrNoop cannot extend!");
4959 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4960 return V; // No conversion
4961 return getTruncateExpr(Op: V, Ty);
4962}
4963
4964const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
4965 const SCEV *RHS) {
4966 const SCEV *PromotedLHS = LHS;
4967 const SCEV *PromotedRHS = RHS;
4968
4969 if (getTypeSizeInBits(Ty: LHS->getType()) > getTypeSizeInBits(Ty: RHS->getType()))
4970 PromotedRHS = getZeroExtendExpr(Op: RHS, Ty: LHS->getType());
4971 else
4972 PromotedLHS = getNoopOrZeroExtend(V: LHS, Ty: RHS->getType());
4973
4974 return getUMaxExpr(LHS: PromotedLHS, RHS: PromotedRHS);
4975}
4976
4977const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
4978 const SCEV *RHS,
4979 bool Sequential) {
4980 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4981 return getUMinFromMismatchedTypes(Ops, Sequential);
4982}
4983
4984const SCEV *
4985ScalarEvolution::getUMinFromMismatchedTypes(SmallVectorImpl<SCEVUse> &Ops,
4986 bool Sequential) {
4987 assert(!Ops.empty() && "At least one operand must be!");
4988 // Trivial case.
4989 if (Ops.size() == 1)
4990 return Ops[0];
4991
4992 // Find the max type first.
4993 Type *MaxType = nullptr;
4994 for (SCEVUse S : Ops)
4995 if (MaxType)
4996 MaxType = getWiderType(T1: MaxType, T2: S->getType());
4997 else
4998 MaxType = S->getType();
4999 assert(MaxType && "Failed to find maximum type!");
5000
5001 // Extend all ops to max type.
5002 SmallVector<SCEVUse, 2> PromotedOps;
5003 for (SCEVUse S : Ops)
5004 PromotedOps.push_back(Elt: getNoopOrZeroExtend(V: S, Ty: MaxType));
5005
5006 // Generate umin.
5007 return getUMinExpr(Ops&: PromotedOps, Sequential);
5008}
5009
5010const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
5011 // A pointer operand may evaluate to a nonpointer expression, such as null.
5012 if (!V->getType()->isPointerTy())
5013 return V;
5014
5015 while (true) {
5016 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: V)) {
5017 V = AddRec->getStart();
5018 } else if (auto *Add = dyn_cast<SCEVAddExpr>(Val: V)) {
5019 const SCEV *PtrOp = nullptr;
5020 for (const SCEV *AddOp : Add->operands()) {
5021 if (AddOp->getType()->isPointerTy()) {
5022 assert(!PtrOp && "Cannot have multiple pointer ops");
5023 PtrOp = AddOp;
5024 }
5025 }
5026 assert(PtrOp && "Must have pointer op");
5027 V = PtrOp;
5028 } else // Not something we can look further into.
5029 return V;
5030 }
5031}
5032
5033/// Push users of the given Instruction onto the given Worklist.
5034static void PushDefUseChildren(Instruction *I,
5035 SmallVectorImpl<Instruction *> &Worklist,
5036 SmallPtrSetImpl<Instruction *> &Visited) {
5037 // Push the def-use children onto the Worklist stack.
5038 for (User *U : I->users()) {
5039 auto *UserInsn = cast<Instruction>(Val: U);
5040 if (Visited.insert(Ptr: UserInsn).second)
5041 Worklist.push_back(Elt: UserInsn);
5042 }
5043}
5044
5045namespace {
5046
5047/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its start
5048/// expression in case its Loop is L. If it is not L then
5049/// if IgnoreOtherLoops is true then use AddRec itself
5050/// otherwise rewrite cannot be done.
5051/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
5052class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> {
5053public:
5054 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
5055 bool IgnoreOtherLoops = true) {
5056 SCEVInitRewriter Rewriter(L, SE);
5057 const SCEV *Result = Rewriter.visit(S);
5058 if (Rewriter.hasSeenLoopVariantSCEVUnknown())
5059 return SE.getCouldNotCompute();
5060 return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
5061 ? SE.getCouldNotCompute()
5062 : Result;
5063 }
5064
5065 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5066 if (!SE.isLoopInvariant(S: Expr, L))
5067 SeenLoopVariantSCEVUnknown = true;
5068 return Expr;
5069 }
5070
5071 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
5072 // Only re-write AddRecExprs for this loop.
5073 if (Expr->getLoop() == L)
5074 return Expr->getStart();
5075 SeenOtherLoops = true;
5076 return Expr;
5077 }
5078
5079 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
5080
5081 bool hasSeenOtherLoops() { return SeenOtherLoops; }
5082
5083private:
5084 explicit SCEVInitRewriter(const Loop *L, ScalarEvolution &SE)
5085 : SCEVRewriteVisitor(SE), L(L) {}
5086
5087 const Loop *L;
5088 bool SeenLoopVariantSCEVUnknown = false;
5089 bool SeenOtherLoops = false;
5090};
5091
5092/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its post
5093/// increment expression in case its Loop is L. If it is not L then
5094/// use AddRec itself.
5095/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
5096class SCEVPostIncRewriter : public SCEVRewriteVisitor<SCEVPostIncRewriter> {
5097public:
5098 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE) {
5099 SCEVPostIncRewriter Rewriter(L, SE);
5100 const SCEV *Result = Rewriter.visit(S);
5101 return Rewriter.hasSeenLoopVariantSCEVUnknown()
5102 ? SE.getCouldNotCompute()
5103 : Result;
5104 }
5105
5106 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5107 if (!SE.isLoopInvariant(S: Expr, L))
5108 SeenLoopVariantSCEVUnknown = true;
5109 return Expr;
5110 }
5111
5112 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
5113 // Only re-write AddRecExprs for this loop.
5114 if (Expr->getLoop() == L)
5115 return Expr->getPostIncExpr(SE);
5116 SeenOtherLoops = true;
5117 return Expr;
5118 }
5119
5120 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
5121
5122 bool hasSeenOtherLoops() { return SeenOtherLoops; }
5123
5124private:
5125 explicit SCEVPostIncRewriter(const Loop *L, ScalarEvolution &SE)
5126 : SCEVRewriteVisitor(SE), L(L) {}
5127
5128 const Loop *L;
5129 bool SeenLoopVariantSCEVUnknown = false;
5130 bool SeenOtherLoops = false;
5131};
5132
5133/// This class evaluates the compare condition by matching it against the
5134/// condition of loop latch. If there is a match we assume a true value
5135/// for the condition while building SCEV nodes.
5136class SCEVBackedgeConditionFolder
5137 : public SCEVRewriteVisitor<SCEVBackedgeConditionFolder> {
5138public:
5139 static const SCEV *rewrite(const SCEV *S, const Loop *L,
5140 ScalarEvolution &SE) {
5141 bool IsPosBECond = false;
5142 Value *BECond = nullptr;
5143 if (BasicBlock *Latch = L->getLoopLatch()) {
5144 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Latch->getTerminator())) {
5145 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
5146 "Both outgoing branches should not target same header!");
5147 BECond = BI->getCondition();
5148 IsPosBECond = BI->getSuccessor(i: 0) == L->getHeader();
5149 } else {
5150 return S;
5151 }
5152 }
5153 SCEVBackedgeConditionFolder Rewriter(L, BECond, IsPosBECond, SE);
5154 return Rewriter.visit(S);
5155 }
5156
5157 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5158 const SCEV *Result = Expr;
5159 bool InvariantF = SE.isLoopInvariant(S: Expr, L);
5160
5161 if (!InvariantF) {
5162 Instruction *I = cast<Instruction>(Val: Expr->getValue());
5163 switch (I->getOpcode()) {
5164 case Instruction::Select: {
5165 SelectInst *SI = cast<SelectInst>(Val: I);
5166 std::optional<const SCEV *> Res =
5167 compareWithBackedgeCondition(IC: SI->getCondition());
5168 if (Res) {
5169 bool IsOne = cast<SCEVConstant>(Val: *Res)->getValue()->isOne();
5170 Result = SE.getSCEV(V: IsOne ? SI->getTrueValue() : SI->getFalseValue());
5171 }
5172 break;
5173 }
5174 default: {
5175 std::optional<const SCEV *> Res = compareWithBackedgeCondition(IC: I);
5176 if (Res)
5177 Result = *Res;
5178 break;
5179 }
5180 }
5181 }
5182 return Result;
5183 }
5184
5185private:
5186 explicit SCEVBackedgeConditionFolder(const Loop *L, Value *BECond,
5187 bool IsPosBECond, ScalarEvolution &SE)
5188 : SCEVRewriteVisitor(SE), L(L), BackedgeCond(BECond),
5189 IsPositiveBECond(IsPosBECond) {}
5190
5191 std::optional<const SCEV *> compareWithBackedgeCondition(Value *IC);
5192
5193 const Loop *L;
5194 /// Loop back condition.
5195 Value *BackedgeCond = nullptr;
5196 /// Set to true if loop back is on positive branch condition.
5197 bool IsPositiveBECond;
5198};
5199
5200std::optional<const SCEV *>
5201SCEVBackedgeConditionFolder::compareWithBackedgeCondition(Value *IC) {
5202
5203 // If value matches the backedge condition for loop latch,
5204 // then return a constant evolution node based on loopback
5205 // branch taken.
5206 if (BackedgeCond == IC)
5207 return IsPositiveBECond ? SE.getOne(Ty: Type::getInt1Ty(C&: SE.getContext()))
5208 : SE.getZero(Ty: Type::getInt1Ty(C&: SE.getContext()));
5209 return std::nullopt;
5210}
5211
5212class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> {
5213public:
5214 static const SCEV *rewrite(const SCEV *S, const Loop *L,
5215 ScalarEvolution &SE) {
5216 SCEVShiftRewriter Rewriter(L, SE);
5217 const SCEV *Result = Rewriter.visit(S);
5218 return Rewriter.isValid() ? Result : SE.getCouldNotCompute();
5219 }
5220
5221 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5222 // Only allow AddRecExprs for this loop.
5223 if (!SE.isLoopInvariant(S: Expr, L))
5224 Valid = false;
5225 return Expr;
5226 }
5227
5228 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
5229 if (Expr->getLoop() == L && Expr->isAffine())
5230 return SE.getMinusSCEV(LHS: Expr, RHS: Expr->getStepRecurrence(SE));
5231 Valid = false;
5232 return Expr;
5233 }
5234
5235 bool isValid() { return Valid; }
5236
5237private:
5238 explicit SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE)
5239 : SCEVRewriteVisitor(SE), L(L) {}
5240
5241 const Loop *L;
5242 bool Valid = true;
5243};
5244
5245} // end anonymous namespace
5246
5247void ScalarEvolution::inferNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) {
5248 if (!AR->isAffine())
5249 return;
5250
5251 // Force computation of ranges, which will also perform range-based flag
5252 // inference.
5253 if (!AR->hasNoSignedWrap())
5254 (void)getSignedRange(S: AR);
5255
5256 if (!AR->hasNoUnsignedWrap())
5257 (void)getUnsignedRange(S: AR);
5258
5259 if (!AR->hasNoSelfWrap()) {
5260 const SCEV *BECount = getConstantMaxBackedgeTakenCount(L: AR->getLoop());
5261 if (const SCEVConstant *BECountMax = dyn_cast<SCEVConstant>(Val: BECount)) {
5262 ConstantRange StepCR = getSignedRange(S: AR->getStepRecurrence(SE&: *this));
5263 const APInt &BECountAP = BECountMax->getAPInt();
5264 unsigned NoOverflowBitWidth =
5265 BECountAP.getActiveBits() + StepCR.getMinSignedBits();
5266 if (NoOverflowBitWidth <= getTypeSizeInBits(Ty: AR->getType()))
5267 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
5268 }
5269 }
5270}
5271
5272SCEV::NoWrapFlags
5273ScalarEvolution::proveNoSignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5274 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
5275
5276 if (AR->hasNoSignedWrap())
5277 return Result;
5278
5279 if (!AR->isAffine())
5280 return Result;
5281
5282 // This function can be expensive, only try to prove NSW once per AddRec.
5283 if (!SignedWrapViaInductionTried.insert(Ptr: AR).second)
5284 return Result;
5285
5286 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5287 const Loop *L = AR->getLoop();
5288
5289 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5290 // Note that this serves two purposes: It filters out loops that are
5291 // simply not analyzable, and it covers the case where this code is
5292 // being called from within backedge-taken count analysis, such that
5293 // attempting to ask for the backedge-taken count would likely result
5294 // in infinite recursion. In the later case, the analysis code will
5295 // cope with a conservative value, and it will take care to purge
5296 // that value once it has finished.
5297 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5298
5299 // Normally, in the cases we can prove no-overflow via a
5300 // backedge guarding condition, we can also compute a backedge
5301 // taken count for the loop. The exceptions are assumptions and
5302 // guards present in the loop -- SCEV is not great at exploiting
5303 // these to compute max backedge taken counts, but can still use
5304 // these to prove lack of overflow. Use this fact to avoid
5305 // doing extra work that may not pay off.
5306
5307 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5308 AC.assumptions().empty())
5309 return Result;
5310
5311 // If the backedge is guarded by a comparison with the pre-inc value the
5312 // addrec is safe. Also, if the entry is guarded by a comparison with the
5313 // start value and the backedge is guarded by a comparison with the post-inc
5314 // value, the addrec is safe.
5315 ICmpInst::Predicate Pred;
5316 const SCEV *OverflowLimit =
5317 getSignedOverflowLimitForStep(Step, Pred: &Pred, SE: this);
5318 if (OverflowLimit &&
5319 (isLoopBackedgeGuardedByCond(L, Pred, LHS: AR, RHS: OverflowLimit) ||
5320 isKnownOnEveryIteration(Pred, LHS: AR, RHS: OverflowLimit))) {
5321 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNSW);
5322 }
5323 return Result;
5324}
5325SCEV::NoWrapFlags
5326ScalarEvolution::proveNoUnsignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5327 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
5328
5329 if (AR->hasNoUnsignedWrap())
5330 return Result;
5331
5332 if (!AR->isAffine())
5333 return Result;
5334
5335 // This function can be expensive, only try to prove NUW once per AddRec.
5336 if (!UnsignedWrapViaInductionTried.insert(Ptr: AR).second)
5337 return Result;
5338
5339 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5340 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
5341 const Loop *L = AR->getLoop();
5342
5343 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5344 // Note that this serves two purposes: It filters out loops that are
5345 // simply not analyzable, and it covers the case where this code is
5346 // being called from within backedge-taken count analysis, such that
5347 // attempting to ask for the backedge-taken count would likely result
5348 // in infinite recursion. In the later case, the analysis code will
5349 // cope with a conservative value, and it will take care to purge
5350 // that value once it has finished.
5351 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5352
5353 // Normally, in the cases we can prove no-overflow via a
5354 // backedge guarding condition, we can also compute a backedge
5355 // taken count for the loop. The exceptions are assumptions and
5356 // guards present in the loop -- SCEV is not great at exploiting
5357 // these to compute max backedge taken counts, but can still use
5358 // these to prove lack of overflow. Use this fact to avoid
5359 // doing extra work that may not pay off.
5360
5361 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5362 AC.assumptions().empty())
5363 return Result;
5364
5365 // If the backedge is guarded by a comparison with the pre-inc value the
5366 // addrec is safe. Also, if the entry is guarded by a comparison with the
5367 // start value and the backedge is guarded by a comparison with the post-inc
5368 // value, the addrec is safe.
5369 if (isKnownPositive(S: Step)) {
5370 const SCEV *N = getConstant(Val: APInt::getMinValue(numBits: BitWidth) -
5371 getUnsignedRangeMax(S: Step));
5372 if (isLoopBackedgeGuardedByCond(L, Pred: ICmpInst::ICMP_ULT, LHS: AR, RHS: N) ||
5373 isKnownOnEveryIteration(Pred: ICmpInst::ICMP_ULT, LHS: AR, RHS: N)) {
5374 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNUW);
5375 }
5376 }
5377
5378 return Result;
5379}
5380
5381namespace {
5382
5383/// Represents an abstract binary operation. This may exist as a
5384/// normal instruction or constant expression, or may have been
5385/// derived from an expression tree.
5386struct BinaryOp {
5387 unsigned Opcode;
5388 Value *LHS;
5389 Value *RHS;
5390 bool IsNSW = false;
5391 bool IsNUW = false;
5392
5393 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
5394 /// constant expression.
5395 Operator *Op = nullptr;
5396
5397 explicit BinaryOp(Operator *Op)
5398 : Opcode(Op->getOpcode()), LHS(Op->getOperand(i: 0)), RHS(Op->getOperand(i: 1)),
5399 Op(Op) {
5400 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: Op)) {
5401 IsNSW = OBO->hasNoSignedWrap();
5402 IsNUW = OBO->hasNoUnsignedWrap();
5403 }
5404 }
5405
5406 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
5407 bool IsNUW = false)
5408 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
5409};
5410
5411} // end anonymous namespace
5412
5413/// Try to map \p V into a BinaryOp, and return \c std::nullopt on failure.
5414static std::optional<BinaryOp> MatchBinaryOp(Value *V, const DataLayout &DL,
5415 AssumptionCache &AC,
5416 const DominatorTree &DT,
5417 const Instruction *CxtI) {
5418 auto *Op = dyn_cast<Operator>(Val: V);
5419 if (!Op)
5420 return std::nullopt;
5421
5422 // Implementation detail: all the cleverness here should happen without
5423 // creating new SCEV expressions -- our caller knowns tricks to avoid creating
5424 // SCEV expressions when possible, and we should not break that.
5425
5426 switch (Op->getOpcode()) {
5427 case Instruction::Add:
5428 case Instruction::Sub:
5429 case Instruction::Mul:
5430 case Instruction::UDiv:
5431 case Instruction::URem:
5432 case Instruction::And:
5433 case Instruction::AShr:
5434 case Instruction::Shl:
5435 return BinaryOp(Op);
5436
5437 case Instruction::Or: {
5438 // Convert or disjoint into add nuw nsw.
5439 if (cast<PossiblyDisjointInst>(Val: Op)->isDisjoint()) {
5440 BinaryOp BinOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1),
5441 /*IsNSW=*/true, /*IsNUW=*/true);
5442 // Keep the reference to the original instruction so that we can later
5443 // check whether it can produce poison value or not.
5444 BinOp.Op = Op;
5445 return BinOp;
5446 }
5447 return BinaryOp(Op);
5448 }
5449
5450 case Instruction::Xor:
5451 if (auto *RHSC = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1)))
5452 // If the RHS of the xor is a signmask, then this is just an add.
5453 // Instcombine turns add of signmask into xor as a strength reduction step.
5454 if (RHSC->getValue().isSignMask())
5455 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5456 // Binary `xor` is a bit-wise `add`.
5457 if (V->getType()->isIntegerTy(BitWidth: 1))
5458 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5459 return BinaryOp(Op);
5460
5461 case Instruction::LShr:
5462 // Turn logical shift right of a constant into a unsigned divide.
5463 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1))) {
5464 uint32_t BitWidth = cast<IntegerType>(Val: Op->getType())->getBitWidth();
5465
5466 // If the shift count is not less than the bitwidth, the result of
5467 // the shift is undefined. Don't try to analyze it, because the
5468 // resolution chosen here may differ from the resolution chosen in
5469 // other parts of the compiler.
5470 if (SA->getValue().ult(RHS: BitWidth)) {
5471 Constant *X =
5472 ConstantInt::get(Context&: SA->getContext(),
5473 V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
5474 return BinaryOp(Instruction::UDiv, Op->getOperand(i: 0), X);
5475 }
5476 }
5477 return BinaryOp(Op);
5478
5479 case Instruction::ExtractValue: {
5480 auto *EVI = cast<ExtractValueInst>(Val: Op);
5481 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
5482 break;
5483
5484 auto *WO = dyn_cast<WithOverflowInst>(Val: EVI->getAggregateOperand());
5485 if (!WO)
5486 break;
5487
5488 Instruction::BinaryOps BinOp = WO->getBinaryOp();
5489 bool Signed = WO->isSigned();
5490 // TODO: Should add nuw/nsw flags for mul as well.
5491 if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT))
5492 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
5493
5494 // Now that we know that all uses of the arithmetic-result component of
5495 // CI are guarded by the overflow check, we can go ahead and pretend
5496 // that the arithmetic is non-overflowing.
5497 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
5498 /* IsNSW = */ Signed, /* IsNUW = */ !Signed);
5499 }
5500
5501 default:
5502 break;
5503 }
5504
5505 // Recognise intrinsic loop.decrement.reg, and as this has exactly the same
5506 // semantics as a Sub, return a binary sub expression.
5507 if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
5508 if (II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
5509 return BinaryOp(Instruction::Sub, II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1));
5510
5511 return std::nullopt;
5512}
5513
5514/// Helper function to createAddRecFromPHIWithCasts. We have a phi
5515/// node whose symbolic (unknown) SCEV is \p SymbolicPHI, which is updated via
5516/// the loop backedge by a SCEVAddExpr, possibly also with a few casts on the
5517/// way. This function checks if \p Op, an operand of this SCEVAddExpr,
5518/// follows one of the following patterns:
5519/// Op == (SExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5520/// Op == (ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5521/// If the SCEV expression of \p Op conforms with one of the expected patterns
5522/// we return the type of the truncation operation, and indicate whether the
5523/// truncated type should be treated as signed/unsigned by setting
5524/// \p Signed to true/false, respectively.
5525static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI,
5526 bool &Signed, ScalarEvolution &SE) {
5527 // The case where Op == SymbolicPHI (that is, with no type conversions on
5528 // the way) is handled by the regular add recurrence creating logic and
5529 // would have already been triggered in createAddRecForPHI. Reaching it here
5530 // means that createAddRecFromPHI had failed for this PHI before (e.g.,
5531 // because one of the other operands of the SCEVAddExpr updating this PHI is
5532 // not invariant).
5533 //
5534 // Here we look for the case where Op = (ext(trunc(SymbolicPHI))), and in
5535 // this case predicates that allow us to prove that Op == SymbolicPHI will
5536 // be added.
5537 if (Op == SymbolicPHI)
5538 return nullptr;
5539
5540 unsigned SourceBits = SE.getTypeSizeInBits(Ty: SymbolicPHI->getType());
5541 unsigned NewBits = SE.getTypeSizeInBits(Ty: Op->getType());
5542 if (SourceBits != NewBits)
5543 return nullptr;
5544
5545 if (match(S: Op, P: m_scev_SExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5546 Signed = true;
5547 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5548 }
5549 if (match(S: Op, P: m_scev_ZExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5550 Signed = false;
5551 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5552 }
5553 return nullptr;
5554}
5555
5556static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
5557 if (!PN->getType()->isIntegerTy())
5558 return nullptr;
5559 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5560 if (!L || L->getHeader() != PN->getParent())
5561 return nullptr;
5562 return L;
5563}
5564
5565// Analyze \p SymbolicPHI, a SCEV expression of a phi node, and check if the
5566// computation that updates the phi follows the following pattern:
5567// (SExt/ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy) + InvariantAccum
5568// which correspond to a phi->trunc->sext/zext->add->phi update chain.
5569// If so, try to see if it can be rewritten as an AddRecExpr under some
5570// Predicates. If successful, return them as a pair. Also cache the results
5571// of the analysis.
5572//
5573// Example usage scenario:
5574// Say the Rewriter is called for the following SCEV:
5575// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5576// where:
5577// %X = phi i64 (%Start, %BEValue)
5578// It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
5579// and call this function with %SymbolicPHI = %X.
5580//
5581// The analysis will find that the value coming around the backedge has
5582// the following SCEV:
5583// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5584// Upon concluding that this matches the desired pattern, the function
5585// will return the pair {NewAddRec, SmallPredsVec} where:
5586// NewAddRec = {%Start,+,%Step}
5587// SmallPredsVec = {P1, P2, P3} as follows:
5588// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
5589// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
5590// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
5591// The returned pair means that SymbolicPHI can be rewritten into NewAddRec
5592// under the predicates {P1,P2,P3}.
5593// This predicated rewrite will be cached in PredicatedSCEVRewrites:
5594// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
5595//
5596// TODO's:
5597//
5598// 1) Extend the Induction descriptor to also support inductions that involve
5599// casts: When needed (namely, when we are called in the context of the
5600// vectorizer induction analysis), a Set of cast instructions will be
5601// populated by this method, and provided back to isInductionPHI. This is
5602// needed to allow the vectorizer to properly record them to be ignored by
5603// the cost model and to avoid vectorizing them (otherwise these casts,
5604// which are redundant under the runtime overflow checks, will be
5605// vectorized, which can be costly).
5606//
5607// 2) Support additional induction/PHISCEV patterns: We also want to support
5608// inductions where the sext-trunc / zext-trunc operations (partly) occur
5609// after the induction update operation (the induction increment):
5610//
5611// (Trunc iy (SExt/ZExt ix (%SymbolicPHI + InvariantAccum) to iy) to ix)
5612// which correspond to a phi->add->trunc->sext/zext->phi update chain.
5613//
5614// (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
5615// which correspond to a phi->trunc->add->sext/zext->phi update chain.
5616//
5617// 3) Outline common code with createAddRecFromPHI to avoid duplication.
5618std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5619ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
5620 SmallVector<const SCEVPredicate *, 3> Predicates;
5621
5622 // *** Part1: Analyze if we have a phi-with-cast pattern for which we can
5623 // return an AddRec expression under some predicate.
5624
5625 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5626 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5627 assert(L && "Expecting an integer loop header phi");
5628
5629 // The loop may have multiple entrances or multiple exits; we can analyze
5630 // this phi as an addrec if it has a unique entry value and a unique
5631 // backedge value.
5632 Value *BEValueV = nullptr, *StartValueV = nullptr;
5633 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5634 Value *V = PN->getIncomingValue(i);
5635 if (L->contains(BB: PN->getIncomingBlock(i))) {
5636 if (!BEValueV) {
5637 BEValueV = V;
5638 } else if (BEValueV != V) {
5639 BEValueV = nullptr;
5640 break;
5641 }
5642 } else if (!StartValueV) {
5643 StartValueV = V;
5644 } else if (StartValueV != V) {
5645 StartValueV = nullptr;
5646 break;
5647 }
5648 }
5649 if (!BEValueV || !StartValueV)
5650 return std::nullopt;
5651
5652 const SCEV *BEValue = getSCEV(V: BEValueV);
5653
5654 // If the value coming around the backedge is an add with the symbolic
5655 // value we just inserted, possibly with casts that we can ignore under
5656 // an appropriate runtime guard, then we found a simple induction variable!
5657 const auto *Add = dyn_cast<SCEVAddExpr>(Val: BEValue);
5658 if (!Add)
5659 return std::nullopt;
5660
5661 // If there is a single occurrence of the symbolic value, possibly
5662 // casted, replace it with a recurrence.
5663 unsigned FoundIndex = Add->getNumOperands();
5664 Type *TruncTy = nullptr;
5665 bool Signed;
5666 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5667 if ((TruncTy =
5668 isSimpleCastedPHI(Op: Add->getOperand(i), SymbolicPHI, Signed, SE&: *this)))
5669 if (FoundIndex == e) {
5670 FoundIndex = i;
5671 break;
5672 }
5673
5674 if (FoundIndex == Add->getNumOperands())
5675 return std::nullopt;
5676
5677 // Create an add with everything but the specified operand.
5678 SmallVector<SCEVUse, 8> Ops;
5679 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5680 if (i != FoundIndex)
5681 Ops.push_back(Elt: Add->getOperand(i));
5682 const SCEV *Accum = getAddExpr(Ops);
5683
5684 // The runtime checks will not be valid if the step amount is
5685 // varying inside the loop.
5686 if (!isLoopInvariant(S: Accum, L))
5687 return std::nullopt;
5688
5689 // *** Part2: Create the predicates
5690
5691 // Analysis was successful: we have a phi-with-cast pattern for which we
5692 // can return an AddRec expression under the following predicates:
5693 //
5694 // P1: A Wrap predicate that guarantees that Trunc(Start) + i*Trunc(Accum)
5695 // fits within the truncated type (does not overflow) for i = 0 to n-1.
5696 // P2: An Equal predicate that guarantees that
5697 // Start = (Ext ix (Trunc iy (Start) to ix) to iy)
5698 // P3: An Equal predicate that guarantees that
5699 // Accum = (Ext ix (Trunc iy (Accum) to ix) to iy)
5700 //
5701 // As we next prove, the above predicates guarantee that:
5702 // Start + i*Accum = (Ext ix (Trunc iy ( Start + i*Accum ) to ix) to iy)
5703 //
5704 //
5705 // More formally, we want to prove that:
5706 // Expr(i+1) = Start + (i+1) * Accum
5707 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5708 //
5709 // Given that:
5710 // 1) Expr(0) = Start
5711 // 2) Expr(1) = Start + Accum
5712 // = (Ext ix (Trunc iy (Start) to ix) to iy) + Accum :: from P2
5713 // 3) Induction hypothesis (step i):
5714 // Expr(i) = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum
5715 //
5716 // Proof:
5717 // Expr(i+1) =
5718 // = Start + (i+1)*Accum
5719 // = (Start + i*Accum) + Accum
5720 // = Expr(i) + Accum
5721 // = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum + Accum
5722 // :: from step i
5723 //
5724 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy) + Accum + Accum
5725 //
5726 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy)
5727 // + (Ext ix (Trunc iy (Accum) to ix) to iy)
5728 // + Accum :: from P3
5729 //
5730 // = (Ext ix (Trunc iy ((Start + (i-1)*Accum) + Accum) to ix) to iy)
5731 // + Accum :: from P1: Ext(x)+Ext(y)=>Ext(x+y)
5732 //
5733 // = (Ext ix (Trunc iy (Start + i*Accum) to ix) to iy) + Accum
5734 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5735 //
5736 // By induction, the same applies to all iterations 1<=i<n:
5737 //
5738
5739 // Create a truncated addrec for which we will add a no overflow check (P1).
5740 const SCEV *StartVal = getSCEV(V: StartValueV);
5741 const SCEV *PHISCEV =
5742 getAddRecExpr(Start: getTruncateExpr(Op: StartVal, Ty: TruncTy),
5743 Step: getTruncateExpr(Op: Accum, Ty: TruncTy), L, Flags: SCEV::FlagAnyWrap);
5744
5745 // PHISCEV can be either a SCEVConstant or a SCEVAddRecExpr.
5746 // ex: If truncated Accum is 0 and StartVal is a constant, then PHISCEV
5747 // will be constant.
5748 //
5749 // If PHISCEV is a constant, then P1 degenerates into P2 or P3, so we don't
5750 // add P1.
5751 if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV)) {
5752 SCEVWrapPredicate::IncrementWrapFlags AddedFlags =
5753 Signed ? SCEVWrapPredicate::IncrementNSSW
5754 : SCEVWrapPredicate::IncrementNUSW;
5755 const SCEVPredicate *AddRecPred = getWrapPredicate(AR, AddedFlags);
5756 Predicates.push_back(Elt: AddRecPred);
5757 }
5758
5759 // Create the Equal Predicates P2,P3:
5760
5761 // It is possible that the predicates P2 and/or P3 are computable at
5762 // compile time due to StartVal and/or Accum being constants.
5763 // If either one is, then we can check that now and escape if either P2
5764 // or P3 is false.
5765
5766 // Construct the extended SCEV: (Ext ix (Trunc iy (Expr) to ix) to iy)
5767 // for each of StartVal and Accum
5768 auto getExtendedExpr = [&](const SCEV *Expr,
5769 bool CreateSignExtend) -> const SCEV * {
5770 assert(isLoopInvariant(Expr, L) && "Expr is expected to be invariant");
5771 const SCEV *TruncatedExpr = getTruncateExpr(Op: Expr, Ty: TruncTy);
5772 const SCEV *ExtendedExpr =
5773 CreateSignExtend ? getSignExtendExpr(Op: TruncatedExpr, Ty: Expr->getType())
5774 : getZeroExtendExpr(Op: TruncatedExpr, Ty: Expr->getType());
5775 return ExtendedExpr;
5776 };
5777
5778 // Given:
5779 // ExtendedExpr = (Ext ix (Trunc iy (Expr) to ix) to iy
5780 // = getExtendedExpr(Expr)
5781 // Determine whether the predicate P: Expr == ExtendedExpr
5782 // is known to be false at compile time
5783 auto PredIsKnownFalse = [&](const SCEV *Expr,
5784 const SCEV *ExtendedExpr) -> bool {
5785 return Expr != ExtendedExpr &&
5786 isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: Expr, RHS: ExtendedExpr);
5787 };
5788
5789 const SCEV *StartExtended = getExtendedExpr(StartVal, Signed);
5790 if (PredIsKnownFalse(StartVal, StartExtended)) {
5791 LLVM_DEBUG(dbgs() << "P2 is compile-time false\n";);
5792 return std::nullopt;
5793 }
5794
5795 // The Step is always Signed (because the overflow checks are either
5796 // NSSW or NUSW)
5797 const SCEV *AccumExtended = getExtendedExpr(Accum, /*CreateSignExtend=*/true);
5798 if (PredIsKnownFalse(Accum, AccumExtended)) {
5799 LLVM_DEBUG(dbgs() << "P3 is compile-time false\n";);
5800 return std::nullopt;
5801 }
5802
5803 auto AppendPredicate = [&](const SCEV *Expr,
5804 const SCEV *ExtendedExpr) -> void {
5805 if (Expr != ExtendedExpr &&
5806 !isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: Expr, RHS: ExtendedExpr)) {
5807 const SCEVPredicate *Pred = getEqualPredicate(LHS: Expr, RHS: ExtendedExpr);
5808 LLVM_DEBUG(dbgs() << "Added Predicate: " << *Pred);
5809 Predicates.push_back(Elt: Pred);
5810 }
5811 };
5812
5813 AppendPredicate(StartVal, StartExtended);
5814 AppendPredicate(Accum, AccumExtended);
5815
5816 // *** Part3: Predicates are ready. Now go ahead and create the new addrec in
5817 // which the casts had been folded away. The caller can rewrite SymbolicPHI
5818 // into NewAR if it will also add the runtime overflow checks specified in
5819 // Predicates.
5820 auto *NewAR = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags: SCEV::FlagAnyWrap);
5821
5822 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
5823 std::make_pair(x&: NewAR, y&: Predicates);
5824 // Remember the result of the analysis for this SCEV at this locayyytion.
5825 PredicatedSCEVRewrites[{SymbolicPHI, L}] = PredRewrite;
5826 return PredRewrite;
5827}
5828
5829std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5830ScalarEvolution::createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI) {
5831 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5832 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5833 if (!L)
5834 return std::nullopt;
5835
5836 // Check to see if we already analyzed this PHI.
5837 auto I = PredicatedSCEVRewrites.find(Val: {SymbolicPHI, L});
5838 if (I != PredicatedSCEVRewrites.end()) {
5839 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
5840 I->second;
5841 // Analysis was done before and failed to create an AddRec:
5842 if (Rewrite.first == SymbolicPHI)
5843 return std::nullopt;
5844 // Analysis was done before and succeeded to create an AddRec under
5845 // a predicate:
5846 assert(isa<SCEVAddRecExpr>(Rewrite.first) && "Expected an AddRec");
5847 assert(!(Rewrite.second).empty() && "Expected to find Predicates");
5848 return Rewrite;
5849 }
5850
5851 std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5852 Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
5853
5854 // Record in the cache that the analysis failed
5855 if (!Rewrite) {
5856 SmallVector<const SCEVPredicate *, 3> Predicates;
5857 PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
5858 return std::nullopt;
5859 }
5860
5861 return Rewrite;
5862}
5863
5864// FIXME: This utility is currently required because the Rewriter currently
5865// does not rewrite this expression:
5866// {0, +, (sext ix (trunc iy to ix) to iy)}
5867// into {0, +, %step},
5868// even when the following Equal predicate exists:
5869// "%step == (sext ix (trunc iy to ix) to iy)".
5870bool PredicatedScalarEvolution::areAddRecsEqualWithPreds(
5871 const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2,
5872 ArrayRef<const SCEVPredicate *> NoWrapPreds) const {
5873 if (AR1 == AR2)
5874 return true;
5875
5876 SCEVUnionPredicate NoWrapUnionPred(NoWrapPreds, SE);
5877 SCEVUnionPredicate AllPreds = Preds->getUnionWith(N: &NoWrapUnionPred, SE);
5878 auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool {
5879 if (Expr1 != Expr2 &&
5880 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr1, RHS: Expr2), SE) &&
5881 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr2, RHS: Expr1), SE))
5882 return false;
5883 return true;
5884 };
5885
5886 if (!areExprsEqual(AR1->getStart(), AR2->getStart()) ||
5887 !areExprsEqual(AR1->getStepRecurrence(SE), AR2->getStepRecurrence(SE)))
5888 return false;
5889 return true;
5890}
5891
5892/// A helper function for createAddRecFromPHI to handle simple cases.
5893///
5894/// This function tries to find an AddRec expression for the simplest (yet most
5895/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
5896/// If it fails, createAddRecFromPHI will use a more general, but slow,
5897/// technique for finding the AddRec expression.
5898const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
5899 Value *BEValueV,
5900 Value *StartValueV) {
5901 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5902 assert(L && L->getHeader() == PN->getParent());
5903 assert(BEValueV && StartValueV);
5904
5905 auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CxtI: PN);
5906 if (!BO)
5907 return nullptr;
5908
5909 if (BO->Opcode != Instruction::Add)
5910 return nullptr;
5911
5912 const SCEV *Accum = nullptr;
5913 if (BO->LHS == PN && L->isLoopInvariant(V: BO->RHS))
5914 Accum = getSCEV(V: BO->RHS);
5915 else if (BO->RHS == PN && L->isLoopInvariant(V: BO->LHS))
5916 Accum = getSCEV(V: BO->LHS);
5917
5918 if (!Accum)
5919 return nullptr;
5920
5921 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
5922 if (BO->IsNUW)
5923 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5924 if (BO->IsNSW)
5925 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5926
5927 const SCEV *StartVal = getSCEV(V: StartValueV);
5928 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5929 insertValueToMap(V: PN, S: PHISCEV);
5930
5931 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
5932 inferNoWrapViaConstantRanges(AR);
5933
5934 // We can add Flags to the post-inc expression only if we
5935 // know that it is *undefined behavior* for BEValueV to
5936 // overflow.
5937 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV)) {
5938 assert(isLoopInvariant(Accum, L) &&
5939 "Accum is defined outside L, but is not invariant?");
5940 if (isAddRecNeverPoison(I: BEInst, L))
5941 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5942 }
5943
5944 return PHISCEV;
5945}
5946
5947const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
5948 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5949 if (!L || L->getHeader() != PN->getParent())
5950 return nullptr;
5951
5952 // The loop may have multiple entrances or multiple exits; we can analyze
5953 // this phi as an addrec if it has a unique entry value and a unique
5954 // backedge value.
5955 Value *BEValueV = nullptr, *StartValueV = nullptr;
5956 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5957 Value *V = PN->getIncomingValue(i);
5958 if (L->contains(BB: PN->getIncomingBlock(i))) {
5959 if (!BEValueV) {
5960 BEValueV = V;
5961 } else if (BEValueV != V) {
5962 BEValueV = nullptr;
5963 break;
5964 }
5965 } else if (!StartValueV) {
5966 StartValueV = V;
5967 } else if (StartValueV != V) {
5968 StartValueV = nullptr;
5969 break;
5970 }
5971 }
5972 if (!BEValueV || !StartValueV)
5973 return nullptr;
5974
5975 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
5976 "PHI node already processed?");
5977
5978 // First, try to find AddRec expression without creating a fictituos symbolic
5979 // value for PN.
5980 if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
5981 return S;
5982
5983 // Handle PHI node value symbolically.
5984 const SCEV *SymbolicName = getUnknown(V: PN);
5985 insertValueToMap(V: PN, S: SymbolicName);
5986
5987 // Using this symbolic name for the PHI, analyze the value coming around
5988 // the back-edge.
5989 const SCEV *BEValue = getSCEV(V: BEValueV);
5990
5991 // NOTE: If BEValue is loop invariant, we know that the PHI node just
5992 // has a special value for the first iteration of the loop.
5993
5994 // If the value coming around the backedge is an add with the symbolic
5995 // value we just inserted, then we found a simple induction variable!
5996 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: BEValue)) {
5997 // If there is a single occurrence of the symbolic value, replace it
5998 // with a recurrence.
5999 unsigned FoundIndex = Add->getNumOperands();
6000 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
6001 if (Add->getOperand(i) == SymbolicName)
6002 if (FoundIndex == e) {
6003 FoundIndex = i;
6004 break;
6005 }
6006
6007 if (FoundIndex != Add->getNumOperands()) {
6008 // Create an add with everything but the specified operand.
6009 SmallVector<SCEVUse, 8> Ops;
6010 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
6011 if (i != FoundIndex)
6012 Ops.push_back(Elt: SCEVBackedgeConditionFolder::rewrite(S: Add->getOperand(i),
6013 L, SE&: *this));
6014 const SCEV *Accum = getAddExpr(Ops);
6015
6016 // This is not a valid addrec if the step amount is varying each
6017 // loop iteration, but is not itself an addrec in this loop.
6018 if (isLoopInvariant(S: Accum, L) ||
6019 (isa<SCEVAddRecExpr>(Val: Accum) &&
6020 cast<SCEVAddRecExpr>(Val: Accum)->getLoop() == L)) {
6021 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
6022
6023 if (auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CxtI: PN)) {
6024 if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
6025 if (BO->IsNUW)
6026 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
6027 if (BO->IsNSW)
6028 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
6029 }
6030 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: BEValueV)) {
6031 if (GEP->getOperand(i_nocapture: 0) == PN) {
6032 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
6033 // If the increment has any nowrap flags, then we know the address
6034 // space cannot be wrapped around.
6035 if (NW != GEPNoWrapFlags::none())
6036 Flags = setFlags(Flags, OnFlags: SCEV::FlagNW);
6037 // If the GEP is nuw or nusw with non-negative offset, we know that
6038 // no unsigned wrap occurs. We cannot set the nsw flag as only the
6039 // offset is treated as signed, while the base is unsigned.
6040 if (NW.hasNoUnsignedWrap() ||
6041 (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(S: Accum)))
6042 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
6043 }
6044
6045 // We cannot transfer nuw and nsw flags from subtraction
6046 // operations -- sub nuw X, Y is not the same as add nuw X, -Y
6047 // for instance.
6048 }
6049
6050 const SCEV *StartVal = getSCEV(V: StartValueV);
6051 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
6052
6053 // Okay, for the entire analysis of this edge we assumed the PHI
6054 // to be symbolic. We now need to go back and purge all of the
6055 // entries for the scalars that use the symbolic expression.
6056 forgetMemoizedResults(SCEVs: {SymbolicName});
6057 insertValueToMap(V: PN, S: PHISCEV);
6058
6059 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
6060 inferNoWrapViaConstantRanges(AR);
6061
6062 // We can add Flags to the post-inc expression only if we
6063 // know that it is *undefined behavior* for BEValueV to
6064 // overflow.
6065 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV))
6066 if (isLoopInvariant(S: Accum, L) && isAddRecNeverPoison(I: BEInst, L))
6067 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
6068
6069 return PHISCEV;
6070 }
6071 }
6072 } else {
6073 // Otherwise, this could be a loop like this:
6074 // i = 0; for (j = 1; ..; ++j) { .... i = j; }
6075 // In this case, j = {1,+,1} and BEValue is j.
6076 // Because the other in-value of i (0) fits the evolution of BEValue
6077 // i really is an addrec evolution.
6078 //
6079 // We can generalize this saying that i is the shifted value of BEValue
6080 // by one iteration:
6081 // PHI(f(0), f({1,+,1})) --> f({0,+,1})
6082
6083 // Do not allow refinement in rewriting of BEValue.
6084 const SCEV *Shifted = SCEVShiftRewriter::rewrite(S: BEValue, L, SE&: *this);
6085 const SCEV *Start = SCEVInitRewriter::rewrite(S: Shifted, L, SE&: *this, IgnoreOtherLoops: false);
6086 if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
6087 isGuaranteedNotToCauseUB(Op: Shifted) && ::impliesPoison(AssumedPoison: Shifted, S: Start)) {
6088 const SCEV *StartVal = getSCEV(V: StartValueV);
6089 if (Start == StartVal) {
6090 // Okay, for the entire analysis of this edge we assumed the PHI
6091 // to be symbolic. We now need to go back and purge all of the
6092 // entries for the scalars that use the symbolic expression.
6093 forgetMemoizedResults(SCEVs: {SymbolicName});
6094 insertValueToMap(V: PN, S: Shifted);
6095 return Shifted;
6096 }
6097 }
6098 }
6099
6100 // Remove the temporary PHI node SCEV that has been inserted while intending
6101 // to create an AddRecExpr for this PHI node. We can not keep this temporary
6102 // as it will prevent later (possibly simpler) SCEV expressions to be added
6103 // to the ValueExprMap.
6104 eraseValueFromMap(V: PN);
6105
6106 return nullptr;
6107}
6108
6109// Try to match a control flow sequence that branches out at BI and merges back
6110// at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful
6111// match.
6112static bool BrPHIToSelect(DominatorTree &DT, CondBrInst *BI, PHINode *Merge,
6113 Value *&C, Value *&LHS, Value *&RHS) {
6114 C = BI->getCondition();
6115
6116 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(i: 0));
6117 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(i: 1));
6118
6119 Use &LeftUse = Merge->getOperandUse(i: 0);
6120 Use &RightUse = Merge->getOperandUse(i: 1);
6121
6122 if (DT.dominates(BBE: LeftEdge, U: LeftUse) && DT.dominates(BBE: RightEdge, U: RightUse)) {
6123 LHS = LeftUse;
6124 RHS = RightUse;
6125 return true;
6126 }
6127
6128 if (DT.dominates(BBE: LeftEdge, U: RightUse) && DT.dominates(BBE: RightEdge, U: LeftUse)) {
6129 LHS = RightUse;
6130 RHS = LeftUse;
6131 return true;
6132 }
6133
6134 return false;
6135}
6136
6137static bool getOperandsForSelectLikePHI(DominatorTree &DT, PHINode *PN,
6138 Value *&Cond, Value *&LHS,
6139 Value *&RHS) {
6140 auto IsReachable =
6141 [&](BasicBlock *BB) { return DT.isReachableFromEntry(A: BB); };
6142 if (PN->getNumIncomingValues() == 2 && all_of(Range: PN->blocks(), P: IsReachable)) {
6143 // Try to match
6144 //
6145 // br %cond, label %left, label %right
6146 // left:
6147 // br label %merge
6148 // right:
6149 // br label %merge
6150 // merge:
6151 // V = phi [ %x, %left ], [ %y, %right ]
6152 //
6153 // as "select %cond, %x, %y"
6154
6155 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock();
6156 assert(IDom && "At least the entry block should dominate PN");
6157
6158 auto *BI = dyn_cast<CondBrInst>(Val: IDom->getTerminator());
6159 return BI && BrPHIToSelect(DT, BI, Merge: PN, C&: Cond, LHS, RHS);
6160 }
6161 return false;
6162}
6163
6164const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
6165 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
6166 if (getOperandsForSelectLikePHI(DT, PN, Cond, LHS, RHS) &&
6167 properlyDominates(S: getSCEV(V: LHS), BB: PN->getParent()) &&
6168 properlyDominates(S: getSCEV(V: RHS), BB: PN->getParent()))
6169 return createNodeForSelectOrPHI(V: PN, Cond, TrueVal: LHS, FalseVal: RHS);
6170
6171 return nullptr;
6172}
6173
6174static BinaryOperator *getCommonInstForPHI(PHINode *PN) {
6175 BinaryOperator *CommonInst = nullptr;
6176 // Check if instructions are identical.
6177 for (Value *Incoming : PN->incoming_values()) {
6178 auto *IncomingInst = dyn_cast<BinaryOperator>(Val: Incoming);
6179 if (!IncomingInst)
6180 return nullptr;
6181 if (CommonInst) {
6182 if (!CommonInst->isIdenticalToWhenDefined(I: IncomingInst))
6183 return nullptr; // Not identical, give up
6184 } else {
6185 // Remember binary operator
6186 CommonInst = IncomingInst;
6187 }
6188 }
6189 return CommonInst;
6190}
6191
6192/// Returns SCEV for the first operand of a phi if all phi operands have
6193/// identical opcodes and operands
6194/// eg.
6195/// a: %add = %a + %b
6196/// br %c
6197/// b: %add1 = %a + %b
6198/// br %c
6199/// c: %phi = phi [%add, a], [%add1, b]
6200/// scev(%phi) => scev(%add)
6201const SCEV *
6202ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
6203 BinaryOperator *CommonInst = getCommonInstForPHI(PN);
6204 if (!CommonInst)
6205 return nullptr;
6206
6207 // Check if SCEV exprs for instructions are identical.
6208 const SCEV *CommonSCEV = getSCEV(V: CommonInst);
6209 bool SCEVExprsIdentical =
6210 all_of(Range: drop_begin(RangeOrContainer: PN->incoming_values()),
6211 P: [this, CommonSCEV](Value *V) { return CommonSCEV == getSCEV(V); });
6212 return SCEVExprsIdentical ? CommonSCEV : nullptr;
6213}
6214
6215const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
6216 if (const SCEV *S = createAddRecFromPHI(PN))
6217 return S;
6218
6219 // We do not allow simplifying phi (undef, X) to X here, to avoid reusing the
6220 // phi node for X.
6221 if (Value *V = simplifyInstruction(
6222 I: PN, Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
6223 /*UseInstrInfo=*/true, /*CanUseUndef=*/false}))
6224 return getSCEV(V);
6225
6226 if (const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
6227 return S;
6228
6229 if (const SCEV *S = createNodeFromSelectLikePHI(PN))
6230 return S;
6231
6232 // If it's not a loop phi, we can't handle it yet.
6233 return getUnknown(V: PN);
6234}
6235
6236bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind,
6237 SCEVTypes RootKind) {
6238 struct FindClosure {
6239 const SCEV *OperandToFind;
6240 const SCEVTypes RootKind; // Must be a sequential min/max expression.
6241 const SCEVTypes NonSequentialRootKind; // Non-seq variant of RootKind.
6242
6243 bool Found = false;
6244
6245 bool canRecurseInto(SCEVTypes Kind) const {
6246 // We can only recurse into the SCEV expression of the same effective type
6247 // as the type of our root SCEV expression, and into zero-extensions.
6248 return RootKind == Kind || NonSequentialRootKind == Kind ||
6249 scZeroExtend == Kind;
6250 };
6251
6252 FindClosure(const SCEV *OperandToFind, SCEVTypes RootKind)
6253 : OperandToFind(OperandToFind), RootKind(RootKind),
6254 NonSequentialRootKind(
6255 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
6256 Ty: RootKind)) {}
6257
6258 bool follow(const SCEV *S) {
6259 Found = S == OperandToFind;
6260
6261 return !isDone() && canRecurseInto(Kind: S->getSCEVType());
6262 }
6263
6264 bool isDone() const { return Found; }
6265 };
6266
6267 FindClosure FC(OperandToFind, RootKind);
6268 visitAll(Root, Visitor&: FC);
6269 return FC.Found;
6270}
6271
6272std::optional<const SCEV *>
6273ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond(Type *Ty,
6274 ICmpInst *Cond,
6275 Value *TrueVal,
6276 Value *FalseVal) {
6277 // Try to match some simple smax or umax patterns.
6278 auto *ICI = Cond;
6279
6280 Value *LHS = ICI->getOperand(i_nocapture: 0);
6281 Value *RHS = ICI->getOperand(i_nocapture: 1);
6282
6283 switch (ICI->getPredicate()) {
6284 case ICmpInst::ICMP_SLT:
6285 case ICmpInst::ICMP_SLE:
6286 case ICmpInst::ICMP_ULT:
6287 case ICmpInst::ICMP_ULE:
6288 std::swap(a&: LHS, b&: RHS);
6289 [[fallthrough]];
6290 case ICmpInst::ICMP_SGT:
6291 case ICmpInst::ICMP_SGE:
6292 case ICmpInst::ICMP_UGT:
6293 case ICmpInst::ICMP_UGE:
6294 // a > b ? a+x : b+x -> max(a, b)+x
6295 // a > b ? b+x : a+x -> min(a, b)+x
6296 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty)) {
6297 bool Signed = ICI->isSigned();
6298 const SCEV *LA = getSCEV(V: TrueVal);
6299 const SCEV *RA = getSCEV(V: FalseVal);
6300 const SCEV *LS = getSCEV(V: LHS);
6301 const SCEV *RS = getSCEV(V: RHS);
6302 if (LA->getType()->isPointerTy()) {
6303 // FIXME: Handle cases where LS/RS are pointers not equal to LA/RA.
6304 // Need to make sure we can't produce weird expressions involving
6305 // negated pointers.
6306 if (LA == LS && RA == RS)
6307 return Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS);
6308 if (LA == RS && RA == LS)
6309 return Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS);
6310 }
6311 auto CoerceOperand = [&](const SCEV *Op) -> const SCEV * {
6312 if (Op->getType()->isPointerTy()) {
6313 Op = getLosslessPtrToIntExpr(Op);
6314 if (isa<SCEVCouldNotCompute>(Val: Op))
6315 return Op;
6316 }
6317 if (Signed)
6318 Op = getNoopOrSignExtend(V: Op, Ty);
6319 else
6320 Op = getNoopOrZeroExtend(V: Op, Ty);
6321 return Op;
6322 };
6323 LS = CoerceOperand(LS);
6324 RS = CoerceOperand(RS);
6325 if (isa<SCEVCouldNotCompute>(Val: LS) || isa<SCEVCouldNotCompute>(Val: RS))
6326 break;
6327 const SCEV *LDiff = getMinusSCEV(LHS: LA, RHS: LS);
6328 const SCEV *RDiff = getMinusSCEV(LHS: RA, RHS: RS);
6329 if (LDiff == RDiff)
6330 return getAddExpr(LHS: Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS),
6331 RHS: LDiff);
6332 LDiff = getMinusSCEV(LHS: LA, RHS: RS);
6333 RDiff = getMinusSCEV(LHS: RA, RHS: LS);
6334 if (LDiff == RDiff)
6335 return getAddExpr(LHS: Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS),
6336 RHS: LDiff);
6337 }
6338 break;
6339 case ICmpInst::ICMP_NE:
6340 // x != 0 ? x+y : C+y -> x == 0 ? C+y : x+y
6341 std::swap(a&: TrueVal, b&: FalseVal);
6342 [[fallthrough]];
6343 case ICmpInst::ICMP_EQ:
6344 // x == 0 ? C+y : x+y -> umax(x, C)+y iff C u<= 1
6345 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty) &&
6346 isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()) {
6347 const SCEV *X = getNoopOrZeroExtend(V: getSCEV(V: LHS), Ty);
6348 const SCEV *TrueValExpr = getSCEV(V: TrueVal); // C+y
6349 const SCEV *FalseValExpr = getSCEV(V: FalseVal); // x+y
6350 const SCEV *Y = getMinusSCEV(LHS: FalseValExpr, RHS: X); // y = (x+y)-x
6351 const SCEV *C = getMinusSCEV(LHS: TrueValExpr, RHS: Y); // C = (C+y)-y
6352 if (isa<SCEVConstant>(Val: C) && cast<SCEVConstant>(Val: C)->getAPInt().ule(RHS: 1))
6353 return getAddExpr(LHS: getUMaxExpr(LHS: X, RHS: C), RHS: Y);
6354 }
6355 // x == 0 ? 0 : umin (..., x, ...) -> umin_seq(x, umin (...))
6356 // x == 0 ? 0 : umin_seq(..., x, ...) -> umin_seq(x, umin_seq(...))
6357 // x == 0 ? 0 : umin (..., umin_seq(..., x, ...), ...)
6358 // -> umin_seq(x, umin (..., umin_seq(...), ...))
6359 if (isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero() &&
6360 isa<ConstantInt>(Val: TrueVal) && cast<ConstantInt>(Val: TrueVal)->isZero()) {
6361 const SCEV *X = getSCEV(V: LHS);
6362 while (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: X))
6363 X = ZExt->getOperand();
6364 if (getTypeSizeInBits(Ty: X->getType()) <= getTypeSizeInBits(Ty)) {
6365 const SCEV *FalseValExpr = getSCEV(V: FalseVal);
6366 if (SCEVMinMaxExprContains(Root: FalseValExpr, OperandToFind: X, RootKind: scSequentialUMinExpr))
6367 return getUMinExpr(LHS: getNoopOrZeroExtend(V: X, Ty), RHS: FalseValExpr,
6368 /*Sequential=*/true);
6369 }
6370 }
6371 break;
6372 default:
6373 break;
6374 }
6375
6376 return std::nullopt;
6377}
6378
6379static std::optional<const SCEV *>
6380createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr,
6381 const SCEV *TrueExpr, const SCEV *FalseExpr) {
6382 assert(CondExpr->getType()->isIntegerTy(1) &&
6383 TrueExpr->getType() == FalseExpr->getType() &&
6384 TrueExpr->getType()->isIntegerTy(1) &&
6385 "Unexpected operands of a select.");
6386
6387 // i1 cond ? i1 x : i1 C --> C + (i1 cond ? (i1 x - i1 C) : i1 0)
6388 // --> C + (umin_seq cond, x - C)
6389 //
6390 // i1 cond ? i1 C : i1 x --> C + (i1 cond ? i1 0 : (i1 x - i1 C))
6391 // --> C + (i1 ~cond ? (i1 x - i1 C) : i1 0)
6392 // --> C + (umin_seq ~cond, x - C)
6393
6394 // FIXME: while we can't legally model the case where both of the hands
6395 // are fully variable, we only require that the *difference* is constant.
6396 if (!isa<SCEVConstant>(Val: TrueExpr) && !isa<SCEVConstant>(Val: FalseExpr))
6397 return std::nullopt;
6398
6399 const SCEV *X, *C;
6400 if (isa<SCEVConstant>(Val: TrueExpr)) {
6401 CondExpr = SE->getNotSCEV(V: CondExpr);
6402 X = FalseExpr;
6403 C = TrueExpr;
6404 } else {
6405 X = TrueExpr;
6406 C = FalseExpr;
6407 }
6408 return SE->getAddExpr(LHS: C, RHS: SE->getUMinExpr(LHS: CondExpr, RHS: SE->getMinusSCEV(LHS: X, RHS: C),
6409 /*Sequential=*/true));
6410}
6411
6412static std::optional<const SCEV *>
6413createNodeForSelectViaUMinSeq(ScalarEvolution *SE, Value *Cond, Value *TrueVal,
6414 Value *FalseVal) {
6415 if (!isa<ConstantInt>(Val: TrueVal) && !isa<ConstantInt>(Val: FalseVal))
6416 return std::nullopt;
6417
6418 const auto *SECond = SE->getSCEV(V: Cond);
6419 const auto *SETrue = SE->getSCEV(V: TrueVal);
6420 const auto *SEFalse = SE->getSCEV(V: FalseVal);
6421 return createNodeForSelectViaUMinSeq(SE, CondExpr: SECond, TrueExpr: SETrue, FalseExpr: SEFalse);
6422}
6423
6424const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq(
6425 Value *V, Value *Cond, Value *TrueVal, Value *FalseVal) {
6426 assert(Cond->getType()->isIntegerTy(1) && "Select condition is not an i1?");
6427 assert(TrueVal->getType() == FalseVal->getType() &&
6428 V->getType() == TrueVal->getType() &&
6429 "Types of select hands and of the result must match.");
6430
6431 // For now, only deal with i1-typed `select`s.
6432 if (!V->getType()->isIntegerTy(BitWidth: 1))
6433 return getUnknown(V);
6434
6435 if (std::optional<const SCEV *> S =
6436 createNodeForSelectViaUMinSeq(SE: this, Cond, TrueVal, FalseVal))
6437 return *S;
6438
6439 return getUnknown(V);
6440}
6441
6442const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Value *V, Value *Cond,
6443 Value *TrueVal,
6444 Value *FalseVal) {
6445 // Handle "constant" branch or select. This can occur for instance when a
6446 // loop pass transforms an inner loop and moves on to process the outer loop.
6447 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond))
6448 return getSCEV(V: CI->isOne() ? TrueVal : FalseVal);
6449
6450 if (auto *I = dyn_cast<Instruction>(Val: V)) {
6451 if (auto *ICI = dyn_cast<ICmpInst>(Val: Cond)) {
6452 if (std::optional<const SCEV *> S =
6453 createNodeForSelectOrPHIInstWithICmpInstCond(Ty: I->getType(), Cond: ICI,
6454 TrueVal, FalseVal))
6455 return *S;
6456 }
6457 }
6458
6459 return createNodeForSelectOrPHIViaUMinSeq(V, Cond, TrueVal, FalseVal);
6460}
6461
6462/// Expand GEP instructions into add and multiply operations. This allows them
6463/// to be analyzed by regular SCEV code.
6464const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
6465 assert(GEP->getSourceElementType()->isSized() &&
6466 "GEP source element type must be sized");
6467
6468 SmallVector<SCEVUse, 4> IndexExprs;
6469 for (Value *Index : GEP->indices())
6470 IndexExprs.push_back(Elt: getSCEV(V: Index));
6471 return getGEPExpr(GEP, IndexExprs);
6472}
6473
6474APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S,
6475 const Instruction *CtxI) {
6476 uint64_t BitWidth = getTypeSizeInBits(Ty: S->getType());
6477 auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) {
6478 return TrailingZeros >= BitWidth
6479 ? APInt::getZero(numBits: BitWidth)
6480 : APInt::getOneBitSet(numBits: BitWidth, BitNo: TrailingZeros);
6481 };
6482 auto GetGCDMultiple = [this, CtxI](const SCEVNAryExpr *N) {
6483 // The result is GCD of all operands results.
6484 APInt Res = getConstantMultiple(S: N->getOperand(i: 0), CtxI);
6485 for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I)
6486 Res = APIntOps::GreatestCommonDivisor(
6487 A: Res, B: getConstantMultiple(S: N->getOperand(i: I), CtxI));
6488 return Res;
6489 };
6490
6491 switch (S->getSCEVType()) {
6492 case scConstant:
6493 return cast<SCEVConstant>(Val: S)->getAPInt();
6494 case scPtrToAddr:
6495 case scPtrToInt:
6496 return getConstantMultiple(S: cast<SCEVCastExpr>(Val: S)->getOperand());
6497 case scUDivExpr:
6498 case scVScale:
6499 return APInt(BitWidth, 1);
6500 case scTruncate: {
6501 // Only multiples that are a power of 2 will hold after truncation.
6502 const SCEVTruncateExpr *T = cast<SCEVTruncateExpr>(Val: S);
6503 uint32_t TZ = getMinTrailingZeros(S: T->getOperand(), CtxI);
6504 return GetShiftedByZeros(TZ);
6505 }
6506 case scZeroExtend: {
6507 const SCEVZeroExtendExpr *Z = cast<SCEVZeroExtendExpr>(Val: S);
6508 return getConstantMultiple(S: Z->getOperand(), CtxI).zext(width: BitWidth);
6509 }
6510 case scSignExtend: {
6511 // Only multiples that are a power of 2 will hold after sext.
6512 const SCEVSignExtendExpr *E = cast<SCEVSignExtendExpr>(Val: S);
6513 uint32_t TZ = getMinTrailingZeros(S: E->getOperand(), CtxI);
6514 return GetShiftedByZeros(TZ);
6515 }
6516 case scMulExpr: {
6517 const SCEVMulExpr *M = cast<SCEVMulExpr>(Val: S);
6518 if (M->hasNoUnsignedWrap()) {
6519 // The result is the product of all operand results.
6520 APInt Res = getConstantMultiple(S: M->getOperand(i: 0), CtxI);
6521 for (const SCEV *Operand : M->operands().drop_front())
6522 Res = Res * getConstantMultiple(S: Operand, CtxI);
6523 return Res;
6524 }
6525
6526 // If there are no wrap guarentees, find the trailing zeros, which is the
6527 // sum of trailing zeros for all its operands.
6528 uint32_t TZ = 0;
6529 for (const SCEV *Operand : M->operands())
6530 TZ += getMinTrailingZeros(S: Operand, CtxI);
6531 return GetShiftedByZeros(TZ);
6532 }
6533 case scAddExpr:
6534 case scAddRecExpr: {
6535 const SCEVNAryExpr *N = cast<SCEVNAryExpr>(Val: S);
6536 if (N->hasNoUnsignedWrap())
6537 return GetGCDMultiple(N);
6538 // Find the trailing bits, which is the minimum of its operands.
6539 uint32_t TZ = getMinTrailingZeros(S: N->getOperand(i: 0), CtxI);
6540 for (const SCEV *Operand : N->operands().drop_front())
6541 TZ = std::min(a: TZ, b: getMinTrailingZeros(S: Operand, CtxI));
6542 return GetShiftedByZeros(TZ);
6543 }
6544 case scUMaxExpr:
6545 case scSMaxExpr:
6546 case scUMinExpr:
6547 case scSMinExpr:
6548 case scSequentialUMinExpr:
6549 return GetGCDMultiple(cast<SCEVNAryExpr>(Val: S));
6550 case scUnknown: {
6551 // Ask ValueTracking for known bits. SCEVUnknown only become available at
6552 // the point their underlying IR instruction has been defined. If CtxI was
6553 // not provided, use:
6554 // * the first instruction in the entry block if it is an argument
6555 // * the instruction itself otherwise.
6556 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6557 if (!CtxI) {
6558 if (isa<Argument>(Val: U->getValue()))
6559 CtxI = &*F.getEntryBlock().begin();
6560 else if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
6561 CtxI = I;
6562 }
6563 unsigned Known =
6564 computeKnownBits(V: U->getValue(),
6565 Q: SimplifyQuery(getDataLayout(), &DT, &AC, CtxI)
6566 .allowEphemerals(AllowEphemerals: true))
6567 .countMinTrailingZeros();
6568 return GetShiftedByZeros(Known);
6569 }
6570 case scCouldNotCompute:
6571 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6572 }
6573 llvm_unreachable("Unknown SCEV kind!");
6574}
6575
6576APInt ScalarEvolution::getConstantMultiple(const SCEV *S,
6577 const Instruction *CtxI) {
6578 // Skip looking up and updating the cache if there is a context instruction,
6579 // as the result will only be valid in the specified context.
6580 if (CtxI)
6581 return getConstantMultipleImpl(S, CtxI);
6582
6583 auto I = ConstantMultipleCache.find(Val: S);
6584 if (I != ConstantMultipleCache.end())
6585 return I->second;
6586
6587 APInt Result = getConstantMultipleImpl(S, CtxI);
6588 auto InsertPair = ConstantMultipleCache.insert(KV: {S, Result});
6589 assert(InsertPair.second && "Should insert a new key");
6590 return InsertPair.first->second;
6591}
6592
6593APInt ScalarEvolution::getNonZeroConstantMultiple(const SCEV *S) {
6594 APInt Multiple = getConstantMultiple(S);
6595 return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple;
6596}
6597
6598uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S,
6599 const Instruction *CtxI) {
6600 return std::min(a: getConstantMultiple(S, CtxI).countTrailingZeros(),
6601 b: (unsigned)getTypeSizeInBits(Ty: S->getType()));
6602}
6603
6604/// Helper method to assign a range to V from metadata present in the IR.
6605static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
6606 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
6607 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_range))
6608 return getConstantRangeFromMetadata(RangeMD: *MD);
6609 if (const auto *CB = dyn_cast<CallBase>(Val: V))
6610 if (std::optional<ConstantRange> Range = CB->getRange())
6611 return Range;
6612 }
6613 if (auto *A = dyn_cast<Argument>(Val: V))
6614 if (std::optional<ConstantRange> Range = A->getRange())
6615 return Range;
6616
6617 return std::nullopt;
6618}
6619
6620void ScalarEvolution::setNoWrapFlags(SCEVAddRecExpr *AddRec,
6621 SCEV::NoWrapFlags Flags) {
6622 if (AddRec->getNoWrapFlags(Mask: Flags) != Flags) {
6623 AddRec->setNoWrapFlags(Flags);
6624 UnsignedRanges.erase(Val: AddRec);
6625 SignedRanges.erase(Val: AddRec);
6626 ConstantMultipleCache.erase(Val: AddRec);
6627 }
6628}
6629
6630ConstantRange ScalarEvolution::
6631getRangeForUnknownRecurrence(const SCEVUnknown *U) {
6632 const DataLayout &DL = getDataLayout();
6633
6634 unsigned BitWidth = getTypeSizeInBits(Ty: U->getType());
6635 const ConstantRange FullSet(BitWidth, /*isFullSet=*/true);
6636
6637 // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
6638 // use information about the trip count to improve our available range. Note
6639 // that the trip count independent cases are already handled by known bits.
6640 // WARNING: The definition of recurrence used here is subtly different than
6641 // the one used by AddRec (and thus most of this file). Step is allowed to
6642 // be arbitrarily loop varying here, where AddRec allows only loop invariant
6643 // and other addrecs in the same loop (for non-affine addrecs). The code
6644 // below intentionally handles the case where step is not loop invariant.
6645 auto *P = dyn_cast<PHINode>(Val: U->getValue());
6646 if (!P)
6647 return FullSet;
6648
6649 // Make sure that no Phi input comes from an unreachable block. Otherwise,
6650 // even the values that are not available in these blocks may come from them,
6651 // and this leads to false-positive recurrence test.
6652 for (auto *Pred : predecessors(BB: P->getParent()))
6653 if (!DT.isReachableFromEntry(A: Pred))
6654 return FullSet;
6655
6656 BinaryOperator *BO;
6657 Value *Start, *Step;
6658 if (!matchSimpleRecurrence(P, BO, Start, Step))
6659 return FullSet;
6660
6661 // If we found a recurrence in reachable code, we must be in a loop. Note
6662 // that BO might be in some subloop of L, and that's completely okay.
6663 auto *L = LI.getLoopFor(BB: P->getParent());
6664 assert(L && L->getHeader() == P->getParent());
6665 if (!L->contains(BB: BO->getParent()))
6666 // NOTE: This bailout should be an assert instead. However, asserting
6667 // the condition here exposes a case where LoopFusion is querying SCEV
6668 // with malformed loop information during the midst of the transform.
6669 // There doesn't appear to be an obvious fix, so for the moment bailout
6670 // until the caller issue can be fixed. PR49566 tracks the bug.
6671 return FullSet;
6672
6673 // TODO: Extend to other opcodes such as mul, and div
6674 switch (BO->getOpcode()) {
6675 default:
6676 return FullSet;
6677 case Instruction::AShr:
6678 case Instruction::LShr:
6679 case Instruction::Shl:
6680 break;
6681 };
6682
6683 if (BO->getOperand(i_nocapture: 0) != P)
6684 // TODO: Handle the power function forms some day.
6685 return FullSet;
6686
6687 unsigned TC = getSmallConstantMaxTripCount(L);
6688 if (!TC || TC >= BitWidth)
6689 return FullSet;
6690
6691 auto KnownStart = computeKnownBits(V: Start, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6692 auto KnownStep = computeKnownBits(V: Step, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6693 assert(KnownStart.getBitWidth() == BitWidth &&
6694 KnownStep.getBitWidth() == BitWidth);
6695
6696 // Compute total shift amount, being careful of overflow and bitwidths.
6697 auto MaxShiftAmt = KnownStep.getMaxValue();
6698 APInt TCAP(BitWidth, TC-1);
6699 bool Overflow = false;
6700 auto TotalShift = MaxShiftAmt.umul_ov(RHS: TCAP, Overflow);
6701 if (Overflow)
6702 return FullSet;
6703
6704 switch (BO->getOpcode()) {
6705 default:
6706 llvm_unreachable("filtered out above");
6707 case Instruction::AShr: {
6708 // For each ashr, three cases:
6709 // shift = 0 => unchanged value
6710 // saturation => 0 or -1
6711 // other => a value closer to zero (of the same sign)
6712 // Thus, the end value is closer to zero than the start.
6713 auto KnownEnd = KnownBits::ashr(LHS: KnownStart,
6714 RHS: KnownBits::makeConstant(C: TotalShift));
6715 if (KnownStart.isNonNegative())
6716 // Analogous to lshr (simply not yet canonicalized)
6717 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6718 Upper: KnownStart.getMaxValue() + 1);
6719 if (KnownStart.isNegative())
6720 // End >=u Start && End <=s Start
6721 return ConstantRange::getNonEmpty(Lower: KnownStart.getMinValue(),
6722 Upper: KnownEnd.getMaxValue() + 1);
6723 break;
6724 }
6725 case Instruction::LShr: {
6726 // For each lshr, three cases:
6727 // shift = 0 => unchanged value
6728 // saturation => 0
6729 // other => a smaller positive number
6730 // Thus, the low end of the unsigned range is the last value produced.
6731 auto KnownEnd = KnownBits::lshr(LHS: KnownStart,
6732 RHS: KnownBits::makeConstant(C: TotalShift));
6733 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6734 Upper: KnownStart.getMaxValue() + 1);
6735 }
6736 case Instruction::Shl: {
6737 // Iff no bits are shifted out, value increases on every shift.
6738 auto KnownEnd = KnownBits::shl(LHS: KnownStart,
6739 RHS: KnownBits::makeConstant(C: TotalShift));
6740 if (TotalShift.ult(RHS: KnownStart.countMinLeadingZeros()))
6741 return ConstantRange(KnownStart.getMinValue(),
6742 KnownEnd.getMaxValue() + 1);
6743 break;
6744 }
6745 };
6746 return FullSet;
6747}
6748
6749// The goal of this function is to check if recursively visiting the operands
6750// of this PHI might lead to an infinite loop. If we do see such a loop,
6751// there's no good way to break it, so we avoid analyzing such cases.
6752//
6753// getRangeRef previously used a visited set to avoid infinite loops, but this
6754// caused other issues: the result was dependent on the order of getRangeRef
6755// calls, and the interaction with createSCEVIter could cause a stack overflow
6756// in some cases (see issue #148253).
6757//
6758// FIXME: The way this is implemented is overly conservative; this checks
6759// for a few obviously safe patterns, but anything that doesn't lead to
6760// recursion is fine.
6761static bool RangeRefPHIAllowedOperands(DominatorTree &DT, PHINode *PHI) {
6762 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
6763 if (getOperandsForSelectLikePHI(DT, PN: PHI, Cond, LHS, RHS))
6764 return true;
6765
6766 if (all_of(Range: PHI->operands(),
6767 P: [&](Value *Operand) { return DT.dominates(Def: Operand, User: PHI); }))
6768 return true;
6769
6770 return false;
6771}
6772
6773const ConstantRange &
6774ScalarEvolution::getRangeRefIter(const SCEV *S,
6775 ScalarEvolution::RangeSignHint SignHint) {
6776 DenseMap<const SCEV *, ConstantRange> &Cache =
6777 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6778 : SignedRanges;
6779 SmallVector<SCEVUse> WorkList;
6780 SmallPtrSet<const SCEV *, 8> Seen;
6781
6782 // Add Expr to the worklist, if Expr is either an N-ary expression or a
6783 // SCEVUnknown PHI node.
6784 auto AddToWorklist = [&WorkList, &Seen, &Cache](const SCEV *Expr) {
6785 if (!Seen.insert(Ptr: Expr).second)
6786 return;
6787 if (Cache.contains(Val: Expr))
6788 return;
6789 switch (Expr->getSCEVType()) {
6790 case scUnknown:
6791 if (!isa<PHINode>(Val: cast<SCEVUnknown>(Val: Expr)->getValue()))
6792 break;
6793 [[fallthrough]];
6794 case scConstant:
6795 case scVScale:
6796 case scTruncate:
6797 case scZeroExtend:
6798 case scSignExtend:
6799 case scPtrToAddr:
6800 case scPtrToInt:
6801 case scAddExpr:
6802 case scMulExpr:
6803 case scUDivExpr:
6804 case scAddRecExpr:
6805 case scUMaxExpr:
6806 case scSMaxExpr:
6807 case scUMinExpr:
6808 case scSMinExpr:
6809 case scSequentialUMinExpr:
6810 WorkList.push_back(Elt: Expr);
6811 break;
6812 case scCouldNotCompute:
6813 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6814 }
6815 };
6816 AddToWorklist(S);
6817
6818 // Build worklist by queuing operands of N-ary expressions and phi nodes.
6819 for (unsigned I = 0; I != WorkList.size(); ++I) {
6820 const SCEV *P = WorkList[I];
6821 auto *UnknownS = dyn_cast<SCEVUnknown>(Val: P);
6822 // If it is not a `SCEVUnknown`, just recurse into operands.
6823 if (!UnknownS) {
6824 for (const SCEV *Op : P->operands())
6825 AddToWorklist(Op);
6826 continue;
6827 }
6828 // `SCEVUnknown`'s require special treatment.
6829 if (PHINode *P = dyn_cast<PHINode>(Val: UnknownS->getValue())) {
6830 if (!RangeRefPHIAllowedOperands(DT, PHI: P))
6831 continue;
6832 for (auto &Op : reverse(C: P->operands()))
6833 AddToWorklist(getSCEV(V: Op));
6834 }
6835 }
6836
6837 if (!WorkList.empty()) {
6838 // Use getRangeRef to compute ranges for items in the worklist in reverse
6839 // order. This will force ranges for earlier operands to be computed before
6840 // their users in most cases.
6841 for (const SCEV *P : reverse(C: drop_begin(RangeOrContainer&: WorkList))) {
6842 getRangeRef(S: P, Hint: SignHint);
6843 }
6844 }
6845
6846 return getRangeRef(S, Hint: SignHint, Depth: 0);
6847}
6848
6849/// Determine the range for a particular SCEV. If SignHint is
6850/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
6851/// with a "cleaner" unsigned (resp. signed) representation.
6852const ConstantRange &ScalarEvolution::getRangeRef(
6853 const SCEV *S, ScalarEvolution::RangeSignHint SignHint, unsigned Depth) {
6854 DenseMap<const SCEV *, ConstantRange> &Cache =
6855 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6856 : SignedRanges;
6857 ConstantRange::PreferredRangeType RangeType =
6858 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? ConstantRange::Unsigned
6859 : ConstantRange::Signed;
6860
6861 // See if we've computed this range already.
6862 auto I = Cache.find(Val: S);
6863 if (I != Cache.end())
6864 return I->second;
6865
6866 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: S))
6867 return setRange(S: C, Hint: SignHint, CR: ConstantRange(C->getAPInt()));
6868
6869 // Switch to iteratively computing the range for S, if it is part of a deeply
6870 // nested expression.
6871 if (Depth > RangeIterThreshold)
6872 return getRangeRefIter(S, SignHint);
6873
6874 unsigned BitWidth = getTypeSizeInBits(Ty: S->getType());
6875 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
6876 using OBO = OverflowingBinaryOperator;
6877
6878 // If the value has known zeros, the maximum value will have those known zeros
6879 // as well.
6880 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
6881 APInt Multiple = getNonZeroConstantMultiple(S);
6882 APInt Remainder = APInt::getMaxValue(numBits: BitWidth).urem(RHS: Multiple);
6883 if (!Remainder.isZero())
6884 ConservativeResult =
6885 ConstantRange(APInt::getMinValue(numBits: BitWidth),
6886 APInt::getMaxValue(numBits: BitWidth) - Remainder + 1);
6887 }
6888 else {
6889 uint32_t TZ = getMinTrailingZeros(S);
6890 if (TZ != 0) {
6891 ConservativeResult = ConstantRange(
6892 APInt::getSignedMinValue(numBits: BitWidth),
6893 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: TZ).shl(shiftAmt: TZ) + 1);
6894 }
6895 }
6896
6897 switch (S->getSCEVType()) {
6898 case scConstant:
6899 llvm_unreachable("Already handled above.");
6900 case scVScale:
6901 return setRange(S, Hint: SignHint, CR: getVScaleRange(F: &F, BitWidth));
6902 case scTruncate: {
6903 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: S);
6904 ConstantRange X = getRangeRef(S: Trunc->getOperand(), SignHint, Depth: Depth + 1);
6905 return setRange(
6906 S: Trunc, Hint: SignHint,
6907 CR: ConservativeResult.intersectWith(CR: X.truncate(BitWidth), Type: RangeType));
6908 }
6909 case scZeroExtend: {
6910 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: S);
6911 ConstantRange X = getRangeRef(S: ZExt->getOperand(), SignHint, Depth: Depth + 1);
6912 return setRange(
6913 S: ZExt, Hint: SignHint,
6914 CR: ConservativeResult.intersectWith(CR: X.zeroExtend(BitWidth), Type: RangeType));
6915 }
6916 case scSignExtend: {
6917 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: S);
6918 ConstantRange X = getRangeRef(S: SExt->getOperand(), SignHint, Depth: Depth + 1);
6919 return setRange(
6920 S: SExt, Hint: SignHint,
6921 CR: ConservativeResult.intersectWith(CR: X.signExtend(BitWidth), Type: RangeType));
6922 }
6923 case scPtrToAddr:
6924 case scPtrToInt: {
6925 const SCEVCastExpr *Cast = cast<SCEVCastExpr>(Val: S);
6926 ConstantRange X = getRangeRef(S: Cast->getOperand(), SignHint, Depth: Depth + 1);
6927 return setRange(S: Cast, Hint: SignHint, CR: X);
6928 }
6929 case scAddExpr: {
6930 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: S);
6931 // Check if this is a URem pattern: A - (A / B) * B, which is always < B.
6932 const SCEV *URemLHS = nullptr, *URemRHS = nullptr;
6933 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED &&
6934 match(S, P: m_scev_URem(LHS: m_SCEV(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE&: *this))) {
6935 ConstantRange LHSRange = getRangeRef(S: URemLHS, SignHint, Depth: Depth + 1);
6936 ConstantRange RHSRange = getRangeRef(S: URemRHS, SignHint, Depth: Depth + 1);
6937 ConservativeResult =
6938 ConservativeResult.intersectWith(CR: LHSRange.urem(Other: RHSRange), Type: RangeType);
6939 }
6940 ConstantRange X = getRangeRef(S: Add->getOperand(i: 0), SignHint, Depth: Depth + 1);
6941 unsigned WrapType = OBO::AnyWrap;
6942 if (Add->hasNoSignedWrap())
6943 WrapType |= OBO::NoSignedWrap;
6944 if (Add->hasNoUnsignedWrap())
6945 WrapType |= OBO::NoUnsignedWrap;
6946 for (const SCEV *Op : drop_begin(RangeOrContainer: Add->operands()))
6947 X = X.addWithNoWrap(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1), NoWrapKind: WrapType,
6948 RangeType);
6949 return setRange(S: Add, Hint: SignHint,
6950 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6951 }
6952 case scMulExpr: {
6953 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val: S);
6954 ConstantRange X = getRangeRef(S: Mul->getOperand(i: 0), SignHint, Depth: Depth + 1);
6955 for (const SCEV *Op : drop_begin(RangeOrContainer: Mul->operands()))
6956 X = X.multiply(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1));
6957 return setRange(S: Mul, Hint: SignHint,
6958 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6959 }
6960 case scUDivExpr: {
6961 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: S);
6962 ConstantRange X = getRangeRef(S: UDiv->getLHS(), SignHint, Depth: Depth + 1);
6963 ConstantRange Y = getRangeRef(S: UDiv->getRHS(), SignHint, Depth: Depth + 1);
6964 return setRange(S: UDiv, Hint: SignHint,
6965 CR: ConservativeResult.intersectWith(CR: X.udiv(Other: Y), Type: RangeType));
6966 }
6967 case scAddRecExpr: {
6968 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: S);
6969 // If there's no unsigned wrap, the value will never be less than its
6970 // initial value.
6971 if (AddRec->hasNoUnsignedWrap()) {
6972 APInt UnsignedMinValue = getUnsignedRangeMin(S: AddRec->getStart());
6973 if (!UnsignedMinValue.isZero())
6974 ConservativeResult = ConservativeResult.intersectWith(
6975 CR: ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), Type: RangeType);
6976 }
6977
6978 // If there's no signed wrap, and all the operands except initial value have
6979 // the same sign or zero, the value won't ever be:
6980 // 1: smaller than initial value if operands are non negative,
6981 // 2: bigger than initial value if operands are non positive.
6982 // For both cases, value can not cross signed min/max boundary.
6983 if (AddRec->hasNoSignedWrap()) {
6984 bool AllNonNeg = true;
6985 bool AllNonPos = true;
6986 for (unsigned i = 1, e = AddRec->getNumOperands(); i != e; ++i) {
6987 if (!isKnownNonNegative(S: AddRec->getOperand(i)))
6988 AllNonNeg = false;
6989 if (!isKnownNonPositive(S: AddRec->getOperand(i)))
6990 AllNonPos = false;
6991 }
6992 if (AllNonNeg)
6993 ConservativeResult = ConservativeResult.intersectWith(
6994 CR: ConstantRange::getNonEmpty(Lower: getSignedRangeMin(S: AddRec->getStart()),
6995 Upper: APInt::getSignedMinValue(numBits: BitWidth)),
6996 Type: RangeType);
6997 else if (AllNonPos)
6998 ConservativeResult = ConservativeResult.intersectWith(
6999 CR: ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
7000 Upper: getSignedRangeMax(S: AddRec->getStart()) +
7001 1),
7002 Type: RangeType);
7003 }
7004
7005 // TODO: non-affine addrec
7006 if (AddRec->isAffine()) {
7007 const SCEV *MaxBEScev =
7008 getConstantMaxBackedgeTakenCount(L: AddRec->getLoop());
7009 if (!isa<SCEVCouldNotCompute>(Val: MaxBEScev)) {
7010 APInt MaxBECount = cast<SCEVConstant>(Val: MaxBEScev)->getAPInt();
7011
7012 // Adjust MaxBECount to the same bitwidth as AddRec. We can truncate if
7013 // MaxBECount's active bits are all <= AddRec's bit width.
7014 if (MaxBECount.getBitWidth() > BitWidth &&
7015 MaxBECount.getActiveBits() <= BitWidth)
7016 MaxBECount = MaxBECount.trunc(width: BitWidth);
7017 else if (MaxBECount.getBitWidth() < BitWidth)
7018 MaxBECount = MaxBECount.zext(width: BitWidth);
7019
7020 if (MaxBECount.getBitWidth() == BitWidth) {
7021 auto [RangeFromAffine, Flags] = getRangeForAffineAR(
7022 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
7023 ConservativeResult =
7024 ConservativeResult.intersectWith(CR: RangeFromAffine, Type: RangeType);
7025 const_cast<SCEVAddRecExpr *>(AddRec)->setNoWrapFlags(Flags);
7026
7027 auto RangeFromFactoring = getRangeViaFactoring(
7028 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
7029 ConservativeResult =
7030 ConservativeResult.intersectWith(CR: RangeFromFactoring, Type: RangeType);
7031 }
7032 }
7033
7034 // Now try symbolic BE count and more powerful methods.
7035 if (UseExpensiveRangeSharpening) {
7036 const SCEV *SymbolicMaxBECount =
7037 getSymbolicMaxBackedgeTakenCount(L: AddRec->getLoop());
7038 if (!isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount) &&
7039 getTypeSizeInBits(Ty: MaxBEScev->getType()) <= BitWidth &&
7040 AddRec->hasNoSelfWrap()) {
7041 auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
7042 AddRec, MaxBECount: SymbolicMaxBECount, BitWidth, SignHint);
7043 ConservativeResult =
7044 ConservativeResult.intersectWith(CR: RangeFromAffineNew, Type: RangeType);
7045 }
7046 }
7047 }
7048
7049 return setRange(S: AddRec, Hint: SignHint, CR: std::move(ConservativeResult));
7050 }
7051 case scUMaxExpr:
7052 case scSMaxExpr:
7053 case scUMinExpr:
7054 case scSMinExpr:
7055 case scSequentialUMinExpr: {
7056 Intrinsic::ID ID;
7057 switch (S->getSCEVType()) {
7058 case scUMaxExpr:
7059 ID = Intrinsic::umax;
7060 break;
7061 case scSMaxExpr:
7062 ID = Intrinsic::smax;
7063 break;
7064 case scUMinExpr:
7065 case scSequentialUMinExpr:
7066 ID = Intrinsic::umin;
7067 break;
7068 case scSMinExpr:
7069 ID = Intrinsic::smin;
7070 break;
7071 default:
7072 llvm_unreachable("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr.");
7073 }
7074
7075 const auto *NAry = cast<SCEVNAryExpr>(Val: S);
7076 ConstantRange X = getRangeRef(S: NAry->getOperand(i: 0), SignHint, Depth: Depth + 1);
7077 for (unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
7078 X = X.intrinsic(
7079 IntrinsicID: ID, Ops: {X, getRangeRef(S: NAry->getOperand(i), SignHint, Depth: Depth + 1)});
7080 return setRange(S, Hint: SignHint,
7081 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
7082 }
7083 case scUnknown: {
7084 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
7085 Value *V = U->getValue();
7086
7087 // Check if the IR explicitly contains !range metadata.
7088 std::optional<ConstantRange> MDRange = GetRangeFromMetadata(V);
7089 if (MDRange)
7090 ConservativeResult =
7091 ConservativeResult.intersectWith(CR: *MDRange, Type: RangeType);
7092
7093 // Use facts about recurrences in the underlying IR. Note that add
7094 // recurrences are AddRecExprs and thus don't hit this path. This
7095 // primarily handles shift recurrences.
7096 auto CR = getRangeForUnknownRecurrence(U);
7097 ConservativeResult = ConservativeResult.intersectWith(CR);
7098
7099 // See if ValueTracking can give us a useful range.
7100 const DataLayout &DL = getDataLayout();
7101 KnownBits Known = computeKnownBits(V, DL, AC: &AC, CxtI: nullptr, DT: &DT);
7102 if (Known.getBitWidth() != BitWidth)
7103 Known = Known.zextOrTrunc(BitWidth);
7104
7105 // ValueTracking may be able to compute a tighter result for the number of
7106 // sign bits than for the value of those sign bits.
7107 unsigned NS = ComputeNumSignBits(Op: V, DL, AC: &AC, CxtI: nullptr, DT: &DT);
7108 if (U->getType()->isPointerTy()) {
7109 // If the pointer size is larger than the index size type, this can cause
7110 // NS to be larger than BitWidth. So compensate for this.
7111 unsigned ptrSize = DL.getPointerTypeSizeInBits(U->getType());
7112 int ptrIdxDiff = ptrSize - BitWidth;
7113 if (ptrIdxDiff > 0 && ptrSize > BitWidth && NS > (unsigned)ptrIdxDiff)
7114 NS -= ptrIdxDiff;
7115 }
7116
7117 if (NS > 1) {
7118 // If we know any of the sign bits, we know all of the sign bits.
7119 if (!Known.Zero.getHiBits(numBits: NS).isZero())
7120 Known.Zero.setHighBits(NS);
7121 if (!Known.One.getHiBits(numBits: NS).isZero())
7122 Known.One.setHighBits(NS);
7123 }
7124
7125 if (Known.getMinValue() != Known.getMaxValue() + 1)
7126 ConservativeResult = ConservativeResult.intersectWith(
7127 CR: ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
7128 Type: RangeType);
7129 if (NS > 1)
7130 ConservativeResult = ConservativeResult.intersectWith(
7131 CR: ConstantRange(APInt::getSignedMinValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1),
7132 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1) + 1),
7133 Type: RangeType);
7134
7135 if (U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
7136 // Strengthen the range if the underlying IR value is a
7137 // global/alloca/heap allocation using the size of the object.
7138 bool CanBeNull;
7139 uint64_t DerefBytes = V->getPointerDereferenceableBytes(
7140 DL, CanBeNull, /*CanBeFreed=*/nullptr);
7141 if (DerefBytes > 1 && isUIntN(N: BitWidth, x: DerefBytes)) {
7142 // The highest address the object can start is DerefBytes bytes before
7143 // the end (unsigned max value). If this value is not a multiple of the
7144 // alignment, the last possible start value is the next lowest multiple
7145 // of the alignment. Note: The computations below cannot overflow,
7146 // because if they would there's no possible start address for the
7147 // object.
7148 APInt MaxVal =
7149 APInt::getMaxValue(numBits: BitWidth) - APInt(BitWidth, DerefBytes);
7150 uint64_t Align = U->getValue()->getPointerAlignment(DL).value();
7151 uint64_t Rem = MaxVal.urem(RHS: Align);
7152 MaxVal -= APInt(BitWidth, Rem);
7153 APInt MinVal = APInt::getZero(numBits: BitWidth);
7154 if (llvm::isKnownNonZero(V, Q: DL))
7155 MinVal = Align;
7156 ConservativeResult = ConservativeResult.intersectWith(
7157 CR: ConstantRange::getNonEmpty(Lower: MinVal, Upper: MaxVal + 1), Type: RangeType);
7158 }
7159 }
7160
7161 // A range of Phi is a subset of union of all ranges of its input.
7162 if (PHINode *Phi = dyn_cast<PHINode>(Val: V)) {
7163 // SCEVExpander sometimes creates SCEVUnknowns that are secretly
7164 // AddRecs; return the range for the corresponding AddRec.
7165 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: getSCEV(V)))
7166 return getRangeRef(S: AR, SignHint, Depth: Depth + 1);
7167
7168 // Make sure that we do not run over cycled Phis.
7169 if (RangeRefPHIAllowedOperands(DT, PHI: Phi)) {
7170 ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false);
7171
7172 for (const auto &Op : Phi->operands()) {
7173 auto OpRange = getRangeRef(S: getSCEV(V: Op), SignHint, Depth: Depth + 1);
7174 RangeFromOps = RangeFromOps.unionWith(CR: OpRange);
7175 // No point to continue if we already have a full set.
7176 if (RangeFromOps.isFullSet())
7177 break;
7178 }
7179 ConservativeResult =
7180 ConservativeResult.intersectWith(CR: RangeFromOps, Type: RangeType);
7181 }
7182 }
7183
7184 // vscale can't be equal to zero
7185 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V))
7186 if (II->getIntrinsicID() == Intrinsic::vscale) {
7187 ConstantRange Disallowed = APInt::getZero(numBits: BitWidth);
7188 ConservativeResult = ConservativeResult.difference(CR: Disallowed);
7189 }
7190
7191 return setRange(S: U, Hint: SignHint, CR: std::move(ConservativeResult));
7192 }
7193 case scCouldNotCompute:
7194 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
7195 }
7196
7197 return setRange(S, Hint: SignHint, CR: std::move(ConservativeResult));
7198}
7199
7200// Given a StartRange, Step and MaxBECount for an expression compute a range of
7201// values that the expression can take. Initially, the expression has a value
7202// from StartRange and then is changed by Step up to MaxBECount times. Signed
7203// argument defines if we treat Step as signed or unsigned. The second return
7204// value indicates that no wrapping occurred.
7205static std::pair<ConstantRange, bool>
7206getRangeForAffineARHelper(APInt Step, const ConstantRange &StartRange,
7207 const APInt &MaxBECount, bool Signed) {
7208 unsigned BitWidth = Step.getBitWidth();
7209 assert(BitWidth == StartRange.getBitWidth() &&
7210 BitWidth == MaxBECount.getBitWidth() && "mismatched bit widths");
7211 // If either Step or MaxBECount is 0, then the expression won't change, and we
7212 // just need to return the initial range.
7213 if (Step == 0 || MaxBECount == 0)
7214 return {StartRange, true};
7215
7216 // If we don't know anything about the initial value (i.e. StartRange is
7217 // FullRange), then we don't know anything about the final range either.
7218 // Return FullRange.
7219 if (StartRange.isFullSet())
7220 return {ConstantRange::getFull(BitWidth), false};
7221
7222 // If Step is signed and negative, then we use its absolute value, but we also
7223 // note that we're moving in the opposite direction.
7224 bool Descending = Signed && Step.isNegative();
7225
7226 if (Signed)
7227 // This is correct even for INT_SMIN. Let's look at i8 to illustrate this:
7228 // abs(INT_SMIN) = abs(-128) = abs(0x80) = -0x80 = 0x80 = 128.
7229 // This equations hold true due to the well-defined wrap-around behavior of
7230 // APInt.
7231 Step = Step.abs();
7232
7233 // Check if Offset is more than full span of BitWidth. If it is, the
7234 // expression is guaranteed to overflow.
7235 if (APInt::getMaxValue(numBits: StartRange.getBitWidth()).udiv(RHS: Step).ult(RHS: MaxBECount))
7236 return {ConstantRange::getFull(BitWidth), false};
7237
7238 // Offset is by how much the expression can change. Checks above guarantee no
7239 // overflow here.
7240 APInt Offset = Step * MaxBECount;
7241
7242 // Minimum value of the final range will match the minimal value of StartRange
7243 // if the expression is increasing and will be decreased by Offset otherwise.
7244 // Maximum value of the final range will match the maximal value of StartRange
7245 // if the expression is decreasing and will be increased by Offset otherwise.
7246 APInt StartLower = StartRange.getLower();
7247 APInt StartUpper = StartRange.getUpper() - 1;
7248 bool Overflow;
7249 APInt MovedBoundary;
7250 if (Signed) {
7251 // This does not use sadd_ov, as we want to check overflow for a signed
7252 // start with an unsigned offset.
7253 if (Descending) {
7254 MovedBoundary = StartLower - std::move(Offset);
7255 Overflow = MovedBoundary.sgt(RHS: StartLower) || StartRange.isSignWrappedSet();
7256 } else {
7257 MovedBoundary = StartUpper + std::move(Offset);
7258 Overflow = MovedBoundary.slt(RHS: StartUpper) || StartRange.isSignWrappedSet();
7259 }
7260 } else {
7261 MovedBoundary = StartUpper.uadd_ov(RHS: std::move(Offset), Overflow);
7262 Overflow |= StartRange.isWrappedSet();
7263 }
7264
7265 // It's possible that the new minimum/maximum value will fall into the initial
7266 // range (due to wrap around). This means that the expression can take any
7267 // value in this bitwidth, and we have to return full range.
7268 if (StartRange.contains(Val: MovedBoundary))
7269 return {ConstantRange::getFull(BitWidth), false};
7270
7271 APInt NewLower =
7272 Descending ? std::move(MovedBoundary) : std::move(StartLower);
7273 APInt NewUpper =
7274 Descending ? std::move(StartUpper) : std::move(MovedBoundary);
7275 NewUpper += 1;
7276
7277 // No overflow detected, return [StartLower, StartUpper + Offset + 1) range.
7278 return {ConstantRange::getNonEmpty(Lower: std::move(NewLower), Upper: std::move(NewUpper)),
7279 !Overflow};
7280}
7281
7282std::pair<ConstantRange, SCEV::NoWrapFlags>
7283ScalarEvolution::getRangeForAffineAR(const SCEV *Start, const SCEV *Step,
7284 const APInt &MaxBECount) {
7285 assert(getTypeSizeInBits(Start->getType()) ==
7286 getTypeSizeInBits(Step->getType()) &&
7287 getTypeSizeInBits(Start->getType()) == MaxBECount.getBitWidth() &&
7288 "mismatched bit widths");
7289
7290 // First, consider step signed.
7291 ConstantRange StartSRange = getSignedRange(S: Start);
7292 ConstantRange StepSRange = getSignedRange(S: Step);
7293
7294 // If Step can be both positive and negative, we need to find ranges for the
7295 // maximum absolute step values in both directions and union them.
7296 auto [SR1, NSW1] = getRangeForAffineARHelper(
7297 Step: StepSRange.getSignedMin(), StartRange: StartSRange, MaxBECount, /*Signed=*/true);
7298 auto [SR2, NSW2] = getRangeForAffineARHelper(Step: StepSRange.getSignedMax(),
7299 StartRange: StartSRange, MaxBECount,
7300 /*Signed=*/true);
7301 ConstantRange SR = SR1.unionWith(CR: SR2);
7302
7303 // Next, consider step unsigned.
7304 auto [UR, NUW] = getRangeForAffineARHelper(
7305 Step: getUnsignedRangeMax(S: Step), StartRange: getUnsignedRange(S: Start), MaxBECount,
7306 /*Signed=*/false);
7307
7308 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7309 if (NUW)
7310 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7311 if (NSW1 && NSW2)
7312 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7313
7314 // Finally, intersect signed and unsigned ranges.
7315 return {SR.intersectWith(CR: UR, Type: ConstantRange::Smallest), Flags};
7316}
7317
7318ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
7319 const SCEVAddRecExpr *AddRec, const SCEV *MaxBECount, unsigned BitWidth,
7320 ScalarEvolution::RangeSignHint SignHint) {
7321 assert(AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n");
7322 assert(AddRec->hasNoSelfWrap() &&
7323 "This only works for non-self-wrapping AddRecs!");
7324 const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
7325 const SCEV *Step = AddRec->getStepRecurrence(SE&: *this);
7326 // Only deal with constant step to save compile time.
7327 if (!isa<SCEVConstant>(Val: Step))
7328 return ConstantRange::getFull(BitWidth);
7329 // Let's make sure that we can prove that we do not self-wrap during
7330 // MaxBECount iterations. We need this because MaxBECount is a maximum
7331 // iteration count estimate, and we might infer nw from some exit for which we
7332 // do not know max exit count (or any other side reasoning).
7333 // TODO: Turn into assert at some point.
7334 if (getTypeSizeInBits(Ty: MaxBECount->getType()) >
7335 getTypeSizeInBits(Ty: AddRec->getType()))
7336 return ConstantRange::getFull(BitWidth);
7337 MaxBECount = getNoopOrZeroExtend(V: MaxBECount, Ty: AddRec->getType());
7338 const SCEV *RangeWidth = getMinusOne(Ty: AddRec->getType());
7339 const SCEV *StepAbs = getUMinExpr(LHS: Step, RHS: getNegativeSCEV(V: Step));
7340 const SCEV *MaxItersWithoutWrap = getUDivExpr(LHS: RangeWidth, RHS: StepAbs);
7341 if (!isKnownPredicateViaConstantRanges(Pred: ICmpInst::ICMP_ULE, LHS: MaxBECount,
7342 RHS: MaxItersWithoutWrap))
7343 return ConstantRange::getFull(BitWidth);
7344
7345 ICmpInst::Predicate LEPred =
7346 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
7347 ICmpInst::Predicate GEPred =
7348 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
7349 const SCEV *End = AddRec->evaluateAtIteration(It: MaxBECount, SE&: *this);
7350
7351 // We know that there is no self-wrap. Let's take Start and End values and
7352 // look at all intermediate values V1, V2, ..., Vn that IndVar takes during
7353 // the iteration. They either lie inside the range [Min(Start, End),
7354 // Max(Start, End)] or outside it:
7355 //
7356 // Case 1: RangeMin ... Start V1 ... VN End ... RangeMax;
7357 // Case 2: RangeMin Vk ... V1 Start ... End Vn ... Vk + 1 RangeMax;
7358 //
7359 // No self wrap flag guarantees that the intermediate values cannot be BOTH
7360 // outside and inside the range [Min(Start, End), Max(Start, End)]. Using that
7361 // knowledge, let's try to prove that we are dealing with Case 1. It is so if
7362 // Start <= End and step is positive, or Start >= End and step is negative.
7363 const SCEV *Start = applyLoopGuards(Expr: AddRec->getStart(), L: AddRec->getLoop());
7364 ConstantRange StartRange = getRangeRef(S: Start, SignHint);
7365 ConstantRange EndRange = getRangeRef(S: End, SignHint);
7366 ConstantRange RangeBetween = StartRange.unionWith(CR: EndRange);
7367 // If they already cover full iteration space, we will know nothing useful
7368 // even if we prove what we want to prove.
7369 if (RangeBetween.isFullSet())
7370 return RangeBetween;
7371 // Only deal with ranges that do not wrap (i.e. RangeMin < RangeMax).
7372 bool IsWrappedSet = IsSigned ? RangeBetween.isSignWrappedSet()
7373 : RangeBetween.isWrappedSet();
7374 if (IsWrappedSet)
7375 return ConstantRange::getFull(BitWidth);
7376
7377 if (isKnownPositive(S: Step) &&
7378 isKnownPredicateViaConstantRanges(Pred: LEPred, LHS: Start, RHS: End))
7379 return RangeBetween;
7380 if (isKnownNegative(S: Step) &&
7381 isKnownPredicateViaConstantRanges(Pred: GEPred, LHS: Start, RHS: End))
7382 return RangeBetween;
7383 return ConstantRange::getFull(BitWidth);
7384}
7385
7386ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start,
7387 const SCEV *Step,
7388 const APInt &MaxBECount) {
7389 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q})
7390 // == RangeOf({A,+,P}) union RangeOf({B,+,Q})
7391
7392 unsigned BitWidth = MaxBECount.getBitWidth();
7393 assert(getTypeSizeInBits(Start->getType()) == BitWidth &&
7394 getTypeSizeInBits(Step->getType()) == BitWidth &&
7395 "mismatched bit widths");
7396
7397 struct SelectPattern {
7398 Value *Condition = nullptr;
7399 APInt TrueValue;
7400 APInt FalseValue;
7401
7402 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth,
7403 const SCEV *S) {
7404 std::optional<unsigned> CastOp;
7405 APInt Offset(BitWidth, 0);
7406
7407 assert(SE.getTypeSizeInBits(S->getType()) == BitWidth &&
7408 "Should be!");
7409
7410 // Peel off a constant offset. In the future we could consider being
7411 // smarter here and handle {Start+Step,+,Step} too.
7412 const APInt *Off;
7413 if (match(S, P: m_scev_Add(Op0: m_scev_APInt(C&: Off), Op1: m_SCEV(V&: S))))
7414 Offset = *Off;
7415
7416 // Peel off a cast operation
7417 if (auto *SCast = dyn_cast<SCEVIntegralCastExpr>(Val: S)) {
7418 CastOp = SCast->getSCEVType();
7419 S = SCast->getOperand();
7420 }
7421
7422 using namespace llvm::PatternMatch;
7423
7424 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
7425 const APInt *TrueVal, *FalseVal;
7426 if (!SU ||
7427 !match(V: SU->getValue(), P: m_Select(C: m_Value(V&: Condition), L: m_APInt(Res&: TrueVal),
7428 R: m_APInt(Res&: FalseVal)))) {
7429 Condition = nullptr;
7430 return;
7431 }
7432
7433 TrueValue = *TrueVal;
7434 FalseValue = *FalseVal;
7435
7436 // Re-apply the cast we peeled off earlier
7437 if (CastOp)
7438 switch (*CastOp) {
7439 default:
7440 llvm_unreachable("Unknown SCEV cast type!");
7441
7442 case scTruncate:
7443 TrueValue = TrueValue.trunc(width: BitWidth);
7444 FalseValue = FalseValue.trunc(width: BitWidth);
7445 break;
7446 case scZeroExtend:
7447 TrueValue = TrueValue.zext(width: BitWidth);
7448 FalseValue = FalseValue.zext(width: BitWidth);
7449 break;
7450 case scSignExtend:
7451 TrueValue = TrueValue.sext(width: BitWidth);
7452 FalseValue = FalseValue.sext(width: BitWidth);
7453 break;
7454 }
7455
7456 // Re-apply the constant offset we peeled off earlier
7457 TrueValue += Offset;
7458 FalseValue += Offset;
7459 }
7460
7461 bool isRecognized() { return Condition != nullptr; }
7462 };
7463
7464 SelectPattern StartPattern(*this, BitWidth, Start);
7465 if (!StartPattern.isRecognized())
7466 return ConstantRange::getFull(BitWidth);
7467
7468 SelectPattern StepPattern(*this, BitWidth, Step);
7469 if (!StepPattern.isRecognized())
7470 return ConstantRange::getFull(BitWidth);
7471
7472 if (StartPattern.Condition != StepPattern.Condition) {
7473 // We don't handle this case today; but we could, by considering four
7474 // possibilities below instead of two. I'm not sure if there are cases where
7475 // that will help over what getRange already does, though.
7476 return ConstantRange::getFull(BitWidth);
7477 }
7478
7479 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to
7480 // construct arbitrary general SCEV expressions here. This function is called
7481 // from deep in the call stack, and calling getSCEV (on a sext instruction,
7482 // say) can end up caching a suboptimal value.
7483
7484 // FIXME: without the explicit `this` receiver below, MSVC errors out with
7485 // C2352 and C2512 (otherwise it isn't needed).
7486
7487 const SCEV *TrueStart = this->getConstant(Val: StartPattern.TrueValue);
7488 const SCEV *TrueStep = this->getConstant(Val: StepPattern.TrueValue);
7489 const SCEV *FalseStart = this->getConstant(Val: StartPattern.FalseValue);
7490 const SCEV *FalseStep = this->getConstant(Val: StepPattern.FalseValue);
7491
7492 ConstantRange TrueRange =
7493 this->getRangeForAffineAR(Start: TrueStart, Step: TrueStep, MaxBECount).first;
7494 ConstantRange FalseRange =
7495 this->getRangeForAffineAR(Start: FalseStart, Step: FalseStep, MaxBECount).first;
7496
7497 return TrueRange.unionWith(CR: FalseRange);
7498}
7499
7500SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
7501 if (isa<ConstantExpr>(Val: V)) return SCEV::FlagAnyWrap;
7502 const BinaryOperator *BinOp = cast<BinaryOperator>(Val: V);
7503
7504 // Return early if there are no flags to propagate to the SCEV.
7505 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7506 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: BinOp);
7507 PDI && PDI->isDisjoint()) {
7508 Flags = ScalarEvolution::setFlags(Flags: SCEV::FlagNUW, OnFlags: SCEV::FlagNSW);
7509 } else {
7510 if (BinOp->hasNoUnsignedWrap())
7511 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7512 if (BinOp->hasNoSignedWrap())
7513 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7514 }
7515 if (Flags == SCEV::FlagAnyWrap)
7516 return SCEV::FlagAnyWrap;
7517
7518 return isSCEVExprNeverPoison(I: BinOp) ? Flags : SCEV::FlagAnyWrap;
7519}
7520
7521const Instruction *
7522ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) {
7523 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S))
7524 return &*AddRec->getLoop()->getHeader()->begin();
7525 if (auto *U = dyn_cast<SCEVUnknown>(Val: S))
7526 if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
7527 return I;
7528 return nullptr;
7529}
7530
7531const Instruction *ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops,
7532 bool &Precise) {
7533 Precise = true;
7534 // Do a bounded search of the def relation of the requested SCEVs.
7535 SmallPtrSet<const SCEV *, 16> Visited;
7536 SmallVector<SCEVUse> Worklist;
7537 auto pushOp = [&](const SCEV *S) {
7538 if (!Visited.insert(Ptr: S).second)
7539 return;
7540 // Threshold of 30 here is arbitrary.
7541 if (Visited.size() > 30) {
7542 Precise = false;
7543 return;
7544 }
7545 Worklist.push_back(Elt: S);
7546 };
7547
7548 for (SCEVUse S : Ops)
7549 pushOp(S);
7550
7551 const Instruction *Bound = nullptr;
7552 while (!Worklist.empty()) {
7553 SCEVUse S = Worklist.pop_back_val();
7554 if (auto *DefI = getNonTrivialDefiningScopeBound(S)) {
7555 if (!Bound || DT.dominates(Def: Bound, User: DefI))
7556 Bound = DefI;
7557 } else {
7558 for (SCEVUse Op : S->operands())
7559 pushOp(Op);
7560 }
7561 }
7562 return Bound ? Bound : &*F.getEntryBlock().begin();
7563}
7564
7565const Instruction *
7566ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops) {
7567 bool Discard;
7568 return getDefiningScopeBound(Ops, Precise&: Discard);
7569}
7570
7571bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
7572 const Instruction *B) {
7573 if (A->getParent() == B->getParent() &&
7574 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7575 End: B->getIterator()))
7576 return true;
7577
7578 auto *BLoop = LI.getLoopFor(BB: B->getParent());
7579 if (BLoop && BLoop->getHeader() == B->getParent() &&
7580 BLoop->getLoopPreheader() == A->getParent() &&
7581 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7582 End: A->getParent()->end()) &&
7583 isGuaranteedToTransferExecutionToSuccessor(Begin: B->getParent()->begin(),
7584 End: B->getIterator()))
7585 return true;
7586 return false;
7587}
7588
7589bool ScalarEvolution::isGuaranteedNotToBePoison(const SCEV *Op) {
7590 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ true);
7591 visitAll(Root: Op, Visitor&: PC);
7592 return PC.MaybePoison.empty();
7593}
7594
7595bool ScalarEvolution::isGuaranteedNotToCauseUB(const SCEV *Op) {
7596 return !SCEVExprContains(Root: Op, Pred: [this](const SCEV *S) {
7597 const SCEV *Op1;
7598 bool M = match(S, P: m_scev_UDiv(Op0: m_SCEV(), Op1: m_SCEV(V&: Op1)));
7599 // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
7600 // is a non-zero constant, we have to assume the UDiv may be UB.
7601 return M && (!isKnownNonZero(S: Op1) || !isGuaranteedNotToBePoison(Op: Op1));
7602 });
7603}
7604
7605bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
7606 // Only proceed if we can prove that I does not yield poison.
7607 if (!programUndefinedIfPoison(Inst: I))
7608 return false;
7609
7610 // At this point we know that if I is executed, then it does not wrap
7611 // according to at least one of NSW or NUW. If I is not executed, then we do
7612 // not know if the calculation that I represents would wrap. Multiple
7613 // instructions can map to the same SCEV. If we apply NSW or NUW from I to
7614 // the SCEV, we must guarantee no wrapping for that SCEV also when it is
7615 // derived from other instructions that map to the same SCEV. We cannot make
7616 // that guarantee for cases where I is not executed. So we need to find a
7617 // upper bound on the defining scope for the SCEV, and prove that I is
7618 // executed every time we enter that scope. When the bounding scope is a
7619 // loop (the common case), this is equivalent to proving I executes on every
7620 // iteration of that loop.
7621 SmallVector<SCEVUse> SCEVOps;
7622 for (const Use &Op : I->operands()) {
7623 // I could be an extractvalue from a call to an overflow intrinsic.
7624 // TODO: We can do better here in some cases.
7625 if (isSCEVable(Ty: Op->getType()))
7626 SCEVOps.push_back(Elt: getSCEV(V: Op));
7627 }
7628 auto *DefI = getDefiningScopeBound(Ops: SCEVOps);
7629 return isGuaranteedToTransferExecutionTo(A: DefI, B: I);
7630}
7631
7632bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) {
7633 // If we know that \c I can never be poison period, then that's enough.
7634 if (isSCEVExprNeverPoison(I))
7635 return true;
7636
7637 // If the loop only has one exit, then we know that, if the loop is entered,
7638 // any instruction dominating that exit will be executed. If any such
7639 // instruction would result in UB, the addrec cannot be poison.
7640 //
7641 // This is basically the same reasoning as in isSCEVExprNeverPoison(), but
7642 // also handles uses outside the loop header (they just need to dominate the
7643 // single exit).
7644
7645 auto *ExitingBB = L->getExitingBlock();
7646 if (!ExitingBB || !loopHasNoAbnormalExits(L))
7647 return false;
7648
7649 SmallPtrSet<const Value *, 16> KnownPoison;
7650 SmallVector<const Instruction *, 8> Worklist;
7651
7652 // We start by assuming \c I, the post-inc add recurrence, is poison. Only
7653 // things that are known to be poison under that assumption go on the
7654 // Worklist.
7655 KnownPoison.insert(Ptr: I);
7656 Worklist.push_back(Elt: I);
7657
7658 while (!Worklist.empty()) {
7659 const Instruction *Poison = Worklist.pop_back_val();
7660
7661 for (const Use &U : Poison->uses()) {
7662 const Instruction *PoisonUser = cast<Instruction>(Val: U.getUser());
7663 if (mustTriggerUB(I: PoisonUser, KnownPoison) &&
7664 DT.dominates(A: PoisonUser->getParent(), B: ExitingBB))
7665 return true;
7666
7667 if (propagatesPoison(PoisonOp: U) && L->contains(Inst: PoisonUser))
7668 if (KnownPoison.insert(Ptr: PoisonUser).second)
7669 Worklist.push_back(Elt: PoisonUser);
7670 }
7671 }
7672
7673 return false;
7674}
7675
7676ScalarEvolution::LoopProperties
7677ScalarEvolution::getLoopProperties(const Loop *L) {
7678 using LoopProperties = ScalarEvolution::LoopProperties;
7679
7680 auto Itr = LoopPropertiesCache.find(Val: L);
7681 if (Itr == LoopPropertiesCache.end()) {
7682 auto HasSideEffects = [](Instruction *I) {
7683 if (auto *SI = dyn_cast<StoreInst>(Val: I))
7684 return !SI->isSimple();
7685
7686 if (I->mayThrow())
7687 return true;
7688
7689 // Non-volatile memset / memcpy do not count as side-effect for forward
7690 // progress.
7691 if (isa<MemIntrinsic>(Val: I) && !I->isVolatile())
7692 return false;
7693
7694 return I->mayWriteToMemory();
7695 };
7696
7697 LoopProperties LP = {/* HasNoAbnormalExits */ true,
7698 /*HasNoSideEffects*/ true};
7699
7700 for (auto *BB : L->getBlocks())
7701 for (auto &I : *BB) {
7702 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7703 LP.HasNoAbnormalExits = false;
7704 if (HasSideEffects(&I))
7705 LP.HasNoSideEffects = false;
7706 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
7707 break; // We're already as pessimistic as we can get.
7708 }
7709
7710 auto InsertPair = LoopPropertiesCache.insert(KV: {L, LP});
7711 assert(InsertPair.second && "We just checked!");
7712 Itr = InsertPair.first;
7713 }
7714
7715 return Itr->second;
7716}
7717
7718bool ScalarEvolution::loopIsFiniteByAssumption(const Loop *L) {
7719 // A mustprogress loop without side effects must be finite.
7720 // TODO: The check used here is very conservative. It's only *specific*
7721 // side effects which are well defined in infinite loops.
7722 return isFinite(L) || (isMustProgress(L) && loopHasNoSideEffects(L));
7723}
7724
7725const SCEV *ScalarEvolution::createSCEVIter(Value *V) {
7726 // Worklist item with a Value and a bool indicating whether all operands have
7727 // been visited already.
7728 using PointerTy = PointerIntPair<Value *, 1, bool>;
7729 SmallVector<PointerTy> Stack;
7730
7731 Stack.emplace_back(Args&: V, Args: false);
7732 while (!Stack.empty()) {
7733 auto E = Stack.back();
7734 Value *CurV = E.getPointer();
7735
7736 if (getExistingSCEV(V: CurV)) {
7737 Stack.pop_back();
7738 continue;
7739 }
7740
7741 SmallVector<Value *> Ops;
7742 const SCEV *CreatedSCEV = nullptr;
7743 // If all operands have been visited already, create the SCEV.
7744 if (E.getInt()) {
7745 CreatedSCEV = createSCEV(V: CurV);
7746 } else {
7747 // Otherwise get the operands we need to create SCEV's for before creating
7748 // the SCEV for CurV. If the SCEV for CurV can be constructed trivially,
7749 // just use it.
7750 CreatedSCEV = getOperandsToCreate(V: CurV, Ops);
7751 }
7752
7753 if (CreatedSCEV) {
7754 insertValueToMap(V: CurV, S: CreatedSCEV);
7755 Stack.pop_back();
7756 } else {
7757 Stack.back().setInt(true);
7758 // Queue its operands which need to be constructed.
7759 for (Value *Op : Ops)
7760 Stack.emplace_back(Args&: Op, Args: false);
7761 }
7762 }
7763
7764 return getExistingSCEV(V);
7765}
7766
7767const SCEV *
7768ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
7769 if (!isSCEVable(Ty: V->getType()))
7770 return getUnknown(V);
7771
7772 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7773 // Don't attempt to analyze instructions in blocks that aren't
7774 // reachable. Such instructions don't matter, and they aren't required
7775 // to obey basic rules for definitions dominating uses which this
7776 // analysis depends on.
7777 if (!DT.isReachableFromEntry(A: I->getParent()))
7778 return getUnknown(V: PoisonValue::get(T: V->getType()));
7779 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7780 return getConstant(V: CI);
7781 else if (isa<GlobalAlias>(Val: V))
7782 return getUnknown(V);
7783 else if (!isa<ConstantExpr>(Val: V))
7784 return getUnknown(V);
7785
7786 Operator *U = cast<Operator>(Val: V);
7787 if (auto BO =
7788 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CxtI: dyn_cast<Instruction>(Val: V))) {
7789 bool IsConstArg = isa<ConstantInt>(Val: BO->RHS);
7790 switch (BO->Opcode) {
7791 case Instruction::Add:
7792 case Instruction::Mul: {
7793 // For additions and multiplications, traverse add/mul chains for which we
7794 // can potentially create a single SCEV, to reduce the number of
7795 // get{Add,Mul}Expr calls.
7796 do {
7797 if (BO->Op) {
7798 if (BO->Op != V && getExistingSCEV(V: BO->Op)) {
7799 Ops.push_back(Elt: BO->Op);
7800 break;
7801 }
7802 }
7803 Ops.push_back(Elt: BO->RHS);
7804 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7805 CxtI: dyn_cast<Instruction>(Val: V));
7806 if (!NewBO ||
7807 (BO->Opcode == Instruction::Add &&
7808 (NewBO->Opcode != Instruction::Add &&
7809 NewBO->Opcode != Instruction::Sub)) ||
7810 (BO->Opcode == Instruction::Mul &&
7811 NewBO->Opcode != Instruction::Mul)) {
7812 Ops.push_back(Elt: BO->LHS);
7813 break;
7814 }
7815 // CreateSCEV calls getNoWrapFlagsFromUB, which under certain conditions
7816 // requires a SCEV for the LHS.
7817 if (BO->Op && (BO->IsNSW || BO->IsNUW)) {
7818 auto *I = dyn_cast<Instruction>(Val: BO->Op);
7819 if (I && programUndefinedIfPoison(Inst: I)) {
7820 Ops.push_back(Elt: BO->LHS);
7821 break;
7822 }
7823 }
7824 BO = NewBO;
7825 } while (true);
7826 return nullptr;
7827 }
7828 case Instruction::Sub:
7829 case Instruction::UDiv:
7830 case Instruction::URem:
7831 break;
7832 case Instruction::AShr:
7833 case Instruction::Shl:
7834 case Instruction::Xor:
7835 if (!IsConstArg)
7836 return nullptr;
7837 break;
7838 case Instruction::And:
7839 case Instruction::Or:
7840 if (!IsConstArg && !BO->LHS->getType()->isIntegerTy(BitWidth: 1))
7841 return nullptr;
7842 break;
7843 case Instruction::LShr:
7844 return getUnknown(V);
7845 default:
7846 llvm_unreachable("Unhandled binop");
7847 break;
7848 }
7849
7850 Ops.push_back(Elt: BO->LHS);
7851 Ops.push_back(Elt: BO->RHS);
7852 return nullptr;
7853 }
7854
7855 switch (U->getOpcode()) {
7856 case Instruction::Trunc:
7857 case Instruction::ZExt:
7858 case Instruction::SExt:
7859 case Instruction::PtrToAddr:
7860 case Instruction::PtrToInt:
7861 Ops.push_back(Elt: U->getOperand(i: 0));
7862 return nullptr;
7863
7864 case Instruction::BitCast:
7865 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType())) {
7866 Ops.push_back(Elt: U->getOperand(i: 0));
7867 return nullptr;
7868 }
7869 return getUnknown(V);
7870
7871 case Instruction::SDiv:
7872 case Instruction::SRem:
7873 Ops.push_back(Elt: U->getOperand(i: 0));
7874 Ops.push_back(Elt: U->getOperand(i: 1));
7875 return nullptr;
7876
7877 case Instruction::GetElementPtr:
7878 assert(cast<GEPOperator>(U)->getSourceElementType()->isSized() &&
7879 "GEP source element type must be sized");
7880 llvm::append_range(C&: Ops, R: U->operands());
7881 return nullptr;
7882
7883 case Instruction::IntToPtr:
7884 return getUnknown(V);
7885
7886 case Instruction::PHI:
7887 // getNodeForPHI has four ways to turn a PHI into a SCEV; retrieve the
7888 // relevant nodes for each of them.
7889 //
7890 // The first is just to call simplifyInstruction, and get something back
7891 // that isn't a PHI.
7892 if (Value *V = simplifyInstruction(
7893 I: cast<PHINode>(Val: U),
7894 Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
7895 /*UseInstrInfo=*/true, /*CanUseUndef=*/false})) {
7896 assert(V);
7897 Ops.push_back(Elt: V);
7898 return nullptr;
7899 }
7900 // The second is createNodeForPHIWithIdenticalOperands: this looks for
7901 // operands which all perform the same operation, but haven't been
7902 // CSE'ed for whatever reason.
7903 if (BinaryOperator *BO = getCommonInstForPHI(PN: cast<PHINode>(Val: U))) {
7904 assert(BO);
7905 Ops.push_back(Elt: BO);
7906 return nullptr;
7907 }
7908 // The third is createNodeFromSelectLikePHI; this takes a PHI which
7909 // is equivalent to a select, and analyzes it like a select.
7910 {
7911 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
7912 if (getOperandsForSelectLikePHI(DT, PN: cast<PHINode>(Val: U), Cond, LHS, RHS)) {
7913 assert(Cond);
7914 assert(LHS);
7915 assert(RHS);
7916 if (auto *CondICmp = dyn_cast<ICmpInst>(Val: Cond)) {
7917 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 0));
7918 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 1));
7919 }
7920 Ops.push_back(Elt: Cond);
7921 Ops.push_back(Elt: LHS);
7922 Ops.push_back(Elt: RHS);
7923 return nullptr;
7924 }
7925 }
7926 // The fourth way is createAddRecFromPHI. It's complicated to handle here,
7927 // so just construct it recursively.
7928 //
7929 // In addition to getNodeForPHI, also construct nodes which might be needed
7930 // by getRangeRef.
7931 if (RangeRefPHIAllowedOperands(DT, PHI: cast<PHINode>(Val: U))) {
7932 for (Value *V : cast<PHINode>(Val: U)->operands())
7933 Ops.push_back(Elt: V);
7934 return nullptr;
7935 }
7936 return nullptr;
7937
7938 case Instruction::Select: {
7939 // Check if U is a select that can be simplified to a SCEVUnknown.
7940 auto CanSimplifyToUnknown = [this, U]() {
7941 if (U->getType()->isIntegerTy(BitWidth: 1) || isa<ConstantInt>(Val: U->getOperand(i: 0)))
7942 return false;
7943
7944 auto *ICI = dyn_cast<ICmpInst>(Val: U->getOperand(i: 0));
7945 if (!ICI)
7946 return false;
7947 Value *LHS = ICI->getOperand(i_nocapture: 0);
7948 Value *RHS = ICI->getOperand(i_nocapture: 1);
7949 if (ICI->getPredicate() == CmpInst::ICMP_EQ ||
7950 ICI->getPredicate() == CmpInst::ICMP_NE) {
7951 if (!(isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()))
7952 return true;
7953 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
7954 getTypeSizeInBits(Ty: U->getType()))
7955 return true;
7956 return false;
7957 };
7958 if (CanSimplifyToUnknown())
7959 return getUnknown(V: U);
7960
7961 llvm::append_range(C&: Ops, R: U->operands());
7962 return nullptr;
7963 break;
7964 }
7965 case Instruction::Call:
7966 case Instruction::Invoke:
7967 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand()) {
7968 Ops.push_back(Elt: RV);
7969 return nullptr;
7970 }
7971
7972 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
7973 switch (II->getIntrinsicID()) {
7974 case Intrinsic::abs:
7975 Ops.push_back(Elt: II->getArgOperand(i: 0));
7976 return nullptr;
7977 case Intrinsic::umax:
7978 case Intrinsic::umin:
7979 case Intrinsic::smax:
7980 case Intrinsic::smin:
7981 case Intrinsic::usub_sat:
7982 case Intrinsic::uadd_sat:
7983 Ops.push_back(Elt: II->getArgOperand(i: 0));
7984 Ops.push_back(Elt: II->getArgOperand(i: 1));
7985 return nullptr;
7986 case Intrinsic::start_loop_iterations:
7987 case Intrinsic::annotation:
7988 case Intrinsic::ptr_annotation:
7989 Ops.push_back(Elt: II->getArgOperand(i: 0));
7990 return nullptr;
7991 default:
7992 break;
7993 }
7994 }
7995 break;
7996 }
7997
7998 return nullptr;
7999}
8000
8001const SCEV *ScalarEvolution::createSCEV(Value *V) {
8002 if (!isSCEVable(Ty: V->getType()))
8003 return getUnknown(V);
8004
8005 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
8006 // Don't attempt to analyze instructions in blocks that aren't
8007 // reachable. Such instructions don't matter, and they aren't required
8008 // to obey basic rules for definitions dominating uses which this
8009 // analysis depends on.
8010 if (!DT.isReachableFromEntry(A: I->getParent()))
8011 return getUnknown(V: PoisonValue::get(T: V->getType()));
8012 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
8013 return getConstant(V: CI);
8014 else if (isa<GlobalAlias>(Val: V))
8015 return getUnknown(V);
8016 else if (!isa<ConstantExpr>(Val: V))
8017 return getUnknown(V);
8018
8019 const SCEV *LHS;
8020 const SCEV *RHS;
8021
8022 Operator *U = cast<Operator>(Val: V);
8023 if (auto BO =
8024 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CxtI: dyn_cast<Instruction>(Val: V))) {
8025 switch (BO->Opcode) {
8026 case Instruction::Add: {
8027 // The simple thing to do would be to just call getSCEV on both operands
8028 // and call getAddExpr with the result. However if we're looking at a
8029 // bunch of things all added together, this can be quite inefficient,
8030 // because it leads to N-1 getAddExpr calls for N ultimate operands.
8031 // Instead, gather up all the operands and make a single getAddExpr call.
8032 // LLVM IR canonical form means we need only traverse the left operands.
8033 SmallVector<SCEVUse, 4> AddOps;
8034 do {
8035 if (BO->Op) {
8036 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
8037 AddOps.push_back(Elt: OpSCEV);
8038 break;
8039 }
8040
8041 // If a NUW or NSW flag can be applied to the SCEV for this
8042 // addition, then compute the SCEV for this addition by itself
8043 // with a separate call to getAddExpr. We need to do that
8044 // instead of pushing the operands of the addition onto AddOps,
8045 // since the flags are only known to apply to this particular
8046 // addition - they may not apply to other additions that can be
8047 // formed with operands from AddOps.
8048 const SCEV *RHS = getSCEV(V: BO->RHS);
8049 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
8050 if (Flags != SCEV::FlagAnyWrap) {
8051 const SCEV *LHS = getSCEV(V: BO->LHS);
8052 if (BO->Opcode == Instruction::Sub)
8053 AddOps.push_back(Elt: getMinusSCEV(LHS, RHS, Flags));
8054 else
8055 AddOps.push_back(Elt: getAddExpr(LHS, RHS, Flags));
8056 break;
8057 }
8058 }
8059
8060 if (BO->Opcode == Instruction::Sub)
8061 AddOps.push_back(Elt: getNegativeSCEV(V: getSCEV(V: BO->RHS)));
8062 else
8063 AddOps.push_back(Elt: getSCEV(V: BO->RHS));
8064
8065 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
8066 CxtI: dyn_cast<Instruction>(Val: V));
8067 if (!NewBO || (NewBO->Opcode != Instruction::Add &&
8068 NewBO->Opcode != Instruction::Sub)) {
8069 AddOps.push_back(Elt: getSCEV(V: BO->LHS));
8070 break;
8071 }
8072 BO = NewBO;
8073 } while (true);
8074
8075 return getAddExpr(Ops&: AddOps);
8076 }
8077
8078 case Instruction::Mul: {
8079 SmallVector<SCEVUse, 4> MulOps;
8080 do {
8081 if (BO->Op) {
8082 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
8083 MulOps.push_back(Elt: OpSCEV);
8084 break;
8085 }
8086
8087 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
8088 if (Flags != SCEV::FlagAnyWrap) {
8089 LHS = getSCEV(V: BO->LHS);
8090 RHS = getSCEV(V: BO->RHS);
8091 MulOps.push_back(Elt: getMulExpr(LHS, RHS, Flags));
8092 break;
8093 }
8094 }
8095
8096 MulOps.push_back(Elt: getSCEV(V: BO->RHS));
8097 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
8098 CxtI: dyn_cast<Instruction>(Val: V));
8099 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
8100 MulOps.push_back(Elt: getSCEV(V: BO->LHS));
8101 break;
8102 }
8103 BO = NewBO;
8104 } while (true);
8105
8106 return getMulExpr(Ops&: MulOps);
8107 }
8108 case Instruction::UDiv:
8109 LHS = getSCEV(V: BO->LHS);
8110 RHS = getSCEV(V: BO->RHS);
8111 return getUDivExpr(LHS, RHS);
8112 case Instruction::URem:
8113 LHS = getSCEV(V: BO->LHS);
8114 RHS = getSCEV(V: BO->RHS);
8115 return getURemExpr(LHS, RHS);
8116 case Instruction::Sub: {
8117 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
8118 if (BO->Op)
8119 Flags = getNoWrapFlagsFromUB(V: BO->Op);
8120 LHS = getSCEV(V: BO->LHS);
8121 RHS = getSCEV(V: BO->RHS);
8122 return getMinusSCEV(LHS, RHS, Flags);
8123 }
8124 case Instruction::And:
8125 // For an expression like x&255 that merely masks off the high bits,
8126 // use zext(trunc(x)) as the SCEV expression.
8127 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8128 if (CI->isZero())
8129 return getSCEV(V: BO->RHS);
8130 if (CI->isMinusOne())
8131 return getSCEV(V: BO->LHS);
8132 const APInt &A = CI->getValue();
8133
8134 // Instcombine's ShrinkDemandedConstant may strip bits out of
8135 // constants, obscuring what would otherwise be a low-bits mask.
8136 // Use computeKnownBits to compute what ShrinkDemandedConstant
8137 // knew about to reconstruct a low-bits mask value.
8138 unsigned LZ = A.countl_zero();
8139 unsigned TZ = A.countr_zero();
8140 unsigned BitWidth = A.getBitWidth();
8141 KnownBits Known(BitWidth);
8142 computeKnownBits(V: BO->LHS, Known, DL: getDataLayout(), AC: &AC, CxtI: nullptr, DT: &DT);
8143
8144 APInt EffectiveMask =
8145 APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - LZ - TZ).shl(shiftAmt: TZ);
8146 if ((LZ != 0 || TZ != 0) && !((~A & ~Known.Zero) & EffectiveMask)) {
8147 const SCEV *MulCount = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ));
8148 const SCEV *LHS = getSCEV(V: BO->LHS);
8149 const SCEV *ShiftedLHS = nullptr;
8150 if (auto *LHSMul = dyn_cast<SCEVMulExpr>(Val: LHS)) {
8151 if (auto *OpC = dyn_cast<SCEVConstant>(Val: LHSMul->getOperand(i: 0))) {
8152 // For an expression like (x * 8) & 8, simplify the multiply.
8153 unsigned MulZeros = OpC->getAPInt().countr_zero();
8154 unsigned GCD = std::min(a: MulZeros, b: TZ);
8155 APInt DivAmt = APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ - GCD);
8156 SmallVector<SCEVUse, 4> MulOps;
8157 MulOps.push_back(Elt: getConstant(Val: OpC->getAPInt().ashr(ShiftAmt: GCD)));
8158 append_range(C&: MulOps, R: LHSMul->operands().drop_front());
8159 auto *NewMul = getMulExpr(Ops&: MulOps, OrigFlags: LHSMul->getNoWrapFlags());
8160 ShiftedLHS = getUDivExpr(LHS: NewMul, RHS: getConstant(Val: DivAmt));
8161 }
8162 }
8163 if (!ShiftedLHS)
8164 ShiftedLHS = getUDivExpr(LHS, RHS: MulCount);
8165 return getMulExpr(
8166 LHS: getZeroExtendExpr(
8167 Op: getTruncateExpr(Op: ShiftedLHS,
8168 Ty: IntegerType::get(C&: getContext(), NumBits: BitWidth - LZ - TZ)),
8169 Ty: BO->LHS->getType()),
8170 RHS: MulCount);
8171 }
8172 }
8173 // Binary `and` is a bit-wise `umin`.
8174 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
8175 LHS = getSCEV(V: BO->LHS);
8176 RHS = getSCEV(V: BO->RHS);
8177 return getUMinExpr(LHS, RHS);
8178 }
8179 break;
8180
8181 case Instruction::Or:
8182 // Binary `or` is a bit-wise `umax`.
8183 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
8184 LHS = getSCEV(V: BO->LHS);
8185 RHS = getSCEV(V: BO->RHS);
8186 return getUMaxExpr(LHS, RHS);
8187 }
8188 break;
8189
8190 case Instruction::Xor:
8191 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8192 // If the RHS of xor is -1, then this is a not operation.
8193 if (CI->isMinusOne())
8194 return getNotSCEV(V: getSCEV(V: BO->LHS));
8195
8196 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
8197 // This is a variant of the check for xor with -1, and it handles
8198 // the case where instcombine has trimmed non-demanded bits out
8199 // of an xor with -1.
8200 if (auto *LBO = dyn_cast<BinaryOperator>(Val: BO->LHS))
8201 if (ConstantInt *LCI = dyn_cast<ConstantInt>(Val: LBO->getOperand(i_nocapture: 1)))
8202 if (LBO->getOpcode() == Instruction::And &&
8203 LCI->getValue() == CI->getValue())
8204 if (const SCEVZeroExtendExpr *Z =
8205 dyn_cast<SCEVZeroExtendExpr>(Val: getSCEV(V: BO->LHS))) {
8206 Type *UTy = BO->LHS->getType();
8207 const SCEV *Z0 = Z->getOperand();
8208 Type *Z0Ty = Z0->getType();
8209 unsigned Z0TySize = getTypeSizeInBits(Ty: Z0Ty);
8210
8211 // If C is a low-bits mask, the zero extend is serving to
8212 // mask off the high bits. Complement the operand and
8213 // re-apply the zext.
8214 if (CI->getValue().isMask(numBits: Z0TySize))
8215 return getZeroExtendExpr(Op: getNotSCEV(V: Z0), Ty: UTy);
8216
8217 // If C is a single bit, it may be in the sign-bit position
8218 // before the zero-extend. In this case, represent the xor
8219 // using an add, which is equivalent, and re-apply the zext.
8220 APInt Trunc = CI->getValue().trunc(width: Z0TySize);
8221 if (Trunc.zext(width: getTypeSizeInBits(Ty: UTy)) == CI->getValue() &&
8222 Trunc.isSignMask())
8223 return getZeroExtendExpr(Op: getAddExpr(LHS: Z0, RHS: getConstant(Val: Trunc)),
8224 Ty: UTy);
8225 }
8226 }
8227 break;
8228
8229 case Instruction::Shl:
8230 // Turn shift left of a constant amount into a multiply.
8231 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: BO->RHS)) {
8232 uint32_t BitWidth = cast<IntegerType>(Val: SA->getType())->getBitWidth();
8233
8234 // If the shift count is not less than the bitwidth, the result of
8235 // the shift is undefined. Don't try to analyze it, because the
8236 // resolution chosen here may differ from the resolution chosen in
8237 // other parts of the compiler.
8238 if (SA->getValue().uge(RHS: BitWidth))
8239 break;
8240
8241 // We can safely preserve the nuw flag in all cases. It's also safe to
8242 // turn a nuw nsw shl into a nuw nsw mul. However, nsw in isolation
8243 // requires special handling. It can be preserved as long as we're not
8244 // left shifting by bitwidth - 1.
8245 auto Flags = SCEV::FlagAnyWrap;
8246 if (BO->Op) {
8247 auto MulFlags = getNoWrapFlagsFromUB(V: BO->Op);
8248 if (any(Val: MulFlags & SCEV::FlagNSW) &&
8249 (any(Val: MulFlags & SCEV::FlagNUW) ||
8250 SA->getValue().ult(RHS: BitWidth - 1)))
8251 Flags = Flags | SCEV::FlagNSW;
8252 if (any(Val: MulFlags & SCEV::FlagNUW))
8253 Flags = Flags | SCEV::FlagNUW;
8254 }
8255
8256 ConstantInt *X = ConstantInt::get(
8257 Context&: getContext(), V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
8258 return getMulExpr(LHS: getSCEV(V: BO->LHS), RHS: getConstant(V: X), Flags);
8259 }
8260 break;
8261
8262 case Instruction::AShr:
8263 // AShr X, C, where C is a constant.
8264 ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS);
8265 if (!CI)
8266 break;
8267
8268 Type *OuterTy = BO->LHS->getType();
8269 uint64_t BitWidth = getTypeSizeInBits(Ty: OuterTy);
8270 // If the shift count is not less than the bitwidth, the result of
8271 // the shift is undefined. Don't try to analyze it, because the
8272 // resolution chosen here may differ from the resolution chosen in
8273 // other parts of the compiler.
8274 if (CI->getValue().uge(RHS: BitWidth))
8275 break;
8276
8277 if (CI->isZero())
8278 return getSCEV(V: BO->LHS); // shift by zero --> noop
8279
8280 uint64_t AShrAmt = CI->getZExtValue();
8281 Type *TruncTy = IntegerType::get(C&: getContext(), NumBits: BitWidth - AShrAmt);
8282
8283 Operator *L = dyn_cast<Operator>(Val: BO->LHS);
8284 const SCEV *AddTruncateExpr = nullptr;
8285 ConstantInt *ShlAmtCI = nullptr;
8286 const SCEV *AddConstant = nullptr;
8287
8288 if (L && L->getOpcode() == Instruction::Add) {
8289 // X = Shl A, n
8290 // Y = Add X, c
8291 // Z = AShr Y, m
8292 // n, c and m are constants.
8293
8294 Operator *LShift = dyn_cast<Operator>(Val: L->getOperand(i: 0));
8295 ConstantInt *AddOperandCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8296 if (LShift && LShift->getOpcode() == Instruction::Shl) {
8297 if (AddOperandCI) {
8298 const SCEV *ShlOp0SCEV = getSCEV(V: LShift->getOperand(i: 0));
8299 ShlAmtCI = dyn_cast<ConstantInt>(Val: LShift->getOperand(i: 1));
8300 // since we truncate to TruncTy, the AddConstant should be of the
8301 // same type, so create a new Constant with type same as TruncTy.
8302 // Also, the Add constant should be shifted right by AShr amount.
8303 APInt AddOperand = AddOperandCI->getValue().ashr(ShiftAmt: AShrAmt);
8304 AddConstant = getConstant(Val: AddOperand.trunc(width: BitWidth - AShrAmt));
8305 // we model the expression as sext(add(trunc(A), c << n)), since the
8306 // sext(trunc) part is already handled below, we create a
8307 // AddExpr(TruncExp) which will be used later.
8308 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8309 }
8310 }
8311 } else if (L && L->getOpcode() == Instruction::Shl) {
8312 // X = Shl A, n
8313 // Y = AShr X, m
8314 // Both n and m are constant.
8315
8316 const SCEV *ShlOp0SCEV = getSCEV(V: L->getOperand(i: 0));
8317 ShlAmtCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8318 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8319 }
8320
8321 if (AddTruncateExpr && ShlAmtCI) {
8322 // We can merge the two given cases into a single SCEV statement,
8323 // incase n = m, the mul expression will be 2^0, so it gets resolved to
8324 // a simpler case. The following code handles the two cases:
8325 //
8326 // 1) For a two-shift sext-inreg, i.e. n = m,
8327 // use sext(trunc(x)) as the SCEV expression.
8328 //
8329 // 2) When n > m, use sext(mul(trunc(x), 2^(n-m)))) as the SCEV
8330 // expression. We already checked that ShlAmt < BitWidth, so
8331 // the multiplier, 1 << (ShlAmt - AShrAmt), fits into TruncTy as
8332 // ShlAmt - AShrAmt < Amt.
8333 const APInt &ShlAmt = ShlAmtCI->getValue();
8334 if (ShlAmt.ult(RHS: BitWidth) && ShlAmt.uge(RHS: AShrAmt)) {
8335 APInt Mul = APInt::getOneBitSet(numBits: BitWidth - AShrAmt,
8336 BitNo: ShlAmtCI->getZExtValue() - AShrAmt);
8337 const SCEV *CompositeExpr =
8338 getMulExpr(LHS: AddTruncateExpr, RHS: getConstant(Val: Mul));
8339 if (L->getOpcode() != Instruction::Shl)
8340 CompositeExpr = getAddExpr(LHS: CompositeExpr, RHS: AddConstant);
8341
8342 return getSignExtendExpr(Op: CompositeExpr, Ty: OuterTy);
8343 }
8344 }
8345 break;
8346 }
8347 }
8348
8349 switch (U->getOpcode()) {
8350 case Instruction::Trunc:
8351 return getTruncateExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8352
8353 case Instruction::ZExt:
8354 return getZeroExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8355
8356 case Instruction::SExt:
8357 if (auto BO = MatchBinaryOp(V: U->getOperand(i: 0), DL: getDataLayout(), AC, DT,
8358 CxtI: dyn_cast<Instruction>(Val: V))) {
8359 // The NSW flag of a subtract does not always survive the conversion to
8360 // A + (-1)*B. By pushing sign extension onto its operands we are much
8361 // more likely to preserve NSW and allow later AddRec optimisations.
8362 //
8363 // NOTE: This is effectively duplicating this logic from getSignExtend:
8364 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
8365 // but by that point the NSW information has potentially been lost.
8366 if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
8367 Type *Ty = U->getType();
8368 auto *V1 = getSignExtendExpr(Op: getSCEV(V: BO->LHS), Ty);
8369 auto *V2 = getSignExtendExpr(Op: getSCEV(V: BO->RHS), Ty);
8370 return getMinusSCEV(LHS: V1, RHS: V2, Flags: SCEV::FlagNSW);
8371 }
8372 }
8373 return getSignExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8374
8375 case Instruction::BitCast:
8376 // BitCasts are no-op casts so we just eliminate the cast.
8377 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType()))
8378 return getSCEV(V: U->getOperand(i: 0));
8379 break;
8380
8381 case Instruction::PtrToAddr: {
8382 const SCEV *IntOp = getPtrToAddrExpr(Op: getSCEV(V: U->getOperand(i: 0)));
8383 if (isa<SCEVCouldNotCompute>(Val: IntOp))
8384 return getUnknown(V);
8385 return IntOp;
8386 }
8387
8388 case Instruction::PtrToInt: {
8389 // Pointer to integer cast is straight-forward, so do model it.
8390 const SCEV *Op = getSCEV(V: U->getOperand(i: 0));
8391 Type *DstIntTy = U->getType();
8392 // But only if effective SCEV (integer) type is wide enough to represent
8393 // all possible pointer values.
8394 const SCEV *IntOp = getPtrToIntExpr(Op, Ty: DstIntTy);
8395 if (isa<SCEVCouldNotCompute>(Val: IntOp))
8396 return getUnknown(V);
8397 return IntOp;
8398 }
8399 case Instruction::IntToPtr:
8400 // Just don't deal with inttoptr casts.
8401 return getUnknown(V);
8402
8403 case Instruction::SDiv:
8404 // If both operands are non-negative, this is just an udiv.
8405 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8406 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8407 return getUDivExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8408 break;
8409
8410 case Instruction::SRem:
8411 // If both operands are non-negative, this is just an urem.
8412 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8413 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8414 return getURemExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8415 break;
8416
8417 case Instruction::GetElementPtr:
8418 return createNodeForGEP(GEP: cast<GEPOperator>(Val: U));
8419
8420 case Instruction::PHI:
8421 return createNodeForPHI(PN: cast<PHINode>(Val: U));
8422
8423 case Instruction::Select:
8424 return createNodeForSelectOrPHI(V: U, Cond: U->getOperand(i: 0), TrueVal: U->getOperand(i: 1),
8425 FalseVal: U->getOperand(i: 2));
8426
8427 case Instruction::Call:
8428 case Instruction::Invoke:
8429 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand())
8430 return getSCEV(V: RV);
8431
8432 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
8433 switch (II->getIntrinsicID()) {
8434 case Intrinsic::abs:
8435 return getAbsExpr(
8436 Op: getSCEV(V: II->getArgOperand(i: 0)),
8437 /*IsNSW=*/cast<ConstantInt>(Val: II->getArgOperand(i: 1))->isOne());
8438 case Intrinsic::umax:
8439 LHS = getSCEV(V: II->getArgOperand(i: 0));
8440 RHS = getSCEV(V: II->getArgOperand(i: 1));
8441 return getUMaxExpr(LHS, RHS);
8442 case Intrinsic::umin:
8443 LHS = getSCEV(V: II->getArgOperand(i: 0));
8444 RHS = getSCEV(V: II->getArgOperand(i: 1));
8445 return getUMinExpr(LHS, RHS);
8446 case Intrinsic::smax:
8447 LHS = getSCEV(V: II->getArgOperand(i: 0));
8448 RHS = getSCEV(V: II->getArgOperand(i: 1));
8449 return getSMaxExpr(LHS, RHS);
8450 case Intrinsic::smin:
8451 LHS = getSCEV(V: II->getArgOperand(i: 0));
8452 RHS = getSCEV(V: II->getArgOperand(i: 1));
8453 return getSMinExpr(LHS, RHS);
8454 case Intrinsic::usub_sat: {
8455 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8456 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8457 const SCEV *ClampedY = getUMinExpr(LHS: X, RHS: Y);
8458 return getMinusSCEV(LHS: X, RHS: ClampedY, Flags: SCEV::FlagNUW);
8459 }
8460 case Intrinsic::uadd_sat: {
8461 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8462 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8463 const SCEV *ClampedX = getUMinExpr(LHS: X, RHS: getNotSCEV(V: Y));
8464 return getAddExpr(LHS: ClampedX, RHS: Y, Flags: SCEV::FlagNUW);
8465 }
8466 case Intrinsic::start_loop_iterations:
8467 case Intrinsic::annotation:
8468 case Intrinsic::ptr_annotation:
8469 // A start_loop_iterations or llvm.annotation or llvm.prt.annotation is
8470 // just eqivalent to the first operand for SCEV purposes.
8471 return getSCEV(V: II->getArgOperand(i: 0));
8472 case Intrinsic::vscale:
8473 return getVScale(Ty: II->getType());
8474 default:
8475 break;
8476 }
8477 }
8478 break;
8479 }
8480
8481 return getUnknown(V);
8482}
8483
8484//===----------------------------------------------------------------------===//
8485// Iteration Count Computation Code
8486//
8487
8488const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount) {
8489 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8490 return getCouldNotCompute();
8491
8492 auto *ExitCountType = ExitCount->getType();
8493 assert(ExitCountType->isIntegerTy());
8494 auto *EvalTy = Type::getIntNTy(C&: ExitCountType->getContext(),
8495 N: 1 + ExitCountType->getScalarSizeInBits());
8496 return getTripCountFromExitCount(ExitCount, EvalTy, L: nullptr);
8497}
8498
8499const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount,
8500 Type *EvalTy,
8501 const Loop *L) {
8502 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8503 return getCouldNotCompute();
8504
8505 unsigned ExitCountSize = getTypeSizeInBits(Ty: ExitCount->getType());
8506 unsigned EvalSize = EvalTy->getPrimitiveSizeInBits();
8507
8508 auto CanAddOneWithoutOverflow = [&]() {
8509 ConstantRange ExitCountRange =
8510 getRangeRef(S: ExitCount, SignHint: RangeSignHint::HINT_RANGE_UNSIGNED);
8511 if (!ExitCountRange.contains(Val: APInt::getMaxValue(numBits: ExitCountSize)))
8512 return true;
8513
8514 return L && isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: ExitCount,
8515 RHS: getMinusOne(Ty: ExitCount->getType()));
8516 };
8517
8518 // If we need to zero extend the backedge count, check if we can add one to
8519 // it prior to zero extending without overflow. Provided this is safe, it
8520 // allows better simplification of the +1.
8521 if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
8522 return getZeroExtendExpr(
8523 Op: getAddExpr(LHS: ExitCount, RHS: getOne(Ty: ExitCount->getType())), Ty: EvalTy);
8524
8525 // Get the total trip count from the count by adding 1. This may wrap.
8526 return getAddExpr(LHS: getTruncateOrZeroExtend(V: ExitCount, Ty: EvalTy), RHS: getOne(Ty: EvalTy));
8527}
8528
8529static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
8530 if (!ExitCount)
8531 return 0;
8532
8533 ConstantInt *ExitConst = ExitCount->getValue();
8534
8535 // Guard against huge trip counts.
8536 if (ExitConst->getValue().getActiveBits() > 32)
8537 return 0;
8538
8539 // In case of integer overflow, this returns 0, which is correct.
8540 return ((unsigned)ExitConst->getZExtValue()) + 1;
8541}
8542
8543unsigned ScalarEvolution::getSmallConstantTripCount(const Loop *L) {
8544 auto *ExitCount = dyn_cast<SCEVConstant>(Val: getBackedgeTakenCount(L, Kind: Exact));
8545 return getConstantTripCount(ExitCount);
8546}
8547
8548unsigned
8549ScalarEvolution::getSmallConstantTripCount(const Loop *L,
8550 const BasicBlock *ExitingBlock) {
8551 assert(ExitingBlock && "Must pass a non-null exiting block!");
8552 assert(L->isLoopExiting(ExitingBlock) &&
8553 "Exiting block must actually branch out of the loop!");
8554 const SCEVConstant *ExitCount =
8555 dyn_cast<SCEVConstant>(Val: getExitCount(L, ExitingBlock));
8556 return getConstantTripCount(ExitCount);
8557}
8558
8559unsigned ScalarEvolution::getSmallConstantMaxTripCount(
8560 const Loop *L, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8561
8562 const auto *MaxExitCount =
8563 Predicates ? getPredicatedConstantMaxBackedgeTakenCount(L, Predicates&: *Predicates)
8564 : getConstantMaxBackedgeTakenCount(L);
8565 return getConstantTripCount(ExitCount: dyn_cast<SCEVConstant>(Val: MaxExitCount));
8566}
8567
8568unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L) {
8569 SmallVector<BasicBlock *, 8> ExitingBlocks;
8570 L->getExitingBlocks(ExitingBlocks);
8571
8572 std::optional<unsigned> Res;
8573 for (auto *ExitingBB : ExitingBlocks) {
8574 unsigned Multiple = getSmallConstantTripMultiple(L, ExitingBlock: ExitingBB);
8575 if (!Res)
8576 Res = Multiple;
8577 Res = std::gcd(m: *Res, n: Multiple);
8578 }
8579 return Res.value_or(u: 1);
8580}
8581
8582unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8583 const SCEV *ExitCount) {
8584 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8585 return 1;
8586
8587 // Get the trip count
8588 const SCEV *TCExpr = getTripCountFromExitCount(ExitCount: applyLoopGuards(Expr: ExitCount, L));
8589
8590 APInt Multiple = getNonZeroConstantMultiple(S: TCExpr);
8591 // If a trip multiple is huge (>=2^32), the trip count is still divisible by
8592 // the greatest power of 2 divisor less than 2^32.
8593 return Multiple.getActiveBits() > 32
8594 ? 1U << std::min(a: 31U, b: Multiple.countTrailingZeros())
8595 : (unsigned)Multiple.getZExtValue();
8596}
8597
8598/// Returns the largest constant divisor of the trip count of this loop as a
8599/// normal unsigned value, if possible. This means that the actual trip count is
8600/// always a multiple of the returned value (don't forget the trip count could
8601/// very well be zero as well!).
8602///
8603/// Returns 1 if the trip count is unknown or not guaranteed to be the
8604/// multiple of a constant (which is also the case if the trip count is simply
8605/// constant, use getSmallConstantTripCount for that case), Will also return 1
8606/// if the trip count is very large (>= 2^32).
8607///
8608/// As explained in the comments for getSmallConstantTripCount, this assumes
8609/// that control exits the loop via ExitingBlock.
8610unsigned
8611ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8612 const BasicBlock *ExitingBlock) {
8613 assert(ExitingBlock && "Must pass a non-null exiting block!");
8614 assert(L->isLoopExiting(ExitingBlock) &&
8615 "Exiting block must actually branch out of the loop!");
8616 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
8617 return getSmallConstantTripMultiple(L, ExitCount);
8618}
8619
8620const SCEV *ScalarEvolution::getExitCount(const Loop *L,
8621 const BasicBlock *ExitingBlock,
8622 ExitCountKind Kind) {
8623 switch (Kind) {
8624 case Exact:
8625 return getBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this);
8626 case SymbolicMaximum:
8627 return getBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this);
8628 case ConstantMaximum:
8629 return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this);
8630 };
8631 llvm_unreachable("Invalid ExitCountKind!");
8632}
8633
8634const SCEV *ScalarEvolution::getPredicatedExitCount(
8635 const Loop *L, const BasicBlock *ExitingBlock,
8636 SmallVectorImpl<const SCEVPredicate *> *Predicates, ExitCountKind Kind) {
8637 switch (Kind) {
8638 case Exact:
8639 return getPredicatedBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this,
8640 Predicates);
8641 case SymbolicMaximum:
8642 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this,
8643 Predicates);
8644 case ConstantMaximum:
8645 return getPredicatedBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this,
8646 Predicates);
8647 };
8648 llvm_unreachable("Invalid ExitCountKind!");
8649}
8650
8651const SCEV *ScalarEvolution::getPredicatedBackedgeTakenCount(
8652 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8653 return getPredicatedBackedgeTakenInfo(L).getExact(L, SE: this, Predicates: &Preds);
8654}
8655
8656const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L,
8657 ExitCountKind Kind) {
8658 switch (Kind) {
8659 case Exact:
8660 return getBackedgeTakenInfo(L).getExact(L, SE: this);
8661 case ConstantMaximum:
8662 return getBackedgeTakenInfo(L).getConstantMax(SE: this);
8663 case SymbolicMaximum:
8664 return getBackedgeTakenInfo(L).getSymbolicMax(L, SE: this);
8665 };
8666 llvm_unreachable("Invalid ExitCountKind!");
8667}
8668
8669const SCEV *ScalarEvolution::getPredicatedSymbolicMaxBackedgeTakenCount(
8670 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8671 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, SE: this, Predicates: &Preds);
8672}
8673
8674const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
8675 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8676 return getPredicatedBackedgeTakenInfo(L).getConstantMax(SE: this, Predicates: &Preds);
8677}
8678
8679bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
8680 return getBackedgeTakenInfo(L).isConstantMaxOrZero(SE: this);
8681}
8682
8683ScalarEvolution::BackedgeTakenInfo &
8684ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) {
8685 auto &BTI = getBackedgeTakenInfo(L);
8686 if (BTI.hasFullInfo())
8687 return BTI;
8688
8689 auto Pair = PredicatedBackedgeTakenCounts.try_emplace(Key: L);
8690
8691 if (!Pair.second)
8692 return Pair.first->second;
8693
8694 BackedgeTakenInfo Result =
8695 computeBackedgeTakenCount(L, /*AllowPredicates=*/true);
8696
8697 return PredicatedBackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8698}
8699
8700ScalarEvolution::BackedgeTakenInfo &
8701ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
8702 // Initially insert an invalid entry for this loop. If the insertion
8703 // succeeds, proceed to actually compute a backedge-taken count and
8704 // update the value. The temporary CouldNotCompute value tells SCEV
8705 // code elsewhere that it shouldn't attempt to request a new
8706 // backedge-taken count, which could result in infinite recursion.
8707 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
8708 BackedgeTakenCounts.try_emplace(Key: L);
8709 if (!Pair.second)
8710 return Pair.first->second;
8711
8712 // computeBackedgeTakenCount may allocate memory for its result. Inserting it
8713 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
8714 // must be cleared in this scope.
8715 BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
8716
8717 // Now that we know more about the trip count for this loop, forget any
8718 // existing SCEV values for PHI nodes in this loop since they are only
8719 // conservative estimates made without the benefit of trip count
8720 // information. This invalidation is not necessary for correctness, and is
8721 // only done to produce more precise results.
8722 if (Result.hasAnyInfo()) {
8723 // Invalidate any expression using an addrec in this loop.
8724 SmallVector<SCEVUse, 8> ToForget;
8725 auto LoopUsersIt = LoopUsers.find(Val: L);
8726 if (LoopUsersIt != LoopUsers.end())
8727 append_range(C&: ToForget, R&: LoopUsersIt->second);
8728 forgetMemoizedResults(SCEVs: ToForget);
8729
8730 // Invalidate constant-evolved loop header phis.
8731 for (PHINode &PN : L->getHeader()->phis())
8732 ConstantEvolutionLoopExitValue.erase(Val: &PN);
8733 }
8734
8735 // Re-lookup the insert position, since the call to
8736 // computeBackedgeTakenCount above could result in a
8737 // recusive call to getBackedgeTakenInfo (on a different
8738 // loop), which would invalidate the iterator computed
8739 // earlier.
8740 return BackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8741}
8742
8743void ScalarEvolution::forgetAllLoops() {
8744 // This method is intended to forget all info about loops. It should
8745 // invalidate caches as if the following happened:
8746 // - The trip counts of all loops have changed arbitrarily
8747 // - Every llvm::Value has been updated in place to produce a different
8748 // result.
8749 BackedgeTakenCounts.clear();
8750 PredicatedBackedgeTakenCounts.clear();
8751 BECountUsers.clear();
8752 LoopPropertiesCache.clear();
8753 ConstantEvolutionLoopExitValue.clear();
8754 ValueExprMap.clear();
8755 ValuesAtScopes.clear();
8756 ValuesAtScopesUsers.clear();
8757 LoopDispositions.clear();
8758 BlockDispositions.clear();
8759 UnsignedRanges.clear();
8760 SignedRanges.clear();
8761 ExprValueMap.clear();
8762 HasRecMap.clear();
8763 ConstantMultipleCache.clear();
8764 PredicatedSCEVRewrites.clear();
8765 FoldCache.clear();
8766 FoldCacheUser.clear();
8767}
8768void ScalarEvolution::visitAndClearUsers(
8769 SmallVectorImpl<Instruction *> &Worklist,
8770 SmallPtrSetImpl<Instruction *> &Visited,
8771 SmallVectorImpl<SCEVUse> &ToForget) {
8772 while (!Worklist.empty()) {
8773 Instruction *I = Worklist.pop_back_val();
8774 if (!isSCEVable(Ty: I->getType()) && !isa<WithOverflowInst>(Val: I))
8775 continue;
8776
8777 ValueExprMapType::iterator It =
8778 ValueExprMap.find_as(Val: static_cast<Value *>(I));
8779 if (It != ValueExprMap.end()) {
8780 ToForget.push_back(Elt: It->second);
8781 eraseValueFromMap(V: It->first);
8782 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
8783 ConstantEvolutionLoopExitValue.erase(Val: PN);
8784 }
8785
8786 PushDefUseChildren(I, Worklist, Visited);
8787 }
8788}
8789
8790void ScalarEvolution::forgetLoop(const Loop *L) {
8791 SmallVector<const Loop *, 16> LoopWorklist(1, L);
8792 SmallVector<SCEVUse, 16> ToForget;
8793
8794 // Iterate over all the loops and sub-loops to drop SCEV information.
8795 while (!LoopWorklist.empty()) {
8796 auto *CurrL = LoopWorklist.pop_back_val();
8797
8798 // Drop any stored trip count value.
8799 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ false);
8800 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ true);
8801
8802 // Drop information about predicated SCEV rewrites for this loop.
8803 PredicatedSCEVRewrites.remove_if(
8804 Pred: [&](const auto &Entry) { return Entry.first.second == CurrL; });
8805
8806 auto LoopUsersItr = LoopUsers.find(Val: CurrL);
8807 if (LoopUsersItr != LoopUsers.end())
8808 llvm::append_range(C&: ToForget, R&: LoopUsersItr->second);
8809
8810 // Drop information about expressions based on loop-header PHIs.
8811 for (PHINode &PN : CurrL->getHeader()->phis()) {
8812 ConstantEvolutionLoopExitValue.erase(Val: &PN);
8813 auto VIt = ValueExprMap.find_as(Val: static_cast<Value *>(&PN));
8814 if (VIt != ValueExprMap.end())
8815 ToForget.push_back(Elt: VIt->second);
8816 }
8817
8818 LoopPropertiesCache.erase(Val: CurrL);
8819 // Forget all contained loops too, to avoid dangling entries in the
8820 // ValuesAtScopes map.
8821 LoopWorklist.append(in_start: CurrL->begin(), in_end: CurrL->end());
8822 }
8823 forgetMemoizedResults(SCEVs: ToForget);
8824}
8825
8826void ScalarEvolution::forgetTopmostLoop(const Loop *L) {
8827 forgetLoop(L: L->getOutermostLoop());
8828}
8829
8830void ScalarEvolution::forgetValue(Value *V) {
8831 Instruction *I = dyn_cast<Instruction>(Val: V);
8832 if (!I) return;
8833
8834 // Drop information about expressions based on loop-header PHIs.
8835 SmallVector<Instruction *, 16> Worklist;
8836 SmallPtrSet<Instruction *, 8> Visited;
8837 SmallVector<SCEVUse, 8> ToForget;
8838 Worklist.push_back(Elt: I);
8839 Visited.insert(Ptr: I);
8840 visitAndClearUsers(Worklist, Visited, ToForget);
8841
8842 forgetMemoizedResults(SCEVs: ToForget);
8843}
8844
8845void ScalarEvolution::forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V) {
8846 if (!isSCEVable(Ty: V->getType()))
8847 return;
8848
8849 // If SCEV looked through a trivial LCSSA phi node, we might have SCEV's
8850 // directly using a SCEVUnknown/SCEVAddRec defined in the loop. After an
8851 // extra predecessor is added, this is no longer valid. Find all Unknowns and
8852 // AddRecs defined in the loop and invalidate any SCEV's making use of them.
8853 if (const SCEV *S = getExistingSCEV(V)) {
8854 struct InvalidationRootCollector {
8855 Loop *L;
8856 SmallVector<SCEVUse, 8> Roots;
8857
8858 InvalidationRootCollector(Loop *L) : L(L) {}
8859
8860 bool follow(const SCEV *S) {
8861 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
8862 if (auto *I = dyn_cast<Instruction>(Val: SU->getValue()))
8863 if (L->contains(Inst: I))
8864 Roots.push_back(Elt: S);
8865 } else if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S)) {
8866 if (L->contains(L: AddRec->getLoop()))
8867 Roots.push_back(Elt: S);
8868 }
8869 return true;
8870 }
8871 bool isDone() const { return false; }
8872 };
8873
8874 InvalidationRootCollector C(L);
8875 visitAll(Root: S, Visitor&: C);
8876 forgetMemoizedResults(SCEVs: C.Roots);
8877 }
8878
8879 // Also perform the normal invalidation.
8880 forgetValue(V);
8881}
8882
8883void ScalarEvolution::forgetLoopDispositions() { LoopDispositions.clear(); }
8884
8885void ScalarEvolution::forgetBlockAndLoopDispositions(Value *V) {
8886 // Unless a specific value is passed to invalidation, completely clear both
8887 // caches.
8888 if (!V) {
8889 BlockDispositions.clear();
8890 LoopDispositions.clear();
8891 return;
8892 }
8893
8894 if (!isSCEVable(Ty: V->getType()))
8895 return;
8896
8897 const SCEV *S = getExistingSCEV(V);
8898 if (!S)
8899 return;
8900
8901 // Invalidate the block and loop dispositions cached for S. Dispositions of
8902 // S's users may change if S's disposition changes (i.e. a user may change to
8903 // loop-invariant, if S changes to loop invariant), so also invalidate
8904 // dispositions of S's users recursively.
8905 SmallVector<SCEVUse, 8> Worklist = {S};
8906 SmallPtrSet<const SCEV *, 8> Seen = {S};
8907 while (!Worklist.empty()) {
8908 const SCEV *Curr = Worklist.pop_back_val();
8909 bool LoopDispoRemoved = LoopDispositions.erase(Val: Curr);
8910 bool BlockDispoRemoved = BlockDispositions.erase(Val: Curr);
8911 if (!LoopDispoRemoved && !BlockDispoRemoved)
8912 continue;
8913 auto Users = SCEVUsers.find(Val: Curr);
8914 if (Users != SCEVUsers.end())
8915 for (const auto *User : Users->second)
8916 if (Seen.insert(Ptr: User).second)
8917 Worklist.push_back(Elt: User);
8918 }
8919}
8920
8921/// Get the exact loop backedge taken count considering all loop exits. A
8922/// computable result can only be returned for loops with all exiting blocks
8923/// dominating the latch. howFarToZero assumes that the limit of each loop test
8924/// is never skipped. This is a valid assumption as long as the loop exits via
8925/// that test. For precise results, it is the caller's responsibility to specify
8926/// the relevant loop exiting block using getExact(ExitingBlock, SE).
8927const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
8928 const Loop *L, ScalarEvolution *SE,
8929 SmallVectorImpl<const SCEVPredicate *> *Preds) const {
8930 // If any exits were not computable, the loop is not computable.
8931 if (!isComplete() || ExitNotTaken.empty())
8932 return SE->getCouldNotCompute();
8933
8934 const BasicBlock *Latch = L->getLoopLatch();
8935 // All exiting blocks we have collected must dominate the only backedge.
8936 if (!Latch)
8937 return SE->getCouldNotCompute();
8938
8939 // All exiting blocks we have gathered dominate loop's latch, so exact trip
8940 // count is simply a minimum out of all these calculated exit counts.
8941 SmallVector<SCEVUse, 2> Ops;
8942 for (const auto &ENT : ExitNotTaken) {
8943 const SCEV *BECount = ENT.ExactNotTaken;
8944 assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!");
8945 assert(SE->DT.dominates(ENT.ExitingBlock, Latch) &&
8946 "We should only have known counts for exiting blocks that dominate "
8947 "latch!");
8948
8949 Ops.push_back(Elt: BECount);
8950
8951 if (Preds)
8952 append_range(C&: *Preds, R: ENT.Predicates);
8953
8954 assert((Preds || ENT.hasAlwaysTruePredicate()) &&
8955 "Predicate should be always true!");
8956 }
8957
8958 // If an earlier exit exits on the first iteration (exit count zero), then
8959 // a later poison exit count should not propagate into the result. This are
8960 // exactly the semantics provided by umin_seq.
8961 return SE->getUMinFromMismatchedTypes(Ops, /* Sequential */ true);
8962}
8963
8964const ScalarEvolution::ExitNotTakenInfo *
8965ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
8966 const BasicBlock *ExitingBlock,
8967 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8968 for (const auto &ENT : ExitNotTaken)
8969 if (ENT.ExitingBlock == ExitingBlock) {
8970 if (ENT.hasAlwaysTruePredicate())
8971 return &ENT;
8972 else if (Predicates) {
8973 append_range(C&: *Predicates, R: ENT.Predicates);
8974 return &ENT;
8975 }
8976 }
8977
8978 return nullptr;
8979}
8980
8981/// getConstantMax - Get the constant max backedge taken count for the loop.
8982const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
8983 ScalarEvolution *SE,
8984 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8985 if (!getConstantMax())
8986 return SE->getCouldNotCompute();
8987
8988 for (const auto &ENT : ExitNotTaken)
8989 if (!ENT.hasAlwaysTruePredicate()) {
8990 if (!Predicates)
8991 return SE->getCouldNotCompute();
8992 append_range(C&: *Predicates, R: ENT.Predicates);
8993 }
8994
8995 assert((isa<SCEVCouldNotCompute>(getConstantMax()) ||
8996 isa<SCEVConstant>(getConstantMax())) &&
8997 "No point in having a non-constant max backedge taken count!");
8998 return getConstantMax();
8999}
9000
9001const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
9002 const Loop *L, ScalarEvolution *SE,
9003 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
9004 if (!SymbolicMax) {
9005 // Form an expression for the maximum exit count possible for this loop. We
9006 // merge the max and exact information to approximate a version of
9007 // getConstantMaxBackedgeTakenCount which isn't restricted to just
9008 // constants.
9009 SmallVector<SCEVUse, 4> ExitCounts;
9010
9011 for (const auto &ENT : ExitNotTaken) {
9012 const SCEV *ExitCount = ENT.SymbolicMaxNotTaken;
9013 if (!isa<SCEVCouldNotCompute>(Val: ExitCount)) {
9014 assert(SE->DT.dominates(ENT.ExitingBlock, L->getLoopLatch()) &&
9015 "We should only have known counts for exiting blocks that "
9016 "dominate latch!");
9017 ExitCounts.push_back(Elt: ExitCount);
9018 if (Predicates)
9019 append_range(C&: *Predicates, R: ENT.Predicates);
9020
9021 assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
9022 "Predicate should be always true!");
9023 }
9024 }
9025 if (ExitCounts.empty())
9026 SymbolicMax = SE->getCouldNotCompute();
9027 else
9028 SymbolicMax =
9029 SE->getUMinFromMismatchedTypes(Ops&: ExitCounts, /*Sequential*/ true);
9030 }
9031 return SymbolicMax;
9032}
9033
9034bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
9035 ScalarEvolution *SE) const {
9036 auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
9037 return !ENT.hasAlwaysTruePredicate();
9038 };
9039 return MaxOrZero && !any_of(Range: ExitNotTaken, P: PredicateNotAlwaysTrue);
9040}
9041
9042ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E)
9043 : ExitLimit(E, E, E, false) {}
9044
9045ScalarEvolution::ExitLimit::ExitLimit(
9046 const SCEV *E, const SCEV *ConstantMaxNotTaken,
9047 const SCEV *SymbolicMaxNotTaken, bool MaxOrZero,
9048 ArrayRef<ArrayRef<const SCEVPredicate *>> PredLists)
9049 : ExactNotTaken(E), ConstantMaxNotTaken(ConstantMaxNotTaken),
9050 SymbolicMaxNotTaken(SymbolicMaxNotTaken), MaxOrZero(MaxOrZero) {
9051 // If we prove the max count is zero, so is the symbolic bound. This happens
9052 // in practice due to differences in a) how context sensitive we've chosen
9053 // to be and b) how we reason about bounds implied by UB.
9054 if (ConstantMaxNotTaken->isZero()) {
9055 this->ExactNotTaken = E = ConstantMaxNotTaken;
9056 this->SymbolicMaxNotTaken = SymbolicMaxNotTaken = ConstantMaxNotTaken;
9057 }
9058
9059 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
9060 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
9061 "Exact is not allowed to be less precise than Constant Max");
9062 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
9063 !isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken)) &&
9064 "Exact is not allowed to be less precise than Symbolic Max");
9065 assert((isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken) ||
9066 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
9067 "Symbolic Max is not allowed to be less precise than Constant Max");
9068 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
9069 isa<SCEVConstant>(ConstantMaxNotTaken)) &&
9070 "No point in having a non-constant max backedge taken count!");
9071 SmallPtrSet<const SCEVPredicate *, 4> SeenPreds;
9072 for (const auto PredList : PredLists)
9073 for (const auto *P : PredList) {
9074 if (SeenPreds.contains(Ptr: P))
9075 continue;
9076 assert(!isa<SCEVUnionPredicate>(P) && "Only add leaf predicates here!");
9077 SeenPreds.insert(Ptr: P);
9078 Predicates.push_back(Elt: P);
9079 }
9080 assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) &&
9081 "Backedge count should be int");
9082 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
9083 !ConstantMaxNotTaken->getType()->isPointerTy()) &&
9084 "Max backedge count should be int");
9085}
9086
9087ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E,
9088 const SCEV *ConstantMaxNotTaken,
9089 const SCEV *SymbolicMaxNotTaken,
9090 bool MaxOrZero,
9091 ArrayRef<const SCEVPredicate *> PredList)
9092 : ExitLimit(E, ConstantMaxNotTaken, SymbolicMaxNotTaken, MaxOrZero,
9093 ArrayRef({PredList})) {}
9094
9095/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
9096/// computable exit into a persistent ExitNotTakenInfo array.
9097ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
9098 ArrayRef<ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo> ExitCounts,
9099 bool IsComplete, const SCEV *ConstantMax, bool MaxOrZero)
9100 : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
9101 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
9102
9103 ExitNotTaken.reserve(N: ExitCounts.size());
9104 std::transform(first: ExitCounts.begin(), last: ExitCounts.end(),
9105 result: std::back_inserter(x&: ExitNotTaken),
9106 unary_op: [&](const EdgeExitInfo &EEI) {
9107 BasicBlock *ExitBB = EEI.first;
9108 const ExitLimit &EL = EEI.second;
9109 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
9110 EL.ConstantMaxNotTaken, EL.SymbolicMaxNotTaken,
9111 EL.Predicates);
9112 });
9113 assert((isa<SCEVCouldNotCompute>(ConstantMax) ||
9114 isa<SCEVConstant>(ConstantMax)) &&
9115 "No point in having a non-constant max backedge taken count!");
9116}
9117
9118/// Compute the number of times the backedge of the specified loop will execute.
9119ScalarEvolution::BackedgeTakenInfo
9120ScalarEvolution::computeBackedgeTakenCount(const Loop *L,
9121 bool AllowPredicates) {
9122 SmallVector<BasicBlock *, 8> ExitingBlocks;
9123 L->getExitingBlocks(ExitingBlocks);
9124
9125 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
9126
9127 SmallVector<EdgeExitInfo, 4> ExitCounts;
9128 bool CouldComputeBECount = true;
9129 BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
9130 const SCEV *MustExitMaxBECount = nullptr;
9131 const SCEV *MayExitMaxBECount = nullptr;
9132 bool MustExitMaxOrZero = false;
9133 bool IsOnlyExit = ExitingBlocks.size() == 1;
9134
9135 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
9136 // and compute maxBECount.
9137 // Do a union of all the predicates here.
9138 for (BasicBlock *ExitBB : ExitingBlocks) {
9139 // We canonicalize untaken exits to br (constant), ignore them so that
9140 // proving an exit untaken doesn't negatively impact our ability to reason
9141 // about the loop as whole.
9142 if (auto *BI = dyn_cast<CondBrInst>(Val: ExitBB->getTerminator()))
9143 if (auto *CI = dyn_cast<ConstantInt>(Val: BI->getCondition())) {
9144 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
9145 if (ExitIfTrue == CI->isZero())
9146 continue;
9147 }
9148
9149 ExitLimit EL = computeExitLimit(L, ExitingBlock: ExitBB, IsOnlyExit, AllowPredicates);
9150
9151 assert((AllowPredicates || EL.Predicates.empty()) &&
9152 "Predicated exit limit when predicates are not allowed!");
9153
9154 // 1. For each exit that can be computed, add an entry to ExitCounts.
9155 // CouldComputeBECount is true only if all exits can be computed.
9156 if (EL.ExactNotTaken != getCouldNotCompute())
9157 ++NumExitCountsComputed;
9158 else
9159 // We couldn't compute an exact value for this exit, so
9160 // we won't be able to compute an exact value for the loop.
9161 CouldComputeBECount = false;
9162 // Remember exit count if either exact or symbolic is known. Because
9163 // Exact always implies symbolic, only check symbolic.
9164 if (EL.SymbolicMaxNotTaken != getCouldNotCompute())
9165 ExitCounts.emplace_back(Args&: ExitBB, Args&: EL);
9166 else {
9167 assert(EL.ExactNotTaken == getCouldNotCompute() &&
9168 "Exact is known but symbolic isn't?");
9169 ++NumExitCountsNotComputed;
9170 }
9171
9172 // 2. Derive the loop's MaxBECount from each exit's max number of
9173 // non-exiting iterations. Partition the loop exits into two kinds:
9174 // LoopMustExits and LoopMayExits.
9175 //
9176 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
9177 // is a LoopMayExit. If any computable LoopMustExit is found, then
9178 // MaxBECount is the minimum EL.ConstantMaxNotTaken of computable
9179 // LoopMustExits. Otherwise, MaxBECount is conservatively the maximum
9180 // EL.ConstantMaxNotTaken, where CouldNotCompute is considered greater than
9181 // any
9182 // computable EL.ConstantMaxNotTaken.
9183 if (EL.ConstantMaxNotTaken != getCouldNotCompute() && Latch &&
9184 DT.dominates(A: ExitBB, B: Latch)) {
9185 if (!MustExitMaxBECount) {
9186 MustExitMaxBECount = EL.ConstantMaxNotTaken;
9187 MustExitMaxOrZero = EL.MaxOrZero;
9188 } else {
9189 MustExitMaxBECount = getUMinFromMismatchedTypes(LHS: MustExitMaxBECount,
9190 RHS: EL.ConstantMaxNotTaken);
9191 }
9192 } else if (MayExitMaxBECount != getCouldNotCompute()) {
9193 if (!MayExitMaxBECount || EL.ConstantMaxNotTaken == getCouldNotCompute())
9194 MayExitMaxBECount = EL.ConstantMaxNotTaken;
9195 else {
9196 MayExitMaxBECount = getUMaxFromMismatchedTypes(LHS: MayExitMaxBECount,
9197 RHS: EL.ConstantMaxNotTaken);
9198 }
9199 }
9200 }
9201 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
9202 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
9203 // The loop backedge will be taken the maximum or zero times if there's
9204 // a single exit that must be taken the maximum or zero times.
9205 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
9206
9207 // Remember which SCEVs are used in exit limits for invalidation purposes.
9208 // We only care about non-constant SCEVs here, so we can ignore
9209 // EL.ConstantMaxNotTaken
9210 // and MaxBECount, which must be SCEVConstant.
9211 for (const auto &Pair : ExitCounts) {
9212 if (!isa<SCEVConstant>(Val: Pair.second.ExactNotTaken))
9213 BECountUsers[Pair.second.ExactNotTaken].insert(Ptr: {L, AllowPredicates});
9214 if (!isa<SCEVConstant>(Val: Pair.second.SymbolicMaxNotTaken))
9215 BECountUsers[Pair.second.SymbolicMaxNotTaken].insert(
9216 Ptr: {L, AllowPredicates});
9217 }
9218 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
9219 MaxBECount, MaxOrZero);
9220}
9221
9222ScalarEvolution::ExitLimit
9223ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock,
9224 bool IsOnlyExit, bool AllowPredicates) {
9225 assert(L->contains(ExitingBlock) && "Exit count for non-loop block?");
9226 // If our exiting block does not dominate the latch, then its connection with
9227 // loop's exit limit may be far from trivial.
9228 const BasicBlock *Latch = L->getLoopLatch();
9229 if (!Latch || !DT.dominates(A: ExitingBlock, B: Latch))
9230 return getCouldNotCompute();
9231
9232 Instruction *Term = ExitingBlock->getTerminator();
9233 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Term)) {
9234 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
9235 assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) &&
9236 "It should have one successor in loop and one exit block!");
9237 // Proceed to the next level to examine the exit condition expression.
9238 return computeExitLimitFromCond(L, ExitCond: BI->getCondition(), ExitIfTrue,
9239 /*ControlsOnlyExit=*/IsOnlyExit,
9240 AllowPredicates);
9241 }
9242
9243 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: Term)) {
9244 // For switch, make sure that there is a single exit from the loop.
9245 BasicBlock *Exit = nullptr;
9246 for (auto *SBB : successors(BB: ExitingBlock))
9247 if (!L->contains(BB: SBB)) {
9248 if (Exit) // Multiple exit successors.
9249 return getCouldNotCompute();
9250 Exit = SBB;
9251 }
9252 assert(Exit && "Exiting block must have at least one exit");
9253 return computeExitLimitFromSingleExitSwitch(
9254 L, Switch: SI, ExitingBB: Exit, /*ControlsOnlyExit=*/IsSubExpr: IsOnlyExit);
9255 }
9256
9257 return getCouldNotCompute();
9258}
9259
9260ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCond(
9261 const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9262 bool AllowPredicates) {
9263 ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
9264 return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
9265 ControlsOnlyExit, AllowPredicates);
9266}
9267
9268std::optional<ScalarEvolution::ExitLimit>
9269ScalarEvolution::ExitLimitCache::find(const Loop *L, Value *ExitCond,
9270 bool ExitIfTrue, bool ControlsOnlyExit,
9271 bool AllowPredicates) {
9272 (void)this->L;
9273 (void)this->ExitIfTrue;
9274 (void)this->AllowPredicates;
9275
9276 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9277 this->AllowPredicates == AllowPredicates &&
9278 "Variance in assumed invariant key components!");
9279 auto Itr = TripCountMap.find(Val: {ExitCond, ControlsOnlyExit});
9280 if (Itr == TripCountMap.end())
9281 return std::nullopt;
9282 return Itr->second;
9283}
9284
9285void ScalarEvolution::ExitLimitCache::insert(const Loop *L, Value *ExitCond,
9286 bool ExitIfTrue,
9287 bool ControlsOnlyExit,
9288 bool AllowPredicates,
9289 const ExitLimit &EL) {
9290 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9291 this->AllowPredicates == AllowPredicates &&
9292 "Variance in assumed invariant key components!");
9293
9294 auto InsertResult = TripCountMap.insert(KV: {{ExitCond, ControlsOnlyExit}, EL});
9295 assert(InsertResult.second && "Expected successful insertion!");
9296 (void)InsertResult;
9297 (void)ExitIfTrue;
9298}
9299
9300ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached(
9301 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9302 bool ControlsOnlyExit, bool AllowPredicates) {
9303
9304 if (auto MaybeEL = Cache.find(L, ExitCond, ExitIfTrue, ControlsOnlyExit,
9305 AllowPredicates))
9306 return *MaybeEL;
9307
9308 ExitLimit EL = computeExitLimitFromCondImpl(
9309 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates);
9310 Cache.insert(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates, EL);
9311 return EL;
9312}
9313
9314ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl(
9315 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9316 bool ControlsOnlyExit, bool AllowPredicates) {
9317 // Handle BinOp conditions (And, Or).
9318 if (auto LimitFromBinOp = computeExitLimitFromCondFromBinOp(
9319 Cache, L, ExitCond, ExitIfTrue, AllowPredicates))
9320 return *LimitFromBinOp;
9321
9322 // With an icmp, it may be feasible to compute an exact backedge-taken count.
9323 // Proceed to the next level to examine the icmp.
9324 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(Val: ExitCond)) {
9325 ExitLimit EL =
9326 computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue, IsSubExpr: ControlsOnlyExit);
9327 if (EL.hasFullInfo() || !AllowPredicates)
9328 return EL;
9329
9330 // Try again, but use SCEV predicates this time.
9331 return computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue,
9332 IsSubExpr: ControlsOnlyExit,
9333 /*AllowPredicates=*/true);
9334 }
9335
9336 // Check for a constant condition. These are normally stripped out by
9337 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
9338 // preserve the CFG and is temporarily leaving constant conditions
9339 // in place.
9340 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: ExitCond)) {
9341 if (ExitIfTrue == !CI->getZExtValue())
9342 // The backedge is always taken.
9343 return getCouldNotCompute();
9344 // The backedge is never taken.
9345 return getZero(Ty: CI->getType());
9346 }
9347
9348 // If we're exiting based on the overflow flag of an x.with.overflow intrinsic
9349 // with a constant step, we can form an equivalent icmp predicate and figure
9350 // out how many iterations will be taken before we exit.
9351 const WithOverflowInst *WO;
9352 const APInt *C;
9353 if (match(V: ExitCond, P: m_ExtractValue<1>(V: m_WithOverflowInst(I&: WO))) &&
9354 match(V: WO->getRHS(), P: m_APInt(Res&: C))) {
9355 ConstantRange NWR =
9356 ConstantRange::makeExactNoWrapRegion(BinOp: WO->getBinaryOp(), Other: *C,
9357 NoWrapKind: WO->getNoWrapKind());
9358 CmpInst::Predicate Pred;
9359 APInt NewRHSC, Offset;
9360 NWR.getEquivalentICmp(Pred, RHS&: NewRHSC, Offset);
9361 if (!ExitIfTrue)
9362 Pred = ICmpInst::getInversePredicate(pred: Pred);
9363 auto *LHS = getSCEV(V: WO->getLHS());
9364 if (Offset != 0)
9365 LHS = getAddExpr(LHS, RHS: getConstant(Val: Offset));
9366 auto EL = computeExitLimitFromICmp(L, Pred, LHS, RHS: getConstant(Val: NewRHSC),
9367 IsSubExpr: ControlsOnlyExit, AllowPredicates);
9368 if (EL.hasAnyInfo())
9369 return EL;
9370 }
9371
9372 // If it's not an integer or pointer comparison then compute it the hard way.
9373 return computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9374}
9375
9376std::optional<ScalarEvolution::ExitLimit>
9377ScalarEvolution::computeExitLimitFromCondFromBinOp(ExitLimitCacheTy &Cache,
9378 const Loop *L,
9379 Value *ExitCond,
9380 bool ExitIfTrue,
9381 bool AllowPredicates) {
9382 // Check if the controlling expression for this loop is an And or Or.
9383 Value *Op0, *Op1;
9384 bool IsAnd;
9385 if (match(V: ExitCond, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9386 IsAnd = true;
9387 else if (match(V: ExitCond, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9388 IsAnd = false;
9389 else
9390 return std::nullopt;
9391
9392 // A sub-condition of a non-trivial binop never solely controls the exit,
9393 // whether we exit always depends on both conditions.
9394 ExitLimit EL0 = computeExitLimitFromCondCached(
9395 Cache, L, ExitCond: Op0, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9396 ExitLimit EL1 = computeExitLimitFromCondCached(
9397 Cache, L, ExitCond: Op1, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9398
9399 // EitherMayExit is true in these two cases:
9400 // br (and Op0 Op1), loop, exit
9401 // br (or Op0 Op1), exit, loop
9402 bool EitherMayExit = IsAnd ^ ExitIfTrue;
9403
9404 const SCEV *BECount = getCouldNotCompute();
9405 const SCEV *ConstantMaxBECount = getCouldNotCompute();
9406 const SCEV *SymbolicMaxBECount = getCouldNotCompute();
9407 if (EitherMayExit) {
9408 bool UseSequentialUMin = !isa<BinaryOperator>(Val: ExitCond);
9409 // Both conditions must be same for the loop to continue executing.
9410 // Choose the less conservative count.
9411 if (EL0.ExactNotTaken != getCouldNotCompute() &&
9412 EL1.ExactNotTaken != getCouldNotCompute()) {
9413 BECount = getUMinFromMismatchedTypes(LHS: EL0.ExactNotTaken, RHS: EL1.ExactNotTaken,
9414 Sequential: UseSequentialUMin);
9415 }
9416 if (EL0.ConstantMaxNotTaken == getCouldNotCompute())
9417 ConstantMaxBECount = EL1.ConstantMaxNotTaken;
9418 else if (EL1.ConstantMaxNotTaken == getCouldNotCompute())
9419 ConstantMaxBECount = EL0.ConstantMaxNotTaken;
9420 else
9421 ConstantMaxBECount = getUMinFromMismatchedTypes(LHS: EL0.ConstantMaxNotTaken,
9422 RHS: EL1.ConstantMaxNotTaken);
9423 if (EL0.SymbolicMaxNotTaken == getCouldNotCompute())
9424 SymbolicMaxBECount = EL1.SymbolicMaxNotTaken;
9425 else if (EL1.SymbolicMaxNotTaken == getCouldNotCompute())
9426 SymbolicMaxBECount = EL0.SymbolicMaxNotTaken;
9427 else
9428 SymbolicMaxBECount = getUMinFromMismatchedTypes(
9429 LHS: EL0.SymbolicMaxNotTaken, RHS: EL1.SymbolicMaxNotTaken, Sequential: UseSequentialUMin);
9430 } else {
9431 // Both conditions must be same at the same time for the loop to exit.
9432 // For now, be conservative.
9433 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
9434 BECount = EL0.ExactNotTaken;
9435 }
9436
9437 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able
9438 // to be more aggressive when computing BECount than when computing
9439 // ConstantMaxBECount. In these cases it is possible for EL0.ExactNotTaken
9440 // and
9441 // EL1.ExactNotTaken to match, but for EL0.ConstantMaxNotTaken and
9442 // EL1.ConstantMaxNotTaken to not.
9443 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
9444 !isa<SCEVCouldNotCompute>(Val: BECount))
9445 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
9446 if (isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount))
9447 SymbolicMaxBECount =
9448 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
9449 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
9450 {ArrayRef(EL0.Predicates), ArrayRef(EL1.Predicates)});
9451}
9452
9453ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9454 const Loop *L, ICmpInst *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9455 bool AllowPredicates) {
9456 // If the condition was exit on true, convert the condition to exit on false
9457 CmpPredicate Pred;
9458 if (!ExitIfTrue)
9459 Pred = ExitCond->getCmpPredicate();
9460 else
9461 Pred = ExitCond->getInverseCmpPredicate();
9462 const ICmpInst::Predicate OriginalPred = Pred;
9463
9464 const SCEV *LHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 0));
9465 const SCEV *RHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 1));
9466
9467 ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, IsSubExpr: ControlsOnlyExit,
9468 AllowPredicates);
9469 if (EL.hasAnyInfo())
9470 return EL;
9471
9472 auto *ExhaustiveCount =
9473 computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9474
9475 if (!isa<SCEVCouldNotCompute>(Val: ExhaustiveCount))
9476 return ExhaustiveCount;
9477
9478 return computeShiftCompareExitLimit(LHS: ExitCond->getOperand(i_nocapture: 0),
9479 RHS: ExitCond->getOperand(i_nocapture: 1), L, Pred: OriginalPred);
9480}
9481ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9482 const Loop *L, CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS,
9483 bool ControlsOnlyExit, bool AllowPredicates) {
9484
9485 // Try to evaluate any dependencies out of the loop.
9486 LHS = getSCEVAtScope(S: LHS, L);
9487 RHS = getSCEVAtScope(S: RHS, L);
9488
9489 // At this point, we would like to compute how many iterations of the
9490 // loop the predicate will return true for these inputs.
9491 if (isLoopInvariant(S: LHS, L) && !isLoopInvariant(S: RHS, L)) {
9492 // If there is a loop-invariant, force it into the RHS.
9493 std::swap(a&: LHS, b&: RHS);
9494 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
9495 }
9496
9497 bool ControllingFiniteLoop = ControlsOnlyExit && loopHasNoAbnormalExits(L) &&
9498 loopIsFiniteByAssumption(L);
9499 // Simplify the operands before analyzing them.
9500 (void)SimplifyICmpOperands(Pred, LHS, RHS, /*Depth=*/0);
9501
9502 // If we have a comparison of a chrec against a constant, try to use value
9503 // ranges to answer this query.
9504 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS))
9505 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
9506 if (AddRec->getLoop() == L) {
9507 // Form the constant range.
9508 ConstantRange CompRange =
9509 ConstantRange::makeExactICmpRegion(Pred, Other: RHSC->getAPInt());
9510
9511 const SCEV *Ret = AddRec->getNumIterationsInRange(Range: CompRange, SE&: *this);
9512 if (!isa<SCEVCouldNotCompute>(Val: Ret)) return Ret;
9513 }
9514
9515 // If this loop must exit based on this condition (or execute undefined
9516 // behaviour), see if we can improve wrap flags. This is essentially
9517 // a must execute style proof.
9518 if (ControllingFiniteLoop && isLoopInvariant(S: RHS, L)) {
9519 // If we can prove the test sequence produced must repeat the same values
9520 // on self-wrap of the IV, then we can infer that IV doesn't self wrap
9521 // because if it did, we'd have an infinite (undefined) loop.
9522 // TODO: We can peel off any functions which are invertible *in L*. Loop
9523 // invariant terms are effectively constants for our purposes here.
9524 SCEVUse InnerLHS = LHS;
9525 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val&: LHS))
9526 InnerLHS = ZExt->getOperand();
9527 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: InnerLHS);
9528 AR && !AR->hasNoSelfWrap() && AR->getLoop() == L && AR->isAffine() &&
9529 isKnownToBeAPowerOfTwo(S: AR->getStepRecurrence(SE&: *this), /*OrZero=*/true,
9530 /*OrNegative=*/true)) {
9531 auto Flags = AR->getNoWrapFlags();
9532 Flags = setFlags(Flags, OnFlags: SCEV::FlagNW);
9533 SmallVector<SCEVUse> Operands{AR->operands()};
9534 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9535 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9536 }
9537
9538 // For a slt/ult condition with a positive step, can we prove nsw/nuw?
9539 // From no-self-wrap, this follows trivially from the fact that every
9540 // (un)signed-wrapped, but not self-wrapped value must be LT than the
9541 // last value before (un)signed wrap. Since we know that last value
9542 // didn't exit, nor will any smaller one.
9543 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT) {
9544 auto WrapType = Pred == ICmpInst::ICMP_SLT ? SCEV::FlagNSW : SCEV::FlagNUW;
9545 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS);
9546 AR && AR->getLoop() == L && AR->isAffine() &&
9547 !AR->getNoWrapFlags(Mask: WrapType) && AR->hasNoSelfWrap() &&
9548 isKnownPositive(S: AR->getStepRecurrence(SE&: *this))) {
9549 auto Flags = AR->getNoWrapFlags();
9550 Flags = setFlags(Flags, OnFlags: WrapType);
9551 SmallVector<SCEVUse> Operands{AR->operands()};
9552 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9553 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9554 }
9555 }
9556 }
9557
9558 switch (Pred) {
9559 case ICmpInst::ICMP_NE: { // while (X != Y)
9560 // Convert to: while (X-Y != 0)
9561 if (LHS->getType()->isPointerTy()) {
9562 LHS = getLosslessPtrToIntExpr(Op: LHS);
9563 if (isa<SCEVCouldNotCompute>(Val: LHS))
9564 return LHS;
9565 }
9566 if (RHS->getType()->isPointerTy()) {
9567 RHS = getLosslessPtrToIntExpr(Op: RHS);
9568 if (isa<SCEVCouldNotCompute>(Val: RHS))
9569 return RHS;
9570 }
9571 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit,
9572 AllowPredicates);
9573 if (EL.hasAnyInfo())
9574 return EL;
9575 break;
9576 }
9577 case ICmpInst::ICMP_EQ: { // while (X == Y)
9578 // Convert to: while (X-Y == 0)
9579 if (LHS->getType()->isPointerTy()) {
9580 LHS = getLosslessPtrToIntExpr(Op: LHS);
9581 if (isa<SCEVCouldNotCompute>(Val: LHS))
9582 return LHS;
9583 }
9584 if (RHS->getType()->isPointerTy()) {
9585 RHS = getLosslessPtrToIntExpr(Op: RHS);
9586 if (isa<SCEVCouldNotCompute>(Val: RHS))
9587 return RHS;
9588 }
9589 ExitLimit EL = howFarToNonZero(V: getMinusSCEV(LHS, RHS), L);
9590 if (EL.hasAnyInfo()) return EL;
9591 break;
9592 }
9593 case ICmpInst::ICMP_SLE:
9594 case ICmpInst::ICMP_ULE:
9595 // Since the loop is finite, an invariant RHS cannot include the boundary
9596 // value, otherwise it would loop forever.
9597 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9598 !isLoopInvariant(S: RHS, L)) {
9599 // Otherwise, perform the addition in a wider type, to avoid overflow.
9600 // If the LHS is an addrec with the appropriate nowrap flag, the
9601 // extension will be sunk into it and the exit count can be analyzed.
9602 auto *OldType = dyn_cast<IntegerType>(Val: LHS->getType());
9603 if (!OldType)
9604 break;
9605 // Prefer doubling the bitwidth over adding a single bit to make it more
9606 // likely that we use a legal type.
9607 auto *NewType =
9608 Type::getIntNTy(C&: OldType->getContext(), N: OldType->getBitWidth() * 2);
9609 if (ICmpInst::isSigned(Pred)) {
9610 LHS = getSignExtendExpr(Op: LHS, Ty: NewType);
9611 RHS = getSignExtendExpr(Op: RHS, Ty: NewType);
9612 } else {
9613 LHS = getZeroExtendExpr(Op: LHS, Ty: NewType);
9614 RHS = getZeroExtendExpr(Op: RHS, Ty: NewType);
9615 }
9616 }
9617 RHS = getAddExpr(LHS: getOne(Ty: RHS->getType()), RHS);
9618 [[fallthrough]];
9619 case ICmpInst::ICMP_SLT:
9620 case ICmpInst::ICMP_ULT: { // while (X < Y)
9621 bool IsSigned = ICmpInst::isSigned(Pred);
9622 ExitLimit EL = howManyLessThans(LHS, RHS, L, isSigned: IsSigned, ControlsOnlyExit,
9623 AllowPredicates);
9624 if (EL.hasAnyInfo())
9625 return EL;
9626 break;
9627 }
9628 case ICmpInst::ICMP_SGE:
9629 case ICmpInst::ICMP_UGE:
9630 // Since the loop is finite, an invariant RHS cannot include the boundary
9631 // value, otherwise it would loop forever.
9632 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9633 !isLoopInvariant(S: RHS, L))
9634 break;
9635 RHS = getAddExpr(LHS: getMinusOne(Ty: RHS->getType()), RHS);
9636 [[fallthrough]];
9637 case ICmpInst::ICMP_SGT:
9638 case ICmpInst::ICMP_UGT: { // while (X > Y)
9639 bool IsSigned = ICmpInst::isSigned(Pred);
9640 ExitLimit EL = howManyGreaterThans(LHS, RHS, L, isSigned: IsSigned, IsSubExpr: ControlsOnlyExit,
9641 AllowPredicates);
9642 if (EL.hasAnyInfo())
9643 return EL;
9644 break;
9645 }
9646 default:
9647 break;
9648 }
9649
9650 return getCouldNotCompute();
9651}
9652
9653ScalarEvolution::ExitLimit
9654ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L,
9655 SwitchInst *Switch,
9656 BasicBlock *ExitingBlock,
9657 bool ControlsOnlyExit) {
9658 assert(!L->contains(ExitingBlock) && "Not an exiting block!");
9659
9660 // Give up if the exit is the default dest of a switch.
9661 if (Switch->getDefaultDest() == ExitingBlock)
9662 return getCouldNotCompute();
9663
9664 assert(L->contains(Switch->getDefaultDest()) &&
9665 "Default case must not exit the loop!");
9666 const SCEV *LHS = getSCEVAtScope(V: Switch->getCondition(), L);
9667 const SCEV *RHS = getConstant(V: Switch->findCaseDest(BB: ExitingBlock));
9668
9669 // while (X != Y) --> while (X-Y != 0)
9670 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit);
9671 if (EL.hasAnyInfo())
9672 return EL;
9673
9674 return getCouldNotCompute();
9675}
9676
9677static ConstantInt *
9678EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
9679 ScalarEvolution &SE) {
9680 const SCEV *InVal = SE.getConstant(V: C);
9681 const SCEV *Val = AddRec->evaluateAtIteration(It: InVal, SE);
9682 assert(isa<SCEVConstant>(Val) &&
9683 "Evaluation of SCEV at constant didn't fold correctly?");
9684 return cast<SCEVConstant>(Val)->getValue();
9685}
9686
9687ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
9688 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) {
9689 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: RHSV);
9690 if (!RHS)
9691 return getCouldNotCompute();
9692
9693 const BasicBlock *Latch = L->getLoopLatch();
9694 if (!Latch)
9695 return getCouldNotCompute();
9696
9697 const BasicBlock *Predecessor = L->getLoopPredecessor();
9698 if (!Predecessor)
9699 return getCouldNotCompute();
9700
9701 // Return true if V is of the form "LHS `shift_op` <positive constant>".
9702 // Return LHS in OutLHS, shift_op in OutOpCode, and the shift amount in
9703 // OutShiftAmt.
9704 auto MatchPositiveShift = [](Value *V, Value *&OutLHS,
9705 Instruction::BinaryOps &OutOpCode,
9706 unsigned &OutShiftAmt) {
9707 using namespace PatternMatch;
9708
9709 ConstantInt *ShiftAmt;
9710 if (match(V, P: m_LShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9711 OutOpCode = Instruction::LShr;
9712 else if (match(V, P: m_AShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9713 OutOpCode = Instruction::AShr;
9714 else if (match(V, P: m_Shl(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9715 OutOpCode = Instruction::Shl;
9716 else
9717 return false;
9718
9719 uint64_t Amt = ShiftAmt->getValue().getLimitedValue();
9720 if (Amt == 0 || Amt >= OutLHS->getType()->getScalarSizeInBits())
9721 return false;
9722 OutShiftAmt = Amt;
9723 return true;
9724 };
9725
9726 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in
9727 //
9728 // loop:
9729 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ]
9730 // %iv.shifted = lshr i32 %iv, <positive constant>
9731 //
9732 // Return true on a successful match. Return the corresponding PHI node (%iv
9733 // above) in PNOut, the opcode of the shift operation in OpCodeOut, and the
9734 // shift amount in ShiftAmtOut.
9735 auto MatchShiftRecurrence = [&](Value *V, PHINode *&PNOut,
9736 Instruction::BinaryOps &OpCodeOut,
9737 unsigned &ShiftAmtOut) {
9738 std::optional<Instruction::BinaryOps> PostShiftOpCode;
9739
9740 {
9741 Instruction::BinaryOps OpC;
9742 Value *V;
9743 unsigned Amt;
9744
9745 // If we encounter a shift instruction, "peel off" the shift operation,
9746 // and remember that we did so. Later when we inspect %iv's backedge
9747 // value, we will make sure that the backedge value uses the same
9748 // operation.
9749 //
9750 // Note: the peeled shift operation does not have to be the same
9751 // instruction as the one feeding into the PHI's backedge value. We only
9752 // really care about it being the same *kind* of shift instruction --
9753 // that's all that is required for our later inferences to hold.
9754 if (MatchPositiveShift(LHS, V, OpC, Amt)) {
9755 PostShiftOpCode = OpC;
9756 LHS = V;
9757 }
9758 }
9759
9760 PNOut = dyn_cast<PHINode>(Val: LHS);
9761 if (!PNOut || PNOut->getParent() != L->getHeader())
9762 return false;
9763
9764 Value *BEValue = PNOut->getIncomingValueForBlock(BB: Latch);
9765 Value *OpLHS;
9766
9767 return
9768 // The backedge value for the PHI node must be a shift by a positive
9769 // amount
9770 MatchPositiveShift(BEValue, OpLHS, OpCodeOut, ShiftAmtOut) &&
9771
9772 // of the PHI node itself
9773 OpLHS == PNOut &&
9774
9775 // and the kind of shift should be match the kind of shift we peeled
9776 // off, if any.
9777 (!PostShiftOpCode || *PostShiftOpCode == OpCodeOut);
9778 };
9779
9780 PHINode *PN;
9781 Instruction::BinaryOps OpCode;
9782 unsigned ShiftAmt;
9783 if (!MatchShiftRecurrence(LHS, PN, OpCode, ShiftAmt))
9784 return getCouldNotCompute();
9785
9786 const DataLayout &DL = getDataLayout();
9787
9788 // The key rationale for this optimization is that for some kinds of shift
9789 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1
9790 // within a finite number of iterations. If the condition guarding the
9791 // backedge (in the sense that the backedge is taken if the condition is true)
9792 // is false for the value the shift recurrence stabilizes to, then we know
9793 // that the backedge is taken only a finite number of times.
9794
9795 ConstantInt *StableValue = nullptr;
9796 switch (OpCode) {
9797 default:
9798 llvm_unreachable("Impossible case!");
9799
9800 case Instruction::AShr: {
9801 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most
9802 // bitwidth(K) iterations.
9803 Value *FirstValue = PN->getIncomingValueForBlock(BB: Predecessor);
9804 KnownBits Known = computeKnownBits(V: FirstValue, DL, AC: &AC,
9805 CxtI: Predecessor->getTerminator(), DT: &DT);
9806 auto *Ty = cast<IntegerType>(Val: RHS->getType());
9807 if (Known.isNonNegative())
9808 StableValue = ConstantInt::get(Ty, V: 0);
9809 else if (Known.isNegative())
9810 StableValue = ConstantInt::get(Ty, V: -1, IsSigned: true);
9811 else
9812 return getCouldNotCompute();
9813
9814 break;
9815 }
9816 case Instruction::LShr:
9817 case Instruction::Shl:
9818 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>}
9819 // stabilize to 0 in at most bitwidth(K) iterations.
9820 StableValue = ConstantInt::get(Ty: cast<IntegerType>(Val: RHS->getType()), V: 0);
9821 break;
9822 }
9823
9824 auto *Result =
9825 ConstantFoldCompareInstOperands(Predicate: Pred, LHS: StableValue, RHS, DL, TLI: &TLI);
9826 assert(Result->getType()->isIntegerTy(1) &&
9827 "Otherwise cannot be an operand to a branch instruction");
9828
9829 if (Result->isNullValue()) {
9830 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
9831 unsigned MaxBTC = BitWidth;
9832
9833 // For right-shift recurrences (lshr/ashr with non-negative start), we can
9834 // compute a tighter max backedge-taken count from the range of the start
9835 // value. After k shifts of ShiftAmt, value = start >> (k * ShiftAmt).
9836 // The value reaches 0 (the stable value) when k * ShiftAmt >=
9837 // activeBits(start), so max BTC = ceil(activeBits(maxStart) / ShiftAmt).
9838 if (OpCode == Instruction::LShr || OpCode == Instruction::AShr) {
9839 Value *StartValue = PN->getIncomingValueForBlock(BB: Predecessor);
9840 const SCEV *StartSCEV = getSCEV(V: StartValue);
9841 APInt MaxStart = getUnsignedRangeMax(S: StartSCEV);
9842 if (MaxStart.isStrictlyPositive()) {
9843 unsigned ActiveBits = MaxStart.getActiveBits();
9844 unsigned RangeBTC = divideCeil(Numerator: ActiveBits, Denominator: ShiftAmt);
9845 MaxBTC = std::min(a: MaxBTC, b: RangeBTC);
9846 }
9847 }
9848
9849 const SCEV *UpperBound =
9850 getConstant(Ty: getEffectiveSCEVType(Ty: RHS->getType()), V: MaxBTC);
9851 return ExitLimit(getCouldNotCompute(), UpperBound, UpperBound, false);
9852 }
9853
9854 return getCouldNotCompute();
9855}
9856
9857/// Return true if we can constant fold an instruction of the specified type,
9858/// assuming that all operands were constants.
9859static bool CanConstantFold(const Instruction *I) {
9860 if (isa<BinaryOperator>(Val: I) || isa<CmpInst>(Val: I) ||
9861 isa<SelectInst>(Val: I) || isa<CastInst>(Val: I) || isa<GetElementPtrInst>(Val: I) ||
9862 isa<LoadInst>(Val: I) || isa<ExtractValueInst>(Val: I))
9863 return true;
9864
9865 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
9866 if (const Function *F = CI->getCalledFunction())
9867 return canConstantFoldCallTo(Call: CI, F);
9868 return false;
9869}
9870
9871/// Determine whether this instruction can constant evolve within this loop
9872/// assuming its operands can all constant evolve.
9873static bool canConstantEvolve(Instruction *I, const Loop *L) {
9874 // An instruction outside of the loop can't be derived from a loop PHI.
9875 if (!L->contains(Inst: I)) return false;
9876
9877 if (isa<PHINode>(Val: I)) {
9878 // We don't currently keep track of the control flow needed to evaluate
9879 // PHIs, so we cannot handle PHIs inside of loops.
9880 return L->getHeader() == I->getParent();
9881 }
9882
9883 // If we won't be able to constant fold this expression even if the operands
9884 // are constants, bail early.
9885 return CanConstantFold(I);
9886}
9887
9888/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
9889/// recursing through each instruction operand until reaching a loop header phi.
9890static PHINode *
9891getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L,
9892 DenseMap<Instruction *, PHINode *> &PHIMap,
9893 unsigned Depth) {
9894 if (Depth > MaxConstantEvolvingDepth)
9895 return nullptr;
9896
9897 // Otherwise, we can evaluate this instruction if all of its operands are
9898 // constant or derived from a PHI node themselves.
9899 PHINode *PHI = nullptr;
9900 for (Value *Op : UseInst->operands()) {
9901 if (isa<Constant>(Val: Op)) continue;
9902
9903 Instruction *OpInst = dyn_cast<Instruction>(Val: Op);
9904 if (!OpInst || !canConstantEvolve(I: OpInst, L)) return nullptr;
9905
9906 PHINode *P = dyn_cast<PHINode>(Val: OpInst);
9907 if (!P)
9908 // If this operand is already visited, reuse the prior result.
9909 // We may have P != PHI if this is the deepest point at which the
9910 // inconsistent paths meet.
9911 P = PHIMap.lookup(Val: OpInst);
9912 if (!P) {
9913 // Recurse and memoize the results, whether a phi is found or not.
9914 // This recursive call invalidates pointers into PHIMap.
9915 P = getConstantEvolvingPHIOperands(UseInst: OpInst, L, PHIMap, Depth: Depth + 1);
9916 PHIMap[OpInst] = P;
9917 }
9918 if (!P)
9919 return nullptr; // Not evolving from PHI
9920 if (PHI && PHI != P)
9921 return nullptr; // Evolving from multiple different PHIs.
9922 PHI = P;
9923 }
9924 // This is a expression evolving from a constant PHI!
9925 return PHI;
9926}
9927
9928/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
9929/// in the loop that V is derived from. We allow arbitrary operations along the
9930/// way, but the operands of an operation must either be constants or a value
9931/// derived from a constant PHI. If this expression does not fit with these
9932/// constraints, return null.
9933static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) {
9934 Instruction *I = dyn_cast<Instruction>(Val: V);
9935 if (!I || !canConstantEvolve(I, L)) return nullptr;
9936
9937 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
9938 return PN;
9939
9940 // Record non-constant instructions contained by the loop.
9941 DenseMap<Instruction *, PHINode *> PHIMap;
9942 return getConstantEvolvingPHIOperands(UseInst: I, L, PHIMap, Depth: 0);
9943}
9944
9945/// EvaluateExpression - Given an expression that passes the
9946/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
9947/// in the loop has the value PHIVal. If we can't fold this expression for some
9948/// reason, return null.
9949static Constant *EvaluateExpression(Value *V, const Loop *L,
9950 DenseMap<Instruction *, Constant *> &Vals,
9951 const DataLayout &DL,
9952 const TargetLibraryInfo *TLI) {
9953 // Convenient constant check, but redundant for recursive calls.
9954 if (Constant *C = dyn_cast<Constant>(Val: V)) return C;
9955 Instruction *I = dyn_cast<Instruction>(Val: V);
9956 if (!I) return nullptr;
9957
9958 if (Constant *C = Vals.lookup(Val: I)) return C;
9959
9960 // An instruction inside the loop depends on a value outside the loop that we
9961 // weren't given a mapping for, or a value such as a call inside the loop.
9962 if (!canConstantEvolve(I, L)) return nullptr;
9963
9964 // An unmapped PHI can be due to a branch or another loop inside this loop,
9965 // or due to this not being the initial iteration through a loop where we
9966 // couldn't compute the evolution of this particular PHI last time.
9967 if (isa<PHINode>(Val: I)) return nullptr;
9968
9969 std::vector<Constant*> Operands(I->getNumOperands());
9970
9971 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
9972 Instruction *Operand = dyn_cast<Instruction>(Val: I->getOperand(i));
9973 if (!Operand) {
9974 Operands[i] = dyn_cast<Constant>(Val: I->getOperand(i));
9975 if (!Operands[i]) return nullptr;
9976 continue;
9977 }
9978 Constant *C = EvaluateExpression(V: Operand, L, Vals, DL, TLI);
9979 Vals[Operand] = C;
9980 if (!C) return nullptr;
9981 Operands[i] = C;
9982 }
9983
9984 return ConstantFoldInstOperands(I, Ops: Operands, DL, TLI,
9985 /*AllowNonDeterministic=*/false);
9986}
9987
9988
9989// If every incoming value to PN except the one for BB is a specific Constant,
9990// return that, else return nullptr.
9991static Constant *getOtherIncomingValue(PHINode *PN, BasicBlock *BB) {
9992 Constant *IncomingVal = nullptr;
9993
9994 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
9995 if (PN->getIncomingBlock(i) == BB)
9996 continue;
9997
9998 auto *CurrentVal = dyn_cast<Constant>(Val: PN->getIncomingValue(i));
9999 if (!CurrentVal)
10000 return nullptr;
10001
10002 if (IncomingVal != CurrentVal) {
10003 if (IncomingVal)
10004 return nullptr;
10005 IncomingVal = CurrentVal;
10006 }
10007 }
10008
10009 return IncomingVal;
10010}
10011
10012/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
10013/// in the header of its containing loop, we know the loop executes a
10014/// constant number of times, and the PHI node is just a recurrence
10015/// involving constants, fold it.
10016Constant *
10017ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
10018 const APInt &BEs,
10019 const Loop *L) {
10020 auto [I, Inserted] = ConstantEvolutionLoopExitValue.try_emplace(Key: PN);
10021 if (!Inserted)
10022 return I->second;
10023
10024 if (BEs.ugt(RHS: MaxBruteForceIterations))
10025 return nullptr; // Not going to evaluate it.
10026
10027 Constant *&RetVal = I->second;
10028
10029 DenseMap<Instruction *, Constant *> CurrentIterVals;
10030 BasicBlock *Header = L->getHeader();
10031 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
10032
10033 BasicBlock *Latch = L->getLoopLatch();
10034 if (!Latch)
10035 return nullptr;
10036
10037 for (PHINode &PHI : Header->phis()) {
10038 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
10039 CurrentIterVals[&PHI] = StartCST;
10040 }
10041 if (!CurrentIterVals.count(Val: PN))
10042 return RetVal = nullptr;
10043
10044 Value *BEValue = PN->getIncomingValueForBlock(BB: Latch);
10045
10046 // Execute the loop symbolically to determine the exit value.
10047 assert(BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) &&
10048 "BEs is <= MaxBruteForceIterations which is an 'unsigned'!");
10049
10050 unsigned NumIterations = BEs.getZExtValue(); // must be in range
10051 unsigned IterationNum = 0;
10052 const DataLayout &DL = getDataLayout();
10053 for (; ; ++IterationNum) {
10054 if (IterationNum == NumIterations)
10055 return RetVal = CurrentIterVals[PN]; // Got exit value!
10056
10057 // Compute the value of the PHIs for the next iteration.
10058 // EvaluateExpression adds non-phi values to the CurrentIterVals map.
10059 DenseMap<Instruction *, Constant *> NextIterVals;
10060 Constant *NextPHI =
10061 EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
10062 if (!NextPHI)
10063 return nullptr; // Couldn't evaluate!
10064 NextIterVals[PN] = NextPHI;
10065
10066 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
10067
10068 // Also evaluate the other PHI nodes. However, we don't get to stop if we
10069 // cease to be able to evaluate one of them or if they stop evolving,
10070 // because that doesn't necessarily prevent us from computing PN.
10071 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute;
10072 for (const auto &I : CurrentIterVals) {
10073 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
10074 if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
10075 PHIsToCompute.emplace_back(Args&: PHI, Args: I.second);
10076 }
10077 // We use two distinct loops because EvaluateExpression may invalidate any
10078 // iterators into CurrentIterVals.
10079 for (const auto &I : PHIsToCompute) {
10080 PHINode *PHI = I.first;
10081 Constant *&NextPHI = NextIterVals[PHI];
10082 if (!NextPHI) { // Not already computed.
10083 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
10084 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
10085 }
10086 if (NextPHI != I.second)
10087 StoppedEvolving = false;
10088 }
10089
10090 // If all entries in CurrentIterVals == NextIterVals then we can stop
10091 // iterating, the loop can't continue to change.
10092 if (StoppedEvolving)
10093 return RetVal = CurrentIterVals[PN];
10094
10095 CurrentIterVals.swap(RHS&: NextIterVals);
10096 }
10097}
10098
10099const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L,
10100 Value *Cond,
10101 bool ExitWhen) {
10102 PHINode *PN = getConstantEvolvingPHI(V: Cond, L);
10103 if (!PN) return getCouldNotCompute();
10104
10105 // If the loop is canonicalized, the PHI will have exactly two entries.
10106 // That's the only form we support here.
10107 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
10108
10109 DenseMap<Instruction *, Constant *> CurrentIterVals;
10110 BasicBlock *Header = L->getHeader();
10111 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
10112
10113 BasicBlock *Latch = L->getLoopLatch();
10114 assert(Latch && "Should follow from NumIncomingValues == 2!");
10115
10116 for (PHINode &PHI : Header->phis()) {
10117 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
10118 CurrentIterVals[&PHI] = StartCST;
10119 }
10120 if (!CurrentIterVals.count(Val: PN))
10121 return getCouldNotCompute();
10122
10123 // Okay, we find a PHI node that defines the trip count of this loop. Execute
10124 // the loop symbolically to determine when the condition gets a value of
10125 // "ExitWhen".
10126 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis.
10127 const DataLayout &DL = getDataLayout();
10128 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
10129 auto *CondVal = dyn_cast_or_null<ConstantInt>(
10130 Val: EvaluateExpression(V: Cond, L, Vals&: CurrentIterVals, DL, TLI: &TLI));
10131
10132 // Couldn't symbolically evaluate.
10133 if (!CondVal) return getCouldNotCompute();
10134
10135 if (CondVal->getValue() == uint64_t(ExitWhen)) {
10136 ++NumBruteForceTripCountsComputed;
10137 return getConstant(Ty: Type::getInt32Ty(C&: getContext()), V: IterationNum);
10138 }
10139
10140 // Update all the PHI nodes for the next iteration.
10141 DenseMap<Instruction *, Constant *> NextIterVals;
10142
10143 // Create a list of which PHIs we need to compute. We want to do this before
10144 // calling EvaluateExpression on them because that may invalidate iterators
10145 // into CurrentIterVals.
10146 SmallVector<PHINode *, 8> PHIsToCompute;
10147 for (const auto &I : CurrentIterVals) {
10148 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
10149 if (!PHI || PHI->getParent() != Header) continue;
10150 PHIsToCompute.push_back(Elt: PHI);
10151 }
10152 for (PHINode *PHI : PHIsToCompute) {
10153 Constant *&NextPHI = NextIterVals[PHI];
10154 if (NextPHI) continue; // Already computed!
10155
10156 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
10157 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
10158 }
10159 CurrentIterVals.swap(RHS&: NextIterVals);
10160 }
10161
10162 // Too many iterations were needed to evaluate.
10163 return getCouldNotCompute();
10164}
10165
10166const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
10167 SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values =
10168 ValuesAtScopes[V];
10169 // Check to see if we've folded this expression at this loop before.
10170 for (auto &LS : Values)
10171 if (LS.first == L)
10172 return LS.second ? LS.second : V;
10173
10174 Values.emplace_back(Args&: L, Args: nullptr);
10175
10176 // Otherwise compute it.
10177 const SCEV *C = computeSCEVAtScope(S: V, L);
10178 for (auto &LS : reverse(C&: ValuesAtScopes[V]))
10179 if (LS.first == L) {
10180 LS.second = C;
10181 if (!isa<SCEVConstant>(Val: C))
10182 ValuesAtScopesUsers[C].push_back(Elt: {L, V});
10183 break;
10184 }
10185 return C;
10186}
10187
10188/// This builds up a Constant using the ConstantExpr interface. That way, we
10189/// will return Constants for objects which aren't represented by a
10190/// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
10191/// Returns NULL if the SCEV isn't representable as a Constant.
10192static Constant *BuildConstantFromSCEV(const SCEV *V) {
10193 switch (V->getSCEVType()) {
10194 case scCouldNotCompute:
10195 case scAddRecExpr:
10196 case scVScale:
10197 return nullptr;
10198 case scConstant:
10199 return cast<SCEVConstant>(Val: V)->getValue();
10200 case scUnknown:
10201 return dyn_cast<Constant>(Val: cast<SCEVUnknown>(Val: V)->getValue());
10202 case scPtrToAddr: {
10203 const SCEVPtrToAddrExpr *P2I = cast<SCEVPtrToAddrExpr>(Val: V);
10204 if (Constant *CastOp = BuildConstantFromSCEV(V: P2I->getOperand()))
10205 return ConstantExpr::getPtrToAddr(C: CastOp, Ty: P2I->getType());
10206
10207 return nullptr;
10208 }
10209 case scPtrToInt: {
10210 const SCEVPtrToIntExpr *P2I = cast<SCEVPtrToIntExpr>(Val: V);
10211 if (Constant *CastOp = BuildConstantFromSCEV(V: P2I->getOperand()))
10212 return ConstantExpr::getPtrToInt(C: CastOp, Ty: P2I->getType());
10213
10214 return nullptr;
10215 }
10216 case scTruncate: {
10217 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(Val: V);
10218 if (Constant *CastOp = BuildConstantFromSCEV(V: ST->getOperand()))
10219 return ConstantExpr::getTrunc(C: CastOp, Ty: ST->getType());
10220 return nullptr;
10221 }
10222 case scAddExpr: {
10223 const SCEVAddExpr *SA = cast<SCEVAddExpr>(Val: V);
10224 Constant *C = nullptr;
10225 for (const SCEV *Op : SA->operands()) {
10226 Constant *OpC = BuildConstantFromSCEV(V: Op);
10227 if (!OpC)
10228 return nullptr;
10229 if (!C) {
10230 C = OpC;
10231 continue;
10232 }
10233 assert(!C->getType()->isPointerTy() &&
10234 "Can only have one pointer, and it must be last");
10235 if (OpC->getType()->isPointerTy()) {
10236 // The offsets have been converted to bytes. We can add bytes using
10237 // an i8 GEP.
10238 C = ConstantExpr::getPtrAdd(Ptr: OpC, Offset: C);
10239 } else {
10240 C = ConstantExpr::getAdd(C1: C, C2: OpC);
10241 }
10242 }
10243 return C;
10244 }
10245 case scMulExpr:
10246 case scSignExtend:
10247 case scZeroExtend:
10248 case scUDivExpr:
10249 case scSMaxExpr:
10250 case scUMaxExpr:
10251 case scSMinExpr:
10252 case scUMinExpr:
10253 case scSequentialUMinExpr:
10254 return nullptr;
10255 }
10256 llvm_unreachable("Unknown SCEV kind!");
10257}
10258
10259const SCEV *ScalarEvolution::getWithOperands(const SCEV *S,
10260 SmallVectorImpl<SCEVUse> &NewOps) {
10261 switch (S->getSCEVType()) {
10262 case scTruncate:
10263 case scZeroExtend:
10264 case scSignExtend:
10265 case scPtrToAddr:
10266 case scPtrToInt:
10267 return getCastExpr(Kind: S->getSCEVType(), Op: NewOps[0], Ty: S->getType());
10268 case scAddRecExpr: {
10269 auto *AddRec = cast<SCEVAddRecExpr>(Val: S);
10270 return getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), Flags: AddRec->getNoWrapFlags());
10271 }
10272 case scAddExpr:
10273 return getAddExpr(Ops&: NewOps, OrigFlags: cast<SCEVAddExpr>(Val: S)->getNoWrapFlags());
10274 case scMulExpr:
10275 return getMulExpr(Ops&: NewOps, OrigFlags: cast<SCEVMulExpr>(Val: S)->getNoWrapFlags());
10276 case scUDivExpr:
10277 return getUDivExpr(LHS: NewOps[0], RHS: NewOps[1]);
10278 case scUMaxExpr:
10279 case scSMaxExpr:
10280 case scUMinExpr:
10281 case scSMinExpr:
10282 return getMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10283 case scSequentialUMinExpr:
10284 return getSequentialMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10285 case scConstant:
10286 case scVScale:
10287 case scUnknown:
10288 return S;
10289 case scCouldNotCompute:
10290 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10291 }
10292 llvm_unreachable("Unknown SCEV kind!");
10293}
10294
10295const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
10296 switch (V->getSCEVType()) {
10297 case scConstant:
10298 case scVScale:
10299 return V;
10300 case scAddRecExpr: {
10301 // If this is a loop recurrence for a loop that does not contain L, then we
10302 // are dealing with the final value computed by the loop.
10303 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: V);
10304 // First, attempt to evaluate each operand.
10305 // Avoid performing the look-up in the common case where the specified
10306 // expression has no loop-variant portions.
10307 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
10308 const SCEV *OpAtScope = getSCEVAtScope(V: AddRec->getOperand(i), L);
10309 if (OpAtScope == AddRec->getOperand(i))
10310 continue;
10311
10312 // Okay, at least one of these operands is loop variant but might be
10313 // foldable. Build a new instance of the folded commutative expression.
10314 SmallVector<SCEVUse, 8> NewOps;
10315 NewOps.reserve(N: AddRec->getNumOperands());
10316 append_range(C&: NewOps, R: AddRec->operands().take_front(N: i));
10317 NewOps.push_back(Elt: OpAtScope);
10318 for (++i; i != e; ++i)
10319 NewOps.push_back(Elt: getSCEVAtScope(V: AddRec->getOperand(i), L));
10320
10321 const SCEV *FoldedRec = getAddRecExpr(
10322 Operands&: NewOps, L: AddRec->getLoop(), Flags: AddRec->getNoWrapFlags(Mask: SCEV::FlagNW));
10323 AddRec = dyn_cast<SCEVAddRecExpr>(Val: FoldedRec);
10324 // The addrec may be folded to a nonrecurrence, for example, if the
10325 // induction variable is multiplied by zero after constant folding. Go
10326 // ahead and return the folded value.
10327 if (!AddRec)
10328 return FoldedRec;
10329 break;
10330 }
10331
10332 // If the scope is outside the addrec's loop, evaluate it by using the
10333 // loop exit value of the addrec.
10334 if (!AddRec->getLoop()->contains(L)) {
10335 // To evaluate this recurrence, we need to know how many times the AddRec
10336 // loop iterates. Compute this now.
10337 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(L: AddRec->getLoop());
10338 if (BackedgeTakenCount == getCouldNotCompute())
10339 return AddRec;
10340
10341 // Then, evaluate the AddRec.
10342 return AddRec->evaluateAtIteration(It: BackedgeTakenCount, SE&: *this);
10343 }
10344
10345 return AddRec;
10346 }
10347 case scTruncate:
10348 case scZeroExtend:
10349 case scSignExtend:
10350 case scPtrToAddr:
10351 case scPtrToInt:
10352 case scAddExpr:
10353 case scMulExpr:
10354 case scUDivExpr:
10355 case scUMaxExpr:
10356 case scSMaxExpr:
10357 case scUMinExpr:
10358 case scSMinExpr:
10359 case scSequentialUMinExpr: {
10360 ArrayRef<SCEVUse> Ops = V->operands();
10361 // Avoid performing the look-up in the common case where the specified
10362 // expression has no loop-variant portions.
10363 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
10364 const SCEV *OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10365 if (OpAtScope != Ops[i].getPointer()) {
10366 // Okay, at least one of these operands is loop variant but might be
10367 // foldable. Build a new instance of the folded commutative expression.
10368 SmallVector<SCEVUse, 8> NewOps;
10369 NewOps.reserve(N: Ops.size());
10370 append_range(C&: NewOps, R: Ops.take_front(N: i));
10371 NewOps.push_back(Elt: OpAtScope);
10372
10373 for (++i; i != e; ++i) {
10374 OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10375 NewOps.push_back(Elt: OpAtScope);
10376 }
10377
10378 return getWithOperands(S: V, NewOps);
10379 }
10380 }
10381 // If we got here, all operands are loop invariant.
10382 return V;
10383 }
10384 case scUnknown: {
10385 // If this instruction is evolved from a constant-evolving PHI, compute the
10386 // exit value from the loop without using SCEVs.
10387 const SCEVUnknown *SU = cast<SCEVUnknown>(Val: V);
10388 Instruction *I = dyn_cast<Instruction>(Val: SU->getValue());
10389 if (!I)
10390 return V; // This is some other type of SCEVUnknown, just return it.
10391
10392 if (PHINode *PN = dyn_cast<PHINode>(Val: I)) {
10393 const Loop *CurrLoop = this->LI[I->getParent()];
10394 // Looking for loop exit value.
10395 if (CurrLoop && CurrLoop->getParentLoop() == L &&
10396 PN->getParent() == CurrLoop->getHeader()) {
10397 // Okay, there is no closed form solution for the PHI node. Check
10398 // to see if the loop that contains it has a known backedge-taken
10399 // count. If so, we may be able to force computation of the exit
10400 // value.
10401 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(L: CurrLoop);
10402 // This trivial case can show up in some degenerate cases where
10403 // the incoming IR has not yet been fully simplified.
10404 if (BackedgeTakenCount->isZero()) {
10405 Value *InitValue = nullptr;
10406 bool MultipleInitValues = false;
10407 for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
10408 if (!CurrLoop->contains(BB: PN->getIncomingBlock(i))) {
10409 if (!InitValue)
10410 InitValue = PN->getIncomingValue(i);
10411 else if (InitValue != PN->getIncomingValue(i)) {
10412 MultipleInitValues = true;
10413 break;
10414 }
10415 }
10416 }
10417 if (!MultipleInitValues && InitValue)
10418 return getSCEV(V: InitValue);
10419 }
10420 // Do we have a loop invariant value flowing around the backedge
10421 // for a loop which must execute the backedge?
10422 if (!isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount) &&
10423 isKnownNonZero(S: BackedgeTakenCount) &&
10424 PN->getNumIncomingValues() == 2) {
10425
10426 unsigned InLoopPred =
10427 CurrLoop->contains(BB: PN->getIncomingBlock(i: 0)) ? 0 : 1;
10428 Value *BackedgeVal = PN->getIncomingValue(i: InLoopPred);
10429 if (CurrLoop->isLoopInvariant(V: BackedgeVal))
10430 return getSCEV(V: BackedgeVal);
10431 }
10432 if (auto *BTCC = dyn_cast<SCEVConstant>(Val: BackedgeTakenCount)) {
10433 // Okay, we know how many times the containing loop executes. If
10434 // this is a constant evolving PHI node, get the final value at
10435 // the specified iteration number.
10436 Constant *RV =
10437 getConstantEvolutionLoopExitValue(PN, BEs: BTCC->getAPInt(), L: CurrLoop);
10438 if (RV)
10439 return getSCEV(V: RV);
10440 }
10441 }
10442 }
10443
10444 // Okay, this is an expression that we cannot symbolically evaluate
10445 // into a SCEV. Check to see if it's possible to symbolically evaluate
10446 // the arguments into constants, and if so, try to constant propagate the
10447 // result. This is particularly useful for computing loop exit values.
10448 if (!CanConstantFold(I))
10449 return V; // This is some other type of SCEVUnknown, just return it.
10450
10451 SmallVector<Constant *, 4> Operands;
10452 Operands.reserve(N: I->getNumOperands());
10453 bool MadeImprovement = false;
10454 for (Value *Op : I->operands()) {
10455 if (Constant *C = dyn_cast<Constant>(Val: Op)) {
10456 Operands.push_back(Elt: C);
10457 continue;
10458 }
10459
10460 // If any of the operands is non-constant and if they are
10461 // non-integer and non-pointer, don't even try to analyze them
10462 // with scev techniques.
10463 if (!isSCEVable(Ty: Op->getType()))
10464 return V;
10465
10466 const SCEV *OrigV = getSCEV(V: Op);
10467 const SCEV *OpV = getSCEVAtScope(V: OrigV, L);
10468 MadeImprovement |= OrigV != OpV;
10469
10470 Constant *C = BuildConstantFromSCEV(V: OpV);
10471 if (!C)
10472 return V;
10473 assert(C->getType() == Op->getType() && "Type mismatch");
10474 Operands.push_back(Elt: C);
10475 }
10476
10477 // Check to see if getSCEVAtScope actually made an improvement.
10478 if (!MadeImprovement)
10479 return V; // This is some other type of SCEVUnknown, just return it.
10480
10481 Constant *C = nullptr;
10482 const DataLayout &DL = getDataLayout();
10483 C = ConstantFoldInstOperands(I, Ops: Operands, DL, TLI: &TLI,
10484 /*AllowNonDeterministic=*/false);
10485 if (!C)
10486 return V;
10487 return getSCEV(V: C);
10488 }
10489 case scCouldNotCompute:
10490 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10491 }
10492 llvm_unreachable("Unknown SCEV type!");
10493}
10494
10495const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
10496 return getSCEVAtScope(V: getSCEV(V), L);
10497}
10498
10499const SCEV *ScalarEvolution::stripInjectiveFunctions(const SCEV *S) const {
10500 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: S))
10501 return stripInjectiveFunctions(S: ZExt->getOperand());
10502 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
10503 return stripInjectiveFunctions(S: SExt->getOperand());
10504 return S;
10505}
10506
10507/// Finds the minimum unsigned root of the following equation:
10508///
10509/// A * X = B (mod N)
10510///
10511/// where N = 2^BW and BW is the common bit width of A and B. The signedness of
10512/// A and B isn't important.
10513///
10514/// If the equation does not have a solution, SCEVCouldNotCompute is returned.
10515static const SCEV *
10516SolveLinEquationWithOverflow(const APInt &A, const SCEV *B,
10517 SmallVectorImpl<const SCEVPredicate *> *Predicates,
10518 ScalarEvolution &SE, const Loop *L) {
10519 uint32_t BW = A.getBitWidth();
10520 assert(BW == SE.getTypeSizeInBits(B->getType()));
10521 assert(A != 0 && "A must be non-zero.");
10522
10523 // 1. D = gcd(A, N)
10524 //
10525 // The gcd of A and N may have only one prime factor: 2. The number of
10526 // trailing zeros in A is its multiplicity
10527 uint32_t Mult2 = A.countr_zero();
10528 // D = 2^Mult2
10529
10530 // 2. Check if B is divisible by D.
10531 //
10532 // B is divisible by D if and only if the multiplicity of prime factor 2 for B
10533 // is not less than multiplicity of this prime factor for D.
10534 unsigned MinTZ = SE.getMinTrailingZeros(S: B);
10535 // Try again with the terminator of the loop predecessor for context-specific
10536 // result, if MinTZ s too small.
10537 if (MinTZ < Mult2 && L->getLoopPredecessor())
10538 MinTZ = SE.getMinTrailingZeros(S: B, CtxI: L->getLoopPredecessor()->getTerminator());
10539 if (MinTZ < Mult2) {
10540 // Check if we can prove there's no remainder using URem.
10541 const SCEV *URem =
10542 SE.getURemExpr(LHS: B, RHS: SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2)));
10543 const SCEV *Zero = SE.getZero(Ty: B->getType());
10544 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: URem, RHS: Zero)) {
10545 // Try to add a predicate ensuring B is a multiple of 1 << Mult2.
10546 if (!Predicates)
10547 return SE.getCouldNotCompute();
10548
10549 // Avoid adding a predicate that is known to be false.
10550 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_NE, LHS: URem, RHS: Zero))
10551 return SE.getCouldNotCompute();
10552 Predicates->push_back(Elt: SE.getEqualPredicate(LHS: URem, RHS: Zero));
10553 }
10554 }
10555
10556 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
10557 // modulo (N / D).
10558 //
10559 // If D == 1, (N / D) == N == 2^BW, so we need one extra bit to represent
10560 // (N / D) in general. The inverse itself always fits into BW bits, though,
10561 // so we immediately truncate it.
10562 APInt AD = A.lshr(shiftAmt: Mult2).trunc(width: BW - Mult2); // AD = A / D
10563 APInt I = AD.multiplicativeInverse().zext(width: BW);
10564
10565 // 4. Compute the minimum unsigned root of the equation:
10566 // I * (B / D) mod (N / D)
10567 // To simplify the computation, we factor out the divide by D:
10568 // (I * B mod N) / D
10569 const SCEV *D = SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2));
10570 return SE.getUDivExactExpr(LHS: SE.getMulExpr(LHS: B, RHS: SE.getConstant(Val: I)), RHS: D);
10571}
10572
10573/// For a given quadratic addrec, generate coefficients of the corresponding
10574/// quadratic equation, multiplied by a common value to ensure that they are
10575/// integers.
10576/// The returned value is a tuple { A, B, C, M, BitWidth }, where
10577/// Ax^2 + Bx + C is the quadratic function, M is the value that A, B and C
10578/// were multiplied by, and BitWidth is the bit width of the original addrec
10579/// coefficients.
10580/// This function returns std::nullopt if the addrec coefficients are not
10581/// compile- time constants.
10582static std::optional<std::tuple<APInt, APInt, APInt, APInt, unsigned>>
10583GetQuadraticEquation(const SCEVAddRecExpr *AddRec) {
10584 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
10585 const SCEVConstant *LC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 0));
10586 const SCEVConstant *MC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 1));
10587 const SCEVConstant *NC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 2));
10588 LLVM_DEBUG(dbgs() << __func__ << ": analyzing quadratic addrec: "
10589 << *AddRec << '\n');
10590
10591 // We currently can only solve this if the coefficients are constants.
10592 if (!LC || !MC || !NC) {
10593 LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n");
10594 return std::nullopt;
10595 }
10596
10597 APInt L = LC->getAPInt();
10598 APInt M = MC->getAPInt();
10599 APInt N = NC->getAPInt();
10600 assert(!N.isZero() && "This is not a quadratic addrec");
10601
10602 unsigned BitWidth = LC->getAPInt().getBitWidth();
10603 unsigned NewWidth = BitWidth + 1;
10604 LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: "
10605 << BitWidth << '\n');
10606 // The sign-extension (as opposed to a zero-extension) here matches the
10607 // extension used in SolveQuadraticEquationWrap (with the same motivation).
10608 N = N.sext(width: NewWidth);
10609 M = M.sext(width: NewWidth);
10610 L = L.sext(width: NewWidth);
10611
10612 // The increments are M, M+N, M+2N, ..., so the accumulated values are
10613 // L+M, (L+M)+(M+N), (L+M)+(M+N)+(M+2N), ..., that is,
10614 // L+M, L+2M+N, L+3M+3N, ...
10615 // After n iterations the accumulated value Acc is L + nM + n(n-1)/2 N.
10616 //
10617 // The equation Acc = 0 is then
10618 // L + nM + n(n-1)/2 N = 0, or 2L + 2M n + n(n-1) N = 0.
10619 // In a quadratic form it becomes:
10620 // N n^2 + (2M-N) n + 2L = 0.
10621
10622 APInt A = N;
10623 APInt B = 2 * M - A;
10624 APInt C = 2 * L;
10625 APInt T = APInt(NewWidth, 2);
10626 LLVM_DEBUG(dbgs() << __func__ << ": equation " << A << "x^2 + " << B
10627 << "x + " << C << ", coeff bw: " << NewWidth
10628 << ", multiplied by " << T << '\n');
10629 return std::make_tuple(args&: A, args&: B, args&: C, args&: T, args&: BitWidth);
10630}
10631
10632/// Helper function to compare optional APInts:
10633/// (a) if X and Y both exist, return min(X, Y),
10634/// (b) if neither X nor Y exist, return std::nullopt,
10635/// (c) if exactly one of X and Y exists, return that value.
10636static std::optional<APInt> MinOptional(std::optional<APInt> X,
10637 std::optional<APInt> Y) {
10638 if (X && Y) {
10639 unsigned W = std::max(a: X->getBitWidth(), b: Y->getBitWidth());
10640 APInt XW = X->sext(width: W);
10641 APInt YW = Y->sext(width: W);
10642 return XW.slt(RHS: YW) ? *X : *Y;
10643 }
10644 if (!X && !Y)
10645 return std::nullopt;
10646 return X ? *X : *Y;
10647}
10648
10649/// Helper function to truncate an optional APInt to a given BitWidth.
10650/// When solving addrec-related equations, it is preferable to return a value
10651/// that has the same bit width as the original addrec's coefficients. If the
10652/// solution fits in the original bit width, truncate it (except for i1).
10653/// Returning a value of a different bit width may inhibit some optimizations.
10654///
10655/// In general, a solution to a quadratic equation generated from an addrec
10656/// may require BW+1 bits, where BW is the bit width of the addrec's
10657/// coefficients. The reason is that the coefficients of the quadratic
10658/// equation are BW+1 bits wide (to avoid truncation when converting from
10659/// the addrec to the equation).
10660static std::optional<APInt> TruncIfPossible(std::optional<APInt> X,
10661 unsigned BitWidth) {
10662 if (!X)
10663 return std::nullopt;
10664 unsigned W = X->getBitWidth();
10665 if (BitWidth > 1 && BitWidth < W && X->isIntN(N: BitWidth))
10666 return X->trunc(width: BitWidth);
10667 return X;
10668}
10669
10670/// Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n
10671/// iterations. The values L, M, N are assumed to be signed, and they
10672/// should all have the same bit widths.
10673/// Find the least n >= 0 such that c(n) = 0 in the arithmetic modulo 2^BW,
10674/// where BW is the bit width of the addrec's coefficients.
10675/// If the calculated value is a BW-bit integer (for BW > 1), it will be
10676/// returned as such, otherwise the bit width of the returned value may
10677/// be greater than BW.
10678///
10679/// This function returns std::nullopt if
10680/// (a) the addrec coefficients are not constant, or
10681/// (b) SolveQuadraticEquationWrap was unable to find a solution. For cases
10682/// like x^2 = 5, no integer solutions exist, in other cases an integer
10683/// solution may exist, but SolveQuadraticEquationWrap may fail to find it.
10684static std::optional<APInt>
10685SolveQuadraticAddRecExact(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
10686 APInt A, B, C, M;
10687 unsigned BitWidth;
10688 auto T = GetQuadraticEquation(AddRec);
10689 if (!T)
10690 return std::nullopt;
10691
10692 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10693 LLVM_DEBUG(dbgs() << __func__ << ": solving for unsigned overflow\n");
10694 std::optional<APInt> X =
10695 APIntOps::SolveQuadraticEquationWrap(A, B, C, RangeWidth: BitWidth + 1);
10696 if (!X)
10697 return std::nullopt;
10698
10699 ConstantInt *CX = ConstantInt::get(Context&: SE.getContext(), V: *X);
10700 ConstantInt *V = EvaluateConstantChrecAtConstant(AddRec, C: CX, SE);
10701 if (!V->isZero())
10702 return std::nullopt;
10703
10704 return TruncIfPossible(X, BitWidth);
10705}
10706
10707/// Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n
10708/// iterations. The values M, N are assumed to be signed, and they
10709/// should all have the same bit widths.
10710/// Find the least n such that c(n) does not belong to the given range,
10711/// while c(n-1) does.
10712///
10713/// This function returns std::nullopt if
10714/// (a) the addrec coefficients are not constant, or
10715/// (b) SolveQuadraticEquationWrap was unable to find a solution for the
10716/// bounds of the range.
10717static std::optional<APInt>
10718SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec,
10719 const ConstantRange &Range, ScalarEvolution &SE) {
10720 assert(AddRec->getOperand(0)->isZero() &&
10721 "Starting value of addrec should be 0");
10722 LLVM_DEBUG(dbgs() << __func__ << ": solving boundary crossing for range "
10723 << Range << ", addrec " << *AddRec << '\n');
10724 // This case is handled in getNumIterationsInRange. Here we can assume that
10725 // we start in the range.
10726 assert(Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) &&
10727 "Addrec's initial value should be in range");
10728
10729 APInt A, B, C, M;
10730 unsigned BitWidth;
10731 auto T = GetQuadraticEquation(AddRec);
10732 if (!T)
10733 return std::nullopt;
10734
10735 // Be careful about the return value: there can be two reasons for not
10736 // returning an actual number. First, if no solutions to the equations
10737 // were found, and second, if the solutions don't leave the given range.
10738 // The first case means that the actual solution is "unknown", the second
10739 // means that it's known, but not valid. If the solution is unknown, we
10740 // cannot make any conclusions.
10741 // Return a pair: the optional solution and a flag indicating if the
10742 // solution was found.
10743 auto SolveForBoundary =
10744 [&](APInt Bound) -> std::pair<std::optional<APInt>, bool> {
10745 // Solve for signed overflow and unsigned overflow, pick the lower
10746 // solution.
10747 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: checking boundary "
10748 << Bound << " (before multiplying by " << M << ")\n");
10749 Bound *= M; // The quadratic equation multiplier.
10750
10751 std::optional<APInt> SO;
10752 if (BitWidth > 1) {
10753 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10754 "signed overflow\n");
10755 SO = APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth);
10756 }
10757 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10758 "unsigned overflow\n");
10759 std::optional<APInt> UO =
10760 APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth + 1);
10761
10762 auto LeavesRange = [&] (const APInt &X) {
10763 ConstantInt *C0 = ConstantInt::get(Context&: SE.getContext(), V: X);
10764 ConstantInt *V0 = EvaluateConstantChrecAtConstant(AddRec, C: C0, SE);
10765 if (Range.contains(Val: V0->getValue()))
10766 return false;
10767 // X should be at least 1, so X-1 is non-negative.
10768 ConstantInt *C1 = ConstantInt::get(Context&: SE.getContext(), V: X-1);
10769 ConstantInt *V1 = EvaluateConstantChrecAtConstant(AddRec, C: C1, SE);
10770 if (Range.contains(Val: V1->getValue()))
10771 return true;
10772 return false;
10773 };
10774
10775 // If SolveQuadraticEquationWrap returns std::nullopt, it means that there
10776 // can be a solution, but the function failed to find it. We cannot treat it
10777 // as "no solution".
10778 if (!SO || !UO)
10779 return {std::nullopt, false};
10780
10781 // Check the smaller value first to see if it leaves the range.
10782 // At this point, both SO and UO must have values.
10783 std::optional<APInt> Min = MinOptional(X: SO, Y: UO);
10784 if (LeavesRange(*Min))
10785 return { Min, true };
10786 std::optional<APInt> Max = Min == SO ? UO : SO;
10787 if (LeavesRange(*Max))
10788 return { Max, true };
10789
10790 // Solutions were found, but were eliminated, hence the "true".
10791 return {std::nullopt, true};
10792 };
10793
10794 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10795 // Lower bound is inclusive, subtract 1 to represent the exiting value.
10796 APInt Lower = Range.getLower().sext(width: A.getBitWidth()) - 1;
10797 APInt Upper = Range.getUpper().sext(width: A.getBitWidth());
10798 auto SL = SolveForBoundary(Lower);
10799 auto SU = SolveForBoundary(Upper);
10800 // If any of the solutions was unknown, no meaninigful conclusions can
10801 // be made.
10802 if (!SL.second || !SU.second)
10803 return std::nullopt;
10804
10805 // Claim: The correct solution is not some value between Min and Max.
10806 //
10807 // Justification: Assuming that Min and Max are different values, one of
10808 // them is when the first signed overflow happens, the other is when the
10809 // first unsigned overflow happens. Crossing the range boundary is only
10810 // possible via an overflow (treating 0 as a special case of it, modeling
10811 // an overflow as crossing k*2^W for some k).
10812 //
10813 // The interesting case here is when Min was eliminated as an invalid
10814 // solution, but Max was not. The argument is that if there was another
10815 // overflow between Min and Max, it would also have been eliminated if
10816 // it was considered.
10817 //
10818 // For a given boundary, it is possible to have two overflows of the same
10819 // type (signed/unsigned) without having the other type in between: this
10820 // can happen when the vertex of the parabola is between the iterations
10821 // corresponding to the overflows. This is only possible when the two
10822 // overflows cross k*2^W for the same k. In such case, if the second one
10823 // left the range (and was the first one to do so), the first overflow
10824 // would have to enter the range, which would mean that either we had left
10825 // the range before or that we started outside of it. Both of these cases
10826 // are contradictions.
10827 //
10828 // Claim: In the case where SolveForBoundary returns std::nullopt, the correct
10829 // solution is not some value between the Max for this boundary and the
10830 // Min of the other boundary.
10831 //
10832 // Justification: Assume that we had such Max_A and Min_B corresponding
10833 // to range boundaries A and B and such that Max_A < Min_B. If there was
10834 // a solution between Max_A and Min_B, it would have to be caused by an
10835 // overflow corresponding to either A or B. It cannot correspond to B,
10836 // since Min_B is the first occurrence of such an overflow. If it
10837 // corresponded to A, it would have to be either a signed or an unsigned
10838 // overflow that is larger than both eliminated overflows for A. But
10839 // between the eliminated overflows and this overflow, the values would
10840 // cover the entire value space, thus crossing the other boundary, which
10841 // is a contradiction.
10842
10843 return TruncIfPossible(X: MinOptional(X: SL.first, Y: SU.first), BitWidth);
10844}
10845
10846ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V,
10847 const Loop *L,
10848 bool ControlsOnlyExit,
10849 bool AllowPredicates) {
10850
10851 // This is only used for loops with a "x != y" exit test. The exit condition
10852 // is now expressed as a single expression, V = x-y. So the exit test is
10853 // effectively V != 0. We know and take advantage of the fact that this
10854 // expression only being used in a comparison by zero context.
10855
10856 SmallVector<const SCEVPredicate *> Predicates;
10857 // If the value is a constant
10858 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10859 // If the value is already zero, the branch will execute zero times.
10860 if (C->getValue()->isZero()) return C;
10861 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10862 }
10863
10864 const SCEVAddRecExpr *AddRec =
10865 dyn_cast<SCEVAddRecExpr>(Val: stripInjectiveFunctions(S: V));
10866
10867 if (!AddRec && AllowPredicates)
10868 // Try to make this an AddRec using runtime tests, in the first X
10869 // iterations of this loop, where X is the SCEV expression found by the
10870 // algorithm below.
10871 AddRec = convertSCEVToAddRecWithPredicates(S: V, L, Preds&: Predicates);
10872
10873 if (!AddRec || AddRec->getLoop() != L)
10874 return getCouldNotCompute();
10875
10876 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
10877 // the quadratic equation to solve it.
10878 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
10879 // We can only use this value if the chrec ends up with an exact zero
10880 // value at this index. When solving for "X*X != 5", for example, we
10881 // should not accept a root of 2.
10882 if (auto S = SolveQuadraticAddRecExact(AddRec, SE&: *this)) {
10883 const auto *R = cast<SCEVConstant>(Val: getConstant(Val: *S));
10884 return ExitLimit(R, R, R, false, Predicates);
10885 }
10886 return getCouldNotCompute();
10887 }
10888
10889 // Otherwise we can only handle this if it is affine.
10890 if (!AddRec->isAffine())
10891 return getCouldNotCompute();
10892
10893 // If this is an affine expression, the execution count of this branch is
10894 // the minimum unsigned root of the following equation:
10895 //
10896 // Start + Step*N = 0 (mod 2^BW)
10897 //
10898 // equivalent to:
10899 //
10900 // Step*N = -Start (mod 2^BW)
10901 //
10902 // where BW is the common bit width of Start and Step.
10903
10904 // Get the initial value for the loop.
10905 const SCEV *Start = getSCEVAtScope(V: AddRec->getStart(), L: L->getParentLoop());
10906 const SCEV *Step = getSCEVAtScope(V: AddRec->getOperand(i: 1), L: L->getParentLoop());
10907
10908 if (!isLoopInvariant(S: Step, L))
10909 return getCouldNotCompute();
10910
10911 LoopGuards Guards = LoopGuards::collect(L, SE&: *this);
10912 // Specialize step for this loop so we get context sensitive facts below.
10913 const SCEV *StepWLG = applyLoopGuards(Expr: Step, Guards);
10914
10915 // For positive steps (counting up until unsigned overflow):
10916 // N = -Start/Step (as unsigned)
10917 // For negative steps (counting down to zero):
10918 // N = Start/-Step
10919 // First compute the unsigned distance from zero in the direction of Step.
10920 bool CountDown = isKnownNegative(S: StepWLG);
10921 if (!CountDown && !isKnownNonNegative(S: StepWLG))
10922 return getCouldNotCompute();
10923
10924 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(V: Start);
10925 // Handle unitary steps, which cannot wraparound.
10926 // 1*N = -Start; -1*N = Start (mod 2^BW), so:
10927 // N = Distance (as unsigned)
10928
10929 if (match(S: Step, P: m_CombineOr(Ps: m_scev_One(), Ps: m_scev_AllOnes()))) {
10930 APInt MaxBECount = getUnsignedRangeMax(S: applyLoopGuards(Expr: Distance, Guards));
10931 MaxBECount = APIntOps::umin(A: MaxBECount, B: getUnsignedRangeMax(S: Distance));
10932
10933 // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated,
10934 // we end up with a loop whose backedge-taken count is n - 1. Detect this
10935 // case, and see if we can improve the bound.
10936 //
10937 // Explicitly handling this here is necessary because getUnsignedRange
10938 // isn't context-sensitive; it doesn't know that we only care about the
10939 // range inside the loop.
10940 const SCEV *Zero = getZero(Ty: Distance->getType());
10941 const SCEV *One = getOne(Ty: Distance->getType());
10942 const SCEV *DistancePlusOne = getAddExpr(LHS: Distance, RHS: One);
10943 if (isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: DistancePlusOne, RHS: Zero)) {
10944 // If Distance + 1 doesn't overflow, we can compute the maximum distance
10945 // as "unsigned_max(Distance + 1) - 1".
10946 ConstantRange CR = getUnsignedRange(S: DistancePlusOne);
10947 MaxBECount = APIntOps::umin(A: MaxBECount, B: CR.getUnsignedMax() - 1);
10948 }
10949 return ExitLimit(Distance, getConstant(Val: MaxBECount), Distance, false,
10950 Predicates);
10951 }
10952
10953 // If the condition controls loop exit (the loop exits only if the expression
10954 // is true) and the addition is no-wrap we can use unsigned divide to
10955 // compute the backedge count. In this case, the step may not divide the
10956 // distance, but we don't care because if the condition is "missed" the loop
10957 // will have undefined behavior due to wrapping.
10958 if (ControlsOnlyExit && AddRec->hasNoSelfWrap() &&
10959 loopHasNoAbnormalExits(L: AddRec->getLoop())) {
10960
10961 // If the stride is zero and the start is non-zero, the loop must be
10962 // infinite. In C++, most loops are finite by assumption, in which case the
10963 // step being zero implies UB must execute if the loop is entered.
10964 if (!(loopIsFiniteByAssumption(L) && isKnownNonZero(S: Start)) &&
10965 !isKnownNonZero(S: StepWLG))
10966 return getCouldNotCompute();
10967
10968 const SCEV *Exact =
10969 getUDivExpr(LHS: Distance, RHS: CountDown ? getNegativeSCEV(V: Step) : Step);
10970 const SCEV *ConstantMax = getCouldNotCompute();
10971 if (Exact != getCouldNotCompute()) {
10972 APInt MaxInt = getUnsignedRangeMax(S: applyLoopGuards(Expr: Exact, Guards));
10973 ConstantMax =
10974 getConstant(Val: APIntOps::umin(A: MaxInt, B: getUnsignedRangeMax(S: Exact)));
10975 }
10976 const SCEV *SymbolicMax =
10977 isa<SCEVCouldNotCompute>(Val: Exact) ? ConstantMax : Exact;
10978 return ExitLimit(Exact, ConstantMax, SymbolicMax, false, Predicates);
10979 }
10980
10981 // Solve the general equation.
10982 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Val: Step);
10983 if (!StepC || StepC->getValue()->isZero())
10984 return getCouldNotCompute();
10985 const SCEV *E = SolveLinEquationWithOverflow(
10986 A: StepC->getAPInt(), B: getNegativeSCEV(V: Start),
10987 Predicates: AllowPredicates ? &Predicates : nullptr, SE&: *this, L);
10988
10989 const SCEV *M = E;
10990 if (E != getCouldNotCompute()) {
10991 APInt MaxWithGuards = getUnsignedRangeMax(S: applyLoopGuards(Expr: E, Guards));
10992 M = getConstant(Val: APIntOps::umin(A: MaxWithGuards, B: getUnsignedRangeMax(S: E)));
10993 }
10994 auto *S = isa<SCEVCouldNotCompute>(Val: E) ? M : E;
10995 return ExitLimit(E, M, S, false, Predicates);
10996}
10997
10998ScalarEvolution::ExitLimit
10999ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) {
11000 // Loops that look like: while (X == 0) are very strange indeed. We don't
11001 // handle them yet except for the trivial case. This could be expanded in the
11002 // future as needed.
11003
11004 // If the value is a constant, check to see if it is known to be non-zero
11005 // already. If so, the backedge will execute zero times.
11006 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
11007 if (!C->getValue()->isZero())
11008 return getZero(Ty: C->getType());
11009 return getCouldNotCompute(); // Otherwise it will loop infinitely.
11010 }
11011
11012 // We could implement others, but I really doubt anyone writes loops like
11013 // this, and if they did, they would already be constant folded.
11014 return getCouldNotCompute();
11015}
11016
11017std::pair<const BasicBlock *, const BasicBlock *>
11018ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(const BasicBlock *BB)
11019 const {
11020 // If the block has a unique predecessor, then there is no path from the
11021 // predecessor to the block that does not go through the direct edge
11022 // from the predecessor to the block.
11023 if (const BasicBlock *Pred = BB->getSinglePredecessor())
11024 return {Pred, BB};
11025
11026 // A loop's header is defined to be a block that dominates the loop.
11027 // If the header has a unique predecessor outside the loop, it must be
11028 // a block that has exactly one successor that can reach the loop.
11029 if (const Loop *L = LI.getLoopFor(BB))
11030 return {L->getLoopPredecessor(), L->getHeader()};
11031
11032 return {nullptr, BB};
11033}
11034
11035/// SCEV structural equivalence is usually sufficient for testing whether two
11036/// expressions are equal, however for the purposes of looking for a condition
11037/// guarding a loop, it can be useful to be a little more general, since a
11038/// front-end may have replicated the controlling expression.
11039static bool HasSameValue(const SCEV *A, const SCEV *B) {
11040 // Quick check to see if they are the same SCEV.
11041 if (A == B) return true;
11042
11043 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) {
11044 // Not all instructions that are "identical" compute the same value. For
11045 // instance, two distinct alloca instructions allocating the same type are
11046 // identical and do not read memory; but compute distinct values.
11047 return A->isIdenticalTo(I: B) && (isa<BinaryOperator>(Val: A) || isa<GetElementPtrInst>(Val: A));
11048 };
11049
11050 // Otherwise, if they're both SCEVUnknown, it's possible that they hold
11051 // two different instructions with the same value. Check for this case.
11052 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(Val: A))
11053 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(Val: B))
11054 if (const Instruction *AI = dyn_cast<Instruction>(Val: AU->getValue()))
11055 if (const Instruction *BI = dyn_cast<Instruction>(Val: BU->getValue()))
11056 if (ComputesEqualValues(AI, BI))
11057 return true;
11058
11059 // Otherwise assume they may have a different value.
11060 return false;
11061}
11062
11063static bool MatchBinarySub(const SCEV *S, SCEVUse &LHS, SCEVUse &RHS) {
11064 const SCEV *Op0, *Op1;
11065 if (!match(S, P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))))
11066 return false;
11067 if (match(S: Op0, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
11068 LHS = Op1;
11069 return true;
11070 }
11071 if (match(S: Op1, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
11072 LHS = Op0;
11073 return true;
11074 }
11075 return false;
11076}
11077
11078bool ScalarEvolution::SimplifyICmpOperands(CmpPredicate &Pred, SCEVUse &LHS,
11079 SCEVUse &RHS, unsigned Depth) {
11080 bool Changed = false;
11081 // Simplifies ICMP to trivial true or false by turning it into '0 == 0' or
11082 // '0 != 0'.
11083 auto TrivialCase = [&](bool TriviallyTrue) {
11084 LHS = RHS = getConstant(V: ConstantInt::getFalse(Context&: getContext()));
11085 Pred = TriviallyTrue ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
11086 return true;
11087 };
11088 // If we hit the max recursion limit bail out.
11089 if (Depth >= 3)
11090 return false;
11091
11092 const SCEV *NewLHS, *NewRHS;
11093 if (match(U: LHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
11094 match(U: RHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale()))) {
11095 const SCEVMulExpr *LMul = cast<SCEVMulExpr>(Val&: LHS);
11096 const SCEVMulExpr *RMul = cast<SCEVMulExpr>(Val&: RHS);
11097
11098 // (X * vscale) pred (Y * vscale) ==> X pred Y
11099 // when both multiples are NSW.
11100 // (X * vscale) uicmp/eq/ne (Y * vscale) ==> X uicmp/eq/ne Y
11101 // when both multiples are NUW.
11102 if ((LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap()) ||
11103 (LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap() &&
11104 !ICmpInst::isSigned(Pred))) {
11105 LHS = NewLHS;
11106 RHS = NewRHS;
11107 Changed = true;
11108 }
11109 }
11110
11111 // Canonicalize a constant to the right side.
11112 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS)) {
11113 // Check for both operands constant.
11114 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
11115 if (!ICmpInst::compare(LHS: LHSC->getAPInt(), RHS: RHSC->getAPInt(), Pred))
11116 return TrivialCase(false);
11117 return TrivialCase(true);
11118 }
11119 // Otherwise swap the operands to put the constant on the right.
11120 std::swap(a&: LHS, b&: RHS);
11121 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11122 Changed = true;
11123 }
11124
11125 // If we're comparing an addrec with a value which is loop-invariant in the
11126 // addrec's loop, put the addrec on the left. Also make a dominance check,
11127 // as both operands could be addrecs loop-invariant in each other's loop.
11128 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: RHS)) {
11129 const Loop *L = AR->getLoop();
11130 if (isLoopInvariant(S: LHS, L) && properlyDominates(S: LHS, BB: L->getHeader())) {
11131 std::swap(a&: LHS, b&: RHS);
11132 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11133 Changed = true;
11134 }
11135 }
11136
11137 // If there's a constant operand, canonicalize comparisons with boundary
11138 // cases, and canonicalize *-or-equal comparisons to regular comparisons.
11139 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(Val&: RHS)) {
11140 const APInt &RA = RC->getAPInt();
11141
11142 bool SimplifiedByConstantRange = false;
11143
11144 if (!ICmpInst::isEquality(P: Pred)) {
11145 ConstantRange ExactCR = ConstantRange::makeExactICmpRegion(Pred, Other: RA);
11146 if (ExactCR.isFullSet())
11147 return TrivialCase(true);
11148 if (ExactCR.isEmptySet())
11149 return TrivialCase(false);
11150
11151 APInt NewRHS;
11152 CmpInst::Predicate NewPred;
11153 if (ExactCR.getEquivalentICmp(Pred&: NewPred, RHS&: NewRHS) &&
11154 ICmpInst::isEquality(P: NewPred)) {
11155 // We were able to convert an inequality to an equality.
11156 Pred = NewPred;
11157 RHS = getConstant(Val: NewRHS);
11158 Changed = SimplifiedByConstantRange = true;
11159 }
11160 }
11161
11162 if (!SimplifiedByConstantRange) {
11163 switch (Pred) {
11164 default:
11165 break;
11166 case ICmpInst::ICMP_EQ:
11167 case ICmpInst::ICMP_NE:
11168 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
11169 if (RA.isZero() && MatchBinarySub(S: LHS, LHS, RHS))
11170 Changed = true;
11171 break;
11172
11173 // The "Should have been caught earlier!" messages refer to the fact
11174 // that the ExactCR.isFullSet() or ExactCR.isEmptySet() check above
11175 // should have fired on the corresponding cases, and canonicalized the
11176 // check to trivial case.
11177
11178 case ICmpInst::ICMP_UGE:
11179 assert(!RA.isMinValue() && "Should have been caught earlier!");
11180 Pred = ICmpInst::ICMP_UGT;
11181 RHS = getConstant(Val: RA - 1);
11182 Changed = true;
11183 break;
11184 case ICmpInst::ICMP_ULE:
11185 assert(!RA.isMaxValue() && "Should have been caught earlier!");
11186 Pred = ICmpInst::ICMP_ULT;
11187 RHS = getConstant(Val: RA + 1);
11188 Changed = true;
11189 break;
11190 case ICmpInst::ICMP_SGE:
11191 assert(!RA.isMinSignedValue() && "Should have been caught earlier!");
11192 Pred = ICmpInst::ICMP_SGT;
11193 RHS = getConstant(Val: RA - 1);
11194 Changed = true;
11195 break;
11196 case ICmpInst::ICMP_SLE:
11197 assert(!RA.isMaxSignedValue() && "Should have been caught earlier!");
11198 Pred = ICmpInst::ICMP_SLT;
11199 RHS = getConstant(Val: RA + 1);
11200 Changed = true;
11201 break;
11202 }
11203 }
11204 }
11205
11206 // Check for obvious equality.
11207 if (HasSameValue(A: LHS, B: RHS)) {
11208 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
11209 return TrivialCase(true);
11210 if (ICmpInst::isFalseWhenEqual(predicate: Pred))
11211 return TrivialCase(false);
11212 }
11213
11214 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
11215 // adding or subtracting 1 from one of the operands.
11216 switch (Pred) {
11217 case ICmpInst::ICMP_SLE:
11218 if (!getSignedRangeMax(S: RHS).isMaxSignedValue()) {
11219 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11220 Flags: SCEV::FlagNSW);
11221 Pred = ICmpInst::ICMP_SLT;
11222 Changed = true;
11223 } else if (!getSignedRangeMin(S: LHS).isMinSignedValue()) {
11224 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS,
11225 Flags: SCEV::FlagNSW);
11226 Pred = ICmpInst::ICMP_SLT;
11227 Changed = true;
11228 }
11229 break;
11230 case ICmpInst::ICMP_SGE:
11231 if (!getSignedRangeMin(S: RHS).isMinSignedValue()) {
11232 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS,
11233 Flags: SCEV::FlagNSW);
11234 Pred = ICmpInst::ICMP_SGT;
11235 Changed = true;
11236 } else if (!getSignedRangeMax(S: LHS).isMaxSignedValue()) {
11237 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11238 Flags: SCEV::FlagNSW);
11239 Pred = ICmpInst::ICMP_SGT;
11240 Changed = true;
11241 }
11242 break;
11243 case ICmpInst::ICMP_ULE:
11244 if (!getUnsignedRangeMax(S: RHS).isMaxValue()) {
11245 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11246 Flags: SCEV::FlagNUW);
11247 Pred = ICmpInst::ICMP_ULT;
11248 Changed = true;
11249 } else if (!getUnsignedRangeMin(S: LHS).isMinValue()) {
11250 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS);
11251 Pred = ICmpInst::ICMP_ULT;
11252 Changed = true;
11253 }
11254 break;
11255 case ICmpInst::ICMP_UGE:
11256 // If RHS is an op we can fold the -1, try that first.
11257 // Otherwise prefer LHS to preserve the nuw flag.
11258 if ((isa<SCEVConstant>(Val: RHS) ||
11259 (isa<SCEVAddExpr, SCEVAddRecExpr>(Val: RHS) &&
11260 isa<SCEVConstant>(Val: cast<SCEVNAryExpr>(Val&: RHS)->getOperand(i: 0)))) &&
11261 !getUnsignedRangeMin(S: RHS).isMinValue()) {
11262 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11263 Pred = ICmpInst::ICMP_UGT;
11264 Changed = true;
11265 } else if (!getUnsignedRangeMax(S: LHS).isMaxValue()) {
11266 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11267 Flags: SCEV::FlagNUW);
11268 Pred = ICmpInst::ICMP_UGT;
11269 Changed = true;
11270 } else if (!getUnsignedRangeMin(S: RHS).isMinValue()) {
11271 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11272 Pred = ICmpInst::ICMP_UGT;
11273 Changed = true;
11274 }
11275 break;
11276 default:
11277 break;
11278 }
11279
11280 // TODO: More simplifications are possible here.
11281
11282 // Recursively simplify until we either hit a recursion limit or nothing
11283 // changes.
11284 if (Changed)
11285 (void)SimplifyICmpOperands(Pred, LHS, RHS, Depth: Depth + 1);
11286
11287 return Changed;
11288}
11289
11290bool ScalarEvolution::isKnownNegative(const SCEV *S) {
11291 return getSignedRangeMax(S).isNegative();
11292}
11293
11294bool ScalarEvolution::isKnownPositive(const SCEV *S) {
11295 return getSignedRangeMin(S).isStrictlyPositive();
11296}
11297
11298bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
11299 return !getSignedRangeMin(S).isNegative();
11300}
11301
11302bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
11303 return !getSignedRangeMax(S).isStrictlyPositive();
11304}
11305
11306bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
11307 // Query push down for cases where the unsigned range is
11308 // less than sufficient.
11309 if (const auto *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
11310 return isKnownNonZero(S: SExt->getOperand(i: 0));
11311 return getUnsignedRangeMin(S) != 0;
11312}
11313
11314bool ScalarEvolution::isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero,
11315 bool OrNegative) {
11316 auto NonRecursive = [OrNegative](const SCEV *S) {
11317 if (auto *C = dyn_cast<SCEVConstant>(Val: S))
11318 return C->getAPInt().isPowerOf2() ||
11319 (OrNegative && C->getAPInt().isNegatedPowerOf2());
11320
11321 // vscale is a power-of-two.
11322 return isa<SCEVVScale>(Val: S);
11323 };
11324
11325 if (NonRecursive(S))
11326 return true;
11327
11328 auto *Mul = dyn_cast<SCEVMulExpr>(Val: S);
11329 if (!Mul)
11330 return false;
11331 return all_of(Range: Mul->operands(), P: NonRecursive) && (OrZero || isKnownNonZero(S));
11332}
11333
11334bool ScalarEvolution::isKnownMultipleOf(
11335 const SCEV *S, uint64_t M,
11336 SmallVectorImpl<const SCEVPredicate *> &Assumptions) {
11337 if (M == 0)
11338 return false;
11339 if (M == 1)
11340 return true;
11341
11342 // Recursively check AddRec operands. An AddRecExpr S is a multiple of M if S
11343 // starts with a multiple of M and at every iteration step S only adds
11344 // multiples of M.
11345 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S))
11346 return isKnownMultipleOf(S: AddRec->getStart(), M, Assumptions) &&
11347 isKnownMultipleOf(S: AddRec->getStepRecurrence(SE&: *this), M, Assumptions);
11348
11349 // For a constant, check that "S % M == 0".
11350 if (auto *Cst = dyn_cast<SCEVConstant>(Val: S)) {
11351 APInt C = Cst->getAPInt();
11352 return C.urem(RHS: M) == 0;
11353 }
11354
11355 // TODO: Also check other SCEV expressions, i.e., SCEVAddRecExpr, etc.
11356
11357 // Basic tests have failed.
11358 // Check "S % M == 0" at compile time and record runtime Assumptions.
11359 auto *STy = dyn_cast<IntegerType>(Val: S->getType());
11360 const SCEV *SmodM =
11361 getURemExpr(LHS: S, RHS: getConstant(V: ConstantInt::get(Ty: STy, V: M, IsSigned: false)));
11362 const SCEV *Zero = getZero(Ty: STy);
11363
11364 // Check whether "S % M == 0" is known at compile time.
11365 if (isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero))
11366 return true;
11367
11368 // Check whether "S % M != 0" is known at compile time.
11369 if (isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: SmodM, RHS: Zero))
11370 return false;
11371
11372 const SCEVPredicate *P = getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero);
11373
11374 // Detect redundant predicates.
11375 for (auto *A : Assumptions)
11376 if (A->implies(N: P, SE&: *this))
11377 return true;
11378
11379 // Only record non-redundant predicates.
11380 Assumptions.push_back(Elt: P);
11381 return true;
11382}
11383
11384bool ScalarEvolution::haveSameSign(const SCEV *S1, const SCEV *S2) {
11385 return ((isKnownNonNegative(S: S1) && isKnownNonNegative(S: S2)) ||
11386 (isKnownNegative(S: S1) && isKnownNegative(S: S2)));
11387}
11388
11389std::pair<const SCEV *, const SCEV *>
11390ScalarEvolution::SplitIntoInitAndPostInc(const Loop *L, const SCEV *S) {
11391 // Compute SCEV on entry of loop L.
11392 const SCEV *Start = SCEVInitRewriter::rewrite(S, L, SE&: *this);
11393 if (Start == getCouldNotCompute())
11394 return { Start, Start };
11395 // Compute post increment SCEV for loop L.
11396 const SCEV *PostInc = SCEVPostIncRewriter::rewrite(S, L, SE&: *this);
11397 assert(PostInc != getCouldNotCompute() && "Unexpected could not compute");
11398 return { Start, PostInc };
11399}
11400
11401bool ScalarEvolution::isKnownViaInduction(CmpPredicate Pred, SCEVUse LHS,
11402 SCEVUse RHS) {
11403 // First collect all loops.
11404 SmallPtrSet<const Loop *, 8> LoopsUsed;
11405 getUsedLoops(S: LHS, LoopsUsed);
11406 getUsedLoops(S: RHS, LoopsUsed);
11407
11408 if (LoopsUsed.empty())
11409 return false;
11410
11411 // Domination relationship must be a linear order on collected loops.
11412#ifndef NDEBUG
11413 for (const auto *L1 : LoopsUsed)
11414 for (const auto *L2 : LoopsUsed)
11415 assert((DT.dominates(L1->getHeader(), L2->getHeader()) ||
11416 DT.dominates(L2->getHeader(), L1->getHeader())) &&
11417 "Domination relationship is not a linear order");
11418#endif
11419
11420 const Loop *MDL =
11421 *llvm::max_element(Range&: LoopsUsed, C: [&](const Loop *L1, const Loop *L2) {
11422 return DT.properlyDominates(A: L1->getHeader(), B: L2->getHeader());
11423 });
11424
11425 // Get init and post increment value for LHS.
11426 auto SplitLHS = SplitIntoInitAndPostInc(L: MDL, S: LHS);
11427 // if LHS contains unknown non-invariant SCEV then bail out.
11428 if (SplitLHS.first == getCouldNotCompute())
11429 return false;
11430 assert (SplitLHS.second != getCouldNotCompute() && "Unexpected CNC");
11431 // Get init and post increment value for RHS.
11432 auto SplitRHS = SplitIntoInitAndPostInc(L: MDL, S: RHS);
11433 // if RHS contains unknown non-invariant SCEV then bail out.
11434 if (SplitRHS.first == getCouldNotCompute())
11435 return false;
11436 assert (SplitRHS.second != getCouldNotCompute() && "Unexpected CNC");
11437 // It is possible that init SCEV contains an invariant load but it does
11438 // not dominate MDL and is not available at MDL loop entry, so we should
11439 // check it here.
11440 if (!isAvailableAtLoopEntry(S: SplitLHS.first, L: MDL) ||
11441 !isAvailableAtLoopEntry(S: SplitRHS.first, L: MDL))
11442 return false;
11443
11444 // It seems backedge guard check is faster than entry one so in some cases
11445 // it can speed up whole estimation by short circuit
11446 return isLoopBackedgeGuardedByCond(L: MDL, Pred, LHS: SplitLHS.second,
11447 RHS: SplitRHS.second) &&
11448 isLoopEntryGuardedByCond(L: MDL, Pred, LHS: SplitLHS.first, RHS: SplitRHS.first);
11449}
11450
11451bool ScalarEvolution::isKnownPredicate(CmpPredicate Pred, SCEVUse LHS,
11452 SCEVUse RHS) {
11453 // Canonicalize the inputs first.
11454 (void)SimplifyICmpOperands(Pred, LHS, RHS);
11455
11456 if (isKnownViaInduction(Pred, LHS, RHS))
11457 return true;
11458
11459 if (isKnownPredicateViaSplitting(Pred, LHS, RHS))
11460 return true;
11461
11462 // Otherwise see what can be done with some simple reasoning.
11463 return isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
11464}
11465
11466std::optional<bool> ScalarEvolution::evaluatePredicate(CmpPredicate Pred,
11467 const SCEV *LHS,
11468 const SCEV *RHS) {
11469 if (isKnownPredicate(Pred, LHS, RHS))
11470 return true;
11471 if (isKnownPredicate(Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11472 return false;
11473 return std::nullopt;
11474}
11475
11476bool ScalarEvolution::isKnownPredicateAt(CmpPredicate Pred, const SCEV *LHS,
11477 const SCEV *RHS,
11478 const Instruction *CtxI) {
11479 // TODO: Analyze guards and assumes from Context's block.
11480 return isKnownPredicate(Pred, LHS, RHS) ||
11481 isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS);
11482}
11483
11484std::optional<bool>
11485ScalarEvolution::evaluatePredicateAt(CmpPredicate Pred, const SCEV *LHS,
11486 const SCEV *RHS, const Instruction *CtxI) {
11487 std::optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS);
11488 if (KnownWithoutContext)
11489 return KnownWithoutContext;
11490
11491 if (isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS))
11492 return true;
11493 if (isBasicBlockEntryGuardedByCond(
11494 BB: CtxI->getParent(), Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11495 return false;
11496 return std::nullopt;
11497}
11498
11499bool ScalarEvolution::isKnownOnEveryIteration(CmpPredicate Pred,
11500 const SCEVAddRecExpr *LHS,
11501 const SCEV *RHS) {
11502 const Loop *L = LHS->getLoop();
11503 return isLoopEntryGuardedByCond(L, Pred, LHS: LHS->getStart(), RHS) &&
11504 isLoopBackedgeGuardedByCond(L, Pred, LHS: LHS->getPostIncExpr(SE&: *this), RHS);
11505}
11506
11507std::optional<ScalarEvolution::MonotonicPredicateType>
11508ScalarEvolution::getMonotonicPredicateType(const SCEVAddRecExpr *LHS,
11509 ICmpInst::Predicate Pred) {
11510 auto Result = getMonotonicPredicateTypeImpl(LHS, Pred);
11511
11512#ifndef NDEBUG
11513 // Verify an invariant: inverting the predicate should turn a monotonically
11514 // increasing change to a monotonically decreasing one, and vice versa.
11515 if (Result) {
11516 auto ResultSwapped =
11517 getMonotonicPredicateTypeImpl(LHS, ICmpInst::getSwappedPredicate(Pred));
11518
11519 assert(*ResultSwapped != *Result &&
11520 "monotonicity should flip as we flip the predicate");
11521 }
11522#endif
11523
11524 return Result;
11525}
11526
11527std::optional<ScalarEvolution::MonotonicPredicateType>
11528ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
11529 ICmpInst::Predicate Pred) {
11530 // A zero step value for LHS means the induction variable is essentially a
11531 // loop invariant value. We don't really depend on the predicate actually
11532 // flipping from false to true (for increasing predicates, and the other way
11533 // around for decreasing predicates), all we care about is that *if* the
11534 // predicate changes then it only changes from false to true.
11535 //
11536 // A zero step value in itself is not very useful, but there may be places
11537 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be
11538 // as general as possible.
11539
11540 // Only handle LE/LT/GE/GT predicates.
11541 if (!ICmpInst::isRelational(P: Pred))
11542 return std::nullopt;
11543
11544 bool IsGreater = ICmpInst::isGE(P: Pred) || ICmpInst::isGT(P: Pred);
11545 assert((IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) &&
11546 "Should be greater or less!");
11547
11548 // Check that AR does not wrap.
11549 if (ICmpInst::isUnsigned(Pred)) {
11550 if (!LHS->hasNoUnsignedWrap())
11551 return std::nullopt;
11552 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11553 }
11554 assert(ICmpInst::isSigned(Pred) &&
11555 "Relational predicate is either signed or unsigned!");
11556 if (!LHS->hasNoSignedWrap())
11557 return std::nullopt;
11558
11559 const SCEV *Step = LHS->getStepRecurrence(SE&: *this);
11560
11561 if (isKnownNonNegative(S: Step))
11562 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11563
11564 if (isKnownNonPositive(S: Step))
11565 return !IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11566
11567 return std::nullopt;
11568}
11569
11570std::optional<ScalarEvolution::LoopInvariantPredicate>
11571ScalarEvolution::getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS,
11572 const SCEV *RHS, const Loop *L,
11573 const Instruction *CtxI) {
11574 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11575 if (!isLoopInvariant(S: RHS, L)) {
11576 if (!isLoopInvariant(S: LHS, L))
11577 return std::nullopt;
11578
11579 std::swap(a&: LHS, b&: RHS);
11580 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11581 }
11582
11583 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11584 if (!ArLHS || ArLHS->getLoop() != L)
11585 return std::nullopt;
11586
11587 auto MonotonicType = getMonotonicPredicateType(LHS: ArLHS, Pred);
11588 if (!MonotonicType)
11589 return std::nullopt;
11590 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
11591 // true as the loop iterates, and the backedge is control dependent on
11592 // "ArLHS `Pred` RHS" == true then we can reason as follows:
11593 //
11594 // * if the predicate was false in the first iteration then the predicate
11595 // is never evaluated again, since the loop exits without taking the
11596 // backedge.
11597 // * if the predicate was true in the first iteration then it will
11598 // continue to be true for all future iterations since it is
11599 // monotonically increasing.
11600 //
11601 // For both the above possibilities, we can replace the loop varying
11602 // predicate with its value on the first iteration of the loop (which is
11603 // loop invariant).
11604 //
11605 // A similar reasoning applies for a monotonically decreasing predicate, by
11606 // replacing true with false and false with true in the above two bullets.
11607 bool Increasing = *MonotonicType == ScalarEvolution::MonotonicallyIncreasing;
11608 auto P = Increasing ? Pred : ICmpInst::getInverseCmpPredicate(Pred);
11609
11610 if (isLoopBackedgeGuardedByCond(L, Pred: P, LHS, RHS))
11611 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11612 RHS);
11613
11614 if (!CtxI)
11615 return std::nullopt;
11616 // Try to prove via context.
11617 // TODO: Support other cases.
11618 switch (Pred) {
11619 default:
11620 break;
11621 case ICmpInst::ICMP_ULE:
11622 case ICmpInst::ICMP_ULT: {
11623 assert(ArLHS->hasNoUnsignedWrap() && "Is a requirement of monotonicity!");
11624 // Given preconditions
11625 // (1) ArLHS does not cross the border of positive and negative parts of
11626 // range because of:
11627 // - Positive step; (TODO: lift this limitation)
11628 // - nuw - does not cross zero boundary;
11629 // - nsw - does not cross SINT_MAX boundary;
11630 // (2) ArLHS <s RHS
11631 // (3) RHS >=s 0
11632 // we can replace the loop variant ArLHS <u RHS condition with loop
11633 // invariant Start(ArLHS) <u RHS.
11634 //
11635 // Because of (1) there are two options:
11636 // - ArLHS is always negative. It means that ArLHS <u RHS is always false;
11637 // - ArLHS is always non-negative. Because of (3) RHS is also non-negative.
11638 // It means that ArLHS <s RHS <=> ArLHS <u RHS.
11639 // Because of (2) ArLHS <u RHS is trivially true.
11640 // All together it means that ArLHS <u RHS <=> Start(ArLHS) >=s 0.
11641 // We can strengthen this to Start(ArLHS) <u RHS.
11642 auto SignFlippedPred = ICmpInst::getFlippedSignednessPredicate(Pred);
11643 if (ArLHS->hasNoSignedWrap() && ArLHS->isAffine() &&
11644 isKnownPositive(S: ArLHS->getStepRecurrence(SE&: *this)) &&
11645 isKnownNonNegative(S: RHS) &&
11646 isKnownPredicateAt(Pred: SignFlippedPred, LHS: ArLHS, RHS, CtxI))
11647 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11648 RHS);
11649 }
11650 }
11651
11652 return std::nullopt;
11653}
11654
11655std::optional<ScalarEvolution::LoopInvariantPredicate>
11656ScalarEvolution::getLoopInvariantExitCondDuringFirstIterations(
11657 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11658 const Instruction *CtxI, const SCEV *MaxIter) {
11659 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11660 Pred, LHS, RHS, L, CtxI, MaxIter))
11661 return LIP;
11662 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: MaxIter))
11663 // Number of iterations expressed as UMIN isn't always great for expressing
11664 // the value on the last iteration. If the straightforward approach didn't
11665 // work, try the following trick: if the a predicate is invariant for X, it
11666 // is also invariant for umin(X, ...). So try to find something that works
11667 // among subexpressions of MaxIter expressed as umin.
11668 for (SCEVUse Op : UMin->operands())
11669 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11670 Pred, LHS, RHS, L, CtxI, MaxIter: Op))
11671 return LIP;
11672 return std::nullopt;
11673}
11674
11675std::optional<ScalarEvolution::LoopInvariantPredicate>
11676ScalarEvolution::getLoopInvariantExitCondDuringFirstIterationsImpl(
11677 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11678 const Instruction *CtxI, const SCEV *MaxIter) {
11679 // Try to prove the following set of facts:
11680 // - The predicate is monotonic in the iteration space.
11681 // - If the check does not fail on the 1st iteration:
11682 // - No overflow will happen during first MaxIter iterations;
11683 // - It will not fail on the MaxIter'th iteration.
11684 // If the check does fail on the 1st iteration, we leave the loop and no
11685 // other checks matter.
11686
11687 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11688 if (!isLoopInvariant(S: RHS, L)) {
11689 if (!isLoopInvariant(S: LHS, L))
11690 return std::nullopt;
11691
11692 std::swap(a&: LHS, b&: RHS);
11693 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11694 }
11695
11696 auto *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11697 if (!AR || AR->getLoop() != L)
11698 return std::nullopt;
11699
11700 // Even if both are valid, we need to consistently chose the unsigned or the
11701 // signed predicate below, not mixtures of both. For now, prefer the unsigned
11702 // predicate.
11703 Pred = Pred.dropSameSign();
11704
11705 // The predicate must be relational (i.e. <, <=, >=, >).
11706 if (!ICmpInst::isRelational(P: Pred))
11707 return std::nullopt;
11708
11709 // TODO: Support steps other than +/- 1.
11710 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
11711 auto *One = getOne(Ty: Step->getType());
11712 auto *MinusOne = getNegativeSCEV(V: One);
11713 if (Step != One && Step != MinusOne)
11714 return std::nullopt;
11715
11716 // Type mismatch here means that MaxIter is potentially larger than max
11717 // unsigned value in start type, which mean we cannot prove no wrap for the
11718 // indvar.
11719 if (AR->getType() != MaxIter->getType())
11720 return std::nullopt;
11721
11722 // Value of IV on suggested last iteration.
11723 const SCEV *Last = AR->evaluateAtIteration(It: MaxIter, SE&: *this);
11724 // Does it still meet the requirement?
11725 if (!isLoopBackedgeGuardedByCond(L, Pred, LHS: Last, RHS))
11726 return std::nullopt;
11727 // Because step is +/- 1 and MaxIter has same type as Start (i.e. it does
11728 // not exceed max unsigned value of this type), this effectively proves
11729 // that there is no wrap during the iteration. To prove that there is no
11730 // signed/unsigned wrap, we need to check that
11731 // Start <= Last for step = 1 or Start >= Last for step = -1.
11732 ICmpInst::Predicate NoOverflowPred =
11733 CmpInst::isSigned(Pred) ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
11734 if (Step == MinusOne)
11735 NoOverflowPred = ICmpInst::getSwappedPredicate(pred: NoOverflowPred);
11736 const SCEV *Start = AR->getStart();
11737 if (!isKnownPredicateAt(Pred: NoOverflowPred, LHS: Start, RHS: Last, CtxI))
11738 return std::nullopt;
11739
11740 // Everything is fine.
11741 return ScalarEvolution::LoopInvariantPredicate(Pred, Start, RHS);
11742}
11743
11744bool ScalarEvolution::isKnownPredicateViaConstantRanges(CmpPredicate Pred,
11745 SCEVUse LHS,
11746 SCEVUse RHS) {
11747 if (HasSameValue(A: LHS, B: RHS))
11748 return ICmpInst::isTrueWhenEqual(predicate: Pred);
11749
11750 auto CheckRange = [&](bool IsSigned) {
11751 auto RangeLHS = IsSigned ? getSignedRange(S: LHS) : getUnsignedRange(S: LHS);
11752 auto RangeRHS = IsSigned ? getSignedRange(S: RHS) : getUnsignedRange(S: RHS);
11753 return RangeLHS.icmp(Pred, Other: RangeRHS);
11754 };
11755
11756 // The check at the top of the function catches the case where the values are
11757 // known to be equal.
11758 if (Pred == CmpInst::ICMP_EQ)
11759 return false;
11760
11761 if (Pred == CmpInst::ICMP_NE) {
11762 if (CheckRange(true) || CheckRange(false))
11763 return true;
11764 auto *Diff = getMinusSCEV(LHS, RHS);
11765 return !isa<SCEVCouldNotCompute>(Val: Diff) && isKnownNonZero(S: Diff);
11766 }
11767
11768 return CheckRange(CmpInst::isSigned(Pred));
11769}
11770
11771bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred,
11772 SCEVUse LHS, SCEVUse RHS) {
11773 // Match X to (A + C1)<ExpectedFlags> and Y to (A + C2)<ExpectedFlags>, where
11774 // C1 and C2 are constant integers. If either X or Y are not add expressions,
11775 // consider them as X + 0 and Y + 0 respectively. C1 and C2 are returned via
11776 // OutC1 and OutC2.
11777 auto MatchBinaryAddToConst = [this](SCEVUse X, SCEVUse Y, APInt &OutC1,
11778 APInt &OutC2,
11779 SCEV::NoWrapFlags ExpectedFlags) {
11780 SCEVUse XNonConstOp, XConstOp;
11781 SCEVUse YNonConstOp, YConstOp;
11782 SCEV::NoWrapFlags XFlagsPresent;
11783 SCEV::NoWrapFlags YFlagsPresent;
11784
11785 if (!splitBinaryAdd(Expr: X, L&: XConstOp, R&: XNonConstOp, Flags&: XFlagsPresent)) {
11786 XConstOp = getZero(Ty: X->getType());
11787 XNonConstOp = X;
11788 XFlagsPresent = ExpectedFlags;
11789 }
11790 if (!isa<SCEVConstant>(Val: XConstOp))
11791 return false;
11792
11793 if (!splitBinaryAdd(Expr: Y, L&: YConstOp, R&: YNonConstOp, Flags&: YFlagsPresent)) {
11794 YConstOp = getZero(Ty: Y->getType());
11795 YNonConstOp = Y;
11796 YFlagsPresent = ExpectedFlags;
11797 }
11798
11799 if (YNonConstOp != XNonConstOp)
11800 return false;
11801
11802 if (!isa<SCEVConstant>(Val: YConstOp))
11803 return false;
11804
11805 // When matching ADDs with NUW flags (and unsigned predicates), only the
11806 // second ADD (with the larger constant) requires NUW.
11807 if ((YFlagsPresent & ExpectedFlags) != ExpectedFlags)
11808 return false;
11809 if (ExpectedFlags != SCEV::FlagNUW &&
11810 (XFlagsPresent & ExpectedFlags) != ExpectedFlags) {
11811 return false;
11812 }
11813
11814 OutC1 = cast<SCEVConstant>(Val&: XConstOp)->getAPInt();
11815 OutC2 = cast<SCEVConstant>(Val&: YConstOp)->getAPInt();
11816
11817 return true;
11818 };
11819
11820 APInt C1;
11821 APInt C2;
11822
11823 switch (Pred) {
11824 default:
11825 break;
11826
11827 case ICmpInst::ICMP_SGE:
11828 std::swap(a&: LHS, b&: RHS);
11829 [[fallthrough]];
11830 case ICmpInst::ICMP_SLE:
11831 // (X + C1)<nsw> s<= (X + C2)<nsw> if C1 s<= C2.
11832 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.sle(RHS: C2))
11833 return true;
11834
11835 break;
11836
11837 case ICmpInst::ICMP_SGT:
11838 std::swap(a&: LHS, b&: RHS);
11839 [[fallthrough]];
11840 case ICmpInst::ICMP_SLT:
11841 // (X + C1)<nsw> s< (X + C2)<nsw> if C1 s< C2.
11842 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.slt(RHS: C2))
11843 return true;
11844
11845 break;
11846
11847 case ICmpInst::ICMP_UGE:
11848 std::swap(a&: LHS, b&: RHS);
11849 [[fallthrough]];
11850 case ICmpInst::ICMP_ULE:
11851 // (X + C1) u<= (X + C2)<nuw> for C1 u<= C2.
11852 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ule(RHS: C2))
11853 return true;
11854
11855 break;
11856
11857 case ICmpInst::ICMP_UGT:
11858 std::swap(a&: LHS, b&: RHS);
11859 [[fallthrough]];
11860 case ICmpInst::ICMP_ULT:
11861 // (X + C1) u< (X + C2)<nuw> if C1 u< C2.
11862 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ult(RHS: C2))
11863 return true;
11864 break;
11865 }
11866
11867 return false;
11868}
11869
11870bool ScalarEvolution::isKnownPredicateViaSplitting(CmpPredicate Pred,
11871 SCEVUse LHS, SCEVUse RHS) {
11872 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate)
11873 return false;
11874
11875 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on
11876 // the stack can result in exponential time complexity.
11877 SaveAndRestore Restore(ProvingSplitPredicate, true);
11878
11879 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L
11880 //
11881 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use
11882 // isKnownPredicate. isKnownPredicate is more powerful, but also more
11883 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the
11884 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to
11885 // use isKnownPredicate later if needed.
11886 return isKnownNonNegative(S: RHS) &&
11887 isKnownPredicate(Pred: CmpInst::ICMP_SGE, LHS, RHS: getZero(Ty: LHS->getType())) &&
11888 isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS, RHS);
11889}
11890
11891bool ScalarEvolution::isImpliedViaGuard(const BasicBlock *BB, CmpPredicate Pred,
11892 const SCEV *LHS, const SCEV *RHS) {
11893 // No need to even try if we know the module has no guards.
11894 if (!HasGuards)
11895 return false;
11896
11897 return any_of(Range: *BB, P: [&](const Instruction &I) {
11898 using namespace llvm::PatternMatch;
11899
11900 Value *Condition;
11901 return match(V: &I, P: m_Intrinsic<Intrinsic::experimental_guard>(
11902 Op0: m_Value(V&: Condition))) &&
11903 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse: false);
11904 });
11905}
11906
11907/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
11908/// protected by a conditional between LHS and RHS. This is used to
11909/// to eliminate casts.
11910bool ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
11911 CmpPredicate Pred,
11912 const SCEV *LHS,
11913 const SCEV *RHS) {
11914 // Interpret a null as meaning no loop, where there is obviously no guard
11915 // (interprocedural conditions notwithstanding). Do not bother about
11916 // unreachable loops.
11917 if (!L || !DT.isReachableFromEntry(A: L->getHeader()))
11918 return true;
11919
11920 if (VerifyIR)
11921 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()) &&
11922 "This cannot be done on broken IR!");
11923
11924
11925 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11926 return true;
11927
11928 BasicBlock *Latch = L->getLoopLatch();
11929 if (!Latch)
11930 return false;
11931
11932 CondBrInst *LoopContinuePredicate =
11933 dyn_cast<CondBrInst>(Val: Latch->getTerminator());
11934 if (LoopContinuePredicate &&
11935 isImpliedCond(Pred, LHS, RHS, FoundCondValue: LoopContinuePredicate->getCondition(),
11936 Inverse: LoopContinuePredicate->getSuccessor(i: 0) != L->getHeader()))
11937 return true;
11938
11939 // We don't want more than one activation of the following loops on the stack
11940 // -- that can lead to O(n!) time complexity.
11941 if (WalkingBEDominatingConds)
11942 return false;
11943
11944 SaveAndRestore ClearOnExit(WalkingBEDominatingConds, true);
11945
11946 // See if we can exploit a trip count to prove the predicate.
11947 const auto &BETakenInfo = getBackedgeTakenInfo(L);
11948 const SCEV *LatchBECount = BETakenInfo.getExact(ExitingBlock: Latch, SE: this);
11949 if (LatchBECount != getCouldNotCompute()) {
11950 // We know that Latch branches back to the loop header exactly
11951 // LatchBECount times. This means the backdege condition at Latch is
11952 // equivalent to "{0,+,1} u< LatchBECount".
11953 Type *Ty = LatchBECount->getType();
11954 auto NoWrapFlags = SCEV::NoWrapFlags(SCEV::FlagNUW | SCEV::FlagNW);
11955 const SCEV *LoopCounter =
11956 getAddRecExpr(Start: getZero(Ty), Step: getOne(Ty), L, Flags: NoWrapFlags);
11957 if (isImpliedCond(Pred, LHS, RHS, FoundPred: ICmpInst::ICMP_ULT, FoundLHS: LoopCounter,
11958 FoundRHS: LatchBECount))
11959 return true;
11960 }
11961
11962 // Check conditions due to any @llvm.assume intrinsics.
11963 for (auto &AssumeVH : AC.assumptions()) {
11964 if (!AssumeVH)
11965 continue;
11966 auto *CI = cast<CallInst>(Val&: AssumeVH);
11967 if (!DT.dominates(Def: CI, User: Latch->getTerminator()))
11968 continue;
11969
11970 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: CI->getArgOperand(i: 0), Inverse: false))
11971 return true;
11972 }
11973
11974 if (isImpliedViaGuard(BB: Latch, Pred, LHS, RHS))
11975 return true;
11976
11977 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()];
11978 DTN != HeaderDTN; DTN = DTN->getIDom()) {
11979 assert(DTN && "should reach the loop header before reaching the root!");
11980
11981 BasicBlock *BB = DTN->getBlock();
11982 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
11983 return true;
11984
11985 BasicBlock *PBB = BB->getSinglePredecessor();
11986 if (!PBB)
11987 continue;
11988
11989 CondBrInst *ContBr = dyn_cast<CondBrInst>(Val: PBB->getTerminator());
11990 if (!ContBr || ContBr->getSuccessor(i: 0) == ContBr->getSuccessor(i: 1))
11991 continue;
11992
11993 // If we have an edge `E` within the loop body that dominates the only
11994 // latch, the condition guarding `E` also guards the backedge. This
11995 // reasoning works only for loops with a single latch.
11996 // We're constructively (and conservatively) enumerating edges within the
11997 // loop body that dominate the latch. The dominator tree better agree
11998 // with us on this:
11999 assert(DT.dominates(BasicBlockEdge(PBB, BB), Latch) && "should be!");
12000 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: ContBr->getCondition(),
12001 Inverse: BB != ContBr->getSuccessor(i: 0)))
12002 return true;
12003 }
12004
12005 return false;
12006}
12007
12008bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
12009 CmpPredicate Pred,
12010 const SCEV *LHS,
12011 const SCEV *RHS) {
12012 // Do not bother proving facts for unreachable code.
12013 if (!DT.isReachableFromEntry(A: BB))
12014 return true;
12015 if (VerifyIR)
12016 assert(!verifyFunction(*BB->getParent(), &dbgs()) &&
12017 "This cannot be done on broken IR!");
12018
12019 // If we cannot prove strict comparison (e.g. a > b), maybe we can prove
12020 // the facts (a >= b && a != b) separately. A typical situation is when the
12021 // non-strict comparison is known from ranges and non-equality is known from
12022 // dominating predicates. If we are proving strict comparison, we always try
12023 // to prove non-equality and non-strict comparison separately.
12024 CmpPredicate NonStrictPredicate = ICmpInst::getNonStrictCmpPredicate(Pred);
12025 const bool ProvingStrictComparison =
12026 Pred != NonStrictPredicate.dropSameSign();
12027 bool ProvedNonStrictComparison = false;
12028 bool ProvedNonEquality = false;
12029
12030 auto SplitAndProve = [&](std::function<bool(CmpPredicate)> Fn) -> bool {
12031 if (!ProvedNonStrictComparison)
12032 ProvedNonStrictComparison = Fn(NonStrictPredicate);
12033 if (!ProvedNonEquality)
12034 ProvedNonEquality = Fn(ICmpInst::ICMP_NE);
12035 if (ProvedNonStrictComparison && ProvedNonEquality)
12036 return true;
12037 return false;
12038 };
12039
12040 if (ProvingStrictComparison) {
12041 auto ProofFn = [&](CmpPredicate P) {
12042 return isKnownViaNonRecursiveReasoning(Pred: P, LHS, RHS);
12043 };
12044 if (SplitAndProve(ProofFn))
12045 return true;
12046 }
12047
12048 // Try to prove (Pred, LHS, RHS) using isImpliedCond.
12049 auto ProveViaCond = [&](const Value *Condition, bool Inverse) {
12050 const Instruction *CtxI = &BB->front();
12051 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI))
12052 return true;
12053 if (ProvingStrictComparison) {
12054 auto ProofFn = [&](CmpPredicate P) {
12055 return isImpliedCond(Pred: P, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI);
12056 };
12057 if (SplitAndProve(ProofFn))
12058 return true;
12059 }
12060 return false;
12061 };
12062
12063 // Starting at the block's predecessor, climb up the predecessor chain, as long
12064 // as there are predecessors that can be found that have unique successors
12065 // leading to the original block.
12066 const Loop *ContainingLoop = LI.getLoopFor(BB);
12067 const BasicBlock *PredBB;
12068 if (ContainingLoop && ContainingLoop->getHeader() == BB)
12069 PredBB = ContainingLoop->getLoopPredecessor();
12070 else
12071 PredBB = BB->getSinglePredecessor();
12072 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB);
12073 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
12074 const CondBrInst *BlockEntryPredicate =
12075 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
12076 if (!BlockEntryPredicate)
12077 continue;
12078
12079 if (ProveViaCond(BlockEntryPredicate->getCondition(),
12080 BlockEntryPredicate->getSuccessor(i: 0) != Pair.second))
12081 return true;
12082 }
12083
12084 // Check conditions due to any @llvm.assume intrinsics.
12085 for (auto &AssumeVH : AC.assumptions()) {
12086 if (!AssumeVH)
12087 continue;
12088 auto *CI = cast<CallInst>(Val&: AssumeVH);
12089 if (!DT.dominates(Def: CI, BB))
12090 continue;
12091
12092 if (ProveViaCond(CI->getArgOperand(i: 0), false))
12093 return true;
12094 }
12095
12096 // Check conditions due to any @llvm.experimental.guard intrinsics.
12097 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
12098 M: F.getParent(), id: Intrinsic::experimental_guard);
12099 if (GuardDecl)
12100 for (const auto *GU : GuardDecl->users())
12101 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
12102 if (Guard->getFunction() == BB->getParent() && DT.dominates(Def: Guard, BB))
12103 if (ProveViaCond(Guard->getArgOperand(i: 0), false))
12104 return true;
12105 return false;
12106}
12107
12108bool ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred,
12109 const SCEV *LHS,
12110 const SCEV *RHS) {
12111 // Interpret a null as meaning no loop, where there is obviously no guard
12112 // (interprocedural conditions notwithstanding).
12113 if (!L)
12114 return false;
12115
12116 // Both LHS and RHS must be available at loop entry.
12117 assert(isAvailableAtLoopEntry(LHS, L) &&
12118 "LHS is not available at Loop Entry");
12119 assert(isAvailableAtLoopEntry(RHS, L) &&
12120 "RHS is not available at Loop Entry");
12121
12122 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
12123 return true;
12124
12125 return isBasicBlockEntryGuardedByCond(BB: L->getHeader(), Pred, LHS, RHS);
12126}
12127
12128bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12129 const SCEV *RHS,
12130 const Value *FoundCondValue, bool Inverse,
12131 const Instruction *CtxI) {
12132 // False conditions implies anything. Do not bother analyzing it further.
12133 if (FoundCondValue ==
12134 ConstantInt::getBool(Context&: FoundCondValue->getContext(), V: Inverse))
12135 return true;
12136
12137 if (!PendingLoopPredicates.insert(Ptr: FoundCondValue).second)
12138 return false;
12139
12140 llvm::scope_exit ClearOnExit(
12141 [&]() { PendingLoopPredicates.erase(Ptr: FoundCondValue); });
12142
12143 // Recursively handle And and Or conditions.
12144 const Value *Op0, *Op1;
12145 if (match(V: FoundCondValue, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
12146 if (!Inverse)
12147 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12148 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12149 } else if (match(V: FoundCondValue, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
12150 if (Inverse)
12151 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12152 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12153 }
12154
12155 const ICmpInst *ICI = dyn_cast<ICmpInst>(Val: FoundCondValue);
12156 if (!ICI) return false;
12157
12158 // Now that we found a conditional branch that dominates the loop or controls
12159 // the loop latch. Check to see if it is the comparison we are looking for.
12160 CmpPredicate FoundPred;
12161 if (Inverse)
12162 FoundPred = ICI->getInverseCmpPredicate();
12163 else
12164 FoundPred = ICI->getCmpPredicate();
12165
12166 const SCEV *FoundLHS = getSCEV(V: ICI->getOperand(i_nocapture: 0));
12167 const SCEV *FoundRHS = getSCEV(V: ICI->getOperand(i_nocapture: 1));
12168
12169 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, Context: CtxI);
12170}
12171
12172bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12173 const SCEV *RHS, CmpPredicate FoundPred,
12174 const SCEV *FoundLHS, const SCEV *FoundRHS,
12175 const Instruction *CtxI) {
12176 // Balance the types.
12177 if (getTypeSizeInBits(Ty: LHS->getType()) <
12178 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12179 // For unsigned and equality predicates, try to prove that both found
12180 // operands fit into narrow unsigned range. If so, try to prove facts in
12181 // narrow types.
12182 if (!CmpInst::isSigned(Pred: FoundPred) && !FoundLHS->getType()->isPointerTy() &&
12183 !FoundRHS->getType()->isPointerTy()) {
12184 auto *NarrowType = LHS->getType();
12185 auto *WideType = FoundLHS->getType();
12186 auto BitWidth = getTypeSizeInBits(Ty: NarrowType);
12187 const SCEV *MaxValue = getZeroExtendExpr(
12188 Op: getConstant(Val: APInt::getMaxValue(numBits: BitWidth)), Ty: WideType);
12189 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundLHS,
12190 RHS: MaxValue) &&
12191 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundRHS,
12192 RHS: MaxValue)) {
12193 const SCEV *TruncFoundLHS = getTruncateExpr(Op: FoundLHS, Ty: NarrowType);
12194 const SCEV *TruncFoundRHS = getTruncateExpr(Op: FoundRHS, Ty: NarrowType);
12195 // We cannot preserve samesign after truncation.
12196 if (isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred: FoundPred.dropSameSign(),
12197 FoundLHS: TruncFoundLHS, FoundRHS: TruncFoundRHS, CtxI))
12198 return true;
12199 }
12200 }
12201
12202 if (LHS->getType()->isPointerTy() || RHS->getType()->isPointerTy())
12203 return false;
12204 if (CmpInst::isSigned(Pred)) {
12205 LHS = getSignExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12206 RHS = getSignExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12207 } else {
12208 LHS = getZeroExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12209 RHS = getZeroExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12210 }
12211 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
12212 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12213 if (FoundLHS->getType()->isPointerTy() || FoundRHS->getType()->isPointerTy())
12214 return false;
12215 if (CmpInst::isSigned(Pred: FoundPred)) {
12216 FoundLHS = getSignExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12217 FoundRHS = getSignExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12218 } else {
12219 FoundLHS = getZeroExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12220 FoundRHS = getZeroExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12221 }
12222 }
12223 return isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, FoundLHS,
12224 FoundRHS, CtxI);
12225}
12226
12227bool ScalarEvolution::isImpliedCondBalancedTypes(
12228 CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS, CmpPredicate FoundPred,
12229 SCEVUse FoundLHS, SCEVUse FoundRHS, const Instruction *CtxI) {
12230 assert(getTypeSizeInBits(LHS->getType()) ==
12231 getTypeSizeInBits(FoundLHS->getType()) &&
12232 "Types should be balanced!");
12233 // Canonicalize the query to match the way instcombine will have
12234 // canonicalized the comparison.
12235 if (SimplifyICmpOperands(Pred, LHS, RHS))
12236 if (LHS == RHS)
12237 return CmpInst::isTrueWhenEqual(predicate: Pred);
12238 if (SimplifyICmpOperands(Pred&: FoundPred, LHS&: FoundLHS, RHS&: FoundRHS))
12239 if (FoundLHS == FoundRHS)
12240 return CmpInst::isFalseWhenEqual(predicate: FoundPred);
12241
12242 // Check to see if we can make the LHS or RHS match.
12243 if (LHS == FoundRHS || RHS == FoundLHS) {
12244 if (isa<SCEVConstant>(Val: RHS)) {
12245 std::swap(a&: FoundLHS, b&: FoundRHS);
12246 FoundPred = ICmpInst::getSwappedCmpPredicate(Pred: FoundPred);
12247 } else {
12248 std::swap(a&: LHS, b&: RHS);
12249 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12250 }
12251 }
12252
12253 // Check whether the found predicate is the same as the desired predicate.
12254 if (auto P = CmpPredicate::getMatching(A: FoundPred, B: Pred))
12255 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12256
12257 // Check whether swapping the found predicate makes it the same as the
12258 // desired predicate.
12259 if (auto P = CmpPredicate::getMatching(
12260 A: ICmpInst::getSwappedCmpPredicate(Pred: FoundPred), B: Pred)) {
12261 // We can write the implication
12262 // 0. LHS Pred RHS <- FoundLHS SwapPred FoundRHS
12263 // using one of the following ways:
12264 // 1. LHS Pred RHS <- FoundRHS Pred FoundLHS
12265 // 2. RHS SwapPred LHS <- FoundLHS SwapPred FoundRHS
12266 // 3. LHS Pred RHS <- ~FoundLHS Pred ~FoundRHS
12267 // 4. ~LHS SwapPred ~RHS <- FoundLHS SwapPred FoundRHS
12268 // Forms 1. and 2. require swapping the operands of one condition. Don't
12269 // do this if it would break canonical constant/addrec ordering.
12270 if (!isa<SCEVConstant>(Val: RHS) && !isa<SCEVAddRecExpr>(Val: LHS))
12271 return isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P), LHS: RHS,
12272 RHS: LHS, FoundLHS, FoundRHS, Context: CtxI);
12273 if (!isa<SCEVConstant>(Val: FoundRHS) && !isa<SCEVAddRecExpr>(Val: FoundLHS))
12274 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: FoundRHS, FoundRHS: FoundLHS, Context: CtxI);
12275
12276 // There's no clear preference between forms 3. and 4., try both. Avoid
12277 // forming getNotSCEV of pointer values as the resulting subtract is
12278 // not legal.
12279 if (!LHS->getType()->isPointerTy() && !RHS->getType()->isPointerTy() &&
12280 isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P),
12281 LHS: getNotSCEV(V: LHS), RHS: getNotSCEV(V: RHS), FoundLHS,
12282 FoundRHS, Context: CtxI))
12283 return true;
12284
12285 if (!FoundLHS->getType()->isPointerTy() &&
12286 !FoundRHS->getType()->isPointerTy() &&
12287 isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: getNotSCEV(V: FoundLHS),
12288 FoundRHS: getNotSCEV(V: FoundRHS), Context: CtxI))
12289 return true;
12290
12291 return false;
12292 }
12293
12294 auto IsSignFlippedPredicate = [](CmpInst::Predicate P1,
12295 CmpInst::Predicate P2) {
12296 assert(P1 != P2 && "Handled earlier!");
12297 return CmpInst::isRelational(P: P2) &&
12298 P1 == ICmpInst::getFlippedSignednessPredicate(Pred: P2);
12299 };
12300 if (IsSignFlippedPredicate(Pred, FoundPred)) {
12301 // Unsigned comparison is the same as signed comparison when both the
12302 // operands are non-negative or negative.
12303 if (haveSameSign(S1: FoundLHS, S2: FoundRHS))
12304 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12305 // Create local copies that we can freely swap and canonicalize our
12306 // conditions to "le/lt".
12307 CmpPredicate CanonicalPred = Pred, CanonicalFoundPred = FoundPred;
12308 const SCEV *CanonicalLHS = LHS, *CanonicalRHS = RHS,
12309 *CanonicalFoundLHS = FoundLHS, *CanonicalFoundRHS = FoundRHS;
12310 if (ICmpInst::isGT(P: CanonicalPred) || ICmpInst::isGE(P: CanonicalPred)) {
12311 CanonicalPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalPred);
12312 CanonicalFoundPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalFoundPred);
12313 std::swap(a&: CanonicalLHS, b&: CanonicalRHS);
12314 std::swap(a&: CanonicalFoundLHS, b&: CanonicalFoundRHS);
12315 }
12316 assert((ICmpInst::isLT(CanonicalPred) || ICmpInst::isLE(CanonicalPred)) &&
12317 "Must be!");
12318 assert((ICmpInst::isLT(CanonicalFoundPred) ||
12319 ICmpInst::isLE(CanonicalFoundPred)) &&
12320 "Must be!");
12321 if (ICmpInst::isSigned(Pred: CanonicalPred) && isKnownNonNegative(S: CanonicalRHS))
12322 // Use implication:
12323 // x <u y && y >=s 0 --> x <s y.
12324 // If we can prove the left part, the right part is also proven.
12325 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
12326 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
12327 FoundRHS: CanonicalFoundRHS);
12328 if (ICmpInst::isUnsigned(Pred: CanonicalPred) && isKnownNegative(S: CanonicalRHS))
12329 // Use implication:
12330 // x <s y && y <s 0 --> x <u y.
12331 // If we can prove the left part, the right part is also proven.
12332 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
12333 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
12334 FoundRHS: CanonicalFoundRHS);
12335 }
12336
12337 // Check if we can make progress by sharpening ranges.
12338 if (FoundPred == ICmpInst::ICMP_NE &&
12339 (isa<SCEVConstant>(Val: FoundLHS) || isa<SCEVConstant>(Val: FoundRHS))) {
12340
12341 const SCEVConstant *C = nullptr;
12342 const SCEV *V = nullptr;
12343
12344 if (isa<SCEVConstant>(Val: FoundLHS)) {
12345 C = cast<SCEVConstant>(Val&: FoundLHS);
12346 V = FoundRHS;
12347 } else {
12348 C = cast<SCEVConstant>(Val&: FoundRHS);
12349 V = FoundLHS;
12350 }
12351
12352 // The guarding predicate tells us that C != V. If the known range
12353 // of V is [C, t), we can sharpen the range to [C + 1, t). The
12354 // range we consider has to correspond to same signedness as the
12355 // predicate we're interested in folding.
12356
12357 APInt Min = ICmpInst::isSigned(Pred) ?
12358 getSignedRangeMin(S: V) : getUnsignedRangeMin(S: V);
12359
12360 if (Min == C->getAPInt()) {
12361 // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
12362 // This is true even if (Min + 1) wraps around -- in case of
12363 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
12364
12365 APInt SharperMin = Min + 1;
12366
12367 switch (Pred) {
12368 case ICmpInst::ICMP_SGE:
12369 case ICmpInst::ICMP_UGE:
12370 // We know V `Pred` SharperMin. If this implies LHS `Pred`
12371 // RHS, we're done.
12372 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin),
12373 Context: CtxI))
12374 return true;
12375 [[fallthrough]];
12376
12377 case ICmpInst::ICMP_SGT:
12378 case ICmpInst::ICMP_UGT:
12379 // We know from the range information that (V `Pred` Min ||
12380 // V == Min). We know from the guarding condition that !(V
12381 // == Min). This gives us
12382 //
12383 // V `Pred` Min || V == Min && !(V == Min)
12384 // => V `Pred` Min
12385 //
12386 // If V `Pred` Min implies LHS `Pred` RHS, we're done.
12387
12388 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12389 return true;
12390 break;
12391
12392 // `LHS < RHS` and `LHS <= RHS` are handled in the same way as `RHS > LHS` and `RHS >= LHS` respectively.
12393 case ICmpInst::ICMP_SLE:
12394 case ICmpInst::ICMP_ULE:
12395 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12396 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin), Context: CtxI))
12397 return true;
12398 [[fallthrough]];
12399
12400 case ICmpInst::ICMP_SLT:
12401 case ICmpInst::ICMP_ULT:
12402 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12403 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12404 return true;
12405 break;
12406
12407 default:
12408 // No change
12409 break;
12410 }
12411 }
12412 }
12413
12414 // Check whether the actual condition is beyond sufficient.
12415 if (FoundPred == ICmpInst::ICMP_EQ)
12416 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
12417 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12418 return true;
12419 if (Pred == ICmpInst::ICMP_NE)
12420 if (!ICmpInst::isTrueWhenEqual(predicate: FoundPred))
12421 if (isImpliedCondOperands(Pred: FoundPred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12422 return true;
12423
12424 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS))
12425 return true;
12426
12427 // Otherwise assume the worst.
12428 return false;
12429}
12430
12431bool ScalarEvolution::splitBinaryAdd(SCEVUse Expr, SCEVUse &L, SCEVUse &R,
12432 SCEV::NoWrapFlags &Flags) {
12433 if (!match(U: Expr, P: m_scev_Add(Op0: m_SCEV(V&: L), Op1: m_SCEV(V&: R))))
12434 return false;
12435
12436 Flags = cast<SCEVAddExpr>(Val&: Expr)->getNoWrapFlags();
12437 return true;
12438}
12439
12440std::optional<APInt>
12441ScalarEvolution::computeConstantDifference(const SCEV *More, const SCEV *Less) {
12442 // We avoid subtracting expressions here because this function is usually
12443 // fairly deep in the call stack (i.e. is called many times).
12444
12445 unsigned BW = getTypeSizeInBits(Ty: More->getType());
12446 APInt Diff(BW, 0);
12447 APInt DiffMul(BW, 1);
12448 // Try various simplifications to reduce the difference to a constant. Limit
12449 // the number of allowed simplifications to keep compile-time low.
12450 for (unsigned I = 0; I < 8; ++I) {
12451 if (More == Less)
12452 return Diff;
12453
12454 // Reduce addrecs with identical steps to their start value.
12455 if (isa<SCEVAddRecExpr>(Val: Less) && isa<SCEVAddRecExpr>(Val: More)) {
12456 const auto *LAR = cast<SCEVAddRecExpr>(Val: Less);
12457 const auto *MAR = cast<SCEVAddRecExpr>(Val: More);
12458
12459 if (LAR->getLoop() != MAR->getLoop())
12460 return std::nullopt;
12461
12462 // We look at affine expressions only; not for correctness but to keep
12463 // getStepRecurrence cheap.
12464 if (!LAR->isAffine() || !MAR->isAffine())
12465 return std::nullopt;
12466
12467 if (LAR->getStepRecurrence(SE&: *this) != MAR->getStepRecurrence(SE&: *this))
12468 return std::nullopt;
12469
12470 Less = LAR->getStart();
12471 More = MAR->getStart();
12472 continue;
12473 }
12474
12475 // Try to match a common constant multiply.
12476 auto MatchConstMul =
12477 [](const SCEV *S) -> std::optional<std::pair<const SCEV *, APInt>> {
12478 const APInt *C;
12479 const SCEV *Op;
12480 if (match(S, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEV(V&: Op))))
12481 return {{Op, *C}};
12482 return std::nullopt;
12483 };
12484 if (auto MatchedMore = MatchConstMul(More)) {
12485 if (auto MatchedLess = MatchConstMul(Less)) {
12486 if (MatchedMore->second == MatchedLess->second) {
12487 More = MatchedMore->first;
12488 Less = MatchedLess->first;
12489 DiffMul *= MatchedMore->second;
12490 continue;
12491 }
12492 }
12493 }
12494
12495 // Try to cancel out common factors in two add expressions.
12496 SmallDenseMap<const SCEV *, int, 8> Multiplicity;
12497 auto Add = [&](const SCEV *S, int Mul) {
12498 if (auto *C = dyn_cast<SCEVConstant>(Val: S)) {
12499 if (Mul == 1) {
12500 Diff += C->getAPInt() * DiffMul;
12501 } else {
12502 assert(Mul == -1);
12503 Diff -= C->getAPInt() * DiffMul;
12504 }
12505 } else
12506 Multiplicity[S] += Mul;
12507 };
12508 auto Decompose = [&](const SCEV *S, int Mul) {
12509 if (isa<SCEVAddExpr>(Val: S)) {
12510 for (const SCEV *Op : S->operands())
12511 Add(Op, Mul);
12512 } else
12513 Add(S, Mul);
12514 };
12515 Decompose(More, 1);
12516 Decompose(Less, -1);
12517
12518 // Check whether all the non-constants cancel out, or reduce to new
12519 // More/Less values.
12520 const SCEV *NewMore = nullptr, *NewLess = nullptr;
12521 for (const auto &[S, Mul] : Multiplicity) {
12522 if (Mul == 0)
12523 continue;
12524 if (Mul == 1) {
12525 if (NewMore)
12526 return std::nullopt;
12527 NewMore = S;
12528 } else if (Mul == -1) {
12529 if (NewLess)
12530 return std::nullopt;
12531 NewLess = S;
12532 } else
12533 return std::nullopt;
12534 }
12535
12536 // Values stayed the same, no point in trying further.
12537 if (NewMore == More || NewLess == Less)
12538 return std::nullopt;
12539
12540 More = NewMore;
12541 Less = NewLess;
12542
12543 // Reduced to constant.
12544 if (!More && !Less)
12545 return Diff;
12546
12547 // Left with variable on only one side, bail out.
12548 if (!More || !Less)
12549 return std::nullopt;
12550 }
12551
12552 // Did not reduce to constant.
12553 return std::nullopt;
12554}
12555
12556bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart(
12557 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12558 const SCEV *FoundRHS, const Instruction *CtxI) {
12559 // Try to recognize the following pattern:
12560 //
12561 // FoundRHS = ...
12562 // ...
12563 // loop:
12564 // FoundLHS = {Start,+,W}
12565 // context_bb: // Basic block from the same loop
12566 // known(Pred, FoundLHS, FoundRHS)
12567 //
12568 // If some predicate is known in the context of a loop, it is also known on
12569 // each iteration of this loop, including the first iteration. Therefore, in
12570 // this case, `FoundLHS Pred FoundRHS` implies `Start Pred FoundRHS`. Try to
12571 // prove the original pred using this fact.
12572 if (!CtxI)
12573 return false;
12574 const BasicBlock *ContextBB = CtxI->getParent();
12575 // Make sure AR varies in the context block.
12576 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS)) {
12577 const Loop *L = AR->getLoop();
12578 const auto *Latch = L->getLoopLatch();
12579 // Make sure that context belongs to the loop and executes on 1st iteration
12580 // (if it ever executes at all).
12581 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12582 return false;
12583 if (!isAvailableAtLoopEntry(S: FoundRHS, L: AR->getLoop()))
12584 return false;
12585 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: AR->getStart(), FoundRHS);
12586 }
12587
12588 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundRHS)) {
12589 const Loop *L = AR->getLoop();
12590 const auto *Latch = L->getLoopLatch();
12591 // Make sure that context belongs to the loop and executes on 1st iteration
12592 // (if it ever executes at all).
12593 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12594 return false;
12595 if (!isAvailableAtLoopEntry(S: FoundLHS, L: AR->getLoop()))
12596 return false;
12597 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS: AR->getStart());
12598 }
12599
12600 return false;
12601}
12602
12603bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(CmpPredicate Pred,
12604 const SCEV *LHS,
12605 const SCEV *RHS,
12606 const SCEV *FoundLHS,
12607 const SCEV *FoundRHS) {
12608 if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT)
12609 return false;
12610
12611 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12612 if (!AddRecLHS)
12613 return false;
12614
12615 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12616 if (!AddRecFoundLHS)
12617 return false;
12618
12619 // We'd like to let SCEV reason about control dependencies, so we constrain
12620 // both the inequalities to be about add recurrences on the same loop. This
12621 // way we can use isLoopEntryGuardedByCond later.
12622
12623 const Loop *L = AddRecFoundLHS->getLoop();
12624 if (L != AddRecLHS->getLoop())
12625 return false;
12626
12627 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1)
12628 //
12629 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C)
12630 // ... (2)
12631 //
12632 // Informal proof for (2), assuming (1) [*]:
12633 //
12634 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**]
12635 //
12636 // Then
12637 //
12638 // FoundLHS s< FoundRHS s< INT_MIN - C
12639 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ]
12640 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ]
12641 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s<
12642 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ]
12643 // <=> FoundLHS + C s< FoundRHS + C
12644 //
12645 // [*]: (1) can be proved by ruling out overflow.
12646 //
12647 // [**]: This can be proved by analyzing all the four possibilities:
12648 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and
12649 // (A s>= 0, B s>= 0).
12650 //
12651 // Note:
12652 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C"
12653 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS
12654 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS
12655 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is
12656 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS +
12657 // C)".
12658
12659 std::optional<APInt> LDiff = computeConstantDifference(More: LHS, Less: FoundLHS);
12660 if (!LDiff)
12661 return false;
12662 std::optional<APInt> RDiff = computeConstantDifference(More: RHS, Less: FoundRHS);
12663 if (!RDiff || *LDiff != *RDiff)
12664 return false;
12665
12666 if (LDiff->isMinValue())
12667 return true;
12668
12669 APInt FoundRHSLimit;
12670
12671 if (Pred == CmpInst::ICMP_ULT) {
12672 FoundRHSLimit = -(*RDiff);
12673 } else {
12674 assert(Pred == CmpInst::ICMP_SLT && "Checked above!");
12675 FoundRHSLimit = APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: RHS->getType())) - *RDiff;
12676 }
12677
12678 // Try to prove (1) or (2), as needed.
12679 return isAvailableAtLoopEntry(S: FoundRHS, L) &&
12680 isLoopEntryGuardedByCond(L, Pred, LHS: FoundRHS,
12681 RHS: getConstant(Val: FoundRHSLimit));
12682}
12683
12684bool ScalarEvolution::isImpliedViaMerge(CmpPredicate Pred, const SCEV *LHS,
12685 const SCEV *RHS, const SCEV *FoundLHS,
12686 const SCEV *FoundRHS, unsigned Depth) {
12687 const PHINode *LPhi = nullptr, *RPhi = nullptr;
12688
12689 llvm::scope_exit ClearOnExit([&]() {
12690 if (LPhi) {
12691 bool Erased = PendingMerges.erase(Ptr: LPhi);
12692 assert(Erased && "Failed to erase LPhi!");
12693 (void)Erased;
12694 }
12695 if (RPhi) {
12696 bool Erased = PendingMerges.erase(Ptr: RPhi);
12697 assert(Erased && "Failed to erase RPhi!");
12698 (void)Erased;
12699 }
12700 });
12701
12702 // Find respective Phis and check that they are not being pending.
12703 if (const SCEVUnknown *LU = dyn_cast<SCEVUnknown>(Val: LHS))
12704 if (auto *Phi = dyn_cast<PHINode>(Val: LU->getValue())) {
12705 if (!PendingMerges.insert(Ptr: Phi).second)
12706 return false;
12707 LPhi = Phi;
12708 }
12709 if (const SCEVUnknown *RU = dyn_cast<SCEVUnknown>(Val: RHS))
12710 if (auto *Phi = dyn_cast<PHINode>(Val: RU->getValue())) {
12711 // If we detect a loop of Phi nodes being processed by this method, for
12712 // example:
12713 //
12714 // %a = phi i32 [ %some1, %preheader ], [ %b, %latch ]
12715 // %b = phi i32 [ %some2, %preheader ], [ %a, %latch ]
12716 //
12717 // we don't want to deal with a case that complex, so return conservative
12718 // answer false.
12719 if (!PendingMerges.insert(Ptr: Phi).second)
12720 return false;
12721 RPhi = Phi;
12722 }
12723
12724 // If none of LHS, RHS is a Phi, nothing to do here.
12725 if (!LPhi && !RPhi)
12726 return false;
12727
12728 // If there is a SCEVUnknown Phi we are interested in, make it left.
12729 if (!LPhi) {
12730 std::swap(a&: LHS, b&: RHS);
12731 std::swap(a&: FoundLHS, b&: FoundRHS);
12732 std::swap(a&: LPhi, b&: RPhi);
12733 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12734 }
12735
12736 assert(LPhi && "LPhi should definitely be a SCEVUnknown Phi!");
12737 const BasicBlock *LBB = LPhi->getParent();
12738 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(Val: RHS);
12739
12740 auto ProvedEasily = [&](const SCEV *S1, const SCEV *S2) {
12741 return isKnownViaNonRecursiveReasoning(Pred, LHS: S1, RHS: S2) ||
12742 isImpliedCondOperandsViaRanges(Pred, LHS: S1, RHS: S2, FoundPred: Pred, FoundLHS, FoundRHS) ||
12743 isImpliedViaOperations(Pred, LHS: S1, RHS: S2, FoundLHS, FoundRHS, Depth);
12744 };
12745
12746 if (RPhi && RPhi->getParent() == LBB) {
12747 // Case one: RHS is also a SCEVUnknown Phi from the same basic block.
12748 // If we compare two Phis from the same block, and for each entry block
12749 // the predicate is true for incoming values from this block, then the
12750 // predicate is also true for the Phis.
12751 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12752 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12753 const SCEV *R = getSCEV(V: RPhi->getIncomingValueForBlock(BB: IncBB));
12754 if (!ProvedEasily(L, R))
12755 return false;
12756 }
12757 } else if (RAR && RAR->getLoop()->getHeader() == LBB) {
12758 // Case two: RHS is also a Phi from the same basic block, and it is an
12759 // AddRec. It means that there is a loop which has both AddRec and Unknown
12760 // PHIs, for it we can compare incoming values of AddRec from above the loop
12761 // and latch with their respective incoming values of LPhi.
12762 // TODO: Generalize to handle loops with many inputs in a header.
12763 if (LPhi->getNumIncomingValues() != 2) return false;
12764
12765 auto *RLoop = RAR->getLoop();
12766 auto *Predecessor = RLoop->getLoopPredecessor();
12767 assert(Predecessor && "Loop with AddRec with no predecessor?");
12768 const SCEV *L1 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Predecessor));
12769 if (!ProvedEasily(L1, RAR->getStart()))
12770 return false;
12771 auto *Latch = RLoop->getLoopLatch();
12772 assert(Latch && "Loop with AddRec with no latch?");
12773 const SCEV *L2 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Latch));
12774 if (!ProvedEasily(L2, RAR->getPostIncExpr(SE&: *this)))
12775 return false;
12776 } else {
12777 // In all other cases go over inputs of LHS and compare each of them to RHS,
12778 // the predicate is true for (LHS, RHS) if it is true for all such pairs.
12779 // At this point RHS is either a non-Phi, or it is a Phi from some block
12780 // different from LBB.
12781 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12782 // Check that RHS is available in this block.
12783 if (!dominates(S: RHS, BB: IncBB))
12784 return false;
12785 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12786 // Make sure L does not refer to a value from a potentially previous
12787 // iteration of a loop.
12788 if (!properlyDominates(S: L, BB: LBB))
12789 return false;
12790 // Addrecs are considered to properly dominate their loop, so are missed
12791 // by the previous check. Discard any values that have computable
12792 // evolution in this loop.
12793 if (auto *Loop = LI.getLoopFor(BB: LBB))
12794 if (hasComputableLoopEvolution(S: L, L: Loop))
12795 return false;
12796 if (!ProvedEasily(L, RHS))
12797 return false;
12798 }
12799 }
12800 return true;
12801}
12802
12803bool ScalarEvolution::isImpliedCondOperandsViaShift(CmpPredicate Pred,
12804 const SCEV *LHS,
12805 const SCEV *RHS,
12806 const SCEV *FoundLHS,
12807 const SCEV *FoundRHS) {
12808 // We want to imply LHS < RHS from LHS < (RHS >> shiftvalue). First, make
12809 // sure that we are dealing with same LHS.
12810 if (RHS == FoundRHS) {
12811 std::swap(a&: LHS, b&: RHS);
12812 std::swap(a&: FoundLHS, b&: FoundRHS);
12813 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12814 }
12815 if (LHS != FoundLHS)
12816 return false;
12817
12818 auto *SUFoundRHS = dyn_cast<SCEVUnknown>(Val: FoundRHS);
12819 if (!SUFoundRHS)
12820 return false;
12821
12822 Value *Shiftee, *ShiftValue;
12823
12824 using namespace PatternMatch;
12825 if (match(V: SUFoundRHS->getValue(),
12826 P: m_LShr(L: m_Value(V&: Shiftee), R: m_Value(V&: ShiftValue)))) {
12827 auto *ShifteeS = getSCEV(V: Shiftee);
12828 // Prove one of the following:
12829 // LHS <u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <u RHS
12830 // LHS <=u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <=u RHS
12831 // LHS <s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12832 // ---> LHS <s RHS
12833 // LHS <=s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12834 // ---> LHS <=s RHS
12835 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
12836 return isKnownPredicate(Pred: ICmpInst::ICMP_ULE, LHS: ShifteeS, RHS);
12837 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
12838 if (isKnownNonNegative(S: ShifteeS))
12839 return isKnownPredicate(Pred: ICmpInst::ICMP_SLE, LHS: ShifteeS, RHS);
12840 }
12841
12842 return false;
12843}
12844
12845bool ScalarEvolution::isImpliedCondOperands(CmpPredicate Pred, const SCEV *LHS,
12846 const SCEV *RHS,
12847 const SCEV *FoundLHS,
12848 const SCEV *FoundRHS,
12849 const Instruction *CtxI) {
12850 return isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred: Pred, FoundLHS,
12851 FoundRHS) ||
12852 isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS,
12853 FoundRHS) ||
12854 isImpliedCondOperandsViaShift(Pred, LHS, RHS, FoundLHS, FoundRHS) ||
12855 isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
12856 CtxI) ||
12857 isImpliedCondOperandsHelper(Pred, LHS, RHS, FoundLHS, FoundRHS);
12858}
12859
12860/// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
12861template <typename MinMaxExprType>
12862static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr,
12863 const SCEV *Candidate) {
12864 const MinMaxExprType *MinMaxExpr = dyn_cast<MinMaxExprType>(MaybeMinMaxExpr);
12865 if (!MinMaxExpr)
12866 return false;
12867
12868 return is_contained(MinMaxExpr->operands(), Candidate);
12869}
12870
12871static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE,
12872 CmpPredicate Pred, const SCEV *LHS,
12873 const SCEV *RHS) {
12874 // If both sides are affine addrecs for the same loop, with equal
12875 // steps, and we know the recurrences don't wrap, then we only
12876 // need to check the predicate on the starting values.
12877
12878 if (!ICmpInst::isRelational(P: Pred))
12879 return false;
12880
12881 const SCEV *LStart, *RStart, *Step;
12882 const Loop *L;
12883 if (!match(S: LHS,
12884 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: LStart), Op1: m_SCEV(V&: Step), L: m_Loop(L))) ||
12885 !match(S: RHS, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: RStart), Op1: m_scev_Specific(S: Step),
12886 L: m_SpecificLoop(L))))
12887 return false;
12888 const SCEVAddRecExpr *LAR = cast<SCEVAddRecExpr>(Val: LHS);
12889 const SCEVAddRecExpr *RAR = cast<SCEVAddRecExpr>(Val: RHS);
12890 SCEV::NoWrapFlags NW = ICmpInst::isSigned(Pred) ?
12891 SCEV::FlagNSW : SCEV::FlagNUW;
12892 if (!LAR->getNoWrapFlags(Mask: NW) || !RAR->getNoWrapFlags(Mask: NW))
12893 return false;
12894
12895 return SE.isKnownPredicate(Pred, LHS: LStart, RHS: RStart);
12896}
12897
12898/// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
12899/// expression?
12900static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, CmpPredicate Pred,
12901 const SCEV *LHS, const SCEV *RHS) {
12902 switch (Pred) {
12903 default:
12904 return false;
12905
12906 case ICmpInst::ICMP_SGE:
12907 std::swap(a&: LHS, b&: RHS);
12908 [[fallthrough]];
12909 case ICmpInst::ICMP_SLE:
12910 return
12911 // min(A, ...) <= A
12912 IsMinMaxConsistingOf<SCEVSMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12913 // A <= max(A, ...)
12914 IsMinMaxConsistingOf<SCEVSMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12915
12916 case ICmpInst::ICMP_UGE:
12917 std::swap(a&: LHS, b&: RHS);
12918 [[fallthrough]];
12919 case ICmpInst::ICMP_ULE:
12920 return
12921 // min(A, ...) <= A
12922 // FIXME: what about umin_seq?
12923 IsMinMaxConsistingOf<SCEVUMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12924 // A <= max(A, ...)
12925 IsMinMaxConsistingOf<SCEVUMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12926 }
12927
12928 llvm_unreachable("covered switch fell through?!");
12929}
12930
12931bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred, const SCEV *LHS,
12932 const SCEV *RHS,
12933 const SCEV *FoundLHS,
12934 const SCEV *FoundRHS,
12935 unsigned Depth) {
12936 assert(getTypeSizeInBits(LHS->getType()) ==
12937 getTypeSizeInBits(RHS->getType()) &&
12938 "LHS and RHS have different sizes?");
12939 assert(getTypeSizeInBits(FoundLHS->getType()) ==
12940 getTypeSizeInBits(FoundRHS->getType()) &&
12941 "FoundLHS and FoundRHS have different sizes?");
12942 // We want to avoid hurting the compile time with analysis of too big trees.
12943 if (Depth > MaxSCEVOperationsImplicationDepth)
12944 return false;
12945
12946 // We only want to work with GT comparison so far.
12947 if (ICmpInst::isLT(P: Pred)) {
12948 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12949 std::swap(a&: LHS, b&: RHS);
12950 std::swap(a&: FoundLHS, b&: FoundRHS);
12951 }
12952
12953 CmpInst::Predicate P = Pred.getPreferredSignedPredicate();
12954
12955 // For unsigned, try to reduce it to corresponding signed comparison.
12956 if (P == ICmpInst::ICMP_UGT)
12957 // We can replace unsigned predicate with its signed counterpart if all
12958 // involved values are non-negative.
12959 // TODO: We could have better support for unsigned.
12960 if (isKnownNonNegative(S: FoundLHS) && isKnownNonNegative(S: FoundRHS)) {
12961 // Knowing that both FoundLHS and FoundRHS are non-negative, and knowing
12962 // FoundLHS >u FoundRHS, we also know that FoundLHS >s FoundRHS. Let us
12963 // use this fact to prove that LHS and RHS are non-negative.
12964 const SCEV *MinusOne = getMinusOne(Ty: LHS->getType());
12965 if (isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS, RHS: MinusOne, FoundLHS,
12966 FoundRHS) &&
12967 isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS: RHS, RHS: MinusOne, FoundLHS,
12968 FoundRHS))
12969 P = ICmpInst::ICMP_SGT;
12970 }
12971
12972 if (P != ICmpInst::ICMP_SGT)
12973 return false;
12974
12975 auto GetOpFromSExt = [&](const SCEV *S) -> const SCEV * {
12976 if (auto *Ext = dyn_cast<SCEVSignExtendExpr>(Val: S))
12977 return Ext->getOperand();
12978 // TODO: If S is a SCEVConstant then you can cheaply "strip" the sext off
12979 // the constant in some cases.
12980 return S;
12981 };
12982
12983 // Acquire values from extensions.
12984 auto *OrigLHS = LHS;
12985 auto *OrigFoundLHS = FoundLHS;
12986 LHS = GetOpFromSExt(LHS);
12987 FoundLHS = GetOpFromSExt(FoundLHS);
12988
12989 // Is the SGT predicate can be proved trivially or using the found context.
12990 auto IsSGTViaContext = [&](const SCEV *S1, const SCEV *S2) {
12991 return isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2) ||
12992 isImpliedViaOperations(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2, FoundLHS: OrigFoundLHS,
12993 FoundRHS, Depth: Depth + 1);
12994 };
12995
12996 if (auto *LHSAddExpr = dyn_cast<SCEVAddExpr>(Val: LHS)) {
12997 // We want to avoid creation of any new non-constant SCEV. Since we are
12998 // going to compare the operands to RHS, we should be certain that we don't
12999 // need any size extensions for this. So let's decline all cases when the
13000 // sizes of types of LHS and RHS do not match.
13001 // TODO: Maybe try to get RHS from sext to catch more cases?
13002 if (getTypeSizeInBits(Ty: LHS->getType()) != getTypeSizeInBits(Ty: RHS->getType()))
13003 return false;
13004
13005 // Should not overflow.
13006 if (!LHSAddExpr->hasNoSignedWrap())
13007 return false;
13008
13009 SCEVUse LL = LHSAddExpr->getOperand(i: 0);
13010 SCEVUse LR = LHSAddExpr->getOperand(i: 1);
13011 auto *MinusOne = getMinusOne(Ty: RHS->getType());
13012
13013 // Checks that S1 >= 0 && S2 > RHS, trivially or using the found context.
13014 auto IsSumGreaterThanRHS = [&](const SCEV *S1, const SCEV *S2) {
13015 return IsSGTViaContext(S1, MinusOne) && IsSGTViaContext(S2, RHS);
13016 };
13017 // Try to prove the following rule:
13018 // (LHS = LL + LR) && (LL >= 0) && (LR > RHS) => (LHS > RHS).
13019 // (LHS = LL + LR) && (LR >= 0) && (LL > RHS) => (LHS > RHS).
13020 if (IsSumGreaterThanRHS(LL, LR) || IsSumGreaterThanRHS(LR, LL))
13021 return true;
13022 } else if (auto *LHSUnknownExpr = dyn_cast<SCEVUnknown>(Val: LHS)) {
13023 Value *LL, *LR;
13024 // FIXME: Once we have SDiv implemented, we can get rid of this matching.
13025
13026 using namespace llvm::PatternMatch;
13027
13028 if (match(V: LHSUnknownExpr->getValue(), P: m_SDiv(L: m_Value(V&: LL), R: m_Value(V&: LR)))) {
13029 // Rules for division.
13030 // We are going to perform some comparisons with Denominator and its
13031 // derivative expressions. In general case, creating a SCEV for it may
13032 // lead to a complex analysis of the entire graph, and in particular it
13033 // can request trip count recalculation for the same loop. This would
13034 // cache as SCEVCouldNotCompute to avoid the infinite recursion. To avoid
13035 // this, we only want to create SCEVs that are constants in this section.
13036 // So we bail if Denominator is not a constant.
13037 if (!isa<ConstantInt>(Val: LR))
13038 return false;
13039
13040 auto *Denominator = cast<SCEVConstant>(Val: getSCEV(V: LR));
13041
13042 // We want to make sure that LHS = FoundLHS / Denominator. If it is so,
13043 // then a SCEV for the numerator already exists and matches with FoundLHS.
13044 auto *Numerator = getExistingSCEV(V: LL);
13045 if (!Numerator || Numerator->getType() != FoundLHS->getType())
13046 return false;
13047
13048 // Make sure that the numerator matches with FoundLHS and the denominator
13049 // is positive.
13050 if (!HasSameValue(A: Numerator, B: FoundLHS) || !isKnownPositive(S: Denominator))
13051 return false;
13052
13053 auto *DTy = Denominator->getType();
13054 auto *FRHSTy = FoundRHS->getType();
13055 if (DTy->isPointerTy() != FRHSTy->isPointerTy())
13056 // One of types is a pointer and another one is not. We cannot extend
13057 // them properly to a wider type, so let us just reject this case.
13058 // TODO: Usage of getEffectiveSCEVType for DTy, FRHSTy etc should help
13059 // to avoid this check.
13060 return false;
13061
13062 // Given that:
13063 // FoundLHS > FoundRHS, LHS = FoundLHS / Denominator, Denominator > 0.
13064 auto *WTy = getWiderType(T1: DTy, T2: FRHSTy);
13065 auto *DenominatorExt = getNoopOrSignExtend(V: Denominator, Ty: WTy);
13066 auto *FoundRHSExt = getNoopOrSignExtend(V: FoundRHS, Ty: WTy);
13067
13068 // Try to prove the following rule:
13069 // (FoundRHS > Denominator - 2) && (RHS <= 0) => (LHS > RHS).
13070 // For example, given that FoundLHS > 2. It means that FoundLHS is at
13071 // least 3. If we divide it by Denominator < 4, we will have at least 1.
13072 auto *DenomMinusTwo = getMinusSCEV(LHS: DenominatorExt, RHS: getConstant(Ty: WTy, V: 2));
13073 if (isKnownNonPositive(S: RHS) &&
13074 IsSGTViaContext(FoundRHSExt, DenomMinusTwo))
13075 return true;
13076
13077 // Try to prove the following rule:
13078 // (FoundRHS > -1 - Denominator) && (RHS < 0) => (LHS > RHS).
13079 // For example, given that FoundLHS > -3. Then FoundLHS is at least -2.
13080 // If we divide it by Denominator > 2, then:
13081 // 1. If FoundLHS is negative, then the result is 0.
13082 // 2. If FoundLHS is non-negative, then the result is non-negative.
13083 // Anyways, the result is non-negative.
13084 auto *MinusOne = getMinusOne(Ty: WTy);
13085 auto *NegDenomMinusOne = getMinusSCEV(LHS: MinusOne, RHS: DenominatorExt);
13086 if (isKnownNegative(S: RHS) &&
13087 IsSGTViaContext(FoundRHSExt, NegDenomMinusOne))
13088 return true;
13089 }
13090 }
13091
13092 // If our expression contained SCEVUnknown Phis, and we split it down and now
13093 // need to prove something for them, try to prove the predicate for every
13094 // possible incoming values of those Phis.
13095 if (isImpliedViaMerge(Pred, LHS: OrigLHS, RHS, FoundLHS: OrigFoundLHS, FoundRHS, Depth: Depth + 1))
13096 return true;
13097
13098 return false;
13099}
13100
13101static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS,
13102 const SCEV *RHS) {
13103 // zext x u<= sext x, sext x s<= zext x
13104 const SCEV *Op;
13105 switch (Pred) {
13106 case ICmpInst::ICMP_SGE:
13107 std::swap(a&: LHS, b&: RHS);
13108 [[fallthrough]];
13109 case ICmpInst::ICMP_SLE: {
13110 // If operand >=s 0 then ZExt == SExt. If operand <s 0 then SExt <s ZExt.
13111 return match(S: LHS, P: m_scev_SExt(Op0: m_SCEV(V&: Op))) &&
13112 match(S: RHS, P: m_scev_ZExt(Op0: m_scev_Specific(S: Op)));
13113 }
13114 case ICmpInst::ICMP_UGE:
13115 std::swap(a&: LHS, b&: RHS);
13116 [[fallthrough]];
13117 case ICmpInst::ICMP_ULE: {
13118 // If operand >=u 0 then ZExt == SExt. If operand <u 0 then ZExt <u SExt.
13119 return match(S: LHS, P: m_scev_ZExt(Op0: m_SCEV(V&: Op))) &&
13120 match(S: RHS, P: m_scev_SExt(Op0: m_scev_Specific(S: Op)));
13121 }
13122 default:
13123 return false;
13124 };
13125 llvm_unreachable("unhandled case");
13126}
13127
13128bool ScalarEvolution::isKnownViaNonRecursiveReasoning(CmpPredicate Pred,
13129 SCEVUse LHS,
13130 SCEVUse RHS) {
13131 return isKnownPredicateExtendIdiom(Pred, LHS, RHS) ||
13132 isKnownPredicateViaConstantRanges(Pred, LHS, RHS) ||
13133 IsKnownPredicateViaMinOrMax(SE&: *this, Pred, LHS, RHS) ||
13134 IsKnownPredicateViaAddRecStart(SE&: *this, Pred, LHS, RHS) ||
13135 isKnownPredicateViaNoOverflow(Pred, LHS, RHS);
13136}
13137
13138bool ScalarEvolution::isImpliedCondOperandsHelper(CmpPredicate Pred,
13139 const SCEV *LHS,
13140 const SCEV *RHS,
13141 const SCEV *FoundLHS,
13142 const SCEV *FoundRHS) {
13143 switch (Pred) {
13144 default:
13145 llvm_unreachable("Unexpected CmpPredicate value!");
13146 case ICmpInst::ICMP_EQ:
13147 case ICmpInst::ICMP_NE:
13148 if (HasSameValue(A: LHS, B: FoundLHS) && HasSameValue(A: RHS, B: FoundRHS))
13149 return true;
13150 break;
13151 case ICmpInst::ICMP_SLT:
13152 case ICmpInst::ICMP_SLE:
13153 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS, RHS: FoundLHS) &&
13154 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS: RHS, RHS: FoundRHS))
13155 return true;
13156 break;
13157 case ICmpInst::ICMP_SGT:
13158 case ICmpInst::ICMP_SGE:
13159 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS, RHS: FoundLHS) &&
13160 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS: RHS, RHS: FoundRHS))
13161 return true;
13162 break;
13163 case ICmpInst::ICMP_ULT:
13164 case ICmpInst::ICMP_ULE:
13165 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS, RHS: FoundLHS) &&
13166 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS: RHS, RHS: FoundRHS))
13167 return true;
13168 break;
13169 case ICmpInst::ICMP_UGT:
13170 case ICmpInst::ICMP_UGE:
13171 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS, RHS: FoundLHS) &&
13172 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: RHS, RHS: FoundRHS))
13173 return true;
13174 break;
13175 }
13176
13177 // Maybe it can be proved via operations?
13178 if (isImpliedViaOperations(Pred, LHS, RHS, FoundLHS, FoundRHS))
13179 return true;
13180
13181 return false;
13182}
13183
13184bool ScalarEvolution::isImpliedCondOperandsViaRanges(
13185 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, CmpPredicate FoundPred,
13186 const SCEV *FoundLHS, const SCEV *FoundRHS) {
13187 if (!isa<SCEVConstant>(Val: RHS) || !isa<SCEVConstant>(Val: FoundRHS))
13188 // The restriction on `FoundRHS` be lifted easily -- it exists only to
13189 // reduce the compile time impact of this optimization.
13190 return false;
13191
13192 std::optional<APInt> Addend = computeConstantDifference(More: LHS, Less: FoundLHS);
13193 if (!Addend)
13194 return false;
13195
13196 const APInt &ConstFoundRHS = cast<SCEVConstant>(Val: FoundRHS)->getAPInt();
13197
13198 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the
13199 // antecedent "`FoundLHS` `FoundPred` `FoundRHS`".
13200 ConstantRange FoundLHSRange =
13201 ConstantRange::makeExactICmpRegion(Pred: FoundPred, Other: ConstFoundRHS);
13202
13203 // Since `LHS` is `FoundLHS` + `Addend`, we can compute a range for `LHS`:
13204 ConstantRange LHSRange = FoundLHSRange.add(Other: ConstantRange(*Addend));
13205
13206 // We can also compute the range of values for `LHS` that satisfy the
13207 // consequent, "`LHS` `Pred` `RHS`":
13208 const APInt &ConstRHS = cast<SCEVConstant>(Val: RHS)->getAPInt();
13209 // The antecedent implies the consequent if every value of `LHS` that
13210 // satisfies the antecedent also satisfies the consequent.
13211 return LHSRange.icmp(Pred, Other: ConstRHS);
13212}
13213
13214bool ScalarEvolution::canIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride,
13215 bool IsSigned) {
13216 assert(isKnownPositive(Stride) && "Positive stride expected!");
13217
13218 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
13219 const SCEV *One = getOne(Ty: Stride->getType());
13220
13221 if (IsSigned) {
13222 APInt MaxRHS = getSignedRangeMax(S: RHS);
13223 APInt MaxValue = APInt::getSignedMaxValue(numBits: BitWidth);
13224 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13225
13226 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow!
13227 return (std::move(MaxValue) - MaxStrideMinusOne).slt(RHS: MaxRHS);
13228 }
13229
13230 APInt MaxRHS = getUnsignedRangeMax(S: RHS);
13231 APInt MaxValue = APInt::getMaxValue(numBits: BitWidth);
13232 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13233
13234 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow!
13235 return (std::move(MaxValue) - MaxStrideMinusOne).ult(RHS: MaxRHS);
13236}
13237
13238bool ScalarEvolution::canIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride,
13239 bool IsSigned) {
13240
13241 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
13242 const SCEV *One = getOne(Ty: Stride->getType());
13243
13244 if (IsSigned) {
13245 APInt MinRHS = getSignedRangeMin(S: RHS);
13246 APInt MinValue = APInt::getSignedMinValue(numBits: BitWidth);
13247 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13248
13249 // SMinRHS - SMaxStrideMinusOne < SMinValue => overflow!
13250 return (std::move(MinValue) + MaxStrideMinusOne).sgt(RHS: MinRHS);
13251 }
13252
13253 APInt MinRHS = getUnsignedRangeMin(S: RHS);
13254 APInt MinValue = APInt::getMinValue(numBits: BitWidth);
13255 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13256
13257 // UMinRHS - UMaxStrideMinusOne < UMinValue => overflow!
13258 return (std::move(MinValue) + MaxStrideMinusOne).ugt(RHS: MinRHS);
13259}
13260
13261const SCEV *ScalarEvolution::getUDivCeilSCEV(const SCEV *N, const SCEV *D) {
13262 // umin(N, 1) + floor((N - umin(N, 1)) / D)
13263 // This is equivalent to "1 + floor((N - 1) / D)" for N != 0. The umin
13264 // expression fixes the case of N=0.
13265 const SCEV *MinNOne = getUMinExpr(LHS: N, RHS: getOne(Ty: N->getType()));
13266 const SCEV *NMinusOne = getMinusSCEV(LHS: N, RHS: MinNOne);
13267 return getAddExpr(LHS: MinNOne, RHS: getUDivExpr(LHS: NMinusOne, RHS: D));
13268}
13269
13270const SCEV *ScalarEvolution::computeMaxBECountForLT(const SCEV *Start,
13271 const SCEV *Stride,
13272 const SCEV *End,
13273 unsigned BitWidth,
13274 bool IsSigned) {
13275 // The logic in this function assumes we can represent a positive stride.
13276 // If we can't, the backedge-taken count must be zero.
13277 if (IsSigned && BitWidth == 1)
13278 return getZero(Ty: Stride->getType());
13279
13280 // This code below only been closely audited for negative strides in the
13281 // unsigned comparison case, it may be correct for signed comparison, but
13282 // that needs to be established.
13283 if (IsSigned && isKnownNegative(S: Stride))
13284 return getCouldNotCompute();
13285
13286 // Calculate the maximum backedge count based on the range of values
13287 // permitted by Start, End, and Stride.
13288 APInt MinStart =
13289 IsSigned ? getSignedRangeMin(S: Start) : getUnsignedRangeMin(S: Start);
13290
13291 APInt MinStride =
13292 IsSigned ? getSignedRangeMin(S: Stride) : getUnsignedRangeMin(S: Stride);
13293
13294 // We assume either the stride is positive, or the backedge-taken count
13295 // is zero. So force StrideForMaxBECount to be at least one.
13296 APInt One(BitWidth, 1);
13297 APInt StrideForMaxBECount = IsSigned ? APIntOps::smax(A: One, B: MinStride)
13298 : APIntOps::umax(A: One, B: MinStride);
13299
13300 APInt MaxValue = IsSigned ? APInt::getSignedMaxValue(numBits: BitWidth)
13301 : APInt::getMaxValue(numBits: BitWidth);
13302 APInt Limit = MaxValue - (StrideForMaxBECount - 1);
13303
13304 // Although End can be a MAX expression we estimate MaxEnd considering only
13305 // the case End = RHS of the loop termination condition. This is safe because
13306 // in the other case (End - Start) is zero, leading to a zero maximum backedge
13307 // taken count.
13308 APInt MaxEnd = IsSigned ? APIntOps::smin(A: getSignedRangeMax(S: End), B: Limit)
13309 : APIntOps::umin(A: getUnsignedRangeMax(S: End), B: Limit);
13310
13311 // MaxBECount = ceil((max(MaxEnd, MinStart) - MinStart) / Stride)
13312 MaxEnd = IsSigned ? APIntOps::smax(A: MaxEnd, B: MinStart)
13313 : APIntOps::umax(A: MaxEnd, B: MinStart);
13314
13315 return getUDivCeilSCEV(N: getConstant(Val: MaxEnd - MinStart) /* Delta */,
13316 D: getConstant(Val: StrideForMaxBECount) /* Step */);
13317}
13318
13319ScalarEvolution::ExitLimit
13320ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
13321 const Loop *L, bool IsSigned,
13322 bool ControlsOnlyExit, bool AllowPredicates) {
13323 SmallVector<const SCEVPredicate *> Predicates;
13324
13325 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
13326 bool PredicatedIV = false;
13327 if (!IV) {
13328 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: LHS)) {
13329 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: ZExt->getOperand());
13330 if (AR && AR->getLoop() == L && AR->isAffine()) {
13331 auto canProveNUW = [&]() {
13332 // We can use the comparison to infer no-wrap flags only if it fully
13333 // controls the loop exit.
13334 if (!ControlsOnlyExit)
13335 return false;
13336
13337 if (!isLoopInvariant(S: RHS, L))
13338 return false;
13339
13340 if (!isKnownNonZero(S: AR->getStepRecurrence(SE&: *this)))
13341 // We need the sequence defined by AR to strictly increase in the
13342 // unsigned integer domain for the logic below to hold.
13343 return false;
13344
13345 const unsigned InnerBitWidth = getTypeSizeInBits(Ty: AR->getType());
13346 const unsigned OuterBitWidth = getTypeSizeInBits(Ty: RHS->getType());
13347 // If RHS <=u Limit, then there must exist a value V in the sequence
13348 // defined by AR (e.g. {Start,+,Step}) such that V >u RHS, and
13349 // V <=u UINT_MAX. Thus, we must exit the loop before unsigned
13350 // overflow occurs. This limit also implies that a signed comparison
13351 // (in the wide bitwidth) is equivalent to an unsigned comparison as
13352 // the high bits on both sides must be zero.
13353 APInt StrideMax = getUnsignedRangeMax(S: AR->getStepRecurrence(SE&: *this));
13354 APInt Limit = APInt::getMaxValue(numBits: InnerBitWidth) - (StrideMax - 1);
13355 Limit = Limit.zext(width: OuterBitWidth);
13356 return getUnsignedRangeMax(S: applyLoopGuards(Expr: RHS, L)).ule(RHS: Limit);
13357 };
13358 auto Flags = AR->getNoWrapFlags();
13359 if (!hasFlags(Flags, TestFlags: SCEV::FlagNUW) && canProveNUW())
13360 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
13361
13362 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
13363 if (AR->hasNoUnsignedWrap()) {
13364 // Emulate what getZeroExtendExpr would have done during construction
13365 // if we'd been able to infer the fact just above at that time.
13366 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
13367 Type *Ty = ZExt->getType();
13368 auto *S = getAddRecExpr(
13369 Start: getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: 0),
13370 Step: getZeroExtendExpr(Op: Step, Ty, Depth: 0), L, Flags: AR->getNoWrapFlags());
13371 IV = dyn_cast<SCEVAddRecExpr>(Val: S);
13372 }
13373 }
13374 }
13375 }
13376
13377
13378 if (!IV && AllowPredicates) {
13379 // Try to make this an AddRec using runtime tests, in the first X
13380 // iterations of this loop, where X is the SCEV expression found by the
13381 // algorithm below.
13382 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13383 PredicatedIV = true;
13384 }
13385
13386 // Avoid weird loops
13387 if (!IV || IV->getLoop() != L || !IV->isAffine())
13388 return getCouldNotCompute();
13389
13390 // A precondition of this method is that the condition being analyzed
13391 // reaches an exiting branch which dominates the latch. Given that, we can
13392 // assume that an increment which violates the nowrap specification and
13393 // produces poison must cause undefined behavior when the resulting poison
13394 // value is branched upon and thus we can conclude that the backedge is
13395 // taken no more often than would be required to produce that poison value.
13396 // Note that a well defined loop can exit on the iteration which violates
13397 // the nowrap specification if there is another exit (either explicit or
13398 // implicit/exceptional) which causes the loop to execute before the
13399 // exiting instruction we're analyzing would trigger UB.
13400 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13401 bool NoWrap = ControlsOnlyExit && any(Val: IV->getNoWrapFlags(Mask: WrapType));
13402 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13403
13404 const SCEV *Stride = IV->getStepRecurrence(SE&: *this);
13405
13406 bool PositiveStride = isKnownPositive(S: Stride);
13407
13408 // Avoid negative or zero stride values.
13409 if (!PositiveStride) {
13410 // We can compute the correct backedge taken count for loops with unknown
13411 // strides if we can prove that the loop is not an infinite loop with side
13412 // effects. Here's the loop structure we are trying to handle -
13413 //
13414 // i = start
13415 // do {
13416 // A[i] = i;
13417 // i += s;
13418 // } while (i < end);
13419 //
13420 // The backedge taken count for such loops is evaluated as -
13421 // (max(end, start + stride) - start - 1) /u stride
13422 //
13423 // The additional preconditions that we need to check to prove correctness
13424 // of the above formula is as follows -
13425 //
13426 // a) IV is either nuw or nsw depending upon signedness (indicated by the
13427 // NoWrap flag).
13428 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13429 // no side effects within the loop)
13430 // c) loop has a single static exit (with no abnormal exits)
13431 //
13432 // Precondition a) implies that if the stride is negative, this is a single
13433 // trip loop. The backedge taken count formula reduces to zero in this case.
13434 //
13435 // Precondition b) and c) combine to imply that if rhs is invariant in L,
13436 // then a zero stride means the backedge can't be taken without executing
13437 // undefined behavior.
13438 //
13439 // The positive stride case is the same as isKnownPositive(Stride) returning
13440 // true (original behavior of the function).
13441 //
13442 if (PredicatedIV || !NoWrap || !loopIsFiniteByAssumption(L) ||
13443 !loopHasNoAbnormalExits(L))
13444 return getCouldNotCompute();
13445
13446 if (!isKnownNonZero(S: Stride)) {
13447 // If we have a step of zero, and RHS isn't invariant in L, we don't know
13448 // if it might eventually be greater than start and if so, on which
13449 // iteration. We can't even produce a useful upper bound.
13450 if (!isLoopInvariant(S: RHS, L))
13451 return getCouldNotCompute();
13452
13453 // We allow a potentially zero stride, but we need to divide by stride
13454 // below. Since the loop can't be infinite and this check must control
13455 // the sole exit, we can infer the exit must be taken on the first
13456 // iteration (e.g. backedge count = 0) if the stride is zero. Given that,
13457 // we know the numerator in the divides below must be zero, so we can
13458 // pick an arbitrary non-zero value for the denominator (e.g. stride)
13459 // and produce the right result.
13460 // FIXME: Handle the case where Stride is poison?
13461 auto wouldZeroStrideBeUB = [&]() {
13462 // Proof by contradiction. Suppose the stride were zero. If we can
13463 // prove that the backedge *is* taken on the first iteration, then since
13464 // we know this condition controls the sole exit, we must have an
13465 // infinite loop. We can't have a (well defined) infinite loop per
13466 // check just above.
13467 // Note: The (Start - Stride) term is used to get the start' term from
13468 // (start' + stride,+,stride). Remember that we only care about the
13469 // result of this expression when stride == 0 at runtime.
13470 auto *StartIfZero = getMinusSCEV(LHS: IV->getStart(), RHS: Stride);
13471 return isLoopEntryGuardedByCond(L, Pred: Cond, LHS: StartIfZero, RHS);
13472 };
13473 if (!wouldZeroStrideBeUB()) {
13474 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13475 }
13476 }
13477 } else if (!NoWrap) {
13478 // Avoid proven overflow cases: this will ensure that the backedge taken
13479 // count will not generate any unsigned overflow.
13480 if (canIVOverflowOnLT(RHS, Stride, IsSigned))
13481 return getCouldNotCompute();
13482 }
13483
13484 // On all paths just preceeding, we established the following invariant:
13485 // IV can be assumed not to overflow up to and including the exiting
13486 // iteration. We proved this in one of two ways:
13487 // 1) We can show overflow doesn't occur before the exiting iteration
13488 // 1a) canIVOverflowOnLT, and b) step of one
13489 // 2) We can show that if overflow occurs, the loop must execute UB
13490 // before any possible exit.
13491 // Note that we have not yet proved RHS invariant (in general).
13492
13493 const SCEV *Start = IV->getStart();
13494
13495 // Preserve pointer-typed Start/RHS to pass to isLoopEntryGuardedByCond.
13496 // If we convert to integers, isLoopEntryGuardedByCond will miss some cases.
13497 // Use integer-typed versions for actual computation; we can't subtract
13498 // pointers in general.
13499 const SCEV *OrigStart = Start;
13500 const SCEV *OrigRHS = RHS;
13501 if (Start->getType()->isPointerTy()) {
13502 Start = getLosslessPtrToIntExpr(Op: Start);
13503 if (isa<SCEVCouldNotCompute>(Val: Start))
13504 return Start;
13505 }
13506 if (RHS->getType()->isPointerTy()) {
13507 RHS = getLosslessPtrToIntExpr(Op: RHS);
13508 if (isa<SCEVCouldNotCompute>(Val: RHS))
13509 return RHS;
13510 }
13511
13512 const SCEV *End = nullptr, *BECount = nullptr,
13513 *BECountIfBackedgeTaken = nullptr;
13514 if (!isLoopInvariant(S: RHS, L)) {
13515 const auto *RHSAddRec = dyn_cast<SCEVAddRecExpr>(Val: RHS);
13516 if (PositiveStride && RHSAddRec != nullptr && RHSAddRec->getLoop() == L &&
13517 any(Val: RHSAddRec->getNoWrapFlags())) {
13518 // The structure of loop we are trying to calculate backedge count of:
13519 //
13520 // left = left_start
13521 // right = right_start
13522 //
13523 // while(left < right){
13524 // ... do something here ...
13525 // left += s1; // stride of left is s1 (s1 > 0)
13526 // right += s2; // stride of right is s2 (s2 < 0)
13527 // }
13528 //
13529
13530 const SCEV *RHSStart = RHSAddRec->getStart();
13531 const SCEV *RHSStride = RHSAddRec->getStepRecurrence(SE&: *this);
13532
13533 // If Stride - RHSStride is positive and does not overflow, we can write
13534 // backedge count as ->
13535 // ceil((End - Start) /u (Stride - RHSStride))
13536 // Where, End = max(RHSStart, Start)
13537
13538 // Check if RHSStride < 0 and Stride - RHSStride will not overflow.
13539 if (isKnownNegative(S: RHSStride) &&
13540 willNotOverflow(BinOp: Instruction::Sub, /*Signed=*/true, LHS: Stride,
13541 RHS: RHSStride)) {
13542
13543 const SCEV *Denominator = getMinusSCEV(LHS: Stride, RHS: RHSStride);
13544 if (isKnownPositive(S: Denominator)) {
13545 End = IsSigned ? getSMaxExpr(LHS: RHSStart, RHS: Start)
13546 : getUMaxExpr(LHS: RHSStart, RHS: Start);
13547
13548 // We can do this because End >= Start, as End = max(RHSStart, Start)
13549 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13550
13551 BECount = getUDivCeilSCEV(N: Delta, D: Denominator);
13552 BECountIfBackedgeTaken =
13553 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHSStart, RHS: Start), D: Denominator);
13554 }
13555 }
13556 }
13557 if (BECount == nullptr) {
13558 // If we cannot calculate ExactBECount, we can calculate the MaxBECount,
13559 // given the start, stride and max value for the end bound of the
13560 // loop (RHS), and the fact that IV does not overflow (which is
13561 // checked above).
13562 const SCEV *MaxBECount = computeMaxBECountForLT(
13563 Start, Stride, End: RHS, BitWidth: getTypeSizeInBits(Ty: LHS->getType()), IsSigned);
13564 return ExitLimit(getCouldNotCompute() /* ExactNotTaken */, MaxBECount,
13565 MaxBECount, false /*MaxOrZero*/, Predicates);
13566 }
13567 } else {
13568 // We use the expression (max(End,Start)-Start)/Stride to describe the
13569 // backedge count, as if the backedge is taken at least once
13570 // max(End,Start) is End and so the result is as above, and if not
13571 // max(End,Start) is Start so we get a backedge count of zero.
13572 auto *OrigStartMinusStride = getMinusSCEV(LHS: OrigStart, RHS: Stride);
13573 assert(isAvailableAtLoopEntry(OrigStartMinusStride, L) && "Must be!");
13574 assert(isAvailableAtLoopEntry(OrigStart, L) && "Must be!");
13575 assert(isAvailableAtLoopEntry(OrigRHS, L) && "Must be!");
13576 // Can we prove (max(RHS,Start) > Start - Stride?
13577 if (isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigStartMinusStride, RHS: OrigStart) &&
13578 isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigStartMinusStride, RHS: OrigRHS)) {
13579 // In this case, we can use a refined formula for computing backedge
13580 // taken count. The general formula remains:
13581 // "End-Start /uceiling Stride" where "End = max(RHS,Start)"
13582 // We want to use the alternate formula:
13583 // "((End - 1) - (Start - Stride)) /u Stride"
13584 // Let's do a quick case analysis to show these are equivalent under
13585 // our precondition that max(RHS,Start) > Start - Stride.
13586 // * For RHS <= Start, the backedge-taken count must be zero.
13587 // "((End - 1) - (Start - Stride)) /u Stride" reduces to
13588 // "((Start - 1) - (Start - Stride)) /u Stride" which simplies to
13589 // "Stride - 1 /u Stride" which is indeed zero for all non-zero values
13590 // of Stride. For 0 stride, we've use umin(1,Stride) above,
13591 // reducing this to the stride of 1 case.
13592 // * For RHS >= Start, the backedge count must be "RHS-Start /uceil
13593 // Stride".
13594 // "((End - 1) - (Start - Stride)) /u Stride" reduces to
13595 // "((RHS - 1) - (Start - Stride)) /u Stride" reassociates to
13596 // "((RHS - (Start - Stride) - 1) /u Stride".
13597 // Our preconditions trivially imply no overflow in that form.
13598 const SCEV *MinusOne = getMinusOne(Ty: Stride->getType());
13599 const SCEV *Numerator =
13600 getMinusSCEV(LHS: getAddExpr(LHS: RHS, RHS: MinusOne), RHS: getMinusSCEV(LHS: Start, RHS: Stride));
13601 BECount = getUDivExpr(LHS: Numerator, RHS: Stride);
13602 }
13603
13604 if (!BECount) {
13605 auto canProveRHSGreaterThanEqualStart = [&]() {
13606 auto CondGE = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13607 const SCEV *GuardedRHS = applyLoopGuards(Expr: OrigRHS, L);
13608 const SCEV *GuardedStart = applyLoopGuards(Expr: OrigStart, L);
13609
13610 if (isLoopEntryGuardedByCond(L, Pred: CondGE, LHS: OrigRHS, RHS: OrigStart) ||
13611 isKnownPredicate(Pred: CondGE, LHS: GuardedRHS, RHS: GuardedStart))
13612 return true;
13613
13614 // (RHS > Start - 1) implies RHS >= Start.
13615 // * "RHS >= Start" is trivially equivalent to "RHS > Start - 1" if
13616 // "Start - 1" doesn't overflow.
13617 // * For signed comparison, if Start - 1 does overflow, it's equal
13618 // to INT_MAX, and "RHS >s INT_MAX" is trivially false.
13619 // * For unsigned comparison, if Start - 1 does overflow, it's equal
13620 // to UINT_MAX, and "RHS >u UINT_MAX" is trivially false.
13621 //
13622 // FIXME: Should isLoopEntryGuardedByCond do this for us?
13623 auto CondGT = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13624 auto *StartMinusOne =
13625 getAddExpr(LHS: OrigStart, RHS: getMinusOne(Ty: OrigStart->getType()));
13626 return isLoopEntryGuardedByCond(L, Pred: CondGT, LHS: OrigRHS, RHS: StartMinusOne);
13627 };
13628
13629 // If we know that RHS >= Start in the context of loop, then we know
13630 // that max(RHS, Start) = RHS at this point.
13631 if (canProveRHSGreaterThanEqualStart()) {
13632 End = RHS;
13633 } else {
13634 // If RHS < Start, the backedge will be taken zero times. So in
13635 // general, we can write the backedge-taken count as:
13636 //
13637 // RHS >= Start ? ceil(RHS - Start) / Stride : 0
13638 //
13639 // We convert it to the following to make it more convenient for SCEV:
13640 //
13641 // ceil(max(RHS, Start) - Start) / Stride
13642 End = IsSigned ? getSMaxExpr(LHS: RHS, RHS: Start) : getUMaxExpr(LHS: RHS, RHS: Start);
13643
13644 // See what would happen if we assume the backedge is taken. This is
13645 // used to compute MaxBECount.
13646 BECountIfBackedgeTaken =
13647 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHS, RHS: Start), D: Stride);
13648 }
13649
13650 // At this point, we know:
13651 //
13652 // 1. If IsSigned, Start <=s End; otherwise, Start <=u End
13653 // 2. The index variable doesn't overflow.
13654 //
13655 // Therefore, we know N exists such that
13656 // (Start + Stride * N) >= End, and computing "(Start + Stride * N)"
13657 // doesn't overflow.
13658 //
13659 // Using this information, try to prove whether the addition in
13660 // "(Start - End) + (Stride - 1)" has unsigned overflow.
13661 const SCEV *One = getOne(Ty: Stride->getType());
13662 bool MayAddOverflow = [&] {
13663 if (isKnownToBeAPowerOfTwo(S: Stride)) {
13664 // Suppose Stride is a power of two, and Start/End are unsigned
13665 // integers. Let UMAX be the largest representable unsigned
13666 // integer.
13667 //
13668 // By the preconditions of this function, we know
13669 // "(Start + Stride * N) >= End", and this doesn't overflow.
13670 // As a formula:
13671 //
13672 // End <= (Start + Stride * N) <= UMAX
13673 //
13674 // Subtracting Start from all the terms:
13675 //
13676 // End - Start <= Stride * N <= UMAX - Start
13677 //
13678 // Since Start is unsigned, UMAX - Start <= UMAX. Therefore:
13679 //
13680 // End - Start <= Stride * N <= UMAX
13681 //
13682 // Stride * N is a multiple of Stride. Therefore,
13683 //
13684 // End - Start <= Stride * N <= UMAX - (UMAX mod Stride)
13685 //
13686 // Since Stride is a power of two, UMAX + 1 is divisible by
13687 // Stride. Therefore, UMAX mod Stride == Stride - 1. So we can
13688 // write:
13689 //
13690 // End - Start <= Stride * N <= UMAX - Stride - 1
13691 //
13692 // Dropping the middle term:
13693 //
13694 // End - Start <= UMAX - Stride - 1
13695 //
13696 // Adding Stride - 1 to both sides:
13697 //
13698 // (End - Start) + (Stride - 1) <= UMAX
13699 //
13700 // In other words, the addition doesn't have unsigned overflow.
13701 //
13702 // A similar proof works if we treat Start/End as signed values.
13703 // Just rewrite steps before "End - Start <= Stride * N <= UMAX"
13704 // to use signed max instead of unsigned max. Note that we're
13705 // trying to prove a lack of unsigned overflow in either case.
13706 return false;
13707 }
13708 if (Start == Stride || Start == getMinusSCEV(LHS: Stride, RHS: One)) {
13709 // If Start is equal to Stride, (End - Start) + (Stride - 1) == End
13710 // - 1. If !IsSigned, 0 <u Stride == Start <=u End; so 0 <u End - 1
13711 // <u End. If IsSigned, 0 <s Stride == Start <=s End; so 0 <s End -
13712 // 1 <s End.
13713 //
13714 // If Start is equal to Stride - 1, (End - Start) + Stride - 1 ==
13715 // End.
13716 return false;
13717 }
13718 return true;
13719 }();
13720
13721 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13722 if (!MayAddOverflow) {
13723 // floor((D + (S - 1)) / S)
13724 // We prefer this formulation if it's legal because it's fewer
13725 // operations.
13726 BECount =
13727 getUDivExpr(LHS: getAddExpr(LHS: Delta, RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13728 } else {
13729 BECount = getUDivCeilSCEV(N: Delta, D: Stride);
13730 }
13731 }
13732 }
13733
13734 const SCEV *ConstantMaxBECount;
13735 bool MaxOrZero = false;
13736 if (isa<SCEVConstant>(Val: BECount)) {
13737 ConstantMaxBECount = BECount;
13738 } else if (BECountIfBackedgeTaken &&
13739 isa<SCEVConstant>(Val: BECountIfBackedgeTaken)) {
13740 // If we know exactly how many times the backedge will be taken if it's
13741 // taken at least once, then the backedge count will either be that or
13742 // zero.
13743 ConstantMaxBECount = BECountIfBackedgeTaken;
13744 MaxOrZero = true;
13745 } else {
13746 ConstantMaxBECount = computeMaxBECountForLT(
13747 Start, Stride, End: RHS, BitWidth: getTypeSizeInBits(Ty: LHS->getType()), IsSigned);
13748 }
13749
13750 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
13751 !isa<SCEVCouldNotCompute>(Val: BECount))
13752 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
13753
13754 const SCEV *SymbolicMaxBECount =
13755 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13756 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, MaxOrZero,
13757 Predicates);
13758}
13759
13760ScalarEvolution::ExitLimit ScalarEvolution::howManyGreaterThans(
13761 const SCEV *LHS, const SCEV *RHS, const Loop *L, bool IsSigned,
13762 bool ControlsOnlyExit, bool AllowPredicates) {
13763 SmallVector<const SCEVPredicate *> Predicates;
13764 // We handle only IV > Invariant
13765 if (!isLoopInvariant(S: RHS, L))
13766 return getCouldNotCompute();
13767
13768 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
13769 if (!IV && AllowPredicates)
13770 // Try to make this an AddRec using runtime tests, in the first X
13771 // iterations of this loop, where X is the SCEV expression found by the
13772 // algorithm below.
13773 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13774
13775 // Avoid weird loops
13776 if (!IV || IV->getLoop() != L || !IV->isAffine())
13777 return getCouldNotCompute();
13778
13779 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13780 bool NoWrap = ControlsOnlyExit && any(Val: IV->getNoWrapFlags(Mask: WrapType));
13781 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13782
13783 const SCEV *Stride = getNegativeSCEV(V: IV->getStepRecurrence(SE&: *this));
13784
13785 // Avoid negative or zero stride values
13786 if (!isKnownPositive(S: Stride))
13787 return getCouldNotCompute();
13788
13789 // Avoid proven overflow cases: this will ensure that the backedge taken count
13790 // will not generate any unsigned overflow. Relaxed no-overflow conditions
13791 // exploit NoWrapFlags, allowing to optimize in presence of undefined
13792 // behaviors like the case of C language.
13793 if (!Stride->isOne() && !NoWrap)
13794 if (canIVOverflowOnGT(RHS, Stride, IsSigned))
13795 return getCouldNotCompute();
13796
13797 const SCEV *Start = IV->getStart();
13798 const SCEV *End = RHS;
13799 if (!isLoopEntryGuardedByCond(L, Pred: Cond, LHS: getAddExpr(LHS: Start, RHS: Stride), RHS)) {
13800 // If we know that Start >= RHS in the context of loop, then we know that
13801 // min(RHS, Start) = RHS at this point.
13802 if (isLoopEntryGuardedByCond(
13803 L, Pred: IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, LHS: Start, RHS))
13804 End = RHS;
13805 else
13806 End = IsSigned ? getSMinExpr(LHS: RHS, RHS: Start) : getUMinExpr(LHS: RHS, RHS: Start);
13807 }
13808
13809 if (Start->getType()->isPointerTy()) {
13810 Start = getLosslessPtrToIntExpr(Op: Start);
13811 if (isa<SCEVCouldNotCompute>(Val: Start))
13812 return Start;
13813 }
13814 if (End->getType()->isPointerTy()) {
13815 End = getLosslessPtrToIntExpr(Op: End);
13816 if (isa<SCEVCouldNotCompute>(Val: End))
13817 return End;
13818 }
13819
13820 // Compute ((Start - End) + (Stride - 1)) / Stride.
13821 // FIXME: This can overflow. Holding off on fixing this for now;
13822 // howManyGreaterThans will hopefully be gone soon.
13823 const SCEV *One = getOne(Ty: Stride->getType());
13824 const SCEV *BECount = getUDivExpr(
13825 LHS: getAddExpr(LHS: getMinusSCEV(LHS: Start, RHS: End), RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13826
13827 APInt MaxStart = IsSigned ? getSignedRangeMax(S: Start)
13828 : getUnsignedRangeMax(S: Start);
13829
13830 APInt MinStride = IsSigned ? getSignedRangeMin(S: Stride)
13831 : getUnsignedRangeMin(S: Stride);
13832
13833 unsigned BitWidth = getTypeSizeInBits(Ty: LHS->getType());
13834 APInt Limit = IsSigned ? APInt::getSignedMinValue(numBits: BitWidth) + (MinStride - 1)
13835 : APInt::getMinValue(numBits: BitWidth) + (MinStride - 1);
13836
13837 // Although End can be a MIN expression we estimate MinEnd considering only
13838 // the case End = RHS. This is safe because in the other case (Start - End)
13839 // is zero, leading to a zero maximum backedge taken count.
13840 APInt MinEnd =
13841 IsSigned ? APIntOps::smax(A: getSignedRangeMin(S: RHS), B: Limit)
13842 : APIntOps::umax(A: getUnsignedRangeMin(S: RHS), B: Limit);
13843
13844 const SCEV *ConstantMaxBECount =
13845 isa<SCEVConstant>(Val: BECount)
13846 ? BECount
13847 : getUDivCeilSCEV(N: getConstant(Val: MaxStart - MinEnd),
13848 D: getConstant(Val: MinStride));
13849
13850 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount))
13851 ConstantMaxBECount = BECount;
13852 const SCEV *SymbolicMaxBECount =
13853 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13854
13855 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
13856 Predicates);
13857}
13858
13859const SCEV *SCEVAddRecExpr::getNumIterationsInRange(const ConstantRange &Range,
13860 ScalarEvolution &SE) const {
13861 if (Range.isFullSet()) // Infinite loop.
13862 return SE.getCouldNotCompute();
13863
13864 // If the start is a non-zero constant, shift the range to simplify things.
13865 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: getStart()))
13866 if (!SC->getValue()->isZero()) {
13867 SmallVector<SCEVUse, 4> Operands(operands());
13868 Operands[0] = SE.getZero(Ty: SC->getType());
13869 const SCEV *Shifted = SE.getAddRecExpr(Operands, L: getLoop(),
13870 Flags: getNoWrapFlags(Mask: FlagNW));
13871 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Val: Shifted))
13872 return ShiftedAddRec->getNumIterationsInRange(
13873 Range: Range.subtract(CI: SC->getAPInt()), SE);
13874 // This is strange and shouldn't happen.
13875 return SE.getCouldNotCompute();
13876 }
13877
13878 // The only time we can solve this is when we have all constant indices.
13879 // Otherwise, we cannot determine the overflow conditions.
13880 if (any_of(Range: operands(), P: [](const SCEV *Op) { return !isa<SCEVConstant>(Val: Op); }))
13881 return SE.getCouldNotCompute();
13882
13883 // Okay at this point we know that all elements of the chrec are constants and
13884 // that the start element is zero.
13885
13886 // First check to see if the range contains zero. If not, the first
13887 // iteration exits.
13888 unsigned BitWidth = SE.getTypeSizeInBits(Ty: getType());
13889 if (!Range.contains(Val: APInt(BitWidth, 0)))
13890 return SE.getZero(Ty: getType());
13891
13892 if (isAffine()) {
13893 // If this is an affine expression then we have this situation:
13894 // Solve {0,+,A} in Range === Ax in Range
13895
13896 // We know that zero is in the range. If A is positive then we know that
13897 // the upper value of the range must be the first possible exit value.
13898 // If A is negative then the lower of the range is the last possible loop
13899 // value. Also note that we already checked for a full range.
13900 APInt A = cast<SCEVConstant>(Val: getOperand(i: 1))->getAPInt();
13901 APInt End = A.sge(RHS: 1) ? (Range.getUpper() - 1) : Range.getLower();
13902
13903 // The exit value should be (End+A)/A.
13904 APInt ExitVal = (End + A).udiv(RHS: A);
13905 ConstantInt *ExitValue = ConstantInt::get(Context&: SE.getContext(), V: ExitVal);
13906
13907 // Evaluate at the exit value. If we really did fall out of the valid
13908 // range, then we computed our trip count, otherwise wrap around or other
13909 // things must have happened.
13910 ConstantInt *Val = EvaluateConstantChrecAtConstant(AddRec: this, C: ExitValue, SE);
13911 if (Range.contains(Val: Val->getValue()))
13912 return SE.getCouldNotCompute(); // Something strange happened
13913
13914 // Ensure that the previous value is in the range.
13915 assert(Range.contains(
13916 EvaluateConstantChrecAtConstant(this,
13917 ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) &&
13918 "Linear scev computation is off in a bad way!");
13919 return SE.getConstant(V: ExitValue);
13920 }
13921
13922 if (isQuadratic()) {
13923 if (auto S = SolveQuadraticAddRecRange(AddRec: this, Range, SE))
13924 return SE.getConstant(Val: *S);
13925 }
13926
13927 return SE.getCouldNotCompute();
13928}
13929
13930const SCEVAddRecExpr *
13931SCEVAddRecExpr::getPostIncExpr(ScalarEvolution &SE) const {
13932 assert(getNumOperands() > 1 && "AddRec with zero step?");
13933 // There is a temptation to just call getAddExpr(this, getStepRecurrence(SE)),
13934 // but in this case we cannot guarantee that the value returned will be an
13935 // AddRec because SCEV does not have a fixed point where it stops
13936 // simplification: it is legal to return ({rec1} + {rec2}). For example, it
13937 // may happen if we reach arithmetic depth limit while simplifying. So we
13938 // construct the returned value explicitly.
13939 SmallVector<SCEVUse, 3> Ops;
13940 // If this is {A,+,B,+,C,...,+,N}, then its step is {B,+,C,+,...,+,N}, and
13941 // (this + Step) is {A+B,+,B+C,+...,+,N}.
13942 for (unsigned i = 0, e = getNumOperands() - 1; i < e; ++i)
13943 Ops.push_back(Elt: SE.getAddExpr(LHS: getOperand(i), RHS: getOperand(i: i + 1)));
13944 // We know that the last operand is not a constant zero (otherwise it would
13945 // have been popped out earlier). This guarantees us that if the result has
13946 // the same last operand, then it will also not be popped out, meaning that
13947 // the returned value will be an AddRec.
13948 const SCEV *Last = getOperand(i: getNumOperands() - 1);
13949 assert(!Last->isZero() && "Recurrency with zero step?");
13950 Ops.push_back(Elt: Last);
13951 return cast<SCEVAddRecExpr>(Val: SE.getAddRecExpr(Operands&: Ops, L: getLoop(),
13952 Flags: SCEV::FlagAnyWrap));
13953}
13954
13955// Return true when S contains at least an undef value.
13956bool ScalarEvolution::containsUndefs(const SCEV *S) const {
13957 return SCEVExprContains(
13958 Root: S, Pred: [](const SCEV *S) { return match(S, P: m_scev_UndefOrPoison()); });
13959}
13960
13961// Return true when S contains a value that is a nullptr.
13962bool ScalarEvolution::containsErasedValue(const SCEV *S) const {
13963 return SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
13964 if (const auto *SU = dyn_cast<SCEVUnknown>(Val: S))
13965 return SU->getValue() == nullptr;
13966 return false;
13967 });
13968}
13969
13970/// Return the size of an element read or written by Inst.
13971const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) {
13972 Type *Ty;
13973 if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst))
13974 Ty = Store->getValueOperand()->getType();
13975 else if (LoadInst *Load = dyn_cast<LoadInst>(Val: Inst))
13976 Ty = Load->getType();
13977 else
13978 return nullptr;
13979
13980 Type *ETy = getEffectiveSCEVType(Ty: PointerType::getUnqual(C&: Inst->getContext()));
13981 return getSizeOfExpr(IntTy: ETy, AllocTy: Ty);
13982}
13983
13984//===----------------------------------------------------------------------===//
13985// SCEVCallbackVH Class Implementation
13986//===----------------------------------------------------------------------===//
13987
13988void ScalarEvolution::SCEVCallbackVH::deleted() {
13989 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
13990 if (PHINode *PN = dyn_cast<PHINode>(Val: getValPtr()))
13991 SE->ConstantEvolutionLoopExitValue.erase(Val: PN);
13992 SE->eraseValueFromMap(V: getValPtr());
13993 // this now dangles!
13994}
13995
13996void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) {
13997 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
13998
13999 // Forget all the expressions associated with users of the old value,
14000 // so that future queries will recompute the expressions using the new
14001 // value.
14002 SE->forgetValue(V: getValPtr());
14003 // this now dangles!
14004}
14005
14006ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se)
14007 : CallbackVH(V), SE(se) {}
14008
14009//===----------------------------------------------------------------------===//
14010// ScalarEvolution Class Implementation
14011//===----------------------------------------------------------------------===//
14012
14013ScalarEvolution::ScalarEvolution(Function &F, TargetLibraryInfo &TLI,
14014 AssumptionCache &AC, DominatorTree &DT,
14015 LoopInfo &LI)
14016 : F(F), DL(F.getDataLayout()), TLI(TLI), AC(AC), DT(DT), LI(LI),
14017 CouldNotCompute(new SCEVCouldNotCompute()), ValuesAtScopes(64),
14018 LoopDispositions(64), BlockDispositions(64) {
14019 // To use guards for proving predicates, we need to scan every instruction in
14020 // relevant basic blocks, and not just terminators. Doing this is a waste of
14021 // time if the IR does not actually contain any calls to
14022 // @llvm.experimental.guard, so do a quick check and remember this beforehand.
14023 //
14024 // This pessimizes the case where a pass that preserves ScalarEvolution wants
14025 // to _add_ guards to the module when there weren't any before, and wants
14026 // ScalarEvolution to optimize based on those guards. For now we prefer to be
14027 // efficient in lieu of being smart in that rather obscure case.
14028
14029 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
14030 M: F.getParent(), id: Intrinsic::experimental_guard);
14031 HasGuards = GuardDecl && !GuardDecl->use_empty();
14032}
14033
14034ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg)
14035 : F(Arg.F), DL(Arg.DL), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC),
14036 DT(Arg.DT), LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)),
14037 ValueExprMap(std::move(Arg.ValueExprMap)),
14038 PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
14039 PendingMerges(std::move(Arg.PendingMerges)),
14040 ConstantMultipleCache(std::move(Arg.ConstantMultipleCache)),
14041 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
14042 PredicatedBackedgeTakenCounts(
14043 std::move(Arg.PredicatedBackedgeTakenCounts)),
14044 BECountUsers(std::move(Arg.BECountUsers)),
14045 ConstantEvolutionLoopExitValue(
14046 std::move(Arg.ConstantEvolutionLoopExitValue)),
14047 ValuesAtScopes(std::move(Arg.ValuesAtScopes)),
14048 ValuesAtScopesUsers(std::move(Arg.ValuesAtScopesUsers)),
14049 LoopDispositions(std::move(Arg.LoopDispositions)),
14050 LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)),
14051 BlockDispositions(std::move(Arg.BlockDispositions)),
14052 SCEVUsers(std::move(Arg.SCEVUsers)),
14053 UnsignedRanges(std::move(Arg.UnsignedRanges)),
14054 SignedRanges(std::move(Arg.SignedRanges)),
14055 UniqueSCEVs(std::move(Arg.UniqueSCEVs)),
14056 UniquePreds(std::move(Arg.UniquePreds)),
14057 SCEVAllocator(std::move(Arg.SCEVAllocator)),
14058 LoopUsers(std::move(Arg.LoopUsers)),
14059 PredicatedSCEVRewrites(std::move(Arg.PredicatedSCEVRewrites)),
14060 FirstUnknown(Arg.FirstUnknown) {
14061 Arg.FirstUnknown = nullptr;
14062}
14063
14064ScalarEvolution::~ScalarEvolution() {
14065 // Iterate through all the SCEVUnknown instances and call their
14066 // destructors, so that they release their references to their values.
14067 for (SCEVUnknown *U = FirstUnknown; U;) {
14068 SCEVUnknown *Tmp = U;
14069 U = U->Next;
14070 Tmp->~SCEVUnknown();
14071 }
14072 FirstUnknown = nullptr;
14073
14074 ExprValueMap.clear();
14075 ValueExprMap.clear();
14076 HasRecMap.clear();
14077 BackedgeTakenCounts.clear();
14078 PredicatedBackedgeTakenCounts.clear();
14079
14080 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage");
14081 assert(PendingMerges.empty() && "isImpliedViaMerge garbage");
14082 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!");
14083 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!");
14084}
14085
14086bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) {
14087 return !isa<SCEVCouldNotCompute>(Val: getBackedgeTakenCount(L));
14088}
14089
14090/// When printing a top-level SCEV for trip counts, it's helpful to include
14091/// a type for constants which are otherwise hard to disambiguate.
14092static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV* S) {
14093 if (isa<SCEVConstant>(Val: S))
14094 OS << *S->getType() << " ";
14095 OS << *S;
14096}
14097
14098static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE,
14099 const Loop *L) {
14100 // Print all inner loops first
14101 for (Loop *I : *L)
14102 PrintLoopInfo(OS, SE, L: I);
14103
14104 OS << "Loop ";
14105 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14106 OS << ": ";
14107
14108 SmallVector<BasicBlock *, 8> ExitingBlocks;
14109 L->getExitingBlocks(ExitingBlocks);
14110 if (ExitingBlocks.size() != 1)
14111 OS << "<multiple exits> ";
14112
14113 auto *BTC = SE->getBackedgeTakenCount(L);
14114 if (!isa<SCEVCouldNotCompute>(Val: BTC)) {
14115 OS << "backedge-taken count is ";
14116 PrintSCEVWithTypeHint(OS, S: BTC);
14117 } else
14118 OS << "Unpredictable backedge-taken count.";
14119 OS << "\n";
14120
14121 if (ExitingBlocks.size() > 1)
14122 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14123 OS << " exit count for " << ExitingBlock->getName() << ": ";
14124 const SCEV *EC = SE->getExitCount(L, ExitingBlock);
14125 PrintSCEVWithTypeHint(OS, S: EC);
14126 if (isa<SCEVCouldNotCompute>(Val: EC)) {
14127 // Retry with predicates.
14128 SmallVector<const SCEVPredicate *> Predicates;
14129 EC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates);
14130 if (!isa<SCEVCouldNotCompute>(Val: EC)) {
14131 OS << "\n predicated exit count for " << ExitingBlock->getName()
14132 << ": ";
14133 PrintSCEVWithTypeHint(OS, S: EC);
14134 OS << "\n Predicates:\n";
14135 for (const auto *P : Predicates)
14136 P->print(OS, Depth: 4);
14137 }
14138 }
14139 OS << "\n";
14140 }
14141
14142 OS << "Loop ";
14143 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14144 OS << ": ";
14145
14146 auto *ConstantBTC = SE->getConstantMaxBackedgeTakenCount(L);
14147 if (!isa<SCEVCouldNotCompute>(Val: ConstantBTC)) {
14148 OS << "constant max backedge-taken count is ";
14149 PrintSCEVWithTypeHint(OS, S: ConstantBTC);
14150 if (SE->isBackedgeTakenCountMaxOrZero(L))
14151 OS << ", actual taken count either this or zero.";
14152 } else {
14153 OS << "Unpredictable constant max backedge-taken count. ";
14154 }
14155
14156 OS << "\n"
14157 "Loop ";
14158 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14159 OS << ": ";
14160
14161 auto *SymbolicBTC = SE->getSymbolicMaxBackedgeTakenCount(L);
14162 if (!isa<SCEVCouldNotCompute>(Val: SymbolicBTC)) {
14163 OS << "symbolic max backedge-taken count is ";
14164 PrintSCEVWithTypeHint(OS, S: SymbolicBTC);
14165 if (SE->isBackedgeTakenCountMaxOrZero(L))
14166 OS << ", actual taken count either this or zero.";
14167 } else {
14168 OS << "Unpredictable symbolic max backedge-taken count. ";
14169 }
14170 OS << "\n";
14171
14172 if (ExitingBlocks.size() > 1)
14173 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14174 OS << " symbolic max exit count for " << ExitingBlock->getName() << ": ";
14175 auto *ExitBTC = SE->getExitCount(L, ExitingBlock,
14176 Kind: ScalarEvolution::SymbolicMaximum);
14177 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14178 if (isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14179 // Retry with predicates.
14180 SmallVector<const SCEVPredicate *> Predicates;
14181 ExitBTC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates,
14182 Kind: ScalarEvolution::SymbolicMaximum);
14183 if (!isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14184 OS << "\n predicated symbolic max exit count for "
14185 << ExitingBlock->getName() << ": ";
14186 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14187 OS << "\n Predicates:\n";
14188 for (const auto *P : Predicates)
14189 P->print(OS, Depth: 4);
14190 }
14191 }
14192 OS << "\n";
14193 }
14194
14195 SmallVector<const SCEVPredicate *, 4> Preds;
14196 auto *PBT = SE->getPredicatedBackedgeTakenCount(L, Preds);
14197 if (PBT != BTC) {
14198 OS << "Loop ";
14199 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14200 OS << ": ";
14201 if (!isa<SCEVCouldNotCompute>(Val: PBT)) {
14202 OS << "Predicated backedge-taken count is ";
14203 PrintSCEVWithTypeHint(OS, S: PBT);
14204 } else
14205 OS << "Unpredictable predicated backedge-taken count.";
14206 OS << "\n";
14207 OS << " Predicates:\n";
14208 for (const auto *P : Preds)
14209 P->print(OS, Depth: 4);
14210 }
14211 Preds.clear();
14212
14213 auto *PredConstantMax =
14214 SE->getPredicatedConstantMaxBackedgeTakenCount(L, Preds);
14215 if (PredConstantMax != ConstantBTC) {
14216 OS << "Loop ";
14217 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14218 OS << ": ";
14219 if (!isa<SCEVCouldNotCompute>(Val: PredConstantMax)) {
14220 OS << "Predicated constant max backedge-taken count is ";
14221 PrintSCEVWithTypeHint(OS, S: PredConstantMax);
14222 } else
14223 OS << "Unpredictable predicated constant max backedge-taken count.";
14224 OS << "\n";
14225 OS << " Predicates:\n";
14226 for (const auto *P : Preds)
14227 P->print(OS, Depth: 4);
14228 }
14229 Preds.clear();
14230
14231 auto *PredSymbolicMax =
14232 SE->getPredicatedSymbolicMaxBackedgeTakenCount(L, Preds);
14233 if (SymbolicBTC != PredSymbolicMax) {
14234 OS << "Loop ";
14235 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14236 OS << ": ";
14237 if (!isa<SCEVCouldNotCompute>(Val: PredSymbolicMax)) {
14238 OS << "Predicated symbolic max backedge-taken count is ";
14239 PrintSCEVWithTypeHint(OS, S: PredSymbolicMax);
14240 } else
14241 OS << "Unpredictable predicated symbolic max backedge-taken count.";
14242 OS << "\n";
14243 OS << " Predicates:\n";
14244 for (const auto *P : Preds)
14245 P->print(OS, Depth: 4);
14246 }
14247
14248 if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
14249 OS << "Loop ";
14250 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14251 OS << ": ";
14252 OS << "Trip multiple is " << SE->getSmallConstantTripMultiple(L) << "\n";
14253 }
14254}
14255
14256namespace llvm {
14257// Note: these overloaded operators need to be in the llvm namespace for them
14258// to be resolved correctly. If we put them outside the llvm namespace, the
14259//
14260// OS << ": " << SE.getLoopDisposition(SV, InnerL);
14261//
14262// code below "breaks" and start printing raw enum values as opposed to the
14263// string values.
14264static raw_ostream &operator<<(raw_ostream &OS,
14265 ScalarEvolution::LoopDisposition LD) {
14266 switch (LD) {
14267 case ScalarEvolution::LoopVariant:
14268 OS << "Variant";
14269 break;
14270 case ScalarEvolution::LoopInvariant:
14271 OS << "Invariant";
14272 break;
14273 case ScalarEvolution::LoopUniform:
14274 OS << "Uniform";
14275 break;
14276 case ScalarEvolution::LoopComputable:
14277 OS << "Computable";
14278 break;
14279 }
14280 return OS;
14281}
14282
14283static raw_ostream &operator<<(raw_ostream &OS,
14284 llvm::ScalarEvolution::BlockDisposition BD) {
14285 switch (BD) {
14286 case ScalarEvolution::DoesNotDominateBlock:
14287 OS << "DoesNotDominate";
14288 break;
14289 case ScalarEvolution::DominatesBlock:
14290 OS << "Dominates";
14291 break;
14292 case ScalarEvolution::ProperlyDominatesBlock:
14293 OS << "ProperlyDominates";
14294 break;
14295 }
14296 return OS;
14297}
14298} // namespace llvm
14299
14300void ScalarEvolution::print(raw_ostream &OS) const {
14301 // ScalarEvolution's implementation of the print method is to print
14302 // out SCEV values of all instructions that are interesting. Doing
14303 // this potentially causes it to create new SCEV objects though,
14304 // which technically conflicts with the const qualifier. This isn't
14305 // observable from outside the class though, so casting away the
14306 // const isn't dangerous.
14307 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14308
14309 if (ClassifyExpressions) {
14310 OS << "Classifying expressions for: ";
14311 F.printAsOperand(O&: OS, /*PrintType=*/false);
14312 OS << "\n";
14313 for (Instruction &I : instructions(F))
14314 if (isSCEVable(Ty: I.getType()) && !isa<CmpInst>(Val: I)) {
14315 OS << I << '\n';
14316 OS << " --> ";
14317 const SCEV *SV = SE.getSCEV(V: &I);
14318 SV->print(OS);
14319 if (!isa<SCEVCouldNotCompute>(Val: SV)) {
14320 OS << " U: ";
14321 SE.getUnsignedRange(S: SV).print(OS);
14322 OS << " S: ";
14323 SE.getSignedRange(S: SV).print(OS);
14324 }
14325
14326 const Loop *L = LI.getLoopFor(BB: I.getParent());
14327
14328 const SCEV *AtUse = SE.getSCEVAtScope(V: SV, L);
14329 if (AtUse != SV) {
14330 OS << " --> ";
14331 AtUse->print(OS);
14332 if (!isa<SCEVCouldNotCompute>(Val: AtUse)) {
14333 OS << " U: ";
14334 SE.getUnsignedRange(S: AtUse).print(OS);
14335 OS << " S: ";
14336 SE.getSignedRange(S: AtUse).print(OS);
14337 }
14338 }
14339
14340 if (L) {
14341 OS << "\t\t" "Exits: ";
14342 const SCEV *ExitValue = SE.getSCEVAtScope(V: SV, L: L->getParentLoop());
14343 if (!SE.isLoopInvariant(S: ExitValue, L)) {
14344 OS << "<<Unknown>>";
14345 } else {
14346 OS << *ExitValue;
14347 }
14348
14349 ListSeparator LS(", ", "\t\tLoopDispositions: { ");
14350 for (const auto *Iter = L; Iter; Iter = Iter->getParentLoop()) {
14351 OS << LS;
14352 Iter->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14353 OS << ": " << SE.getLoopDisposition(S: SV, L: Iter);
14354 }
14355
14356 for (const auto *InnerL : depth_first(G: L)) {
14357 if (InnerL == L)
14358 continue;
14359 OS << LS;
14360 InnerL->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14361 OS << ": " << SE.getLoopDisposition(S: SV, L: InnerL);
14362 }
14363
14364 OS << " }";
14365 }
14366
14367 OS << "\n";
14368 }
14369 }
14370
14371 OS << "Determining loop execution counts for: ";
14372 F.printAsOperand(O&: OS, /*PrintType=*/false);
14373 OS << "\n";
14374 for (Loop *I : LI)
14375 PrintLoopInfo(OS, SE: &SE, L: I);
14376}
14377
14378ScalarEvolution::LoopDisposition
14379ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) {
14380 auto &Values = LoopDispositions[S];
14381 for (auto &V : Values) {
14382 if (V.getPointer() == L)
14383 return V.getInt();
14384 }
14385 Values.emplace_back(Args&: L, Args: LoopVariant);
14386 LoopDisposition D = computeLoopDisposition(S, L);
14387 auto &Values2 = LoopDispositions[S];
14388 for (auto &V : llvm::reverse(C&: Values2)) {
14389 if (V.getPointer() == L) {
14390 V.setInt(D);
14391 break;
14392 }
14393 }
14394 return D;
14395}
14396
14397ScalarEvolution::LoopDisposition
14398ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
14399 switch (S->getSCEVType()) {
14400 case scConstant:
14401 case scVScale:
14402 return LoopInvariant;
14403 case scAddRecExpr: {
14404 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14405
14406 // If L is the addrec's loop, it's computable.
14407 if (AR->getLoop() == L)
14408 return LoopComputable;
14409
14410 // Add recurrences are never invariant in the function-body (null loop).
14411 if (!L)
14412 return LoopVariant;
14413
14414 // Everything that is not defined at loop entry is variant.
14415 if (DT.dominates(A: L->getHeader(), B: AR->getLoop()->getHeader())) {
14416 if (L->contains(L: AR->getLoop()) &&
14417 llvm::all_of(Range: AR->operands(),
14418 P: [&](const SCEV *Op) { return isLoopUniform(S: Op, L); }))
14419 return LoopUniform;
14420
14421 return LoopVariant;
14422 }
14423 assert(!L->contains(AR->getLoop()) && "Containing loop's header does not"
14424 " dominate the contained loop's header?");
14425
14426 // This recurrence is invariant w.r.t. L if AR's loop contains L.
14427 if (AR->getLoop()->contains(L))
14428 return LoopInvariant;
14429
14430 // This recurrence is variant w.r.t. L if any of its operands
14431 // are variant.
14432 for (SCEVUse Op : AR->operands())
14433 if (!isLoopInvariant(S: Op, L))
14434 return LoopVariant;
14435
14436 // Otherwise it's loop-invariant.
14437 return LoopInvariant;
14438 }
14439 case scTruncate:
14440 case scZeroExtend:
14441 case scSignExtend:
14442 case scPtrToAddr:
14443 case scPtrToInt:
14444 case scAddExpr:
14445 case scMulExpr:
14446 case scUDivExpr:
14447 case scUMaxExpr:
14448 case scSMaxExpr:
14449 case scUMinExpr:
14450 case scSMinExpr:
14451 case scSequentialUMinExpr: {
14452 bool HasVarying = false;
14453 bool HasUniform = false;
14454 for (SCEVUse Op : S->operands()) {
14455 LoopDisposition D = getLoopDisposition(S: Op, L);
14456 if (D == LoopVariant)
14457 return LoopVariant;
14458 if (D == LoopComputable)
14459 HasVarying = true;
14460 if (D == LoopUniform)
14461 HasUniform = true;
14462 }
14463 return HasVarying ? (HasUniform ? LoopVariant : LoopComputable)
14464 : (HasUniform ? LoopUniform : LoopInvariant);
14465 }
14466 case scUnknown:
14467 // All non-instruction values are loop invariant. All instructions are loop
14468 // invariant if they are not contained in the specified loop.
14469 // Instructions are never considered invariant in the function body
14470 // (null loop) because they are defined within the "loop".
14471 if (auto *I = dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue()))
14472 return (L && !L->contains(Inst: I)) ? LoopInvariant : LoopVariant;
14473 return LoopInvariant;
14474 case scCouldNotCompute:
14475 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14476 }
14477 llvm_unreachable("Unknown SCEV kind!");
14478}
14479
14480bool ScalarEvolution::isLoopUniform(const SCEV *S, const Loop *L) {
14481 LoopDisposition D = getLoopDisposition(S, L);
14482 return D == LoopUniform || D == LoopInvariant;
14483}
14484
14485bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) {
14486 return getLoopDisposition(S, L) == LoopInvariant;
14487}
14488
14489bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) {
14490 return getLoopDisposition(S, L) == LoopComputable;
14491}
14492
14493ScalarEvolution::BlockDisposition
14494ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14495 auto &Values = BlockDispositions[S];
14496 for (auto &V : Values) {
14497 if (V.getPointer() == BB)
14498 return V.getInt();
14499 }
14500 Values.emplace_back(Args&: BB, Args: DoesNotDominateBlock);
14501 BlockDisposition D = computeBlockDisposition(S, BB);
14502 auto &Values2 = BlockDispositions[S];
14503 for (auto &V : llvm::reverse(C&: Values2)) {
14504 if (V.getPointer() == BB) {
14505 V.setInt(D);
14506 break;
14507 }
14508 }
14509 return D;
14510}
14511
14512ScalarEvolution::BlockDisposition
14513ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14514 switch (S->getSCEVType()) {
14515 case scConstant:
14516 case scVScale:
14517 return ProperlyDominatesBlock;
14518 case scAddRecExpr: {
14519 // This uses a "dominates" query instead of "properly dominates" query
14520 // to test for proper dominance too, because the instruction which
14521 // produces the addrec's value is a PHI, and a PHI effectively properly
14522 // dominates its entire containing block.
14523 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14524 if (!DT.dominates(A: AR->getLoop()->getHeader(), B: BB))
14525 return DoesNotDominateBlock;
14526
14527 // Fall through into SCEVNAryExpr handling.
14528 [[fallthrough]];
14529 }
14530 case scTruncate:
14531 case scZeroExtend:
14532 case scSignExtend:
14533 case scPtrToAddr:
14534 case scPtrToInt:
14535 case scAddExpr:
14536 case scMulExpr:
14537 case scUDivExpr:
14538 case scUMaxExpr:
14539 case scSMaxExpr:
14540 case scUMinExpr:
14541 case scSMinExpr:
14542 case scSequentialUMinExpr: {
14543 bool Proper = true;
14544 for (const SCEV *NAryOp : S->operands()) {
14545 BlockDisposition D = getBlockDisposition(S: NAryOp, BB);
14546 if (D == DoesNotDominateBlock)
14547 return DoesNotDominateBlock;
14548 if (D == DominatesBlock)
14549 Proper = false;
14550 }
14551 return Proper ? ProperlyDominatesBlock : DominatesBlock;
14552 }
14553 case scUnknown:
14554 if (Instruction *I =
14555 dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue())) {
14556 if (I->getParent() == BB)
14557 return DominatesBlock;
14558 if (DT.properlyDominates(A: I->getParent(), B: BB))
14559 return ProperlyDominatesBlock;
14560 return DoesNotDominateBlock;
14561 }
14562 return ProperlyDominatesBlock;
14563 case scCouldNotCompute:
14564 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14565 }
14566 llvm_unreachable("Unknown SCEV kind!");
14567}
14568
14569bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
14570 return getBlockDisposition(S, BB) >= DominatesBlock;
14571}
14572
14573bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) {
14574 return getBlockDisposition(S, BB) == ProperlyDominatesBlock;
14575}
14576
14577bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const {
14578 return SCEVExprContains(Root: S, Pred: [&](const SCEV *Expr) { return Expr == Op; });
14579}
14580
14581void ScalarEvolution::forgetBackedgeTakenCounts(const Loop *L,
14582 bool Predicated) {
14583 auto &BECounts =
14584 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14585 auto It = BECounts.find(Val: L);
14586 if (It != BECounts.end()) {
14587 for (const ExitNotTakenInfo &ENT : It->second.ExitNotTaken) {
14588 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14589 if (!isa<SCEVConstant>(Val: S)) {
14590 auto UserIt = BECountUsers.find(Val: S);
14591 assert(UserIt != BECountUsers.end());
14592 UserIt->second.erase(Ptr: {L, Predicated});
14593 }
14594 }
14595 }
14596 BECounts.erase(I: It);
14597 }
14598}
14599
14600void ScalarEvolution::forgetMemoizedResults(ArrayRef<SCEVUse> SCEVs) {
14601 SmallPtrSet<const SCEV *, 8> ToForget(llvm::from_range, SCEVs);
14602 SmallVector<SCEVUse, 8> Worklist(ToForget.begin(), ToForget.end());
14603
14604 while (!Worklist.empty()) {
14605 const SCEV *Curr = Worklist.pop_back_val();
14606 auto Users = SCEVUsers.find(Val: Curr);
14607 if (Users != SCEVUsers.end())
14608 for (const auto *User : Users->second)
14609 if (ToForget.insert(Ptr: User).second)
14610 Worklist.push_back(Elt: User);
14611 }
14612
14613 for (const auto *S : ToForget)
14614 forgetMemoizedResultsImpl(S);
14615
14616 PredicatedSCEVRewrites.remove_if(
14617 Pred: [&](const auto &Entry) { return ToForget.count(Ptr: Entry.first.first); });
14618}
14619
14620void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) {
14621 LoopDispositions.erase(Val: S);
14622 BlockDispositions.erase(Val: S);
14623 UnsignedRanges.erase(Val: S);
14624 SignedRanges.erase(Val: S);
14625 HasRecMap.erase(Val: S);
14626 ConstantMultipleCache.erase(Val: S);
14627
14628 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
14629 UnsignedWrapViaInductionTried.erase(Ptr: AR);
14630 SignedWrapViaInductionTried.erase(Ptr: AR);
14631 }
14632
14633 auto ExprIt = ExprValueMap.find(Val: S);
14634 if (ExprIt != ExprValueMap.end()) {
14635 for (Value *V : ExprIt->second) {
14636 auto ValueIt = ValueExprMap.find_as(Val: V);
14637 if (ValueIt != ValueExprMap.end())
14638 ValueExprMap.erase(I: ValueIt);
14639 }
14640 ExprValueMap.erase(I: ExprIt);
14641 }
14642
14643 auto ScopeIt = ValuesAtScopes.find(Val: S);
14644 if (ScopeIt != ValuesAtScopes.end()) {
14645 for (const auto &Pair : ScopeIt->second)
14646 if (!isa_and_nonnull<SCEVConstant>(Val: Pair.second))
14647 llvm::erase(C&: ValuesAtScopesUsers[Pair.second],
14648 V: std::make_pair(x: Pair.first, y&: S));
14649 ValuesAtScopes.erase(I: ScopeIt);
14650 }
14651
14652 auto ScopeUserIt = ValuesAtScopesUsers.find(Val: S);
14653 if (ScopeUserIt != ValuesAtScopesUsers.end()) {
14654 for (const auto &Pair : ScopeUserIt->second)
14655 llvm::erase(C&: ValuesAtScopes[Pair.second], V: std::make_pair(x: Pair.first, y&: S));
14656 ValuesAtScopesUsers.erase(I: ScopeUserIt);
14657 }
14658
14659 auto BEUsersIt = BECountUsers.find(Val: S);
14660 if (BEUsersIt != BECountUsers.end()) {
14661 // Work on a copy, as forgetBackedgeTakenCounts() will modify the original.
14662 auto Copy = BEUsersIt->second;
14663 for (const auto &Pair : Copy)
14664 forgetBackedgeTakenCounts(L: Pair.getPointer(), Predicated: Pair.getInt());
14665 BECountUsers.erase(I: BEUsersIt);
14666 }
14667
14668 auto FoldUser = FoldCacheUser.find(Val: S);
14669 if (FoldUser != FoldCacheUser.end())
14670 for (auto &KV : FoldUser->second)
14671 FoldCache.erase(Val: KV);
14672 FoldCacheUser.erase(Val: S);
14673}
14674
14675void
14676ScalarEvolution::getUsedLoops(const SCEV *S,
14677 SmallPtrSetImpl<const Loop *> &LoopsUsed) {
14678 struct FindUsedLoops {
14679 FindUsedLoops(SmallPtrSetImpl<const Loop *> &LoopsUsed)
14680 : LoopsUsed(LoopsUsed) {}
14681 SmallPtrSetImpl<const Loop *> &LoopsUsed;
14682 bool follow(const SCEV *S) {
14683 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S))
14684 LoopsUsed.insert(Ptr: AR->getLoop());
14685 return true;
14686 }
14687
14688 bool isDone() const { return false; }
14689 };
14690
14691 FindUsedLoops F(LoopsUsed);
14692 SCEVTraversal<FindUsedLoops>(F).visitAll(Root: S);
14693}
14694
14695void ScalarEvolution::getReachableBlocks(
14696 SmallPtrSetImpl<BasicBlock *> &Reachable, Function &F) {
14697 SmallVector<BasicBlock *> Worklist;
14698 Worklist.push_back(Elt: &F.getEntryBlock());
14699 while (!Worklist.empty()) {
14700 BasicBlock *BB = Worklist.pop_back_val();
14701 if (!Reachable.insert(Ptr: BB).second)
14702 continue;
14703
14704 Value *Cond;
14705 BasicBlock *TrueBB, *FalseBB;
14706 if (match(V: BB->getTerminator(), P: m_Br(C: m_Value(V&: Cond), T: m_BasicBlock(V&: TrueBB),
14707 F: m_BasicBlock(V&: FalseBB)))) {
14708 if (auto *C = dyn_cast<ConstantInt>(Val: Cond)) {
14709 Worklist.push_back(Elt: C->isOne() ? TrueBB : FalseBB);
14710 continue;
14711 }
14712
14713 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
14714 const SCEV *L = getSCEV(V: Cmp->getOperand(i_nocapture: 0));
14715 const SCEV *R = getSCEV(V: Cmp->getOperand(i_nocapture: 1));
14716 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getCmpPredicate(), LHS: L, RHS: R)) {
14717 Worklist.push_back(Elt: TrueBB);
14718 continue;
14719 }
14720 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getInverseCmpPredicate(), LHS: L,
14721 RHS: R)) {
14722 Worklist.push_back(Elt: FalseBB);
14723 continue;
14724 }
14725 }
14726 }
14727
14728 append_range(C&: Worklist, R: successors(BB));
14729 }
14730}
14731
14732void ScalarEvolution::verify() const {
14733 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14734 ScalarEvolution SE2(F, TLI, AC, DT, LI);
14735
14736 SmallVector<Loop *, 8> LoopStack(LI.begin(), LI.end());
14737
14738 // Map's SCEV expressions from one ScalarEvolution "universe" to another.
14739 struct SCEVMapper : public SCEVRewriteVisitor<SCEVMapper> {
14740 SCEVMapper(ScalarEvolution &SE) : SCEVRewriteVisitor<SCEVMapper>(SE) {}
14741
14742 const SCEV *visitConstant(const SCEVConstant *Constant) {
14743 return SE.getConstant(Val: Constant->getAPInt());
14744 }
14745
14746 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14747 return SE.getUnknown(V: Expr->getValue());
14748 }
14749
14750 const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) {
14751 return SE.getCouldNotCompute();
14752 }
14753 };
14754
14755 SCEVMapper SCM(SE2);
14756 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
14757 SE2.getReachableBlocks(Reachable&: ReachableBlocks, F);
14758
14759 auto GetDelta = [&](const SCEV *Old, const SCEV *New) -> const SCEV * {
14760 if (containsUndefs(S: Old) || containsUndefs(S: New)) {
14761 // SCEV treats "undef" as an unknown but consistent value (i.e. it does
14762 // not propagate undef aggressively). This means we can (and do) fail
14763 // verification in cases where a transform makes a value go from "undef"
14764 // to "undef+1" (say). The transform is fine, since in both cases the
14765 // result is "undef", but SCEV thinks the value increased by 1.
14766 return nullptr;
14767 }
14768
14769 // Unless VerifySCEVStrict is set, we only compare constant deltas.
14770 const SCEV *Delta = SE2.getMinusSCEV(LHS: Old, RHS: New);
14771 if (!VerifySCEVStrict && !isa<SCEVConstant>(Val: Delta))
14772 return nullptr;
14773
14774 return Delta;
14775 };
14776
14777 while (!LoopStack.empty()) {
14778 auto *L = LoopStack.pop_back_val();
14779 llvm::append_range(C&: LoopStack, R&: *L);
14780
14781 // Only verify BECounts in reachable loops. For an unreachable loop,
14782 // any BECount is legal.
14783 if (!ReachableBlocks.contains(Ptr: L->getHeader()))
14784 continue;
14785
14786 // Only verify cached BECounts. Computing new BECounts may change the
14787 // results of subsequent SCEV uses.
14788 auto It = BackedgeTakenCounts.find(Val: L);
14789 if (It == BackedgeTakenCounts.end())
14790 continue;
14791
14792 auto *CurBECount =
14793 SCM.visit(S: It->second.getExact(L, SE: const_cast<ScalarEvolution *>(this)));
14794 auto *NewBECount = SE2.getBackedgeTakenCount(L);
14795
14796 if (CurBECount == SE2.getCouldNotCompute() ||
14797 NewBECount == SE2.getCouldNotCompute()) {
14798 // NB! This situation is legal, but is very suspicious -- whatever pass
14799 // change the loop to make a trip count go from could not compute to
14800 // computable or vice-versa *should have* invalidated SCEV. However, we
14801 // choose not to assert here (for now) since we don't want false
14802 // positives.
14803 continue;
14804 }
14805
14806 if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) >
14807 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14808 NewBECount = SE2.getZeroExtendExpr(Op: NewBECount, Ty: CurBECount->getType());
14809 else if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) <
14810 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14811 CurBECount = SE2.getZeroExtendExpr(Op: CurBECount, Ty: NewBECount->getType());
14812
14813 const SCEV *Delta = GetDelta(CurBECount, NewBECount);
14814 if (Delta && !Delta->isZero()) {
14815 dbgs() << "Trip Count for " << *L << " Changed!\n";
14816 dbgs() << "Old: " << *CurBECount << "\n";
14817 dbgs() << "New: " << *NewBECount << "\n";
14818 dbgs() << "Delta: " << *Delta << "\n";
14819 std::abort();
14820 }
14821 }
14822
14823 // Collect all valid loops currently in LoopInfo.
14824 SmallPtrSet<Loop *, 32> ValidLoops;
14825 SmallVector<Loop *, 32> Worklist(LI.begin(), LI.end());
14826 while (!Worklist.empty()) {
14827 Loop *L = Worklist.pop_back_val();
14828 if (ValidLoops.insert(Ptr: L).second)
14829 Worklist.append(in_start: L->begin(), in_end: L->end());
14830 }
14831 for (const auto &KV : ValueExprMap) {
14832#ifndef NDEBUG
14833 // Check for SCEV expressions referencing invalid/deleted loops.
14834 if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) {
14835 assert(ValidLoops.contains(AR->getLoop()) &&
14836 "AddRec references invalid loop");
14837 }
14838#endif
14839
14840 // Check that the value is also part of the reverse map.
14841 auto It = ExprValueMap.find(Val: KV.second);
14842 if (It == ExprValueMap.end() || !It->second.contains(key: KV.first)) {
14843 dbgs() << "Value " << *KV.first
14844 << " is in ValueExprMap but not in ExprValueMap\n";
14845 std::abort();
14846 }
14847
14848 if (auto *I = dyn_cast<Instruction>(Val: &*KV.first)) {
14849 if (!ReachableBlocks.contains(Ptr: I->getParent()))
14850 continue;
14851 const SCEV *OldSCEV = SCM.visit(S: KV.second);
14852 const SCEV *NewSCEV = SE2.getSCEV(V: I);
14853 const SCEV *Delta = GetDelta(OldSCEV, NewSCEV);
14854 if (Delta && !Delta->isZero()) {
14855 dbgs() << "SCEV for value " << *I << " changed!\n"
14856 << "Old: " << *OldSCEV << "\n"
14857 << "New: " << *NewSCEV << "\n"
14858 << "Delta: " << *Delta << "\n";
14859 std::abort();
14860 }
14861 }
14862 }
14863
14864 for (const auto &KV : ExprValueMap) {
14865 for (Value *V : KV.second) {
14866 const SCEV *S = ValueExprMap.lookup(Val: V);
14867 if (!S) {
14868 dbgs() << "Value " << *V
14869 << " is in ExprValueMap but not in ValueExprMap\n";
14870 std::abort();
14871 }
14872 if (S != KV.first) {
14873 dbgs() << "Value " << *V << " mapped to " << *S << " rather than "
14874 << *KV.first << "\n";
14875 std::abort();
14876 }
14877 }
14878 }
14879
14880 // Verify integrity of SCEV users.
14881 for (const auto &S : UniqueSCEVs) {
14882 for (SCEVUse Op : S.operands()) {
14883 // We do not store dependencies of constants.
14884 if (isa<SCEVConstant>(Val: Op))
14885 continue;
14886 auto It = SCEVUsers.find(Val: Op);
14887 if (It != SCEVUsers.end() && It->second.count(Ptr: &S))
14888 continue;
14889 dbgs() << "Use of operand " << *Op << " by user " << S
14890 << " is not being tracked!\n";
14891 std::abort();
14892 }
14893 }
14894
14895 // Verify integrity of ValuesAtScopes users.
14896 for (const auto &ValueAndVec : ValuesAtScopes) {
14897 const SCEV *Value = ValueAndVec.first;
14898 for (const auto &LoopAndValueAtScope : ValueAndVec.second) {
14899 const Loop *L = LoopAndValueAtScope.first;
14900 const SCEV *ValueAtScope = LoopAndValueAtScope.second;
14901 if (!isa<SCEVConstant>(Val: ValueAtScope)) {
14902 auto It = ValuesAtScopesUsers.find(Val: ValueAtScope);
14903 if (It != ValuesAtScopesUsers.end() &&
14904 is_contained(Range: It->second, Element: std::make_pair(x&: L, y&: Value)))
14905 continue;
14906 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14907 << *ValueAtScope << " missing in ValuesAtScopesUsers\n";
14908 std::abort();
14909 }
14910 }
14911 }
14912
14913 for (const auto &ValueAtScopeAndVec : ValuesAtScopesUsers) {
14914 const SCEV *ValueAtScope = ValueAtScopeAndVec.first;
14915 for (const auto &LoopAndValue : ValueAtScopeAndVec.second) {
14916 const Loop *L = LoopAndValue.first;
14917 const SCEV *Value = LoopAndValue.second;
14918 assert(!isa<SCEVConstant>(Value));
14919 auto It = ValuesAtScopes.find(Val: Value);
14920 if (It != ValuesAtScopes.end() &&
14921 is_contained(Range: It->second, Element: std::make_pair(x&: L, y&: ValueAtScope)))
14922 continue;
14923 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14924 << *ValueAtScope << " missing in ValuesAtScopes\n";
14925 std::abort();
14926 }
14927 }
14928
14929 // Verify integrity of BECountUsers.
14930 auto VerifyBECountUsers = [&](bool Predicated) {
14931 auto &BECounts =
14932 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14933 for (const auto &LoopAndBEInfo : BECounts) {
14934 for (const ExitNotTakenInfo &ENT : LoopAndBEInfo.second.ExitNotTaken) {
14935 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14936 if (!isa<SCEVConstant>(Val: S)) {
14937 auto UserIt = BECountUsers.find(Val: S);
14938 if (UserIt != BECountUsers.end() &&
14939 UserIt->second.contains(Ptr: { LoopAndBEInfo.first, Predicated }))
14940 continue;
14941 dbgs() << "Value " << *S << " for loop " << *LoopAndBEInfo.first
14942 << " missing from BECountUsers\n";
14943 std::abort();
14944 }
14945 }
14946 }
14947 }
14948 };
14949 VerifyBECountUsers(/* Predicated */ false);
14950 VerifyBECountUsers(/* Predicated */ true);
14951
14952 // Verify intergity of loop disposition cache.
14953 for (auto &[S, Values] : LoopDispositions) {
14954 for (auto [Loop, CachedDisposition] : Values) {
14955 const auto RecomputedDisposition = SE2.getLoopDisposition(S, L: Loop);
14956 if (CachedDisposition != RecomputedDisposition) {
14957 dbgs() << "Cached disposition of " << *S << " for loop " << *Loop
14958 << " is incorrect: cached " << CachedDisposition << ", actual "
14959 << RecomputedDisposition << "\n";
14960 std::abort();
14961 }
14962 }
14963 }
14964
14965 // Verify integrity of the block disposition cache.
14966 for (auto &[S, Values] : BlockDispositions) {
14967 for (auto [BB, CachedDisposition] : Values) {
14968 const auto RecomputedDisposition = SE2.getBlockDisposition(S, BB);
14969 if (CachedDisposition != RecomputedDisposition) {
14970 dbgs() << "Cached disposition of " << *S << " for block %"
14971 << BB->getName() << " is incorrect: cached " << CachedDisposition
14972 << ", actual " << RecomputedDisposition << "\n";
14973 std::abort();
14974 }
14975 }
14976 }
14977
14978 // Verify FoldCache/FoldCacheUser caches.
14979 for (auto [FoldID, Expr] : FoldCache) {
14980 auto I = FoldCacheUser.find(Val: Expr);
14981 if (I == FoldCacheUser.end()) {
14982 dbgs() << "Missing entry in FoldCacheUser for cached expression " << *Expr
14983 << "!\n";
14984 std::abort();
14985 }
14986 if (!is_contained(Range: I->second, Element: FoldID)) {
14987 dbgs() << "Missing FoldID in cached users of " << *Expr << "!\n";
14988 std::abort();
14989 }
14990 }
14991 for (auto [Expr, IDs] : FoldCacheUser) {
14992 for (auto &FoldID : IDs) {
14993 const SCEV *S = FoldCache.lookup(Val: FoldID);
14994 if (!S) {
14995 dbgs() << "Missing entry in FoldCache for expression " << *Expr
14996 << "!\n";
14997 std::abort();
14998 }
14999 if (S != Expr) {
15000 dbgs() << "Entry in FoldCache doesn't match FoldCacheUser: " << *S
15001 << " != " << *Expr << "!\n";
15002 std::abort();
15003 }
15004 }
15005 }
15006
15007 // Verify that ConstantMultipleCache computations are correct. We check that
15008 // cached multiples and recomputed multiples are multiples of each other to
15009 // verify correctness. It is possible that a recomputed multiple is different
15010 // from the cached multiple due to strengthened no wrap flags or changes in
15011 // KnownBits computations.
15012 for (auto [S, Multiple] : ConstantMultipleCache) {
15013 APInt RecomputedMultiple = SE2.getConstantMultiple(S);
15014 if ((Multiple != 0 && RecomputedMultiple != 0 &&
15015 Multiple.urem(RHS: RecomputedMultiple) != 0 &&
15016 RecomputedMultiple.urem(RHS: Multiple) != 0)) {
15017 dbgs() << "Incorrect cached computation in ConstantMultipleCache for "
15018 << *S << " : Computed " << RecomputedMultiple
15019 << " but cache contains " << Multiple << "!\n";
15020 std::abort();
15021 }
15022 }
15023}
15024
15025bool ScalarEvolution::invalidate(
15026 Function &F, const PreservedAnalyses &PA,
15027 FunctionAnalysisManager::Invalidator &Inv) {
15028 // Invalidate the ScalarEvolution object whenever it isn't preserved or one
15029 // of its dependencies is invalidated.
15030 auto PAC = PA.getChecker<ScalarEvolutionAnalysis>();
15031 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>()) ||
15032 Inv.invalidate<AssumptionAnalysis>(IR&: F, PA) ||
15033 Inv.invalidate<DominatorTreeAnalysis>(IR&: F, PA) ||
15034 Inv.invalidate<LoopAnalysis>(IR&: F, PA);
15035}
15036
15037AnalysisKey ScalarEvolutionAnalysis::Key;
15038
15039ScalarEvolution ScalarEvolutionAnalysis::run(Function &F,
15040 FunctionAnalysisManager &AM) {
15041 auto &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
15042 auto &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
15043 auto &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
15044 auto &LI = AM.getResult<LoopAnalysis>(IR&: F);
15045 return ScalarEvolution(F, TLI, AC, DT, LI);
15046}
15047
15048PreservedAnalyses
15049ScalarEvolutionVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
15050 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).verify();
15051 return PreservedAnalyses::all();
15052}
15053
15054PreservedAnalyses
15055ScalarEvolutionPrinterPass::run(Function &F, FunctionAnalysisManager &AM) {
15056 // For compatibility with opt's -analyze feature under legacy pass manager
15057 // which was not ported to NPM. This keeps tests using
15058 // update_analyze_test_checks.py working.
15059 OS << "Printing analysis 'Scalar Evolution Analysis' for function '"
15060 << F.getName() << "':\n";
15061 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).print(OS);
15062 return PreservedAnalyses::all();
15063}
15064
15065INITIALIZE_PASS_BEGIN(ScalarEvolutionWrapperPass, "scalar-evolution",
15066 "Scalar Evolution Analysis", false, true)
15067INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
15068INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
15069INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
15070INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
15071INITIALIZE_PASS_END(ScalarEvolutionWrapperPass, "scalar-evolution",
15072 "Scalar Evolution Analysis", false, true)
15073
15074char ScalarEvolutionWrapperPass::ID = 0;
15075
15076ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) {}
15077
15078bool ScalarEvolutionWrapperPass::runOnFunction(Function &F) {
15079 SE.reset(p: new ScalarEvolution(
15080 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F),
15081 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
15082 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
15083 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()));
15084 return false;
15085}
15086
15087void ScalarEvolutionWrapperPass::releaseMemory() { SE.reset(); }
15088
15089void ScalarEvolutionWrapperPass::print(raw_ostream &OS, const Module *) const {
15090 SE->print(OS);
15091}
15092
15093void ScalarEvolutionWrapperPass::verifyAnalysis() const {
15094 if (!VerifySCEV)
15095 return;
15096
15097 SE->verify();
15098}
15099
15100void ScalarEvolutionWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
15101 AU.setPreservesAll();
15102 AU.addRequiredTransitive<AssumptionCacheTracker>();
15103 AU.addRequiredTransitive<LoopInfoWrapperPass>();
15104 AU.addRequiredTransitive<DominatorTreeWrapperPass>();
15105 AU.addRequiredTransitive<TargetLibraryInfoWrapperPass>();
15106}
15107
15108const SCEVPredicate *ScalarEvolution::getEqualPredicate(const SCEV *LHS,
15109 const SCEV *RHS) {
15110 return getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS, RHS);
15111}
15112
15113const SCEVPredicate *
15114ScalarEvolution::getComparePredicate(const ICmpInst::Predicate Pred,
15115 const SCEV *LHS, const SCEV *RHS) {
15116 FoldingSetNodeID ID;
15117 assert(LHS->getType() == RHS->getType() &&
15118 "Type mismatch between LHS and RHS");
15119 // Unique this node based on the arguments
15120 ID.AddInteger(I: SCEVPredicate::P_Compare);
15121 ID.AddInteger(I: Pred);
15122 ID.AddPointer(Ptr: LHS);
15123 ID.AddPointer(Ptr: RHS);
15124 void *IP = nullptr;
15125 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, InsertPos&: IP))
15126 return S;
15127 SCEVComparePredicate *Eq = new (SCEVAllocator)
15128 SCEVComparePredicate(ID.Intern(Allocator&: SCEVAllocator), Pred, LHS, RHS);
15129 UniquePreds.InsertNode(N: Eq, InsertPos: IP);
15130 return Eq;
15131}
15132
15133const SCEVPredicate *ScalarEvolution::getWrapPredicate(
15134 const SCEVAddRecExpr *AR,
15135 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15136 FoldingSetNodeID ID;
15137 // Unique this node based on the arguments
15138 ID.AddInteger(I: SCEVPredicate::P_Wrap);
15139 ID.AddPointer(Ptr: AR);
15140 ID.AddInteger(I: AddedFlags);
15141 void *IP = nullptr;
15142 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, InsertPos&: IP))
15143 return S;
15144 auto *OF = new (SCEVAllocator)
15145 SCEVWrapPredicate(ID.Intern(Allocator&: SCEVAllocator), AR, AddedFlags);
15146 UniquePreds.InsertNode(N: OF, InsertPos: IP);
15147 return OF;
15148}
15149
15150namespace {
15151
15152class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
15153public:
15154
15155 /// Rewrites \p S in the context of a loop L and the SCEV predication
15156 /// infrastructure.
15157 ///
15158 /// If \p Pred is non-null, the SCEV expression is rewritten to respect the
15159 /// equivalences present in \p Pred.
15160 ///
15161 /// If \p NewPreds is non-null, rewrite is free to add further predicates to
15162 /// \p NewPreds such that the result will be an AddRecExpr.
15163 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
15164 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15165 const SCEVPredicate *Pred) {
15166 SCEVPredicateRewriter Rewriter(L, SE, NewPreds, Pred);
15167 return Rewriter.visit(S);
15168 }
15169
15170 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
15171 if (Pred) {
15172 if (auto *U = dyn_cast<SCEVUnionPredicate>(Val: Pred)) {
15173 for (const auto *Pred : U->getPredicates())
15174 if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred))
15175 if (IPred->getLHS() == Expr &&
15176 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15177 return IPred->getRHS();
15178 } else if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred)) {
15179 if (IPred->getLHS() == Expr &&
15180 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15181 return IPred->getRHS();
15182 }
15183 }
15184 return convertToAddRecWithPreds(Expr);
15185 }
15186
15187 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
15188 const SCEV *Operand = visit(S: Expr->getOperand());
15189 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15190 if (AR && AR->getLoop() == L && AR->isAffine()) {
15191 // This couldn't be folded because the operand didn't have the nuw
15192 // flag. Add the nusw flag as an assumption that we could make.
15193 const SCEV *Step = AR->getStepRecurrence(SE);
15194 Type *Ty = Expr->getType();
15195 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNUSW))
15196 return SE.getAddRecExpr(Start: SE.getZeroExtendExpr(Op: AR->getStart(), Ty),
15197 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15198 Flags: AR->getNoWrapFlags());
15199 }
15200 return SE.getZeroExtendExpr(Op: Operand, Ty: Expr->getType());
15201 }
15202
15203 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
15204 const SCEV *Operand = visit(S: Expr->getOperand());
15205 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15206 if (AR && AR->getLoop() == L && AR->isAffine()) {
15207 // This couldn't be folded because the operand didn't have the nsw
15208 // flag. Add the nssw flag as an assumption that we could make.
15209 const SCEV *Step = AR->getStepRecurrence(SE);
15210 Type *Ty = Expr->getType();
15211 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNSSW))
15212 return SE.getAddRecExpr(Start: SE.getSignExtendExpr(Op: AR->getStart(), Ty),
15213 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15214 Flags: AR->getNoWrapFlags());
15215 }
15216 return SE.getSignExtendExpr(Op: Operand, Ty: Expr->getType());
15217 }
15218
15219private:
15220 explicit SCEVPredicateRewriter(
15221 const Loop *L, ScalarEvolution &SE,
15222 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15223 const SCEVPredicate *Pred)
15224 : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred), L(L) {}
15225
15226 bool addOverflowAssumption(const SCEVPredicate *P) {
15227 if (!NewPreds) {
15228 // Check if we've already made this assumption.
15229 return Pred && Pred->implies(N: P, SE);
15230 }
15231 NewPreds->push_back(Elt: P);
15232 return true;
15233 }
15234
15235 bool addOverflowAssumption(const SCEVAddRecExpr *AR,
15236 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15237 auto *A = SE.getWrapPredicate(AR, AddedFlags);
15238 return addOverflowAssumption(P: A);
15239 }
15240
15241 // If \p Expr represents a PHINode, we try to see if it can be represented
15242 // as an AddRec, possibly under a predicate (PHISCEVPred). If it is possible
15243 // to add this predicate as a runtime overflow check, we return the AddRec.
15244 // If \p Expr does not meet these conditions (is not a PHI node, or we
15245 // couldn't create an AddRec for it, or couldn't add the predicate), we just
15246 // return \p Expr.
15247 const SCEV *convertToAddRecWithPreds(const SCEVUnknown *Expr) {
15248 if (!isa<PHINode>(Val: Expr->getValue()))
15249 return Expr;
15250 std::optional<
15251 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
15252 PredicatedRewrite = SE.createAddRecFromPHIWithCasts(SymbolicPHI: Expr);
15253 if (!PredicatedRewrite)
15254 return Expr;
15255 for (const auto *P : PredicatedRewrite->second){
15256 // Wrap predicates from outer loops are not supported.
15257 if (auto *WP = dyn_cast<const SCEVWrapPredicate>(Val: P)) {
15258 if (L != WP->getExpr()->getLoop())
15259 return Expr;
15260 }
15261 if (!addOverflowAssumption(P))
15262 return Expr;
15263 }
15264 return PredicatedRewrite->first;
15265 }
15266
15267 SmallVectorImpl<const SCEVPredicate *> *NewPreds;
15268 const SCEVPredicate *Pred;
15269 const Loop *L;
15270};
15271
15272} // end anonymous namespace
15273
15274const SCEV *
15275ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L,
15276 const SCEVPredicate &Preds) {
15277 return SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: nullptr, Pred: &Preds);
15278}
15279
15280const SCEVAddRecExpr *ScalarEvolution::convertSCEVToAddRecWithPredicates(
15281 const SCEV *S, const Loop *L,
15282 SmallVectorImpl<const SCEVPredicate *> &Preds) {
15283 SmallVector<const SCEVPredicate *> TransformPreds;
15284 S = SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: &TransformPreds, Pred: nullptr);
15285 auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S);
15286
15287 if (!AddRec)
15288 return nullptr;
15289
15290 // Check if any of the transformed predicates is known to be false. In that
15291 // case, it doesn't make sense to convert to a predicated AddRec, as the
15292 // versioned loop will never execute.
15293 for (const SCEVPredicate *Pred : TransformPreds) {
15294 auto *WrapPred = dyn_cast<SCEVWrapPredicate>(Val: Pred);
15295 if (!WrapPred || WrapPred->getFlags() != SCEVWrapPredicate::IncrementNSSW)
15296 continue;
15297
15298 const SCEVAddRecExpr *AddRecToCheck = WrapPred->getExpr();
15299 const SCEV *ExitCount = getBackedgeTakenCount(L: AddRecToCheck->getLoop());
15300 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
15301 continue;
15302
15303 const SCEV *Step = AddRecToCheck->getStepRecurrence(SE&: *this);
15304 if (!Step->isOne())
15305 continue;
15306
15307 ExitCount = getTruncateOrSignExtend(V: ExitCount, Ty: Step->getType());
15308 const SCEV *Add = getAddExpr(LHS: AddRecToCheck->getStart(), RHS: ExitCount);
15309 if (isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS: Add, RHS: AddRecToCheck->getStart()))
15310 return nullptr;
15311 }
15312
15313 // Since the transformation was successful, we can now transfer the SCEV
15314 // predicates.
15315 Preds.append(in_start: TransformPreds.begin(), in_end: TransformPreds.end());
15316
15317 return AddRec;
15318}
15319
15320/// SCEV predicates
15321SCEVPredicate::SCEVPredicate(const FoldingSetNodeIDRef ID,
15322 SCEVPredicateKind Kind)
15323 : FastID(ID), Kind(Kind) {}
15324
15325SCEVComparePredicate::SCEVComparePredicate(const FoldingSetNodeIDRef ID,
15326 const ICmpInst::Predicate Pred,
15327 const SCEV *LHS, const SCEV *RHS)
15328 : SCEVPredicate(ID, P_Compare), Pred(Pred), LHS(LHS), RHS(RHS) {
15329 assert(LHS->getType() == RHS->getType() && "LHS and RHS types don't match");
15330 assert(LHS != RHS && "LHS and RHS are the same SCEV");
15331}
15332
15333bool SCEVComparePredicate::implies(const SCEVPredicate *N,
15334 ScalarEvolution &SE) const {
15335 const auto *Op = dyn_cast<SCEVComparePredicate>(Val: N);
15336
15337 if (!Op)
15338 return false;
15339
15340 if (Pred != ICmpInst::ICMP_EQ)
15341 return false;
15342
15343 return Op->LHS == LHS && Op->RHS == RHS;
15344}
15345
15346bool SCEVComparePredicate::isAlwaysTrue() const { return false; }
15347
15348void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const {
15349 if (Pred == ICmpInst::ICMP_EQ)
15350 OS.indent(NumSpaces: Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n";
15351 else
15352 OS.indent(NumSpaces: Depth) << "Compare predicate: " << *LHS << " " << Pred << ") "
15353 << *RHS << "\n";
15354
15355}
15356
15357SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID,
15358 const SCEVAddRecExpr *AR,
15359 IncrementWrapFlags Flags)
15360 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {}
15361
15362const SCEVAddRecExpr *SCEVWrapPredicate::getExpr() const { return AR; }
15363
15364bool SCEVWrapPredicate::implies(const SCEVPredicate *N,
15365 ScalarEvolution &SE) const {
15366 const auto *Op = dyn_cast<SCEVWrapPredicate>(Val: N);
15367 if (!Op || setFlags(Flags, OnFlags: Op->Flags) != Flags)
15368 return false;
15369
15370 if (Op->AR == AR)
15371 return true;
15372
15373 if (Flags != SCEVWrapPredicate::IncrementNSSW &&
15374 Flags != SCEVWrapPredicate::IncrementNUSW)
15375 return false;
15376
15377 const SCEV *Start = AR->getStart();
15378 const SCEV *OpStart = Op->AR->getStart();
15379 if (Start->getType()->isPointerTy() != OpStart->getType()->isPointerTy())
15380 return false;
15381
15382 // Reject pointers to different address spaces.
15383 if (Start->getType()->isPointerTy() && Start->getType() != OpStart->getType())
15384 return false;
15385
15386 // NUSW/NSSW on a wider-type AddRec does not imply the same on a
15387 // narrower-type AddRec.
15388 if (SE.getTypeSizeInBits(Ty: AR->getType()) >
15389 SE.getTypeSizeInBits(Ty: Op->AR->getType()))
15390 return false;
15391
15392 const SCEV *Step = AR->getStepRecurrence(SE);
15393 const SCEV *OpStep = Op->AR->getStepRecurrence(SE);
15394 if (!SE.isKnownPositive(S: Step) || !SE.isKnownPositive(S: OpStep))
15395 return false;
15396
15397 // If both steps are positive, this implies N, if N's start and step are
15398 // ULE/SLE (for NSUW/NSSW) than this'.
15399 Type *WiderTy = SE.getWiderType(T1: Step->getType(), T2: OpStep->getType());
15400 Step = SE.getNoopOrZeroExtend(V: Step, Ty: WiderTy);
15401 OpStep = SE.getNoopOrZeroExtend(V: OpStep, Ty: WiderTy);
15402
15403 bool IsNUW = Flags == SCEVWrapPredicate::IncrementNUSW;
15404 OpStart = IsNUW ? SE.getNoopOrZeroExtend(V: OpStart, Ty: WiderTy)
15405 : SE.getNoopOrSignExtend(V: OpStart, Ty: WiderTy);
15406 Start = IsNUW ? SE.getNoopOrZeroExtend(V: Start, Ty: WiderTy)
15407 : SE.getNoopOrSignExtend(V: Start, Ty: WiderTy);
15408 CmpInst::Predicate Pred = IsNUW ? CmpInst::ICMP_ULE : CmpInst::ICMP_SLE;
15409 return SE.isKnownPredicate(Pred, LHS: OpStep, RHS: Step) &&
15410 SE.isKnownPredicate(Pred, LHS: OpStart, RHS: Start);
15411}
15412
15413bool SCEVWrapPredicate::isAlwaysTrue() const {
15414 SCEV::NoWrapFlags ScevFlags = AR->getNoWrapFlags();
15415 IncrementWrapFlags IFlags = Flags;
15416
15417 if (ScalarEvolution::setFlags(Flags: ScevFlags, OnFlags: SCEV::FlagNSW) == ScevFlags)
15418 IFlags = clearFlags(Flags: IFlags, OffFlags: IncrementNSSW);
15419
15420 return IFlags == IncrementAnyWrap;
15421}
15422
15423void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const {
15424 OS.indent(NumSpaces: Depth) << *getExpr() << " Added Flags: ";
15425 if (SCEVWrapPredicate::IncrementNUSW & getFlags())
15426 OS << "<nusw>";
15427 if (SCEVWrapPredicate::IncrementNSSW & getFlags())
15428 OS << "<nssw>";
15429 OS << "\n";
15430}
15431
15432SCEVWrapPredicate::IncrementWrapFlags
15433SCEVWrapPredicate::getImpliedFlags(const SCEVAddRecExpr *AR,
15434 ScalarEvolution &SE) {
15435 IncrementWrapFlags ImpliedFlags = IncrementAnyWrap;
15436 SCEV::NoWrapFlags StaticFlags = AR->getNoWrapFlags();
15437
15438 // We can safely transfer the NSW flag as NSSW.
15439 if (ScalarEvolution::setFlags(Flags: StaticFlags, OnFlags: SCEV::FlagNSW) == StaticFlags)
15440 ImpliedFlags = IncrementNSSW;
15441
15442 if (ScalarEvolution::setFlags(Flags: StaticFlags, OnFlags: SCEV::FlagNUW) == StaticFlags) {
15443 // If the increment is positive, the SCEV NUW flag will also imply the
15444 // WrapPredicate NUSW flag.
15445 if (const auto *Step = dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE)))
15446 if (Step->getValue()->getValue().isNonNegative())
15447 ImpliedFlags = setFlags(Flags: ImpliedFlags, OnFlags: IncrementNUSW);
15448 }
15449
15450 return ImpliedFlags;
15451}
15452
15453/// Union predicates don't get cached so create a dummy set ID for it.
15454SCEVUnionPredicate::SCEVUnionPredicate(ArrayRef<const SCEVPredicate *> Preds,
15455 ScalarEvolution &SE)
15456 : SCEVPredicate(FoldingSetNodeIDRef(nullptr, 0), P_Union) {
15457 for (const auto *P : Preds)
15458 add(N: P, SE);
15459}
15460
15461bool SCEVUnionPredicate::isAlwaysTrue() const {
15462 return all_of(Range: Preds,
15463 P: [](const SCEVPredicate *I) { return I->isAlwaysTrue(); });
15464}
15465
15466bool SCEVUnionPredicate::implies(const SCEVPredicate *N,
15467 ScalarEvolution &SE) const {
15468 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N))
15469 return all_of(Range: Set->Preds, P: [this, &SE](const SCEVPredicate *I) {
15470 return this->implies(N: I, SE);
15471 });
15472
15473 if (any_of(Range: Preds,
15474 P: [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); }))
15475 return true;
15476
15477 // A wrap predicate may be implied by a wrap predicate in Preds after applying
15478 // equal predicates.
15479 const auto *NWrap = dyn_cast<SCEVWrapPredicate>(Val: N);
15480 if (!NWrap)
15481 return false;
15482 const Loop *L = NWrap->getExpr()->getLoop();
15483 return any_of(Range: Preds, P: [&](const SCEVPredicate *I) {
15484 const auto *IWrap = dyn_cast<SCEVWrapPredicate>(Val: I);
15485 if (!IWrap)
15486 return false;
15487 const auto *RewrittenAR = dyn_cast<SCEVAddRecExpr>(
15488 Val: SE.rewriteUsingPredicate(S: IWrap->getExpr(), L, Preds: *this));
15489 return RewrittenAR &&
15490 SE.getWrapPredicate(AR: RewrittenAR, AddedFlags: IWrap->getFlags())->implies(N, SE);
15491 });
15492}
15493
15494void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const {
15495 for (const auto *Pred : Preds)
15496 Pred->print(OS, Depth);
15497}
15498
15499void SCEVUnionPredicate::add(const SCEVPredicate *N, ScalarEvolution &SE) {
15500 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N)) {
15501 for (const auto *Pred : Set->Preds)
15502 add(N: Pred, SE);
15503 return;
15504 }
15505
15506 // Implication checks are quadratic in the number of predicates. Stop doing
15507 // them if there are many predicates, as they should be too expensive to use
15508 // anyway at that point.
15509 bool CheckImplies = Preds.size() < 16;
15510
15511 // Only add predicate if it is not already implied by this union predicate.
15512 if (CheckImplies && implies(N, SE))
15513 return;
15514
15515 // Build a new vector containing the current predicates, except the ones that
15516 // are implied by the new predicate N.
15517 SmallVector<const SCEVPredicate *> PrunedPreds;
15518 for (auto *P : Preds) {
15519 if (CheckImplies && N->implies(N: P, SE))
15520 continue;
15521 PrunedPreds.push_back(Elt: P);
15522 }
15523 Preds = std::move(PrunedPreds);
15524 Preds.push_back(Elt: N);
15525}
15526
15527PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE,
15528 Loop &L)
15529 : SE(SE), L(L) {
15530 SmallVector<const SCEVPredicate*, 4> Empty;
15531 Preds = std::make_unique<SCEVUnionPredicate>(args&: Empty, args&: SE);
15532}
15533
15534void ScalarEvolution::registerUser(const SCEV *User,
15535 ArrayRef<const SCEV *> Ops) {
15536 for (const auto *Op : Ops)
15537 // We do not expect that forgetting cached data for SCEVConstants will ever
15538 // open any prospects for sharpening or introduce any correctness issues,
15539 // so we don't bother storing their dependencies.
15540 if (!isa<SCEVConstant>(Val: Op))
15541 SCEVUsers[Op].insert(Ptr: User);
15542}
15543
15544void ScalarEvolution::registerUser(const SCEV *User, ArrayRef<SCEVUse> Ops) {
15545 for (const SCEV *Op : Ops)
15546 // We do not expect that forgetting cached data for SCEVConstants will ever
15547 // open any prospects for sharpening or introduce any correctness issues,
15548 // so we don't bother storing their dependencies.
15549 if (!isa<SCEVConstant>(Val: Op))
15550 SCEVUsers[Op].insert(Ptr: User);
15551}
15552
15553const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) {
15554 const SCEV *Expr = SE.getSCEV(V);
15555 return getPredicatedSCEV(Expr);
15556}
15557
15558const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
15559 RewriteEntry &Entry = RewriteMap[Expr];
15560
15561 // If we already have an entry and the version matches, return it.
15562 if (Entry.second && Generation == Entry.first)
15563 return Entry.second;
15564
15565 // We found an entry but it's stale. Rewrite the stale entry
15566 // according to the current predicate.
15567 if (Entry.second)
15568 Expr = Entry.second;
15569
15570 const SCEV *NewSCEV = SE.rewriteUsingPredicate(S: Expr, L: &L, Preds: *Preds);
15571 Entry = {Generation, NewSCEV};
15572
15573 return NewSCEV;
15574}
15575
15576const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
15577 if (!BackedgeCount) {
15578 SmallVector<const SCEVPredicate *, 4> Preds;
15579 BackedgeCount = SE.getPredicatedBackedgeTakenCount(L: &L, Preds);
15580 for (const auto *P : Preds)
15581 addPredicate(Pred: *P);
15582 }
15583 return BackedgeCount;
15584}
15585
15586const SCEV *PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount() {
15587 if (!SymbolicMaxBackedgeCount) {
15588 SmallVector<const SCEVPredicate *, 4> Preds;
15589 SymbolicMaxBackedgeCount =
15590 SE.getPredicatedSymbolicMaxBackedgeTakenCount(L: &L, Preds);
15591 for (const auto *P : Preds)
15592 addPredicate(Pred: *P);
15593 }
15594 return SymbolicMaxBackedgeCount;
15595}
15596
15597unsigned PredicatedScalarEvolution::getSmallConstantMaxTripCount() {
15598 if (!SmallConstantMaxTripCount) {
15599 SmallVector<const SCEVPredicate *, 4> Preds;
15600 SmallConstantMaxTripCount = SE.getSmallConstantMaxTripCount(L: &L, Predicates: &Preds);
15601 for (const auto *P : Preds)
15602 addPredicate(Pred: *P);
15603 }
15604 return *SmallConstantMaxTripCount;
15605}
15606
15607void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
15608 if (Preds->implies(N: &Pred, SE))
15609 return;
15610
15611 SmallVector<const SCEVPredicate *, 4> NewPreds(Preds->getPredicates());
15612 NewPreds.push_back(Elt: &Pred);
15613 Preds = std::make_unique<SCEVUnionPredicate>(args&: NewPreds, args&: SE);
15614 updateGeneration();
15615}
15616
15617void PredicatedScalarEvolution::addPredicates(
15618 ArrayRef<const SCEVPredicate *> Preds) {
15619 for (const SCEVPredicate *P : Preds)
15620 addPredicate(Pred: *P);
15621}
15622
15623const SCEVPredicate &PredicatedScalarEvolution::getPredicate() const {
15624 return *Preds;
15625}
15626
15627void PredicatedScalarEvolution::updateGeneration() {
15628 // If the generation number wrapped recompute everything.
15629 if (++Generation == 0) {
15630 for (auto &II : RewriteMap) {
15631 const SCEV *Rewritten = II.second.second;
15632 II.second = {Generation, SE.rewriteUsingPredicate(S: Rewritten, L: &L, Preds: *Preds)};
15633 }
15634 }
15635}
15636
15637bool PredicatedScalarEvolution::hasNoOverflow(
15638 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) {
15639 const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: getSCEV(V));
15640 if (!AR)
15641 return false;
15642
15643 Flags = SCEVWrapPredicate::clearFlags(
15644 Flags, OffFlags: SCEVWrapPredicate::getImpliedFlags(AR, SE));
15645
15646 return Flags == SCEVWrapPredicate::IncrementAnyWrap;
15647}
15648
15649const SCEVAddRecExpr *PredicatedScalarEvolution::getAsAddRec(
15650 Value *V, SmallVectorImpl<const SCEVPredicate *> *ExtraPreds) {
15651 const SCEV *Expr = this->getSCEV(V);
15652 SmallVector<const SCEVPredicate *, 4> NewPreds;
15653 auto *New = SE.convertSCEVToAddRecWithPredicates(S: Expr, L: &L, Preds&: NewPreds);
15654
15655 if (!New)
15656 return nullptr;
15657
15658 if (ExtraPreds) {
15659 ExtraPreds->append(RHS: NewPreds);
15660 return New;
15661 }
15662
15663 addPredicates(Preds: NewPreds);
15664
15665 RewriteMap[SE.getSCEV(V)] = {Generation, New};
15666 return New;
15667}
15668
15669PredicatedScalarEvolution::PredicatedScalarEvolution(
15670 const PredicatedScalarEvolution &Init)
15671 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L),
15672 Preds(std::make_unique<SCEVUnionPredicate>(args: Init.Preds->getPredicates(),
15673 args&: SE)),
15674 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {}
15675
15676void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const {
15677 // For each block.
15678 for (auto *BB : L.getBlocks())
15679 for (auto &I : *BB) {
15680 if (!SE.isSCEVable(Ty: I.getType()))
15681 continue;
15682
15683 auto *Expr = SE.getSCEV(V: &I);
15684 auto II = RewriteMap.find(Val: Expr);
15685
15686 if (II == RewriteMap.end())
15687 continue;
15688
15689 // Don't print things that are not interesting.
15690 if (II->second.second == Expr)
15691 continue;
15692
15693 OS.indent(NumSpaces: Depth) << "[PSE]" << I << ":\n";
15694 OS.indent(NumSpaces: Depth + 2) << *Expr << "\n";
15695 OS.indent(NumSpaces: Depth + 2) << "--> " << *II->second.second << "\n";
15696 }
15697}
15698
15699ScalarEvolution::LoopGuards
15700ScalarEvolution::LoopGuards::collect(const Loop *L, ScalarEvolution &SE) {
15701 BasicBlock *Header = L->getHeader();
15702 BasicBlock *Pred = L->getLoopPredecessor();
15703 LoopGuards Guards(SE);
15704 if (!Pred)
15705 return Guards;
15706 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
15707 collectFromBlock(SE, Guards, Block: Header, Pred, VisitedBlocks);
15708 return Guards;
15709}
15710
15711void ScalarEvolution::LoopGuards::collectFromPHI(
15712 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15713 const PHINode &Phi, SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks,
15714 SmallDenseMap<const BasicBlock *, LoopGuards> &IncomingGuards,
15715 unsigned Depth) {
15716 if (!SE.isSCEVable(Ty: Phi.getType()))
15717 return;
15718
15719 using MinMaxPattern = std::pair<const SCEVConstant *, SCEVTypes>;
15720 auto GetMinMaxConst = [&](unsigned IncomingIdx) -> MinMaxPattern {
15721 const BasicBlock *InBlock = Phi.getIncomingBlock(i: IncomingIdx);
15722 if (!VisitedBlocks.insert(Ptr: InBlock).second)
15723 return {nullptr, scCouldNotCompute};
15724
15725 // Avoid analyzing unreachable blocks so that we don't get trapped
15726 // traversing cycles with ill-formed dominance or infinite cycles
15727 if (!SE.DT.isReachableFromEntry(A: InBlock))
15728 return {nullptr, scCouldNotCompute};
15729
15730 auto [G, Inserted] = IncomingGuards.try_emplace(Key: InBlock, Args: LoopGuards(SE));
15731 if (Inserted)
15732 collectFromBlock(SE, Guards&: G->second, Block: Phi.getParent(), Pred: InBlock, VisitedBlocks,
15733 Depth: Depth + 1);
15734 auto &RewriteMap = G->second.RewriteMap;
15735 if (RewriteMap.empty())
15736 return {nullptr, scCouldNotCompute};
15737 auto S = RewriteMap.find(Val: SE.getSCEV(V: Phi.getIncomingValue(i: IncomingIdx)));
15738 if (S == RewriteMap.end())
15739 return {nullptr, scCouldNotCompute};
15740 auto *SM = dyn_cast_if_present<SCEVMinMaxExpr>(Val: S->second);
15741 if (!SM)
15742 return {nullptr, scCouldNotCompute};
15743 if (const SCEVConstant *C0 = dyn_cast<SCEVConstant>(Val: SM->getOperand(i: 0)))
15744 return {C0, SM->getSCEVType()};
15745 return {nullptr, scCouldNotCompute};
15746 };
15747 auto MergeMinMaxConst = [](MinMaxPattern P1,
15748 MinMaxPattern P2) -> MinMaxPattern {
15749 auto [C1, T1] = P1;
15750 auto [C2, T2] = P2;
15751 if (!C1 || !C2 || T1 != T2)
15752 return {nullptr, scCouldNotCompute};
15753 switch (T1) {
15754 case scUMaxExpr:
15755 return {C1->getAPInt().ult(RHS: C2->getAPInt()) ? C1 : C2, T1};
15756 case scSMaxExpr:
15757 return {C1->getAPInt().slt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15758 case scUMinExpr:
15759 return {C1->getAPInt().ugt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15760 case scSMinExpr:
15761 return {C1->getAPInt().sgt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15762 default:
15763 llvm_unreachable("Trying to merge non-MinMaxExpr SCEVs.");
15764 }
15765 };
15766 auto P = GetMinMaxConst(0);
15767 for (unsigned int In = 1; In < Phi.getNumIncomingValues(); In++) {
15768 if (!P.first)
15769 break;
15770 P = MergeMinMaxConst(P, GetMinMaxConst(In));
15771 }
15772 if (P.first) {
15773 const SCEV *LHS = SE.getSCEV(V: const_cast<PHINode *>(&Phi));
15774 SmallVector<SCEVUse, 2> Ops({P.first, LHS});
15775 const SCEV *RHS = SE.getMinMaxExpr(Kind: P.second, Ops);
15776 Guards.RewriteMap.insert(KV: {LHS, RHS});
15777 }
15778}
15779
15780// Return a new SCEV that modifies \p Expr to the closest number divides by
15781// \p Divisor and less or equal than Expr. For now, only handle constant
15782// Expr.
15783static const SCEV *getPreviousSCEVDivisibleByDivisor(const SCEV *Expr,
15784 const APInt &DivisorVal,
15785 ScalarEvolution &SE) {
15786 const APInt *ExprVal;
15787 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15788 DivisorVal.isNonPositive())
15789 return Expr;
15790 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15791 // return the SCEV: Expr - Expr % Divisor
15792 return SE.getConstant(Val: *ExprVal - Rem);
15793}
15794
15795// Return a new SCEV that modifies \p Expr to the closest number divides by
15796// \p Divisor and greater or equal than Expr. For now, only handle constant
15797// Expr.
15798static const SCEV *getNextSCEVDivisibleByDivisor(const SCEV *Expr,
15799 const APInt &DivisorVal,
15800 ScalarEvolution &SE) {
15801 const APInt *ExprVal;
15802 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15803 DivisorVal.isNonPositive())
15804 return Expr;
15805 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15806 if (Rem.isZero())
15807 return Expr;
15808 // return the SCEV: Expr + Divisor - Expr % Divisor
15809 return SE.getConstant(Val: *ExprVal + DivisorVal - Rem);
15810}
15811
15812static bool collectDivisibilityInformation(
15813 ICmpInst::Predicate Predicate, const SCEV *LHS, const SCEV *RHS,
15814 DenseMap<const SCEV *, const SCEV *> &DivInfo,
15815 DenseMap<const SCEV *, APInt> &Multiples, ScalarEvolution &SE) {
15816 // If we have LHS == 0, check if LHS is computing a property of some unknown
15817 // SCEV %v which we can rewrite %v to express explicitly.
15818 if (Predicate != CmpInst::ICMP_EQ || !match(S: RHS, P: m_scev_Zero()))
15819 return false;
15820 // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to
15821 // explicitly express that.
15822 const SCEVUnknown *URemLHS = nullptr;
15823 const SCEV *URemRHS = nullptr;
15824 if (!match(S: LHS, P: m_scev_URem(LHS: m_SCEVUnknown(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE)))
15825 return false;
15826
15827 const SCEV *Multiple =
15828 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: URemLHS, RHS: URemRHS), RHS: URemRHS);
15829 DivInfo[URemLHS] = Multiple;
15830 if (auto *C = dyn_cast<SCEVConstant>(Val: URemRHS))
15831 Multiples[URemLHS] = C->getAPInt();
15832 return true;
15833}
15834
15835// Check if the condition is a divisibility guard (A % B == 0).
15836static bool isDivisibilityGuard(const SCEV *LHS, const SCEV *RHS,
15837 ScalarEvolution &SE) {
15838 const SCEV *X, *Y;
15839 return match(S: LHS, P: m_scev_URem(LHS: m_SCEV(V&: X), RHS: m_SCEV(V&: Y), SE)) && RHS->isZero();
15840}
15841
15842// Apply divisibility by \p Divisor on MinMaxExpr with constant values,
15843// recursively. This is done by aligning up/down the constant value to the
15844// Divisor.
15845static const SCEV *applyDivisibilityOnMinMaxExpr(const SCEV *MinMaxExpr,
15846 APInt Divisor,
15847 ScalarEvolution &SE) {
15848 // Return true if \p Expr is a MinMax SCEV expression with a non-negative
15849 // constant operand. If so, return in \p SCTy the SCEV type and in \p RHS
15850 // the non-constant operand and in \p LHS the constant operand.
15851 auto IsMinMaxSCEVWithNonNegativeConstant =
15852 [&](const SCEV *Expr, SCEVTypes &SCTy, const SCEV *&LHS,
15853 const SCEV *&RHS) {
15854 if (auto *MinMax = dyn_cast<SCEVMinMaxExpr>(Val: Expr)) {
15855 if (MinMax->getNumOperands() != 2)
15856 return false;
15857 if (auto *C = dyn_cast<SCEVConstant>(Val: MinMax->getOperand(i: 0))) {
15858 if (C->getAPInt().isNegative())
15859 return false;
15860 SCTy = MinMax->getSCEVType();
15861 LHS = MinMax->getOperand(i: 0);
15862 RHS = MinMax->getOperand(i: 1);
15863 return true;
15864 }
15865 }
15866 return false;
15867 };
15868
15869 const SCEV *MinMaxLHS = nullptr, *MinMaxRHS = nullptr;
15870 SCEVTypes SCTy;
15871 if (!IsMinMaxSCEVWithNonNegativeConstant(MinMaxExpr, SCTy, MinMaxLHS,
15872 MinMaxRHS))
15873 return MinMaxExpr;
15874 auto IsMin = isa<SCEVSMinExpr>(Val: MinMaxExpr) || isa<SCEVUMinExpr>(Val: MinMaxExpr);
15875 assert(SE.isKnownNonNegative(MinMaxLHS) && "Expected non-negative operand!");
15876 auto *DivisibleExpr =
15877 IsMin ? getPreviousSCEVDivisibleByDivisor(Expr: MinMaxLHS, DivisorVal: Divisor, SE)
15878 : getNextSCEVDivisibleByDivisor(Expr: MinMaxLHS, DivisorVal: Divisor, SE);
15879 SmallVector<SCEVUse> Ops = {
15880 applyDivisibilityOnMinMaxExpr(MinMaxExpr: MinMaxRHS, Divisor, SE), DivisibleExpr};
15881 return SE.getMinMaxExpr(Kind: SCTy, Ops);
15882}
15883
15884void ScalarEvolution::LoopGuards::collectFromBlock(
15885 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15886 const BasicBlock *Block, const BasicBlock *Pred,
15887 SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks, unsigned Depth) {
15888
15889 assert(SE.DT.isReachableFromEntry(Block) && SE.DT.isReachableFromEntry(Pred));
15890
15891 SmallVector<SCEVUse> ExprsToRewrite;
15892 auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS,
15893 const SCEV *RHS,
15894 DenseMap<const SCEV *, const SCEV *> &RewriteMap,
15895 const LoopGuards &DivGuards) {
15896 // WARNING: It is generally unsound to apply any wrap flags to the proposed
15897 // replacement SCEV which isn't directly implied by the structure of that
15898 // SCEV. In particular, using contextual facts to imply flags is *NOT*
15899 // legal. See the scoping rules for flags in the header to understand why.
15900
15901 // Check for a condition of the form (-C1 + X < C2). InstCombine will
15902 // create this form when combining two checks of the form (X u< C2 + C1) and
15903 // (X >=u C1).
15904 auto MatchRangeCheckIdiom = [&SE, Predicate, LHS, RHS, &RewriteMap,
15905 &ExprsToRewrite]() {
15906 const SCEVConstant *C1;
15907 const SCEVUnknown *LHSUnknown;
15908 auto *C2 = dyn_cast<SCEVConstant>(Val: RHS);
15909 if (!match(S: LHS,
15910 P: m_scev_Add(Op0: m_SCEVConstant(V&: C1), Op1: m_SCEVUnknown(V&: LHSUnknown))) ||
15911 !C2)
15912 return false;
15913
15914 auto ExactRegion =
15915 ConstantRange::makeExactICmpRegion(Pred: Predicate, Other: C2->getAPInt())
15916 .sub(Other: C1->getAPInt());
15917
15918 // Bail out, unless we have a non-wrapping, monotonic range.
15919 if (ExactRegion.isWrappedSet() || ExactRegion.isFullSet())
15920 return false;
15921 auto [I, Inserted] = RewriteMap.try_emplace(Key: LHSUnknown);
15922 const SCEV *RewrittenLHS = Inserted ? LHSUnknown : I->second;
15923 I->second = SE.getUMaxExpr(
15924 LHS: SE.getConstant(Val: ExactRegion.getUnsignedMin()),
15925 RHS: SE.getUMinExpr(LHS: RewrittenLHS,
15926 RHS: SE.getConstant(Val: ExactRegion.getUnsignedMax())));
15927 ExprsToRewrite.push_back(Elt: LHSUnknown);
15928 return true;
15929 };
15930 if (MatchRangeCheckIdiom())
15931 return;
15932
15933 // Do not apply information for constants or if RHS contains an AddRec.
15934 if (isa<SCEVConstant>(Val: LHS) || SE.containsAddRecurrence(S: RHS))
15935 return;
15936
15937 // If RHS is SCEVUnknown, make sure the information is applied to it.
15938 if (!isa<SCEVUnknown>(Val: LHS) && isa<SCEVUnknown>(Val: RHS)) {
15939 std::swap(a&: LHS, b&: RHS);
15940 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
15941 }
15942
15943 // Puts rewrite rule \p From -> \p To into the rewrite map. Also if \p From
15944 // and \p FromRewritten are the same (i.e. there has been no rewrite
15945 // registered for \p From), then puts this value in the list of rewritten
15946 // expressions.
15947 auto AddRewrite = [&](const SCEV *From, const SCEV *FromRewritten,
15948 const SCEV *To) {
15949 if (From == FromRewritten)
15950 ExprsToRewrite.push_back(Elt: From);
15951 RewriteMap[From] = To;
15952 };
15953
15954 // Checks whether \p S has already been rewritten. In that case returns the
15955 // existing rewrite because we want to chain further rewrites onto the
15956 // already rewritten value. Otherwise returns \p S.
15957 auto GetMaybeRewritten = [&](const SCEV *S) {
15958 return RewriteMap.lookup_or(Val: S, Default&: S);
15959 };
15960
15961 const SCEV *RewrittenLHS = GetMaybeRewritten(LHS);
15962 // Apply divisibility information when computing the constant multiple.
15963 const APInt &DividesBy =
15964 SE.getConstantMultiple(S: DivGuards.rewrite(Expr: RewrittenLHS));
15965
15966 // Collect rewrites for LHS and its transitive operands based on the
15967 // condition.
15968 // For min/max expressions, also apply the guard to its operands:
15969 // 'min(a, b) >= c' -> '(a >= c) and (b >= c)',
15970 // 'min(a, b) > c' -> '(a > c) and (b > c)',
15971 // 'max(a, b) <= c' -> '(a <= c) and (b <= c)',
15972 // 'max(a, b) < c' -> '(a < c) and (b < c)'.
15973
15974 // We cannot express strict predicates in SCEV, so instead we replace them
15975 // with non-strict ones against plus or minus one of RHS depending on the
15976 // predicate.
15977 const SCEV *One = SE.getOne(Ty: RHS->getType());
15978 switch (Predicate) {
15979 case CmpInst::ICMP_ULT:
15980 if (RHS->getType()->isPointerTy())
15981 return;
15982 RHS = SE.getUMaxExpr(LHS: RHS, RHS: One);
15983 [[fallthrough]];
15984 case CmpInst::ICMP_SLT: {
15985 RHS = SE.getMinusSCEV(LHS: RHS, RHS: One);
15986 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
15987 break;
15988 }
15989 case CmpInst::ICMP_UGT:
15990 case CmpInst::ICMP_SGT:
15991 RHS = SE.getAddExpr(LHS: RHS, RHS: One);
15992 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
15993 break;
15994 case CmpInst::ICMP_ULE:
15995 case CmpInst::ICMP_SLE:
15996 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
15997 break;
15998 case CmpInst::ICMP_UGE:
15999 case CmpInst::ICMP_SGE:
16000 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
16001 break;
16002 default:
16003 break;
16004 }
16005
16006 SmallVector<SCEVUse, 16> Worklist(1, LHS);
16007 SmallPtrSet<const SCEV *, 16> Visited;
16008
16009 auto EnqueueOperands = [&Worklist](const SCEVNAryExpr *S) {
16010 append_range(C&: Worklist, R: S->operands());
16011 };
16012
16013 while (!Worklist.empty()) {
16014 const SCEV *From = Worklist.pop_back_val();
16015 if (isa<SCEVConstant>(Val: From))
16016 continue;
16017 if (!Visited.insert(Ptr: From).second)
16018 continue;
16019 const SCEV *FromRewritten = GetMaybeRewritten(From);
16020 const SCEV *To = nullptr;
16021
16022 switch (Predicate) {
16023 case CmpInst::ICMP_ULT:
16024 case CmpInst::ICMP_ULE:
16025 To = SE.getUMinExpr(LHS: FromRewritten, RHS);
16026 if (auto *UMax = dyn_cast<SCEVUMaxExpr>(Val: FromRewritten))
16027 EnqueueOperands(UMax);
16028 break;
16029 case CmpInst::ICMP_SLT:
16030 case CmpInst::ICMP_SLE:
16031 To = SE.getSMinExpr(LHS: FromRewritten, RHS);
16032 if (auto *SMax = dyn_cast<SCEVSMaxExpr>(Val: FromRewritten))
16033 EnqueueOperands(SMax);
16034 break;
16035 case CmpInst::ICMP_UGT:
16036 case CmpInst::ICMP_UGE:
16037 To = SE.getUMaxExpr(LHS: FromRewritten, RHS);
16038 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: FromRewritten))
16039 EnqueueOperands(UMin);
16040 break;
16041 case CmpInst::ICMP_SGT:
16042 case CmpInst::ICMP_SGE:
16043 To = SE.getSMaxExpr(LHS: FromRewritten, RHS);
16044 if (auto *SMin = dyn_cast<SCEVSMinExpr>(Val: FromRewritten))
16045 EnqueueOperands(SMin);
16046 break;
16047 case CmpInst::ICMP_EQ:
16048 if (isa<SCEVConstant>(Val: RHS))
16049 To = RHS;
16050 break;
16051 case CmpInst::ICMP_NE:
16052 if (match(S: RHS, P: m_scev_Zero())) {
16053 const SCEV *OneAlignedUp =
16054 getNextSCEVDivisibleByDivisor(Expr: One, DivisorVal: DividesBy, SE);
16055 To = SE.getUMaxExpr(LHS: FromRewritten, RHS: OneAlignedUp);
16056 } else {
16057 // LHS != RHS can be rewritten as (LHS - RHS) = UMax(1, LHS - RHS),
16058 // but creating the subtraction eagerly is expensive. Track the
16059 // inequalities in a separate map, and materialize the rewrite lazily
16060 // when encountering a suitable subtraction while re-writing.
16061 if (LHS->getType()->isPointerTy()) {
16062 LHS = SE.getLosslessPtrToIntExpr(Op: LHS);
16063 RHS = SE.getLosslessPtrToIntExpr(Op: RHS);
16064 if (isa<SCEVCouldNotCompute>(Val: LHS) || isa<SCEVCouldNotCompute>(Val: RHS))
16065 break;
16066 }
16067 const SCEVConstant *C;
16068 const SCEV *A, *B;
16069 if (match(S: RHS, P: m_scev_Add(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: A))) &&
16070 match(S: LHS, P: m_scev_Add(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: B)))) {
16071 RHS = A;
16072 LHS = B;
16073 }
16074 if (LHS > RHS)
16075 std::swap(a&: LHS, b&: RHS);
16076 Guards.NotEqual.insert(V: {LHS, RHS});
16077 continue;
16078 }
16079 break;
16080 default:
16081 break;
16082 }
16083
16084 if (To)
16085 AddRewrite(From, FromRewritten, To);
16086 }
16087 };
16088
16089 SmallVector<PointerIntPair<Value *, 1, bool>> Terms;
16090 // First, collect information from assumptions dominating the loop.
16091 for (auto &AssumeVH : SE.AC.assumptions()) {
16092 if (!AssumeVH)
16093 continue;
16094 auto *AssumeI = cast<CallInst>(Val&: AssumeVH);
16095 if (!SE.DT.dominates(Def: AssumeI, BB: Block))
16096 continue;
16097 Terms.emplace_back(Args: AssumeI->getOperand(i_nocapture: 0), Args: true);
16098 }
16099
16100 // Second, collect information from llvm.experimental.guards dominating the loop.
16101 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
16102 M: SE.F.getParent(), id: Intrinsic::experimental_guard);
16103 if (GuardDecl)
16104 for (const auto *GU : GuardDecl->users())
16105 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
16106 if (Guard->getFunction() == Block->getParent() &&
16107 SE.DT.dominates(Def: Guard, BB: Block))
16108 Terms.emplace_back(Args: Guard->getArgOperand(i: 0), Args: true);
16109
16110 // Third, collect conditions from dominating branches. Starting at the loop
16111 // predecessor, climb up the predecessor chain, as long as there are
16112 // predecessors that can be found that have unique successors leading to the
16113 // original header.
16114 // TODO: share this logic with isLoopEntryGuardedByCond.
16115 unsigned NumCollectedConditions = 0;
16116 VisitedBlocks.insert(Ptr: Block);
16117 std::pair<const BasicBlock *, const BasicBlock *> Pair(Pred, Block);
16118 for (; Pair.first;
16119 Pair = SE.getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
16120 VisitedBlocks.insert(Ptr: Pair.second);
16121 const CondBrInst *LoopEntryPredicate =
16122 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
16123 if (!LoopEntryPredicate)
16124 continue;
16125
16126 Terms.emplace_back(Args: LoopEntryPredicate->getCondition(),
16127 Args: LoopEntryPredicate->getSuccessor(i: 0) == Pair.second);
16128 NumCollectedConditions++;
16129
16130 // If we are recursively collecting guards stop after 2
16131 // conditions to limit compile-time impact for now.
16132 if (Depth > 0 && NumCollectedConditions == 2)
16133 break;
16134 }
16135 // Finally, if we stopped climbing the predecessor chain because
16136 // there wasn't a unique one to continue, try to collect conditions
16137 // for PHINodes by recursively following all of their incoming
16138 // blocks and try to merge the found conditions to build a new one
16139 // for the Phi.
16140 if (Pair.second->hasNPredecessorsOrMore(N: 2) &&
16141 Depth < MaxLoopGuardCollectionDepth) {
16142 SmallDenseMap<const BasicBlock *, LoopGuards> IncomingGuards;
16143 for (auto &Phi : Pair.second->phis())
16144 collectFromPHI(SE, Guards, Phi, VisitedBlocks, IncomingGuards, Depth);
16145 }
16146
16147 // Now apply the information from the collected conditions to
16148 // Guards.RewriteMap. Conditions are processed in reverse order, so the
16149 // earliest conditions is processed first, except guards with divisibility
16150 // information, which are moved to the back. This ensures the SCEVs with the
16151 // shortest dependency chains are constructed first.
16152 SmallVector<std::tuple<CmpInst::Predicate, const SCEV *, const SCEV *>>
16153 GuardsToProcess;
16154 for (auto [Term, EnterIfTrue] : reverse(C&: Terms)) {
16155 SmallVector<Value *, 8> Worklist;
16156 SmallPtrSet<Value *, 8> Visited;
16157 Worklist.push_back(Elt: Term);
16158 while (!Worklist.empty()) {
16159 Value *Cond = Worklist.pop_back_val();
16160 if (!Visited.insert(Ptr: Cond).second)
16161 continue;
16162
16163 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
16164 auto Predicate =
16165 EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate();
16166 const auto *LHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 0));
16167 const auto *RHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 1));
16168 // If LHS is a constant, apply information to the other expression.
16169 // TODO: If LHS is not a constant, check if using CompareSCEVComplexity
16170 // can improve results.
16171 if (isa<SCEVConstant>(Val: LHS)) {
16172 std::swap(a&: LHS, b&: RHS);
16173 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
16174 }
16175 GuardsToProcess.emplace_back(Args&: Predicate, Args&: LHS, Args&: RHS);
16176 continue;
16177 }
16178
16179 Value *L, *R;
16180 if (EnterIfTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: L), R: m_Value(V&: R)))
16181 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: L), R: m_Value(V&: R)))) {
16182 Worklist.push_back(Elt: L);
16183 Worklist.push_back(Elt: R);
16184 }
16185 }
16186 }
16187
16188 // Process divisibility guards in reverse order to populate DivGuards early.
16189 DenseMap<const SCEV *, APInt> Multiples;
16190 LoopGuards DivGuards(SE);
16191 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess) {
16192 if (!isDivisibilityGuard(LHS, RHS, SE))
16193 continue;
16194 collectDivisibilityInformation(Predicate, LHS, RHS, DivInfo&: DivGuards.RewriteMap,
16195 Multiples, SE);
16196 }
16197
16198 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess)
16199 CollectCondition(Predicate, LHS, RHS, Guards.RewriteMap, DivGuards);
16200
16201 // Apply divisibility information last. This ensures it is applied to the
16202 // outermost expression after other rewrites for the given value.
16203 for (const auto &[K, Divisor] : Multiples) {
16204 const SCEV *DivisorSCEV = SE.getConstant(Val: Divisor);
16205 Guards.RewriteMap[K] =
16206 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: applyDivisibilityOnMinMaxExpr(
16207 MinMaxExpr: Guards.rewrite(Expr: K), Divisor, SE),
16208 RHS: DivisorSCEV),
16209 RHS: DivisorSCEV);
16210 ExprsToRewrite.push_back(Elt: K);
16211 }
16212
16213 // Let the rewriter preserve NUW/NSW flags if the unsigned/signed ranges of
16214 // the replacement expressions are contained in the ranges of the replaced
16215 // expressions.
16216 Guards.PreserveNUW = true;
16217 Guards.PreserveNSW = true;
16218 for (const SCEV *Expr : ExprsToRewrite) {
16219 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16220 Guards.PreserveNUW &=
16221 SE.getUnsignedRange(S: Expr).contains(CR: SE.getUnsignedRange(S: RewriteTo));
16222 Guards.PreserveNSW &=
16223 SE.getSignedRange(S: Expr).contains(CR: SE.getSignedRange(S: RewriteTo));
16224 }
16225
16226 // Now that all rewrite information is collect, rewrite the collected
16227 // expressions with the information in the map. This applies information to
16228 // sub-expressions.
16229 if (ExprsToRewrite.size() > 1) {
16230 for (const SCEV *Expr : ExprsToRewrite) {
16231 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16232 Guards.RewriteMap.erase(Val: Expr);
16233 Guards.RewriteMap.insert(KV: {Expr, Guards.rewrite(Expr: RewriteTo)});
16234 }
16235 }
16236}
16237
16238const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const {
16239 /// A rewriter to replace SCEV expressions in Map with the corresponding entry
16240 /// in the map. It skips AddRecExpr because we cannot guarantee that the
16241 /// replacement is loop invariant in the loop of the AddRec.
16242 class SCEVLoopGuardRewriter
16243 : public SCEVRewriteVisitor<SCEVLoopGuardRewriter> {
16244 const DenseMap<const SCEV *, const SCEV *> &Map;
16245 const SmallDenseSet<std::pair<const SCEV *, const SCEV *>> &NotEqual;
16246
16247 SCEV::NoWrapFlags FlagMask = SCEV::FlagAnyWrap;
16248
16249 public:
16250 SCEVLoopGuardRewriter(ScalarEvolution &SE,
16251 const ScalarEvolution::LoopGuards &Guards)
16252 : SCEVRewriteVisitor(SE), Map(Guards.RewriteMap),
16253 NotEqual(Guards.NotEqual) {
16254 if (Guards.PreserveNUW)
16255 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNUW);
16256 if (Guards.PreserveNSW)
16257 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNSW);
16258 }
16259
16260 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
16261
16262 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
16263 return Map.lookup_or(Val: Expr, Default&: Expr);
16264 }
16265
16266 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
16267 if (const SCEV *S = Map.lookup(Val: Expr))
16268 return S;
16269
16270 // If we didn't find the extact ZExt expr in the map, check if there's
16271 // an entry for a smaller ZExt we can use instead.
16272 Type *Ty = Expr->getType();
16273 const SCEV *Op = Expr->getOperand(i: 0);
16274 unsigned Bitwidth = Ty->getScalarSizeInBits() / 2;
16275 while (Bitwidth % 8 == 0 && Bitwidth >= 8 &&
16276 Bitwidth > Op->getType()->getScalarSizeInBits()) {
16277 Type *NarrowTy = IntegerType::get(C&: SE.getContext(), NumBits: Bitwidth);
16278 auto *NarrowExt = SE.getZeroExtendExpr(Op, Ty: NarrowTy);
16279 if (const SCEV *S = Map.lookup(Val: NarrowExt))
16280 return SE.getZeroExtendExpr(Op: S, Ty);
16281 Bitwidth = Bitwidth / 2;
16282 }
16283
16284 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitZeroExtendExpr(
16285 Expr);
16286 }
16287
16288 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
16289 if (const SCEV *S = Map.lookup(Val: Expr))
16290 return S;
16291 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSignExtendExpr(
16292 Expr);
16293 }
16294
16295 const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) {
16296 if (const SCEV *S = Map.lookup(Val: Expr))
16297 return S;
16298 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitUMinExpr(Expr);
16299 }
16300
16301 const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) {
16302 if (const SCEV *S = Map.lookup(Val: Expr))
16303 return S;
16304 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSMinExpr(Expr);
16305 }
16306
16307 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
16308 // Helper to check if S is a subtraction (A - B) where A != B, and if so,
16309 // return UMax(S, 1).
16310 auto RewriteSubtraction = [&](const SCEV *S) -> const SCEV * {
16311 SCEVUse LHS, RHS;
16312 if (MatchBinarySub(S, LHS, RHS)) {
16313 if (LHS > RHS)
16314 std::swap(a&: LHS, b&: RHS);
16315 if (NotEqual.contains(V: {LHS, RHS})) {
16316 const SCEV *OneAlignedUp = getNextSCEVDivisibleByDivisor(
16317 Expr: SE.getOne(Ty: S->getType()), DivisorVal: SE.getConstantMultiple(S), SE);
16318 return SE.getUMaxExpr(LHS: OneAlignedUp, RHS: S);
16319 }
16320 }
16321 return nullptr;
16322 };
16323
16324 // Check if Expr itself is a subtraction pattern with guard info.
16325 if (const SCEV *Rewritten = RewriteSubtraction(Expr))
16326 return Rewritten;
16327
16328 // Trip count expressions sometimes consist of adding 3 operands, i.e.
16329 // (Const + A + B). There may be guard info for A + B, and if so, apply
16330 // it.
16331 // TODO: Could more generally apply guards to Add sub-expressions.
16332 if (isa<SCEVConstant>(Val: Expr->getOperand(i: 0)) &&
16333 Expr->getNumOperands() == 3) {
16334 const SCEV *Add =
16335 SE.getAddExpr(LHS: Expr->getOperand(i: 1), RHS: Expr->getOperand(i: 2));
16336 if (const SCEV *Rewritten = RewriteSubtraction(Add))
16337 return SE.getAddExpr(
16338 LHS: Expr->getOperand(i: 0), RHS: Rewritten,
16339 Flags: ScalarEvolution::maskFlags(Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16340 if (const SCEV *S = Map.lookup(Val: Add))
16341 return SE.getAddExpr(LHS: Expr->getOperand(i: 0), RHS: S);
16342 }
16343 SmallVector<SCEVUse, 2> Operands;
16344 bool Changed = false;
16345 for (SCEVUse Op : Expr->operands()) {
16346 Operands.push_back(
16347 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
16348 Changed |= Op != Operands.back();
16349 }
16350 // We are only replacing operands with equivalent values, so transfer the
16351 // flags from the original expression.
16352 return !Changed ? Expr
16353 : SE.getAddExpr(Ops&: Operands,
16354 OrigFlags: ScalarEvolution::maskFlags(
16355 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16356 }
16357
16358 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
16359 SmallVector<SCEVUse, 2> Operands;
16360 bool Changed = false;
16361 for (SCEVUse Op : Expr->operands()) {
16362 Operands.push_back(
16363 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
16364 Changed |= Op != Operands.back();
16365 }
16366 // We are only replacing operands with equivalent values, so transfer the
16367 // flags from the original expression.
16368 return !Changed ? Expr
16369 : SE.getMulExpr(Ops&: Operands,
16370 OrigFlags: ScalarEvolution::maskFlags(
16371 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16372 }
16373 };
16374
16375 if (RewriteMap.empty() && NotEqual.empty())
16376 return Expr;
16377
16378 SCEVLoopGuardRewriter Rewriter(SE, *this);
16379 return Rewriter.visit(S: Expr);
16380}
16381
16382const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
16383 return applyLoopGuards(Expr, Guards: LoopGuards::collect(L, SE&: *this));
16384}
16385
16386const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr,
16387 const LoopGuards &Guards) {
16388 return Guards.rewrite(Expr);
16389}
16390