1//===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the scalar evolution analysis
10// engine, which is used primarily to analyze expressions involving induction
11// variables in loops.
12//
13// There are several aspects to this library. First is the representation of
14// scalar expressions, which are represented as subclasses of the SCEV class.
15// These classes are used to represent certain types of subexpressions that we
16// can handle. We only create one SCEV of a particular shape, so
17// pointer-comparisons for equality are legal.
18//
19// One important aspect of the SCEV objects is that they are never cyclic, even
20// if there is a cycle in the dataflow for an expression (ie, a PHI node). If
21// the PHI node is one of the idioms that we can represent (e.g., a polynomial
22// recurrence) then we represent it directly as a recurrence node, otherwise we
23// represent it as a SCEVUnknown node.
24//
25// In addition to being able to represent expressions of various types, we also
26// have folders that are used to build the *canonical* representation for a
27// particular expression. These folders are capable of using a variety of
28// rewrite rules to simplify the expressions.
29//
30// Once the folders are defined, we can implement the more interesting
31// higher-level code, such as the code that recognizes PHI nodes of various
32// types, computes the execution count of a loop, etc.
33//
34// TODO: We should use these routines and value representations to implement
35// dependence analysis!
36//
37//===----------------------------------------------------------------------===//
38//
39// There are several good references for the techniques used in this analysis.
40//
41// Chains of recurrences -- a method to expedite the evaluation
42// of closed-form functions
43// Olaf Bachmann, Paul S. Wang, Eugene V. Zima
44//
45// On computational properties of chains of recurrences
46// Eugene V. Zima
47//
48// Symbolic Evaluation of Chains of Recurrences for Loop Optimization
49// Robert A. van Engelen
50//
51// Efficient Symbolic Analysis for Optimizing Compilers
52// Robert A. van Engelen
53//
54// Using the chains of recurrences algebra for data dependence testing and
55// induction variable substitution
56// MS Thesis, Johnie Birch
57//
58//===----------------------------------------------------------------------===//
59
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "llvm/ADT/APInt.h"
62#include "llvm/ADT/ArrayRef.h"
63#include "llvm/ADT/DenseMap.h"
64#include "llvm/ADT/DepthFirstIterator.h"
65#include "llvm/ADT/FoldingSet.h"
66#include "llvm/ADT/STLExtras.h"
67#include "llvm/ADT/ScopeExit.h"
68#include "llvm/ADT/Sequence.h"
69#include "llvm/ADT/SmallPtrSet.h"
70#include "llvm/ADT/SmallSet.h"
71#include "llvm/ADT/SmallVector.h"
72#include "llvm/ADT/Statistic.h"
73#include "llvm/ADT/StringExtras.h"
74#include "llvm/ADT/StringRef.h"
75#include "llvm/Analysis/AssumptionCache.h"
76#include "llvm/Analysis/ConstantFolding.h"
77#include "llvm/Analysis/InstructionSimplify.h"
78#include "llvm/Analysis/LoopInfo.h"
79#include "llvm/Analysis/MemoryBuiltins.h"
80#include "llvm/Analysis/ScalarEvolutionExpressions.h"
81#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
82#include "llvm/Analysis/TargetLibraryInfo.h"
83#include "llvm/Analysis/ValueTracking.h"
84#include "llvm/Config/llvm-config.h"
85#include "llvm/IR/Argument.h"
86#include "llvm/IR/BasicBlock.h"
87#include "llvm/IR/CFG.h"
88#include "llvm/IR/Constant.h"
89#include "llvm/IR/ConstantRange.h"
90#include "llvm/IR/Constants.h"
91#include "llvm/IR/DataLayout.h"
92#include "llvm/IR/DerivedTypes.h"
93#include "llvm/IR/Dominators.h"
94#include "llvm/IR/Function.h"
95#include "llvm/IR/GlobalAlias.h"
96#include "llvm/IR/GlobalValue.h"
97#include "llvm/IR/InstIterator.h"
98#include "llvm/IR/InstrTypes.h"
99#include "llvm/IR/Instruction.h"
100#include "llvm/IR/Instructions.h"
101#include "llvm/IR/IntrinsicInst.h"
102#include "llvm/IR/Intrinsics.h"
103#include "llvm/IR/LLVMContext.h"
104#include "llvm/IR/Operator.h"
105#include "llvm/IR/PatternMatch.h"
106#include "llvm/IR/Type.h"
107#include "llvm/IR/Use.h"
108#include "llvm/IR/User.h"
109#include "llvm/IR/Value.h"
110#include "llvm/IR/Verifier.h"
111#include "llvm/InitializePasses.h"
112#include "llvm/Pass.h"
113#include "llvm/Support/Casting.h"
114#include "llvm/Support/CommandLine.h"
115#include "llvm/Support/Compiler.h"
116#include "llvm/Support/Debug.h"
117#include "llvm/Support/ErrorHandling.h"
118#include "llvm/Support/InterleavedRange.h"
119#include "llvm/Support/KnownBits.h"
120#include "llvm/Support/SaveAndRestore.h"
121#include "llvm/Support/raw_ostream.h"
122#include <algorithm>
123#include <cassert>
124#include <climits>
125#include <cstdint>
126#include <cstdlib>
127#include <map>
128#include <memory>
129#include <numeric>
130#include <optional>
131#include <tuple>
132#include <utility>
133#include <vector>
134
135using namespace llvm;
136using namespace PatternMatch;
137using namespace SCEVPatternMatch;
138
139#define DEBUG_TYPE "scalar-evolution"
140
141STATISTIC(NumExitCountsComputed,
142 "Number of loop exits with predictable exit counts");
143STATISTIC(NumExitCountsNotComputed,
144 "Number of loop exits without predictable exit counts");
145STATISTIC(NumBruteForceTripCountsComputed,
146 "Number of loops with trip counts computed by force");
147
148#ifdef EXPENSIVE_CHECKS
149bool llvm::VerifySCEV = true;
150#else
151bool llvm::VerifySCEV = false;
152#endif
153
154static cl::opt<unsigned>
155 MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
156 cl::desc("Maximum number of iterations SCEV will "
157 "symbolically execute a constant "
158 "derived loop"),
159 cl::init(Val: 100));
160
161static cl::opt<bool, true> VerifySCEVOpt(
162 "verify-scev", cl::Hidden, cl::location(L&: VerifySCEV),
163 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
164static cl::opt<bool> VerifySCEVStrict(
165 "verify-scev-strict", cl::Hidden,
166 cl::desc("Enable stricter verification with -verify-scev is passed"));
167
168static cl::opt<bool> VerifyIR(
169 "scev-verify-ir", cl::Hidden,
170 cl::desc("Verify IR correctness when making sensitive SCEV queries (slow)"),
171 cl::init(Val: false));
172
173static cl::opt<unsigned> MulOpsInlineThreshold(
174 "scev-mulops-inline-threshold", cl::Hidden,
175 cl::desc("Threshold for inlining multiplication operands into a SCEV"),
176 cl::init(Val: 32));
177
178static cl::opt<unsigned> AddOpsInlineThreshold(
179 "scev-addops-inline-threshold", cl::Hidden,
180 cl::desc("Threshold for inlining addition operands into a SCEV"),
181 cl::init(Val: 500));
182
183static cl::opt<unsigned> MaxSCEVCompareDepth(
184 "scalar-evolution-max-scev-compare-depth", cl::Hidden,
185 cl::desc("Maximum depth of recursive SCEV complexity comparisons"),
186 cl::init(Val: 32));
187
188static cl::opt<unsigned> MaxSCEVOperationsImplicationDepth(
189 "scalar-evolution-max-scev-operations-implication-depth", cl::Hidden,
190 cl::desc("Maximum depth of recursive SCEV operations implication analysis"),
191 cl::init(Val: 2));
192
193static cl::opt<unsigned> MaxValueCompareDepth(
194 "scalar-evolution-max-value-compare-depth", cl::Hidden,
195 cl::desc("Maximum depth of recursive value complexity comparisons"),
196 cl::init(Val: 2));
197
198static cl::opt<unsigned>
199 MaxArithDepth("scalar-evolution-max-arith-depth", cl::Hidden,
200 cl::desc("Maximum depth of recursive arithmetics"),
201 cl::init(Val: 32));
202
203static cl::opt<unsigned> MaxConstantEvolvingDepth(
204 "scalar-evolution-max-constant-evolving-depth", cl::Hidden,
205 cl::desc("Maximum depth of recursive constant evolving"), cl::init(Val: 32));
206
207static cl::opt<unsigned>
208 MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden,
209 cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"),
210 cl::init(Val: 8));
211
212static cl::opt<unsigned>
213 MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden,
214 cl::desc("Max coefficients in AddRec during evolving"),
215 cl::init(Val: 8));
216
217static cl::opt<unsigned>
218 HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden,
219 cl::desc("Size of the expression which is considered huge"),
220 cl::init(Val: 4096));
221
222static cl::opt<unsigned> RangeIterThreshold(
223 "scev-range-iter-threshold", cl::Hidden,
224 cl::desc("Threshold for switching to iteratively computing SCEV ranges"),
225 cl::init(Val: 32));
226
227static cl::opt<unsigned> MaxLoopGuardCollectionDepth(
228 "scalar-evolution-max-loop-guard-collection-depth", cl::Hidden,
229 cl::desc("Maximum depth for recursive loop guard collection"), cl::init(Val: 1));
230
231static cl::opt<bool>
232ClassifyExpressions("scalar-evolution-classify-expressions",
233 cl::Hidden, cl::init(Val: true),
234 cl::desc("When printing analysis, include information on every instruction"));
235
236static cl::opt<bool> UseExpensiveRangeSharpening(
237 "scalar-evolution-use-expensive-range-sharpening", cl::Hidden,
238 cl::init(Val: false),
239 cl::desc("Use more powerful methods of sharpening expression ranges. May "
240 "be costly in terms of compile time"));
241
242static cl::opt<unsigned> MaxPhiSCCAnalysisSize(
243 "scalar-evolution-max-scc-analysis-depth", cl::Hidden,
244 cl::desc("Maximum amount of nodes to process while searching SCEVUnknown "
245 "Phi strongly connected components"),
246 cl::init(Val: 8));
247
248static cl::opt<bool>
249 EnableFiniteLoopControl("scalar-evolution-finite-loop", cl::Hidden,
250 cl::desc("Handle <= and >= in finite loops"),
251 cl::init(Val: true));
252
253static cl::opt<bool> UseContextForNoWrapFlagInference(
254 "scalar-evolution-use-context-for-no-wrap-flag-strenghening", cl::Hidden,
255 cl::desc("Infer nuw/nsw flags using context where suitable"),
256 cl::init(Val: true));
257
258//===----------------------------------------------------------------------===//
259// SCEV class definitions
260//===----------------------------------------------------------------------===//
261
262//===----------------------------------------------------------------------===//
263// Implementation of the SCEV class.
264//
265
266#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
267LLVM_DUMP_METHOD void SCEV::dump() const {
268 print(dbgs());
269 dbgs() << '\n';
270}
271#endif
272
273void SCEV::print(raw_ostream &OS) const {
274 switch (getSCEVType()) {
275 case scConstant:
276 cast<SCEVConstant>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
277 return;
278 case scVScale:
279 OS << "vscale";
280 return;
281 case scPtrToInt: {
282 const SCEVPtrToIntExpr *PtrToInt = cast<SCEVPtrToIntExpr>(Val: this);
283 const SCEV *Op = PtrToInt->getOperand();
284 OS << "(ptrtoint " << *Op->getType() << " " << *Op << " to "
285 << *PtrToInt->getType() << ")";
286 return;
287 }
288 case scTruncate: {
289 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: this);
290 const SCEV *Op = Trunc->getOperand();
291 OS << "(trunc " << *Op->getType() << " " << *Op << " to "
292 << *Trunc->getType() << ")";
293 return;
294 }
295 case scZeroExtend: {
296 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: this);
297 const SCEV *Op = ZExt->getOperand();
298 OS << "(zext " << *Op->getType() << " " << *Op << " to "
299 << *ZExt->getType() << ")";
300 return;
301 }
302 case scSignExtend: {
303 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: this);
304 const SCEV *Op = SExt->getOperand();
305 OS << "(sext " << *Op->getType() << " " << *Op << " to "
306 << *SExt->getType() << ")";
307 return;
308 }
309 case scAddRecExpr: {
310 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: this);
311 OS << "{" << *AR->getOperand(i: 0);
312 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
313 OS << ",+," << *AR->getOperand(i);
314 OS << "}<";
315 if (AR->hasNoUnsignedWrap())
316 OS << "nuw><";
317 if (AR->hasNoSignedWrap())
318 OS << "nsw><";
319 if (AR->hasNoSelfWrap() &&
320 !AR->getNoWrapFlags(Mask: (NoWrapFlags)(FlagNUW | FlagNSW)))
321 OS << "nw><";
322 AR->getLoop()->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
323 OS << ">";
324 return;
325 }
326 case scAddExpr:
327 case scMulExpr:
328 case scUMaxExpr:
329 case scSMaxExpr:
330 case scUMinExpr:
331 case scSMinExpr:
332 case scSequentialUMinExpr: {
333 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(Val: this);
334 const char *OpStr = nullptr;
335 switch (NAry->getSCEVType()) {
336 case scAddExpr: OpStr = " + "; break;
337 case scMulExpr: OpStr = " * "; break;
338 case scUMaxExpr: OpStr = " umax "; break;
339 case scSMaxExpr: OpStr = " smax "; break;
340 case scUMinExpr:
341 OpStr = " umin ";
342 break;
343 case scSMinExpr:
344 OpStr = " smin ";
345 break;
346 case scSequentialUMinExpr:
347 OpStr = " umin_seq ";
348 break;
349 default:
350 llvm_unreachable("There are no other nary expression types.");
351 }
352 OS << "("
353 << llvm::interleaved(R: llvm::make_pointee_range(Range: NAry->operands()), Separator: OpStr)
354 << ")";
355 switch (NAry->getSCEVType()) {
356 case scAddExpr:
357 case scMulExpr:
358 if (NAry->hasNoUnsignedWrap())
359 OS << "<nuw>";
360 if (NAry->hasNoSignedWrap())
361 OS << "<nsw>";
362 break;
363 default:
364 // Nothing to print for other nary expressions.
365 break;
366 }
367 return;
368 }
369 case scUDivExpr: {
370 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: this);
371 OS << "(" << *UDiv->getLHS() << " /u " << *UDiv->getRHS() << ")";
372 return;
373 }
374 case scUnknown:
375 cast<SCEVUnknown>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
376 return;
377 case scCouldNotCompute:
378 OS << "***COULDNOTCOMPUTE***";
379 return;
380 }
381 llvm_unreachable("Unknown SCEV kind!");
382}
383
384Type *SCEV::getType() const {
385 switch (getSCEVType()) {
386 case scConstant:
387 return cast<SCEVConstant>(Val: this)->getType();
388 case scVScale:
389 return cast<SCEVVScale>(Val: this)->getType();
390 case scPtrToInt:
391 case scTruncate:
392 case scZeroExtend:
393 case scSignExtend:
394 return cast<SCEVCastExpr>(Val: this)->getType();
395 case scAddRecExpr:
396 return cast<SCEVAddRecExpr>(Val: this)->getType();
397 case scMulExpr:
398 return cast<SCEVMulExpr>(Val: this)->getType();
399 case scUMaxExpr:
400 case scSMaxExpr:
401 case scUMinExpr:
402 case scSMinExpr:
403 return cast<SCEVMinMaxExpr>(Val: this)->getType();
404 case scSequentialUMinExpr:
405 return cast<SCEVSequentialMinMaxExpr>(Val: this)->getType();
406 case scAddExpr:
407 return cast<SCEVAddExpr>(Val: this)->getType();
408 case scUDivExpr:
409 return cast<SCEVUDivExpr>(Val: this)->getType();
410 case scUnknown:
411 return cast<SCEVUnknown>(Val: this)->getType();
412 case scCouldNotCompute:
413 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
414 }
415 llvm_unreachable("Unknown SCEV kind!");
416}
417
418ArrayRef<const SCEV *> SCEV::operands() const {
419 switch (getSCEVType()) {
420 case scConstant:
421 case scVScale:
422 case scUnknown:
423 return {};
424 case scPtrToInt:
425 case scTruncate:
426 case scZeroExtend:
427 case scSignExtend:
428 return cast<SCEVCastExpr>(Val: this)->operands();
429 case scAddRecExpr:
430 case scAddExpr:
431 case scMulExpr:
432 case scUMaxExpr:
433 case scSMaxExpr:
434 case scUMinExpr:
435 case scSMinExpr:
436 case scSequentialUMinExpr:
437 return cast<SCEVNAryExpr>(Val: this)->operands();
438 case scUDivExpr:
439 return cast<SCEVUDivExpr>(Val: this)->operands();
440 case scCouldNotCompute:
441 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
442 }
443 llvm_unreachable("Unknown SCEV kind!");
444}
445
446bool SCEV::isZero() const { return match(S: this, P: m_scev_Zero()); }
447
448bool SCEV::isOne() const { return match(S: this, P: m_scev_One()); }
449
450bool SCEV::isAllOnesValue() const { return match(S: this, P: m_scev_AllOnes()); }
451
452bool SCEV::isNonConstantNegative() const {
453 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: this);
454 if (!Mul) return false;
455
456 // If there is a constant factor, it will be first.
457 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Mul->getOperand(i: 0));
458 if (!SC) return false;
459
460 // Return true if the value is negative, this matches things like (-42 * V).
461 return SC->getAPInt().isNegative();
462}
463
464SCEVCouldNotCompute::SCEVCouldNotCompute() :
465 SCEV(FoldingSetNodeIDRef(), scCouldNotCompute, 0) {}
466
467bool SCEVCouldNotCompute::classof(const SCEV *S) {
468 return S->getSCEVType() == scCouldNotCompute;
469}
470
471const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
472 FoldingSetNodeID ID;
473 ID.AddInteger(I: scConstant);
474 ID.AddPointer(Ptr: V);
475 void *IP = nullptr;
476 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
477 SCEV *S = new (SCEVAllocator) SCEVConstant(ID.Intern(Allocator&: SCEVAllocator), V);
478 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
479 return S;
480}
481
482const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
483 return getConstant(V: ConstantInt::get(Context&: getContext(), V: Val));
484}
485
486const SCEV *
487ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
488 IntegerType *ITy = cast<IntegerType>(Val: getEffectiveSCEVType(Ty));
489 return getConstant(V: ConstantInt::get(Ty: ITy, V, IsSigned: isSigned));
490}
491
492const SCEV *ScalarEvolution::getVScale(Type *Ty) {
493 FoldingSetNodeID ID;
494 ID.AddInteger(I: scVScale);
495 ID.AddPointer(Ptr: Ty);
496 void *IP = nullptr;
497 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
498 return S;
499 SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(Allocator&: SCEVAllocator), Ty);
500 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
501 return S;
502}
503
504const SCEV *ScalarEvolution::getElementCount(Type *Ty, ElementCount EC) {
505 const SCEV *Res = getConstant(Ty, V: EC.getKnownMinValue());
506 if (EC.isScalable())
507 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty));
508 return Res;
509}
510
511SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
512 const SCEV *op, Type *ty)
513 : SCEV(ID, SCEVTy, computeExpressionSize(Args: op)), Op(op), Ty(ty) {}
514
515SCEVPtrToIntExpr::SCEVPtrToIntExpr(const FoldingSetNodeIDRef ID, const SCEV *Op,
516 Type *ITy)
517 : SCEVCastExpr(ID, scPtrToInt, Op, ITy) {
518 assert(getOperand()->getType()->isPointerTy() && Ty->isIntegerTy() &&
519 "Must be a non-bit-width-changing pointer-to-integer cast!");
520}
521
522SCEVIntegralCastExpr::SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID,
523 SCEVTypes SCEVTy, const SCEV *op,
524 Type *ty)
525 : SCEVCastExpr(ID, SCEVTy, op, ty) {}
526
527SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, const SCEV *op,
528 Type *ty)
529 : SCEVIntegralCastExpr(ID, scTruncate, op, ty) {
530 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
531 "Cannot truncate non-integer value!");
532}
533
534SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID,
535 const SCEV *op, Type *ty)
536 : SCEVIntegralCastExpr(ID, scZeroExtend, op, ty) {
537 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
538 "Cannot zero extend non-integer value!");
539}
540
541SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID,
542 const SCEV *op, Type *ty)
543 : SCEVIntegralCastExpr(ID, scSignExtend, op, ty) {
544 assert(getOperand()->getType()->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
545 "Cannot sign extend non-integer value!");
546}
547
548void SCEVUnknown::deleted() {
549 // Clear this SCEVUnknown from various maps.
550 SE->forgetMemoizedResults(SCEVs: this);
551
552 // Remove this SCEVUnknown from the uniquing map.
553 SE->UniqueSCEVs.RemoveNode(N: this);
554
555 // Release the value.
556 setValPtr(nullptr);
557}
558
559void SCEVUnknown::allUsesReplacedWith(Value *New) {
560 // Clear this SCEVUnknown from various maps.
561 SE->forgetMemoizedResults(SCEVs: this);
562
563 // Remove this SCEVUnknown from the uniquing map.
564 SE->UniqueSCEVs.RemoveNode(N: this);
565
566 // Replace the value pointer in case someone is still using this SCEVUnknown.
567 setValPtr(New);
568}
569
570//===----------------------------------------------------------------------===//
571// SCEV Utilities
572//===----------------------------------------------------------------------===//
573
574/// Compare the two values \p LV and \p RV in terms of their "complexity" where
575/// "complexity" is a partial (and somewhat ad-hoc) relation used to order
576/// operands in SCEV expressions.
577static int CompareValueComplexity(const LoopInfo *const LI, Value *LV,
578 Value *RV, unsigned Depth) {
579 if (Depth > MaxValueCompareDepth)
580 return 0;
581
582 // Order pointer values after integer values. This helps SCEVExpander form
583 // GEPs.
584 bool LIsPointer = LV->getType()->isPointerTy(),
585 RIsPointer = RV->getType()->isPointerTy();
586 if (LIsPointer != RIsPointer)
587 return (int)LIsPointer - (int)RIsPointer;
588
589 // Compare getValueID values.
590 unsigned LID = LV->getValueID(), RID = RV->getValueID();
591 if (LID != RID)
592 return (int)LID - (int)RID;
593
594 // Sort arguments by their position.
595 if (const auto *LA = dyn_cast<Argument>(Val: LV)) {
596 const auto *RA = cast<Argument>(Val: RV);
597 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
598 return (int)LArgNo - (int)RArgNo;
599 }
600
601 if (const auto *LGV = dyn_cast<GlobalValue>(Val: LV)) {
602 const auto *RGV = cast<GlobalValue>(Val: RV);
603
604 const auto IsGVNameSemantic = [&](const GlobalValue *GV) {
605 auto LT = GV->getLinkage();
606 return !(GlobalValue::isPrivateLinkage(Linkage: LT) ||
607 GlobalValue::isInternalLinkage(Linkage: LT));
608 };
609
610 // Use the names to distinguish the two values, but only if the
611 // names are semantically important.
612 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
613 return LGV->getName().compare(RHS: RGV->getName());
614 }
615
616 // For instructions, compare their loop depth, and their operand count. This
617 // is pretty loose.
618 if (const auto *LInst = dyn_cast<Instruction>(Val: LV)) {
619 const auto *RInst = cast<Instruction>(Val: RV);
620
621 // Compare loop depths.
622 const BasicBlock *LParent = LInst->getParent(),
623 *RParent = RInst->getParent();
624 if (LParent != RParent) {
625 unsigned LDepth = LI->getLoopDepth(BB: LParent),
626 RDepth = LI->getLoopDepth(BB: RParent);
627 if (LDepth != RDepth)
628 return (int)LDepth - (int)RDepth;
629 }
630
631 // Compare the number of operands.
632 unsigned LNumOps = LInst->getNumOperands(),
633 RNumOps = RInst->getNumOperands();
634 if (LNumOps != RNumOps)
635 return (int)LNumOps - (int)RNumOps;
636
637 for (unsigned Idx : seq(Size: LNumOps)) {
638 int Result = CompareValueComplexity(LI, LV: LInst->getOperand(i: Idx),
639 RV: RInst->getOperand(i: Idx), Depth: Depth + 1);
640 if (Result != 0)
641 return Result;
642 }
643 }
644
645 return 0;
646}
647
648// Return negative, zero, or positive, if LHS is less than, equal to, or greater
649// than RHS, respectively. A three-way result allows recursive comparisons to be
650// more efficient.
651// If the max analysis depth was reached, return std::nullopt, assuming we do
652// not know if they are equivalent for sure.
653static std::optional<int>
654CompareSCEVComplexity(const LoopInfo *const LI, const SCEV *LHS,
655 const SCEV *RHS, DominatorTree &DT, unsigned Depth = 0) {
656 // Fast-path: SCEVs are uniqued so we can do a quick equality check.
657 if (LHS == RHS)
658 return 0;
659
660 // Primarily, sort the SCEVs by their getSCEVType().
661 SCEVTypes LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
662 if (LType != RType)
663 return (int)LType - (int)RType;
664
665 if (Depth > MaxSCEVCompareDepth)
666 return std::nullopt;
667
668 // Aside from the getSCEVType() ordering, the particular ordering
669 // isn't very important except that it's beneficial to be consistent,
670 // so that (a + b) and (b + a) don't end up as different expressions.
671 switch (LType) {
672 case scUnknown: {
673 const SCEVUnknown *LU = cast<SCEVUnknown>(Val: LHS);
674 const SCEVUnknown *RU = cast<SCEVUnknown>(Val: RHS);
675
676 int X =
677 CompareValueComplexity(LI, LV: LU->getValue(), RV: RU->getValue(), Depth: Depth + 1);
678 return X;
679 }
680
681 case scConstant: {
682 const SCEVConstant *LC = cast<SCEVConstant>(Val: LHS);
683 const SCEVConstant *RC = cast<SCEVConstant>(Val: RHS);
684
685 // Compare constant values.
686 const APInt &LA = LC->getAPInt();
687 const APInt &RA = RC->getAPInt();
688 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
689 if (LBitWidth != RBitWidth)
690 return (int)LBitWidth - (int)RBitWidth;
691 return LA.ult(RHS: RA) ? -1 : 1;
692 }
693
694 case scVScale: {
695 const auto *LTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: LHS)->getType());
696 const auto *RTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val: RHS)->getType());
697 return LTy->getBitWidth() - RTy->getBitWidth();
698 }
699
700 case scAddRecExpr: {
701 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(Val: LHS);
702 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(Val: RHS);
703
704 // There is always a dominance between two recs that are used by one SCEV,
705 // so we can safely sort recs by loop header dominance. We require such
706 // order in getAddExpr.
707 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
708 if (LLoop != RLoop) {
709 const BasicBlock *LHead = LLoop->getHeader(), *RHead = RLoop->getHeader();
710 assert(LHead != RHead && "Two loops share the same header?");
711 if (DT.dominates(A: LHead, B: RHead))
712 return 1;
713 assert(DT.dominates(RHead, LHead) &&
714 "No dominance between recurrences used by one SCEV?");
715 return -1;
716 }
717
718 [[fallthrough]];
719 }
720
721 case scTruncate:
722 case scZeroExtend:
723 case scSignExtend:
724 case scPtrToInt:
725 case scAddExpr:
726 case scMulExpr:
727 case scUDivExpr:
728 case scSMaxExpr:
729 case scUMaxExpr:
730 case scSMinExpr:
731 case scUMinExpr:
732 case scSequentialUMinExpr: {
733 ArrayRef<const SCEV *> LOps = LHS->operands();
734 ArrayRef<const SCEV *> ROps = RHS->operands();
735
736 // Lexicographically compare n-ary-like expressions.
737 unsigned LNumOps = LOps.size(), RNumOps = ROps.size();
738 if (LNumOps != RNumOps)
739 return (int)LNumOps - (int)RNumOps;
740
741 for (unsigned i = 0; i != LNumOps; ++i) {
742 auto X = CompareSCEVComplexity(LI, LHS: LOps[i], RHS: ROps[i], DT, Depth: Depth + 1);
743 if (X != 0)
744 return X;
745 }
746 return 0;
747 }
748
749 case scCouldNotCompute:
750 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
751 }
752 llvm_unreachable("Unknown SCEV kind!");
753}
754
755/// Given a list of SCEV objects, order them by their complexity, and group
756/// objects of the same complexity together by value. When this routine is
757/// finished, we know that any duplicates in the vector are consecutive and that
758/// complexity is monotonically increasing.
759///
760/// Note that we go take special precautions to ensure that we get deterministic
761/// results from this routine. In other words, we don't want the results of
762/// this to depend on where the addresses of various SCEV objects happened to
763/// land in memory.
764static void GroupByComplexity(SmallVectorImpl<const SCEV *> &Ops,
765 LoopInfo *LI, DominatorTree &DT) {
766 if (Ops.size() < 2) return; // Noop
767
768 // Whether LHS has provably less complexity than RHS.
769 auto IsLessComplex = [&](const SCEV *LHS, const SCEV *RHS) {
770 auto Complexity = CompareSCEVComplexity(LI, LHS, RHS, DT);
771 return Complexity && *Complexity < 0;
772 };
773 if (Ops.size() == 2) {
774 // This is the common case, which also happens to be trivially simple.
775 // Special case it.
776 const SCEV *&LHS = Ops[0], *&RHS = Ops[1];
777 if (IsLessComplex(RHS, LHS))
778 std::swap(a&: LHS, b&: RHS);
779 return;
780 }
781
782 // Do the rough sort by complexity.
783 llvm::stable_sort(Range&: Ops, C: [&](const SCEV *LHS, const SCEV *RHS) {
784 return IsLessComplex(LHS, RHS);
785 });
786
787 // Now that we are sorted by complexity, group elements of the same
788 // complexity. Note that this is, at worst, N^2, but the vector is likely to
789 // be extremely short in practice. Note that we take this approach because we
790 // do not want to depend on the addresses of the objects we are grouping.
791 for (unsigned i = 0, e = Ops.size(); i != e-2; ++i) {
792 const SCEV *S = Ops[i];
793 unsigned Complexity = S->getSCEVType();
794
795 // If there are any objects of the same complexity and same value as this
796 // one, group them.
797 for (unsigned j = i+1; j != e && Ops[j]->getSCEVType() == Complexity; ++j) {
798 if (Ops[j] == S) { // Found a duplicate.
799 // Move it to immediately after i'th element.
800 std::swap(a&: Ops[i+1], b&: Ops[j]);
801 ++i; // no need to rescan it.
802 if (i == e-2) return; // Done!
803 }
804 }
805 }
806}
807
808/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
809/// least HugeExprThreshold nodes).
810static bool hasHugeExpression(ArrayRef<const SCEV *> Ops) {
811 return any_of(Range&: Ops, P: [](const SCEV *S) {
812 return S->getExpressionSize() >= HugeExprThreshold;
813 });
814}
815
816/// Performs a number of common optimizations on the passed \p Ops. If the
817/// whole expression reduces down to a single operand, it will be returned.
818///
819/// The following optimizations are performed:
820/// * Fold constants using the \p Fold function.
821/// * Remove identity constants satisfying \p IsIdentity.
822/// * If a constant satisfies \p IsAbsorber, return it.
823/// * Sort operands by complexity.
824template <typename FoldT, typename IsIdentityT, typename IsAbsorberT>
825static const SCEV *
826constantFoldAndGroupOps(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT,
827 SmallVectorImpl<const SCEV *> &Ops, FoldT Fold,
828 IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
829 const SCEVConstant *Folded = nullptr;
830 for (unsigned Idx = 0; Idx < Ops.size();) {
831 const SCEV *Op = Ops[Idx];
832 if (const auto *C = dyn_cast<SCEVConstant>(Val: Op)) {
833 if (!Folded)
834 Folded = C;
835 else
836 Folded = cast<SCEVConstant>(
837 SE.getConstant(Fold(Folded->getAPInt(), C->getAPInt())));
838 Ops.erase(CI: Ops.begin() + Idx);
839 continue;
840 }
841 ++Idx;
842 }
843
844 if (Ops.empty()) {
845 assert(Folded && "Must have folded value");
846 return Folded;
847 }
848
849 if (Folded && IsAbsorber(Folded->getAPInt()))
850 return Folded;
851
852 GroupByComplexity(Ops, LI: &LI, DT);
853 if (Folded && !IsIdentity(Folded->getAPInt()))
854 Ops.insert(I: Ops.begin(), Elt: Folded);
855
856 return Ops.size() == 1 ? Ops[0] : nullptr;
857}
858
859//===----------------------------------------------------------------------===//
860// Simple SCEV method implementations
861//===----------------------------------------------------------------------===//
862
863/// Compute BC(It, K). The result has width W. Assume, K > 0.
864static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
865 ScalarEvolution &SE,
866 Type *ResultTy) {
867 // Handle the simplest case efficiently.
868 if (K == 1)
869 return SE.getTruncateOrZeroExtend(V: It, Ty: ResultTy);
870
871 // We are using the following formula for BC(It, K):
872 //
873 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
874 //
875 // Suppose, W is the bitwidth of the return value. We must be prepared for
876 // overflow. Hence, we must assure that the result of our computation is
877 // equal to the accurate one modulo 2^W. Unfortunately, division isn't
878 // safe in modular arithmetic.
879 //
880 // However, this code doesn't use exactly that formula; the formula it uses
881 // is something like the following, where T is the number of factors of 2 in
882 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
883 // exponentiation:
884 //
885 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
886 //
887 // This formula is trivially equivalent to the previous formula. However,
888 // this formula can be implemented much more efficiently. The trick is that
889 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
890 // arithmetic. To do exact division in modular arithmetic, all we have
891 // to do is multiply by the inverse. Therefore, this step can be done at
892 // width W.
893 //
894 // The next issue is how to safely do the division by 2^T. The way this
895 // is done is by doing the multiplication step at a width of at least W + T
896 // bits. This way, the bottom W+T bits of the product are accurate. Then,
897 // when we perform the division by 2^T (which is equivalent to a right shift
898 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get
899 // truncated out after the division by 2^T.
900 //
901 // In comparison to just directly using the first formula, this technique
902 // is much more efficient; using the first formula requires W * K bits,
903 // but this formula less than W + K bits. Also, the first formula requires
904 // a division step, whereas this formula only requires multiplies and shifts.
905 //
906 // It doesn't matter whether the subtraction step is done in the calculation
907 // width or the input iteration count's width; if the subtraction overflows,
908 // the result must be zero anyway. We prefer here to do it in the width of
909 // the induction variable because it helps a lot for certain cases; CodeGen
910 // isn't smart enough to ignore the overflow, which leads to much less
911 // efficient code if the width of the subtraction is wider than the native
912 // register width.
913 //
914 // (It's possible to not widen at all by pulling out factors of 2 before
915 // the multiplication; for example, K=2 can be calculated as
916 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
917 // extra arithmetic, so it's not an obvious win, and it gets
918 // much more complicated for K > 3.)
919
920 // Protection from insane SCEVs; this bound is conservative,
921 // but it probably doesn't matter.
922 if (K > 1000)
923 return SE.getCouldNotCompute();
924
925 unsigned W = SE.getTypeSizeInBits(Ty: ResultTy);
926
927 // Calculate K! / 2^T and T; we divide out the factors of two before
928 // multiplying for calculating K! / 2^T to avoid overflow.
929 // Other overflow doesn't matter because we only care about the bottom
930 // W bits of the result.
931 APInt OddFactorial(W, 1);
932 unsigned T = 1;
933 for (unsigned i = 3; i <= K; ++i) {
934 unsigned TwoFactors = countr_zero(Val: i);
935 T += TwoFactors;
936 OddFactorial *= (i >> TwoFactors);
937 }
938
939 // We need at least W + T bits for the multiplication step
940 unsigned CalculationBits = W + T;
941
942 // Calculate 2^T, at width T+W.
943 APInt DivFactor = APInt::getOneBitSet(numBits: CalculationBits, BitNo: T);
944
945 // Calculate the multiplicative inverse of K! / 2^T;
946 // this multiplication factor will perform the exact division by
947 // K! / 2^T.
948 APInt MultiplyFactor = OddFactorial.multiplicativeInverse();
949
950 // Calculate the product, at width T+W
951 IntegerType *CalculationTy = IntegerType::get(C&: SE.getContext(),
952 NumBits: CalculationBits);
953 const SCEV *Dividend = SE.getTruncateOrZeroExtend(V: It, Ty: CalculationTy);
954 for (unsigned i = 1; i != K; ++i) {
955 const SCEV *S = SE.getMinusSCEV(LHS: It, RHS: SE.getConstant(Ty: It->getType(), V: i));
956 Dividend = SE.getMulExpr(LHS: Dividend,
957 RHS: SE.getTruncateOrZeroExtend(V: S, Ty: CalculationTy));
958 }
959
960 // Divide by 2^T
961 const SCEV *DivResult = SE.getUDivExpr(LHS: Dividend, RHS: SE.getConstant(Val: DivFactor));
962
963 // Truncate the result, and divide by K! / 2^T.
964
965 return SE.getMulExpr(LHS: SE.getConstant(Val: MultiplyFactor),
966 RHS: SE.getTruncateOrZeroExtend(V: DivResult, Ty: ResultTy));
967}
968
969/// Return the value of this chain of recurrences at the specified iteration
970/// number. We can evaluate this recurrence by multiplying each element in the
971/// chain by the binomial coefficient corresponding to it. In other words, we
972/// can evaluate {A,+,B,+,C,+,D} as:
973///
974/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
975///
976/// where BC(It, k) stands for binomial coefficient.
977const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
978 ScalarEvolution &SE) const {
979 return evaluateAtIteration(Operands: operands(), It, SE);
980}
981
982const SCEV *
983SCEVAddRecExpr::evaluateAtIteration(ArrayRef<const SCEV *> Operands,
984 const SCEV *It, ScalarEvolution &SE) {
985 assert(Operands.size() > 0);
986 const SCEV *Result = Operands[0];
987 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
988 // The computation is correct in the face of overflow provided that the
989 // multiplication is performed _after_ the evaluation of the binomial
990 // coefficient.
991 const SCEV *Coeff = BinomialCoefficient(It, K: i, SE, ResultTy: Result->getType());
992 if (isa<SCEVCouldNotCompute>(Val: Coeff))
993 return Coeff;
994
995 Result = SE.getAddExpr(LHS: Result, RHS: SE.getMulExpr(LHS: Operands[i], RHS: Coeff));
996 }
997 return Result;
998}
999
1000//===----------------------------------------------------------------------===//
1001// SCEV Expression folder implementations
1002//===----------------------------------------------------------------------===//
1003
1004const SCEV *ScalarEvolution::getLosslessPtrToIntExpr(const SCEV *Op,
1005 unsigned Depth) {
1006 assert(Depth <= 1 &&
1007 "getLosslessPtrToIntExpr() should self-recurse at most once.");
1008
1009 // We could be called with an integer-typed operands during SCEV rewrites.
1010 // Since the operand is an integer already, just perform zext/trunc/self cast.
1011 if (!Op->getType()->isPointerTy())
1012 return Op;
1013
1014 // What would be an ID for such a SCEV cast expression?
1015 FoldingSetNodeID ID;
1016 ID.AddInteger(I: scPtrToInt);
1017 ID.AddPointer(Ptr: Op);
1018
1019 void *IP = nullptr;
1020
1021 // Is there already an expression for such a cast?
1022 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
1023 return S;
1024
1025 // It isn't legal for optimizations to construct new ptrtoint expressions
1026 // for non-integral pointers.
1027 if (getDataLayout().isNonIntegralPointerType(Ty: Op->getType()))
1028 return getCouldNotCompute();
1029
1030 Type *IntPtrTy = getDataLayout().getIntPtrType(Op->getType());
1031
1032 // We can only trivially model ptrtoint if SCEV's effective (integer) type
1033 // is sufficiently wide to represent all possible pointer values.
1034 // We could theoretically teach SCEV to truncate wider pointers, but
1035 // that isn't implemented for now.
1036 if (getDataLayout().getTypeSizeInBits(Ty: getEffectiveSCEVType(Ty: Op->getType())) !=
1037 getDataLayout().getTypeSizeInBits(Ty: IntPtrTy))
1038 return getCouldNotCompute();
1039
1040 // If not, is this expression something we can't reduce any further?
1041 if (auto *U = dyn_cast<SCEVUnknown>(Val: Op)) {
1042 // Perform some basic constant folding. If the operand of the ptr2int cast
1043 // is a null pointer, don't create a ptr2int SCEV expression (that will be
1044 // left as-is), but produce a zero constant.
1045 // NOTE: We could handle a more general case, but lack motivational cases.
1046 if (isa<ConstantPointerNull>(Val: U->getValue()))
1047 return getZero(Ty: IntPtrTy);
1048
1049 // Create an explicit cast node.
1050 // We can reuse the existing insert position since if we get here,
1051 // we won't have made any changes which would invalidate it.
1052 SCEV *S = new (SCEVAllocator)
1053 SCEVPtrToIntExpr(ID.Intern(Allocator&: SCEVAllocator), Op, IntPtrTy);
1054 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1055 registerUser(User: S, Ops: Op);
1056 return S;
1057 }
1058
1059 assert(Depth == 0 && "getLosslessPtrToIntExpr() should not self-recurse for "
1060 "non-SCEVUnknown's.");
1061
1062 // Otherwise, we've got some expression that is more complex than just a
1063 // single SCEVUnknown. But we don't want to have a SCEVPtrToIntExpr of an
1064 // arbitrary expression, we want to have SCEVPtrToIntExpr of an SCEVUnknown
1065 // only, and the expressions must otherwise be integer-typed.
1066 // So sink the cast down to the SCEVUnknown's.
1067
1068 /// The SCEVPtrToIntSinkingRewriter takes a scalar evolution expression,
1069 /// which computes a pointer-typed value, and rewrites the whole expression
1070 /// tree so that *all* the computations are done on integers, and the only
1071 /// pointer-typed operands in the expression are SCEVUnknown.
1072 class SCEVPtrToIntSinkingRewriter
1073 : public SCEVRewriteVisitor<SCEVPtrToIntSinkingRewriter> {
1074 using Base = SCEVRewriteVisitor<SCEVPtrToIntSinkingRewriter>;
1075
1076 public:
1077 SCEVPtrToIntSinkingRewriter(ScalarEvolution &SE) : SCEVRewriteVisitor(SE) {}
1078
1079 static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE) {
1080 SCEVPtrToIntSinkingRewriter Rewriter(SE);
1081 return Rewriter.visit(S: Scev);
1082 }
1083
1084 const SCEV *visit(const SCEV *S) {
1085 Type *STy = S->getType();
1086 // If the expression is not pointer-typed, just keep it as-is.
1087 if (!STy->isPointerTy())
1088 return S;
1089 // Else, recursively sink the cast down into it.
1090 return Base::visit(S);
1091 }
1092
1093 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
1094 SmallVector<const SCEV *, 2> Operands;
1095 bool Changed = false;
1096 for (const auto *Op : Expr->operands()) {
1097 Operands.push_back(Elt: visit(S: Op));
1098 Changed |= Op != Operands.back();
1099 }
1100 return !Changed ? Expr : SE.getAddExpr(Ops&: Operands, Flags: Expr->getNoWrapFlags());
1101 }
1102
1103 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
1104 SmallVector<const SCEV *, 2> Operands;
1105 bool Changed = false;
1106 for (const auto *Op : Expr->operands()) {
1107 Operands.push_back(Elt: visit(S: Op));
1108 Changed |= Op != Operands.back();
1109 }
1110 return !Changed ? Expr : SE.getMulExpr(Ops&: Operands, Flags: Expr->getNoWrapFlags());
1111 }
1112
1113 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
1114 assert(Expr->getType()->isPointerTy() &&
1115 "Should only reach pointer-typed SCEVUnknown's.");
1116 return SE.getLosslessPtrToIntExpr(Op: Expr, /*Depth=*/1);
1117 }
1118 };
1119
1120 // And actually perform the cast sinking.
1121 const SCEV *IntOp = SCEVPtrToIntSinkingRewriter::rewrite(Scev: Op, SE&: *this);
1122 assert(IntOp->getType()->isIntegerTy() &&
1123 "We must have succeeded in sinking the cast, "
1124 "and ending up with an integer-typed expression!");
1125 return IntOp;
1126}
1127
1128const SCEV *ScalarEvolution::getPtrToIntExpr(const SCEV *Op, Type *Ty) {
1129 assert(Ty->isIntegerTy() && "Target type must be an integer type!");
1130
1131 const SCEV *IntOp = getLosslessPtrToIntExpr(Op);
1132 if (isa<SCEVCouldNotCompute>(Val: IntOp))
1133 return IntOp;
1134
1135 return getTruncateOrZeroExtend(V: IntOp, Ty);
1136}
1137
1138const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, Type *Ty,
1139 unsigned Depth) {
1140 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
1141 "This is not a truncating conversion!");
1142 assert(isSCEVable(Ty) &&
1143 "This is not a conversion to a SCEVable type!");
1144 assert(!Op->getType()->isPointerTy() && "Can't truncate pointer!");
1145 Ty = getEffectiveSCEVType(Ty);
1146
1147 FoldingSetNodeID ID;
1148 ID.AddInteger(I: scTruncate);
1149 ID.AddPointer(Ptr: Op);
1150 ID.AddPointer(Ptr: Ty);
1151 void *IP = nullptr;
1152 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
1153
1154 // Fold if the operand is constant.
1155 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
1156 return getConstant(
1157 V: cast<ConstantInt>(Val: ConstantExpr::getTrunc(C: SC->getValue(), Ty)));
1158
1159 // trunc(trunc(x)) --> trunc(x)
1160 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val: Op))
1161 return getTruncateExpr(Op: ST->getOperand(), Ty, Depth: Depth + 1);
1162
1163 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
1164 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val: Op))
1165 return getTruncateOrSignExtend(V: SS->getOperand(), Ty, Depth: Depth + 1);
1166
1167 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
1168 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val: Op))
1169 return getTruncateOrZeroExtend(V: SZ->getOperand(), Ty, Depth: Depth + 1);
1170
1171 if (Depth > MaxCastDepth) {
1172 SCEV *S =
1173 new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator), Op, Ty);
1174 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1175 registerUser(User: S, Ops: Op);
1176 return S;
1177 }
1178
1179 // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and
1180 // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN),
1181 // if after transforming we have at most one truncate, not counting truncates
1182 // that replace other casts.
1183 if (isa<SCEVAddExpr>(Val: Op) || isa<SCEVMulExpr>(Val: Op)) {
1184 auto *CommOp = cast<SCEVCommutativeExpr>(Val: Op);
1185 SmallVector<const SCEV *, 4> Operands;
1186 unsigned numTruncs = 0;
1187 for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
1188 ++i) {
1189 const SCEV *S = getTruncateExpr(Op: CommOp->getOperand(i), Ty, Depth: Depth + 1);
1190 if (!isa<SCEVIntegralCastExpr>(Val: CommOp->getOperand(i)) &&
1191 isa<SCEVTruncateExpr>(Val: S))
1192 numTruncs++;
1193 Operands.push_back(Elt: S);
1194 }
1195 if (numTruncs < 2) {
1196 if (isa<SCEVAddExpr>(Val: Op))
1197 return getAddExpr(Ops&: Operands);
1198 if (isa<SCEVMulExpr>(Val: Op))
1199 return getMulExpr(Ops&: Operands);
1200 llvm_unreachable("Unexpected SCEV type for Op.");
1201 }
1202 // Although we checked in the beginning that ID is not in the cache, it is
1203 // possible that during recursion and different modification ID was inserted
1204 // into the cache. So if we find it, just return it.
1205 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
1206 return S;
1207 }
1208
1209 // If the input value is a chrec scev, truncate the chrec's operands.
1210 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val: Op)) {
1211 SmallVector<const SCEV *, 4> Operands;
1212 for (const SCEV *Op : AddRec->operands())
1213 Operands.push_back(Elt: getTruncateExpr(Op, Ty, Depth: Depth + 1));
1214 return getAddRecExpr(Operands, L: AddRec->getLoop(), Flags: SCEV::FlagAnyWrap);
1215 }
1216
1217 // Return zero if truncating to known zeros.
1218 uint32_t MinTrailingZeros = getMinTrailingZeros(S: Op);
1219 if (MinTrailingZeros >= getTypeSizeInBits(Ty))
1220 return getZero(Ty);
1221
1222 // The cast wasn't folded; create an explicit cast node. We can reuse
1223 // the existing insert position since if we get here, we won't have
1224 // made any changes which would invalidate it.
1225 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator),
1226 Op, Ty);
1227 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1228 registerUser(User: S, Ops: Op);
1229 return S;
1230}
1231
1232// Get the limit of a recurrence such that incrementing by Step cannot cause
1233// signed overflow as long as the value of the recurrence within the
1234// loop does not exceed this limit before incrementing.
1235static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
1236 ICmpInst::Predicate *Pred,
1237 ScalarEvolution *SE) {
1238 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1239 if (SE->isKnownPositive(S: Step)) {
1240 *Pred = ICmpInst::ICMP_SLT;
1241 return SE->getConstant(Val: APInt::getSignedMinValue(numBits: BitWidth) -
1242 SE->getSignedRangeMax(S: Step));
1243 }
1244 if (SE->isKnownNegative(S: Step)) {
1245 *Pred = ICmpInst::ICMP_SGT;
1246 return SE->getConstant(Val: APInt::getSignedMaxValue(numBits: BitWidth) -
1247 SE->getSignedRangeMin(S: Step));
1248 }
1249 return nullptr;
1250}
1251
1252// Get the limit of a recurrence such that incrementing by Step cannot cause
1253// unsigned overflow as long as the value of the recurrence within the loop does
1254// not exceed this limit before incrementing.
1255static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step,
1256 ICmpInst::Predicate *Pred,
1257 ScalarEvolution *SE) {
1258 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1259 *Pred = ICmpInst::ICMP_ULT;
1260
1261 return SE->getConstant(Val: APInt::getMinValue(numBits: BitWidth) -
1262 SE->getUnsignedRangeMax(S: Step));
1263}
1264
1265namespace {
1266
1267struct ExtendOpTraitsBase {
1268 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(const SCEV *, Type *,
1269 unsigned);
1270};
1271
1272// Used to make code generic over signed and unsigned overflow.
1273template <typename ExtendOp> struct ExtendOpTraits {
1274 // Members present:
1275 //
1276 // static const SCEV::NoWrapFlags WrapType;
1277 //
1278 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
1279 //
1280 // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1281 // ICmpInst::Predicate *Pred,
1282 // ScalarEvolution *SE);
1283};
1284
1285template <>
1286struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
1287 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNSW;
1288
1289 static const GetExtendExprTy GetExtendExpr;
1290
1291 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1292 ICmpInst::Predicate *Pred,
1293 ScalarEvolution *SE) {
1294 return getSignedOverflowLimitForStep(Step, Pred, SE);
1295 }
1296};
1297
1298const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1299 SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr;
1300
1301template <>
1302struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
1303 static const SCEV::NoWrapFlags WrapType = SCEV::FlagNUW;
1304
1305 static const GetExtendExprTy GetExtendExpr;
1306
1307 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1308 ICmpInst::Predicate *Pred,
1309 ScalarEvolution *SE) {
1310 return getUnsignedOverflowLimitForStep(Step, Pred, SE);
1311 }
1312};
1313
1314const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1315 SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr;
1316
1317} // end anonymous namespace
1318
1319// The recurrence AR has been shown to have no signed/unsigned wrap or something
1320// close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
1321// easily prove NSW/NUW for its preincrement or postincrement sibling. This
1322// allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
1323// Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
1324// expression "Step + sext/zext(PreIncAR)" is congruent with
1325// "sext/zext(PostIncAR)"
1326template <typename ExtendOpTy>
1327static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR, Type *Ty,
1328 ScalarEvolution *SE, unsigned Depth) {
1329 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1330 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1331
1332 const Loop *L = AR->getLoop();
1333 const SCEV *Start = AR->getStart();
1334 const SCEV *Step = AR->getStepRecurrence(SE&: *SE);
1335
1336 // Check for a simple looking step prior to loop entry.
1337 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Val: Start);
1338 if (!SA)
1339 return nullptr;
1340
1341 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
1342 // subtraction is expensive. For this purpose, perform a quick and dirty
1343 // difference, by checking for Step in the operand list. Note, that
1344 // SA might have repeated ops, like %a + %a + ..., so only remove one.
1345 SmallVector<const SCEV *, 4> DiffOps(SA->operands());
1346 for (auto It = DiffOps.begin(); It != DiffOps.end(); ++It)
1347 if (*It == Step) {
1348 DiffOps.erase(CI: It);
1349 break;
1350 }
1351
1352 if (DiffOps.size() == SA->getNumOperands())
1353 return nullptr;
1354
1355 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
1356 // `Step`:
1357
1358 // 1. NSW/NUW flags on the step increment.
1359 auto PreStartFlags =
1360 ScalarEvolution::maskFlags(Flags: SA->getNoWrapFlags(), Mask: SCEV::FlagNUW);
1361 const SCEV *PreStart = SE->getAddExpr(Ops&: DiffOps, Flags: PreStartFlags);
1362 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
1363 Val: SE->getAddRecExpr(Start: PreStart, Step, L, Flags: SCEV::FlagAnyWrap));
1364
1365 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
1366 // "S+X does not sign/unsign-overflow".
1367 //
1368
1369 const SCEV *BECount = SE->getBackedgeTakenCount(L);
1370 if (PreAR && PreAR->getNoWrapFlags(Mask: WrapType) &&
1371 !isa<SCEVCouldNotCompute>(Val: BECount) && SE->isKnownPositive(S: BECount))
1372 return PreStart;
1373
1374 // 2. Direct overflow check on the step operation's expression.
1375 unsigned BitWidth = SE->getTypeSizeInBits(Ty: AR->getType());
1376 Type *WideTy = IntegerType::get(C&: SE->getContext(), NumBits: BitWidth * 2);
1377 const SCEV *OperandExtendedStart =
1378 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
1379 (SE->*GetExtendExpr)(Step, WideTy, Depth));
1380 if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
1381 if (PreAR && AR->getNoWrapFlags(Mask: WrapType)) {
1382 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
1383 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
1384 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact.
1385 SE->setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(PreAR), Flags: WrapType);
1386 }
1387 return PreStart;
1388 }
1389
1390 // 3. Loop precondition.
1391 ICmpInst::Predicate Pred;
1392 const SCEV *OverflowLimit =
1393 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1394
1395 if (OverflowLimit &&
1396 SE->isLoopEntryGuardedByCond(L, Pred, LHS: PreStart, RHS: OverflowLimit))
1397 return PreStart;
1398
1399 return nullptr;
1400}
1401
1402// Get the normalized zero or sign extended expression for this AddRec's Start.
1403template <typename ExtendOpTy>
1404static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
1405 ScalarEvolution *SE,
1406 unsigned Depth) {
1407 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1408
1409 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, Ty, SE, Depth);
1410 if (!PreStart)
1411 return (SE->*GetExtendExpr)(AR->getStart(), Ty, Depth);
1412
1413 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(SE&: *SE), Ty,
1414 Depth),
1415 (SE->*GetExtendExpr)(PreStart, Ty, Depth));
1416}
1417
1418// Try to prove away overflow by looking at "nearby" add recurrences. A
1419// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
1420// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
1421//
1422// Formally:
1423//
1424// {S,+,X} == {S-T,+,X} + T
1425// => Ext({S,+,X}) == Ext({S-T,+,X} + T)
1426//
1427// If ({S-T,+,X} + T) does not overflow ... (1)
1428//
1429// RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
1430//
1431// If {S-T,+,X} does not overflow ... (2)
1432//
1433// RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
1434// == {Ext(S-T)+Ext(T),+,Ext(X)}
1435//
1436// If (S-T)+T does not overflow ... (3)
1437//
1438// RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
1439// == {Ext(S),+,Ext(X)} == LHS
1440//
1441// Thus, if (1), (2) and (3) are true for some T, then
1442// Ext({S,+,X}) == {Ext(S),+,Ext(X)}
1443//
1444// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
1445// does not overflow" restricted to the 0th iteration. Therefore we only need
1446// to check for (1) and (2).
1447//
1448// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
1449// is `Delta` (defined below).
1450template <typename ExtendOpTy>
1451bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
1452 const SCEV *Step,
1453 const Loop *L) {
1454 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1455
1456 // We restrict `Start` to a constant to prevent SCEV from spending too much
1457 // time here. It is correct (but more expensive) to continue with a
1458 // non-constant `Start` and do a general SCEV subtraction to compute
1459 // `PreStart` below.
1460 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Val: Start);
1461 if (!StartC)
1462 return false;
1463
1464 APInt StartAI = StartC->getAPInt();
1465
1466 for (unsigned Delta : {-2, -1, 1, 2}) {
1467 const SCEV *PreStart = getConstant(Val: StartAI - Delta);
1468
1469 FoldingSetNodeID ID;
1470 ID.AddInteger(I: scAddRecExpr);
1471 ID.AddPointer(Ptr: PreStart);
1472 ID.AddPointer(Ptr: Step);
1473 ID.AddPointer(Ptr: L);
1474 void *IP = nullptr;
1475 const auto *PreAR =
1476 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
1477
1478 // Give up if we don't already have the add recurrence we need because
1479 // actually constructing an add recurrence is relatively expensive.
1480 if (PreAR && PreAR->getNoWrapFlags(Mask: WrapType)) { // proves (2)
1481 const SCEV *DeltaS = getConstant(Ty: StartC->getType(), V: Delta);
1482 ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
1483 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1484 DeltaS, &Pred, this);
1485 if (Limit && isKnownPredicate(Pred, LHS: PreAR, RHS: Limit)) // proves (1)
1486 return true;
1487 }
1488 }
1489
1490 return false;
1491}
1492
1493// Finds an integer D for an expression (C + x + y + ...) such that the top
1494// level addition in (D + (C - D + x + y + ...)) would not wrap (signed or
1495// unsigned) and the number of trailing zeros of (C - D + x + y + ...) is
1496// maximized, where C is the \p ConstantTerm, x, y, ... are arbitrary SCEVs, and
1497// the (C + x + y + ...) expression is \p WholeAddExpr.
1498static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1499 const SCEVConstant *ConstantTerm,
1500 const SCEVAddExpr *WholeAddExpr) {
1501 const APInt &C = ConstantTerm->getAPInt();
1502 const unsigned BitWidth = C.getBitWidth();
1503 // Find number of trailing zeros of (x + y + ...) w/o the C first:
1504 uint32_t TZ = BitWidth;
1505 for (unsigned I = 1, E = WholeAddExpr->getNumOperands(); I < E && TZ; ++I)
1506 TZ = std::min(a: TZ, b: SE.getMinTrailingZeros(S: WholeAddExpr->getOperand(i: I)));
1507 if (TZ) {
1508 // Set D to be as many least significant bits of C as possible while still
1509 // guaranteeing that adding D to (C - D + x + y + ...) won't cause a wrap:
1510 return TZ < BitWidth ? C.trunc(width: TZ).zext(width: BitWidth) : C;
1511 }
1512 return APInt(BitWidth, 0);
1513}
1514
1515// Finds an integer D for an affine AddRec expression {C,+,x} such that the top
1516// level addition in (D + {C-D,+,x}) would not wrap (signed or unsigned) and the
1517// number of trailing zeros of (C - D + x * n) is maximized, where C is the \p
1518// ConstantStart, x is an arbitrary \p Step, and n is the loop trip count.
1519static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1520 const APInt &ConstantStart,
1521 const SCEV *Step) {
1522 const unsigned BitWidth = ConstantStart.getBitWidth();
1523 const uint32_t TZ = SE.getMinTrailingZeros(S: Step);
1524 if (TZ)
1525 return TZ < BitWidth ? ConstantStart.trunc(width: TZ).zext(width: BitWidth)
1526 : ConstantStart;
1527 return APInt(BitWidth, 0);
1528}
1529
1530static void insertFoldCacheEntry(
1531 const ScalarEvolution::FoldID &ID, const SCEV *S,
1532 DenseMap<ScalarEvolution::FoldID, const SCEV *> &FoldCache,
1533 DenseMap<const SCEV *, SmallVector<ScalarEvolution::FoldID, 2>>
1534 &FoldCacheUser) {
1535 auto I = FoldCache.insert(KV: {ID, S});
1536 if (!I.second) {
1537 // Remove FoldCacheUser entry for ID when replacing an existing FoldCache
1538 // entry.
1539 auto &UserIDs = FoldCacheUser[I.first->second];
1540 assert(count(UserIDs, ID) == 1 && "unexpected duplicates in UserIDs");
1541 for (unsigned I = 0; I != UserIDs.size(); ++I)
1542 if (UserIDs[I] == ID) {
1543 std::swap(a&: UserIDs[I], b&: UserIDs.back());
1544 break;
1545 }
1546 UserIDs.pop_back();
1547 I.first->second = S;
1548 }
1549 FoldCacheUser[S].push_back(Elt: ID);
1550}
1551
1552const SCEV *
1553ScalarEvolution::getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) {
1554 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1555 "This is not an extending conversion!");
1556 assert(isSCEVable(Ty) &&
1557 "This is not a conversion to a SCEVable type!");
1558 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1559 Ty = getEffectiveSCEVType(Ty);
1560
1561 FoldID ID(scZeroExtend, Op, Ty);
1562 auto Iter = FoldCache.find(Val: ID);
1563 if (Iter != FoldCache.end())
1564 return Iter->second;
1565
1566 const SCEV *S = getZeroExtendExprImpl(Op, Ty, Depth);
1567 if (!isa<SCEVZeroExtendExpr>(Val: S))
1568 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1569 return S;
1570}
1571
1572const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty,
1573 unsigned Depth) {
1574 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1575 "This is not an extending conversion!");
1576 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1577 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1578
1579 // Fold if the operand is constant.
1580 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
1581 return getConstant(Val: SC->getAPInt().zext(width: getTypeSizeInBits(Ty)));
1582
1583 // zext(zext(x)) --> zext(x)
1584 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val: Op))
1585 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1586
1587 // Before doing any expensive analysis, check to see if we've already
1588 // computed a SCEV for this Op and Ty.
1589 FoldingSetNodeID ID;
1590 ID.AddInteger(I: scZeroExtend);
1591 ID.AddPointer(Ptr: Op);
1592 ID.AddPointer(Ptr: Ty);
1593 void *IP = nullptr;
1594 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
1595 if (Depth > MaxCastDepth) {
1596 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1597 Op, Ty);
1598 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1599 registerUser(User: S, Ops: Op);
1600 return S;
1601 }
1602
1603 // zext(trunc(x)) --> zext(x) or x or trunc(x)
1604 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
1605 // It's possible the bits taken off by the truncate were all zero bits. If
1606 // so, we should be able to simplify this further.
1607 const SCEV *X = ST->getOperand();
1608 ConstantRange CR = getUnsignedRange(S: X);
1609 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1610 unsigned NewBits = getTypeSizeInBits(Ty);
1611 if (CR.truncate(BitWidth: TruncBits).zeroExtend(BitWidth: NewBits).contains(
1612 CR: CR.zextOrTrunc(BitWidth: NewBits)))
1613 return getTruncateOrZeroExtend(V: X, Ty, Depth);
1614 }
1615
1616 // If the input value is a chrec scev, and we can prove that the value
1617 // did not overflow the old, smaller, value, we can zero extend all of the
1618 // operands (often constants). This allows analysis of something like
1619 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
1620 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Op))
1621 if (AR->isAffine()) {
1622 const SCEV *Start = AR->getStart();
1623 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
1624 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
1625 const Loop *L = AR->getLoop();
1626
1627 // If we have special knowledge that this addrec won't overflow,
1628 // we don't need to do any further analysis.
1629 if (AR->hasNoUnsignedWrap()) {
1630 Start =
1631 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1632 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1633 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1634 }
1635
1636 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1637 // Note that this serves two purposes: It filters out loops that are
1638 // simply not analyzable, and it covers the case where this code is
1639 // being called from within backedge-taken count analysis, such that
1640 // attempting to ask for the backedge-taken count would likely result
1641 // in infinite recursion. In the later case, the analysis code will
1642 // cope with a conservative value, and it will take care to purge
1643 // that value once it has finished.
1644 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1645 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
1646 // Manually compute the final value for AR, checking for overflow.
1647
1648 // Check whether the backedge-taken count can be losslessly casted to
1649 // the addrec's type. The count is always unsigned.
1650 const SCEV *CastedMaxBECount =
1651 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
1652 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1653 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
1654 if (MaxBECount == RecastedMaxBECount) {
1655 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
1656 // Check whether Start+Step*MaxBECount has no unsigned overflow.
1657 const SCEV *ZMul = getMulExpr(LHS: CastedMaxBECount, RHS: Step,
1658 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1659 const SCEV *ZAdd = getZeroExtendExpr(Op: getAddExpr(LHS: Start, RHS: ZMul,
1660 Flags: SCEV::FlagAnyWrap,
1661 Depth: Depth + 1),
1662 Ty: WideTy, Depth: Depth + 1);
1663 const SCEV *WideStart = getZeroExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
1664 const SCEV *WideMaxBECount =
1665 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
1666 const SCEV *OperandExtendedAdd =
1667 getAddExpr(LHS: WideStart,
1668 RHS: getMulExpr(LHS: WideMaxBECount,
1669 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1670 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
1671 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1672 if (ZAdd == OperandExtendedAdd) {
1673 // Cache knowledge of AR NUW, which is propagated to this AddRec.
1674 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1675 // Return the expression with the addrec on the outside.
1676 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this,
1677 Depth: Depth + 1);
1678 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1679 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1680 }
1681 // Similar to above, only this time treat the step value as signed.
1682 // This covers loops that count down.
1683 OperandExtendedAdd =
1684 getAddExpr(LHS: WideStart,
1685 RHS: getMulExpr(LHS: WideMaxBECount,
1686 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1687 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
1688 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
1689 if (ZAdd == OperandExtendedAdd) {
1690 // Cache knowledge of AR NW, which is propagated to this AddRec.
1691 // Negative step causes unsigned wrap, but it still can't self-wrap.
1692 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1693 // Return the expression with the addrec on the outside.
1694 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this,
1695 Depth: Depth + 1);
1696 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1697 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1698 }
1699 }
1700 }
1701
1702 // Normally, in the cases we can prove no-overflow via a
1703 // backedge guarding condition, we can also compute a backedge
1704 // taken count for the loop. The exceptions are assumptions and
1705 // guards present in the loop -- SCEV is not great at exploiting
1706 // these to compute max backedge taken counts, but can still use
1707 // these to prove lack of overflow. Use this fact to avoid
1708 // doing extra work that may not pay off.
1709 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount) || HasGuards ||
1710 !AC.assumptions().empty()) {
1711
1712 auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
1713 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
1714 if (AR->hasNoUnsignedWrap()) {
1715 // Same as nuw case above - duplicated here to avoid a compile time
1716 // issue. It's not clear that the order of checks does matter, but
1717 // it's one of two issue possible causes for a change which was
1718 // reverted. Be conservative for the moment.
1719 Start =
1720 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1721 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1722 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1723 }
1724
1725 // For a negative step, we can extend the operands iff doing so only
1726 // traverses values in the range zext([0,UINT_MAX]).
1727 if (isKnownNegative(S: Step)) {
1728 const SCEV *N = getConstant(Val: APInt::getMaxValue(numBits: BitWidth) -
1729 getSignedRangeMin(S: Step));
1730 if (isLoopBackedgeGuardedByCond(L, Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N) ||
1731 isKnownOnEveryIteration(Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N)) {
1732 // Cache knowledge of AR NW, which is propagated to this
1733 // AddRec. Negative step causes unsigned wrap, but it
1734 // still can't self-wrap.
1735 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1736 // Return the expression with the addrec on the outside.
1737 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this,
1738 Depth: Depth + 1);
1739 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1740 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1741 }
1742 }
1743 }
1744
1745 // zext({C,+,Step}) --> (zext(D) + zext({C-D,+,Step}))<nuw><nsw>
1746 // if D + (C - D + Step * n) could be proven to not unsigned wrap
1747 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1748 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
1749 const APInt &C = SC->getAPInt();
1750 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
1751 if (D != 0) {
1752 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1753 const SCEV *SResidual =
1754 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
1755 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1756 return getAddExpr(LHS: SZExtD, RHS: SZExtR,
1757 Flags: (SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
1758 Depth: Depth + 1);
1759 }
1760 }
1761
1762 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1763 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1764 Start =
1765 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1766 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1767 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1768 }
1769 }
1770
1771 // zext(A % B) --> zext(A) % zext(B)
1772 {
1773 const SCEV *LHS;
1774 const SCEV *RHS;
1775 if (matchURem(Expr: Op, LHS, RHS))
1776 return getURemExpr(LHS: getZeroExtendExpr(Op: LHS, Ty, Depth: Depth + 1),
1777 RHS: getZeroExtendExpr(Op: RHS, Ty, Depth: Depth + 1));
1778 }
1779
1780 // zext(A / B) --> zext(A) / zext(B).
1781 if (auto *Div = dyn_cast<SCEVUDivExpr>(Val: Op))
1782 return getUDivExpr(LHS: getZeroExtendExpr(Op: Div->getLHS(), Ty, Depth: Depth + 1),
1783 RHS: getZeroExtendExpr(Op: Div->getRHS(), Ty, Depth: Depth + 1));
1784
1785 if (auto *SA = dyn_cast<SCEVAddExpr>(Val: Op)) {
1786 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw>
1787 if (SA->hasNoUnsignedWrap()) {
1788 // If the addition does not unsign overflow then we can, by definition,
1789 // commute the zero extension with the addition operation.
1790 SmallVector<const SCEV *, 4> Ops;
1791 for (const auto *Op : SA->operands())
1792 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1793 return getAddExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1794 }
1795
1796 // zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
1797 // if D + (C - D + x + y + ...) could be proven to not unsigned wrap
1798 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1799 //
1800 // Often address arithmetics contain expressions like
1801 // (zext (add (shl X, C1), C2)), for instance, (zext (5 + (4 * X))).
1802 // This transformation is useful while proving that such expressions are
1803 // equal or differ by a small constant amount, see LoadStoreVectorizer pass.
1804 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1805 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1806 if (D != 0) {
1807 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1808 const SCEV *SResidual =
1809 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagAnyWrap, Depth);
1810 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1811 return getAddExpr(LHS: SZExtD, RHS: SZExtR,
1812 Flags: (SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
1813 Depth: Depth + 1);
1814 }
1815 }
1816 }
1817
1818 if (auto *SM = dyn_cast<SCEVMulExpr>(Val: Op)) {
1819 // zext((A * B * ...)<nuw>) --> (zext(A) * zext(B) * ...)<nuw>
1820 if (SM->hasNoUnsignedWrap()) {
1821 // If the multiply does not unsign overflow then we can, by definition,
1822 // commute the zero extension with the multiply operation.
1823 SmallVector<const SCEV *, 4> Ops;
1824 for (const auto *Op : SM->operands())
1825 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1826 return getMulExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1827 }
1828
1829 // zext(2^K * (trunc X to iN)) to iM ->
1830 // 2^K * (zext(trunc X to i{N-K}) to iM)<nuw>
1831 //
1832 // Proof:
1833 //
1834 // zext(2^K * (trunc X to iN)) to iM
1835 // = zext((trunc X to iN) << K) to iM
1836 // = zext((trunc X to i{N-K}) << K)<nuw> to iM
1837 // (because shl removes the top K bits)
1838 // = zext((2^K * (trunc X to i{N-K}))<nuw>) to iM
1839 // = (2^K * (zext(trunc X to i{N-K}) to iM))<nuw>.
1840 //
1841 if (SM->getNumOperands() == 2)
1842 if (auto *MulLHS = dyn_cast<SCEVConstant>(Val: SM->getOperand(i: 0)))
1843 if (MulLHS->getAPInt().isPowerOf2())
1844 if (auto *TruncRHS = dyn_cast<SCEVTruncateExpr>(Val: SM->getOperand(i: 1))) {
1845 int NewTruncBits = getTypeSizeInBits(Ty: TruncRHS->getType()) -
1846 MulLHS->getAPInt().logBase2();
1847 Type *NewTruncTy = IntegerType::get(C&: getContext(), NumBits: NewTruncBits);
1848 return getMulExpr(
1849 LHS: getZeroExtendExpr(Op: MulLHS, Ty),
1850 RHS: getZeroExtendExpr(
1851 Op: getTruncateExpr(Op: TruncRHS->getOperand(), Ty: NewTruncTy), Ty),
1852 Flags: SCEV::FlagNUW, Depth: Depth + 1);
1853 }
1854 }
1855
1856 // zext(umin(x, y)) -> umin(zext(x), zext(y))
1857 // zext(umax(x, y)) -> umax(zext(x), zext(y))
1858 if (isa<SCEVUMinExpr>(Val: Op) || isa<SCEVUMaxExpr>(Val: Op)) {
1859 auto *MinMax = cast<SCEVMinMaxExpr>(Val: Op);
1860 SmallVector<const SCEV *, 4> Operands;
1861 for (auto *Operand : MinMax->operands())
1862 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1863 if (isa<SCEVUMinExpr>(Val: MinMax))
1864 return getUMinExpr(Operands);
1865 return getUMaxExpr(Operands);
1866 }
1867
1868 // zext(umin_seq(x, y)) -> umin_seq(zext(x), zext(y))
1869 if (auto *MinMax = dyn_cast<SCEVSequentialMinMaxExpr>(Val: Op)) {
1870 assert(isa<SCEVSequentialUMinExpr>(MinMax) && "Not supported!");
1871 SmallVector<const SCEV *, 4> Operands;
1872 for (auto *Operand : MinMax->operands())
1873 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1874 return getUMinExpr(Operands, /*Sequential*/ true);
1875 }
1876
1877 // The cast wasn't folded; create an explicit cast node.
1878 // Recompute the insert position, as it may have been invalidated.
1879 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
1880 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1881 Op, Ty);
1882 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1883 registerUser(User: S, Ops: Op);
1884 return S;
1885}
1886
1887const SCEV *
1888ScalarEvolution::getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) {
1889 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1890 "This is not an extending conversion!");
1891 assert(isSCEVable(Ty) &&
1892 "This is not a conversion to a SCEVable type!");
1893 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1894 Ty = getEffectiveSCEVType(Ty);
1895
1896 FoldID ID(scSignExtend, Op, Ty);
1897 auto Iter = FoldCache.find(Val: ID);
1898 if (Iter != FoldCache.end())
1899 return Iter->second;
1900
1901 const SCEV *S = getSignExtendExprImpl(Op, Ty, Depth);
1902 if (!isa<SCEVSignExtendExpr>(Val: S))
1903 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1904 return S;
1905}
1906
1907const SCEV *ScalarEvolution::getSignExtendExprImpl(const SCEV *Op, Type *Ty,
1908 unsigned Depth) {
1909 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1910 "This is not an extending conversion!");
1911 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1912 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1913 Ty = getEffectiveSCEVType(Ty);
1914
1915 // Fold if the operand is constant.
1916 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
1917 return getConstant(Val: SC->getAPInt().sext(width: getTypeSizeInBits(Ty)));
1918
1919 // sext(sext(x)) --> sext(x)
1920 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val: Op))
1921 return getSignExtendExpr(Op: SS->getOperand(), Ty, Depth: Depth + 1);
1922
1923 // sext(zext(x)) --> zext(x)
1924 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val: Op))
1925 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1926
1927 // Before doing any expensive analysis, check to see if we've already
1928 // computed a SCEV for this Op and Ty.
1929 FoldingSetNodeID ID;
1930 ID.AddInteger(I: scSignExtend);
1931 ID.AddPointer(Ptr: Op);
1932 ID.AddPointer(Ptr: Ty);
1933 void *IP = nullptr;
1934 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
1935 // Limit recursion depth.
1936 if (Depth > MaxCastDepth) {
1937 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1938 Op, Ty);
1939 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
1940 registerUser(User: S, Ops: Op);
1941 return S;
1942 }
1943
1944 // sext(trunc(x)) --> sext(x) or x or trunc(x)
1945 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
1946 // It's possible the bits taken off by the truncate were all sign bits. If
1947 // so, we should be able to simplify this further.
1948 const SCEV *X = ST->getOperand();
1949 ConstantRange CR = getSignedRange(S: X);
1950 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1951 unsigned NewBits = getTypeSizeInBits(Ty);
1952 if (CR.truncate(BitWidth: TruncBits).signExtend(BitWidth: NewBits).contains(
1953 CR: CR.sextOrTrunc(BitWidth: NewBits)))
1954 return getTruncateOrSignExtend(V: X, Ty, Depth);
1955 }
1956
1957 if (auto *SA = dyn_cast<SCEVAddExpr>(Val: Op)) {
1958 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
1959 if (SA->hasNoSignedWrap()) {
1960 // If the addition does not sign overflow then we can, by definition,
1961 // commute the sign extension with the addition operation.
1962 SmallVector<const SCEV *, 4> Ops;
1963 for (const auto *Op : SA->operands())
1964 Ops.push_back(Elt: getSignExtendExpr(Op, Ty, Depth: Depth + 1));
1965 return getAddExpr(Ops, Flags: SCEV::FlagNSW, Depth: Depth + 1);
1966 }
1967
1968 // sext(C + x + y + ...) --> (sext(D) + sext((C - D) + x + y + ...))
1969 // if D + (C - D + x + y + ...) could be proven to not signed wrap
1970 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1971 //
1972 // For instance, this will bring two seemingly different expressions:
1973 // 1 + sext(5 + 20 * %x + 24 * %y) and
1974 // sext(6 + 20 * %x + 24 * %y)
1975 // to the same form:
1976 // 2 + sext(4 + 20 * %x + 24 * %y)
1977 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1978 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1979 if (D != 0) {
1980 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1981 const SCEV *SResidual =
1982 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagAnyWrap, Depth);
1983 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1984 return getAddExpr(LHS: SSExtD, RHS: SSExtR,
1985 Flags: (SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
1986 Depth: Depth + 1);
1987 }
1988 }
1989 }
1990 // If the input value is a chrec scev, and we can prove that the value
1991 // did not overflow the old, smaller, value, we can sign extend all of the
1992 // operands (often constants). This allows analysis of something like
1993 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; }
1994 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Op))
1995 if (AR->isAffine()) {
1996 const SCEV *Start = AR->getStart();
1997 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
1998 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
1999 const Loop *L = AR->getLoop();
2000
2001 // If we have special knowledge that this addrec won't overflow,
2002 // we don't need to do any further analysis.
2003 if (AR->hasNoSignedWrap()) {
2004 Start =
2005 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2006 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2007 return getAddRecExpr(Start, Step, L, Flags: SCEV::FlagNSW);
2008 }
2009
2010 // Check whether the backedge-taken count is SCEVCouldNotCompute.
2011 // Note that this serves two purposes: It filters out loops that are
2012 // simply not analyzable, and it covers the case where this code is
2013 // being called from within backedge-taken count analysis, such that
2014 // attempting to ask for the backedge-taken count would likely result
2015 // in infinite recursion. In the later case, the analysis code will
2016 // cope with a conservative value, and it will take care to purge
2017 // that value once it has finished.
2018 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
2019 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
2020 // Manually compute the final value for AR, checking for
2021 // overflow.
2022
2023 // Check whether the backedge-taken count can be losslessly casted to
2024 // the addrec's type. The count is always unsigned.
2025 const SCEV *CastedMaxBECount =
2026 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
2027 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
2028 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
2029 if (MaxBECount == RecastedMaxBECount) {
2030 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
2031 // Check whether Start+Step*MaxBECount has no signed overflow.
2032 const SCEV *SMul = getMulExpr(LHS: CastedMaxBECount, RHS: Step,
2033 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2034 const SCEV *SAdd = getSignExtendExpr(Op: getAddExpr(LHS: Start, RHS: SMul,
2035 Flags: SCEV::FlagAnyWrap,
2036 Depth: Depth + 1),
2037 Ty: WideTy, Depth: Depth + 1);
2038 const SCEV *WideStart = getSignExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
2039 const SCEV *WideMaxBECount =
2040 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
2041 const SCEV *OperandExtendedAdd =
2042 getAddExpr(LHS: WideStart,
2043 RHS: getMulExpr(LHS: WideMaxBECount,
2044 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2045 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2046 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2047 if (SAdd == OperandExtendedAdd) {
2048 // Cache knowledge of AR NSW, which is propagated to this AddRec.
2049 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2050 // Return the expression with the addrec on the outside.
2051 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this,
2052 Depth: Depth + 1);
2053 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2054 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2055 }
2056 // Similar to above, only this time treat the step value as unsigned.
2057 // This covers loops that count up with an unsigned step.
2058 OperandExtendedAdd =
2059 getAddExpr(LHS: WideStart,
2060 RHS: getMulExpr(LHS: WideMaxBECount,
2061 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
2062 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2063 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2064 if (SAdd == OperandExtendedAdd) {
2065 // If AR wraps around then
2066 //
2067 // abs(Step) * MaxBECount > unsigned-max(AR->getType())
2068 // => SAdd != OperandExtendedAdd
2069 //
2070 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
2071 // (SAdd == OperandExtendedAdd => AR is NW)
2072
2073 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
2074
2075 // Return the expression with the addrec on the outside.
2076 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this,
2077 Depth: Depth + 1);
2078 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2079 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2080 }
2081 }
2082 }
2083
2084 auto NewFlags = proveNoSignedWrapViaInduction(AR);
2085 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
2086 if (AR->hasNoSignedWrap()) {
2087 // Same as nsw case above - duplicated here to avoid a compile time
2088 // issue. It's not clear that the order of checks does matter, but
2089 // it's one of two issue possible causes for a change which was
2090 // reverted. Be conservative for the moment.
2091 Start =
2092 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2093 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2094 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2095 }
2096
2097 // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw>
2098 // if D + (C - D + Step * n) could be proven to not signed wrap
2099 // where D maximizes the number of trailing zeros of (C - D + Step * n)
2100 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
2101 const APInt &C = SC->getAPInt();
2102 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
2103 if (D != 0) {
2104 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
2105 const SCEV *SResidual =
2106 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
2107 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
2108 return getAddExpr(LHS: SSExtD, RHS: SSExtR,
2109 Flags: (SCEV::NoWrapFlags)(SCEV::FlagNSW | SCEV::FlagNUW),
2110 Depth: Depth + 1);
2111 }
2112 }
2113
2114 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
2115 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
2116 Start =
2117 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
2118 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
2119 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
2120 }
2121 }
2122
2123 // If the input value is provably positive and we could not simplify
2124 // away the sext build a zext instead.
2125 if (isKnownNonNegative(S: Op))
2126 return getZeroExtendExpr(Op, Ty, Depth: Depth + 1);
2127
2128 // sext(smin(x, y)) -> smin(sext(x), sext(y))
2129 // sext(smax(x, y)) -> smax(sext(x), sext(y))
2130 if (isa<SCEVSMinExpr>(Val: Op) || isa<SCEVSMaxExpr>(Val: Op)) {
2131 auto *MinMax = cast<SCEVMinMaxExpr>(Val: Op);
2132 SmallVector<const SCEV *, 4> Operands;
2133 for (auto *Operand : MinMax->operands())
2134 Operands.push_back(Elt: getSignExtendExpr(Op: Operand, Ty));
2135 if (isa<SCEVSMinExpr>(Val: MinMax))
2136 return getSMinExpr(Operands);
2137 return getSMaxExpr(Operands);
2138 }
2139
2140 // The cast wasn't folded; create an explicit cast node.
2141 // Recompute the insert position, as it may have been invalidated.
2142 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
2143 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
2144 Op, Ty);
2145 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
2146 registerUser(User: S, Ops: { Op });
2147 return S;
2148}
2149
2150const SCEV *ScalarEvolution::getCastExpr(SCEVTypes Kind, const SCEV *Op,
2151 Type *Ty) {
2152 switch (Kind) {
2153 case scTruncate:
2154 return getTruncateExpr(Op, Ty);
2155 case scZeroExtend:
2156 return getZeroExtendExpr(Op, Ty);
2157 case scSignExtend:
2158 return getSignExtendExpr(Op, Ty);
2159 case scPtrToInt:
2160 return getPtrToIntExpr(Op, Ty);
2161 default:
2162 llvm_unreachable("Not a SCEV cast expression!");
2163 }
2164}
2165
2166/// getAnyExtendExpr - Return a SCEV for the given operand extended with
2167/// unspecified bits out to the given type.
2168const SCEV *ScalarEvolution::getAnyExtendExpr(const SCEV *Op,
2169 Type *Ty) {
2170 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2171 "This is not an extending conversion!");
2172 assert(isSCEVable(Ty) &&
2173 "This is not a conversion to a SCEVable type!");
2174 Ty = getEffectiveSCEVType(Ty);
2175
2176 // Sign-extend negative constants.
2177 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Op))
2178 if (SC->getAPInt().isNegative())
2179 return getSignExtendExpr(Op, Ty);
2180
2181 // Peel off a truncate cast.
2182 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2183 const SCEV *NewOp = T->getOperand();
2184 if (getTypeSizeInBits(Ty: NewOp->getType()) < getTypeSizeInBits(Ty))
2185 return getAnyExtendExpr(Op: NewOp, Ty);
2186 return getTruncateOrNoop(V: NewOp, Ty);
2187 }
2188
2189 // Next try a zext cast. If the cast is folded, use it.
2190 const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
2191 if (!isa<SCEVZeroExtendExpr>(Val: ZExt))
2192 return ZExt;
2193
2194 // Next try a sext cast. If the cast is folded, use it.
2195 const SCEV *SExt = getSignExtendExpr(Op, Ty);
2196 if (!isa<SCEVSignExtendExpr>(Val: SExt))
2197 return SExt;
2198
2199 // Force the cast to be folded into the operands of an addrec.
2200 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Op)) {
2201 SmallVector<const SCEV *, 4> Ops;
2202 for (const SCEV *Op : AR->operands())
2203 Ops.push_back(Elt: getAnyExtendExpr(Op, Ty));
2204 return getAddRecExpr(Operands&: Ops, L: AR->getLoop(), Flags: SCEV::FlagNW);
2205 }
2206
2207 // If the expression is obviously signed, use the sext cast value.
2208 if (isa<SCEVSMaxExpr>(Val: Op))
2209 return SExt;
2210
2211 // Absent any other information, use the zext cast value.
2212 return ZExt;
2213}
2214
2215/// Process the given Ops list, which is a list of operands to be added under
2216/// the given scale, update the given map. This is a helper function for
2217/// getAddRecExpr. As an example of what it does, given a sequence of operands
2218/// that would form an add expression like this:
2219///
2220/// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
2221///
2222/// where A and B are constants, update the map with these values:
2223///
2224/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
2225///
2226/// and add 13 + A*B*29 to AccumulatedConstant.
2227/// This will allow getAddRecExpr to produce this:
2228///
2229/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
2230///
2231/// This form often exposes folding opportunities that are hidden in
2232/// the original operand list.
2233///
2234/// Return true iff it appears that any interesting folding opportunities
2235/// may be exposed. This helps getAddRecExpr short-circuit extra work in
2236/// the common case where no interesting opportunities are present, and
2237/// is also used as a check to avoid infinite recursion.
2238static bool
2239CollectAddOperandsWithScales(SmallDenseMap<const SCEV *, APInt, 16> &M,
2240 SmallVectorImpl<const SCEV *> &NewOps,
2241 APInt &AccumulatedConstant,
2242 ArrayRef<const SCEV *> Ops, const APInt &Scale,
2243 ScalarEvolution &SE) {
2244 bool Interesting = false;
2245
2246 // Iterate over the add operands. They are sorted, with constants first.
2247 unsigned i = 0;
2248 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Ops[i])) {
2249 ++i;
2250 // Pull a buried constant out to the outside.
2251 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
2252 Interesting = true;
2253 AccumulatedConstant += Scale * C->getAPInt();
2254 }
2255
2256 // Next comes everything else. We're especially interested in multiplies
2257 // here, but they're in the middle, so just visit the rest with one loop.
2258 for (; i != Ops.size(); ++i) {
2259 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: Ops[i]);
2260 if (Mul && isa<SCEVConstant>(Val: Mul->getOperand(i: 0))) {
2261 APInt NewScale =
2262 Scale * cast<SCEVConstant>(Val: Mul->getOperand(i: 0))->getAPInt();
2263 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Val: Mul->getOperand(i: 1))) {
2264 // A multiplication of a constant with another add; recurse.
2265 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: Mul->getOperand(i: 1));
2266 Interesting |=
2267 CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2268 Ops: Add->operands(), Scale: NewScale, SE);
2269 } else {
2270 // A multiplication of a constant with some other value. Update
2271 // the map.
2272 SmallVector<const SCEV *, 4> MulOps(drop_begin(RangeOrContainer: Mul->operands()));
2273 const SCEV *Key = SE.getMulExpr(Ops&: MulOps);
2274 auto Pair = M.insert(KV: {Key, NewScale});
2275 if (Pair.second) {
2276 NewOps.push_back(Elt: Pair.first->first);
2277 } else {
2278 Pair.first->second += NewScale;
2279 // The map already had an entry for this value, which may indicate
2280 // a folding opportunity.
2281 Interesting = true;
2282 }
2283 }
2284 } else {
2285 // An ordinary operand. Update the map.
2286 std::pair<DenseMap<const SCEV *, APInt>::iterator, bool> Pair =
2287 M.insert(KV: {Ops[i], Scale});
2288 if (Pair.second) {
2289 NewOps.push_back(Elt: Pair.first->first);
2290 } else {
2291 Pair.first->second += Scale;
2292 // The map already had an entry for this value, which may indicate
2293 // a folding opportunity.
2294 Interesting = true;
2295 }
2296 }
2297 }
2298
2299 return Interesting;
2300}
2301
2302bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed,
2303 const SCEV *LHS, const SCEV *RHS,
2304 const Instruction *CtxI) {
2305 const SCEV *(ScalarEvolution::*Operation)(const SCEV *, const SCEV *,
2306 SCEV::NoWrapFlags, unsigned);
2307 switch (BinOp) {
2308 default:
2309 llvm_unreachable("Unsupported binary op");
2310 case Instruction::Add:
2311 Operation = &ScalarEvolution::getAddExpr;
2312 break;
2313 case Instruction::Sub:
2314 Operation = &ScalarEvolution::getMinusSCEV;
2315 break;
2316 case Instruction::Mul:
2317 Operation = &ScalarEvolution::getMulExpr;
2318 break;
2319 }
2320
2321 const SCEV *(ScalarEvolution::*Extension)(const SCEV *, Type *, unsigned) =
2322 Signed ? &ScalarEvolution::getSignExtendExpr
2323 : &ScalarEvolution::getZeroExtendExpr;
2324
2325 // Check ext(LHS op RHS) == ext(LHS) op ext(RHS)
2326 auto *NarrowTy = cast<IntegerType>(Val: LHS->getType());
2327 auto *WideTy =
2328 IntegerType::get(C&: NarrowTy->getContext(), NumBits: NarrowTy->getBitWidth() * 2);
2329
2330 const SCEV *A = (this->*Extension)(
2331 (this->*Operation)(LHS, RHS, SCEV::FlagAnyWrap, 0), WideTy, 0);
2332 const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0);
2333 const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0);
2334 const SCEV *B = (this->*Operation)(LHSB, RHSB, SCEV::FlagAnyWrap, 0);
2335 if (A == B)
2336 return true;
2337 // Can we use context to prove the fact we need?
2338 if (!CtxI)
2339 return false;
2340 // TODO: Support mul.
2341 if (BinOp == Instruction::Mul)
2342 return false;
2343 auto *RHSC = dyn_cast<SCEVConstant>(Val: RHS);
2344 // TODO: Lift this limitation.
2345 if (!RHSC)
2346 return false;
2347 APInt C = RHSC->getAPInt();
2348 unsigned NumBits = C.getBitWidth();
2349 bool IsSub = (BinOp == Instruction::Sub);
2350 bool IsNegativeConst = (Signed && C.isNegative());
2351 // Compute the direction and magnitude by which we need to check overflow.
2352 bool OverflowDown = IsSub ^ IsNegativeConst;
2353 APInt Magnitude = C;
2354 if (IsNegativeConst) {
2355 if (C == APInt::getSignedMinValue(numBits: NumBits))
2356 // TODO: SINT_MIN on inversion gives the same negative value, we don't
2357 // want to deal with that.
2358 return false;
2359 Magnitude = -C;
2360 }
2361
2362 ICmpInst::Predicate Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2363 if (OverflowDown) {
2364 // To avoid overflow down, we need to make sure that MIN + Magnitude <= LHS.
2365 APInt Min = Signed ? APInt::getSignedMinValue(numBits: NumBits)
2366 : APInt::getMinValue(numBits: NumBits);
2367 APInt Limit = Min + Magnitude;
2368 return isKnownPredicateAt(Pred, LHS: getConstant(Val: Limit), RHS: LHS, CtxI);
2369 } else {
2370 // To avoid overflow up, we need to make sure that LHS <= MAX - Magnitude.
2371 APInt Max = Signed ? APInt::getSignedMaxValue(numBits: NumBits)
2372 : APInt::getMaxValue(numBits: NumBits);
2373 APInt Limit = Max - Magnitude;
2374 return isKnownPredicateAt(Pred, LHS, RHS: getConstant(Val: Limit), CtxI);
2375 }
2376}
2377
2378std::optional<SCEV::NoWrapFlags>
2379ScalarEvolution::getStrengthenedNoWrapFlagsFromBinOp(
2380 const OverflowingBinaryOperator *OBO) {
2381 // It cannot be done any better.
2382 if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap())
2383 return std::nullopt;
2384
2385 SCEV::NoWrapFlags Flags = SCEV::NoWrapFlags::FlagAnyWrap;
2386
2387 if (OBO->hasNoUnsignedWrap())
2388 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2389 if (OBO->hasNoSignedWrap())
2390 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2391
2392 bool Deduced = false;
2393
2394 if (OBO->getOpcode() != Instruction::Add &&
2395 OBO->getOpcode() != Instruction::Sub &&
2396 OBO->getOpcode() != Instruction::Mul)
2397 return std::nullopt;
2398
2399 const SCEV *LHS = getSCEV(V: OBO->getOperand(i_nocapture: 0));
2400 const SCEV *RHS = getSCEV(V: OBO->getOperand(i_nocapture: 1));
2401
2402 const Instruction *CtxI =
2403 UseContextForNoWrapFlagInference ? dyn_cast<Instruction>(Val: OBO) : nullptr;
2404 if (!OBO->hasNoUnsignedWrap() &&
2405 willNotOverflow(BinOp: (Instruction::BinaryOps)OBO->getOpcode(),
2406 /* Signed */ false, LHS, RHS, CtxI)) {
2407 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2408 Deduced = true;
2409 }
2410
2411 if (!OBO->hasNoSignedWrap() &&
2412 willNotOverflow(BinOp: (Instruction::BinaryOps)OBO->getOpcode(),
2413 /* Signed */ true, LHS, RHS, CtxI)) {
2414 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2415 Deduced = true;
2416 }
2417
2418 if (Deduced)
2419 return Flags;
2420 return std::nullopt;
2421}
2422
2423// We're trying to construct a SCEV of type `Type' with `Ops' as operands and
2424// `OldFlags' as can't-wrap behavior. Infer a more aggressive set of
2425// can't-overflow flags for the operation if possible.
2426static SCEV::NoWrapFlags
2427StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type,
2428 const ArrayRef<const SCEV *> Ops,
2429 SCEV::NoWrapFlags Flags) {
2430 using namespace std::placeholders;
2431
2432 using OBO = OverflowingBinaryOperator;
2433
2434 bool CanAnalyze =
2435 Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr;
2436 (void)CanAnalyze;
2437 assert(CanAnalyze && "don't call from other places!");
2438
2439 int SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
2440 SCEV::NoWrapFlags SignOrUnsignWrap =
2441 ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2442
2443 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
2444 auto IsKnownNonNegative = [&](const SCEV *S) {
2445 return SE->isKnownNonNegative(S);
2446 };
2447
2448 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Range: Ops, P: IsKnownNonNegative))
2449 Flags =
2450 ScalarEvolution::setFlags(Flags, OnFlags: (SCEV::NoWrapFlags)SignOrUnsignMask);
2451
2452 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2453
2454 if (SignOrUnsignWrap != SignOrUnsignMask &&
2455 (Type == scAddExpr || Type == scMulExpr) && Ops.size() == 2 &&
2456 isa<SCEVConstant>(Val: Ops[0])) {
2457
2458 auto Opcode = [&] {
2459 switch (Type) {
2460 case scAddExpr:
2461 return Instruction::Add;
2462 case scMulExpr:
2463 return Instruction::Mul;
2464 default:
2465 llvm_unreachable("Unexpected SCEV op.");
2466 }
2467 }();
2468
2469 const APInt &C = cast<SCEVConstant>(Val: Ops[0])->getAPInt();
2470
2471 // (A <opcode> C) --> (A <opcode> C)<nsw> if the op doesn't sign overflow.
2472 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) {
2473 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2474 BinOp: Opcode, Other: C, NoWrapKind: OBO::NoSignedWrap);
2475 if (NSWRegion.contains(CR: SE->getSignedRange(S: Ops[1])))
2476 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2477 }
2478
2479 // (A <opcode> C) --> (A <opcode> C)<nuw> if the op doesn't unsign overflow.
2480 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) {
2481 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
2482 BinOp: Opcode, Other: C, NoWrapKind: OBO::NoUnsignedWrap);
2483 if (NUWRegion.contains(CR: SE->getUnsignedRange(S: Ops[1])))
2484 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2485 }
2486 }
2487
2488 // <0,+,nonnegative><nw> is also nuw
2489 // TODO: Add corresponding nsw case
2490 if (Type == scAddRecExpr && ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNW) &&
2491 !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) && Ops.size() == 2 &&
2492 Ops[0]->isZero() && IsKnownNonNegative(Ops[1]))
2493 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2494
2495 // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW
2496 if (Type == scMulExpr && !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) &&
2497 Ops.size() == 2) {
2498 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[0]))
2499 if (UDiv->getOperand(i: 1) == Ops[1])
2500 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2501 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[1]))
2502 if (UDiv->getOperand(i: 1) == Ops[0])
2503 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2504 }
2505
2506 return Flags;
2507}
2508
2509bool ScalarEvolution::isAvailableAtLoopEntry(const SCEV *S, const Loop *L) {
2510 return isLoopInvariant(S, L) && properlyDominates(S, BB: L->getHeader());
2511}
2512
2513/// Get a canonical add expression, or something simpler if possible.
2514const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl<const SCEV *> &Ops,
2515 SCEV::NoWrapFlags OrigFlags,
2516 unsigned Depth) {
2517 assert(!(OrigFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2518 "only nuw or nsw allowed");
2519 assert(!Ops.empty() && "Cannot get empty add!");
2520 if (Ops.size() == 1) return Ops[0];
2521#ifndef NDEBUG
2522 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
2523 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
2524 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
2525 "SCEVAddExpr operand types don't match!");
2526 unsigned NumPtrs = count_if(
2527 Ops, [](const SCEV *Op) { return Op->getType()->isPointerTy(); });
2528 assert(NumPtrs <= 1 && "add has at most one pointer operand");
2529#endif
2530
2531 const SCEV *Folded = constantFoldAndGroupOps(
2532 SE&: *this, LI, DT, Ops,
2533 Fold: [](const APInt &C1, const APInt &C2) { return C1 + C2; },
2534 IsIdentity: [](const APInt &C) { return C.isZero(); }, // identity
2535 IsAbsorber: [](const APInt &C) { return false; }); // absorber
2536 if (Folded)
2537 return Folded;
2538
2539 unsigned Idx = isa<SCEVConstant>(Val: Ops[0]) ? 1 : 0;
2540
2541 // Delay expensive flag strengthening until necessary.
2542 auto ComputeFlags = [this, OrigFlags](const ArrayRef<const SCEV *> Ops) {
2543 return StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops, Flags: OrigFlags);
2544 };
2545
2546 // Limit recursion calls depth.
2547 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
2548 return getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops));
2549
2550 if (SCEV *S = findExistingSCEVInCache(SCEVType: scAddExpr, Ops)) {
2551 // Don't strengthen flags if we have no new information.
2552 SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S);
2553 if (Add->getNoWrapFlags(Mask: OrigFlags) != OrigFlags)
2554 Add->setNoWrapFlags(ComputeFlags(Ops));
2555 return S;
2556 }
2557
2558 // Okay, check to see if the same value occurs in the operand list more than
2559 // once. If so, merge them together into an multiply expression. Since we
2560 // sorted the list, these values are required to be adjacent.
2561 Type *Ty = Ops[0]->getType();
2562 bool FoundMatch = false;
2563 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
2564 if (Ops[i] == Ops[i+1]) { // X + Y + Y --> X + Y*2
2565 // Scan ahead to count how many equal operands there are.
2566 unsigned Count = 2;
2567 while (i+Count != e && Ops[i+Count] == Ops[i])
2568 ++Count;
2569 // Merge the values into a multiply.
2570 const SCEV *Scale = getConstant(Ty, V: Count);
2571 const SCEV *Mul = getMulExpr(LHS: Scale, RHS: Ops[i], Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2572 if (Ops.size() == Count)
2573 return Mul;
2574 Ops[i] = Mul;
2575 Ops.erase(CS: Ops.begin()+i+1, CE: Ops.begin()+i+Count);
2576 --i; e -= Count - 1;
2577 FoundMatch = true;
2578 }
2579 if (FoundMatch)
2580 return getAddExpr(Ops, OrigFlags, Depth: Depth + 1);
2581
2582 // Check for truncates. If all the operands are truncated from the same
2583 // type, see if factoring out the truncate would permit the result to be
2584 // folded. eg., n*trunc(x) + m*trunc(y) --> trunc(trunc(m)*x + trunc(n)*y)
2585 // if the contents of the resulting outer trunc fold to something simple.
2586 auto FindTruncSrcType = [&]() -> Type * {
2587 // We're ultimately looking to fold an addrec of truncs and muls of only
2588 // constants and truncs, so if we find any other types of SCEV
2589 // as operands of the addrec then we bail and return nullptr here.
2590 // Otherwise, we return the type of the operand of a trunc that we find.
2591 if (auto *T = dyn_cast<SCEVTruncateExpr>(Val: Ops[Idx]))
2592 return T->getOperand()->getType();
2593 if (const auto *Mul = dyn_cast<SCEVMulExpr>(Val: Ops[Idx])) {
2594 const auto *LastOp = Mul->getOperand(i: Mul->getNumOperands() - 1);
2595 if (const auto *T = dyn_cast<SCEVTruncateExpr>(Val: LastOp))
2596 return T->getOperand()->getType();
2597 }
2598 return nullptr;
2599 };
2600 if (auto *SrcType = FindTruncSrcType()) {
2601 SmallVector<const SCEV *, 8> LargeOps;
2602 bool Ok = true;
2603 // Check all the operands to see if they can be represented in the
2604 // source type of the truncate.
2605 for (const SCEV *Op : Ops) {
2606 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2607 if (T->getOperand()->getType() != SrcType) {
2608 Ok = false;
2609 break;
2610 }
2611 LargeOps.push_back(Elt: T->getOperand());
2612 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Op)) {
2613 LargeOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2614 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val: Op)) {
2615 SmallVector<const SCEV *, 8> LargeMulOps;
2616 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2617 if (const SCEVTruncateExpr *T =
2618 dyn_cast<SCEVTruncateExpr>(Val: M->getOperand(i: j))) {
2619 if (T->getOperand()->getType() != SrcType) {
2620 Ok = false;
2621 break;
2622 }
2623 LargeMulOps.push_back(Elt: T->getOperand());
2624 } else if (const auto *C = dyn_cast<SCEVConstant>(Val: M->getOperand(i: j))) {
2625 LargeMulOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2626 } else {
2627 Ok = false;
2628 break;
2629 }
2630 }
2631 if (Ok)
2632 LargeOps.push_back(Elt: getMulExpr(Ops&: LargeMulOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2633 } else {
2634 Ok = false;
2635 break;
2636 }
2637 }
2638 if (Ok) {
2639 // Evaluate the expression in the larger type.
2640 const SCEV *Fold = getAddExpr(Ops&: LargeOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2641 // If it folds to something simple, use it. Otherwise, don't.
2642 if (isa<SCEVConstant>(Val: Fold) || isa<SCEVUnknown>(Val: Fold))
2643 return getTruncateExpr(Op: Fold, Ty);
2644 }
2645 }
2646
2647 if (Ops.size() == 2) {
2648 // Check if we have an expression of the form ((X + C1) - C2), where C1 and
2649 // C2 can be folded in a way that allows retaining wrapping flags of (X +
2650 // C1).
2651 const SCEV *A = Ops[0];
2652 const SCEV *B = Ops[1];
2653 auto *AddExpr = dyn_cast<SCEVAddExpr>(Val: B);
2654 auto *C = dyn_cast<SCEVConstant>(Val: A);
2655 if (AddExpr && C && isa<SCEVConstant>(Val: AddExpr->getOperand(i: 0))) {
2656 auto C1 = cast<SCEVConstant>(Val: AddExpr->getOperand(i: 0))->getAPInt();
2657 auto C2 = C->getAPInt();
2658 SCEV::NoWrapFlags PreservedFlags = SCEV::FlagAnyWrap;
2659
2660 APInt ConstAdd = C1 + C2;
2661 auto AddFlags = AddExpr->getNoWrapFlags();
2662 // Adding a smaller constant is NUW if the original AddExpr was NUW.
2663 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNUW) &&
2664 ConstAdd.ule(RHS: C1)) {
2665 PreservedFlags =
2666 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNUW);
2667 }
2668
2669 // Adding a constant with the same sign and small magnitude is NSW, if the
2670 // original AddExpr was NSW.
2671 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNSW) &&
2672 C1.isSignBitSet() == ConstAdd.isSignBitSet() &&
2673 ConstAdd.abs().ule(RHS: C1.abs())) {
2674 PreservedFlags =
2675 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNSW);
2676 }
2677
2678 if (PreservedFlags != SCEV::FlagAnyWrap) {
2679 SmallVector<const SCEV *, 4> NewOps(AddExpr->operands());
2680 NewOps[0] = getConstant(Val: ConstAdd);
2681 return getAddExpr(Ops&: NewOps, OrigFlags: PreservedFlags);
2682 }
2683 }
2684 }
2685
2686 // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
2687 if (Ops.size() == 2) {
2688 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: Ops[0]);
2689 if (Mul && Mul->getNumOperands() == 2 &&
2690 Mul->getOperand(i: 0)->isAllOnesValue()) {
2691 const SCEV *X;
2692 const SCEV *Y;
2693 if (matchURem(Expr: Mul->getOperand(i: 1), LHS&: X, RHS&: Y) && X == Ops[1]) {
2694 return getMulExpr(LHS: Y, RHS: getUDivExpr(LHS: X, RHS: Y));
2695 }
2696 }
2697 }
2698
2699 // Skip past any other cast SCEVs.
2700 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
2701 ++Idx;
2702
2703 // If there are add operands they would be next.
2704 if (Idx < Ops.size()) {
2705 bool DeletedAdd = false;
2706 // If the original flags and all inlined SCEVAddExprs are NUW, use the
2707 // common NUW flag for expression after inlining. Other flags cannot be
2708 // preserved, because they may depend on the original order of operations.
2709 SCEV::NoWrapFlags CommonFlags = maskFlags(Flags: OrigFlags, Mask: SCEV::FlagNUW);
2710 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: Ops[Idx])) {
2711 if (Ops.size() > AddOpsInlineThreshold ||
2712 Add->getNumOperands() > AddOpsInlineThreshold)
2713 break;
2714 // If we have an add, expand the add operands onto the end of the operands
2715 // list.
2716 Ops.erase(CI: Ops.begin()+Idx);
2717 append_range(C&: Ops, R: Add->operands());
2718 DeletedAdd = true;
2719 CommonFlags = maskFlags(Flags: CommonFlags, Mask: Add->getNoWrapFlags());
2720 }
2721
2722 // If we deleted at least one add, we added operands to the end of the list,
2723 // and they are not necessarily sorted. Recurse to resort and resimplify
2724 // any operands we just acquired.
2725 if (DeletedAdd)
2726 return getAddExpr(Ops, OrigFlags: CommonFlags, Depth: Depth + 1);
2727 }
2728
2729 // Skip over the add expression until we get to a multiply.
2730 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
2731 ++Idx;
2732
2733 // Check to see if there are any folding opportunities present with
2734 // operands multiplied by constant values.
2735 if (Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx])) {
2736 uint64_t BitWidth = getTypeSizeInBits(Ty);
2737 SmallDenseMap<const SCEV *, APInt, 16> M;
2738 SmallVector<const SCEV *, 8> NewOps;
2739 APInt AccumulatedConstant(BitWidth, 0);
2740 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2741 Ops, Scale: APInt(BitWidth, 1), SE&: *this)) {
2742 struct APIntCompare {
2743 bool operator()(const APInt &LHS, const APInt &RHS) const {
2744 return LHS.ult(RHS);
2745 }
2746 };
2747
2748 // Some interesting folding opportunity is present, so its worthwhile to
2749 // re-generate the operands list. Group the operands by constant scale,
2750 // to avoid multiplying by the same constant scale multiple times.
2751 std::map<APInt, SmallVector<const SCEV *, 4>, APIntCompare> MulOpLists;
2752 for (const SCEV *NewOp : NewOps)
2753 MulOpLists[M.find(Val: NewOp)->second].push_back(Elt: NewOp);
2754 // Re-generate the operands list.
2755 Ops.clear();
2756 if (AccumulatedConstant != 0)
2757 Ops.push_back(Elt: getConstant(Val: AccumulatedConstant));
2758 for (auto &MulOp : MulOpLists) {
2759 if (MulOp.first == 1) {
2760 Ops.push_back(Elt: getAddExpr(Ops&: MulOp.second, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2761 } else if (MulOp.first != 0) {
2762 Ops.push_back(Elt: getMulExpr(
2763 LHS: getConstant(Val: MulOp.first),
2764 RHS: getAddExpr(Ops&: MulOp.second, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1),
2765 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
2766 }
2767 }
2768 if (Ops.empty())
2769 return getZero(Ty);
2770 if (Ops.size() == 1)
2771 return Ops[0];
2772 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2773 }
2774 }
2775
2776 // If we are adding something to a multiply expression, make sure the
2777 // something is not already an operand of the multiply. If so, merge it into
2778 // the multiply.
2779 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx]); ++Idx) {
2780 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val: Ops[Idx]);
2781 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
2782 const SCEV *MulOpSCEV = Mul->getOperand(i: MulOp);
2783 if (isa<SCEVConstant>(Val: MulOpSCEV))
2784 continue;
2785 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp)
2786 if (MulOpSCEV == Ops[AddOp]) {
2787 // Fold W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1))
2788 const SCEV *InnerMul = Mul->getOperand(i: MulOp == 0);
2789 if (Mul->getNumOperands() != 2) {
2790 // If the multiply has more than two operands, we must get the
2791 // Y*Z term.
2792 SmallVector<const SCEV *, 4> MulOps(
2793 Mul->operands().take_front(N: MulOp));
2794 append_range(C&: MulOps, R: Mul->operands().drop_front(N: MulOp + 1));
2795 InnerMul = getMulExpr(Ops&: MulOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2796 }
2797 SmallVector<const SCEV *, 2> TwoOps = {getOne(Ty), InnerMul};
2798 const SCEV *AddOne = getAddExpr(Ops&: TwoOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2799 const SCEV *OuterMul = getMulExpr(LHS: AddOne, RHS: MulOpSCEV,
2800 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2801 if (Ops.size() == 2) return OuterMul;
2802 if (AddOp < Idx) {
2803 Ops.erase(CI: Ops.begin()+AddOp);
2804 Ops.erase(CI: Ops.begin()+Idx-1);
2805 } else {
2806 Ops.erase(CI: Ops.begin()+Idx);
2807 Ops.erase(CI: Ops.begin()+AddOp-1);
2808 }
2809 Ops.push_back(Elt: OuterMul);
2810 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2811 }
2812
2813 // Check this multiply against other multiplies being added together.
2814 for (unsigned OtherMulIdx = Idx+1;
2815 OtherMulIdx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[OtherMulIdx]);
2816 ++OtherMulIdx) {
2817 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Val: Ops[OtherMulIdx]);
2818 // If MulOp occurs in OtherMul, we can fold the two multiplies
2819 // together.
2820 for (unsigned OMulOp = 0, e = OtherMul->getNumOperands();
2821 OMulOp != e; ++OMulOp)
2822 if (OtherMul->getOperand(i: OMulOp) == MulOpSCEV) {
2823 // Fold X + (A*B*C) + (A*D*E) --> X + (A*(B*C+D*E))
2824 const SCEV *InnerMul1 = Mul->getOperand(i: MulOp == 0);
2825 if (Mul->getNumOperands() != 2) {
2826 SmallVector<const SCEV *, 4> MulOps(
2827 Mul->operands().take_front(N: MulOp));
2828 append_range(C&: MulOps, R: Mul->operands().drop_front(N: MulOp+1));
2829 InnerMul1 = getMulExpr(Ops&: MulOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2830 }
2831 const SCEV *InnerMul2 = OtherMul->getOperand(i: OMulOp == 0);
2832 if (OtherMul->getNumOperands() != 2) {
2833 SmallVector<const SCEV *, 4> MulOps(
2834 OtherMul->operands().take_front(N: OMulOp));
2835 append_range(C&: MulOps, R: OtherMul->operands().drop_front(N: OMulOp+1));
2836 InnerMul2 = getMulExpr(Ops&: MulOps, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2837 }
2838 SmallVector<const SCEV *, 2> TwoOps = {InnerMul1, InnerMul2};
2839 const SCEV *InnerMulSum =
2840 getAddExpr(Ops&: TwoOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2841 const SCEV *OuterMul = getMulExpr(LHS: MulOpSCEV, RHS: InnerMulSum,
2842 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2843 if (Ops.size() == 2) return OuterMul;
2844 Ops.erase(CI: Ops.begin()+Idx);
2845 Ops.erase(CI: Ops.begin()+OtherMulIdx-1);
2846 Ops.push_back(Elt: OuterMul);
2847 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2848 }
2849 }
2850 }
2851 }
2852
2853 // If there are any add recurrences in the operands list, see if any other
2854 // added values are loop invariant. If so, we can fold them into the
2855 // recurrence.
2856 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
2857 ++Idx;
2858
2859 // Scan over all recurrences, trying to fold loop invariants into them.
2860 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
2861 // Scan all of the other operands to this add and add them to the vector if
2862 // they are loop invariant w.r.t. the recurrence.
2863 SmallVector<const SCEV *, 8> LIOps;
2864 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: Ops[Idx]);
2865 const Loop *AddRecLoop = AddRec->getLoop();
2866 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2867 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRecLoop)) {
2868 LIOps.push_back(Elt: Ops[i]);
2869 Ops.erase(CI: Ops.begin()+i);
2870 --i; --e;
2871 }
2872
2873 // If we found some loop invariants, fold them into the recurrence.
2874 if (!LIOps.empty()) {
2875 // Compute nowrap flags for the addition of the loop-invariant ops and
2876 // the addrec. Temporarily push it as an operand for that purpose. These
2877 // flags are valid in the scope of the addrec only.
2878 LIOps.push_back(Elt: AddRec);
2879 SCEV::NoWrapFlags Flags = ComputeFlags(LIOps);
2880 LIOps.pop_back();
2881
2882 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step}
2883 LIOps.push_back(Elt: AddRec->getStart());
2884
2885 SmallVector<const SCEV *, 4> AddRecOps(AddRec->operands());
2886
2887 // It is not in general safe to propagate flags valid on an add within
2888 // the addrec scope to one outside it. We must prove that the inner
2889 // scope is guaranteed to execute if the outer one does to be able to
2890 // safely propagate. We know the program is undefined if poison is
2891 // produced on the inner scoped addrec. We also know that *for this use*
2892 // the outer scoped add can't overflow (because of the flags we just
2893 // computed for the inner scoped add) without the program being undefined.
2894 // Proving that entry to the outer scope neccesitates entry to the inner
2895 // scope, thus proves the program undefined if the flags would be violated
2896 // in the outer scope.
2897 SCEV::NoWrapFlags AddFlags = Flags;
2898 if (AddFlags != SCEV::FlagAnyWrap) {
2899 auto *DefI = getDefiningScopeBound(Ops: LIOps);
2900 auto *ReachI = &*AddRecLoop->getHeader()->begin();
2901 if (!isGuaranteedToTransferExecutionTo(A: DefI, B: ReachI))
2902 AddFlags = SCEV::FlagAnyWrap;
2903 }
2904 AddRecOps[0] = getAddExpr(Ops&: LIOps, OrigFlags: AddFlags, Depth: Depth + 1);
2905
2906 // Build the new addrec. Propagate the NUW and NSW flags if both the
2907 // outer add and the inner addrec are guaranteed to have no overflow.
2908 // Always propagate NW.
2909 Flags = AddRec->getNoWrapFlags(Mask: setFlags(Flags, OnFlags: SCEV::FlagNW));
2910 const SCEV *NewRec = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags);
2911
2912 // If all of the other operands were loop invariant, we are done.
2913 if (Ops.size() == 1) return NewRec;
2914
2915 // Otherwise, add the folded AddRec by the non-invariant parts.
2916 for (unsigned i = 0;; ++i)
2917 if (Ops[i] == AddRec) {
2918 Ops[i] = NewRec;
2919 break;
2920 }
2921 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2922 }
2923
2924 // Okay, if there weren't any loop invariants to be folded, check to see if
2925 // there are multiple AddRec's with the same loop induction variable being
2926 // added together. If so, we can fold them.
2927 for (unsigned OtherIdx = Idx+1;
2928 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
2929 ++OtherIdx) {
2930 // We expect the AddRecExpr's to be sorted in reverse dominance order,
2931 // so that the 1st found AddRecExpr is dominated by all others.
2932 assert(DT.dominates(
2933 cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(),
2934 AddRec->getLoop()->getHeader()) &&
2935 "AddRecExprs are not sorted in reverse dominance order?");
2936 if (AddRecLoop == cast<SCEVAddRecExpr>(Val: Ops[OtherIdx])->getLoop()) {
2937 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L>
2938 SmallVector<const SCEV *, 4> AddRecOps(AddRec->operands());
2939 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
2940 ++OtherIdx) {
2941 const auto *OtherAddRec = cast<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
2942 if (OtherAddRec->getLoop() == AddRecLoop) {
2943 for (unsigned i = 0, e = OtherAddRec->getNumOperands();
2944 i != e; ++i) {
2945 if (i >= AddRecOps.size()) {
2946 append_range(C&: AddRecOps, R: OtherAddRec->operands().drop_front(N: i));
2947 break;
2948 }
2949 SmallVector<const SCEV *, 2> TwoOps = {
2950 AddRecOps[i], OtherAddRec->getOperand(i)};
2951 AddRecOps[i] = getAddExpr(Ops&: TwoOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2952 }
2953 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
2954 }
2955 }
2956 // Step size has changed, so we cannot guarantee no self-wraparound.
2957 Ops[Idx] = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags: SCEV::FlagAnyWrap);
2958 return getAddExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
2959 }
2960 }
2961
2962 // Otherwise couldn't fold anything into this recurrence. Move onto the
2963 // next one.
2964 }
2965
2966 // Okay, it looks like we really DO need an add expr. Check to see if we
2967 // already have one, otherwise create a new one.
2968 return getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops));
2969}
2970
2971const SCEV *
2972ScalarEvolution::getOrCreateAddExpr(ArrayRef<const SCEV *> Ops,
2973 SCEV::NoWrapFlags Flags) {
2974 FoldingSetNodeID ID;
2975 ID.AddInteger(I: scAddExpr);
2976 for (const SCEV *Op : Ops)
2977 ID.AddPointer(Ptr: Op);
2978 void *IP = nullptr;
2979 SCEVAddExpr *S =
2980 static_cast<SCEVAddExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
2981 if (!S) {
2982 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Num: Ops.size());
2983 llvm::uninitialized_copy(Src&: Ops, Dst: O);
2984 S = new (SCEVAllocator)
2985 SCEVAddExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size());
2986 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
2987 registerUser(User: S, Ops);
2988 }
2989 S->setNoWrapFlags(Flags);
2990 return S;
2991}
2992
2993const SCEV *
2994ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<const SCEV *> Ops,
2995 const Loop *L, SCEV::NoWrapFlags Flags) {
2996 FoldingSetNodeID ID;
2997 ID.AddInteger(I: scAddRecExpr);
2998 for (const SCEV *Op : Ops)
2999 ID.AddPointer(Ptr: Op);
3000 ID.AddPointer(Ptr: L);
3001 void *IP = nullptr;
3002 SCEVAddRecExpr *S =
3003 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
3004 if (!S) {
3005 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Num: Ops.size());
3006 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3007 S = new (SCEVAllocator)
3008 SCEVAddRecExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size(), L);
3009 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3010 LoopUsers[L].push_back(Elt: S);
3011 registerUser(User: S, Ops);
3012 }
3013 setNoWrapFlags(AddRec: S, Flags);
3014 return S;
3015}
3016
3017const SCEV *
3018ScalarEvolution::getOrCreateMulExpr(ArrayRef<const SCEV *> Ops,
3019 SCEV::NoWrapFlags Flags) {
3020 FoldingSetNodeID ID;
3021 ID.AddInteger(I: scMulExpr);
3022 for (const SCEV *Op : Ops)
3023 ID.AddPointer(Ptr: Op);
3024 void *IP = nullptr;
3025 SCEVMulExpr *S =
3026 static_cast<SCEVMulExpr *>(UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP));
3027 if (!S) {
3028 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Num: Ops.size());
3029 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3030 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(Allocator&: SCEVAllocator),
3031 O, Ops.size());
3032 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3033 registerUser(User: S, Ops);
3034 }
3035 S->setNoWrapFlags(Flags);
3036 return S;
3037}
3038
3039static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
3040 uint64_t k = i*j;
3041 if (j > 1 && k / j != i) Overflow = true;
3042 return k;
3043}
3044
3045/// Compute the result of "n choose k", the binomial coefficient. If an
3046/// intermediate computation overflows, Overflow will be set and the return will
3047/// be garbage. Overflow is not cleared on absence of overflow.
3048static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
3049 // We use the multiplicative formula:
3050 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
3051 // At each iteration, we take the n-th term of the numeral and divide by the
3052 // (k-n)th term of the denominator. This division will always produce an
3053 // integral result, and helps reduce the chance of overflow in the
3054 // intermediate computations. However, we can still overflow even when the
3055 // final result would fit.
3056
3057 if (n == 0 || n == k) return 1;
3058 if (k > n) return 0;
3059
3060 if (k > n/2)
3061 k = n-k;
3062
3063 uint64_t r = 1;
3064 for (uint64_t i = 1; i <= k; ++i) {
3065 r = umul_ov(i: r, j: n-(i-1), Overflow);
3066 r /= i;
3067 }
3068 return r;
3069}
3070
3071/// Determine if any of the operands in this SCEV are a constant or if
3072/// any of the add or multiply expressions in this SCEV contain a constant.
3073static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
3074 struct FindConstantInAddMulChain {
3075 bool FoundConstant = false;
3076
3077 bool follow(const SCEV *S) {
3078 FoundConstant |= isa<SCEVConstant>(Val: S);
3079 return isa<SCEVAddExpr>(Val: S) || isa<SCEVMulExpr>(Val: S);
3080 }
3081
3082 bool isDone() const {
3083 return FoundConstant;
3084 }
3085 };
3086
3087 FindConstantInAddMulChain F;
3088 SCEVTraversal<FindConstantInAddMulChain> ST(F);
3089 ST.visitAll(Root: StartExpr);
3090 return F.FoundConstant;
3091}
3092
3093/// Get a canonical multiply expression, or something simpler if possible.
3094const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl<const SCEV *> &Ops,
3095 SCEV::NoWrapFlags OrigFlags,
3096 unsigned Depth) {
3097 assert(OrigFlags == maskFlags(OrigFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3098 "only nuw or nsw allowed");
3099 assert(!Ops.empty() && "Cannot get empty mul!");
3100 if (Ops.size() == 1) return Ops[0];
3101#ifndef NDEBUG
3102 Type *ETy = Ops[0]->getType();
3103 assert(!ETy->isPointerTy());
3104 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
3105 assert(Ops[i]->getType() == ETy &&
3106 "SCEVMulExpr operand types don't match!");
3107#endif
3108
3109 const SCEV *Folded = constantFoldAndGroupOps(
3110 SE&: *this, LI, DT, Ops,
3111 Fold: [](const APInt &C1, const APInt &C2) { return C1 * C2; },
3112 IsIdentity: [](const APInt &C) { return C.isOne(); }, // identity
3113 IsAbsorber: [](const APInt &C) { return C.isZero(); }); // absorber
3114 if (Folded)
3115 return Folded;
3116
3117 // Delay expensive flag strengthening until necessary.
3118 auto ComputeFlags = [this, OrigFlags](const ArrayRef<const SCEV *> Ops) {
3119 return StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops, Flags: OrigFlags);
3120 };
3121
3122 // Limit recursion calls depth.
3123 if (Depth > MaxArithDepth || hasHugeExpression(Ops))
3124 return getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops));
3125
3126 if (SCEV *S = findExistingSCEVInCache(SCEVType: scMulExpr, Ops)) {
3127 // Don't strengthen flags if we have no new information.
3128 SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
3129 if (Mul->getNoWrapFlags(Mask: OrigFlags) != OrigFlags)
3130 Mul->setNoWrapFlags(ComputeFlags(Ops));
3131 return S;
3132 }
3133
3134 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val: Ops[0])) {
3135 if (Ops.size() == 2) {
3136 // C1*(C2+V) -> C1*C2 + C1*V
3137 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: Ops[1]))
3138 // If any of Add's ops are Adds or Muls with a constant, apply this
3139 // transformation as well.
3140 //
3141 // TODO: There are some cases where this transformation is not
3142 // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of
3143 // this transformation should be narrowed down.
3144 if (Add->getNumOperands() == 2 && containsConstantInAddMulChain(StartExpr: Add)) {
3145 const SCEV *LHS = getMulExpr(LHS: LHSC, RHS: Add->getOperand(i: 0),
3146 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3147 const SCEV *RHS = getMulExpr(LHS: LHSC, RHS: Add->getOperand(i: 1),
3148 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3149 return getAddExpr(LHS, RHS, Flags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3150 }
3151
3152 if (Ops[0]->isAllOnesValue()) {
3153 // If we have a mul by -1 of an add, try distributing the -1 among the
3154 // add operands.
3155 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: Ops[1])) {
3156 SmallVector<const SCEV *, 4> NewOps;
3157 bool AnyFolded = false;
3158 for (const SCEV *AddOp : Add->operands()) {
3159 const SCEV *Mul = getMulExpr(LHS: Ops[0], RHS: AddOp, Flags: SCEV::FlagAnyWrap,
3160 Depth: Depth + 1);
3161 if (!isa<SCEVMulExpr>(Val: Mul)) AnyFolded = true;
3162 NewOps.push_back(Elt: Mul);
3163 }
3164 if (AnyFolded)
3165 return getAddExpr(Ops&: NewOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3166 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: Ops[1])) {
3167 // Negation preserves a recurrence's no self-wrap property.
3168 SmallVector<const SCEV *, 4> Operands;
3169 for (const SCEV *AddRecOp : AddRec->operands())
3170 Operands.push_back(Elt: getMulExpr(LHS: Ops[0], RHS: AddRecOp, Flags: SCEV::FlagAnyWrap,
3171 Depth: Depth + 1));
3172 // Let M be the minimum representable signed value. AddRec with nsw
3173 // multiplied by -1 can have signed overflow if and only if it takes a
3174 // value of M: M * (-1) would stay M and (M + 1) * (-1) would be the
3175 // maximum signed value. In all other cases signed overflow is
3176 // impossible.
3177 auto FlagsMask = SCEV::FlagNW;
3178 if (hasFlags(Flags: AddRec->getNoWrapFlags(), TestFlags: SCEV::FlagNSW)) {
3179 auto MinInt =
3180 APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: AddRec->getType()));
3181 if (getSignedRangeMin(S: AddRec) != MinInt)
3182 FlagsMask = setFlags(Flags: FlagsMask, OnFlags: SCEV::FlagNSW);
3183 }
3184 return getAddRecExpr(Operands, L: AddRec->getLoop(),
3185 Flags: AddRec->getNoWrapFlags(Mask: FlagsMask));
3186 }
3187 }
3188 }
3189 }
3190
3191 // Skip over the add expression until we get to a multiply.
3192 unsigned Idx = 0;
3193 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
3194 ++Idx;
3195
3196 // If there are mul operands inline them all into this expression.
3197 if (Idx < Ops.size()) {
3198 bool DeletedMul = false;
3199 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: Ops[Idx])) {
3200 if (Ops.size() > MulOpsInlineThreshold)
3201 break;
3202 // If we have an mul, expand the mul operands onto the end of the
3203 // operands list.
3204 Ops.erase(CI: Ops.begin()+Idx);
3205 append_range(C&: Ops, R: Mul->operands());
3206 DeletedMul = true;
3207 }
3208
3209 // If we deleted at least one mul, we added operands to the end of the
3210 // list, and they are not necessarily sorted. Recurse to resort and
3211 // resimplify any operands we just acquired.
3212 if (DeletedMul)
3213 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3214 }
3215
3216 // If there are any add recurrences in the operands list, see if any other
3217 // added values are loop invariant. If so, we can fold them into the
3218 // recurrence.
3219 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
3220 ++Idx;
3221
3222 // Scan over all recurrences, trying to fold loop invariants into them.
3223 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
3224 // Scan all of the other operands to this mul and add them to the vector
3225 // if they are loop invariant w.r.t. the recurrence.
3226 SmallVector<const SCEV *, 8> LIOps;
3227 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: Ops[Idx]);
3228 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3229 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRec->getLoop())) {
3230 LIOps.push_back(Elt: Ops[i]);
3231 Ops.erase(CI: Ops.begin()+i);
3232 --i; --e;
3233 }
3234
3235 // If we found some loop invariants, fold them into the recurrence.
3236 if (!LIOps.empty()) {
3237 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step}
3238 SmallVector<const SCEV *, 4> NewOps;
3239 NewOps.reserve(N: AddRec->getNumOperands());
3240 const SCEV *Scale = getMulExpr(Ops&: LIOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3241
3242 // If both the mul and addrec are nuw, we can preserve nuw.
3243 // If both the mul and addrec are nsw, we can only preserve nsw if either
3244 // a) they are also nuw, or
3245 // b) all multiplications of addrec operands with scale are nsw.
3246 SCEV::NoWrapFlags Flags =
3247 AddRec->getNoWrapFlags(Mask: ComputeFlags({Scale, AddRec}));
3248
3249 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
3250 NewOps.push_back(Elt: getMulExpr(LHS: Scale, RHS: AddRec->getOperand(i),
3251 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3252
3253 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW) && !hasFlags(Flags, TestFlags: SCEV::FlagNUW)) {
3254 ConstantRange NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
3255 BinOp: Instruction::Mul, Other: getSignedRange(S: Scale),
3256 NoWrapKind: OverflowingBinaryOperator::NoSignedWrap);
3257 if (!NSWRegion.contains(CR: getSignedRange(S: AddRec->getOperand(i))))
3258 Flags = clearFlags(Flags, OffFlags: SCEV::FlagNSW);
3259 }
3260 }
3261
3262 const SCEV *NewRec = getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), Flags);
3263
3264 // If all of the other operands were loop invariant, we are done.
3265 if (Ops.size() == 1) return NewRec;
3266
3267 // Otherwise, multiply the folded AddRec by the non-invariant parts.
3268 for (unsigned i = 0;; ++i)
3269 if (Ops[i] == AddRec) {
3270 Ops[i] = NewRec;
3271 break;
3272 }
3273 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3274 }
3275
3276 // Okay, if there weren't any loop invariants to be folded, check to see
3277 // if there are multiple AddRec's with the same loop induction variable
3278 // being multiplied together. If so, we can fold them.
3279
3280 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
3281 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
3282 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
3283 // ]]],+,...up to x=2n}.
3284 // Note that the arguments to choose() are always integers with values
3285 // known at compile time, never SCEV objects.
3286 //
3287 // The implementation avoids pointless extra computations when the two
3288 // addrec's are of different length (mathematically, it's equivalent to
3289 // an infinite stream of zeros on the right).
3290 bool OpsModified = false;
3291 for (unsigned OtherIdx = Idx+1;
3292 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3293 ++OtherIdx) {
3294 const SCEVAddRecExpr *OtherAddRec =
3295 dyn_cast<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3296 if (!OtherAddRec || OtherAddRec->getLoop() != AddRec->getLoop())
3297 continue;
3298
3299 // Limit max number of arguments to avoid creation of unreasonably big
3300 // SCEVAddRecs with very complex operands.
3301 if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
3302 MaxAddRecSize || hasHugeExpression(Ops: {AddRec, OtherAddRec}))
3303 continue;
3304
3305 bool Overflow = false;
3306 Type *Ty = AddRec->getType();
3307 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
3308 SmallVector<const SCEV*, 7> AddRecOps;
3309 for (int x = 0, xe = AddRec->getNumOperands() +
3310 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
3311 SmallVector <const SCEV *, 7> SumOps;
3312 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
3313 uint64_t Coeff1 = Choose(n: x, k: 2*x - y, Overflow);
3314 for (int z = std::max(a: y-x, b: y-(int)AddRec->getNumOperands()+1),
3315 ze = std::min(a: x+1, b: (int)OtherAddRec->getNumOperands());
3316 z < ze && !Overflow; ++z) {
3317 uint64_t Coeff2 = Choose(n: 2*x - y, k: x-z, Overflow);
3318 uint64_t Coeff;
3319 if (LargerThan64Bits)
3320 Coeff = umul_ov(i: Coeff1, j: Coeff2, Overflow);
3321 else
3322 Coeff = Coeff1*Coeff2;
3323 const SCEV *CoeffTerm = getConstant(Ty, V: Coeff);
3324 const SCEV *Term1 = AddRec->getOperand(i: y-z);
3325 const SCEV *Term2 = OtherAddRec->getOperand(i: z);
3326 SumOps.push_back(Elt: getMulExpr(Op0: CoeffTerm, Op1: Term1, Op2: Term2,
3327 Flags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3328 }
3329 }
3330 if (SumOps.empty())
3331 SumOps.push_back(Elt: getZero(Ty));
3332 AddRecOps.push_back(Elt: getAddExpr(Ops&: SumOps, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1));
3333 }
3334 if (!Overflow) {
3335 const SCEV *NewAddRec = getAddRecExpr(Operands&: AddRecOps, L: AddRec->getLoop(),
3336 Flags: SCEV::FlagAnyWrap);
3337 if (Ops.size() == 2) return NewAddRec;
3338 Ops[Idx] = NewAddRec;
3339 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
3340 OpsModified = true;
3341 AddRec = dyn_cast<SCEVAddRecExpr>(Val: NewAddRec);
3342 if (!AddRec)
3343 break;
3344 }
3345 }
3346 if (OpsModified)
3347 return getMulExpr(Ops, OrigFlags: SCEV::FlagAnyWrap, Depth: Depth + 1);
3348
3349 // Otherwise couldn't fold anything into this recurrence. Move onto the
3350 // next one.
3351 }
3352
3353 // Okay, it looks like we really DO need an mul expr. Check to see if we
3354 // already have one, otherwise create a new one.
3355 return getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops));
3356}
3357
3358/// Represents an unsigned remainder expression based on unsigned division.
3359const SCEV *ScalarEvolution::getURemExpr(const SCEV *LHS,
3360 const SCEV *RHS) {
3361 assert(getEffectiveSCEVType(LHS->getType()) ==
3362 getEffectiveSCEVType(RHS->getType()) &&
3363 "SCEVURemExpr operand types don't match!");
3364
3365 // Short-circuit easy cases
3366 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val: RHS)) {
3367 // If constant is one, the result is trivial
3368 if (RHSC->getValue()->isOne())
3369 return getZero(Ty: LHS->getType()); // X urem 1 --> 0
3370
3371 // If constant is a power of two, fold into a zext(trunc(LHS)).
3372 if (RHSC->getAPInt().isPowerOf2()) {
3373 Type *FullTy = LHS->getType();
3374 Type *TruncTy =
3375 IntegerType::get(C&: getContext(), NumBits: RHSC->getAPInt().logBase2());
3376 return getZeroExtendExpr(Op: getTruncateExpr(Op: LHS, Ty: TruncTy), Ty: FullTy);
3377 }
3378 }
3379
3380 // Fallback to %a == %x urem %y == %x -<nuw> ((%x udiv %y) *<nuw> %y)
3381 const SCEV *UDiv = getUDivExpr(LHS, RHS);
3382 const SCEV *Mult = getMulExpr(LHS: UDiv, RHS, Flags: SCEV::FlagNUW);
3383 return getMinusSCEV(LHS, RHS: Mult, Flags: SCEV::FlagNUW);
3384}
3385
3386/// Get a canonical unsigned division expression, or something simpler if
3387/// possible.
3388const SCEV *ScalarEvolution::getUDivExpr(const SCEV *LHS,
3389 const SCEV *RHS) {
3390 assert(!LHS->getType()->isPointerTy() &&
3391 "SCEVUDivExpr operand can't be pointer!");
3392 assert(LHS->getType() == RHS->getType() &&
3393 "SCEVUDivExpr operand types don't match!");
3394
3395 FoldingSetNodeID ID;
3396 ID.AddInteger(I: scUDivExpr);
3397 ID.AddPointer(Ptr: LHS);
3398 ID.AddPointer(Ptr: RHS);
3399 void *IP = nullptr;
3400 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
3401 return S;
3402
3403 // 0 udiv Y == 0
3404 if (match(S: LHS, P: m_scev_Zero()))
3405 return LHS;
3406
3407 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val: RHS)) {
3408 if (RHSC->getValue()->isOne())
3409 return LHS; // X udiv 1 --> x
3410 // If the denominator is zero, the result of the udiv is undefined. Don't
3411 // try to analyze it, because the resolution chosen here may differ from
3412 // the resolution chosen in other parts of the compiler.
3413 if (!RHSC->getValue()->isZero()) {
3414 // Determine if the division can be folded into the operands of
3415 // its operands.
3416 // TODO: Generalize this to non-constants by using known-bits information.
3417 Type *Ty = LHS->getType();
3418 unsigned LZ = RHSC->getAPInt().countl_zero();
3419 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
3420 // For non-power-of-two values, effectively round the value up to the
3421 // nearest power of two.
3422 if (!RHSC->getAPInt().isPowerOf2())
3423 ++MaxShiftAmt;
3424 IntegerType *ExtTy =
3425 IntegerType::get(C&: getContext(), NumBits: getTypeSizeInBits(Ty) + MaxShiftAmt);
3426 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS))
3427 if (const SCEVConstant *Step =
3428 dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE&: *this))) {
3429 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
3430 const APInt &StepInt = Step->getAPInt();
3431 const APInt &DivInt = RHSC->getAPInt();
3432 if (!StepInt.urem(RHS: DivInt) &&
3433 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3434 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3435 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy),
3436 L: AR->getLoop(), Flags: SCEV::FlagAnyWrap)) {
3437 SmallVector<const SCEV *, 4> Operands;
3438 for (const SCEV *Op : AR->operands())
3439 Operands.push_back(Elt: getUDivExpr(LHS: Op, RHS));
3440 return getAddRecExpr(Operands, L: AR->getLoop(), Flags: SCEV::FlagNW);
3441 }
3442 /// Get a canonical UDivExpr for a recurrence.
3443 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
3444 // We can currently only fold X%N if X is constant.
3445 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Val: AR->getStart());
3446 if (StartC && !DivInt.urem(RHS: StepInt) &&
3447 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3448 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3449 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy),
3450 L: AR->getLoop(), Flags: SCEV::FlagAnyWrap)) {
3451 const APInt &StartInt = StartC->getAPInt();
3452 const APInt &StartRem = StartInt.urem(RHS: StepInt);
3453 if (StartRem != 0) {
3454 const SCEV *NewLHS =
3455 getAddRecExpr(Start: getConstant(Val: StartInt - StartRem), Step,
3456 L: AR->getLoop(), Flags: SCEV::FlagNW);
3457 if (LHS != NewLHS) {
3458 LHS = NewLHS;
3459
3460 // Reset the ID to include the new LHS, and check if it is
3461 // already cached.
3462 ID.clear();
3463 ID.AddInteger(I: scUDivExpr);
3464 ID.AddPointer(Ptr: LHS);
3465 ID.AddPointer(Ptr: RHS);
3466 IP = nullptr;
3467 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP))
3468 return S;
3469 }
3470 }
3471 }
3472 }
3473 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
3474 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val: LHS)) {
3475 SmallVector<const SCEV *, 4> Operands;
3476 for (const SCEV *Op : M->operands())
3477 Operands.push_back(Elt: getZeroExtendExpr(Op, Ty: ExtTy));
3478 if (getZeroExtendExpr(Op: M, Ty: ExtTy) == getMulExpr(Ops&: Operands))
3479 // Find an operand that's safely divisible.
3480 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
3481 const SCEV *Op = M->getOperand(i);
3482 const SCEV *Div = getUDivExpr(LHS: Op, RHS: RHSC);
3483 if (!isa<SCEVUDivExpr>(Val: Div) && getMulExpr(LHS: Div, RHS: RHSC) == Op) {
3484 Operands = SmallVector<const SCEV *, 4>(M->operands());
3485 Operands[i] = Div;
3486 return getMulExpr(Ops&: Operands);
3487 }
3488 }
3489 }
3490
3491 // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
3492 if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(Val: LHS)) {
3493 if (auto *DivisorConstant =
3494 dyn_cast<SCEVConstant>(Val: OtherDiv->getRHS())) {
3495 bool Overflow = false;
3496 APInt NewRHS =
3497 DivisorConstant->getAPInt().umul_ov(RHS: RHSC->getAPInt(), Overflow);
3498 if (Overflow) {
3499 return getConstant(Ty: RHSC->getType(), V: 0, isSigned: false);
3500 }
3501 return getUDivExpr(LHS: OtherDiv->getLHS(), RHS: getConstant(Val: NewRHS));
3502 }
3503 }
3504
3505 // (A+B)/C --> (A/C + B/C) if safe and A/C and B/C can be folded.
3506 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(Val: LHS)) {
3507 SmallVector<const SCEV *, 4> Operands;
3508 for (const SCEV *Op : A->operands())
3509 Operands.push_back(Elt: getZeroExtendExpr(Op, Ty: ExtTy));
3510 if (getZeroExtendExpr(Op: A, Ty: ExtTy) == getAddExpr(Ops&: Operands)) {
3511 Operands.clear();
3512 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
3513 const SCEV *Op = getUDivExpr(LHS: A->getOperand(i), RHS);
3514 if (isa<SCEVUDivExpr>(Val: Op) ||
3515 getMulExpr(LHS: Op, RHS) != A->getOperand(i))
3516 break;
3517 Operands.push_back(Elt: Op);
3518 }
3519 if (Operands.size() == A->getNumOperands())
3520 return getAddExpr(Ops&: Operands);
3521 }
3522 }
3523
3524 // Fold if both operands are constant.
3525 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val: LHS))
3526 return getConstant(Val: LHSC->getAPInt().udiv(RHS: RHSC->getAPInt()));
3527 }
3528 }
3529
3530 // ((-C + (C smax %x)) /u %x) evaluates to zero, for any positive constant C.
3531 if (const auto *AE = dyn_cast<SCEVAddExpr>(Val: LHS);
3532 AE && AE->getNumOperands() == 2) {
3533 if (const auto *VC = dyn_cast<SCEVConstant>(Val: AE->getOperand(i: 0))) {
3534 const APInt &NegC = VC->getAPInt();
3535 if (NegC.isNegative() && !NegC.isMinSignedValue()) {
3536 const auto *MME = dyn_cast<SCEVSMaxExpr>(Val: AE->getOperand(i: 1));
3537 if (MME && MME->getNumOperands() == 2 &&
3538 isa<SCEVConstant>(Val: MME->getOperand(i: 0)) &&
3539 cast<SCEVConstant>(Val: MME->getOperand(i: 0))->getAPInt() == -NegC &&
3540 MME->getOperand(i: 1) == RHS)
3541 return getZero(Ty: LHS->getType());
3542 }
3543 }
3544 }
3545
3546 // The Insertion Point (IP) might be invalid by now (due to UniqueSCEVs
3547 // changes). Make sure we get a new one.
3548 IP = nullptr;
3549 if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) return S;
3550 SCEV *S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(Allocator&: SCEVAllocator),
3551 LHS, RHS);
3552 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3553 registerUser(User: S, Ops: {LHS, RHS});
3554 return S;
3555}
3556
3557APInt gcd(const SCEVConstant *C1, const SCEVConstant *C2) {
3558 APInt A = C1->getAPInt().abs();
3559 APInt B = C2->getAPInt().abs();
3560 uint32_t ABW = A.getBitWidth();
3561 uint32_t BBW = B.getBitWidth();
3562
3563 if (ABW > BBW)
3564 B = B.zext(width: ABW);
3565 else if (ABW < BBW)
3566 A = A.zext(width: BBW);
3567
3568 return APIntOps::GreatestCommonDivisor(A: std::move(A), B: std::move(B));
3569}
3570
3571/// Get a canonical unsigned division expression, or something simpler if
3572/// possible. There is no representation for an exact udiv in SCEV IR, but we
3573/// can attempt to remove factors from the LHS and RHS. We can't do this when
3574/// it's not exact because the udiv may be clearing bits.
3575const SCEV *ScalarEvolution::getUDivExactExpr(const SCEV *LHS,
3576 const SCEV *RHS) {
3577 // TODO: we could try to find factors in all sorts of things, but for now we
3578 // just deal with u/exact (multiply, constant). See SCEVDivision towards the
3579 // end of this file for inspiration.
3580
3581 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: LHS);
3582 if (!Mul || !Mul->hasNoUnsignedWrap())
3583 return getUDivExpr(LHS, RHS);
3584
3585 if (const SCEVConstant *RHSCst = dyn_cast<SCEVConstant>(Val: RHS)) {
3586 // If the mulexpr multiplies by a constant, then that constant must be the
3587 // first element of the mulexpr.
3588 if (const auto *LHSCst = dyn_cast<SCEVConstant>(Val: Mul->getOperand(i: 0))) {
3589 if (LHSCst == RHSCst) {
3590 SmallVector<const SCEV *, 2> Operands(drop_begin(RangeOrContainer: Mul->operands()));
3591 return getMulExpr(Ops&: Operands);
3592 }
3593
3594 // We can't just assume that LHSCst divides RHSCst cleanly, it could be
3595 // that there's a factor provided by one of the other terms. We need to
3596 // check.
3597 APInt Factor = gcd(C1: LHSCst, C2: RHSCst);
3598 if (!Factor.isIntN(N: 1)) {
3599 LHSCst =
3600 cast<SCEVConstant>(Val: getConstant(Val: LHSCst->getAPInt().udiv(RHS: Factor)));
3601 RHSCst =
3602 cast<SCEVConstant>(Val: getConstant(Val: RHSCst->getAPInt().udiv(RHS: Factor)));
3603 SmallVector<const SCEV *, 2> Operands;
3604 Operands.push_back(Elt: LHSCst);
3605 append_range(C&: Operands, R: Mul->operands().drop_front());
3606 LHS = getMulExpr(Ops&: Operands);
3607 RHS = RHSCst;
3608 Mul = dyn_cast<SCEVMulExpr>(Val: LHS);
3609 if (!Mul)
3610 return getUDivExactExpr(LHS, RHS);
3611 }
3612 }
3613 }
3614
3615 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
3616 if (Mul->getOperand(i) == RHS) {
3617 SmallVector<const SCEV *, 2> Operands;
3618 append_range(C&: Operands, R: Mul->operands().take_front(N: i));
3619 append_range(C&: Operands, R: Mul->operands().drop_front(N: i + 1));
3620 return getMulExpr(Ops&: Operands);
3621 }
3622 }
3623
3624 return getUDivExpr(LHS, RHS);
3625}
3626
3627/// Get an add recurrence expression for the specified loop. Simplify the
3628/// expression as much as possible.
3629const SCEV *ScalarEvolution::getAddRecExpr(const SCEV *Start, const SCEV *Step,
3630 const Loop *L,
3631 SCEV::NoWrapFlags Flags) {
3632 SmallVector<const SCEV *, 4> Operands;
3633 Operands.push_back(Elt: Start);
3634 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Val: Step))
3635 if (StepChrec->getLoop() == L) {
3636 append_range(C&: Operands, R: StepChrec->operands());
3637 return getAddRecExpr(Operands, L, Flags: maskFlags(Flags, Mask: SCEV::FlagNW));
3638 }
3639
3640 Operands.push_back(Elt: Step);
3641 return getAddRecExpr(Operands, L, Flags);
3642}
3643
3644/// Get an add recurrence expression for the specified loop. Simplify the
3645/// expression as much as possible.
3646const SCEV *
3647ScalarEvolution::getAddRecExpr(SmallVectorImpl<const SCEV *> &Operands,
3648 const Loop *L, SCEV::NoWrapFlags Flags) {
3649 if (Operands.size() == 1) return Operands[0];
3650#ifndef NDEBUG
3651 Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
3652 for (const SCEV *Op : llvm::drop_begin(Operands)) {
3653 assert(getEffectiveSCEVType(Op->getType()) == ETy &&
3654 "SCEVAddRecExpr operand types don't match!");
3655 assert(!Op->getType()->isPointerTy() && "Step must be integer");
3656 }
3657 for (const SCEV *Op : Operands)
3658 assert(isAvailableAtLoopEntry(Op, L) &&
3659 "SCEVAddRecExpr operand is not available at loop entry!");
3660#endif
3661
3662 if (Operands.back()->isZero()) {
3663 Operands.pop_back();
3664 return getAddRecExpr(Operands, L, Flags: SCEV::FlagAnyWrap); // {X,+,0} --> X
3665 }
3666
3667 // It's tempting to want to call getConstantMaxBackedgeTakenCount count here and
3668 // use that information to infer NUW and NSW flags. However, computing a
3669 // BE count requires calling getAddRecExpr, so we may not yet have a
3670 // meaningful BE count at this point (and if we don't, we'd be stuck
3671 // with a SCEVCouldNotCompute as the cached BE count).
3672
3673 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
3674
3675 // Canonicalize nested AddRecs in by nesting them in order of loop depth.
3676 if (const SCEVAddRecExpr *NestedAR = dyn_cast<SCEVAddRecExpr>(Val: Operands[0])) {
3677 const Loop *NestedLoop = NestedAR->getLoop();
3678 if (L->contains(L: NestedLoop)
3679 ? (L->getLoopDepth() < NestedLoop->getLoopDepth())
3680 : (!NestedLoop->contains(L) &&
3681 DT.dominates(A: L->getHeader(), B: NestedLoop->getHeader()))) {
3682 SmallVector<const SCEV *, 4> NestedOperands(NestedAR->operands());
3683 Operands[0] = NestedAR->getStart();
3684 // AddRecs require their operands be loop-invariant with respect to their
3685 // loops. Don't perform this transformation if it would break this
3686 // requirement.
3687 bool AllInvariant = all_of(
3688 Range&: Operands, P: [&](const SCEV *Op) { return isLoopInvariant(S: Op, L); });
3689
3690 if (AllInvariant) {
3691 // Create a recurrence for the outer loop with the same step size.
3692 //
3693 // The outer recurrence keeps its NW flag but only keeps NUW/NSW if the
3694 // inner recurrence has the same property.
3695 SCEV::NoWrapFlags OuterFlags =
3696 maskFlags(Flags, Mask: SCEV::FlagNW | NestedAR->getNoWrapFlags());
3697
3698 NestedOperands[0] = getAddRecExpr(Operands, L, Flags: OuterFlags);
3699 AllInvariant = all_of(Range&: NestedOperands, P: [&](const SCEV *Op) {
3700 return isLoopInvariant(S: Op, L: NestedLoop);
3701 });
3702
3703 if (AllInvariant) {
3704 // Ok, both add recurrences are valid after the transformation.
3705 //
3706 // The inner recurrence keeps its NW flag but only keeps NUW/NSW if
3707 // the outer recurrence has the same property.
3708 SCEV::NoWrapFlags InnerFlags =
3709 maskFlags(Flags: NestedAR->getNoWrapFlags(), Mask: SCEV::FlagNW | Flags);
3710 return getAddRecExpr(Operands&: NestedOperands, L: NestedLoop, Flags: InnerFlags);
3711 }
3712 }
3713 // Reset Operands to its original state.
3714 Operands[0] = NestedAR;
3715 }
3716 }
3717
3718 // Okay, it looks like we really DO need an addrec expr. Check to see if we
3719 // already have one, otherwise create a new one.
3720 return getOrCreateAddRecExpr(Ops: Operands, L, Flags);
3721}
3722
3723const SCEV *
3724ScalarEvolution::getGEPExpr(GEPOperator *GEP,
3725 const SmallVectorImpl<const SCEV *> &IndexExprs) {
3726 const SCEV *BaseExpr = getSCEV(V: GEP->getPointerOperand());
3727 // getSCEV(Base)->getType() has the same address space as Base->getType()
3728 // because SCEV::getType() preserves the address space.
3729 Type *IntIdxTy = getEffectiveSCEVType(Ty: BaseExpr->getType());
3730 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
3731 if (NW != GEPNoWrapFlags::none()) {
3732 // We'd like to propagate flags from the IR to the corresponding SCEV nodes,
3733 // but to do that, we have to ensure that said flag is valid in the entire
3734 // defined scope of the SCEV.
3735 // TODO: non-instructions have global scope. We might be able to prove
3736 // some global scope cases
3737 auto *GEPI = dyn_cast<Instruction>(Val: GEP);
3738 if (!GEPI || !isSCEVExprNeverPoison(I: GEPI))
3739 NW = GEPNoWrapFlags::none();
3740 }
3741
3742 SCEV::NoWrapFlags OffsetWrap = SCEV::FlagAnyWrap;
3743 if (NW.hasNoUnsignedSignedWrap())
3744 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNSW);
3745 if (NW.hasNoUnsignedWrap())
3746 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNUW);
3747
3748 Type *CurTy = GEP->getType();
3749 bool FirstIter = true;
3750 SmallVector<const SCEV *, 4> Offsets;
3751 for (const SCEV *IndexExpr : IndexExprs) {
3752 // Compute the (potentially symbolic) offset in bytes for this index.
3753 if (StructType *STy = dyn_cast<StructType>(Val: CurTy)) {
3754 // For a struct, add the member offset.
3755 ConstantInt *Index = cast<SCEVConstant>(Val: IndexExpr)->getValue();
3756 unsigned FieldNo = Index->getZExtValue();
3757 const SCEV *FieldOffset = getOffsetOfExpr(IntTy: IntIdxTy, STy, FieldNo);
3758 Offsets.push_back(Elt: FieldOffset);
3759
3760 // Update CurTy to the type of the field at Index.
3761 CurTy = STy->getTypeAtIndex(V: Index);
3762 } else {
3763 // Update CurTy to its element type.
3764 if (FirstIter) {
3765 assert(isa<PointerType>(CurTy) &&
3766 "The first index of a GEP indexes a pointer");
3767 CurTy = GEP->getSourceElementType();
3768 FirstIter = false;
3769 } else {
3770 CurTy = GetElementPtrInst::getTypeAtIndex(Ty: CurTy, Idx: (uint64_t)0);
3771 }
3772 // For an array, add the element offset, explicitly scaled.
3773 const SCEV *ElementSize = getSizeOfExpr(IntTy: IntIdxTy, AllocTy: CurTy);
3774 // Getelementptr indices are signed.
3775 IndexExpr = getTruncateOrSignExtend(V: IndexExpr, Ty: IntIdxTy);
3776
3777 // Multiply the index by the element size to compute the element offset.
3778 const SCEV *LocalOffset = getMulExpr(LHS: IndexExpr, RHS: ElementSize, Flags: OffsetWrap);
3779 Offsets.push_back(Elt: LocalOffset);
3780 }
3781 }
3782
3783 // Handle degenerate case of GEP without offsets.
3784 if (Offsets.empty())
3785 return BaseExpr;
3786
3787 // Add the offsets together, assuming nsw if inbounds.
3788 const SCEV *Offset = getAddExpr(Ops&: Offsets, OrigFlags: OffsetWrap);
3789 // Add the base address and the offset. We cannot use the nsw flag, as the
3790 // base address is unsigned. However, if we know that the offset is
3791 // non-negative, we can use nuw.
3792 bool NUW = NW.hasNoUnsignedWrap() ||
3793 (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(S: Offset));
3794 SCEV::NoWrapFlags BaseWrap = NUW ? SCEV::FlagNUW : SCEV::FlagAnyWrap;
3795 auto *GEPExpr = getAddExpr(LHS: BaseExpr, RHS: Offset, Flags: BaseWrap);
3796 assert(BaseExpr->getType() == GEPExpr->getType() &&
3797 "GEP should not change type mid-flight.");
3798 return GEPExpr;
3799}
3800
3801SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
3802 ArrayRef<const SCEV *> Ops) {
3803 FoldingSetNodeID ID;
3804 ID.AddInteger(I: SCEVType);
3805 for (const SCEV *Op : Ops)
3806 ID.AddPointer(Ptr: Op);
3807 void *IP = nullptr;
3808 return UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP);
3809}
3810
3811const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
3812 SCEV::NoWrapFlags Flags = IsNSW ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
3813 return getSMaxExpr(LHS: Op, RHS: getNegativeSCEV(V: Op, Flags));
3814}
3815
3816const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
3817 SmallVectorImpl<const SCEV *> &Ops) {
3818 assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!");
3819 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
3820 if (Ops.size() == 1) return Ops[0];
3821#ifndef NDEBUG
3822 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
3823 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
3824 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
3825 "Operand types don't match!");
3826 assert(Ops[0]->getType()->isPointerTy() ==
3827 Ops[i]->getType()->isPointerTy() &&
3828 "min/max should be consistently pointerish");
3829 }
3830#endif
3831
3832 bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr;
3833 bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr;
3834
3835 const SCEV *Folded = constantFoldAndGroupOps(
3836 SE&: *this, LI, DT, Ops,
3837 Fold: [&](const APInt &C1, const APInt &C2) {
3838 switch (Kind) {
3839 case scSMaxExpr:
3840 return APIntOps::smax(A: C1, B: C2);
3841 case scSMinExpr:
3842 return APIntOps::smin(A: C1, B: C2);
3843 case scUMaxExpr:
3844 return APIntOps::umax(A: C1, B: C2);
3845 case scUMinExpr:
3846 return APIntOps::umin(A: C1, B: C2);
3847 default:
3848 llvm_unreachable("Unknown SCEV min/max opcode");
3849 }
3850 },
3851 IsIdentity: [&](const APInt &C) {
3852 // identity
3853 if (IsMax)
3854 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3855 else
3856 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3857 },
3858 IsAbsorber: [&](const APInt &C) {
3859 // absorber
3860 if (IsMax)
3861 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3862 else
3863 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3864 });
3865 if (Folded)
3866 return Folded;
3867
3868 // Check if we have created the same expression before.
3869 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops)) {
3870 return S;
3871 }
3872
3873 // Find the first operation of the same kind
3874 unsigned Idx = 0;
3875 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind)
3876 ++Idx;
3877
3878 // Check to see if one of the operands is of the same kind. If so, expand its
3879 // operands onto our operand list, and recurse to simplify.
3880 if (Idx < Ops.size()) {
3881 bool DeletedAny = false;
3882 while (Ops[Idx]->getSCEVType() == Kind) {
3883 const SCEVMinMaxExpr *SMME = cast<SCEVMinMaxExpr>(Val: Ops[Idx]);
3884 Ops.erase(CI: Ops.begin()+Idx);
3885 append_range(C&: Ops, R: SMME->operands());
3886 DeletedAny = true;
3887 }
3888
3889 if (DeletedAny)
3890 return getMinMaxExpr(Kind, Ops);
3891 }
3892
3893 // Okay, check to see if the same value occurs in the operand list twice. If
3894 // so, delete one. Since we sorted the list, these values are required to
3895 // be adjacent.
3896 llvm::CmpInst::Predicate GEPred =
3897 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
3898 llvm::CmpInst::Predicate LEPred =
3899 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
3900 llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred;
3901 llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred;
3902 for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) {
3903 if (Ops[i] == Ops[i + 1] ||
3904 isKnownViaNonRecursiveReasoning(Pred: FirstPred, LHS: Ops[i], RHS: Ops[i + 1])) {
3905 // X op Y op Y --> X op Y
3906 // X op Y --> X, if we know X, Y are ordered appropriately
3907 Ops.erase(CS: Ops.begin() + i + 1, CE: Ops.begin() + i + 2);
3908 --i;
3909 --e;
3910 } else if (isKnownViaNonRecursiveReasoning(Pred: SecondPred, LHS: Ops[i],
3911 RHS: Ops[i + 1])) {
3912 // X op Y --> Y, if we know X, Y are ordered appropriately
3913 Ops.erase(CS: Ops.begin() + i, CE: Ops.begin() + i + 1);
3914 --i;
3915 --e;
3916 }
3917 }
3918
3919 if (Ops.size() == 1) return Ops[0];
3920
3921 assert(!Ops.empty() && "Reduced smax down to nothing!");
3922
3923 // Okay, it looks like we really DO need an expr. Check to see if we
3924 // already have one, otherwise create a new one.
3925 FoldingSetNodeID ID;
3926 ID.AddInteger(I: Kind);
3927 for (const SCEV *Op : Ops)
3928 ID.AddPointer(Ptr: Op);
3929 void *IP = nullptr;
3930 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP);
3931 if (ExistingSCEV)
3932 return ExistingSCEV;
3933 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Num: Ops.size());
3934 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3935 SCEV *S = new (SCEVAllocator)
3936 SCEVMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
3937
3938 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
3939 registerUser(User: S, Ops);
3940 return S;
3941}
3942
3943namespace {
3944
3945class SCEVSequentialMinMaxDeduplicatingVisitor final
3946 : public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
3947 std::optional<const SCEV *>> {
3948 using RetVal = std::optional<const SCEV *>;
3949 using Base = SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor, RetVal>;
3950
3951 ScalarEvolution &SE;
3952 const SCEVTypes RootKind; // Must be a sequential min/max expression.
3953 const SCEVTypes NonSequentialRootKind; // Non-sequential variant of RootKind.
3954 SmallPtrSet<const SCEV *, 16> SeenOps;
3955
3956 bool canRecurseInto(SCEVTypes Kind) const {
3957 // We can only recurse into the SCEV expression of the same effective type
3958 // as the type of our root SCEV expression.
3959 return RootKind == Kind || NonSequentialRootKind == Kind;
3960 };
3961
3962 RetVal visitAnyMinMaxExpr(const SCEV *S) {
3963 assert((isa<SCEVMinMaxExpr>(S) || isa<SCEVSequentialMinMaxExpr>(S)) &&
3964 "Only for min/max expressions.");
3965 SCEVTypes Kind = S->getSCEVType();
3966
3967 if (!canRecurseInto(Kind))
3968 return S;
3969
3970 auto *NAry = cast<SCEVNAryExpr>(Val: S);
3971 SmallVector<const SCEV *> NewOps;
3972 bool Changed = visit(Kind, OrigOps: NAry->operands(), NewOps);
3973
3974 if (!Changed)
3975 return S;
3976 if (NewOps.empty())
3977 return std::nullopt;
3978
3979 return isa<SCEVSequentialMinMaxExpr>(Val: S)
3980 ? SE.getSequentialMinMaxExpr(Kind, Operands&: NewOps)
3981 : SE.getMinMaxExpr(Kind, Ops&: NewOps);
3982 }
3983
3984 RetVal visit(const SCEV *S) {
3985 // Has the whole operand been seen already?
3986 if (!SeenOps.insert(Ptr: S).second)
3987 return std::nullopt;
3988 return Base::visit(S);
3989 }
3990
3991public:
3992 SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
3993 SCEVTypes RootKind)
3994 : SE(SE), RootKind(RootKind),
3995 NonSequentialRootKind(
3996 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
3997 Ty: RootKind)) {}
3998
3999 bool /*Changed*/ visit(SCEVTypes Kind, ArrayRef<const SCEV *> OrigOps,
4000 SmallVectorImpl<const SCEV *> &NewOps) {
4001 bool Changed = false;
4002 SmallVector<const SCEV *> Ops;
4003 Ops.reserve(N: OrigOps.size());
4004
4005 for (const SCEV *Op : OrigOps) {
4006 RetVal NewOp = visit(S: Op);
4007 if (NewOp != Op)
4008 Changed = true;
4009 if (NewOp)
4010 Ops.emplace_back(Args&: *NewOp);
4011 }
4012
4013 if (Changed)
4014 NewOps = std::move(Ops);
4015 return Changed;
4016 }
4017
4018 RetVal visitConstant(const SCEVConstant *Constant) { return Constant; }
4019
4020 RetVal visitVScale(const SCEVVScale *VScale) { return VScale; }
4021
4022 RetVal visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) { return Expr; }
4023
4024 RetVal visitTruncateExpr(const SCEVTruncateExpr *Expr) { return Expr; }
4025
4026 RetVal visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) { return Expr; }
4027
4028 RetVal visitSignExtendExpr(const SCEVSignExtendExpr *Expr) { return Expr; }
4029
4030 RetVal visitAddExpr(const SCEVAddExpr *Expr) { return Expr; }
4031
4032 RetVal visitMulExpr(const SCEVMulExpr *Expr) { return Expr; }
4033
4034 RetVal visitUDivExpr(const SCEVUDivExpr *Expr) { return Expr; }
4035
4036 RetVal visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
4037
4038 RetVal visitSMaxExpr(const SCEVSMaxExpr *Expr) {
4039 return visitAnyMinMaxExpr(S: Expr);
4040 }
4041
4042 RetVal visitUMaxExpr(const SCEVUMaxExpr *Expr) {
4043 return visitAnyMinMaxExpr(S: Expr);
4044 }
4045
4046 RetVal visitSMinExpr(const SCEVSMinExpr *Expr) {
4047 return visitAnyMinMaxExpr(S: Expr);
4048 }
4049
4050 RetVal visitUMinExpr(const SCEVUMinExpr *Expr) {
4051 return visitAnyMinMaxExpr(S: Expr);
4052 }
4053
4054 RetVal visitSequentialUMinExpr(const SCEVSequentialUMinExpr *Expr) {
4055 return visitAnyMinMaxExpr(S: Expr);
4056 }
4057
4058 RetVal visitUnknown(const SCEVUnknown *Expr) { return Expr; }
4059
4060 RetVal visitCouldNotCompute(const SCEVCouldNotCompute *Expr) { return Expr; }
4061};
4062
4063} // namespace
4064
4065static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind) {
4066 switch (Kind) {
4067 case scConstant:
4068 case scVScale:
4069 case scTruncate:
4070 case scZeroExtend:
4071 case scSignExtend:
4072 case scPtrToInt:
4073 case scAddExpr:
4074 case scMulExpr:
4075 case scUDivExpr:
4076 case scAddRecExpr:
4077 case scUMaxExpr:
4078 case scSMaxExpr:
4079 case scUMinExpr:
4080 case scSMinExpr:
4081 case scUnknown:
4082 // If any operand is poison, the whole expression is poison.
4083 return true;
4084 case scSequentialUMinExpr:
4085 // FIXME: if the *first* operand is poison, the whole expression is poison.
4086 return false; // Pessimistically, say that it does not propagate poison.
4087 case scCouldNotCompute:
4088 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
4089 }
4090 llvm_unreachable("Unknown SCEV kind!");
4091}
4092
4093namespace {
4094// The only way poison may be introduced in a SCEV expression is from a
4095// poison SCEVUnknown (ConstantExprs are also represented as SCEVUnknown,
4096// not SCEVConstant). Notably, nowrap flags in SCEV nodes can *not*
4097// introduce poison -- they encode guaranteed, non-speculated knowledge.
4098//
4099// Additionally, all SCEV nodes propagate poison from inputs to outputs,
4100// with the notable exception of umin_seq, where only poison from the first
4101// operand is (unconditionally) propagated.
4102struct SCEVPoisonCollector {
4103 bool LookThroughMaybePoisonBlocking;
4104 SmallPtrSet<const SCEVUnknown *, 4> MaybePoison;
4105 SCEVPoisonCollector(bool LookThroughMaybePoisonBlocking)
4106 : LookThroughMaybePoisonBlocking(LookThroughMaybePoisonBlocking) {}
4107
4108 bool follow(const SCEV *S) {
4109 if (!LookThroughMaybePoisonBlocking &&
4110 !scevUnconditionallyPropagatesPoisonFromOperands(Kind: S->getSCEVType()))
4111 return false;
4112
4113 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
4114 if (!isGuaranteedNotToBePoison(V: SU->getValue()))
4115 MaybePoison.insert(Ptr: SU);
4116 }
4117 return true;
4118 }
4119 bool isDone() const { return false; }
4120};
4121} // namespace
4122
4123/// Return true if V is poison given that AssumedPoison is already poison.
4124static bool impliesPoison(const SCEV *AssumedPoison, const SCEV *S) {
4125 // First collect all SCEVs that might result in AssumedPoison to be poison.
4126 // We need to look through potentially poison-blocking operations here,
4127 // because we want to find all SCEVs that *might* result in poison, not only
4128 // those that are *required* to.
4129 SCEVPoisonCollector PC1(/* LookThroughMaybePoisonBlocking */ true);
4130 visitAll(Root: AssumedPoison, Visitor&: PC1);
4131
4132 // AssumedPoison is never poison. As the assumption is false, the implication
4133 // is true. Don't bother walking the other SCEV in this case.
4134 if (PC1.MaybePoison.empty())
4135 return true;
4136
4137 // Collect all SCEVs in S that, if poison, *will* result in S being poison
4138 // as well. We cannot look through potentially poison-blocking operations
4139 // here, as their arguments only *may* make the result poison.
4140 SCEVPoisonCollector PC2(/* LookThroughMaybePoisonBlocking */ false);
4141 visitAll(Root: S, Visitor&: PC2);
4142
4143 // Make sure that no matter which SCEV in PC1.MaybePoison is actually poison,
4144 // it will also make S poison by being part of PC2.MaybePoison.
4145 return llvm::set_is_subset(S1: PC1.MaybePoison, S2: PC2.MaybePoison);
4146}
4147
4148void ScalarEvolution::getPoisonGeneratingValues(
4149 SmallPtrSetImpl<const Value *> &Result, const SCEV *S) {
4150 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ false);
4151 visitAll(Root: S, Visitor&: PC);
4152 for (const SCEVUnknown *SU : PC.MaybePoison)
4153 Result.insert(Ptr: SU->getValue());
4154}
4155
4156bool ScalarEvolution::canReuseInstruction(
4157 const SCEV *S, Instruction *I,
4158 SmallVectorImpl<Instruction *> &DropPoisonGeneratingInsts) {
4159 // If the instruction cannot be poison, it's always safe to reuse.
4160 if (programUndefinedIfPoison(Inst: I))
4161 return true;
4162
4163 // Otherwise, it is possible that I is more poisonous that S. Collect the
4164 // poison-contributors of S, and then check whether I has any additional
4165 // poison-contributors. Poison that is contributed through poison-generating
4166 // flags is handled by dropping those flags instead.
4167 SmallPtrSet<const Value *, 8> PoisonVals;
4168 getPoisonGeneratingValues(Result&: PoisonVals, S);
4169
4170 SmallVector<Value *> Worklist;
4171 SmallPtrSet<Value *, 8> Visited;
4172 Worklist.push_back(Elt: I);
4173 while (!Worklist.empty()) {
4174 Value *V = Worklist.pop_back_val();
4175 if (!Visited.insert(Ptr: V).second)
4176 continue;
4177
4178 // Avoid walking large instruction graphs.
4179 if (Visited.size() > 16)
4180 return false;
4181
4182 // Either the value can't be poison, or the S would also be poison if it
4183 // is.
4184 if (PoisonVals.contains(Ptr: V) || ::isGuaranteedNotToBePoison(V))
4185 continue;
4186
4187 auto *I = dyn_cast<Instruction>(Val: V);
4188 if (!I)
4189 return false;
4190
4191 // Disjoint or instructions are interpreted as adds by SCEV. However, we
4192 // can't replace an arbitrary add with disjoint or, even if we drop the
4193 // flag. We would need to convert the or into an add.
4194 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: I))
4195 if (PDI->isDisjoint())
4196 return false;
4197
4198 // FIXME: Ignore vscale, even though it technically could be poison. Do this
4199 // because SCEV currently assumes it can't be poison. Remove this special
4200 // case once we proper model when vscale can be poison.
4201 if (auto *II = dyn_cast<IntrinsicInst>(Val: I);
4202 II && II->getIntrinsicID() == Intrinsic::vscale)
4203 continue;
4204
4205 if (canCreatePoison(Op: cast<Operator>(Val: I), /*ConsiderFlagsAndMetadata*/ false))
4206 return false;
4207
4208 // If the instruction can't create poison, we can recurse to its operands.
4209 if (I->hasPoisonGeneratingAnnotations())
4210 DropPoisonGeneratingInsts.push_back(Elt: I);
4211
4212 llvm::append_range(C&: Worklist, R: I->operands());
4213 }
4214 return true;
4215}
4216
4217const SCEV *
4218ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
4219 SmallVectorImpl<const SCEV *> &Ops) {
4220 assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&
4221 "Not a SCEVSequentialMinMaxExpr!");
4222 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
4223 if (Ops.size() == 1)
4224 return Ops[0];
4225#ifndef NDEBUG
4226 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4227 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4228 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
4229 "Operand types don't match!");
4230 assert(Ops[0]->getType()->isPointerTy() ==
4231 Ops[i]->getType()->isPointerTy() &&
4232 "min/max should be consistently pointerish");
4233 }
4234#endif
4235
4236 // Note that SCEVSequentialMinMaxExpr is *NOT* commutative,
4237 // so we can *NOT* do any kind of sorting of the expressions!
4238
4239 // Check if we have created the same expression before.
4240 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops))
4241 return S;
4242
4243 // FIXME: there are *some* simplifications that we can do here.
4244
4245 // Keep only the first instance of an operand.
4246 {
4247 SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*this, Kind);
4248 bool Changed = Deduplicator.visit(Kind, OrigOps: Ops, NewOps&: Ops);
4249 if (Changed)
4250 return getSequentialMinMaxExpr(Kind, Ops);
4251 }
4252
4253 // Check to see if one of the operands is of the same kind. If so, expand its
4254 // operands onto our operand list, and recurse to simplify.
4255 {
4256 unsigned Idx = 0;
4257 bool DeletedAny = false;
4258 while (Idx < Ops.size()) {
4259 if (Ops[Idx]->getSCEVType() != Kind) {
4260 ++Idx;
4261 continue;
4262 }
4263 const auto *SMME = cast<SCEVSequentialMinMaxExpr>(Val: Ops[Idx]);
4264 Ops.erase(CI: Ops.begin() + Idx);
4265 Ops.insert(I: Ops.begin() + Idx, From: SMME->operands().begin(),
4266 To: SMME->operands().end());
4267 DeletedAny = true;
4268 }
4269
4270 if (DeletedAny)
4271 return getSequentialMinMaxExpr(Kind, Ops);
4272 }
4273
4274 const SCEV *SaturationPoint;
4275 ICmpInst::Predicate Pred;
4276 switch (Kind) {
4277 case scSequentialUMinExpr:
4278 SaturationPoint = getZero(Ty: Ops[0]->getType());
4279 Pred = ICmpInst::ICMP_ULE;
4280 break;
4281 default:
4282 llvm_unreachable("Not a sequential min/max type.");
4283 }
4284
4285 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4286 if (!isGuaranteedNotToCauseUB(Op: Ops[i]))
4287 continue;
4288 // We can replace %x umin_seq %y with %x umin %y if either:
4289 // * %y being poison implies %x is also poison.
4290 // * %x cannot be the saturating value (e.g. zero for umin).
4291 if (::impliesPoison(AssumedPoison: Ops[i], S: Ops[i - 1]) ||
4292 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_NE, LHS: Ops[i - 1],
4293 RHS: SaturationPoint)) {
4294 SmallVector<const SCEV *> SeqOps = {Ops[i - 1], Ops[i]};
4295 Ops[i - 1] = getMinMaxExpr(
4296 Kind: SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Ty: Kind),
4297 Ops&: SeqOps);
4298 Ops.erase(CI: Ops.begin() + i);
4299 return getSequentialMinMaxExpr(Kind, Ops);
4300 }
4301 // Fold %x umin_seq %y to %x if %x ule %y.
4302 // TODO: We might be able to prove the predicate for a later operand.
4303 if (isKnownViaNonRecursiveReasoning(Pred, LHS: Ops[i - 1], RHS: Ops[i])) {
4304 Ops.erase(CI: Ops.begin() + i);
4305 return getSequentialMinMaxExpr(Kind, Ops);
4306 }
4307 }
4308
4309 // Okay, it looks like we really DO need an expr. Check to see if we
4310 // already have one, otherwise create a new one.
4311 FoldingSetNodeID ID;
4312 ID.AddInteger(I: Kind);
4313 for (const SCEV *Op : Ops)
4314 ID.AddPointer(Ptr: Op);
4315 void *IP = nullptr;
4316 const SCEV *ExistingSCEV = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP);
4317 if (ExistingSCEV)
4318 return ExistingSCEV;
4319
4320 const SCEV **O = SCEVAllocator.Allocate<const SCEV *>(Num: Ops.size());
4321 llvm::uninitialized_copy(Src&: Ops, Dst: O);
4322 SCEV *S = new (SCEVAllocator)
4323 SCEVSequentialMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
4324
4325 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
4326 registerUser(User: S, Ops);
4327 return S;
4328}
4329
4330const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS, const SCEV *RHS) {
4331 SmallVector<const SCEV *, 2> Ops = {LHS, RHS};
4332 return getSMaxExpr(Operands&: Ops);
4333}
4334
4335const SCEV *ScalarEvolution::getSMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
4336 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4337}
4338
4339const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS, const SCEV *RHS) {
4340 SmallVector<const SCEV *, 2> Ops = {LHS, RHS};
4341 return getUMaxExpr(Operands&: Ops);
4342}
4343
4344const SCEV *ScalarEvolution::getUMaxExpr(SmallVectorImpl<const SCEV *> &Ops) {
4345 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4346}
4347
4348const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS,
4349 const SCEV *RHS) {
4350 SmallVector<const SCEV *, 2> Ops = { LHS, RHS };
4351 return getSMinExpr(Operands&: Ops);
4352}
4353
4354const SCEV *ScalarEvolution::getSMinExpr(SmallVectorImpl<const SCEV *> &Ops) {
4355 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4356}
4357
4358const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS, const SCEV *RHS,
4359 bool Sequential) {
4360 SmallVector<const SCEV *, 2> Ops = { LHS, RHS };
4361 return getUMinExpr(Operands&: Ops, Sequential);
4362}
4363
4364const SCEV *ScalarEvolution::getUMinExpr(SmallVectorImpl<const SCEV *> &Ops,
4365 bool Sequential) {
4366 return Sequential ? getSequentialMinMaxExpr(Kind: scSequentialUMinExpr, Ops)
4367 : getMinMaxExpr(Kind: scUMinExpr, Ops);
4368}
4369
4370const SCEV *
4371ScalarEvolution::getSizeOfExpr(Type *IntTy, TypeSize Size) {
4372 const SCEV *Res = getConstant(Ty: IntTy, V: Size.getKnownMinValue());
4373 if (Size.isScalable())
4374 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty: IntTy));
4375 return Res;
4376}
4377
4378const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
4379 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeAllocSize(Ty: AllocTy));
4380}
4381
4382const SCEV *ScalarEvolution::getStoreSizeOfExpr(Type *IntTy, Type *StoreTy) {
4383 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeStoreSize(Ty: StoreTy));
4384}
4385
4386const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
4387 StructType *STy,
4388 unsigned FieldNo) {
4389 // We can bypass creating a target-independent constant expression and then
4390 // folding it back into a ConstantInt. This is just a compile-time
4391 // optimization.
4392 const StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
4393 assert(!SL->getSizeInBits().isScalable() &&
4394 "Cannot get offset for structure containing scalable vector types");
4395 return getConstant(Ty: IntTy, V: SL->getElementOffset(Idx: FieldNo));
4396}
4397
4398const SCEV *ScalarEvolution::getUnknown(Value *V) {
4399 // Don't attempt to do anything other than create a SCEVUnknown object
4400 // here. createSCEV only calls getUnknown after checking for all other
4401 // interesting possibilities, and any other code that calls getUnknown
4402 // is doing so in order to hide a value from SCEV canonicalization.
4403
4404 FoldingSetNodeID ID;
4405 ID.AddInteger(I: scUnknown);
4406 ID.AddPointer(Ptr: V);
4407 void *IP = nullptr;
4408 if (SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, InsertPos&: IP)) {
4409 assert(cast<SCEVUnknown>(S)->getValue() == V &&
4410 "Stale SCEVUnknown in uniquing map!");
4411 return S;
4412 }
4413 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(Allocator&: SCEVAllocator), V, this,
4414 FirstUnknown);
4415 FirstUnknown = cast<SCEVUnknown>(Val: S);
4416 UniqueSCEVs.InsertNode(N: S, InsertPos: IP);
4417 return S;
4418}
4419
4420//===----------------------------------------------------------------------===//
4421// Basic SCEV Analysis and PHI Idiom Recognition Code
4422//
4423
4424/// Test if values of the given type are analyzable within the SCEV
4425/// framework. This primarily includes integer types, and it can optionally
4426/// include pointer types if the ScalarEvolution class has access to
4427/// target-specific information.
4428bool ScalarEvolution::isSCEVable(Type *Ty) const {
4429 // Integers and pointers are always SCEVable.
4430 return Ty->isIntOrPtrTy();
4431}
4432
4433/// Return the size in bits of the specified type, for which isSCEVable must
4434/// return true.
4435uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
4436 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4437 if (Ty->isPointerTy())
4438 return getDataLayout().getIndexTypeSizeInBits(Ty);
4439 return getDataLayout().getTypeSizeInBits(Ty);
4440}
4441
4442/// Return a type with the same bitwidth as the given type and which represents
4443/// how SCEV will treat the given type, for which isSCEVable must return
4444/// true. For pointer types, this is the pointer index sized integer type.
4445Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
4446 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4447
4448 if (Ty->isIntegerTy())
4449 return Ty;
4450
4451 // The only other support type is pointer.
4452 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
4453 return getDataLayout().getIndexType(PtrTy: Ty);
4454}
4455
4456Type *ScalarEvolution::getWiderType(Type *T1, Type *T2) const {
4457 return getTypeSizeInBits(Ty: T1) >= getTypeSizeInBits(Ty: T2) ? T1 : T2;
4458}
4459
4460bool ScalarEvolution::instructionCouldExistWithOperands(const SCEV *A,
4461 const SCEV *B) {
4462 /// For a valid use point to exist, the defining scope of one operand
4463 /// must dominate the other.
4464 bool PreciseA, PreciseB;
4465 auto *ScopeA = getDefiningScopeBound(Ops: {A}, Precise&: PreciseA);
4466 auto *ScopeB = getDefiningScopeBound(Ops: {B}, Precise&: PreciseB);
4467 if (!PreciseA || !PreciseB)
4468 // Can't tell.
4469 return false;
4470 return (ScopeA == ScopeB) || DT.dominates(Def: ScopeA, User: ScopeB) ||
4471 DT.dominates(Def: ScopeB, User: ScopeA);
4472}
4473
4474const SCEV *ScalarEvolution::getCouldNotCompute() {
4475 return CouldNotCompute.get();
4476}
4477
4478bool ScalarEvolution::checkValidity(const SCEV *S) const {
4479 bool ContainsNulls = SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
4480 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
4481 return SU && SU->getValue() == nullptr;
4482 });
4483
4484 return !ContainsNulls;
4485}
4486
4487bool ScalarEvolution::containsAddRecurrence(const SCEV *S) {
4488 HasRecMapType::iterator I = HasRecMap.find(Val: S);
4489 if (I != HasRecMap.end())
4490 return I->second;
4491
4492 bool FoundAddRec =
4493 SCEVExprContains(Root: S, Pred: [](const SCEV *S) { return isa<SCEVAddRecExpr>(Val: S); });
4494 HasRecMap.insert(KV: {S, FoundAddRec});
4495 return FoundAddRec;
4496}
4497
4498/// Return the ValueOffsetPair set for \p S. \p S can be represented
4499/// by the value and offset from any ValueOffsetPair in the set.
4500ArrayRef<Value *> ScalarEvolution::getSCEVValues(const SCEV *S) {
4501 ExprValueMapType::iterator SI = ExprValueMap.find_as(Val: S);
4502 if (SI == ExprValueMap.end())
4503 return {};
4504 return SI->second.getArrayRef();
4505}
4506
4507/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V)
4508/// cannot be used separately. eraseValueFromMap should be used to remove
4509/// V from ValueExprMap and ExprValueMap at the same time.
4510void ScalarEvolution::eraseValueFromMap(Value *V) {
4511 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4512 if (I != ValueExprMap.end()) {
4513 auto EVIt = ExprValueMap.find(Val: I->second);
4514 bool Removed = EVIt->second.remove(X: V);
4515 (void) Removed;
4516 assert(Removed && "Value not in ExprValueMap?");
4517 ValueExprMap.erase(I);
4518 }
4519}
4520
4521void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
4522 // A recursive query may have already computed the SCEV. It should be
4523 // equivalent, but may not necessarily be exactly the same, e.g. due to lazily
4524 // inferred nowrap flags.
4525 auto It = ValueExprMap.find_as(Val: V);
4526 if (It == ValueExprMap.end()) {
4527 ValueExprMap.insert(KV: {SCEVCallbackVH(V, this), S});
4528 ExprValueMap[S].insert(X: V);
4529 }
4530}
4531
4532/// Return an existing SCEV if it exists, otherwise analyze the expression and
4533/// create a new one.
4534const SCEV *ScalarEvolution::getSCEV(Value *V) {
4535 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4536
4537 if (const SCEV *S = getExistingSCEV(V))
4538 return S;
4539 return createSCEVIter(V);
4540}
4541
4542const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
4543 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4544
4545 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4546 if (I != ValueExprMap.end()) {
4547 const SCEV *S = I->second;
4548 assert(checkValidity(S) &&
4549 "existing SCEV has not been properly invalidated");
4550 return S;
4551 }
4552 return nullptr;
4553}
4554
4555/// Return a SCEV corresponding to -V = -1*V
4556const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V,
4557 SCEV::NoWrapFlags Flags) {
4558 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4559 return getConstant(
4560 V: cast<ConstantInt>(Val: ConstantExpr::getNeg(C: VC->getValue())));
4561
4562 Type *Ty = V->getType();
4563 Ty = getEffectiveSCEVType(Ty);
4564 return getMulExpr(LHS: V, RHS: getMinusOne(Ty), Flags);
4565}
4566
4567/// If Expr computes ~A, return A else return nullptr
4568static const SCEV *MatchNotExpr(const SCEV *Expr) {
4569 const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: Expr);
4570 if (!Add || Add->getNumOperands() != 2 ||
4571 !Add->getOperand(i: 0)->isAllOnesValue())
4572 return nullptr;
4573
4574 const SCEVMulExpr *AddRHS = dyn_cast<SCEVMulExpr>(Val: Add->getOperand(i: 1));
4575 if (!AddRHS || AddRHS->getNumOperands() != 2 ||
4576 !AddRHS->getOperand(i: 0)->isAllOnesValue())
4577 return nullptr;
4578
4579 return AddRHS->getOperand(i: 1);
4580}
4581
4582/// Return a SCEV corresponding to ~V = -1-V
4583const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
4584 assert(!V->getType()->isPointerTy() && "Can't negate pointer");
4585
4586 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4587 return getConstant(
4588 V: cast<ConstantInt>(Val: ConstantExpr::getNot(C: VC->getValue())));
4589
4590 // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
4591 if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(Val: V)) {
4592 auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
4593 SmallVector<const SCEV *, 2> MatchedOperands;
4594 for (const SCEV *Operand : MME->operands()) {
4595 const SCEV *Matched = MatchNotExpr(Expr: Operand);
4596 if (!Matched)
4597 return (const SCEV *)nullptr;
4598 MatchedOperands.push_back(Elt: Matched);
4599 }
4600 return getMinMaxExpr(Kind: SCEVMinMaxExpr::negate(T: MME->getSCEVType()),
4601 Ops&: MatchedOperands);
4602 };
4603 if (const SCEV *Replaced = MatchMinMaxNegation(MME))
4604 return Replaced;
4605 }
4606
4607 Type *Ty = V->getType();
4608 Ty = getEffectiveSCEVType(Ty);
4609 return getMinusSCEV(LHS: getMinusOne(Ty), RHS: V);
4610}
4611
4612const SCEV *ScalarEvolution::removePointerBase(const SCEV *P) {
4613 assert(P->getType()->isPointerTy());
4614
4615 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: P)) {
4616 // The base of an AddRec is the first operand.
4617 SmallVector<const SCEV *> Ops{AddRec->operands()};
4618 Ops[0] = removePointerBase(P: Ops[0]);
4619 // Don't try to transfer nowrap flags for now. We could in some cases
4620 // (for example, if pointer operand of the AddRec is a SCEVUnknown).
4621 return getAddRecExpr(Operands&: Ops, L: AddRec->getLoop(), Flags: SCEV::FlagAnyWrap);
4622 }
4623 if (auto *Add = dyn_cast<SCEVAddExpr>(Val: P)) {
4624 // The base of an Add is the pointer operand.
4625 SmallVector<const SCEV *> Ops{Add->operands()};
4626 const SCEV **PtrOp = nullptr;
4627 for (const SCEV *&AddOp : Ops) {
4628 if (AddOp->getType()->isPointerTy()) {
4629 assert(!PtrOp && "Cannot have multiple pointer ops");
4630 PtrOp = &AddOp;
4631 }
4632 }
4633 *PtrOp = removePointerBase(P: *PtrOp);
4634 // Don't try to transfer nowrap flags for now. We could in some cases
4635 // (for example, if the pointer operand of the Add is a SCEVUnknown).
4636 return getAddExpr(Ops);
4637 }
4638 // Any other expression must be a pointer base.
4639 return getZero(Ty: P->getType());
4640}
4641
4642const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS,
4643 SCEV::NoWrapFlags Flags,
4644 unsigned Depth) {
4645 // Fast path: X - X --> 0.
4646 if (LHS == RHS)
4647 return getZero(Ty: LHS->getType());
4648
4649 // If we subtract two pointers with different pointer bases, bail.
4650 // Eventually, we're going to add an assertion to getMulExpr that we
4651 // can't multiply by a pointer.
4652 if (RHS->getType()->isPointerTy()) {
4653 if (!LHS->getType()->isPointerTy() ||
4654 getPointerBase(V: LHS) != getPointerBase(V: RHS))
4655 return getCouldNotCompute();
4656 LHS = removePointerBase(P: LHS);
4657 RHS = removePointerBase(P: RHS);
4658 }
4659
4660 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation
4661 // makes it so that we cannot make much use of NUW.
4662 auto AddFlags = SCEV::FlagAnyWrap;
4663 const bool RHSIsNotMinSigned =
4664 !getSignedRangeMin(S: RHS).isMinSignedValue();
4665 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW)) {
4666 // Let M be the minimum representable signed value. Then (-1)*RHS
4667 // signed-wraps if and only if RHS is M. That can happen even for
4668 // a NSW subtraction because e.g. (-1)*M signed-wraps even though
4669 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS +
4670 // (-1)*RHS, we need to prove that RHS != M.
4671 //
4672 // If LHS is non-negative and we know that LHS - RHS does not
4673 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap
4674 // either by proving that RHS > M or that LHS >= 0.
4675 if (RHSIsNotMinSigned || isKnownNonNegative(S: LHS)) {
4676 AddFlags = SCEV::FlagNSW;
4677 }
4678 }
4679
4680 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS -
4681 // RHS is NSW and LHS >= 0.
4682 //
4683 // The difficulty here is that the NSW flag may have been proven
4684 // relative to a loop that is to be found in a recurrence in LHS and
4685 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a
4686 // larger scope than intended.
4687 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagAnyWrap;
4688
4689 return getAddExpr(LHS, RHS: getNegativeSCEV(V: RHS, Flags: NegFlags), Flags: AddFlags, Depth);
4690}
4691
4692const SCEV *ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty,
4693 unsigned Depth) {
4694 Type *SrcTy = V->getType();
4695 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4696 "Cannot truncate or zero extend with non-integer arguments!");
4697 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4698 return V; // No conversion
4699 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4700 return getTruncateExpr(Op: V, Ty, Depth);
4701 return getZeroExtendExpr(Op: V, Ty, Depth);
4702}
4703
4704const SCEV *ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, Type *Ty,
4705 unsigned Depth) {
4706 Type *SrcTy = V->getType();
4707 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4708 "Cannot truncate or zero extend with non-integer arguments!");
4709 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4710 return V; // No conversion
4711 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4712 return getTruncateExpr(Op: V, Ty, Depth);
4713 return getSignExtendExpr(Op: V, Ty, Depth);
4714}
4715
4716const SCEV *
4717ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
4718 Type *SrcTy = V->getType();
4719 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4720 "Cannot noop or zero extend with non-integer arguments!");
4721 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4722 "getNoopOrZeroExtend cannot truncate!");
4723 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4724 return V; // No conversion
4725 return getZeroExtendExpr(Op: V, Ty);
4726}
4727
4728const SCEV *
4729ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
4730 Type *SrcTy = V->getType();
4731 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4732 "Cannot noop or sign extend with non-integer arguments!");
4733 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4734 "getNoopOrSignExtend cannot truncate!");
4735 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4736 return V; // No conversion
4737 return getSignExtendExpr(Op: V, Ty);
4738}
4739
4740const SCEV *
4741ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
4742 Type *SrcTy = V->getType();
4743 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4744 "Cannot noop or any extend with non-integer arguments!");
4745 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4746 "getNoopOrAnyExtend cannot truncate!");
4747 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4748 return V; // No conversion
4749 return getAnyExtendExpr(Op: V, Ty);
4750}
4751
4752const SCEV *
4753ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
4754 Type *SrcTy = V->getType();
4755 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4756 "Cannot truncate or noop with non-integer arguments!");
4757 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
4758 "getTruncateOrNoop cannot extend!");
4759 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4760 return V; // No conversion
4761 return getTruncateExpr(Op: V, Ty);
4762}
4763
4764const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
4765 const SCEV *RHS) {
4766 const SCEV *PromotedLHS = LHS;
4767 const SCEV *PromotedRHS = RHS;
4768
4769 if (getTypeSizeInBits(Ty: LHS->getType()) > getTypeSizeInBits(Ty: RHS->getType()))
4770 PromotedRHS = getZeroExtendExpr(Op: RHS, Ty: LHS->getType());
4771 else
4772 PromotedLHS = getNoopOrZeroExtend(V: LHS, Ty: RHS->getType());
4773
4774 return getUMaxExpr(LHS: PromotedLHS, RHS: PromotedRHS);
4775}
4776
4777const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
4778 const SCEV *RHS,
4779 bool Sequential) {
4780 SmallVector<const SCEV *, 2> Ops = { LHS, RHS };
4781 return getUMinFromMismatchedTypes(Ops, Sequential);
4782}
4783
4784const SCEV *
4785ScalarEvolution::getUMinFromMismatchedTypes(SmallVectorImpl<const SCEV *> &Ops,
4786 bool Sequential) {
4787 assert(!Ops.empty() && "At least one operand must be!");
4788 // Trivial case.
4789 if (Ops.size() == 1)
4790 return Ops[0];
4791
4792 // Find the max type first.
4793 Type *MaxType = nullptr;
4794 for (const auto *S : Ops)
4795 if (MaxType)
4796 MaxType = getWiderType(T1: MaxType, T2: S->getType());
4797 else
4798 MaxType = S->getType();
4799 assert(MaxType && "Failed to find maximum type!");
4800
4801 // Extend all ops to max type.
4802 SmallVector<const SCEV *, 2> PromotedOps;
4803 for (const auto *S : Ops)
4804 PromotedOps.push_back(Elt: getNoopOrZeroExtend(V: S, Ty: MaxType));
4805
4806 // Generate umin.
4807 return getUMinExpr(Ops&: PromotedOps, Sequential);
4808}
4809
4810const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
4811 // A pointer operand may evaluate to a nonpointer expression, such as null.
4812 if (!V->getType()->isPointerTy())
4813 return V;
4814
4815 while (true) {
4816 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: V)) {
4817 V = AddRec->getStart();
4818 } else if (auto *Add = dyn_cast<SCEVAddExpr>(Val: V)) {
4819 const SCEV *PtrOp = nullptr;
4820 for (const SCEV *AddOp : Add->operands()) {
4821 if (AddOp->getType()->isPointerTy()) {
4822 assert(!PtrOp && "Cannot have multiple pointer ops");
4823 PtrOp = AddOp;
4824 }
4825 }
4826 assert(PtrOp && "Must have pointer op");
4827 V = PtrOp;
4828 } else // Not something we can look further into.
4829 return V;
4830 }
4831}
4832
4833/// Push users of the given Instruction onto the given Worklist.
4834static void PushDefUseChildren(Instruction *I,
4835 SmallVectorImpl<Instruction *> &Worklist,
4836 SmallPtrSetImpl<Instruction *> &Visited) {
4837 // Push the def-use children onto the Worklist stack.
4838 for (User *U : I->users()) {
4839 auto *UserInsn = cast<Instruction>(Val: U);
4840 if (Visited.insert(Ptr: UserInsn).second)
4841 Worklist.push_back(Elt: UserInsn);
4842 }
4843}
4844
4845namespace {
4846
4847/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its start
4848/// expression in case its Loop is L. If it is not L then
4849/// if IgnoreOtherLoops is true then use AddRec itself
4850/// otherwise rewrite cannot be done.
4851/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4852class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> {
4853public:
4854 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
4855 bool IgnoreOtherLoops = true) {
4856 SCEVInitRewriter Rewriter(L, SE);
4857 const SCEV *Result = Rewriter.visit(S);
4858 if (Rewriter.hasSeenLoopVariantSCEVUnknown())
4859 return SE.getCouldNotCompute();
4860 return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
4861 ? SE.getCouldNotCompute()
4862 : Result;
4863 }
4864
4865 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4866 if (!SE.isLoopInvariant(S: Expr, L))
4867 SeenLoopVariantSCEVUnknown = true;
4868 return Expr;
4869 }
4870
4871 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4872 // Only re-write AddRecExprs for this loop.
4873 if (Expr->getLoop() == L)
4874 return Expr->getStart();
4875 SeenOtherLoops = true;
4876 return Expr;
4877 }
4878
4879 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4880
4881 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4882
4883private:
4884 explicit SCEVInitRewriter(const Loop *L, ScalarEvolution &SE)
4885 : SCEVRewriteVisitor(SE), L(L) {}
4886
4887 const Loop *L;
4888 bool SeenLoopVariantSCEVUnknown = false;
4889 bool SeenOtherLoops = false;
4890};
4891
4892/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its post
4893/// increment expression in case its Loop is L. If it is not L then
4894/// use AddRec itself.
4895/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4896class SCEVPostIncRewriter : public SCEVRewriteVisitor<SCEVPostIncRewriter> {
4897public:
4898 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE) {
4899 SCEVPostIncRewriter Rewriter(L, SE);
4900 const SCEV *Result = Rewriter.visit(S);
4901 return Rewriter.hasSeenLoopVariantSCEVUnknown()
4902 ? SE.getCouldNotCompute()
4903 : Result;
4904 }
4905
4906 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4907 if (!SE.isLoopInvariant(S: Expr, L))
4908 SeenLoopVariantSCEVUnknown = true;
4909 return Expr;
4910 }
4911
4912 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4913 // Only re-write AddRecExprs for this loop.
4914 if (Expr->getLoop() == L)
4915 return Expr->getPostIncExpr(SE);
4916 SeenOtherLoops = true;
4917 return Expr;
4918 }
4919
4920 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4921
4922 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4923
4924private:
4925 explicit SCEVPostIncRewriter(const Loop *L, ScalarEvolution &SE)
4926 : SCEVRewriteVisitor(SE), L(L) {}
4927
4928 const Loop *L;
4929 bool SeenLoopVariantSCEVUnknown = false;
4930 bool SeenOtherLoops = false;
4931};
4932
4933/// This class evaluates the compare condition by matching it against the
4934/// condition of loop latch. If there is a match we assume a true value
4935/// for the condition while building SCEV nodes.
4936class SCEVBackedgeConditionFolder
4937 : public SCEVRewriteVisitor<SCEVBackedgeConditionFolder> {
4938public:
4939 static const SCEV *rewrite(const SCEV *S, const Loop *L,
4940 ScalarEvolution &SE) {
4941 bool IsPosBECond = false;
4942 Value *BECond = nullptr;
4943 if (BasicBlock *Latch = L->getLoopLatch()) {
4944 BranchInst *BI = dyn_cast<BranchInst>(Val: Latch->getTerminator());
4945 if (BI && BI->isConditional()) {
4946 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
4947 "Both outgoing branches should not target same header!");
4948 BECond = BI->getCondition();
4949 IsPosBECond = BI->getSuccessor(i: 0) == L->getHeader();
4950 } else {
4951 return S;
4952 }
4953 }
4954 SCEVBackedgeConditionFolder Rewriter(L, BECond, IsPosBECond, SE);
4955 return Rewriter.visit(S);
4956 }
4957
4958 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4959 const SCEV *Result = Expr;
4960 bool InvariantF = SE.isLoopInvariant(S: Expr, L);
4961
4962 if (!InvariantF) {
4963 Instruction *I = cast<Instruction>(Val: Expr->getValue());
4964 switch (I->getOpcode()) {
4965 case Instruction::Select: {
4966 SelectInst *SI = cast<SelectInst>(Val: I);
4967 std::optional<const SCEV *> Res =
4968 compareWithBackedgeCondition(IC: SI->getCondition());
4969 if (Res) {
4970 bool IsOne = cast<SCEVConstant>(Val: *Res)->getValue()->isOne();
4971 Result = SE.getSCEV(V: IsOne ? SI->getTrueValue() : SI->getFalseValue());
4972 }
4973 break;
4974 }
4975 default: {
4976 std::optional<const SCEV *> Res = compareWithBackedgeCondition(IC: I);
4977 if (Res)
4978 Result = *Res;
4979 break;
4980 }
4981 }
4982 }
4983 return Result;
4984 }
4985
4986private:
4987 explicit SCEVBackedgeConditionFolder(const Loop *L, Value *BECond,
4988 bool IsPosBECond, ScalarEvolution &SE)
4989 : SCEVRewriteVisitor(SE), L(L), BackedgeCond(BECond),
4990 IsPositiveBECond(IsPosBECond) {}
4991
4992 std::optional<const SCEV *> compareWithBackedgeCondition(Value *IC);
4993
4994 const Loop *L;
4995 /// Loop back condition.
4996 Value *BackedgeCond = nullptr;
4997 /// Set to true if loop back is on positive branch condition.
4998 bool IsPositiveBECond;
4999};
5000
5001std::optional<const SCEV *>
5002SCEVBackedgeConditionFolder::compareWithBackedgeCondition(Value *IC) {
5003
5004 // If value matches the backedge condition for loop latch,
5005 // then return a constant evolution node based on loopback
5006 // branch taken.
5007 if (BackedgeCond == IC)
5008 return IsPositiveBECond ? SE.getOne(Ty: Type::getInt1Ty(C&: SE.getContext()))
5009 : SE.getZero(Ty: Type::getInt1Ty(C&: SE.getContext()));
5010 return std::nullopt;
5011}
5012
5013class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> {
5014public:
5015 static const SCEV *rewrite(const SCEV *S, const Loop *L,
5016 ScalarEvolution &SE) {
5017 SCEVShiftRewriter Rewriter(L, SE);
5018 const SCEV *Result = Rewriter.visit(S);
5019 return Rewriter.isValid() ? Result : SE.getCouldNotCompute();
5020 }
5021
5022 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
5023 // Only allow AddRecExprs for this loop.
5024 if (!SE.isLoopInvariant(S: Expr, L))
5025 Valid = false;
5026 return Expr;
5027 }
5028
5029 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
5030 if (Expr->getLoop() == L && Expr->isAffine())
5031 return SE.getMinusSCEV(LHS: Expr, RHS: Expr->getStepRecurrence(SE));
5032 Valid = false;
5033 return Expr;
5034 }
5035
5036 bool isValid() { return Valid; }
5037
5038private:
5039 explicit SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE)
5040 : SCEVRewriteVisitor(SE), L(L) {}
5041
5042 const Loop *L;
5043 bool Valid = true;
5044};
5045
5046} // end anonymous namespace
5047
5048SCEV::NoWrapFlags
5049ScalarEvolution::proveNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) {
5050 if (!AR->isAffine())
5051 return SCEV::FlagAnyWrap;
5052
5053 using OBO = OverflowingBinaryOperator;
5054
5055 SCEV::NoWrapFlags Result = SCEV::FlagAnyWrap;
5056
5057 if (!AR->hasNoSelfWrap()) {
5058 const SCEV *BECount = getConstantMaxBackedgeTakenCount(L: AR->getLoop());
5059 if (const SCEVConstant *BECountMax = dyn_cast<SCEVConstant>(Val: BECount)) {
5060 ConstantRange StepCR = getSignedRange(S: AR->getStepRecurrence(SE&: *this));
5061 const APInt &BECountAP = BECountMax->getAPInt();
5062 unsigned NoOverflowBitWidth =
5063 BECountAP.getActiveBits() + StepCR.getMinSignedBits();
5064 if (NoOverflowBitWidth <= getTypeSizeInBits(Ty: AR->getType()))
5065 Result = ScalarEvolution::setFlags(Flags: Result, OnFlags: SCEV::FlagNW);
5066 }
5067 }
5068
5069 if (!AR->hasNoSignedWrap()) {
5070 ConstantRange AddRecRange = getSignedRange(S: AR);
5071 ConstantRange IncRange = getSignedRange(S: AR->getStepRecurrence(SE&: *this));
5072
5073 auto NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
5074 BinOp: Instruction::Add, Other: IncRange, NoWrapKind: OBO::NoSignedWrap);
5075 if (NSWRegion.contains(CR: AddRecRange))
5076 Result = ScalarEvolution::setFlags(Flags: Result, OnFlags: SCEV::FlagNSW);
5077 }
5078
5079 if (!AR->hasNoUnsignedWrap()) {
5080 ConstantRange AddRecRange = getUnsignedRange(S: AR);
5081 ConstantRange IncRange = getUnsignedRange(S: AR->getStepRecurrence(SE&: *this));
5082
5083 auto NUWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
5084 BinOp: Instruction::Add, Other: IncRange, NoWrapKind: OBO::NoUnsignedWrap);
5085 if (NUWRegion.contains(CR: AddRecRange))
5086 Result = ScalarEvolution::setFlags(Flags: Result, OnFlags: SCEV::FlagNUW);
5087 }
5088
5089 return Result;
5090}
5091
5092SCEV::NoWrapFlags
5093ScalarEvolution::proveNoSignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5094 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
5095
5096 if (AR->hasNoSignedWrap())
5097 return Result;
5098
5099 if (!AR->isAffine())
5100 return Result;
5101
5102 // This function can be expensive, only try to prove NSW once per AddRec.
5103 if (!SignedWrapViaInductionTried.insert(Ptr: AR).second)
5104 return Result;
5105
5106 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5107 const Loop *L = AR->getLoop();
5108
5109 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5110 // Note that this serves two purposes: It filters out loops that are
5111 // simply not analyzable, and it covers the case where this code is
5112 // being called from within backedge-taken count analysis, such that
5113 // attempting to ask for the backedge-taken count would likely result
5114 // in infinite recursion. In the later case, the analysis code will
5115 // cope with a conservative value, and it will take care to purge
5116 // that value once it has finished.
5117 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5118
5119 // Normally, in the cases we can prove no-overflow via a
5120 // backedge guarding condition, we can also compute a backedge
5121 // taken count for the loop. The exceptions are assumptions and
5122 // guards present in the loop -- SCEV is not great at exploiting
5123 // these to compute max backedge taken counts, but can still use
5124 // these to prove lack of overflow. Use this fact to avoid
5125 // doing extra work that may not pay off.
5126
5127 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5128 AC.assumptions().empty())
5129 return Result;
5130
5131 // If the backedge is guarded by a comparison with the pre-inc value the
5132 // addrec is safe. Also, if the entry is guarded by a comparison with the
5133 // start value and the backedge is guarded by a comparison with the post-inc
5134 // value, the addrec is safe.
5135 ICmpInst::Predicate Pred;
5136 const SCEV *OverflowLimit =
5137 getSignedOverflowLimitForStep(Step, Pred: &Pred, SE: this);
5138 if (OverflowLimit &&
5139 (isLoopBackedgeGuardedByCond(L, Pred, LHS: AR, RHS: OverflowLimit) ||
5140 isKnownOnEveryIteration(Pred, LHS: AR, RHS: OverflowLimit))) {
5141 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNSW);
5142 }
5143 return Result;
5144}
5145SCEV::NoWrapFlags
5146ScalarEvolution::proveNoUnsignedWrapViaInduction(const SCEVAddRecExpr *AR) {
5147 SCEV::NoWrapFlags Result = AR->getNoWrapFlags();
5148
5149 if (AR->hasNoUnsignedWrap())
5150 return Result;
5151
5152 if (!AR->isAffine())
5153 return Result;
5154
5155 // This function can be expensive, only try to prove NUW once per AddRec.
5156 if (!UnsignedWrapViaInductionTried.insert(Ptr: AR).second)
5157 return Result;
5158
5159 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5160 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
5161 const Loop *L = AR->getLoop();
5162
5163 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5164 // Note that this serves two purposes: It filters out loops that are
5165 // simply not analyzable, and it covers the case where this code is
5166 // being called from within backedge-taken count analysis, such that
5167 // attempting to ask for the backedge-taken count would likely result
5168 // in infinite recursion. In the later case, the analysis code will
5169 // cope with a conservative value, and it will take care to purge
5170 // that value once it has finished.
5171 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5172
5173 // Normally, in the cases we can prove no-overflow via a
5174 // backedge guarding condition, we can also compute a backedge
5175 // taken count for the loop. The exceptions are assumptions and
5176 // guards present in the loop -- SCEV is not great at exploiting
5177 // these to compute max backedge taken counts, but can still use
5178 // these to prove lack of overflow. Use this fact to avoid
5179 // doing extra work that may not pay off.
5180
5181 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5182 AC.assumptions().empty())
5183 return Result;
5184
5185 // If the backedge is guarded by a comparison with the pre-inc value the
5186 // addrec is safe. Also, if the entry is guarded by a comparison with the
5187 // start value and the backedge is guarded by a comparison with the post-inc
5188 // value, the addrec is safe.
5189 if (isKnownPositive(S: Step)) {
5190 const SCEV *N = getConstant(Val: APInt::getMinValue(numBits: BitWidth) -
5191 getUnsignedRangeMax(S: Step));
5192 if (isLoopBackedgeGuardedByCond(L, Pred: ICmpInst::ICMP_ULT, LHS: AR, RHS: N) ||
5193 isKnownOnEveryIteration(Pred: ICmpInst::ICMP_ULT, LHS: AR, RHS: N)) {
5194 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNUW);
5195 }
5196 }
5197
5198 return Result;
5199}
5200
5201namespace {
5202
5203/// Represents an abstract binary operation. This may exist as a
5204/// normal instruction or constant expression, or may have been
5205/// derived from an expression tree.
5206struct BinaryOp {
5207 unsigned Opcode;
5208 Value *LHS;
5209 Value *RHS;
5210 bool IsNSW = false;
5211 bool IsNUW = false;
5212
5213 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
5214 /// constant expression.
5215 Operator *Op = nullptr;
5216
5217 explicit BinaryOp(Operator *Op)
5218 : Opcode(Op->getOpcode()), LHS(Op->getOperand(i: 0)), RHS(Op->getOperand(i: 1)),
5219 Op(Op) {
5220 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: Op)) {
5221 IsNSW = OBO->hasNoSignedWrap();
5222 IsNUW = OBO->hasNoUnsignedWrap();
5223 }
5224 }
5225
5226 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
5227 bool IsNUW = false)
5228 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
5229};
5230
5231} // end anonymous namespace
5232
5233/// Try to map \p V into a BinaryOp, and return \c std::nullopt on failure.
5234static std::optional<BinaryOp> MatchBinaryOp(Value *V, const DataLayout &DL,
5235 AssumptionCache &AC,
5236 const DominatorTree &DT,
5237 const Instruction *CxtI) {
5238 auto *Op = dyn_cast<Operator>(Val: V);
5239 if (!Op)
5240 return std::nullopt;
5241
5242 // Implementation detail: all the cleverness here should happen without
5243 // creating new SCEV expressions -- our caller knowns tricks to avoid creating
5244 // SCEV expressions when possible, and we should not break that.
5245
5246 switch (Op->getOpcode()) {
5247 case Instruction::Add:
5248 case Instruction::Sub:
5249 case Instruction::Mul:
5250 case Instruction::UDiv:
5251 case Instruction::URem:
5252 case Instruction::And:
5253 case Instruction::AShr:
5254 case Instruction::Shl:
5255 return BinaryOp(Op);
5256
5257 case Instruction::Or: {
5258 // Convert or disjoint into add nuw nsw.
5259 if (cast<PossiblyDisjointInst>(Val: Op)->isDisjoint())
5260 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1),
5261 /*IsNSW=*/true, /*IsNUW=*/true);
5262 return BinaryOp(Op);
5263 }
5264
5265 case Instruction::Xor:
5266 if (auto *RHSC = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1)))
5267 // If the RHS of the xor is a signmask, then this is just an add.
5268 // Instcombine turns add of signmask into xor as a strength reduction step.
5269 if (RHSC->getValue().isSignMask())
5270 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5271 // Binary `xor` is a bit-wise `add`.
5272 if (V->getType()->isIntegerTy(Bitwidth: 1))
5273 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5274 return BinaryOp(Op);
5275
5276 case Instruction::LShr:
5277 // Turn logical shift right of a constant into a unsigned divide.
5278 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1))) {
5279 uint32_t BitWidth = cast<IntegerType>(Val: Op->getType())->getBitWidth();
5280
5281 // If the shift count is not less than the bitwidth, the result of
5282 // the shift is undefined. Don't try to analyze it, because the
5283 // resolution chosen here may differ from the resolution chosen in
5284 // other parts of the compiler.
5285 if (SA->getValue().ult(RHS: BitWidth)) {
5286 Constant *X =
5287 ConstantInt::get(Context&: SA->getContext(),
5288 V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
5289 return BinaryOp(Instruction::UDiv, Op->getOperand(i: 0), X);
5290 }
5291 }
5292 return BinaryOp(Op);
5293
5294 case Instruction::ExtractValue: {
5295 auto *EVI = cast<ExtractValueInst>(Val: Op);
5296 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
5297 break;
5298
5299 auto *WO = dyn_cast<WithOverflowInst>(Val: EVI->getAggregateOperand());
5300 if (!WO)
5301 break;
5302
5303 Instruction::BinaryOps BinOp = WO->getBinaryOp();
5304 bool Signed = WO->isSigned();
5305 // TODO: Should add nuw/nsw flags for mul as well.
5306 if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT))
5307 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
5308
5309 // Now that we know that all uses of the arithmetic-result component of
5310 // CI are guarded by the overflow check, we can go ahead and pretend
5311 // that the arithmetic is non-overflowing.
5312 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
5313 /* IsNSW = */ Signed, /* IsNUW = */ !Signed);
5314 }
5315
5316 default:
5317 break;
5318 }
5319
5320 // Recognise intrinsic loop.decrement.reg, and as this has exactly the same
5321 // semantics as a Sub, return a binary sub expression.
5322 if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
5323 if (II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
5324 return BinaryOp(Instruction::Sub, II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1));
5325
5326 return std::nullopt;
5327}
5328
5329/// Helper function to createAddRecFromPHIWithCasts. We have a phi
5330/// node whose symbolic (unknown) SCEV is \p SymbolicPHI, which is updated via
5331/// the loop backedge by a SCEVAddExpr, possibly also with a few casts on the
5332/// way. This function checks if \p Op, an operand of this SCEVAddExpr,
5333/// follows one of the following patterns:
5334/// Op == (SExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5335/// Op == (ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5336/// If the SCEV expression of \p Op conforms with one of the expected patterns
5337/// we return the type of the truncation operation, and indicate whether the
5338/// truncated type should be treated as signed/unsigned by setting
5339/// \p Signed to true/false, respectively.
5340static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI,
5341 bool &Signed, ScalarEvolution &SE) {
5342 // The case where Op == SymbolicPHI (that is, with no type conversions on
5343 // the way) is handled by the regular add recurrence creating logic and
5344 // would have already been triggered in createAddRecForPHI. Reaching it here
5345 // means that createAddRecFromPHI had failed for this PHI before (e.g.,
5346 // because one of the other operands of the SCEVAddExpr updating this PHI is
5347 // not invariant).
5348 //
5349 // Here we look for the case where Op = (ext(trunc(SymbolicPHI))), and in
5350 // this case predicates that allow us to prove that Op == SymbolicPHI will
5351 // be added.
5352 if (Op == SymbolicPHI)
5353 return nullptr;
5354
5355 unsigned SourceBits = SE.getTypeSizeInBits(Ty: SymbolicPHI->getType());
5356 unsigned NewBits = SE.getTypeSizeInBits(Ty: Op->getType());
5357 if (SourceBits != NewBits)
5358 return nullptr;
5359
5360 const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(Val: Op);
5361 const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: Op);
5362 if (!SExt && !ZExt)
5363 return nullptr;
5364 const SCEVTruncateExpr *Trunc =
5365 SExt ? dyn_cast<SCEVTruncateExpr>(Val: SExt->getOperand())
5366 : dyn_cast<SCEVTruncateExpr>(Val: ZExt->getOperand());
5367 if (!Trunc)
5368 return nullptr;
5369 const SCEV *X = Trunc->getOperand();
5370 if (X != SymbolicPHI)
5371 return nullptr;
5372 Signed = SExt != nullptr;
5373 return Trunc->getType();
5374}
5375
5376static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
5377 if (!PN->getType()->isIntegerTy())
5378 return nullptr;
5379 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5380 if (!L || L->getHeader() != PN->getParent())
5381 return nullptr;
5382 return L;
5383}
5384
5385// Analyze \p SymbolicPHI, a SCEV expression of a phi node, and check if the
5386// computation that updates the phi follows the following pattern:
5387// (SExt/ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy) + InvariantAccum
5388// which correspond to a phi->trunc->sext/zext->add->phi update chain.
5389// If so, try to see if it can be rewritten as an AddRecExpr under some
5390// Predicates. If successful, return them as a pair. Also cache the results
5391// of the analysis.
5392//
5393// Example usage scenario:
5394// Say the Rewriter is called for the following SCEV:
5395// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5396// where:
5397// %X = phi i64 (%Start, %BEValue)
5398// It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
5399// and call this function with %SymbolicPHI = %X.
5400//
5401// The analysis will find that the value coming around the backedge has
5402// the following SCEV:
5403// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5404// Upon concluding that this matches the desired pattern, the function
5405// will return the pair {NewAddRec, SmallPredsVec} where:
5406// NewAddRec = {%Start,+,%Step}
5407// SmallPredsVec = {P1, P2, P3} as follows:
5408// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
5409// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
5410// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
5411// The returned pair means that SymbolicPHI can be rewritten into NewAddRec
5412// under the predicates {P1,P2,P3}.
5413// This predicated rewrite will be cached in PredicatedSCEVRewrites:
5414// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
5415//
5416// TODO's:
5417//
5418// 1) Extend the Induction descriptor to also support inductions that involve
5419// casts: When needed (namely, when we are called in the context of the
5420// vectorizer induction analysis), a Set of cast instructions will be
5421// populated by this method, and provided back to isInductionPHI. This is
5422// needed to allow the vectorizer to properly record them to be ignored by
5423// the cost model and to avoid vectorizing them (otherwise these casts,
5424// which are redundant under the runtime overflow checks, will be
5425// vectorized, which can be costly).
5426//
5427// 2) Support additional induction/PHISCEV patterns: We also want to support
5428// inductions where the sext-trunc / zext-trunc operations (partly) occur
5429// after the induction update operation (the induction increment):
5430//
5431// (Trunc iy (SExt/ZExt ix (%SymbolicPHI + InvariantAccum) to iy) to ix)
5432// which correspond to a phi->add->trunc->sext/zext->phi update chain.
5433//
5434// (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
5435// which correspond to a phi->trunc->add->sext/zext->phi update chain.
5436//
5437// 3) Outline common code with createAddRecFromPHI to avoid duplication.
5438std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5439ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
5440 SmallVector<const SCEVPredicate *, 3> Predicates;
5441
5442 // *** Part1: Analyze if we have a phi-with-cast pattern for which we can
5443 // return an AddRec expression under some predicate.
5444
5445 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5446 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5447 assert(L && "Expecting an integer loop header phi");
5448
5449 // The loop may have multiple entrances or multiple exits; we can analyze
5450 // this phi as an addrec if it has a unique entry value and a unique
5451 // backedge value.
5452 Value *BEValueV = nullptr, *StartValueV = nullptr;
5453 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5454 Value *V = PN->getIncomingValue(i);
5455 if (L->contains(BB: PN->getIncomingBlock(i))) {
5456 if (!BEValueV) {
5457 BEValueV = V;
5458 } else if (BEValueV != V) {
5459 BEValueV = nullptr;
5460 break;
5461 }
5462 } else if (!StartValueV) {
5463 StartValueV = V;
5464 } else if (StartValueV != V) {
5465 StartValueV = nullptr;
5466 break;
5467 }
5468 }
5469 if (!BEValueV || !StartValueV)
5470 return std::nullopt;
5471
5472 const SCEV *BEValue = getSCEV(V: BEValueV);
5473
5474 // If the value coming around the backedge is an add with the symbolic
5475 // value we just inserted, possibly with casts that we can ignore under
5476 // an appropriate runtime guard, then we found a simple induction variable!
5477 const auto *Add = dyn_cast<SCEVAddExpr>(Val: BEValue);
5478 if (!Add)
5479 return std::nullopt;
5480
5481 // If there is a single occurrence of the symbolic value, possibly
5482 // casted, replace it with a recurrence.
5483 unsigned FoundIndex = Add->getNumOperands();
5484 Type *TruncTy = nullptr;
5485 bool Signed;
5486 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5487 if ((TruncTy =
5488 isSimpleCastedPHI(Op: Add->getOperand(i), SymbolicPHI, Signed, SE&: *this)))
5489 if (FoundIndex == e) {
5490 FoundIndex = i;
5491 break;
5492 }
5493
5494 if (FoundIndex == Add->getNumOperands())
5495 return std::nullopt;
5496
5497 // Create an add with everything but the specified operand.
5498 SmallVector<const SCEV *, 8> Ops;
5499 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5500 if (i != FoundIndex)
5501 Ops.push_back(Elt: Add->getOperand(i));
5502 const SCEV *Accum = getAddExpr(Ops);
5503
5504 // The runtime checks will not be valid if the step amount is
5505 // varying inside the loop.
5506 if (!isLoopInvariant(S: Accum, L))
5507 return std::nullopt;
5508
5509 // *** Part2: Create the predicates
5510
5511 // Analysis was successful: we have a phi-with-cast pattern for which we
5512 // can return an AddRec expression under the following predicates:
5513 //
5514 // P1: A Wrap predicate that guarantees that Trunc(Start) + i*Trunc(Accum)
5515 // fits within the truncated type (does not overflow) for i = 0 to n-1.
5516 // P2: An Equal predicate that guarantees that
5517 // Start = (Ext ix (Trunc iy (Start) to ix) to iy)
5518 // P3: An Equal predicate that guarantees that
5519 // Accum = (Ext ix (Trunc iy (Accum) to ix) to iy)
5520 //
5521 // As we next prove, the above predicates guarantee that:
5522 // Start + i*Accum = (Ext ix (Trunc iy ( Start + i*Accum ) to ix) to iy)
5523 //
5524 //
5525 // More formally, we want to prove that:
5526 // Expr(i+1) = Start + (i+1) * Accum
5527 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5528 //
5529 // Given that:
5530 // 1) Expr(0) = Start
5531 // 2) Expr(1) = Start + Accum
5532 // = (Ext ix (Trunc iy (Start) to ix) to iy) + Accum :: from P2
5533 // 3) Induction hypothesis (step i):
5534 // Expr(i) = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum
5535 //
5536 // Proof:
5537 // Expr(i+1) =
5538 // = Start + (i+1)*Accum
5539 // = (Start + i*Accum) + Accum
5540 // = Expr(i) + Accum
5541 // = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum + Accum
5542 // :: from step i
5543 //
5544 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy) + Accum + Accum
5545 //
5546 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy)
5547 // + (Ext ix (Trunc iy (Accum) to ix) to iy)
5548 // + Accum :: from P3
5549 //
5550 // = (Ext ix (Trunc iy ((Start + (i-1)*Accum) + Accum) to ix) to iy)
5551 // + Accum :: from P1: Ext(x)+Ext(y)=>Ext(x+y)
5552 //
5553 // = (Ext ix (Trunc iy (Start + i*Accum) to ix) to iy) + Accum
5554 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5555 //
5556 // By induction, the same applies to all iterations 1<=i<n:
5557 //
5558
5559 // Create a truncated addrec for which we will add a no overflow check (P1).
5560 const SCEV *StartVal = getSCEV(V: StartValueV);
5561 const SCEV *PHISCEV =
5562 getAddRecExpr(Start: getTruncateExpr(Op: StartVal, Ty: TruncTy),
5563 Step: getTruncateExpr(Op: Accum, Ty: TruncTy), L, Flags: SCEV::FlagAnyWrap);
5564
5565 // PHISCEV can be either a SCEVConstant or a SCEVAddRecExpr.
5566 // ex: If truncated Accum is 0 and StartVal is a constant, then PHISCEV
5567 // will be constant.
5568 //
5569 // If PHISCEV is a constant, then P1 degenerates into P2 or P3, so we don't
5570 // add P1.
5571 if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV)) {
5572 SCEVWrapPredicate::IncrementWrapFlags AddedFlags =
5573 Signed ? SCEVWrapPredicate::IncrementNSSW
5574 : SCEVWrapPredicate::IncrementNUSW;
5575 const SCEVPredicate *AddRecPred = getWrapPredicate(AR, AddedFlags);
5576 Predicates.push_back(Elt: AddRecPred);
5577 }
5578
5579 // Create the Equal Predicates P2,P3:
5580
5581 // It is possible that the predicates P2 and/or P3 are computable at
5582 // compile time due to StartVal and/or Accum being constants.
5583 // If either one is, then we can check that now and escape if either P2
5584 // or P3 is false.
5585
5586 // Construct the extended SCEV: (Ext ix (Trunc iy (Expr) to ix) to iy)
5587 // for each of StartVal and Accum
5588 auto getExtendedExpr = [&](const SCEV *Expr,
5589 bool CreateSignExtend) -> const SCEV * {
5590 assert(isLoopInvariant(Expr, L) && "Expr is expected to be invariant");
5591 const SCEV *TruncatedExpr = getTruncateExpr(Op: Expr, Ty: TruncTy);
5592 const SCEV *ExtendedExpr =
5593 CreateSignExtend ? getSignExtendExpr(Op: TruncatedExpr, Ty: Expr->getType())
5594 : getZeroExtendExpr(Op: TruncatedExpr, Ty: Expr->getType());
5595 return ExtendedExpr;
5596 };
5597
5598 // Given:
5599 // ExtendedExpr = (Ext ix (Trunc iy (Expr) to ix) to iy
5600 // = getExtendedExpr(Expr)
5601 // Determine whether the predicate P: Expr == ExtendedExpr
5602 // is known to be false at compile time
5603 auto PredIsKnownFalse = [&](const SCEV *Expr,
5604 const SCEV *ExtendedExpr) -> bool {
5605 return Expr != ExtendedExpr &&
5606 isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: Expr, RHS: ExtendedExpr);
5607 };
5608
5609 const SCEV *StartExtended = getExtendedExpr(StartVal, Signed);
5610 if (PredIsKnownFalse(StartVal, StartExtended)) {
5611 LLVM_DEBUG(dbgs() << "P2 is compile-time false\n";);
5612 return std::nullopt;
5613 }
5614
5615 // The Step is always Signed (because the overflow checks are either
5616 // NSSW or NUSW)
5617 const SCEV *AccumExtended = getExtendedExpr(Accum, /*CreateSignExtend=*/true);
5618 if (PredIsKnownFalse(Accum, AccumExtended)) {
5619 LLVM_DEBUG(dbgs() << "P3 is compile-time false\n";);
5620 return std::nullopt;
5621 }
5622
5623 auto AppendPredicate = [&](const SCEV *Expr,
5624 const SCEV *ExtendedExpr) -> void {
5625 if (Expr != ExtendedExpr &&
5626 !isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: Expr, RHS: ExtendedExpr)) {
5627 const SCEVPredicate *Pred = getEqualPredicate(LHS: Expr, RHS: ExtendedExpr);
5628 LLVM_DEBUG(dbgs() << "Added Predicate: " << *Pred);
5629 Predicates.push_back(Elt: Pred);
5630 }
5631 };
5632
5633 AppendPredicate(StartVal, StartExtended);
5634 AppendPredicate(Accum, AccumExtended);
5635
5636 // *** Part3: Predicates are ready. Now go ahead and create the new addrec in
5637 // which the casts had been folded away. The caller can rewrite SymbolicPHI
5638 // into NewAR if it will also add the runtime overflow checks specified in
5639 // Predicates.
5640 auto *NewAR = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags: SCEV::FlagAnyWrap);
5641
5642 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
5643 std::make_pair(x&: NewAR, y&: Predicates);
5644 // Remember the result of the analysis for this SCEV at this locayyytion.
5645 PredicatedSCEVRewrites[{SymbolicPHI, L}] = PredRewrite;
5646 return PredRewrite;
5647}
5648
5649std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5650ScalarEvolution::createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI) {
5651 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5652 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5653 if (!L)
5654 return std::nullopt;
5655
5656 // Check to see if we already analyzed this PHI.
5657 auto I = PredicatedSCEVRewrites.find(Val: {SymbolicPHI, L});
5658 if (I != PredicatedSCEVRewrites.end()) {
5659 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
5660 I->second;
5661 // Analysis was done before and failed to create an AddRec:
5662 if (Rewrite.first == SymbolicPHI)
5663 return std::nullopt;
5664 // Analysis was done before and succeeded to create an AddRec under
5665 // a predicate:
5666 assert(isa<SCEVAddRecExpr>(Rewrite.first) && "Expected an AddRec");
5667 assert(!(Rewrite.second).empty() && "Expected to find Predicates");
5668 return Rewrite;
5669 }
5670
5671 std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5672 Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
5673
5674 // Record in the cache that the analysis failed
5675 if (!Rewrite) {
5676 SmallVector<const SCEVPredicate *, 3> Predicates;
5677 PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
5678 return std::nullopt;
5679 }
5680
5681 return Rewrite;
5682}
5683
5684// FIXME: This utility is currently required because the Rewriter currently
5685// does not rewrite this expression:
5686// {0, +, (sext ix (trunc iy to ix) to iy)}
5687// into {0, +, %step},
5688// even when the following Equal predicate exists:
5689// "%step == (sext ix (trunc iy to ix) to iy)".
5690bool PredicatedScalarEvolution::areAddRecsEqualWithPreds(
5691 const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2) const {
5692 if (AR1 == AR2)
5693 return true;
5694
5695 auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool {
5696 if (Expr1 != Expr2 &&
5697 !Preds->implies(N: SE.getEqualPredicate(LHS: Expr1, RHS: Expr2), SE) &&
5698 !Preds->implies(N: SE.getEqualPredicate(LHS: Expr2, RHS: Expr1), SE))
5699 return false;
5700 return true;
5701 };
5702
5703 if (!areExprsEqual(AR1->getStart(), AR2->getStart()) ||
5704 !areExprsEqual(AR1->getStepRecurrence(SE), AR2->getStepRecurrence(SE)))
5705 return false;
5706 return true;
5707}
5708
5709/// A helper function for createAddRecFromPHI to handle simple cases.
5710///
5711/// This function tries to find an AddRec expression for the simplest (yet most
5712/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
5713/// If it fails, createAddRecFromPHI will use a more general, but slow,
5714/// technique for finding the AddRec expression.
5715const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
5716 Value *BEValueV,
5717 Value *StartValueV) {
5718 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5719 assert(L && L->getHeader() == PN->getParent());
5720 assert(BEValueV && StartValueV);
5721
5722 auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CxtI: PN);
5723 if (!BO)
5724 return nullptr;
5725
5726 if (BO->Opcode != Instruction::Add)
5727 return nullptr;
5728
5729 const SCEV *Accum = nullptr;
5730 if (BO->LHS == PN && L->isLoopInvariant(V: BO->RHS))
5731 Accum = getSCEV(V: BO->RHS);
5732 else if (BO->RHS == PN && L->isLoopInvariant(V: BO->LHS))
5733 Accum = getSCEV(V: BO->LHS);
5734
5735 if (!Accum)
5736 return nullptr;
5737
5738 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
5739 if (BO->IsNUW)
5740 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5741 if (BO->IsNSW)
5742 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5743
5744 const SCEV *StartVal = getSCEV(V: StartValueV);
5745 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5746 insertValueToMap(V: PN, S: PHISCEV);
5747
5748 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV)) {
5749 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR),
5750 Flags: (SCEV::NoWrapFlags)(AR->getNoWrapFlags() |
5751 proveNoWrapViaConstantRanges(AR)));
5752 }
5753
5754 // We can add Flags to the post-inc expression only if we
5755 // know that it is *undefined behavior* for BEValueV to
5756 // overflow.
5757 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV)) {
5758 assert(isLoopInvariant(Accum, L) &&
5759 "Accum is defined outside L, but is not invariant?");
5760 if (isAddRecNeverPoison(I: BEInst, L))
5761 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5762 }
5763
5764 return PHISCEV;
5765}
5766
5767const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
5768 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5769 if (!L || L->getHeader() != PN->getParent())
5770 return nullptr;
5771
5772 // The loop may have multiple entrances or multiple exits; we can analyze
5773 // this phi as an addrec if it has a unique entry value and a unique
5774 // backedge value.
5775 Value *BEValueV = nullptr, *StartValueV = nullptr;
5776 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5777 Value *V = PN->getIncomingValue(i);
5778 if (L->contains(BB: PN->getIncomingBlock(i))) {
5779 if (!BEValueV) {
5780 BEValueV = V;
5781 } else if (BEValueV != V) {
5782 BEValueV = nullptr;
5783 break;
5784 }
5785 } else if (!StartValueV) {
5786 StartValueV = V;
5787 } else if (StartValueV != V) {
5788 StartValueV = nullptr;
5789 break;
5790 }
5791 }
5792 if (!BEValueV || !StartValueV)
5793 return nullptr;
5794
5795 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
5796 "PHI node already processed?");
5797
5798 // First, try to find AddRec expression without creating a fictituos symbolic
5799 // value for PN.
5800 if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
5801 return S;
5802
5803 // Handle PHI node value symbolically.
5804 const SCEV *SymbolicName = getUnknown(V: PN);
5805 insertValueToMap(V: PN, S: SymbolicName);
5806
5807 // Using this symbolic name for the PHI, analyze the value coming around
5808 // the back-edge.
5809 const SCEV *BEValue = getSCEV(V: BEValueV);
5810
5811 // NOTE: If BEValue is loop invariant, we know that the PHI node just
5812 // has a special value for the first iteration of the loop.
5813
5814 // If the value coming around the backedge is an add with the symbolic
5815 // value we just inserted, then we found a simple induction variable!
5816 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: BEValue)) {
5817 // If there is a single occurrence of the symbolic value, replace it
5818 // with a recurrence.
5819 unsigned FoundIndex = Add->getNumOperands();
5820 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5821 if (Add->getOperand(i) == SymbolicName)
5822 if (FoundIndex == e) {
5823 FoundIndex = i;
5824 break;
5825 }
5826
5827 if (FoundIndex != Add->getNumOperands()) {
5828 // Create an add with everything but the specified operand.
5829 SmallVector<const SCEV *, 8> Ops;
5830 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5831 if (i != FoundIndex)
5832 Ops.push_back(Elt: SCEVBackedgeConditionFolder::rewrite(S: Add->getOperand(i),
5833 L, SE&: *this));
5834 const SCEV *Accum = getAddExpr(Ops);
5835
5836 // This is not a valid addrec if the step amount is varying each
5837 // loop iteration, but is not itself an addrec in this loop.
5838 if (isLoopInvariant(S: Accum, L) ||
5839 (isa<SCEVAddRecExpr>(Val: Accum) &&
5840 cast<SCEVAddRecExpr>(Val: Accum)->getLoop() == L)) {
5841 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
5842
5843 if (auto BO = MatchBinaryOp(V: BEValueV, DL: getDataLayout(), AC, DT, CxtI: PN)) {
5844 if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
5845 if (BO->IsNUW)
5846 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5847 if (BO->IsNSW)
5848 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5849 }
5850 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: BEValueV)) {
5851 if (GEP->getOperand(i_nocapture: 0) == PN) {
5852 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
5853 // If the increment has any nowrap flags, then we know the address
5854 // space cannot be wrapped around.
5855 if (NW != GEPNoWrapFlags::none())
5856 Flags = setFlags(Flags, OnFlags: SCEV::FlagNW);
5857 // If the GEP is nuw or nusw with non-negative offset, we know that
5858 // no unsigned wrap occurs. We cannot set the nsw flag as only the
5859 // offset is treated as signed, while the base is unsigned.
5860 if (NW.hasNoUnsignedWrap() ||
5861 (NW.hasNoUnsignedSignedWrap() && isKnownNonNegative(S: Accum)))
5862 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5863 }
5864
5865 // We cannot transfer nuw and nsw flags from subtraction
5866 // operations -- sub nuw X, Y is not the same as add nuw X, -Y
5867 // for instance.
5868 }
5869
5870 const SCEV *StartVal = getSCEV(V: StartValueV);
5871 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5872
5873 // Okay, for the entire analysis of this edge we assumed the PHI
5874 // to be symbolic. We now need to go back and purge all of the
5875 // entries for the scalars that use the symbolic expression.
5876 forgetMemoizedResults(SCEVs: SymbolicName);
5877 insertValueToMap(V: PN, S: PHISCEV);
5878
5879 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV)) {
5880 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR),
5881 Flags: (SCEV::NoWrapFlags)(AR->getNoWrapFlags() |
5882 proveNoWrapViaConstantRanges(AR)));
5883 }
5884
5885 // We can add Flags to the post-inc expression only if we
5886 // know that it is *undefined behavior* for BEValueV to
5887 // overflow.
5888 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV))
5889 if (isLoopInvariant(S: Accum, L) && isAddRecNeverPoison(I: BEInst, L))
5890 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5891
5892 return PHISCEV;
5893 }
5894 }
5895 } else {
5896 // Otherwise, this could be a loop like this:
5897 // i = 0; for (j = 1; ..; ++j) { .... i = j; }
5898 // In this case, j = {1,+,1} and BEValue is j.
5899 // Because the other in-value of i (0) fits the evolution of BEValue
5900 // i really is an addrec evolution.
5901 //
5902 // We can generalize this saying that i is the shifted value of BEValue
5903 // by one iteration:
5904 // PHI(f(0), f({1,+,1})) --> f({0,+,1})
5905
5906 // Do not allow refinement in rewriting of BEValue.
5907 const SCEV *Shifted = SCEVShiftRewriter::rewrite(S: BEValue, L, SE&: *this);
5908 const SCEV *Start = SCEVInitRewriter::rewrite(S: Shifted, L, SE&: *this, IgnoreOtherLoops: false);
5909 if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
5910 isGuaranteedNotToCauseUB(Op: Shifted) && ::impliesPoison(AssumedPoison: Shifted, S: Start)) {
5911 const SCEV *StartVal = getSCEV(V: StartValueV);
5912 if (Start == StartVal) {
5913 // Okay, for the entire analysis of this edge we assumed the PHI
5914 // to be symbolic. We now need to go back and purge all of the
5915 // entries for the scalars that use the symbolic expression.
5916 forgetMemoizedResults(SCEVs: SymbolicName);
5917 insertValueToMap(V: PN, S: Shifted);
5918 return Shifted;
5919 }
5920 }
5921 }
5922
5923 // Remove the temporary PHI node SCEV that has been inserted while intending
5924 // to create an AddRecExpr for this PHI node. We can not keep this temporary
5925 // as it will prevent later (possibly simpler) SCEV expressions to be added
5926 // to the ValueExprMap.
5927 eraseValueFromMap(V: PN);
5928
5929 return nullptr;
5930}
5931
5932// Try to match a control flow sequence that branches out at BI and merges back
5933// at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful
5934// match.
5935static bool BrPHIToSelect(DominatorTree &DT, BranchInst *BI, PHINode *Merge,
5936 Value *&C, Value *&LHS, Value *&RHS) {
5937 C = BI->getCondition();
5938
5939 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(i: 0));
5940 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(i: 1));
5941
5942 if (!LeftEdge.isSingleEdge())
5943 return false;
5944
5945 assert(RightEdge.isSingleEdge() && "Follows from LeftEdge.isSingleEdge()");
5946
5947 Use &LeftUse = Merge->getOperandUse(i: 0);
5948 Use &RightUse = Merge->getOperandUse(i: 1);
5949
5950 if (DT.dominates(BBE: LeftEdge, U: LeftUse) && DT.dominates(BBE: RightEdge, U: RightUse)) {
5951 LHS = LeftUse;
5952 RHS = RightUse;
5953 return true;
5954 }
5955
5956 if (DT.dominates(BBE: LeftEdge, U: RightUse) && DT.dominates(BBE: RightEdge, U: LeftUse)) {
5957 LHS = RightUse;
5958 RHS = LeftUse;
5959 return true;
5960 }
5961
5962 return false;
5963}
5964
5965const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
5966 auto IsReachable =
5967 [&](BasicBlock *BB) { return DT.isReachableFromEntry(A: BB); };
5968 if (PN->getNumIncomingValues() == 2 && all_of(Range: PN->blocks(), P: IsReachable)) {
5969 // Try to match
5970 //
5971 // br %cond, label %left, label %right
5972 // left:
5973 // br label %merge
5974 // right:
5975 // br label %merge
5976 // merge:
5977 // V = phi [ %x, %left ], [ %y, %right ]
5978 //
5979 // as "select %cond, %x, %y"
5980
5981 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock();
5982 assert(IDom && "At least the entry block should dominate PN");
5983
5984 auto *BI = dyn_cast<BranchInst>(Val: IDom->getTerminator());
5985 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
5986
5987 if (BI && BI->isConditional() &&
5988 BrPHIToSelect(DT, BI, Merge: PN, C&: Cond, LHS, RHS) &&
5989 properlyDominates(S: getSCEV(V: LHS), BB: PN->getParent()) &&
5990 properlyDominates(S: getSCEV(V: RHS), BB: PN->getParent()))
5991 return createNodeForSelectOrPHI(V: PN, Cond, TrueVal: LHS, FalseVal: RHS);
5992 }
5993
5994 return nullptr;
5995}
5996
5997/// Returns SCEV for the first operand of a phi if all phi operands have
5998/// identical opcodes and operands
5999/// eg.
6000/// a: %add = %a + %b
6001/// br %c
6002/// b: %add1 = %a + %b
6003/// br %c
6004/// c: %phi = phi [%add, a], [%add1, b]
6005/// scev(%phi) => scev(%add)
6006const SCEV *
6007ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
6008 BinaryOperator *CommonInst = nullptr;
6009 // Check if instructions are identical.
6010 for (Value *Incoming : PN->incoming_values()) {
6011 auto *IncomingInst = dyn_cast<BinaryOperator>(Val: Incoming);
6012 if (!IncomingInst)
6013 return nullptr;
6014 if (CommonInst) {
6015 if (!CommonInst->isIdenticalToWhenDefined(I: IncomingInst))
6016 return nullptr; // Not identical, give up
6017 } else {
6018 // Remember binary operator
6019 CommonInst = IncomingInst;
6020 }
6021 }
6022 if (!CommonInst)
6023 return nullptr;
6024
6025 // Check if SCEV exprs for instructions are identical.
6026 const SCEV *CommonSCEV = getSCEV(V: CommonInst);
6027 bool SCEVExprsIdentical =
6028 all_of(Range: drop_begin(RangeOrContainer: PN->incoming_values()),
6029 P: [this, CommonSCEV](Value *V) { return CommonSCEV == getSCEV(V); });
6030 return SCEVExprsIdentical ? CommonSCEV : nullptr;
6031}
6032
6033const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
6034 if (const SCEV *S = createAddRecFromPHI(PN))
6035 return S;
6036
6037 // We do not allow simplifying phi (undef, X) to X here, to avoid reusing the
6038 // phi node for X.
6039 if (Value *V = simplifyInstruction(
6040 I: PN, Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
6041 /*UseInstrInfo=*/true, /*CanUseUndef=*/false}))
6042 return getSCEV(V);
6043
6044 if (const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
6045 return S;
6046
6047 if (const SCEV *S = createNodeFromSelectLikePHI(PN))
6048 return S;
6049
6050 // If it's not a loop phi, we can't handle it yet.
6051 return getUnknown(V: PN);
6052}
6053
6054bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind,
6055 SCEVTypes RootKind) {
6056 struct FindClosure {
6057 const SCEV *OperandToFind;
6058 const SCEVTypes RootKind; // Must be a sequential min/max expression.
6059 const SCEVTypes NonSequentialRootKind; // Non-seq variant of RootKind.
6060
6061 bool Found = false;
6062
6063 bool canRecurseInto(SCEVTypes Kind) const {
6064 // We can only recurse into the SCEV expression of the same effective type
6065 // as the type of our root SCEV expression, and into zero-extensions.
6066 return RootKind == Kind || NonSequentialRootKind == Kind ||
6067 scZeroExtend == Kind;
6068 };
6069
6070 FindClosure(const SCEV *OperandToFind, SCEVTypes RootKind)
6071 : OperandToFind(OperandToFind), RootKind(RootKind),
6072 NonSequentialRootKind(
6073 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
6074 Ty: RootKind)) {}
6075
6076 bool follow(const SCEV *S) {
6077 Found = S == OperandToFind;
6078
6079 return !isDone() && canRecurseInto(Kind: S->getSCEVType());
6080 }
6081
6082 bool isDone() const { return Found; }
6083 };
6084
6085 FindClosure FC(OperandToFind, RootKind);
6086 visitAll(Root, Visitor&: FC);
6087 return FC.Found;
6088}
6089
6090std::optional<const SCEV *>
6091ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond(Type *Ty,
6092 ICmpInst *Cond,
6093 Value *TrueVal,
6094 Value *FalseVal) {
6095 // Try to match some simple smax or umax patterns.
6096 auto *ICI = Cond;
6097
6098 Value *LHS = ICI->getOperand(i_nocapture: 0);
6099 Value *RHS = ICI->getOperand(i_nocapture: 1);
6100
6101 switch (ICI->getPredicate()) {
6102 case ICmpInst::ICMP_SLT:
6103 case ICmpInst::ICMP_SLE:
6104 case ICmpInst::ICMP_ULT:
6105 case ICmpInst::ICMP_ULE:
6106 std::swap(a&: LHS, b&: RHS);
6107 [[fallthrough]];
6108 case ICmpInst::ICMP_SGT:
6109 case ICmpInst::ICMP_SGE:
6110 case ICmpInst::ICMP_UGT:
6111 case ICmpInst::ICMP_UGE:
6112 // a > b ? a+x : b+x -> max(a, b)+x
6113 // a > b ? b+x : a+x -> min(a, b)+x
6114 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty)) {
6115 bool Signed = ICI->isSigned();
6116 const SCEV *LA = getSCEV(V: TrueVal);
6117 const SCEV *RA = getSCEV(V: FalseVal);
6118 const SCEV *LS = getSCEV(V: LHS);
6119 const SCEV *RS = getSCEV(V: RHS);
6120 if (LA->getType()->isPointerTy()) {
6121 // FIXME: Handle cases where LS/RS are pointers not equal to LA/RA.
6122 // Need to make sure we can't produce weird expressions involving
6123 // negated pointers.
6124 if (LA == LS && RA == RS)
6125 return Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS);
6126 if (LA == RS && RA == LS)
6127 return Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS);
6128 }
6129 auto CoerceOperand = [&](const SCEV *Op) -> const SCEV * {
6130 if (Op->getType()->isPointerTy()) {
6131 Op = getLosslessPtrToIntExpr(Op);
6132 if (isa<SCEVCouldNotCompute>(Val: Op))
6133 return Op;
6134 }
6135 if (Signed)
6136 Op = getNoopOrSignExtend(V: Op, Ty);
6137 else
6138 Op = getNoopOrZeroExtend(V: Op, Ty);
6139 return Op;
6140 };
6141 LS = CoerceOperand(LS);
6142 RS = CoerceOperand(RS);
6143 if (isa<SCEVCouldNotCompute>(Val: LS) || isa<SCEVCouldNotCompute>(Val: RS))
6144 break;
6145 const SCEV *LDiff = getMinusSCEV(LHS: LA, RHS: LS);
6146 const SCEV *RDiff = getMinusSCEV(LHS: RA, RHS: RS);
6147 if (LDiff == RDiff)
6148 return getAddExpr(LHS: Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS),
6149 RHS: LDiff);
6150 LDiff = getMinusSCEV(LHS: LA, RHS: RS);
6151 RDiff = getMinusSCEV(LHS: RA, RHS: LS);
6152 if (LDiff == RDiff)
6153 return getAddExpr(LHS: Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS),
6154 RHS: LDiff);
6155 }
6156 break;
6157 case ICmpInst::ICMP_NE:
6158 // x != 0 ? x+y : C+y -> x == 0 ? C+y : x+y
6159 std::swap(a&: TrueVal, b&: FalseVal);
6160 [[fallthrough]];
6161 case ICmpInst::ICMP_EQ:
6162 // x == 0 ? C+y : x+y -> umax(x, C)+y iff C u<= 1
6163 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty) &&
6164 isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()) {
6165 const SCEV *X = getNoopOrZeroExtend(V: getSCEV(V: LHS), Ty);
6166 const SCEV *TrueValExpr = getSCEV(V: TrueVal); // C+y
6167 const SCEV *FalseValExpr = getSCEV(V: FalseVal); // x+y
6168 const SCEV *Y = getMinusSCEV(LHS: FalseValExpr, RHS: X); // y = (x+y)-x
6169 const SCEV *C = getMinusSCEV(LHS: TrueValExpr, RHS: Y); // C = (C+y)-y
6170 if (isa<SCEVConstant>(Val: C) && cast<SCEVConstant>(Val: C)->getAPInt().ule(RHS: 1))
6171 return getAddExpr(LHS: getUMaxExpr(LHS: X, RHS: C), RHS: Y);
6172 }
6173 // x == 0 ? 0 : umin (..., x, ...) -> umin_seq(x, umin (...))
6174 // x == 0 ? 0 : umin_seq(..., x, ...) -> umin_seq(x, umin_seq(...))
6175 // x == 0 ? 0 : umin (..., umin_seq(..., x, ...), ...)
6176 // -> umin_seq(x, umin (..., umin_seq(...), ...))
6177 if (isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero() &&
6178 isa<ConstantInt>(Val: TrueVal) && cast<ConstantInt>(Val: TrueVal)->isZero()) {
6179 const SCEV *X = getSCEV(V: LHS);
6180 while (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: X))
6181 X = ZExt->getOperand();
6182 if (getTypeSizeInBits(Ty: X->getType()) <= getTypeSizeInBits(Ty)) {
6183 const SCEV *FalseValExpr = getSCEV(V: FalseVal);
6184 if (SCEVMinMaxExprContains(Root: FalseValExpr, OperandToFind: X, RootKind: scSequentialUMinExpr))
6185 return getUMinExpr(LHS: getNoopOrZeroExtend(V: X, Ty), RHS: FalseValExpr,
6186 /*Sequential=*/true);
6187 }
6188 }
6189 break;
6190 default:
6191 break;
6192 }
6193
6194 return std::nullopt;
6195}
6196
6197static std::optional<const SCEV *>
6198createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr,
6199 const SCEV *TrueExpr, const SCEV *FalseExpr) {
6200 assert(CondExpr->getType()->isIntegerTy(1) &&
6201 TrueExpr->getType() == FalseExpr->getType() &&
6202 TrueExpr->getType()->isIntegerTy(1) &&
6203 "Unexpected operands of a select.");
6204
6205 // i1 cond ? i1 x : i1 C --> C + (i1 cond ? (i1 x - i1 C) : i1 0)
6206 // --> C + (umin_seq cond, x - C)
6207 //
6208 // i1 cond ? i1 C : i1 x --> C + (i1 cond ? i1 0 : (i1 x - i1 C))
6209 // --> C + (i1 ~cond ? (i1 x - i1 C) : i1 0)
6210 // --> C + (umin_seq ~cond, x - C)
6211
6212 // FIXME: while we can't legally model the case where both of the hands
6213 // are fully variable, we only require that the *difference* is constant.
6214 if (!isa<SCEVConstant>(Val: TrueExpr) && !isa<SCEVConstant>(Val: FalseExpr))
6215 return std::nullopt;
6216
6217 const SCEV *X, *C;
6218 if (isa<SCEVConstant>(Val: TrueExpr)) {
6219 CondExpr = SE->getNotSCEV(V: CondExpr);
6220 X = FalseExpr;
6221 C = TrueExpr;
6222 } else {
6223 X = TrueExpr;
6224 C = FalseExpr;
6225 }
6226 return SE->getAddExpr(LHS: C, RHS: SE->getUMinExpr(LHS: CondExpr, RHS: SE->getMinusSCEV(LHS: X, RHS: C),
6227 /*Sequential=*/true));
6228}
6229
6230static std::optional<const SCEV *>
6231createNodeForSelectViaUMinSeq(ScalarEvolution *SE, Value *Cond, Value *TrueVal,
6232 Value *FalseVal) {
6233 if (!isa<ConstantInt>(Val: TrueVal) && !isa<ConstantInt>(Val: FalseVal))
6234 return std::nullopt;
6235
6236 const auto *SECond = SE->getSCEV(V: Cond);
6237 const auto *SETrue = SE->getSCEV(V: TrueVal);
6238 const auto *SEFalse = SE->getSCEV(V: FalseVal);
6239 return createNodeForSelectViaUMinSeq(SE, CondExpr: SECond, TrueExpr: SETrue, FalseExpr: SEFalse);
6240}
6241
6242const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq(
6243 Value *V, Value *Cond, Value *TrueVal, Value *FalseVal) {
6244 assert(Cond->getType()->isIntegerTy(1) && "Select condition is not an i1?");
6245 assert(TrueVal->getType() == FalseVal->getType() &&
6246 V->getType() == TrueVal->getType() &&
6247 "Types of select hands and of the result must match.");
6248
6249 // For now, only deal with i1-typed `select`s.
6250 if (!V->getType()->isIntegerTy(Bitwidth: 1))
6251 return getUnknown(V);
6252
6253 if (std::optional<const SCEV *> S =
6254 createNodeForSelectViaUMinSeq(SE: this, Cond, TrueVal, FalseVal))
6255 return *S;
6256
6257 return getUnknown(V);
6258}
6259
6260const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Value *V, Value *Cond,
6261 Value *TrueVal,
6262 Value *FalseVal) {
6263 // Handle "constant" branch or select. This can occur for instance when a
6264 // loop pass transforms an inner loop and moves on to process the outer loop.
6265 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond))
6266 return getSCEV(V: CI->isOne() ? TrueVal : FalseVal);
6267
6268 if (auto *I = dyn_cast<Instruction>(Val: V)) {
6269 if (auto *ICI = dyn_cast<ICmpInst>(Val: Cond)) {
6270 if (std::optional<const SCEV *> S =
6271 createNodeForSelectOrPHIInstWithICmpInstCond(Ty: I->getType(), Cond: ICI,
6272 TrueVal, FalseVal))
6273 return *S;
6274 }
6275 }
6276
6277 return createNodeForSelectOrPHIViaUMinSeq(V, Cond, TrueVal, FalseVal);
6278}
6279
6280/// Expand GEP instructions into add and multiply operations. This allows them
6281/// to be analyzed by regular SCEV code.
6282const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
6283 assert(GEP->getSourceElementType()->isSized() &&
6284 "GEP source element type must be sized");
6285
6286 SmallVector<const SCEV *, 4> IndexExprs;
6287 for (Value *Index : GEP->indices())
6288 IndexExprs.push_back(Elt: getSCEV(V: Index));
6289 return getGEPExpr(GEP, IndexExprs);
6290}
6291
6292APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S) {
6293 uint64_t BitWidth = getTypeSizeInBits(Ty: S->getType());
6294 auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) {
6295 return TrailingZeros >= BitWidth
6296 ? APInt::getZero(numBits: BitWidth)
6297 : APInt::getOneBitSet(numBits: BitWidth, BitNo: TrailingZeros);
6298 };
6299 auto GetGCDMultiple = [this](const SCEVNAryExpr *N) {
6300 // The result is GCD of all operands results.
6301 APInt Res = getConstantMultiple(S: N->getOperand(i: 0));
6302 for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I)
6303 Res = APIntOps::GreatestCommonDivisor(
6304 A: Res, B: getConstantMultiple(S: N->getOperand(i: I)));
6305 return Res;
6306 };
6307
6308 switch (S->getSCEVType()) {
6309 case scConstant:
6310 return cast<SCEVConstant>(Val: S)->getAPInt();
6311 case scPtrToInt:
6312 return getConstantMultiple(S: cast<SCEVPtrToIntExpr>(Val: S)->getOperand());
6313 case scUDivExpr:
6314 case scVScale:
6315 return APInt(BitWidth, 1);
6316 case scTruncate: {
6317 // Only multiples that are a power of 2 will hold after truncation.
6318 const SCEVTruncateExpr *T = cast<SCEVTruncateExpr>(Val: S);
6319 uint32_t TZ = getMinTrailingZeros(S: T->getOperand());
6320 return GetShiftedByZeros(TZ);
6321 }
6322 case scZeroExtend: {
6323 const SCEVZeroExtendExpr *Z = cast<SCEVZeroExtendExpr>(Val: S);
6324 return getConstantMultiple(S: Z->getOperand()).zext(width: BitWidth);
6325 }
6326 case scSignExtend: {
6327 // Only multiples that are a power of 2 will hold after sext.
6328 const SCEVSignExtendExpr *E = cast<SCEVSignExtendExpr>(Val: S);
6329 uint32_t TZ = getMinTrailingZeros(S: E->getOperand());
6330 return GetShiftedByZeros(TZ);
6331 }
6332 case scMulExpr: {
6333 const SCEVMulExpr *M = cast<SCEVMulExpr>(Val: S);
6334 if (M->hasNoUnsignedWrap()) {
6335 // The result is the product of all operand results.
6336 APInt Res = getConstantMultiple(S: M->getOperand(i: 0));
6337 for (const SCEV *Operand : M->operands().drop_front())
6338 Res = Res * getConstantMultiple(S: Operand);
6339 return Res;
6340 }
6341
6342 // If there are no wrap guarentees, find the trailing zeros, which is the
6343 // sum of trailing zeros for all its operands.
6344 uint32_t TZ = 0;
6345 for (const SCEV *Operand : M->operands())
6346 TZ += getMinTrailingZeros(S: Operand);
6347 return GetShiftedByZeros(TZ);
6348 }
6349 case scAddExpr:
6350 case scAddRecExpr: {
6351 const SCEVNAryExpr *N = cast<SCEVNAryExpr>(Val: S);
6352 if (N->hasNoUnsignedWrap())
6353 return GetGCDMultiple(N);
6354 // Find the trailing bits, which is the minimum of its operands.
6355 uint32_t TZ = getMinTrailingZeros(S: N->getOperand(i: 0));
6356 for (const SCEV *Operand : N->operands().drop_front())
6357 TZ = std::min(a: TZ, b: getMinTrailingZeros(S: Operand));
6358 return GetShiftedByZeros(TZ);
6359 }
6360 case scUMaxExpr:
6361 case scSMaxExpr:
6362 case scUMinExpr:
6363 case scSMinExpr:
6364 case scSequentialUMinExpr:
6365 return GetGCDMultiple(cast<SCEVNAryExpr>(Val: S));
6366 case scUnknown: {
6367 // ask ValueTracking for known bits
6368 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6369 unsigned Known =
6370 computeKnownBits(V: U->getValue(), DL: getDataLayout(), AC: &AC, CxtI: nullptr, DT: &DT)
6371 .countMinTrailingZeros();
6372 return GetShiftedByZeros(Known);
6373 }
6374 case scCouldNotCompute:
6375 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6376 }
6377 llvm_unreachable("Unknown SCEV kind!");
6378}
6379
6380APInt ScalarEvolution::getConstantMultiple(const SCEV *S) {
6381 auto I = ConstantMultipleCache.find(Val: S);
6382 if (I != ConstantMultipleCache.end())
6383 return I->second;
6384
6385 APInt Result = getConstantMultipleImpl(S);
6386 auto InsertPair = ConstantMultipleCache.insert(KV: {S, Result});
6387 assert(InsertPair.second && "Should insert a new key");
6388 return InsertPair.first->second;
6389}
6390
6391APInt ScalarEvolution::getNonZeroConstantMultiple(const SCEV *S) {
6392 APInt Multiple = getConstantMultiple(S);
6393 return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple;
6394}
6395
6396uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S) {
6397 return std::min(a: getConstantMultiple(S).countTrailingZeros(),
6398 b: (unsigned)getTypeSizeInBits(Ty: S->getType()));
6399}
6400
6401/// Helper method to assign a range to V from metadata present in the IR.
6402static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
6403 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
6404 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_range))
6405 return getConstantRangeFromMetadata(RangeMD: *MD);
6406 if (const auto *CB = dyn_cast<CallBase>(Val: V))
6407 if (std::optional<ConstantRange> Range = CB->getRange())
6408 return Range;
6409 }
6410 if (auto *A = dyn_cast<Argument>(Val: V))
6411 if (std::optional<ConstantRange> Range = A->getRange())
6412 return Range;
6413
6414 return std::nullopt;
6415}
6416
6417void ScalarEvolution::setNoWrapFlags(SCEVAddRecExpr *AddRec,
6418 SCEV::NoWrapFlags Flags) {
6419 if (AddRec->getNoWrapFlags(Mask: Flags) != Flags) {
6420 AddRec->setNoWrapFlags(Flags);
6421 UnsignedRanges.erase(Val: AddRec);
6422 SignedRanges.erase(Val: AddRec);
6423 ConstantMultipleCache.erase(Val: AddRec);
6424 }
6425}
6426
6427ConstantRange ScalarEvolution::
6428getRangeForUnknownRecurrence(const SCEVUnknown *U) {
6429 const DataLayout &DL = getDataLayout();
6430
6431 unsigned BitWidth = getTypeSizeInBits(Ty: U->getType());
6432 const ConstantRange FullSet(BitWidth, /*isFullSet=*/true);
6433
6434 // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
6435 // use information about the trip count to improve our available range. Note
6436 // that the trip count independent cases are already handled by known bits.
6437 // WARNING: The definition of recurrence used here is subtly different than
6438 // the one used by AddRec (and thus most of this file). Step is allowed to
6439 // be arbitrarily loop varying here, where AddRec allows only loop invariant
6440 // and other addrecs in the same loop (for non-affine addrecs). The code
6441 // below intentionally handles the case where step is not loop invariant.
6442 auto *P = dyn_cast<PHINode>(Val: U->getValue());
6443 if (!P)
6444 return FullSet;
6445
6446 // Make sure that no Phi input comes from an unreachable block. Otherwise,
6447 // even the values that are not available in these blocks may come from them,
6448 // and this leads to false-positive recurrence test.
6449 for (auto *Pred : predecessors(BB: P->getParent()))
6450 if (!DT.isReachableFromEntry(A: Pred))
6451 return FullSet;
6452
6453 BinaryOperator *BO;
6454 Value *Start, *Step;
6455 if (!matchSimpleRecurrence(P, BO, Start, Step))
6456 return FullSet;
6457
6458 // If we found a recurrence in reachable code, we must be in a loop. Note
6459 // that BO might be in some subloop of L, and that's completely okay.
6460 auto *L = LI.getLoopFor(BB: P->getParent());
6461 assert(L && L->getHeader() == P->getParent());
6462 if (!L->contains(BB: BO->getParent()))
6463 // NOTE: This bailout should be an assert instead. However, asserting
6464 // the condition here exposes a case where LoopFusion is querying SCEV
6465 // with malformed loop information during the midst of the transform.
6466 // There doesn't appear to be an obvious fix, so for the moment bailout
6467 // until the caller issue can be fixed. PR49566 tracks the bug.
6468 return FullSet;
6469
6470 // TODO: Extend to other opcodes such as mul, and div
6471 switch (BO->getOpcode()) {
6472 default:
6473 return FullSet;
6474 case Instruction::AShr:
6475 case Instruction::LShr:
6476 case Instruction::Shl:
6477 break;
6478 };
6479
6480 if (BO->getOperand(i_nocapture: 0) != P)
6481 // TODO: Handle the power function forms some day.
6482 return FullSet;
6483
6484 unsigned TC = getSmallConstantMaxTripCount(L);
6485 if (!TC || TC >= BitWidth)
6486 return FullSet;
6487
6488 auto KnownStart = computeKnownBits(V: Start, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6489 auto KnownStep = computeKnownBits(V: Step, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6490 assert(KnownStart.getBitWidth() == BitWidth &&
6491 KnownStep.getBitWidth() == BitWidth);
6492
6493 // Compute total shift amount, being careful of overflow and bitwidths.
6494 auto MaxShiftAmt = KnownStep.getMaxValue();
6495 APInt TCAP(BitWidth, TC-1);
6496 bool Overflow = false;
6497 auto TotalShift = MaxShiftAmt.umul_ov(RHS: TCAP, Overflow);
6498 if (Overflow)
6499 return FullSet;
6500
6501 switch (BO->getOpcode()) {
6502 default:
6503 llvm_unreachable("filtered out above");
6504 case Instruction::AShr: {
6505 // For each ashr, three cases:
6506 // shift = 0 => unchanged value
6507 // saturation => 0 or -1
6508 // other => a value closer to zero (of the same sign)
6509 // Thus, the end value is closer to zero than the start.
6510 auto KnownEnd = KnownBits::ashr(LHS: KnownStart,
6511 RHS: KnownBits::makeConstant(C: TotalShift));
6512 if (KnownStart.isNonNegative())
6513 // Analogous to lshr (simply not yet canonicalized)
6514 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6515 Upper: KnownStart.getMaxValue() + 1);
6516 if (KnownStart.isNegative())
6517 // End >=u Start && End <=s Start
6518 return ConstantRange::getNonEmpty(Lower: KnownStart.getMinValue(),
6519 Upper: KnownEnd.getMaxValue() + 1);
6520 break;
6521 }
6522 case Instruction::LShr: {
6523 // For each lshr, three cases:
6524 // shift = 0 => unchanged value
6525 // saturation => 0
6526 // other => a smaller positive number
6527 // Thus, the low end of the unsigned range is the last value produced.
6528 auto KnownEnd = KnownBits::lshr(LHS: KnownStart,
6529 RHS: KnownBits::makeConstant(C: TotalShift));
6530 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6531 Upper: KnownStart.getMaxValue() + 1);
6532 }
6533 case Instruction::Shl: {
6534 // Iff no bits are shifted out, value increases on every shift.
6535 auto KnownEnd = KnownBits::shl(LHS: KnownStart,
6536 RHS: KnownBits::makeConstant(C: TotalShift));
6537 if (TotalShift.ult(RHS: KnownStart.countMinLeadingZeros()))
6538 return ConstantRange(KnownStart.getMinValue(),
6539 KnownEnd.getMaxValue() + 1);
6540 break;
6541 }
6542 };
6543 return FullSet;
6544}
6545
6546const ConstantRange &
6547ScalarEvolution::getRangeRefIter(const SCEV *S,
6548 ScalarEvolution::RangeSignHint SignHint) {
6549 DenseMap<const SCEV *, ConstantRange> &Cache =
6550 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6551 : SignedRanges;
6552 SmallVector<const SCEV *> WorkList;
6553 SmallPtrSet<const SCEV *, 8> Seen;
6554
6555 // Add Expr to the worklist, if Expr is either an N-ary expression or a
6556 // SCEVUnknown PHI node.
6557 auto AddToWorklist = [&WorkList, &Seen, &Cache](const SCEV *Expr) {
6558 if (!Seen.insert(Ptr: Expr).second)
6559 return;
6560 if (Cache.contains(Val: Expr))
6561 return;
6562 switch (Expr->getSCEVType()) {
6563 case scUnknown:
6564 if (!isa<PHINode>(Val: cast<SCEVUnknown>(Val: Expr)->getValue()))
6565 break;
6566 [[fallthrough]];
6567 case scConstant:
6568 case scVScale:
6569 case scTruncate:
6570 case scZeroExtend:
6571 case scSignExtend:
6572 case scPtrToInt:
6573 case scAddExpr:
6574 case scMulExpr:
6575 case scUDivExpr:
6576 case scAddRecExpr:
6577 case scUMaxExpr:
6578 case scSMaxExpr:
6579 case scUMinExpr:
6580 case scSMinExpr:
6581 case scSequentialUMinExpr:
6582 WorkList.push_back(Elt: Expr);
6583 break;
6584 case scCouldNotCompute:
6585 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6586 }
6587 };
6588 AddToWorklist(S);
6589
6590 // Build worklist by queuing operands of N-ary expressions and phi nodes.
6591 for (unsigned I = 0; I != WorkList.size(); ++I) {
6592 const SCEV *P = WorkList[I];
6593 auto *UnknownS = dyn_cast<SCEVUnknown>(Val: P);
6594 // If it is not a `SCEVUnknown`, just recurse into operands.
6595 if (!UnknownS) {
6596 for (const SCEV *Op : P->operands())
6597 AddToWorklist(Op);
6598 continue;
6599 }
6600 // `SCEVUnknown`'s require special treatment.
6601 if (const PHINode *P = dyn_cast<PHINode>(Val: UnknownS->getValue())) {
6602 if (!PendingPhiRangesIter.insert(Ptr: P).second)
6603 continue;
6604 for (auto &Op : reverse(C: P->operands()))
6605 AddToWorklist(getSCEV(V: Op));
6606 }
6607 }
6608
6609 if (!WorkList.empty()) {
6610 // Use getRangeRef to compute ranges for items in the worklist in reverse
6611 // order. This will force ranges for earlier operands to be computed before
6612 // their users in most cases.
6613 for (const SCEV *P : reverse(C: drop_begin(RangeOrContainer&: WorkList))) {
6614 getRangeRef(S: P, Hint: SignHint);
6615
6616 if (auto *UnknownS = dyn_cast<SCEVUnknown>(Val: P))
6617 if (const PHINode *P = dyn_cast<PHINode>(Val: UnknownS->getValue()))
6618 PendingPhiRangesIter.erase(Ptr: P);
6619 }
6620 }
6621
6622 return getRangeRef(S, Hint: SignHint, Depth: 0);
6623}
6624
6625/// Determine the range for a particular SCEV. If SignHint is
6626/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
6627/// with a "cleaner" unsigned (resp. signed) representation.
6628const ConstantRange &ScalarEvolution::getRangeRef(
6629 const SCEV *S, ScalarEvolution::RangeSignHint SignHint, unsigned Depth) {
6630 DenseMap<const SCEV *, ConstantRange> &Cache =
6631 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6632 : SignedRanges;
6633 ConstantRange::PreferredRangeType RangeType =
6634 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? ConstantRange::Unsigned
6635 : ConstantRange::Signed;
6636
6637 // See if we've computed this range already.
6638 DenseMap<const SCEV *, ConstantRange>::iterator I = Cache.find(Val: S);
6639 if (I != Cache.end())
6640 return I->second;
6641
6642 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: S))
6643 return setRange(S: C, Hint: SignHint, CR: ConstantRange(C->getAPInt()));
6644
6645 // Switch to iteratively computing the range for S, if it is part of a deeply
6646 // nested expression.
6647 if (Depth > RangeIterThreshold)
6648 return getRangeRefIter(S, SignHint);
6649
6650 unsigned BitWidth = getTypeSizeInBits(Ty: S->getType());
6651 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
6652 using OBO = OverflowingBinaryOperator;
6653
6654 // If the value has known zeros, the maximum value will have those known zeros
6655 // as well.
6656 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
6657 APInt Multiple = getNonZeroConstantMultiple(S);
6658 APInt Remainder = APInt::getMaxValue(numBits: BitWidth).urem(RHS: Multiple);
6659 if (!Remainder.isZero())
6660 ConservativeResult =
6661 ConstantRange(APInt::getMinValue(numBits: BitWidth),
6662 APInt::getMaxValue(numBits: BitWidth) - Remainder + 1);
6663 }
6664 else {
6665 uint32_t TZ = getMinTrailingZeros(S);
6666 if (TZ != 0) {
6667 ConservativeResult = ConstantRange(
6668 APInt::getSignedMinValue(numBits: BitWidth),
6669 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: TZ).shl(shiftAmt: TZ) + 1);
6670 }
6671 }
6672
6673 switch (S->getSCEVType()) {
6674 case scConstant:
6675 llvm_unreachable("Already handled above.");
6676 case scVScale:
6677 return setRange(S, Hint: SignHint, CR: getVScaleRange(F: &F, BitWidth));
6678 case scTruncate: {
6679 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: S);
6680 ConstantRange X = getRangeRef(S: Trunc->getOperand(), SignHint, Depth: Depth + 1);
6681 return setRange(
6682 S: Trunc, Hint: SignHint,
6683 CR: ConservativeResult.intersectWith(CR: X.truncate(BitWidth), Type: RangeType));
6684 }
6685 case scZeroExtend: {
6686 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: S);
6687 ConstantRange X = getRangeRef(S: ZExt->getOperand(), SignHint, Depth: Depth + 1);
6688 return setRange(
6689 S: ZExt, Hint: SignHint,
6690 CR: ConservativeResult.intersectWith(CR: X.zeroExtend(BitWidth), Type: RangeType));
6691 }
6692 case scSignExtend: {
6693 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: S);
6694 ConstantRange X = getRangeRef(S: SExt->getOperand(), SignHint, Depth: Depth + 1);
6695 return setRange(
6696 S: SExt, Hint: SignHint,
6697 CR: ConservativeResult.intersectWith(CR: X.signExtend(BitWidth), Type: RangeType));
6698 }
6699 case scPtrToInt: {
6700 const SCEVPtrToIntExpr *PtrToInt = cast<SCEVPtrToIntExpr>(Val: S);
6701 ConstantRange X = getRangeRef(S: PtrToInt->getOperand(), SignHint, Depth: Depth + 1);
6702 return setRange(S: PtrToInt, Hint: SignHint, CR: X);
6703 }
6704 case scAddExpr: {
6705 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: S);
6706 ConstantRange X = getRangeRef(S: Add->getOperand(i: 0), SignHint, Depth: Depth + 1);
6707 unsigned WrapType = OBO::AnyWrap;
6708 if (Add->hasNoSignedWrap())
6709 WrapType |= OBO::NoSignedWrap;
6710 if (Add->hasNoUnsignedWrap())
6711 WrapType |= OBO::NoUnsignedWrap;
6712 for (const SCEV *Op : drop_begin(RangeOrContainer: Add->operands()))
6713 X = X.addWithNoWrap(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1), NoWrapKind: WrapType,
6714 RangeType);
6715 return setRange(S: Add, Hint: SignHint,
6716 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6717 }
6718 case scMulExpr: {
6719 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val: S);
6720 ConstantRange X = getRangeRef(S: Mul->getOperand(i: 0), SignHint, Depth: Depth + 1);
6721 for (const SCEV *Op : drop_begin(RangeOrContainer: Mul->operands()))
6722 X = X.multiply(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1));
6723 return setRange(S: Mul, Hint: SignHint,
6724 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6725 }
6726 case scUDivExpr: {
6727 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: S);
6728 ConstantRange X = getRangeRef(S: UDiv->getLHS(), SignHint, Depth: Depth + 1);
6729 ConstantRange Y = getRangeRef(S: UDiv->getRHS(), SignHint, Depth: Depth + 1);
6730 return setRange(S: UDiv, Hint: SignHint,
6731 CR: ConservativeResult.intersectWith(CR: X.udiv(Other: Y), Type: RangeType));
6732 }
6733 case scAddRecExpr: {
6734 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: S);
6735 // If there's no unsigned wrap, the value will never be less than its
6736 // initial value.
6737 if (AddRec->hasNoUnsignedWrap()) {
6738 APInt UnsignedMinValue = getUnsignedRangeMin(S: AddRec->getStart());
6739 if (!UnsignedMinValue.isZero())
6740 ConservativeResult = ConservativeResult.intersectWith(
6741 CR: ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), Type: RangeType);
6742 }
6743
6744 // If there's no signed wrap, and all the operands except initial value have
6745 // the same sign or zero, the value won't ever be:
6746 // 1: smaller than initial value if operands are non negative,
6747 // 2: bigger than initial value if operands are non positive.
6748 // For both cases, value can not cross signed min/max boundary.
6749 if (AddRec->hasNoSignedWrap()) {
6750 bool AllNonNeg = true;
6751 bool AllNonPos = true;
6752 for (unsigned i = 1, e = AddRec->getNumOperands(); i != e; ++i) {
6753 if (!isKnownNonNegative(S: AddRec->getOperand(i)))
6754 AllNonNeg = false;
6755 if (!isKnownNonPositive(S: AddRec->getOperand(i)))
6756 AllNonPos = false;
6757 }
6758 if (AllNonNeg)
6759 ConservativeResult = ConservativeResult.intersectWith(
6760 CR: ConstantRange::getNonEmpty(Lower: getSignedRangeMin(S: AddRec->getStart()),
6761 Upper: APInt::getSignedMinValue(numBits: BitWidth)),
6762 Type: RangeType);
6763 else if (AllNonPos)
6764 ConservativeResult = ConservativeResult.intersectWith(
6765 CR: ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
6766 Upper: getSignedRangeMax(S: AddRec->getStart()) +
6767 1),
6768 Type: RangeType);
6769 }
6770
6771 // TODO: non-affine addrec
6772 if (AddRec->isAffine()) {
6773 const SCEV *MaxBEScev =
6774 getConstantMaxBackedgeTakenCount(L: AddRec->getLoop());
6775 if (!isa<SCEVCouldNotCompute>(Val: MaxBEScev)) {
6776 APInt MaxBECount = cast<SCEVConstant>(Val: MaxBEScev)->getAPInt();
6777
6778 // Adjust MaxBECount to the same bitwidth as AddRec. We can truncate if
6779 // MaxBECount's active bits are all <= AddRec's bit width.
6780 if (MaxBECount.getBitWidth() > BitWidth &&
6781 MaxBECount.getActiveBits() <= BitWidth)
6782 MaxBECount = MaxBECount.trunc(width: BitWidth);
6783 else if (MaxBECount.getBitWidth() < BitWidth)
6784 MaxBECount = MaxBECount.zext(width: BitWidth);
6785
6786 if (MaxBECount.getBitWidth() == BitWidth) {
6787 auto RangeFromAffine = getRangeForAffineAR(
6788 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
6789 ConservativeResult =
6790 ConservativeResult.intersectWith(CR: RangeFromAffine, Type: RangeType);
6791
6792 auto RangeFromFactoring = getRangeViaFactoring(
6793 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
6794 ConservativeResult =
6795 ConservativeResult.intersectWith(CR: RangeFromFactoring, Type: RangeType);
6796 }
6797 }
6798
6799 // Now try symbolic BE count and more powerful methods.
6800 if (UseExpensiveRangeSharpening) {
6801 const SCEV *SymbolicMaxBECount =
6802 getSymbolicMaxBackedgeTakenCount(L: AddRec->getLoop());
6803 if (!isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount) &&
6804 getTypeSizeInBits(Ty: MaxBEScev->getType()) <= BitWidth &&
6805 AddRec->hasNoSelfWrap()) {
6806 auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
6807 AddRec, MaxBECount: SymbolicMaxBECount, BitWidth, SignHint);
6808 ConservativeResult =
6809 ConservativeResult.intersectWith(CR: RangeFromAffineNew, Type: RangeType);
6810 }
6811 }
6812 }
6813
6814 return setRange(S: AddRec, Hint: SignHint, CR: std::move(ConservativeResult));
6815 }
6816 case scUMaxExpr:
6817 case scSMaxExpr:
6818 case scUMinExpr:
6819 case scSMinExpr:
6820 case scSequentialUMinExpr: {
6821 Intrinsic::ID ID;
6822 switch (S->getSCEVType()) {
6823 case scUMaxExpr:
6824 ID = Intrinsic::umax;
6825 break;
6826 case scSMaxExpr:
6827 ID = Intrinsic::smax;
6828 break;
6829 case scUMinExpr:
6830 case scSequentialUMinExpr:
6831 ID = Intrinsic::umin;
6832 break;
6833 case scSMinExpr:
6834 ID = Intrinsic::smin;
6835 break;
6836 default:
6837 llvm_unreachable("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr.");
6838 }
6839
6840 const auto *NAry = cast<SCEVNAryExpr>(Val: S);
6841 ConstantRange X = getRangeRef(S: NAry->getOperand(i: 0), SignHint, Depth: Depth + 1);
6842 for (unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
6843 X = X.intrinsic(
6844 IntrinsicID: ID, Ops: {X, getRangeRef(S: NAry->getOperand(i), SignHint, Depth: Depth + 1)});
6845 return setRange(S, Hint: SignHint,
6846 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6847 }
6848 case scUnknown: {
6849 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6850 Value *V = U->getValue();
6851
6852 // Check if the IR explicitly contains !range metadata.
6853 std::optional<ConstantRange> MDRange = GetRangeFromMetadata(V);
6854 if (MDRange)
6855 ConservativeResult =
6856 ConservativeResult.intersectWith(CR: *MDRange, Type: RangeType);
6857
6858 // Use facts about recurrences in the underlying IR. Note that add
6859 // recurrences are AddRecExprs and thus don't hit this path. This
6860 // primarily handles shift recurrences.
6861 auto CR = getRangeForUnknownRecurrence(U);
6862 ConservativeResult = ConservativeResult.intersectWith(CR);
6863
6864 // See if ValueTracking can give us a useful range.
6865 const DataLayout &DL = getDataLayout();
6866 KnownBits Known = computeKnownBits(V, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6867 if (Known.getBitWidth() != BitWidth)
6868 Known = Known.zextOrTrunc(BitWidth);
6869
6870 // ValueTracking may be able to compute a tighter result for the number of
6871 // sign bits than for the value of those sign bits.
6872 unsigned NS = ComputeNumSignBits(Op: V, DL, AC: &AC, CxtI: nullptr, DT: &DT);
6873 if (U->getType()->isPointerTy()) {
6874 // If the pointer size is larger than the index size type, this can cause
6875 // NS to be larger than BitWidth. So compensate for this.
6876 unsigned ptrSize = DL.getPointerTypeSizeInBits(U->getType());
6877 int ptrIdxDiff = ptrSize - BitWidth;
6878 if (ptrIdxDiff > 0 && ptrSize > BitWidth && NS > (unsigned)ptrIdxDiff)
6879 NS -= ptrIdxDiff;
6880 }
6881
6882 if (NS > 1) {
6883 // If we know any of the sign bits, we know all of the sign bits.
6884 if (!Known.Zero.getHiBits(numBits: NS).isZero())
6885 Known.Zero.setHighBits(NS);
6886 if (!Known.One.getHiBits(numBits: NS).isZero())
6887 Known.One.setHighBits(NS);
6888 }
6889
6890 if (Known.getMinValue() != Known.getMaxValue() + 1)
6891 ConservativeResult = ConservativeResult.intersectWith(
6892 CR: ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
6893 Type: RangeType);
6894 if (NS > 1)
6895 ConservativeResult = ConservativeResult.intersectWith(
6896 CR: ConstantRange(APInt::getSignedMinValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1),
6897 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1) + 1),
6898 Type: RangeType);
6899
6900 if (U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
6901 // Strengthen the range if the underlying IR value is a
6902 // global/alloca/heap allocation using the size of the object.
6903 bool CanBeNull, CanBeFreed;
6904 uint64_t DerefBytes =
6905 V->getPointerDereferenceableBytes(DL, CanBeNull, CanBeFreed);
6906 if (DerefBytes > 1 && isUIntN(N: BitWidth, x: DerefBytes)) {
6907 // The highest address the object can start is DerefBytes bytes before
6908 // the end (unsigned max value). If this value is not a multiple of the
6909 // alignment, the last possible start value is the next lowest multiple
6910 // of the alignment. Note: The computations below cannot overflow,
6911 // because if they would there's no possible start address for the
6912 // object.
6913 APInt MaxVal =
6914 APInt::getMaxValue(numBits: BitWidth) - APInt(BitWidth, DerefBytes);
6915 uint64_t Align = U->getValue()->getPointerAlignment(DL).value();
6916 uint64_t Rem = MaxVal.urem(RHS: Align);
6917 MaxVal -= APInt(BitWidth, Rem);
6918 APInt MinVal = APInt::getZero(numBits: BitWidth);
6919 if (llvm::isKnownNonZero(V, Q: DL))
6920 MinVal = Align;
6921 ConservativeResult = ConservativeResult.intersectWith(
6922 CR: ConstantRange::getNonEmpty(Lower: MinVal, Upper: MaxVal + 1), Type: RangeType);
6923 }
6924 }
6925
6926 // A range of Phi is a subset of union of all ranges of its input.
6927 if (PHINode *Phi = dyn_cast<PHINode>(Val: V)) {
6928 // Make sure that we do not run over cycled Phis.
6929 if (PendingPhiRanges.insert(Ptr: Phi).second) {
6930 ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false);
6931
6932 for (const auto &Op : Phi->operands()) {
6933 auto OpRange = getRangeRef(S: getSCEV(V: Op), SignHint, Depth: Depth + 1);
6934 RangeFromOps = RangeFromOps.unionWith(CR: OpRange);
6935 // No point to continue if we already have a full set.
6936 if (RangeFromOps.isFullSet())
6937 break;
6938 }
6939 ConservativeResult =
6940 ConservativeResult.intersectWith(CR: RangeFromOps, Type: RangeType);
6941 bool Erased = PendingPhiRanges.erase(Ptr: Phi);
6942 assert(Erased && "Failed to erase Phi properly?");
6943 (void)Erased;
6944 }
6945 }
6946
6947 // vscale can't be equal to zero
6948 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V))
6949 if (II->getIntrinsicID() == Intrinsic::vscale) {
6950 ConstantRange Disallowed = APInt::getZero(numBits: BitWidth);
6951 ConservativeResult = ConservativeResult.difference(CR: Disallowed);
6952 }
6953
6954 return setRange(S: U, Hint: SignHint, CR: std::move(ConservativeResult));
6955 }
6956 case scCouldNotCompute:
6957 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6958 }
6959
6960 return setRange(S, Hint: SignHint, CR: std::move(ConservativeResult));
6961}
6962
6963// Given a StartRange, Step and MaxBECount for an expression compute a range of
6964// values that the expression can take. Initially, the expression has a value
6965// from StartRange and then is changed by Step up to MaxBECount times. Signed
6966// argument defines if we treat Step as signed or unsigned.
6967static ConstantRange getRangeForAffineARHelper(APInt Step,
6968 const ConstantRange &StartRange,
6969 const APInt &MaxBECount,
6970 bool Signed) {
6971 unsigned BitWidth = Step.getBitWidth();
6972 assert(BitWidth == StartRange.getBitWidth() &&
6973 BitWidth == MaxBECount.getBitWidth() && "mismatched bit widths");
6974 // If either Step or MaxBECount is 0, then the expression won't change, and we
6975 // just need to return the initial range.
6976 if (Step == 0 || MaxBECount == 0)
6977 return StartRange;
6978
6979 // If we don't know anything about the initial value (i.e. StartRange is
6980 // FullRange), then we don't know anything about the final range either.
6981 // Return FullRange.
6982 if (StartRange.isFullSet())
6983 return ConstantRange::getFull(BitWidth);
6984
6985 // If Step is signed and negative, then we use its absolute value, but we also
6986 // note that we're moving in the opposite direction.
6987 bool Descending = Signed && Step.isNegative();
6988
6989 if (Signed)
6990 // This is correct even for INT_SMIN. Let's look at i8 to illustrate this:
6991 // abs(INT_SMIN) = abs(-128) = abs(0x80) = -0x80 = 0x80 = 128.
6992 // This equations hold true due to the well-defined wrap-around behavior of
6993 // APInt.
6994 Step = Step.abs();
6995
6996 // Check if Offset is more than full span of BitWidth. If it is, the
6997 // expression is guaranteed to overflow.
6998 if (APInt::getMaxValue(numBits: StartRange.getBitWidth()).udiv(RHS: Step).ult(RHS: MaxBECount))
6999 return ConstantRange::getFull(BitWidth);
7000
7001 // Offset is by how much the expression can change. Checks above guarantee no
7002 // overflow here.
7003 APInt Offset = Step * MaxBECount;
7004
7005 // Minimum value of the final range will match the minimal value of StartRange
7006 // if the expression is increasing and will be decreased by Offset otherwise.
7007 // Maximum value of the final range will match the maximal value of StartRange
7008 // if the expression is decreasing and will be increased by Offset otherwise.
7009 APInt StartLower = StartRange.getLower();
7010 APInt StartUpper = StartRange.getUpper() - 1;
7011 APInt MovedBoundary = Descending ? (StartLower - std::move(Offset))
7012 : (StartUpper + std::move(Offset));
7013
7014 // It's possible that the new minimum/maximum value will fall into the initial
7015 // range (due to wrap around). This means that the expression can take any
7016 // value in this bitwidth, and we have to return full range.
7017 if (StartRange.contains(Val: MovedBoundary))
7018 return ConstantRange::getFull(BitWidth);
7019
7020 APInt NewLower =
7021 Descending ? std::move(MovedBoundary) : std::move(StartLower);
7022 APInt NewUpper =
7023 Descending ? std::move(StartUpper) : std::move(MovedBoundary);
7024 NewUpper += 1;
7025
7026 // No overflow detected, return [StartLower, StartUpper + Offset + 1) range.
7027 return ConstantRange::getNonEmpty(Lower: std::move(NewLower), Upper: std::move(NewUpper));
7028}
7029
7030ConstantRange ScalarEvolution::getRangeForAffineAR(const SCEV *Start,
7031 const SCEV *Step,
7032 const APInt &MaxBECount) {
7033 assert(getTypeSizeInBits(Start->getType()) ==
7034 getTypeSizeInBits(Step->getType()) &&
7035 getTypeSizeInBits(Start->getType()) == MaxBECount.getBitWidth() &&
7036 "mismatched bit widths");
7037
7038 // First, consider step signed.
7039 ConstantRange StartSRange = getSignedRange(S: Start);
7040 ConstantRange StepSRange = getSignedRange(S: Step);
7041
7042 // If Step can be both positive and negative, we need to find ranges for the
7043 // maximum absolute step values in both directions and union them.
7044 ConstantRange SR = getRangeForAffineARHelper(
7045 Step: StepSRange.getSignedMin(), StartRange: StartSRange, MaxBECount, /* Signed = */ true);
7046 SR = SR.unionWith(CR: getRangeForAffineARHelper(Step: StepSRange.getSignedMax(),
7047 StartRange: StartSRange, MaxBECount,
7048 /* Signed = */ true));
7049
7050 // Next, consider step unsigned.
7051 ConstantRange UR = getRangeForAffineARHelper(
7052 Step: getUnsignedRangeMax(S: Step), StartRange: getUnsignedRange(S: Start), MaxBECount,
7053 /* Signed = */ false);
7054
7055 // Finally, intersect signed and unsigned ranges.
7056 return SR.intersectWith(CR: UR, Type: ConstantRange::Smallest);
7057}
7058
7059ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
7060 const SCEVAddRecExpr *AddRec, const SCEV *MaxBECount, unsigned BitWidth,
7061 ScalarEvolution::RangeSignHint SignHint) {
7062 assert(AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n");
7063 assert(AddRec->hasNoSelfWrap() &&
7064 "This only works for non-self-wrapping AddRecs!");
7065 const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
7066 const SCEV *Step = AddRec->getStepRecurrence(SE&: *this);
7067 // Only deal with constant step to save compile time.
7068 if (!isa<SCEVConstant>(Val: Step))
7069 return ConstantRange::getFull(BitWidth);
7070 // Let's make sure that we can prove that we do not self-wrap during
7071 // MaxBECount iterations. We need this because MaxBECount is a maximum
7072 // iteration count estimate, and we might infer nw from some exit for which we
7073 // do not know max exit count (or any other side reasoning).
7074 // TODO: Turn into assert at some point.
7075 if (getTypeSizeInBits(Ty: MaxBECount->getType()) >
7076 getTypeSizeInBits(Ty: AddRec->getType()))
7077 return ConstantRange::getFull(BitWidth);
7078 MaxBECount = getNoopOrZeroExtend(V: MaxBECount, Ty: AddRec->getType());
7079 const SCEV *RangeWidth = getMinusOne(Ty: AddRec->getType());
7080 const SCEV *StepAbs = getUMinExpr(LHS: Step, RHS: getNegativeSCEV(V: Step));
7081 const SCEV *MaxItersWithoutWrap = getUDivExpr(LHS: RangeWidth, RHS: StepAbs);
7082 if (!isKnownPredicateViaConstantRanges(Pred: ICmpInst::ICMP_ULE, LHS: MaxBECount,
7083 RHS: MaxItersWithoutWrap))
7084 return ConstantRange::getFull(BitWidth);
7085
7086 ICmpInst::Predicate LEPred =
7087 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
7088 ICmpInst::Predicate GEPred =
7089 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
7090 const SCEV *End = AddRec->evaluateAtIteration(It: MaxBECount, SE&: *this);
7091
7092 // We know that there is no self-wrap. Let's take Start and End values and
7093 // look at all intermediate values V1, V2, ..., Vn that IndVar takes during
7094 // the iteration. They either lie inside the range [Min(Start, End),
7095 // Max(Start, End)] or outside it:
7096 //
7097 // Case 1: RangeMin ... Start V1 ... VN End ... RangeMax;
7098 // Case 2: RangeMin Vk ... V1 Start ... End Vn ... Vk + 1 RangeMax;
7099 //
7100 // No self wrap flag guarantees that the intermediate values cannot be BOTH
7101 // outside and inside the range [Min(Start, End), Max(Start, End)]. Using that
7102 // knowledge, let's try to prove that we are dealing with Case 1. It is so if
7103 // Start <= End and step is positive, or Start >= End and step is negative.
7104 const SCEV *Start = applyLoopGuards(Expr: AddRec->getStart(), L: AddRec->getLoop());
7105 ConstantRange StartRange = getRangeRef(S: Start, SignHint);
7106 ConstantRange EndRange = getRangeRef(S: End, SignHint);
7107 ConstantRange RangeBetween = StartRange.unionWith(CR: EndRange);
7108 // If they already cover full iteration space, we will know nothing useful
7109 // even if we prove what we want to prove.
7110 if (RangeBetween.isFullSet())
7111 return RangeBetween;
7112 // Only deal with ranges that do not wrap (i.e. RangeMin < RangeMax).
7113 bool IsWrappedSet = IsSigned ? RangeBetween.isSignWrappedSet()
7114 : RangeBetween.isWrappedSet();
7115 if (IsWrappedSet)
7116 return ConstantRange::getFull(BitWidth);
7117
7118 if (isKnownPositive(S: Step) &&
7119 isKnownPredicateViaConstantRanges(Pred: LEPred, LHS: Start, RHS: End))
7120 return RangeBetween;
7121 if (isKnownNegative(S: Step) &&
7122 isKnownPredicateViaConstantRanges(Pred: GEPred, LHS: Start, RHS: End))
7123 return RangeBetween;
7124 return ConstantRange::getFull(BitWidth);
7125}
7126
7127ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start,
7128 const SCEV *Step,
7129 const APInt &MaxBECount) {
7130 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q})
7131 // == RangeOf({A,+,P}) union RangeOf({B,+,Q})
7132
7133 unsigned BitWidth = MaxBECount.getBitWidth();
7134 assert(getTypeSizeInBits(Start->getType()) == BitWidth &&
7135 getTypeSizeInBits(Step->getType()) == BitWidth &&
7136 "mismatched bit widths");
7137
7138 struct SelectPattern {
7139 Value *Condition = nullptr;
7140 APInt TrueValue;
7141 APInt FalseValue;
7142
7143 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth,
7144 const SCEV *S) {
7145 std::optional<unsigned> CastOp;
7146 APInt Offset(BitWidth, 0);
7147
7148 assert(SE.getTypeSizeInBits(S->getType()) == BitWidth &&
7149 "Should be!");
7150
7151 // Peel off a constant offset. In the future we could consider being
7152 // smarter here and handle {Start+Step,+,Step} too.
7153 const APInt *Off;
7154 if (match(S, P: m_scev_Add(Op0: m_scev_APInt(C&: Off), Op1: m_SCEV(V&: S))))
7155 Offset = *Off;
7156
7157 // Peel off a cast operation
7158 if (auto *SCast = dyn_cast<SCEVIntegralCastExpr>(Val: S)) {
7159 CastOp = SCast->getSCEVType();
7160 S = SCast->getOperand();
7161 }
7162
7163 using namespace llvm::PatternMatch;
7164
7165 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
7166 const APInt *TrueVal, *FalseVal;
7167 if (!SU ||
7168 !match(V: SU->getValue(), P: m_Select(C: m_Value(V&: Condition), L: m_APInt(Res&: TrueVal),
7169 R: m_APInt(Res&: FalseVal)))) {
7170 Condition = nullptr;
7171 return;
7172 }
7173
7174 TrueValue = *TrueVal;
7175 FalseValue = *FalseVal;
7176
7177 // Re-apply the cast we peeled off earlier
7178 if (CastOp)
7179 switch (*CastOp) {
7180 default:
7181 llvm_unreachable("Unknown SCEV cast type!");
7182
7183 case scTruncate:
7184 TrueValue = TrueValue.trunc(width: BitWidth);
7185 FalseValue = FalseValue.trunc(width: BitWidth);
7186 break;
7187 case scZeroExtend:
7188 TrueValue = TrueValue.zext(width: BitWidth);
7189 FalseValue = FalseValue.zext(width: BitWidth);
7190 break;
7191 case scSignExtend:
7192 TrueValue = TrueValue.sext(width: BitWidth);
7193 FalseValue = FalseValue.sext(width: BitWidth);
7194 break;
7195 }
7196
7197 // Re-apply the constant offset we peeled off earlier
7198 TrueValue += Offset;
7199 FalseValue += Offset;
7200 }
7201
7202 bool isRecognized() { return Condition != nullptr; }
7203 };
7204
7205 SelectPattern StartPattern(*this, BitWidth, Start);
7206 if (!StartPattern.isRecognized())
7207 return ConstantRange::getFull(BitWidth);
7208
7209 SelectPattern StepPattern(*this, BitWidth, Step);
7210 if (!StepPattern.isRecognized())
7211 return ConstantRange::getFull(BitWidth);
7212
7213 if (StartPattern.Condition != StepPattern.Condition) {
7214 // We don't handle this case today; but we could, by considering four
7215 // possibilities below instead of two. I'm not sure if there are cases where
7216 // that will help over what getRange already does, though.
7217 return ConstantRange::getFull(BitWidth);
7218 }
7219
7220 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to
7221 // construct arbitrary general SCEV expressions here. This function is called
7222 // from deep in the call stack, and calling getSCEV (on a sext instruction,
7223 // say) can end up caching a suboptimal value.
7224
7225 // FIXME: without the explicit `this` receiver below, MSVC errors out with
7226 // C2352 and C2512 (otherwise it isn't needed).
7227
7228 const SCEV *TrueStart = this->getConstant(Val: StartPattern.TrueValue);
7229 const SCEV *TrueStep = this->getConstant(Val: StepPattern.TrueValue);
7230 const SCEV *FalseStart = this->getConstant(Val: StartPattern.FalseValue);
7231 const SCEV *FalseStep = this->getConstant(Val: StepPattern.FalseValue);
7232
7233 ConstantRange TrueRange =
7234 this->getRangeForAffineAR(Start: TrueStart, Step: TrueStep, MaxBECount);
7235 ConstantRange FalseRange =
7236 this->getRangeForAffineAR(Start: FalseStart, Step: FalseStep, MaxBECount);
7237
7238 return TrueRange.unionWith(CR: FalseRange);
7239}
7240
7241SCEV::NoWrapFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
7242 if (isa<ConstantExpr>(Val: V)) return SCEV::FlagAnyWrap;
7243 const BinaryOperator *BinOp = cast<BinaryOperator>(Val: V);
7244
7245 // Return early if there are no flags to propagate to the SCEV.
7246 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7247 if (BinOp->hasNoUnsignedWrap())
7248 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7249 if (BinOp->hasNoSignedWrap())
7250 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7251 if (Flags == SCEV::FlagAnyWrap)
7252 return SCEV::FlagAnyWrap;
7253
7254 return isSCEVExprNeverPoison(I: BinOp) ? Flags : SCEV::FlagAnyWrap;
7255}
7256
7257const Instruction *
7258ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) {
7259 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S))
7260 return &*AddRec->getLoop()->getHeader()->begin();
7261 if (auto *U = dyn_cast<SCEVUnknown>(Val: S))
7262 if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
7263 return I;
7264 return nullptr;
7265}
7266
7267const Instruction *
7268ScalarEvolution::getDefiningScopeBound(ArrayRef<const SCEV *> Ops,
7269 bool &Precise) {
7270 Precise = true;
7271 // Do a bounded search of the def relation of the requested SCEVs.
7272 SmallSet<const SCEV *, 16> Visited;
7273 SmallVector<const SCEV *> Worklist;
7274 auto pushOp = [&](const SCEV *S) {
7275 if (!Visited.insert(Ptr: S).second)
7276 return;
7277 // Threshold of 30 here is arbitrary.
7278 if (Visited.size() > 30) {
7279 Precise = false;
7280 return;
7281 }
7282 Worklist.push_back(Elt: S);
7283 };
7284
7285 for (const auto *S : Ops)
7286 pushOp(S);
7287
7288 const Instruction *Bound = nullptr;
7289 while (!Worklist.empty()) {
7290 auto *S = Worklist.pop_back_val();
7291 if (auto *DefI = getNonTrivialDefiningScopeBound(S)) {
7292 if (!Bound || DT.dominates(Def: Bound, User: DefI))
7293 Bound = DefI;
7294 } else {
7295 for (const auto *Op : S->operands())
7296 pushOp(Op);
7297 }
7298 }
7299 return Bound ? Bound : &*F.getEntryBlock().begin();
7300}
7301
7302const Instruction *
7303ScalarEvolution::getDefiningScopeBound(ArrayRef<const SCEV *> Ops) {
7304 bool Discard;
7305 return getDefiningScopeBound(Ops, Precise&: Discard);
7306}
7307
7308bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
7309 const Instruction *B) {
7310 if (A->getParent() == B->getParent() &&
7311 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7312 End: B->getIterator()))
7313 return true;
7314
7315 auto *BLoop = LI.getLoopFor(BB: B->getParent());
7316 if (BLoop && BLoop->getHeader() == B->getParent() &&
7317 BLoop->getLoopPreheader() == A->getParent() &&
7318 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7319 End: A->getParent()->end()) &&
7320 isGuaranteedToTransferExecutionToSuccessor(Begin: B->getParent()->begin(),
7321 End: B->getIterator()))
7322 return true;
7323 return false;
7324}
7325
7326bool ScalarEvolution::isGuaranteedNotToBePoison(const SCEV *Op) {
7327 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ true);
7328 visitAll(Root: Op, Visitor&: PC);
7329 return PC.MaybePoison.empty();
7330}
7331
7332bool ScalarEvolution::isGuaranteedNotToCauseUB(const SCEV *Op) {
7333 return !SCEVExprContains(Root: Op, Pred: [this](const SCEV *S) {
7334 const SCEV *Op1;
7335 bool M = match(S, P: m_scev_UDiv(Op0: m_SCEV(), Op1: m_SCEV(V&: Op1)));
7336 // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
7337 // is a non-zero constant, we have to assume the UDiv may be UB.
7338 return M && (!isKnownNonZero(S: Op1) || !isGuaranteedNotToBePoison(Op: Op1));
7339 });
7340}
7341
7342bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
7343 // Only proceed if we can prove that I does not yield poison.
7344 if (!programUndefinedIfPoison(Inst: I))
7345 return false;
7346
7347 // At this point we know that if I is executed, then it does not wrap
7348 // according to at least one of NSW or NUW. If I is not executed, then we do
7349 // not know if the calculation that I represents would wrap. Multiple
7350 // instructions can map to the same SCEV. If we apply NSW or NUW from I to
7351 // the SCEV, we must guarantee no wrapping for that SCEV also when it is
7352 // derived from other instructions that map to the same SCEV. We cannot make
7353 // that guarantee for cases where I is not executed. So we need to find a
7354 // upper bound on the defining scope for the SCEV, and prove that I is
7355 // executed every time we enter that scope. When the bounding scope is a
7356 // loop (the common case), this is equivalent to proving I executes on every
7357 // iteration of that loop.
7358 SmallVector<const SCEV *> SCEVOps;
7359 for (const Use &Op : I->operands()) {
7360 // I could be an extractvalue from a call to an overflow intrinsic.
7361 // TODO: We can do better here in some cases.
7362 if (isSCEVable(Ty: Op->getType()))
7363 SCEVOps.push_back(Elt: getSCEV(V: Op));
7364 }
7365 auto *DefI = getDefiningScopeBound(Ops: SCEVOps);
7366 return isGuaranteedToTransferExecutionTo(A: DefI, B: I);
7367}
7368
7369bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) {
7370 // If we know that \c I can never be poison period, then that's enough.
7371 if (isSCEVExprNeverPoison(I))
7372 return true;
7373
7374 // If the loop only has one exit, then we know that, if the loop is entered,
7375 // any instruction dominating that exit will be executed. If any such
7376 // instruction would result in UB, the addrec cannot be poison.
7377 //
7378 // This is basically the same reasoning as in isSCEVExprNeverPoison(), but
7379 // also handles uses outside the loop header (they just need to dominate the
7380 // single exit).
7381
7382 auto *ExitingBB = L->getExitingBlock();
7383 if (!ExitingBB || !loopHasNoAbnormalExits(L))
7384 return false;
7385
7386 SmallPtrSet<const Value *, 16> KnownPoison;
7387 SmallVector<const Instruction *, 8> Worklist;
7388
7389 // We start by assuming \c I, the post-inc add recurrence, is poison. Only
7390 // things that are known to be poison under that assumption go on the
7391 // Worklist.
7392 KnownPoison.insert(Ptr: I);
7393 Worklist.push_back(Elt: I);
7394
7395 while (!Worklist.empty()) {
7396 const Instruction *Poison = Worklist.pop_back_val();
7397
7398 for (const Use &U : Poison->uses()) {
7399 const Instruction *PoisonUser = cast<Instruction>(Val: U.getUser());
7400 if (mustTriggerUB(I: PoisonUser, KnownPoison) &&
7401 DT.dominates(A: PoisonUser->getParent(), B: ExitingBB))
7402 return true;
7403
7404 if (propagatesPoison(PoisonOp: U) && L->contains(Inst: PoisonUser))
7405 if (KnownPoison.insert(Ptr: PoisonUser).second)
7406 Worklist.push_back(Elt: PoisonUser);
7407 }
7408 }
7409
7410 return false;
7411}
7412
7413ScalarEvolution::LoopProperties
7414ScalarEvolution::getLoopProperties(const Loop *L) {
7415 using LoopProperties = ScalarEvolution::LoopProperties;
7416
7417 auto Itr = LoopPropertiesCache.find(Val: L);
7418 if (Itr == LoopPropertiesCache.end()) {
7419 auto HasSideEffects = [](Instruction *I) {
7420 if (auto *SI = dyn_cast<StoreInst>(Val: I))
7421 return !SI->isSimple();
7422
7423 return I->mayThrow() || I->mayWriteToMemory();
7424 };
7425
7426 LoopProperties LP = {/* HasNoAbnormalExits */ true,
7427 /*HasNoSideEffects*/ true};
7428
7429 for (auto *BB : L->getBlocks())
7430 for (auto &I : *BB) {
7431 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7432 LP.HasNoAbnormalExits = false;
7433 if (HasSideEffects(&I))
7434 LP.HasNoSideEffects = false;
7435 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
7436 break; // We're already as pessimistic as we can get.
7437 }
7438
7439 auto InsertPair = LoopPropertiesCache.insert(KV: {L, LP});
7440 assert(InsertPair.second && "We just checked!");
7441 Itr = InsertPair.first;
7442 }
7443
7444 return Itr->second;
7445}
7446
7447bool ScalarEvolution::loopIsFiniteByAssumption(const Loop *L) {
7448 // A mustprogress loop without side effects must be finite.
7449 // TODO: The check used here is very conservative. It's only *specific*
7450 // side effects which are well defined in infinite loops.
7451 return isFinite(L) || (isMustProgress(L) && loopHasNoSideEffects(L));
7452}
7453
7454const SCEV *ScalarEvolution::createSCEVIter(Value *V) {
7455 // Worklist item with a Value and a bool indicating whether all operands have
7456 // been visited already.
7457 using PointerTy = PointerIntPair<Value *, 1, bool>;
7458 SmallVector<PointerTy> Stack;
7459
7460 Stack.emplace_back(Args&: V, Args: true);
7461 Stack.emplace_back(Args&: V, Args: false);
7462 while (!Stack.empty()) {
7463 auto E = Stack.pop_back_val();
7464 Value *CurV = E.getPointer();
7465
7466 if (getExistingSCEV(V: CurV))
7467 continue;
7468
7469 SmallVector<Value *> Ops;
7470 const SCEV *CreatedSCEV = nullptr;
7471 // If all operands have been visited already, create the SCEV.
7472 if (E.getInt()) {
7473 CreatedSCEV = createSCEV(V: CurV);
7474 } else {
7475 // Otherwise get the operands we need to create SCEV's for before creating
7476 // the SCEV for CurV. If the SCEV for CurV can be constructed trivially,
7477 // just use it.
7478 CreatedSCEV = getOperandsToCreate(V: CurV, Ops);
7479 }
7480
7481 if (CreatedSCEV) {
7482 insertValueToMap(V: CurV, S: CreatedSCEV);
7483 } else {
7484 // Queue CurV for SCEV creation, followed by its's operands which need to
7485 // be constructed first.
7486 Stack.emplace_back(Args&: CurV, Args: true);
7487 for (Value *Op : Ops)
7488 Stack.emplace_back(Args&: Op, Args: false);
7489 }
7490 }
7491
7492 return getExistingSCEV(V);
7493}
7494
7495const SCEV *
7496ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
7497 if (!isSCEVable(Ty: V->getType()))
7498 return getUnknown(V);
7499
7500 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7501 // Don't attempt to analyze instructions in blocks that aren't
7502 // reachable. Such instructions don't matter, and they aren't required
7503 // to obey basic rules for definitions dominating uses which this
7504 // analysis depends on.
7505 if (!DT.isReachableFromEntry(A: I->getParent()))
7506 return getUnknown(V: PoisonValue::get(T: V->getType()));
7507 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7508 return getConstant(V: CI);
7509 else if (isa<GlobalAlias>(Val: V))
7510 return getUnknown(V);
7511 else if (!isa<ConstantExpr>(Val: V))
7512 return getUnknown(V);
7513
7514 Operator *U = cast<Operator>(Val: V);
7515 if (auto BO =
7516 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CxtI: dyn_cast<Instruction>(Val: V))) {
7517 bool IsConstArg = isa<ConstantInt>(Val: BO->RHS);
7518 switch (BO->Opcode) {
7519 case Instruction::Add:
7520 case Instruction::Mul: {
7521 // For additions and multiplications, traverse add/mul chains for which we
7522 // can potentially create a single SCEV, to reduce the number of
7523 // get{Add,Mul}Expr calls.
7524 do {
7525 if (BO->Op) {
7526 if (BO->Op != V && getExistingSCEV(V: BO->Op)) {
7527 Ops.push_back(Elt: BO->Op);
7528 break;
7529 }
7530 }
7531 Ops.push_back(Elt: BO->RHS);
7532 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7533 CxtI: dyn_cast<Instruction>(Val: V));
7534 if (!NewBO ||
7535 (BO->Opcode == Instruction::Add &&
7536 (NewBO->Opcode != Instruction::Add &&
7537 NewBO->Opcode != Instruction::Sub)) ||
7538 (BO->Opcode == Instruction::Mul &&
7539 NewBO->Opcode != Instruction::Mul)) {
7540 Ops.push_back(Elt: BO->LHS);
7541 break;
7542 }
7543 // CreateSCEV calls getNoWrapFlagsFromUB, which under certain conditions
7544 // requires a SCEV for the LHS.
7545 if (BO->Op && (BO->IsNSW || BO->IsNUW)) {
7546 auto *I = dyn_cast<Instruction>(Val: BO->Op);
7547 if (I && programUndefinedIfPoison(Inst: I)) {
7548 Ops.push_back(Elt: BO->LHS);
7549 break;
7550 }
7551 }
7552 BO = NewBO;
7553 } while (true);
7554 return nullptr;
7555 }
7556 case Instruction::Sub:
7557 case Instruction::UDiv:
7558 case Instruction::URem:
7559 break;
7560 case Instruction::AShr:
7561 case Instruction::Shl:
7562 case Instruction::Xor:
7563 if (!IsConstArg)
7564 return nullptr;
7565 break;
7566 case Instruction::And:
7567 case Instruction::Or:
7568 if (!IsConstArg && !BO->LHS->getType()->isIntegerTy(Bitwidth: 1))
7569 return nullptr;
7570 break;
7571 case Instruction::LShr:
7572 return getUnknown(V);
7573 default:
7574 llvm_unreachable("Unhandled binop");
7575 break;
7576 }
7577
7578 Ops.push_back(Elt: BO->LHS);
7579 Ops.push_back(Elt: BO->RHS);
7580 return nullptr;
7581 }
7582
7583 switch (U->getOpcode()) {
7584 case Instruction::Trunc:
7585 case Instruction::ZExt:
7586 case Instruction::SExt:
7587 case Instruction::PtrToInt:
7588 Ops.push_back(Elt: U->getOperand(i: 0));
7589 return nullptr;
7590
7591 case Instruction::BitCast:
7592 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType())) {
7593 Ops.push_back(Elt: U->getOperand(i: 0));
7594 return nullptr;
7595 }
7596 return getUnknown(V);
7597
7598 case Instruction::SDiv:
7599 case Instruction::SRem:
7600 Ops.push_back(Elt: U->getOperand(i: 0));
7601 Ops.push_back(Elt: U->getOperand(i: 1));
7602 return nullptr;
7603
7604 case Instruction::GetElementPtr:
7605 assert(cast<GEPOperator>(U)->getSourceElementType()->isSized() &&
7606 "GEP source element type must be sized");
7607 llvm::append_range(C&: Ops, R: U->operands());
7608 return nullptr;
7609
7610 case Instruction::IntToPtr:
7611 return getUnknown(V);
7612
7613 case Instruction::PHI:
7614 // Keep constructing SCEVs' for phis recursively for now.
7615 return nullptr;
7616
7617 case Instruction::Select: {
7618 // Check if U is a select that can be simplified to a SCEVUnknown.
7619 auto CanSimplifyToUnknown = [this, U]() {
7620 if (U->getType()->isIntegerTy(Bitwidth: 1) || isa<ConstantInt>(Val: U->getOperand(i: 0)))
7621 return false;
7622
7623 auto *ICI = dyn_cast<ICmpInst>(Val: U->getOperand(i: 0));
7624 if (!ICI)
7625 return false;
7626 Value *LHS = ICI->getOperand(i_nocapture: 0);
7627 Value *RHS = ICI->getOperand(i_nocapture: 1);
7628 if (ICI->getPredicate() == CmpInst::ICMP_EQ ||
7629 ICI->getPredicate() == CmpInst::ICMP_NE) {
7630 if (!(isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()))
7631 return true;
7632 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
7633 getTypeSizeInBits(Ty: U->getType()))
7634 return true;
7635 return false;
7636 };
7637 if (CanSimplifyToUnknown())
7638 return getUnknown(V: U);
7639
7640 llvm::append_range(C&: Ops, R: U->operands());
7641 return nullptr;
7642 break;
7643 }
7644 case Instruction::Call:
7645 case Instruction::Invoke:
7646 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand()) {
7647 Ops.push_back(Elt: RV);
7648 return nullptr;
7649 }
7650
7651 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
7652 switch (II->getIntrinsicID()) {
7653 case Intrinsic::abs:
7654 Ops.push_back(Elt: II->getArgOperand(i: 0));
7655 return nullptr;
7656 case Intrinsic::umax:
7657 case Intrinsic::umin:
7658 case Intrinsic::smax:
7659 case Intrinsic::smin:
7660 case Intrinsic::usub_sat:
7661 case Intrinsic::uadd_sat:
7662 Ops.push_back(Elt: II->getArgOperand(i: 0));
7663 Ops.push_back(Elt: II->getArgOperand(i: 1));
7664 return nullptr;
7665 case Intrinsic::start_loop_iterations:
7666 case Intrinsic::annotation:
7667 case Intrinsic::ptr_annotation:
7668 Ops.push_back(Elt: II->getArgOperand(i: 0));
7669 return nullptr;
7670 default:
7671 break;
7672 }
7673 }
7674 break;
7675 }
7676
7677 return nullptr;
7678}
7679
7680const SCEV *ScalarEvolution::createSCEV(Value *V) {
7681 if (!isSCEVable(Ty: V->getType()))
7682 return getUnknown(V);
7683
7684 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7685 // Don't attempt to analyze instructions in blocks that aren't
7686 // reachable. Such instructions don't matter, and they aren't required
7687 // to obey basic rules for definitions dominating uses which this
7688 // analysis depends on.
7689 if (!DT.isReachableFromEntry(A: I->getParent()))
7690 return getUnknown(V: PoisonValue::get(T: V->getType()));
7691 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7692 return getConstant(V: CI);
7693 else if (isa<GlobalAlias>(Val: V))
7694 return getUnknown(V);
7695 else if (!isa<ConstantExpr>(Val: V))
7696 return getUnknown(V);
7697
7698 const SCEV *LHS;
7699 const SCEV *RHS;
7700
7701 Operator *U = cast<Operator>(Val: V);
7702 if (auto BO =
7703 MatchBinaryOp(V: U, DL: getDataLayout(), AC, DT, CxtI: dyn_cast<Instruction>(Val: V))) {
7704 switch (BO->Opcode) {
7705 case Instruction::Add: {
7706 // The simple thing to do would be to just call getSCEV on both operands
7707 // and call getAddExpr with the result. However if we're looking at a
7708 // bunch of things all added together, this can be quite inefficient,
7709 // because it leads to N-1 getAddExpr calls for N ultimate operands.
7710 // Instead, gather up all the operands and make a single getAddExpr call.
7711 // LLVM IR canonical form means we need only traverse the left operands.
7712 SmallVector<const SCEV *, 4> AddOps;
7713 do {
7714 if (BO->Op) {
7715 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
7716 AddOps.push_back(Elt: OpSCEV);
7717 break;
7718 }
7719
7720 // If a NUW or NSW flag can be applied to the SCEV for this
7721 // addition, then compute the SCEV for this addition by itself
7722 // with a separate call to getAddExpr. We need to do that
7723 // instead of pushing the operands of the addition onto AddOps,
7724 // since the flags are only known to apply to this particular
7725 // addition - they may not apply to other additions that can be
7726 // formed with operands from AddOps.
7727 const SCEV *RHS = getSCEV(V: BO->RHS);
7728 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
7729 if (Flags != SCEV::FlagAnyWrap) {
7730 const SCEV *LHS = getSCEV(V: BO->LHS);
7731 if (BO->Opcode == Instruction::Sub)
7732 AddOps.push_back(Elt: getMinusSCEV(LHS, RHS, Flags));
7733 else
7734 AddOps.push_back(Elt: getAddExpr(LHS, RHS, Flags));
7735 break;
7736 }
7737 }
7738
7739 if (BO->Opcode == Instruction::Sub)
7740 AddOps.push_back(Elt: getNegativeSCEV(V: getSCEV(V: BO->RHS)));
7741 else
7742 AddOps.push_back(Elt: getSCEV(V: BO->RHS));
7743
7744 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7745 CxtI: dyn_cast<Instruction>(Val: V));
7746 if (!NewBO || (NewBO->Opcode != Instruction::Add &&
7747 NewBO->Opcode != Instruction::Sub)) {
7748 AddOps.push_back(Elt: getSCEV(V: BO->LHS));
7749 break;
7750 }
7751 BO = NewBO;
7752 } while (true);
7753
7754 return getAddExpr(Ops&: AddOps);
7755 }
7756
7757 case Instruction::Mul: {
7758 SmallVector<const SCEV *, 4> MulOps;
7759 do {
7760 if (BO->Op) {
7761 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
7762 MulOps.push_back(Elt: OpSCEV);
7763 break;
7764 }
7765
7766 SCEV::NoWrapFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
7767 if (Flags != SCEV::FlagAnyWrap) {
7768 LHS = getSCEV(V: BO->LHS);
7769 RHS = getSCEV(V: BO->RHS);
7770 MulOps.push_back(Elt: getMulExpr(LHS, RHS, Flags));
7771 break;
7772 }
7773 }
7774
7775 MulOps.push_back(Elt: getSCEV(V: BO->RHS));
7776 auto NewBO = MatchBinaryOp(V: BO->LHS, DL: getDataLayout(), AC, DT,
7777 CxtI: dyn_cast<Instruction>(Val: V));
7778 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
7779 MulOps.push_back(Elt: getSCEV(V: BO->LHS));
7780 break;
7781 }
7782 BO = NewBO;
7783 } while (true);
7784
7785 return getMulExpr(Ops&: MulOps);
7786 }
7787 case Instruction::UDiv:
7788 LHS = getSCEV(V: BO->LHS);
7789 RHS = getSCEV(V: BO->RHS);
7790 return getUDivExpr(LHS, RHS);
7791 case Instruction::URem:
7792 LHS = getSCEV(V: BO->LHS);
7793 RHS = getSCEV(V: BO->RHS);
7794 return getURemExpr(LHS, RHS);
7795 case Instruction::Sub: {
7796 SCEV::NoWrapFlags Flags = SCEV::FlagAnyWrap;
7797 if (BO->Op)
7798 Flags = getNoWrapFlagsFromUB(V: BO->Op);
7799 LHS = getSCEV(V: BO->LHS);
7800 RHS = getSCEV(V: BO->RHS);
7801 return getMinusSCEV(LHS, RHS, Flags);
7802 }
7803 case Instruction::And:
7804 // For an expression like x&255 that merely masks off the high bits,
7805 // use zext(trunc(x)) as the SCEV expression.
7806 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
7807 if (CI->isZero())
7808 return getSCEV(V: BO->RHS);
7809 if (CI->isMinusOne())
7810 return getSCEV(V: BO->LHS);
7811 const APInt &A = CI->getValue();
7812
7813 // Instcombine's ShrinkDemandedConstant may strip bits out of
7814 // constants, obscuring what would otherwise be a low-bits mask.
7815 // Use computeKnownBits to compute what ShrinkDemandedConstant
7816 // knew about to reconstruct a low-bits mask value.
7817 unsigned LZ = A.countl_zero();
7818 unsigned TZ = A.countr_zero();
7819 unsigned BitWidth = A.getBitWidth();
7820 KnownBits Known(BitWidth);
7821 computeKnownBits(V: BO->LHS, Known, DL: getDataLayout(), AC: &AC, CxtI: nullptr, DT: &DT);
7822
7823 APInt EffectiveMask =
7824 APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - LZ - TZ).shl(shiftAmt: TZ);
7825 if ((LZ != 0 || TZ != 0) && !((~A & ~Known.Zero) & EffectiveMask)) {
7826 const SCEV *MulCount = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ));
7827 const SCEV *LHS = getSCEV(V: BO->LHS);
7828 const SCEV *ShiftedLHS = nullptr;
7829 if (auto *LHSMul = dyn_cast<SCEVMulExpr>(Val: LHS)) {
7830 if (auto *OpC = dyn_cast<SCEVConstant>(Val: LHSMul->getOperand(i: 0))) {
7831 // For an expression like (x * 8) & 8, simplify the multiply.
7832 unsigned MulZeros = OpC->getAPInt().countr_zero();
7833 unsigned GCD = std::min(a: MulZeros, b: TZ);
7834 APInt DivAmt = APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ - GCD);
7835 SmallVector<const SCEV*, 4> MulOps;
7836 MulOps.push_back(Elt: getConstant(Val: OpC->getAPInt().ashr(ShiftAmt: GCD)));
7837 append_range(C&: MulOps, R: LHSMul->operands().drop_front());
7838 auto *NewMul = getMulExpr(Ops&: MulOps, OrigFlags: LHSMul->getNoWrapFlags());
7839 ShiftedLHS = getUDivExpr(LHS: NewMul, RHS: getConstant(Val: DivAmt));
7840 }
7841 }
7842 if (!ShiftedLHS)
7843 ShiftedLHS = getUDivExpr(LHS, RHS: MulCount);
7844 return getMulExpr(
7845 LHS: getZeroExtendExpr(
7846 Op: getTruncateExpr(Op: ShiftedLHS,
7847 Ty: IntegerType::get(C&: getContext(), NumBits: BitWidth - LZ - TZ)),
7848 Ty: BO->LHS->getType()),
7849 RHS: MulCount);
7850 }
7851 }
7852 // Binary `and` is a bit-wise `umin`.
7853 if (BO->LHS->getType()->isIntegerTy(Bitwidth: 1)) {
7854 LHS = getSCEV(V: BO->LHS);
7855 RHS = getSCEV(V: BO->RHS);
7856 return getUMinExpr(LHS, RHS);
7857 }
7858 break;
7859
7860 case Instruction::Or:
7861 // Binary `or` is a bit-wise `umax`.
7862 if (BO->LHS->getType()->isIntegerTy(Bitwidth: 1)) {
7863 LHS = getSCEV(V: BO->LHS);
7864 RHS = getSCEV(V: BO->RHS);
7865 return getUMaxExpr(LHS, RHS);
7866 }
7867 break;
7868
7869 case Instruction::Xor:
7870 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
7871 // If the RHS of xor is -1, then this is a not operation.
7872 if (CI->isMinusOne())
7873 return getNotSCEV(V: getSCEV(V: BO->LHS));
7874
7875 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
7876 // This is a variant of the check for xor with -1, and it handles
7877 // the case where instcombine has trimmed non-demanded bits out
7878 // of an xor with -1.
7879 if (auto *LBO = dyn_cast<BinaryOperator>(Val: BO->LHS))
7880 if (ConstantInt *LCI = dyn_cast<ConstantInt>(Val: LBO->getOperand(i_nocapture: 1)))
7881 if (LBO->getOpcode() == Instruction::And &&
7882 LCI->getValue() == CI->getValue())
7883 if (const SCEVZeroExtendExpr *Z =
7884 dyn_cast<SCEVZeroExtendExpr>(Val: getSCEV(V: BO->LHS))) {
7885 Type *UTy = BO->LHS->getType();
7886 const SCEV *Z0 = Z->getOperand();
7887 Type *Z0Ty = Z0->getType();
7888 unsigned Z0TySize = getTypeSizeInBits(Ty: Z0Ty);
7889
7890 // If C is a low-bits mask, the zero extend is serving to
7891 // mask off the high bits. Complement the operand and
7892 // re-apply the zext.
7893 if (CI->getValue().isMask(numBits: Z0TySize))
7894 return getZeroExtendExpr(Op: getNotSCEV(V: Z0), Ty: UTy);
7895
7896 // If C is a single bit, it may be in the sign-bit position
7897 // before the zero-extend. In this case, represent the xor
7898 // using an add, which is equivalent, and re-apply the zext.
7899 APInt Trunc = CI->getValue().trunc(width: Z0TySize);
7900 if (Trunc.zext(width: getTypeSizeInBits(Ty: UTy)) == CI->getValue() &&
7901 Trunc.isSignMask())
7902 return getZeroExtendExpr(Op: getAddExpr(LHS: Z0, RHS: getConstant(Val: Trunc)),
7903 Ty: UTy);
7904 }
7905 }
7906 break;
7907
7908 case Instruction::Shl:
7909 // Turn shift left of a constant amount into a multiply.
7910 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: BO->RHS)) {
7911 uint32_t BitWidth = cast<IntegerType>(Val: SA->getType())->getBitWidth();
7912
7913 // If the shift count is not less than the bitwidth, the result of
7914 // the shift is undefined. Don't try to analyze it, because the
7915 // resolution chosen here may differ from the resolution chosen in
7916 // other parts of the compiler.
7917 if (SA->getValue().uge(RHS: BitWidth))
7918 break;
7919
7920 // We can safely preserve the nuw flag in all cases. It's also safe to
7921 // turn a nuw nsw shl into a nuw nsw mul. However, nsw in isolation
7922 // requires special handling. It can be preserved as long as we're not
7923 // left shifting by bitwidth - 1.
7924 auto Flags = SCEV::FlagAnyWrap;
7925 if (BO->Op) {
7926 auto MulFlags = getNoWrapFlagsFromUB(V: BO->Op);
7927 if ((MulFlags & SCEV::FlagNSW) &&
7928 ((MulFlags & SCEV::FlagNUW) || SA->getValue().ult(RHS: BitWidth - 1)))
7929 Flags = (SCEV::NoWrapFlags)(Flags | SCEV::FlagNSW);
7930 if (MulFlags & SCEV::FlagNUW)
7931 Flags = (SCEV::NoWrapFlags)(Flags | SCEV::FlagNUW);
7932 }
7933
7934 ConstantInt *X = ConstantInt::get(
7935 Context&: getContext(), V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
7936 return getMulExpr(LHS: getSCEV(V: BO->LHS), RHS: getConstant(V: X), Flags);
7937 }
7938 break;
7939
7940 case Instruction::AShr:
7941 // AShr X, C, where C is a constant.
7942 ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS);
7943 if (!CI)
7944 break;
7945
7946 Type *OuterTy = BO->LHS->getType();
7947 uint64_t BitWidth = getTypeSizeInBits(Ty: OuterTy);
7948 // If the shift count is not less than the bitwidth, the result of
7949 // the shift is undefined. Don't try to analyze it, because the
7950 // resolution chosen here may differ from the resolution chosen in
7951 // other parts of the compiler.
7952 if (CI->getValue().uge(RHS: BitWidth))
7953 break;
7954
7955 if (CI->isZero())
7956 return getSCEV(V: BO->LHS); // shift by zero --> noop
7957
7958 uint64_t AShrAmt = CI->getZExtValue();
7959 Type *TruncTy = IntegerType::get(C&: getContext(), NumBits: BitWidth - AShrAmt);
7960
7961 Operator *L = dyn_cast<Operator>(Val: BO->LHS);
7962 const SCEV *AddTruncateExpr = nullptr;
7963 ConstantInt *ShlAmtCI = nullptr;
7964 const SCEV *AddConstant = nullptr;
7965
7966 if (L && L->getOpcode() == Instruction::Add) {
7967 // X = Shl A, n
7968 // Y = Add X, c
7969 // Z = AShr Y, m
7970 // n, c and m are constants.
7971
7972 Operator *LShift = dyn_cast<Operator>(Val: L->getOperand(i: 0));
7973 ConstantInt *AddOperandCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
7974 if (LShift && LShift->getOpcode() == Instruction::Shl) {
7975 if (AddOperandCI) {
7976 const SCEV *ShlOp0SCEV = getSCEV(V: LShift->getOperand(i: 0));
7977 ShlAmtCI = dyn_cast<ConstantInt>(Val: LShift->getOperand(i: 1));
7978 // since we truncate to TruncTy, the AddConstant should be of the
7979 // same type, so create a new Constant with type same as TruncTy.
7980 // Also, the Add constant should be shifted right by AShr amount.
7981 APInt AddOperand = AddOperandCI->getValue().ashr(ShiftAmt: AShrAmt);
7982 AddConstant = getConstant(Val: AddOperand.trunc(width: BitWidth - AShrAmt));
7983 // we model the expression as sext(add(trunc(A), c << n)), since the
7984 // sext(trunc) part is already handled below, we create a
7985 // AddExpr(TruncExp) which will be used later.
7986 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
7987 }
7988 }
7989 } else if (L && L->getOpcode() == Instruction::Shl) {
7990 // X = Shl A, n
7991 // Y = AShr X, m
7992 // Both n and m are constant.
7993
7994 const SCEV *ShlOp0SCEV = getSCEV(V: L->getOperand(i: 0));
7995 ShlAmtCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
7996 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
7997 }
7998
7999 if (AddTruncateExpr && ShlAmtCI) {
8000 // We can merge the two given cases into a single SCEV statement,
8001 // incase n = m, the mul expression will be 2^0, so it gets resolved to
8002 // a simpler case. The following code handles the two cases:
8003 //
8004 // 1) For a two-shift sext-inreg, i.e. n = m,
8005 // use sext(trunc(x)) as the SCEV expression.
8006 //
8007 // 2) When n > m, use sext(mul(trunc(x), 2^(n-m)))) as the SCEV
8008 // expression. We already checked that ShlAmt < BitWidth, so
8009 // the multiplier, 1 << (ShlAmt - AShrAmt), fits into TruncTy as
8010 // ShlAmt - AShrAmt < Amt.
8011 const APInt &ShlAmt = ShlAmtCI->getValue();
8012 if (ShlAmt.ult(RHS: BitWidth) && ShlAmt.uge(RHS: AShrAmt)) {
8013 APInt Mul = APInt::getOneBitSet(numBits: BitWidth - AShrAmt,
8014 BitNo: ShlAmtCI->getZExtValue() - AShrAmt);
8015 const SCEV *CompositeExpr =
8016 getMulExpr(LHS: AddTruncateExpr, RHS: getConstant(Val: Mul));
8017 if (L->getOpcode() != Instruction::Shl)
8018 CompositeExpr = getAddExpr(LHS: CompositeExpr, RHS: AddConstant);
8019
8020 return getSignExtendExpr(Op: CompositeExpr, Ty: OuterTy);
8021 }
8022 }
8023 break;
8024 }
8025 }
8026
8027 switch (U->getOpcode()) {
8028 case Instruction::Trunc:
8029 return getTruncateExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8030
8031 case Instruction::ZExt:
8032 return getZeroExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8033
8034 case Instruction::SExt:
8035 if (auto BO = MatchBinaryOp(V: U->getOperand(i: 0), DL: getDataLayout(), AC, DT,
8036 CxtI: dyn_cast<Instruction>(Val: V))) {
8037 // The NSW flag of a subtract does not always survive the conversion to
8038 // A + (-1)*B. By pushing sign extension onto its operands we are much
8039 // more likely to preserve NSW and allow later AddRec optimisations.
8040 //
8041 // NOTE: This is effectively duplicating this logic from getSignExtend:
8042 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
8043 // but by that point the NSW information has potentially been lost.
8044 if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
8045 Type *Ty = U->getType();
8046 auto *V1 = getSignExtendExpr(Op: getSCEV(V: BO->LHS), Ty);
8047 auto *V2 = getSignExtendExpr(Op: getSCEV(V: BO->RHS), Ty);
8048 return getMinusSCEV(LHS: V1, RHS: V2, Flags: SCEV::FlagNSW);
8049 }
8050 }
8051 return getSignExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8052
8053 case Instruction::BitCast:
8054 // BitCasts are no-op casts so we just eliminate the cast.
8055 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType()))
8056 return getSCEV(V: U->getOperand(i: 0));
8057 break;
8058
8059 case Instruction::PtrToInt: {
8060 // Pointer to integer cast is straight-forward, so do model it.
8061 const SCEV *Op = getSCEV(V: U->getOperand(i: 0));
8062 Type *DstIntTy = U->getType();
8063 // But only if effective SCEV (integer) type is wide enough to represent
8064 // all possible pointer values.
8065 const SCEV *IntOp = getPtrToIntExpr(Op, Ty: DstIntTy);
8066 if (isa<SCEVCouldNotCompute>(Val: IntOp))
8067 return getUnknown(V);
8068 return IntOp;
8069 }
8070 case Instruction::IntToPtr:
8071 // Just don't deal with inttoptr casts.
8072 return getUnknown(V);
8073
8074 case Instruction::SDiv:
8075 // If both operands are non-negative, this is just an udiv.
8076 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8077 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8078 return getUDivExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8079 break;
8080
8081 case Instruction::SRem:
8082 // If both operands are non-negative, this is just an urem.
8083 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8084 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8085 return getURemExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8086 break;
8087
8088 case Instruction::GetElementPtr:
8089 return createNodeForGEP(GEP: cast<GEPOperator>(Val: U));
8090
8091 case Instruction::PHI:
8092 return createNodeForPHI(PN: cast<PHINode>(Val: U));
8093
8094 case Instruction::Select:
8095 return createNodeForSelectOrPHI(V: U, Cond: U->getOperand(i: 0), TrueVal: U->getOperand(i: 1),
8096 FalseVal: U->getOperand(i: 2));
8097
8098 case Instruction::Call:
8099 case Instruction::Invoke:
8100 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand())
8101 return getSCEV(V: RV);
8102
8103 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
8104 switch (II->getIntrinsicID()) {
8105 case Intrinsic::abs:
8106 return getAbsExpr(
8107 Op: getSCEV(V: II->getArgOperand(i: 0)),
8108 /*IsNSW=*/cast<ConstantInt>(Val: II->getArgOperand(i: 1))->isOne());
8109 case Intrinsic::umax:
8110 LHS = getSCEV(V: II->getArgOperand(i: 0));
8111 RHS = getSCEV(V: II->getArgOperand(i: 1));
8112 return getUMaxExpr(LHS, RHS);
8113 case Intrinsic::umin:
8114 LHS = getSCEV(V: II->getArgOperand(i: 0));
8115 RHS = getSCEV(V: II->getArgOperand(i: 1));
8116 return getUMinExpr(LHS, RHS);
8117 case Intrinsic::smax:
8118 LHS = getSCEV(V: II->getArgOperand(i: 0));
8119 RHS = getSCEV(V: II->getArgOperand(i: 1));
8120 return getSMaxExpr(LHS, RHS);
8121 case Intrinsic::smin:
8122 LHS = getSCEV(V: II->getArgOperand(i: 0));
8123 RHS = getSCEV(V: II->getArgOperand(i: 1));
8124 return getSMinExpr(LHS, RHS);
8125 case Intrinsic::usub_sat: {
8126 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8127 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8128 const SCEV *ClampedY = getUMinExpr(LHS: X, RHS: Y);
8129 return getMinusSCEV(LHS: X, RHS: ClampedY, Flags: SCEV::FlagNUW);
8130 }
8131 case Intrinsic::uadd_sat: {
8132 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8133 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8134 const SCEV *ClampedX = getUMinExpr(LHS: X, RHS: getNotSCEV(V: Y));
8135 return getAddExpr(LHS: ClampedX, RHS: Y, Flags: SCEV::FlagNUW);
8136 }
8137 case Intrinsic::start_loop_iterations:
8138 case Intrinsic::annotation:
8139 case Intrinsic::ptr_annotation:
8140 // A start_loop_iterations or llvm.annotation or llvm.prt.annotation is
8141 // just eqivalent to the first operand for SCEV purposes.
8142 return getSCEV(V: II->getArgOperand(i: 0));
8143 case Intrinsic::vscale:
8144 return getVScale(Ty: II->getType());
8145 default:
8146 break;
8147 }
8148 }
8149 break;
8150 }
8151
8152 return getUnknown(V);
8153}
8154
8155//===----------------------------------------------------------------------===//
8156// Iteration Count Computation Code
8157//
8158
8159const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount) {
8160 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8161 return getCouldNotCompute();
8162
8163 auto *ExitCountType = ExitCount->getType();
8164 assert(ExitCountType->isIntegerTy());
8165 auto *EvalTy = Type::getIntNTy(C&: ExitCountType->getContext(),
8166 N: 1 + ExitCountType->getScalarSizeInBits());
8167 return getTripCountFromExitCount(ExitCount, EvalTy, L: nullptr);
8168}
8169
8170const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount,
8171 Type *EvalTy,
8172 const Loop *L) {
8173 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8174 return getCouldNotCompute();
8175
8176 unsigned ExitCountSize = getTypeSizeInBits(Ty: ExitCount->getType());
8177 unsigned EvalSize = EvalTy->getPrimitiveSizeInBits();
8178
8179 auto CanAddOneWithoutOverflow = [&]() {
8180 ConstantRange ExitCountRange =
8181 getRangeRef(S: ExitCount, SignHint: RangeSignHint::HINT_RANGE_UNSIGNED);
8182 if (!ExitCountRange.contains(Val: APInt::getMaxValue(numBits: ExitCountSize)))
8183 return true;
8184
8185 return L && isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: ExitCount,
8186 RHS: getMinusOne(Ty: ExitCount->getType()));
8187 };
8188
8189 // If we need to zero extend the backedge count, check if we can add one to
8190 // it prior to zero extending without overflow. Provided this is safe, it
8191 // allows better simplification of the +1.
8192 if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
8193 return getZeroExtendExpr(
8194 Op: getAddExpr(LHS: ExitCount, RHS: getOne(Ty: ExitCount->getType())), Ty: EvalTy);
8195
8196 // Get the total trip count from the count by adding 1. This may wrap.
8197 return getAddExpr(LHS: getTruncateOrZeroExtend(V: ExitCount, Ty: EvalTy), RHS: getOne(Ty: EvalTy));
8198}
8199
8200static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
8201 if (!ExitCount)
8202 return 0;
8203
8204 ConstantInt *ExitConst = ExitCount->getValue();
8205
8206 // Guard against huge trip counts.
8207 if (ExitConst->getValue().getActiveBits() > 32)
8208 return 0;
8209
8210 // In case of integer overflow, this returns 0, which is correct.
8211 return ((unsigned)ExitConst->getZExtValue()) + 1;
8212}
8213
8214unsigned ScalarEvolution::getSmallConstantTripCount(const Loop *L) {
8215 auto *ExitCount = dyn_cast<SCEVConstant>(Val: getBackedgeTakenCount(L, Kind: Exact));
8216 return getConstantTripCount(ExitCount);
8217}
8218
8219unsigned
8220ScalarEvolution::getSmallConstantTripCount(const Loop *L,
8221 const BasicBlock *ExitingBlock) {
8222 assert(ExitingBlock && "Must pass a non-null exiting block!");
8223 assert(L->isLoopExiting(ExitingBlock) &&
8224 "Exiting block must actually branch out of the loop!");
8225 const SCEVConstant *ExitCount =
8226 dyn_cast<SCEVConstant>(Val: getExitCount(L, ExitingBlock));
8227 return getConstantTripCount(ExitCount);
8228}
8229
8230unsigned ScalarEvolution::getSmallConstantMaxTripCount(
8231 const Loop *L, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8232
8233 const auto *MaxExitCount =
8234 Predicates ? getPredicatedConstantMaxBackedgeTakenCount(L, Predicates&: *Predicates)
8235 : getConstantMaxBackedgeTakenCount(L);
8236 return getConstantTripCount(ExitCount: dyn_cast<SCEVConstant>(Val: MaxExitCount));
8237}
8238
8239unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L) {
8240 SmallVector<BasicBlock *, 8> ExitingBlocks;
8241 L->getExitingBlocks(ExitingBlocks);
8242
8243 std::optional<unsigned> Res;
8244 for (auto *ExitingBB : ExitingBlocks) {
8245 unsigned Multiple = getSmallConstantTripMultiple(L, ExitingBlock: ExitingBB);
8246 if (!Res)
8247 Res = Multiple;
8248 Res = std::gcd(m: *Res, n: Multiple);
8249 }
8250 return Res.value_or(u: 1);
8251}
8252
8253unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8254 const SCEV *ExitCount) {
8255 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8256 return 1;
8257
8258 // Get the trip count
8259 const SCEV *TCExpr = getTripCountFromExitCount(ExitCount: applyLoopGuards(Expr: ExitCount, L));
8260
8261 APInt Multiple = getNonZeroConstantMultiple(S: TCExpr);
8262 // If a trip multiple is huge (>=2^32), the trip count is still divisible by
8263 // the greatest power of 2 divisor less than 2^32.
8264 return Multiple.getActiveBits() > 32
8265 ? 1U << std::min(a: 31U, b: Multiple.countTrailingZeros())
8266 : (unsigned)Multiple.getZExtValue();
8267}
8268
8269/// Returns the largest constant divisor of the trip count of this loop as a
8270/// normal unsigned value, if possible. This means that the actual trip count is
8271/// always a multiple of the returned value (don't forget the trip count could
8272/// very well be zero as well!).
8273///
8274/// Returns 1 if the trip count is unknown or not guaranteed to be the
8275/// multiple of a constant (which is also the case if the trip count is simply
8276/// constant, use getSmallConstantTripCount for that case), Will also return 1
8277/// if the trip count is very large (>= 2^32).
8278///
8279/// As explained in the comments for getSmallConstantTripCount, this assumes
8280/// that control exits the loop via ExitingBlock.
8281unsigned
8282ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8283 const BasicBlock *ExitingBlock) {
8284 assert(ExitingBlock && "Must pass a non-null exiting block!");
8285 assert(L->isLoopExiting(ExitingBlock) &&
8286 "Exiting block must actually branch out of the loop!");
8287 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
8288 return getSmallConstantTripMultiple(L, ExitCount);
8289}
8290
8291const SCEV *ScalarEvolution::getExitCount(const Loop *L,
8292 const BasicBlock *ExitingBlock,
8293 ExitCountKind Kind) {
8294 switch (Kind) {
8295 case Exact:
8296 return getBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this);
8297 case SymbolicMaximum:
8298 return getBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this);
8299 case ConstantMaximum:
8300 return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this);
8301 };
8302 llvm_unreachable("Invalid ExitCountKind!");
8303}
8304
8305const SCEV *ScalarEvolution::getPredicatedExitCount(
8306 const Loop *L, const BasicBlock *ExitingBlock,
8307 SmallVectorImpl<const SCEVPredicate *> *Predicates, ExitCountKind Kind) {
8308 switch (Kind) {
8309 case Exact:
8310 return getPredicatedBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this,
8311 Predicates);
8312 case SymbolicMaximum:
8313 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this,
8314 Predicates);
8315 case ConstantMaximum:
8316 return getPredicatedBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this,
8317 Predicates);
8318 };
8319 llvm_unreachable("Invalid ExitCountKind!");
8320}
8321
8322const SCEV *ScalarEvolution::getPredicatedBackedgeTakenCount(
8323 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8324 return getPredicatedBackedgeTakenInfo(L).getExact(L, SE: this, Predicates: &Preds);
8325}
8326
8327const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L,
8328 ExitCountKind Kind) {
8329 switch (Kind) {
8330 case Exact:
8331 return getBackedgeTakenInfo(L).getExact(L, SE: this);
8332 case ConstantMaximum:
8333 return getBackedgeTakenInfo(L).getConstantMax(SE: this);
8334 case SymbolicMaximum:
8335 return getBackedgeTakenInfo(L).getSymbolicMax(L, SE: this);
8336 };
8337 llvm_unreachable("Invalid ExitCountKind!");
8338}
8339
8340const SCEV *ScalarEvolution::getPredicatedSymbolicMaxBackedgeTakenCount(
8341 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8342 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, SE: this, Predicates: &Preds);
8343}
8344
8345const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
8346 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8347 return getPredicatedBackedgeTakenInfo(L).getConstantMax(SE: this, Predicates: &Preds);
8348}
8349
8350bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
8351 return getBackedgeTakenInfo(L).isConstantMaxOrZero(SE: this);
8352}
8353
8354/// Push PHI nodes in the header of the given loop onto the given Worklist.
8355static void PushLoopPHIs(const Loop *L,
8356 SmallVectorImpl<Instruction *> &Worklist,
8357 SmallPtrSetImpl<Instruction *> &Visited) {
8358 BasicBlock *Header = L->getHeader();
8359
8360 // Push all Loop-header PHIs onto the Worklist stack.
8361 for (PHINode &PN : Header->phis())
8362 if (Visited.insert(Ptr: &PN).second)
8363 Worklist.push_back(Elt: &PN);
8364}
8365
8366ScalarEvolution::BackedgeTakenInfo &
8367ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) {
8368 auto &BTI = getBackedgeTakenInfo(L);
8369 if (BTI.hasFullInfo())
8370 return BTI;
8371
8372 auto Pair = PredicatedBackedgeTakenCounts.try_emplace(Key: L);
8373
8374 if (!Pair.second)
8375 return Pair.first->second;
8376
8377 BackedgeTakenInfo Result =
8378 computeBackedgeTakenCount(L, /*AllowPredicates=*/true);
8379
8380 return PredicatedBackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8381}
8382
8383ScalarEvolution::BackedgeTakenInfo &
8384ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
8385 // Initially insert an invalid entry for this loop. If the insertion
8386 // succeeds, proceed to actually compute a backedge-taken count and
8387 // update the value. The temporary CouldNotCompute value tells SCEV
8388 // code elsewhere that it shouldn't attempt to request a new
8389 // backedge-taken count, which could result in infinite recursion.
8390 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
8391 BackedgeTakenCounts.try_emplace(Key: L);
8392 if (!Pair.second)
8393 return Pair.first->second;
8394
8395 // computeBackedgeTakenCount may allocate memory for its result. Inserting it
8396 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
8397 // must be cleared in this scope.
8398 BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
8399
8400 // Now that we know more about the trip count for this loop, forget any
8401 // existing SCEV values for PHI nodes in this loop since they are only
8402 // conservative estimates made without the benefit of trip count
8403 // information. This invalidation is not necessary for correctness, and is
8404 // only done to produce more precise results.
8405 if (Result.hasAnyInfo()) {
8406 // Invalidate any expression using an addrec in this loop.
8407 SmallVector<const SCEV *, 8> ToForget;
8408 auto LoopUsersIt = LoopUsers.find(Val: L);
8409 if (LoopUsersIt != LoopUsers.end())
8410 append_range(C&: ToForget, R&: LoopUsersIt->second);
8411 forgetMemoizedResults(SCEVs: ToForget);
8412
8413 // Invalidate constant-evolved loop header phis.
8414 for (PHINode &PN : L->getHeader()->phis())
8415 ConstantEvolutionLoopExitValue.erase(Val: &PN);
8416 }
8417
8418 // Re-lookup the insert position, since the call to
8419 // computeBackedgeTakenCount above could result in a
8420 // recusive call to getBackedgeTakenInfo (on a different
8421 // loop), which would invalidate the iterator computed
8422 // earlier.
8423 return BackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8424}
8425
8426void ScalarEvolution::forgetAllLoops() {
8427 // This method is intended to forget all info about loops. It should
8428 // invalidate caches as if the following happened:
8429 // - The trip counts of all loops have changed arbitrarily
8430 // - Every llvm::Value has been updated in place to produce a different
8431 // result.
8432 BackedgeTakenCounts.clear();
8433 PredicatedBackedgeTakenCounts.clear();
8434 BECountUsers.clear();
8435 LoopPropertiesCache.clear();
8436 ConstantEvolutionLoopExitValue.clear();
8437 ValueExprMap.clear();
8438 ValuesAtScopes.clear();
8439 ValuesAtScopesUsers.clear();
8440 LoopDispositions.clear();
8441 BlockDispositions.clear();
8442 UnsignedRanges.clear();
8443 SignedRanges.clear();
8444 ExprValueMap.clear();
8445 HasRecMap.clear();
8446 ConstantMultipleCache.clear();
8447 PredicatedSCEVRewrites.clear();
8448 FoldCache.clear();
8449 FoldCacheUser.clear();
8450}
8451void ScalarEvolution::visitAndClearUsers(
8452 SmallVectorImpl<Instruction *> &Worklist,
8453 SmallPtrSetImpl<Instruction *> &Visited,
8454 SmallVectorImpl<const SCEV *> &ToForget) {
8455 while (!Worklist.empty()) {
8456 Instruction *I = Worklist.pop_back_val();
8457 if (!isSCEVable(Ty: I->getType()) && !isa<WithOverflowInst>(Val: I))
8458 continue;
8459
8460 ValueExprMapType::iterator It =
8461 ValueExprMap.find_as(Val: static_cast<Value *>(I));
8462 if (It != ValueExprMap.end()) {
8463 eraseValueFromMap(V: It->first);
8464 ToForget.push_back(Elt: It->second);
8465 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
8466 ConstantEvolutionLoopExitValue.erase(Val: PN);
8467 }
8468
8469 PushDefUseChildren(I, Worklist, Visited);
8470 }
8471}
8472
8473void ScalarEvolution::forgetLoop(const Loop *L) {
8474 SmallVector<const Loop *, 16> LoopWorklist(1, L);
8475 SmallVector<Instruction *, 32> Worklist;
8476 SmallPtrSet<Instruction *, 16> Visited;
8477 SmallVector<const SCEV *, 16> ToForget;
8478
8479 // Iterate over all the loops and sub-loops to drop SCEV information.
8480 while (!LoopWorklist.empty()) {
8481 auto *CurrL = LoopWorklist.pop_back_val();
8482
8483 // Drop any stored trip count value.
8484 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ false);
8485 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ true);
8486
8487 // Drop information about predicated SCEV rewrites for this loop.
8488 for (auto I = PredicatedSCEVRewrites.begin();
8489 I != PredicatedSCEVRewrites.end();) {
8490 std::pair<const SCEV *, const Loop *> Entry = I->first;
8491 if (Entry.second == CurrL)
8492 PredicatedSCEVRewrites.erase(I: I++);
8493 else
8494 ++I;
8495 }
8496
8497 auto LoopUsersItr = LoopUsers.find(Val: CurrL);
8498 if (LoopUsersItr != LoopUsers.end())
8499 llvm::append_range(C&: ToForget, R&: LoopUsersItr->second);
8500
8501 // Drop information about expressions based on loop-header PHIs.
8502 PushLoopPHIs(L: CurrL, Worklist, Visited);
8503 visitAndClearUsers(Worklist, Visited, ToForget);
8504
8505 LoopPropertiesCache.erase(Val: CurrL);
8506 // Forget all contained loops too, to avoid dangling entries in the
8507 // ValuesAtScopes map.
8508 LoopWorklist.append(in_start: CurrL->begin(), in_end: CurrL->end());
8509 }
8510 forgetMemoizedResults(SCEVs: ToForget);
8511}
8512
8513void ScalarEvolution::forgetTopmostLoop(const Loop *L) {
8514 forgetLoop(L: L->getOutermostLoop());
8515}
8516
8517void ScalarEvolution::forgetValue(Value *V) {
8518 Instruction *I = dyn_cast<Instruction>(Val: V);
8519 if (!I) return;
8520
8521 // Drop information about expressions based on loop-header PHIs.
8522 SmallVector<Instruction *, 16> Worklist;
8523 SmallPtrSet<Instruction *, 8> Visited;
8524 SmallVector<const SCEV *, 8> ToForget;
8525 Worklist.push_back(Elt: I);
8526 Visited.insert(Ptr: I);
8527 visitAndClearUsers(Worklist, Visited, ToForget);
8528
8529 forgetMemoizedResults(SCEVs: ToForget);
8530}
8531
8532void ScalarEvolution::forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V) {
8533 if (!isSCEVable(Ty: V->getType()))
8534 return;
8535
8536 // If SCEV looked through a trivial LCSSA phi node, we might have SCEV's
8537 // directly using a SCEVUnknown/SCEVAddRec defined in the loop. After an
8538 // extra predecessor is added, this is no longer valid. Find all Unknowns and
8539 // AddRecs defined in the loop and invalidate any SCEV's making use of them.
8540 if (const SCEV *S = getExistingSCEV(V)) {
8541 struct InvalidationRootCollector {
8542 Loop *L;
8543 SmallVector<const SCEV *, 8> Roots;
8544
8545 InvalidationRootCollector(Loop *L) : L(L) {}
8546
8547 bool follow(const SCEV *S) {
8548 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
8549 if (auto *I = dyn_cast<Instruction>(Val: SU->getValue()))
8550 if (L->contains(Inst: I))
8551 Roots.push_back(Elt: S);
8552 } else if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S)) {
8553 if (L->contains(L: AddRec->getLoop()))
8554 Roots.push_back(Elt: S);
8555 }
8556 return true;
8557 }
8558 bool isDone() const { return false; }
8559 };
8560
8561 InvalidationRootCollector C(L);
8562 visitAll(Root: S, Visitor&: C);
8563 forgetMemoizedResults(SCEVs: C.Roots);
8564 }
8565
8566 // Also perform the normal invalidation.
8567 forgetValue(V);
8568}
8569
8570void ScalarEvolution::forgetLoopDispositions() { LoopDispositions.clear(); }
8571
8572void ScalarEvolution::forgetBlockAndLoopDispositions(Value *V) {
8573 // Unless a specific value is passed to invalidation, completely clear both
8574 // caches.
8575 if (!V) {
8576 BlockDispositions.clear();
8577 LoopDispositions.clear();
8578 return;
8579 }
8580
8581 if (!isSCEVable(Ty: V->getType()))
8582 return;
8583
8584 const SCEV *S = getExistingSCEV(V);
8585 if (!S)
8586 return;
8587
8588 // Invalidate the block and loop dispositions cached for S. Dispositions of
8589 // S's users may change if S's disposition changes (i.e. a user may change to
8590 // loop-invariant, if S changes to loop invariant), so also invalidate
8591 // dispositions of S's users recursively.
8592 SmallVector<const SCEV *, 8> Worklist = {S};
8593 SmallPtrSet<const SCEV *, 8> Seen = {S};
8594 while (!Worklist.empty()) {
8595 const SCEV *Curr = Worklist.pop_back_val();
8596 bool LoopDispoRemoved = LoopDispositions.erase(Val: Curr);
8597 bool BlockDispoRemoved = BlockDispositions.erase(Val: Curr);
8598 if (!LoopDispoRemoved && !BlockDispoRemoved)
8599 continue;
8600 auto Users = SCEVUsers.find(Val: Curr);
8601 if (Users != SCEVUsers.end())
8602 for (const auto *User : Users->second)
8603 if (Seen.insert(Ptr: User).second)
8604 Worklist.push_back(Elt: User);
8605 }
8606}
8607
8608/// Get the exact loop backedge taken count considering all loop exits. A
8609/// computable result can only be returned for loops with all exiting blocks
8610/// dominating the latch. howFarToZero assumes that the limit of each loop test
8611/// is never skipped. This is a valid assumption as long as the loop exits via
8612/// that test. For precise results, it is the caller's responsibility to specify
8613/// the relevant loop exiting block using getExact(ExitingBlock, SE).
8614const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
8615 const Loop *L, ScalarEvolution *SE,
8616 SmallVectorImpl<const SCEVPredicate *> *Preds) const {
8617 // If any exits were not computable, the loop is not computable.
8618 if (!isComplete() || ExitNotTaken.empty())
8619 return SE->getCouldNotCompute();
8620
8621 const BasicBlock *Latch = L->getLoopLatch();
8622 // All exiting blocks we have collected must dominate the only backedge.
8623 if (!Latch)
8624 return SE->getCouldNotCompute();
8625
8626 // All exiting blocks we have gathered dominate loop's latch, so exact trip
8627 // count is simply a minimum out of all these calculated exit counts.
8628 SmallVector<const SCEV *, 2> Ops;
8629 for (const auto &ENT : ExitNotTaken) {
8630 const SCEV *BECount = ENT.ExactNotTaken;
8631 assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!");
8632 assert(SE->DT.dominates(ENT.ExitingBlock, Latch) &&
8633 "We should only have known counts for exiting blocks that dominate "
8634 "latch!");
8635
8636 Ops.push_back(Elt: BECount);
8637
8638 if (Preds)
8639 append_range(C&: *Preds, R: ENT.Predicates);
8640
8641 assert((Preds || ENT.hasAlwaysTruePredicate()) &&
8642 "Predicate should be always true!");
8643 }
8644
8645 // If an earlier exit exits on the first iteration (exit count zero), then
8646 // a later poison exit count should not propagate into the result. This are
8647 // exactly the semantics provided by umin_seq.
8648 return SE->getUMinFromMismatchedTypes(Ops, /* Sequential */ true);
8649}
8650
8651const ScalarEvolution::ExitNotTakenInfo *
8652ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
8653 const BasicBlock *ExitingBlock,
8654 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8655 for (const auto &ENT : ExitNotTaken)
8656 if (ENT.ExitingBlock == ExitingBlock) {
8657 if (ENT.hasAlwaysTruePredicate())
8658 return &ENT;
8659 else if (Predicates) {
8660 append_range(C&: *Predicates, R: ENT.Predicates);
8661 return &ENT;
8662 }
8663 }
8664
8665 return nullptr;
8666}
8667
8668/// getConstantMax - Get the constant max backedge taken count for the loop.
8669const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
8670 ScalarEvolution *SE,
8671 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8672 if (!getConstantMax())
8673 return SE->getCouldNotCompute();
8674
8675 for (const auto &ENT : ExitNotTaken)
8676 if (!ENT.hasAlwaysTruePredicate()) {
8677 if (!Predicates)
8678 return SE->getCouldNotCompute();
8679 append_range(C&: *Predicates, R: ENT.Predicates);
8680 }
8681
8682 assert((isa<SCEVCouldNotCompute>(getConstantMax()) ||
8683 isa<SCEVConstant>(getConstantMax())) &&
8684 "No point in having a non-constant max backedge taken count!");
8685 return getConstantMax();
8686}
8687
8688const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
8689 const Loop *L, ScalarEvolution *SE,
8690 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8691 if (!SymbolicMax) {
8692 // Form an expression for the maximum exit count possible for this loop. We
8693 // merge the max and exact information to approximate a version of
8694 // getConstantMaxBackedgeTakenCount which isn't restricted to just
8695 // constants.
8696 SmallVector<const SCEV *, 4> ExitCounts;
8697
8698 for (const auto &ENT : ExitNotTaken) {
8699 const SCEV *ExitCount = ENT.SymbolicMaxNotTaken;
8700 if (!isa<SCEVCouldNotCompute>(Val: ExitCount)) {
8701 assert(SE->DT.dominates(ENT.ExitingBlock, L->getLoopLatch()) &&
8702 "We should only have known counts for exiting blocks that "
8703 "dominate latch!");
8704 ExitCounts.push_back(Elt: ExitCount);
8705 if (Predicates)
8706 append_range(C&: *Predicates, R: ENT.Predicates);
8707
8708 assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
8709 "Predicate should be always true!");
8710 }
8711 }
8712 if (ExitCounts.empty())
8713 SymbolicMax = SE->getCouldNotCompute();
8714 else
8715 SymbolicMax =
8716 SE->getUMinFromMismatchedTypes(Ops&: ExitCounts, /*Sequential*/ true);
8717 }
8718 return SymbolicMax;
8719}
8720
8721bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
8722 ScalarEvolution *SE) const {
8723 auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
8724 return !ENT.hasAlwaysTruePredicate();
8725 };
8726 return MaxOrZero && !any_of(Range: ExitNotTaken, P: PredicateNotAlwaysTrue);
8727}
8728
8729ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E)
8730 : ExitLimit(E, E, E, false) {}
8731
8732ScalarEvolution::ExitLimit::ExitLimit(
8733 const SCEV *E, const SCEV *ConstantMaxNotTaken,
8734 const SCEV *SymbolicMaxNotTaken, bool MaxOrZero,
8735 ArrayRef<ArrayRef<const SCEVPredicate *>> PredLists)
8736 : ExactNotTaken(E), ConstantMaxNotTaken(ConstantMaxNotTaken),
8737 SymbolicMaxNotTaken(SymbolicMaxNotTaken), MaxOrZero(MaxOrZero) {
8738 // If we prove the max count is zero, so is the symbolic bound. This happens
8739 // in practice due to differences in a) how context sensitive we've chosen
8740 // to be and b) how we reason about bounds implied by UB.
8741 if (ConstantMaxNotTaken->isZero()) {
8742 this->ExactNotTaken = E = ConstantMaxNotTaken;
8743 this->SymbolicMaxNotTaken = SymbolicMaxNotTaken = ConstantMaxNotTaken;
8744 }
8745
8746 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
8747 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
8748 "Exact is not allowed to be less precise than Constant Max");
8749 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
8750 !isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken)) &&
8751 "Exact is not allowed to be less precise than Symbolic Max");
8752 assert((isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken) ||
8753 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
8754 "Symbolic Max is not allowed to be less precise than Constant Max");
8755 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
8756 isa<SCEVConstant>(ConstantMaxNotTaken)) &&
8757 "No point in having a non-constant max backedge taken count!");
8758 SmallPtrSet<const SCEVPredicate *, 4> SeenPreds;
8759 for (const auto PredList : PredLists)
8760 for (const auto *P : PredList) {
8761 if (SeenPreds.contains(Ptr: P))
8762 continue;
8763 assert(!isa<SCEVUnionPredicate>(P) && "Only add leaf predicates here!");
8764 SeenPreds.insert(Ptr: P);
8765 Predicates.push_back(Elt: P);
8766 }
8767 assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) &&
8768 "Backedge count should be int");
8769 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
8770 !ConstantMaxNotTaken->getType()->isPointerTy()) &&
8771 "Max backedge count should be int");
8772}
8773
8774ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E,
8775 const SCEV *ConstantMaxNotTaken,
8776 const SCEV *SymbolicMaxNotTaken,
8777 bool MaxOrZero,
8778 ArrayRef<const SCEVPredicate *> PredList)
8779 : ExitLimit(E, ConstantMaxNotTaken, SymbolicMaxNotTaken, MaxOrZero,
8780 ArrayRef({PredList})) {}
8781
8782/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
8783/// computable exit into a persistent ExitNotTakenInfo array.
8784ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
8785 ArrayRef<ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo> ExitCounts,
8786 bool IsComplete, const SCEV *ConstantMax, bool MaxOrZero)
8787 : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
8788 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8789
8790 ExitNotTaken.reserve(N: ExitCounts.size());
8791 std::transform(first: ExitCounts.begin(), last: ExitCounts.end(),
8792 result: std::back_inserter(x&: ExitNotTaken),
8793 unary_op: [&](const EdgeExitInfo &EEI) {
8794 BasicBlock *ExitBB = EEI.first;
8795 const ExitLimit &EL = EEI.second;
8796 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
8797 EL.ConstantMaxNotTaken, EL.SymbolicMaxNotTaken,
8798 EL.Predicates);
8799 });
8800 assert((isa<SCEVCouldNotCompute>(ConstantMax) ||
8801 isa<SCEVConstant>(ConstantMax)) &&
8802 "No point in having a non-constant max backedge taken count!");
8803}
8804
8805/// Compute the number of times the backedge of the specified loop will execute.
8806ScalarEvolution::BackedgeTakenInfo
8807ScalarEvolution::computeBackedgeTakenCount(const Loop *L,
8808 bool AllowPredicates) {
8809 SmallVector<BasicBlock *, 8> ExitingBlocks;
8810 L->getExitingBlocks(ExitingBlocks);
8811
8812 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8813
8814 SmallVector<EdgeExitInfo, 4> ExitCounts;
8815 bool CouldComputeBECount = true;
8816 BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
8817 const SCEV *MustExitMaxBECount = nullptr;
8818 const SCEV *MayExitMaxBECount = nullptr;
8819 bool MustExitMaxOrZero = false;
8820 bool IsOnlyExit = ExitingBlocks.size() == 1;
8821
8822 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
8823 // and compute maxBECount.
8824 // Do a union of all the predicates here.
8825 for (BasicBlock *ExitBB : ExitingBlocks) {
8826 // We canonicalize untaken exits to br (constant), ignore them so that
8827 // proving an exit untaken doesn't negatively impact our ability to reason
8828 // about the loop as whole.
8829 if (auto *BI = dyn_cast<BranchInst>(Val: ExitBB->getTerminator()))
8830 if (auto *CI = dyn_cast<ConstantInt>(Val: BI->getCondition())) {
8831 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
8832 if (ExitIfTrue == CI->isZero())
8833 continue;
8834 }
8835
8836 ExitLimit EL = computeExitLimit(L, ExitingBlock: ExitBB, IsOnlyExit, AllowPredicates);
8837
8838 assert((AllowPredicates || EL.Predicates.empty()) &&
8839 "Predicated exit limit when predicates are not allowed!");
8840
8841 // 1. For each exit that can be computed, add an entry to ExitCounts.
8842 // CouldComputeBECount is true only if all exits can be computed.
8843 if (EL.ExactNotTaken != getCouldNotCompute())
8844 ++NumExitCountsComputed;
8845 else
8846 // We couldn't compute an exact value for this exit, so
8847 // we won't be able to compute an exact value for the loop.
8848 CouldComputeBECount = false;
8849 // Remember exit count if either exact or symbolic is known. Because
8850 // Exact always implies symbolic, only check symbolic.
8851 if (EL.SymbolicMaxNotTaken != getCouldNotCompute())
8852 ExitCounts.emplace_back(Args&: ExitBB, Args&: EL);
8853 else {
8854 assert(EL.ExactNotTaken == getCouldNotCompute() &&
8855 "Exact is known but symbolic isn't?");
8856 ++NumExitCountsNotComputed;
8857 }
8858
8859 // 2. Derive the loop's MaxBECount from each exit's max number of
8860 // non-exiting iterations. Partition the loop exits into two kinds:
8861 // LoopMustExits and LoopMayExits.
8862 //
8863 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
8864 // is a LoopMayExit. If any computable LoopMustExit is found, then
8865 // MaxBECount is the minimum EL.ConstantMaxNotTaken of computable
8866 // LoopMustExits. Otherwise, MaxBECount is conservatively the maximum
8867 // EL.ConstantMaxNotTaken, where CouldNotCompute is considered greater than
8868 // any
8869 // computable EL.ConstantMaxNotTaken.
8870 if (EL.ConstantMaxNotTaken != getCouldNotCompute() && Latch &&
8871 DT.dominates(A: ExitBB, B: Latch)) {
8872 if (!MustExitMaxBECount) {
8873 MustExitMaxBECount = EL.ConstantMaxNotTaken;
8874 MustExitMaxOrZero = EL.MaxOrZero;
8875 } else {
8876 MustExitMaxBECount = getUMinFromMismatchedTypes(LHS: MustExitMaxBECount,
8877 RHS: EL.ConstantMaxNotTaken);
8878 }
8879 } else if (MayExitMaxBECount != getCouldNotCompute()) {
8880 if (!MayExitMaxBECount || EL.ConstantMaxNotTaken == getCouldNotCompute())
8881 MayExitMaxBECount = EL.ConstantMaxNotTaken;
8882 else {
8883 MayExitMaxBECount = getUMaxFromMismatchedTypes(LHS: MayExitMaxBECount,
8884 RHS: EL.ConstantMaxNotTaken);
8885 }
8886 }
8887 }
8888 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
8889 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
8890 // The loop backedge will be taken the maximum or zero times if there's
8891 // a single exit that must be taken the maximum or zero times.
8892 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
8893
8894 // Remember which SCEVs are used in exit limits for invalidation purposes.
8895 // We only care about non-constant SCEVs here, so we can ignore
8896 // EL.ConstantMaxNotTaken
8897 // and MaxBECount, which must be SCEVConstant.
8898 for (const auto &Pair : ExitCounts) {
8899 if (!isa<SCEVConstant>(Val: Pair.second.ExactNotTaken))
8900 BECountUsers[Pair.second.ExactNotTaken].insert(Ptr: {L, AllowPredicates});
8901 if (!isa<SCEVConstant>(Val: Pair.second.SymbolicMaxNotTaken))
8902 BECountUsers[Pair.second.SymbolicMaxNotTaken].insert(
8903 Ptr: {L, AllowPredicates});
8904 }
8905 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
8906 MaxBECount, MaxOrZero);
8907}
8908
8909ScalarEvolution::ExitLimit
8910ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock,
8911 bool IsOnlyExit, bool AllowPredicates) {
8912 assert(L->contains(ExitingBlock) && "Exit count for non-loop block?");
8913 // If our exiting block does not dominate the latch, then its connection with
8914 // loop's exit limit may be far from trivial.
8915 const BasicBlock *Latch = L->getLoopLatch();
8916 if (!Latch || !DT.dominates(A: ExitingBlock, B: Latch))
8917 return getCouldNotCompute();
8918
8919 Instruction *Term = ExitingBlock->getTerminator();
8920 if (BranchInst *BI = dyn_cast<BranchInst>(Val: Term)) {
8921 assert(BI->isConditional() && "If unconditional, it can't be in loop!");
8922 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
8923 assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) &&
8924 "It should have one successor in loop and one exit block!");
8925 // Proceed to the next level to examine the exit condition expression.
8926 return computeExitLimitFromCond(L, ExitCond: BI->getCondition(), ExitIfTrue,
8927 /*ControlsOnlyExit=*/IsOnlyExit,
8928 AllowPredicates);
8929 }
8930
8931 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: Term)) {
8932 // For switch, make sure that there is a single exit from the loop.
8933 BasicBlock *Exit = nullptr;
8934 for (auto *SBB : successors(BB: ExitingBlock))
8935 if (!L->contains(BB: SBB)) {
8936 if (Exit) // Multiple exit successors.
8937 return getCouldNotCompute();
8938 Exit = SBB;
8939 }
8940 assert(Exit && "Exiting block must have at least one exit");
8941 return computeExitLimitFromSingleExitSwitch(
8942 L, Switch: SI, ExitingBB: Exit, /*ControlsOnlyExit=*/IsSubExpr: IsOnlyExit);
8943 }
8944
8945 return getCouldNotCompute();
8946}
8947
8948ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCond(
8949 const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
8950 bool AllowPredicates) {
8951 ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
8952 return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
8953 ControlsOnlyExit, AllowPredicates);
8954}
8955
8956std::optional<ScalarEvolution::ExitLimit>
8957ScalarEvolution::ExitLimitCache::find(const Loop *L, Value *ExitCond,
8958 bool ExitIfTrue, bool ControlsOnlyExit,
8959 bool AllowPredicates) {
8960 (void)this->L;
8961 (void)this->ExitIfTrue;
8962 (void)this->AllowPredicates;
8963
8964 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
8965 this->AllowPredicates == AllowPredicates &&
8966 "Variance in assumed invariant key components!");
8967 auto Itr = TripCountMap.find(Val: {ExitCond, ControlsOnlyExit});
8968 if (Itr == TripCountMap.end())
8969 return std::nullopt;
8970 return Itr->second;
8971}
8972
8973void ScalarEvolution::ExitLimitCache::insert(const Loop *L, Value *ExitCond,
8974 bool ExitIfTrue,
8975 bool ControlsOnlyExit,
8976 bool AllowPredicates,
8977 const ExitLimit &EL) {
8978 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
8979 this->AllowPredicates == AllowPredicates &&
8980 "Variance in assumed invariant key components!");
8981
8982 auto InsertResult = TripCountMap.insert(KV: {{ExitCond, ControlsOnlyExit}, EL});
8983 assert(InsertResult.second && "Expected successful insertion!");
8984 (void)InsertResult;
8985 (void)ExitIfTrue;
8986}
8987
8988ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached(
8989 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
8990 bool ControlsOnlyExit, bool AllowPredicates) {
8991
8992 if (auto MaybeEL = Cache.find(L, ExitCond, ExitIfTrue, ControlsOnlyExit,
8993 AllowPredicates))
8994 return *MaybeEL;
8995
8996 ExitLimit EL = computeExitLimitFromCondImpl(
8997 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates);
8998 Cache.insert(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates, EL);
8999 return EL;
9000}
9001
9002ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl(
9003 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9004 bool ControlsOnlyExit, bool AllowPredicates) {
9005 // Handle BinOp conditions (And, Or).
9006 if (auto LimitFromBinOp = computeExitLimitFromCondFromBinOp(
9007 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates))
9008 return *LimitFromBinOp;
9009
9010 // With an icmp, it may be feasible to compute an exact backedge-taken count.
9011 // Proceed to the next level to examine the icmp.
9012 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(Val: ExitCond)) {
9013 ExitLimit EL =
9014 computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue, IsSubExpr: ControlsOnlyExit);
9015 if (EL.hasFullInfo() || !AllowPredicates)
9016 return EL;
9017
9018 // Try again, but use SCEV predicates this time.
9019 return computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue,
9020 IsSubExpr: ControlsOnlyExit,
9021 /*AllowPredicates=*/true);
9022 }
9023
9024 // Check for a constant condition. These are normally stripped out by
9025 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
9026 // preserve the CFG and is temporarily leaving constant conditions
9027 // in place.
9028 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: ExitCond)) {
9029 if (ExitIfTrue == !CI->getZExtValue())
9030 // The backedge is always taken.
9031 return getCouldNotCompute();
9032 // The backedge is never taken.
9033 return getZero(Ty: CI->getType());
9034 }
9035
9036 // If we're exiting based on the overflow flag of an x.with.overflow intrinsic
9037 // with a constant step, we can form an equivalent icmp predicate and figure
9038 // out how many iterations will be taken before we exit.
9039 const WithOverflowInst *WO;
9040 const APInt *C;
9041 if (match(V: ExitCond, P: m_ExtractValue<1>(V: m_WithOverflowInst(I&: WO))) &&
9042 match(V: WO->getRHS(), P: m_APInt(Res&: C))) {
9043 ConstantRange NWR =
9044 ConstantRange::makeExactNoWrapRegion(BinOp: WO->getBinaryOp(), Other: *C,
9045 NoWrapKind: WO->getNoWrapKind());
9046 CmpInst::Predicate Pred;
9047 APInt NewRHSC, Offset;
9048 NWR.getEquivalentICmp(Pred, RHS&: NewRHSC, Offset);
9049 if (!ExitIfTrue)
9050 Pred = ICmpInst::getInversePredicate(pred: Pred);
9051 auto *LHS = getSCEV(V: WO->getLHS());
9052 if (Offset != 0)
9053 LHS = getAddExpr(LHS, RHS: getConstant(Val: Offset));
9054 auto EL = computeExitLimitFromICmp(L, Pred, LHS, RHS: getConstant(Val: NewRHSC),
9055 IsSubExpr: ControlsOnlyExit, AllowPredicates);
9056 if (EL.hasAnyInfo())
9057 return EL;
9058 }
9059
9060 // If it's not an integer or pointer comparison then compute it the hard way.
9061 return computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9062}
9063
9064std::optional<ScalarEvolution::ExitLimit>
9065ScalarEvolution::computeExitLimitFromCondFromBinOp(
9066 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9067 bool ControlsOnlyExit, bool AllowPredicates) {
9068 // Check if the controlling expression for this loop is an And or Or.
9069 Value *Op0, *Op1;
9070 bool IsAnd = false;
9071 if (match(V: ExitCond, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9072 IsAnd = true;
9073 else if (match(V: ExitCond, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9074 IsAnd = false;
9075 else
9076 return std::nullopt;
9077
9078 // EitherMayExit is true in these two cases:
9079 // br (and Op0 Op1), loop, exit
9080 // br (or Op0 Op1), exit, loop
9081 bool EitherMayExit = IsAnd ^ ExitIfTrue;
9082 ExitLimit EL0 = computeExitLimitFromCondCached(
9083 Cache, L, ExitCond: Op0, ExitIfTrue, ControlsOnlyExit: ControlsOnlyExit && !EitherMayExit,
9084 AllowPredicates);
9085 ExitLimit EL1 = computeExitLimitFromCondCached(
9086 Cache, L, ExitCond: Op1, ExitIfTrue, ControlsOnlyExit: ControlsOnlyExit && !EitherMayExit,
9087 AllowPredicates);
9088
9089 // Be robust against unsimplified IR for the form "op i1 X, NeutralElement"
9090 const Constant *NeutralElement = ConstantInt::get(Ty: ExitCond->getType(), V: IsAnd);
9091 if (isa<ConstantInt>(Val: Op1))
9092 return Op1 == NeutralElement ? EL0 : EL1;
9093 if (isa<ConstantInt>(Val: Op0))
9094 return Op0 == NeutralElement ? EL1 : EL0;
9095
9096 const SCEV *BECount = getCouldNotCompute();
9097 const SCEV *ConstantMaxBECount = getCouldNotCompute();
9098 const SCEV *SymbolicMaxBECount = getCouldNotCompute();
9099 if (EitherMayExit) {
9100 bool UseSequentialUMin = !isa<BinaryOperator>(Val: ExitCond);
9101 // Both conditions must be same for the loop to continue executing.
9102 // Choose the less conservative count.
9103 if (EL0.ExactNotTaken != getCouldNotCompute() &&
9104 EL1.ExactNotTaken != getCouldNotCompute()) {
9105 BECount = getUMinFromMismatchedTypes(LHS: EL0.ExactNotTaken, RHS: EL1.ExactNotTaken,
9106 Sequential: UseSequentialUMin);
9107 }
9108 if (EL0.ConstantMaxNotTaken == getCouldNotCompute())
9109 ConstantMaxBECount = EL1.ConstantMaxNotTaken;
9110 else if (EL1.ConstantMaxNotTaken == getCouldNotCompute())
9111 ConstantMaxBECount = EL0.ConstantMaxNotTaken;
9112 else
9113 ConstantMaxBECount = getUMinFromMismatchedTypes(LHS: EL0.ConstantMaxNotTaken,
9114 RHS: EL1.ConstantMaxNotTaken);
9115 if (EL0.SymbolicMaxNotTaken == getCouldNotCompute())
9116 SymbolicMaxBECount = EL1.SymbolicMaxNotTaken;
9117 else if (EL1.SymbolicMaxNotTaken == getCouldNotCompute())
9118 SymbolicMaxBECount = EL0.SymbolicMaxNotTaken;
9119 else
9120 SymbolicMaxBECount = getUMinFromMismatchedTypes(
9121 LHS: EL0.SymbolicMaxNotTaken, RHS: EL1.SymbolicMaxNotTaken, Sequential: UseSequentialUMin);
9122 } else {
9123 // Both conditions must be same at the same time for the loop to exit.
9124 // For now, be conservative.
9125 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
9126 BECount = EL0.ExactNotTaken;
9127 }
9128
9129 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able
9130 // to be more aggressive when computing BECount than when computing
9131 // ConstantMaxBECount. In these cases it is possible for EL0.ExactNotTaken
9132 // and
9133 // EL1.ExactNotTaken to match, but for EL0.ConstantMaxNotTaken and
9134 // EL1.ConstantMaxNotTaken to not.
9135 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
9136 !isa<SCEVCouldNotCompute>(Val: BECount))
9137 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
9138 if (isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount))
9139 SymbolicMaxBECount =
9140 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
9141 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
9142 {ArrayRef(EL0.Predicates), ArrayRef(EL1.Predicates)});
9143}
9144
9145ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9146 const Loop *L, ICmpInst *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9147 bool AllowPredicates) {
9148 // If the condition was exit on true, convert the condition to exit on false
9149 CmpPredicate Pred;
9150 if (!ExitIfTrue)
9151 Pred = ExitCond->getCmpPredicate();
9152 else
9153 Pred = ExitCond->getInverseCmpPredicate();
9154 const ICmpInst::Predicate OriginalPred = Pred;
9155
9156 const SCEV *LHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 0));
9157 const SCEV *RHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 1));
9158
9159 ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, IsSubExpr: ControlsOnlyExit,
9160 AllowPredicates);
9161 if (EL.hasAnyInfo())
9162 return EL;
9163
9164 auto *ExhaustiveCount =
9165 computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9166
9167 if (!isa<SCEVCouldNotCompute>(Val: ExhaustiveCount))
9168 return ExhaustiveCount;
9169
9170 return computeShiftCompareExitLimit(LHS: ExitCond->getOperand(i_nocapture: 0),
9171 RHS: ExitCond->getOperand(i_nocapture: 1), L, Pred: OriginalPred);
9172}
9173ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9174 const Loop *L, CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS,
9175 bool ControlsOnlyExit, bool AllowPredicates) {
9176
9177 // Try to evaluate any dependencies out of the loop.
9178 LHS = getSCEVAtScope(S: LHS, L);
9179 RHS = getSCEVAtScope(S: RHS, L);
9180
9181 // At this point, we would like to compute how many iterations of the
9182 // loop the predicate will return true for these inputs.
9183 if (isLoopInvariant(S: LHS, L) && !isLoopInvariant(S: RHS, L)) {
9184 // If there is a loop-invariant, force it into the RHS.
9185 std::swap(a&: LHS, b&: RHS);
9186 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
9187 }
9188
9189 bool ControllingFiniteLoop = ControlsOnlyExit && loopHasNoAbnormalExits(L) &&
9190 loopIsFiniteByAssumption(L);
9191 // Simplify the operands before analyzing them.
9192 (void)SimplifyICmpOperands(Pred, LHS, RHS, /*Depth=*/0);
9193
9194 // If we have a comparison of a chrec against a constant, try to use value
9195 // ranges to answer this query.
9196 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val: RHS))
9197 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val: LHS))
9198 if (AddRec->getLoop() == L) {
9199 // Form the constant range.
9200 ConstantRange CompRange =
9201 ConstantRange::makeExactICmpRegion(Pred, Other: RHSC->getAPInt());
9202
9203 const SCEV *Ret = AddRec->getNumIterationsInRange(Range: CompRange, SE&: *this);
9204 if (!isa<SCEVCouldNotCompute>(Val: Ret)) return Ret;
9205 }
9206
9207 // If this loop must exit based on this condition (or execute undefined
9208 // behaviour), see if we can improve wrap flags. This is essentially
9209 // a must execute style proof.
9210 if (ControllingFiniteLoop && isLoopInvariant(S: RHS, L)) {
9211 // If we can prove the test sequence produced must repeat the same values
9212 // on self-wrap of the IV, then we can infer that IV doesn't self wrap
9213 // because if it did, we'd have an infinite (undefined) loop.
9214 // TODO: We can peel off any functions which are invertible *in L*. Loop
9215 // invariant terms are effectively constants for our purposes here.
9216 auto *InnerLHS = LHS;
9217 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: LHS))
9218 InnerLHS = ZExt->getOperand();
9219 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: InnerLHS);
9220 AR && !AR->hasNoSelfWrap() && AR->getLoop() == L && AR->isAffine() &&
9221 isKnownToBeAPowerOfTwo(S: AR->getStepRecurrence(SE&: *this), /*OrZero=*/true,
9222 /*OrNegative=*/true)) {
9223 auto Flags = AR->getNoWrapFlags();
9224 Flags = setFlags(Flags, OnFlags: SCEV::FlagNW);
9225 SmallVector<const SCEV *> Operands{AR->operands()};
9226 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9227 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9228 }
9229
9230 // For a slt/ult condition with a positive step, can we prove nsw/nuw?
9231 // From no-self-wrap, this follows trivially from the fact that every
9232 // (un)signed-wrapped, but not self-wrapped value must be LT than the
9233 // last value before (un)signed wrap. Since we know that last value
9234 // didn't exit, nor will any smaller one.
9235 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT) {
9236 auto WrapType = Pred == ICmpInst::ICMP_SLT ? SCEV::FlagNSW : SCEV::FlagNUW;
9237 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS);
9238 AR && AR->getLoop() == L && AR->isAffine() &&
9239 !AR->getNoWrapFlags(Mask: WrapType) && AR->hasNoSelfWrap() &&
9240 isKnownPositive(S: AR->getStepRecurrence(SE&: *this))) {
9241 auto Flags = AR->getNoWrapFlags();
9242 Flags = setFlags(Flags, OnFlags: WrapType);
9243 SmallVector<const SCEV*> Operands{AR->operands()};
9244 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9245 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9246 }
9247 }
9248 }
9249
9250 switch (Pred) {
9251 case ICmpInst::ICMP_NE: { // while (X != Y)
9252 // Convert to: while (X-Y != 0)
9253 if (LHS->getType()->isPointerTy()) {
9254 LHS = getLosslessPtrToIntExpr(Op: LHS);
9255 if (isa<SCEVCouldNotCompute>(Val: LHS))
9256 return LHS;
9257 }
9258 if (RHS->getType()->isPointerTy()) {
9259 RHS = getLosslessPtrToIntExpr(Op: RHS);
9260 if (isa<SCEVCouldNotCompute>(Val: RHS))
9261 return RHS;
9262 }
9263 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit,
9264 AllowPredicates);
9265 if (EL.hasAnyInfo())
9266 return EL;
9267 break;
9268 }
9269 case ICmpInst::ICMP_EQ: { // while (X == Y)
9270 // Convert to: while (X-Y == 0)
9271 if (LHS->getType()->isPointerTy()) {
9272 LHS = getLosslessPtrToIntExpr(Op: LHS);
9273 if (isa<SCEVCouldNotCompute>(Val: LHS))
9274 return LHS;
9275 }
9276 if (RHS->getType()->isPointerTy()) {
9277 RHS = getLosslessPtrToIntExpr(Op: RHS);
9278 if (isa<SCEVCouldNotCompute>(Val: RHS))
9279 return RHS;
9280 }
9281 ExitLimit EL = howFarToNonZero(V: getMinusSCEV(LHS, RHS), L);
9282 if (EL.hasAnyInfo()) return EL;
9283 break;
9284 }
9285 case ICmpInst::ICMP_SLE:
9286 case ICmpInst::ICMP_ULE:
9287 // Since the loop is finite, an invariant RHS cannot include the boundary
9288 // value, otherwise it would loop forever.
9289 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9290 !isLoopInvariant(S: RHS, L)) {
9291 // Otherwise, perform the addition in a wider type, to avoid overflow.
9292 // If the LHS is an addrec with the appropriate nowrap flag, the
9293 // extension will be sunk into it and the exit count can be analyzed.
9294 auto *OldType = dyn_cast<IntegerType>(Val: LHS->getType());
9295 if (!OldType)
9296 break;
9297 // Prefer doubling the bitwidth over adding a single bit to make it more
9298 // likely that we use a legal type.
9299 auto *NewType =
9300 Type::getIntNTy(C&: OldType->getContext(), N: OldType->getBitWidth() * 2);
9301 if (ICmpInst::isSigned(predicate: Pred)) {
9302 LHS = getSignExtendExpr(Op: LHS, Ty: NewType);
9303 RHS = getSignExtendExpr(Op: RHS, Ty: NewType);
9304 } else {
9305 LHS = getZeroExtendExpr(Op: LHS, Ty: NewType);
9306 RHS = getZeroExtendExpr(Op: RHS, Ty: NewType);
9307 }
9308 }
9309 RHS = getAddExpr(LHS: getOne(Ty: RHS->getType()), RHS);
9310 [[fallthrough]];
9311 case ICmpInst::ICMP_SLT:
9312 case ICmpInst::ICMP_ULT: { // while (X < Y)
9313 bool IsSigned = ICmpInst::isSigned(predicate: Pred);
9314 ExitLimit EL = howManyLessThans(LHS, RHS, L, isSigned: IsSigned, ControlsOnlyExit,
9315 AllowPredicates);
9316 if (EL.hasAnyInfo())
9317 return EL;
9318 break;
9319 }
9320 case ICmpInst::ICMP_SGE:
9321 case ICmpInst::ICMP_UGE:
9322 // Since the loop is finite, an invariant RHS cannot include the boundary
9323 // value, otherwise it would loop forever.
9324 if (!EnableFiniteLoopControl || !ControllingFiniteLoop ||
9325 !isLoopInvariant(S: RHS, L))
9326 break;
9327 RHS = getAddExpr(LHS: getMinusOne(Ty: RHS->getType()), RHS);
9328 [[fallthrough]];
9329 case ICmpInst::ICMP_SGT:
9330 case ICmpInst::ICMP_UGT: { // while (X > Y)
9331 bool IsSigned = ICmpInst::isSigned(predicate: Pred);
9332 ExitLimit EL = howManyGreaterThans(LHS, RHS, L, isSigned: IsSigned, IsSubExpr: ControlsOnlyExit,
9333 AllowPredicates);
9334 if (EL.hasAnyInfo())
9335 return EL;
9336 break;
9337 }
9338 default:
9339 break;
9340 }
9341
9342 return getCouldNotCompute();
9343}
9344
9345ScalarEvolution::ExitLimit
9346ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L,
9347 SwitchInst *Switch,
9348 BasicBlock *ExitingBlock,
9349 bool ControlsOnlyExit) {
9350 assert(!L->contains(ExitingBlock) && "Not an exiting block!");
9351
9352 // Give up if the exit is the default dest of a switch.
9353 if (Switch->getDefaultDest() == ExitingBlock)
9354 return getCouldNotCompute();
9355
9356 assert(L->contains(Switch->getDefaultDest()) &&
9357 "Default case must not exit the loop!");
9358 const SCEV *LHS = getSCEVAtScope(V: Switch->getCondition(), L);
9359 const SCEV *RHS = getConstant(V: Switch->findCaseDest(BB: ExitingBlock));
9360
9361 // while (X != Y) --> while (X-Y != 0)
9362 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit);
9363 if (EL.hasAnyInfo())
9364 return EL;
9365
9366 return getCouldNotCompute();
9367}
9368
9369static ConstantInt *
9370EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
9371 ScalarEvolution &SE) {
9372 const SCEV *InVal = SE.getConstant(V: C);
9373 const SCEV *Val = AddRec->evaluateAtIteration(It: InVal, SE);
9374 assert(isa<SCEVConstant>(Val) &&
9375 "Evaluation of SCEV at constant didn't fold correctly?");
9376 return cast<SCEVConstant>(Val)->getValue();
9377}
9378
9379ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
9380 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) {
9381 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: RHSV);
9382 if (!RHS)
9383 return getCouldNotCompute();
9384
9385 const BasicBlock *Latch = L->getLoopLatch();
9386 if (!Latch)
9387 return getCouldNotCompute();
9388
9389 const BasicBlock *Predecessor = L->getLoopPredecessor();
9390 if (!Predecessor)
9391 return getCouldNotCompute();
9392
9393 // Return true if V is of the form "LHS `shift_op` <positive constant>".
9394 // Return LHS in OutLHS and shift_opt in OutOpCode.
9395 auto MatchPositiveShift =
9396 [](Value *V, Value *&OutLHS, Instruction::BinaryOps &OutOpCode) {
9397
9398 using namespace PatternMatch;
9399
9400 ConstantInt *ShiftAmt;
9401 if (match(V, P: m_LShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9402 OutOpCode = Instruction::LShr;
9403 else if (match(V, P: m_AShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9404 OutOpCode = Instruction::AShr;
9405 else if (match(V, P: m_Shl(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9406 OutOpCode = Instruction::Shl;
9407 else
9408 return false;
9409
9410 return ShiftAmt->getValue().isStrictlyPositive();
9411 };
9412
9413 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in
9414 //
9415 // loop:
9416 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ]
9417 // %iv.shifted = lshr i32 %iv, <positive constant>
9418 //
9419 // Return true on a successful match. Return the corresponding PHI node (%iv
9420 // above) in PNOut and the opcode of the shift operation in OpCodeOut.
9421 auto MatchShiftRecurrence =
9422 [&](Value *V, PHINode *&PNOut, Instruction::BinaryOps &OpCodeOut) {
9423 std::optional<Instruction::BinaryOps> PostShiftOpCode;
9424
9425 {
9426 Instruction::BinaryOps OpC;
9427 Value *V;
9428
9429 // If we encounter a shift instruction, "peel off" the shift operation,
9430 // and remember that we did so. Later when we inspect %iv's backedge
9431 // value, we will make sure that the backedge value uses the same
9432 // operation.
9433 //
9434 // Note: the peeled shift operation does not have to be the same
9435 // instruction as the one feeding into the PHI's backedge value. We only
9436 // really care about it being the same *kind* of shift instruction --
9437 // that's all that is required for our later inferences to hold.
9438 if (MatchPositiveShift(LHS, V, OpC)) {
9439 PostShiftOpCode = OpC;
9440 LHS = V;
9441 }
9442 }
9443
9444 PNOut = dyn_cast<PHINode>(Val: LHS);
9445 if (!PNOut || PNOut->getParent() != L->getHeader())
9446 return false;
9447
9448 Value *BEValue = PNOut->getIncomingValueForBlock(BB: Latch);
9449 Value *OpLHS;
9450
9451 return
9452 // The backedge value for the PHI node must be a shift by a positive
9453 // amount
9454 MatchPositiveShift(BEValue, OpLHS, OpCodeOut) &&
9455
9456 // of the PHI node itself
9457 OpLHS == PNOut &&
9458
9459 // and the kind of shift should be match the kind of shift we peeled
9460 // off, if any.
9461 (!PostShiftOpCode || *PostShiftOpCode == OpCodeOut);
9462 };
9463
9464 PHINode *PN;
9465 Instruction::BinaryOps OpCode;
9466 if (!MatchShiftRecurrence(LHS, PN, OpCode))
9467 return getCouldNotCompute();
9468
9469 const DataLayout &DL = getDataLayout();
9470
9471 // The key rationale for this optimization is that for some kinds of shift
9472 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1
9473 // within a finite number of iterations. If the condition guarding the
9474 // backedge (in the sense that the backedge is taken if the condition is true)
9475 // is false for the value the shift recurrence stabilizes to, then we know
9476 // that the backedge is taken only a finite number of times.
9477
9478 ConstantInt *StableValue = nullptr;
9479 switch (OpCode) {
9480 default:
9481 llvm_unreachable("Impossible case!");
9482
9483 case Instruction::AShr: {
9484 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most
9485 // bitwidth(K) iterations.
9486 Value *FirstValue = PN->getIncomingValueForBlock(BB: Predecessor);
9487 KnownBits Known = computeKnownBits(V: FirstValue, DL, AC: &AC,
9488 CxtI: Predecessor->getTerminator(), DT: &DT);
9489 auto *Ty = cast<IntegerType>(Val: RHS->getType());
9490 if (Known.isNonNegative())
9491 StableValue = ConstantInt::get(Ty, V: 0);
9492 else if (Known.isNegative())
9493 StableValue = ConstantInt::get(Ty, V: -1, IsSigned: true);
9494 else
9495 return getCouldNotCompute();
9496
9497 break;
9498 }
9499 case Instruction::LShr:
9500 case Instruction::Shl:
9501 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>}
9502 // stabilize to 0 in at most bitwidth(K) iterations.
9503 StableValue = ConstantInt::get(Ty: cast<IntegerType>(Val: RHS->getType()), V: 0);
9504 break;
9505 }
9506
9507 auto *Result =
9508 ConstantFoldCompareInstOperands(Predicate: Pred, LHS: StableValue, RHS, DL, TLI: &TLI);
9509 assert(Result->getType()->isIntegerTy(1) &&
9510 "Otherwise cannot be an operand to a branch instruction");
9511
9512 if (Result->isZeroValue()) {
9513 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
9514 const SCEV *UpperBound =
9515 getConstant(Ty: getEffectiveSCEVType(Ty: RHS->getType()), V: BitWidth);
9516 return ExitLimit(getCouldNotCompute(), UpperBound, UpperBound, false);
9517 }
9518
9519 return getCouldNotCompute();
9520}
9521
9522/// Return true if we can constant fold an instruction of the specified type,
9523/// assuming that all operands were constants.
9524static bool CanConstantFold(const Instruction *I) {
9525 if (isa<BinaryOperator>(Val: I) || isa<CmpInst>(Val: I) ||
9526 isa<SelectInst>(Val: I) || isa<CastInst>(Val: I) || isa<GetElementPtrInst>(Val: I) ||
9527 isa<LoadInst>(Val: I) || isa<ExtractValueInst>(Val: I))
9528 return true;
9529
9530 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
9531 if (const Function *F = CI->getCalledFunction())
9532 return canConstantFoldCallTo(Call: CI, F);
9533 return false;
9534}
9535
9536/// Determine whether this instruction can constant evolve within this loop
9537/// assuming its operands can all constant evolve.
9538static bool canConstantEvolve(Instruction *I, const Loop *L) {
9539 // An instruction outside of the loop can't be derived from a loop PHI.
9540 if (!L->contains(Inst: I)) return false;
9541
9542 if (isa<PHINode>(Val: I)) {
9543 // We don't currently keep track of the control flow needed to evaluate
9544 // PHIs, so we cannot handle PHIs inside of loops.
9545 return L->getHeader() == I->getParent();
9546 }
9547
9548 // If we won't be able to constant fold this expression even if the operands
9549 // are constants, bail early.
9550 return CanConstantFold(I);
9551}
9552
9553/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
9554/// recursing through each instruction operand until reaching a loop header phi.
9555static PHINode *
9556getConstantEvolvingPHIOperands(Instruction *UseInst, const Loop *L,
9557 DenseMap<Instruction *, PHINode *> &PHIMap,
9558 unsigned Depth) {
9559 if (Depth > MaxConstantEvolvingDepth)
9560 return nullptr;
9561
9562 // Otherwise, we can evaluate this instruction if all of its operands are
9563 // constant or derived from a PHI node themselves.
9564 PHINode *PHI = nullptr;
9565 for (Value *Op : UseInst->operands()) {
9566 if (isa<Constant>(Val: Op)) continue;
9567
9568 Instruction *OpInst = dyn_cast<Instruction>(Val: Op);
9569 if (!OpInst || !canConstantEvolve(I: OpInst, L)) return nullptr;
9570
9571 PHINode *P = dyn_cast<PHINode>(Val: OpInst);
9572 if (!P)
9573 // If this operand is already visited, reuse the prior result.
9574 // We may have P != PHI if this is the deepest point at which the
9575 // inconsistent paths meet.
9576 P = PHIMap.lookup(Val: OpInst);
9577 if (!P) {
9578 // Recurse and memoize the results, whether a phi is found or not.
9579 // This recursive call invalidates pointers into PHIMap.
9580 P = getConstantEvolvingPHIOperands(UseInst: OpInst, L, PHIMap, Depth: Depth + 1);
9581 PHIMap[OpInst] = P;
9582 }
9583 if (!P)
9584 return nullptr; // Not evolving from PHI
9585 if (PHI && PHI != P)
9586 return nullptr; // Evolving from multiple different PHIs.
9587 PHI = P;
9588 }
9589 // This is a expression evolving from a constant PHI!
9590 return PHI;
9591}
9592
9593/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
9594/// in the loop that V is derived from. We allow arbitrary operations along the
9595/// way, but the operands of an operation must either be constants or a value
9596/// derived from a constant PHI. If this expression does not fit with these
9597/// constraints, return null.
9598static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) {
9599 Instruction *I = dyn_cast<Instruction>(Val: V);
9600 if (!I || !canConstantEvolve(I, L)) return nullptr;
9601
9602 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
9603 return PN;
9604
9605 // Record non-constant instructions contained by the loop.
9606 DenseMap<Instruction *, PHINode *> PHIMap;
9607 return getConstantEvolvingPHIOperands(UseInst: I, L, PHIMap, Depth: 0);
9608}
9609
9610/// EvaluateExpression - Given an expression that passes the
9611/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
9612/// in the loop has the value PHIVal. If we can't fold this expression for some
9613/// reason, return null.
9614static Constant *EvaluateExpression(Value *V, const Loop *L,
9615 DenseMap<Instruction *, Constant *> &Vals,
9616 const DataLayout &DL,
9617 const TargetLibraryInfo *TLI) {
9618 // Convenient constant check, but redundant for recursive calls.
9619 if (Constant *C = dyn_cast<Constant>(Val: V)) return C;
9620 Instruction *I = dyn_cast<Instruction>(Val: V);
9621 if (!I) return nullptr;
9622
9623 if (Constant *C = Vals.lookup(Val: I)) return C;
9624
9625 // An instruction inside the loop depends on a value outside the loop that we
9626 // weren't given a mapping for, or a value such as a call inside the loop.
9627 if (!canConstantEvolve(I, L)) return nullptr;
9628
9629 // An unmapped PHI can be due to a branch or another loop inside this loop,
9630 // or due to this not being the initial iteration through a loop where we
9631 // couldn't compute the evolution of this particular PHI last time.
9632 if (isa<PHINode>(Val: I)) return nullptr;
9633
9634 std::vector<Constant*> Operands(I->getNumOperands());
9635
9636 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
9637 Instruction *Operand = dyn_cast<Instruction>(Val: I->getOperand(i));
9638 if (!Operand) {
9639 Operands[i] = dyn_cast<Constant>(Val: I->getOperand(i));
9640 if (!Operands[i]) return nullptr;
9641 continue;
9642 }
9643 Constant *C = EvaluateExpression(V: Operand, L, Vals, DL, TLI);
9644 Vals[Operand] = C;
9645 if (!C) return nullptr;
9646 Operands[i] = C;
9647 }
9648
9649 return ConstantFoldInstOperands(I, Ops: Operands, DL, TLI,
9650 /*AllowNonDeterministic=*/false);
9651}
9652
9653
9654// If every incoming value to PN except the one for BB is a specific Constant,
9655// return that, else return nullptr.
9656static Constant *getOtherIncomingValue(PHINode *PN, BasicBlock *BB) {
9657 Constant *IncomingVal = nullptr;
9658
9659 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
9660 if (PN->getIncomingBlock(i) == BB)
9661 continue;
9662
9663 auto *CurrentVal = dyn_cast<Constant>(Val: PN->getIncomingValue(i));
9664 if (!CurrentVal)
9665 return nullptr;
9666
9667 if (IncomingVal != CurrentVal) {
9668 if (IncomingVal)
9669 return nullptr;
9670 IncomingVal = CurrentVal;
9671 }
9672 }
9673
9674 return IncomingVal;
9675}
9676
9677/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
9678/// in the header of its containing loop, we know the loop executes a
9679/// constant number of times, and the PHI node is just a recurrence
9680/// involving constants, fold it.
9681Constant *
9682ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
9683 const APInt &BEs,
9684 const Loop *L) {
9685 auto [I, Inserted] = ConstantEvolutionLoopExitValue.try_emplace(Key: PN);
9686 if (!Inserted)
9687 return I->second;
9688
9689 if (BEs.ugt(RHS: MaxBruteForceIterations))
9690 return nullptr; // Not going to evaluate it.
9691
9692 Constant *&RetVal = I->second;
9693
9694 DenseMap<Instruction *, Constant *> CurrentIterVals;
9695 BasicBlock *Header = L->getHeader();
9696 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9697
9698 BasicBlock *Latch = L->getLoopLatch();
9699 if (!Latch)
9700 return nullptr;
9701
9702 for (PHINode &PHI : Header->phis()) {
9703 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
9704 CurrentIterVals[&PHI] = StartCST;
9705 }
9706 if (!CurrentIterVals.count(Val: PN))
9707 return RetVal = nullptr;
9708
9709 Value *BEValue = PN->getIncomingValueForBlock(BB: Latch);
9710
9711 // Execute the loop symbolically to determine the exit value.
9712 assert(BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) &&
9713 "BEs is <= MaxBruteForceIterations which is an 'unsigned'!");
9714
9715 unsigned NumIterations = BEs.getZExtValue(); // must be in range
9716 unsigned IterationNum = 0;
9717 const DataLayout &DL = getDataLayout();
9718 for (; ; ++IterationNum) {
9719 if (IterationNum == NumIterations)
9720 return RetVal = CurrentIterVals[PN]; // Got exit value!
9721
9722 // Compute the value of the PHIs for the next iteration.
9723 // EvaluateExpression adds non-phi values to the CurrentIterVals map.
9724 DenseMap<Instruction *, Constant *> NextIterVals;
9725 Constant *NextPHI =
9726 EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9727 if (!NextPHI)
9728 return nullptr; // Couldn't evaluate!
9729 NextIterVals[PN] = NextPHI;
9730
9731 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
9732
9733 // Also evaluate the other PHI nodes. However, we don't get to stop if we
9734 // cease to be able to evaluate one of them or if they stop evolving,
9735 // because that doesn't necessarily prevent us from computing PN.
9736 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute;
9737 for (const auto &I : CurrentIterVals) {
9738 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
9739 if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
9740 PHIsToCompute.emplace_back(Args&: PHI, Args: I.second);
9741 }
9742 // We use two distinct loops because EvaluateExpression may invalidate any
9743 // iterators into CurrentIterVals.
9744 for (const auto &I : PHIsToCompute) {
9745 PHINode *PHI = I.first;
9746 Constant *&NextPHI = NextIterVals[PHI];
9747 if (!NextPHI) { // Not already computed.
9748 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
9749 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9750 }
9751 if (NextPHI != I.second)
9752 StoppedEvolving = false;
9753 }
9754
9755 // If all entries in CurrentIterVals == NextIterVals then we can stop
9756 // iterating, the loop can't continue to change.
9757 if (StoppedEvolving)
9758 return RetVal = CurrentIterVals[PN];
9759
9760 CurrentIterVals.swap(RHS&: NextIterVals);
9761 }
9762}
9763
9764const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L,
9765 Value *Cond,
9766 bool ExitWhen) {
9767 PHINode *PN = getConstantEvolvingPHI(V: Cond, L);
9768 if (!PN) return getCouldNotCompute();
9769
9770 // If the loop is canonicalized, the PHI will have exactly two entries.
9771 // That's the only form we support here.
9772 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
9773
9774 DenseMap<Instruction *, Constant *> CurrentIterVals;
9775 BasicBlock *Header = L->getHeader();
9776 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9777
9778 BasicBlock *Latch = L->getLoopLatch();
9779 assert(Latch && "Should follow from NumIncomingValues == 2!");
9780
9781 for (PHINode &PHI : Header->phis()) {
9782 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
9783 CurrentIterVals[&PHI] = StartCST;
9784 }
9785 if (!CurrentIterVals.count(Val: PN))
9786 return getCouldNotCompute();
9787
9788 // Okay, we find a PHI node that defines the trip count of this loop. Execute
9789 // the loop symbolically to determine when the condition gets a value of
9790 // "ExitWhen".
9791 unsigned MaxIterations = MaxBruteForceIterations; // Limit analysis.
9792 const DataLayout &DL = getDataLayout();
9793 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
9794 auto *CondVal = dyn_cast_or_null<ConstantInt>(
9795 Val: EvaluateExpression(V: Cond, L, Vals&: CurrentIterVals, DL, TLI: &TLI));
9796
9797 // Couldn't symbolically evaluate.
9798 if (!CondVal) return getCouldNotCompute();
9799
9800 if (CondVal->getValue() == uint64_t(ExitWhen)) {
9801 ++NumBruteForceTripCountsComputed;
9802 return getConstant(Ty: Type::getInt32Ty(C&: getContext()), V: IterationNum);
9803 }
9804
9805 // Update all the PHI nodes for the next iteration.
9806 DenseMap<Instruction *, Constant *> NextIterVals;
9807
9808 // Create a list of which PHIs we need to compute. We want to do this before
9809 // calling EvaluateExpression on them because that may invalidate iterators
9810 // into CurrentIterVals.
9811 SmallVector<PHINode *, 8> PHIsToCompute;
9812 for (const auto &I : CurrentIterVals) {
9813 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
9814 if (!PHI || PHI->getParent() != Header) continue;
9815 PHIsToCompute.push_back(Elt: PHI);
9816 }
9817 for (PHINode *PHI : PHIsToCompute) {
9818 Constant *&NextPHI = NextIterVals[PHI];
9819 if (NextPHI) continue; // Already computed!
9820
9821 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
9822 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9823 }
9824 CurrentIterVals.swap(RHS&: NextIterVals);
9825 }
9826
9827 // Too many iterations were needed to evaluate.
9828 return getCouldNotCompute();
9829}
9830
9831const SCEV *ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
9832 SmallVector<std::pair<const Loop *, const SCEV *>, 2> &Values =
9833 ValuesAtScopes[V];
9834 // Check to see if we've folded this expression at this loop before.
9835 for (auto &LS : Values)
9836 if (LS.first == L)
9837 return LS.second ? LS.second : V;
9838
9839 Values.emplace_back(Args&: L, Args: nullptr);
9840
9841 // Otherwise compute it.
9842 const SCEV *C = computeSCEVAtScope(S: V, L);
9843 for (auto &LS : reverse(C&: ValuesAtScopes[V]))
9844 if (LS.first == L) {
9845 LS.second = C;
9846 if (!isa<SCEVConstant>(Val: C))
9847 ValuesAtScopesUsers[C].push_back(Elt: {L, V});
9848 break;
9849 }
9850 return C;
9851}
9852
9853/// This builds up a Constant using the ConstantExpr interface. That way, we
9854/// will return Constants for objects which aren't represented by a
9855/// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
9856/// Returns NULL if the SCEV isn't representable as a Constant.
9857static Constant *BuildConstantFromSCEV(const SCEV *V) {
9858 switch (V->getSCEVType()) {
9859 case scCouldNotCompute:
9860 case scAddRecExpr:
9861 case scVScale:
9862 return nullptr;
9863 case scConstant:
9864 return cast<SCEVConstant>(Val: V)->getValue();
9865 case scUnknown:
9866 return dyn_cast<Constant>(Val: cast<SCEVUnknown>(Val: V)->getValue());
9867 case scPtrToInt: {
9868 const SCEVPtrToIntExpr *P2I = cast<SCEVPtrToIntExpr>(Val: V);
9869 if (Constant *CastOp = BuildConstantFromSCEV(V: P2I->getOperand()))
9870 return ConstantExpr::getPtrToInt(C: CastOp, Ty: P2I->getType());
9871
9872 return nullptr;
9873 }
9874 case scTruncate: {
9875 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(Val: V);
9876 if (Constant *CastOp = BuildConstantFromSCEV(V: ST->getOperand()))
9877 return ConstantExpr::getTrunc(C: CastOp, Ty: ST->getType());
9878 return nullptr;
9879 }
9880 case scAddExpr: {
9881 const SCEVAddExpr *SA = cast<SCEVAddExpr>(Val: V);
9882 Constant *C = nullptr;
9883 for (const SCEV *Op : SA->operands()) {
9884 Constant *OpC = BuildConstantFromSCEV(V: Op);
9885 if (!OpC)
9886 return nullptr;
9887 if (!C) {
9888 C = OpC;
9889 continue;
9890 }
9891 assert(!C->getType()->isPointerTy() &&
9892 "Can only have one pointer, and it must be last");
9893 if (OpC->getType()->isPointerTy()) {
9894 // The offsets have been converted to bytes. We can add bytes using
9895 // an i8 GEP.
9896 C = ConstantExpr::getGetElementPtr(Ty: Type::getInt8Ty(C&: C->getContext()),
9897 C: OpC, Idx: C);
9898 } else {
9899 C = ConstantExpr::getAdd(C1: C, C2: OpC);
9900 }
9901 }
9902 return C;
9903 }
9904 case scMulExpr:
9905 case scSignExtend:
9906 case scZeroExtend:
9907 case scUDivExpr:
9908 case scSMaxExpr:
9909 case scUMaxExpr:
9910 case scSMinExpr:
9911 case scUMinExpr:
9912 case scSequentialUMinExpr:
9913 return nullptr;
9914 }
9915 llvm_unreachable("Unknown SCEV kind!");
9916}
9917
9918const SCEV *
9919ScalarEvolution::getWithOperands(const SCEV *S,
9920 SmallVectorImpl<const SCEV *> &NewOps) {
9921 switch (S->getSCEVType()) {
9922 case scTruncate:
9923 case scZeroExtend:
9924 case scSignExtend:
9925 case scPtrToInt:
9926 return getCastExpr(Kind: S->getSCEVType(), Op: NewOps[0], Ty: S->getType());
9927 case scAddRecExpr: {
9928 auto *AddRec = cast<SCEVAddRecExpr>(Val: S);
9929 return getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), Flags: AddRec->getNoWrapFlags());
9930 }
9931 case scAddExpr:
9932 return getAddExpr(Ops&: NewOps, OrigFlags: cast<SCEVAddExpr>(Val: S)->getNoWrapFlags());
9933 case scMulExpr:
9934 return getMulExpr(Ops&: NewOps, OrigFlags: cast<SCEVMulExpr>(Val: S)->getNoWrapFlags());
9935 case scUDivExpr:
9936 return getUDivExpr(LHS: NewOps[0], RHS: NewOps[1]);
9937 case scUMaxExpr:
9938 case scSMaxExpr:
9939 case scUMinExpr:
9940 case scSMinExpr:
9941 return getMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
9942 case scSequentialUMinExpr:
9943 return getSequentialMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
9944 case scConstant:
9945 case scVScale:
9946 case scUnknown:
9947 return S;
9948 case scCouldNotCompute:
9949 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
9950 }
9951 llvm_unreachable("Unknown SCEV kind!");
9952}
9953
9954const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
9955 switch (V->getSCEVType()) {
9956 case scConstant:
9957 case scVScale:
9958 return V;
9959 case scAddRecExpr: {
9960 // If this is a loop recurrence for a loop that does not contain L, then we
9961 // are dealing with the final value computed by the loop.
9962 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: V);
9963 // First, attempt to evaluate each operand.
9964 // Avoid performing the look-up in the common case where the specified
9965 // expression has no loop-variant portions.
9966 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
9967 const SCEV *OpAtScope = getSCEVAtScope(V: AddRec->getOperand(i), L);
9968 if (OpAtScope == AddRec->getOperand(i))
9969 continue;
9970
9971 // Okay, at least one of these operands is loop variant but might be
9972 // foldable. Build a new instance of the folded commutative expression.
9973 SmallVector<const SCEV *, 8> NewOps;
9974 NewOps.reserve(N: AddRec->getNumOperands());
9975 append_range(C&: NewOps, R: AddRec->operands().take_front(N: i));
9976 NewOps.push_back(Elt: OpAtScope);
9977 for (++i; i != e; ++i)
9978 NewOps.push_back(Elt: getSCEVAtScope(V: AddRec->getOperand(i), L));
9979
9980 const SCEV *FoldedRec = getAddRecExpr(
9981 Operands&: NewOps, L: AddRec->getLoop(), Flags: AddRec->getNoWrapFlags(Mask: SCEV::FlagNW));
9982 AddRec = dyn_cast<SCEVAddRecExpr>(Val: FoldedRec);
9983 // The addrec may be folded to a nonrecurrence, for example, if the
9984 // induction variable is multiplied by zero after constant folding. Go
9985 // ahead and return the folded value.
9986 if (!AddRec)
9987 return FoldedRec;
9988 break;
9989 }
9990
9991 // If the scope is outside the addrec's loop, evaluate it by using the
9992 // loop exit value of the addrec.
9993 if (!AddRec->getLoop()->contains(L)) {
9994 // To evaluate this recurrence, we need to know how many times the AddRec
9995 // loop iterates. Compute this now.
9996 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(L: AddRec->getLoop());
9997 if (BackedgeTakenCount == getCouldNotCompute())
9998 return AddRec;
9999
10000 // Then, evaluate the AddRec.
10001 return AddRec->evaluateAtIteration(It: BackedgeTakenCount, SE&: *this);
10002 }
10003
10004 return AddRec;
10005 }
10006 case scTruncate:
10007 case scZeroExtend:
10008 case scSignExtend:
10009 case scPtrToInt:
10010 case scAddExpr:
10011 case scMulExpr:
10012 case scUDivExpr:
10013 case scUMaxExpr:
10014 case scSMaxExpr:
10015 case scUMinExpr:
10016 case scSMinExpr:
10017 case scSequentialUMinExpr: {
10018 ArrayRef<const SCEV *> Ops = V->operands();
10019 // Avoid performing the look-up in the common case where the specified
10020 // expression has no loop-variant portions.
10021 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
10022 const SCEV *OpAtScope = getSCEVAtScope(V: Ops[i], L);
10023 if (OpAtScope != Ops[i]) {
10024 // Okay, at least one of these operands is loop variant but might be
10025 // foldable. Build a new instance of the folded commutative expression.
10026 SmallVector<const SCEV *, 8> NewOps;
10027 NewOps.reserve(N: Ops.size());
10028 append_range(C&: NewOps, R: Ops.take_front(N: i));
10029 NewOps.push_back(Elt: OpAtScope);
10030
10031 for (++i; i != e; ++i) {
10032 OpAtScope = getSCEVAtScope(V: Ops[i], L);
10033 NewOps.push_back(Elt: OpAtScope);
10034 }
10035
10036 return getWithOperands(S: V, NewOps);
10037 }
10038 }
10039 // If we got here, all operands are loop invariant.
10040 return V;
10041 }
10042 case scUnknown: {
10043 // If this instruction is evolved from a constant-evolving PHI, compute the
10044 // exit value from the loop without using SCEVs.
10045 const SCEVUnknown *SU = cast<SCEVUnknown>(Val: V);
10046 Instruction *I = dyn_cast<Instruction>(Val: SU->getValue());
10047 if (!I)
10048 return V; // This is some other type of SCEVUnknown, just return it.
10049
10050 if (PHINode *PN = dyn_cast<PHINode>(Val: I)) {
10051 const Loop *CurrLoop = this->LI[I->getParent()];
10052 // Looking for loop exit value.
10053 if (CurrLoop && CurrLoop->getParentLoop() == L &&
10054 PN->getParent() == CurrLoop->getHeader()) {
10055 // Okay, there is no closed form solution for the PHI node. Check
10056 // to see if the loop that contains it has a known backedge-taken
10057 // count. If so, we may be able to force computation of the exit
10058 // value.
10059 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(L: CurrLoop);
10060 // This trivial case can show up in some degenerate cases where
10061 // the incoming IR has not yet been fully simplified.
10062 if (BackedgeTakenCount->isZero()) {
10063 Value *InitValue = nullptr;
10064 bool MultipleInitValues = false;
10065 for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
10066 if (!CurrLoop->contains(BB: PN->getIncomingBlock(i))) {
10067 if (!InitValue)
10068 InitValue = PN->getIncomingValue(i);
10069 else if (InitValue != PN->getIncomingValue(i)) {
10070 MultipleInitValues = true;
10071 break;
10072 }
10073 }
10074 }
10075 if (!MultipleInitValues && InitValue)
10076 return getSCEV(V: InitValue);
10077 }
10078 // Do we have a loop invariant value flowing around the backedge
10079 // for a loop which must execute the backedge?
10080 if (!isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount) &&
10081 isKnownNonZero(S: BackedgeTakenCount) &&
10082 PN->getNumIncomingValues() == 2) {
10083
10084 unsigned InLoopPred =
10085 CurrLoop->contains(BB: PN->getIncomingBlock(i: 0)) ? 0 : 1;
10086 Value *BackedgeVal = PN->getIncomingValue(i: InLoopPred);
10087 if (CurrLoop->isLoopInvariant(V: BackedgeVal))
10088 return getSCEV(V: BackedgeVal);
10089 }
10090 if (auto *BTCC = dyn_cast<SCEVConstant>(Val: BackedgeTakenCount)) {
10091 // Okay, we know how many times the containing loop executes. If
10092 // this is a constant evolving PHI node, get the final value at
10093 // the specified iteration number.
10094 Constant *RV =
10095 getConstantEvolutionLoopExitValue(PN, BEs: BTCC->getAPInt(), L: CurrLoop);
10096 if (RV)
10097 return getSCEV(V: RV);
10098 }
10099 }
10100 }
10101
10102 // Okay, this is an expression that we cannot symbolically evaluate
10103 // into a SCEV. Check to see if it's possible to symbolically evaluate
10104 // the arguments into constants, and if so, try to constant propagate the
10105 // result. This is particularly useful for computing loop exit values.
10106 if (!CanConstantFold(I))
10107 return V; // This is some other type of SCEVUnknown, just return it.
10108
10109 SmallVector<Constant *, 4> Operands;
10110 Operands.reserve(N: I->getNumOperands());
10111 bool MadeImprovement = false;
10112 for (Value *Op : I->operands()) {
10113 if (Constant *C = dyn_cast<Constant>(Val: Op)) {
10114 Operands.push_back(Elt: C);
10115 continue;
10116 }
10117
10118 // If any of the operands is non-constant and if they are
10119 // non-integer and non-pointer, don't even try to analyze them
10120 // with scev techniques.
10121 if (!isSCEVable(Ty: Op->getType()))
10122 return V;
10123
10124 const SCEV *OrigV = getSCEV(V: Op);
10125 const SCEV *OpV = getSCEVAtScope(V: OrigV, L);
10126 MadeImprovement |= OrigV != OpV;
10127
10128 Constant *C = BuildConstantFromSCEV(V: OpV);
10129 if (!C)
10130 return V;
10131 assert(C->getType() == Op->getType() && "Type mismatch");
10132 Operands.push_back(Elt: C);
10133 }
10134
10135 // Check to see if getSCEVAtScope actually made an improvement.
10136 if (!MadeImprovement)
10137 return V; // This is some other type of SCEVUnknown, just return it.
10138
10139 Constant *C = nullptr;
10140 const DataLayout &DL = getDataLayout();
10141 C = ConstantFoldInstOperands(I, Ops: Operands, DL, TLI: &TLI,
10142 /*AllowNonDeterministic=*/false);
10143 if (!C)
10144 return V;
10145 return getSCEV(V: C);
10146 }
10147 case scCouldNotCompute:
10148 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10149 }
10150 llvm_unreachable("Unknown SCEV type!");
10151}
10152
10153const SCEV *ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
10154 return getSCEVAtScope(V: getSCEV(V), L);
10155}
10156
10157const SCEV *ScalarEvolution::stripInjectiveFunctions(const SCEV *S) const {
10158 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: S))
10159 return stripInjectiveFunctions(S: ZExt->getOperand());
10160 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
10161 return stripInjectiveFunctions(S: SExt->getOperand());
10162 return S;
10163}
10164
10165/// Finds the minimum unsigned root of the following equation:
10166///
10167/// A * X = B (mod N)
10168///
10169/// where N = 2^BW and BW is the common bit width of A and B. The signedness of
10170/// A and B isn't important.
10171///
10172/// If the equation does not have a solution, SCEVCouldNotCompute is returned.
10173static const SCEV *
10174SolveLinEquationWithOverflow(const APInt &A, const SCEV *B,
10175 SmallVectorImpl<const SCEVPredicate *> *Predicates,
10176
10177 ScalarEvolution &SE) {
10178 uint32_t BW = A.getBitWidth();
10179 assert(BW == SE.getTypeSizeInBits(B->getType()));
10180 assert(A != 0 && "A must be non-zero.");
10181
10182 // 1. D = gcd(A, N)
10183 //
10184 // The gcd of A and N may have only one prime factor: 2. The number of
10185 // trailing zeros in A is its multiplicity
10186 uint32_t Mult2 = A.countr_zero();
10187 // D = 2^Mult2
10188
10189 // 2. Check if B is divisible by D.
10190 //
10191 // B is divisible by D if and only if the multiplicity of prime factor 2 for B
10192 // is not less than multiplicity of this prime factor for D.
10193 if (SE.getMinTrailingZeros(S: B) < Mult2) {
10194 // Check if we can prove there's no remainder using URem.
10195 const SCEV *URem =
10196 SE.getURemExpr(LHS: B, RHS: SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2)));
10197 const SCEV *Zero = SE.getZero(Ty: B->getType());
10198 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: URem, RHS: Zero)) {
10199 // Try to add a predicate ensuring B is a multiple of 1 << Mult2.
10200 if (!Predicates)
10201 return SE.getCouldNotCompute();
10202
10203 // Avoid adding a predicate that is known to be false.
10204 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_NE, LHS: URem, RHS: Zero))
10205 return SE.getCouldNotCompute();
10206 Predicates->push_back(Elt: SE.getEqualPredicate(LHS: URem, RHS: Zero));
10207 }
10208 }
10209
10210 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
10211 // modulo (N / D).
10212 //
10213 // If D == 1, (N / D) == N == 2^BW, so we need one extra bit to represent
10214 // (N / D) in general. The inverse itself always fits into BW bits, though,
10215 // so we immediately truncate it.
10216 APInt AD = A.lshr(shiftAmt: Mult2).trunc(width: BW - Mult2); // AD = A / D
10217 APInt I = AD.multiplicativeInverse().zext(width: BW);
10218
10219 // 4. Compute the minimum unsigned root of the equation:
10220 // I * (B / D) mod (N / D)
10221 // To simplify the computation, we factor out the divide by D:
10222 // (I * B mod N) / D
10223 const SCEV *D = SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2));
10224 return SE.getUDivExactExpr(LHS: SE.getMulExpr(LHS: B, RHS: SE.getConstant(Val: I)), RHS: D);
10225}
10226
10227/// For a given quadratic addrec, generate coefficients of the corresponding
10228/// quadratic equation, multiplied by a common value to ensure that they are
10229/// integers.
10230/// The returned value is a tuple { A, B, C, M, BitWidth }, where
10231/// Ax^2 + Bx + C is the quadratic function, M is the value that A, B and C
10232/// were multiplied by, and BitWidth is the bit width of the original addrec
10233/// coefficients.
10234/// This function returns std::nullopt if the addrec coefficients are not
10235/// compile- time constants.
10236static std::optional<std::tuple<APInt, APInt, APInt, APInt, unsigned>>
10237GetQuadraticEquation(const SCEVAddRecExpr *AddRec) {
10238 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
10239 const SCEVConstant *LC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 0));
10240 const SCEVConstant *MC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 1));
10241 const SCEVConstant *NC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 2));
10242 LLVM_DEBUG(dbgs() << __func__ << ": analyzing quadratic addrec: "
10243 << *AddRec << '\n');
10244
10245 // We currently can only solve this if the coefficients are constants.
10246 if (!LC || !MC || !NC) {
10247 LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n");
10248 return std::nullopt;
10249 }
10250
10251 APInt L = LC->getAPInt();
10252 APInt M = MC->getAPInt();
10253 APInt N = NC->getAPInt();
10254 assert(!N.isZero() && "This is not a quadratic addrec");
10255
10256 unsigned BitWidth = LC->getAPInt().getBitWidth();
10257 unsigned NewWidth = BitWidth + 1;
10258 LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: "
10259 << BitWidth << '\n');
10260 // The sign-extension (as opposed to a zero-extension) here matches the
10261 // extension used in SolveQuadraticEquationWrap (with the same motivation).
10262 N = N.sext(width: NewWidth);
10263 M = M.sext(width: NewWidth);
10264 L = L.sext(width: NewWidth);
10265
10266 // The increments are M, M+N, M+2N, ..., so the accumulated values are
10267 // L+M, (L+M)+(M+N), (L+M)+(M+N)+(M+2N), ..., that is,
10268 // L+M, L+2M+N, L+3M+3N, ...
10269 // After n iterations the accumulated value Acc is L + nM + n(n-1)/2 N.
10270 //
10271 // The equation Acc = 0 is then
10272 // L + nM + n(n-1)/2 N = 0, or 2L + 2M n + n(n-1) N = 0.
10273 // In a quadratic form it becomes:
10274 // N n^2 + (2M-N) n + 2L = 0.
10275
10276 APInt A = N;
10277 APInt B = 2 * M - A;
10278 APInt C = 2 * L;
10279 APInt T = APInt(NewWidth, 2);
10280 LLVM_DEBUG(dbgs() << __func__ << ": equation " << A << "x^2 + " << B
10281 << "x + " << C << ", coeff bw: " << NewWidth
10282 << ", multiplied by " << T << '\n');
10283 return std::make_tuple(args&: A, args&: B, args&: C, args&: T, args&: BitWidth);
10284}
10285
10286/// Helper function to compare optional APInts:
10287/// (a) if X and Y both exist, return min(X, Y),
10288/// (b) if neither X nor Y exist, return std::nullopt,
10289/// (c) if exactly one of X and Y exists, return that value.
10290static std::optional<APInt> MinOptional(std::optional<APInt> X,
10291 std::optional<APInt> Y) {
10292 if (X && Y) {
10293 unsigned W = std::max(a: X->getBitWidth(), b: Y->getBitWidth());
10294 APInt XW = X->sext(width: W);
10295 APInt YW = Y->sext(width: W);
10296 return XW.slt(RHS: YW) ? *X : *Y;
10297 }
10298 if (!X && !Y)
10299 return std::nullopt;
10300 return X ? *X : *Y;
10301}
10302
10303/// Helper function to truncate an optional APInt to a given BitWidth.
10304/// When solving addrec-related equations, it is preferable to return a value
10305/// that has the same bit width as the original addrec's coefficients. If the
10306/// solution fits in the original bit width, truncate it (except for i1).
10307/// Returning a value of a different bit width may inhibit some optimizations.
10308///
10309/// In general, a solution to a quadratic equation generated from an addrec
10310/// may require BW+1 bits, where BW is the bit width of the addrec's
10311/// coefficients. The reason is that the coefficients of the quadratic
10312/// equation are BW+1 bits wide (to avoid truncation when converting from
10313/// the addrec to the equation).
10314static std::optional<APInt> TruncIfPossible(std::optional<APInt> X,
10315 unsigned BitWidth) {
10316 if (!X)
10317 return std::nullopt;
10318 unsigned W = X->getBitWidth();
10319 if (BitWidth > 1 && BitWidth < W && X->isIntN(N: BitWidth))
10320 return X->trunc(width: BitWidth);
10321 return X;
10322}
10323
10324/// Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n
10325/// iterations. The values L, M, N are assumed to be signed, and they
10326/// should all have the same bit widths.
10327/// Find the least n >= 0 such that c(n) = 0 in the arithmetic modulo 2^BW,
10328/// where BW is the bit width of the addrec's coefficients.
10329/// If the calculated value is a BW-bit integer (for BW > 1), it will be
10330/// returned as such, otherwise the bit width of the returned value may
10331/// be greater than BW.
10332///
10333/// This function returns std::nullopt if
10334/// (a) the addrec coefficients are not constant, or
10335/// (b) SolveQuadraticEquationWrap was unable to find a solution. For cases
10336/// like x^2 = 5, no integer solutions exist, in other cases an integer
10337/// solution may exist, but SolveQuadraticEquationWrap may fail to find it.
10338static std::optional<APInt>
10339SolveQuadraticAddRecExact(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
10340 APInt A, B, C, M;
10341 unsigned BitWidth;
10342 auto T = GetQuadraticEquation(AddRec);
10343 if (!T)
10344 return std::nullopt;
10345
10346 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10347 LLVM_DEBUG(dbgs() << __func__ << ": solving for unsigned overflow\n");
10348 std::optional<APInt> X =
10349 APIntOps::SolveQuadraticEquationWrap(A, B, C, RangeWidth: BitWidth + 1);
10350 if (!X)
10351 return std::nullopt;
10352
10353 ConstantInt *CX = ConstantInt::get(Context&: SE.getContext(), V: *X);
10354 ConstantInt *V = EvaluateConstantChrecAtConstant(AddRec, C: CX, SE);
10355 if (!V->isZero())
10356 return std::nullopt;
10357
10358 return TruncIfPossible(X, BitWidth);
10359}
10360
10361/// Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n
10362/// iterations. The values M, N are assumed to be signed, and they
10363/// should all have the same bit widths.
10364/// Find the least n such that c(n) does not belong to the given range,
10365/// while c(n-1) does.
10366///
10367/// This function returns std::nullopt if
10368/// (a) the addrec coefficients are not constant, or
10369/// (b) SolveQuadraticEquationWrap was unable to find a solution for the
10370/// bounds of the range.
10371static std::optional<APInt>
10372SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec,
10373 const ConstantRange &Range, ScalarEvolution &SE) {
10374 assert(AddRec->getOperand(0)->isZero() &&
10375 "Starting value of addrec should be 0");
10376 LLVM_DEBUG(dbgs() << __func__ << ": solving boundary crossing for range "
10377 << Range << ", addrec " << *AddRec << '\n');
10378 // This case is handled in getNumIterationsInRange. Here we can assume that
10379 // we start in the range.
10380 assert(Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) &&
10381 "Addrec's initial value should be in range");
10382
10383 APInt A, B, C, M;
10384 unsigned BitWidth;
10385 auto T = GetQuadraticEquation(AddRec);
10386 if (!T)
10387 return std::nullopt;
10388
10389 // Be careful about the return value: there can be two reasons for not
10390 // returning an actual number. First, if no solutions to the equations
10391 // were found, and second, if the solutions don't leave the given range.
10392 // The first case means that the actual solution is "unknown", the second
10393 // means that it's known, but not valid. If the solution is unknown, we
10394 // cannot make any conclusions.
10395 // Return a pair: the optional solution and a flag indicating if the
10396 // solution was found.
10397 auto SolveForBoundary =
10398 [&](APInt Bound) -> std::pair<std::optional<APInt>, bool> {
10399 // Solve for signed overflow and unsigned overflow, pick the lower
10400 // solution.
10401 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: checking boundary "
10402 << Bound << " (before multiplying by " << M << ")\n");
10403 Bound *= M; // The quadratic equation multiplier.
10404
10405 std::optional<APInt> SO;
10406 if (BitWidth > 1) {
10407 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10408 "signed overflow\n");
10409 SO = APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth);
10410 }
10411 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10412 "unsigned overflow\n");
10413 std::optional<APInt> UO =
10414 APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth + 1);
10415
10416 auto LeavesRange = [&] (const APInt &X) {
10417 ConstantInt *C0 = ConstantInt::get(Context&: SE.getContext(), V: X);
10418 ConstantInt *V0 = EvaluateConstantChrecAtConstant(AddRec, C: C0, SE);
10419 if (Range.contains(Val: V0->getValue()))
10420 return false;
10421 // X should be at least 1, so X-1 is non-negative.
10422 ConstantInt *C1 = ConstantInt::get(Context&: SE.getContext(), V: X-1);
10423 ConstantInt *V1 = EvaluateConstantChrecAtConstant(AddRec, C: C1, SE);
10424 if (Range.contains(Val: V1->getValue()))
10425 return true;
10426 return false;
10427 };
10428
10429 // If SolveQuadraticEquationWrap returns std::nullopt, it means that there
10430 // can be a solution, but the function failed to find it. We cannot treat it
10431 // as "no solution".
10432 if (!SO || !UO)
10433 return {std::nullopt, false};
10434
10435 // Check the smaller value first to see if it leaves the range.
10436 // At this point, both SO and UO must have values.
10437 std::optional<APInt> Min = MinOptional(X: SO, Y: UO);
10438 if (LeavesRange(*Min))
10439 return { Min, true };
10440 std::optional<APInt> Max = Min == SO ? UO : SO;
10441 if (LeavesRange(*Max))
10442 return { Max, true };
10443
10444 // Solutions were found, but were eliminated, hence the "true".
10445 return {std::nullopt, true};
10446 };
10447
10448 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10449 // Lower bound is inclusive, subtract 1 to represent the exiting value.
10450 APInt Lower = Range.getLower().sext(width: A.getBitWidth()) - 1;
10451 APInt Upper = Range.getUpper().sext(width: A.getBitWidth());
10452 auto SL = SolveForBoundary(Lower);
10453 auto SU = SolveForBoundary(Upper);
10454 // If any of the solutions was unknown, no meaninigful conclusions can
10455 // be made.
10456 if (!SL.second || !SU.second)
10457 return std::nullopt;
10458
10459 // Claim: The correct solution is not some value between Min and Max.
10460 //
10461 // Justification: Assuming that Min and Max are different values, one of
10462 // them is when the first signed overflow happens, the other is when the
10463 // first unsigned overflow happens. Crossing the range boundary is only
10464 // possible via an overflow (treating 0 as a special case of it, modeling
10465 // an overflow as crossing k*2^W for some k).
10466 //
10467 // The interesting case here is when Min was eliminated as an invalid
10468 // solution, but Max was not. The argument is that if there was another
10469 // overflow between Min and Max, it would also have been eliminated if
10470 // it was considered.
10471 //
10472 // For a given boundary, it is possible to have two overflows of the same
10473 // type (signed/unsigned) without having the other type in between: this
10474 // can happen when the vertex of the parabola is between the iterations
10475 // corresponding to the overflows. This is only possible when the two
10476 // overflows cross k*2^W for the same k. In such case, if the second one
10477 // left the range (and was the first one to do so), the first overflow
10478 // would have to enter the range, which would mean that either we had left
10479 // the range before or that we started outside of it. Both of these cases
10480 // are contradictions.
10481 //
10482 // Claim: In the case where SolveForBoundary returns std::nullopt, the correct
10483 // solution is not some value between the Max for this boundary and the
10484 // Min of the other boundary.
10485 //
10486 // Justification: Assume that we had such Max_A and Min_B corresponding
10487 // to range boundaries A and B and such that Max_A < Min_B. If there was
10488 // a solution between Max_A and Min_B, it would have to be caused by an
10489 // overflow corresponding to either A or B. It cannot correspond to B,
10490 // since Min_B is the first occurrence of such an overflow. If it
10491 // corresponded to A, it would have to be either a signed or an unsigned
10492 // overflow that is larger than both eliminated overflows for A. But
10493 // between the eliminated overflows and this overflow, the values would
10494 // cover the entire value space, thus crossing the other boundary, which
10495 // is a contradiction.
10496
10497 return TruncIfPossible(X: MinOptional(X: SL.first, Y: SU.first), BitWidth);
10498}
10499
10500ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V,
10501 const Loop *L,
10502 bool ControlsOnlyExit,
10503 bool AllowPredicates) {
10504
10505 // This is only used for loops with a "x != y" exit test. The exit condition
10506 // is now expressed as a single expression, V = x-y. So the exit test is
10507 // effectively V != 0. We know and take advantage of the fact that this
10508 // expression only being used in a comparison by zero context.
10509
10510 SmallVector<const SCEVPredicate *> Predicates;
10511 // If the value is a constant
10512 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10513 // If the value is already zero, the branch will execute zero times.
10514 if (C->getValue()->isZero()) return C;
10515 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10516 }
10517
10518 const SCEVAddRecExpr *AddRec =
10519 dyn_cast<SCEVAddRecExpr>(Val: stripInjectiveFunctions(S: V));
10520
10521 if (!AddRec && AllowPredicates)
10522 // Try to make this an AddRec using runtime tests, in the first X
10523 // iterations of this loop, where X is the SCEV expression found by the
10524 // algorithm below.
10525 AddRec = convertSCEVToAddRecWithPredicates(S: V, L, Preds&: Predicates);
10526
10527 if (!AddRec || AddRec->getLoop() != L)
10528 return getCouldNotCompute();
10529
10530 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
10531 // the quadratic equation to solve it.
10532 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
10533 // We can only use this value if the chrec ends up with an exact zero
10534 // value at this index. When solving for "X*X != 5", for example, we
10535 // should not accept a root of 2.
10536 if (auto S = SolveQuadraticAddRecExact(AddRec, SE&: *this)) {
10537 const auto *R = cast<SCEVConstant>(Val: getConstant(Val: *S));
10538 return ExitLimit(R, R, R, false, Predicates);
10539 }
10540 return getCouldNotCompute();
10541 }
10542
10543 // Otherwise we can only handle this if it is affine.
10544 if (!AddRec->isAffine())
10545 return getCouldNotCompute();
10546
10547 // If this is an affine expression, the execution count of this branch is
10548 // the minimum unsigned root of the following equation:
10549 //
10550 // Start + Step*N = 0 (mod 2^BW)
10551 //
10552 // equivalent to:
10553 //
10554 // Step*N = -Start (mod 2^BW)
10555 //
10556 // where BW is the common bit width of Start and Step.
10557
10558 // Get the initial value for the loop.
10559 const SCEV *Start = getSCEVAtScope(V: AddRec->getStart(), L: L->getParentLoop());
10560 const SCEV *Step = getSCEVAtScope(V: AddRec->getOperand(i: 1), L: L->getParentLoop());
10561
10562 if (!isLoopInvariant(S: Step, L))
10563 return getCouldNotCompute();
10564
10565 LoopGuards Guards = LoopGuards::collect(L, SE&: *this);
10566 // Specialize step for this loop so we get context sensitive facts below.
10567 const SCEV *StepWLG = applyLoopGuards(Expr: Step, Guards);
10568
10569 // For positive steps (counting up until unsigned overflow):
10570 // N = -Start/Step (as unsigned)
10571 // For negative steps (counting down to zero):
10572 // N = Start/-Step
10573 // First compute the unsigned distance from zero in the direction of Step.
10574 bool CountDown = isKnownNegative(S: StepWLG);
10575 if (!CountDown && !isKnownNonNegative(S: StepWLG))
10576 return getCouldNotCompute();
10577
10578 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(V: Start);
10579 // Handle unitary steps, which cannot wraparound.
10580 // 1*N = -Start; -1*N = Start (mod 2^BW), so:
10581 // N = Distance (as unsigned)
10582
10583 if (match(S: Step, P: m_CombineOr(L: m_scev_One(), R: m_scev_AllOnes()))) {
10584 APInt MaxBECount = getUnsignedRangeMax(S: applyLoopGuards(Expr: Distance, Guards));
10585 MaxBECount = APIntOps::umin(A: MaxBECount, B: getUnsignedRangeMax(S: Distance));
10586
10587 // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated,
10588 // we end up with a loop whose backedge-taken count is n - 1. Detect this
10589 // case, and see if we can improve the bound.
10590 //
10591 // Explicitly handling this here is necessary because getUnsignedRange
10592 // isn't context-sensitive; it doesn't know that we only care about the
10593 // range inside the loop.
10594 const SCEV *Zero = getZero(Ty: Distance->getType());
10595 const SCEV *One = getOne(Ty: Distance->getType());
10596 const SCEV *DistancePlusOne = getAddExpr(LHS: Distance, RHS: One);
10597 if (isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: DistancePlusOne, RHS: Zero)) {
10598 // If Distance + 1 doesn't overflow, we can compute the maximum distance
10599 // as "unsigned_max(Distance + 1) - 1".
10600 ConstantRange CR = getUnsignedRange(S: DistancePlusOne);
10601 MaxBECount = APIntOps::umin(A: MaxBECount, B: CR.getUnsignedMax() - 1);
10602 }
10603 return ExitLimit(Distance, getConstant(Val: MaxBECount), Distance, false,
10604 Predicates);
10605 }
10606
10607 // If the condition controls loop exit (the loop exits only if the expression
10608 // is true) and the addition is no-wrap we can use unsigned divide to
10609 // compute the backedge count. In this case, the step may not divide the
10610 // distance, but we don't care because if the condition is "missed" the loop
10611 // will have undefined behavior due to wrapping.
10612 if (ControlsOnlyExit && AddRec->hasNoSelfWrap() &&
10613 loopHasNoAbnormalExits(L: AddRec->getLoop())) {
10614
10615 // If the stride is zero and the start is non-zero, the loop must be
10616 // infinite. In C++, most loops are finite by assumption, in which case the
10617 // step being zero implies UB must execute if the loop is entered.
10618 if (!(loopIsFiniteByAssumption(L) && isKnownNonZero(S: Start)) &&
10619 !isKnownNonZero(S: StepWLG))
10620 return getCouldNotCompute();
10621
10622 const SCEV *Exact =
10623 getUDivExpr(LHS: Distance, RHS: CountDown ? getNegativeSCEV(V: Step) : Step);
10624 const SCEV *ConstantMax = getCouldNotCompute();
10625 if (Exact != getCouldNotCompute()) {
10626 APInt MaxInt = getUnsignedRangeMax(S: applyLoopGuards(Expr: Exact, Guards));
10627 ConstantMax =
10628 getConstant(Val: APIntOps::umin(A: MaxInt, B: getUnsignedRangeMax(S: Exact)));
10629 }
10630 const SCEV *SymbolicMax =
10631 isa<SCEVCouldNotCompute>(Val: Exact) ? ConstantMax : Exact;
10632 return ExitLimit(Exact, ConstantMax, SymbolicMax, false, Predicates);
10633 }
10634
10635 // Solve the general equation.
10636 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Val: Step);
10637 if (!StepC || StepC->getValue()->isZero())
10638 return getCouldNotCompute();
10639 const SCEV *E = SolveLinEquationWithOverflow(
10640 A: StepC->getAPInt(), B: getNegativeSCEV(V: Start),
10641 Predicates: AllowPredicates ? &Predicates : nullptr, SE&: *this);
10642
10643 const SCEV *M = E;
10644 if (E != getCouldNotCompute()) {
10645 APInt MaxWithGuards = getUnsignedRangeMax(S: applyLoopGuards(Expr: E, Guards));
10646 M = getConstant(Val: APIntOps::umin(A: MaxWithGuards, B: getUnsignedRangeMax(S: E)));
10647 }
10648 auto *S = isa<SCEVCouldNotCompute>(Val: E) ? M : E;
10649 return ExitLimit(E, M, S, false, Predicates);
10650}
10651
10652ScalarEvolution::ExitLimit
10653ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) {
10654 // Loops that look like: while (X == 0) are very strange indeed. We don't
10655 // handle them yet except for the trivial case. This could be expanded in the
10656 // future as needed.
10657
10658 // If the value is a constant, check to see if it is known to be non-zero
10659 // already. If so, the backedge will execute zero times.
10660 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10661 if (!C->getValue()->isZero())
10662 return getZero(Ty: C->getType());
10663 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10664 }
10665
10666 // We could implement others, but I really doubt anyone writes loops like
10667 // this, and if they did, they would already be constant folded.
10668 return getCouldNotCompute();
10669}
10670
10671std::pair<const BasicBlock *, const BasicBlock *>
10672ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(const BasicBlock *BB)
10673 const {
10674 // If the block has a unique predecessor, then there is no path from the
10675 // predecessor to the block that does not go through the direct edge
10676 // from the predecessor to the block.
10677 if (const BasicBlock *Pred = BB->getSinglePredecessor())
10678 return {Pred, BB};
10679
10680 // A loop's header is defined to be a block that dominates the loop.
10681 // If the header has a unique predecessor outside the loop, it must be
10682 // a block that has exactly one successor that can reach the loop.
10683 if (const Loop *L = LI.getLoopFor(BB))
10684 return {L->getLoopPredecessor(), L->getHeader()};
10685
10686 return {nullptr, BB};
10687}
10688
10689/// SCEV structural equivalence is usually sufficient for testing whether two
10690/// expressions are equal, however for the purposes of looking for a condition
10691/// guarding a loop, it can be useful to be a little more general, since a
10692/// front-end may have replicated the controlling expression.
10693static bool HasSameValue(const SCEV *A, const SCEV *B) {
10694 // Quick check to see if they are the same SCEV.
10695 if (A == B) return true;
10696
10697 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) {
10698 // Not all instructions that are "identical" compute the same value. For
10699 // instance, two distinct alloca instructions allocating the same type are
10700 // identical and do not read memory; but compute distinct values.
10701 return A->isIdenticalTo(I: B) && (isa<BinaryOperator>(Val: A) || isa<GetElementPtrInst>(Val: A));
10702 };
10703
10704 // Otherwise, if they're both SCEVUnknown, it's possible that they hold
10705 // two different instructions with the same value. Check for this case.
10706 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(Val: A))
10707 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(Val: B))
10708 if (const Instruction *AI = dyn_cast<Instruction>(Val: AU->getValue()))
10709 if (const Instruction *BI = dyn_cast<Instruction>(Val: BU->getValue()))
10710 if (ComputesEqualValues(AI, BI))
10711 return true;
10712
10713 // Otherwise assume they may have a different value.
10714 return false;
10715}
10716
10717static bool MatchBinarySub(const SCEV *S, const SCEV *&LHS, const SCEV *&RHS) {
10718 const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S);
10719 if (!Add || Add->getNumOperands() != 2)
10720 return false;
10721 if (auto *ME = dyn_cast<SCEVMulExpr>(Val: Add->getOperand(i: 0));
10722 ME && ME->getNumOperands() == 2 && ME->getOperand(i: 0)->isAllOnesValue()) {
10723 LHS = Add->getOperand(i: 1);
10724 RHS = ME->getOperand(i: 1);
10725 return true;
10726 }
10727 if (auto *ME = dyn_cast<SCEVMulExpr>(Val: Add->getOperand(i: 1));
10728 ME && ME->getNumOperands() == 2 && ME->getOperand(i: 0)->isAllOnesValue()) {
10729 LHS = Add->getOperand(i: 0);
10730 RHS = ME->getOperand(i: 1);
10731 return true;
10732 }
10733 return false;
10734}
10735
10736bool ScalarEvolution::SimplifyICmpOperands(CmpPredicate &Pred, const SCEV *&LHS,
10737 const SCEV *&RHS, unsigned Depth) {
10738 bool Changed = false;
10739 // Simplifies ICMP to trivial true or false by turning it into '0 == 0' or
10740 // '0 != 0'.
10741 auto TrivialCase = [&](bool TriviallyTrue) {
10742 LHS = RHS = getConstant(V: ConstantInt::getFalse(Context&: getContext()));
10743 Pred = TriviallyTrue ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
10744 return true;
10745 };
10746 // If we hit the max recursion limit bail out.
10747 if (Depth >= 3)
10748 return false;
10749
10750 // Canonicalize a constant to the right side.
10751 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val: LHS)) {
10752 // Check for both operands constant.
10753 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val: RHS)) {
10754 if (!ICmpInst::compare(LHS: LHSC->getAPInt(), RHS: RHSC->getAPInt(), Pred))
10755 return TrivialCase(false);
10756 return TrivialCase(true);
10757 }
10758 // Otherwise swap the operands to put the constant on the right.
10759 std::swap(a&: LHS, b&: RHS);
10760 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
10761 Changed = true;
10762 }
10763
10764 // If we're comparing an addrec with a value which is loop-invariant in the
10765 // addrec's loop, put the addrec on the left. Also make a dominance check,
10766 // as both operands could be addrecs loop-invariant in each other's loop.
10767 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: RHS)) {
10768 const Loop *L = AR->getLoop();
10769 if (isLoopInvariant(S: LHS, L) && properlyDominates(S: LHS, BB: L->getHeader())) {
10770 std::swap(a&: LHS, b&: RHS);
10771 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
10772 Changed = true;
10773 }
10774 }
10775
10776 // If there's a constant operand, canonicalize comparisons with boundary
10777 // cases, and canonicalize *-or-equal comparisons to regular comparisons.
10778 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(Val: RHS)) {
10779 const APInt &RA = RC->getAPInt();
10780
10781 bool SimplifiedByConstantRange = false;
10782
10783 if (!ICmpInst::isEquality(P: Pred)) {
10784 ConstantRange ExactCR = ConstantRange::makeExactICmpRegion(Pred, Other: RA);
10785 if (ExactCR.isFullSet())
10786 return TrivialCase(true);
10787 if (ExactCR.isEmptySet())
10788 return TrivialCase(false);
10789
10790 APInt NewRHS;
10791 CmpInst::Predicate NewPred;
10792 if (ExactCR.getEquivalentICmp(Pred&: NewPred, RHS&: NewRHS) &&
10793 ICmpInst::isEquality(P: NewPred)) {
10794 // We were able to convert an inequality to an equality.
10795 Pred = NewPred;
10796 RHS = getConstant(Val: NewRHS);
10797 Changed = SimplifiedByConstantRange = true;
10798 }
10799 }
10800
10801 if (!SimplifiedByConstantRange) {
10802 switch (Pred) {
10803 default:
10804 break;
10805 case ICmpInst::ICMP_EQ:
10806 case ICmpInst::ICMP_NE:
10807 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
10808 if (RA.isZero() && MatchBinarySub(S: LHS, LHS, RHS))
10809 Changed = true;
10810 break;
10811
10812 // The "Should have been caught earlier!" messages refer to the fact
10813 // that the ExactCR.isFullSet() or ExactCR.isEmptySet() check above
10814 // should have fired on the corresponding cases, and canonicalized the
10815 // check to trivial case.
10816
10817 case ICmpInst::ICMP_UGE:
10818 assert(!RA.isMinValue() && "Should have been caught earlier!");
10819 Pred = ICmpInst::ICMP_UGT;
10820 RHS = getConstant(Val: RA - 1);
10821 Changed = true;
10822 break;
10823 case ICmpInst::ICMP_ULE:
10824 assert(!RA.isMaxValue() && "Should have been caught earlier!");
10825 Pred = ICmpInst::ICMP_ULT;
10826 RHS = getConstant(Val: RA + 1);
10827 Changed = true;
10828 break;
10829 case ICmpInst::ICMP_SGE:
10830 assert(!RA.isMinSignedValue() && "Should have been caught earlier!");
10831 Pred = ICmpInst::ICMP_SGT;
10832 RHS = getConstant(Val: RA - 1);
10833 Changed = true;
10834 break;
10835 case ICmpInst::ICMP_SLE:
10836 assert(!RA.isMaxSignedValue() && "Should have been caught earlier!");
10837 Pred = ICmpInst::ICMP_SLT;
10838 RHS = getConstant(Val: RA + 1);
10839 Changed = true;
10840 break;
10841 }
10842 }
10843 }
10844
10845 // Check for obvious equality.
10846 if (HasSameValue(A: LHS, B: RHS)) {
10847 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
10848 return TrivialCase(true);
10849 if (ICmpInst::isFalseWhenEqual(predicate: Pred))
10850 return TrivialCase(false);
10851 }
10852
10853 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
10854 // adding or subtracting 1 from one of the operands.
10855 switch (Pred) {
10856 case ICmpInst::ICMP_SLE:
10857 if (!getSignedRangeMax(S: RHS).isMaxSignedValue()) {
10858 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
10859 Flags: SCEV::FlagNSW);
10860 Pred = ICmpInst::ICMP_SLT;
10861 Changed = true;
10862 } else if (!getSignedRangeMin(S: LHS).isMinSignedValue()) {
10863 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS,
10864 Flags: SCEV::FlagNSW);
10865 Pred = ICmpInst::ICMP_SLT;
10866 Changed = true;
10867 }
10868 break;
10869 case ICmpInst::ICMP_SGE:
10870 if (!getSignedRangeMin(S: RHS).isMinSignedValue()) {
10871 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS,
10872 Flags: SCEV::FlagNSW);
10873 Pred = ICmpInst::ICMP_SGT;
10874 Changed = true;
10875 } else if (!getSignedRangeMax(S: LHS).isMaxSignedValue()) {
10876 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
10877 Flags: SCEV::FlagNSW);
10878 Pred = ICmpInst::ICMP_SGT;
10879 Changed = true;
10880 }
10881 break;
10882 case ICmpInst::ICMP_ULE:
10883 if (!getUnsignedRangeMax(S: RHS).isMaxValue()) {
10884 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
10885 Flags: SCEV::FlagNUW);
10886 Pred = ICmpInst::ICMP_ULT;
10887 Changed = true;
10888 } else if (!getUnsignedRangeMin(S: LHS).isMinValue()) {
10889 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS);
10890 Pred = ICmpInst::ICMP_ULT;
10891 Changed = true;
10892 }
10893 break;
10894 case ICmpInst::ICMP_UGE:
10895 // If RHS is an op we can fold the -1, try that first.
10896 // Otherwise prefer LHS to preserve the nuw flag.
10897 if ((isa<SCEVConstant>(Val: RHS) ||
10898 (isa<SCEVAddExpr, SCEVAddRecExpr>(Val: RHS) &&
10899 isa<SCEVConstant>(Val: cast<SCEVNAryExpr>(Val: RHS)->getOperand(i: 0)))) &&
10900 !getUnsignedRangeMin(S: RHS).isMinValue()) {
10901 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
10902 Pred = ICmpInst::ICMP_UGT;
10903 Changed = true;
10904 } else if (!getUnsignedRangeMax(S: LHS).isMaxValue()) {
10905 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
10906 Flags: SCEV::FlagNUW);
10907 Pred = ICmpInst::ICMP_UGT;
10908 Changed = true;
10909 } else if (!getUnsignedRangeMin(S: RHS).isMinValue()) {
10910 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
10911 Pred = ICmpInst::ICMP_UGT;
10912 Changed = true;
10913 }
10914 break;
10915 default:
10916 break;
10917 }
10918
10919 // TODO: More simplifications are possible here.
10920
10921 // Recursively simplify until we either hit a recursion limit or nothing
10922 // changes.
10923 if (Changed)
10924 return SimplifyICmpOperands(Pred, LHS, RHS, Depth: Depth + 1);
10925
10926 return Changed;
10927}
10928
10929bool ScalarEvolution::isKnownNegative(const SCEV *S) {
10930 return getSignedRangeMax(S).isNegative();
10931}
10932
10933bool ScalarEvolution::isKnownPositive(const SCEV *S) {
10934 return getSignedRangeMin(S).isStrictlyPositive();
10935}
10936
10937bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
10938 return !getSignedRangeMin(S).isNegative();
10939}
10940
10941bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
10942 return !getSignedRangeMax(S).isStrictlyPositive();
10943}
10944
10945bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
10946 // Query push down for cases where the unsigned range is
10947 // less than sufficient.
10948 if (const auto *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
10949 return isKnownNonZero(S: SExt->getOperand(i: 0));
10950 return getUnsignedRangeMin(S) != 0;
10951}
10952
10953bool ScalarEvolution::isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero,
10954 bool OrNegative) {
10955 auto NonRecursive = [this, OrNegative](const SCEV *S) {
10956 if (auto *C = dyn_cast<SCEVConstant>(Val: S))
10957 return C->getAPInt().isPowerOf2() ||
10958 (OrNegative && C->getAPInt().isNegatedPowerOf2());
10959
10960 // The vscale_range indicates vscale is a power-of-two.
10961 return isa<SCEVVScale>(Val: S) && F.hasFnAttribute(Kind: Attribute::VScaleRange);
10962 };
10963
10964 if (NonRecursive(S))
10965 return true;
10966
10967 auto *Mul = dyn_cast<SCEVMulExpr>(Val: S);
10968 if (!Mul)
10969 return false;
10970 return all_of(Range: Mul->operands(), P: NonRecursive) && (OrZero || isKnownNonZero(S));
10971}
10972
10973bool ScalarEvolution::isKnownMultipleOf(
10974 const SCEV *S, uint64_t M,
10975 SmallVectorImpl<const SCEVPredicate *> &Assumptions) {
10976 if (M == 0)
10977 return false;
10978 if (M == 1)
10979 return true;
10980
10981 // Recursively check AddRec operands. An AddRecExpr S is a multiple of M if S
10982 // starts with a multiple of M and at every iteration step S only adds
10983 // multiples of M.
10984 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S))
10985 return isKnownMultipleOf(S: AddRec->getStart(), M, Assumptions) &&
10986 isKnownMultipleOf(S: AddRec->getStepRecurrence(SE&: *this), M, Assumptions);
10987
10988 // For a constant, check that "S % M == 0".
10989 if (auto *Cst = dyn_cast<SCEVConstant>(Val: S)) {
10990 APInt C = Cst->getAPInt();
10991 return C.urem(RHS: M) == 0;
10992 }
10993
10994 // TODO: Also check other SCEV expressions, i.e., SCEVAddRecExpr, etc.
10995
10996 // Basic tests have failed.
10997 // Check "S % M == 0" at compile time and record runtime Assumptions.
10998 auto *STy = dyn_cast<IntegerType>(Val: S->getType());
10999 const SCEV *SmodM =
11000 getURemExpr(LHS: S, RHS: getConstant(V: ConstantInt::get(Ty: STy, V: M, IsSigned: false)));
11001 const SCEV *Zero = getZero(Ty: STy);
11002
11003 // Check whether "S % M == 0" is known at compile time.
11004 if (isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero))
11005 return true;
11006
11007 // Check whether "S % M != 0" is known at compile time.
11008 if (isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: SmodM, RHS: Zero))
11009 return false;
11010
11011 const SCEVPredicate *P = getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero);
11012
11013 // Detect redundant predicates.
11014 for (auto *A : Assumptions)
11015 if (A->implies(N: P, SE&: *this))
11016 return true;
11017
11018 // Only record non-redundant predicates.
11019 Assumptions.push_back(Elt: P);
11020 return true;
11021}
11022
11023std::pair<const SCEV *, const SCEV *>
11024ScalarEvolution::SplitIntoInitAndPostInc(const Loop *L, const SCEV *S) {
11025 // Compute SCEV on entry of loop L.
11026 const SCEV *Start = SCEVInitRewriter::rewrite(S, L, SE&: *this);
11027 if (Start == getCouldNotCompute())
11028 return { Start, Start };
11029 // Compute post increment SCEV for loop L.
11030 const SCEV *PostInc = SCEVPostIncRewriter::rewrite(S, L, SE&: *this);
11031 assert(PostInc != getCouldNotCompute() && "Unexpected could not compute");
11032 return { Start, PostInc };
11033}
11034
11035bool ScalarEvolution::isKnownViaInduction(CmpPredicate Pred, const SCEV *LHS,
11036 const SCEV *RHS) {
11037 // First collect all loops.
11038 SmallPtrSet<const Loop *, 8> LoopsUsed;
11039 getUsedLoops(S: LHS, LoopsUsed);
11040 getUsedLoops(S: RHS, LoopsUsed);
11041
11042 if (LoopsUsed.empty())
11043 return false;
11044
11045 // Domination relationship must be a linear order on collected loops.
11046#ifndef NDEBUG
11047 for (const auto *L1 : LoopsUsed)
11048 for (const auto *L2 : LoopsUsed)
11049 assert((DT.dominates(L1->getHeader(), L2->getHeader()) ||
11050 DT.dominates(L2->getHeader(), L1->getHeader())) &&
11051 "Domination relationship is not a linear order");
11052#endif
11053
11054 const Loop *MDL =
11055 *llvm::max_element(Range&: LoopsUsed, C: [&](const Loop *L1, const Loop *L2) {
11056 return DT.properlyDominates(A: L1->getHeader(), B: L2->getHeader());
11057 });
11058
11059 // Get init and post increment value for LHS.
11060 auto SplitLHS = SplitIntoInitAndPostInc(L: MDL, S: LHS);
11061 // if LHS contains unknown non-invariant SCEV then bail out.
11062 if (SplitLHS.first == getCouldNotCompute())
11063 return false;
11064 assert (SplitLHS.second != getCouldNotCompute() && "Unexpected CNC");
11065 // Get init and post increment value for RHS.
11066 auto SplitRHS = SplitIntoInitAndPostInc(L: MDL, S: RHS);
11067 // if RHS contains unknown non-invariant SCEV then bail out.
11068 if (SplitRHS.first == getCouldNotCompute())
11069 return false;
11070 assert (SplitRHS.second != getCouldNotCompute() && "Unexpected CNC");
11071 // It is possible that init SCEV contains an invariant load but it does
11072 // not dominate MDL and is not available at MDL loop entry, so we should
11073 // check it here.
11074 if (!isAvailableAtLoopEntry(S: SplitLHS.first, L: MDL) ||
11075 !isAvailableAtLoopEntry(S: SplitRHS.first, L: MDL))
11076 return false;
11077
11078 // It seems backedge guard check is faster than entry one so in some cases
11079 // it can speed up whole estimation by short circuit
11080 return isLoopBackedgeGuardedByCond(L: MDL, Pred, LHS: SplitLHS.second,
11081 RHS: SplitRHS.second) &&
11082 isLoopEntryGuardedByCond(L: MDL, Pred, LHS: SplitLHS.first, RHS: SplitRHS.first);
11083}
11084
11085bool ScalarEvolution::isKnownPredicate(CmpPredicate Pred, const SCEV *LHS,
11086 const SCEV *RHS) {
11087 // Canonicalize the inputs first.
11088 (void)SimplifyICmpOperands(Pred, LHS, RHS);
11089
11090 if (isKnownViaInduction(Pred, LHS, RHS))
11091 return true;
11092
11093 if (isKnownPredicateViaSplitting(Pred, LHS, RHS))
11094 return true;
11095
11096 // Otherwise see what can be done with some simple reasoning.
11097 return isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
11098}
11099
11100std::optional<bool> ScalarEvolution::evaluatePredicate(CmpPredicate Pred,
11101 const SCEV *LHS,
11102 const SCEV *RHS) {
11103 if (isKnownPredicate(Pred, LHS, RHS))
11104 return true;
11105 if (isKnownPredicate(Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11106 return false;
11107 return std::nullopt;
11108}
11109
11110bool ScalarEvolution::isKnownPredicateAt(CmpPredicate Pred, const SCEV *LHS,
11111 const SCEV *RHS,
11112 const Instruction *CtxI) {
11113 // TODO: Analyze guards and assumes from Context's block.
11114 return isKnownPredicate(Pred, LHS, RHS) ||
11115 isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS);
11116}
11117
11118std::optional<bool>
11119ScalarEvolution::evaluatePredicateAt(CmpPredicate Pred, const SCEV *LHS,
11120 const SCEV *RHS, const Instruction *CtxI) {
11121 std::optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS);
11122 if (KnownWithoutContext)
11123 return KnownWithoutContext;
11124
11125 if (isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS))
11126 return true;
11127 if (isBasicBlockEntryGuardedByCond(
11128 BB: CtxI->getParent(), Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11129 return false;
11130 return std::nullopt;
11131}
11132
11133bool ScalarEvolution::isKnownOnEveryIteration(CmpPredicate Pred,
11134 const SCEVAddRecExpr *LHS,
11135 const SCEV *RHS) {
11136 const Loop *L = LHS->getLoop();
11137 return isLoopEntryGuardedByCond(L, Pred, LHS: LHS->getStart(), RHS) &&
11138 isLoopBackedgeGuardedByCond(L, Pred, LHS: LHS->getPostIncExpr(SE&: *this), RHS);
11139}
11140
11141std::optional<ScalarEvolution::MonotonicPredicateType>
11142ScalarEvolution::getMonotonicPredicateType(const SCEVAddRecExpr *LHS,
11143 ICmpInst::Predicate Pred) {
11144 auto Result = getMonotonicPredicateTypeImpl(LHS, Pred);
11145
11146#ifndef NDEBUG
11147 // Verify an invariant: inverting the predicate should turn a monotonically
11148 // increasing change to a monotonically decreasing one, and vice versa.
11149 if (Result) {
11150 auto ResultSwapped =
11151 getMonotonicPredicateTypeImpl(LHS, ICmpInst::getSwappedPredicate(Pred));
11152
11153 assert(*ResultSwapped != *Result &&
11154 "monotonicity should flip as we flip the predicate");
11155 }
11156#endif
11157
11158 return Result;
11159}
11160
11161std::optional<ScalarEvolution::MonotonicPredicateType>
11162ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
11163 ICmpInst::Predicate Pred) {
11164 // A zero step value for LHS means the induction variable is essentially a
11165 // loop invariant value. We don't really depend on the predicate actually
11166 // flipping from false to true (for increasing predicates, and the other way
11167 // around for decreasing predicates), all we care about is that *if* the
11168 // predicate changes then it only changes from false to true.
11169 //
11170 // A zero step value in itself is not very useful, but there may be places
11171 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be
11172 // as general as possible.
11173
11174 // Only handle LE/LT/GE/GT predicates.
11175 if (!ICmpInst::isRelational(P: Pred))
11176 return std::nullopt;
11177
11178 bool IsGreater = ICmpInst::isGE(P: Pred) || ICmpInst::isGT(P: Pred);
11179 assert((IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) &&
11180 "Should be greater or less!");
11181
11182 // Check that AR does not wrap.
11183 if (ICmpInst::isUnsigned(predicate: Pred)) {
11184 if (!LHS->hasNoUnsignedWrap())
11185 return std::nullopt;
11186 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11187 }
11188 assert(ICmpInst::isSigned(Pred) &&
11189 "Relational predicate is either signed or unsigned!");
11190 if (!LHS->hasNoSignedWrap())
11191 return std::nullopt;
11192
11193 const SCEV *Step = LHS->getStepRecurrence(SE&: *this);
11194
11195 if (isKnownNonNegative(S: Step))
11196 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11197
11198 if (isKnownNonPositive(S: Step))
11199 return !IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11200
11201 return std::nullopt;
11202}
11203
11204std::optional<ScalarEvolution::LoopInvariantPredicate>
11205ScalarEvolution::getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS,
11206 const SCEV *RHS, const Loop *L,
11207 const Instruction *CtxI) {
11208 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11209 if (!isLoopInvariant(S: RHS, L)) {
11210 if (!isLoopInvariant(S: LHS, L))
11211 return std::nullopt;
11212
11213 std::swap(a&: LHS, b&: RHS);
11214 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11215 }
11216
11217 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11218 if (!ArLHS || ArLHS->getLoop() != L)
11219 return std::nullopt;
11220
11221 auto MonotonicType = getMonotonicPredicateType(LHS: ArLHS, Pred);
11222 if (!MonotonicType)
11223 return std::nullopt;
11224 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
11225 // true as the loop iterates, and the backedge is control dependent on
11226 // "ArLHS `Pred` RHS" == true then we can reason as follows:
11227 //
11228 // * if the predicate was false in the first iteration then the predicate
11229 // is never evaluated again, since the loop exits without taking the
11230 // backedge.
11231 // * if the predicate was true in the first iteration then it will
11232 // continue to be true for all future iterations since it is
11233 // monotonically increasing.
11234 //
11235 // For both the above possibilities, we can replace the loop varying
11236 // predicate with its value on the first iteration of the loop (which is
11237 // loop invariant).
11238 //
11239 // A similar reasoning applies for a monotonically decreasing predicate, by
11240 // replacing true with false and false with true in the above two bullets.
11241 bool Increasing = *MonotonicType == ScalarEvolution::MonotonicallyIncreasing;
11242 auto P = Increasing ? Pred : ICmpInst::getInverseCmpPredicate(Pred);
11243
11244 if (isLoopBackedgeGuardedByCond(L, Pred: P, LHS, RHS))
11245 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11246 RHS);
11247
11248 if (!CtxI)
11249 return std::nullopt;
11250 // Try to prove via context.
11251 // TODO: Support other cases.
11252 switch (Pred) {
11253 default:
11254 break;
11255 case ICmpInst::ICMP_ULE:
11256 case ICmpInst::ICMP_ULT: {
11257 assert(ArLHS->hasNoUnsignedWrap() && "Is a requirement of monotonicity!");
11258 // Given preconditions
11259 // (1) ArLHS does not cross the border of positive and negative parts of
11260 // range because of:
11261 // - Positive step; (TODO: lift this limitation)
11262 // - nuw - does not cross zero boundary;
11263 // - nsw - does not cross SINT_MAX boundary;
11264 // (2) ArLHS <s RHS
11265 // (3) RHS >=s 0
11266 // we can replace the loop variant ArLHS <u RHS condition with loop
11267 // invariant Start(ArLHS) <u RHS.
11268 //
11269 // Because of (1) there are two options:
11270 // - ArLHS is always negative. It means that ArLHS <u RHS is always false;
11271 // - ArLHS is always non-negative. Because of (3) RHS is also non-negative.
11272 // It means that ArLHS <s RHS <=> ArLHS <u RHS.
11273 // Because of (2) ArLHS <u RHS is trivially true.
11274 // All together it means that ArLHS <u RHS <=> Start(ArLHS) >=s 0.
11275 // We can strengthen this to Start(ArLHS) <u RHS.
11276 auto SignFlippedPred = ICmpInst::getFlippedSignednessPredicate(Pred);
11277 if (ArLHS->hasNoSignedWrap() && ArLHS->isAffine() &&
11278 isKnownPositive(S: ArLHS->getStepRecurrence(SE&: *this)) &&
11279 isKnownNonNegative(S: RHS) &&
11280 isKnownPredicateAt(Pred: SignFlippedPred, LHS: ArLHS, RHS, CtxI))
11281 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11282 RHS);
11283 }
11284 }
11285
11286 return std::nullopt;
11287}
11288
11289std::optional<ScalarEvolution::LoopInvariantPredicate>
11290ScalarEvolution::getLoopInvariantExitCondDuringFirstIterations(
11291 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11292 const Instruction *CtxI, const SCEV *MaxIter) {
11293 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11294 Pred, LHS, RHS, L, CtxI, MaxIter))
11295 return LIP;
11296 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: MaxIter))
11297 // Number of iterations expressed as UMIN isn't always great for expressing
11298 // the value on the last iteration. If the straightforward approach didn't
11299 // work, try the following trick: if the a predicate is invariant for X, it
11300 // is also invariant for umin(X, ...). So try to find something that works
11301 // among subexpressions of MaxIter expressed as umin.
11302 for (auto *Op : UMin->operands())
11303 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11304 Pred, LHS, RHS, L, CtxI, MaxIter: Op))
11305 return LIP;
11306 return std::nullopt;
11307}
11308
11309std::optional<ScalarEvolution::LoopInvariantPredicate>
11310ScalarEvolution::getLoopInvariantExitCondDuringFirstIterationsImpl(
11311 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11312 const Instruction *CtxI, const SCEV *MaxIter) {
11313 // Try to prove the following set of facts:
11314 // - The predicate is monotonic in the iteration space.
11315 // - If the check does not fail on the 1st iteration:
11316 // - No overflow will happen during first MaxIter iterations;
11317 // - It will not fail on the MaxIter'th iteration.
11318 // If the check does fail on the 1st iteration, we leave the loop and no
11319 // other checks matter.
11320
11321 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11322 if (!isLoopInvariant(S: RHS, L)) {
11323 if (!isLoopInvariant(S: LHS, L))
11324 return std::nullopt;
11325
11326 std::swap(a&: LHS, b&: RHS);
11327 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11328 }
11329
11330 auto *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11331 if (!AR || AR->getLoop() != L)
11332 return std::nullopt;
11333
11334 // The predicate must be relational (i.e. <, <=, >=, >).
11335 if (!ICmpInst::isRelational(P: Pred))
11336 return std::nullopt;
11337
11338 // TODO: Support steps other than +/- 1.
11339 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
11340 auto *One = getOne(Ty: Step->getType());
11341 auto *MinusOne = getNegativeSCEV(V: One);
11342 if (Step != One && Step != MinusOne)
11343 return std::nullopt;
11344
11345 // Type mismatch here means that MaxIter is potentially larger than max
11346 // unsigned value in start type, which mean we cannot prove no wrap for the
11347 // indvar.
11348 if (AR->getType() != MaxIter->getType())
11349 return std::nullopt;
11350
11351 // Value of IV on suggested last iteration.
11352 const SCEV *Last = AR->evaluateAtIteration(It: MaxIter, SE&: *this);
11353 // Does it still meet the requirement?
11354 if (!isLoopBackedgeGuardedByCond(L, Pred, LHS: Last, RHS))
11355 return std::nullopt;
11356 // Because step is +/- 1 and MaxIter has same type as Start (i.e. it does
11357 // not exceed max unsigned value of this type), this effectively proves
11358 // that there is no wrap during the iteration. To prove that there is no
11359 // signed/unsigned wrap, we need to check that
11360 // Start <= Last for step = 1 or Start >= Last for step = -1.
11361 ICmpInst::Predicate NoOverflowPred =
11362 CmpInst::isSigned(predicate: Pred) ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
11363 if (Step == MinusOne)
11364 NoOverflowPred = ICmpInst::getSwappedCmpPredicate(Pred: NoOverflowPred);
11365 const SCEV *Start = AR->getStart();
11366 if (!isKnownPredicateAt(Pred: NoOverflowPred, LHS: Start, RHS: Last, CtxI))
11367 return std::nullopt;
11368
11369 // Everything is fine.
11370 return ScalarEvolution::LoopInvariantPredicate(Pred, Start, RHS);
11371}
11372
11373bool ScalarEvolution::isKnownPredicateViaConstantRanges(CmpPredicate Pred,
11374 const SCEV *LHS,
11375 const SCEV *RHS) {
11376 if (HasSameValue(A: LHS, B: RHS))
11377 return ICmpInst::isTrueWhenEqual(predicate: Pred);
11378
11379 // This code is split out from isKnownPredicate because it is called from
11380 // within isLoopEntryGuardedByCond.
11381
11382 auto CheckRanges = [&](const ConstantRange &RangeLHS,
11383 const ConstantRange &RangeRHS) {
11384 return RangeLHS.icmp(Pred, Other: RangeRHS);
11385 };
11386
11387 // The check at the top of the function catches the case where the values are
11388 // known to be equal.
11389 if (Pred == CmpInst::ICMP_EQ)
11390 return false;
11391
11392 if (Pred == CmpInst::ICMP_NE) {
11393 auto SL = getSignedRange(S: LHS);
11394 auto SR = getSignedRange(S: RHS);
11395 if (CheckRanges(SL, SR))
11396 return true;
11397 auto UL = getUnsignedRange(S: LHS);
11398 auto UR = getUnsignedRange(S: RHS);
11399 if (CheckRanges(UL, UR))
11400 return true;
11401 auto *Diff = getMinusSCEV(LHS, RHS);
11402 return !isa<SCEVCouldNotCompute>(Val: Diff) && isKnownNonZero(S: Diff);
11403 }
11404
11405 if (CmpInst::isSigned(predicate: Pred)) {
11406 auto SL = getSignedRange(S: LHS);
11407 auto SR = getSignedRange(S: RHS);
11408 return CheckRanges(SL, SR);
11409 }
11410
11411 auto UL = getUnsignedRange(S: LHS);
11412 auto UR = getUnsignedRange(S: RHS);
11413 return CheckRanges(UL, UR);
11414}
11415
11416bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred,
11417 const SCEV *LHS,
11418 const SCEV *RHS) {
11419 // Match X to (A + C1)<ExpectedFlags> and Y to (A + C2)<ExpectedFlags>, where
11420 // C1 and C2 are constant integers. If either X or Y are not add expressions,
11421 // consider them as X + 0 and Y + 0 respectively. C1 and C2 are returned via
11422 // OutC1 and OutC2.
11423 auto MatchBinaryAddToConst = [this](const SCEV *X, const SCEV *Y,
11424 APInt &OutC1, APInt &OutC2,
11425 SCEV::NoWrapFlags ExpectedFlags) {
11426 const SCEV *XNonConstOp, *XConstOp;
11427 const SCEV *YNonConstOp, *YConstOp;
11428 SCEV::NoWrapFlags XFlagsPresent;
11429 SCEV::NoWrapFlags YFlagsPresent;
11430
11431 if (!splitBinaryAdd(Expr: X, L&: XConstOp, R&: XNonConstOp, Flags&: XFlagsPresent)) {
11432 XConstOp = getZero(Ty: X->getType());
11433 XNonConstOp = X;
11434 XFlagsPresent = ExpectedFlags;
11435 }
11436 if (!isa<SCEVConstant>(Val: XConstOp) ||
11437 (XFlagsPresent & ExpectedFlags) != ExpectedFlags)
11438 return false;
11439
11440 if (!splitBinaryAdd(Expr: Y, L&: YConstOp, R&: YNonConstOp, Flags&: YFlagsPresent)) {
11441 YConstOp = getZero(Ty: Y->getType());
11442 YNonConstOp = Y;
11443 YFlagsPresent = ExpectedFlags;
11444 }
11445
11446 if (!isa<SCEVConstant>(Val: YConstOp) ||
11447 (YFlagsPresent & ExpectedFlags) != ExpectedFlags)
11448 return false;
11449
11450 if (YNonConstOp != XNonConstOp)
11451 return false;
11452
11453 OutC1 = cast<SCEVConstant>(Val: XConstOp)->getAPInt();
11454 OutC2 = cast<SCEVConstant>(Val: YConstOp)->getAPInt();
11455
11456 return true;
11457 };
11458
11459 APInt C1;
11460 APInt C2;
11461
11462 switch (Pred) {
11463 default:
11464 break;
11465
11466 case ICmpInst::ICMP_SGE:
11467 std::swap(a&: LHS, b&: RHS);
11468 [[fallthrough]];
11469 case ICmpInst::ICMP_SLE:
11470 // (X + C1)<nsw> s<= (X + C2)<nsw> if C1 s<= C2.
11471 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.sle(RHS: C2))
11472 return true;
11473
11474 break;
11475
11476 case ICmpInst::ICMP_SGT:
11477 std::swap(a&: LHS, b&: RHS);
11478 [[fallthrough]];
11479 case ICmpInst::ICMP_SLT:
11480 // (X + C1)<nsw> s< (X + C2)<nsw> if C1 s< C2.
11481 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.slt(RHS: C2))
11482 return true;
11483
11484 break;
11485
11486 case ICmpInst::ICMP_UGE:
11487 std::swap(a&: LHS, b&: RHS);
11488 [[fallthrough]];
11489 case ICmpInst::ICMP_ULE:
11490 // (X + C1)<nuw> u<= (X + C2)<nuw> for C1 u<= C2.
11491 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ule(RHS: C2))
11492 return true;
11493
11494 break;
11495
11496 case ICmpInst::ICMP_UGT:
11497 std::swap(a&: LHS, b&: RHS);
11498 [[fallthrough]];
11499 case ICmpInst::ICMP_ULT:
11500 // (X + C1)<nuw> u< (X + C2)<nuw> if C1 u< C2.
11501 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ult(RHS: C2))
11502 return true;
11503 break;
11504 }
11505
11506 return false;
11507}
11508
11509bool ScalarEvolution::isKnownPredicateViaSplitting(CmpPredicate Pred,
11510 const SCEV *LHS,
11511 const SCEV *RHS) {
11512 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate)
11513 return false;
11514
11515 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on
11516 // the stack can result in exponential time complexity.
11517 SaveAndRestore Restore(ProvingSplitPredicate, true);
11518
11519 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L
11520 //
11521 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use
11522 // isKnownPredicate. isKnownPredicate is more powerful, but also more
11523 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the
11524 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to
11525 // use isKnownPredicate later if needed.
11526 return isKnownNonNegative(S: RHS) &&
11527 isKnownPredicate(Pred: CmpInst::ICMP_SGE, LHS, RHS: getZero(Ty: LHS->getType())) &&
11528 isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS, RHS);
11529}
11530
11531bool ScalarEvolution::isImpliedViaGuard(const BasicBlock *BB, CmpPredicate Pred,
11532 const SCEV *LHS, const SCEV *RHS) {
11533 // No need to even try if we know the module has no guards.
11534 if (!HasGuards)
11535 return false;
11536
11537 return any_of(Range: *BB, P: [&](const Instruction &I) {
11538 using namespace llvm::PatternMatch;
11539
11540 Value *Condition;
11541 return match(V: &I, P: m_Intrinsic<Intrinsic::experimental_guard>(
11542 Op0: m_Value(V&: Condition))) &&
11543 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse: false);
11544 });
11545}
11546
11547/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
11548/// protected by a conditional between LHS and RHS. This is used to
11549/// to eliminate casts.
11550bool ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
11551 CmpPredicate Pred,
11552 const SCEV *LHS,
11553 const SCEV *RHS) {
11554 // Interpret a null as meaning no loop, where there is obviously no guard
11555 // (interprocedural conditions notwithstanding). Do not bother about
11556 // unreachable loops.
11557 if (!L || !DT.isReachableFromEntry(A: L->getHeader()))
11558 return true;
11559
11560 if (VerifyIR)
11561 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()) &&
11562 "This cannot be done on broken IR!");
11563
11564
11565 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11566 return true;
11567
11568 BasicBlock *Latch = L->getLoopLatch();
11569 if (!Latch)
11570 return false;
11571
11572 BranchInst *LoopContinuePredicate =
11573 dyn_cast<BranchInst>(Val: Latch->getTerminator());
11574 if (LoopContinuePredicate && LoopContinuePredicate->isConditional() &&
11575 isImpliedCond(Pred, LHS, RHS,
11576 FoundCondValue: LoopContinuePredicate->getCondition(),
11577 Inverse: LoopContinuePredicate->getSuccessor(i: 0) != L->getHeader()))
11578 return true;
11579
11580 // We don't want more than one activation of the following loops on the stack
11581 // -- that can lead to O(n!) time complexity.
11582 if (WalkingBEDominatingConds)
11583 return false;
11584
11585 SaveAndRestore ClearOnExit(WalkingBEDominatingConds, true);
11586
11587 // See if we can exploit a trip count to prove the predicate.
11588 const auto &BETakenInfo = getBackedgeTakenInfo(L);
11589 const SCEV *LatchBECount = BETakenInfo.getExact(ExitingBlock: Latch, SE: this);
11590 if (LatchBECount != getCouldNotCompute()) {
11591 // We know that Latch branches back to the loop header exactly
11592 // LatchBECount times. This means the backdege condition at Latch is
11593 // equivalent to "{0,+,1} u< LatchBECount".
11594 Type *Ty = LatchBECount->getType();
11595 auto NoWrapFlags = SCEV::NoWrapFlags(SCEV::FlagNUW | SCEV::FlagNW);
11596 const SCEV *LoopCounter =
11597 getAddRecExpr(Start: getZero(Ty), Step: getOne(Ty), L, Flags: NoWrapFlags);
11598 if (isImpliedCond(Pred, LHS, RHS, FoundPred: ICmpInst::ICMP_ULT, FoundLHS: LoopCounter,
11599 FoundRHS: LatchBECount))
11600 return true;
11601 }
11602
11603 // Check conditions due to any @llvm.assume intrinsics.
11604 for (auto &AssumeVH : AC.assumptions()) {
11605 if (!AssumeVH)
11606 continue;
11607 auto *CI = cast<CallInst>(Val&: AssumeVH);
11608 if (!DT.dominates(Def: CI, User: Latch->getTerminator()))
11609 continue;
11610
11611 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: CI->getArgOperand(i: 0), Inverse: false))
11612 return true;
11613 }
11614
11615 if (isImpliedViaGuard(BB: Latch, Pred, LHS, RHS))
11616 return true;
11617
11618 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()];
11619 DTN != HeaderDTN; DTN = DTN->getIDom()) {
11620 assert(DTN && "should reach the loop header before reaching the root!");
11621
11622 BasicBlock *BB = DTN->getBlock();
11623 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
11624 return true;
11625
11626 BasicBlock *PBB = BB->getSinglePredecessor();
11627 if (!PBB)
11628 continue;
11629
11630 BranchInst *ContinuePredicate = dyn_cast<BranchInst>(Val: PBB->getTerminator());
11631 if (!ContinuePredicate || !ContinuePredicate->isConditional())
11632 continue;
11633
11634 Value *Condition = ContinuePredicate->getCondition();
11635
11636 // If we have an edge `E` within the loop body that dominates the only
11637 // latch, the condition guarding `E` also guards the backedge. This
11638 // reasoning works only for loops with a single latch.
11639
11640 BasicBlockEdge DominatingEdge(PBB, BB);
11641 if (DominatingEdge.isSingleEdge()) {
11642 // We're constructively (and conservatively) enumerating edges within the
11643 // loop body that dominate the latch. The dominator tree better agree
11644 // with us on this:
11645 assert(DT.dominates(DominatingEdge, Latch) && "should be!");
11646
11647 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition,
11648 Inverse: BB != ContinuePredicate->getSuccessor(i: 0)))
11649 return true;
11650 }
11651 }
11652
11653 return false;
11654}
11655
11656bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
11657 CmpPredicate Pred,
11658 const SCEV *LHS,
11659 const SCEV *RHS) {
11660 // Do not bother proving facts for unreachable code.
11661 if (!DT.isReachableFromEntry(A: BB))
11662 return true;
11663 if (VerifyIR)
11664 assert(!verifyFunction(*BB->getParent(), &dbgs()) &&
11665 "This cannot be done on broken IR!");
11666
11667 // If we cannot prove strict comparison (e.g. a > b), maybe we can prove
11668 // the facts (a >= b && a != b) separately. A typical situation is when the
11669 // non-strict comparison is known from ranges and non-equality is known from
11670 // dominating predicates. If we are proving strict comparison, we always try
11671 // to prove non-equality and non-strict comparison separately.
11672 CmpPredicate NonStrictPredicate = ICmpInst::getNonStrictCmpPredicate(Pred);
11673 const bool ProvingStrictComparison =
11674 Pred != NonStrictPredicate.dropSameSign();
11675 bool ProvedNonStrictComparison = false;
11676 bool ProvedNonEquality = false;
11677
11678 auto SplitAndProve = [&](std::function<bool(CmpPredicate)> Fn) -> bool {
11679 if (!ProvedNonStrictComparison)
11680 ProvedNonStrictComparison = Fn(NonStrictPredicate);
11681 if (!ProvedNonEquality)
11682 ProvedNonEquality = Fn(ICmpInst::ICMP_NE);
11683 if (ProvedNonStrictComparison && ProvedNonEquality)
11684 return true;
11685 return false;
11686 };
11687
11688 if (ProvingStrictComparison) {
11689 auto ProofFn = [&](CmpPredicate P) {
11690 return isKnownViaNonRecursiveReasoning(Pred: P, LHS, RHS);
11691 };
11692 if (SplitAndProve(ProofFn))
11693 return true;
11694 }
11695
11696 // Try to prove (Pred, LHS, RHS) using isImpliedCond.
11697 auto ProveViaCond = [&](const Value *Condition, bool Inverse) {
11698 const Instruction *CtxI = &BB->front();
11699 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI))
11700 return true;
11701 if (ProvingStrictComparison) {
11702 auto ProofFn = [&](CmpPredicate P) {
11703 return isImpliedCond(Pred: P, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI);
11704 };
11705 if (SplitAndProve(ProofFn))
11706 return true;
11707 }
11708 return false;
11709 };
11710
11711 // Starting at the block's predecessor, climb up the predecessor chain, as long
11712 // as there are predecessors that can be found that have unique successors
11713 // leading to the original block.
11714 const Loop *ContainingLoop = LI.getLoopFor(BB);
11715 const BasicBlock *PredBB;
11716 if (ContainingLoop && ContainingLoop->getHeader() == BB)
11717 PredBB = ContainingLoop->getLoopPredecessor();
11718 else
11719 PredBB = BB->getSinglePredecessor();
11720 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB);
11721 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
11722 const BranchInst *BlockEntryPredicate =
11723 dyn_cast<BranchInst>(Val: Pair.first->getTerminator());
11724 if (!BlockEntryPredicate || BlockEntryPredicate->isUnconditional())
11725 continue;
11726
11727 if (ProveViaCond(BlockEntryPredicate->getCondition(),
11728 BlockEntryPredicate->getSuccessor(i: 0) != Pair.second))
11729 return true;
11730 }
11731
11732 // Check conditions due to any @llvm.assume intrinsics.
11733 for (auto &AssumeVH : AC.assumptions()) {
11734 if (!AssumeVH)
11735 continue;
11736 auto *CI = cast<CallInst>(Val&: AssumeVH);
11737 if (!DT.dominates(Def: CI, BB))
11738 continue;
11739
11740 if (ProveViaCond(CI->getArgOperand(i: 0), false))
11741 return true;
11742 }
11743
11744 // Check conditions due to any @llvm.experimental.guard intrinsics.
11745 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
11746 M: F.getParent(), id: Intrinsic::experimental_guard);
11747 if (GuardDecl)
11748 for (const auto *GU : GuardDecl->users())
11749 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
11750 if (Guard->getFunction() == BB->getParent() && DT.dominates(Def: Guard, BB))
11751 if (ProveViaCond(Guard->getArgOperand(i: 0), false))
11752 return true;
11753 return false;
11754}
11755
11756bool ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred,
11757 const SCEV *LHS,
11758 const SCEV *RHS) {
11759 // Interpret a null as meaning no loop, where there is obviously no guard
11760 // (interprocedural conditions notwithstanding).
11761 if (!L)
11762 return false;
11763
11764 // Both LHS and RHS must be available at loop entry.
11765 assert(isAvailableAtLoopEntry(LHS, L) &&
11766 "LHS is not available at Loop Entry");
11767 assert(isAvailableAtLoopEntry(RHS, L) &&
11768 "RHS is not available at Loop Entry");
11769
11770 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11771 return true;
11772
11773 return isBasicBlockEntryGuardedByCond(BB: L->getHeader(), Pred, LHS, RHS);
11774}
11775
11776bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
11777 const SCEV *RHS,
11778 const Value *FoundCondValue, bool Inverse,
11779 const Instruction *CtxI) {
11780 // False conditions implies anything. Do not bother analyzing it further.
11781 if (FoundCondValue ==
11782 ConstantInt::getBool(Context&: FoundCondValue->getContext(), V: Inverse))
11783 return true;
11784
11785 if (!PendingLoopPredicates.insert(Ptr: FoundCondValue).second)
11786 return false;
11787
11788 auto ClearOnExit =
11789 make_scope_exit(F: [&]() { PendingLoopPredicates.erase(Ptr: FoundCondValue); });
11790
11791 // Recursively handle And and Or conditions.
11792 const Value *Op0, *Op1;
11793 if (match(V: FoundCondValue, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
11794 if (!Inverse)
11795 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
11796 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
11797 } else if (match(V: FoundCondValue, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
11798 if (Inverse)
11799 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
11800 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
11801 }
11802
11803 const ICmpInst *ICI = dyn_cast<ICmpInst>(Val: FoundCondValue);
11804 if (!ICI) return false;
11805
11806 // Now that we found a conditional branch that dominates the loop or controls
11807 // the loop latch. Check to see if it is the comparison we are looking for.
11808 CmpPredicate FoundPred;
11809 if (Inverse)
11810 FoundPred = ICI->getInverseCmpPredicate();
11811 else
11812 FoundPred = ICI->getCmpPredicate();
11813
11814 const SCEV *FoundLHS = getSCEV(V: ICI->getOperand(i_nocapture: 0));
11815 const SCEV *FoundRHS = getSCEV(V: ICI->getOperand(i_nocapture: 1));
11816
11817 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, Context: CtxI);
11818}
11819
11820bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
11821 const SCEV *RHS, CmpPredicate FoundPred,
11822 const SCEV *FoundLHS, const SCEV *FoundRHS,
11823 const Instruction *CtxI) {
11824 // Balance the types.
11825 if (getTypeSizeInBits(Ty: LHS->getType()) <
11826 getTypeSizeInBits(Ty: FoundLHS->getType())) {
11827 // For unsigned and equality predicates, try to prove that both found
11828 // operands fit into narrow unsigned range. If so, try to prove facts in
11829 // narrow types.
11830 if (!CmpInst::isSigned(predicate: FoundPred) && !FoundLHS->getType()->isPointerTy() &&
11831 !FoundRHS->getType()->isPointerTy()) {
11832 auto *NarrowType = LHS->getType();
11833 auto *WideType = FoundLHS->getType();
11834 auto BitWidth = getTypeSizeInBits(Ty: NarrowType);
11835 const SCEV *MaxValue = getZeroExtendExpr(
11836 Op: getConstant(Val: APInt::getMaxValue(numBits: BitWidth)), Ty: WideType);
11837 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundLHS,
11838 RHS: MaxValue) &&
11839 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundRHS,
11840 RHS: MaxValue)) {
11841 const SCEV *TruncFoundLHS = getTruncateExpr(Op: FoundLHS, Ty: NarrowType);
11842 const SCEV *TruncFoundRHS = getTruncateExpr(Op: FoundRHS, Ty: NarrowType);
11843 // We cannot preserve samesign after truncation.
11844 if (isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred: FoundPred.dropSameSign(),
11845 FoundLHS: TruncFoundLHS, FoundRHS: TruncFoundRHS, CtxI))
11846 return true;
11847 }
11848 }
11849
11850 if (LHS->getType()->isPointerTy() || RHS->getType()->isPointerTy())
11851 return false;
11852 if (CmpInst::isSigned(predicate: Pred)) {
11853 LHS = getSignExtendExpr(Op: LHS, Ty: FoundLHS->getType());
11854 RHS = getSignExtendExpr(Op: RHS, Ty: FoundLHS->getType());
11855 } else {
11856 LHS = getZeroExtendExpr(Op: LHS, Ty: FoundLHS->getType());
11857 RHS = getZeroExtendExpr(Op: RHS, Ty: FoundLHS->getType());
11858 }
11859 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
11860 getTypeSizeInBits(Ty: FoundLHS->getType())) {
11861 if (FoundLHS->getType()->isPointerTy() || FoundRHS->getType()->isPointerTy())
11862 return false;
11863 if (CmpInst::isSigned(predicate: FoundPred)) {
11864 FoundLHS = getSignExtendExpr(Op: FoundLHS, Ty: LHS->getType());
11865 FoundRHS = getSignExtendExpr(Op: FoundRHS, Ty: LHS->getType());
11866 } else {
11867 FoundLHS = getZeroExtendExpr(Op: FoundLHS, Ty: LHS->getType());
11868 FoundRHS = getZeroExtendExpr(Op: FoundRHS, Ty: LHS->getType());
11869 }
11870 }
11871 return isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, FoundLHS,
11872 FoundRHS, CtxI);
11873}
11874
11875bool ScalarEvolution::isImpliedCondBalancedTypes(
11876 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, CmpPredicate FoundPred,
11877 const SCEV *FoundLHS, const SCEV *FoundRHS, const Instruction *CtxI) {
11878 assert(getTypeSizeInBits(LHS->getType()) ==
11879 getTypeSizeInBits(FoundLHS->getType()) &&
11880 "Types should be balanced!");
11881 // Canonicalize the query to match the way instcombine will have
11882 // canonicalized the comparison.
11883 if (SimplifyICmpOperands(Pred, LHS, RHS))
11884 if (LHS == RHS)
11885 return CmpInst::isTrueWhenEqual(predicate: Pred);
11886 if (SimplifyICmpOperands(Pred&: FoundPred, LHS&: FoundLHS, RHS&: FoundRHS))
11887 if (FoundLHS == FoundRHS)
11888 return CmpInst::isFalseWhenEqual(predicate: FoundPred);
11889
11890 // Check to see if we can make the LHS or RHS match.
11891 if (LHS == FoundRHS || RHS == FoundLHS) {
11892 if (isa<SCEVConstant>(Val: RHS)) {
11893 std::swap(a&: FoundLHS, b&: FoundRHS);
11894 FoundPred = ICmpInst::getSwappedCmpPredicate(Pred: FoundPred);
11895 } else {
11896 std::swap(a&: LHS, b&: RHS);
11897 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11898 }
11899 }
11900
11901 // Check whether the found predicate is the same as the desired predicate.
11902 if (auto P = CmpPredicate::getMatching(A: FoundPred, B: Pred))
11903 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
11904
11905 // Check whether swapping the found predicate makes it the same as the
11906 // desired predicate.
11907 if (auto P = CmpPredicate::getMatching(
11908 A: ICmpInst::getSwappedCmpPredicate(Pred: FoundPred), B: Pred)) {
11909 // We can write the implication
11910 // 0. LHS Pred RHS <- FoundLHS SwapPred FoundRHS
11911 // using one of the following ways:
11912 // 1. LHS Pred RHS <- FoundRHS Pred FoundLHS
11913 // 2. RHS SwapPred LHS <- FoundLHS SwapPred FoundRHS
11914 // 3. LHS Pred RHS <- ~FoundLHS Pred ~FoundRHS
11915 // 4. ~LHS SwapPred ~RHS <- FoundLHS SwapPred FoundRHS
11916 // Forms 1. and 2. require swapping the operands of one condition. Don't
11917 // do this if it would break canonical constant/addrec ordering.
11918 if (!isa<SCEVConstant>(Val: RHS) && !isa<SCEVAddRecExpr>(Val: LHS))
11919 return isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P), LHS: RHS,
11920 RHS: LHS, FoundLHS, FoundRHS, Context: CtxI);
11921 if (!isa<SCEVConstant>(Val: FoundRHS) && !isa<SCEVAddRecExpr>(Val: FoundLHS))
11922 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: FoundRHS, FoundRHS: FoundLHS, Context: CtxI);
11923
11924 // There's no clear preference between forms 3. and 4., try both. Avoid
11925 // forming getNotSCEV of pointer values as the resulting subtract is
11926 // not legal.
11927 if (!LHS->getType()->isPointerTy() && !RHS->getType()->isPointerTy() &&
11928 isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P),
11929 LHS: getNotSCEV(V: LHS), RHS: getNotSCEV(V: RHS), FoundLHS,
11930 FoundRHS, Context: CtxI))
11931 return true;
11932
11933 if (!FoundLHS->getType()->isPointerTy() &&
11934 !FoundRHS->getType()->isPointerTy() &&
11935 isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: getNotSCEV(V: FoundLHS),
11936 FoundRHS: getNotSCEV(V: FoundRHS), Context: CtxI))
11937 return true;
11938
11939 return false;
11940 }
11941
11942 auto IsSignFlippedPredicate = [](CmpInst::Predicate P1,
11943 CmpInst::Predicate P2) {
11944 assert(P1 != P2 && "Handled earlier!");
11945 return CmpInst::isRelational(P: P2) &&
11946 P1 == ICmpInst::getFlippedSignednessPredicate(Pred: P2);
11947 };
11948 if (IsSignFlippedPredicate(Pred, FoundPred)) {
11949 // Unsigned comparison is the same as signed comparison when both the
11950 // operands are non-negative or negative.
11951 if ((isKnownNonNegative(S: FoundLHS) && isKnownNonNegative(S: FoundRHS)) ||
11952 (isKnownNegative(S: FoundLHS) && isKnownNegative(S: FoundRHS)))
11953 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
11954 // Create local copies that we can freely swap and canonicalize our
11955 // conditions to "le/lt".
11956 CmpPredicate CanonicalPred = Pred, CanonicalFoundPred = FoundPred;
11957 const SCEV *CanonicalLHS = LHS, *CanonicalRHS = RHS,
11958 *CanonicalFoundLHS = FoundLHS, *CanonicalFoundRHS = FoundRHS;
11959 if (ICmpInst::isGT(P: CanonicalPred) || ICmpInst::isGE(P: CanonicalPred)) {
11960 CanonicalPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalPred);
11961 CanonicalFoundPred = ICmpInst::getSwappedCmpPredicate(Pred: CanonicalFoundPred);
11962 std::swap(a&: CanonicalLHS, b&: CanonicalRHS);
11963 std::swap(a&: CanonicalFoundLHS, b&: CanonicalFoundRHS);
11964 }
11965 assert((ICmpInst::isLT(CanonicalPred) || ICmpInst::isLE(CanonicalPred)) &&
11966 "Must be!");
11967 assert((ICmpInst::isLT(CanonicalFoundPred) ||
11968 ICmpInst::isLE(CanonicalFoundPred)) &&
11969 "Must be!");
11970 if (ICmpInst::isSigned(predicate: CanonicalPred) && isKnownNonNegative(S: CanonicalRHS))
11971 // Use implication:
11972 // x <u y && y >=s 0 --> x <s y.
11973 // If we can prove the left part, the right part is also proven.
11974 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
11975 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
11976 FoundRHS: CanonicalFoundRHS);
11977 if (ICmpInst::isUnsigned(predicate: CanonicalPred) && isKnownNegative(S: CanonicalRHS))
11978 // Use implication:
11979 // x <s y && y <s 0 --> x <u y.
11980 // If we can prove the left part, the right part is also proven.
11981 return isImpliedCondOperands(Pred: CanonicalFoundPred, LHS: CanonicalLHS,
11982 RHS: CanonicalRHS, FoundLHS: CanonicalFoundLHS,
11983 FoundRHS: CanonicalFoundRHS);
11984 }
11985
11986 // Check if we can make progress by sharpening ranges.
11987 if (FoundPred == ICmpInst::ICMP_NE &&
11988 (isa<SCEVConstant>(Val: FoundLHS) || isa<SCEVConstant>(Val: FoundRHS))) {
11989
11990 const SCEVConstant *C = nullptr;
11991 const SCEV *V = nullptr;
11992
11993 if (isa<SCEVConstant>(Val: FoundLHS)) {
11994 C = cast<SCEVConstant>(Val: FoundLHS);
11995 V = FoundRHS;
11996 } else {
11997 C = cast<SCEVConstant>(Val: FoundRHS);
11998 V = FoundLHS;
11999 }
12000
12001 // The guarding predicate tells us that C != V. If the known range
12002 // of V is [C, t), we can sharpen the range to [C + 1, t). The
12003 // range we consider has to correspond to same signedness as the
12004 // predicate we're interested in folding.
12005
12006 APInt Min = ICmpInst::isSigned(predicate: Pred) ?
12007 getSignedRangeMin(S: V) : getUnsignedRangeMin(S: V);
12008
12009 if (Min == C->getAPInt()) {
12010 // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
12011 // This is true even if (Min + 1) wraps around -- in case of
12012 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
12013
12014 APInt SharperMin = Min + 1;
12015
12016 switch (Pred) {
12017 case ICmpInst::ICMP_SGE:
12018 case ICmpInst::ICMP_UGE:
12019 // We know V `Pred` SharperMin. If this implies LHS `Pred`
12020 // RHS, we're done.
12021 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin),
12022 Context: CtxI))
12023 return true;
12024 [[fallthrough]];
12025
12026 case ICmpInst::ICMP_SGT:
12027 case ICmpInst::ICMP_UGT:
12028 // We know from the range information that (V `Pred` Min ||
12029 // V == Min). We know from the guarding condition that !(V
12030 // == Min). This gives us
12031 //
12032 // V `Pred` Min || V == Min && !(V == Min)
12033 // => V `Pred` Min
12034 //
12035 // If V `Pred` Min implies LHS `Pred` RHS, we're done.
12036
12037 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12038 return true;
12039 break;
12040
12041 // `LHS < RHS` and `LHS <= RHS` are handled in the same way as `RHS > LHS` and `RHS >= LHS` respectively.
12042 case ICmpInst::ICMP_SLE:
12043 case ICmpInst::ICMP_ULE:
12044 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12045 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin), Context: CtxI))
12046 return true;
12047 [[fallthrough]];
12048
12049 case ICmpInst::ICMP_SLT:
12050 case ICmpInst::ICMP_ULT:
12051 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12052 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12053 return true;
12054 break;
12055
12056 default:
12057 // No change
12058 break;
12059 }
12060 }
12061 }
12062
12063 // Check whether the actual condition is beyond sufficient.
12064 if (FoundPred == ICmpInst::ICMP_EQ)
12065 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
12066 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12067 return true;
12068 if (Pred == ICmpInst::ICMP_NE)
12069 if (!ICmpInst::isTrueWhenEqual(predicate: FoundPred))
12070 if (isImpliedCondOperands(Pred: FoundPred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12071 return true;
12072
12073 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS))
12074 return true;
12075
12076 // Otherwise assume the worst.
12077 return false;
12078}
12079
12080bool ScalarEvolution::splitBinaryAdd(const SCEV *Expr,
12081 const SCEV *&L, const SCEV *&R,
12082 SCEV::NoWrapFlags &Flags) {
12083 const auto *AE = dyn_cast<SCEVAddExpr>(Val: Expr);
12084 if (!AE || AE->getNumOperands() != 2)
12085 return false;
12086
12087 L = AE->getOperand(i: 0);
12088 R = AE->getOperand(i: 1);
12089 Flags = AE->getNoWrapFlags();
12090 return true;
12091}
12092
12093std::optional<APInt>
12094ScalarEvolution::computeConstantDifference(const SCEV *More, const SCEV *Less) {
12095 // We avoid subtracting expressions here because this function is usually
12096 // fairly deep in the call stack (i.e. is called many times).
12097
12098 unsigned BW = getTypeSizeInBits(Ty: More->getType());
12099 APInt Diff(BW, 0);
12100 APInt DiffMul(BW, 1);
12101 // Try various simplifications to reduce the difference to a constant. Limit
12102 // the number of allowed simplifications to keep compile-time low.
12103 for (unsigned I = 0; I < 8; ++I) {
12104 if (More == Less)
12105 return Diff;
12106
12107 // Reduce addrecs with identical steps to their start value.
12108 if (isa<SCEVAddRecExpr>(Val: Less) && isa<SCEVAddRecExpr>(Val: More)) {
12109 const auto *LAR = cast<SCEVAddRecExpr>(Val: Less);
12110 const auto *MAR = cast<SCEVAddRecExpr>(Val: More);
12111
12112 if (LAR->getLoop() != MAR->getLoop())
12113 return std::nullopt;
12114
12115 // We look at affine expressions only; not for correctness but to keep
12116 // getStepRecurrence cheap.
12117 if (!LAR->isAffine() || !MAR->isAffine())
12118 return std::nullopt;
12119
12120 if (LAR->getStepRecurrence(SE&: *this) != MAR->getStepRecurrence(SE&: *this))
12121 return std::nullopt;
12122
12123 Less = LAR->getStart();
12124 More = MAR->getStart();
12125 continue;
12126 }
12127
12128 // Try to match a common constant multiply.
12129 auto MatchConstMul =
12130 [](const SCEV *S) -> std::optional<std::pair<const SCEV *, APInt>> {
12131 auto *M = dyn_cast<SCEVMulExpr>(Val: S);
12132 if (!M || M->getNumOperands() != 2 ||
12133 !isa<SCEVConstant>(Val: M->getOperand(i: 0)))
12134 return std::nullopt;
12135 return {
12136 {M->getOperand(i: 1), cast<SCEVConstant>(Val: M->getOperand(i: 0))->getAPInt()}};
12137 };
12138 if (auto MatchedMore = MatchConstMul(More)) {
12139 if (auto MatchedLess = MatchConstMul(Less)) {
12140 if (MatchedMore->second == MatchedLess->second) {
12141 More = MatchedMore->first;
12142 Less = MatchedLess->first;
12143 DiffMul *= MatchedMore->second;
12144 continue;
12145 }
12146 }
12147 }
12148
12149 // Try to cancel out common factors in two add expressions.
12150 SmallDenseMap<const SCEV *, int, 8> Multiplicity;
12151 auto Add = [&](const SCEV *S, int Mul) {
12152 if (auto *C = dyn_cast<SCEVConstant>(Val: S)) {
12153 if (Mul == 1) {
12154 Diff += C->getAPInt() * DiffMul;
12155 } else {
12156 assert(Mul == -1);
12157 Diff -= C->getAPInt() * DiffMul;
12158 }
12159 } else
12160 Multiplicity[S] += Mul;
12161 };
12162 auto Decompose = [&](const SCEV *S, int Mul) {
12163 if (isa<SCEVAddExpr>(Val: S)) {
12164 for (const SCEV *Op : S->operands())
12165 Add(Op, Mul);
12166 } else
12167 Add(S, Mul);
12168 };
12169 Decompose(More, 1);
12170 Decompose(Less, -1);
12171
12172 // Check whether all the non-constants cancel out, or reduce to new
12173 // More/Less values.
12174 const SCEV *NewMore = nullptr, *NewLess = nullptr;
12175 for (const auto &[S, Mul] : Multiplicity) {
12176 if (Mul == 0)
12177 continue;
12178 if (Mul == 1) {
12179 if (NewMore)
12180 return std::nullopt;
12181 NewMore = S;
12182 } else if (Mul == -1) {
12183 if (NewLess)
12184 return std::nullopt;
12185 NewLess = S;
12186 } else
12187 return std::nullopt;
12188 }
12189
12190 // Values stayed the same, no point in trying further.
12191 if (NewMore == More || NewLess == Less)
12192 return std::nullopt;
12193
12194 More = NewMore;
12195 Less = NewLess;
12196
12197 // Reduced to constant.
12198 if (!More && !Less)
12199 return Diff;
12200
12201 // Left with variable on only one side, bail out.
12202 if (!More || !Less)
12203 return std::nullopt;
12204 }
12205
12206 // Did not reduce to constant.
12207 return std::nullopt;
12208}
12209
12210bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart(
12211 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12212 const SCEV *FoundRHS, const Instruction *CtxI) {
12213 // Try to recognize the following pattern:
12214 //
12215 // FoundRHS = ...
12216 // ...
12217 // loop:
12218 // FoundLHS = {Start,+,W}
12219 // context_bb: // Basic block from the same loop
12220 // known(Pred, FoundLHS, FoundRHS)
12221 //
12222 // If some predicate is known in the context of a loop, it is also known on
12223 // each iteration of this loop, including the first iteration. Therefore, in
12224 // this case, `FoundLHS Pred FoundRHS` implies `Start Pred FoundRHS`. Try to
12225 // prove the original pred using this fact.
12226 if (!CtxI)
12227 return false;
12228 const BasicBlock *ContextBB = CtxI->getParent();
12229 // Make sure AR varies in the context block.
12230 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS)) {
12231 const Loop *L = AR->getLoop();
12232 // Make sure that context belongs to the loop and executes on 1st iteration
12233 // (if it ever executes at all).
12234 if (!L->contains(BB: ContextBB) || !DT.dominates(A: ContextBB, B: L->getLoopLatch()))
12235 return false;
12236 if (!isAvailableAtLoopEntry(S: FoundRHS, L: AR->getLoop()))
12237 return false;
12238 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: AR->getStart(), FoundRHS);
12239 }
12240
12241 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundRHS)) {
12242 const Loop *L = AR->getLoop();
12243 // Make sure that context belongs to the loop and executes on 1st iteration
12244 // (if it ever executes at all).
12245 if (!L->contains(BB: ContextBB) || !DT.dominates(A: ContextBB, B: L->getLoopLatch()))
12246 return false;
12247 if (!isAvailableAtLoopEntry(S: FoundLHS, L: AR->getLoop()))
12248 return false;
12249 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS: AR->getStart());
12250 }
12251
12252 return false;
12253}
12254
12255bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(CmpPredicate Pred,
12256 const SCEV *LHS,
12257 const SCEV *RHS,
12258 const SCEV *FoundLHS,
12259 const SCEV *FoundRHS) {
12260 if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT)
12261 return false;
12262
12263 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12264 if (!AddRecLHS)
12265 return false;
12266
12267 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12268 if (!AddRecFoundLHS)
12269 return false;
12270
12271 // We'd like to let SCEV reason about control dependencies, so we constrain
12272 // both the inequalities to be about add recurrences on the same loop. This
12273 // way we can use isLoopEntryGuardedByCond later.
12274
12275 const Loop *L = AddRecFoundLHS->getLoop();
12276 if (L != AddRecLHS->getLoop())
12277 return false;
12278
12279 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1)
12280 //
12281 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C)
12282 // ... (2)
12283 //
12284 // Informal proof for (2), assuming (1) [*]:
12285 //
12286 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**]
12287 //
12288 // Then
12289 //
12290 // FoundLHS s< FoundRHS s< INT_MIN - C
12291 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ]
12292 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ]
12293 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s<
12294 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ]
12295 // <=> FoundLHS + C s< FoundRHS + C
12296 //
12297 // [*]: (1) can be proved by ruling out overflow.
12298 //
12299 // [**]: This can be proved by analyzing all the four possibilities:
12300 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and
12301 // (A s>= 0, B s>= 0).
12302 //
12303 // Note:
12304 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C"
12305 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS
12306 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS
12307 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is
12308 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS +
12309 // C)".
12310
12311 std::optional<APInt> LDiff = computeConstantDifference(More: LHS, Less: FoundLHS);
12312 if (!LDiff)
12313 return false;
12314 std::optional<APInt> RDiff = computeConstantDifference(More: RHS, Less: FoundRHS);
12315 if (!RDiff || *LDiff != *RDiff)
12316 return false;
12317
12318 if (LDiff->isMinValue())
12319 return true;
12320
12321 APInt FoundRHSLimit;
12322
12323 if (Pred == CmpInst::ICMP_ULT) {
12324 FoundRHSLimit = -(*RDiff);
12325 } else {
12326 assert(Pred == CmpInst::ICMP_SLT && "Checked above!");
12327 FoundRHSLimit = APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: RHS->getType())) - *RDiff;
12328 }
12329
12330 // Try to prove (1) or (2), as needed.
12331 return isAvailableAtLoopEntry(S: FoundRHS, L) &&
12332 isLoopEntryGuardedByCond(L, Pred, LHS: FoundRHS,
12333 RHS: getConstant(Val: FoundRHSLimit));
12334}
12335
12336bool ScalarEvolution::isImpliedViaMerge(CmpPredicate Pred, const SCEV *LHS,
12337 const SCEV *RHS, const SCEV *FoundLHS,
12338 const SCEV *FoundRHS, unsigned Depth) {
12339 const PHINode *LPhi = nullptr, *RPhi = nullptr;
12340
12341 auto ClearOnExit = make_scope_exit(F: [&]() {
12342 if (LPhi) {
12343 bool Erased = PendingMerges.erase(Ptr: LPhi);
12344 assert(Erased && "Failed to erase LPhi!");
12345 (void)Erased;
12346 }
12347 if (RPhi) {
12348 bool Erased = PendingMerges.erase(Ptr: RPhi);
12349 assert(Erased && "Failed to erase RPhi!");
12350 (void)Erased;
12351 }
12352 });
12353
12354 // Find respective Phis and check that they are not being pending.
12355 if (const SCEVUnknown *LU = dyn_cast<SCEVUnknown>(Val: LHS))
12356 if (auto *Phi = dyn_cast<PHINode>(Val: LU->getValue())) {
12357 if (!PendingMerges.insert(Ptr: Phi).second)
12358 return false;
12359 LPhi = Phi;
12360 }
12361 if (const SCEVUnknown *RU = dyn_cast<SCEVUnknown>(Val: RHS))
12362 if (auto *Phi = dyn_cast<PHINode>(Val: RU->getValue())) {
12363 // If we detect a loop of Phi nodes being processed by this method, for
12364 // example:
12365 //
12366 // %a = phi i32 [ %some1, %preheader ], [ %b, %latch ]
12367 // %b = phi i32 [ %some2, %preheader ], [ %a, %latch ]
12368 //
12369 // we don't want to deal with a case that complex, so return conservative
12370 // answer false.
12371 if (!PendingMerges.insert(Ptr: Phi).second)
12372 return false;
12373 RPhi = Phi;
12374 }
12375
12376 // If none of LHS, RHS is a Phi, nothing to do here.
12377 if (!LPhi && !RPhi)
12378 return false;
12379
12380 // If there is a SCEVUnknown Phi we are interested in, make it left.
12381 if (!LPhi) {
12382 std::swap(a&: LHS, b&: RHS);
12383 std::swap(a&: FoundLHS, b&: FoundRHS);
12384 std::swap(a&: LPhi, b&: RPhi);
12385 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12386 }
12387
12388 assert(LPhi && "LPhi should definitely be a SCEVUnknown Phi!");
12389 const BasicBlock *LBB = LPhi->getParent();
12390 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(Val: RHS);
12391
12392 auto ProvedEasily = [&](const SCEV *S1, const SCEV *S2) {
12393 return isKnownViaNonRecursiveReasoning(Pred, LHS: S1, RHS: S2) ||
12394 isImpliedCondOperandsViaRanges(Pred, LHS: S1, RHS: S2, FoundPred: Pred, FoundLHS, FoundRHS) ||
12395 isImpliedViaOperations(Pred, LHS: S1, RHS: S2, FoundLHS, FoundRHS, Depth);
12396 };
12397
12398 if (RPhi && RPhi->getParent() == LBB) {
12399 // Case one: RHS is also a SCEVUnknown Phi from the same basic block.
12400 // If we compare two Phis from the same block, and for each entry block
12401 // the predicate is true for incoming values from this block, then the
12402 // predicate is also true for the Phis.
12403 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12404 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12405 const SCEV *R = getSCEV(V: RPhi->getIncomingValueForBlock(BB: IncBB));
12406 if (!ProvedEasily(L, R))
12407 return false;
12408 }
12409 } else if (RAR && RAR->getLoop()->getHeader() == LBB) {
12410 // Case two: RHS is also a Phi from the same basic block, and it is an
12411 // AddRec. It means that there is a loop which has both AddRec and Unknown
12412 // PHIs, for it we can compare incoming values of AddRec from above the loop
12413 // and latch with their respective incoming values of LPhi.
12414 // TODO: Generalize to handle loops with many inputs in a header.
12415 if (LPhi->getNumIncomingValues() != 2) return false;
12416
12417 auto *RLoop = RAR->getLoop();
12418 auto *Predecessor = RLoop->getLoopPredecessor();
12419 assert(Predecessor && "Loop with AddRec with no predecessor?");
12420 const SCEV *L1 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Predecessor));
12421 if (!ProvedEasily(L1, RAR->getStart()))
12422 return false;
12423 auto *Latch = RLoop->getLoopLatch();
12424 assert(Latch && "Loop with AddRec with no latch?");
12425 const SCEV *L2 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Latch));
12426 if (!ProvedEasily(L2, RAR->getPostIncExpr(SE&: *this)))
12427 return false;
12428 } else {
12429 // In all other cases go over inputs of LHS and compare each of them to RHS,
12430 // the predicate is true for (LHS, RHS) if it is true for all such pairs.
12431 // At this point RHS is either a non-Phi, or it is a Phi from some block
12432 // different from LBB.
12433 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12434 // Check that RHS is available in this block.
12435 if (!dominates(S: RHS, BB: IncBB))
12436 return false;
12437 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12438 // Make sure L does not refer to a value from a potentially previous
12439 // iteration of a loop.
12440 if (!properlyDominates(S: L, BB: LBB))
12441 return false;
12442 // Addrecs are considered to properly dominate their loop, so are missed
12443 // by the previous check. Discard any values that have computable
12444 // evolution in this loop.
12445 if (auto *Loop = LI.getLoopFor(BB: LBB))
12446 if (hasComputableLoopEvolution(S: L, L: Loop))
12447 return false;
12448 if (!ProvedEasily(L, RHS))
12449 return false;
12450 }
12451 }
12452 return true;
12453}
12454
12455bool ScalarEvolution::isImpliedCondOperandsViaShift(CmpPredicate Pred,
12456 const SCEV *LHS,
12457 const SCEV *RHS,
12458 const SCEV *FoundLHS,
12459 const SCEV *FoundRHS) {
12460 // We want to imply LHS < RHS from LHS < (RHS >> shiftvalue). First, make
12461 // sure that we are dealing with same LHS.
12462 if (RHS == FoundRHS) {
12463 std::swap(a&: LHS, b&: RHS);
12464 std::swap(a&: FoundLHS, b&: FoundRHS);
12465 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12466 }
12467 if (LHS != FoundLHS)
12468 return false;
12469
12470 auto *SUFoundRHS = dyn_cast<SCEVUnknown>(Val: FoundRHS);
12471 if (!SUFoundRHS)
12472 return false;
12473
12474 Value *Shiftee, *ShiftValue;
12475
12476 using namespace PatternMatch;
12477 if (match(V: SUFoundRHS->getValue(),
12478 P: m_LShr(L: m_Value(V&: Shiftee), R: m_Value(V&: ShiftValue)))) {
12479 auto *ShifteeS = getSCEV(V: Shiftee);
12480 // Prove one of the following:
12481 // LHS <u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <u RHS
12482 // LHS <=u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <=u RHS
12483 // LHS <s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12484 // ---> LHS <s RHS
12485 // LHS <=s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12486 // ---> LHS <=s RHS
12487 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
12488 return isKnownPredicate(Pred: ICmpInst::ICMP_ULE, LHS: ShifteeS, RHS);
12489 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
12490 if (isKnownNonNegative(S: ShifteeS))
12491 return isKnownPredicate(Pred: ICmpInst::ICMP_SLE, LHS: ShifteeS, RHS);
12492 }
12493
12494 return false;
12495}
12496
12497bool ScalarEvolution::isImpliedCondOperands(CmpPredicate Pred, const SCEV *LHS,
12498 const SCEV *RHS,
12499 const SCEV *FoundLHS,
12500 const SCEV *FoundRHS,
12501 const Instruction *CtxI) {
12502 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred: Pred, FoundLHS, FoundRHS))
12503 return true;
12504
12505 if (isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS, FoundRHS))
12506 return true;
12507
12508 if (isImpliedCondOperandsViaShift(Pred, LHS, RHS, FoundLHS, FoundRHS))
12509 return true;
12510
12511 if (isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
12512 CtxI))
12513 return true;
12514
12515 return isImpliedCondOperandsHelper(Pred, LHS, RHS,
12516 FoundLHS, FoundRHS);
12517}
12518
12519/// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
12520template <typename MinMaxExprType>
12521static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr,
12522 const SCEV *Candidate) {
12523 const MinMaxExprType *MinMaxExpr = dyn_cast<MinMaxExprType>(MaybeMinMaxExpr);
12524 if (!MinMaxExpr)
12525 return false;
12526
12527 return is_contained(MinMaxExpr->operands(), Candidate);
12528}
12529
12530static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE,
12531 CmpPredicate Pred, const SCEV *LHS,
12532 const SCEV *RHS) {
12533 // If both sides are affine addrecs for the same loop, with equal
12534 // steps, and we know the recurrences don't wrap, then we only
12535 // need to check the predicate on the starting values.
12536
12537 if (!ICmpInst::isRelational(P: Pred))
12538 return false;
12539
12540 const SCEV *LStart, *RStart, *Step;
12541 const Loop *L;
12542 if (!match(S: LHS,
12543 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: LStart), Op1: m_SCEV(V&: Step), L: m_Loop(L))) ||
12544 !match(S: RHS, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: RStart), Op1: m_scev_Specific(S: Step),
12545 L: m_SpecificLoop(L))))
12546 return false;
12547 const SCEVAddRecExpr *LAR = cast<SCEVAddRecExpr>(Val: LHS);
12548 const SCEVAddRecExpr *RAR = cast<SCEVAddRecExpr>(Val: RHS);
12549 SCEV::NoWrapFlags NW = ICmpInst::isSigned(predicate: Pred) ?
12550 SCEV::FlagNSW : SCEV::FlagNUW;
12551 if (!LAR->getNoWrapFlags(Mask: NW) || !RAR->getNoWrapFlags(Mask: NW))
12552 return false;
12553
12554 return SE.isKnownPredicate(Pred, LHS: LStart, RHS: RStart);
12555}
12556
12557/// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
12558/// expression?
12559static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, CmpPredicate Pred,
12560 const SCEV *LHS, const SCEV *RHS) {
12561 switch (Pred) {
12562 default:
12563 return false;
12564
12565 case ICmpInst::ICMP_SGE:
12566 std::swap(a&: LHS, b&: RHS);
12567 [[fallthrough]];
12568 case ICmpInst::ICMP_SLE:
12569 return
12570 // min(A, ...) <= A
12571 IsMinMaxConsistingOf<SCEVSMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12572 // A <= max(A, ...)
12573 IsMinMaxConsistingOf<SCEVSMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12574
12575 case ICmpInst::ICMP_UGE:
12576 std::swap(a&: LHS, b&: RHS);
12577 [[fallthrough]];
12578 case ICmpInst::ICMP_ULE:
12579 return
12580 // min(A, ...) <= A
12581 // FIXME: what about umin_seq?
12582 IsMinMaxConsistingOf<SCEVUMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12583 // A <= max(A, ...)
12584 IsMinMaxConsistingOf<SCEVUMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12585 }
12586
12587 llvm_unreachable("covered switch fell through?!");
12588}
12589
12590bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred, const SCEV *LHS,
12591 const SCEV *RHS,
12592 const SCEV *FoundLHS,
12593 const SCEV *FoundRHS,
12594 unsigned Depth) {
12595 assert(getTypeSizeInBits(LHS->getType()) ==
12596 getTypeSizeInBits(RHS->getType()) &&
12597 "LHS and RHS have different sizes?");
12598 assert(getTypeSizeInBits(FoundLHS->getType()) ==
12599 getTypeSizeInBits(FoundRHS->getType()) &&
12600 "FoundLHS and FoundRHS have different sizes?");
12601 // We want to avoid hurting the compile time with analysis of too big trees.
12602 if (Depth > MaxSCEVOperationsImplicationDepth)
12603 return false;
12604
12605 // We only want to work with GT comparison so far.
12606 if (ICmpInst::isLT(P: Pred)) {
12607 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12608 std::swap(a&: LHS, b&: RHS);
12609 std::swap(a&: FoundLHS, b&: FoundRHS);
12610 }
12611
12612 CmpInst::Predicate P = Pred.getPreferredSignedPredicate();
12613
12614 // For unsigned, try to reduce it to corresponding signed comparison.
12615 if (P == ICmpInst::ICMP_UGT)
12616 // We can replace unsigned predicate with its signed counterpart if all
12617 // involved values are non-negative.
12618 // TODO: We could have better support for unsigned.
12619 if (isKnownNonNegative(S: FoundLHS) && isKnownNonNegative(S: FoundRHS)) {
12620 // Knowing that both FoundLHS and FoundRHS are non-negative, and knowing
12621 // FoundLHS >u FoundRHS, we also know that FoundLHS >s FoundRHS. Let us
12622 // use this fact to prove that LHS and RHS are non-negative.
12623 const SCEV *MinusOne = getMinusOne(Ty: LHS->getType());
12624 if (isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS, RHS: MinusOne, FoundLHS,
12625 FoundRHS) &&
12626 isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS: RHS, RHS: MinusOne, FoundLHS,
12627 FoundRHS))
12628 P = ICmpInst::ICMP_SGT;
12629 }
12630
12631 if (P != ICmpInst::ICMP_SGT)
12632 return false;
12633
12634 auto GetOpFromSExt = [&](const SCEV *S) {
12635 if (auto *Ext = dyn_cast<SCEVSignExtendExpr>(Val: S))
12636 return Ext->getOperand();
12637 // TODO: If S is a SCEVConstant then you can cheaply "strip" the sext off
12638 // the constant in some cases.
12639 return S;
12640 };
12641
12642 // Acquire values from extensions.
12643 auto *OrigLHS = LHS;
12644 auto *OrigFoundLHS = FoundLHS;
12645 LHS = GetOpFromSExt(LHS);
12646 FoundLHS = GetOpFromSExt(FoundLHS);
12647
12648 // Is the SGT predicate can be proved trivially or using the found context.
12649 auto IsSGTViaContext = [&](const SCEV *S1, const SCEV *S2) {
12650 return isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2) ||
12651 isImpliedViaOperations(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2, FoundLHS: OrigFoundLHS,
12652 FoundRHS, Depth: Depth + 1);
12653 };
12654
12655 if (auto *LHSAddExpr = dyn_cast<SCEVAddExpr>(Val: LHS)) {
12656 // We want to avoid creation of any new non-constant SCEV. Since we are
12657 // going to compare the operands to RHS, we should be certain that we don't
12658 // need any size extensions for this. So let's decline all cases when the
12659 // sizes of types of LHS and RHS do not match.
12660 // TODO: Maybe try to get RHS from sext to catch more cases?
12661 if (getTypeSizeInBits(Ty: LHS->getType()) != getTypeSizeInBits(Ty: RHS->getType()))
12662 return false;
12663
12664 // Should not overflow.
12665 if (!LHSAddExpr->hasNoSignedWrap())
12666 return false;
12667
12668 auto *LL = LHSAddExpr->getOperand(i: 0);
12669 auto *LR = LHSAddExpr->getOperand(i: 1);
12670 auto *MinusOne = getMinusOne(Ty: RHS->getType());
12671
12672 // Checks that S1 >= 0 && S2 > RHS, trivially or using the found context.
12673 auto IsSumGreaterThanRHS = [&](const SCEV *S1, const SCEV *S2) {
12674 return IsSGTViaContext(S1, MinusOne) && IsSGTViaContext(S2, RHS);
12675 };
12676 // Try to prove the following rule:
12677 // (LHS = LL + LR) && (LL >= 0) && (LR > RHS) => (LHS > RHS).
12678 // (LHS = LL + LR) && (LR >= 0) && (LL > RHS) => (LHS > RHS).
12679 if (IsSumGreaterThanRHS(LL, LR) || IsSumGreaterThanRHS(LR, LL))
12680 return true;
12681 } else if (auto *LHSUnknownExpr = dyn_cast<SCEVUnknown>(Val: LHS)) {
12682 Value *LL, *LR;
12683 // FIXME: Once we have SDiv implemented, we can get rid of this matching.
12684
12685 using namespace llvm::PatternMatch;
12686
12687 if (match(V: LHSUnknownExpr->getValue(), P: m_SDiv(L: m_Value(V&: LL), R: m_Value(V&: LR)))) {
12688 // Rules for division.
12689 // We are going to perform some comparisons with Denominator and its
12690 // derivative expressions. In general case, creating a SCEV for it may
12691 // lead to a complex analysis of the entire graph, and in particular it
12692 // can request trip count recalculation for the same loop. This would
12693 // cache as SCEVCouldNotCompute to avoid the infinite recursion. To avoid
12694 // this, we only want to create SCEVs that are constants in this section.
12695 // So we bail if Denominator is not a constant.
12696 if (!isa<ConstantInt>(Val: LR))
12697 return false;
12698
12699 auto *Denominator = cast<SCEVConstant>(Val: getSCEV(V: LR));
12700
12701 // We want to make sure that LHS = FoundLHS / Denominator. If it is so,
12702 // then a SCEV for the numerator already exists and matches with FoundLHS.
12703 auto *Numerator = getExistingSCEV(V: LL);
12704 if (!Numerator || Numerator->getType() != FoundLHS->getType())
12705 return false;
12706
12707 // Make sure that the numerator matches with FoundLHS and the denominator
12708 // is positive.
12709 if (!HasSameValue(A: Numerator, B: FoundLHS) || !isKnownPositive(S: Denominator))
12710 return false;
12711
12712 auto *DTy = Denominator->getType();
12713 auto *FRHSTy = FoundRHS->getType();
12714 if (DTy->isPointerTy() != FRHSTy->isPointerTy())
12715 // One of types is a pointer and another one is not. We cannot extend
12716 // them properly to a wider type, so let us just reject this case.
12717 // TODO: Usage of getEffectiveSCEVType for DTy, FRHSTy etc should help
12718 // to avoid this check.
12719 return false;
12720
12721 // Given that:
12722 // FoundLHS > FoundRHS, LHS = FoundLHS / Denominator, Denominator > 0.
12723 auto *WTy = getWiderType(T1: DTy, T2: FRHSTy);
12724 auto *DenominatorExt = getNoopOrSignExtend(V: Denominator, Ty: WTy);
12725 auto *FoundRHSExt = getNoopOrSignExtend(V: FoundRHS, Ty: WTy);
12726
12727 // Try to prove the following rule:
12728 // (FoundRHS > Denominator - 2) && (RHS <= 0) => (LHS > RHS).
12729 // For example, given that FoundLHS > 2. It means that FoundLHS is at
12730 // least 3. If we divide it by Denominator < 4, we will have at least 1.
12731 auto *DenomMinusTwo = getMinusSCEV(LHS: DenominatorExt, RHS: getConstant(Ty: WTy, V: 2));
12732 if (isKnownNonPositive(S: RHS) &&
12733 IsSGTViaContext(FoundRHSExt, DenomMinusTwo))
12734 return true;
12735
12736 // Try to prove the following rule:
12737 // (FoundRHS > -1 - Denominator) && (RHS < 0) => (LHS > RHS).
12738 // For example, given that FoundLHS > -3. Then FoundLHS is at least -2.
12739 // If we divide it by Denominator > 2, then:
12740 // 1. If FoundLHS is negative, then the result is 0.
12741 // 2. If FoundLHS is non-negative, then the result is non-negative.
12742 // Anyways, the result is non-negative.
12743 auto *MinusOne = getMinusOne(Ty: WTy);
12744 auto *NegDenomMinusOne = getMinusSCEV(LHS: MinusOne, RHS: DenominatorExt);
12745 if (isKnownNegative(S: RHS) &&
12746 IsSGTViaContext(FoundRHSExt, NegDenomMinusOne))
12747 return true;
12748 }
12749 }
12750
12751 // If our expression contained SCEVUnknown Phis, and we split it down and now
12752 // need to prove something for them, try to prove the predicate for every
12753 // possible incoming values of those Phis.
12754 if (isImpliedViaMerge(Pred, LHS: OrigLHS, RHS, FoundLHS: OrigFoundLHS, FoundRHS, Depth: Depth + 1))
12755 return true;
12756
12757 return false;
12758}
12759
12760static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS,
12761 const SCEV *RHS) {
12762 // zext x u<= sext x, sext x s<= zext x
12763 const SCEV *Op;
12764 switch (Pred) {
12765 case ICmpInst::ICMP_SGE:
12766 std::swap(a&: LHS, b&: RHS);
12767 [[fallthrough]];
12768 case ICmpInst::ICMP_SLE: {
12769 // If operand >=s 0 then ZExt == SExt. If operand <s 0 then SExt <s ZExt.
12770 return match(S: LHS, P: m_scev_SExt(Op0: m_SCEV(V&: Op))) &&
12771 match(S: RHS, P: m_scev_ZExt(Op0: m_scev_Specific(S: Op)));
12772 }
12773 case ICmpInst::ICMP_UGE:
12774 std::swap(a&: LHS, b&: RHS);
12775 [[fallthrough]];
12776 case ICmpInst::ICMP_ULE: {
12777 // If operand >=u 0 then ZExt == SExt. If operand <u 0 then ZExt <u SExt.
12778 return match(S: LHS, P: m_scev_ZExt(Op0: m_SCEV(V&: Op))) &&
12779 match(S: RHS, P: m_scev_SExt(Op0: m_scev_Specific(S: Op)));
12780 }
12781 default:
12782 return false;
12783 };
12784 llvm_unreachable("unhandled case");
12785}
12786
12787bool ScalarEvolution::isKnownViaNonRecursiveReasoning(CmpPredicate Pred,
12788 const SCEV *LHS,
12789 const SCEV *RHS) {
12790 return isKnownPredicateExtendIdiom(Pred, LHS, RHS) ||
12791 isKnownPredicateViaConstantRanges(Pred, LHS, RHS) ||
12792 IsKnownPredicateViaMinOrMax(SE&: *this, Pred, LHS, RHS) ||
12793 IsKnownPredicateViaAddRecStart(SE&: *this, Pred, LHS, RHS) ||
12794 isKnownPredicateViaNoOverflow(Pred, LHS, RHS);
12795}
12796
12797bool ScalarEvolution::isImpliedCondOperandsHelper(CmpPredicate Pred,
12798 const SCEV *LHS,
12799 const SCEV *RHS,
12800 const SCEV *FoundLHS,
12801 const SCEV *FoundRHS) {
12802 switch (Pred) {
12803 default:
12804 llvm_unreachable("Unexpected CmpPredicate value!");
12805 case ICmpInst::ICMP_EQ:
12806 case ICmpInst::ICMP_NE:
12807 if (HasSameValue(A: LHS, B: FoundLHS) && HasSameValue(A: RHS, B: FoundRHS))
12808 return true;
12809 break;
12810 case ICmpInst::ICMP_SLT:
12811 case ICmpInst::ICMP_SLE:
12812 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS, RHS: FoundLHS) &&
12813 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS: RHS, RHS: FoundRHS))
12814 return true;
12815 break;
12816 case ICmpInst::ICMP_SGT:
12817 case ICmpInst::ICMP_SGE:
12818 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGE, LHS, RHS: FoundLHS) &&
12819 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SLE, LHS: RHS, RHS: FoundRHS))
12820 return true;
12821 break;
12822 case ICmpInst::ICMP_ULT:
12823 case ICmpInst::ICMP_ULE:
12824 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS, RHS: FoundLHS) &&
12825 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS: RHS, RHS: FoundRHS))
12826 return true;
12827 break;
12828 case ICmpInst::ICMP_UGT:
12829 case ICmpInst::ICMP_UGE:
12830 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_UGE, LHS, RHS: FoundLHS) &&
12831 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: RHS, RHS: FoundRHS))
12832 return true;
12833 break;
12834 }
12835
12836 // Maybe it can be proved via operations?
12837 if (isImpliedViaOperations(Pred, LHS, RHS, FoundLHS, FoundRHS))
12838 return true;
12839
12840 return false;
12841}
12842
12843bool ScalarEvolution::isImpliedCondOperandsViaRanges(
12844 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, CmpPredicate FoundPred,
12845 const SCEV *FoundLHS, const SCEV *FoundRHS) {
12846 if (!isa<SCEVConstant>(Val: RHS) || !isa<SCEVConstant>(Val: FoundRHS))
12847 // The restriction on `FoundRHS` be lifted easily -- it exists only to
12848 // reduce the compile time impact of this optimization.
12849 return false;
12850
12851 std::optional<APInt> Addend = computeConstantDifference(More: LHS, Less: FoundLHS);
12852 if (!Addend)
12853 return false;
12854
12855 const APInt &ConstFoundRHS = cast<SCEVConstant>(Val: FoundRHS)->getAPInt();
12856
12857 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the
12858 // antecedent "`FoundLHS` `FoundPred` `FoundRHS`".
12859 ConstantRange FoundLHSRange =
12860 ConstantRange::makeExactICmpRegion(Pred: FoundPred, Other: ConstFoundRHS);
12861
12862 // Since `LHS` is `FoundLHS` + `Addend`, we can compute a range for `LHS`:
12863 ConstantRange LHSRange = FoundLHSRange.add(Other: ConstantRange(*Addend));
12864
12865 // We can also compute the range of values for `LHS` that satisfy the
12866 // consequent, "`LHS` `Pred` `RHS`":
12867 const APInt &ConstRHS = cast<SCEVConstant>(Val: RHS)->getAPInt();
12868 // The antecedent implies the consequent if every value of `LHS` that
12869 // satisfies the antecedent also satisfies the consequent.
12870 return LHSRange.icmp(Pred, Other: ConstRHS);
12871}
12872
12873bool ScalarEvolution::canIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride,
12874 bool IsSigned) {
12875 assert(isKnownPositive(Stride) && "Positive stride expected!");
12876
12877 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
12878 const SCEV *One = getOne(Ty: Stride->getType());
12879
12880 if (IsSigned) {
12881 APInt MaxRHS = getSignedRangeMax(S: RHS);
12882 APInt MaxValue = APInt::getSignedMaxValue(numBits: BitWidth);
12883 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
12884
12885 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow!
12886 return (std::move(MaxValue) - MaxStrideMinusOne).slt(RHS: MaxRHS);
12887 }
12888
12889 APInt MaxRHS = getUnsignedRangeMax(S: RHS);
12890 APInt MaxValue = APInt::getMaxValue(numBits: BitWidth);
12891 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
12892
12893 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow!
12894 return (std::move(MaxValue) - MaxStrideMinusOne).ult(RHS: MaxRHS);
12895}
12896
12897bool ScalarEvolution::canIVOverflowOnGT(const SCEV *RHS, const SCEV *Stride,
12898 bool IsSigned) {
12899
12900 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
12901 const SCEV *One = getOne(Ty: Stride->getType());
12902
12903 if (IsSigned) {
12904 APInt MinRHS = getSignedRangeMin(S: RHS);
12905 APInt MinValue = APInt::getSignedMinValue(numBits: BitWidth);
12906 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
12907
12908 // SMinRHS - SMaxStrideMinusOne < SMinValue => overflow!
12909 return (std::move(MinValue) + MaxStrideMinusOne).sgt(RHS: MinRHS);
12910 }
12911
12912 APInt MinRHS = getUnsignedRangeMin(S: RHS);
12913 APInt MinValue = APInt::getMinValue(numBits: BitWidth);
12914 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
12915
12916 // UMinRHS - UMaxStrideMinusOne < UMinValue => overflow!
12917 return (std::move(MinValue) + MaxStrideMinusOne).ugt(RHS: MinRHS);
12918}
12919
12920const SCEV *ScalarEvolution::getUDivCeilSCEV(const SCEV *N, const SCEV *D) {
12921 // umin(N, 1) + floor((N - umin(N, 1)) / D)
12922 // This is equivalent to "1 + floor((N - 1) / D)" for N != 0. The umin
12923 // expression fixes the case of N=0.
12924 const SCEV *MinNOne = getUMinExpr(LHS: N, RHS: getOne(Ty: N->getType()));
12925 const SCEV *NMinusOne = getMinusSCEV(LHS: N, RHS: MinNOne);
12926 return getAddExpr(LHS: MinNOne, RHS: getUDivExpr(LHS: NMinusOne, RHS: D));
12927}
12928
12929const SCEV *ScalarEvolution::computeMaxBECountForLT(const SCEV *Start,
12930 const SCEV *Stride,
12931 const SCEV *End,
12932 unsigned BitWidth,
12933 bool IsSigned) {
12934 // The logic in this function assumes we can represent a positive stride.
12935 // If we can't, the backedge-taken count must be zero.
12936 if (IsSigned && BitWidth == 1)
12937 return getZero(Ty: Stride->getType());
12938
12939 // This code below only been closely audited for negative strides in the
12940 // unsigned comparison case, it may be correct for signed comparison, but
12941 // that needs to be established.
12942 if (IsSigned && isKnownNegative(S: Stride))
12943 return getCouldNotCompute();
12944
12945 // Calculate the maximum backedge count based on the range of values
12946 // permitted by Start, End, and Stride.
12947 APInt MinStart =
12948 IsSigned ? getSignedRangeMin(S: Start) : getUnsignedRangeMin(S: Start);
12949
12950 APInt MinStride =
12951 IsSigned ? getSignedRangeMin(S: Stride) : getUnsignedRangeMin(S: Stride);
12952
12953 // We assume either the stride is positive, or the backedge-taken count
12954 // is zero. So force StrideForMaxBECount to be at least one.
12955 APInt One(BitWidth, 1);
12956 APInt StrideForMaxBECount = IsSigned ? APIntOps::smax(A: One, B: MinStride)
12957 : APIntOps::umax(A: One, B: MinStride);
12958
12959 APInt MaxValue = IsSigned ? APInt::getSignedMaxValue(numBits: BitWidth)
12960 : APInt::getMaxValue(numBits: BitWidth);
12961 APInt Limit = MaxValue - (StrideForMaxBECount - 1);
12962
12963 // Although End can be a MAX expression we estimate MaxEnd considering only
12964 // the case End = RHS of the loop termination condition. This is safe because
12965 // in the other case (End - Start) is zero, leading to a zero maximum backedge
12966 // taken count.
12967 APInt MaxEnd = IsSigned ? APIntOps::smin(A: getSignedRangeMax(S: End), B: Limit)
12968 : APIntOps::umin(A: getUnsignedRangeMax(S: End), B: Limit);
12969
12970 // MaxBECount = ceil((max(MaxEnd, MinStart) - MinStart) / Stride)
12971 MaxEnd = IsSigned ? APIntOps::smax(A: MaxEnd, B: MinStart)
12972 : APIntOps::umax(A: MaxEnd, B: MinStart);
12973
12974 return getUDivCeilSCEV(N: getConstant(Val: MaxEnd - MinStart) /* Delta */,
12975 D: getConstant(Val: StrideForMaxBECount) /* Step */);
12976}
12977
12978ScalarEvolution::ExitLimit
12979ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
12980 const Loop *L, bool IsSigned,
12981 bool ControlsOnlyExit, bool AllowPredicates) {
12982 SmallVector<const SCEVPredicate *> Predicates;
12983
12984 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12985 bool PredicatedIV = false;
12986 if (!IV) {
12987 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: LHS)) {
12988 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: ZExt->getOperand());
12989 if (AR && AR->getLoop() == L && AR->isAffine()) {
12990 auto canProveNUW = [&]() {
12991 // We can use the comparison to infer no-wrap flags only if it fully
12992 // controls the loop exit.
12993 if (!ControlsOnlyExit)
12994 return false;
12995
12996 if (!isLoopInvariant(S: RHS, L))
12997 return false;
12998
12999 if (!isKnownNonZero(S: AR->getStepRecurrence(SE&: *this)))
13000 // We need the sequence defined by AR to strictly increase in the
13001 // unsigned integer domain for the logic below to hold.
13002 return false;
13003
13004 const unsigned InnerBitWidth = getTypeSizeInBits(Ty: AR->getType());
13005 const unsigned OuterBitWidth = getTypeSizeInBits(Ty: RHS->getType());
13006 // If RHS <=u Limit, then there must exist a value V in the sequence
13007 // defined by AR (e.g. {Start,+,Step}) such that V >u RHS, and
13008 // V <=u UINT_MAX. Thus, we must exit the loop before unsigned
13009 // overflow occurs. This limit also implies that a signed comparison
13010 // (in the wide bitwidth) is equivalent to an unsigned comparison as
13011 // the high bits on both sides must be zero.
13012 APInt StrideMax = getUnsignedRangeMax(S: AR->getStepRecurrence(SE&: *this));
13013 APInt Limit = APInt::getMaxValue(numBits: InnerBitWidth) - (StrideMax - 1);
13014 Limit = Limit.zext(width: OuterBitWidth);
13015 return getUnsignedRangeMax(S: applyLoopGuards(Expr: RHS, L)).ule(RHS: Limit);
13016 };
13017 auto Flags = AR->getNoWrapFlags();
13018 if (!hasFlags(Flags, TestFlags: SCEV::FlagNUW) && canProveNUW())
13019 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
13020
13021 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
13022 if (AR->hasNoUnsignedWrap()) {
13023 // Emulate what getZeroExtendExpr would have done during construction
13024 // if we'd been able to infer the fact just above at that time.
13025 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
13026 Type *Ty = ZExt->getType();
13027 auto *S = getAddRecExpr(
13028 Start: getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: 0),
13029 Step: getZeroExtendExpr(Op: Step, Ty, Depth: 0), L, Flags: AR->getNoWrapFlags());
13030 IV = dyn_cast<SCEVAddRecExpr>(Val: S);
13031 }
13032 }
13033 }
13034 }
13035
13036
13037 if (!IV && AllowPredicates) {
13038 // Try to make this an AddRec using runtime tests, in the first X
13039 // iterations of this loop, where X is the SCEV expression found by the
13040 // algorithm below.
13041 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13042 PredicatedIV = true;
13043 }
13044
13045 // Avoid weird loops
13046 if (!IV || IV->getLoop() != L || !IV->isAffine())
13047 return getCouldNotCompute();
13048
13049 // A precondition of this method is that the condition being analyzed
13050 // reaches an exiting branch which dominates the latch. Given that, we can
13051 // assume that an increment which violates the nowrap specification and
13052 // produces poison must cause undefined behavior when the resulting poison
13053 // value is branched upon and thus we can conclude that the backedge is
13054 // taken no more often than would be required to produce that poison value.
13055 // Note that a well defined loop can exit on the iteration which violates
13056 // the nowrap specification if there is another exit (either explicit or
13057 // implicit/exceptional) which causes the loop to execute before the
13058 // exiting instruction we're analyzing would trigger UB.
13059 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13060 bool NoWrap = ControlsOnlyExit && IV->getNoWrapFlags(Mask: WrapType);
13061 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13062
13063 const SCEV *Stride = IV->getStepRecurrence(SE&: *this);
13064
13065 bool PositiveStride = isKnownPositive(S: Stride);
13066
13067 // Avoid negative or zero stride values.
13068 if (!PositiveStride) {
13069 // We can compute the correct backedge taken count for loops with unknown
13070 // strides if we can prove that the loop is not an infinite loop with side
13071 // effects. Here's the loop structure we are trying to handle -
13072 //
13073 // i = start
13074 // do {
13075 // A[i] = i;
13076 // i += s;
13077 // } while (i < end);
13078 //
13079 // The backedge taken count for such loops is evaluated as -
13080 // (max(end, start + stride) - start - 1) /u stride
13081 //
13082 // The additional preconditions that we need to check to prove correctness
13083 // of the above formula is as follows -
13084 //
13085 // a) IV is either nuw or nsw depending upon signedness (indicated by the
13086 // NoWrap flag).
13087 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13088 // no side effects within the loop)
13089 // c) loop has a single static exit (with no abnormal exits)
13090 //
13091 // Precondition a) implies that if the stride is negative, this is a single
13092 // trip loop. The backedge taken count formula reduces to zero in this case.
13093 //
13094 // Precondition b) and c) combine to imply that if rhs is invariant in L,
13095 // then a zero stride means the backedge can't be taken without executing
13096 // undefined behavior.
13097 //
13098 // The positive stride case is the same as isKnownPositive(Stride) returning
13099 // true (original behavior of the function).
13100 //
13101 if (PredicatedIV || !NoWrap || !loopIsFiniteByAssumption(L) ||
13102 !loopHasNoAbnormalExits(L))
13103 return getCouldNotCompute();
13104
13105 if (!isKnownNonZero(S: Stride)) {
13106 // If we have a step of zero, and RHS isn't invariant in L, we don't know
13107 // if it might eventually be greater than start and if so, on which
13108 // iteration. We can't even produce a useful upper bound.
13109 if (!isLoopInvariant(S: RHS, L))
13110 return getCouldNotCompute();
13111
13112 // We allow a potentially zero stride, but we need to divide by stride
13113 // below. Since the loop can't be infinite and this check must control
13114 // the sole exit, we can infer the exit must be taken on the first
13115 // iteration (e.g. backedge count = 0) if the stride is zero. Given that,
13116 // we know the numerator in the divides below must be zero, so we can
13117 // pick an arbitrary non-zero value for the denominator (e.g. stride)
13118 // and produce the right result.
13119 // FIXME: Handle the case where Stride is poison?
13120 auto wouldZeroStrideBeUB = [&]() {
13121 // Proof by contradiction. Suppose the stride were zero. If we can
13122 // prove that the backedge *is* taken on the first iteration, then since
13123 // we know this condition controls the sole exit, we must have an
13124 // infinite loop. We can't have a (well defined) infinite loop per
13125 // check just above.
13126 // Note: The (Start - Stride) term is used to get the start' term from
13127 // (start' + stride,+,stride). Remember that we only care about the
13128 // result of this expression when stride == 0 at runtime.
13129 auto *StartIfZero = getMinusSCEV(LHS: IV->getStart(), RHS: Stride);
13130 return isLoopEntryGuardedByCond(L, Pred: Cond, LHS: StartIfZero, RHS);
13131 };
13132 if (!wouldZeroStrideBeUB()) {
13133 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13134 }
13135 }
13136 } else if (!NoWrap) {
13137 // Avoid proven overflow cases: this will ensure that the backedge taken
13138 // count will not generate any unsigned overflow.
13139 if (canIVOverflowOnLT(RHS, Stride, IsSigned))
13140 return getCouldNotCompute();
13141 }
13142
13143 // On all paths just preceeding, we established the following invariant:
13144 // IV can be assumed not to overflow up to and including the exiting
13145 // iteration. We proved this in one of two ways:
13146 // 1) We can show overflow doesn't occur before the exiting iteration
13147 // 1a) canIVOverflowOnLT, and b) step of one
13148 // 2) We can show that if overflow occurs, the loop must execute UB
13149 // before any possible exit.
13150 // Note that we have not yet proved RHS invariant (in general).
13151
13152 const SCEV *Start = IV->getStart();
13153
13154 // Preserve pointer-typed Start/RHS to pass to isLoopEntryGuardedByCond.
13155 // If we convert to integers, isLoopEntryGuardedByCond will miss some cases.
13156 // Use integer-typed versions for actual computation; we can't subtract
13157 // pointers in general.
13158 const SCEV *OrigStart = Start;
13159 const SCEV *OrigRHS = RHS;
13160 if (Start->getType()->isPointerTy()) {
13161 Start = getLosslessPtrToIntExpr(Op: Start);
13162 if (isa<SCEVCouldNotCompute>(Val: Start))
13163 return Start;
13164 }
13165 if (RHS->getType()->isPointerTy()) {
13166 RHS = getLosslessPtrToIntExpr(Op: RHS);
13167 if (isa<SCEVCouldNotCompute>(Val: RHS))
13168 return RHS;
13169 }
13170
13171 const SCEV *End = nullptr, *BECount = nullptr,
13172 *BECountIfBackedgeTaken = nullptr;
13173 if (!isLoopInvariant(S: RHS, L)) {
13174 const auto *RHSAddRec = dyn_cast<SCEVAddRecExpr>(Val: RHS);
13175 if (PositiveStride && RHSAddRec != nullptr && RHSAddRec->getLoop() == L &&
13176 RHSAddRec->getNoWrapFlags()) {
13177 // The structure of loop we are trying to calculate backedge count of:
13178 //
13179 // left = left_start
13180 // right = right_start
13181 //
13182 // while(left < right){
13183 // ... do something here ...
13184 // left += s1; // stride of left is s1 (s1 > 0)
13185 // right += s2; // stride of right is s2 (s2 < 0)
13186 // }
13187 //
13188
13189 const SCEV *RHSStart = RHSAddRec->getStart();
13190 const SCEV *RHSStride = RHSAddRec->getStepRecurrence(SE&: *this);
13191
13192 // If Stride - RHSStride is positive and does not overflow, we can write
13193 // backedge count as ->
13194 // ceil((End - Start) /u (Stride - RHSStride))
13195 // Where, End = max(RHSStart, Start)
13196
13197 // Check if RHSStride < 0 and Stride - RHSStride will not overflow.
13198 if (isKnownNegative(S: RHSStride) &&
13199 willNotOverflow(BinOp: Instruction::Sub, /*Signed=*/true, LHS: Stride,
13200 RHS: RHSStride)) {
13201
13202 const SCEV *Denominator = getMinusSCEV(LHS: Stride, RHS: RHSStride);
13203 if (isKnownPositive(S: Denominator)) {
13204 End = IsSigned ? getSMaxExpr(LHS: RHSStart, RHS: Start)
13205 : getUMaxExpr(LHS: RHSStart, RHS: Start);
13206
13207 // We can do this because End >= Start, as End = max(RHSStart, Start)
13208 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13209
13210 BECount = getUDivCeilSCEV(N: Delta, D: Denominator);
13211 BECountIfBackedgeTaken =
13212 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHSStart, RHS: Start), D: Denominator);
13213 }
13214 }
13215 }
13216 if (BECount == nullptr) {
13217 // If we cannot calculate ExactBECount, we can calculate the MaxBECount,
13218 // given the start, stride and max value for the end bound of the
13219 // loop (RHS), and the fact that IV does not overflow (which is
13220 // checked above).
13221 const SCEV *MaxBECount = computeMaxBECountForLT(
13222 Start, Stride, End: RHS, BitWidth: getTypeSizeInBits(Ty: LHS->getType()), IsSigned);
13223 return ExitLimit(getCouldNotCompute() /* ExactNotTaken */, MaxBECount,
13224 MaxBECount, false /*MaxOrZero*/, Predicates);
13225 }
13226 } else {
13227 // We use the expression (max(End,Start)-Start)/Stride to describe the
13228 // backedge count, as if the backedge is taken at least once
13229 // max(End,Start) is End and so the result is as above, and if not
13230 // max(End,Start) is Start so we get a backedge count of zero.
13231 auto *OrigStartMinusStride = getMinusSCEV(LHS: OrigStart, RHS: Stride);
13232 assert(isAvailableAtLoopEntry(OrigStartMinusStride, L) && "Must be!");
13233 assert(isAvailableAtLoopEntry(OrigStart, L) && "Must be!");
13234 assert(isAvailableAtLoopEntry(OrigRHS, L) && "Must be!");
13235 // Can we prove (max(RHS,Start) > Start - Stride?
13236 if (isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigStartMinusStride, RHS: OrigStart) &&
13237 isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigStartMinusStride, RHS: OrigRHS)) {
13238 // In this case, we can use a refined formula for computing backedge
13239 // taken count. The general formula remains:
13240 // "End-Start /uceiling Stride" where "End = max(RHS,Start)"
13241 // We want to use the alternate formula:
13242 // "((End - 1) - (Start - Stride)) /u Stride"
13243 // Let's do a quick case analysis to show these are equivalent under
13244 // our precondition that max(RHS,Start) > Start - Stride.
13245 // * For RHS <= Start, the backedge-taken count must be zero.
13246 // "((End - 1) - (Start - Stride)) /u Stride" reduces to
13247 // "((Start - 1) - (Start - Stride)) /u Stride" which simplies to
13248 // "Stride - 1 /u Stride" which is indeed zero for all non-zero values
13249 // of Stride. For 0 stride, we've use umin(1,Stride) above,
13250 // reducing this to the stride of 1 case.
13251 // * For RHS >= Start, the backedge count must be "RHS-Start /uceil
13252 // Stride".
13253 // "((End - 1) - (Start - Stride)) /u Stride" reduces to
13254 // "((RHS - 1) - (Start - Stride)) /u Stride" reassociates to
13255 // "((RHS - (Start - Stride) - 1) /u Stride".
13256 // Our preconditions trivially imply no overflow in that form.
13257 const SCEV *MinusOne = getMinusOne(Ty: Stride->getType());
13258 const SCEV *Numerator =
13259 getMinusSCEV(LHS: getAddExpr(LHS: RHS, RHS: MinusOne), RHS: getMinusSCEV(LHS: Start, RHS: Stride));
13260 BECount = getUDivExpr(LHS: Numerator, RHS: Stride);
13261 }
13262
13263 if (!BECount) {
13264 auto canProveRHSGreaterThanEqualStart = [&]() {
13265 auto CondGE = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13266 const SCEV *GuardedRHS = applyLoopGuards(Expr: OrigRHS, L);
13267 const SCEV *GuardedStart = applyLoopGuards(Expr: OrigStart, L);
13268
13269 if (isLoopEntryGuardedByCond(L, Pred: CondGE, LHS: OrigRHS, RHS: OrigStart) ||
13270 isKnownPredicate(Pred: CondGE, LHS: GuardedRHS, RHS: GuardedStart))
13271 return true;
13272
13273 // (RHS > Start - 1) implies RHS >= Start.
13274 // * "RHS >= Start" is trivially equivalent to "RHS > Start - 1" if
13275 // "Start - 1" doesn't overflow.
13276 // * For signed comparison, if Start - 1 does overflow, it's equal
13277 // to INT_MAX, and "RHS >s INT_MAX" is trivially false.
13278 // * For unsigned comparison, if Start - 1 does overflow, it's equal
13279 // to UINT_MAX, and "RHS >u UINT_MAX" is trivially false.
13280 //
13281 // FIXME: Should isLoopEntryGuardedByCond do this for us?
13282 auto CondGT = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13283 auto *StartMinusOne =
13284 getAddExpr(LHS: OrigStart, RHS: getMinusOne(Ty: OrigStart->getType()));
13285 return isLoopEntryGuardedByCond(L, Pred: CondGT, LHS: OrigRHS, RHS: StartMinusOne);
13286 };
13287
13288 // If we know that RHS >= Start in the context of loop, then we know
13289 // that max(RHS, Start) = RHS at this point.
13290 if (canProveRHSGreaterThanEqualStart()) {
13291 End = RHS;
13292 } else {
13293 // If RHS < Start, the backedge will be taken zero times. So in
13294 // general, we can write the backedge-taken count as:
13295 //
13296 // RHS >= Start ? ceil(RHS - Start) / Stride : 0
13297 //
13298 // We convert it to the following to make it more convenient for SCEV:
13299 //
13300 // ceil(max(RHS, Start) - Start) / Stride
13301 End = IsSigned ? getSMaxExpr(LHS: RHS, RHS: Start) : getUMaxExpr(LHS: RHS, RHS: Start);
13302
13303 // See what would happen if we assume the backedge is taken. This is
13304 // used to compute MaxBECount.
13305 BECountIfBackedgeTaken =
13306 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHS, RHS: Start), D: Stride);
13307 }
13308
13309 // At this point, we know:
13310 //
13311 // 1. If IsSigned, Start <=s End; otherwise, Start <=u End
13312 // 2. The index variable doesn't overflow.
13313 //
13314 // Therefore, we know N exists such that
13315 // (Start + Stride * N) >= End, and computing "(Start + Stride * N)"
13316 // doesn't overflow.
13317 //
13318 // Using this information, try to prove whether the addition in
13319 // "(Start - End) + (Stride - 1)" has unsigned overflow.
13320 const SCEV *One = getOne(Ty: Stride->getType());
13321 bool MayAddOverflow = [&] {
13322 if (isKnownToBeAPowerOfTwo(S: Stride)) {
13323 // Suppose Stride is a power of two, and Start/End are unsigned
13324 // integers. Let UMAX be the largest representable unsigned
13325 // integer.
13326 //
13327 // By the preconditions of this function, we know
13328 // "(Start + Stride * N) >= End", and this doesn't overflow.
13329 // As a formula:
13330 //
13331 // End <= (Start + Stride * N) <= UMAX
13332 //
13333 // Subtracting Start from all the terms:
13334 //
13335 // End - Start <= Stride * N <= UMAX - Start
13336 //
13337 // Since Start is unsigned, UMAX - Start <= UMAX. Therefore:
13338 //
13339 // End - Start <= Stride * N <= UMAX
13340 //
13341 // Stride * N is a multiple of Stride. Therefore,
13342 //
13343 // End - Start <= Stride * N <= UMAX - (UMAX mod Stride)
13344 //
13345 // Since Stride is a power of two, UMAX + 1 is divisible by
13346 // Stride. Therefore, UMAX mod Stride == Stride - 1. So we can
13347 // write:
13348 //
13349 // End - Start <= Stride * N <= UMAX - Stride - 1
13350 //
13351 // Dropping the middle term:
13352 //
13353 // End - Start <= UMAX - Stride - 1
13354 //
13355 // Adding Stride - 1 to both sides:
13356 //
13357 // (End - Start) + (Stride - 1) <= UMAX
13358 //
13359 // In other words, the addition doesn't have unsigned overflow.
13360 //
13361 // A similar proof works if we treat Start/End as signed values.
13362 // Just rewrite steps before "End - Start <= Stride * N <= UMAX"
13363 // to use signed max instead of unsigned max. Note that we're
13364 // trying to prove a lack of unsigned overflow in either case.
13365 return false;
13366 }
13367 if (Start == Stride || Start == getMinusSCEV(LHS: Stride, RHS: One)) {
13368 // If Start is equal to Stride, (End - Start) + (Stride - 1) == End
13369 // - 1. If !IsSigned, 0 <u Stride == Start <=u End; so 0 <u End - 1
13370 // <u End. If IsSigned, 0 <s Stride == Start <=s End; so 0 <s End -
13371 // 1 <s End.
13372 //
13373 // If Start is equal to Stride - 1, (End - Start) + Stride - 1 ==
13374 // End.
13375 return false;
13376 }
13377 return true;
13378 }();
13379
13380 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13381 if (!MayAddOverflow) {
13382 // floor((D + (S - 1)) / S)
13383 // We prefer this formulation if it's legal because it's fewer
13384 // operations.
13385 BECount =
13386 getUDivExpr(LHS: getAddExpr(LHS: Delta, RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13387 } else {
13388 BECount = getUDivCeilSCEV(N: Delta, D: Stride);
13389 }
13390 }
13391 }
13392
13393 const SCEV *ConstantMaxBECount;
13394 bool MaxOrZero = false;
13395 if (isa<SCEVConstant>(Val: BECount)) {
13396 ConstantMaxBECount = BECount;
13397 } else if (BECountIfBackedgeTaken &&
13398 isa<SCEVConstant>(Val: BECountIfBackedgeTaken)) {
13399 // If we know exactly how many times the backedge will be taken if it's
13400 // taken at least once, then the backedge count will either be that or
13401 // zero.
13402 ConstantMaxBECount = BECountIfBackedgeTaken;
13403 MaxOrZero = true;
13404 } else {
13405 ConstantMaxBECount = computeMaxBECountForLT(
13406 Start, Stride, End: RHS, BitWidth: getTypeSizeInBits(Ty: LHS->getType()), IsSigned);
13407 }
13408
13409 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
13410 !isa<SCEVCouldNotCompute>(Val: BECount))
13411 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
13412
13413 const SCEV *SymbolicMaxBECount =
13414 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13415 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, MaxOrZero,
13416 Predicates);
13417}
13418
13419ScalarEvolution::ExitLimit ScalarEvolution::howManyGreaterThans(
13420 const SCEV *LHS, const SCEV *RHS, const Loop *L, bool IsSigned,
13421 bool ControlsOnlyExit, bool AllowPredicates) {
13422 SmallVector<const SCEVPredicate *> Predicates;
13423 // We handle only IV > Invariant
13424 if (!isLoopInvariant(S: RHS, L))
13425 return getCouldNotCompute();
13426
13427 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
13428 if (!IV && AllowPredicates)
13429 // Try to make this an AddRec using runtime tests, in the first X
13430 // iterations of this loop, where X is the SCEV expression found by the
13431 // algorithm below.
13432 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13433
13434 // Avoid weird loops
13435 if (!IV || IV->getLoop() != L || !IV->isAffine())
13436 return getCouldNotCompute();
13437
13438 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13439 bool NoWrap = ControlsOnlyExit && IV->getNoWrapFlags(Mask: WrapType);
13440 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13441
13442 const SCEV *Stride = getNegativeSCEV(V: IV->getStepRecurrence(SE&: *this));
13443
13444 // Avoid negative or zero stride values
13445 if (!isKnownPositive(S: Stride))
13446 return getCouldNotCompute();
13447
13448 // Avoid proven overflow cases: this will ensure that the backedge taken count
13449 // will not generate any unsigned overflow. Relaxed no-overflow conditions
13450 // exploit NoWrapFlags, allowing to optimize in presence of undefined
13451 // behaviors like the case of C language.
13452 if (!Stride->isOne() && !NoWrap)
13453 if (canIVOverflowOnGT(RHS, Stride, IsSigned))
13454 return getCouldNotCompute();
13455
13456 const SCEV *Start = IV->getStart();
13457 const SCEV *End = RHS;
13458 if (!isLoopEntryGuardedByCond(L, Pred: Cond, LHS: getAddExpr(LHS: Start, RHS: Stride), RHS)) {
13459 // If we know that Start >= RHS in the context of loop, then we know that
13460 // min(RHS, Start) = RHS at this point.
13461 if (isLoopEntryGuardedByCond(
13462 L, Pred: IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE, LHS: Start, RHS))
13463 End = RHS;
13464 else
13465 End = IsSigned ? getSMinExpr(LHS: RHS, RHS: Start) : getUMinExpr(LHS: RHS, RHS: Start);
13466 }
13467
13468 if (Start->getType()->isPointerTy()) {
13469 Start = getLosslessPtrToIntExpr(Op: Start);
13470 if (isa<SCEVCouldNotCompute>(Val: Start))
13471 return Start;
13472 }
13473 if (End->getType()->isPointerTy()) {
13474 End = getLosslessPtrToIntExpr(Op: End);
13475 if (isa<SCEVCouldNotCompute>(Val: End))
13476 return End;
13477 }
13478
13479 // Compute ((Start - End) + (Stride - 1)) / Stride.
13480 // FIXME: This can overflow. Holding off on fixing this for now;
13481 // howManyGreaterThans will hopefully be gone soon.
13482 const SCEV *One = getOne(Ty: Stride->getType());
13483 const SCEV *BECount = getUDivExpr(
13484 LHS: getAddExpr(LHS: getMinusSCEV(LHS: Start, RHS: End), RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13485
13486 APInt MaxStart = IsSigned ? getSignedRangeMax(S: Start)
13487 : getUnsignedRangeMax(S: Start);
13488
13489 APInt MinStride = IsSigned ? getSignedRangeMin(S: Stride)
13490 : getUnsignedRangeMin(S: Stride);
13491
13492 unsigned BitWidth = getTypeSizeInBits(Ty: LHS->getType());
13493 APInt Limit = IsSigned ? APInt::getSignedMinValue(numBits: BitWidth) + (MinStride - 1)
13494 : APInt::getMinValue(numBits: BitWidth) + (MinStride - 1);
13495
13496 // Although End can be a MIN expression we estimate MinEnd considering only
13497 // the case End = RHS. This is safe because in the other case (Start - End)
13498 // is zero, leading to a zero maximum backedge taken count.
13499 APInt MinEnd =
13500 IsSigned ? APIntOps::smax(A: getSignedRangeMin(S: RHS), B: Limit)
13501 : APIntOps::umax(A: getUnsignedRangeMin(S: RHS), B: Limit);
13502
13503 const SCEV *ConstantMaxBECount =
13504 isa<SCEVConstant>(Val: BECount)
13505 ? BECount
13506 : getUDivCeilSCEV(N: getConstant(Val: MaxStart - MinEnd),
13507 D: getConstant(Val: MinStride));
13508
13509 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount))
13510 ConstantMaxBECount = BECount;
13511 const SCEV *SymbolicMaxBECount =
13512 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13513
13514 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
13515 Predicates);
13516}
13517
13518const SCEV *SCEVAddRecExpr::getNumIterationsInRange(const ConstantRange &Range,
13519 ScalarEvolution &SE) const {
13520 if (Range.isFullSet()) // Infinite loop.
13521 return SE.getCouldNotCompute();
13522
13523 // If the start is a non-zero constant, shift the range to simplify things.
13524 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: getStart()))
13525 if (!SC->getValue()->isZero()) {
13526 SmallVector<const SCEV *, 4> Operands(operands());
13527 Operands[0] = SE.getZero(Ty: SC->getType());
13528 const SCEV *Shifted = SE.getAddRecExpr(Operands, L: getLoop(),
13529 Flags: getNoWrapFlags(Mask: FlagNW));
13530 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Val: Shifted))
13531 return ShiftedAddRec->getNumIterationsInRange(
13532 Range: Range.subtract(CI: SC->getAPInt()), SE);
13533 // This is strange and shouldn't happen.
13534 return SE.getCouldNotCompute();
13535 }
13536
13537 // The only time we can solve this is when we have all constant indices.
13538 // Otherwise, we cannot determine the overflow conditions.
13539 if (any_of(Range: operands(), P: [](const SCEV *Op) { return !isa<SCEVConstant>(Val: Op); }))
13540 return SE.getCouldNotCompute();
13541
13542 // Okay at this point we know that all elements of the chrec are constants and
13543 // that the start element is zero.
13544
13545 // First check to see if the range contains zero. If not, the first
13546 // iteration exits.
13547 unsigned BitWidth = SE.getTypeSizeInBits(Ty: getType());
13548 if (!Range.contains(Val: APInt(BitWidth, 0)))
13549 return SE.getZero(Ty: getType());
13550
13551 if (isAffine()) {
13552 // If this is an affine expression then we have this situation:
13553 // Solve {0,+,A} in Range === Ax in Range
13554
13555 // We know that zero is in the range. If A is positive then we know that
13556 // the upper value of the range must be the first possible exit value.
13557 // If A is negative then the lower of the range is the last possible loop
13558 // value. Also note that we already checked for a full range.
13559 APInt A = cast<SCEVConstant>(Val: getOperand(i: 1))->getAPInt();
13560 APInt End = A.sge(RHS: 1) ? (Range.getUpper() - 1) : Range.getLower();
13561
13562 // The exit value should be (End+A)/A.
13563 APInt ExitVal = (End + A).udiv(RHS: A);
13564 ConstantInt *ExitValue = ConstantInt::get(Context&: SE.getContext(), V: ExitVal);
13565
13566 // Evaluate at the exit value. If we really did fall out of the valid
13567 // range, then we computed our trip count, otherwise wrap around or other
13568 // things must have happened.
13569 ConstantInt *Val = EvaluateConstantChrecAtConstant(AddRec: this, C: ExitValue, SE);
13570 if (Range.contains(Val: Val->getValue()))
13571 return SE.getCouldNotCompute(); // Something strange happened
13572
13573 // Ensure that the previous value is in the range.
13574 assert(Range.contains(
13575 EvaluateConstantChrecAtConstant(this,
13576 ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) &&
13577 "Linear scev computation is off in a bad way!");
13578 return SE.getConstant(V: ExitValue);
13579 }
13580
13581 if (isQuadratic()) {
13582 if (auto S = SolveQuadraticAddRecRange(AddRec: this, Range, SE))
13583 return SE.getConstant(Val: *S);
13584 }
13585
13586 return SE.getCouldNotCompute();
13587}
13588
13589const SCEVAddRecExpr *
13590SCEVAddRecExpr::getPostIncExpr(ScalarEvolution &SE) const {
13591 assert(getNumOperands() > 1 && "AddRec with zero step?");
13592 // There is a temptation to just call getAddExpr(this, getStepRecurrence(SE)),
13593 // but in this case we cannot guarantee that the value returned will be an
13594 // AddRec because SCEV does not have a fixed point where it stops
13595 // simplification: it is legal to return ({rec1} + {rec2}). For example, it
13596 // may happen if we reach arithmetic depth limit while simplifying. So we
13597 // construct the returned value explicitly.
13598 SmallVector<const SCEV *, 3> Ops;
13599 // If this is {A,+,B,+,C,...,+,N}, then its step is {B,+,C,+,...,+,N}, and
13600 // (this + Step) is {A+B,+,B+C,+...,+,N}.
13601 for (unsigned i = 0, e = getNumOperands() - 1; i < e; ++i)
13602 Ops.push_back(Elt: SE.getAddExpr(LHS: getOperand(i), RHS: getOperand(i: i + 1)));
13603 // We know that the last operand is not a constant zero (otherwise it would
13604 // have been popped out earlier). This guarantees us that if the result has
13605 // the same last operand, then it will also not be popped out, meaning that
13606 // the returned value will be an AddRec.
13607 const SCEV *Last = getOperand(i: getNumOperands() - 1);
13608 assert(!Last->isZero() && "Recurrency with zero step?");
13609 Ops.push_back(Elt: Last);
13610 return cast<SCEVAddRecExpr>(Val: SE.getAddRecExpr(Operands&: Ops, L: getLoop(),
13611 Flags: SCEV::FlagAnyWrap));
13612}
13613
13614// Return true when S contains at least an undef value.
13615bool ScalarEvolution::containsUndefs(const SCEV *S) const {
13616 return SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
13617 if (const auto *SU = dyn_cast<SCEVUnknown>(Val: S))
13618 return isa<UndefValue>(Val: SU->getValue());
13619 return false;
13620 });
13621}
13622
13623// Return true when S contains a value that is a nullptr.
13624bool ScalarEvolution::containsErasedValue(const SCEV *S) const {
13625 return SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
13626 if (const auto *SU = dyn_cast<SCEVUnknown>(Val: S))
13627 return SU->getValue() == nullptr;
13628 return false;
13629 });
13630}
13631
13632/// Return the size of an element read or written by Inst.
13633const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) {
13634 Type *Ty;
13635 if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst))
13636 Ty = Store->getValueOperand()->getType();
13637 else if (LoadInst *Load = dyn_cast<LoadInst>(Val: Inst))
13638 Ty = Load->getType();
13639 else
13640 return nullptr;
13641
13642 Type *ETy = getEffectiveSCEVType(Ty: PointerType::getUnqual(C&: Inst->getContext()));
13643 return getSizeOfExpr(IntTy: ETy, AllocTy: Ty);
13644}
13645
13646//===----------------------------------------------------------------------===//
13647// SCEVCallbackVH Class Implementation
13648//===----------------------------------------------------------------------===//
13649
13650void ScalarEvolution::SCEVCallbackVH::deleted() {
13651 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
13652 if (PHINode *PN = dyn_cast<PHINode>(Val: getValPtr()))
13653 SE->ConstantEvolutionLoopExitValue.erase(Val: PN);
13654 SE->eraseValueFromMap(V: getValPtr());
13655 // this now dangles!
13656}
13657
13658void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) {
13659 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
13660
13661 // Forget all the expressions associated with users of the old value,
13662 // so that future queries will recompute the expressions using the new
13663 // value.
13664 SE->forgetValue(V: getValPtr());
13665 // this now dangles!
13666}
13667
13668ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se)
13669 : CallbackVH(V), SE(se) {}
13670
13671//===----------------------------------------------------------------------===//
13672// ScalarEvolution Class Implementation
13673//===----------------------------------------------------------------------===//
13674
13675ScalarEvolution::ScalarEvolution(Function &F, TargetLibraryInfo &TLI,
13676 AssumptionCache &AC, DominatorTree &DT,
13677 LoopInfo &LI)
13678 : F(F), DL(F.getDataLayout()), TLI(TLI), AC(AC), DT(DT), LI(LI),
13679 CouldNotCompute(new SCEVCouldNotCompute()), ValuesAtScopes(64),
13680 LoopDispositions(64), BlockDispositions(64) {
13681 // To use guards for proving predicates, we need to scan every instruction in
13682 // relevant basic blocks, and not just terminators. Doing this is a waste of
13683 // time if the IR does not actually contain any calls to
13684 // @llvm.experimental.guard, so do a quick check and remember this beforehand.
13685 //
13686 // This pessimizes the case where a pass that preserves ScalarEvolution wants
13687 // to _add_ guards to the module when there weren't any before, and wants
13688 // ScalarEvolution to optimize based on those guards. For now we prefer to be
13689 // efficient in lieu of being smart in that rather obscure case.
13690
13691 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
13692 M: F.getParent(), id: Intrinsic::experimental_guard);
13693 HasGuards = GuardDecl && !GuardDecl->use_empty();
13694}
13695
13696ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg)
13697 : F(Arg.F), DL(Arg.DL), HasGuards(Arg.HasGuards), TLI(Arg.TLI), AC(Arg.AC),
13698 DT(Arg.DT), LI(Arg.LI), CouldNotCompute(std::move(Arg.CouldNotCompute)),
13699 ValueExprMap(std::move(Arg.ValueExprMap)),
13700 PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
13701 PendingPhiRanges(std::move(Arg.PendingPhiRanges)),
13702 PendingMerges(std::move(Arg.PendingMerges)),
13703 ConstantMultipleCache(std::move(Arg.ConstantMultipleCache)),
13704 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
13705 PredicatedBackedgeTakenCounts(
13706 std::move(Arg.PredicatedBackedgeTakenCounts)),
13707 BECountUsers(std::move(Arg.BECountUsers)),
13708 ConstantEvolutionLoopExitValue(
13709 std::move(Arg.ConstantEvolutionLoopExitValue)),
13710 ValuesAtScopes(std::move(Arg.ValuesAtScopes)),
13711 ValuesAtScopesUsers(std::move(Arg.ValuesAtScopesUsers)),
13712 LoopDispositions(std::move(Arg.LoopDispositions)),
13713 LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)),
13714 BlockDispositions(std::move(Arg.BlockDispositions)),
13715 SCEVUsers(std::move(Arg.SCEVUsers)),
13716 UnsignedRanges(std::move(Arg.UnsignedRanges)),
13717 SignedRanges(std::move(Arg.SignedRanges)),
13718 UniqueSCEVs(std::move(Arg.UniqueSCEVs)),
13719 UniquePreds(std::move(Arg.UniquePreds)),
13720 SCEVAllocator(std::move(Arg.SCEVAllocator)),
13721 LoopUsers(std::move(Arg.LoopUsers)),
13722 PredicatedSCEVRewrites(std::move(Arg.PredicatedSCEVRewrites)),
13723 FirstUnknown(Arg.FirstUnknown) {
13724 Arg.FirstUnknown = nullptr;
13725}
13726
13727ScalarEvolution::~ScalarEvolution() {
13728 // Iterate through all the SCEVUnknown instances and call their
13729 // destructors, so that they release their references to their values.
13730 for (SCEVUnknown *U = FirstUnknown; U;) {
13731 SCEVUnknown *Tmp = U;
13732 U = U->Next;
13733 Tmp->~SCEVUnknown();
13734 }
13735 FirstUnknown = nullptr;
13736
13737 ExprValueMap.clear();
13738 ValueExprMap.clear();
13739 HasRecMap.clear();
13740 BackedgeTakenCounts.clear();
13741 PredicatedBackedgeTakenCounts.clear();
13742
13743 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage");
13744 assert(PendingPhiRanges.empty() && "getRangeRef garbage");
13745 assert(PendingMerges.empty() && "isImpliedViaMerge garbage");
13746 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!");
13747 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!");
13748}
13749
13750bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) {
13751 return !isa<SCEVCouldNotCompute>(Val: getBackedgeTakenCount(L));
13752}
13753
13754/// When printing a top-level SCEV for trip counts, it's helpful to include
13755/// a type for constants which are otherwise hard to disambiguate.
13756static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV* S) {
13757 if (isa<SCEVConstant>(Val: S))
13758 OS << *S->getType() << " ";
13759 OS << *S;
13760}
13761
13762static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE,
13763 const Loop *L) {
13764 // Print all inner loops first
13765 for (Loop *I : *L)
13766 PrintLoopInfo(OS, SE, L: I);
13767
13768 OS << "Loop ";
13769 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
13770 OS << ": ";
13771
13772 SmallVector<BasicBlock *, 8> ExitingBlocks;
13773 L->getExitingBlocks(ExitingBlocks);
13774 if (ExitingBlocks.size() != 1)
13775 OS << "<multiple exits> ";
13776
13777 auto *BTC = SE->getBackedgeTakenCount(L);
13778 if (!isa<SCEVCouldNotCompute>(Val: BTC)) {
13779 OS << "backedge-taken count is ";
13780 PrintSCEVWithTypeHint(OS, S: BTC);
13781 } else
13782 OS << "Unpredictable backedge-taken count.";
13783 OS << "\n";
13784
13785 if (ExitingBlocks.size() > 1)
13786 for (BasicBlock *ExitingBlock : ExitingBlocks) {
13787 OS << " exit count for " << ExitingBlock->getName() << ": ";
13788 const SCEV *EC = SE->getExitCount(L, ExitingBlock);
13789 PrintSCEVWithTypeHint(OS, S: EC);
13790 if (isa<SCEVCouldNotCompute>(Val: EC)) {
13791 // Retry with predicates.
13792 SmallVector<const SCEVPredicate *> Predicates;
13793 EC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates);
13794 if (!isa<SCEVCouldNotCompute>(Val: EC)) {
13795 OS << "\n predicated exit count for " << ExitingBlock->getName()
13796 << ": ";
13797 PrintSCEVWithTypeHint(OS, S: EC);
13798 OS << "\n Predicates:\n";
13799 for (const auto *P : Predicates)
13800 P->print(OS, Depth: 4);
13801 }
13802 }
13803 OS << "\n";
13804 }
13805
13806 OS << "Loop ";
13807 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
13808 OS << ": ";
13809
13810 auto *ConstantBTC = SE->getConstantMaxBackedgeTakenCount(L);
13811 if (!isa<SCEVCouldNotCompute>(Val: ConstantBTC)) {
13812 OS << "constant max backedge-taken count is ";
13813 PrintSCEVWithTypeHint(OS, S: ConstantBTC);
13814 if (SE->isBackedgeTakenCountMaxOrZero(L))
13815 OS << ", actual taken count either this or zero.";
13816 } else {
13817 OS << "Unpredictable constant max backedge-taken count. ";
13818 }
13819
13820 OS << "\n"
13821 "Loop ";
13822 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
13823 OS << ": ";
13824
13825 auto *SymbolicBTC = SE->getSymbolicMaxBackedgeTakenCount(L);
13826 if (!isa<SCEVCouldNotCompute>(Val: SymbolicBTC)) {
13827 OS << "symbolic max backedge-taken count is ";
13828 PrintSCEVWithTypeHint(OS, S: SymbolicBTC);
13829 if (SE->isBackedgeTakenCountMaxOrZero(L))
13830 OS << ", actual taken count either this or zero.";
13831 } else {
13832 OS << "Unpredictable symbolic max backedge-taken count. ";
13833 }
13834 OS << "\n";
13835
13836 if (ExitingBlocks.size() > 1)
13837 for (BasicBlock *ExitingBlock : ExitingBlocks) {
13838 OS << " symbolic max exit count for " << ExitingBlock->getName() << ": ";
13839 auto *ExitBTC = SE->getExitCount(L, ExitingBlock,
13840 Kind: ScalarEvolution::SymbolicMaximum);
13841 PrintSCEVWithTypeHint(OS, S: ExitBTC);
13842 if (isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
13843 // Retry with predicates.
13844 SmallVector<const SCEVPredicate *> Predicates;
13845 ExitBTC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates,
13846 Kind: ScalarEvolution::SymbolicMaximum);
13847 if (!isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
13848 OS << "\n predicated symbolic max exit count for "
13849 << ExitingBlock->getName() << ": ";
13850 PrintSCEVWithTypeHint(OS, S: ExitBTC);
13851 OS << "\n Predicates:\n";
13852 for (const auto *P : Predicates)
13853 P->print(OS, Depth: 4);
13854 }
13855 }
13856 OS << "\n";
13857 }
13858
13859 SmallVector<const SCEVPredicate *, 4> Preds;
13860 auto *PBT = SE->getPredicatedBackedgeTakenCount(L, Preds);
13861 if (PBT != BTC) {
13862 assert(!Preds.empty() && "Different predicated BTC, but no predicates");
13863 OS << "Loop ";
13864 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
13865 OS << ": ";
13866 if (!isa<SCEVCouldNotCompute>(Val: PBT)) {
13867 OS << "Predicated backedge-taken count is ";
13868 PrintSCEVWithTypeHint(OS, S: PBT);
13869 } else
13870 OS << "Unpredictable predicated backedge-taken count.";
13871 OS << "\n";
13872 OS << " Predicates:\n";
13873 for (const auto *P : Preds)
13874 P->print(OS, Depth: 4);
13875 }
13876 Preds.clear();
13877
13878 auto *PredConstantMax =
13879 SE->getPredicatedConstantMaxBackedgeTakenCount(L, Preds);
13880 if (PredConstantMax != ConstantBTC) {
13881 assert(!Preds.empty() &&
13882 "different predicated constant max BTC but no predicates");
13883 OS << "Loop ";
13884 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
13885 OS << ": ";
13886 if (!isa<SCEVCouldNotCompute>(Val: PredConstantMax)) {
13887 OS << "Predicated constant max backedge-taken count is ";
13888 PrintSCEVWithTypeHint(OS, S: PredConstantMax);
13889 } else
13890 OS << "Unpredictable predicated constant max backedge-taken count.";
13891 OS << "\n";
13892 OS << " Predicates:\n";
13893 for (const auto *P : Preds)
13894 P->print(OS, Depth: 4);
13895 }
13896 Preds.clear();
13897
13898 auto *PredSymbolicMax =
13899 SE->getPredicatedSymbolicMaxBackedgeTakenCount(L, Preds);
13900 if (SymbolicBTC != PredSymbolicMax) {
13901 assert(!Preds.empty() &&
13902 "Different predicated symbolic max BTC, but no predicates");
13903 OS << "Loop ";
13904 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
13905 OS << ": ";
13906 if (!isa<SCEVCouldNotCompute>(Val: PredSymbolicMax)) {
13907 OS << "Predicated symbolic max backedge-taken count is ";
13908 PrintSCEVWithTypeHint(OS, S: PredSymbolicMax);
13909 } else
13910 OS << "Unpredictable predicated symbolic max backedge-taken count.";
13911 OS << "\n";
13912 OS << " Predicates:\n";
13913 for (const auto *P : Preds)
13914 P->print(OS, Depth: 4);
13915 }
13916
13917 if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
13918 OS << "Loop ";
13919 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
13920 OS << ": ";
13921 OS << "Trip multiple is " << SE->getSmallConstantTripMultiple(L) << "\n";
13922 }
13923}
13924
13925namespace llvm {
13926raw_ostream &operator<<(raw_ostream &OS, ScalarEvolution::LoopDisposition LD) {
13927 switch (LD) {
13928 case ScalarEvolution::LoopVariant:
13929 OS << "Variant";
13930 break;
13931 case ScalarEvolution::LoopInvariant:
13932 OS << "Invariant";
13933 break;
13934 case ScalarEvolution::LoopComputable:
13935 OS << "Computable";
13936 break;
13937 }
13938 return OS;
13939}
13940
13941raw_ostream &operator<<(raw_ostream &OS, ScalarEvolution::BlockDisposition BD) {
13942 switch (BD) {
13943 case ScalarEvolution::DoesNotDominateBlock:
13944 OS << "DoesNotDominate";
13945 break;
13946 case ScalarEvolution::DominatesBlock:
13947 OS << "Dominates";
13948 break;
13949 case ScalarEvolution::ProperlyDominatesBlock:
13950 OS << "ProperlyDominates";
13951 break;
13952 }
13953 return OS;
13954}
13955} // namespace llvm
13956
13957void ScalarEvolution::print(raw_ostream &OS) const {
13958 // ScalarEvolution's implementation of the print method is to print
13959 // out SCEV values of all instructions that are interesting. Doing
13960 // this potentially causes it to create new SCEV objects though,
13961 // which technically conflicts with the const qualifier. This isn't
13962 // observable from outside the class though, so casting away the
13963 // const isn't dangerous.
13964 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
13965
13966 if (ClassifyExpressions) {
13967 OS << "Classifying expressions for: ";
13968 F.printAsOperand(O&: OS, /*PrintType=*/false);
13969 OS << "\n";
13970 for (Instruction &I : instructions(F))
13971 if (isSCEVable(Ty: I.getType()) && !isa<CmpInst>(Val: I)) {
13972 OS << I << '\n';
13973 OS << " --> ";
13974 const SCEV *SV = SE.getSCEV(V: &I);
13975 SV->print(OS);
13976 if (!isa<SCEVCouldNotCompute>(Val: SV)) {
13977 OS << " U: ";
13978 SE.getUnsignedRange(S: SV).print(OS);
13979 OS << " S: ";
13980 SE.getSignedRange(S: SV).print(OS);
13981 }
13982
13983 const Loop *L = LI.getLoopFor(BB: I.getParent());
13984
13985 const SCEV *AtUse = SE.getSCEVAtScope(V: SV, L);
13986 if (AtUse != SV) {
13987 OS << " --> ";
13988 AtUse->print(OS);
13989 if (!isa<SCEVCouldNotCompute>(Val: AtUse)) {
13990 OS << " U: ";
13991 SE.getUnsignedRange(S: AtUse).print(OS);
13992 OS << " S: ";
13993 SE.getSignedRange(S: AtUse).print(OS);
13994 }
13995 }
13996
13997 if (L) {
13998 OS << "\t\t" "Exits: ";
13999 const SCEV *ExitValue = SE.getSCEVAtScope(V: SV, L: L->getParentLoop());
14000 if (!SE.isLoopInvariant(S: ExitValue, L)) {
14001 OS << "<<Unknown>>";
14002 } else {
14003 OS << *ExitValue;
14004 }
14005
14006 bool First = true;
14007 for (const auto *Iter = L; Iter; Iter = Iter->getParentLoop()) {
14008 if (First) {
14009 OS << "\t\t" "LoopDispositions: { ";
14010 First = false;
14011 } else {
14012 OS << ", ";
14013 }
14014
14015 Iter->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14016 OS << ": " << SE.getLoopDisposition(S: SV, L: Iter);
14017 }
14018
14019 for (const auto *InnerL : depth_first(G: L)) {
14020 if (InnerL == L)
14021 continue;
14022 if (First) {
14023 OS << "\t\t" "LoopDispositions: { ";
14024 First = false;
14025 } else {
14026 OS << ", ";
14027 }
14028
14029 InnerL->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14030 OS << ": " << SE.getLoopDisposition(S: SV, L: InnerL);
14031 }
14032
14033 OS << " }";
14034 }
14035
14036 OS << "\n";
14037 }
14038 }
14039
14040 OS << "Determining loop execution counts for: ";
14041 F.printAsOperand(O&: OS, /*PrintType=*/false);
14042 OS << "\n";
14043 for (Loop *I : LI)
14044 PrintLoopInfo(OS, SE: &SE, L: I);
14045}
14046
14047ScalarEvolution::LoopDisposition
14048ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) {
14049 auto &Values = LoopDispositions[S];
14050 for (auto &V : Values) {
14051 if (V.getPointer() == L)
14052 return V.getInt();
14053 }
14054 Values.emplace_back(Args&: L, Args: LoopVariant);
14055 LoopDisposition D = computeLoopDisposition(S, L);
14056 auto &Values2 = LoopDispositions[S];
14057 for (auto &V : llvm::reverse(C&: Values2)) {
14058 if (V.getPointer() == L) {
14059 V.setInt(D);
14060 break;
14061 }
14062 }
14063 return D;
14064}
14065
14066ScalarEvolution::LoopDisposition
14067ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
14068 switch (S->getSCEVType()) {
14069 case scConstant:
14070 case scVScale:
14071 return LoopInvariant;
14072 case scAddRecExpr: {
14073 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14074
14075 // If L is the addrec's loop, it's computable.
14076 if (AR->getLoop() == L)
14077 return LoopComputable;
14078
14079 // Add recurrences are never invariant in the function-body (null loop).
14080 if (!L)
14081 return LoopVariant;
14082
14083 // Everything that is not defined at loop entry is variant.
14084 if (DT.dominates(A: L->getHeader(), B: AR->getLoop()->getHeader()))
14085 return LoopVariant;
14086 assert(!L->contains(AR->getLoop()) && "Containing loop's header does not"
14087 " dominate the contained loop's header?");
14088
14089 // This recurrence is invariant w.r.t. L if AR's loop contains L.
14090 if (AR->getLoop()->contains(L))
14091 return LoopInvariant;
14092
14093 // This recurrence is variant w.r.t. L if any of its operands
14094 // are variant.
14095 for (const auto *Op : AR->operands())
14096 if (!isLoopInvariant(S: Op, L))
14097 return LoopVariant;
14098
14099 // Otherwise it's loop-invariant.
14100 return LoopInvariant;
14101 }
14102 case scTruncate:
14103 case scZeroExtend:
14104 case scSignExtend:
14105 case scPtrToInt:
14106 case scAddExpr:
14107 case scMulExpr:
14108 case scUDivExpr:
14109 case scUMaxExpr:
14110 case scSMaxExpr:
14111 case scUMinExpr:
14112 case scSMinExpr:
14113 case scSequentialUMinExpr: {
14114 bool HasVarying = false;
14115 for (const auto *Op : S->operands()) {
14116 LoopDisposition D = getLoopDisposition(S: Op, L);
14117 if (D == LoopVariant)
14118 return LoopVariant;
14119 if (D == LoopComputable)
14120 HasVarying = true;
14121 }
14122 return HasVarying ? LoopComputable : LoopInvariant;
14123 }
14124 case scUnknown:
14125 // All non-instruction values are loop invariant. All instructions are loop
14126 // invariant if they are not contained in the specified loop.
14127 // Instructions are never considered invariant in the function body
14128 // (null loop) because they are defined within the "loop".
14129 if (auto *I = dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue()))
14130 return (L && !L->contains(Inst: I)) ? LoopInvariant : LoopVariant;
14131 return LoopInvariant;
14132 case scCouldNotCompute:
14133 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14134 }
14135 llvm_unreachable("Unknown SCEV kind!");
14136}
14137
14138bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) {
14139 return getLoopDisposition(S, L) == LoopInvariant;
14140}
14141
14142bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) {
14143 return getLoopDisposition(S, L) == LoopComputable;
14144}
14145
14146ScalarEvolution::BlockDisposition
14147ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14148 auto &Values = BlockDispositions[S];
14149 for (auto &V : Values) {
14150 if (V.getPointer() == BB)
14151 return V.getInt();
14152 }
14153 Values.emplace_back(Args&: BB, Args: DoesNotDominateBlock);
14154 BlockDisposition D = computeBlockDisposition(S, BB);
14155 auto &Values2 = BlockDispositions[S];
14156 for (auto &V : llvm::reverse(C&: Values2)) {
14157 if (V.getPointer() == BB) {
14158 V.setInt(D);
14159 break;
14160 }
14161 }
14162 return D;
14163}
14164
14165ScalarEvolution::BlockDisposition
14166ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14167 switch (S->getSCEVType()) {
14168 case scConstant:
14169 case scVScale:
14170 return ProperlyDominatesBlock;
14171 case scAddRecExpr: {
14172 // This uses a "dominates" query instead of "properly dominates" query
14173 // to test for proper dominance too, because the instruction which
14174 // produces the addrec's value is a PHI, and a PHI effectively properly
14175 // dominates its entire containing block.
14176 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14177 if (!DT.dominates(A: AR->getLoop()->getHeader(), B: BB))
14178 return DoesNotDominateBlock;
14179
14180 // Fall through into SCEVNAryExpr handling.
14181 [[fallthrough]];
14182 }
14183 case scTruncate:
14184 case scZeroExtend:
14185 case scSignExtend:
14186 case scPtrToInt:
14187 case scAddExpr:
14188 case scMulExpr:
14189 case scUDivExpr:
14190 case scUMaxExpr:
14191 case scSMaxExpr:
14192 case scUMinExpr:
14193 case scSMinExpr:
14194 case scSequentialUMinExpr: {
14195 bool Proper = true;
14196 for (const SCEV *NAryOp : S->operands()) {
14197 BlockDisposition D = getBlockDisposition(S: NAryOp, BB);
14198 if (D == DoesNotDominateBlock)
14199 return DoesNotDominateBlock;
14200 if (D == DominatesBlock)
14201 Proper = false;
14202 }
14203 return Proper ? ProperlyDominatesBlock : DominatesBlock;
14204 }
14205 case scUnknown:
14206 if (Instruction *I =
14207 dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue())) {
14208 if (I->getParent() == BB)
14209 return DominatesBlock;
14210 if (DT.properlyDominates(A: I->getParent(), B: BB))
14211 return ProperlyDominatesBlock;
14212 return DoesNotDominateBlock;
14213 }
14214 return ProperlyDominatesBlock;
14215 case scCouldNotCompute:
14216 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14217 }
14218 llvm_unreachable("Unknown SCEV kind!");
14219}
14220
14221bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
14222 return getBlockDisposition(S, BB) >= DominatesBlock;
14223}
14224
14225bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) {
14226 return getBlockDisposition(S, BB) == ProperlyDominatesBlock;
14227}
14228
14229bool ScalarEvolution::hasOperand(const SCEV *S, const SCEV *Op) const {
14230 return SCEVExprContains(Root: S, Pred: [&](const SCEV *Expr) { return Expr == Op; });
14231}
14232
14233void ScalarEvolution::forgetBackedgeTakenCounts(const Loop *L,
14234 bool Predicated) {
14235 auto &BECounts =
14236 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14237 auto It = BECounts.find(Val: L);
14238 if (It != BECounts.end()) {
14239 for (const ExitNotTakenInfo &ENT : It->second.ExitNotTaken) {
14240 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14241 if (!isa<SCEVConstant>(Val: S)) {
14242 auto UserIt = BECountUsers.find(Val: S);
14243 assert(UserIt != BECountUsers.end());
14244 UserIt->second.erase(Ptr: {L, Predicated});
14245 }
14246 }
14247 }
14248 BECounts.erase(I: It);
14249 }
14250}
14251
14252void ScalarEvolution::forgetMemoizedResults(ArrayRef<const SCEV *> SCEVs) {
14253 SmallPtrSet<const SCEV *, 8> ToForget(llvm::from_range, SCEVs);
14254 SmallVector<const SCEV *, 8> Worklist(ToForget.begin(), ToForget.end());
14255
14256 while (!Worklist.empty()) {
14257 const SCEV *Curr = Worklist.pop_back_val();
14258 auto Users = SCEVUsers.find(Val: Curr);
14259 if (Users != SCEVUsers.end())
14260 for (const auto *User : Users->second)
14261 if (ToForget.insert(Ptr: User).second)
14262 Worklist.push_back(Elt: User);
14263 }
14264
14265 for (const auto *S : ToForget)
14266 forgetMemoizedResultsImpl(S);
14267
14268 for (auto I = PredicatedSCEVRewrites.begin();
14269 I != PredicatedSCEVRewrites.end();) {
14270 std::pair<const SCEV *, const Loop *> Entry = I->first;
14271 if (ToForget.count(Ptr: Entry.first))
14272 PredicatedSCEVRewrites.erase(I: I++);
14273 else
14274 ++I;
14275 }
14276}
14277
14278void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) {
14279 LoopDispositions.erase(Val: S);
14280 BlockDispositions.erase(Val: S);
14281 UnsignedRanges.erase(Val: S);
14282 SignedRanges.erase(Val: S);
14283 HasRecMap.erase(Val: S);
14284 ConstantMultipleCache.erase(Val: S);
14285
14286 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
14287 UnsignedWrapViaInductionTried.erase(Ptr: AR);
14288 SignedWrapViaInductionTried.erase(Ptr: AR);
14289 }
14290
14291 auto ExprIt = ExprValueMap.find(Val: S);
14292 if (ExprIt != ExprValueMap.end()) {
14293 for (Value *V : ExprIt->second) {
14294 auto ValueIt = ValueExprMap.find_as(Val: V);
14295 if (ValueIt != ValueExprMap.end())
14296 ValueExprMap.erase(I: ValueIt);
14297 }
14298 ExprValueMap.erase(I: ExprIt);
14299 }
14300
14301 auto ScopeIt = ValuesAtScopes.find(Val: S);
14302 if (ScopeIt != ValuesAtScopes.end()) {
14303 for (const auto &Pair : ScopeIt->second)
14304 if (!isa_and_nonnull<SCEVConstant>(Val: Pair.second))
14305 llvm::erase(C&: ValuesAtScopesUsers[Pair.second],
14306 V: std::make_pair(x: Pair.first, y&: S));
14307 ValuesAtScopes.erase(I: ScopeIt);
14308 }
14309
14310 auto ScopeUserIt = ValuesAtScopesUsers.find(Val: S);
14311 if (ScopeUserIt != ValuesAtScopesUsers.end()) {
14312 for (const auto &Pair : ScopeUserIt->second)
14313 llvm::erase(C&: ValuesAtScopes[Pair.second], V: std::make_pair(x: Pair.first, y&: S));
14314 ValuesAtScopesUsers.erase(I: ScopeUserIt);
14315 }
14316
14317 auto BEUsersIt = BECountUsers.find(Val: S);
14318 if (BEUsersIt != BECountUsers.end()) {
14319 // Work on a copy, as forgetBackedgeTakenCounts() will modify the original.
14320 auto Copy = BEUsersIt->second;
14321 for (const auto &Pair : Copy)
14322 forgetBackedgeTakenCounts(L: Pair.getPointer(), Predicated: Pair.getInt());
14323 BECountUsers.erase(I: BEUsersIt);
14324 }
14325
14326 auto FoldUser = FoldCacheUser.find(Val: S);
14327 if (FoldUser != FoldCacheUser.end())
14328 for (auto &KV : FoldUser->second)
14329 FoldCache.erase(Val: KV);
14330 FoldCacheUser.erase(Val: S);
14331}
14332
14333void
14334ScalarEvolution::getUsedLoops(const SCEV *S,
14335 SmallPtrSetImpl<const Loop *> &LoopsUsed) {
14336 struct FindUsedLoops {
14337 FindUsedLoops(SmallPtrSetImpl<const Loop *> &LoopsUsed)
14338 : LoopsUsed(LoopsUsed) {}
14339 SmallPtrSetImpl<const Loop *> &LoopsUsed;
14340 bool follow(const SCEV *S) {
14341 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S))
14342 LoopsUsed.insert(Ptr: AR->getLoop());
14343 return true;
14344 }
14345
14346 bool isDone() const { return false; }
14347 };
14348
14349 FindUsedLoops F(LoopsUsed);
14350 SCEVTraversal<FindUsedLoops>(F).visitAll(Root: S);
14351}
14352
14353void ScalarEvolution::getReachableBlocks(
14354 SmallPtrSetImpl<BasicBlock *> &Reachable, Function &F) {
14355 SmallVector<BasicBlock *> Worklist;
14356 Worklist.push_back(Elt: &F.getEntryBlock());
14357 while (!Worklist.empty()) {
14358 BasicBlock *BB = Worklist.pop_back_val();
14359 if (!Reachable.insert(Ptr: BB).second)
14360 continue;
14361
14362 Value *Cond;
14363 BasicBlock *TrueBB, *FalseBB;
14364 if (match(V: BB->getTerminator(), P: m_Br(C: m_Value(V&: Cond), T: m_BasicBlock(V&: TrueBB),
14365 F: m_BasicBlock(V&: FalseBB)))) {
14366 if (auto *C = dyn_cast<ConstantInt>(Val: Cond)) {
14367 Worklist.push_back(Elt: C->isOne() ? TrueBB : FalseBB);
14368 continue;
14369 }
14370
14371 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
14372 const SCEV *L = getSCEV(V: Cmp->getOperand(i_nocapture: 0));
14373 const SCEV *R = getSCEV(V: Cmp->getOperand(i_nocapture: 1));
14374 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getCmpPredicate(), LHS: L, RHS: R)) {
14375 Worklist.push_back(Elt: TrueBB);
14376 continue;
14377 }
14378 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getInverseCmpPredicate(), LHS: L,
14379 RHS: R)) {
14380 Worklist.push_back(Elt: FalseBB);
14381 continue;
14382 }
14383 }
14384 }
14385
14386 append_range(C&: Worklist, R: successors(BB));
14387 }
14388}
14389
14390void ScalarEvolution::verify() const {
14391 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14392 ScalarEvolution SE2(F, TLI, AC, DT, LI);
14393
14394 SmallVector<Loop *, 8> LoopStack(LI.begin(), LI.end());
14395
14396 // Map's SCEV expressions from one ScalarEvolution "universe" to another.
14397 struct SCEVMapper : public SCEVRewriteVisitor<SCEVMapper> {
14398 SCEVMapper(ScalarEvolution &SE) : SCEVRewriteVisitor<SCEVMapper>(SE) {}
14399
14400 const SCEV *visitConstant(const SCEVConstant *Constant) {
14401 return SE.getConstant(Val: Constant->getAPInt());
14402 }
14403
14404 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14405 return SE.getUnknown(V: Expr->getValue());
14406 }
14407
14408 const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) {
14409 return SE.getCouldNotCompute();
14410 }
14411 };
14412
14413 SCEVMapper SCM(SE2);
14414 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
14415 SE2.getReachableBlocks(Reachable&: ReachableBlocks, F);
14416
14417 auto GetDelta = [&](const SCEV *Old, const SCEV *New) -> const SCEV * {
14418 if (containsUndefs(S: Old) || containsUndefs(S: New)) {
14419 // SCEV treats "undef" as an unknown but consistent value (i.e. it does
14420 // not propagate undef aggressively). This means we can (and do) fail
14421 // verification in cases where a transform makes a value go from "undef"
14422 // to "undef+1" (say). The transform is fine, since in both cases the
14423 // result is "undef", but SCEV thinks the value increased by 1.
14424 return nullptr;
14425 }
14426
14427 // Unless VerifySCEVStrict is set, we only compare constant deltas.
14428 const SCEV *Delta = SE2.getMinusSCEV(LHS: Old, RHS: New);
14429 if (!VerifySCEVStrict && !isa<SCEVConstant>(Val: Delta))
14430 return nullptr;
14431
14432 return Delta;
14433 };
14434
14435 while (!LoopStack.empty()) {
14436 auto *L = LoopStack.pop_back_val();
14437 llvm::append_range(C&: LoopStack, R&: *L);
14438
14439 // Only verify BECounts in reachable loops. For an unreachable loop,
14440 // any BECount is legal.
14441 if (!ReachableBlocks.contains(Ptr: L->getHeader()))
14442 continue;
14443
14444 // Only verify cached BECounts. Computing new BECounts may change the
14445 // results of subsequent SCEV uses.
14446 auto It = BackedgeTakenCounts.find(Val: L);
14447 if (It == BackedgeTakenCounts.end())
14448 continue;
14449
14450 auto *CurBECount =
14451 SCM.visit(S: It->second.getExact(L, SE: const_cast<ScalarEvolution *>(this)));
14452 auto *NewBECount = SE2.getBackedgeTakenCount(L);
14453
14454 if (CurBECount == SE2.getCouldNotCompute() ||
14455 NewBECount == SE2.getCouldNotCompute()) {
14456 // NB! This situation is legal, but is very suspicious -- whatever pass
14457 // change the loop to make a trip count go from could not compute to
14458 // computable or vice-versa *should have* invalidated SCEV. However, we
14459 // choose not to assert here (for now) since we don't want false
14460 // positives.
14461 continue;
14462 }
14463
14464 if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) >
14465 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14466 NewBECount = SE2.getZeroExtendExpr(Op: NewBECount, Ty: CurBECount->getType());
14467 else if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) <
14468 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14469 CurBECount = SE2.getZeroExtendExpr(Op: CurBECount, Ty: NewBECount->getType());
14470
14471 const SCEV *Delta = GetDelta(CurBECount, NewBECount);
14472 if (Delta && !Delta->isZero()) {
14473 dbgs() << "Trip Count for " << *L << " Changed!\n";
14474 dbgs() << "Old: " << *CurBECount << "\n";
14475 dbgs() << "New: " << *NewBECount << "\n";
14476 dbgs() << "Delta: " << *Delta << "\n";
14477 std::abort();
14478 }
14479 }
14480
14481 // Collect all valid loops currently in LoopInfo.
14482 SmallPtrSet<Loop *, 32> ValidLoops;
14483 SmallVector<Loop *, 32> Worklist(LI.begin(), LI.end());
14484 while (!Worklist.empty()) {
14485 Loop *L = Worklist.pop_back_val();
14486 if (ValidLoops.insert(Ptr: L).second)
14487 Worklist.append(in_start: L->begin(), in_end: L->end());
14488 }
14489 for (const auto &KV : ValueExprMap) {
14490#ifndef NDEBUG
14491 // Check for SCEV expressions referencing invalid/deleted loops.
14492 if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) {
14493 assert(ValidLoops.contains(AR->getLoop()) &&
14494 "AddRec references invalid loop");
14495 }
14496#endif
14497
14498 // Check that the value is also part of the reverse map.
14499 auto It = ExprValueMap.find(Val: KV.second);
14500 if (It == ExprValueMap.end() || !It->second.contains(key: KV.first)) {
14501 dbgs() << "Value " << *KV.first
14502 << " is in ValueExprMap but not in ExprValueMap\n";
14503 std::abort();
14504 }
14505
14506 if (auto *I = dyn_cast<Instruction>(Val: &*KV.first)) {
14507 if (!ReachableBlocks.contains(Ptr: I->getParent()))
14508 continue;
14509 const SCEV *OldSCEV = SCM.visit(S: KV.second);
14510 const SCEV *NewSCEV = SE2.getSCEV(V: I);
14511 const SCEV *Delta = GetDelta(OldSCEV, NewSCEV);
14512 if (Delta && !Delta->isZero()) {
14513 dbgs() << "SCEV for value " << *I << " changed!\n"
14514 << "Old: " << *OldSCEV << "\n"
14515 << "New: " << *NewSCEV << "\n"
14516 << "Delta: " << *Delta << "\n";
14517 std::abort();
14518 }
14519 }
14520 }
14521
14522 for (const auto &KV : ExprValueMap) {
14523 for (Value *V : KV.second) {
14524 auto It = ValueExprMap.find_as(Val: V);
14525 if (It == ValueExprMap.end()) {
14526 dbgs() << "Value " << *V
14527 << " is in ExprValueMap but not in ValueExprMap\n";
14528 std::abort();
14529 }
14530 if (It->second != KV.first) {
14531 dbgs() << "Value " << *V << " mapped to " << *It->second
14532 << " rather than " << *KV.first << "\n";
14533 std::abort();
14534 }
14535 }
14536 }
14537
14538 // Verify integrity of SCEV users.
14539 for (const auto &S : UniqueSCEVs) {
14540 for (const auto *Op : S.operands()) {
14541 // We do not store dependencies of constants.
14542 if (isa<SCEVConstant>(Val: Op))
14543 continue;
14544 auto It = SCEVUsers.find(Val: Op);
14545 if (It != SCEVUsers.end() && It->second.count(Ptr: &S))
14546 continue;
14547 dbgs() << "Use of operand " << *Op << " by user " << S
14548 << " is not being tracked!\n";
14549 std::abort();
14550 }
14551 }
14552
14553 // Verify integrity of ValuesAtScopes users.
14554 for (const auto &ValueAndVec : ValuesAtScopes) {
14555 const SCEV *Value = ValueAndVec.first;
14556 for (const auto &LoopAndValueAtScope : ValueAndVec.second) {
14557 const Loop *L = LoopAndValueAtScope.first;
14558 const SCEV *ValueAtScope = LoopAndValueAtScope.second;
14559 if (!isa<SCEVConstant>(Val: ValueAtScope)) {
14560 auto It = ValuesAtScopesUsers.find(Val: ValueAtScope);
14561 if (It != ValuesAtScopesUsers.end() &&
14562 is_contained(Range: It->second, Element: std::make_pair(x&: L, y&: Value)))
14563 continue;
14564 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14565 << *ValueAtScope << " missing in ValuesAtScopesUsers\n";
14566 std::abort();
14567 }
14568 }
14569 }
14570
14571 for (const auto &ValueAtScopeAndVec : ValuesAtScopesUsers) {
14572 const SCEV *ValueAtScope = ValueAtScopeAndVec.first;
14573 for (const auto &LoopAndValue : ValueAtScopeAndVec.second) {
14574 const Loop *L = LoopAndValue.first;
14575 const SCEV *Value = LoopAndValue.second;
14576 assert(!isa<SCEVConstant>(Value));
14577 auto It = ValuesAtScopes.find(Val: Value);
14578 if (It != ValuesAtScopes.end() &&
14579 is_contained(Range: It->second, Element: std::make_pair(x&: L, y&: ValueAtScope)))
14580 continue;
14581 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14582 << *ValueAtScope << " missing in ValuesAtScopes\n";
14583 std::abort();
14584 }
14585 }
14586
14587 // Verify integrity of BECountUsers.
14588 auto VerifyBECountUsers = [&](bool Predicated) {
14589 auto &BECounts =
14590 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14591 for (const auto &LoopAndBEInfo : BECounts) {
14592 for (const ExitNotTakenInfo &ENT : LoopAndBEInfo.second.ExitNotTaken) {
14593 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14594 if (!isa<SCEVConstant>(Val: S)) {
14595 auto UserIt = BECountUsers.find(Val: S);
14596 if (UserIt != BECountUsers.end() &&
14597 UserIt->second.contains(Ptr: { LoopAndBEInfo.first, Predicated }))
14598 continue;
14599 dbgs() << "Value " << *S << " for loop " << *LoopAndBEInfo.first
14600 << " missing from BECountUsers\n";
14601 std::abort();
14602 }
14603 }
14604 }
14605 }
14606 };
14607 VerifyBECountUsers(/* Predicated */ false);
14608 VerifyBECountUsers(/* Predicated */ true);
14609
14610 // Verify intergity of loop disposition cache.
14611 for (auto &[S, Values] : LoopDispositions) {
14612 for (auto [Loop, CachedDisposition] : Values) {
14613 const auto RecomputedDisposition = SE2.getLoopDisposition(S, L: Loop);
14614 if (CachedDisposition != RecomputedDisposition) {
14615 dbgs() << "Cached disposition of " << *S << " for loop " << *Loop
14616 << " is incorrect: cached " << CachedDisposition << ", actual "
14617 << RecomputedDisposition << "\n";
14618 std::abort();
14619 }
14620 }
14621 }
14622
14623 // Verify integrity of the block disposition cache.
14624 for (auto &[S, Values] : BlockDispositions) {
14625 for (auto [BB, CachedDisposition] : Values) {
14626 const auto RecomputedDisposition = SE2.getBlockDisposition(S, BB);
14627 if (CachedDisposition != RecomputedDisposition) {
14628 dbgs() << "Cached disposition of " << *S << " for block %"
14629 << BB->getName() << " is incorrect: cached " << CachedDisposition
14630 << ", actual " << RecomputedDisposition << "\n";
14631 std::abort();
14632 }
14633 }
14634 }
14635
14636 // Verify FoldCache/FoldCacheUser caches.
14637 for (auto [FoldID, Expr] : FoldCache) {
14638 auto I = FoldCacheUser.find(Val: Expr);
14639 if (I == FoldCacheUser.end()) {
14640 dbgs() << "Missing entry in FoldCacheUser for cached expression " << *Expr
14641 << "!\n";
14642 std::abort();
14643 }
14644 if (!is_contained(Range: I->second, Element: FoldID)) {
14645 dbgs() << "Missing FoldID in cached users of " << *Expr << "!\n";
14646 std::abort();
14647 }
14648 }
14649 for (auto [Expr, IDs] : FoldCacheUser) {
14650 for (auto &FoldID : IDs) {
14651 auto I = FoldCache.find(Val: FoldID);
14652 if (I == FoldCache.end()) {
14653 dbgs() << "Missing entry in FoldCache for expression " << *Expr
14654 << "!\n";
14655 std::abort();
14656 }
14657 if (I->second != Expr) {
14658 dbgs() << "Entry in FoldCache doesn't match FoldCacheUser: "
14659 << *I->second << " != " << *Expr << "!\n";
14660 std::abort();
14661 }
14662 }
14663 }
14664
14665 // Verify that ConstantMultipleCache computations are correct. We check that
14666 // cached multiples and recomputed multiples are multiples of each other to
14667 // verify correctness. It is possible that a recomputed multiple is different
14668 // from the cached multiple due to strengthened no wrap flags or changes in
14669 // KnownBits computations.
14670 for (auto [S, Multiple] : ConstantMultipleCache) {
14671 APInt RecomputedMultiple = SE2.getConstantMultiple(S);
14672 if ((Multiple != 0 && RecomputedMultiple != 0 &&
14673 Multiple.urem(RHS: RecomputedMultiple) != 0 &&
14674 RecomputedMultiple.urem(RHS: Multiple) != 0)) {
14675 dbgs() << "Incorrect cached computation in ConstantMultipleCache for "
14676 << *S << " : Computed " << RecomputedMultiple
14677 << " but cache contains " << Multiple << "!\n";
14678 std::abort();
14679 }
14680 }
14681}
14682
14683bool ScalarEvolution::invalidate(
14684 Function &F, const PreservedAnalyses &PA,
14685 FunctionAnalysisManager::Invalidator &Inv) {
14686 // Invalidate the ScalarEvolution object whenever it isn't preserved or one
14687 // of its dependencies is invalidated.
14688 auto PAC = PA.getChecker<ScalarEvolutionAnalysis>();
14689 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>()) ||
14690 Inv.invalidate<AssumptionAnalysis>(IR&: F, PA) ||
14691 Inv.invalidate<DominatorTreeAnalysis>(IR&: F, PA) ||
14692 Inv.invalidate<LoopAnalysis>(IR&: F, PA);
14693}
14694
14695AnalysisKey ScalarEvolutionAnalysis::Key;
14696
14697ScalarEvolution ScalarEvolutionAnalysis::run(Function &F,
14698 FunctionAnalysisManager &AM) {
14699 auto &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
14700 auto &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
14701 auto &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
14702 auto &LI = AM.getResult<LoopAnalysis>(IR&: F);
14703 return ScalarEvolution(F, TLI, AC, DT, LI);
14704}
14705
14706PreservedAnalyses
14707ScalarEvolutionVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
14708 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).verify();
14709 return PreservedAnalyses::all();
14710}
14711
14712PreservedAnalyses
14713ScalarEvolutionPrinterPass::run(Function &F, FunctionAnalysisManager &AM) {
14714 // For compatibility with opt's -analyze feature under legacy pass manager
14715 // which was not ported to NPM. This keeps tests using
14716 // update_analyze_test_checks.py working.
14717 OS << "Printing analysis 'Scalar Evolution Analysis' for function '"
14718 << F.getName() << "':\n";
14719 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).print(OS);
14720 return PreservedAnalyses::all();
14721}
14722
14723INITIALIZE_PASS_BEGIN(ScalarEvolutionWrapperPass, "scalar-evolution",
14724 "Scalar Evolution Analysis", false, true)
14725INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
14726INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
14727INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
14728INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
14729INITIALIZE_PASS_END(ScalarEvolutionWrapperPass, "scalar-evolution",
14730 "Scalar Evolution Analysis", false, true)
14731
14732char ScalarEvolutionWrapperPass::ID = 0;
14733
14734ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) {}
14735
14736bool ScalarEvolutionWrapperPass::runOnFunction(Function &F) {
14737 SE.reset(p: new ScalarEvolution(
14738 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F),
14739 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
14740 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
14741 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()));
14742 return false;
14743}
14744
14745void ScalarEvolutionWrapperPass::releaseMemory() { SE.reset(); }
14746
14747void ScalarEvolutionWrapperPass::print(raw_ostream &OS, const Module *) const {
14748 SE->print(OS);
14749}
14750
14751void ScalarEvolutionWrapperPass::verifyAnalysis() const {
14752 if (!VerifySCEV)
14753 return;
14754
14755 SE->verify();
14756}
14757
14758void ScalarEvolutionWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
14759 AU.setPreservesAll();
14760 AU.addRequiredTransitive<AssumptionCacheTracker>();
14761 AU.addRequiredTransitive<LoopInfoWrapperPass>();
14762 AU.addRequiredTransitive<DominatorTreeWrapperPass>();
14763 AU.addRequiredTransitive<TargetLibraryInfoWrapperPass>();
14764}
14765
14766const SCEVPredicate *ScalarEvolution::getEqualPredicate(const SCEV *LHS,
14767 const SCEV *RHS) {
14768 return getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS, RHS);
14769}
14770
14771const SCEVPredicate *
14772ScalarEvolution::getComparePredicate(const ICmpInst::Predicate Pred,
14773 const SCEV *LHS, const SCEV *RHS) {
14774 FoldingSetNodeID ID;
14775 assert(LHS->getType() == RHS->getType() &&
14776 "Type mismatch between LHS and RHS");
14777 // Unique this node based on the arguments
14778 ID.AddInteger(I: SCEVPredicate::P_Compare);
14779 ID.AddInteger(I: Pred);
14780 ID.AddPointer(Ptr: LHS);
14781 ID.AddPointer(Ptr: RHS);
14782 void *IP = nullptr;
14783 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, InsertPos&: IP))
14784 return S;
14785 SCEVComparePredicate *Eq = new (SCEVAllocator)
14786 SCEVComparePredicate(ID.Intern(Allocator&: SCEVAllocator), Pred, LHS, RHS);
14787 UniquePreds.InsertNode(N: Eq, InsertPos: IP);
14788 return Eq;
14789}
14790
14791const SCEVPredicate *ScalarEvolution::getWrapPredicate(
14792 const SCEVAddRecExpr *AR,
14793 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
14794 FoldingSetNodeID ID;
14795 // Unique this node based on the arguments
14796 ID.AddInteger(I: SCEVPredicate::P_Wrap);
14797 ID.AddPointer(Ptr: AR);
14798 ID.AddInteger(I: AddedFlags);
14799 void *IP = nullptr;
14800 if (const auto *S = UniquePreds.FindNodeOrInsertPos(ID, InsertPos&: IP))
14801 return S;
14802 auto *OF = new (SCEVAllocator)
14803 SCEVWrapPredicate(ID.Intern(Allocator&: SCEVAllocator), AR, AddedFlags);
14804 UniquePreds.InsertNode(N: OF, InsertPos: IP);
14805 return OF;
14806}
14807
14808namespace {
14809
14810class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
14811public:
14812
14813 /// Rewrites \p S in the context of a loop L and the SCEV predication
14814 /// infrastructure.
14815 ///
14816 /// If \p Pred is non-null, the SCEV expression is rewritten to respect the
14817 /// equivalences present in \p Pred.
14818 ///
14819 /// If \p NewPreds is non-null, rewrite is free to add further predicates to
14820 /// \p NewPreds such that the result will be an AddRecExpr.
14821 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
14822 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
14823 const SCEVPredicate *Pred) {
14824 SCEVPredicateRewriter Rewriter(L, SE, NewPreds, Pred);
14825 return Rewriter.visit(S);
14826 }
14827
14828 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14829 if (Pred) {
14830 if (auto *U = dyn_cast<SCEVUnionPredicate>(Val: Pred)) {
14831 for (const auto *Pred : U->getPredicates())
14832 if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred))
14833 if (IPred->getLHS() == Expr &&
14834 IPred->getPredicate() == ICmpInst::ICMP_EQ)
14835 return IPred->getRHS();
14836 } else if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred)) {
14837 if (IPred->getLHS() == Expr &&
14838 IPred->getPredicate() == ICmpInst::ICMP_EQ)
14839 return IPred->getRHS();
14840 }
14841 }
14842 return convertToAddRecWithPreds(Expr);
14843 }
14844
14845 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
14846 const SCEV *Operand = visit(S: Expr->getOperand());
14847 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
14848 if (AR && AR->getLoop() == L && AR->isAffine()) {
14849 // This couldn't be folded because the operand didn't have the nuw
14850 // flag. Add the nusw flag as an assumption that we could make.
14851 const SCEV *Step = AR->getStepRecurrence(SE);
14852 Type *Ty = Expr->getType();
14853 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNUSW))
14854 return SE.getAddRecExpr(Start: SE.getZeroExtendExpr(Op: AR->getStart(), Ty),
14855 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
14856 Flags: AR->getNoWrapFlags());
14857 }
14858 return SE.getZeroExtendExpr(Op: Operand, Ty: Expr->getType());
14859 }
14860
14861 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
14862 const SCEV *Operand = visit(S: Expr->getOperand());
14863 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
14864 if (AR && AR->getLoop() == L && AR->isAffine()) {
14865 // This couldn't be folded because the operand didn't have the nsw
14866 // flag. Add the nssw flag as an assumption that we could make.
14867 const SCEV *Step = AR->getStepRecurrence(SE);
14868 Type *Ty = Expr->getType();
14869 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNSSW))
14870 return SE.getAddRecExpr(Start: SE.getSignExtendExpr(Op: AR->getStart(), Ty),
14871 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
14872 Flags: AR->getNoWrapFlags());
14873 }
14874 return SE.getSignExtendExpr(Op: Operand, Ty: Expr->getType());
14875 }
14876
14877private:
14878 explicit SCEVPredicateRewriter(
14879 const Loop *L, ScalarEvolution &SE,
14880 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
14881 const SCEVPredicate *Pred)
14882 : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred), L(L) {}
14883
14884 bool addOverflowAssumption(const SCEVPredicate *P) {
14885 if (!NewPreds) {
14886 // Check if we've already made this assumption.
14887 return Pred && Pred->implies(N: P, SE);
14888 }
14889 NewPreds->push_back(Elt: P);
14890 return true;
14891 }
14892
14893 bool addOverflowAssumption(const SCEVAddRecExpr *AR,
14894 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
14895 auto *A = SE.getWrapPredicate(AR, AddedFlags);
14896 return addOverflowAssumption(P: A);
14897 }
14898
14899 // If \p Expr represents a PHINode, we try to see if it can be represented
14900 // as an AddRec, possibly under a predicate (PHISCEVPred). If it is possible
14901 // to add this predicate as a runtime overflow check, we return the AddRec.
14902 // If \p Expr does not meet these conditions (is not a PHI node, or we
14903 // couldn't create an AddRec for it, or couldn't add the predicate), we just
14904 // return \p Expr.
14905 const SCEV *convertToAddRecWithPreds(const SCEVUnknown *Expr) {
14906 if (!isa<PHINode>(Val: Expr->getValue()))
14907 return Expr;
14908 std::optional<
14909 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
14910 PredicatedRewrite = SE.createAddRecFromPHIWithCasts(SymbolicPHI: Expr);
14911 if (!PredicatedRewrite)
14912 return Expr;
14913 for (const auto *P : PredicatedRewrite->second){
14914 // Wrap predicates from outer loops are not supported.
14915 if (auto *WP = dyn_cast<const SCEVWrapPredicate>(Val: P)) {
14916 if (L != WP->getExpr()->getLoop())
14917 return Expr;
14918 }
14919 if (!addOverflowAssumption(P))
14920 return Expr;
14921 }
14922 return PredicatedRewrite->first;
14923 }
14924
14925 SmallVectorImpl<const SCEVPredicate *> *NewPreds;
14926 const SCEVPredicate *Pred;
14927 const Loop *L;
14928};
14929
14930} // end anonymous namespace
14931
14932const SCEV *
14933ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L,
14934 const SCEVPredicate &Preds) {
14935 return SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: nullptr, Pred: &Preds);
14936}
14937
14938const SCEVAddRecExpr *ScalarEvolution::convertSCEVToAddRecWithPredicates(
14939 const SCEV *S, const Loop *L,
14940 SmallVectorImpl<const SCEVPredicate *> &Preds) {
14941 SmallVector<const SCEVPredicate *> TransformPreds;
14942 S = SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: &TransformPreds, Pred: nullptr);
14943 auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S);
14944
14945 if (!AddRec)
14946 return nullptr;
14947
14948 // Since the transformation was successful, we can now transfer the SCEV
14949 // predicates.
14950 Preds.append(in_start: TransformPreds.begin(), in_end: TransformPreds.end());
14951
14952 return AddRec;
14953}
14954
14955/// SCEV predicates
14956SCEVPredicate::SCEVPredicate(const FoldingSetNodeIDRef ID,
14957 SCEVPredicateKind Kind)
14958 : FastID(ID), Kind(Kind) {}
14959
14960SCEVComparePredicate::SCEVComparePredicate(const FoldingSetNodeIDRef ID,
14961 const ICmpInst::Predicate Pred,
14962 const SCEV *LHS, const SCEV *RHS)
14963 : SCEVPredicate(ID, P_Compare), Pred(Pred), LHS(LHS), RHS(RHS) {
14964 assert(LHS->getType() == RHS->getType() && "LHS and RHS types don't match");
14965 assert(LHS != RHS && "LHS and RHS are the same SCEV");
14966}
14967
14968bool SCEVComparePredicate::implies(const SCEVPredicate *N,
14969 ScalarEvolution &SE) const {
14970 const auto *Op = dyn_cast<SCEVComparePredicate>(Val: N);
14971
14972 if (!Op)
14973 return false;
14974
14975 if (Pred != ICmpInst::ICMP_EQ)
14976 return false;
14977
14978 return Op->LHS == LHS && Op->RHS == RHS;
14979}
14980
14981bool SCEVComparePredicate::isAlwaysTrue() const { return false; }
14982
14983void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const {
14984 if (Pred == ICmpInst::ICMP_EQ)
14985 OS.indent(NumSpaces: Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n";
14986 else
14987 OS.indent(NumSpaces: Depth) << "Compare predicate: " << *LHS << " " << Pred << ") "
14988 << *RHS << "\n";
14989
14990}
14991
14992SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID,
14993 const SCEVAddRecExpr *AR,
14994 IncrementWrapFlags Flags)
14995 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {}
14996
14997const SCEVAddRecExpr *SCEVWrapPredicate::getExpr() const { return AR; }
14998
14999bool SCEVWrapPredicate::implies(const SCEVPredicate *N,
15000 ScalarEvolution &SE) const {
15001 const auto *Op = dyn_cast<SCEVWrapPredicate>(Val: N);
15002 if (!Op || setFlags(Flags, OnFlags: Op->Flags) != Flags)
15003 return false;
15004
15005 if (Op->AR == AR)
15006 return true;
15007
15008 if (Flags != SCEVWrapPredicate::IncrementNSSW &&
15009 Flags != SCEVWrapPredicate::IncrementNUSW)
15010 return false;
15011
15012 const SCEV *Start = AR->getStart();
15013 const SCEV *OpStart = Op->AR->getStart();
15014 if (Start->getType()->isPointerTy() != OpStart->getType()->isPointerTy())
15015 return false;
15016
15017 // Reject pointers to different address spaces.
15018 if (Start->getType()->isPointerTy() && Start->getType() != OpStart->getType())
15019 return false;
15020
15021 const SCEV *Step = AR->getStepRecurrence(SE);
15022 const SCEV *OpStep = Op->AR->getStepRecurrence(SE);
15023 if (!SE.isKnownPositive(S: Step) || !SE.isKnownPositive(S: OpStep))
15024 return false;
15025
15026 // If both steps are positive, this implies N, if N's start and step are
15027 // ULE/SLE (for NSUW/NSSW) than this'.
15028 Type *WiderTy = SE.getWiderType(T1: Step->getType(), T2: OpStep->getType());
15029 Step = SE.getNoopOrZeroExtend(V: Step, Ty: WiderTy);
15030 OpStep = SE.getNoopOrZeroExtend(V: OpStep, Ty: WiderTy);
15031
15032 bool IsNUW = Flags == SCEVWrapPredicate::IncrementNUSW;
15033 OpStart = IsNUW ? SE.getNoopOrZeroExtend(V: OpStart, Ty: WiderTy)
15034 : SE.getNoopOrSignExtend(V: OpStart, Ty: WiderTy);
15035 Start = IsNUW ? SE.getNoopOrZeroExtend(V: Start, Ty: WiderTy)
15036 : SE.getNoopOrSignExtend(V: Start, Ty: WiderTy);
15037 CmpInst::Predicate Pred = IsNUW ? CmpInst::ICMP_ULE : CmpInst::ICMP_SLE;
15038 return SE.isKnownPredicate(Pred, LHS: OpStep, RHS: Step) &&
15039 SE.isKnownPredicate(Pred, LHS: OpStart, RHS: Start);
15040}
15041
15042bool SCEVWrapPredicate::isAlwaysTrue() const {
15043 SCEV::NoWrapFlags ScevFlags = AR->getNoWrapFlags();
15044 IncrementWrapFlags IFlags = Flags;
15045
15046 if (ScalarEvolution::setFlags(Flags: ScevFlags, OnFlags: SCEV::FlagNSW) == ScevFlags)
15047 IFlags = clearFlags(Flags: IFlags, OffFlags: IncrementNSSW);
15048
15049 return IFlags == IncrementAnyWrap;
15050}
15051
15052void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const {
15053 OS.indent(NumSpaces: Depth) << *getExpr() << " Added Flags: ";
15054 if (SCEVWrapPredicate::IncrementNUSW & getFlags())
15055 OS << "<nusw>";
15056 if (SCEVWrapPredicate::IncrementNSSW & getFlags())
15057 OS << "<nssw>";
15058 OS << "\n";
15059}
15060
15061SCEVWrapPredicate::IncrementWrapFlags
15062SCEVWrapPredicate::getImpliedFlags(const SCEVAddRecExpr *AR,
15063 ScalarEvolution &SE) {
15064 IncrementWrapFlags ImpliedFlags = IncrementAnyWrap;
15065 SCEV::NoWrapFlags StaticFlags = AR->getNoWrapFlags();
15066
15067 // We can safely transfer the NSW flag as NSSW.
15068 if (ScalarEvolution::setFlags(Flags: StaticFlags, OnFlags: SCEV::FlagNSW) == StaticFlags)
15069 ImpliedFlags = IncrementNSSW;
15070
15071 if (ScalarEvolution::setFlags(Flags: StaticFlags, OnFlags: SCEV::FlagNUW) == StaticFlags) {
15072 // If the increment is positive, the SCEV NUW flag will also imply the
15073 // WrapPredicate NUSW flag.
15074 if (const auto *Step = dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE)))
15075 if (Step->getValue()->getValue().isNonNegative())
15076 ImpliedFlags = setFlags(Flags: ImpliedFlags, OnFlags: IncrementNUSW);
15077 }
15078
15079 return ImpliedFlags;
15080}
15081
15082/// Union predicates don't get cached so create a dummy set ID for it.
15083SCEVUnionPredicate::SCEVUnionPredicate(ArrayRef<const SCEVPredicate *> Preds,
15084 ScalarEvolution &SE)
15085 : SCEVPredicate(FoldingSetNodeIDRef(nullptr, 0), P_Union) {
15086 for (const auto *P : Preds)
15087 add(N: P, SE);
15088}
15089
15090bool SCEVUnionPredicate::isAlwaysTrue() const {
15091 return all_of(Range: Preds,
15092 P: [](const SCEVPredicate *I) { return I->isAlwaysTrue(); });
15093}
15094
15095bool SCEVUnionPredicate::implies(const SCEVPredicate *N,
15096 ScalarEvolution &SE) const {
15097 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N))
15098 return all_of(Range: Set->Preds, P: [this, &SE](const SCEVPredicate *I) {
15099 return this->implies(N: I, SE);
15100 });
15101
15102 return any_of(Range: Preds,
15103 P: [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); });
15104}
15105
15106void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const {
15107 for (const auto *Pred : Preds)
15108 Pred->print(OS, Depth);
15109}
15110
15111void SCEVUnionPredicate::add(const SCEVPredicate *N, ScalarEvolution &SE) {
15112 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N)) {
15113 for (const auto *Pred : Set->Preds)
15114 add(N: Pred, SE);
15115 return;
15116 }
15117
15118 // Only add predicate if it is not already implied by this union predicate.
15119 if (implies(N, SE))
15120 return;
15121
15122 // Build a new vector containing the current predicates, except the ones that
15123 // are implied by the new predicate N.
15124 SmallVector<const SCEVPredicate *> PrunedPreds;
15125 for (auto *P : Preds) {
15126 if (N->implies(N: P, SE))
15127 continue;
15128 PrunedPreds.push_back(Elt: P);
15129 }
15130 Preds = std::move(PrunedPreds);
15131 Preds.push_back(Elt: N);
15132}
15133
15134PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE,
15135 Loop &L)
15136 : SE(SE), L(L) {
15137 SmallVector<const SCEVPredicate*, 4> Empty;
15138 Preds = std::make_unique<SCEVUnionPredicate>(args&: Empty, args&: SE);
15139}
15140
15141void ScalarEvolution::registerUser(const SCEV *User,
15142 ArrayRef<const SCEV *> Ops) {
15143 for (const auto *Op : Ops)
15144 // We do not expect that forgetting cached data for SCEVConstants will ever
15145 // open any prospects for sharpening or introduce any correctness issues,
15146 // so we don't bother storing their dependencies.
15147 if (!isa<SCEVConstant>(Val: Op))
15148 SCEVUsers[Op].insert(Ptr: User);
15149}
15150
15151const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) {
15152 const SCEV *Expr = SE.getSCEV(V);
15153 RewriteEntry &Entry = RewriteMap[Expr];
15154
15155 // If we already have an entry and the version matches, return it.
15156 if (Entry.second && Generation == Entry.first)
15157 return Entry.second;
15158
15159 // We found an entry but it's stale. Rewrite the stale entry
15160 // according to the current predicate.
15161 if (Entry.second)
15162 Expr = Entry.second;
15163
15164 const SCEV *NewSCEV = SE.rewriteUsingPredicate(S: Expr, L: &L, Preds: *Preds);
15165 Entry = {Generation, NewSCEV};
15166
15167 return NewSCEV;
15168}
15169
15170const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
15171 if (!BackedgeCount) {
15172 SmallVector<const SCEVPredicate *, 4> Preds;
15173 BackedgeCount = SE.getPredicatedBackedgeTakenCount(L: &L, Preds);
15174 for (const auto *P : Preds)
15175 addPredicate(Pred: *P);
15176 }
15177 return BackedgeCount;
15178}
15179
15180const SCEV *PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount() {
15181 if (!SymbolicMaxBackedgeCount) {
15182 SmallVector<const SCEVPredicate *, 4> Preds;
15183 SymbolicMaxBackedgeCount =
15184 SE.getPredicatedSymbolicMaxBackedgeTakenCount(L: &L, Preds);
15185 for (const auto *P : Preds)
15186 addPredicate(Pred: *P);
15187 }
15188 return SymbolicMaxBackedgeCount;
15189}
15190
15191unsigned PredicatedScalarEvolution::getSmallConstantMaxTripCount() {
15192 if (!SmallConstantMaxTripCount) {
15193 SmallVector<const SCEVPredicate *, 4> Preds;
15194 SmallConstantMaxTripCount = SE.getSmallConstantMaxTripCount(L: &L, Predicates: &Preds);
15195 for (const auto *P : Preds)
15196 addPredicate(Pred: *P);
15197 }
15198 return *SmallConstantMaxTripCount;
15199}
15200
15201void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
15202 if (Preds->implies(N: &Pred, SE))
15203 return;
15204
15205 SmallVector<const SCEVPredicate *, 4> NewPreds(Preds->getPredicates());
15206 NewPreds.push_back(Elt: &Pred);
15207 Preds = std::make_unique<SCEVUnionPredicate>(args&: NewPreds, args&: SE);
15208 updateGeneration();
15209}
15210
15211const SCEVPredicate &PredicatedScalarEvolution::getPredicate() const {
15212 return *Preds;
15213}
15214
15215void PredicatedScalarEvolution::updateGeneration() {
15216 // If the generation number wrapped recompute everything.
15217 if (++Generation == 0) {
15218 for (auto &II : RewriteMap) {
15219 const SCEV *Rewritten = II.second.second;
15220 II.second = {Generation, SE.rewriteUsingPredicate(S: Rewritten, L: &L, Preds: *Preds)};
15221 }
15222 }
15223}
15224
15225void PredicatedScalarEvolution::setNoOverflow(
15226 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) {
15227 const SCEV *Expr = getSCEV(V);
15228 const auto *AR = cast<SCEVAddRecExpr>(Val: Expr);
15229
15230 auto ImpliedFlags = SCEVWrapPredicate::getImpliedFlags(AR, SE);
15231
15232 // Clear the statically implied flags.
15233 Flags = SCEVWrapPredicate::clearFlags(Flags, OffFlags: ImpliedFlags);
15234 addPredicate(Pred: *SE.getWrapPredicate(AR, AddedFlags: Flags));
15235
15236 auto II = FlagsMap.insert(KV: {V, Flags});
15237 if (!II.second)
15238 II.first->second = SCEVWrapPredicate::setFlags(Flags, OnFlags: II.first->second);
15239}
15240
15241bool PredicatedScalarEvolution::hasNoOverflow(
15242 Value *V, SCEVWrapPredicate::IncrementWrapFlags Flags) {
15243 const SCEV *Expr = getSCEV(V);
15244 const auto *AR = cast<SCEVAddRecExpr>(Val: Expr);
15245
15246 Flags = SCEVWrapPredicate::clearFlags(
15247 Flags, OffFlags: SCEVWrapPredicate::getImpliedFlags(AR, SE));
15248
15249 auto II = FlagsMap.find(Val: V);
15250
15251 if (II != FlagsMap.end())
15252 Flags = SCEVWrapPredicate::clearFlags(Flags, OffFlags: II->second);
15253
15254 return Flags == SCEVWrapPredicate::IncrementAnyWrap;
15255}
15256
15257const SCEVAddRecExpr *PredicatedScalarEvolution::getAsAddRec(Value *V) {
15258 const SCEV *Expr = this->getSCEV(V);
15259 SmallVector<const SCEVPredicate *, 4> NewPreds;
15260 auto *New = SE.convertSCEVToAddRecWithPredicates(S: Expr, L: &L, Preds&: NewPreds);
15261
15262 if (!New)
15263 return nullptr;
15264
15265 for (const auto *P : NewPreds)
15266 addPredicate(Pred: *P);
15267
15268 RewriteMap[SE.getSCEV(V)] = {Generation, New};
15269 return New;
15270}
15271
15272PredicatedScalarEvolution::PredicatedScalarEvolution(
15273 const PredicatedScalarEvolution &Init)
15274 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L),
15275 Preds(std::make_unique<SCEVUnionPredicate>(args: Init.Preds->getPredicates(),
15276 args&: SE)),
15277 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {
15278 for (auto I : Init.FlagsMap)
15279 FlagsMap.insert(KV: I);
15280}
15281
15282void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const {
15283 // For each block.
15284 for (auto *BB : L.getBlocks())
15285 for (auto &I : *BB) {
15286 if (!SE.isSCEVable(Ty: I.getType()))
15287 continue;
15288
15289 auto *Expr = SE.getSCEV(V: &I);
15290 auto II = RewriteMap.find(Val: Expr);
15291
15292 if (II == RewriteMap.end())
15293 continue;
15294
15295 // Don't print things that are not interesting.
15296 if (II->second.second == Expr)
15297 continue;
15298
15299 OS.indent(NumSpaces: Depth) << "[PSE]" << I << ":\n";
15300 OS.indent(NumSpaces: Depth + 2) << *Expr << "\n";
15301 OS.indent(NumSpaces: Depth + 2) << "--> " << *II->second.second << "\n";
15302 }
15303}
15304
15305// Match the mathematical pattern A - (A / B) * B, where A and B can be
15306// arbitrary expressions. Also match zext (trunc A to iB) to iY, which is used
15307// for URem with constant power-of-2 second operands.
15308// It's not always easy, as A and B can be folded (imagine A is X / 2, and B is
15309// 4, A / B becomes X / 8).
15310bool ScalarEvolution::matchURem(const SCEV *Expr, const SCEV *&LHS,
15311 const SCEV *&RHS) {
15312 if (Expr->getType()->isPointerTy())
15313 return false;
15314
15315 // Try to match 'zext (trunc A to iB) to iY', which is used
15316 // for URem with constant power-of-2 second operands. Make sure the size of
15317 // the operand A matches the size of the whole expressions.
15318 if (const auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: Expr))
15319 if (const auto *Trunc = dyn_cast<SCEVTruncateExpr>(Val: ZExt->getOperand(i: 0))) {
15320 LHS = Trunc->getOperand();
15321 // Bail out if the type of the LHS is larger than the type of the
15322 // expression for now.
15323 if (getTypeSizeInBits(Ty: LHS->getType()) >
15324 getTypeSizeInBits(Ty: Expr->getType()))
15325 return false;
15326 if (LHS->getType() != Expr->getType())
15327 LHS = getZeroExtendExpr(Op: LHS, Ty: Expr->getType());
15328 RHS = getConstant(Val: APInt(getTypeSizeInBits(Ty: Expr->getType()), 1)
15329 << getTypeSizeInBits(Ty: Trunc->getType()));
15330 return true;
15331 }
15332 const auto *Add = dyn_cast<SCEVAddExpr>(Val: Expr);
15333 if (Add == nullptr || Add->getNumOperands() != 2)
15334 return false;
15335
15336 const SCEV *A = Add->getOperand(i: 1);
15337 const auto *Mul = dyn_cast<SCEVMulExpr>(Val: Add->getOperand(i: 0));
15338
15339 if (Mul == nullptr)
15340 return false;
15341
15342 const auto MatchURemWithDivisor = [&](const SCEV *B) {
15343 // (SomeExpr + (-(SomeExpr / B) * B)).
15344 if (Expr == getURemExpr(LHS: A, RHS: B)) {
15345 LHS = A;
15346 RHS = B;
15347 return true;
15348 }
15349 return false;
15350 };
15351
15352 // (SomeExpr + (-1 * (SomeExpr / B) * B)).
15353 if (Mul->getNumOperands() == 3 && isa<SCEVConstant>(Val: Mul->getOperand(i: 0)))
15354 return MatchURemWithDivisor(Mul->getOperand(i: 1)) ||
15355 MatchURemWithDivisor(Mul->getOperand(i: 2));
15356
15357 // (SomeExpr + ((-SomeExpr / B) * B)) or (SomeExpr + ((SomeExpr / B) * -B)).
15358 if (Mul->getNumOperands() == 2)
15359 return MatchURemWithDivisor(Mul->getOperand(i: 1)) ||
15360 MatchURemWithDivisor(Mul->getOperand(i: 0)) ||
15361 MatchURemWithDivisor(getNegativeSCEV(V: Mul->getOperand(i: 1))) ||
15362 MatchURemWithDivisor(getNegativeSCEV(V: Mul->getOperand(i: 0)));
15363 return false;
15364}
15365
15366ScalarEvolution::LoopGuards
15367ScalarEvolution::LoopGuards::collect(const Loop *L, ScalarEvolution &SE) {
15368 BasicBlock *Header = L->getHeader();
15369 BasicBlock *Pred = L->getLoopPredecessor();
15370 LoopGuards Guards(SE);
15371 if (!Pred)
15372 return Guards;
15373 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
15374 collectFromBlock(SE, Guards, Block: Header, Pred, VisitedBlocks);
15375 return Guards;
15376}
15377
15378void ScalarEvolution::LoopGuards::collectFromPHI(
15379 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15380 const PHINode &Phi, SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks,
15381 SmallDenseMap<const BasicBlock *, LoopGuards> &IncomingGuards,
15382 unsigned Depth) {
15383 if (!SE.isSCEVable(Ty: Phi.getType()))
15384 return;
15385
15386 using MinMaxPattern = std::pair<const SCEVConstant *, SCEVTypes>;
15387 auto GetMinMaxConst = [&](unsigned IncomingIdx) -> MinMaxPattern {
15388 const BasicBlock *InBlock = Phi.getIncomingBlock(i: IncomingIdx);
15389 if (!VisitedBlocks.insert(Ptr: InBlock).second)
15390 return {nullptr, scCouldNotCompute};
15391 auto [G, Inserted] = IncomingGuards.try_emplace(Key: InBlock, Args: LoopGuards(SE));
15392 if (Inserted)
15393 collectFromBlock(SE, Guards&: G->second, Block: Phi.getParent(), Pred: InBlock, VisitedBlocks,
15394 Depth: Depth + 1);
15395 auto &RewriteMap = G->second.RewriteMap;
15396 if (RewriteMap.empty())
15397 return {nullptr, scCouldNotCompute};
15398 auto S = RewriteMap.find(Val: SE.getSCEV(V: Phi.getIncomingValue(i: IncomingIdx)));
15399 if (S == RewriteMap.end())
15400 return {nullptr, scCouldNotCompute};
15401 auto *SM = dyn_cast_if_present<SCEVMinMaxExpr>(Val: S->second);
15402 if (!SM)
15403 return {nullptr, scCouldNotCompute};
15404 if (const SCEVConstant *C0 = dyn_cast<SCEVConstant>(Val: SM->getOperand(i: 0)))
15405 return {C0, SM->getSCEVType()};
15406 return {nullptr, scCouldNotCompute};
15407 };
15408 auto MergeMinMaxConst = [](MinMaxPattern P1,
15409 MinMaxPattern P2) -> MinMaxPattern {
15410 auto [C1, T1] = P1;
15411 auto [C2, T2] = P2;
15412 if (!C1 || !C2 || T1 != T2)
15413 return {nullptr, scCouldNotCompute};
15414 switch (T1) {
15415 case scUMaxExpr:
15416 return {C1->getAPInt().ult(RHS: C2->getAPInt()) ? C1 : C2, T1};
15417 case scSMaxExpr:
15418 return {C1->getAPInt().slt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15419 case scUMinExpr:
15420 return {C1->getAPInt().ugt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15421 case scSMinExpr:
15422 return {C1->getAPInt().sgt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15423 default:
15424 llvm_unreachable("Trying to merge non-MinMaxExpr SCEVs.");
15425 }
15426 };
15427 auto P = GetMinMaxConst(0);
15428 for (unsigned int In = 1; In < Phi.getNumIncomingValues(); In++) {
15429 if (!P.first)
15430 break;
15431 P = MergeMinMaxConst(P, GetMinMaxConst(In));
15432 }
15433 if (P.first) {
15434 const SCEV *LHS = SE.getSCEV(V: const_cast<PHINode *>(&Phi));
15435 SmallVector<const SCEV *, 2> Ops({P.first, LHS});
15436 const SCEV *RHS = SE.getMinMaxExpr(Kind: P.second, Ops);
15437 Guards.RewriteMap.insert(KV: {LHS, RHS});
15438 }
15439}
15440
15441void ScalarEvolution::LoopGuards::collectFromBlock(
15442 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15443 const BasicBlock *Block, const BasicBlock *Pred,
15444 SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks, unsigned Depth) {
15445 SmallVector<const SCEV *> ExprsToRewrite;
15446 auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS,
15447 const SCEV *RHS,
15448 DenseMap<const SCEV *, const SCEV *>
15449 &RewriteMap) {
15450 // WARNING: It is generally unsound to apply any wrap flags to the proposed
15451 // replacement SCEV which isn't directly implied by the structure of that
15452 // SCEV. In particular, using contextual facts to imply flags is *NOT*
15453 // legal. See the scoping rules for flags in the header to understand why.
15454
15455 // If LHS is a constant, apply information to the other expression.
15456 if (isa<SCEVConstant>(Val: LHS)) {
15457 std::swap(a&: LHS, b&: RHS);
15458 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
15459 }
15460
15461 // Check for a condition of the form (-C1 + X < C2). InstCombine will
15462 // create this form when combining two checks of the form (X u< C2 + C1) and
15463 // (X >=u C1).
15464 auto MatchRangeCheckIdiom = [&SE, Predicate, LHS, RHS, &RewriteMap,
15465 &ExprsToRewrite]() {
15466 const SCEVConstant *C1;
15467 const SCEVUnknown *LHSUnknown;
15468 auto *C2 = dyn_cast<SCEVConstant>(Val: RHS);
15469 if (!match(S: LHS,
15470 P: m_scev_Add(Op0: m_SCEVConstant(V&: C1), Op1: m_SCEVUnknown(V&: LHSUnknown))) ||
15471 !C2)
15472 return false;
15473
15474 auto ExactRegion =
15475 ConstantRange::makeExactICmpRegion(Pred: Predicate, Other: C2->getAPInt())
15476 .sub(Other: C1->getAPInt());
15477
15478 // Bail out, unless we have a non-wrapping, monotonic range.
15479 if (ExactRegion.isWrappedSet() || ExactRegion.isFullSet())
15480 return false;
15481 auto [I, Inserted] = RewriteMap.try_emplace(Key: LHSUnknown);
15482 const SCEV *RewrittenLHS = Inserted ? LHSUnknown : I->second;
15483 I->second = SE.getUMaxExpr(
15484 LHS: SE.getConstant(Val: ExactRegion.getUnsignedMin()),
15485 RHS: SE.getUMinExpr(LHS: RewrittenLHS,
15486 RHS: SE.getConstant(Val: ExactRegion.getUnsignedMax())));
15487 ExprsToRewrite.push_back(Elt: LHSUnknown);
15488 return true;
15489 };
15490 if (MatchRangeCheckIdiom())
15491 return;
15492
15493 // Return true if \p Expr is a MinMax SCEV expression with a non-negative
15494 // constant operand. If so, return in \p SCTy the SCEV type and in \p RHS
15495 // the non-constant operand and in \p LHS the constant operand.
15496 auto IsMinMaxSCEVWithNonNegativeConstant =
15497 [&](const SCEV *Expr, SCEVTypes &SCTy, const SCEV *&LHS,
15498 const SCEV *&RHS) {
15499 if (auto *MinMax = dyn_cast<SCEVMinMaxExpr>(Val: Expr)) {
15500 if (MinMax->getNumOperands() != 2)
15501 return false;
15502 if (auto *C = dyn_cast<SCEVConstant>(Val: MinMax->getOperand(i: 0))) {
15503 if (C->getAPInt().isNegative())
15504 return false;
15505 SCTy = MinMax->getSCEVType();
15506 LHS = MinMax->getOperand(i: 0);
15507 RHS = MinMax->getOperand(i: 1);
15508 return true;
15509 }
15510 }
15511 return false;
15512 };
15513
15514 // Checks whether Expr is a non-negative constant, and Divisor is a positive
15515 // constant, and returns their APInt in ExprVal and in DivisorVal.
15516 auto GetNonNegExprAndPosDivisor = [&](const SCEV *Expr, const SCEV *Divisor,
15517 APInt &ExprVal, APInt &DivisorVal) {
15518 auto *ConstExpr = dyn_cast<SCEVConstant>(Val: Expr);
15519 auto *ConstDivisor = dyn_cast<SCEVConstant>(Val: Divisor);
15520 if (!ConstExpr || !ConstDivisor)
15521 return false;
15522 ExprVal = ConstExpr->getAPInt();
15523 DivisorVal = ConstDivisor->getAPInt();
15524 return ExprVal.isNonNegative() && !DivisorVal.isNonPositive();
15525 };
15526
15527 // Return a new SCEV that modifies \p Expr to the closest number divides by
15528 // \p Divisor and greater or equal than Expr.
15529 // For now, only handle constant Expr and Divisor.
15530 auto GetNextSCEVDividesByDivisor = [&](const SCEV *Expr,
15531 const SCEV *Divisor) {
15532 APInt ExprVal;
15533 APInt DivisorVal;
15534 if (!GetNonNegExprAndPosDivisor(Expr, Divisor, ExprVal, DivisorVal))
15535 return Expr;
15536 APInt Rem = ExprVal.urem(RHS: DivisorVal);
15537 if (!Rem.isZero())
15538 // return the SCEV: Expr + Divisor - Expr % Divisor
15539 return SE.getConstant(Val: ExprVal + DivisorVal - Rem);
15540 return Expr;
15541 };
15542
15543 // Return a new SCEV that modifies \p Expr to the closest number divides by
15544 // \p Divisor and less or equal than Expr.
15545 // For now, only handle constant Expr and Divisor.
15546 auto GetPreviousSCEVDividesByDivisor = [&](const SCEV *Expr,
15547 const SCEV *Divisor) {
15548 APInt ExprVal;
15549 APInt DivisorVal;
15550 if (!GetNonNegExprAndPosDivisor(Expr, Divisor, ExprVal, DivisorVal))
15551 return Expr;
15552 APInt Rem = ExprVal.urem(RHS: DivisorVal);
15553 // return the SCEV: Expr - Expr % Divisor
15554 return SE.getConstant(Val: ExprVal - Rem);
15555 };
15556
15557 // Apply divisibilty by \p Divisor on MinMaxExpr with constant values,
15558 // recursively. This is done by aligning up/down the constant value to the
15559 // Divisor.
15560 std::function<const SCEV *(const SCEV *, const SCEV *)>
15561 ApplyDivisibiltyOnMinMaxExpr = [&](const SCEV *MinMaxExpr,
15562 const SCEV *Divisor) {
15563 const SCEV *MinMaxLHS = nullptr, *MinMaxRHS = nullptr;
15564 SCEVTypes SCTy;
15565 if (!IsMinMaxSCEVWithNonNegativeConstant(MinMaxExpr, SCTy, MinMaxLHS,
15566 MinMaxRHS))
15567 return MinMaxExpr;
15568 auto IsMin =
15569 isa<SCEVSMinExpr>(Val: MinMaxExpr) || isa<SCEVUMinExpr>(Val: MinMaxExpr);
15570 assert(SE.isKnownNonNegative(MinMaxLHS) &&
15571 "Expected non-negative operand!");
15572 auto *DivisibleExpr =
15573 IsMin ? GetPreviousSCEVDividesByDivisor(MinMaxLHS, Divisor)
15574 : GetNextSCEVDividesByDivisor(MinMaxLHS, Divisor);
15575 SmallVector<const SCEV *> Ops = {
15576 ApplyDivisibiltyOnMinMaxExpr(MinMaxRHS, Divisor), DivisibleExpr};
15577 return SE.getMinMaxExpr(Kind: SCTy, Ops);
15578 };
15579
15580 // If we have LHS == 0, check if LHS is computing a property of some unknown
15581 // SCEV %v which we can rewrite %v to express explicitly.
15582 if (Predicate == CmpInst::ICMP_EQ && match(S: RHS, P: m_scev_Zero())) {
15583 // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to
15584 // explicitly express that.
15585 const SCEV *URemLHS = nullptr;
15586 const SCEV *URemRHS = nullptr;
15587 if (SE.matchURem(Expr: LHS, LHS&: URemLHS, RHS&: URemRHS)) {
15588 if (const SCEVUnknown *LHSUnknown = dyn_cast<SCEVUnknown>(Val: URemLHS)) {
15589 auto I = RewriteMap.find(Val: LHSUnknown);
15590 const SCEV *RewrittenLHS =
15591 I != RewriteMap.end() ? I->second : LHSUnknown;
15592 RewrittenLHS = ApplyDivisibiltyOnMinMaxExpr(RewrittenLHS, URemRHS);
15593 const auto *Multiple =
15594 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: RewrittenLHS, RHS: URemRHS), RHS: URemRHS);
15595 RewriteMap[LHSUnknown] = Multiple;
15596 ExprsToRewrite.push_back(Elt: LHSUnknown);
15597 return;
15598 }
15599 }
15600 }
15601
15602 // Do not apply information for constants or if RHS contains an AddRec.
15603 if (isa<SCEVConstant>(Val: LHS) || SE.containsAddRecurrence(S: RHS))
15604 return;
15605
15606 // If RHS is SCEVUnknown, make sure the information is applied to it.
15607 if (!isa<SCEVUnknown>(Val: LHS) && isa<SCEVUnknown>(Val: RHS)) {
15608 std::swap(a&: LHS, b&: RHS);
15609 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
15610 }
15611
15612 // Puts rewrite rule \p From -> \p To into the rewrite map. Also if \p From
15613 // and \p FromRewritten are the same (i.e. there has been no rewrite
15614 // registered for \p From), then puts this value in the list of rewritten
15615 // expressions.
15616 auto AddRewrite = [&](const SCEV *From, const SCEV *FromRewritten,
15617 const SCEV *To) {
15618 if (From == FromRewritten)
15619 ExprsToRewrite.push_back(Elt: From);
15620 RewriteMap[From] = To;
15621 };
15622
15623 // Checks whether \p S has already been rewritten. In that case returns the
15624 // existing rewrite because we want to chain further rewrites onto the
15625 // already rewritten value. Otherwise returns \p S.
15626 auto GetMaybeRewritten = [&](const SCEV *S) {
15627 auto I = RewriteMap.find(Val: S);
15628 return I != RewriteMap.end() ? I->second : S;
15629 };
15630
15631 // Check for the SCEV expression (A /u B) * B while B is a constant, inside
15632 // \p Expr. The check is done recuresively on \p Expr, which is assumed to
15633 // be a composition of Min/Max SCEVs. Return whether the SCEV expression (A
15634 // /u B) * B was found, and return the divisor B in \p DividesBy. For
15635 // example, if Expr = umin (umax ((A /u 8) * 8, 16), 64), return true since
15636 // (A /u 8) * 8 matched the pattern, and return the constant SCEV 8 in \p
15637 // DividesBy.
15638 std::function<bool(const SCEV *, const SCEV *&)> HasDivisibiltyInfo =
15639 [&](const SCEV *Expr, const SCEV *&DividesBy) {
15640 if (auto *Mul = dyn_cast<SCEVMulExpr>(Val: Expr)) {
15641 if (Mul->getNumOperands() != 2)
15642 return false;
15643 auto *MulLHS = Mul->getOperand(i: 0);
15644 auto *MulRHS = Mul->getOperand(i: 1);
15645 if (isa<SCEVConstant>(Val: MulLHS))
15646 std::swap(a&: MulLHS, b&: MulRHS);
15647 if (auto *Div = dyn_cast<SCEVUDivExpr>(Val: MulLHS))
15648 if (Div->getOperand(i: 1) == MulRHS) {
15649 DividesBy = MulRHS;
15650 return true;
15651 }
15652 }
15653 if (auto *MinMax = dyn_cast<SCEVMinMaxExpr>(Val: Expr))
15654 return HasDivisibiltyInfo(MinMax->getOperand(i: 0), DividesBy) ||
15655 HasDivisibiltyInfo(MinMax->getOperand(i: 1), DividesBy);
15656 return false;
15657 };
15658
15659 // Return true if Expr known to divide by \p DividesBy.
15660 std::function<bool(const SCEV *, const SCEV *&)> IsKnownToDivideBy =
15661 [&](const SCEV *Expr, const SCEV *DividesBy) {
15662 if (SE.getURemExpr(LHS: Expr, RHS: DividesBy)->isZero())
15663 return true;
15664 if (auto *MinMax = dyn_cast<SCEVMinMaxExpr>(Val: Expr))
15665 return IsKnownToDivideBy(MinMax->getOperand(i: 0), DividesBy) &&
15666 IsKnownToDivideBy(MinMax->getOperand(i: 1), DividesBy);
15667 return false;
15668 };
15669
15670 const SCEV *RewrittenLHS = GetMaybeRewritten(LHS);
15671 const SCEV *DividesBy = nullptr;
15672 if (HasDivisibiltyInfo(RewrittenLHS, DividesBy))
15673 // Check that the whole expression is divided by DividesBy
15674 DividesBy =
15675 IsKnownToDivideBy(RewrittenLHS, DividesBy) ? DividesBy : nullptr;
15676
15677 // Collect rewrites for LHS and its transitive operands based on the
15678 // condition.
15679 // For min/max expressions, also apply the guard to its operands:
15680 // 'min(a, b) >= c' -> '(a >= c) and (b >= c)',
15681 // 'min(a, b) > c' -> '(a > c) and (b > c)',
15682 // 'max(a, b) <= c' -> '(a <= c) and (b <= c)',
15683 // 'max(a, b) < c' -> '(a < c) and (b < c)'.
15684
15685 // We cannot express strict predicates in SCEV, so instead we replace them
15686 // with non-strict ones against plus or minus one of RHS depending on the
15687 // predicate.
15688 const SCEV *One = SE.getOne(Ty: RHS->getType());
15689 switch (Predicate) {
15690 case CmpInst::ICMP_ULT:
15691 if (RHS->getType()->isPointerTy())
15692 return;
15693 RHS = SE.getUMaxExpr(LHS: RHS, RHS: One);
15694 [[fallthrough]];
15695 case CmpInst::ICMP_SLT: {
15696 RHS = SE.getMinusSCEV(LHS: RHS, RHS: One);
15697 RHS = DividesBy ? GetPreviousSCEVDividesByDivisor(RHS, DividesBy) : RHS;
15698 break;
15699 }
15700 case CmpInst::ICMP_UGT:
15701 case CmpInst::ICMP_SGT:
15702 RHS = SE.getAddExpr(LHS: RHS, RHS: One);
15703 RHS = DividesBy ? GetNextSCEVDividesByDivisor(RHS, DividesBy) : RHS;
15704 break;
15705 case CmpInst::ICMP_ULE:
15706 case CmpInst::ICMP_SLE:
15707 RHS = DividesBy ? GetPreviousSCEVDividesByDivisor(RHS, DividesBy) : RHS;
15708 break;
15709 case CmpInst::ICMP_UGE:
15710 case CmpInst::ICMP_SGE:
15711 RHS = DividesBy ? GetNextSCEVDividesByDivisor(RHS, DividesBy) : RHS;
15712 break;
15713 default:
15714 break;
15715 }
15716
15717 SmallVector<const SCEV *, 16> Worklist(1, LHS);
15718 SmallPtrSet<const SCEV *, 16> Visited;
15719
15720 auto EnqueueOperands = [&Worklist](const SCEVNAryExpr *S) {
15721 append_range(C&: Worklist, R: S->operands());
15722 };
15723
15724 while (!Worklist.empty()) {
15725 const SCEV *From = Worklist.pop_back_val();
15726 if (isa<SCEVConstant>(Val: From))
15727 continue;
15728 if (!Visited.insert(Ptr: From).second)
15729 continue;
15730 const SCEV *FromRewritten = GetMaybeRewritten(From);
15731 const SCEV *To = nullptr;
15732
15733 switch (Predicate) {
15734 case CmpInst::ICMP_ULT:
15735 case CmpInst::ICMP_ULE:
15736 To = SE.getUMinExpr(LHS: FromRewritten, RHS);
15737 if (auto *UMax = dyn_cast<SCEVUMaxExpr>(Val: FromRewritten))
15738 EnqueueOperands(UMax);
15739 break;
15740 case CmpInst::ICMP_SLT:
15741 case CmpInst::ICMP_SLE:
15742 To = SE.getSMinExpr(LHS: FromRewritten, RHS);
15743 if (auto *SMax = dyn_cast<SCEVSMaxExpr>(Val: FromRewritten))
15744 EnqueueOperands(SMax);
15745 break;
15746 case CmpInst::ICMP_UGT:
15747 case CmpInst::ICMP_UGE:
15748 To = SE.getUMaxExpr(LHS: FromRewritten, RHS);
15749 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: FromRewritten))
15750 EnqueueOperands(UMin);
15751 break;
15752 case CmpInst::ICMP_SGT:
15753 case CmpInst::ICMP_SGE:
15754 To = SE.getSMaxExpr(LHS: FromRewritten, RHS);
15755 if (auto *SMin = dyn_cast<SCEVSMinExpr>(Val: FromRewritten))
15756 EnqueueOperands(SMin);
15757 break;
15758 case CmpInst::ICMP_EQ:
15759 if (isa<SCEVConstant>(Val: RHS))
15760 To = RHS;
15761 break;
15762 case CmpInst::ICMP_NE:
15763 if (match(S: RHS, P: m_scev_Zero())) {
15764 const SCEV *OneAlignedUp =
15765 DividesBy ? GetNextSCEVDividesByDivisor(One, DividesBy) : One;
15766 To = SE.getUMaxExpr(LHS: FromRewritten, RHS: OneAlignedUp);
15767 }
15768 break;
15769 default:
15770 break;
15771 }
15772
15773 if (To)
15774 AddRewrite(From, FromRewritten, To);
15775 }
15776 };
15777
15778 SmallVector<PointerIntPair<Value *, 1, bool>> Terms;
15779 // First, collect information from assumptions dominating the loop.
15780 for (auto &AssumeVH : SE.AC.assumptions()) {
15781 if (!AssumeVH)
15782 continue;
15783 auto *AssumeI = cast<CallInst>(Val&: AssumeVH);
15784 if (!SE.DT.dominates(Def: AssumeI, BB: Block))
15785 continue;
15786 Terms.emplace_back(Args: AssumeI->getOperand(i_nocapture: 0), Args: true);
15787 }
15788
15789 // Second, collect information from llvm.experimental.guards dominating the loop.
15790 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
15791 M: SE.F.getParent(), id: Intrinsic::experimental_guard);
15792 if (GuardDecl)
15793 for (const auto *GU : GuardDecl->users())
15794 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
15795 if (Guard->getFunction() == Block->getParent() &&
15796 SE.DT.dominates(Def: Guard, BB: Block))
15797 Terms.emplace_back(Args: Guard->getArgOperand(i: 0), Args: true);
15798
15799 // Third, collect conditions from dominating branches. Starting at the loop
15800 // predecessor, climb up the predecessor chain, as long as there are
15801 // predecessors that can be found that have unique successors leading to the
15802 // original header.
15803 // TODO: share this logic with isLoopEntryGuardedByCond.
15804 unsigned NumCollectedConditions = 0;
15805 VisitedBlocks.insert(Ptr: Block);
15806 std::pair<const BasicBlock *, const BasicBlock *> Pair(Pred, Block);
15807 for (; Pair.first;
15808 Pair = SE.getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
15809 VisitedBlocks.insert(Ptr: Pair.second);
15810 const BranchInst *LoopEntryPredicate =
15811 dyn_cast<BranchInst>(Val: Pair.first->getTerminator());
15812 if (!LoopEntryPredicate || LoopEntryPredicate->isUnconditional())
15813 continue;
15814
15815 Terms.emplace_back(Args: LoopEntryPredicate->getCondition(),
15816 Args: LoopEntryPredicate->getSuccessor(i: 0) == Pair.second);
15817 NumCollectedConditions++;
15818
15819 // If we are recursively collecting guards stop after 2
15820 // conditions to limit compile-time impact for now.
15821 if (Depth > 0 && NumCollectedConditions == 2)
15822 break;
15823 }
15824 // Finally, if we stopped climbing the predecessor chain because
15825 // there wasn't a unique one to continue, try to collect conditions
15826 // for PHINodes by recursively following all of their incoming
15827 // blocks and try to merge the found conditions to build a new one
15828 // for the Phi.
15829 if (Pair.second->hasNPredecessorsOrMore(N: 2) &&
15830 Depth < MaxLoopGuardCollectionDepth) {
15831 SmallDenseMap<const BasicBlock *, LoopGuards> IncomingGuards;
15832 for (auto &Phi : Pair.second->phis())
15833 collectFromPHI(SE, Guards, Phi, VisitedBlocks, IncomingGuards, Depth);
15834 }
15835
15836 // Now apply the information from the collected conditions to
15837 // Guards.RewriteMap. Conditions are processed in reverse order, so the
15838 // earliest conditions is processed first. This ensures the SCEVs with the
15839 // shortest dependency chains are constructed first.
15840 for (auto [Term, EnterIfTrue] : reverse(C&: Terms)) {
15841 SmallVector<Value *, 8> Worklist;
15842 SmallPtrSet<Value *, 8> Visited;
15843 Worklist.push_back(Elt: Term);
15844 while (!Worklist.empty()) {
15845 Value *Cond = Worklist.pop_back_val();
15846 if (!Visited.insert(Ptr: Cond).second)
15847 continue;
15848
15849 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
15850 auto Predicate =
15851 EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate();
15852 const auto *LHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 0));
15853 const auto *RHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 1));
15854 CollectCondition(Predicate, LHS, RHS, Guards.RewriteMap);
15855 continue;
15856 }
15857
15858 Value *L, *R;
15859 if (EnterIfTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: L), R: m_Value(V&: R)))
15860 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: L), R: m_Value(V&: R)))) {
15861 Worklist.push_back(Elt: L);
15862 Worklist.push_back(Elt: R);
15863 }
15864 }
15865 }
15866
15867 // Let the rewriter preserve NUW/NSW flags if the unsigned/signed ranges of
15868 // the replacement expressions are contained in the ranges of the replaced
15869 // expressions.
15870 Guards.PreserveNUW = true;
15871 Guards.PreserveNSW = true;
15872 for (const SCEV *Expr : ExprsToRewrite) {
15873 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
15874 Guards.PreserveNUW &=
15875 SE.getUnsignedRange(S: Expr).contains(CR: SE.getUnsignedRange(S: RewriteTo));
15876 Guards.PreserveNSW &=
15877 SE.getSignedRange(S: Expr).contains(CR: SE.getSignedRange(S: RewriteTo));
15878 }
15879
15880 // Now that all rewrite information is collect, rewrite the collected
15881 // expressions with the information in the map. This applies information to
15882 // sub-expressions.
15883 if (ExprsToRewrite.size() > 1) {
15884 for (const SCEV *Expr : ExprsToRewrite) {
15885 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
15886 Guards.RewriteMap.erase(Val: Expr);
15887 Guards.RewriteMap.insert(KV: {Expr, Guards.rewrite(Expr: RewriteTo)});
15888 }
15889 }
15890}
15891
15892const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const {
15893 /// A rewriter to replace SCEV expressions in Map with the corresponding entry
15894 /// in the map. It skips AddRecExpr because we cannot guarantee that the
15895 /// replacement is loop invariant in the loop of the AddRec.
15896 class SCEVLoopGuardRewriter
15897 : public SCEVRewriteVisitor<SCEVLoopGuardRewriter> {
15898 const DenseMap<const SCEV *, const SCEV *> &Map;
15899
15900 SCEV::NoWrapFlags FlagMask = SCEV::FlagAnyWrap;
15901
15902 public:
15903 SCEVLoopGuardRewriter(ScalarEvolution &SE,
15904 const ScalarEvolution::LoopGuards &Guards)
15905 : SCEVRewriteVisitor(SE), Map(Guards.RewriteMap) {
15906 if (Guards.PreserveNUW)
15907 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNUW);
15908 if (Guards.PreserveNSW)
15909 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNSW);
15910 }
15911
15912 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
15913
15914 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
15915 return Map.lookup_or(Val: Expr, Default&: Expr);
15916 }
15917
15918 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
15919 if (const SCEV *S = Map.lookup(Val: Expr))
15920 return S;
15921
15922 // If we didn't find the extact ZExt expr in the map, check if there's
15923 // an entry for a smaller ZExt we can use instead.
15924 Type *Ty = Expr->getType();
15925 const SCEV *Op = Expr->getOperand(i: 0);
15926 unsigned Bitwidth = Ty->getScalarSizeInBits() / 2;
15927 while (Bitwidth % 8 == 0 && Bitwidth >= 8 &&
15928 Bitwidth > Op->getType()->getScalarSizeInBits()) {
15929 Type *NarrowTy = IntegerType::get(C&: SE.getContext(), NumBits: Bitwidth);
15930 auto *NarrowExt = SE.getZeroExtendExpr(Op, Ty: NarrowTy);
15931 auto I = Map.find(Val: NarrowExt);
15932 if (I != Map.end())
15933 return SE.getZeroExtendExpr(Op: I->second, Ty);
15934 Bitwidth = Bitwidth / 2;
15935 }
15936
15937 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitZeroExtendExpr(
15938 Expr);
15939 }
15940
15941 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
15942 if (const SCEV *S = Map.lookup(Val: Expr))
15943 return S;
15944 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSignExtendExpr(
15945 Expr);
15946 }
15947
15948 const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) {
15949 if (const SCEV *S = Map.lookup(Val: Expr))
15950 return S;
15951 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitUMinExpr(Expr);
15952 }
15953
15954 const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) {
15955 if (const SCEV *S = Map.lookup(Val: Expr))
15956 return S;
15957 return SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visitSMinExpr(Expr);
15958 }
15959
15960 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
15961 SmallVector<const SCEV *, 2> Operands;
15962 bool Changed = false;
15963 for (const auto *Op : Expr->operands()) {
15964 Operands.push_back(
15965 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
15966 Changed |= Op != Operands.back();
15967 }
15968 // We are only replacing operands with equivalent values, so transfer the
15969 // flags from the original expression.
15970 return !Changed ? Expr
15971 : SE.getAddExpr(Ops&: Operands,
15972 OrigFlags: ScalarEvolution::maskFlags(
15973 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
15974 }
15975
15976 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
15977 SmallVector<const SCEV *, 2> Operands;
15978 bool Changed = false;
15979 for (const auto *Op : Expr->operands()) {
15980 Operands.push_back(
15981 Elt: SCEVRewriteVisitor<SCEVLoopGuardRewriter>::visit(S: Op));
15982 Changed |= Op != Operands.back();
15983 }
15984 // We are only replacing operands with equivalent values, so transfer the
15985 // flags from the original expression.
15986 return !Changed ? Expr
15987 : SE.getMulExpr(Ops&: Operands,
15988 OrigFlags: ScalarEvolution::maskFlags(
15989 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
15990 }
15991 };
15992
15993 if (RewriteMap.empty())
15994 return Expr;
15995
15996 SCEVLoopGuardRewriter Rewriter(SE, *this);
15997 return Rewriter.visit(S: Expr);
15998}
15999
16000const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
16001 return applyLoopGuards(Expr, Guards: LoopGuards::collect(L, SE&: *this));
16002}
16003
16004const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr,
16005 const LoopGuards &Guards) {
16006 return Guards.rewrite(Expr);
16007}
16008