1//===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the scalar evolution analysis
10// engine, which is used primarily to analyze expressions involving induction
11// variables in loops.
12//
13// There are several aspects to this library. First is the representation of
14// scalar expressions, which are represented as subclasses of the SCEV class.
15// These classes are used to represent certain types of subexpressions that we
16// can handle. We only create one SCEV of a particular shape, so
17// pointer-comparisons for equality are legal.
18//
19// One important aspect of the SCEV objects is that they are never cyclic, even
20// if there is a cycle in the dataflow for an expression (ie, a PHI node). If
21// the PHI node is one of the idioms that we can represent (e.g., a polynomial
22// recurrence) then we represent it directly as a recurrence node, otherwise we
23// represent it as a SCEVUnknown node.
24//
25// In addition to being able to represent expressions of various types, we also
26// have folders that are used to build the *canonical* representation for a
27// particular expression. These folders are capable of using a variety of
28// rewrite rules to simplify the expressions.
29//
30// Once the folders are defined, we can implement the more interesting
31// higher-level code, such as the code that recognizes PHI nodes of various
32// types, computes the execution count of a loop, etc.
33//
34// TODO: We should use these routines and value representations to implement
35// dependence analysis!
36//
37//===----------------------------------------------------------------------===//
38//
39// There are several good references for the techniques used in this analysis.
40//
41// Chains of recurrences -- a method to expedite the evaluation
42// of closed-form functions
43// Olaf Bachmann, Paul S. Wang, Eugene V. Zima
44//
45// On computational properties of chains of recurrences
46// Eugene V. Zima
47//
48// Symbolic Evaluation of Chains of Recurrences for Loop Optimization
49// Robert A. van Engelen
50//
51// Efficient Symbolic Analysis for Optimizing Compilers
52// Robert A. van Engelen
53//
54// Using the chains of recurrences algebra for data dependence testing and
55// induction variable substitution
56// MS Thesis, Johnie Birch
57//
58//===----------------------------------------------------------------------===//
59
60#include "llvm/Analysis/ScalarEvolution.h"
61#include "AnalysisOptions.h"
62#include "llvm/ADT/APInt.h"
63#include "llvm/ADT/ArrayRef.h"
64#include "llvm/ADT/DenseMap.h"
65#include "llvm/ADT/DepthFirstIterator.h"
66#include "llvm/ADT/FoldingSet.h"
67#include "llvm/ADT/STLExtras.h"
68#include "llvm/ADT/ScopeExit.h"
69#include "llvm/ADT/Sequence.h"
70#include "llvm/ADT/SmallPtrSet.h"
71#include "llvm/ADT/SmallVector.h"
72#include "llvm/ADT/Statistic.h"
73#include "llvm/ADT/StringExtras.h"
74#include "llvm/ADT/StringRef.h"
75#include "llvm/Analysis/AssumptionCache.h"
76#include "llvm/Analysis/ConstantFolding.h"
77#include "llvm/Analysis/InstructionSimplify.h"
78#include "llvm/Analysis/LoopInfo.h"
79#include "llvm/Analysis/MemoryBuiltins.h"
80#include "llvm/Analysis/ScalarEvolutionExpressions.h"
81#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
82#include "llvm/Analysis/TargetLibraryInfo.h"
83#include "llvm/Analysis/ValueTracking.h"
84#include "llvm/Config/llvm-config.h"
85#include "llvm/IR/Argument.h"
86#include "llvm/IR/BasicBlock.h"
87#include "llvm/IR/CFG.h"
88#include "llvm/IR/Constant.h"
89#include "llvm/IR/ConstantRange.h"
90#include "llvm/IR/Constants.h"
91#include "llvm/IR/DataLayout.h"
92#include "llvm/IR/DerivedTypes.h"
93#include "llvm/IR/Dominators.h"
94#include "llvm/IR/Function.h"
95#include "llvm/IR/GlobalAlias.h"
96#include "llvm/IR/GlobalValue.h"
97#include "llvm/IR/InstIterator.h"
98#include "llvm/IR/InstrTypes.h"
99#include "llvm/IR/Instruction.h"
100#include "llvm/IR/Instructions.h"
101#include "llvm/IR/IntrinsicInst.h"
102#include "llvm/IR/Intrinsics.h"
103#include "llvm/IR/LLVMContext.h"
104#include "llvm/IR/Operator.h"
105#include "llvm/IR/PatternMatch.h"
106#include "llvm/IR/Type.h"
107#include "llvm/IR/Use.h"
108#include "llvm/IR/User.h"
109#include "llvm/IR/Value.h"
110#include "llvm/IR/Verifier.h"
111#include "llvm/InitializePasses.h"
112#include "llvm/Pass.h"
113#include "llvm/Support/Casting.h"
114#include "llvm/Support/CommandLine.h"
115#include "llvm/Support/Compiler.h"
116#include "llvm/Support/Debug.h"
117#include "llvm/Support/ErrorHandling.h"
118#include "llvm/Support/InterleavedRange.h"
119#include "llvm/Support/KnownBits.h"
120#include "llvm/Support/SaveAndRestore.h"
121#include "llvm/Support/raw_ostream.h"
122#include <algorithm>
123#include <cassert>
124#include <climits>
125#include <cstdint>
126#include <cstdlib>
127#include <map>
128#include <memory>
129#include <numeric>
130#include <optional>
131#include <tuple>
132#include <utility>
133#include <vector>
134
135using namespace llvm;
136using namespace PatternMatch;
137using namespace SCEVPatternMatch;
138
139#define DEBUG_TYPE "scalar-evolution"
140
141STATISTIC(NumExitCountsComputed,
142 "Number of loop exits with predictable exit counts");
143STATISTIC(NumExitCountsNotComputed,
144 "Number of loop exits without predictable exit counts");
145STATISTIC(NumBruteForceTripCountsComputed,
146 "Number of loops with trip counts computed by force");
147
148#ifdef EXPENSIVE_CHECKS
149bool llvm::VerifySCEV = true;
150#else
151bool llvm::VerifySCEV = false;
152#endif
153
154static cl::opt<bool, true> VerifySCEVOpt(
155 "verify-scev", cl::Hidden, cl::location(L&: VerifySCEV),
156 cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
157
158//===----------------------------------------------------------------------===//
159// SCEV class definitions
160//===----------------------------------------------------------------------===//
161
162void SCEV::computeAndSetCanonical(ScalarEvolution &SE) {
163 // Leaf nodes are always their own canonical.
164 switch (getSCEVType()) {
165 case scConstant:
166 case scVScale:
167 case scUnknown:
168 CanonicalSCEV = this;
169 return;
170 default:
171 break;
172 }
173
174 // For all other expressions, check whether any immediate operand has a
175 // different canonical. Since operands are always created before their parent,
176 // their canonical pointers are already set — no recursion needed.
177 if (all_of(Range: operands(), P: [](SCEVUse Op) { return Op.isCanonical(); })) {
178 CanonicalSCEV = this;
179 return;
180 }
181
182 SmallVector<SCEVUse, 4> CanonOps(
183 map_range(C: operands(), F: [](SCEVUse Op) { return Op.getCanonical(); }));
184 // Rebuild the expression from the canonical operands, stripping use flags.
185 CanonicalSCEV = SE.getWithOperands(S: this, NewOps&: CanonOps);
186}
187
188//===----------------------------------------------------------------------===//
189// Implementation of the SCEV class.
190//
191
192#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
193LLVM_DUMP_METHOD void SCEV::dump() const {
194 print(dbgs());
195 dbgs() << '\n';
196}
197#endif
198
199void SCEV::print(raw_ostream &OS) const {
200 switch (getSCEVType()) {
201 case scConstant:
202 cast<SCEVConstant>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
203 return;
204 case scVScale:
205 OS << "vscale";
206 return;
207 case scPtrToAddr: {
208 const SCEVCastExpr *PtrCast = cast<SCEVCastExpr>(Val: this);
209 SCEVUse Op = PtrCast->getOperand();
210 OS << "(ptrtoaddr " << *Op->getType() << " " << Op << " to "
211 << *PtrCast->getType() << ")";
212 return;
213 }
214 case scTruncate: {
215 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: this);
216 SCEVUse Op = Trunc->getOperand();
217 OS << "(trunc " << *Op->getType() << " " << Op << " to "
218 << *Trunc->getType() << ")";
219 return;
220 }
221 case scZeroExtend: {
222 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: this);
223 SCEVUse Op = ZExt->getOperand();
224 OS << "(zext " << *Op->getType() << " " << Op << " to " << *ZExt->getType()
225 << ")";
226 return;
227 }
228 case scSignExtend: {
229 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: this);
230 SCEVUse Op = SExt->getOperand();
231 OS << "(sext " << *Op->getType() << " " << Op << " to " << *SExt->getType()
232 << ")";
233 return;
234 }
235 case scAddRecExpr: {
236 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: this);
237 OS << "{" << AR->getOperand(i: 0);
238 for (unsigned i = 1, e = AR->getNumOperands(); i != e; ++i)
239 OS << ",+," << AR->getOperand(i);
240 OS << "}<";
241 if (AR->hasNoUnsignedWrap())
242 OS << "nuw><";
243 if (AR->hasNoSignedWrap())
244 OS << "nsw><";
245 if (AR->hasNoSelfWrap() && !AR->hasNoUnsignedWrap() &&
246 !AR->hasNoSignedWrap())
247 OS << "nw><";
248 AR->getLoop()->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
249 OS << ">";
250 return;
251 }
252 case scAddExpr:
253 case scMulExpr:
254 case scUMaxExpr:
255 case scSMaxExpr:
256 case scUMinExpr:
257 case scSMinExpr:
258 case scSequentialUMinExpr: {
259 const SCEVNAryExpr *NAry = cast<SCEVNAryExpr>(Val: this);
260 const char *OpStr = nullptr;
261 switch (NAry->getSCEVType()) {
262 case scAddExpr: OpStr = " + "; break;
263 case scMulExpr: OpStr = " * "; break;
264 case scUMaxExpr: OpStr = " umax "; break;
265 case scSMaxExpr: OpStr = " smax "; break;
266 case scUMinExpr:
267 OpStr = " umin ";
268 break;
269 case scSMinExpr:
270 OpStr = " smin ";
271 break;
272 case scSequentialUMinExpr:
273 OpStr = " umin_seq ";
274 break;
275 default:
276 llvm_unreachable("There are no other nary expression types.");
277 }
278 OS << "(" << llvm::interleaved(R: NAry->operands(), Separator: OpStr) << ")";
279 switch (NAry->getSCEVType()) {
280 case scAddExpr:
281 case scMulExpr:
282 if (NAry->hasNoUnsignedWrap())
283 OS << "<nuw>";
284 if (NAry->hasNoSignedWrap())
285 OS << "<nsw>";
286 break;
287 default:
288 // Nothing to print for other nary expressions.
289 break;
290 }
291 return;
292 }
293 case scUDivExpr: {
294 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: this);
295 OS << "(" << UDiv->getLHS() << " /u " << UDiv->getRHS() << ")";
296 return;
297 }
298 case scUnknown:
299 cast<SCEVUnknown>(Val: this)->getValue()->printAsOperand(O&: OS, PrintType: false);
300 return;
301 case scCouldNotCompute:
302 OS << "***COULDNOTCOMPUTE***";
303 return;
304 }
305 llvm_unreachable("Unknown SCEV kind!");
306}
307
308ArrayRef<SCEVUse> SCEV::operands() const {
309 switch (getSCEVType()) {
310 case scConstant:
311 case scVScale:
312 case scUnknown:
313 return {};
314 case scPtrToAddr:
315 case scTruncate:
316 case scZeroExtend:
317 case scSignExtend:
318 return cast<SCEVCastExpr>(Val: this)->operands();
319 case scAddRecExpr:
320 case scAddExpr:
321 case scMulExpr:
322 case scUMaxExpr:
323 case scSMaxExpr:
324 case scUMinExpr:
325 case scSMinExpr:
326 case scSequentialUMinExpr:
327 return cast<SCEVNAryExpr>(Val: this)->operands();
328 case scUDivExpr:
329 return cast<SCEVUDivExpr>(Val: this)->operands();
330 case scCouldNotCompute:
331 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
332 }
333 llvm_unreachable("Unknown SCEV kind!");
334}
335
336bool SCEV::isZero() const { return match(S: this, P: m_scev_Zero()); }
337
338bool SCEV::isOne() const { return match(S: this, P: m_scev_One()); }
339
340bool SCEV::isAllOnesValue() const { return match(S: this, P: m_scev_AllOnes()); }
341
342bool SCEV::isNonConstantNegative() const {
343 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: this);
344 if (!Mul) return false;
345
346 // If there is a constant factor, it will be first.
347 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: Mul->getOperand(i: 0));
348 if (!SC) return false;
349
350 // Return true if the value is negative, this matches things like (-42 * V).
351 return SC->getAPInt().isNegative();
352}
353
354SCEVCouldNotCompute::SCEVCouldNotCompute()
355 : SCEV(FoldingSetNodeIDRef(), scCouldNotCompute, 0, nullptr) {}
356
357bool SCEVCouldNotCompute::classof(const SCEV *S) {
358 return S->getSCEVType() == scCouldNotCompute;
359}
360
361const SCEV *ScalarEvolution::getConstant(ConstantInt *V) {
362 auto &Entry = ConstantSCEVs[V];
363 if (Entry)
364 return Entry;
365
366 FoldingSetNodeID ID;
367 ID.AddInteger(I: scConstant);
368 ID.AddPointer(Ptr: V);
369 FoldingSetInsertToken Token;
370 if (SCEVConstant *S =
371 static_cast<SCEVConstant *>(UniqueSCEVs.lookup(ID, Token)))
372 return Entry = S;
373 SCEVConstant *S =
374 new (SCEVAllocator) SCEVConstant(ID.Intern(Allocator&: SCEVAllocator), V);
375 UniqueSCEVs.insert(N: S, Token);
376 S->computeAndSetCanonical(SE&: *this);
377 return Entry = S;
378}
379
380const SCEV *ScalarEvolution::getConstant(const APInt &Val) {
381 return getConstant(V: ConstantInt::get(Context&: getContext(), V: Val));
382}
383
384const SCEV *
385ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) {
386 IntegerType *ITy = cast<IntegerType>(Val: getEffectiveSCEVType(Ty));
387 // TODO: Avoid implicit trunc?
388 // See https://github.com/llvm/llvm-project/issues/112510.
389 return getConstant(
390 V: ConstantInt::get(Ty: ITy, V, IsSigned: isSigned, /*ImplicitTrunc=*/true));
391}
392
393const SCEV *ScalarEvolution::getVScale(Type *Ty) {
394 FoldingSetNodeID ID;
395 ID.AddInteger(I: scVScale);
396 ID.AddPointer(Ptr: Ty);
397 FoldingSetInsertToken Token;
398 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
399 return S;
400 SCEV *S = new (SCEVAllocator) SCEVVScale(ID.Intern(Allocator&: SCEVAllocator), Ty);
401 UniqueSCEVs.insert(N: S, Token);
402 S->computeAndSetCanonical(SE&: *this);
403 return S;
404}
405
406const SCEV *ScalarEvolution::getElementCount(Type *Ty, ElementCount EC,
407 SCEVFlags Flags) {
408 const SCEV *Res = getConstant(Ty, V: EC.getKnownMinValue());
409 if (EC.isScalable())
410 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty), Flags);
411 return Res;
412}
413
414SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, SCEVTypes SCEVTy,
415 SCEVUse op, Type *ty)
416 : SCEV(ID, SCEVTy, computeExpressionSize(Args: op), ty), Op(op) {}
417
418SCEVPtrToAddrExpr::SCEVPtrToAddrExpr(const FoldingSetNodeIDRef ID,
419 const SCEV *Op, Type *ITy)
420 : SCEVCastExpr(ID, scPtrToAddr, Op, ITy) {
421 assert(getOperand()->getType()->isPointerTy() && getType()->isIntegerTy() &&
422 "Must be a non-bit-width-changing pointer-to-integer cast!");
423}
424
425SCEVIntegralCastExpr::SCEVIntegralCastExpr(const FoldingSetNodeIDRef ID,
426 SCEVTypes SCEVTy, SCEVUse op,
427 Type *ty)
428 : SCEVCastExpr(ID, SCEVTy, op, ty) {}
429
430SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
431 Type *ty)
432 : SCEVIntegralCastExpr(ID, scTruncate, op, ty) {
433 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
434 "Cannot truncate non-integer value!");
435}
436
437SCEVZeroExtendExpr::SCEVZeroExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
438 Type *ty)
439 : SCEVIntegralCastExpr(ID, scZeroExtend, op, ty) {
440 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
441 "Cannot zero extend non-integer value!");
442}
443
444SCEVSignExtendExpr::SCEVSignExtendExpr(const FoldingSetNodeIDRef ID, SCEVUse op,
445 Type *ty)
446 : SCEVIntegralCastExpr(ID, scSignExtend, op, ty) {
447 assert(getOperand()->getType()->isIntOrPtrTy() && getType()->isIntOrPtrTy() &&
448 "Cannot sign extend non-integer value!");
449}
450
451void SCEVUnknown::deleted() {
452 // Clear this SCEVUnknown from various maps.
453 SE->forgetMemoizedResults(SCEVs: {this});
454
455 // Remove this SCEVUnknown from the uniquing map.
456 SE->UniqueSCEVs.erase(N: this);
457
458 // Release the value.
459 setValPtr(nullptr);
460}
461
462void SCEVUnknown::allUsesReplacedWith(Value *New) {
463 // Clear this SCEVUnknown from various maps.
464 SE->forgetMemoizedResults(SCEVs: {this});
465
466 // Remove this SCEVUnknown from the uniquing map.
467 SE->UniqueSCEVs.erase(N: this);
468
469 // Replace the value pointer in case someone is still using this SCEVUnknown.
470 setValPtr(New);
471}
472
473//===----------------------------------------------------------------------===//
474// SCEV Utilities
475//===----------------------------------------------------------------------===//
476
477/// Compare the two values \p LV and \p RV in terms of their "complexity" where
478/// "complexity" is a partial (and somewhat ad-hoc) relation used to order
479/// operands in SCEV expressions.
480static int compareValueComplexity(const AnalysisOptions &Opts,
481 const LoopInfo *const LI, Value *LV,
482 Value *RV, unsigned Depth) {
483 if (Depth > Opts.scalar_evolution_max_value_compare_depth)
484 return 0;
485
486 // Order pointer values after integer values. This helps SCEVExpander form
487 // GEPs.
488 bool LIsPointer = LV->getType()->isPointerTy(),
489 RIsPointer = RV->getType()->isPointerTy();
490 if (LIsPointer != RIsPointer)
491 return (int)LIsPointer - (int)RIsPointer;
492
493 // Compare getValueID values.
494 unsigned LID = LV->getValueID(), RID = RV->getValueID();
495 if (LID != RID)
496 return (int)LID - (int)RID;
497
498 // Sort arguments by their position.
499 if (const auto *LA = dyn_cast<Argument>(Val: LV)) {
500 const auto *RA = cast<Argument>(Val: RV);
501 unsigned LArgNo = LA->getArgNo(), RArgNo = RA->getArgNo();
502 return (int)LArgNo - (int)RArgNo;
503 }
504
505 if (const auto *LGV = dyn_cast<GlobalValue>(Val: LV)) {
506 const auto *RGV = cast<GlobalValue>(Val: RV);
507
508 if (auto L = LGV->getLinkage() - RGV->getLinkage())
509 return L;
510
511 const auto IsGVNameSemantic = [&](const GlobalValue *GV) {
512 auto LT = GV->getLinkage();
513 return !(GlobalValue::isPrivateLinkage(Linkage: LT) ||
514 GlobalValue::isInternalLinkage(Linkage: LT));
515 };
516
517 // Use the names to distinguish the two values, but only if the
518 // names are semantically important.
519 if (IsGVNameSemantic(LGV) && IsGVNameSemantic(RGV))
520 return LGV->getName().compare(RHS: RGV->getName());
521 }
522
523 // For instructions, compare their loop depth, and their operand count. This
524 // is pretty loose.
525 if (const auto *LInst = dyn_cast<Instruction>(Val: LV)) {
526 const auto *RInst = cast<Instruction>(Val: RV);
527
528 // Compare loop depths.
529 const BasicBlock *LParent = LInst->getParent(),
530 *RParent = RInst->getParent();
531 if (LParent != RParent) {
532 unsigned LDepth = LI->getLoopDepth(BB: LParent),
533 RDepth = LI->getLoopDepth(BB: RParent);
534 if (LDepth != RDepth)
535 return (int)LDepth - (int)RDepth;
536 }
537
538 // Compare the number of operands.
539 unsigned LNumOps = LInst->getNumOperands(),
540 RNumOps = RInst->getNumOperands();
541 if (LNumOps != RNumOps)
542 return (int)LNumOps - (int)RNumOps;
543
544 for (unsigned Idx : seq(Size: LNumOps)) {
545 int Result = compareValueComplexity(Opts, LI, LV: LInst->getOperand(i: Idx),
546 RV: RInst->getOperand(i: Idx), Depth: Depth + 1);
547 if (Result != 0)
548 return Result;
549 }
550 }
551
552 return 0;
553}
554
555// Return negative, zero, or positive, if LHS is less than, equal to, or greater
556// than RHS, respectively. A three-way result allows recursive comparisons to be
557// more efficient. The result is a total order.
558static int compareSCEVComplexity(const AnalysisOptions &Opts,
559 const LoopInfo *const LI, SCEVUse LHS,
560 SCEVUse RHS, DominatorTree &DT) {
561 // Fast-path: SCEVs are uniqued so we can do a quick equality check.
562 if (LHS == RHS)
563 return 0;
564
565 if (LHS.getPointer() == RHS.getPointer())
566 return (int)LHS.getUseNoWrapFlags() - (int)RHS.getUseNoWrapFlags();
567
568 // Primarily, sort the SCEVs by their getSCEVType().
569 SCEVTypes LType = LHS->getSCEVType(), RType = RHS->getSCEVType();
570 if (LType != RType)
571 return (int)LType - (int)RType;
572
573 // Aside from the getSCEVType() ordering, the particular ordering
574 // isn't very important except that it's beneficial to be consistent,
575 // so that (a + b) and (b + a) don't end up as different expressions.
576 switch (LType) {
577 case scUnknown: {
578 const SCEVUnknown *LU = cast<SCEVUnknown>(Val&: LHS);
579 const SCEVUnknown *RU = cast<SCEVUnknown>(Val&: RHS);
580
581 if (int X = compareValueComplexity(Opts, LI, LV: LU->getValue(), RV: RU->getValue(),
582 /*Depth=*/0))
583 return X;
584 return LU->getOrderID() < RU->getOrderID() ? -1 : 1;
585 }
586
587 case scConstant: {
588 const SCEVConstant *LC = cast<SCEVConstant>(Val&: LHS);
589 const SCEVConstant *RC = cast<SCEVConstant>(Val&: RHS);
590
591 // Compare constant values.
592 const APInt &LA = LC->getAPInt();
593 const APInt &RA = RC->getAPInt();
594 unsigned LBitWidth = LA.getBitWidth(), RBitWidth = RA.getBitWidth();
595 if (LBitWidth != RBitWidth)
596 return (int)LBitWidth - (int)RBitWidth;
597 return LA.ult(RHS: RA) ? -1 : 1;
598 }
599
600 case scVScale: {
601 const auto *LTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val&: LHS)->getType());
602 const auto *RTy = cast<IntegerType>(Val: cast<SCEVVScale>(Val&: RHS)->getType());
603 return LTy->getBitWidth() - RTy->getBitWidth();
604 }
605
606 case scAddRecExpr: {
607 const SCEVAddRecExpr *LA = cast<SCEVAddRecExpr>(Val&: LHS);
608 const SCEVAddRecExpr *RA = cast<SCEVAddRecExpr>(Val&: RHS);
609
610 // There is always a dominance between two recs that are used by one SCEV,
611 // so we can safely sort recs by loop header dominance. We require such
612 // order in getAddExpr.
613 const Loop *LLoop = LA->getLoop(), *RLoop = RA->getLoop();
614 if (LLoop != RLoop) {
615 const BasicBlock *LHead = LLoop->getHeader(), *RHead = RLoop->getHeader();
616 assert(LHead != RHead && "Two loops share the same header?");
617 if (DT.dominates(A: LHead, B: RHead))
618 return 1;
619 assert(DT.dominates(RHead, LHead) &&
620 "No dominance between recurrences used by one SCEV?");
621 return -1;
622 }
623
624 [[fallthrough]];
625 }
626
627 case scTruncate:
628 case scZeroExtend:
629 case scSignExtend:
630 case scPtrToAddr:
631 case scAddExpr:
632 case scMulExpr:
633 case scUDivExpr:
634 case scSMaxExpr:
635 case scUMaxExpr:
636 case scSMinExpr:
637 case scUMinExpr:
638 case scSequentialUMinExpr: {
639 ArrayRef<SCEVUse> LOps = LHS->operands();
640 ArrayRef<SCEVUse> ROps = RHS->operands();
641
642 // Lexicographically compare n-ary-like expressions.
643 unsigned LNumOps = LOps.size(), RNumOps = ROps.size();
644 if (LNumOps != RNumOps)
645 return (int)LNumOps - (int)RNumOps;
646
647 for (unsigned i = 0; i != LNumOps; ++i) {
648 auto X = compareSCEVComplexity(Opts, LI, LHS: LOps[i], RHS: ROps[i], DT);
649 if (X != 0)
650 return X;
651 }
652
653 assert(isa<SCEVCastExpr>(LHS) && "distinct SCEVs compare equal");
654 unsigned LBitWidth = cast<IntegerType>(Val: LHS->getType())->getBitWidth();
655 unsigned RBitWidth = cast<IntegerType>(Val: RHS->getType())->getBitWidth();
656 assert(LBitWidth != RBitWidth && "distinct SCEVs compare equal");
657 return (int)LBitWidth - (int)RBitWidth;
658 }
659
660 case scCouldNotCompute:
661 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
662 }
663 llvm_unreachable("Unknown SCEV kind!");
664}
665
666/// Given a list of SCEV objects, order them by their complexity, and group
667/// objects of the same complexity together by value. When this routine is
668/// finished, we know that any duplicates in the vector are consecutive and that
669/// complexity is monotonically increasing.
670///
671/// Note that we go take special precautions to ensure that we get deterministic
672/// results from this routine. In other words, we don't want the results of
673/// this to depend on where the addresses of various SCEV objects happened to
674/// land in memory.
675static void groupByComplexity(const AnalysisOptions &Opts,
676 SmallVectorImpl<SCEVUse> &Ops, LoopInfo *LI,
677 DominatorTree &DT) {
678 if (Ops.size() < 2) return; // Noop
679
680 // Whether LHS has provably less complexity than RHS.
681 auto IsLessComplex = [&](SCEVUse LHS, SCEVUse RHS) {
682 return compareSCEVComplexity(Opts, LI, LHS, RHS, DT) < 0;
683 };
684 if (Ops.size() == 2) {
685 // This is the common case, which also happens to be trivially simple.
686 // Special case it.
687 SCEVUse &LHS = Ops[0], &RHS = Ops[1];
688 if (IsLessComplex(RHS, LHS))
689 std::swap(a&: LHS, b&: RHS);
690 return;
691 }
692
693 llvm::sort(C&: Ops, Comp: IsLessComplex);
694}
695
696/// Returns true if \p Ops contains a huge SCEV (the subtree of S contains at
697/// least -scalar-evolution-huge-expr-threshold nodes).
698static bool hasHugeExpression(const AnalysisOptions &Opts,
699 ArrayRef<SCEVUse> Ops) {
700 return any_of(Range&: Ops, P: [&](const SCEV *S) {
701 return S->getExpressionSize() >= Opts.scalar_evolution_huge_expr_threshold;
702 });
703}
704
705/// Performs a number of common optimizations on the passed \p Ops. If the
706/// whole expression reduces down to a single operand, it will be returned.
707///
708/// The following optimizations are performed:
709/// * Fold constants using the \p Fold function.
710/// * Remove identity constants satisfying \p IsIdentity.
711/// * If a constant satisfies \p IsAbsorber, return it.
712/// * Sort operands by complexity.
713template <typename FoldT, typename IsIdentityT, typename IsAbsorberT>
714static const SCEV *
715constantFoldAndGroupOps(const AnalysisOptions &Opts, ScalarEvolution &SE,
716 LoopInfo &LI, DominatorTree &DT,
717 SmallVectorImpl<SCEVUse> &Ops, FoldT Fold,
718 IsIdentityT IsIdentity, IsAbsorberT IsAbsorber) {
719 const SCEVConstant *Folded = nullptr;
720 for (unsigned Idx = 0; Idx < Ops.size();) {
721 const SCEV *Op = Ops[Idx];
722 if (const auto *C = dyn_cast<SCEVConstant>(Val: Op)) {
723 if (!Folded)
724 Folded = C;
725 else
726 Folded = cast<SCEVConstant>(
727 SE.getConstant(Fold(Folded->getAPInt(), C->getAPInt())));
728 Ops.erase(CI: Ops.begin() + Idx);
729 continue;
730 }
731 ++Idx;
732 }
733
734 if (Ops.empty()) {
735 assert(Folded && "Must have folded value");
736 return Folded;
737 }
738
739 if (Folded && IsAbsorber(Folded->getAPInt()))
740 return Folded;
741
742 groupByComplexity(Opts, Ops, LI: &LI, DT);
743 if (Folded && !IsIdentity(Folded->getAPInt()))
744 Ops.insert(I: Ops.begin(), Elt: Folded);
745
746 return Ops.size() == 1 ? Ops[0] : nullptr;
747}
748
749//===----------------------------------------------------------------------===//
750// Simple SCEV method implementations
751//===----------------------------------------------------------------------===//
752
753/// Compute BC(It, K). The result has width W. Assume, K > 0.
754static const SCEV *BinomialCoefficient(const SCEV *It, unsigned K,
755 ScalarEvolution &SE,
756 Type *ResultTy) {
757 // Handle the simplest case efficiently.
758 if (K == 1)
759 return SE.getTruncateOrZeroExtend(V: It, Ty: ResultTy);
760
761 // We are using the following formula for BC(It, K):
762 //
763 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / K!
764 //
765 // Suppose, W is the bitwidth of the return value. We must be prepared for
766 // overflow. Hence, we must assure that the result of our computation is
767 // equal to the accurate one modulo 2^W. Unfortunately, division isn't
768 // safe in modular arithmetic.
769 //
770 // However, this code doesn't use exactly that formula; the formula it uses
771 // is something like the following, where T is the number of factors of 2 in
772 // K! (i.e. trailing zeros in the binary representation of K!), and ^ is
773 // exponentiation:
774 //
775 // BC(It, K) = (It * (It - 1) * ... * (It - K + 1)) / 2^T / (K! / 2^T)
776 //
777 // This formula is trivially equivalent to the previous formula. However,
778 // this formula can be implemented much more efficiently. The trick is that
779 // K! / 2^T is odd, and exact division by an odd number *is* safe in modular
780 // arithmetic. To do exact division in modular arithmetic, all we have
781 // to do is multiply by the inverse. Therefore, this step can be done at
782 // width W.
783 //
784 // The next issue is how to safely do the division by 2^T. The way this
785 // is done is by doing the multiplication step at a width of at least W + T
786 // bits. This way, the bottom W+T bits of the product are accurate. Then,
787 // when we perform the division by 2^T (which is equivalent to a right shift
788 // by T), the bottom W bits are accurate. Extra bits are okay; they'll get
789 // truncated out after the division by 2^T.
790 //
791 // In comparison to just directly using the first formula, this technique
792 // is much more efficient; using the first formula requires W * K bits,
793 // but this formula less than W + K bits. Also, the first formula requires
794 // a division step, whereas this formula only requires multiplies and shifts.
795 //
796 // It doesn't matter whether the subtraction step is done in the calculation
797 // width or the input iteration count's width; if the subtraction overflows,
798 // the result must be zero anyway. We prefer here to do it in the width of
799 // the induction variable because it helps a lot for certain cases; CodeGen
800 // isn't smart enough to ignore the overflow, which leads to much less
801 // efficient code if the width of the subtraction is wider than the native
802 // register width.
803 //
804 // (It's possible to not widen at all by pulling out factors of 2 before
805 // the multiplication; for example, K=2 can be calculated as
806 // It/2*(It+(It*INT_MIN/INT_MIN)+-1). However, it requires
807 // extra arithmetic, so it's not an obvious win, and it gets
808 // much more complicated for K > 3.)
809
810 // Protection from insane SCEVs; this bound is conservative,
811 // but it probably doesn't matter.
812 if (K > 1000)
813 return SE.getCouldNotCompute();
814
815 unsigned W = SE.getTypeSizeInBits(Ty: ResultTy);
816
817 // Calculate K! / 2^T and T; we divide out the factors of two before
818 // multiplying for calculating K! / 2^T to avoid overflow.
819 // Other overflow doesn't matter because we only care about the bottom
820 // W bits of the result.
821 APInt OddFactorial(W, 1);
822 unsigned T = 1;
823 for (unsigned i = 3; i <= K; ++i) {
824 unsigned TwoFactors = countr_zero(Val: i);
825 T += TwoFactors;
826 OddFactorial *= (i >> TwoFactors);
827 }
828
829 // We need at least W + T bits for the multiplication step
830 unsigned CalculationBits = W + T;
831
832 // Calculate 2^T, at width T+W.
833 APInt DivFactor = APInt::getOneBitSet(numBits: CalculationBits, BitNo: T);
834
835 // Calculate the multiplicative inverse of K! / 2^T;
836 // this multiplication factor will perform the exact division by
837 // K! / 2^T.
838 APInt MultiplyFactor = OddFactorial.multiplicativeInverse();
839
840 // Calculate the product, at width T+W
841 IntegerType *CalculationTy = IntegerType::get(C&: SE.getContext(),
842 NumBits: CalculationBits);
843 const SCEV *Dividend = SE.getTruncateOrZeroExtend(V: It, Ty: CalculationTy);
844 for (unsigned i = 1; i != K; ++i) {
845 const SCEV *S = SE.getMinusSCEV(LHS: It, RHS: SE.getConstant(Ty: It->getType(), V: i));
846 Dividend = SE.getMulExpr(LHS: Dividend,
847 RHS: SE.getTruncateOrZeroExtend(V: S, Ty: CalculationTy));
848 }
849
850 // Divide by 2^T
851 const SCEV *DivResult = SE.getUDivExpr(LHS: Dividend, RHS: SE.getConstant(Val: DivFactor));
852
853 // Truncate the result, and divide by K! / 2^T.
854
855 return SE.getMulExpr(LHS: SE.getConstant(Val: MultiplyFactor),
856 RHS: SE.getTruncateOrZeroExtend(V: DivResult, Ty: ResultTy));
857}
858
859/// Return the value of this chain of recurrences at the specified iteration
860/// number. We can evaluate this recurrence by multiplying each element in the
861/// chain by the binomial coefficient corresponding to it. In other words, we
862/// can evaluate {A,+,B,+,C,+,D} as:
863///
864/// A*BC(It, 0) + B*BC(It, 1) + C*BC(It, 2) + D*BC(It, 3)
865///
866/// where BC(It, k) stands for binomial coefficient.
867const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It,
868 ScalarEvolution &SE) const {
869 return evaluateAtIteration(Operands: operands(), It, SE);
870}
871
872SCEVUse SCEVAddRecExpr::evaluateAtIteration(ArrayRef<SCEVUse> Operands,
873 const SCEV *It, ScalarEvolution &SE,
874 SCEVFlags UseFlags) {
875 assert(Operands.size() > 0);
876 assert((Operands.size() == 2 || UseFlags == SCEV::FlagNone) &&
877 "use-specific flags only supported for affine AddRecs");
878 SCEVUse Result = Operands[0].getPointer();
879 for (unsigned i = 1, e = Operands.size(); i != e; ++i) {
880 // The computation is correct in the face of overflow provided that the
881 // multiplication is performed _after_ the evaluation of the binomial
882 // coefficient.
883 const SCEV *Coeff = BinomialCoefficient(It, K: i, SE, ResultTy: Result->getType());
884 if (isa<SCEVCouldNotCompute>(Val: Coeff))
885 return Coeff;
886
887 SCEVUse Mul = SE.getMulExpr(LHS: Operands[i].getPointer(), RHS: Coeff,
888 Flags: {SCEV::FlagNone, UseFlags});
889 Result = SE.getAddExpr(LHS: Result, RHS: Mul, Flags: {SCEV::FlagNone, UseFlags});
890 }
891 return Result;
892}
893
894SCEVUse SCEVAddRecExpr::getExitValue(ScalarEvolution &SE) const {
895 const SCEV *BTC = SE.getBackedgeTakenCount(L: getLoop());
896 if (isa<SCEVCouldNotCompute>(Val: BTC))
897 return BTC;
898 // The loop reaches iteration BTC, so the value this recurrence computes there
899 // is the value it had, and that did not wrap.
900 return evaluateAtIteration(Operands: operands(), It: BTC, SE,
901 UseFlags: isAffine() ? getNoWrapFlags(Mask: SCEV::FlagNUW)
902 : SCEV::FlagNone);
903}
904
905//===----------------------------------------------------------------------===//
906// SCEV Expression folder implementations
907//===----------------------------------------------------------------------===//
908
909/// The SCEVCastSinkingRewriter takes a scalar evolution expression,
910/// which computes a pointer-typed value, and rewrites the whole expression
911/// tree so that *all* the computations are done on integers, and the only
912/// pointer-typed operands in the expression are SCEVUnknown.
913/// The CreatePtrCast callback is invoked to create the actual conversion
914/// (ptrtoint or ptrtoaddr) at the SCEVUnknown leaves.
915class SCEVCastSinkingRewriter
916 : public SCEVRewriteVisitor<SCEVCastSinkingRewriter> {
917 using Base = SCEVRewriteVisitor<SCEVCastSinkingRewriter>;
918 using ConversionFn = function_ref<const SCEV *(const SCEVUnknown *)>;
919 Type *TargetTy;
920 ConversionFn CreatePtrCast;
921
922public:
923 SCEVCastSinkingRewriter(ScalarEvolution &SE, Type *TargetTy,
924 ConversionFn CreatePtrCast)
925 : Base(SE), TargetTy(TargetTy), CreatePtrCast(std::move(CreatePtrCast)) {}
926
927 static const SCEV *rewrite(const SCEV *Scev, ScalarEvolution &SE,
928 Type *TargetTy, ConversionFn CreatePtrCast) {
929 SCEVCastSinkingRewriter Rewriter(SE, TargetTy, std::move(CreatePtrCast));
930 return Rewriter.visit(S: Scev);
931 }
932
933 const SCEV *visit(const SCEV *S) {
934 Type *STy = S->getType();
935 // If the expression is not pointer-typed, just keep it as-is.
936 if (!STy->isPointerTy())
937 return S;
938 // Else, recursively sink the cast down into it.
939 return Base::visit(S);
940 }
941
942 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
943 // Preserve wrap flags on rewritten SCEVAddExpr, which the default
944 // implementation drops.
945 SmallVector<SCEVUse, 2> Operands;
946 bool Changed = false;
947 for (SCEVUse Op : Expr->operands()) {
948 Operands.push_back(Elt: visit(S: Op.getPointer()));
949 Changed |= Op.getPointer() != Operands.back();
950 }
951 return !Changed ? Expr : SE.getAddExpr(Ops&: Operands, Flags: Expr->getNoWrapFlags());
952 }
953
954 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
955 assert(Expr->getType()->isPointerTy() &&
956 "Should only reach pointer-typed SCEVUnknown's.");
957 // Perform some basic constant folding. If the operand of the cast is a
958 // null pointer, don't create a cast SCEV expression (that will be left
959 // as-is), but produce a zero constant.
960 if (isa<ConstantPointerNull>(Val: Expr->getValue()))
961 return SE.getZero(Ty: TargetTy);
962 return CreatePtrCast(Expr);
963 }
964};
965
966const SCEV *ScalarEvolution::getPtrToAddrExpr(const SCEV *Op) {
967 assert(Op->getType()->isPointerTy() && "Op must be a pointer");
968
969 // Treat pointers with unstable representation conservatively, since the
970 // address bits may change.
971 if (DL.hasUnstableRepresentation(Ty: Op->getType()))
972 return getCouldNotCompute();
973
974 Type *Ty = DL.getAddressType(PtrTy: Op->getType());
975
976 // Use the rewriter to sink the cast down to SCEVUnknown leaves.
977 // The rewriter handles null pointer constant folding.
978 const SCEV *IntOp = SCEVCastSinkingRewriter::rewrite(
979 Scev: Op, SE&: *this, TargetTy: Ty, CreatePtrCast: [this, Ty](const SCEVUnknown *U) {
980 FoldingSetNodeID ID;
981 ID.AddInteger(I: scPtrToAddr);
982 ID.AddPointer(Ptr: U);
983 ID.AddPointer(Ptr: Ty);
984 FoldingSetInsertToken Token;
985 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
986 return S;
987 SCEV *S = new (SCEVAllocator)
988 SCEVPtrToAddrExpr(ID.Intern(Allocator&: SCEVAllocator), U, Ty);
989 UniqueSCEVs.insert(N: S, Token);
990 S->computeAndSetCanonical(SE&: *this);
991 registerUser(User: S, Ops: {U});
992 return static_cast<const SCEV *>(S);
993 });
994 assert(IntOp->getType()->isIntegerTy() &&
995 "We must have succeeded in sinking the cast, "
996 "and ending up with an integer-typed expression!");
997 return IntOp;
998}
999
1000const SCEV *ScalarEvolution::getTruncateExpr(SCEVUse Op, Type *Ty,
1001 unsigned Depth) {
1002 assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) &&
1003 "This is not a truncating conversion!");
1004 assert(isSCEVable(Ty) &&
1005 "This is not a conversion to a SCEVable type!");
1006 assert(!Op->getType()->isPointerTy() && "Can't truncate pointer!");
1007 Ty = getEffectiveSCEVType(Ty);
1008
1009 FoldingSetNodeID ID;
1010 ID.AddInteger(I: scTruncate);
1011 ID.AddPointer(Ptr: Op.getOpaqueValue());
1012 ID.AddPointer(Ptr: Ty);
1013 FoldingSetInsertToken Token;
1014 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1015 return S;
1016
1017 // Fold if the operand is constant.
1018 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1019 return getConstant(
1020 V: cast<ConstantInt>(Val: ConstantExpr::getTrunc(C: SC->getValue(), Ty)));
1021
1022 // trunc(trunc(x)) --> trunc(x)
1023 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op))
1024 return getTruncateExpr(Op: ST->getOperand(), Ty, Depth: Depth + 1);
1025
1026 // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing
1027 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val&: Op))
1028 return getTruncateOrSignExtend(V: SS->getOperand(), Ty, Depth: Depth + 1);
1029
1030 // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing
1031 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1032 return getTruncateOrZeroExtend(V: SZ->getOperand(), Ty, Depth: Depth + 1);
1033
1034 if (Depth > Opts.scalar_evolution_max_cast_depth) {
1035 SCEV *S =
1036 new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator), Op, Ty);
1037 UniqueSCEVs.insert(N: S, Token);
1038 S->computeAndSetCanonical(SE&: *this);
1039 registerUser(User: S, Ops: Op);
1040 return S;
1041 }
1042
1043 // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and
1044 // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN),
1045 // if after transforming we have at most one truncate, not counting truncates
1046 // that replace other casts.
1047 if (isa<SCEVAddExpr>(Val: Op) || isa<SCEVMulExpr>(Val: Op)) {
1048 auto *CommOp = cast<SCEVCommutativeExpr>(Val&: Op);
1049 SmallVector<SCEVUse, 4> Operands;
1050 unsigned numTruncs = 0;
1051 for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2;
1052 ++i) {
1053 const SCEV *S = getTruncateExpr(Op: CommOp->getOperand(i), Ty, Depth: Depth + 1);
1054 if (!isa<SCEVIntegralCastExpr>(Val: CommOp->getOperand(i)) &&
1055 isa<SCEVTruncateExpr>(Val: S))
1056 numTruncs++;
1057 Operands.push_back(Elt: S);
1058 }
1059 if (numTruncs < 2) {
1060 if (isa<SCEVAddExpr>(Val: Op))
1061 return getAddExpr(Ops&: Operands);
1062 if (isa<SCEVMulExpr>(Val: Op))
1063 return getMulExpr(Ops&: Operands);
1064 llvm_unreachable("Unexpected SCEV type for Op.");
1065 }
1066 // Although we checked in the beginning that ID is not in the cache, it is
1067 // possible that during recursion and different modification ID was inserted
1068 // into the cache. So if we find it, just return it.
1069 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1070 return S;
1071 }
1072
1073 // If the input value is a chrec scev, truncate the chrec's operands.
1074 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: Op)) {
1075 SmallVector<SCEVUse, 4> Operands;
1076 for (const SCEV *Op : AddRec->operands())
1077 Operands.push_back(Elt: getTruncateExpr(Op, Ty, Depth: Depth + 1));
1078 return getAddRecExpr(Operands, L: AddRec->getLoop(), Flags: SCEV::FlagNone);
1079 }
1080
1081 // Return zero if truncating to known zeros.
1082 uint32_t MinTrailingZeros = getMinTrailingZeros(S: Op);
1083 if (MinTrailingZeros >= getTypeSizeInBits(Ty))
1084 return getZero(Ty);
1085
1086 // The cast wasn't folded; create an explicit cast node. We can reuse
1087 // the existing insert position since if we get here, we won't have
1088 // made any changes which would invalidate it.
1089 SCEV *S = new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(Allocator&: SCEVAllocator),
1090 Op, Ty);
1091 UniqueSCEVs.insert(N: S, Token);
1092 S->computeAndSetCanonical(SE&: *this);
1093 registerUser(User: S, Ops: Op);
1094 return S;
1095}
1096
1097// Get the limit of a recurrence such that incrementing by Step cannot cause
1098// signed overflow as long as the value of the recurrence within the
1099// loop does not exceed this limit before incrementing.
1100static const SCEV *getSignedOverflowLimitForStep(const SCEV *Step,
1101 ICmpInst::Predicate *Pred,
1102 ScalarEvolution *SE) {
1103 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1104 if (SE->isKnownPositive(S: Step)) {
1105 *Pred = ICmpInst::ICMP_SLT;
1106 return SE->getConstant(Val: APInt::getSignedMinValue(numBits: BitWidth) -
1107 SE->getSignedRangeMax(S: Step));
1108 }
1109 if (SE->isKnownNegative(S: Step)) {
1110 *Pred = ICmpInst::ICMP_SGT;
1111 return SE->getConstant(Val: APInt::getSignedMaxValue(numBits: BitWidth) -
1112 SE->getSignedRangeMin(S: Step));
1113 }
1114 return nullptr;
1115}
1116
1117// Get the limit of a recurrence such that incrementing by Step cannot cause
1118// unsigned overflow as long as the value of the recurrence within the loop does
1119// not exceed this limit before incrementing.
1120static const SCEV *getUnsignedOverflowLimitForStep(const SCEV *Step,
1121 ICmpInst::Predicate *Pred,
1122 ScalarEvolution *SE) {
1123 unsigned BitWidth = SE->getTypeSizeInBits(Ty: Step->getType());
1124 *Pred = ICmpInst::ICMP_ULT;
1125
1126 return SE->getConstant(Val: APInt::getMinValue(numBits: BitWidth) -
1127 SE->getUnsignedRangeMax(S: Step));
1128}
1129
1130namespace {
1131
1132struct ExtendOpTraitsBase {
1133 typedef const SCEV *(ScalarEvolution::*GetExtendExprTy)(SCEVUse, Type *,
1134 unsigned);
1135};
1136
1137// Used to make code generic over signed and unsigned overflow.
1138template <typename ExtendOp> struct ExtendOpTraits {
1139 // Members present:
1140 //
1141 // static const SCEVFlags WrapType;
1142 //
1143 // static const ExtendOpTraitsBase::GetExtendExprTy GetExtendExpr;
1144 //
1145 // static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1146 // ICmpInst::Predicate *Pred,
1147 // ScalarEvolution *SE);
1148};
1149
1150template <>
1151struct ExtendOpTraits<SCEVSignExtendExpr> : public ExtendOpTraitsBase {
1152 static const SCEVFlags WrapType = SCEV::FlagNSW;
1153
1154 static const GetExtendExprTy GetExtendExpr;
1155
1156 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1157 ICmpInst::Predicate *Pred,
1158 ScalarEvolution *SE) {
1159 return getSignedOverflowLimitForStep(Step, Pred, SE);
1160 }
1161};
1162
1163const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1164 SCEVSignExtendExpr>::GetExtendExpr = &ScalarEvolution::getSignExtendExpr;
1165
1166template <>
1167struct ExtendOpTraits<SCEVZeroExtendExpr> : public ExtendOpTraitsBase {
1168 static const SCEVFlags WrapType = SCEV::FlagNUW;
1169
1170 static const GetExtendExprTy GetExtendExpr;
1171
1172 static const SCEV *getOverflowLimitForStep(const SCEV *Step,
1173 ICmpInst::Predicate *Pred,
1174 ScalarEvolution *SE) {
1175 return getUnsignedOverflowLimitForStep(Step, Pred, SE);
1176 }
1177};
1178
1179const ExtendOpTraitsBase::GetExtendExprTy ExtendOpTraits<
1180 SCEVZeroExtendExpr>::GetExtendExpr = &ScalarEvolution::getZeroExtendExpr;
1181
1182} // end anonymous namespace
1183
1184// The recurrence AR has been shown to have no signed/unsigned wrap or something
1185// close to it. Typically, if we can prove NSW/NUW for AR, then we can just as
1186// easily prove NSW/NUW for its preincrement or postincrement sibling. This
1187// allows normalizing a sign/zero extended AddRec as such: {sext/zext(Step +
1188// Start),+,Step} => {(Step + sext/zext(Start),+,Step} As a result, the
1189// expression "Step + sext/zext(PreIncAR)" is congruent with
1190// "sext/zext(PostIncAR)"
1191template <typename ExtendOpTy>
1192static const SCEV *getPreStartForExtend(const SCEVAddRecExpr *AR,
1193 ScalarEvolution *SE, unsigned Depth) {
1194 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1195 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1196
1197 const Loop *L = AR->getLoop();
1198 const SCEV *Start = AR->getStart();
1199 const SCEV *Step = AR->getStepRecurrence(SE&: *SE);
1200
1201 // Check for a simple looking step prior to loop entry.
1202 const SCEVAddExpr *SA = dyn_cast<SCEVAddExpr>(Val: Start);
1203 if (!SA)
1204 return nullptr;
1205
1206 // Create an AddExpr for "PreStart" after subtracting Step. Full SCEV
1207 // subtraction is expensive. For this purpose, perform a quick and dirty
1208 // difference, by checking for Step in the operand list. Note, that
1209 // SA might have repeated ops, like %a + %a + ..., so only remove one.
1210 SmallVector<SCEVUse, 4> DiffOps(SA->operands());
1211 for (auto It = DiffOps.begin(); It != DiffOps.end(); ++It)
1212 if (*It == Step) {
1213 DiffOps.erase(CI: It);
1214 break;
1215 }
1216
1217 if (DiffOps.size() == SA->getNumOperands())
1218 return nullptr;
1219
1220 // Try to prove `WrapType` (SCEV::FlagNSW or SCEV::FlagNUW) on `PreStart` +
1221 // `Step`:
1222
1223 // 1. NSW/NUW flags on the step increment.
1224 auto PreStartFlags =
1225 ScalarEvolution::maskFlags(Flags: SA->getNoWrapFlags(), Mask: SCEV::FlagNUW);
1226 const SCEV *PreStart = SE->getAddExpr(Ops&: DiffOps, Flags: PreStartFlags);
1227 const SCEVAddRecExpr *PreAR = dyn_cast<SCEVAddRecExpr>(
1228 Val: SE->getAddRecExpr(Start: PreStart, Step, L, Flags: SCEV::FlagNone));
1229
1230 // "{S,+,X} is <nsw>/<nuw>" and "the backedge is taken at least once" implies
1231 // "S+X does not sign/unsign-overflow".
1232 //
1233
1234 const SCEV *BECount = SE->getBackedgeTakenCount(L);
1235 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType)) &&
1236 !isa<SCEVCouldNotCompute>(Val: BECount) && SE->isKnownPositive(S: BECount))
1237 return PreStart;
1238
1239 // 2. Direct overflow check on the step operation's expression.
1240 unsigned BitWidth = SE->getTypeSizeInBits(Ty: AR->getType());
1241 Type *WideTy = IntegerType::get(C&: SE->getContext(), NumBits: BitWidth * 2);
1242 const SCEV *OperandExtendedStart =
1243 SE->getAddExpr((SE->*GetExtendExpr)(PreStart, WideTy, Depth),
1244 (SE->*GetExtendExpr)(Step, WideTy, Depth));
1245 if ((SE->*GetExtendExpr)(Start, WideTy, Depth) == OperandExtendedStart) {
1246 if (PreAR && any(AR->getNoWrapFlags(Mask: WrapType))) {
1247 // If we know `AR` == {`PreStart`+`Step`,+,`Step`} is `WrapType` (FlagNSW
1248 // or FlagNUW) and that `PreStart` + `Step` is `WrapType` too, then
1249 // `PreAR` == {`PreStart`,+,`Step`} is also `WrapType`. Cache this fact.
1250 SE->setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(PreAR), Flags: WrapType);
1251 }
1252 return PreStart;
1253 }
1254
1255 // 3. Loop precondition.
1256 ICmpInst::Predicate Pred;
1257 const SCEV *OverflowLimit =
1258 ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(Step, &Pred, SE);
1259
1260 if (OverflowLimit &&
1261 SE->isLoopEntryGuardedByCond(L, Pred, LHS: PreStart, RHS: OverflowLimit))
1262 return PreStart;
1263
1264 return nullptr;
1265}
1266
1267// Get the normalized zero or sign extended expression for this AddRec's Start.
1268template <typename ExtendOpTy>
1269static const SCEV *getExtendAddRecStart(const SCEVAddRecExpr *AR, Type *Ty,
1270 ScalarEvolution *SE,
1271 unsigned Depth) {
1272 auto GetExtendExpr = ExtendOpTraits<ExtendOpTy>::GetExtendExpr;
1273
1274 const SCEV *PreStart = getPreStartForExtend<ExtendOpTy>(AR, SE, Depth);
1275 if (!PreStart)
1276 return (SE->*GetExtendExpr)(AR->getStart(), Ty, Depth);
1277
1278 return SE->getAddExpr((SE->*GetExtendExpr)(AR->getStepRecurrence(SE&: *SE), Ty,
1279 Depth),
1280 (SE->*GetExtendExpr)(PreStart, Ty, Depth));
1281}
1282
1283// Try to prove away overflow by looking at "nearby" add recurrences. A
1284// motivating example for this rule: if we know `{0,+,4}` is `ult` `-1` and it
1285// does not itself wrap then we can conclude that `{1,+,4}` is `nuw`.
1286//
1287// Formally:
1288//
1289// {S,+,X} == {S-T,+,X} + T
1290// => Ext({S,+,X}) == Ext({S-T,+,X} + T)
1291//
1292// If ({S-T,+,X} + T) does not overflow ... (1)
1293//
1294// RHS == Ext({S-T,+,X} + T) == Ext({S-T,+,X}) + Ext(T)
1295//
1296// If {S-T,+,X} does not overflow ... (2)
1297//
1298// RHS == Ext({S-T,+,X}) + Ext(T) == {Ext(S-T),+,Ext(X)} + Ext(T)
1299// == {Ext(S-T)+Ext(T),+,Ext(X)}
1300//
1301// If (S-T)+T does not overflow ... (3)
1302//
1303// RHS == {Ext(S-T)+Ext(T),+,Ext(X)} == {Ext(S-T+T),+,Ext(X)}
1304// == {Ext(S),+,Ext(X)} == LHS
1305//
1306// Thus, if (1), (2) and (3) are true for some T, then
1307// Ext({S,+,X}) == {Ext(S),+,Ext(X)}
1308//
1309// (3) is implied by (1) -- "(S-T)+T does not overflow" is simply "({S-T,+,X}+T)
1310// does not overflow" restricted to the 0th iteration. Therefore we only need
1311// to check for (1) and (2).
1312//
1313// In the current context, S is `Start`, X is `Step`, Ext is `ExtendOpTy` and T
1314// is `Delta` (defined below).
1315template <typename ExtendOpTy>
1316bool ScalarEvolution::proveNoWrapByVaryingStart(const SCEV *Start,
1317 const SCEV *Step,
1318 const Loop *L) {
1319 auto WrapType = ExtendOpTraits<ExtendOpTy>::WrapType;
1320
1321 // We restrict `Start` to a constant to prevent SCEV from spending too much
1322 // time here. It is correct (but more expensive) to continue with a
1323 // non-constant `Start` and do a general SCEV subtraction to compute
1324 // `PreStart` below.
1325 const SCEVConstant *StartC = dyn_cast<SCEVConstant>(Val: Start);
1326 if (!StartC)
1327 return false;
1328
1329 APInt StartAI = StartC->getAPInt();
1330
1331 for (unsigned Delta : {-2, -1, 1, 2}) {
1332 const SCEV *PreStart = getConstant(Val: StartAI - Delta);
1333 const auto *PreAR = static_cast<SCEVAddRecExpr *>(
1334 findExistingSCEVInCache(SCEVType: scAddRecExpr, Ops: {PreStart, Step}, L));
1335
1336 // Give up if we don't already have the add recurrence we need because
1337 // actually constructing an add recurrence is relatively expensive.
1338 if (PreAR && any(PreAR->getNoWrapFlags(Mask: WrapType))) { // proves (2)
1339 const SCEV *DeltaS = getConstant(Ty: StartC->getType(), V: Delta);
1340 ICmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
1341 const SCEV *Limit = ExtendOpTraits<ExtendOpTy>::getOverflowLimitForStep(
1342 DeltaS, &Pred, this);
1343 if (Limit && isKnownPredicate(Pred, LHS: PreAR, RHS: Limit)) // proves (1)
1344 return true;
1345 }
1346 }
1347
1348 return false;
1349}
1350
1351// Finds an integer D for an expression (C + x + y + ...) such that the top
1352// level addition in (D + (C - D + x + y + ...)) would not wrap (signed or
1353// unsigned) and the number of trailing zeros of (C - D + x + y + ...) is
1354// maximized, where C is the \p ConstantTerm, x, y, ... are arbitrary SCEVs, and
1355// the (C + x + y + ...) expression is \p WholeAddExpr.
1356static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1357 const SCEVConstant *ConstantTerm,
1358 const SCEVAddExpr *WholeAddExpr) {
1359 const APInt &C = ConstantTerm->getAPInt();
1360 const unsigned BitWidth = C.getBitWidth();
1361 // Find number of trailing zeros of (x + y + ...) w/o the C first:
1362 uint32_t TZ = BitWidth;
1363 for (unsigned I = 1, E = WholeAddExpr->getNumOperands(); I < E && TZ; ++I)
1364 TZ = std::min(a: TZ, b: SE.getMinTrailingZeros(S: WholeAddExpr->getOperand(i: I)));
1365 if (TZ) {
1366 // Set D to be as many least significant bits of C as possible while still
1367 // guaranteeing that adding D to (C - D + x + y + ...) won't cause a wrap:
1368 return TZ < BitWidth ? C.trunc(width: TZ).zext(width: BitWidth) : C;
1369 }
1370 return APInt(BitWidth, 0);
1371}
1372
1373// Finds an integer D for an affine AddRec expression {C,+,x} such that the top
1374// level addition in (D + {C-D,+,x}) would not wrap (signed or unsigned) and the
1375// number of trailing zeros of (C - D + x * n) is maximized, where C is the \p
1376// ConstantStart, x is an arbitrary \p Step, and n is the loop trip count.
1377static APInt extractConstantWithoutWrapping(ScalarEvolution &SE,
1378 const APInt &ConstantStart,
1379 const SCEV *Step) {
1380 const unsigned BitWidth = ConstantStart.getBitWidth();
1381 const uint32_t TZ = SE.getMinTrailingZeros(S: Step);
1382 if (TZ)
1383 return TZ < BitWidth ? ConstantStart.trunc(width: TZ).zext(width: BitWidth)
1384 : ConstantStart;
1385 return APInt(BitWidth, 0);
1386}
1387
1388static void insertFoldCacheEntry(
1389 const ScalarEvolution::FoldID &ID, const SCEV *S,
1390 DenseMap<ScalarEvolution::FoldID, const SCEV *> &FoldCache,
1391 DenseMap<const SCEV *, SmallVector<ScalarEvolution::FoldID, 2>>
1392 &FoldCacheUser) {
1393 auto I = FoldCache.insert(KV: {ID, S});
1394 if (!I.second) {
1395 // Remove FoldCacheUser entry for ID when replacing an existing FoldCache
1396 // entry.
1397 auto &UserIDs = FoldCacheUser[I.first->second];
1398 assert(count(UserIDs, ID) == 1 && "unexpected duplicates in UserIDs");
1399 for (unsigned I = 0; I != UserIDs.size(); ++I)
1400 if (UserIDs[I] == ID) {
1401 std::swap(a&: UserIDs[I], b&: UserIDs.back());
1402 break;
1403 }
1404 UserIDs.pop_back();
1405 I.first->second = S;
1406 }
1407 FoldCacheUser[S].push_back(Elt: ID);
1408}
1409
1410const SCEV *ScalarEvolution::getZeroExtendExpr(SCEVUse Op, Type *Ty,
1411 unsigned Depth) {
1412 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1413 "This is not an extending conversion!");
1414 assert(isSCEVable(Ty) &&
1415 "This is not a conversion to a SCEVable type!");
1416 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1417 Ty = getEffectiveSCEVType(Ty);
1418
1419 FoldID ID(scZeroExtend, Op, Ty);
1420 if (const SCEV *S = FoldCache.lookup(Val: ID))
1421 return S;
1422
1423 const SCEV *S = getZeroExtendExprImpl(Op, Ty, Depth);
1424 if (!isa<SCEVZeroExtendExpr>(Val: S))
1425 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1426 return S;
1427}
1428
1429const SCEV *ScalarEvolution::getZeroExtendExprImpl(SCEVUse Op, Type *Ty,
1430 unsigned Depth) {
1431 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1432 "This is not an extending conversion!");
1433 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1434 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1435
1436 // Fold if the operand is constant.
1437 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1438 return getConstant(Val: SC->getAPInt().zext(width: getTypeSizeInBits(Ty)));
1439
1440 // zext(zext(x)) --> zext(x)
1441 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1442 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1443
1444 // If the operand is an affine AddRec with the no-unsigned-wrap flag, the
1445 // zero-extension distributes over the recurrence.
1446 const SCEV *Start, *Step;
1447 const Loop *L;
1448 if (Depth <= Opts.scalar_evolution_max_cast_depth &&
1449 match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1450 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1451 if (AR->hasNoUnsignedWrap()) {
1452 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1453 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1454 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1455 }
1456 }
1457
1458 // Before doing any expensive analysis, check to see if we've already
1459 // computed a SCEV for this Op and Ty.
1460 FoldingSetNodeID ID;
1461 ID.AddInteger(I: scZeroExtend);
1462 ID.AddPointer(Ptr: Op.getOpaqueValue());
1463 ID.AddPointer(Ptr: Ty);
1464 FoldingSetInsertToken Token;
1465 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1466 return S;
1467 if (Depth > Opts.scalar_evolution_max_cast_depth) {
1468 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1469 Op, Ty);
1470 UniqueSCEVs.insert(N: S, Token);
1471 S->computeAndSetCanonical(SE&: *this);
1472 registerUser(User: S, Ops: Op);
1473 return S;
1474 }
1475
1476 // zext(trunc(x)) --> zext(x) or x or trunc(x)
1477 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
1478 // It's possible the bits taken off by the truncate were all zero bits. If
1479 // so, we should be able to simplify this further.
1480 const SCEV *X = ST->getOperand();
1481 ConstantRange CR = getUnsignedRange(S: X);
1482 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1483 unsigned NewBits = getTypeSizeInBits(Ty);
1484 if (CR.truncate(BitWidth: TruncBits).zeroExtend(BitWidth: NewBits).contains(
1485 CR: CR.zextOrTrunc(BitWidth: NewBits)))
1486 return getTruncateOrZeroExtend(V: X, Ty, Depth);
1487 }
1488
1489 // If the input value is a chrec scev, and we can prove that the value
1490 // did not overflow the old, smaller, value, we can zero extend all of the
1491 // operands (often constants). This allows analysis of something like
1492 // this: for (unsigned char X = 0; X < 100; ++X) { int Y = X; }
1493 if (match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1494 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1495 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
1496
1497 // The no-unsigned-wrap case is handled before the uniquing lookup above.
1498
1499 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1500 // Note that this serves two purposes: It filters out loops that are
1501 // simply not analyzable, and it covers the case where this code is
1502 // being called from within backedge-taken count analysis, such that
1503 // attempting to ask for the backedge-taken count would likely result
1504 // in infinite recursion. In the later case, the analysis code will
1505 // cope with a conservative value, and it will take care to purge
1506 // that value once it has finished.
1507 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1508 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
1509 // Manually compute the final value for AR, checking for overflow.
1510
1511 // Check whether the backedge-taken count can be losslessly casted to
1512 // the addrec's type. The count is always unsigned.
1513 const SCEV *CastedMaxBECount =
1514 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
1515 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1516 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
1517 if (MaxBECount == RecastedMaxBECount) {
1518 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
1519 // Check whether Start+Step*MaxBECount has no unsigned overflow.
1520 const SCEV *ZMul =
1521 getMulExpr(LHS: CastedMaxBECount, RHS: Step, Flags: SCEV::FlagNone, Depth: Depth + 1);
1522 const SCEV *ZAdd = getZeroExtendExpr(
1523 Op: getAddExpr(LHS: Start, RHS: ZMul, Flags: SCEV::FlagNone, Depth: Depth + 1), Ty: WideTy,
1524 Depth: Depth + 1);
1525 const SCEV *WideStart = getZeroExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
1526 const SCEV *WideMaxBECount =
1527 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
1528 const SCEV *OperandExtendedAdd =
1529 getAddExpr(LHS: WideStart,
1530 RHS: getMulExpr(LHS: WideMaxBECount,
1531 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1532 Flags: SCEV::FlagNone, Depth: Depth + 1),
1533 Flags: SCEV::FlagNone, Depth: Depth + 1);
1534 if (ZAdd == OperandExtendedAdd) {
1535 // Cache knowledge of AR NUW, which is propagated to this AddRec.
1536 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1537 // Return the expression with the addrec on the outside.
1538 Start =
1539 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1540 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1541 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1542 }
1543 // Similar to above, only this time treat the step value as signed.
1544 // This covers loops that count down.
1545 OperandExtendedAdd =
1546 getAddExpr(LHS: WideStart,
1547 RHS: getMulExpr(LHS: WideMaxBECount,
1548 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1549 Flags: SCEV::FlagNone, Depth: Depth + 1),
1550 Flags: SCEV::FlagNone, Depth: Depth + 1);
1551 if (ZAdd == OperandExtendedAdd) {
1552 // Cache knowledge of AR NW, which is propagated to this AddRec.
1553 // Negative step causes unsigned wrap, but it still can't self-wrap.
1554 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1555 // Return the expression with the addrec on the outside.
1556 Start =
1557 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1558 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1559 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1560 }
1561 }
1562 }
1563
1564 // Normally, in the cases we can prove no-overflow via a
1565 // backedge guarding condition, we can also compute a backedge
1566 // taken count for the loop. The exceptions are assumptions and
1567 // guards present in the loop -- SCEV is not great at exploiting
1568 // these to compute max backedge taken counts, but can still use
1569 // these to prove lack of overflow. Use this fact to avoid
1570 // doing extra work that may not pay off.
1571 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount) || HasGuards ||
1572 !AC.assumptions().empty()) {
1573
1574 auto NewFlags = proveNoUnsignedWrapViaInduction(AR);
1575 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
1576 if (AR->hasNoUnsignedWrap()) {
1577 // Same as nuw case above - duplicated here to avoid a compile time
1578 // issue. It's not clear that the order of checks does matter, but
1579 // it's one of two issue possible causes for a change which was
1580 // reverted. Be conservative for the moment.
1581 Start =
1582 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1583 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1584 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1585 }
1586
1587 // For a negative step, we can extend the operands iff doing so only
1588 // traverses values in the range zext([0,UINT_MAX]).
1589 if (isKnownNegative(S: Step)) {
1590 const SCEV *N =
1591 getConstant(Val: APInt::getMaxValue(numBits: BitWidth) - getSignedRangeMin(S: Step));
1592 if (isLoopBackedgeGuardedByCond(L, Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N) ||
1593 isKnownOnEveryIteration(Pred: ICmpInst::ICMP_UGT, LHS: AR, RHS: N)) {
1594 // Cache knowledge of AR NW, which is propagated to this
1595 // AddRec. Negative step causes unsigned wrap, but it
1596 // still can't self-wrap.
1597 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1598 // Return the expression with the addrec on the outside.
1599 Start =
1600 getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1601 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1602 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1603 }
1604 }
1605 }
1606
1607 // zext({C,+,Step}) --> (zext(D) + zext({C-D,+,Step}))<nuw><nsw>
1608 // if D + (C - D + Step * n) could be proven to not unsigned wrap
1609 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1610 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
1611 const APInt &C = SC->getAPInt();
1612 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
1613 if (D != 0) {
1614 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1615 const SCEV *SResidual =
1616 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
1617 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1618 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: SCEV::FlagNSW | SCEV::FlagNUW,
1619 Depth: Depth + 1);
1620 }
1621 }
1622
1623 if (proveNoWrapByVaryingStart<SCEVZeroExtendExpr>(Start, Step, L)) {
1624 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNUW);
1625 Start = getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1626 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1627 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1628 }
1629 }
1630
1631 // zext(A % B) --> zext(A) % zext(B)
1632 {
1633 const SCEV *LHS;
1634 const SCEV *RHS;
1635 if (match(U: Op, P: m_scev_URem(LHS: m_SCEV(V&: LHS), RHS: m_SCEV(V&: RHS), SE&: *this)))
1636 return getURemExpr(LHS: getZeroExtendExpr(Op: LHS, Ty, Depth: Depth + 1),
1637 RHS: getZeroExtendExpr(Op: RHS, Ty, Depth: Depth + 1));
1638 }
1639
1640 // zext(A / B) --> zext(A) / zext(B).
1641 if (auto *Div = dyn_cast<SCEVUDivExpr>(Val&: Op))
1642 return getUDivExpr(LHS: getZeroExtendExpr(Op: Div->getLHS(), Ty, Depth: Depth + 1),
1643 RHS: getZeroExtendExpr(Op: Div->getRHS(), Ty, Depth: Depth + 1));
1644
1645 if (auto *SA = dyn_cast<SCEVAddExpr>(Val&: Op)) {
1646 // zext((A + B + ...)<nuw>) --> (zext(A) + zext(B) + ...)<nuw>
1647 if (SA->hasNoUnsignedWrap()) {
1648 // If the addition does not unsign overflow then we can, by definition,
1649 // commute the zero extension with the addition operation.
1650 SmallVector<SCEVUse, 4> Ops;
1651 for (SCEVUse Op : SA->operands())
1652 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1653 return getAddExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1654 }
1655
1656 const APInt *C, *C2;
1657 // zext (C + A)<nsw> -> (sext(C) + sext(A))<nsw> if zext (C + A)<nsw> >=s 0.
1658 // Currently the non-negative check is done manually, as isKnownNonNegative
1659 // is too expensive.
1660 if (SA->hasNoSignedWrap() &&
1661 match(V: SA, P: m_scev_Add(Op0: m_scev_APInt(C),
1662 Op1: m_scev_SMax(Op0: m_scev_APInt(C&: C2), Op1: m_SCEV()))) &&
1663 C->isNegative() && !C->isMinSignedValue() && C2->sge(RHS: C->abs())) {
1664 assert(isKnownNonNegative(SA) && "incorrectly determined non-negative");
1665 return getAddExpr(LHS: getSignExtendExpr(Op: SA->getOperand(i: 0), Ty, Depth: Depth + 1),
1666 RHS: getSignExtendExpr(Op: SA->getOperand(i: 1), Ty, Depth: Depth + 1),
1667 Flags: SCEV::FlagNSW, Depth: Depth + 1);
1668 }
1669
1670 // zext(C + x + y + ...) --> (zext(D) + zext((C - D) + x + y + ...))
1671 // if D + (C - D + x + y + ...) could be proven to not unsigned wrap
1672 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1673 //
1674 // Often address arithmetics contain expressions like
1675 // (zext (add (shl X, C1), C2)), for instance, (zext (5 + (4 * X))).
1676 // This transformation is useful while proving that such expressions are
1677 // equal or differ by a small constant amount, see LoadStoreVectorizer pass.
1678 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1679 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1680 if (D != 0) {
1681 const SCEV *SZExtD = getZeroExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1682 const SCEV *SResidual =
1683 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagNone, Depth);
1684 const SCEV *SZExtR = getZeroExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1685 return getAddExpr(LHS: SZExtD, RHS: SZExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
1686 Depth: Depth + 1);
1687 }
1688 }
1689 }
1690
1691 if (auto *SM = dyn_cast<SCEVMulExpr>(Val&: Op)) {
1692 // zext((A * B * ...)<nuw>) --> (zext(A) * zext(B) * ...)<nuw>
1693 if (SM->hasNoUnsignedWrap()) {
1694 // If the multiply does not unsign overflow then we can, by definition,
1695 // commute the zero extension with the multiply operation.
1696 SmallVector<SCEVUse, 4> Ops;
1697 for (SCEVUse Op : SM->operands())
1698 Ops.push_back(Elt: getZeroExtendExpr(Op, Ty, Depth: Depth + 1));
1699 return getMulExpr(Ops, Flags: SCEV::FlagNUW, Depth: Depth + 1);
1700 }
1701
1702 // zext(2^K * (trunc X to iN)) to iM ->
1703 // 2^K * (zext(trunc X to i{N-K}) to iM)<nuw>
1704 //
1705 // Proof:
1706 //
1707 // zext(2^K * (trunc X to iN)) to iM
1708 // = zext((trunc X to iN) << K) to iM
1709 // = zext((trunc X to i{N-K}) << K)<nuw> to iM
1710 // (because shl removes the top K bits)
1711 // = zext((2^K * (trunc X to i{N-K}))<nuw>) to iM
1712 // = (2^K * (zext(trunc X to i{N-K}) to iM))<nuw>.
1713 //
1714 const APInt *C;
1715 const SCEV *TruncRHS;
1716 if (match(V: SM,
1717 P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_scev_Trunc(Op0: m_SCEV(V&: TruncRHS)))) &&
1718 C->isPowerOf2()) {
1719 int NewTruncBits =
1720 getTypeSizeInBits(Ty: SM->getOperand(i: 1)->getType()) - C->logBase2();
1721 Type *NewTruncTy = IntegerType::get(C&: getContext(), NumBits: NewTruncBits);
1722 return getMulExpr(
1723 LHS: getZeroExtendExpr(Op: SM->getOperand(i: 0), Ty),
1724 RHS: getZeroExtendExpr(Op: getTruncateExpr(Op: TruncRHS, Ty: NewTruncTy), Ty),
1725 Flags: SCEV::FlagNUW, Depth: Depth + 1);
1726 }
1727 }
1728
1729 // zext(umin(x, y)) -> umin(zext(x), zext(y))
1730 // zext(umax(x, y)) -> umax(zext(x), zext(y))
1731 if (isa<SCEVUMinExpr>(Val: Op) || isa<SCEVUMaxExpr>(Val: Op)) {
1732 auto *MinMax = cast<SCEVMinMaxExpr>(Val&: Op);
1733 SmallVector<SCEVUse, 4> Operands;
1734 for (SCEVUse Operand : MinMax->operands())
1735 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1736 if (isa<SCEVUMinExpr>(Val: MinMax))
1737 return getUMinExpr(Operands);
1738 return getUMaxExpr(Operands);
1739 }
1740
1741 // zext(umin_seq(x, y)) -> umin_seq(zext(x), zext(y))
1742 if (auto *MinMax = dyn_cast<SCEVSequentialMinMaxExpr>(Val&: Op)) {
1743 assert(isa<SCEVSequentialUMinExpr>(MinMax) && "Not supported!");
1744 SmallVector<SCEVUse, 4> Operands;
1745 for (SCEVUse Operand : MinMax->operands())
1746 Operands.push_back(Elt: getZeroExtendExpr(Op: Operand, Ty));
1747 return getUMinExpr(Operands, /*Sequential*/ true);
1748 }
1749
1750 // The cast wasn't folded; create an explicit cast node.
1751 // Recompute the insert position, as it may have been invalidated.
1752 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1753 return S;
1754 SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1755 Op, Ty);
1756 UniqueSCEVs.insert(N: S, Token);
1757 S->computeAndSetCanonical(SE&: *this);
1758 registerUser(User: S, Ops: Op);
1759 return S;
1760}
1761
1762const SCEV *ScalarEvolution::getSignExtendExpr(SCEVUse Op, Type *Ty,
1763 unsigned Depth) {
1764 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1765 "This is not an extending conversion!");
1766 assert(isSCEVable(Ty) &&
1767 "This is not a conversion to a SCEVable type!");
1768 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1769 Ty = getEffectiveSCEVType(Ty);
1770
1771 FoldID ID(scSignExtend, Op, Ty);
1772 if (const SCEV *S = FoldCache.lookup(Val: ID))
1773 return S;
1774
1775 const SCEV *S = getSignExtendExprImpl(Op, Ty, Depth);
1776 if (!isa<SCEVSignExtendExpr>(Val: S))
1777 insertFoldCacheEntry(ID, S, FoldCache, FoldCacheUser);
1778 return S;
1779}
1780
1781const SCEV *ScalarEvolution::getSignExtendExprImpl(SCEVUse Op, Type *Ty,
1782 unsigned Depth) {
1783 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
1784 "This is not an extending conversion!");
1785 assert(isSCEVable(Ty) && "This is not a conversion to a SCEVable type!");
1786 assert(!Op->getType()->isPointerTy() && "Can't extend pointer!");
1787 Ty = getEffectiveSCEVType(Ty);
1788
1789 // Fold if the operand is constant.
1790 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
1791 return getConstant(Val: SC->getAPInt().sext(width: getTypeSizeInBits(Ty)));
1792
1793 // sext(sext(x)) --> sext(x)
1794 if (const SCEVSignExtendExpr *SS = dyn_cast<SCEVSignExtendExpr>(Val&: Op))
1795 return getSignExtendExpr(Op: SS->getOperand(), Ty, Depth: Depth + 1);
1796
1797 // sext(zext(x)) --> zext(x)
1798 if (const SCEVZeroExtendExpr *SZ = dyn_cast<SCEVZeroExtendExpr>(Val&: Op))
1799 return getZeroExtendExpr(Op: SZ->getOperand(), Ty, Depth: Depth + 1);
1800
1801 // If the operand is an affine AddRec with the no-signed-wrap flag, the
1802 // sign-extension distributes over the recurrence.
1803 const SCEV *Start, *Step;
1804 const Loop *L;
1805 if (Depth <= Opts.scalar_evolution_max_cast_depth &&
1806 match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1807 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1808 if (AR->hasNoSignedWrap()) {
1809 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1810 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1811 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1812 }
1813 }
1814
1815 // Before doing any expensive analysis, check to see if we've already
1816 // computed a SCEV for this Op and Ty.
1817 FoldingSetNodeID ID;
1818 ID.AddInteger(I: scSignExtend);
1819 ID.AddPointer(Ptr: Op.getOpaqueValue());
1820 ID.AddPointer(Ptr: Ty);
1821 FoldingSetInsertToken Token;
1822 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
1823 return S;
1824 // Limit recursion depth.
1825 if (Depth > Opts.scalar_evolution_max_cast_depth) {
1826 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
1827 Op, Ty);
1828 UniqueSCEVs.insert(N: S, Token);
1829 S->computeAndSetCanonical(SE&: *this);
1830 registerUser(User: S, Ops: Op);
1831 return S;
1832 }
1833
1834 // sext(trunc(x)) --> sext(x) or x or trunc(x)
1835 if (const SCEVTruncateExpr *ST = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
1836 // It's possible the bits taken off by the truncate were all sign bits. If
1837 // so, we should be able to simplify this further.
1838 const SCEV *X = ST->getOperand();
1839 ConstantRange CR = getSignedRange(S: X);
1840 unsigned TruncBits = getTypeSizeInBits(Ty: ST->getType());
1841 unsigned NewBits = getTypeSizeInBits(Ty);
1842 if (CR.truncate(BitWidth: TruncBits).signExtend(BitWidth: NewBits).contains(
1843 CR: CR.sextOrTrunc(BitWidth: NewBits)))
1844 return getTruncateOrSignExtend(V: X, Ty, Depth);
1845 }
1846
1847 if (auto *SA = dyn_cast<SCEVAddExpr>(Val&: Op)) {
1848 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
1849 if (SA->hasNoSignedWrap()) {
1850 // If the addition does not sign overflow then we can, by definition,
1851 // commute the sign extension with the addition operation.
1852 SmallVector<SCEVUse, 4> Ops;
1853 for (SCEVUse Op : SA->operands())
1854 Ops.push_back(Elt: getSignExtendExpr(Op, Ty, Depth: Depth + 1));
1855 return getAddExpr(Ops, Flags: SCEV::FlagNSW, Depth: Depth + 1);
1856 }
1857
1858 // sext(C + x + y + ...) --> (sext(D) + sext((C - D) + x + y + ...))
1859 // if D + (C - D + x + y + ...) could be proven to not signed wrap
1860 // where D maximizes the number of trailing zeros of (C - D + x + y + ...)
1861 //
1862 // For instance, this will bring two seemingly different expressions:
1863 // 1 + sext(5 + 20 * %x + 24 * %y) and
1864 // sext(6 + 20 * %x + 24 * %y)
1865 // to the same form:
1866 // 2 + sext(4 + 20 * %x + 24 * %y)
1867 if (const auto *SC = dyn_cast<SCEVConstant>(Val: SA->getOperand(i: 0))) {
1868 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantTerm: SC, WholeAddExpr: SA);
1869 if (D != 0) {
1870 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1871 const SCEV *SResidual =
1872 getAddExpr(LHS: getConstant(Val: -D), RHS: SA, Flags: SCEV::FlagNone, Depth);
1873 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1874 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
1875 Depth: Depth + 1);
1876 }
1877 }
1878 }
1879 // If the input value is a chrec scev, and we can prove that the value
1880 // did not overflow the old, smaller, value, we can sign extend all of the
1881 // operands (often constants). This allows analysis of something like
1882 // this: for (signed char X = 0; X < 100; ++X) { int Y = X; }
1883 if (match(U: Op, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L)))) {
1884 const auto *AR = cast<SCEVAddRecExpr>(Val&: Op);
1885 unsigned BitWidth = getTypeSizeInBits(Ty: AR->getType());
1886
1887 // The no-signed-wrap case is handled before the uniquing lookup above.
1888
1889 // Check whether the backedge-taken count is SCEVCouldNotCompute.
1890 // Note that this serves two purposes: It filters out loops that are
1891 // simply not analyzable, and it covers the case where this code is
1892 // being called from within backedge-taken count analysis, such that
1893 // attempting to ask for the backedge-taken count would likely result
1894 // in infinite recursion. In the later case, the analysis code will
1895 // cope with a conservative value, and it will take care to purge
1896 // that value once it has finished.
1897 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
1898 if (!isa<SCEVCouldNotCompute>(Val: MaxBECount)) {
1899 // Manually compute the final value for AR, checking for
1900 // overflow.
1901
1902 // Check whether the backedge-taken count can be losslessly casted to
1903 // the addrec's type. The count is always unsigned.
1904 const SCEV *CastedMaxBECount =
1905 getTruncateOrZeroExtend(V: MaxBECount, Ty: Start->getType(), Depth);
1906 const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend(
1907 V: CastedMaxBECount, Ty: MaxBECount->getType(), Depth);
1908 if (MaxBECount == RecastedMaxBECount) {
1909 Type *WideTy = IntegerType::get(C&: getContext(), NumBits: BitWidth * 2);
1910 // Check whether Start+Step*MaxBECount has no signed overflow.
1911 const SCEV *SMul =
1912 getMulExpr(LHS: CastedMaxBECount, RHS: Step, Flags: SCEV::FlagNone, Depth: Depth + 1);
1913 const SCEV *SAdd = getSignExtendExpr(
1914 Op: getAddExpr(LHS: Start, RHS: SMul, Flags: SCEV::FlagNone, Depth: Depth + 1), Ty: WideTy,
1915 Depth: Depth + 1);
1916 const SCEV *WideStart = getSignExtendExpr(Op: Start, Ty: WideTy, Depth: Depth + 1);
1917 const SCEV *WideMaxBECount =
1918 getZeroExtendExpr(Op: CastedMaxBECount, Ty: WideTy, Depth: Depth + 1);
1919 const SCEV *OperandExtendedAdd =
1920 getAddExpr(LHS: WideStart,
1921 RHS: getMulExpr(LHS: WideMaxBECount,
1922 RHS: getSignExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1923 Flags: SCEV::FlagNone, Depth: Depth + 1),
1924 Flags: SCEV::FlagNone, Depth: Depth + 1);
1925 if (SAdd == OperandExtendedAdd) {
1926 // Cache knowledge of AR NSW, which is propagated to this AddRec.
1927 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
1928 // Return the expression with the addrec on the outside.
1929 Start =
1930 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1931 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1932 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1933 }
1934 // Similar to above, only this time treat the step value as unsigned.
1935 // This covers loops that count up with an unsigned step.
1936 OperandExtendedAdd =
1937 getAddExpr(LHS: WideStart,
1938 RHS: getMulExpr(LHS: WideMaxBECount,
1939 RHS: getZeroExtendExpr(Op: Step, Ty: WideTy, Depth: Depth + 1),
1940 Flags: SCEV::FlagNone, Depth: Depth + 1),
1941 Flags: SCEV::FlagNone, Depth: Depth + 1);
1942 if (SAdd == OperandExtendedAdd) {
1943 // If AR wraps around then
1944 //
1945 // abs(Step) * MaxBECount > unsigned-max(AR->getType())
1946 // => SAdd != OperandExtendedAdd
1947 //
1948 // Thus (AR is not NW => SAdd != OperandExtendedAdd) <=>
1949 // (SAdd == OperandExtendedAdd => AR is NW)
1950
1951 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNW);
1952
1953 // Return the expression with the addrec on the outside.
1954 Start =
1955 getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1956 Step = getZeroExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1957 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1958 }
1959 }
1960 }
1961
1962 auto NewFlags = proveNoSignedWrapViaInduction(AR);
1963 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: NewFlags);
1964 if (AR->hasNoSignedWrap()) {
1965 // Same as nsw case above - duplicated here to avoid a compile time
1966 // issue. It's not clear that the order of checks does matter, but
1967 // it's one of two issue possible causes for a change which was
1968 // reverted. Be conservative for the moment.
1969 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1970 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1971 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1972 }
1973
1974 // sext({C,+,Step}) --> (sext(D) + sext({C-D,+,Step}))<nuw><nsw>
1975 // if D + (C - D + Step * n) could be proven to not signed wrap
1976 // where D maximizes the number of trailing zeros of (C - D + Step * n)
1977 if (const auto *SC = dyn_cast<SCEVConstant>(Val: Start)) {
1978 const APInt &C = SC->getAPInt();
1979 const APInt &D = extractConstantWithoutWrapping(SE&: *this, ConstantStart: C, Step);
1980 if (D != 0) {
1981 const SCEV *SSExtD = getSignExtendExpr(Op: getConstant(Val: D), Ty, Depth);
1982 const SCEV *SResidual =
1983 getAddRecExpr(Start: getConstant(Val: C - D), Step, L, Flags: AR->getNoWrapFlags());
1984 const SCEV *SSExtR = getSignExtendExpr(Op: SResidual, Ty, Depth: Depth + 1);
1985 return getAddExpr(LHS: SSExtD, RHS: SSExtR, Flags: (SCEV::FlagNSW | SCEV::FlagNUW),
1986 Depth: Depth + 1);
1987 }
1988 }
1989
1990 if (proveNoWrapByVaryingStart<SCEVSignExtendExpr>(Start, Step, L)) {
1991 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags: SCEV::FlagNSW);
1992 Start = getExtendAddRecStart<SCEVSignExtendExpr>(AR, Ty, SE: this, Depth: Depth + 1);
1993 Step = getSignExtendExpr(Op: Step, Ty, Depth: Depth + 1);
1994 return getAddRecExpr(Start, Step, L, Flags: AR->getNoWrapFlags());
1995 }
1996 }
1997
1998 // If the input value is provably positive and we could not simplify
1999 // away the sext build a zext instead.
2000 if (isKnownNonNegative(S: Op))
2001 return getZeroExtendExpr(Op, Ty, Depth: Depth + 1);
2002
2003 // sext(smin(x, y)) -> smin(sext(x), sext(y))
2004 // sext(smax(x, y)) -> smax(sext(x), sext(y))
2005 if (isa<SCEVSMinExpr>(Val: Op) || isa<SCEVSMaxExpr>(Val: Op)) {
2006 auto *MinMax = cast<SCEVMinMaxExpr>(Val&: Op);
2007 SmallVector<SCEVUse, 4> Operands;
2008 for (SCEVUse Operand : MinMax->operands())
2009 Operands.push_back(Elt: getSignExtendExpr(Op: Operand, Ty));
2010 if (isa<SCEVSMinExpr>(Val: MinMax))
2011 return getSMinExpr(Operands);
2012 return getSMaxExpr(Operands);
2013 }
2014
2015 // The cast wasn't folded; create an explicit cast node.
2016 // Recompute the insert position, as it may have been invalidated.
2017 if (const SCEV *S = UniqueSCEVs.lookup(ID, Token))
2018 return S;
2019 SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(Allocator&: SCEVAllocator),
2020 Op, Ty);
2021 UniqueSCEVs.insert(N: S, Token);
2022 S->computeAndSetCanonical(SE&: *this);
2023 registerUser(User: S, Ops: Op);
2024 return S;
2025}
2026
2027const SCEV *ScalarEvolution::getCastExpr(SCEVTypes Kind, SCEVUse Op, Type *Ty) {
2028 switch (Kind) {
2029 case scTruncate:
2030 return getTruncateExpr(Op, Ty);
2031 case scZeroExtend:
2032 return getZeroExtendExpr(Op, Ty);
2033 case scSignExtend:
2034 return getSignExtendExpr(Op, Ty);
2035 case scPtrToAddr: {
2036 const SCEV *Expr = getPtrToAddrExpr(Op);
2037 assert(Expr->getType() == Ty && "requested type must match");
2038 return Expr;
2039 }
2040 default:
2041 llvm_unreachable("Not a SCEV cast expression!");
2042 }
2043}
2044
2045/// getAnyExtendExpr - Return a SCEV for the given operand extended with
2046/// unspecified bits out to the given type.
2047const SCEV *ScalarEvolution::getAnyExtendExpr(SCEVUse Op, Type *Ty) {
2048 assert(getTypeSizeInBits(Op->getType()) < getTypeSizeInBits(Ty) &&
2049 "This is not an extending conversion!");
2050 assert(isSCEVable(Ty) &&
2051 "This is not a conversion to a SCEVable type!");
2052 Ty = getEffectiveSCEVType(Ty);
2053
2054 // Sign-extend negative constants.
2055 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val&: Op))
2056 if (SC->getAPInt().isNegative())
2057 return getSignExtendExpr(Op, Ty);
2058
2059 // Peel off a truncate cast.
2060 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val&: Op)) {
2061 const SCEV *NewOp = T->getOperand();
2062 if (getTypeSizeInBits(Ty: NewOp->getType()) < getTypeSizeInBits(Ty))
2063 return getAnyExtendExpr(Op: NewOp, Ty);
2064 return getTruncateOrNoop(V: NewOp, Ty);
2065 }
2066
2067 // Next try a zext cast. If the cast is folded, use it.
2068 const SCEV *ZExt = getZeroExtendExpr(Op, Ty);
2069 if (!isa<SCEVZeroExtendExpr>(Val: ZExt))
2070 return ZExt;
2071
2072 // Next try a sext cast. If the cast is folded, use it.
2073 const SCEV *SExt = getSignExtendExpr(Op, Ty);
2074 if (!isa<SCEVSignExtendExpr>(Val: SExt))
2075 return SExt;
2076
2077 // Force the cast to be folded into the operands of an addrec.
2078 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: Op)) {
2079 SmallVector<SCEVUse, 4> Ops;
2080 for (const SCEV *Op : AR->operands())
2081 Ops.push_back(Elt: getAnyExtendExpr(Op, Ty));
2082 return getAddRecExpr(Operands&: Ops, L: AR->getLoop(), Flags: SCEV::FlagNW);
2083 }
2084
2085 // If the expression is obviously signed, use the sext cast value.
2086 if (isa<SCEVSMaxExpr>(Val: Op))
2087 return SExt;
2088
2089 // Absent any other information, use the zext cast value.
2090 return ZExt;
2091}
2092
2093/// Process the given Ops list, which is a list of operands to be added under
2094/// the given scale, update the given map. This is a helper function for
2095/// getAddRecExpr. As an example of what it does, given a sequence of operands
2096/// that would form an add expression like this:
2097///
2098/// m + n + 13 + (A * (o + p + (B * (q + m + 29)))) + r + (-1 * r)
2099///
2100/// where A and B are constants, update the map with these values:
2101///
2102/// (m, 1+A*B), (n, 1), (o, A), (p, A), (q, A*B), (r, 0)
2103///
2104/// and add 13 + A*B*29 to AccumulatedConstant.
2105/// This will allow getAddRecExpr to produce this:
2106///
2107/// 13+A*B*29 + n + (m * (1+A*B)) + ((o + p) * A) + (q * A*B)
2108///
2109/// This form often exposes folding opportunities that are hidden in
2110/// the original operand list.
2111///
2112/// Return true iff it appears that any interesting folding opportunities
2113/// may be exposed. This helps getAddRecExpr short-circuit extra work in
2114/// the common case where no interesting opportunities are present, and
2115/// is also used as a check to avoid infinite recursion.
2116static bool
2117CollectAddOperandsWithScales(SmallDenseMap<const SCEV *, APInt, 16> &M,
2118 SmallVectorImpl<SCEVUse> &NewOps,
2119 APInt &AccumulatedConstant, ArrayRef<SCEVUse> Ops,
2120 const APInt &Scale, ScalarEvolution &SE) {
2121 bool Interesting = false;
2122
2123 // Iterate over the add operands. They are sorted, with constants first.
2124 unsigned i = 0;
2125 while (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Ops[i])) {
2126 ++i;
2127 // Pull a buried constant out to the outside.
2128 if (Scale != 1 || AccumulatedConstant != 0 || C->getValue()->isZero())
2129 Interesting = true;
2130 AccumulatedConstant += Scale * C->getAPInt();
2131 }
2132
2133 // Next comes everything else. We're especially interested in multiplies
2134 // here, but they're in the middle, so just visit the rest with one loop.
2135 for (; i != Ops.size(); ++i) {
2136 const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: Ops[i]);
2137 if (Mul && isa<SCEVConstant>(Val: Mul->getOperand(i: 0))) {
2138 APInt NewScale =
2139 Scale * cast<SCEVConstant>(Val: Mul->getOperand(i: 0))->getAPInt();
2140 if (Mul->getNumOperands() == 2 && isa<SCEVAddExpr>(Val: Mul->getOperand(i: 1))) {
2141 // A multiplication of a constant with another add; recurse.
2142 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: Mul->getOperand(i: 1));
2143 Interesting |= CollectAddOperandsWithScales(
2144 M, NewOps, AccumulatedConstant, Ops: Add->operands(), Scale: NewScale, SE);
2145 } else {
2146 // A multiplication of a constant with some other value. Update
2147 // the map.
2148 SmallVector<SCEVUse, 4> MulOps(drop_begin(RangeOrContainer: Mul->operands()));
2149 const SCEV *Key = SE.getMulExpr(Ops&: MulOps)->getCanonical();
2150 auto Pair = M.insert(KV: {Key, NewScale});
2151 if (Pair.second) {
2152 NewOps.push_back(Elt: Pair.first->first);
2153 } else {
2154 Pair.first->second += NewScale;
2155 // The map already had an entry for this value, which may indicate
2156 // a folding opportunity.
2157 Interesting = true;
2158 }
2159 }
2160 } else {
2161 // An ordinary operand. Update the map.
2162 auto Pair = M.insert(KV: {Ops[i]->getCanonical(), Scale});
2163 if (Pair.second) {
2164 NewOps.push_back(Elt: Pair.first->first);
2165 } else {
2166 Pair.first->second += Scale;
2167 // The map already had an entry for this value, which may indicate
2168 // a folding opportunity.
2169 Interesting = true;
2170 }
2171 }
2172 }
2173
2174 return Interesting;
2175}
2176
2177bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed,
2178 const SCEV *LHS, const SCEV *RHS,
2179 const Instruction *CtxI) {
2180 auto Operation = [this, BinOp](SCEVUse L, SCEVUse R) -> const SCEV * {
2181 switch (BinOp) {
2182 default:
2183 llvm_unreachable("Unsupported binary op");
2184 case Instruction::Add:
2185 return getAddExpr(LHS: L, RHS: R);
2186 case Instruction::Sub:
2187 return getMinusSCEV(LHS: L, RHS: R);
2188 case Instruction::Mul:
2189 return getMulExpr(LHS: L, RHS: R);
2190 }
2191 };
2192
2193 const SCEV *(ScalarEvolution::*Extension)(SCEVUse, Type *, unsigned) =
2194 Signed ? &ScalarEvolution::getSignExtendExpr
2195 : &ScalarEvolution::getZeroExtendExpr;
2196
2197 // Check ext(LHS op RHS) == ext(LHS) op ext(RHS)
2198 auto *NarrowTy = cast<IntegerType>(Val: LHS->getType());
2199 auto *WideTy =
2200 IntegerType::get(C&: NarrowTy->getContext(), NumBits: NarrowTy->getBitWidth() * 2);
2201
2202 const SCEV *A = (this->*Extension)(Operation(LHS, RHS), WideTy, 0);
2203 const SCEV *LHSB = (this->*Extension)(LHS, WideTy, 0);
2204 const SCEV *RHSB = (this->*Extension)(RHS, WideTy, 0);
2205 const SCEV *B = Operation(LHSB, RHSB);
2206 if (A == B)
2207 return true;
2208 // Can we use context to prove the fact we need?
2209 if (!CtxI)
2210 return false;
2211 // TODO: Support mul.
2212 if (BinOp == Instruction::Mul)
2213 return false;
2214 auto *RHSC = dyn_cast<SCEVConstant>(Val: RHS);
2215 // TODO: Lift this limitation.
2216 if (!RHSC)
2217 return false;
2218 APInt C = RHSC->getAPInt();
2219 unsigned NumBits = C.getBitWidth();
2220 bool IsSub = (BinOp == Instruction::Sub);
2221 bool IsNegativeConst = (Signed && C.isNegative());
2222 // Compute the direction and magnitude by which we need to check overflow.
2223 bool OverflowDown = IsSub ^ IsNegativeConst;
2224 APInt Magnitude = C;
2225 if (IsNegativeConst) {
2226 if (C == APInt::getSignedMinValue(numBits: NumBits))
2227 // TODO: SINT_MIN on inversion gives the same negative value, we don't
2228 // want to deal with that.
2229 return false;
2230 Magnitude = -C;
2231 }
2232
2233 ICmpInst::Predicate Pred = Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
2234 if (OverflowDown) {
2235 // To avoid overflow down, we need to make sure that MIN + Magnitude <= LHS.
2236 APInt Min = Signed ? APInt::getSignedMinValue(numBits: NumBits)
2237 : APInt::getMinValue(numBits: NumBits);
2238 APInt Limit = Min + Magnitude;
2239 return isKnownPredicateAt(Pred, LHS: getConstant(Val: Limit), RHS: LHS, CtxI);
2240 } else {
2241 // To avoid overflow up, we need to make sure that LHS <= MAX - Magnitude.
2242 APInt Max = Signed ? APInt::getSignedMaxValue(numBits: NumBits)
2243 : APInt::getMaxValue(numBits: NumBits);
2244 APInt Limit = Max - Magnitude;
2245 return isKnownPredicateAt(Pred, LHS, RHS: getConstant(Val: Limit), CtxI);
2246 }
2247}
2248
2249std::optional<SCEVFlags> ScalarEvolution::getStrengthenedNoWrapFlagsFromBinOp(
2250 const OverflowingBinaryOperator *OBO) {
2251 // It cannot be done any better.
2252 if (OBO->hasNoUnsignedWrap() && OBO->hasNoSignedWrap())
2253 return std::nullopt;
2254
2255 SCEVFlags Flags = SCEVFlags::FlagNone;
2256
2257 if (OBO->hasNoUnsignedWrap())
2258 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2259 if (OBO->hasNoSignedWrap())
2260 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2261
2262 bool Deduced = false;
2263
2264 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)OBO->getOpcode();
2265 const SCEV *LHS = getSCEV(V: OBO->getOperand(i_nocapture: 0));
2266 const SCEV *RHS = getSCEV(V: OBO->getOperand(i_nocapture: 1));
2267
2268 bool CanUseNSW = true;
2269 const APInt *ShiftAmt;
2270 // Treat `shl %a, C` as `mul %a, 1 << C`.
2271 if (match(V: OBO, P: m_Shl(L: m_Value(), R: m_APInt(Res&: ShiftAmt)))) {
2272 unsigned BitWidth = ShiftAmt->getBitWidth();
2273 if (ShiftAmt->uge(RHS: BitWidth))
2274 return std::nullopt;
2275 // NSW only transfers if the shift amount is < BitWidth - 1, as INT_MIN * -1
2276 // overflows.
2277 CanUseNSW = ShiftAmt->ult(RHS: BitWidth - 1);
2278 Opcode = Instruction::Mul;
2279 RHS = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: ShiftAmt->getZExtValue()));
2280 } else if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
2281 Opcode != Instruction::Mul) {
2282 return std::nullopt;
2283 }
2284
2285 const Instruction *CtxI =
2286 Opts.scalar_evolution_use_context_for_no_wrap_flag_strenghening
2287 ? dyn_cast<Instruction>(Val: OBO)
2288 : nullptr;
2289 if (!OBO->hasNoUnsignedWrap() &&
2290 willNotOverflow(BinOp: Opcode, /* Signed */ false, LHS, RHS, CtxI)) {
2291 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2292 Deduced = true;
2293 }
2294
2295 if (CanUseNSW && !OBO->hasNoSignedWrap() &&
2296 willNotOverflow(BinOp: Opcode, /* Signed */ true, LHS, RHS, CtxI)) {
2297 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2298 Deduced = true;
2299 }
2300
2301 if (Deduced)
2302 return Flags;
2303 return std::nullopt;
2304}
2305
2306// We're trying to construct a SCEV of type `Type' with `Ops' as operands and
2307// `OldFlags' as can't-wrap behavior. Infer a more aggressive set of
2308// can't-overflow flags for the operation if possible.
2309static SCEVFlags StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type,
2310 ArrayRef<SCEVUse> Ops, SCEVFlags Flags) {
2311 using namespace std::placeholders;
2312
2313 using OBO = OverflowingBinaryOperator;
2314
2315 bool CanAnalyze =
2316 Type == scAddExpr || Type == scAddRecExpr || Type == scMulExpr;
2317 (void)CanAnalyze;
2318 assert(CanAnalyze && "don't call from other places!");
2319
2320 SCEVFlags SignOrUnsignMask = SCEV::FlagNUW | SCEV::FlagNSW;
2321 SCEVFlags SignOrUnsignWrap =
2322 ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2323
2324 // If FlagNSW is true and all the operands are non-negative, infer FlagNUW.
2325 auto IsKnownNonNegative = [&](SCEVUse U) {
2326 return SE->isKnownNonNegative(S: U);
2327 };
2328
2329 if (SignOrUnsignWrap == SCEV::FlagNSW && all_of(Range&: Ops, P: IsKnownNonNegative))
2330 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SignOrUnsignMask);
2331
2332 SignOrUnsignWrap = ScalarEvolution::maskFlags(Flags, Mask: SignOrUnsignMask);
2333
2334 if (SignOrUnsignWrap != SignOrUnsignMask &&
2335 (Type == scAddExpr || Type == scMulExpr) && Ops.size() == 2 &&
2336 isa<SCEVConstant>(Val: Ops[0])) {
2337
2338 auto Opcode = [&] {
2339 switch (Type) {
2340 case scAddExpr:
2341 return Instruction::Add;
2342 case scMulExpr:
2343 return Instruction::Mul;
2344 default:
2345 llvm_unreachable("Unexpected SCEV op.");
2346 }
2347 }();
2348
2349 const APInt &C = cast<SCEVConstant>(Val: Ops[0])->getAPInt();
2350
2351 // (A <opcode> C) --> (A <opcode> C)<nsw> if the op doesn't sign overflow.
2352 if (!(SignOrUnsignWrap & SCEV::FlagNSW)) {
2353 auto NSWRegion =
2354 ConstantRange::makeExactNoWrapRegion(BinOp: Opcode, Other: C, NoWrapKind: OBO::NoSignedWrap);
2355 if (NSWRegion.contains(CR: SE->getSignedRange(S: Ops[1])))
2356 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
2357 }
2358
2359 // (A <opcode> C) --> (A <opcode> C)<nuw> if the op doesn't unsign overflow.
2360 if (!(SignOrUnsignWrap & SCEV::FlagNUW)) {
2361 auto NUWRegion =
2362 ConstantRange::makeExactNoWrapRegion(BinOp: Opcode, Other: C, NoWrapKind: OBO::NoUnsignedWrap);
2363 if (NUWRegion.contains(CR: SE->getUnsignedRange(S: Ops[1])))
2364 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2365 }
2366 }
2367
2368 // <0,+,nonnegative><nw> is also nuw
2369 // TODO: Add corresponding nsw case
2370 if (Type == scAddRecExpr && ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNW) &&
2371 !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) && Ops.size() == 2 &&
2372 Ops[0]->isZero() && IsKnownNonNegative(Ops[1]))
2373 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2374
2375 // both (udiv X, Y) * Y and Y * (udiv X, Y) are always NUW
2376 if (Type == scMulExpr && !ScalarEvolution::hasFlags(Flags, TestFlags: SCEV::FlagNUW) &&
2377 Ops.size() == 2) {
2378 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[0]))
2379 if (UDiv->getOperand(i: 1) == Ops[1])
2380 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2381 if (auto *UDiv = dyn_cast<SCEVUDivExpr>(Val: Ops[1]))
2382 if (UDiv->getOperand(i: 1) == Ops[0])
2383 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
2384 }
2385
2386 return Flags;
2387}
2388
2389bool ScalarEvolution::isAvailableAtLoopEntry(const SCEV *S, const Loop *L) {
2390 return isLoopInvariant(S, L) && properlyDominates(S, BB: L->getHeader());
2391}
2392
2393/// Get a canonical add expression, or something simpler if possible.
2394SCEVUse ScalarEvolution::getAddExpr(SmallVectorImpl<SCEVUse> &Ops,
2395 SCEVFlagsPair Flags, unsigned Depth) {
2396 SCEVFlags ExprFlags = Flags.ExprFlags;
2397 SCEVFlags UseFlags = Flags.UseFlags;
2398 assert(!(ExprFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2399 "only nuw or nsw allowed");
2400 assert(!(UseFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
2401 "only nuw or nsw allowed");
2402 assert(!Ops.empty() && "Cannot get empty add!");
2403 if (Ops.size() == 1) return Ops[0];
2404#ifndef NDEBUG
2405 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
2406 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
2407 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
2408 "SCEVAddExpr operand types don't match!");
2409 unsigned NumPtrs = count_if(
2410 Ops, [](const SCEV *Op) { return Op->getType()->isPointerTy(); });
2411 assert(NumPtrs <= 1 && "add has at most one pointer operand");
2412#endif
2413
2414 const SCEV *Folded = constantFoldAndGroupOps(
2415 Opts, SE&: *this, LI, DT, Ops,
2416 Fold: [](const APInt &C1, const APInt &C2) { return C1 + C2; },
2417 IsIdentity: [](const APInt &C) { return C.isZero(); }, // identity
2418 IsAbsorber: [](const APInt &C) { return false; }); // absorber
2419 if (Folded)
2420 return Folded;
2421
2422#ifndef NDEBUG
2423 // Keep track of operands after constant folding, for verification when adding
2424 // use-specific flags.
2425 const SmallVector<SCEVUse, 8> OrigOps(Ops.begin(), Ops.end());
2426#endif
2427
2428 unsigned Idx = isa<SCEVConstant>(Val: Ops[0]) ? 1 : 0;
2429
2430 // Delay expensive flag strengthening until necessary.
2431 auto ComputeFlags = [this, ExprFlags](ArrayRef<SCEVUse> Ops) {
2432 return StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops, Flags: ExprFlags);
2433 };
2434
2435 // Limit recursion calls depth.
2436 if (Depth > Opts.scalar_evolution_max_arith_depth ||
2437 hasHugeExpression(Opts, Ops))
2438 return {getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
2439
2440 if (SCEV *S = findExistingSCEVInCache(SCEVType: scAddExpr, Ops)) {
2441 // Don't strengthen flags if we have no new information.
2442 SCEVAddExpr *Add = static_cast<SCEVAddExpr *>(S);
2443 if (Add->getNoWrapFlags(Mask: ExprFlags) != ExprFlags)
2444 Add->setNoWrapFlags(ComputeFlags(Ops));
2445 return {S, UseFlags};
2446 }
2447
2448 // Okay, check to see if the same value occurs in the operand list more than
2449 // once. If so, merge them together into an multiply expression. Since we
2450 // sorted the list, these values are required to be adjacent.
2451 Type *Ty = Ops[0]->getType();
2452 bool FoundMatch = false;
2453 for (unsigned i = 0, e = Ops.size(); i != e-1; ++i)
2454 if (SCEVUse Op = SCEVUse::getCommon(A: Ops[i], B: Ops[i + 1])) {
2455 // X + Y + Y --> X + Y*2
2456 // Scan ahead to count how many equal operands there are.
2457 unsigned Count = 2;
2458 for (; i + Count != e; ++Count) {
2459 SCEVUse Common = SCEVUse::getCommon(A: Op, B: Ops[i + Count]);
2460 if (!Common)
2461 break;
2462 Op = Common;
2463 }
2464 // Merge the values into a multiply.
2465 SCEVUse Scale = getConstant(Ty, V: Count);
2466 SCEVUse Mul = getMulExpr(LHS: Scale, RHS: Op, Flags: SCEV::FlagNone, Depth: Depth + 1);
2467 if (Ops.size() == Count)
2468 return Mul;
2469 Ops[i] = Mul;
2470 Ops.erase(CS: Ops.begin()+i+1, CE: Ops.begin()+i+Count);
2471 --i; e -= Count - 1;
2472 FoundMatch = true;
2473 }
2474 if (FoundMatch)
2475 return getAddExpr(Ops, Flags: ExprFlags, Depth: Depth + 1);
2476
2477 // Check for truncates. If all the operands are truncated from the same
2478 // type, see if factoring out the truncate would permit the result to be
2479 // folded. eg., n*trunc(x) + m*trunc(y) --> trunc(trunc(m)*x + trunc(n)*y)
2480 // if the contents of the resulting outer trunc fold to something simple.
2481 auto FindTruncSrcType = [&]() -> Type * {
2482 // We're ultimately looking to fold an addrec of truncs and muls of only
2483 // constants and truncs, so if we find any other types of SCEV
2484 // as operands of the addrec then we bail and return nullptr here.
2485 // Otherwise, we return the type of the operand of a trunc that we find.
2486 if (auto *T = dyn_cast<SCEVTruncateExpr>(Val&: Ops[Idx]))
2487 return T->getOperand()->getType();
2488 if (const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
2489 SCEVUse LastOp = Mul->getOperand(i: Mul->getNumOperands() - 1);
2490 if (const auto *T = dyn_cast<SCEVTruncateExpr>(Val&: LastOp))
2491 return T->getOperand()->getType();
2492 }
2493 return nullptr;
2494 };
2495 if (auto *SrcType = FindTruncSrcType()) {
2496 SmallVector<SCEVUse, 8> LargeOps;
2497 bool Ok = true;
2498 // Check all the operands to see if they can be represented in the
2499 // source type of the truncate.
2500 for (const SCEV *Op : Ops) {
2501 if (const SCEVTruncateExpr *T = dyn_cast<SCEVTruncateExpr>(Val: Op)) {
2502 if (T->getOperand()->getType() != SrcType) {
2503 Ok = false;
2504 break;
2505 }
2506 LargeOps.push_back(Elt: T->getOperand());
2507 } else if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: Op)) {
2508 LargeOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2509 } else if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val: Op)) {
2510 SmallVector<SCEVUse, 8> LargeMulOps;
2511 for (unsigned j = 0, f = M->getNumOperands(); j != f && Ok; ++j) {
2512 if (const SCEVTruncateExpr *T =
2513 dyn_cast<SCEVTruncateExpr>(Val: M->getOperand(i: j))) {
2514 if (T->getOperand()->getType() != SrcType) {
2515 Ok = false;
2516 break;
2517 }
2518 LargeMulOps.push_back(Elt: T->getOperand());
2519 } else if (const auto *C = dyn_cast<SCEVConstant>(Val: M->getOperand(i: j))) {
2520 LargeMulOps.push_back(Elt: getAnyExtendExpr(Op: C, Ty: SrcType));
2521 } else {
2522 Ok = false;
2523 break;
2524 }
2525 }
2526 if (Ok)
2527 LargeOps.push_back(
2528 Elt: getMulExpr(Ops&: LargeMulOps, Flags: SCEV::FlagNone, Depth: Depth + 1));
2529 } else {
2530 Ok = false;
2531 break;
2532 }
2533 }
2534 if (Ok) {
2535 // Evaluate the expression in the larger type.
2536 const SCEV *Fold = getAddExpr(Ops&: LargeOps, Flags: SCEV::FlagNone, Depth: Depth + 1);
2537 // If it folds to something simple, use it. Otherwise, don't.
2538 if (isa<SCEVConstant>(Val: Fold) || isa<SCEVUnknown>(Val: Fold))
2539 return getTruncateExpr(Op: Fold, Ty);
2540 }
2541 }
2542
2543 if (Ops.size() == 2) {
2544 // Check if we have an expression of the form ((X + C1) - C2), where C1 and
2545 // C2 can be folded in a way that allows retaining wrapping flags of (X +
2546 // C1).
2547 const SCEV *A = Ops[0];
2548 const SCEV *B = Ops[1];
2549 auto *AddExpr = dyn_cast<SCEVAddExpr>(Val: B);
2550 auto *C = dyn_cast<SCEVConstant>(Val: A);
2551 if (AddExpr && C && isa<SCEVConstant>(Val: AddExpr->getOperand(i: 0))) {
2552 auto C1 = cast<SCEVConstant>(Val: AddExpr->getOperand(i: 0))->getAPInt();
2553 auto C2 = C->getAPInt();
2554 SCEVFlags PreservedFlags = SCEV::FlagNone;
2555
2556 APInt ConstAdd = C1 + C2;
2557 auto AddFlags = AddExpr->getNoWrapFlags();
2558 // Adding a smaller constant is NUW if the original AddExpr was NUW.
2559 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNUW) &&
2560 ConstAdd.ule(RHS: C1)) {
2561 PreservedFlags =
2562 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNUW);
2563 }
2564
2565 // Adding a constant with the same sign and small magnitude is NSW, if the
2566 // original AddExpr was NSW.
2567 if (ScalarEvolution::hasFlags(Flags: AddFlags, TestFlags: SCEV::FlagNSW) &&
2568 C1.isSignBitSet() == ConstAdd.isSignBitSet() &&
2569 ConstAdd.abs().ule(RHS: C1.abs())) {
2570 PreservedFlags =
2571 ScalarEvolution::setFlags(Flags: PreservedFlags, OnFlags: SCEV::FlagNSW);
2572 }
2573
2574 if (PreservedFlags != SCEV::FlagNone) {
2575 SmallVector<SCEVUse, 4> NewOps(AddExpr->operands());
2576 NewOps[0] = getConstant(Val: ConstAdd);
2577 return getAddExpr(Ops&: NewOps, Flags: PreservedFlags);
2578 }
2579 }
2580
2581 // Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext
2582 // (B), if trunc (A) + -A + B does not unsigned-wrap.
2583 const SCEVAddExpr *InnerAdd;
2584 if (match(S: B, P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
2585 const SCEV *NarrowA = getTruncateExpr(Op: A, Ty: InnerAdd->getType());
2586 if (NarrowA == getNegativeSCEV(V: InnerAdd->getOperand(i: 0)) &&
2587 getZeroExtendExpr(Op: NarrowA, Ty: B->getType()) == A &&
2588 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scAddExpr, Ops: {NarrowA, InnerAdd},
2589 Flags: SCEV::FlagNone),
2590 TestFlags: SCEV::FlagNUW)) {
2591 return getZeroExtendExpr(Op: getAddExpr(LHS: NarrowA, RHS: InnerAdd), Ty: B->getType());
2592 }
2593 }
2594 }
2595
2596 // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y)
2597 const SCEV *Y;
2598 if (Ops.size() == 2 &&
2599 match(U: Ops[0],
2600 P: m_scev_Mul(Op0: m_scev_AllOnes(),
2601 Op1: m_scev_URem(LHS: m_scev_Specific(S: Ops[1]), RHS: m_SCEV(V&: Y), SE&: *this))))
2602 return getMulExpr(LHS: Y, RHS: getUDivExpr(LHS: Ops[1], RHS: Y));
2603
2604 // Skip past any other cast SCEVs.
2605 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr)
2606 ++Idx;
2607
2608 // If there are add operands they would be next.
2609 if (Idx < Ops.size()) {
2610 bool DeletedAdd = false;
2611 // If the original flags and all inlined SCEVAddExprs are NUW, use the
2612 // common NUW flag for expression after inlining. Other flags cannot be
2613 // preserved, because they may depend on the original order of operations.
2614 SCEVFlags CommonFlags = maskFlags(Flags: ExprFlags, Mask: SCEV::FlagNUW);
2615 while (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[Idx])) {
2616 if (Ops.size() > Opts.scev_addops_inline_threshold ||
2617 Add->getNumOperands() > Opts.scev_addops_inline_threshold)
2618 break;
2619 // If we have an add, expand the add operands onto the end of the operands
2620 // list.
2621 Ops.erase(CI: Ops.begin()+Idx);
2622 append_range(C&: Ops, R: Add->operands());
2623 DeletedAdd = true;
2624 CommonFlags = maskFlags(Flags: CommonFlags, Mask: Add->getNoWrapFlags());
2625 }
2626
2627 // If we deleted at least one add, we added operands to the end of the list,
2628 // and they are not necessarily sorted. Recurse to resort and resimplify
2629 // any operands we just acquired.
2630 if (DeletedAdd)
2631 return getAddExpr(Ops, Flags: CommonFlags, Depth: Depth + 1);
2632 }
2633
2634 // Skip over the add expression until we get to a multiply.
2635 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
2636 ++Idx;
2637
2638 // Check to see if there are any folding opportunities present with
2639 // operands multiplied by constant values.
2640 if (Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx])) {
2641 uint64_t BitWidth = getTypeSizeInBits(Ty);
2642 SmallDenseMap<const SCEV *, APInt, 16> M;
2643 SmallVector<SCEVUse, 8> NewOps;
2644 APInt AccumulatedConstant(BitWidth, 0);
2645 if (CollectAddOperandsWithScales(M, NewOps, AccumulatedConstant,
2646 Ops, Scale: APInt(BitWidth, 1), SE&: *this)) {
2647 struct APIntCompare {
2648 bool operator()(const APInt &LHS, const APInt &RHS) const {
2649 return LHS.ult(RHS);
2650 }
2651 };
2652
2653 // Some interesting folding opportunity is present, so its worthwhile to
2654 // re-generate the operands list. Group the operands by constant scale,
2655 // to avoid multiplying by the same constant scale multiple times.
2656 std::map<APInt, SmallVector<SCEVUse, 4>, APIntCompare> MulOpLists;
2657 for (SCEVUse NewOp : NewOps)
2658 MulOpLists[M.find(Val: NewOp)->second].push_back(Elt: NewOp);
2659 // Re-generate the operands list.
2660 Ops.clear();
2661 if (AccumulatedConstant != 0)
2662 Ops.push_back(Elt: getConstant(Val: AccumulatedConstant));
2663 for (auto &MulOp : MulOpLists) {
2664 if (MulOp.first == 1) {
2665 Ops.push_back(Elt: getAddExpr(Ops&: MulOp.second, Flags: SCEV::FlagNone, Depth: Depth + 1));
2666 } else if (MulOp.first != 0) {
2667 Ops.push_back(
2668 Elt: getMulExpr(LHS: getConstant(Val: MulOp.first),
2669 RHS: getAddExpr(Ops&: MulOp.second, Flags: SCEV::FlagNone, Depth: Depth + 1),
2670 Flags: SCEV::FlagNone, Depth: Depth + 1));
2671 }
2672 }
2673 if (Ops.empty())
2674 return getZero(Ty);
2675 if (Ops.size() == 1)
2676 return Ops[0];
2677 return getAddExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
2678 }
2679 }
2680
2681 // Given a SCEVMulExpr and an operand index, return the product of all
2682 // operands except the one at OpIdx.
2683 auto StripFactor = [&](const SCEVMulExpr *M, unsigned OpIdx) -> SCEVUse {
2684 if (M->getNumOperands() == 2)
2685 return M->getOperand(i: OpIdx == 0);
2686 SmallVector<SCEVUse, 4> Remaining(M->operands().take_front(N: OpIdx));
2687 append_range(C&: Remaining, R: M->operands().drop_front(N: OpIdx + 1));
2688 return getMulExpr(Ops&: Remaining, Flags: SCEV::FlagNone, Depth: Depth + 1);
2689 };
2690
2691 // If we are adding something to a multiply expression, make sure the
2692 // something is not already an operand of the multiply. If so, merge it into
2693 // the multiply.
2694 for (; Idx < Ops.size() && isa<SCEVMulExpr>(Val: Ops[Idx]); ++Idx) {
2695 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val&: Ops[Idx]);
2696 for (unsigned MulOp = 0, e = Mul->getNumOperands(); MulOp != e; ++MulOp) {
2697 // Scan all terms to find every occurrence of common factor MulOpSCEV
2698 // and fold them in one shot:
2699 // A1*X + A2*X + ... + An*X --> X * (A1 + A2 + ... + An)
2700 SCEVUse MulOpSCEV = Mul->getOperand(i: MulOp);
2701 if (isa<SCEVConstant>(Val: MulOpSCEV))
2702 continue;
2703
2704 // Cofactors: 1 for bare addends matching MulOpSCEV, or the
2705 // remaining product for multiply terms containing MulOpSCEV.
2706 SmallVector<SCEVUse, 4> Cofactors;
2707 SmallVector<unsigned, 4> DeadIndices;
2708 for (unsigned AddOp = 0, e = Ops.size(); AddOp != e; ++AddOp) {
2709 if (SCEVUse Common = SCEVUse::getCommon(A: MulOpSCEV, B: Ops[AddOp])) {
2710 // W + X + (X * Y * Z) --> W + (X * ((Y*Z)+1))
2711 MulOpSCEV = Common;
2712 Cofactors.push_back(Elt: getOne(Ty));
2713 DeadIndices.push_back(Elt: AddOp);
2714 continue;
2715 }
2716
2717 if (AddOp <= Idx || !isa<SCEVMulExpr>(Val: Ops[AddOp]))
2718 continue;
2719
2720 const SCEVMulExpr *OtherMul = cast<SCEVMulExpr>(Val&: Ops[AddOp]);
2721 for (unsigned OMulOp = 0, OE = OtherMul->getNumOperands(); OMulOp != OE;
2722 ++OMulOp) {
2723 if (SCEVUse Common =
2724 SCEVUse::getCommon(A: MulOpSCEV, B: OtherMul->getOperand(i: OMulOp))) {
2725 // (A*B*C) + (A*D*E) --> A * (B*C + D*E)
2726 MulOpSCEV = Common;
2727 Cofactors.push_back(Elt: StripFactor(OtherMul, OMulOp));
2728 DeadIndices.push_back(Elt: AddOp);
2729 break;
2730 }
2731 }
2732 }
2733
2734 // Fold all collected cofactors with the anchor multiply's cofactor:
2735 // MulOpSCEV * (Cofactor_1 + ... + Cofactor_n + AnchorCofactor)
2736 if (!Cofactors.empty()) {
2737 Cofactors.push_back(Elt: StripFactor(Mul, MulOp));
2738
2739 SCEVUse InnerSum = getAddExpr(Ops&: Cofactors, Flags: SCEV::FlagNone, Depth: Depth + 1);
2740 SCEVUse OuterMul =
2741 getMulExpr(LHS: MulOpSCEV, RHS: InnerSum, Flags: SCEV::FlagNone, Depth: Depth + 1);
2742
2743 // DeadIndices does not include Idx (the anchor), hence +1.
2744 if (Ops.size() == DeadIndices.size() + 1)
2745 return OuterMul;
2746
2747 // Erase Ops[Idx] first, then erase DeadIndices in reverse order.
2748 // The -1 adjustment accounts for the shift from removing Idx;
2749 // reverse order means each erasure only shifts later positions,
2750 // which have already been processed.
2751 Ops.erase(CI: Ops.begin() + Idx);
2752 for (unsigned Dead : reverse(C&: DeadIndices))
2753 Ops.erase(CI: Ops.begin() + (Dead > Idx ? Dead - 1 : Dead));
2754
2755 Ops.push_back(Elt: OuterMul);
2756 return getAddExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
2757 }
2758 }
2759 }
2760
2761 // If there are any add recurrences in the operands list, see if any other
2762 // added values are loop invariant. If so, we can fold them into the
2763 // recurrence.
2764 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
2765 ++Idx;
2766
2767 // Scan over all recurrences, trying to fold loop invariants into them.
2768 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
2769 // Scan all of the other operands to this add and add them to the vector if
2770 // they are loop invariant w.r.t. the recurrence.
2771 SmallVector<SCEVUse, 8> LIOps;
2772 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
2773 const Loop *AddRecLoop = AddRec->getLoop();
2774 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
2775 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRecLoop)) {
2776 LIOps.push_back(Elt: Ops[i]);
2777 Ops.erase(CI: Ops.begin()+i);
2778 --i; --e;
2779 }
2780
2781 // If we found some loop invariants, fold them into the recurrence.
2782 if (!LIOps.empty()) {
2783 // Compute nowrap flags for the addition of the loop-invariant ops and
2784 // the addrec. Temporarily push it as an operand for that purpose. These
2785 // flags are valid in the scope of the addrec only.
2786 LIOps.push_back(Elt: AddRec);
2787 SCEVFlags Flags = ComputeFlags(LIOps);
2788 LIOps.pop_back();
2789
2790 // NLI + LI + {Start,+,Step} --> NLI + {LI+Start,+,Step}
2791 LIOps.push_back(Elt: AddRec->getStart());
2792
2793 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
2794
2795 // It is not in general safe to propagate flags valid on an add within
2796 // the addrec scope to one outside it. We must prove that the inner
2797 // scope is guaranteed to execute if the outer one does to be able to
2798 // safely propagate. We know the program is undefined if poison is
2799 // produced on the inner scoped addrec. We also know that *for this use*
2800 // the outer scoped add can't overflow (because of the flags we just
2801 // computed for the inner scoped add) without the program being undefined.
2802 // Proving that entry to the outer scope neccesitates entry to the inner
2803 // scope, thus proves the program undefined if the flags would be violated
2804 // in the outer scope.
2805 SCEVFlags AddFlags = Flags;
2806 if (AddFlags != SCEV::FlagNone) {
2807 auto *DefI = getDefiningScopeBound(Ops: LIOps);
2808 auto *ReachI = &*AddRecLoop->getHeader()->begin();
2809 if (!isGuaranteedToTransferExecutionTo(A: DefI, B: ReachI))
2810 AddFlags = SCEV::FlagNone;
2811 }
2812 AddRecOps[0] = getAddExpr(Ops&: LIOps, Flags: AddFlags, Depth: Depth + 1);
2813
2814 // Build the new addrec. Propagate the NUW and NSW flags if both the
2815 // outer add and the inner addrec are guaranteed to have no overflow.
2816 // Always propagate NW.
2817 Flags = AddRec->getNoWrapFlags(Mask: setFlags(Flags, OnFlags: SCEV::FlagNW));
2818 const SCEV *NewRec = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags);
2819
2820 // If all of the other operands were loop invariant, we are done.
2821 if (Ops.size() == 1) return NewRec;
2822
2823 // Otherwise, add the folded AddRec by the non-invariant parts.
2824 for (unsigned i = 0;; ++i)
2825 if (Ops[i] == AddRec) {
2826 Ops[i] = NewRec;
2827 break;
2828 }
2829 return getAddExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
2830 }
2831
2832 // Okay, if there weren't any loop invariants to be folded, check to see if
2833 // there are multiple AddRec's with the same loop induction variable being
2834 // added together. If so, we can fold them.
2835 for (unsigned OtherIdx = Idx+1;
2836 OtherIdx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
2837 ++OtherIdx) {
2838 // We expect the AddRecExpr's to be sorted in reverse dominance order,
2839 // so that the 1st found AddRecExpr is dominated by all others.
2840 assert(DT.dominates(
2841 cast<SCEVAddRecExpr>(Ops[OtherIdx])->getLoop()->getHeader(),
2842 AddRec->getLoop()->getHeader()) &&
2843 "AddRecExprs are not sorted in reverse dominance order?");
2844 if (AddRecLoop == cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx])->getLoop()) {
2845 // Other + {A,+,B}<L> + {C,+,D}<L> --> Other + {A+C,+,B+D}<L>
2846 SmallVector<SCEVUse, 4> AddRecOps(AddRec->operands());
2847 for (; OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
2848 ++OtherIdx) {
2849 const auto *OtherAddRec = cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
2850 if (OtherAddRec->getLoop() == AddRecLoop) {
2851 for (unsigned i = 0, e = OtherAddRec->getNumOperands();
2852 i != e; ++i) {
2853 if (i >= AddRecOps.size()) {
2854 append_range(C&: AddRecOps, R: OtherAddRec->operands().drop_front(N: i));
2855 break;
2856 }
2857 AddRecOps[i] =
2858 getAddExpr(LHS: AddRecOps[i], RHS: OtherAddRec->getOperand(i),
2859 Flags: SCEV::FlagNone, Depth: Depth + 1);
2860 }
2861 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
2862 }
2863 }
2864 // Step size has changed, so we cannot guarantee no self-wraparound.
2865 Ops[Idx] = getAddRecExpr(Operands&: AddRecOps, L: AddRecLoop, Flags: SCEV::FlagNone);
2866 return getAddExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
2867 }
2868 }
2869
2870 // Otherwise couldn't fold anything into this recurrence. Move onto the
2871 // next one.
2872 }
2873
2874 // Okay, it looks like we really DO need an add expr. Check to see if we
2875 // already have one, otherwise create a new one.
2876 assert((UseFlags == SCEV::FlagNone || equal(OrigOps, Ops)) &&
2877 "Tried to add SCEVUse flags after operands changed");
2878 return {getOrCreateAddExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
2879}
2880
2881const SCEV *ScalarEvolution::getOrCreateAddExpr(ArrayRef<SCEVUse> Ops,
2882 SCEVFlags Flags) {
2883 FoldingSetNodeID ID;
2884 ID.AddInteger(I: scAddExpr);
2885 for (SCEVUse Op : Ops)
2886 ID.AddPointer(Ptr: Op.getOpaqueValue());
2887 FoldingSetInsertToken Token;
2888 SCEVAddExpr *S = static_cast<SCEVAddExpr *>(UniqueSCEVs.lookup(ID, Token));
2889 if (!S) {
2890 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
2891 llvm::uninitialized_copy(Src&: Ops, Dst: O);
2892 S = new (SCEVAllocator)
2893 SCEVAddExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size());
2894 UniqueSCEVs.insert(N: S, Token);
2895 S->computeAndSetCanonical(SE&: *this);
2896 registerUser(User: S, Ops);
2897 }
2898 S->setNoWrapFlags(Flags);
2899 return S;
2900}
2901
2902const SCEV *ScalarEvolution::getOrCreateAddRecExpr(ArrayRef<SCEVUse> Ops,
2903 const Loop *L,
2904 SCEVFlags Flags) {
2905 FoldingSetNodeID ID;
2906 ID.AddInteger(I: scAddRecExpr);
2907 for (SCEVUse Op : Ops)
2908 ID.AddPointer(Ptr: Op.getOpaqueValue());
2909 ID.AddPointer(Ptr: L);
2910 FoldingSetInsertToken Token;
2911 SCEVAddRecExpr *S =
2912 static_cast<SCEVAddRecExpr *>(UniqueSCEVs.lookup(ID, Token));
2913 if (!S) {
2914 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
2915 llvm::uninitialized_copy(Src&: Ops, Dst: O);
2916 S = new (SCEVAllocator)
2917 SCEVAddRecExpr(ID.Intern(Allocator&: SCEVAllocator), O, Ops.size(), L);
2918 UniqueSCEVs.insert(N: S, Token);
2919 S->computeAndSetCanonical(SE&: *this);
2920 LoopUsers[L].push_back(Elt: S);
2921 registerUser(User: S, Ops);
2922 }
2923 setNoWrapFlags(AddRec: S, Flags);
2924 return S;
2925}
2926
2927const SCEV *ScalarEvolution::getOrCreateMulExpr(ArrayRef<SCEVUse> Ops,
2928 SCEVFlags Flags) {
2929 FoldingSetNodeID ID;
2930 ID.AddInteger(I: scMulExpr);
2931 for (SCEVUse Op : Ops)
2932 ID.AddPointer(Ptr: Op.getOpaqueValue());
2933 FoldingSetInsertToken Token;
2934 SCEVMulExpr *S = static_cast<SCEVMulExpr *>(UniqueSCEVs.lookup(ID, Token));
2935 if (!S) {
2936 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
2937 llvm::uninitialized_copy(Src&: Ops, Dst: O);
2938 S = new (SCEVAllocator) SCEVMulExpr(ID.Intern(Allocator&: SCEVAllocator),
2939 O, Ops.size());
2940 UniqueSCEVs.insert(N: S, Token);
2941 S->computeAndSetCanonical(SE&: *this);
2942 registerUser(User: S, Ops);
2943 }
2944 S->setNoWrapFlags(Flags);
2945 return S;
2946}
2947
2948const SCEV *ScalarEvolution::getOrCreateUDivExpr(SCEVUse LHS, SCEVUse RHS) {
2949 FoldingSetNodeID ID;
2950 ID.AddInteger(I: scUDivExpr);
2951 ID.AddPointer(Ptr: LHS.getOpaqueValue());
2952 ID.AddPointer(Ptr: RHS.getOpaqueValue());
2953 FoldingSetInsertToken Token;
2954 SCEV *S = UniqueSCEVs.lookup(ID, Token);
2955 if (!S) {
2956 S = new (SCEVAllocator) SCEVUDivExpr(ID.Intern(Allocator&: SCEVAllocator), LHS, RHS);
2957 UniqueSCEVs.insert(N: S, Token);
2958 S->computeAndSetCanonical(SE&: *this);
2959 registerUser(User: S, Ops: {LHS, RHS});
2960 }
2961 return S;
2962}
2963
2964static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow) {
2965 uint64_t k = i*j;
2966 if (j > 1 && k / j != i) Overflow = true;
2967 return k;
2968}
2969
2970/// Compute the result of "n choose k", the binomial coefficient. If an
2971/// intermediate computation overflows, Overflow will be set and the return will
2972/// be garbage. Overflow is not cleared on absence of overflow.
2973static uint64_t Choose(uint64_t n, uint64_t k, bool &Overflow) {
2974 // We use the multiplicative formula:
2975 // n(n-1)(n-2)...(n-(k-1)) / k(k-1)(k-2)...1 .
2976 // At each iteration, we take the n-th term of the numeral and divide by the
2977 // (k-n)th term of the denominator. This division will always produce an
2978 // integral result, and helps reduce the chance of overflow in the
2979 // intermediate computations. However, we can still overflow even when the
2980 // final result would fit.
2981
2982 if (n == 0 || n == k) return 1;
2983 if (k > n) return 0;
2984
2985 if (k > n/2)
2986 k = n-k;
2987
2988 uint64_t r = 1;
2989 for (uint64_t i = 1; i <= k; ++i) {
2990 r = umul_ov(i: r, j: n-(i-1), Overflow);
2991 r /= i;
2992 }
2993 return r;
2994}
2995
2996/// Determine if any of the operands in this SCEV are a constant or if
2997/// any of the add or multiply expressions in this SCEV contain a constant.
2998static bool containsConstantInAddMulChain(const SCEV *StartExpr) {
2999 struct FindConstantInAddMulChain {
3000 bool FoundConstant = false;
3001
3002 bool follow(const SCEV *S) {
3003 FoundConstant |= isa<SCEVConstant>(Val: S);
3004 return isa<SCEVAddExpr>(Val: S) || isa<SCEVMulExpr>(Val: S);
3005 }
3006
3007 bool isDone() const {
3008 return FoundConstant;
3009 }
3010 };
3011
3012 FindConstantInAddMulChain F;
3013 SCEVTraversal<FindConstantInAddMulChain> ST(F);
3014 ST.visitAll(Root: StartExpr);
3015 return F.FoundConstant;
3016}
3017
3018/// Get a canonical multiply expression, or something simpler if possible.
3019SCEVUse ScalarEvolution::getMulExpr(SmallVectorImpl<SCEVUse> &Ops,
3020 SCEVFlagsPair Flags, unsigned Depth) {
3021 SCEVFlags ExprFlags = Flags.ExprFlags;
3022 SCEVFlags UseFlags = Flags.UseFlags;
3023 assert(ExprFlags == maskFlags(ExprFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3024 "only nuw or nsw allowed");
3025 assert(UseFlags == maskFlags(UseFlags, SCEV::FlagNUW | SCEV::FlagNSW) &&
3026 "only nuw or nsw allowed");
3027 assert(!Ops.empty() && "Cannot get empty mul!");
3028 if (Ops.size() == 1) return Ops[0];
3029#ifndef NDEBUG
3030 Type *ETy = Ops[0]->getType();
3031 assert(!ETy->isPointerTy());
3032 for (unsigned i = 1, e = Ops.size(); i != e; ++i)
3033 assert(Ops[i]->getType() == ETy &&
3034 "SCEVMulExpr operand types don't match!");
3035#endif
3036
3037 const SCEV *Folded = constantFoldAndGroupOps(
3038 Opts, SE&: *this, LI, DT, Ops,
3039 Fold: [](const APInt &C1, const APInt &C2) { return C1 * C2; },
3040 IsIdentity: [](const APInt &C) { return C.isOne(); }, // identity
3041 IsAbsorber: [](const APInt &C) { return C.isZero(); }); // absorber
3042 if (Folded)
3043 return Folded;
3044
3045#ifndef NDEBUG
3046 // Keep track of operands after constant folding, for verification when adding
3047 // use-specific flags.
3048 const SmallVector<SCEVUse, 8> OrigOps(Ops.begin(), Ops.end());
3049#endif
3050
3051 // Delay expensive flag strengthening until necessary.
3052 auto ComputeFlags = [this, ExprFlags](const ArrayRef<SCEVUse> Ops) {
3053 return StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops, Flags: ExprFlags);
3054 };
3055
3056 // Limit recursion calls depth.
3057 if (Depth > Opts.scalar_evolution_max_arith_depth ||
3058 hasHugeExpression(Opts, Ops))
3059 return {getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
3060
3061 if (SCEV *S = findExistingSCEVInCache(SCEVType: scMulExpr, Ops)) {
3062 // Don't strengthen flags if we have no new information.
3063 SCEVMulExpr *Mul = static_cast<SCEVMulExpr *>(S);
3064 if (Mul->getNoWrapFlags(Mask: ExprFlags) != ExprFlags)
3065 Mul->setNoWrapFlags(ComputeFlags(Ops));
3066 return {S, UseFlags};
3067 }
3068
3069 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: Ops[0])) {
3070 if (Ops.size() == 2) {
3071 // C1*(C2+V) -> C1*C2 + C1*V
3072 // If any of Add's ops are Adds or Muls with a constant, apply this
3073 // transformation as well.
3074 //
3075 // TODO: There are some cases where this transformation is not
3076 // profitable; for example, Add = (C0 + X) * Y + Z. Maybe the scope of
3077 // this transformation should be narrowed down.
3078 const SCEV *Op0, *Op1;
3079 if (match(U: Ops[1], P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))) &&
3080 containsConstantInAddMulChain(StartExpr: Ops[1])) {
3081 const SCEV *LHS = getMulExpr(LHS: LHSC, RHS: Op0, Flags: SCEV::FlagNone, Depth: Depth + 1);
3082 const SCEV *RHS = getMulExpr(LHS: LHSC, RHS: Op1, Flags: SCEV::FlagNone, Depth: Depth + 1);
3083 return getAddExpr(LHS, RHS, Flags: SCEV::FlagNone, Depth: Depth + 1);
3084 }
3085
3086 if (Ops[0]->isAllOnesValue()) {
3087 // If we have a mul by -1 of an add, try distributing the -1 among the
3088 // add operands.
3089 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: Ops[1])) {
3090 SmallVector<SCEVUse, 4> NewOps;
3091 bool AnyFolded = false;
3092 for (const SCEV *AddOp : Add->operands()) {
3093 const SCEV *Mul =
3094 getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddOp), Flags: SCEV::FlagNone, Depth: Depth + 1);
3095 if (!isa<SCEVMulExpr>(Val: Mul)) AnyFolded = true;
3096 NewOps.push_back(Elt: Mul);
3097 }
3098 if (AnyFolded)
3099 return getAddExpr(Ops&: NewOps, Flags: SCEV::FlagNone, Depth: Depth + 1);
3100 } else if (const auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: Ops[1])) {
3101 // Negation preserves a recurrence's no self-wrap property.
3102 SmallVector<SCEVUse, 4> Operands;
3103 for (const SCEV *AddRecOp : AddRec->operands())
3104 Operands.push_back(Elt: getMulExpr(LHS: Ops[0], RHS: SCEVUse(AddRecOp),
3105 Flags: SCEV::FlagNone, Depth: Depth + 1));
3106 // Let M be the minimum representable signed value. AddRec with nsw
3107 // multiplied by -1 can have signed overflow if and only if it takes a
3108 // value of M: M * (-1) would stay M and (M + 1) * (-1) would be the
3109 // maximum signed value. In all other cases signed overflow is
3110 // impossible.
3111 auto FlagsMask = SCEV::FlagNW;
3112 if (AddRec->hasNoSignedWrap()) {
3113 auto MinInt =
3114 APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: AddRec->getType()));
3115 if (getSignedRangeMin(S: AddRec) != MinInt)
3116 FlagsMask = setFlags(Flags: FlagsMask, OnFlags: SCEV::FlagNSW);
3117 }
3118 return getAddRecExpr(Operands, L: AddRec->getLoop(),
3119 Flags: AddRec->getNoWrapFlags(Mask: FlagsMask));
3120 }
3121 }
3122
3123 // Try to push the constant operand into a ZExt: C * zext (A + B) ->
3124 // zext (C*A + C*B) if trunc (C) * (A + B) does not unsigned-wrap.
3125 const SCEVAddExpr *InnerAdd;
3126 if (match(U: Ops[1], P: m_scev_ZExt(Op0: m_scev_Add(V&: InnerAdd)))) {
3127 const SCEV *NarrowC = getTruncateExpr(Op: LHSC, Ty: InnerAdd->getType());
3128 if (isa<SCEVConstant>(Val: InnerAdd->getOperand(i: 0)) &&
3129 getZeroExtendExpr(Op: NarrowC, Ty: Ops[1]->getType()) == LHSC &&
3130 hasFlags(Flags: StrengthenNoWrapFlags(SE: this, Type: scMulExpr, Ops: {NarrowC, InnerAdd},
3131 Flags: SCEV::FlagNone),
3132 TestFlags: SCEV::FlagNUW)) {
3133 const SCEV *Res =
3134 getMulExpr(LHS: NarrowC, RHS: InnerAdd, Flags: SCEV::FlagNUW, Depth: Depth + 1);
3135 return getZeroExtendExpr(Op: Res, Ty: Ops[1]->getType(), Depth: Depth + 1);
3136 };
3137 }
3138
3139 // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2,
3140 // D is a multiple of C2, and C1 is a multiple of C2. If C2 is a multiple
3141 // of C1, fold to (D /u (C2 /u C1)).
3142 const SCEV *D;
3143 APInt C1V = LHSC->getAPInt();
3144 // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. Don't treat -1
3145 // as -1 * 1, as it won't enable additional folds.
3146 if (C1V.isNegative() && !C1V.isMinSignedValue() && !C1V.isAllOnes())
3147 C1V = C1V.abs();
3148 const SCEVConstant *C2;
3149 if (C1V.isPowerOf2() &&
3150 match(U: Ops[1], P: m_scev_UDiv(Op0: m_SCEV(V&: D), Op1: m_SCEVConstant(V&: C2))) &&
3151 C2->getAPInt().isPowerOf2() &&
3152 C1V.logBase2() <= getMinTrailingZeros(S: D)) {
3153 const SCEV *NewMul = nullptr;
3154 if (C1V.uge(RHS: C2->getAPInt())) {
3155 NewMul = getMulExpr(LHS: getUDivExpr(LHS: getConstant(Val: C1V), RHS: C2), RHS: D);
3156 } else if (C2->getAPInt().logBase2() <= getMinTrailingZeros(S: D)) {
3157 assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier");
3158 NewMul = getUDivExpr(LHS: D, RHS: getUDivExpr(LHS: C2, RHS: getConstant(Val: C1V)));
3159 }
3160 if (NewMul)
3161 return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(V: NewMul);
3162 }
3163 }
3164 }
3165
3166 // Skip over the add expression until we get to a multiply.
3167 unsigned Idx = 0;
3168 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scMulExpr)
3169 ++Idx;
3170
3171 // If there are mul operands inline them all into this expression.
3172 if (Idx < Ops.size()) {
3173 bool DeletedMul = false;
3174 while (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val&: Ops[Idx])) {
3175 if (Ops.size() > Opts.scev_mulops_inline_threshold)
3176 break;
3177 // If we have an mul, expand the mul operands onto the end of the
3178 // operands list.
3179 Ops.erase(CI: Ops.begin()+Idx);
3180 append_range(C&: Ops, R: Mul->operands());
3181 DeletedMul = true;
3182 }
3183
3184 // If we deleted at least one mul, we added operands to the end of the
3185 // list, and they are not necessarily sorted. Recurse to resort and
3186 // resimplify any operands we just acquired.
3187 if (DeletedMul)
3188 return getMulExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
3189 }
3190
3191 // If there are any add recurrences in the operands list, see if any other
3192 // added values are loop invariant. If so, we can fold them into the
3193 // recurrence.
3194 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddRecExpr)
3195 ++Idx;
3196
3197 // Scan over all recurrences, trying to fold loop invariants into them.
3198 for (; Idx < Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[Idx]); ++Idx) {
3199 // Scan all of the other operands to this mul and add them to the vector
3200 // if they are loop invariant w.r.t. the recurrence.
3201 SmallVector<SCEVUse, 8> LIOps;
3202 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val&: Ops[Idx]);
3203 for (unsigned i = 0, e = Ops.size(); i != e; ++i)
3204 if (isAvailableAtLoopEntry(S: Ops[i], L: AddRec->getLoop())) {
3205 LIOps.push_back(Elt: Ops[i]);
3206 Ops.erase(CI: Ops.begin()+i);
3207 --i; --e;
3208 }
3209
3210 // If we found some loop invariants, fold them into the recurrence.
3211 if (!LIOps.empty()) {
3212 // NLI * LI * {Start,+,Step} --> NLI * {LI*Start,+,LI*Step}
3213 SmallVector<SCEVUse, 4> NewOps;
3214 NewOps.reserve(N: AddRec->getNumOperands());
3215 const SCEV *Scale = getMulExpr(Ops&: LIOps, Flags: SCEV::FlagNone, Depth: Depth + 1);
3216
3217 // If both the mul and addrec are nuw, we can preserve nuw.
3218 // If both the mul and addrec are nsw, we can only preserve nsw if either
3219 // a) they are also nuw, or
3220 // b) all multiplications of addrec operands with scale are nsw.
3221 SCEVFlags Flags = AddRec->getNoWrapFlags(Mask: ComputeFlags({Scale, AddRec}));
3222
3223 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
3224 NewOps.push_back(Elt: getMulExpr(LHS: Scale, RHS: AddRec->getOperand(i),
3225 Flags: SCEV::FlagNone, Depth: Depth + 1));
3226
3227 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW) && !hasFlags(Flags, TestFlags: SCEV::FlagNUW)) {
3228 ConstantRange NSWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
3229 BinOp: Instruction::Mul, Other: getSignedRange(S: Scale),
3230 NoWrapKind: OverflowingBinaryOperator::NoSignedWrap);
3231 if (!NSWRegion.contains(CR: getSignedRange(S: AddRec->getOperand(i))))
3232 Flags = clearFlags(Flags, OffFlags: SCEV::FlagNSW);
3233 }
3234 }
3235
3236 const SCEV *NewRec = getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), Flags);
3237
3238 // If all of the other operands were loop invariant, we are done.
3239 if (Ops.size() == 1) return NewRec;
3240
3241 // Otherwise, multiply the folded AddRec by the non-invariant parts.
3242 for (unsigned i = 0;; ++i)
3243 if (Ops[i] == AddRec) {
3244 Ops[i] = NewRec;
3245 break;
3246 }
3247 return getMulExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
3248 }
3249
3250 // Okay, if there weren't any loop invariants to be folded, check to see
3251 // if there are multiple AddRec's with the same loop induction variable
3252 // being multiplied together. If so, we can fold them.
3253
3254 // {A1,+,A2,+,...,+,An}<L> * {B1,+,B2,+,...,+,Bn}<L>
3255 // = {x=1 in [ sum y=x..2x [ sum z=max(y-x, y-n)..min(x,n) [
3256 // choose(x, 2x)*choose(2x-y, x-z)*A_{y-z}*B_z
3257 // ]]],+,...up to x=2n}.
3258 // Note that the arguments to choose() are always integers with values
3259 // known at compile time, never SCEV objects.
3260 //
3261 // The implementation avoids pointless extra computations when the two
3262 // addrec's are of different length (mathematically, it's equivalent to
3263 // an infinite stream of zeros on the right).
3264 bool OpsModified = false;
3265 for (unsigned OtherIdx = Idx+1;
3266 OtherIdx != Ops.size() && isa<SCEVAddRecExpr>(Val: Ops[OtherIdx]);
3267 ++OtherIdx) {
3268 const SCEVAddRecExpr *OtherAddRec =
3269 dyn_cast<SCEVAddRecExpr>(Val&: Ops[OtherIdx]);
3270 if (!OtherAddRec || OtherAddRec->getLoop() != AddRec->getLoop())
3271 continue;
3272
3273 // Limit max number of arguments to avoid creation of unreasonably big
3274 // SCEVAddRecs with very complex operands.
3275 if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 >
3276 Opts.scalar_evolution_max_add_rec_size ||
3277 hasHugeExpression(Opts, Ops: {AddRec, OtherAddRec}))
3278 continue;
3279
3280 bool Overflow = false;
3281 Type *Ty = AddRec->getType();
3282 bool LargerThan64Bits = getTypeSizeInBits(Ty) > 64;
3283 SmallVector<SCEVUse, 7> AddRecOps;
3284 for (int x = 0, xe = AddRec->getNumOperands() +
3285 OtherAddRec->getNumOperands() - 1; x != xe && !Overflow; ++x) {
3286 SmallVector<SCEVUse, 7> SumOps;
3287 for (int y = x, ye = 2*x+1; y != ye && !Overflow; ++y) {
3288 uint64_t Coeff1 = Choose(n: x, k: 2*x - y, Overflow);
3289 for (int z = std::max(a: y-x, b: y-(int)AddRec->getNumOperands()+1),
3290 ze = std::min(a: x+1, b: (int)OtherAddRec->getNumOperands());
3291 z < ze && !Overflow; ++z) {
3292 uint64_t Coeff2 = Choose(n: 2*x - y, k: x-z, Overflow);
3293 uint64_t Coeff;
3294 if (LargerThan64Bits)
3295 Coeff = umul_ov(i: Coeff1, j: Coeff2, Overflow);
3296 else
3297 Coeff = Coeff1*Coeff2;
3298 const SCEV *CoeffTerm = getConstant(Ty, V: Coeff);
3299 const SCEV *Term1 = AddRec->getOperand(i: y-z);
3300 const SCEV *Term2 = OtherAddRec->getOperand(i: z);
3301 SumOps.push_back(
3302 Elt: getMulExpr(Op0: CoeffTerm, Op1: Term1, Op2: Term2, Flags: SCEV::FlagNone, Depth: Depth + 1));
3303 }
3304 }
3305 if (SumOps.empty())
3306 SumOps.push_back(Elt: getZero(Ty));
3307 AddRecOps.push_back(Elt: getAddExpr(Ops&: SumOps, Flags: SCEV::FlagNone, Depth: Depth + 1));
3308 }
3309 if (!Overflow) {
3310 const SCEV *NewAddRec =
3311 getAddRecExpr(Operands&: AddRecOps, L: AddRec->getLoop(), Flags: SCEV::FlagNone);
3312 if (Ops.size() == 2) return NewAddRec;
3313 Ops[Idx] = NewAddRec;
3314 Ops.erase(CI: Ops.begin() + OtherIdx); --OtherIdx;
3315 OpsModified = true;
3316 AddRec = dyn_cast<SCEVAddRecExpr>(Val: NewAddRec);
3317 if (!AddRec)
3318 break;
3319 }
3320 }
3321 if (OpsModified)
3322 return getMulExpr(Ops, Flags: SCEV::FlagNone, Depth: Depth + 1);
3323
3324 // Otherwise couldn't fold anything into this recurrence. Move onto the
3325 // next one.
3326 }
3327
3328 // Okay, it looks like we really DO need an mul expr. Check to see if we
3329 // already have one, otherwise create a new one.
3330 assert((UseFlags == SCEV::FlagNone || equal(OrigOps, Ops)) &&
3331 "Tried to add SCEVUse flags after operands changed");
3332 return {getOrCreateMulExpr(Ops, Flags: ComputeFlags(Ops)), UseFlags};
3333}
3334
3335/// Represents an unsigned remainder expression based on unsigned division.
3336const SCEV *ScalarEvolution::getURemExpr(SCEVUse LHS, SCEVUse RHS) {
3337 assert(getEffectiveSCEVType(LHS->getType()) ==
3338 getEffectiveSCEVType(RHS->getType()) &&
3339 "SCEVURemExpr operand types don't match!");
3340
3341 // Short-circuit easy cases
3342 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3343 // If constant is one, the result is trivial
3344 if (RHSC->getValue()->isOne())
3345 return getZero(Ty: LHS->getType()); // X urem 1 --> 0
3346
3347 // If constant is a power of two, fold into a zext(trunc(LHS)).
3348 if (RHSC->getAPInt().isPowerOf2()) {
3349 Type *FullTy = LHS->getType();
3350 Type *TruncTy =
3351 IntegerType::get(C&: getContext(), NumBits: RHSC->getAPInt().logBase2());
3352 return getZeroExtendExpr(Op: getTruncateExpr(Op: LHS, Ty: TruncTy), Ty: FullTy);
3353 }
3354 }
3355
3356 // Fallback to %a == %x urem %y == %x -<nuw> ((%x udiv %y) *<nuw> %y)
3357 const SCEV *UDiv = getUDivExpr(LHS, RHS);
3358 const SCEV *Mult = getMulExpr(LHS: UDiv, RHS, Flags: SCEV::FlagNUW);
3359 return getMinusSCEV(LHS, RHS: Mult, Flags: SCEV::FlagNUW);
3360}
3361
3362/// Get a canonical unsigned division expression, or something simpler if
3363/// possible.
3364const SCEV *ScalarEvolution::getUDivExpr(SCEVUse LHS, SCEVUse RHS) {
3365 assert(!LHS->getType()->isPointerTy() &&
3366 "SCEVUDivExpr operand can't be pointer!");
3367 assert(LHS->getType() == RHS->getType() &&
3368 "SCEVUDivExpr operand types don't match!");
3369
3370 if (SCEV *S = findExistingSCEVInCache(SCEVType: scUDivExpr, Ops: {LHS, RHS}))
3371 return S;
3372
3373 // 0 udiv Y == 0
3374 if (match(U: LHS, P: m_scev_Zero()))
3375 return LHS;
3376
3377 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
3378 if (RHSC->getValue()->isOne())
3379 return LHS; // X udiv 1 --> x
3380 // If the denominator is zero, the result of the udiv is undefined. Don't
3381 // try to analyze it, because the resolution chosen here may differ from
3382 // the resolution chosen in other parts of the compiler.
3383 if (!RHSC->getValue()->isZero()) {
3384 // Determine if the division can be folded into the operands of
3385 // its operands.
3386 // TODO: Generalize this to non-constants by using known-bits information.
3387 Type *Ty = LHS->getType();
3388 unsigned LZ = RHSC->getAPInt().countl_zero();
3389 unsigned MaxShiftAmt = getTypeSizeInBits(Ty) - LZ - 1;
3390 // For non-power-of-two values, effectively round the value up to the
3391 // nearest power of two.
3392 if (!RHSC->getAPInt().isPowerOf2())
3393 ++MaxShiftAmt;
3394 IntegerType *ExtTy =
3395 IntegerType::get(C&: getContext(), NumBits: getTypeSizeInBits(Ty) + MaxShiftAmt);
3396 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
3397 if (const SCEVConstant *Step =
3398 dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE&: *this))) {
3399 // {X,+,N}/C --> {X/C,+,N/C} if safe and N/C can be folded.
3400 const APInt &StepInt = Step->getAPInt();
3401 const APInt &DivInt = RHSC->getAPInt();
3402 if (!StepInt.urem(RHS: DivInt) &&
3403 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3404 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3405 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy), L: AR->getLoop(),
3406 Flags: SCEV::FlagNone)) {
3407 SmallVector<SCEVUse, 4> Operands;
3408 for (const SCEV *Op : AR->operands())
3409 Operands.push_back(Elt: getUDivExpr(LHS: Op, RHS));
3410 return getAddRecExpr(Operands, L: AR->getLoop(), Flags: SCEV::FlagNW);
3411 }
3412 /// Get a canonical UDivExpr for a recurrence.
3413 /// {X,+,N}/C => {Y,+,N}/C where Y=X-(X%N). Safe when C%N=0.
3414 const APInt *StartRem;
3415 if (!DivInt.urem(RHS: StepInt) && match(S: getURemExpr(LHS: AR->getStart(), RHS: Step),
3416 P: m_scev_APInt(C&: StartRem))) {
3417 bool NoWrap =
3418 getZeroExtendExpr(Op: AR, Ty: ExtTy) ==
3419 getAddRecExpr(Start: getZeroExtendExpr(Op: AR->getStart(), Ty: ExtTy),
3420 Step: getZeroExtendExpr(Op: Step, Ty: ExtTy), L: AR->getLoop(),
3421 Flags: SCEV::FlagNone);
3422
3423 // With N <= C and both N, C as powers-of-2, the transformation
3424 // {X,+,N}/C => {(X - X%N),+,N}/C preserves division results even
3425 // if wrapping occurs, as the division results remain equivalent for
3426 // all offsets in [[(X - X%N), X).
3427 bool CanFoldWithWrap = StepInt.ule(RHS: DivInt) && // N <= C
3428 StepInt.isPowerOf2() && DivInt.isPowerOf2();
3429 // Only fold if the subtraction can be folded in the start
3430 // expression.
3431 const SCEV *NewStart =
3432 getMinusSCEV(LHS: AR->getStart(), RHS: getConstant(Val: *StartRem));
3433 if (*StartRem != 0 && (NoWrap || CanFoldWithWrap) &&
3434 !isa<SCEVAddExpr>(Val: NewStart)) {
3435 const SCEV *NewLHS =
3436 getAddRecExpr(Start: NewStart, Step, L: AR->getLoop(),
3437 Flags: NoWrap ? SCEV::FlagNW : SCEV::FlagNone);
3438 if (LHS != NewLHS)
3439 return getUDivExpr(LHS: NewLHS, RHS);
3440 }
3441 }
3442 }
3443 // (A*B)/C --> A*(B/C) if safe and B/C can be folded.
3444 if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(Val&: LHS)) {
3445 if (M->hasNoUnsignedWrap()) {
3446 // Find an operand that's safely divisible.
3447 for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
3448 const SCEV *Op = M->getOperand(i);
3449 const SCEV *Div = getUDivExpr(LHS: Op, RHS: RHSC);
3450 if (!isa<SCEVUDivExpr>(Val: Div) && getMulExpr(LHS: Div, RHS: RHSC) == Op) {
3451 SmallVector<SCEVUse, 4> Operands(M->operands());
3452 Operands[i] = Div;
3453 return getMulExpr(Ops&: Operands);
3454 }
3455 }
3456
3457 // Even if it's not divisible, try to remove a common factor.
3458 if (const auto *LHSC = dyn_cast<SCEVConstant>(Val: M->getOperand(i: 0))) {
3459 APInt Factor = APIntOps::GreatestCommonDivisor(A: LHSC->getAPInt(),
3460 B: RHSC->getAPInt());
3461 if (!Factor.isIntN(N: 1)) {
3462 SmallVector<SCEVUse, 2> NewOperands;
3463 NewOperands.push_back(Elt: getConstant(Val: LHSC->getAPInt().udiv(RHS: Factor)));
3464 append_range(C&: NewOperands, R: M->operands().drop_front());
3465 const SCEV *NewMul = getMulExpr(Ops&: NewOperands);
3466 return getUDivExpr(LHS: NewMul,
3467 RHS: getConstant(Val: RHSC->getAPInt().udiv(RHS: Factor)));
3468 }
3469 }
3470 }
3471 }
3472
3473 // (A/B)/C --> A/(B*C) if safe and B*C can be folded.
3474 if (const SCEVUDivExpr *OtherDiv = dyn_cast<SCEVUDivExpr>(Val&: LHS)) {
3475 if (auto *DivisorConstant =
3476 dyn_cast<SCEVConstant>(Val: OtherDiv->getRHS())) {
3477 bool Overflow = false;
3478 APInt NewRHS =
3479 DivisorConstant->getAPInt().umul_ov(RHS: RHSC->getAPInt(), Overflow);
3480 if (Overflow) {
3481 return getConstant(Ty: RHSC->getType(), V: 0, isSigned: false);
3482 }
3483 return getUDivExpr(LHS: OtherDiv->getLHS(), RHS: getConstant(Val: NewRHS));
3484 }
3485 }
3486
3487 // (A+B)/C --> (A/C + B/C) if the add does not unsigned wrap and A/C and
3488 // B/C can be folded.
3489 if (const SCEVAddExpr *A = dyn_cast<SCEVAddExpr>(Val&: LHS)) {
3490 if (A->hasNoUnsignedWrap()) {
3491 SmallVector<SCEVUse, 4> Operands;
3492 for (unsigned i = 0, e = A->getNumOperands(); i != e; ++i) {
3493 const SCEV *Op = getUDivExpr(LHS: A->getOperand(i), RHS);
3494 if (isa<SCEVUDivExpr>(Val: Op) ||
3495 getMulExpr(LHS: Op, RHS) != A->getOperand(i))
3496 break;
3497 Operands.push_back(Elt: Op);
3498 }
3499 if (Operands.size() == A->getNumOperands())
3500 return getAddExpr(Ops&: Operands);
3501 }
3502 }
3503
3504 // ((N - M) + (M * A)) / N --> ((N - 1) + (M * A)) / N
3505 // This is an idiom for rounding A up to the next multiple of N, where A
3506 // is aready known to be a multiple of M. In this case, instcombine can
3507 // see that some low bits of the added constant are unused, so can clear
3508 // them, but we want to canonicalise to set the low bits. This makes the
3509 // pattern easier to match, without needing to check for known bits in
3510 // A*M.
3511 const APInt &N = RHSC->getAPInt();
3512 const APInt *NMinusM, *M;
3513 const SCEV *A;
3514 if (match(U: LHS, P: m_scev_Add(Op0: m_scev_APInt(C&: NMinusM),
3515 Op1: m_scev_Mul(Op0: m_scev_APInt(C&: M), Op1: m_SCEV(V&: A))))) {
3516 if (N.isPowerOf2() && M->isPowerOf2() && M->ult(RHS: N) &&
3517 *NMinusM == N - *M) {
3518 return getUDivExpr(
3519 LHS: getAddExpr(LHS: getConstant(Val: N - 1), RHS: getMulExpr(LHS: getConstant(Val: *M), RHS: A)),
3520 RHS);
3521 }
3522 }
3523
3524 // Fold if both operands are constant.
3525 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS))
3526 return getConstant(Val: LHSC->getAPInt().udiv(RHS: RHSC->getAPInt()));
3527 }
3528 }
3529
3530 // ((-C + (C smax %x)) /u %x) evaluates to zero, for any positive constant C.
3531 const APInt *NegC, *C;
3532 if (match(U: LHS,
3533 P: m_scev_Add(Op0: m_scev_APInt(C&: NegC),
3534 Op1: m_scev_SMax(Op0: m_scev_APInt(C), Op1: m_scev_Specific(S: RHS)))) &&
3535 NegC->isNegative() && !NegC->isMinSignedValue() && *C == -*NegC)
3536 return getZero(Ty: LHS->getType());
3537
3538 // (%a * %b)<nuw> / %b -> %a
3539 const auto *Mul = dyn_cast<SCEVMulExpr>(Val&: LHS);
3540 if (Mul && Mul->hasNoUnsignedWrap()) {
3541 for (int i = 0, e = Mul->getNumOperands(); i != e; ++i) {
3542 if (Mul->getOperand(i) == RHS) {
3543 SmallVector<SCEVUse, 2> Operands;
3544 append_range(C&: Operands, R: Mul->operands().take_front(N: i));
3545 append_range(C&: Operands, R: Mul->operands().drop_front(N: i + 1));
3546 return getMulExpr(Ops&: Operands);
3547 }
3548 }
3549 }
3550
3551 // TODO: Generalize to handle any common factors.
3552 // udiv (mul nuw a, vscale), (mul nuw b, vscale) --> udiv a, b
3553 const SCEV *NewLHS, *NewRHS;
3554 if (match(U: LHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
3555 match(U: RHS, P: m_scev_c_NUWMul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale())))
3556 return getUDivExpr(LHS: NewLHS, RHS: NewRHS);
3557
3558 return getOrCreateUDivExpr(LHS, RHS);
3559}
3560
3561/// Get a canonical unsigned division expression, or something simpler if
3562/// possible. There is no representation for an exact udiv in SCEV IR, but we
3563/// can attempt to optimize it prior to construction.
3564const SCEV *ScalarEvolution::getUDivExactExpr(SCEVUse LHS, SCEVUse RHS) {
3565 // Currently there is no exact specific logic.
3566
3567 return getUDivExpr(LHS, RHS);
3568}
3569
3570/// Get an add recurrence expression for the specified loop. Simplify the
3571/// expression as much as possible.
3572SCEVUse ScalarEvolution::getAddRecExpr(SCEVUse Start, SCEVUse Step,
3573 const Loop *L, SCEVFlagsPair Flags) {
3574 SmallVector<SCEVUse, 4> Operands;
3575 Operands.push_back(Elt: Start);
3576 if (const SCEVAddRecExpr *StepChrec = dyn_cast<SCEVAddRecExpr>(Val&: Step))
3577 if (StepChrec->getLoop() == L) {
3578 append_range(C&: Operands, R: StepChrec->operands());
3579 // The use flags describe the two-operand recurrence, not the flattened
3580 // one built here, so drop them just like the expression's NUW/NSW.
3581 return getAddRecExpr(Operands, L,
3582 Flags: maskFlags(Flags: Flags.ExprFlags, Mask: SCEV::FlagNW));
3583 }
3584
3585 Operands.push_back(Elt: Step);
3586 return getAddRecExpr(Operands, L, Flags);
3587}
3588
3589/// Get an add recurrence expression for the specified loop. Simplify the
3590/// expression as much as possible.
3591SCEVUse ScalarEvolution::getAddRecExpr(SmallVectorImpl<SCEVUse> &Operands,
3592 const Loop *L, SCEVFlagsPair NWFlags) {
3593 SCEVFlags ExprFlags = NWFlags.ExprFlags;
3594 SCEVFlags UseFlags = NWFlags.UseFlags;
3595 assert(!(UseFlags & ~(SCEV::FlagNUW | SCEV::FlagNSW)) &&
3596 "only nuw or nsw allowed");
3597 if (Operands.size() == 1) return Operands[0];
3598#ifndef NDEBUG
3599 Type *ETy = getEffectiveSCEVType(Operands[0]->getType());
3600 for (const SCEV *Op : llvm::drop_begin(Operands)) {
3601 assert(getEffectiveSCEVType(Op->getType()) == ETy &&
3602 "SCEVAddRecExpr operand types don't match!");
3603 assert(!Op->getType()->isPointerTy() && "Step must be integer");
3604 }
3605 for (const SCEV *Op : Operands)
3606 assert(isAvailableAtLoopEntry(Op, L) &&
3607 "SCEVAddRecExpr operand is not available at loop entry!");
3608#endif
3609
3610 if (Operands.back()->isZero()) {
3611 Operands.pop_back();
3612 return getAddRecExpr(Operands, L, NWFlags: SCEV::FlagNone); // {X,+,0} --> X
3613 }
3614
3615 // It's tempting to want to call getConstantMaxBackedgeTakenCount count here and
3616 // use that information to infer NUW and NSW flags. However, computing a
3617 // BE count requires calling getAddRecExpr, so we may not yet have a
3618 // meaningful BE count at this point (and if we don't, we'd be stuck
3619 // with a SCEVCouldNotCompute as the cached BE count).
3620
3621 ExprFlags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags: ExprFlags);
3622
3623 return {getOrCreateAddRecExpr(Ops: Operands, L, Flags: ExprFlags), UseFlags};
3624}
3625
3626const SCEV *ScalarEvolution::getGEPExpr(GEPOperator *GEP,
3627 ArrayRef<SCEVUse> IndexExprs) {
3628 const SCEV *BaseExpr = getSCEV(V: GEP->getPointerOperand());
3629 // getSCEV(Base)->getType() has the same address space as Base->getType()
3630 // because SCEV::getType() preserves the address space.
3631 GEPNoWrapFlags BaseNW = GEP->getNoWrapFlags();
3632 GEPNoWrapFlags OffsetNW = GEP->getNoWrapFlags();
3633 if (BaseNW != GEPNoWrapFlags::none()) {
3634 // We'd like to propagate flags from the IR to the corresponding SCEV nodes,
3635 // but to do that, we have to ensure that said flag is valid in the entire
3636 // defined scope of the SCEV.
3637 // TODO: non-instructions have global scope. We might be able to prove
3638 // some global scope cases
3639 auto *GEPI = dyn_cast<Instruction>(Val: GEP);
3640 if (!GEPI || !isSCEVExprNeverPoison(I: GEPI)) {
3641 BaseNW = OffsetNW = GEPNoWrapFlags::none();
3642 } else if (any_of(Range&: IndexExprs, P: [&](const SCEVUse &IndexExpr) {
3643 const SCEV *Idx = IndexExpr;
3644 return !isa<SCEVConstant>(Val: Idx) &&
3645 !isGuaranteedToTransferExecutionTo(
3646 A: getDefiningScopeBound(Ops: {Idx}), B: GEPI);
3647 })) {
3648 // The offset flags should not take into account the base pointer
3649 // defining scope, as the offset only depends on the indices. Thus,
3650 // derive the scope bound for the offset flags from each index operand
3651 // individually.
3652 OffsetNW = GEPNoWrapFlags::none();
3653 }
3654 }
3655
3656 return getGEPExpr(BaseExpr, IndexExprs, SrcElementTy: GEP->getSourceElementType(), BaseNW,
3657 OffsetNW);
3658}
3659
3660const SCEV *ScalarEvolution::getGEPExpr(SCEVUse BaseExpr,
3661 ArrayRef<SCEVUse> IndexExprs,
3662 Type *SrcElementTy,
3663 GEPNoWrapFlags BaseNW,
3664 GEPNoWrapFlags OffsetNW) {
3665 SCEVFlags OffsetWrap = SCEV::FlagNone;
3666 if (OffsetNW.hasNoUnsignedSignedWrap())
3667 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNSW);
3668 if (OffsetNW.hasNoUnsignedWrap())
3669 OffsetWrap = setFlags(Flags: OffsetWrap, OnFlags: SCEV::FlagNUW);
3670
3671 Type *CurTy = BaseExpr->getType();
3672 Type *IntIdxTy = getEffectiveSCEVType(Ty: BaseExpr->getType());
3673 bool FirstIter = true;
3674 SmallVector<SCEVUse, 4> Offsets;
3675 for (SCEVUse IndexExpr : IndexExprs) {
3676 // Compute the (potentially symbolic) offset in bytes for this index.
3677 if (StructType *STy = dyn_cast<StructType>(Val: CurTy)) {
3678 // For a struct, add the member offset.
3679 ConstantInt *Index = cast<SCEVConstant>(Val&: IndexExpr)->getValue();
3680 unsigned FieldNo = Index->getZExtValue();
3681 const SCEV *FieldOffset = getOffsetOfExpr(IntTy: IntIdxTy, STy, FieldNo);
3682 Offsets.push_back(Elt: FieldOffset);
3683
3684 // Update CurTy to the type of the field at Index.
3685 CurTy = STy->getTypeAtIndex(V: Index);
3686 } else {
3687 // Update CurTy to its element type.
3688 if (FirstIter) {
3689 assert(isa<PointerType>(CurTy) &&
3690 "The first index of a GEP indexes a pointer");
3691 CurTy = SrcElementTy;
3692 FirstIter = false;
3693 } else {
3694 CurTy = GetElementPtrInst::getTypeAtIndex(Ty: CurTy, Idx: (uint64_t)0);
3695 }
3696 // For an array, add the element offset, explicitly scaled.
3697 const SCEV *ElementSize = getSizeOfExpr(IntTy: IntIdxTy, AllocTy: CurTy);
3698 // Getelementptr indices are signed.
3699 IndexExpr = getTruncateOrSignExtend(V: IndexExpr, Ty: IntIdxTy);
3700
3701 // Multiply the index by the element size to compute the element offset.
3702 const SCEV *LocalOffset = getMulExpr(LHS: IndexExpr, RHS: ElementSize, Flags: OffsetWrap);
3703 Offsets.push_back(Elt: LocalOffset);
3704 }
3705 }
3706
3707 // Handle degenerate case of GEP without offsets.
3708 if (Offsets.empty())
3709 return BaseExpr;
3710
3711 // Add the offsets together, assuming nsw if inbounds.
3712 const SCEV *Offset = getAddExpr(Ops&: Offsets, Flags: OffsetWrap);
3713 // Add the base address and the offset. We cannot use the nsw flag, as the
3714 // base address is unsigned. However, if we know that the offset is
3715 // non-negative, we can use nuw.
3716 bool NUW = BaseNW.hasNoUnsignedWrap() ||
3717 (BaseNW.hasNoUnsignedSignedWrap() && isKnownNonNegative(S: Offset));
3718 SCEVFlags BaseWrap = NUW ? SCEV::FlagNUW : SCEV::FlagNone;
3719 const SCEV *GEPExpr = getAddExpr(LHS: BaseExpr, RHS: Offset, Flags: BaseWrap);
3720 assert(BaseExpr->getType() == GEPExpr->getType() &&
3721 "GEP should not change type mid-flight.");
3722 return GEPExpr;
3723}
3724
3725SCEV *ScalarEvolution::findExistingSCEVInCache(SCEVTypes SCEVType,
3726 ArrayRef<SCEVUse> Ops,
3727 const Loop *L) {
3728 assert((SCEVType != scAddRecExpr || L) &&
3729 "L must be passed to find existing AddRecs");
3730 FoldingSetNodeID ID;
3731 ID.AddInteger(I: SCEVType);
3732 for (SCEVUse Op : Ops)
3733 ID.AddPointer(Ptr: Op.getOpaqueValue());
3734 if (L)
3735 ID.AddPointer(Ptr: L);
3736 FoldingSetInsertToken Token;
3737 return UniqueSCEVs.lookup(ID, Token);
3738}
3739
3740const SCEV *ScalarEvolution::getAbsExpr(const SCEV *Op, bool IsNSW) {
3741 SCEVFlags Flags = IsNSW ? SCEV::FlagNSW : SCEV::FlagNone;
3742 return getSMaxExpr(LHS: Op, RHS: getNegativeSCEV(V: Op, Flags));
3743}
3744
3745const SCEV *ScalarEvolution::getMinMaxExpr(SCEVTypes Kind,
3746 SmallVectorImpl<SCEVUse> &Ops) {
3747 assert(SCEVMinMaxExpr::isMinMaxType(Kind) && "Not a SCEVMinMaxExpr!");
3748 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
3749 if (Ops.size() == 1) return Ops[0];
3750#ifndef NDEBUG
3751 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
3752 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
3753 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
3754 "Operand types don't match!");
3755 assert(Ops[0]->getType()->isPointerTy() ==
3756 Ops[i]->getType()->isPointerTy() &&
3757 "min/max should be consistently pointerish");
3758 }
3759#endif
3760
3761 bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr;
3762 bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr;
3763
3764 const SCEV *Folded = constantFoldAndGroupOps(
3765 Opts, SE&: *this, LI, DT, Ops,
3766 Fold: [&](const APInt &C1, const APInt &C2) {
3767 switch (Kind) {
3768 case scSMaxExpr:
3769 return APIntOps::smax(A: C1, B: C2);
3770 case scSMinExpr:
3771 return APIntOps::smin(A: C1, B: C2);
3772 case scUMaxExpr:
3773 return APIntOps::umax(A: C1, B: C2);
3774 case scUMinExpr:
3775 return APIntOps::umin(A: C1, B: C2);
3776 default:
3777 llvm_unreachable("Unknown SCEV min/max opcode");
3778 }
3779 },
3780 IsIdentity: [&](const APInt &C) {
3781 // identity
3782 if (IsMax)
3783 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3784 else
3785 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3786 },
3787 IsAbsorber: [&](const APInt &C) {
3788 // absorber
3789 if (IsMax)
3790 return IsSigned ? C.isMaxSignedValue() : C.isMaxValue();
3791 else
3792 return IsSigned ? C.isMinSignedValue() : C.isMinValue();
3793 });
3794 if (Folded)
3795 return Folded;
3796
3797 // Check if we have created the same expression before.
3798 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops)) {
3799 return S;
3800 }
3801
3802 // Find the first operation of the same kind
3803 unsigned Idx = 0;
3804 while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind)
3805 ++Idx;
3806
3807 // Check to see if one of the operands is of the same kind. If so, expand its
3808 // operands onto our operand list, and recurse to simplify.
3809 if (Idx < Ops.size()) {
3810 bool DeletedAny = false;
3811 while (Ops[Idx]->getSCEVType() == Kind) {
3812 const SCEVMinMaxExpr *SMME = cast<SCEVMinMaxExpr>(Val&: Ops[Idx]);
3813 Ops.erase(CI: Ops.begin()+Idx);
3814 append_range(C&: Ops, R: SMME->operands());
3815 DeletedAny = true;
3816 }
3817
3818 if (DeletedAny)
3819 return getMinMaxExpr(Kind, Ops);
3820 }
3821
3822 // Okay, check to see if the same value occurs in the operand list twice. If
3823 // so, delete one. Since we sorted the list, these values are required to
3824 // be adjacent.
3825 llvm::CmpInst::Predicate GEPred =
3826 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
3827 llvm::CmpInst::Predicate LEPred =
3828 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
3829 llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred;
3830 llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred;
3831 for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) {
3832 if (SCEVUse Common = SCEVUse::getCommon(A: Ops[i], B: Ops[i + 1]))
3833 Ops[i] = Ops[i + 1] = Common;
3834 if (Ops[i] == Ops[i + 1] ||
3835 isKnownViaNonRecursiveReasoning(Pred: FirstPred, LHS: Ops[i], RHS: Ops[i + 1])) {
3836 // X op Y op Y --> X op Y
3837 // X op Y --> X, if we know X, Y are ordered appropriately
3838 Ops.erase(CS: Ops.begin() + i + 1, CE: Ops.begin() + i + 2);
3839 --i;
3840 --e;
3841 } else if (isKnownViaNonRecursiveReasoning(Pred: SecondPred, LHS: Ops[i],
3842 RHS: Ops[i + 1])) {
3843 // X op Y --> Y, if we know X, Y are ordered appropriately
3844 Ops.erase(CS: Ops.begin() + i, CE: Ops.begin() + i + 1);
3845 --i;
3846 --e;
3847 }
3848 }
3849
3850 if (Ops.size() == 1) return Ops[0];
3851
3852 assert(!Ops.empty() && "Reduced smax down to nothing!");
3853
3854 // Okay, it looks like we really DO need an expr. Check to see if we
3855 // already have one, otherwise create a new one.
3856 FoldingSetNodeID ID;
3857 ID.AddInteger(I: Kind);
3858 for (SCEVUse Op : Ops)
3859 ID.AddPointer(Ptr: Op.getOpaqueValue());
3860 FoldingSetInsertToken Token;
3861 const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
3862 if (ExistingSCEV)
3863 return ExistingSCEV;
3864 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
3865 llvm::uninitialized_copy(Src&: Ops, Dst: O);
3866 SCEV *S = new (SCEVAllocator)
3867 SCEVMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
3868
3869 UniqueSCEVs.insert(N: S, Token);
3870 S->computeAndSetCanonical(SE&: *this);
3871 registerUser(User: S, Ops);
3872 return S;
3873}
3874
3875namespace {
3876
3877class SCEVSequentialMinMaxDeduplicatingVisitor final
3878 : public SCEVVisitor<SCEVSequentialMinMaxDeduplicatingVisitor,
3879 std::optional<const SCEV *>> {
3880 using RetVal = std::optional<const SCEV *>;
3881
3882 ScalarEvolution &SE;
3883 const SCEVTypes RootKind; // Must be a sequential min/max expression.
3884 const SCEVTypes NonSequentialRootKind; // Non-sequential variant of RootKind.
3885 SmallPtrSet<const SCEV *, 16> SeenOps;
3886
3887 bool canRecurseInto(SCEVTypes Kind) const {
3888 // We can only recurse into the SCEV expression of the same effective type
3889 // as the type of our root SCEV expression.
3890 return RootKind == Kind || NonSequentialRootKind == Kind;
3891 };
3892
3893 RetVal visit(const SCEV *S) {
3894 // Has the whole operand been seen already?
3895 if (!SeenOps.insert(Ptr: S).second)
3896 return std::nullopt;
3897 if (isa<SCEVMinMaxExpr, SCEVSequentialMinMaxExpr>(Val: S)) {
3898 SCEVTypes Kind = S->getSCEVType();
3899
3900 if (!canRecurseInto(Kind))
3901 return S;
3902
3903 auto *NAry = cast<SCEVNAryExpr>(Val: S);
3904 SmallVector<SCEVUse> NewOps;
3905 bool Changed = visit(Kind, OrigOps: NAry->operands(), NewOps);
3906
3907 if (!Changed)
3908 return S;
3909 if (NewOps.empty())
3910 return std::nullopt;
3911
3912 return isa<SCEVSequentialMinMaxExpr>(Val: S)
3913 ? SE.getSequentialMinMaxExpr(Kind, Operands&: NewOps)
3914 : SE.getMinMaxExpr(Kind, Ops&: NewOps);
3915 }
3916 return S;
3917 }
3918
3919public:
3920 SCEVSequentialMinMaxDeduplicatingVisitor(ScalarEvolution &SE,
3921 SCEVTypes RootKind)
3922 : SE(SE), RootKind(RootKind),
3923 NonSequentialRootKind(
3924 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
3925 Ty: RootKind)) {}
3926
3927 bool /*Changed*/ visit(SCEVTypes Kind, ArrayRef<SCEVUse> OrigOps,
3928 SmallVectorImpl<SCEVUse> &NewOps) {
3929 bool Changed = false;
3930 SmallVector<SCEVUse> Ops;
3931 Ops.reserve(N: OrigOps.size());
3932
3933 for (const SCEV *Op : OrigOps) {
3934 RetVal NewOp = visit(S: Op);
3935 if (NewOp != Op)
3936 Changed = true;
3937 if (NewOp)
3938 Ops.emplace_back(Args&: *NewOp);
3939 }
3940
3941 if (Changed)
3942 NewOps = std::move(Ops);
3943 return Changed;
3944 }
3945};
3946
3947} // namespace
3948
3949static bool scevUnconditionallyPropagatesPoisonFromOperands(SCEVTypes Kind) {
3950 switch (Kind) {
3951 case scConstant:
3952 case scVScale:
3953 case scTruncate:
3954 case scZeroExtend:
3955 case scSignExtend:
3956 case scPtrToAddr:
3957 case scAddExpr:
3958 case scMulExpr:
3959 case scUDivExpr:
3960 case scAddRecExpr:
3961 case scUMaxExpr:
3962 case scSMaxExpr:
3963 case scUMinExpr:
3964 case scSMinExpr:
3965 case scUnknown:
3966 // If any operand is poison, the whole expression is poison.
3967 return true;
3968 case scSequentialUMinExpr:
3969 // FIXME: if the *first* operand is poison, the whole expression is poison.
3970 return false; // Pessimistically, say that it does not propagate poison.
3971 case scCouldNotCompute:
3972 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
3973 }
3974 llvm_unreachable("Unknown SCEV kind!");
3975}
3976
3977namespace {
3978// The only way poison may be introduced in a SCEV expression is from a
3979// poison SCEVUnknown (ConstantExprs are also represented as SCEVUnknown,
3980// not SCEVConstant). Notably, SCEVFlags on SCEV nodes can *not*
3981// introduce poison -- they encode guaranteed, non-speculated knowledge.
3982//
3983// Additionally, all SCEV nodes propagate poison from inputs to outputs,
3984// with the notable exception of umin_seq, where only poison from the first
3985// operand is (unconditionally) propagated.
3986struct SCEVPoisonCollector {
3987 bool LookThroughMaybePoisonBlocking;
3988 SmallPtrSet<const SCEVUnknown *, 4> MaybePoison;
3989 SCEVPoisonCollector(bool LookThroughMaybePoisonBlocking)
3990 : LookThroughMaybePoisonBlocking(LookThroughMaybePoisonBlocking) {}
3991
3992 bool follow(const SCEV *S) {
3993 if (!LookThroughMaybePoisonBlocking &&
3994 !scevUnconditionallyPropagatesPoisonFromOperands(Kind: S->getSCEVType()))
3995 return false;
3996
3997 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
3998 if (!isGuaranteedNotToBePoison(V: SU->getValue()))
3999 MaybePoison.insert(Ptr: SU);
4000 }
4001 return true;
4002 }
4003 bool isDone() const { return false; }
4004};
4005} // namespace
4006
4007/// Return true if V is poison given that AssumedPoison is already poison.
4008static bool impliesPoison(const SCEV *AssumedPoison, const SCEV *S) {
4009 // First collect all SCEVs that might result in AssumedPoison to be poison.
4010 // We need to look through potentially poison-blocking operations here,
4011 // because we want to find all SCEVs that *might* result in poison, not only
4012 // those that are *required* to.
4013 SCEVPoisonCollector PC1(/* LookThroughMaybePoisonBlocking */ true);
4014 visitAll(Root: AssumedPoison, Visitor&: PC1);
4015
4016 // AssumedPoison is never poison. As the assumption is false, the implication
4017 // is true. Don't bother walking the other SCEV in this case.
4018 if (PC1.MaybePoison.empty())
4019 return true;
4020
4021 // Collect all SCEVs in S that, if poison, *will* result in S being poison
4022 // as well. We cannot look through potentially poison-blocking operations
4023 // here, as their arguments only *may* make the result poison.
4024 SCEVPoisonCollector PC2(/* LookThroughMaybePoisonBlocking */ false);
4025 visitAll(Root: S, Visitor&: PC2);
4026
4027 // Make sure that no matter which SCEV in PC1.MaybePoison is actually poison,
4028 // it will also make S poison by being part of PC2.MaybePoison.
4029 return llvm::set_is_subset(S1: PC1.MaybePoison, S2: PC2.MaybePoison);
4030}
4031
4032void ScalarEvolution::getPoisonGeneratingValues(
4033 SmallPtrSetImpl<const Value *> &Result, const SCEV *S) {
4034 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ false);
4035 visitAll(Root: S, Visitor&: PC);
4036 for (const SCEVUnknown *SU : PC.MaybePoison)
4037 Result.insert(Ptr: SU->getValue());
4038}
4039
4040bool ScalarEvolution::canReuseInstruction(
4041 const SCEV *S, Instruction *I,
4042 SmallVectorImpl<Instruction *> &DropPoisonGeneratingInsts) {
4043 // If the instruction cannot be poison, it's always safe to reuse.
4044 if (programUndefinedIfPoison(Inst: I))
4045 return true;
4046
4047 // Otherwise, it is possible that I is more poisonous that S. Collect the
4048 // poison-contributors of S, and then check whether I has any additional
4049 // poison-contributors. Poison that is contributed through poison-generating
4050 // flags is handled by dropping those flags instead.
4051 SmallPtrSet<const Value *, 8> PoisonVals;
4052 getPoisonGeneratingValues(Result&: PoisonVals, S);
4053
4054 SmallVector<Value *> Worklist;
4055 SmallPtrSet<Value *, 8> Visited;
4056 Worklist.push_back(Elt: I);
4057 unsigned NumVisitedInsts = 0;
4058 const unsigned InstLimit = std::max<unsigned>(a: 16, b: S->getExpressionSize());
4059 while (!Worklist.empty()) {
4060 Value *V = Worklist.pop_back_val();
4061 if (!Visited.insert(Ptr: V).second)
4062 continue;
4063
4064 // Either the value can't be poison, or the S would also be poison if it
4065 // is.
4066 if (PoisonVals.contains(Ptr: V) || ::isGuaranteedNotToBePoison(V))
4067 continue;
4068
4069 // Avoid walking large instruction graphs.
4070 if (++NumVisitedInsts > InstLimit)
4071 return false;
4072
4073 auto *I = dyn_cast<Instruction>(Val: V);
4074 if (!I)
4075 return false;
4076
4077 // Disjoint or instructions are interpreted as adds by SCEV. However, we
4078 // can't replace an arbitrary add with disjoint or, even if we drop the
4079 // flag. We would need to convert the or into an add.
4080 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: I))
4081 if (PDI->isDisjoint())
4082 return false;
4083
4084 // FIXME: Ignore vscale, even though it technically could be poison. Do this
4085 // because SCEV currently assumes it can't be poison. Remove this special
4086 // case once we proper model when vscale can be poison.
4087 if (auto *II = dyn_cast<IntrinsicInst>(Val: I);
4088 II && II->getIntrinsicID() == Intrinsic::vscale)
4089 continue;
4090
4091 if (canCreatePoison(Op: cast<Operator>(Val: I), /*ConsiderFlagsAndMetadata*/ false))
4092 return false;
4093
4094 // If the instruction can't create poison, we can recurse to its operands.
4095 if (I->hasPoisonGeneratingAnnotations())
4096 DropPoisonGeneratingInsts.push_back(Elt: I);
4097
4098 llvm::append_range(C&: Worklist, R: I->operands());
4099 }
4100 return true;
4101}
4102
4103const SCEV *
4104ScalarEvolution::getSequentialMinMaxExpr(SCEVTypes Kind,
4105 SmallVectorImpl<SCEVUse> &Ops) {
4106 assert(SCEVSequentialMinMaxExpr::isSequentialMinMaxType(Kind) &&
4107 "Not a SCEVSequentialMinMaxExpr!");
4108 assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!");
4109 if (Ops.size() == 1)
4110 return Ops[0];
4111#ifndef NDEBUG
4112 Type *ETy = getEffectiveSCEVType(Ops[0]->getType());
4113 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4114 assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy &&
4115 "Operand types don't match!");
4116 assert(Ops[0]->getType()->isPointerTy() ==
4117 Ops[i]->getType()->isPointerTy() &&
4118 "min/max should be consistently pointerish");
4119 }
4120#endif
4121
4122 // Note that SCEVSequentialMinMaxExpr is *NOT* commutative,
4123 // so we can *NOT* do any kind of sorting of the expressions!
4124
4125 // Check if we have created the same expression before.
4126 if (const SCEV *S = findExistingSCEVInCache(SCEVType: Kind, Ops))
4127 return S;
4128
4129 // FIXME: there are *some* simplifications that we can do here.
4130
4131 // Keep only the first instance of an operand.
4132 {
4133 SCEVSequentialMinMaxDeduplicatingVisitor Deduplicator(*this, Kind);
4134 bool Changed = Deduplicator.visit(Kind, OrigOps: Ops, NewOps&: Ops);
4135 if (Changed)
4136 return getSequentialMinMaxExpr(Kind, Ops);
4137 }
4138
4139 // Check to see if one of the operands is of the same kind. If so, expand its
4140 // operands onto our operand list, and recurse to simplify.
4141 {
4142 unsigned Idx = 0;
4143 bool DeletedAny = false;
4144 while (Idx < Ops.size()) {
4145 if (Ops[Idx]->getSCEVType() != Kind) {
4146 ++Idx;
4147 continue;
4148 }
4149 const auto *SMME = cast<SCEVSequentialMinMaxExpr>(Val&: Ops[Idx]);
4150 Ops.erase(CI: Ops.begin() + Idx);
4151 Ops.insert(I: Ops.begin() + Idx, From: SMME->operands().begin(),
4152 To: SMME->operands().end());
4153 DeletedAny = true;
4154 }
4155
4156 if (DeletedAny)
4157 return getSequentialMinMaxExpr(Kind, Ops);
4158 }
4159
4160 const SCEV *SaturationPoint;
4161 ICmpInst::Predicate Pred;
4162 switch (Kind) {
4163 case scSequentialUMinExpr:
4164 SaturationPoint = getZero(Ty: Ops[0]->getType());
4165 Pred = ICmpInst::ICMP_ULE;
4166 break;
4167 default:
4168 llvm_unreachable("Not a sequential min/max type.");
4169 }
4170
4171 for (unsigned i = 1, e = Ops.size(); i != e; ++i) {
4172 if (!isGuaranteedNotToCauseUB(Op: Ops[i]))
4173 continue;
4174 // We can replace %x umin_seq %y with %x umin %y if either:
4175 // * %y being poison implies %x is also poison.
4176 // * %x cannot be the saturating value (e.g. zero for umin).
4177 if (::impliesPoison(AssumedPoison: Ops[i], S: Ops[i - 1]) ||
4178 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_NE, LHS: Ops[i - 1],
4179 RHS: SaturationPoint)) {
4180 SmallVector<SCEVUse, 2> SeqOps = {Ops[i - 1], Ops[i]};
4181 Ops[i - 1] = getMinMaxExpr(
4182 Kind: SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(Ty: Kind),
4183 Ops&: SeqOps);
4184 Ops.erase(CI: Ops.begin() + i);
4185 return getSequentialMinMaxExpr(Kind, Ops);
4186 }
4187 // Fold %x umin_seq %y to %x if %x ule %y.
4188 // TODO: We might be able to prove the predicate for a later operand.
4189 if (isKnownViaNonRecursiveReasoning(Pred, LHS: Ops[i - 1], RHS: Ops[i])) {
4190 Ops.erase(CI: Ops.begin() + i);
4191 return getSequentialMinMaxExpr(Kind, Ops);
4192 }
4193 }
4194
4195 // Okay, it looks like we really DO need an expr. Check to see if we
4196 // already have one, otherwise create a new one.
4197 FoldingSetNodeID ID;
4198 ID.AddInteger(I: Kind);
4199 for (SCEVUse Op : Ops)
4200 ID.AddPointer(Ptr: Op.getOpaqueValue());
4201 FoldingSetInsertToken Token;
4202 const SCEV *ExistingSCEV = UniqueSCEVs.lookup(ID, Token);
4203 if (ExistingSCEV)
4204 return ExistingSCEV;
4205
4206 SCEVUse *O = SCEVAllocator.Allocate<SCEVUse>(Num: Ops.size());
4207 llvm::uninitialized_copy(Src&: Ops, Dst: O);
4208 SCEV *S = new (SCEVAllocator)
4209 SCEVSequentialMinMaxExpr(ID.Intern(Allocator&: SCEVAllocator), Kind, O, Ops.size());
4210
4211 UniqueSCEVs.insert(N: S, Token);
4212 S->computeAndSetCanonical(SE&: *this);
4213 registerUser(User: S, Ops);
4214 return S;
4215}
4216
4217const SCEV *ScalarEvolution::getSMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4218 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4219 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4220}
4221
4222const SCEV *ScalarEvolution::getSMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4223 return getMinMaxExpr(Kind: scSMaxExpr, Ops);
4224}
4225
4226const SCEV *ScalarEvolution::getUMaxExpr(SCEVUse LHS, SCEVUse RHS) {
4227 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4228 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4229}
4230
4231const SCEV *ScalarEvolution::getUMaxExpr(SmallVectorImpl<SCEVUse> &Ops) {
4232 return getMinMaxExpr(Kind: scUMaxExpr, Ops);
4233}
4234
4235const SCEV *ScalarEvolution::getSMinExpr(SCEVUse LHS, SCEVUse RHS) {
4236 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4237 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4238}
4239
4240const SCEV *ScalarEvolution::getSMinExpr(SmallVectorImpl<SCEVUse> &Ops) {
4241 return getMinMaxExpr(Kind: scSMinExpr, Ops);
4242}
4243
4244const SCEV *ScalarEvolution::getUMinExpr(SCEVUse LHS, SCEVUse RHS,
4245 bool Sequential) {
4246 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4247 return getUMinExpr(Operands&: Ops, Sequential);
4248}
4249
4250const SCEV *ScalarEvolution::getUMinExpr(SmallVectorImpl<SCEVUse> &Ops,
4251 bool Sequential) {
4252 return Sequential ? getSequentialMinMaxExpr(Kind: scSequentialUMinExpr, Ops)
4253 : getMinMaxExpr(Kind: scUMinExpr, Ops);
4254}
4255
4256const SCEV *
4257ScalarEvolution::getSizeOfExpr(Type *IntTy, TypeSize Size) {
4258 const SCEV *Res = getConstant(Ty: IntTy, V: Size.getKnownMinValue());
4259 if (Size.isScalable())
4260 Res = getMulExpr(LHS: Res, RHS: getVScale(Ty: IntTy));
4261 return Res;
4262}
4263
4264const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) {
4265 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeAllocSize(Ty: AllocTy));
4266}
4267
4268const SCEV *ScalarEvolution::getStoreSizeOfExpr(Type *IntTy, Type *StoreTy) {
4269 return getSizeOfExpr(IntTy, Size: getDataLayout().getTypeStoreSize(Ty: StoreTy));
4270}
4271
4272const SCEV *ScalarEvolution::getOffsetOfExpr(Type *IntTy,
4273 StructType *STy,
4274 unsigned FieldNo) {
4275 // We can bypass creating a target-independent constant expression and then
4276 // folding it back into a ConstantInt. This is just a compile-time
4277 // optimization.
4278 const StructLayout *SL = getDataLayout().getStructLayout(Ty: STy);
4279 assert(!SL->getSizeInBits().isScalable() &&
4280 "Cannot get offset for structure containing scalable vector types");
4281 return getConstant(Ty: IntTy, V: SL->getElementOffset(Idx: FieldNo));
4282}
4283
4284const SCEV *ScalarEvolution::getUnknown(Value *V) {
4285 // Don't attempt to do anything other than create a SCEVUnknown object
4286 // here. createSCEV only calls getUnknown after checking for all other
4287 // interesting possibilities, and any other code that calls getUnknown
4288 // is doing so in order to hide a value from SCEV canonicalization.
4289
4290 FoldingSetNodeID ID;
4291 ID.AddInteger(I: scUnknown);
4292 ID.AddPointer(Ptr: V);
4293 FoldingSetInsertToken Token;
4294 if (SCEV *S = UniqueSCEVs.lookup(ID, Token)) {
4295 assert(cast<SCEVUnknown>(S)->getValue() == V &&
4296 "Stale SCEVUnknown in uniquing map!");
4297 return S;
4298 }
4299 SCEV *S = new (SCEVAllocator) SCEVUnknown(ID.Intern(Allocator&: SCEVAllocator), V, this,
4300 FirstUnknown, NextUnknownOrderID++);
4301 FirstUnknown = cast<SCEVUnknown>(Val: S);
4302 UniqueSCEVs.insert(N: S, Token);
4303 S->computeAndSetCanonical(SE&: *this);
4304 return S;
4305}
4306
4307//===----------------------------------------------------------------------===//
4308// Basic SCEV Analysis and PHI Idiom Recognition Code
4309//
4310
4311/// Test if values of the given type are analyzable within the SCEV
4312/// framework. This primarily includes integer types, and it can optionally
4313/// include pointer types if the ScalarEvolution class has access to
4314/// target-specific information.
4315bool ScalarEvolution::isSCEVable(Type *Ty) const {
4316 // Integers and pointers are always SCEVable.
4317 return Ty->isIntOrPtrTy();
4318}
4319
4320/// Return the size in bits of the specified type, for which isSCEVable must
4321/// return true.
4322uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
4323 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4324 if (Ty->isPointerTy())
4325 return getDataLayout().getIndexTypeSizeInBits(Ty);
4326 return getDataLayout().getTypeSizeInBits(Ty);
4327}
4328
4329/// Return a type with the same bitwidth as the given type and which represents
4330/// how SCEV will treat the given type, for which isSCEVable must return
4331/// true. For pointer types, this is the pointer index sized integer type.
4332Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
4333 assert(isSCEVable(Ty) && "Type is not SCEVable!");
4334
4335 if (Ty->isIntegerTy())
4336 return Ty;
4337
4338 // The only other support type is pointer.
4339 assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
4340 return getDataLayout().getIndexType(PtrTy: Ty);
4341}
4342
4343Type *ScalarEvolution::getWiderType(Type *T1, Type *T2) const {
4344 return getTypeSizeInBits(Ty: T1) >= getTypeSizeInBits(Ty: T2) ? T1 : T2;
4345}
4346
4347bool ScalarEvolution::instructionCouldExistWithOperands(const SCEV *A,
4348 const SCEV *B) {
4349 /// For a valid use point to exist, the defining scope of one operand
4350 /// must dominate the other.
4351 bool PreciseA, PreciseB;
4352 auto *ScopeA = getDefiningScopeBound(Ops: {A}, Precise&: PreciseA);
4353 auto *ScopeB = getDefiningScopeBound(Ops: {B}, Precise&: PreciseB);
4354 if (!PreciseA || !PreciseB)
4355 // Can't tell.
4356 return false;
4357 return (ScopeA == ScopeB) || DT.dominates(Def: ScopeA, User: ScopeB) ||
4358 DT.dominates(Def: ScopeB, User: ScopeA);
4359}
4360
4361const SCEV *ScalarEvolution::getCouldNotCompute() {
4362 return CouldNotCompute.get();
4363}
4364
4365bool ScalarEvolution::checkValidity(const SCEV *S) const {
4366 bool ContainsNulls = SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
4367 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
4368 return SU && SU->getValue() == nullptr;
4369 });
4370
4371 return !ContainsNulls;
4372}
4373
4374bool ScalarEvolution::containsAddRecurrence(const SCEV *S) {
4375 auto [I, Inserted] = HasRecMap.try_emplace(Key: S);
4376 if (Inserted)
4377 I->second = SCEVExprContains(
4378 Root: S, Pred: [](const SCEV *S) { return isa<SCEVAddRecExpr>(Val: S); });
4379 return I->second;
4380}
4381
4382/// Return the ValueOffsetPair set for \p S. \p S can be represented
4383/// by the value and offset from any ValueOffsetPair in the set.
4384ArrayRef<Value *> ScalarEvolution::getSCEVValues(const SCEV *S) {
4385 ExprValueMapType::iterator SI = ExprValueMap.find_as(Val: S);
4386 if (SI == ExprValueMap.end())
4387 return {};
4388 return SI->second;
4389}
4390
4391/// Erase Value from ValueExprMap and ExprValueMap. ValueExprMap.erase(V)
4392/// cannot be used separately. eraseValueFromMap should be used to remove
4393/// V from ValueExprMap and ExprValueMap at the same time.
4394void ScalarEvolution::eraseValueFromMap(Value *V) {
4395 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4396 if (I != ValueExprMap.end()) {
4397 auto EVIt = ExprValueMap.find(Val: I->second);
4398 assert(is_contained(EVIt->second, V) && "Value not in ExprValueMap?");
4399 llvm::erase(C&: EVIt->second, V);
4400 ValueExprMap.erase(I);
4401 }
4402}
4403
4404void ScalarEvolution::insertValueToMap(Value *V, const SCEV *S) {
4405 // A recursive query may have already computed the SCEV. It should be
4406 // equivalent, but may not necessarily be exactly the same, e.g. due to lazily
4407 // inferred nowrap flags.
4408 if (ValueExprMap.insert(KV: {SCEVCallbackVH(V, this), S}).second)
4409 ExprValueMap[S].push_back(NewVal: V);
4410}
4411
4412/// Return an existing SCEV if it exists, otherwise analyze the expression and
4413/// create a new one.
4414const SCEV *ScalarEvolution::getSCEV(Value *V) {
4415 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4416
4417 if (const SCEV *S = getExistingSCEV(V))
4418 return S;
4419 return createSCEVIter(V);
4420}
4421
4422const SCEV *ScalarEvolution::getExistingSCEV(Value *V) {
4423 assert(isSCEVable(V->getType()) && "Value is not SCEVable!");
4424
4425 ValueExprMapType::iterator I = ValueExprMap.find_as(Val: V);
4426 if (I != ValueExprMap.end()) {
4427 const SCEV *S = I->second;
4428 assert(checkValidity(S) &&
4429 "existing SCEV has not been properly invalidated");
4430 return S;
4431 }
4432 return nullptr;
4433}
4434
4435/// Return a SCEV corresponding to -V = -1*V
4436const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V, SCEVFlags Flags) {
4437 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4438 return getConstant(
4439 V: cast<ConstantInt>(Val: ConstantExpr::getNeg(C: VC->getValue())));
4440
4441 Type *Ty = V->getType();
4442 Ty = getEffectiveSCEVType(Ty);
4443 return getMulExpr(LHS: V, RHS: getMinusOne(Ty), Flags);
4444}
4445
4446/// If Expr computes ~A, return A else return nullptr
4447static const SCEV *MatchNotExpr(const SCEV *Expr) {
4448 const SCEV *MulOp;
4449 if (match(S: Expr, P: m_scev_Add(Op0: m_scev_AllOnes(),
4450 Op1: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: MulOp)))))
4451 return MulOp;
4452 return nullptr;
4453}
4454
4455/// Return a SCEV corresponding to ~V = -1-V
4456const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) {
4457 assert(!V->getType()->isPointerTy() && "Can't negate pointer");
4458
4459 if (const SCEVConstant *VC = dyn_cast<SCEVConstant>(Val: V))
4460 return getConstant(
4461 V: cast<ConstantInt>(Val: ConstantExpr::getNot(C: VC->getValue())));
4462
4463 // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y)
4464 if (const SCEVMinMaxExpr *MME = dyn_cast<SCEVMinMaxExpr>(Val: V)) {
4465 auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) {
4466 SmallVector<SCEVUse, 2> MatchedOperands;
4467 for (const SCEV *Operand : MME->operands()) {
4468 const SCEV *Matched = MatchNotExpr(Expr: Operand);
4469 if (!Matched)
4470 return (const SCEV *)nullptr;
4471 MatchedOperands.push_back(Elt: Matched);
4472 }
4473 return getMinMaxExpr(Kind: SCEVMinMaxExpr::negate(T: MME->getSCEVType()),
4474 Ops&: MatchedOperands);
4475 };
4476 if (const SCEV *Replaced = MatchMinMaxNegation(MME))
4477 return Replaced;
4478 }
4479
4480 Type *Ty = V->getType();
4481 Ty = getEffectiveSCEVType(Ty);
4482 return getMinusSCEV(LHS: getMinusOne(Ty), RHS: V);
4483}
4484
4485const SCEV *ScalarEvolution::removePointerBase(const SCEV *P) {
4486 assert(P->getType()->isPointerTy());
4487
4488 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: P)) {
4489 // The base of an AddRec is the first operand.
4490 SmallVector<SCEVUse> Ops{AddRec->operands()};
4491 Ops[0] = removePointerBase(P: Ops[0]);
4492 // Don't try to transfer nowrap flags for now. We could in some cases
4493 // (for example, if pointer operand of the AddRec is a SCEVUnknown).
4494 return getAddRecExpr(Operands&: Ops, L: AddRec->getLoop(), NWFlags: SCEV::FlagNone);
4495 }
4496 if (auto *Add = dyn_cast<SCEVAddExpr>(Val: P)) {
4497 // The base of an Add is the pointer operand.
4498 SmallVector<SCEVUse> Ops{Add->operands()};
4499 SCEVUse *PtrOp = nullptr;
4500 for (SCEVUse &AddOp : Ops) {
4501 if (AddOp->getType()->isPointerTy()) {
4502 assert(!PtrOp && "Cannot have multiple pointer ops");
4503 PtrOp = &AddOp;
4504 }
4505 }
4506 *PtrOp = removePointerBase(P: *PtrOp);
4507 // Don't try to transfer nowrap flags for now. We could in some cases
4508 // (for example, if the pointer operand of the Add is a SCEVUnknown).
4509 return getAddExpr(Ops);
4510 }
4511 // Any other expression must be a pointer base.
4512 return getZero(Ty: P->getType());
4513}
4514
4515const SCEV *ScalarEvolution::getMinusSCEV(SCEVUse LHS, SCEVUse RHS,
4516 SCEVFlags Flags, unsigned Depth) {
4517 // Fast path: X - X --> 0.
4518 if (SCEVUse::getCommon(A: LHS, B: RHS))
4519 return getZero(Ty: LHS->getType());
4520
4521 // If we subtract two pointers with different pointer bases, bail.
4522 // Eventually, we're going to add an assertion to getMulExpr that we
4523 // can't multiply by a pointer.
4524 if (RHS->getType()->isPointerTy()) {
4525 if (!LHS->getType()->isPointerTy() ||
4526 getPointerBase(V: LHS) != getPointerBase(V: RHS))
4527 return getCouldNotCompute();
4528 LHS = removePointerBase(P: LHS);
4529 RHS = removePointerBase(P: RHS);
4530 }
4531
4532 // We represent LHS - RHS as LHS + (-1)*RHS. This transformation
4533 // makes it so that we cannot make much use of NUW.
4534 auto AddFlags = SCEV::FlagNone;
4535 const bool RHSIsNotMinSigned =
4536 !getSignedRangeMin(S: RHS).isMinSignedValue();
4537 if (hasFlags(Flags, TestFlags: SCEV::FlagNSW)) {
4538 // Let M be the minimum representable signed value. Then (-1)*RHS
4539 // signed-wraps if and only if RHS is M. That can happen even for
4540 // a NSW subtraction because e.g. (-1)*M signed-wraps even though
4541 // -1 - M does not. So to transfer NSW from LHS - RHS to LHS +
4542 // (-1)*RHS, we need to prove that RHS != M.
4543 //
4544 // If LHS is non-negative and we know that LHS - RHS does not
4545 // signed-wrap, then RHS cannot be M. So we can rule out signed-wrap
4546 // either by proving that RHS > M or that LHS >= 0.
4547 if (RHSIsNotMinSigned || isKnownNonNegative(S: LHS)) {
4548 AddFlags = SCEV::FlagNSW;
4549 }
4550 }
4551
4552 // FIXME: Find a correct way to transfer NSW to (-1)*M when LHS -
4553 // RHS is NSW and LHS >= 0.
4554 //
4555 // The difficulty here is that the NSW flag may have been proven
4556 // relative to a loop that is to be found in a recurrence in LHS and
4557 // not in RHS. Applying NSW to (-1)*M may then let the NSW have a
4558 // larger scope than intended.
4559 auto NegFlags = RHSIsNotMinSigned ? SCEV::FlagNSW : SCEV::FlagNone;
4560
4561 return getAddExpr(LHS, RHS: getNegativeSCEV(V: RHS, Flags: NegFlags), Flags: AddFlags, Depth);
4562}
4563
4564const SCEV *ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty,
4565 unsigned Depth) {
4566 Type *SrcTy = V->getType();
4567 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4568 "Cannot truncate or zero extend with non-integer arguments!");
4569 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4570 return V; // No conversion
4571 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4572 return getTruncateExpr(Op: V, Ty, Depth);
4573 return getZeroExtendExpr(Op: V, Ty, Depth);
4574}
4575
4576const SCEV *ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, Type *Ty,
4577 unsigned Depth) {
4578 Type *SrcTy = V->getType();
4579 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4580 "Cannot truncate or zero extend with non-integer arguments!");
4581 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4582 return V; // No conversion
4583 if (getTypeSizeInBits(Ty: SrcTy) > getTypeSizeInBits(Ty))
4584 return getTruncateExpr(Op: V, Ty, Depth);
4585 return getSignExtendExpr(Op: V, Ty, Depth);
4586}
4587
4588const SCEV *ScalarEvolution::getNoopOrZeroExtend(const SCEV *V, Type *Ty) {
4589 Type *SrcTy = V->getType();
4590 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4591 "Cannot noop or zero extend with non-integer arguments!");
4592 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4593 "getNoopOrZeroExtend cannot truncate!");
4594 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4595 return V; // No conversion
4596 return getZeroExtendExpr(Op: V, Ty);
4597}
4598
4599const SCEV *ScalarEvolution::getNoopOrSignExtend(const SCEV *V, Type *Ty) {
4600 Type *SrcTy = V->getType();
4601 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4602 "Cannot noop or sign extend with non-integer arguments!");
4603 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4604 "getNoopOrSignExtend cannot truncate!");
4605 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4606 return V; // No conversion
4607 return getSignExtendExpr(Op: V, Ty);
4608}
4609
4610const SCEV *ScalarEvolution::getNoopOrAnyExtend(const SCEV *V, Type *Ty) {
4611 Type *SrcTy = V->getType();
4612 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4613 "Cannot noop or any extend with non-integer arguments!");
4614 assert(getTypeSizeInBits(SrcTy) <= getTypeSizeInBits(Ty) &&
4615 "getNoopOrAnyExtend cannot truncate!");
4616 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4617 return V; // No conversion
4618 return getAnyExtendExpr(Op: V, Ty);
4619}
4620
4621const SCEV *ScalarEvolution::getTruncateOrNoop(const SCEV *V, Type *Ty) {
4622 Type *SrcTy = V->getType();
4623 assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() &&
4624 "Cannot truncate or noop with non-integer arguments!");
4625 assert(getTypeSizeInBits(SrcTy) >= getTypeSizeInBits(Ty) &&
4626 "getTruncateOrNoop cannot extend!");
4627 if (getTypeSizeInBits(Ty: SrcTy) == getTypeSizeInBits(Ty))
4628 return V; // No conversion
4629 return getTruncateExpr(Op: V, Ty);
4630}
4631
4632const SCEV *ScalarEvolution::getUMaxFromMismatchedTypes(const SCEV *LHS,
4633 const SCEV *RHS) {
4634 const SCEV *PromotedLHS = LHS;
4635 const SCEV *PromotedRHS = RHS;
4636
4637 if (getTypeSizeInBits(Ty: LHS->getType()) > getTypeSizeInBits(Ty: RHS->getType()))
4638 PromotedRHS = getZeroExtendExpr(Op: RHS, Ty: LHS->getType());
4639 else
4640 PromotedLHS = getNoopOrZeroExtend(V: LHS, Ty: RHS->getType());
4641
4642 return getUMaxExpr(LHS: PromotedLHS, RHS: PromotedRHS);
4643}
4644
4645const SCEV *ScalarEvolution::getUMinFromMismatchedTypes(const SCEV *LHS,
4646 const SCEV *RHS,
4647 bool Sequential) {
4648 SmallVector<SCEVUse, 2> Ops = {LHS, RHS};
4649 return getUMinFromMismatchedTypes(Ops, Sequential);
4650}
4651
4652const SCEV *
4653ScalarEvolution::getUMinFromMismatchedTypes(SmallVectorImpl<SCEVUse> &Ops,
4654 bool Sequential) {
4655 assert(!Ops.empty() && "At least one operand must be!");
4656 // Trivial case.
4657 if (Ops.size() == 1)
4658 return Ops[0];
4659
4660 // Find the max type first.
4661 Type *MaxType = nullptr;
4662 for (SCEVUse S : Ops)
4663 if (MaxType)
4664 MaxType = getWiderType(T1: MaxType, T2: S->getType());
4665 else
4666 MaxType = S->getType();
4667 assert(MaxType && "Failed to find maximum type!");
4668
4669 // Extend all ops to max type.
4670 SmallVector<SCEVUse, 2> PromotedOps;
4671 for (SCEVUse S : Ops)
4672 PromotedOps.push_back(Elt: getNoopOrZeroExtend(V: S, Ty: MaxType));
4673
4674 // Generate umin.
4675 return getUMinExpr(Ops&: PromotedOps, Sequential);
4676}
4677
4678const SCEV *ScalarEvolution::getPointerBase(const SCEV *V) {
4679 // A pointer operand may evaluate to a nonpointer expression, such as null.
4680 if (!V->getType()->isPointerTy())
4681 return V;
4682
4683 while (true) {
4684 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: V)) {
4685 V = AddRec->getStart();
4686 } else if (auto *Add = dyn_cast<SCEVAddExpr>(Val: V)) {
4687 const SCEV *PtrOp = nullptr;
4688 for (const SCEV *AddOp : Add->operands()) {
4689 if (AddOp->getType()->isPointerTy()) {
4690 assert(!PtrOp && "Cannot have multiple pointer ops");
4691 PtrOp = AddOp;
4692 }
4693 }
4694 assert(PtrOp && "Must have pointer op");
4695 V = PtrOp;
4696 } else // Not something we can look further into.
4697 return V;
4698 }
4699}
4700
4701/// Push users of the given Instruction onto the given Worklist.
4702static void PushDefUseChildren(Instruction *I,
4703 SmallVectorImpl<Instruction *> &Worklist,
4704 SmallPtrSetImpl<Instruction *> &Visited) {
4705 // Push the def-use children onto the Worklist stack.
4706 for (User *U : I->users()) {
4707 auto *UserInsn = cast<Instruction>(Val: U);
4708 if (Visited.insert(Ptr: UserInsn).second)
4709 Worklist.push_back(Elt: UserInsn);
4710 }
4711}
4712
4713namespace {
4714
4715/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its start
4716/// expression in case its Loop is L. If it is not L then
4717/// if IgnoreOtherLoops is true then use AddRec itself
4718/// otherwise rewrite cannot be done.
4719/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4720class SCEVInitRewriter : public SCEVRewriteVisitor<SCEVInitRewriter> {
4721public:
4722 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
4723 bool IgnoreOtherLoops = true) {
4724 SCEVInitRewriter Rewriter(L, SE);
4725 const SCEV *Result = Rewriter.visit(S);
4726 if (Rewriter.hasSeenLoopVariantSCEVUnknown())
4727 return SE.getCouldNotCompute();
4728 return Rewriter.hasSeenOtherLoops() && !IgnoreOtherLoops
4729 ? SE.getCouldNotCompute()
4730 : Result;
4731 }
4732
4733 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4734 if (!SE.isLoopInvariant(S: Expr, L))
4735 SeenLoopVariantSCEVUnknown = true;
4736 return Expr;
4737 }
4738
4739 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4740 // Only re-write AddRecExprs for this loop.
4741 if (Expr->getLoop() == L)
4742 return Expr->getStart();
4743 SeenOtherLoops = true;
4744 return Expr;
4745 }
4746
4747 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4748
4749 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4750
4751private:
4752 explicit SCEVInitRewriter(const Loop *L, ScalarEvolution &SE)
4753 : SCEVRewriteVisitor(SE), L(L) {}
4754
4755 const Loop *L;
4756 bool SeenLoopVariantSCEVUnknown = false;
4757 bool SeenOtherLoops = false;
4758};
4759
4760/// Takes SCEV S and Loop L. For each AddRec sub-expression, use its post
4761/// increment expression in case its Loop is L. If it is not L then
4762/// use AddRec itself.
4763/// If SCEV contains non-invariant unknown SCEV rewrite cannot be done.
4764class SCEVPostIncRewriter : public SCEVRewriteVisitor<SCEVPostIncRewriter> {
4765public:
4766 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE) {
4767 SCEVPostIncRewriter Rewriter(L, SE);
4768 const SCEV *Result = Rewriter.visit(S);
4769 return Rewriter.hasSeenLoopVariantSCEVUnknown()
4770 ? SE.getCouldNotCompute()
4771 : Result;
4772 }
4773
4774 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4775 if (!SE.isLoopInvariant(S: Expr, L))
4776 SeenLoopVariantSCEVUnknown = true;
4777 return Expr;
4778 }
4779
4780 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4781 // Only re-write AddRecExprs for this loop.
4782 if (Expr->getLoop() == L)
4783 return Expr->getPostIncExpr(SE);
4784 SeenOtherLoops = true;
4785 return Expr;
4786 }
4787
4788 bool hasSeenLoopVariantSCEVUnknown() { return SeenLoopVariantSCEVUnknown; }
4789
4790 bool hasSeenOtherLoops() { return SeenOtherLoops; }
4791
4792private:
4793 explicit SCEVPostIncRewriter(const Loop *L, ScalarEvolution &SE)
4794 : SCEVRewriteVisitor(SE), L(L) {}
4795
4796 const Loop *L;
4797 bool SeenLoopVariantSCEVUnknown = false;
4798 bool SeenOtherLoops = false;
4799};
4800
4801/// This class evaluates the compare condition by matching it against the
4802/// condition of loop latch. If there is a match we assume a true value
4803/// for the condition while building SCEV nodes.
4804class SCEVBackedgeConditionFolder
4805 : public SCEVRewriteVisitor<SCEVBackedgeConditionFolder> {
4806public:
4807 static const SCEV *rewrite(const SCEV *S, const Loop *L,
4808 ScalarEvolution &SE) {
4809 bool IsPosBECond = false;
4810 Value *BECond = nullptr;
4811 if (BasicBlock *Latch = L->getLoopLatch()) {
4812 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Latch->getTerminator())) {
4813 assert(BI->getSuccessor(0) != BI->getSuccessor(1) &&
4814 "Both outgoing branches should not target same header!");
4815 BECond = BI->getCondition();
4816 IsPosBECond = BI->getSuccessor(i: 0) == L->getHeader();
4817 } else {
4818 return S;
4819 }
4820 }
4821 SCEVBackedgeConditionFolder Rewriter(L, BECond, IsPosBECond, SE);
4822 return Rewriter.visit(S);
4823 }
4824
4825 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4826 const SCEV *Result = Expr;
4827 bool InvariantF = SE.isLoopInvariant(S: Expr, L);
4828
4829 if (!InvariantF) {
4830 Instruction *I = cast<Instruction>(Val: Expr->getValue());
4831 switch (I->getOpcode()) {
4832 case Instruction::Select: {
4833 SelectInst *SI = cast<SelectInst>(Val: I);
4834 std::optional<const SCEV *> Res =
4835 compareWithBackedgeCondition(IC: SI->getCondition());
4836 if (Res) {
4837 bool IsOne = cast<SCEVConstant>(Val: *Res)->getValue()->isOne();
4838 Result = SE.getSCEV(V: IsOne ? SI->getTrueValue() : SI->getFalseValue());
4839 }
4840 break;
4841 }
4842 default: {
4843 std::optional<const SCEV *> Res = compareWithBackedgeCondition(IC: I);
4844 if (Res)
4845 Result = *Res;
4846 break;
4847 }
4848 }
4849 }
4850 return Result;
4851 }
4852
4853private:
4854 explicit SCEVBackedgeConditionFolder(const Loop *L, Value *BECond,
4855 bool IsPosBECond, ScalarEvolution &SE)
4856 : SCEVRewriteVisitor(SE), L(L), BackedgeCond(BECond),
4857 IsPositiveBECond(IsPosBECond) {}
4858
4859 std::optional<const SCEV *> compareWithBackedgeCondition(Value *IC);
4860
4861 const Loop *L;
4862 /// Loop back condition.
4863 Value *BackedgeCond = nullptr;
4864 /// Set to true if loop back is on positive branch condition.
4865 bool IsPositiveBECond;
4866};
4867
4868std::optional<const SCEV *>
4869SCEVBackedgeConditionFolder::compareWithBackedgeCondition(Value *IC) {
4870
4871 // If value matches the backedge condition for loop latch,
4872 // then return a constant evolution node based on loopback
4873 // branch taken.
4874 if (BackedgeCond == IC)
4875 return IsPositiveBECond ? SE.getOne(Ty: Type::getInt1Ty(C&: SE.getContext()))
4876 : SE.getZero(Ty: Type::getInt1Ty(C&: SE.getContext()));
4877 return std::nullopt;
4878}
4879
4880class SCEVShiftRewriter : public SCEVRewriteVisitor<SCEVShiftRewriter> {
4881public:
4882 static const SCEV *rewrite(const SCEV *S, const Loop *L,
4883 ScalarEvolution &SE) {
4884 SCEVShiftRewriter Rewriter(L, SE);
4885 const SCEV *Result = Rewriter.visit(S);
4886 return Rewriter.isValid() ? Result : SE.getCouldNotCompute();
4887 }
4888
4889 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
4890 // Only allow AddRecExprs for this loop.
4891 if (!SE.isLoopInvariant(S: Expr, L))
4892 Valid = false;
4893 return Expr;
4894 }
4895
4896 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) {
4897 if (Expr->getLoop() == L && Expr->isAffine())
4898 return SE.getMinusSCEV(LHS: Expr, RHS: Expr->getStepRecurrence(SE));
4899 Valid = false;
4900 return Expr;
4901 }
4902
4903 bool isValid() { return Valid; }
4904
4905private:
4906 explicit SCEVShiftRewriter(const Loop *L, ScalarEvolution &SE)
4907 : SCEVRewriteVisitor(SE), L(L) {}
4908
4909 const Loop *L;
4910 bool Valid = true;
4911};
4912
4913} // end anonymous namespace
4914
4915void ScalarEvolution::inferNoWrapViaConstantRanges(const SCEVAddRecExpr *AR) {
4916 if (!AR->isAffine())
4917 return;
4918
4919 // Force computation of ranges, which will also perform range-based flag
4920 // inference.
4921 if (!AR->hasNoSignedWrap())
4922 (void)getSignedRange(S: AR);
4923
4924 if (!AR->hasNoUnsignedWrap())
4925 (void)getUnsignedRange(S: AR);
4926
4927 if (!AR->hasNoSelfWrap()) {
4928 const SCEV *BECount = getConstantMaxBackedgeTakenCount(L: AR->getLoop());
4929 if (const SCEVConstant *BECountMax = dyn_cast<SCEVConstant>(Val: BECount)) {
4930 ConstantRange StepCR = getSignedRange(S: AR->getStepRecurrence(SE&: *this));
4931 const APInt &BECountAP = BECountMax->getAPInt();
4932 unsigned NoOverflowBitWidth =
4933 BECountAP.getActiveBits() + StepCR.getMinSignedBits();
4934 if (NoOverflowBitWidth <= getTypeSizeInBits(Ty: AR->getType()))
4935 const_cast<SCEVAddRecExpr *>(AR)->setNoWrapFlags(SCEV::FlagNW);
4936 }
4937 }
4938}
4939
4940SCEVFlags
4941ScalarEvolution::proveNoSignedWrapViaInduction(const SCEVAddRecExpr *AR) {
4942 SCEVFlags Result = AR->getNoWrapFlags();
4943
4944 if (AR->hasNoSignedWrap())
4945 return Result;
4946
4947 if (!AR->isAffine())
4948 return Result;
4949
4950 // This function can be expensive, only try to prove NSW once per AddRec.
4951 if (!SignedWrapViaInductionTried.insert(Ptr: AR).second)
4952 return Result;
4953
4954 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
4955 const Loop *L = AR->getLoop();
4956
4957 // Check whether the backedge-taken count is SCEVCouldNotCompute.
4958 // Note that this serves two purposes: It filters out loops that are
4959 // simply not analyzable, and it covers the case where this code is
4960 // being called from within backedge-taken count analysis, such that
4961 // attempting to ask for the backedge-taken count would likely result
4962 // in infinite recursion. In the later case, the analysis code will
4963 // cope with a conservative value, and it will take care to purge
4964 // that value once it has finished.
4965 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
4966
4967 // Normally, in the cases we can prove no-overflow via a
4968 // backedge guarding condition, we can also compute a backedge
4969 // taken count for the loop. The exceptions are assumptions and
4970 // guards present in the loop -- SCEV is not great at exploiting
4971 // these to compute max backedge taken counts, but can still use
4972 // these to prove lack of overflow. Use this fact to avoid
4973 // doing extra work that may not pay off.
4974
4975 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
4976 AC.assumptions().empty())
4977 return Result;
4978
4979 // If the backedge is guarded by a comparison with the pre-inc value the
4980 // addrec is safe. Also, if the entry is guarded by a comparison with the
4981 // start value and the backedge is guarded by a comparison with the post-inc
4982 // value, the addrec is safe.
4983 ICmpInst::Predicate Pred;
4984 const SCEV *OverflowLimit =
4985 getSignedOverflowLimitForStep(Step, Pred: &Pred, SE: this);
4986 if (OverflowLimit &&
4987 (isLoopBackedgeGuardedByCond(L, Pred, LHS: AR, RHS: OverflowLimit) ||
4988 isKnownOnEveryIteration(Pred, LHS: AR, RHS: OverflowLimit))) {
4989 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNSW);
4990 }
4991 return Result;
4992}
4993SCEVFlags
4994ScalarEvolution::proveNoUnsignedWrapViaInduction(const SCEVAddRecExpr *AR) {
4995 SCEVFlags Result = AR->getNoWrapFlags();
4996
4997 if (AR->hasNoUnsignedWrap())
4998 return Result;
4999
5000 if (!AR->isAffine())
5001 return Result;
5002
5003 // This function can be expensive, only try to prove NUW once per AddRec.
5004 if (!UnsignedWrapViaInductionTried.insert(Ptr: AR).second)
5005 return Result;
5006
5007 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
5008 const Loop *L = AR->getLoop();
5009
5010 // Check whether the backedge-taken count is SCEVCouldNotCompute.
5011 // Note that this serves two purposes: It filters out loops that are
5012 // simply not analyzable, and it covers the case where this code is
5013 // being called from within backedge-taken count analysis, such that
5014 // attempting to ask for the backedge-taken count would likely result
5015 // in infinite recursion. In the later case, the analysis code will
5016 // cope with a conservative value, and it will take care to purge
5017 // that value once it has finished.
5018 const SCEV *MaxBECount = getConstantMaxBackedgeTakenCount(L);
5019
5020 // Normally, in the cases we can prove no-overflow via a
5021 // backedge guarding condition, we can also compute a backedge
5022 // taken count for the loop. The exceptions are assumptions and
5023 // guards present in the loop -- SCEV is not great at exploiting
5024 // these to compute max backedge taken counts, but can still use
5025 // these to prove lack of overflow. Use this fact to avoid
5026 // doing extra work that may not pay off.
5027
5028 if (isa<SCEVCouldNotCompute>(Val: MaxBECount) && !HasGuards &&
5029 AC.assumptions().empty())
5030 return Result;
5031
5032 // If the backedge is guarded by a comparison with the pre-inc value the
5033 // addrec is safe. Also, if the entry is guarded by a comparison with the
5034 // start value and the backedge is guarded by a comparison with the post-inc
5035 // value, the addrec is safe.
5036 if (isKnownPositive(S: Step)) {
5037 ICmpInst::Predicate Pred;
5038 const SCEV *OverflowLimit =
5039 getUnsignedOverflowLimitForStep(Step, Pred: &Pred, SE: this);
5040 if (isLoopBackedgeGuardedByCond(L, Pred, LHS: AR, RHS: OverflowLimit) ||
5041 isKnownOnEveryIteration(Pred, LHS: AR, RHS: OverflowLimit))
5042 Result = setFlags(Flags: Result, OnFlags: SCEV::FlagNUW);
5043 }
5044 return Result;
5045}
5046
5047namespace {
5048
5049/// Represents an abstract binary operation. This may exist as a
5050/// normal instruction or constant expression, or may have been
5051/// derived from an expression tree.
5052struct BinaryOp {
5053 unsigned Opcode;
5054 Value *LHS;
5055 Value *RHS;
5056 bool IsNSW = false;
5057 bool IsNUW = false;
5058
5059 /// Op is set if this BinaryOp corresponds to a concrete LLVM instruction or
5060 /// constant expression.
5061 Operator *Op = nullptr;
5062
5063 explicit BinaryOp(Operator *Op)
5064 : Opcode(Op->getOpcode()), LHS(Op->getOperand(i: 0)), RHS(Op->getOperand(i: 1)),
5065 Op(Op) {
5066 if (auto *OBO = dyn_cast<OverflowingBinaryOperator>(Val: Op)) {
5067 IsNSW = OBO->hasNoSignedWrap();
5068 IsNUW = OBO->hasNoUnsignedWrap();
5069 }
5070 }
5071
5072 explicit BinaryOp(unsigned Opcode, Value *LHS, Value *RHS, bool IsNSW = false,
5073 bool IsNUW = false)
5074 : Opcode(Opcode), LHS(LHS), RHS(RHS), IsNSW(IsNSW), IsNUW(IsNUW) {}
5075};
5076
5077} // end anonymous namespace
5078
5079/// Try to map \p V into a BinaryOp, and return \c std::nullopt on failure.
5080static std::optional<BinaryOp> MatchBinaryOp(Value *V,
5081 const DominatorTree &DT) {
5082 auto *Op = dyn_cast<Operator>(Val: V);
5083 if (!Op)
5084 return std::nullopt;
5085
5086 // Implementation detail: all the cleverness here should happen without
5087 // creating new SCEV expressions -- our caller knowns tricks to avoid creating
5088 // SCEV expressions when possible, and we should not break that.
5089
5090 switch (Op->getOpcode()) {
5091 case Instruction::Add:
5092 case Instruction::Sub:
5093 case Instruction::Mul:
5094 case Instruction::UDiv:
5095 case Instruction::URem:
5096 case Instruction::And:
5097 case Instruction::AShr:
5098 case Instruction::Shl:
5099 return BinaryOp(Op);
5100
5101 case Instruction::Or: {
5102 // Convert or disjoint into add nuw nsw.
5103 if (cast<PossiblyDisjointInst>(Val: Op)->isDisjoint()) {
5104 BinaryOp BinOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1),
5105 /*IsNSW=*/true, /*IsNUW=*/true);
5106 // Keep the reference to the original instruction so that we can later
5107 // check whether it can produce poison value or not.
5108 BinOp.Op = Op;
5109 return BinOp;
5110 }
5111 return BinaryOp(Op);
5112 }
5113
5114 case Instruction::Xor:
5115 if (auto *RHSC = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1)))
5116 // If the RHS of the xor is a signmask, then this is just an add.
5117 // Instcombine turns add of signmask into xor as a strength reduction step.
5118 if (RHSC->getValue().isSignMask())
5119 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5120 // Binary `xor` is a bit-wise `add`.
5121 if (V->getType()->isIntegerTy(BitWidth: 1))
5122 return BinaryOp(Instruction::Add, Op->getOperand(i: 0), Op->getOperand(i: 1));
5123 return BinaryOp(Op);
5124
5125 case Instruction::LShr:
5126 // Turn logical shift right of a constant into a unsigned divide.
5127 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: Op->getOperand(i: 1))) {
5128 uint32_t BitWidth = cast<IntegerType>(Val: Op->getType())->getBitWidth();
5129
5130 // If the shift count is not less than the bitwidth, the result of
5131 // the shift is undefined. Don't try to analyze it, because the
5132 // resolution chosen here may differ from the resolution chosen in
5133 // other parts of the compiler.
5134 if (SA->getValue().ult(RHS: BitWidth)) {
5135 Constant *X =
5136 ConstantInt::get(Context&: SA->getContext(),
5137 V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
5138 return BinaryOp(Instruction::UDiv, Op->getOperand(i: 0), X);
5139 }
5140 }
5141 return BinaryOp(Op);
5142
5143 case Instruction::ExtractValue: {
5144 auto *EVI = cast<ExtractValueInst>(Val: Op);
5145 if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0)
5146 break;
5147
5148 auto *WO = dyn_cast<WithOverflowInst>(Val: EVI->getAggregateOperand());
5149 if (!WO)
5150 break;
5151
5152 Instruction::BinaryOps BinOp = WO->getBinaryOp();
5153 bool Signed = WO->isSigned();
5154 // TODO: Should add nuw/nsw flags for mul as well.
5155 if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT))
5156 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS());
5157
5158 // Now that we know that all uses of the arithmetic-result component of
5159 // CI are guarded by the overflow check, we can go ahead and pretend
5160 // that the arithmetic is non-overflowing.
5161 return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(),
5162 /* IsNSW = */ Signed, /* IsNUW = */ !Signed);
5163 }
5164
5165 default:
5166 break;
5167 }
5168
5169 // Recognise intrinsic loop.decrement.reg, and as this has exactly the same
5170 // semantics as a Sub, return a binary sub expression.
5171 if (auto *II = dyn_cast<IntrinsicInst>(Val: V))
5172 if (II->getIntrinsicID() == Intrinsic::loop_decrement_reg)
5173 return BinaryOp(Instruction::Sub, II->getOperand(i_nocapture: 0), II->getOperand(i_nocapture: 1));
5174
5175 return std::nullopt;
5176}
5177
5178/// Helper function to createAddRecFromPHIWithCasts. We have a phi
5179/// node whose symbolic (unknown) SCEV is \p SymbolicPHI, which is updated via
5180/// the loop backedge by a SCEVAddExpr, possibly also with a few casts on the
5181/// way. This function checks if \p Op, an operand of this SCEVAddExpr,
5182/// follows one of the following patterns:
5183/// Op == (SExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5184/// Op == (ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy)
5185/// If the SCEV expression of \p Op conforms with one of the expected patterns
5186/// we return the type of the truncation operation, and indicate whether the
5187/// truncated type should be treated as signed/unsigned by setting
5188/// \p Signed to true/false, respectively.
5189static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI,
5190 bool &Signed, ScalarEvolution &SE) {
5191 // The case where Op == SymbolicPHI (that is, with no type conversions on
5192 // the way) is handled by the regular add recurrence creating logic and
5193 // would have already been triggered in createAddRecForPHI. Reaching it here
5194 // means that createAddRecFromPHI had failed for this PHI before (e.g.,
5195 // because one of the other operands of the SCEVAddExpr updating this PHI is
5196 // not invariant).
5197 //
5198 // Here we look for the case where Op = (ext(trunc(SymbolicPHI))), and in
5199 // this case predicates that allow us to prove that Op == SymbolicPHI will
5200 // be added.
5201 if (Op == SymbolicPHI)
5202 return nullptr;
5203
5204 unsigned SourceBits = SE.getTypeSizeInBits(Ty: SymbolicPHI->getType());
5205 unsigned NewBits = SE.getTypeSizeInBits(Ty: Op->getType());
5206 if (SourceBits != NewBits)
5207 return nullptr;
5208
5209 if (match(S: Op, P: m_scev_SExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5210 Signed = true;
5211 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5212 }
5213 if (match(S: Op, P: m_scev_ZExt(Op0: m_scev_Trunc(Op0: m_scev_Specific(S: SymbolicPHI))))) {
5214 Signed = false;
5215 return cast<SCEVCastExpr>(Val: Op)->getOperand()->getType();
5216 }
5217 return nullptr;
5218}
5219
5220static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) {
5221 if (!PN->getType()->isIntegerTy())
5222 return nullptr;
5223 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5224 if (!L || L->getHeader() != PN->getParent())
5225 return nullptr;
5226 return L;
5227}
5228
5229// Analyze \p SymbolicPHI, a SCEV expression of a phi node, and check if the
5230// computation that updates the phi follows the following pattern:
5231// (SExt/ZExt ix (Trunc iy (%SymbolicPHI) to ix) to iy) + InvariantAccum
5232// which correspond to a phi->trunc->sext/zext->add->phi update chain.
5233// If so, try to see if it can be rewritten as an AddRecExpr under some
5234// Predicates. If successful, return them as a pair. Also cache the results
5235// of the analysis.
5236//
5237// Example usage scenario:
5238// Say the Rewriter is called for the following SCEV:
5239// 8 * ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5240// where:
5241// %X = phi i64 (%Start, %BEValue)
5242// It will visitMul->visitAdd->visitSExt->visitTrunc->visitUnknown(%X),
5243// and call this function with %SymbolicPHI = %X.
5244//
5245// The analysis will find that the value coming around the backedge has
5246// the following SCEV:
5247// BEValue = ((sext i32 (trunc i64 %X to i32) to i64) + %Step)
5248// Upon concluding that this matches the desired pattern, the function
5249// will return the pair {NewAddRec, SmallPredsVec} where:
5250// NewAddRec = {%Start,+,%Step}
5251// SmallPredsVec = {P1, P2, P3} as follows:
5252// P1(WrapPred): AR: {trunc(%Start),+,(trunc %Step)}<nsw> Flags: <nssw>
5253// P2(EqualPred): %Start == (sext i32 (trunc i64 %Start to i32) to i64)
5254// P3(EqualPred): %Step == (sext i32 (trunc i64 %Step to i32) to i64)
5255// The returned pair means that SymbolicPHI can be rewritten into NewAddRec
5256// under the predicates {P1,P2,P3}.
5257// This predicated rewrite will be cached in PredicatedSCEVRewrites:
5258// PredicatedSCEVRewrites[{%X,L}] = {NewAddRec, {P1,P2,P3)}
5259//
5260// TODO's:
5261//
5262// 1) Extend the Induction descriptor to also support inductions that involve
5263// casts: When needed (namely, when we are called in the context of the
5264// vectorizer induction analysis), a Set of cast instructions will be
5265// populated by this method, and provided back to isInductionPHI. This is
5266// needed to allow the vectorizer to properly record them to be ignored by
5267// the cost model and to avoid vectorizing them (otherwise these casts,
5268// which are redundant under the runtime overflow checks, will be
5269// vectorized, which can be costly).
5270//
5271// 2) Support additional induction/PHISCEV patterns: We also want to support
5272// inductions where the sext-trunc / zext-trunc operations (partly) occur
5273// after the induction update operation (the induction increment):
5274//
5275// (Trunc iy (SExt/ZExt ix (%SymbolicPHI + InvariantAccum) to iy) to ix)
5276// which correspond to a phi->add->trunc->sext/zext->phi update chain.
5277//
5278// (Trunc iy ((SExt/ZExt ix (%SymbolicPhi) to iy) + InvariantAccum) to ix)
5279// which correspond to a phi->trunc->add->sext/zext->phi update chain.
5280//
5281// 3) Outline common code with createAddRecFromPHI to avoid duplication.
5282std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5283ScalarEvolution::createAddRecFromPHIWithCastsImpl(const SCEVUnknown *SymbolicPHI) {
5284 SmallVector<const SCEVPredicate *, 3> Predicates;
5285
5286 // *** Part1: Analyze if we have a phi-with-cast pattern for which we can
5287 // return an AddRec expression under some predicate.
5288
5289 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5290 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5291 assert(L && "Expecting an integer loop header phi");
5292
5293 // The loop may have multiple entrances or multiple exits; we can analyze
5294 // this phi as an addrec if it has a unique entry value and a unique
5295 // backedge value.
5296 Value *BEValueV = nullptr, *StartValueV = nullptr;
5297 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5298 Value *V = PN->getIncomingValue(i);
5299 if (L->contains(BB: PN->getIncomingBlock(i))) {
5300 if (!BEValueV) {
5301 BEValueV = V;
5302 } else if (BEValueV != V) {
5303 BEValueV = nullptr;
5304 break;
5305 }
5306 } else if (!StartValueV) {
5307 StartValueV = V;
5308 } else if (StartValueV != V) {
5309 StartValueV = nullptr;
5310 break;
5311 }
5312 }
5313 if (!BEValueV || !StartValueV)
5314 return std::nullopt;
5315
5316 const SCEV *BEValue = getSCEV(V: BEValueV);
5317
5318 // If the value coming around the backedge is an add with the symbolic
5319 // value we just inserted, possibly with casts that we can ignore under
5320 // an appropriate runtime guard, then we found a simple induction variable!
5321 const auto *Add = dyn_cast<SCEVAddExpr>(Val: BEValue);
5322 if (!Add)
5323 return std::nullopt;
5324
5325 // If there is a single occurrence of the symbolic value, possibly
5326 // casted, replace it with a recurrence.
5327 unsigned FoundIndex = Add->getNumOperands();
5328 Type *TruncTy = nullptr;
5329 bool Signed;
5330 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5331 if ((TruncTy =
5332 isSimpleCastedPHI(Op: Add->getOperand(i), SymbolicPHI, Signed, SE&: *this)))
5333 if (FoundIndex == e) {
5334 FoundIndex = i;
5335 break;
5336 }
5337
5338 if (FoundIndex == Add->getNumOperands())
5339 return std::nullopt;
5340
5341 // Create an add with everything but the specified operand.
5342 SmallVector<SCEVUse, 8> Ops;
5343 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5344 if (i != FoundIndex)
5345 Ops.push_back(Elt: Add->getOperand(i));
5346 const SCEV *Accum = getAddExpr(Ops);
5347
5348 // The runtime checks will not be valid if the step amount is
5349 // varying inside the loop.
5350 if (!isLoopInvariant(S: Accum, L))
5351 return std::nullopt;
5352
5353 // *** Part2: Create the predicates
5354
5355 // Analysis was successful: we have a phi-with-cast pattern for which we
5356 // can return an AddRec expression under the following predicates:
5357 //
5358 // P1: A Wrap predicate that guarantees that Trunc(Start) + i*Trunc(Accum)
5359 // fits within the truncated type (does not overflow) for i = 0 to n-1.
5360 // P2: An Equal predicate that guarantees that
5361 // Start = (Ext ix (Trunc iy (Start) to ix) to iy)
5362 // P3: An Equal predicate that guarantees that
5363 // Accum = (Ext ix (Trunc iy (Accum) to ix) to iy)
5364 //
5365 // As we next prove, the above predicates guarantee that:
5366 // Start + i*Accum = (Ext ix (Trunc iy ( Start + i*Accum ) to ix) to iy)
5367 //
5368 //
5369 // More formally, we want to prove that:
5370 // Expr(i+1) = Start + (i+1) * Accum
5371 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5372 //
5373 // Given that:
5374 // 1) Expr(0) = Start
5375 // 2) Expr(1) = Start + Accum
5376 // = (Ext ix (Trunc iy (Start) to ix) to iy) + Accum :: from P2
5377 // 3) Induction hypothesis (step i):
5378 // Expr(i) = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum
5379 //
5380 // Proof:
5381 // Expr(i+1) =
5382 // = Start + (i+1)*Accum
5383 // = (Start + i*Accum) + Accum
5384 // = Expr(i) + Accum
5385 // = (Ext ix (Trunc iy (Expr(i-1)) to ix) to iy) + Accum + Accum
5386 // :: from step i
5387 //
5388 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy) + Accum + Accum
5389 //
5390 // = (Ext ix (Trunc iy (Start + (i-1)*Accum) to ix) to iy)
5391 // + (Ext ix (Trunc iy (Accum) to ix) to iy)
5392 // + Accum :: from P3
5393 //
5394 // = (Ext ix (Trunc iy ((Start + (i-1)*Accum) + Accum) to ix) to iy)
5395 // + Accum :: from P1: Ext(x)+Ext(y)=>Ext(x+y)
5396 //
5397 // = (Ext ix (Trunc iy (Start + i*Accum) to ix) to iy) + Accum
5398 // = (Ext ix (Trunc iy (Expr(i)) to ix) to iy) + Accum
5399 //
5400 // By induction, the same applies to all iterations 1<=i<n:
5401 //
5402
5403 // Create a truncated addrec for which we will add a no overflow check (P1).
5404 const SCEV *StartVal = getSCEV(V: StartValueV);
5405 const SCEV *PHISCEV =
5406 getAddRecExpr(Start: getTruncateExpr(Op: StartVal, Ty: TruncTy),
5407 Step: getTruncateExpr(Op: Accum, Ty: TruncTy), L, Flags: SCEV::FlagNone);
5408
5409 // PHISCEV can be either a SCEVConstant or a SCEVAddRecExpr.
5410 // ex: If truncated Accum is 0 and StartVal is a constant, then PHISCEV
5411 // will be constant.
5412 //
5413 // If PHISCEV is a constant, then P1 degenerates into P2 or P3, so we don't
5414 // add P1.
5415 if (const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV)) {
5416 SCEVWrapPredicate::IncrementWrapFlags AddedFlags =
5417 Signed ? SCEVWrapPredicate::IncrementNSSW
5418 : SCEVWrapPredicate::IncrementNUSW;
5419 const SCEVPredicate *AddRecPred = getWrapPredicate(AR, AddedFlags);
5420 Predicates.push_back(Elt: AddRecPred);
5421 }
5422
5423 // Create the Equal Predicates P2,P3:
5424
5425 // It is possible that the predicates P2 and/or P3 are computable at
5426 // compile time due to StartVal and/or Accum being constants.
5427 // If either one is, then we can check that now and escape if either P2
5428 // or P3 is false.
5429
5430 // Construct the extended SCEV: (Ext ix (Trunc iy (Expr) to ix) to iy)
5431 // for each of StartVal and Accum
5432 auto getExtendedExpr = [&](const SCEV *Expr,
5433 bool CreateSignExtend) -> const SCEV * {
5434 assert(isLoopInvariant(Expr, L) && "Expr is expected to be invariant");
5435 const SCEV *TruncatedExpr = getTruncateExpr(Op: Expr, Ty: TruncTy);
5436 const SCEV *ExtendedExpr =
5437 CreateSignExtend ? getSignExtendExpr(Op: TruncatedExpr, Ty: Expr->getType())
5438 : getZeroExtendExpr(Op: TruncatedExpr, Ty: Expr->getType());
5439 return ExtendedExpr;
5440 };
5441
5442 // Given:
5443 // ExtendedExpr = (Ext ix (Trunc iy (Expr) to ix) to iy
5444 // = getExtendedExpr(Expr)
5445 // Determine whether the predicate P: Expr == ExtendedExpr
5446 // is known to be false at compile time
5447 auto PredIsKnownFalse = [&](const SCEV *Expr,
5448 const SCEV *ExtendedExpr) -> bool {
5449 return Expr != ExtendedExpr &&
5450 isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: Expr, RHS: ExtendedExpr);
5451 };
5452
5453 const SCEV *StartExtended = getExtendedExpr(StartVal, Signed);
5454 if (PredIsKnownFalse(StartVal, StartExtended)) {
5455 LLVM_DEBUG(dbgs() << "P2 is compile-time false\n";);
5456 return std::nullopt;
5457 }
5458
5459 // The Step is always Signed (because the overflow checks are either
5460 // NSSW or NUSW)
5461 const SCEV *AccumExtended = getExtendedExpr(Accum, /*CreateSignExtend=*/true);
5462 if (PredIsKnownFalse(Accum, AccumExtended)) {
5463 LLVM_DEBUG(dbgs() << "P3 is compile-time false\n";);
5464 return std::nullopt;
5465 }
5466
5467 auto AppendPredicate = [&](const SCEV *Expr,
5468 const SCEV *ExtendedExpr) -> void {
5469 if (Expr != ExtendedExpr &&
5470 !isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: Expr, RHS: ExtendedExpr)) {
5471 const SCEVPredicate *Pred = getEqualPredicate(LHS: Expr, RHS: ExtendedExpr);
5472 LLVM_DEBUG(dbgs() << "Added Predicate: " << *Pred);
5473 Predicates.push_back(Elt: Pred);
5474 }
5475 };
5476
5477 AppendPredicate(StartVal, StartExtended);
5478 AppendPredicate(Accum, AccumExtended);
5479
5480 // *** Part3: Predicates are ready. Now go ahead and create the new addrec in
5481 // which the casts had been folded away. The caller can rewrite SymbolicPHI
5482 // into NewAR if it will also add the runtime overflow checks specified in
5483 // Predicates.
5484 const SCEV *NewAR = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags: SCEV::FlagNone);
5485
5486 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> PredRewrite =
5487 std::make_pair(x&: NewAR, y&: Predicates);
5488 // Remember the result of the analysis for this SCEV at this locayyytion.
5489 PredicatedSCEVRewrites[{SymbolicPHI, L}] = PredRewrite;
5490 return PredRewrite;
5491}
5492
5493std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5494ScalarEvolution::createAddRecFromPHIWithCasts(const SCEVUnknown *SymbolicPHI) {
5495 auto *PN = cast<PHINode>(Val: SymbolicPHI->getValue());
5496 const Loop *L = isIntegerLoopHeaderPHI(PN, LI);
5497 if (!L)
5498 return std::nullopt;
5499
5500 // Check to see if we already analyzed this PHI.
5501 auto I = PredicatedSCEVRewrites.find(Val: {SymbolicPHI, L});
5502 if (I != PredicatedSCEVRewrites.end()) {
5503 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>> Rewrite =
5504 I->second;
5505 // Analysis was done before and failed to create an AddRec:
5506 if (Rewrite.first == SymbolicPHI)
5507 return std::nullopt;
5508 // Analysis was done before and succeeded to create an AddRec under
5509 // a predicate:
5510 assert(isa<SCEVAddRecExpr>(Rewrite.first) && "Expected an AddRec");
5511 assert(!(Rewrite.second).empty() && "Expected to find Predicates");
5512 return Rewrite;
5513 }
5514
5515 std::optional<std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
5516 Rewrite = createAddRecFromPHIWithCastsImpl(SymbolicPHI);
5517
5518 // Record in the cache that the analysis failed
5519 if (!Rewrite) {
5520 SmallVector<const SCEVPredicate *, 3> Predicates;
5521 PredicatedSCEVRewrites[{SymbolicPHI, L}] = {SymbolicPHI, Predicates};
5522 return std::nullopt;
5523 }
5524
5525 return Rewrite;
5526}
5527
5528// FIXME: This utility is currently required because the Rewriter currently
5529// does not rewrite this expression:
5530// {0, +, (sext ix (trunc iy to ix) to iy)}
5531// into {0, +, %step},
5532// even when the following Equal predicate exists:
5533// "%step == (sext ix (trunc iy to ix) to iy)".
5534bool PredicatedScalarEvolution::areAddRecsEqualWithPreds(
5535 const SCEVAddRecExpr *AR1, const SCEVAddRecExpr *AR2,
5536 ArrayRef<const SCEVPredicate *> NoWrapPreds) const {
5537 if (AR1 == AR2)
5538 return true;
5539
5540 SCEVUnionPredicate NoWrapUnionPred(NoWrapPreds, SE);
5541 SCEVUnionPredicate AllPreds = Preds->getUnionWith(N: &NoWrapUnionPred, SE);
5542 auto areExprsEqual = [&](const SCEV *Expr1, const SCEV *Expr2) -> bool {
5543 if (Expr1 != Expr2 &&
5544 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr1, RHS: Expr2), SE) &&
5545 !AllPreds.implies(N: SE.getEqualPredicate(LHS: Expr2, RHS: Expr1), SE))
5546 return false;
5547 return true;
5548 };
5549
5550 if (!areExprsEqual(AR1->getStart(), AR2->getStart()) ||
5551 !areExprsEqual(AR1->getStepRecurrence(SE), AR2->getStepRecurrence(SE)))
5552 return false;
5553 return true;
5554}
5555
5556static SCEVFlags getNoWrapFlagsForGEP(GEPOperator *GEP, const SCEV *Accum,
5557 ScalarEvolution &SE) {
5558 SCEVFlags Flags = SCEV::FlagNone;
5559 GEPNoWrapFlags NW = GEP->getNoWrapFlags();
5560 // If the increment has any nowrap flags, then we know the address
5561 // space cannot be wrapped around.
5562 if (NW != GEPNoWrapFlags::none())
5563 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNW);
5564 // If the GEP is nuw or nusw with non-negative offset, we know that
5565 // no unsigned wrap occurs. We cannot set the nsw flag as only the
5566 // offset is treated as signed, while the base is unsigned.
5567 if (NW.hasNoUnsignedWrap() ||
5568 (NW.hasNoUnsignedSignedWrap() && SE.isKnownNonNegative(S: Accum)))
5569 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
5570
5571 return Flags;
5572}
5573
5574/// A helper function for createAddRecFromPHI to handle simple cases.
5575///
5576/// This function tries to find an AddRec expression for the simplest (yet most
5577/// common) cases: PN = PHI(Start, OP(Self, LoopInvariant)).
5578/// If it fails, createAddRecFromPHI will use a more general, but slow,
5579/// technique for finding the AddRec expression.
5580const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN,
5581 Value *BEValueV,
5582 Value *StartValueV) {
5583 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5584 assert(L && L->getHeader() == PN->getParent());
5585 assert(BEValueV && StartValueV);
5586
5587 const SCEV *Accum = nullptr;
5588 SCEVFlags Flags = SCEV::FlagNone;
5589 if (auto BO = MatchBinaryOp(V: BEValueV, DT)) {
5590 if (BO->Opcode != Instruction::Add)
5591 return nullptr;
5592
5593 if (BO->LHS == PN && L->isLoopInvariant(V: BO->RHS))
5594 Accum = getSCEV(V: BO->RHS);
5595 else if (BO->RHS == PN && L->isLoopInvariant(V: BO->LHS))
5596 Accum = getSCEV(V: BO->LHS);
5597
5598 if (!Accum)
5599 return nullptr;
5600
5601 if (BO->IsNUW)
5602 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5603 if (BO->IsNSW)
5604 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5605 } else {
5606 // Handle pointer induction variable: PN = PHI(Start, gep PN,
5607 // LoopInvariant).
5608 auto *GEP = dyn_cast<GEPOperator>(Val: BEValueV);
5609 if (!GEP || GEP->getPointerOperand() != PN || GEP->getNumIndices() != 1)
5610 return nullptr;
5611 Value *Idx = *GEP->idx_begin();
5612 if (!L->isLoopInvariant(V: Idx))
5613 return nullptr;
5614
5615 Type *IntIdxTy = getEffectiveSCEVType(Ty: GEP->getType());
5616 Accum = getMulExpr(LHS: getTruncateOrSignExtend(V: getSCEV(V: Idx), Ty: IntIdxTy),
5617 RHS: getSizeOfExpr(IntTy: IntIdxTy, AllocTy: GEP->getSourceElementType()));
5618 Flags = getNoWrapFlagsForGEP(GEP, Accum, SE&: *this);
5619 }
5620
5621 const SCEV *StartVal = getSCEV(V: StartValueV);
5622 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5623 insertValueToMap(V: PN, S: PHISCEV);
5624
5625 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
5626 inferNoWrapViaConstantRanges(AR);
5627
5628 // We can add Flags to the post-inc expression only if we
5629 // know that it is *undefined behavior* for BEValueV to
5630 // overflow.
5631 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV)) {
5632 assert(isLoopInvariant(Accum, L) &&
5633 "Accum is defined outside L, but is not invariant?");
5634 if (isAddRecNeverPoison(I: BEInst, L))
5635 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5636 }
5637
5638 return PHISCEV;
5639}
5640
5641const SCEV *ScalarEvolution::createAddRecFromPHI(PHINode *PN) {
5642 const Loop *L = LI.getLoopFor(BB: PN->getParent());
5643 if (!L || L->getHeader() != PN->getParent())
5644 return nullptr;
5645
5646 // The loop may have multiple entrances or multiple exits; we can analyze
5647 // this phi as an addrec if it has a unique entry value and a unique
5648 // backedge value.
5649 Value *BEValueV = nullptr, *StartValueV = nullptr;
5650 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
5651 Value *V = PN->getIncomingValue(i);
5652 if (L->contains(BB: PN->getIncomingBlock(i))) {
5653 if (!BEValueV) {
5654 BEValueV = V;
5655 } else if (BEValueV != V) {
5656 BEValueV = nullptr;
5657 break;
5658 }
5659 } else if (!StartValueV) {
5660 StartValueV = V;
5661 } else if (StartValueV != V) {
5662 StartValueV = nullptr;
5663 break;
5664 }
5665 }
5666 if (!BEValueV || !StartValueV)
5667 return nullptr;
5668
5669 assert(ValueExprMap.find_as(PN) == ValueExprMap.end() &&
5670 "PHI node already processed?");
5671
5672 // First, try to find AddRec expression without creating a fictituos symbolic
5673 // value for PN.
5674 if (auto *S = createSimpleAffineAddRec(PN, BEValueV, StartValueV))
5675 return S;
5676
5677 // Handle PHI node value symbolically.
5678 const SCEV *SymbolicName = getUnknown(V: PN);
5679 insertValueToMap(V: PN, S: SymbolicName);
5680
5681 // Using this symbolic name for the PHI, analyze the value coming around
5682 // the back-edge.
5683 const SCEV *BEValue = getSCEV(V: BEValueV);
5684
5685 // NOTE: If BEValue is loop invariant, we know that the PHI node just
5686 // has a special value for the first iteration of the loop.
5687
5688 // If the value coming around the backedge is an add with the symbolic
5689 // value we just inserted, then we found a simple induction variable!
5690 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: BEValue)) {
5691 // If there is a single occurrence of the symbolic value, replace it
5692 // with a recurrence.
5693 unsigned FoundIndex = Add->getNumOperands();
5694 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5695 if (Add->getOperand(i) == SymbolicName)
5696 if (FoundIndex == e) {
5697 FoundIndex = i;
5698 break;
5699 }
5700
5701 if (FoundIndex != Add->getNumOperands()) {
5702 // Create an add with everything but the specified operand.
5703 SmallVector<SCEVUse, 8> Ops;
5704 for (unsigned i = 0, e = Add->getNumOperands(); i != e; ++i)
5705 if (i != FoundIndex)
5706 Ops.push_back(Elt: SCEVBackedgeConditionFolder::rewrite(S: Add->getOperand(i),
5707 L, SE&: *this));
5708 const SCEV *Accum = getAddExpr(Ops);
5709
5710 // This is not a valid addrec if the step amount is varying each
5711 // loop iteration, but is not itself an addrec in this loop.
5712 if (isLoopInvariant(S: Accum, L) ||
5713 (isa<SCEVAddRecExpr>(Val: Accum) &&
5714 cast<SCEVAddRecExpr>(Val: Accum)->getLoop() == L)) {
5715 SCEVFlags Flags = SCEV::FlagNone;
5716
5717 if (auto BO = MatchBinaryOp(V: BEValueV, DT)) {
5718 if (BO->Opcode == Instruction::Add && BO->LHS == PN) {
5719 if (BO->IsNUW)
5720 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
5721 if (BO->IsNSW)
5722 Flags = setFlags(Flags, OnFlags: SCEV::FlagNSW);
5723 }
5724 } else if (GEPOperator *GEP = dyn_cast<GEPOperator>(Val: BEValueV)) {
5725 if (GEP->getOperand(i_nocapture: 0) == PN)
5726 Flags = getNoWrapFlagsForGEP(GEP, Accum, SE&: *this);
5727
5728 // We cannot transfer nuw and nsw flags from subtraction
5729 // operations -- sub nuw X, Y is not the same as add nuw X, -Y
5730 // for instance.
5731 }
5732
5733 const SCEV *StartVal = getSCEV(V: StartValueV);
5734 const SCEV *PHISCEV = getAddRecExpr(Start: StartVal, Step: Accum, L, Flags);
5735
5736 // Okay, for the entire analysis of this edge we assumed the PHI
5737 // to be symbolic. We now need to go back and purge all of the
5738 // entries for the scalars that use the symbolic expression.
5739 forgetMemoizedResults(SCEVs: {SymbolicName});
5740 insertValueToMap(V: PN, S: PHISCEV);
5741
5742 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: PHISCEV))
5743 inferNoWrapViaConstantRanges(AR);
5744
5745 // We can add Flags to the post-inc expression only if we
5746 // know that it is *undefined behavior* for BEValueV to
5747 // overflow.
5748 if (auto *BEInst = dyn_cast<Instruction>(Val: BEValueV))
5749 if (isLoopInvariant(S: Accum, L) && isAddRecNeverPoison(I: BEInst, L))
5750 (void)getAddRecExpr(Start: getAddExpr(LHS: StartVal, RHS: Accum), Step: Accum, L, Flags);
5751
5752 return PHISCEV;
5753 }
5754 }
5755 } else {
5756 // Otherwise, this could be a loop like this:
5757 // i = 0; for (j = 1; ..; ++j) { .... i = j; }
5758 // In this case, j = {1,+,1} and BEValue is j.
5759 // Because the other in-value of i (0) fits the evolution of BEValue
5760 // i really is an addrec evolution.
5761 //
5762 // We can generalize this saying that i is the shifted value of BEValue
5763 // by one iteration:
5764 // PHI(f(0), f({1,+,1})) --> f({0,+,1})
5765
5766 // Do not allow refinement in rewriting of BEValue.
5767 const SCEV *Shifted = SCEVShiftRewriter::rewrite(S: BEValue, L, SE&: *this);
5768 const SCEV *Start = SCEVInitRewriter::rewrite(S: Shifted, L, SE&: *this, IgnoreOtherLoops: false);
5769 if (Shifted != getCouldNotCompute() && Start != getCouldNotCompute() &&
5770 isGuaranteedNotToCauseUB(Op: Shifted) && ::impliesPoison(AssumedPoison: Shifted, S: Start)) {
5771 const SCEV *StartVal = getSCEV(V: StartValueV);
5772 if (Start == StartVal) {
5773 // Okay, for the entire analysis of this edge we assumed the PHI
5774 // to be symbolic. We now need to go back and purge all of the
5775 // entries for the scalars that use the symbolic expression.
5776 forgetMemoizedResults(SCEVs: {SymbolicName});
5777 insertValueToMap(V: PN, S: Shifted);
5778 return Shifted;
5779 }
5780 }
5781 }
5782
5783 // Remove the temporary PHI node SCEV that has been inserted while intending
5784 // to create an AddRecExpr for this PHI node. We can not keep this temporary
5785 // as it will prevent later (possibly simpler) SCEV expressions to be added
5786 // to the ValueExprMap.
5787 eraseValueFromMap(V: PN);
5788
5789 return nullptr;
5790}
5791
5792// Try to match a control flow sequence that branches out at BI and merges back
5793// at Merge into a "C ? LHS : RHS" select pattern. Return true on a successful
5794// match.
5795static bool BrPHIToSelect(DominatorTree &DT, CondBrInst *BI, PHINode *Merge,
5796 Value *&C, Value *&LHS, Value *&RHS) {
5797 C = BI->getCondition();
5798
5799 BasicBlockEdge LeftEdge(BI->getParent(), BI->getSuccessor(i: 0));
5800 BasicBlockEdge RightEdge(BI->getParent(), BI->getSuccessor(i: 1));
5801
5802 Use &LeftUse = Merge->getOperandUse(i: 0);
5803 Use &RightUse = Merge->getOperandUse(i: 1);
5804
5805 if (DT.dominates(BBE: LeftEdge, U: LeftUse) && DT.dominates(BBE: RightEdge, U: RightUse)) {
5806 LHS = LeftUse;
5807 RHS = RightUse;
5808 return true;
5809 }
5810
5811 if (DT.dominates(BBE: LeftEdge, U: RightUse) && DT.dominates(BBE: RightEdge, U: LeftUse)) {
5812 LHS = RightUse;
5813 RHS = LeftUse;
5814 return true;
5815 }
5816
5817 return false;
5818}
5819
5820static bool getOperandsForSelectLikePHI(DominatorTree &DT, PHINode *PN,
5821 Value *&Cond, Value *&LHS,
5822 Value *&RHS) {
5823 auto IsReachable =
5824 [&](BasicBlock *BB) { return DT.isReachableFromEntry(A: BB); };
5825 if (PN->getNumIncomingValues() == 2 && all_of(Range: PN->blocks(), P: IsReachable)) {
5826 // Try to match
5827 //
5828 // br %cond, label %left, label %right
5829 // left:
5830 // br label %merge
5831 // right:
5832 // br label %merge
5833 // merge:
5834 // V = phi [ %x, %left ], [ %y, %right ]
5835 //
5836 // as "select %cond, %x, %y"
5837
5838 BasicBlock *IDom = DT[PN->getParent()]->getIDom()->getBlock();
5839 assert(IDom && "At least the entry block should dominate PN");
5840
5841 auto *BI = dyn_cast<CondBrInst>(Val: IDom->getTerminator());
5842 return BI && BrPHIToSelect(DT, BI, Merge: PN, C&: Cond, LHS, RHS);
5843 }
5844 return false;
5845}
5846
5847const SCEV *ScalarEvolution::createNodeFromSelectLikePHI(PHINode *PN) {
5848 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
5849 if (getOperandsForSelectLikePHI(DT, PN, Cond, LHS, RHS) &&
5850 properlyDominates(S: getSCEV(V: LHS), BB: PN->getParent()) &&
5851 properlyDominates(S: getSCEV(V: RHS), BB: PN->getParent()))
5852 return createNodeForSelectOrPHI(V: PN, Cond, TrueVal: LHS, FalseVal: RHS);
5853
5854 return nullptr;
5855}
5856
5857static BinaryOperator *getCommonInstForPHI(PHINode *PN) {
5858 BinaryOperator *CommonInst = nullptr;
5859 // Check if instructions are identical.
5860 for (Value *Incoming : PN->incoming_values()) {
5861 auto *IncomingInst = dyn_cast<BinaryOperator>(Val: Incoming);
5862 if (!IncomingInst)
5863 return nullptr;
5864 if (CommonInst) {
5865 if (!CommonInst->isIdenticalToWhenDefined(I: IncomingInst))
5866 return nullptr; // Not identical, give up
5867 } else {
5868 // Remember binary operator
5869 CommonInst = IncomingInst;
5870 }
5871 }
5872 return CommonInst;
5873}
5874
5875/// Returns SCEV for the first operand of a phi if all phi operands have
5876/// identical opcodes and operands
5877/// eg.
5878/// a: %add = %a + %b
5879/// br %c
5880/// b: %add1 = %a + %b
5881/// br %c
5882/// c: %phi = phi [%add, a], [%add1, b]
5883/// scev(%phi) => scev(%add)
5884const SCEV *
5885ScalarEvolution::createNodeForPHIWithIdenticalOperands(PHINode *PN) {
5886 BinaryOperator *CommonInst = getCommonInstForPHI(PN);
5887 if (!CommonInst)
5888 return nullptr;
5889
5890 // Check if SCEV exprs for instructions are identical.
5891 const SCEV *CommonSCEV = getSCEV(V: CommonInst);
5892 bool SCEVExprsIdentical =
5893 all_of(Range: drop_begin(RangeOrContainer: PN->incoming_values()),
5894 P: [this, CommonSCEV](Value *V) { return CommonSCEV == getSCEV(V); });
5895 return SCEVExprsIdentical ? CommonSCEV : nullptr;
5896}
5897
5898const SCEV *ScalarEvolution::createNodeForPHI(PHINode *PN) {
5899 if (const SCEV *S = createAddRecFromPHI(PN))
5900 return S;
5901
5902 // We do not allow simplifying phi (undef, X) to X here, to avoid reusing the
5903 // phi node for X.
5904 if (Value *V = simplifyInstruction(
5905 I: PN, Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
5906 /*UseInstrInfo=*/true, /*CanUseUndef=*/false}))
5907 return getSCEV(V);
5908
5909 if (const SCEV *S = createNodeForPHIWithIdenticalOperands(PN))
5910 return S;
5911
5912 if (const SCEV *S = createNodeFromSelectLikePHI(PN))
5913 return S;
5914
5915 // If it's not a loop phi, we can't handle it yet.
5916 return getUnknown(V: PN);
5917}
5918
5919bool SCEVMinMaxExprContains(const SCEV *Root, const SCEV *OperandToFind,
5920 SCEVTypes RootKind) {
5921 struct FindClosure {
5922 const SCEV *OperandToFind;
5923 const SCEVTypes RootKind; // Must be a sequential min/max expression.
5924 const SCEVTypes NonSequentialRootKind; // Non-seq variant of RootKind.
5925
5926 bool Found = false;
5927
5928 bool canRecurseInto(SCEVTypes Kind) const {
5929 // We can only recurse into the SCEV expression of the same effective type
5930 // as the type of our root SCEV expression, and into zero-extensions.
5931 return RootKind == Kind || NonSequentialRootKind == Kind ||
5932 scZeroExtend == Kind;
5933 };
5934
5935 FindClosure(const SCEV *OperandToFind, SCEVTypes RootKind)
5936 : OperandToFind(OperandToFind), RootKind(RootKind),
5937 NonSequentialRootKind(
5938 SCEVSequentialMinMaxExpr::getEquivalentNonSequentialSCEVType(
5939 Ty: RootKind)) {}
5940
5941 bool follow(const SCEV *S) {
5942 Found = S == OperandToFind;
5943
5944 return !isDone() && canRecurseInto(Kind: S->getSCEVType());
5945 }
5946
5947 bool isDone() const { return Found; }
5948 };
5949
5950 FindClosure FC(OperandToFind, RootKind);
5951 visitAll(Root, Visitor&: FC);
5952 return FC.Found;
5953}
5954
5955std::optional<const SCEV *>
5956ScalarEvolution::createNodeForSelectOrPHIInstWithICmpInstCond(Type *Ty,
5957 ICmpInst *Cond,
5958 Value *TrueVal,
5959 Value *FalseVal) {
5960 // Try to match some simple smax or umax patterns.
5961 auto *ICI = Cond;
5962
5963 Value *LHS = ICI->getOperand(i_nocapture: 0);
5964 Value *RHS = ICI->getOperand(i_nocapture: 1);
5965
5966 switch (ICI->getPredicate()) {
5967 case ICmpInst::ICMP_SLT:
5968 case ICmpInst::ICMP_SLE:
5969 case ICmpInst::ICMP_ULT:
5970 case ICmpInst::ICMP_ULE:
5971 std::swap(a&: LHS, b&: RHS);
5972 [[fallthrough]];
5973 case ICmpInst::ICMP_SGT:
5974 case ICmpInst::ICMP_SGE:
5975 case ICmpInst::ICMP_UGT:
5976 case ICmpInst::ICMP_UGE:
5977 // a > b ? a+x : b+x -> max(a, b)+x
5978 // a > b ? b+x : a+x -> min(a, b)+x
5979 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty)) {
5980 bool Signed = ICI->isSigned();
5981 const SCEV *LA = getSCEV(V: TrueVal);
5982 const SCEV *RA = getSCEV(V: FalseVal);
5983 const SCEV *LS = getSCEV(V: LHS);
5984 const SCEV *RS = getSCEV(V: RHS);
5985 if (LA->getType()->isPointerTy()) {
5986 // FIXME: Handle cases where LS/RS are pointers not equal to LA/RA.
5987 // Need to make sure we can't produce weird expressions involving
5988 // negated pointers.
5989 if (LA == LS && RA == RS)
5990 return Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS);
5991 if (LA == RS && RA == LS)
5992 return Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS);
5993 }
5994 auto CoerceOperand = [&](const SCEV *Op) -> const SCEV * {
5995 if (Op->getType()->isPointerTy()) {
5996 Op = getPtrToAddrExpr(Op);
5997 if (isa<SCEVCouldNotCompute>(Val: Op))
5998 return Op;
5999 }
6000 if (Signed)
6001 Op = getNoopOrSignExtend(V: Op, Ty);
6002 else
6003 Op = getNoopOrZeroExtend(V: Op, Ty);
6004 return Op;
6005 };
6006 LS = CoerceOperand(LS);
6007 RS = CoerceOperand(RS);
6008 if (isa<SCEVCouldNotCompute>(Val: LS) || isa<SCEVCouldNotCompute>(Val: RS))
6009 break;
6010 const SCEV *LDiff = getMinusSCEV(LHS: LA, RHS: LS);
6011 const SCEV *RDiff = getMinusSCEV(LHS: RA, RHS: RS);
6012 if (LDiff == RDiff)
6013 return getAddExpr(LHS: Signed ? getSMaxExpr(LHS: LS, RHS: RS) : getUMaxExpr(LHS: LS, RHS: RS),
6014 RHS: LDiff);
6015 LDiff = getMinusSCEV(LHS: LA, RHS: RS);
6016 RDiff = getMinusSCEV(LHS: RA, RHS: LS);
6017 if (LDiff == RDiff)
6018 return getAddExpr(LHS: Signed ? getSMinExpr(LHS: LS, RHS: RS) : getUMinExpr(LHS: LS, RHS: RS),
6019 RHS: LDiff);
6020 }
6021 break;
6022 case ICmpInst::ICMP_NE:
6023 // x != 0 ? x+y : C+y -> x == 0 ? C+y : x+y
6024 std::swap(a&: TrueVal, b&: FalseVal);
6025 [[fallthrough]];
6026 case ICmpInst::ICMP_EQ:
6027 // x == 0 ? C+y : x+y -> umax(x, C)+y iff C u<= 1
6028 if (getTypeSizeInBits(Ty: LHS->getType()) <= getTypeSizeInBits(Ty) &&
6029 isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()) {
6030 const SCEV *X = getNoopOrZeroExtend(V: getSCEV(V: LHS), Ty);
6031 const SCEV *TrueValExpr = getSCEV(V: TrueVal); // C+y
6032 const SCEV *FalseValExpr = getSCEV(V: FalseVal); // x+y
6033 const SCEV *Y = getMinusSCEV(LHS: FalseValExpr, RHS: X); // y = (x+y)-x
6034 const SCEV *C = getMinusSCEV(LHS: TrueValExpr, RHS: Y); // C = (C+y)-y
6035 if (isa<SCEVConstant>(Val: C) && cast<SCEVConstant>(Val: C)->getAPInt().ule(RHS: 1))
6036 return getAddExpr(LHS: getUMaxExpr(LHS: X, RHS: C), RHS: Y);
6037 }
6038 // x == 0 ? 0 : umin (..., x, ...) -> umin_seq(x, umin (...))
6039 // x == 0 ? 0 : umin_seq(..., x, ...) -> umin_seq(x, umin_seq(...))
6040 // x == 0 ? 0 : umin (..., umin_seq(..., x, ...), ...)
6041 // -> umin_seq(x, umin (..., umin_seq(...), ...))
6042 if (isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero() &&
6043 isa<ConstantInt>(Val: TrueVal) && cast<ConstantInt>(Val: TrueVal)->isZero()) {
6044 const SCEV *X = getSCEV(V: LHS);
6045 while (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: X))
6046 X = ZExt->getOperand();
6047 if (getTypeSizeInBits(Ty: X->getType()) <= getTypeSizeInBits(Ty)) {
6048 const SCEV *FalseValExpr = getSCEV(V: FalseVal);
6049 if (SCEVMinMaxExprContains(Root: FalseValExpr, OperandToFind: X, RootKind: scSequentialUMinExpr))
6050 return getUMinExpr(LHS: getNoopOrZeroExtend(V: X, Ty), RHS: FalseValExpr,
6051 /*Sequential=*/true);
6052 }
6053 }
6054 break;
6055 default:
6056 break;
6057 }
6058
6059 return std::nullopt;
6060}
6061
6062static std::optional<const SCEV *>
6063createNodeForSelectViaUMinSeq(ScalarEvolution *SE, const SCEV *CondExpr,
6064 const SCEV *TrueExpr, const SCEV *FalseExpr) {
6065 assert(CondExpr->getType()->isIntegerTy(1) &&
6066 TrueExpr->getType() == FalseExpr->getType() &&
6067 TrueExpr->getType()->isIntegerTy(1) &&
6068 "Unexpected operands of a select.");
6069
6070 // i1 cond ? i1 x : i1 C --> C + (i1 cond ? (i1 x - i1 C) : i1 0)
6071 // --> C + (umin_seq cond, x - C)
6072 //
6073 // i1 cond ? i1 C : i1 x --> C + (i1 cond ? i1 0 : (i1 x - i1 C))
6074 // --> C + (i1 ~cond ? (i1 x - i1 C) : i1 0)
6075 // --> C + (umin_seq ~cond, x - C)
6076
6077 // FIXME: while we can't legally model the case where both of the hands
6078 // are fully variable, we only require that the *difference* is constant.
6079 if (!isa<SCEVConstant>(Val: TrueExpr) && !isa<SCEVConstant>(Val: FalseExpr))
6080 return std::nullopt;
6081
6082 const SCEV *X, *C;
6083 if (isa<SCEVConstant>(Val: TrueExpr)) {
6084 CondExpr = SE->getNotSCEV(V: CondExpr);
6085 X = FalseExpr;
6086 C = TrueExpr;
6087 } else {
6088 X = TrueExpr;
6089 C = FalseExpr;
6090 }
6091 return SE->getAddExpr(LHS: C, RHS: SE->getUMinExpr(LHS: CondExpr, RHS: SE->getMinusSCEV(LHS: X, RHS: C),
6092 /*Sequential=*/true));
6093}
6094
6095static std::optional<const SCEV *>
6096createNodeForSelectViaUMinSeq(ScalarEvolution *SE, Value *Cond, Value *TrueVal,
6097 Value *FalseVal) {
6098 if (!isa<ConstantInt>(Val: TrueVal) && !isa<ConstantInt>(Val: FalseVal))
6099 return std::nullopt;
6100
6101 const auto *SECond = SE->getSCEV(V: Cond);
6102 const auto *SETrue = SE->getSCEV(V: TrueVal);
6103 const auto *SEFalse = SE->getSCEV(V: FalseVal);
6104 return createNodeForSelectViaUMinSeq(SE, CondExpr: SECond, TrueExpr: SETrue, FalseExpr: SEFalse);
6105}
6106
6107const SCEV *ScalarEvolution::createNodeForSelectOrPHIViaUMinSeq(
6108 Value *V, Value *Cond, Value *TrueVal, Value *FalseVal) {
6109 assert(Cond->getType()->isIntegerTy(1) && "Select condition is not an i1?");
6110 assert(TrueVal->getType() == FalseVal->getType() &&
6111 V->getType() == TrueVal->getType() &&
6112 "Types of select hands and of the result must match.");
6113
6114 // For now, only deal with i1-typed `select`s.
6115 if (!V->getType()->isIntegerTy(BitWidth: 1))
6116 return getUnknown(V);
6117
6118 if (std::optional<const SCEV *> S =
6119 createNodeForSelectViaUMinSeq(SE: this, Cond, TrueVal, FalseVal))
6120 return *S;
6121
6122 return getUnknown(V);
6123}
6124
6125const SCEV *ScalarEvolution::createNodeForSelectOrPHI(Value *V, Value *Cond,
6126 Value *TrueVal,
6127 Value *FalseVal) {
6128 // Handle "constant" branch or select. This can occur for instance when a
6129 // loop pass transforms an inner loop and moves on to process the outer loop.
6130 if (auto *CI = dyn_cast<ConstantInt>(Val: Cond))
6131 return getSCEV(V: CI->isOne() ? TrueVal : FalseVal);
6132
6133 if (auto *I = dyn_cast<Instruction>(Val: V)) {
6134 if (auto *ICI = dyn_cast<ICmpInst>(Val: Cond)) {
6135 if (std::optional<const SCEV *> S =
6136 createNodeForSelectOrPHIInstWithICmpInstCond(Ty: I->getType(), Cond: ICI,
6137 TrueVal, FalseVal))
6138 return *S;
6139 }
6140 }
6141
6142 return createNodeForSelectOrPHIViaUMinSeq(V, Cond, TrueVal, FalseVal);
6143}
6144
6145/// Expand GEP instructions into add and multiply operations. This allows them
6146/// to be analyzed by regular SCEV code.
6147const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) {
6148 assert(GEP->getSourceElementType()->isSized() &&
6149 "GEP source element type must be sized");
6150
6151 SmallVector<SCEVUse, 4> IndexExprs;
6152 for (Value *Index : GEP->indices())
6153 IndexExprs.push_back(Elt: getSCEV(V: Index));
6154 return getGEPExpr(GEP, IndexExprs);
6155}
6156
6157APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S,
6158 const Instruction *CtxI) {
6159 uint64_t BitWidth = getTypeSizeInBits(Ty: S->getType());
6160 auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) {
6161 return TrailingZeros >= BitWidth
6162 ? APInt::getZero(numBits: BitWidth)
6163 : APInt::getOneBitSet(numBits: BitWidth, BitNo: TrailingZeros);
6164 };
6165 auto GetGCDMultiple = [this, CtxI](const SCEVNAryExpr *N) {
6166 // The result is GCD of all operands results.
6167 APInt Res = getConstantMultiple(S: N->getOperand(i: 0), CtxI);
6168 for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I)
6169 Res = APIntOps::GreatestCommonDivisor(
6170 A: Res, B: getConstantMultiple(S: N->getOperand(i: I), CtxI));
6171 return Res;
6172 };
6173
6174 switch (S->getSCEVType()) {
6175 case scConstant:
6176 return cast<SCEVConstant>(Val: S)->getAPInt();
6177 case scPtrToAddr:
6178 return getConstantMultiple(S: cast<SCEVCastExpr>(Val: S)->getOperand());
6179 case scUDivExpr:
6180 case scVScale:
6181 return APInt(BitWidth, 1);
6182 case scTruncate: {
6183 // Only multiples that are a power of 2 will hold after truncation.
6184 const SCEVTruncateExpr *T = cast<SCEVTruncateExpr>(Val: S);
6185 uint32_t TZ = getMinTrailingZeros(S: T->getOperand(), CtxI);
6186 return GetShiftedByZeros(TZ);
6187 }
6188 case scZeroExtend: {
6189 const SCEVZeroExtendExpr *Z = cast<SCEVZeroExtendExpr>(Val: S);
6190 return getConstantMultiple(S: Z->getOperand(), CtxI).zext(width: BitWidth);
6191 }
6192 case scSignExtend: {
6193 // Only multiples that are a power of 2 will hold after sext.
6194 const SCEVSignExtendExpr *E = cast<SCEVSignExtendExpr>(Val: S);
6195 uint32_t TZ = getMinTrailingZeros(S: E->getOperand(), CtxI);
6196 return GetShiftedByZeros(TZ);
6197 }
6198 case scMulExpr: {
6199 const SCEVMulExpr *M = cast<SCEVMulExpr>(Val: S);
6200 if (M->hasNoUnsignedWrap()) {
6201 // The result is the product of all operand results.
6202 APInt Res = getConstantMultiple(S: M->getOperand(i: 0), CtxI);
6203 for (const SCEV *Operand : M->operands().drop_front())
6204 Res = Res * getConstantMultiple(S: Operand, CtxI);
6205 return Res;
6206 }
6207
6208 // If there are no wrap guarentees, find the trailing zeros, which is the
6209 // sum of trailing zeros for all its operands.
6210 uint32_t TZ = 0;
6211 for (const SCEV *Operand : M->operands())
6212 TZ += getMinTrailingZeros(S: Operand, CtxI);
6213 return GetShiftedByZeros(TZ);
6214 }
6215 case scAddExpr:
6216 case scAddRecExpr: {
6217 const SCEVNAryExpr *N = cast<SCEVNAryExpr>(Val: S);
6218 if (N->hasNoUnsignedWrap())
6219 return GetGCDMultiple(N);
6220 // Find the trailing bits, which is the minimum of its operands.
6221 uint32_t TZ = getMinTrailingZeros(S: N->getOperand(i: 0), CtxI);
6222 for (const SCEV *Operand : N->operands().drop_front())
6223 TZ = std::min(a: TZ, b: getMinTrailingZeros(S: Operand, CtxI));
6224 return GetShiftedByZeros(TZ);
6225 }
6226 case scUMaxExpr:
6227 case scSMaxExpr:
6228 case scUMinExpr:
6229 case scSMinExpr:
6230 case scSequentialUMinExpr:
6231 return GetGCDMultiple(cast<SCEVNAryExpr>(Val: S));
6232 case scUnknown: {
6233 // Ask ValueTracking for known bits. SCEVUnknown only become available at
6234 // the point their underlying IR instruction has been defined. If CtxI was
6235 // not provided, use:
6236 // * the first instruction in the entry block if it is an argument
6237 // * the instruction itself otherwise.
6238 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6239 if (!CtxI) {
6240 if (isa<Argument>(Val: U->getValue()))
6241 CtxI = &*F.getEntryBlock().begin();
6242 else if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
6243 CtxI = I;
6244 }
6245 unsigned Known =
6246 computeKnownBits(V: U->getValue(),
6247 Q: SimplifyQuery(getDataLayout(), &DT, &AC, CtxI)
6248 .allowEphemerals(AllowEphemerals: true))
6249 .countMinTrailingZeros();
6250 return GetShiftedByZeros(Known);
6251 }
6252 case scCouldNotCompute:
6253 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6254 }
6255 llvm_unreachable("Unknown SCEV kind!");
6256}
6257
6258APInt ScalarEvolution::getConstantMultiple(const SCEV *S,
6259 const Instruction *CtxI) {
6260 // Skip looking up and updating the cache if there is a context instruction,
6261 // as the result will only be valid in the specified context.
6262 if (CtxI)
6263 return getConstantMultipleImpl(S, CtxI);
6264
6265 auto I = ConstantMultipleCache.find(Val: S);
6266 if (I != ConstantMultipleCache.end())
6267 return I->second;
6268
6269 APInt Result = getConstantMultipleImpl(S, CtxI);
6270 auto InsertPair = ConstantMultipleCache.insert(KV: {S, Result});
6271 assert(InsertPair.second && "Should insert a new key");
6272 return InsertPair.first->second;
6273}
6274
6275APInt ScalarEvolution::getNonZeroConstantMultiple(const SCEV *S) {
6276 APInt Multiple = getConstantMultiple(S);
6277 return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple;
6278}
6279
6280uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S,
6281 const Instruction *CtxI) {
6282 return std::min(a: getConstantMultiple(S, CtxI).countTrailingZeros(),
6283 b: (unsigned)getTypeSizeInBits(Ty: S->getType()));
6284}
6285
6286/// Helper method to assign a range to V from metadata present in the IR.
6287static std::optional<ConstantRange> GetRangeFromMetadata(Value *V) {
6288 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
6289 if (MDNode *MD = I->getMetadata(KindID: LLVMContext::MD_range))
6290 return getConstantRangeFromMetadata(RangeMD: *MD);
6291 if (const auto *CB = dyn_cast<CallBase>(Val: V))
6292 if (std::optional<ConstantRange> Range = CB->getRange())
6293 return Range;
6294 }
6295 if (auto *A = dyn_cast<Argument>(Val: V))
6296 if (std::optional<ConstantRange> Range = A->getRange())
6297 return Range;
6298
6299 return std::nullopt;
6300}
6301
6302void ScalarEvolution::setNoWrapFlags(SCEVAddRecExpr *AddRec, SCEVFlags Flags) {
6303 SCEVFlags NWFlags = Flags & SCEV::FlagsNoWrapMask;
6304 if (AddRec->getNoWrapFlags(Mask: NWFlags) != NWFlags) {
6305 AddRec->setNoWrapFlags(NWFlags);
6306 UnsignedRanges.erase(Val: AddRec);
6307 SignedRanges.erase(Val: AddRec);
6308 ConstantMultipleCache.erase(Val: AddRec);
6309 }
6310}
6311
6312ConstantRange ScalarEvolution::
6313getRangeForUnknownRecurrence(const SCEVUnknown *U) {
6314 const DataLayout &DL = getDataLayout();
6315
6316 unsigned BitWidth = getTypeSizeInBits(Ty: U->getType());
6317 const ConstantRange FullSet(BitWidth, /*isFullSet=*/true);
6318
6319 // Match a simple recurrence of the form: <start, ShiftOp, Step>, and then
6320 // use information about the trip count to improve our available range. Note
6321 // that the trip count independent cases are already handled by known bits.
6322 // WARNING: The definition of recurrence used here is subtly different than
6323 // the one used by AddRec (and thus most of this file). Step is allowed to
6324 // be arbitrarily loop varying here, where AddRec allows only loop invariant
6325 // and other addrecs in the same loop (for non-affine addrecs). The code
6326 // below intentionally handles the case where step is not loop invariant.
6327 auto *P = dyn_cast<PHINode>(Val: U->getValue());
6328 if (!P)
6329 return FullSet;
6330
6331 // Make sure that no Phi input comes from an unreachable block. Otherwise,
6332 // even the values that are not available in these blocks may come from them,
6333 // and this leads to false-positive recurrence test.
6334 for (auto *Pred : predecessors(BB: P->getParent()))
6335 if (!DT.isReachableFromEntry(A: Pred))
6336 return FullSet;
6337
6338 BinaryOperator *BO;
6339 Value *Start, *Step;
6340 if (!matchSimpleRecurrence(P, BO, Start, Step))
6341 return FullSet;
6342
6343 // If we found a recurrence in reachable code, we must be in a loop. Note
6344 // that BO might be in some subloop of L, and that's completely okay.
6345 auto *L = LI.getLoopFor(BB: P->getParent());
6346 assert(L && L->getHeader() == P->getParent());
6347 if (!L->contains(BB: BO->getParent()))
6348 // NOTE: This bailout should be an assert instead. However, asserting
6349 // the condition here exposes a case where LoopFusion is querying SCEV
6350 // with malformed loop information during the midst of the transform.
6351 // There doesn't appear to be an obvious fix, so for the moment bailout
6352 // until the caller issue can be fixed. PR49566 tracks the bug.
6353 return FullSet;
6354
6355 // TODO: Extend to other opcodes such as mul, and div
6356 switch (BO->getOpcode()) {
6357 default:
6358 return FullSet;
6359 case Instruction::AShr:
6360 case Instruction::LShr:
6361 case Instruction::Shl:
6362 break;
6363 };
6364
6365 if (BO->getOperand(i_nocapture: 0) != P)
6366 // TODO: Handle the power function forms some day.
6367 return FullSet;
6368
6369 unsigned TC = getSmallConstantMaxTripCount(L);
6370 if (!TC || TC >= BitWidth)
6371 return FullSet;
6372
6373 auto KnownStart = computeKnownBits(V: Start, DL, AC: &AC, CtxI: nullptr, DT: &DT);
6374 auto KnownStep = computeKnownBits(V: Step, DL, AC: &AC, CtxI: nullptr, DT: &DT);
6375 assert(KnownStart.getBitWidth() == BitWidth &&
6376 KnownStep.getBitWidth() == BitWidth);
6377
6378 // Compute total shift amount, being careful of overflow and bitwidths.
6379 auto MaxShiftAmt = KnownStep.getMaxValue();
6380 APInt TCAP(BitWidth, TC-1);
6381 bool Overflow = false;
6382 auto TotalShift = MaxShiftAmt.umul_ov(RHS: TCAP, Overflow);
6383 if (Overflow)
6384 return FullSet;
6385
6386 switch (BO->getOpcode()) {
6387 default:
6388 llvm_unreachable("filtered out above");
6389 case Instruction::AShr: {
6390 // For each ashr, three cases:
6391 // shift = 0 => unchanged value
6392 // saturation => 0 or -1
6393 // other => a value closer to zero (of the same sign)
6394 // Thus, the end value is closer to zero than the start.
6395 auto KnownEnd = KnownBits::ashr(LHS: KnownStart,
6396 RHS: KnownBits::makeConstant(C: TotalShift));
6397 if (KnownStart.isNonNegative())
6398 // Analogous to lshr (simply not yet canonicalized)
6399 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6400 Upper: KnownStart.getMaxValue() + 1);
6401 if (KnownStart.isNegative())
6402 // End >=u Start && End <=s Start
6403 return ConstantRange::getNonEmpty(Lower: KnownStart.getMinValue(),
6404 Upper: KnownEnd.getMaxValue() + 1);
6405 break;
6406 }
6407 case Instruction::LShr: {
6408 // For each lshr, three cases:
6409 // shift = 0 => unchanged value
6410 // saturation => 0
6411 // other => a smaller positive number
6412 // Thus, the low end of the unsigned range is the last value produced.
6413 auto KnownEnd = KnownBits::lshr(LHS: KnownStart,
6414 RHS: KnownBits::makeConstant(C: TotalShift));
6415 return ConstantRange::getNonEmpty(Lower: KnownEnd.getMinValue(),
6416 Upper: KnownStart.getMaxValue() + 1);
6417 }
6418 case Instruction::Shl: {
6419 // Iff no bits are shifted out, value increases on every shift.
6420 auto KnownEnd = KnownBits::shl(LHS: KnownStart,
6421 RHS: KnownBits::makeConstant(C: TotalShift));
6422 if (TotalShift.ult(RHS: KnownStart.countMinLeadingZeros()))
6423 return ConstantRange(KnownStart.getMinValue(),
6424 KnownEnd.getMaxValue() + 1);
6425 break;
6426 }
6427 };
6428 return FullSet;
6429}
6430
6431// The goal of this function is to check if recursively visiting the operands
6432// of this PHI might lead to an infinite loop. If we do see such a loop,
6433// there's no good way to break it, so we avoid analyzing such cases.
6434//
6435// getRangeRef previously used a visited set to avoid infinite loops, but this
6436// caused other issues: the result was dependent on the order of getRangeRef
6437// calls, and the interaction with createSCEVIter could cause a stack overflow
6438// in some cases (see issue #148253).
6439//
6440// FIXME: The way this is implemented is overly conservative; this checks
6441// for a few obviously safe patterns, but anything that doesn't lead to
6442// recursion is fine.
6443static bool RangeRefPHIAllowedOperands(DominatorTree &DT, PHINode *PHI) {
6444 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
6445 if (getOperandsForSelectLikePHI(DT, PN: PHI, Cond, LHS, RHS))
6446 return true;
6447
6448 if (all_of(Range: PHI->operands(),
6449 P: [&](Value *Operand) { return DT.dominates(Def: Operand, User: PHI); }))
6450 return true;
6451
6452 return false;
6453}
6454
6455const ConstantRange &
6456ScalarEvolution::getRangeRefIter(const SCEV *S,
6457 ScalarEvolution::RangeSignHint SignHint) {
6458 DenseMap<const SCEV *, ConstantRange> &Cache =
6459 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6460 : SignedRanges;
6461 SmallVector<SCEVUse> WorkList;
6462 SmallPtrSet<const SCEV *, 8> Seen;
6463
6464 // Add Expr to the worklist, if Expr is either an N-ary expression or a
6465 // SCEVUnknown PHI node.
6466 auto AddToWorklist = [&WorkList, &Seen, &Cache](const SCEV *Expr) {
6467 if (!Seen.insert(Ptr: Expr).second)
6468 return;
6469 if (Cache.contains(Val: Expr))
6470 return;
6471 switch (Expr->getSCEVType()) {
6472 case scUnknown:
6473 if (!isa<PHINode>(Val: cast<SCEVUnknown>(Val: Expr)->getValue()))
6474 break;
6475 [[fallthrough]];
6476 case scConstant:
6477 case scVScale:
6478 case scTruncate:
6479 case scZeroExtend:
6480 case scSignExtend:
6481 case scPtrToAddr:
6482 case scAddExpr:
6483 case scMulExpr:
6484 case scUDivExpr:
6485 case scAddRecExpr:
6486 case scUMaxExpr:
6487 case scSMaxExpr:
6488 case scUMinExpr:
6489 case scSMinExpr:
6490 case scSequentialUMinExpr:
6491 WorkList.push_back(Elt: Expr);
6492 break;
6493 case scCouldNotCompute:
6494 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6495 }
6496 };
6497 AddToWorklist(S);
6498
6499 // Build worklist by queuing operands of N-ary expressions and phi nodes.
6500 for (unsigned I = 0; I != WorkList.size(); ++I) {
6501 const SCEV *P = WorkList[I];
6502 auto *UnknownS = dyn_cast<SCEVUnknown>(Val: P);
6503 // If it is not a `SCEVUnknown`, just recurse into operands.
6504 if (!UnknownS) {
6505 for (const SCEV *Op : P->operands())
6506 AddToWorklist(Op);
6507 continue;
6508 }
6509 // `SCEVUnknown`'s require special treatment.
6510 if (PHINode *P = dyn_cast<PHINode>(Val: UnknownS->getValue())) {
6511 if (!RangeRefPHIAllowedOperands(DT, PHI: P))
6512 continue;
6513 for (auto &Op : reverse(C: P->operands()))
6514 AddToWorklist(getSCEV(V: Op));
6515 }
6516 }
6517
6518 if (!WorkList.empty()) {
6519 // Use getRangeRef to compute ranges for items in the worklist in reverse
6520 // order. This will force ranges for earlier operands to be computed before
6521 // their users in most cases.
6522 for (const SCEV *P : reverse(C: drop_begin(RangeOrContainer&: WorkList))) {
6523 getRangeRef(S: P, Hint: SignHint);
6524 }
6525 }
6526
6527 return getRangeRef(S, Hint: SignHint, Depth: 0);
6528}
6529
6530const APInt *ScalarEvolution::getConstantAPIntOrNull(const SCEV *S) {
6531 if (const auto *C = dyn_cast<SCEVConstant>(Val: S))
6532 return &C->getAPInt();
6533 return nullptr;
6534}
6535
6536/// Determine the range for a particular SCEV. If SignHint is
6537/// HINT_RANGE_UNSIGNED (resp. HINT_RANGE_SIGNED) then getRange prefers ranges
6538/// with a "cleaner" unsigned (resp. signed) representation.
6539const ConstantRange &ScalarEvolution::getRangeRef(
6540 const SCEV *S, ScalarEvolution::RangeSignHint SignHint, unsigned Depth) {
6541 DenseMap<const SCEV *, ConstantRange> &Cache =
6542 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges
6543 : SignedRanges;
6544 ConstantRange::PreferredRangeType RangeType =
6545 SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? ConstantRange::Unsigned
6546 : ConstantRange::Signed;
6547
6548 // See if we've computed this range already.
6549 auto I = Cache.find(Val: S);
6550 if (I != Cache.end())
6551 return I->second;
6552
6553 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: S))
6554 return setRange(S: C, Hint: SignHint, CR: ConstantRange(C->getAPInt()));
6555
6556 // Switch to iteratively computing the range for S, if it is part of a deeply
6557 // nested expression.
6558 if (Depth > Opts.scev_range_iter_threshold)
6559 return getRangeRefIter(S, SignHint);
6560
6561 unsigned BitWidth = getTypeSizeInBits(Ty: S->getType());
6562 ConstantRange ConservativeResult(BitWidth, /*isFullSet=*/true);
6563 using OBO = OverflowingBinaryOperator;
6564
6565 // If the value has known zeros, the maximum value will have those known zeros
6566 // as well.
6567 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED) {
6568 APInt Multiple = getNonZeroConstantMultiple(S);
6569 APInt Remainder = APInt::getMaxValue(numBits: BitWidth).urem(RHS: Multiple);
6570 if (!Remainder.isZero())
6571 ConservativeResult =
6572 ConstantRange(APInt::getMinValue(numBits: BitWidth),
6573 APInt::getMaxValue(numBits: BitWidth) - Remainder + 1);
6574 }
6575 else {
6576 uint32_t TZ = getMinTrailingZeros(S);
6577 if (TZ != 0) {
6578 ConservativeResult = ConstantRange(
6579 APInt::getSignedMinValue(numBits: BitWidth),
6580 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: TZ).shl(shiftAmt: TZ) + 1);
6581 }
6582 }
6583
6584 switch (S->getSCEVType()) {
6585 case scConstant:
6586 llvm_unreachable("Already handled above.");
6587 case scVScale:
6588 return setRange(S, Hint: SignHint, CR: getVScaleRange(F: &F, BitWidth));
6589 case scTruncate: {
6590 const SCEVTruncateExpr *Trunc = cast<SCEVTruncateExpr>(Val: S);
6591 ConstantRange X = getRangeRef(S: Trunc->getOperand(), SignHint, Depth: Depth + 1);
6592 return setRange(
6593 S: Trunc, Hint: SignHint,
6594 CR: ConservativeResult.intersectWith(CR: X.truncate(BitWidth), Type: RangeType));
6595 }
6596 case scZeroExtend: {
6597 const SCEVZeroExtendExpr *ZExt = cast<SCEVZeroExtendExpr>(Val: S);
6598 ConstantRange X = getRangeRef(S: ZExt->getOperand(), SignHint, Depth: Depth + 1);
6599 return setRange(
6600 S: ZExt, Hint: SignHint,
6601 CR: ConservativeResult.intersectWith(CR: X.zeroExtend(BitWidth), Type: RangeType));
6602 }
6603 case scSignExtend: {
6604 const SCEVSignExtendExpr *SExt = cast<SCEVSignExtendExpr>(Val: S);
6605 ConstantRange X = getRangeRef(S: SExt->getOperand(), SignHint, Depth: Depth + 1);
6606 return setRange(
6607 S: SExt, Hint: SignHint,
6608 CR: ConservativeResult.intersectWith(CR: X.signExtend(BitWidth), Type: RangeType));
6609 }
6610 case scPtrToAddr: {
6611 const SCEVCastExpr *Cast = cast<SCEVCastExpr>(Val: S);
6612 ConstantRange X = getRangeRef(S: Cast->getOperand(), SignHint, Depth: Depth + 1);
6613 return setRange(S: Cast, Hint: SignHint, CR: X);
6614 }
6615 case scAddExpr: {
6616 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: S);
6617 // Check if this is a URem pattern: A - (A / B) * B, which is always < B.
6618 const SCEV *URemLHS = nullptr, *URemRHS = nullptr;
6619 if (SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED &&
6620 match(S, P: m_scev_URem(LHS: m_SCEV(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE&: *this))) {
6621 ConstantRange LHSRange = getRangeRef(S: URemLHS, SignHint, Depth: Depth + 1);
6622 ConstantRange RHSRange = getRangeRef(S: URemRHS, SignHint, Depth: Depth + 1);
6623 ConservativeResult =
6624 ConservativeResult.intersectWith(CR: LHSRange.urem(Other: RHSRange), Type: RangeType);
6625 }
6626 ConstantRange X = getRangeRef(S: Add->getOperand(i: 0), SignHint, Depth: Depth + 1);
6627 unsigned WrapType = OBO::AnyWrap;
6628 if (Add->hasNoSignedWrap())
6629 WrapType |= OBO::NoSignedWrap;
6630 if (Add->hasNoUnsignedWrap())
6631 WrapType |= OBO::NoUnsignedWrap;
6632 for (const SCEV *Op : drop_begin(RangeOrContainer: Add->operands()))
6633 X = X.addWithNoWrap(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1), NoWrapKind: WrapType,
6634 RangeType);
6635 return setRange(S: Add, Hint: SignHint,
6636 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6637 }
6638 case scMulExpr: {
6639 const SCEVMulExpr *Mul = cast<SCEVMulExpr>(Val: S);
6640 ConstantRange X = getRangeRef(S: Mul->getOperand(i: 0), SignHint, Depth: Depth + 1);
6641 for (const SCEV *Op : drop_begin(RangeOrContainer: Mul->operands()))
6642 X = X.multiply(Other: getRangeRef(S: Op, SignHint, Depth: Depth + 1));
6643 return setRange(S: Mul, Hint: SignHint,
6644 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6645 }
6646 case scUDivExpr: {
6647 const SCEVUDivExpr *UDiv = cast<SCEVUDivExpr>(Val: S);
6648 ConstantRange X = getRangeRef(S: UDiv->getLHS(), SignHint, Depth: Depth + 1);
6649 ConstantRange Y = getRangeRef(S: UDiv->getRHS(), SignHint, Depth: Depth + 1);
6650 return setRange(S: UDiv, Hint: SignHint,
6651 CR: ConservativeResult.intersectWith(CR: X.udiv(Other: Y), Type: RangeType));
6652 }
6653 case scAddRecExpr: {
6654 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: S);
6655 // If there's no unsigned wrap, the value will never be less than its
6656 // initial value.
6657 if (AddRec->hasNoUnsignedWrap()) {
6658 APInt UnsignedMinValue = getUnsignedRangeMin(S: AddRec->getStart());
6659 if (!UnsignedMinValue.isZero())
6660 ConservativeResult = ConservativeResult.intersectWith(
6661 CR: ConstantRange(UnsignedMinValue, APInt(BitWidth, 0)), Type: RangeType);
6662 }
6663
6664 // If there's no signed wrap, and all the operands except initial value have
6665 // the same sign or zero, the value won't ever be:
6666 // 1: smaller than initial value if operands are non negative,
6667 // 2: bigger than initial value if operands are non positive.
6668 // For both cases, value can not cross signed min/max boundary.
6669 if (AddRec->hasNoSignedWrap()) {
6670 bool AllNonNeg = true;
6671 bool AllNonPos = true;
6672 for (unsigned i = 1, e = AddRec->getNumOperands(); i != e; ++i) {
6673 if (!isKnownNonNegative(S: AddRec->getOperand(i)))
6674 AllNonNeg = false;
6675 if (!isKnownNonPositive(S: AddRec->getOperand(i)))
6676 AllNonPos = false;
6677 }
6678 if (AllNonNeg)
6679 ConservativeResult = ConservativeResult.intersectWith(
6680 CR: ConstantRange::getNonEmpty(Lower: getSignedRangeMin(S: AddRec->getStart()),
6681 Upper: APInt::getSignedMinValue(numBits: BitWidth)),
6682 Type: RangeType);
6683 else if (AllNonPos)
6684 ConservativeResult = ConservativeResult.intersectWith(
6685 CR: ConstantRange::getNonEmpty(Lower: APInt::getSignedMinValue(numBits: BitWidth),
6686 Upper: getSignedRangeMax(S: AddRec->getStart()) +
6687 1),
6688 Type: RangeType);
6689 }
6690
6691 // TODO: non-affine addrec
6692 if (AddRec->isAffine()) {
6693 const SCEV *MaxBEScev =
6694 getConstantMaxBackedgeTakenCount(L: AddRec->getLoop());
6695 if (!isa<SCEVCouldNotCompute>(Val: MaxBEScev)) {
6696 APInt MaxBECount = cast<SCEVConstant>(Val: MaxBEScev)->getAPInt();
6697
6698 // Adjust MaxBECount to the same bitwidth as AddRec. We can truncate if
6699 // MaxBECount's active bits are all <= AddRec's bit width.
6700 if (MaxBECount.getBitWidth() > BitWidth &&
6701 MaxBECount.getActiveBits() <= BitWidth)
6702 MaxBECount = MaxBECount.trunc(width: BitWidth);
6703 else if (MaxBECount.getBitWidth() < BitWidth)
6704 MaxBECount = MaxBECount.zext(width: BitWidth);
6705
6706 if (MaxBECount.getBitWidth() == BitWidth) {
6707 auto [RangeFromAffine, Flags] = getRangeForAffineAR(
6708 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
6709 ConservativeResult =
6710 ConservativeResult.intersectWith(CR: RangeFromAffine, Type: RangeType);
6711 const_cast<SCEVAddRecExpr *>(AddRec)->setNoWrapFlags(Flags);
6712
6713 auto RangeFromFactoring = getRangeViaFactoring(
6714 Start: AddRec->getStart(), Step: AddRec->getStepRecurrence(SE&: *this), MaxBECount);
6715 ConservativeResult =
6716 ConservativeResult.intersectWith(CR: RangeFromFactoring, Type: RangeType);
6717 }
6718 }
6719
6720 // Now try symbolic BE count and more powerful methods.
6721 if (Opts.scalar_evolution_use_expensive_range_sharpening) {
6722 const SCEV *SymbolicMaxBECount =
6723 getSymbolicMaxBackedgeTakenCount(L: AddRec->getLoop());
6724 if (!isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount) &&
6725 getTypeSizeInBits(Ty: MaxBEScev->getType()) <= BitWidth &&
6726 AddRec->hasNoSelfWrap()) {
6727 auto RangeFromAffineNew = getRangeForAffineNoSelfWrappingAR(
6728 AddRec, MaxBECount: SymbolicMaxBECount, BitWidth, SignHint);
6729 ConservativeResult =
6730 ConservativeResult.intersectWith(CR: RangeFromAffineNew, Type: RangeType);
6731 }
6732 }
6733 }
6734
6735 return setRange(S: AddRec, Hint: SignHint, CR: std::move(ConservativeResult));
6736 }
6737 case scUMaxExpr:
6738 case scSMaxExpr:
6739 case scUMinExpr:
6740 case scSMinExpr:
6741 case scSequentialUMinExpr: {
6742 Intrinsic::ID ID;
6743 switch (S->getSCEVType()) {
6744 case scUMaxExpr:
6745 ID = Intrinsic::umax;
6746 break;
6747 case scSMaxExpr:
6748 ID = Intrinsic::smax;
6749 break;
6750 case scUMinExpr:
6751 case scSequentialUMinExpr:
6752 ID = Intrinsic::umin;
6753 break;
6754 case scSMinExpr:
6755 ID = Intrinsic::smin;
6756 break;
6757 default:
6758 llvm_unreachable("Unknown SCEVMinMaxExpr/SCEVSequentialMinMaxExpr.");
6759 }
6760
6761 const auto *NAry = cast<SCEVNAryExpr>(Val: S);
6762 ConstantRange X = getRangeRef(S: NAry->getOperand(i: 0), SignHint, Depth: Depth + 1);
6763 for (unsigned i = 1, e = NAry->getNumOperands(); i != e; ++i)
6764 X = X.intrinsic(
6765 IntrinsicID: ID, Ops: {X, getRangeRef(S: NAry->getOperand(i), SignHint, Depth: Depth + 1)});
6766 return setRange(S, Hint: SignHint,
6767 CR: ConservativeResult.intersectWith(CR: X, Type: RangeType));
6768 }
6769 case scUnknown: {
6770 const SCEVUnknown *U = cast<SCEVUnknown>(Val: S);
6771 Value *V = U->getValue();
6772
6773 // Check if the IR explicitly contains !range metadata.
6774 std::optional<ConstantRange> MDRange = GetRangeFromMetadata(V);
6775 if (MDRange)
6776 ConservativeResult =
6777 ConservativeResult.intersectWith(CR: *MDRange, Type: RangeType);
6778
6779 // Use facts about recurrences in the underlying IR. Note that add
6780 // recurrences are AddRecExprs and thus don't hit this path. This
6781 // primarily handles shift recurrences.
6782 auto CR = getRangeForUnknownRecurrence(U);
6783 ConservativeResult = ConservativeResult.intersectWith(CR);
6784
6785 // See if ValueTracking can give us a useful range.
6786 const DataLayout &DL = getDataLayout();
6787 KnownBits Known = computeKnownBits(V, DL, AC: &AC, CtxI: nullptr, DT: &DT);
6788 if (Known.getBitWidth() != BitWidth)
6789 Known = Known.zextOrTrunc(BitWidth);
6790
6791 // ValueTracking may be able to compute a tighter result for the number of
6792 // sign bits than for the value of those sign bits.
6793 unsigned NS = ComputeNumSignBits(Op: V, DL, AC: &AC, CtxI: nullptr, DT: &DT);
6794 if (U->getType()->isPointerTy()) {
6795 // NS counts the sign bits of the whole pointer; drop those above the
6796 // index bits.
6797 unsigned PtrIdxDiff =
6798 DL.getPointerTypeSizeInBits(U->getType()) - BitWidth;
6799 NS = NS > PtrIdxDiff ? NS - PtrIdxDiff : 1;
6800 }
6801
6802 if (NS > 1) {
6803 // If we know any of the sign bits, we know all of the sign bits.
6804 if (!Known.Zero.getHiBits(numBits: NS).isZero())
6805 Known.Zero.setHighBits(NS);
6806 if (!Known.One.getHiBits(numBits: NS).isZero())
6807 Known.One.setHighBits(NS);
6808 }
6809
6810 if (Known.getMinValue() != Known.getMaxValue() + 1)
6811 ConservativeResult = ConservativeResult.intersectWith(
6812 CR: ConstantRange(Known.getMinValue(), Known.getMaxValue() + 1),
6813 Type: RangeType);
6814 if (NS > 1)
6815 ConservativeResult = ConservativeResult.intersectWith(
6816 CR: ConstantRange(APInt::getSignedMinValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1),
6817 APInt::getSignedMaxValue(numBits: BitWidth).ashr(ShiftAmt: NS - 1) + 1),
6818 Type: RangeType);
6819
6820 if (U->getType()->isPointerTy() && SignHint == HINT_RANGE_UNSIGNED) {
6821 // Strengthen the range if the underlying IR value is a
6822 // global/alloca/heap allocation using the size of the object.
6823 bool CanBeNull;
6824 uint64_t DerefBytes = V->getPointerDereferenceableBytes(
6825 DL, CanBeNull, /*CanBeFreed=*/nullptr);
6826 if (DerefBytes > 1 && isUIntN(N: BitWidth, x: DerefBytes)) {
6827 // The highest address the object can start is DerefBytes bytes before
6828 // the end (unsigned max value). If this value is not a multiple of the
6829 // alignment, the last possible start value is the next lowest multiple
6830 // of the alignment. Note: The computations below cannot overflow,
6831 // because if they would there's no possible start address for the
6832 // object.
6833 APInt MaxVal =
6834 APInt::getMaxValue(numBits: BitWidth) - APInt(BitWidth, DerefBytes);
6835 uint64_t Align = U->getValue()->getPointerAlignment(DL).value();
6836 uint64_t Rem = MaxVal.urem(RHS: Align);
6837 MaxVal -= APInt(BitWidth, Rem);
6838 APInt MinVal = APInt::getZero(numBits: BitWidth);
6839 if (llvm::isKnownNonZero(V, Q: DL))
6840 MinVal = Align;
6841 ConservativeResult = ConservativeResult.intersectWith(
6842 CR: ConstantRange::getNonEmpty(Lower: MinVal, Upper: MaxVal + 1), Type: RangeType);
6843 }
6844 }
6845
6846 // A range of Phi is a subset of union of all ranges of its input.
6847 if (PHINode *Phi = dyn_cast<PHINode>(Val: V)) {
6848 // SCEVExpander sometimes creates SCEVUnknowns that are secretly
6849 // AddRecs; return the range for the corresponding AddRec.
6850 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: getSCEV(V)))
6851 return getRangeRef(S: AR, SignHint, Depth: Depth + 1);
6852
6853 // Make sure that we do not run over cycled Phis.
6854 if (RangeRefPHIAllowedOperands(DT, PHI: Phi)) {
6855 ConstantRange RangeFromOps(BitWidth, /*isFullSet=*/false);
6856
6857 for (const auto &Op : Phi->operands()) {
6858 auto OpRange = getRangeRef(S: getSCEV(V: Op), SignHint, Depth: Depth + 1);
6859 RangeFromOps = RangeFromOps.unionWith(CR: OpRange, Type: RangeType);
6860 // No point to continue if we already have a full set.
6861 if (RangeFromOps.isFullSet())
6862 break;
6863 }
6864 ConservativeResult =
6865 ConservativeResult.intersectWith(CR: RangeFromOps, Type: RangeType);
6866 }
6867 }
6868
6869 const APInt *TrueC, *FalseC;
6870 if (match(V, P: m_Select(C: m_Value(), L: m_APInt(Res&: TrueC), R: m_APInt(Res&: FalseC))))
6871 ConservativeResult = ConservativeResult.intersectWith(
6872 CR: ConstantRange(*TrueC).unionWith(CR: *FalseC, Type: RangeType), Type: RangeType);
6873
6874 // vscale can't be equal to zero
6875 if (const auto *II = dyn_cast<IntrinsicInst>(Val: V))
6876 if (II->getIntrinsicID() == Intrinsic::vscale) {
6877 ConstantRange Disallowed = APInt::getZero(numBits: BitWidth);
6878 ConservativeResult = ConservativeResult.difference(CR: Disallowed);
6879 }
6880
6881 return setRange(S: U, Hint: SignHint, CR: std::move(ConservativeResult));
6882 }
6883 case scCouldNotCompute:
6884 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
6885 }
6886
6887 return setRange(S, Hint: SignHint, CR: std::move(ConservativeResult));
6888}
6889
6890// Given a StartRange, Step and MaxBECount for an expression compute a range of
6891// values that the expression can take. Initially, the expression has a value
6892// from StartRange and then is changed by Step up to MaxBECount times. Signed
6893// argument defines if we treat Step as signed or unsigned. The second return
6894// value indicates that no wrapping occurred.
6895static std::pair<ConstantRange, bool>
6896getRangeForAffineARHelper(APInt Step, const ConstantRange &StartRange,
6897 const APInt &MaxBECount, bool Signed) {
6898 unsigned BitWidth = Step.getBitWidth();
6899 assert(BitWidth == StartRange.getBitWidth() &&
6900 BitWidth == MaxBECount.getBitWidth() && "mismatched bit widths");
6901 // If either Step or MaxBECount is 0, then the expression won't change, and we
6902 // just need to return the initial range.
6903 if (Step == 0 || MaxBECount == 0)
6904 return {StartRange, true};
6905
6906 // If we don't know anything about the initial value (i.e. StartRange is
6907 // FullRange), then we don't know anything about the final range either.
6908 // Return FullRange.
6909 if (StartRange.isFullSet())
6910 return {ConstantRange::getFull(BitWidth), false};
6911
6912 // If Step is signed and negative, then we use its absolute value, but we also
6913 // note that we're moving in the opposite direction.
6914 bool Descending = Signed && Step.isNegative();
6915
6916 if (Signed)
6917 // This is correct even for INT_SMIN. Let's look at i8 to illustrate this:
6918 // abs(INT_SMIN) = abs(-128) = abs(0x80) = -0x80 = 0x80 = 128.
6919 // This equations hold true due to the well-defined wrap-around behavior of
6920 // APInt.
6921 Step = Step.abs();
6922
6923 // Check if Offset is more than full span of BitWidth. If it is, the
6924 // expression is guaranteed to overflow.
6925 if (APInt::getMaxValue(numBits: StartRange.getBitWidth()).udiv(RHS: Step).ult(RHS: MaxBECount))
6926 return {ConstantRange::getFull(BitWidth), false};
6927
6928 // Offset is by how much the expression can change. Checks above guarantee no
6929 // overflow here.
6930 APInt Offset = Step * MaxBECount;
6931
6932 // Minimum value of the final range will match the minimal value of StartRange
6933 // if the expression is increasing and will be decreased by Offset otherwise.
6934 // Maximum value of the final range will match the maximal value of StartRange
6935 // if the expression is decreasing and will be increased by Offset otherwise.
6936 APInt StartLower = StartRange.getLower();
6937 APInt StartUpper = StartRange.getUpper() - 1;
6938 bool Overflow;
6939 APInt MovedBoundary;
6940 if (Signed) {
6941 // This does not use sadd_ov, as we want to check overflow for a signed
6942 // start with an unsigned offset.
6943 if (Descending) {
6944 MovedBoundary = StartLower - std::move(Offset);
6945 Overflow = MovedBoundary.sgt(RHS: StartLower) || StartRange.isSignWrappedSet();
6946 } else {
6947 MovedBoundary = StartUpper + std::move(Offset);
6948 Overflow = MovedBoundary.slt(RHS: StartUpper) || StartRange.isSignWrappedSet();
6949 }
6950 } else {
6951 MovedBoundary = StartUpper.uadd_ov(RHS: std::move(Offset), Overflow);
6952 Overflow |= StartRange.isWrappedSet();
6953 }
6954
6955 // It's possible that the new minimum/maximum value will fall into the initial
6956 // range (due to wrap around). This means that the expression can take any
6957 // value in this bitwidth, and we have to return full range.
6958 if (StartRange.contains(Val: MovedBoundary))
6959 return {ConstantRange::getFull(BitWidth), false};
6960
6961 APInt NewLower =
6962 Descending ? std::move(MovedBoundary) : std::move(StartLower);
6963 APInt NewUpper =
6964 Descending ? std::move(StartUpper) : std::move(MovedBoundary);
6965 NewUpper += 1;
6966
6967 // No overflow detected, return [StartLower, StartUpper + Offset + 1) range.
6968 return {ConstantRange::getNonEmpty(Lower: std::move(NewLower), Upper: std::move(NewUpper)),
6969 !Overflow};
6970}
6971
6972std::pair<ConstantRange, SCEVFlags>
6973ScalarEvolution::getRangeForAffineAR(const SCEV *Start, const SCEV *Step,
6974 const APInt &MaxBECount) {
6975 assert(getTypeSizeInBits(Start->getType()) ==
6976 getTypeSizeInBits(Step->getType()) &&
6977 getTypeSizeInBits(Start->getType()) == MaxBECount.getBitWidth() &&
6978 "mismatched bit widths");
6979
6980 // First, consider step signed.
6981 ConstantRange StartSRange = getSignedRange(S: Start);
6982 ConstantRange StepSRange = getSignedRange(S: Step);
6983
6984 // If Step can be both positive and negative, we need to find ranges for the
6985 // maximum absolute step values in both directions and union them.
6986 auto [SR, NSW] = getRangeForAffineARHelper(
6987 Step: StepSRange.getSignedMin(), StartRange: StartSRange, MaxBECount, /*Signed=*/true);
6988 // A singleton step has identical signed bounds, so the second computation
6989 // would produce the same range and no-wrap result.
6990 if (!StepSRange.isSingleElement()) {
6991 auto [SRMax, NSWMax] = getRangeForAffineARHelper(
6992 Step: StepSRange.getSignedMax(), StartRange: StartSRange, MaxBECount, /*Signed=*/true);
6993 SR = SR.unionWith(CR: SRMax);
6994 NSW &= NSWMax;
6995 }
6996
6997 // Next, consider step unsigned.
6998 auto [UR, NUW] = getRangeForAffineARHelper(
6999 Step: getUnsignedRangeMax(S: Step), StartRange: getUnsignedRange(S: Start), MaxBECount,
7000 /*Signed=*/false);
7001
7002 SCEVFlags Flags = SCEV::FlagNone;
7003 if (NUW)
7004 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7005 if (NSW)
7006 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7007
7008 // Finally, intersect signed and unsigned ranges.
7009 return {SR.intersectWith(CR: UR, Type: ConstantRange::Smallest), Flags};
7010}
7011
7012ConstantRange ScalarEvolution::getRangeForAffineNoSelfWrappingAR(
7013 const SCEVAddRecExpr *AddRec, const SCEV *MaxBECount, unsigned BitWidth,
7014 ScalarEvolution::RangeSignHint SignHint) {
7015 assert(AddRec->isAffine() && "Non-affine AddRecs are not suppored!\n");
7016 assert(AddRec->hasNoSelfWrap() &&
7017 "This only works for non-self-wrapping AddRecs!");
7018 const bool IsSigned = SignHint == HINT_RANGE_SIGNED;
7019 const SCEV *Step = AddRec->getStepRecurrence(SE&: *this);
7020 // Only deal with constant step to save compile time.
7021 if (!isa<SCEVConstant>(Val: Step))
7022 return ConstantRange::getFull(BitWidth);
7023 // Let's make sure that we can prove that we do not self-wrap during
7024 // MaxBECount iterations. We need this because MaxBECount is a maximum
7025 // iteration count estimate, and we might infer nw from some exit for which we
7026 // do not know max exit count (or any other side reasoning).
7027 // TODO: Turn into assert at some point.
7028 if (getTypeSizeInBits(Ty: MaxBECount->getType()) >
7029 getTypeSizeInBits(Ty: AddRec->getType()))
7030 return ConstantRange::getFull(BitWidth);
7031 MaxBECount = getNoopOrZeroExtend(V: MaxBECount, Ty: AddRec->getType());
7032 const SCEV *RangeWidth = getMinusOne(Ty: AddRec->getType());
7033 const SCEV *StepAbs = getUMinExpr(LHS: Step, RHS: getNegativeSCEV(V: Step));
7034 const SCEV *MaxItersWithoutWrap = getUDivExpr(LHS: RangeWidth, RHS: StepAbs);
7035 if (!isKnownPredicateViaConstantRanges(Pred: ICmpInst::ICMP_ULE, LHS: MaxBECount,
7036 RHS: MaxItersWithoutWrap))
7037 return ConstantRange::getFull(BitWidth);
7038
7039 ICmpInst::Predicate LEPred =
7040 IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
7041 ICmpInst::Predicate GEPred =
7042 IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
7043 const SCEV *End = AddRec->evaluateAtIteration(It: MaxBECount, SE&: *this);
7044
7045 // We know that there is no self-wrap. Let's take Start and End values and
7046 // look at all intermediate values V1, V2, ..., Vn that IndVar takes during
7047 // the iteration. They either lie inside the range [Min(Start, End),
7048 // Max(Start, End)] or outside it:
7049 //
7050 // Case 1: RangeMin ... Start V1 ... VN End ... RangeMax;
7051 // Case 2: RangeMin Vk ... V1 Start ... End Vn ... Vk + 1 RangeMax;
7052 //
7053 // No self wrap flag guarantees that the intermediate values cannot be BOTH
7054 // outside and inside the range [Min(Start, End), Max(Start, End)]. Using that
7055 // knowledge, let's try to prove that we are dealing with Case 1. It is so if
7056 // Start <= End and step is positive, or Start >= End and step is negative.
7057 const SCEV *Start = applyLoopGuards(Expr: AddRec->getStart(), L: AddRec->getLoop());
7058 ConstantRange StartRange = getRangeRef(S: Start, SignHint);
7059 ConstantRange EndRange = getRangeRef(S: End, SignHint);
7060 ConstantRange RangeBetween = StartRange.unionWith(CR: EndRange);
7061 // If they already cover full iteration space, we will know nothing useful
7062 // even if we prove what we want to prove.
7063 if (RangeBetween.isFullSet())
7064 return RangeBetween;
7065 // Only deal with ranges that do not wrap (i.e. RangeMin < RangeMax).
7066 bool IsWrappedSet = IsSigned ? RangeBetween.isSignWrappedSet()
7067 : RangeBetween.isWrappedSet();
7068 if (IsWrappedSet)
7069 return ConstantRange::getFull(BitWidth);
7070
7071 if (isKnownPositive(S: Step) &&
7072 isKnownPredicateViaConstantRanges(Pred: LEPred, LHS: Start, RHS: End))
7073 return RangeBetween;
7074 if (isKnownNegative(S: Step) &&
7075 isKnownPredicateViaConstantRanges(Pred: GEPred, LHS: Start, RHS: End))
7076 return RangeBetween;
7077 return ConstantRange::getFull(BitWidth);
7078}
7079
7080ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start,
7081 const SCEV *Step,
7082 const APInt &MaxBECount) {
7083 // RangeOf({C?A:B,+,C?P:Q}) == RangeOf(C?{A,+,P}:{B,+,Q})
7084 // == RangeOf({A,+,P}) union RangeOf({B,+,Q})
7085
7086 unsigned BitWidth = MaxBECount.getBitWidth();
7087 assert(getTypeSizeInBits(Start->getType()) == BitWidth &&
7088 getTypeSizeInBits(Step->getType()) == BitWidth &&
7089 "mismatched bit widths");
7090
7091 struct SelectPattern {
7092 Value *Condition = nullptr;
7093 APInt TrueValue;
7094 APInt FalseValue;
7095
7096 explicit SelectPattern(ScalarEvolution &SE, unsigned BitWidth,
7097 const SCEV *S) {
7098 std::optional<unsigned> CastOp;
7099 APInt Offset(BitWidth, 0);
7100
7101 assert(SE.getTypeSizeInBits(S->getType()) == BitWidth &&
7102 "Should be!");
7103
7104 // Peel off a constant offset. In the future we could consider being
7105 // smarter here and handle {Start+Step,+,Step} too.
7106 const APInt *Off;
7107 if (match(S, P: m_scev_Add(Op0: m_scev_APInt(C&: Off), Op1: m_SCEV(V&: S))))
7108 Offset = *Off;
7109
7110 // Peel off a cast operation
7111 if (auto *SCast = dyn_cast<SCEVIntegralCastExpr>(Val: S)) {
7112 CastOp = SCast->getSCEVType();
7113 S = SCast->getOperand();
7114 }
7115
7116 using namespace llvm::PatternMatch;
7117
7118 auto *SU = dyn_cast<SCEVUnknown>(Val: S);
7119 const APInt *TrueVal, *FalseVal;
7120 if (!SU ||
7121 !match(V: SU->getValue(), P: m_Select(C: m_Value(V&: Condition), L: m_APInt(Res&: TrueVal),
7122 R: m_APInt(Res&: FalseVal)))) {
7123 Condition = nullptr;
7124 return;
7125 }
7126
7127 TrueValue = *TrueVal;
7128 FalseValue = *FalseVal;
7129
7130 // Re-apply the cast we peeled off earlier
7131 if (CastOp)
7132 switch (*CastOp) {
7133 default:
7134 llvm_unreachable("Unknown SCEV cast type!");
7135
7136 case scTruncate:
7137 TrueValue = TrueValue.trunc(width: BitWidth);
7138 FalseValue = FalseValue.trunc(width: BitWidth);
7139 break;
7140 case scZeroExtend:
7141 TrueValue = TrueValue.zext(width: BitWidth);
7142 FalseValue = FalseValue.zext(width: BitWidth);
7143 break;
7144 case scSignExtend:
7145 TrueValue = TrueValue.sext(width: BitWidth);
7146 FalseValue = FalseValue.sext(width: BitWidth);
7147 break;
7148 }
7149
7150 // Re-apply the constant offset we peeled off earlier
7151 TrueValue += Offset;
7152 FalseValue += Offset;
7153 }
7154
7155 bool isRecognized() { return Condition != nullptr; }
7156 };
7157
7158 SelectPattern StartPattern(*this, BitWidth, Start);
7159 if (!StartPattern.isRecognized())
7160 return ConstantRange::getFull(BitWidth);
7161
7162 SelectPattern StepPattern(*this, BitWidth, Step);
7163 if (!StepPattern.isRecognized())
7164 return ConstantRange::getFull(BitWidth);
7165
7166 if (StartPattern.Condition != StepPattern.Condition) {
7167 // We don't handle this case today; but we could, by considering four
7168 // possibilities below instead of two. I'm not sure if there are cases where
7169 // that will help over what getRange already does, though.
7170 return ConstantRange::getFull(BitWidth);
7171 }
7172
7173 // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to
7174 // construct arbitrary general SCEV expressions here. This function is called
7175 // from deep in the call stack, and calling getSCEV (on a sext instruction,
7176 // say) can end up caching a suboptimal value.
7177
7178 // FIXME: without the explicit `this` receiver below, MSVC errors out with
7179 // C2352 and C2512 (otherwise it isn't needed).
7180
7181 const SCEV *TrueStart = this->getConstant(Val: StartPattern.TrueValue);
7182 const SCEV *TrueStep = this->getConstant(Val: StepPattern.TrueValue);
7183 const SCEV *FalseStart = this->getConstant(Val: StartPattern.FalseValue);
7184 const SCEV *FalseStep = this->getConstant(Val: StepPattern.FalseValue);
7185
7186 ConstantRange TrueRange =
7187 this->getRangeForAffineAR(Start: TrueStart, Step: TrueStep, MaxBECount).first;
7188 ConstantRange FalseRange =
7189 this->getRangeForAffineAR(Start: FalseStart, Step: FalseStep, MaxBECount).first;
7190
7191 return TrueRange.unionWith(CR: FalseRange);
7192}
7193
7194SCEVFlags ScalarEvolution::getNoWrapFlagsFromUB(const Value *V) {
7195 if (isa<ConstantExpr>(Val: V))
7196 return SCEV::FlagNone;
7197 const BinaryOperator *BinOp = cast<BinaryOperator>(Val: V);
7198
7199 // Return early if there are no flags to propagate to the SCEV.
7200 SCEVFlags Flags = SCEV::FlagNone;
7201 if (auto *PDI = dyn_cast<PossiblyDisjointInst>(Val: BinOp);
7202 PDI && PDI->isDisjoint()) {
7203 Flags = ScalarEvolution::setFlags(Flags: SCEV::FlagNUW, OnFlags: SCEV::FlagNSW);
7204 } else {
7205 if (BinOp->hasNoUnsignedWrap())
7206 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNUW);
7207 if (BinOp->hasNoSignedWrap())
7208 Flags = ScalarEvolution::setFlags(Flags, OnFlags: SCEV::FlagNSW);
7209 }
7210 if (Flags == SCEV::FlagNone)
7211 return SCEV::FlagNone;
7212
7213 return isSCEVExprNeverPoison(I: BinOp) ? Flags : SCEV::FlagNone;
7214}
7215
7216const Instruction *
7217ScalarEvolution::getNonTrivialDefiningScopeBound(const SCEV *S) {
7218 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S))
7219 return &*AddRec->getLoop()->getHeader()->begin();
7220 if (auto *U = dyn_cast<SCEVUnknown>(Val: S))
7221 if (auto *I = dyn_cast<Instruction>(Val: U->getValue()))
7222 return I;
7223 return nullptr;
7224}
7225
7226const Instruction *ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops,
7227 bool &Precise) {
7228 Precise = true;
7229 // Do a bounded search of the def relation of the requested SCEVs.
7230 SmallPtrSet<const SCEV *, 16> Visited;
7231 SmallVector<SCEVUse> Worklist;
7232 auto pushOp = [&](const SCEV *S) {
7233 if (!Visited.insert(Ptr: S).second)
7234 return;
7235 // Threshold of 30 here is arbitrary.
7236 if (Visited.size() > 30) {
7237 Precise = false;
7238 return;
7239 }
7240 Worklist.push_back(Elt: S);
7241 };
7242
7243 for (SCEVUse S : Ops)
7244 pushOp(S);
7245
7246 const Instruction *Bound = nullptr;
7247 while (!Worklist.empty()) {
7248 SCEVUse S = Worklist.pop_back_val();
7249 if (auto *DefI = getNonTrivialDefiningScopeBound(S)) {
7250 if (!Bound || DT.dominates(Def: Bound, User: DefI))
7251 Bound = DefI;
7252 } else {
7253 for (SCEVUse Op : S->operands())
7254 pushOp(Op);
7255 }
7256 }
7257 return Bound ? Bound : &*F.getEntryBlock().begin();
7258}
7259
7260const Instruction *
7261ScalarEvolution::getDefiningScopeBound(ArrayRef<SCEVUse> Ops) {
7262 bool Discard;
7263 return getDefiningScopeBound(Ops, Precise&: Discard);
7264}
7265
7266bool ScalarEvolution::isGuaranteedToTransferExecutionTo(const Instruction *A,
7267 const Instruction *B) {
7268 if (A->getParent() == B->getParent() &&
7269 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7270 End: B->getIterator()))
7271 return true;
7272
7273 auto *BLoop = LI.getLoopFor(BB: B->getParent());
7274 if (BLoop && BLoop->getHeader() == B->getParent() &&
7275 BLoop->getLoopPreheader() == A->getParent() &&
7276 isGuaranteedToTransferExecutionToSuccessor(Begin: A->getIterator(),
7277 End: A->getParent()->end()) &&
7278 isGuaranteedToTransferExecutionToSuccessor(Begin: B->getParent()->begin(),
7279 End: B->getIterator()))
7280 return true;
7281 return false;
7282}
7283
7284bool ScalarEvolution::isGuaranteedNotToBePoison(const SCEV *Op) {
7285 SCEVPoisonCollector PC(/* LookThroughMaybePoisonBlocking */ true);
7286 visitAll(Root: Op, Visitor&: PC);
7287 return PC.MaybePoison.empty();
7288}
7289
7290bool ScalarEvolution::isGuaranteedNotToCauseUB(const SCEV *Op) {
7291 return !SCEVExprContains(Root: Op, Pred: [this](const SCEV *S) {
7292 const SCEV *Op1;
7293 bool M = match(S, P: m_scev_UDiv(Op0: m_SCEV(), Op1: m_SCEV(V&: Op1)));
7294 // The UDiv may be UB if the divisor is poison or zero. Unless the divisor
7295 // is a non-zero constant, we have to assume the UDiv may be UB.
7296 return M && (!isKnownNonZero(S: Op1) || !isGuaranteedNotToBePoison(Op: Op1));
7297 });
7298}
7299
7300bool ScalarEvolution::isSCEVExprNeverPoison(const Instruction *I) {
7301 // Only proceed if we can prove that I does not yield poison.
7302 if (!programUndefinedIfPoison(Inst: I))
7303 return false;
7304
7305 // At this point we know that if I is executed, then it does not wrap
7306 // according to at least one of NSW or NUW. If I is not executed, then we do
7307 // not know if the calculation that I represents would wrap. Multiple
7308 // instructions can map to the same SCEV. If we apply NSW or NUW from I to
7309 // the SCEV, we must guarantee no wrapping for that SCEV also when it is
7310 // derived from other instructions that map to the same SCEV. We cannot make
7311 // that guarantee for cases where I is not executed. So we need to find a
7312 // upper bound on the defining scope for the SCEV, and prove that I is
7313 // executed every time we enter that scope. When the bounding scope is a
7314 // loop (the common case), this is equivalent to proving I executes on every
7315 // iteration of that loop.
7316 SmallVector<SCEVUse> SCEVOps;
7317 for (const Use &Op : I->operands()) {
7318 // I could be an extractvalue from a call to an overflow intrinsic.
7319 // TODO: We can do better here in some cases.
7320 if (isSCEVable(Ty: Op->getType()))
7321 SCEVOps.push_back(Elt: getSCEV(V: Op));
7322 }
7323 auto *DefI = getDefiningScopeBound(Ops: SCEVOps);
7324 return isGuaranteedToTransferExecutionTo(A: DefI, B: I);
7325}
7326
7327bool ScalarEvolution::isAddRecNeverPoison(const Instruction *I, const Loop *L) {
7328 // If we know that \c I can never be poison period, then that's enough.
7329 if (isSCEVExprNeverPoison(I))
7330 return true;
7331
7332 // If the loop only has one exit, then we know that, if the loop is entered,
7333 // any instruction dominating that exit will be executed. If any such
7334 // instruction would result in UB, the addrec cannot be poison.
7335 //
7336 // This is basically the same reasoning as in isSCEVExprNeverPoison(), but
7337 // also handles uses outside the loop header (they just need to dominate the
7338 // single exit).
7339
7340 auto *ExitingBB = L->getExitingBlock();
7341 if (!ExitingBB || !loopHasNoAbnormalExits(L))
7342 return false;
7343
7344 SmallPtrSet<const Value *, 16> KnownPoison;
7345 SmallVector<const Instruction *, 8> Worklist;
7346
7347 // We start by assuming \c I, the post-inc add recurrence, is poison. Only
7348 // things that are known to be poison under that assumption go on the
7349 // Worklist.
7350 KnownPoison.insert(Ptr: I);
7351 Worklist.push_back(Elt: I);
7352
7353 while (!Worklist.empty()) {
7354 const Instruction *Poison = Worklist.pop_back_val();
7355
7356 for (const Use &U : Poison->uses()) {
7357 const Instruction *PoisonUser = cast<Instruction>(Val: U.getUser());
7358 if (mustTriggerUB(I: PoisonUser, KnownPoison) &&
7359 DT.dominates(A: PoisonUser->getParent(), B: ExitingBB))
7360 return true;
7361
7362 if (propagatesPoison(PoisonOp: U) && L->contains(Inst: PoisonUser))
7363 if (KnownPoison.insert(Ptr: PoisonUser).second)
7364 Worklist.push_back(Elt: PoisonUser);
7365 }
7366 }
7367
7368 return false;
7369}
7370
7371ScalarEvolution::LoopProperties
7372ScalarEvolution::getLoopProperties(const Loop *L) {
7373 using LoopProperties = ScalarEvolution::LoopProperties;
7374
7375 auto Itr = LoopPropertiesCache.find(Val: L);
7376 if (Itr == LoopPropertiesCache.end()) {
7377 auto HasSideEffects = [](Instruction *I) {
7378 if (auto *SI = dyn_cast<StoreInst>(Val: I))
7379 return !SI->isSimple();
7380
7381 if (I->mayThrow())
7382 return true;
7383
7384 // Non-volatile memset / memcpy do not count as side-effect for forward
7385 // progress.
7386 if (auto *MI = dyn_cast<MemIntrinsic>(Val: I))
7387 if (!MI->isVolatile())
7388 return false;
7389
7390 // Droppable intrinsics only model a memory write; they are not progress.
7391 if (I->isDroppable())
7392 return false;
7393
7394 return I->mayWriteToMemory();
7395 };
7396
7397 LoopProperties LP = {/* HasNoAbnormalExits */ true,
7398 /*HasNoSideEffects*/ true};
7399
7400 for (auto *BB : L->getBlocks())
7401 for (auto &I : *BB) {
7402 if (!isGuaranteedToTransferExecutionToSuccessor(I: &I))
7403 LP.HasNoAbnormalExits = false;
7404 if (HasSideEffects(&I))
7405 LP.HasNoSideEffects = false;
7406 if (!LP.HasNoAbnormalExits && !LP.HasNoSideEffects)
7407 break; // We're already as pessimistic as we can get.
7408 }
7409
7410 auto InsertPair = LoopPropertiesCache.insert(KV: {L, LP});
7411 assert(InsertPair.second && "We just checked!");
7412 Itr = InsertPair.first;
7413 }
7414
7415 return Itr->second;
7416}
7417
7418bool ScalarEvolution::loopIsFiniteByAssumption(const Loop *L) {
7419 // A mustprogress loop without side effects must be finite.
7420 // TODO: The check used here is very conservative. It's only *specific*
7421 // side effects which are well defined in infinite loops.
7422 return isFinite(L) || (isMustProgress(L) && loopHasNoSideEffects(L));
7423}
7424
7425const SCEV *ScalarEvolution::createSCEVIter(Value *V) {
7426 // Worklist item with a Value and a bool indicating whether all operands have
7427 // been visited already.
7428 using PointerTy = PointerIntPair<Value *, 1, bool>;
7429 SmallVector<PointerTy> Stack;
7430
7431 Stack.emplace_back(Args&: V, Args: false);
7432 while (!Stack.empty()) {
7433 auto E = Stack.back();
7434 Value *CurV = E.getPointer();
7435
7436 if (getExistingSCEV(V: CurV)) {
7437 Stack.pop_back();
7438 continue;
7439 }
7440
7441 SmallVector<Value *> Ops;
7442 const SCEV *CreatedSCEV = nullptr;
7443 // If all operands have been visited already, create the SCEV.
7444 if (E.getInt()) {
7445 CreatedSCEV = createSCEV(V: CurV);
7446 } else {
7447 // Otherwise get the operands we need to create SCEV's for before creating
7448 // the SCEV for CurV. If the SCEV for CurV can be constructed trivially,
7449 // just use it.
7450 CreatedSCEV = getOperandsToCreate(V: CurV, Ops);
7451 }
7452
7453 if (CreatedSCEV) {
7454 insertValueToMap(V: CurV, S: CreatedSCEV);
7455 Stack.pop_back();
7456 } else {
7457 Stack.back().setInt(true);
7458 // Queue its operands which need to be constructed.
7459 for (Value *Op : Ops)
7460 Stack.emplace_back(Args&: Op, Args: false);
7461 }
7462 }
7463
7464 return getExistingSCEV(V);
7465}
7466
7467const SCEV *
7468ScalarEvolution::getOperandsToCreate(Value *V, SmallVectorImpl<Value *> &Ops) {
7469 if (!isSCEVable(Ty: V->getType()))
7470 return getUnknown(V);
7471
7472 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7473 // Don't attempt to analyze instructions in blocks that aren't
7474 // reachable. Such instructions don't matter, and they aren't required
7475 // to obey basic rules for definitions dominating uses which this
7476 // analysis depends on.
7477 if (!DT.isReachableFromEntry(A: I->getParent()))
7478 return getUnknown(V: PoisonValue::get(T: V->getType()));
7479 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7480 return getConstant(V: CI);
7481 else if (isa<GlobalAlias>(Val: V))
7482 return getUnknown(V);
7483 else if (!isa<ConstantExpr>(Val: V))
7484 return getUnknown(V);
7485
7486 Operator *U = cast<Operator>(Val: V);
7487 if (auto BO = MatchBinaryOp(V: U, DT)) {
7488 bool IsConstArg = isa<ConstantInt>(Val: BO->RHS);
7489 switch (BO->Opcode) {
7490 case Instruction::Add:
7491 case Instruction::Mul: {
7492 // For additions and multiplications, traverse add/mul chains for which we
7493 // can potentially create a single SCEV, to reduce the number of
7494 // get{Add,Mul}Expr calls.
7495 do {
7496 if (BO->Op) {
7497 if (BO->Op != V && getExistingSCEV(V: BO->Op)) {
7498 Ops.push_back(Elt: BO->Op);
7499 break;
7500 }
7501 }
7502 Ops.push_back(Elt: BO->RHS);
7503 auto NewBO = MatchBinaryOp(V: BO->LHS, DT);
7504 if (!NewBO ||
7505 (BO->Opcode == Instruction::Add &&
7506 (NewBO->Opcode != Instruction::Add &&
7507 NewBO->Opcode != Instruction::Sub)) ||
7508 (BO->Opcode == Instruction::Mul &&
7509 NewBO->Opcode != Instruction::Mul)) {
7510 Ops.push_back(Elt: BO->LHS);
7511 break;
7512 }
7513 // CreateSCEV calls getNoWrapFlagsFromUB, which under certain conditions
7514 // requires a SCEV for the LHS.
7515 if (BO->Op && (BO->IsNSW || BO->IsNUW)) {
7516 auto *I = dyn_cast<Instruction>(Val: BO->Op);
7517 if (I && programUndefinedIfPoison(Inst: I)) {
7518 Ops.push_back(Elt: BO->LHS);
7519 break;
7520 }
7521 }
7522 BO = NewBO;
7523 } while (true);
7524 return nullptr;
7525 }
7526 case Instruction::Sub:
7527 case Instruction::UDiv:
7528 case Instruction::URem:
7529 break;
7530 case Instruction::AShr:
7531 case Instruction::Shl:
7532 case Instruction::Xor:
7533 if (!IsConstArg)
7534 return nullptr;
7535 break;
7536 case Instruction::And:
7537 case Instruction::Or:
7538 if (!IsConstArg && !BO->LHS->getType()->isIntegerTy(BitWidth: 1))
7539 return nullptr;
7540 break;
7541 case Instruction::LShr:
7542 return getUnknown(V);
7543 default:
7544 llvm_unreachable("Unhandled binop");
7545 break;
7546 }
7547
7548 Ops.push_back(Elt: BO->LHS);
7549 Ops.push_back(Elt: BO->RHS);
7550 return nullptr;
7551 }
7552
7553 switch (U->getOpcode()) {
7554 case Instruction::Trunc:
7555 case Instruction::ZExt:
7556 case Instruction::SExt:
7557 case Instruction::PtrToAddr:
7558 case Instruction::PtrToInt:
7559 Ops.push_back(Elt: U->getOperand(i: 0));
7560 return nullptr;
7561
7562 case Instruction::BitCast:
7563 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType())) {
7564 Ops.push_back(Elt: U->getOperand(i: 0));
7565 return nullptr;
7566 }
7567 return getUnknown(V);
7568
7569 case Instruction::SDiv:
7570 case Instruction::SRem:
7571 Ops.push_back(Elt: U->getOperand(i: 0));
7572 Ops.push_back(Elt: U->getOperand(i: 1));
7573 return nullptr;
7574
7575 case Instruction::GetElementPtr:
7576 assert(cast<GEPOperator>(U)->getSourceElementType()->isSized() &&
7577 "GEP source element type must be sized");
7578 llvm::append_range(C&: Ops, R: U->operands());
7579 return nullptr;
7580
7581 case Instruction::IntToPtr:
7582 return getUnknown(V);
7583
7584 case Instruction::PHI:
7585 // getNodeForPHI has four ways to turn a PHI into a SCEV; retrieve the
7586 // relevant nodes for each of them.
7587 //
7588 // The first is just to call simplifyInstruction, and get something back
7589 // that isn't a PHI.
7590 if (Value *V = simplifyInstruction(
7591 I: cast<PHINode>(Val: U),
7592 Q: {getDataLayout(), &TLI, &DT, &AC, /*CtxI=*/nullptr,
7593 /*UseInstrInfo=*/true, /*CanUseUndef=*/false})) {
7594 assert(V);
7595 Ops.push_back(Elt: V);
7596 return nullptr;
7597 }
7598 // The second is createNodeForPHIWithIdenticalOperands: this looks for
7599 // operands which all perform the same operation, but haven't been
7600 // CSE'ed for whatever reason.
7601 if (BinaryOperator *BO = getCommonInstForPHI(PN: cast<PHINode>(Val: U))) {
7602 assert(BO);
7603 Ops.push_back(Elt: BO);
7604 return nullptr;
7605 }
7606 // The third is createNodeFromSelectLikePHI; this takes a PHI which
7607 // is equivalent to a select, and analyzes it like a select.
7608 {
7609 Value *Cond = nullptr, *LHS = nullptr, *RHS = nullptr;
7610 if (getOperandsForSelectLikePHI(DT, PN: cast<PHINode>(Val: U), Cond, LHS, RHS)) {
7611 assert(Cond);
7612 assert(LHS);
7613 assert(RHS);
7614 if (auto *CondICmp = dyn_cast<ICmpInst>(Val: Cond)) {
7615 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 0));
7616 Ops.push_back(Elt: CondICmp->getOperand(i_nocapture: 1));
7617 }
7618 Ops.push_back(Elt: Cond);
7619 Ops.push_back(Elt: LHS);
7620 Ops.push_back(Elt: RHS);
7621 return nullptr;
7622 }
7623 }
7624 // The fourth way is createAddRecFromPHI. It's complicated to handle here,
7625 // so just construct it recursively.
7626 //
7627 // In addition to getNodeForPHI, also construct nodes which might be needed
7628 // by getRangeRef.
7629 if (RangeRefPHIAllowedOperands(DT, PHI: cast<PHINode>(Val: U))) {
7630 for (Value *V : cast<PHINode>(Val: U)->operands())
7631 Ops.push_back(Elt: V);
7632 return nullptr;
7633 }
7634 return nullptr;
7635
7636 case Instruction::Select: {
7637 // Check if U is a select that can be simplified to a SCEVUnknown.
7638 auto CanSimplifyToUnknown = [this, U]() {
7639 if (U->getType()->isIntegerTy(BitWidth: 1) || isa<ConstantInt>(Val: U->getOperand(i: 0)))
7640 return false;
7641
7642 auto *ICI = dyn_cast<ICmpInst>(Val: U->getOperand(i: 0));
7643 if (!ICI)
7644 return false;
7645 Value *LHS = ICI->getOperand(i_nocapture: 0);
7646 Value *RHS = ICI->getOperand(i_nocapture: 1);
7647 if (ICI->getPredicate() == CmpInst::ICMP_EQ ||
7648 ICI->getPredicate() == CmpInst::ICMP_NE) {
7649 if (!(isa<ConstantInt>(Val: RHS) && cast<ConstantInt>(Val: RHS)->isZero()))
7650 return true;
7651 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
7652 getTypeSizeInBits(Ty: U->getType()))
7653 return true;
7654 return false;
7655 };
7656 if (CanSimplifyToUnknown())
7657 return getUnknown(V: U);
7658
7659 llvm::append_range(C&: Ops, R: U->operands());
7660 return nullptr;
7661 break;
7662 }
7663 case Instruction::Call:
7664 case Instruction::Invoke:
7665 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand()) {
7666 Ops.push_back(Elt: RV);
7667 return nullptr;
7668 }
7669
7670 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
7671 switch (II->getIntrinsicID()) {
7672 case Intrinsic::abs:
7673 Ops.push_back(Elt: II->getArgOperand(i: 0));
7674 return nullptr;
7675 case Intrinsic::umax:
7676 case Intrinsic::umin:
7677 case Intrinsic::smax:
7678 case Intrinsic::smin:
7679 case Intrinsic::usub_sat:
7680 case Intrinsic::uadd_sat:
7681 Ops.push_back(Elt: II->getArgOperand(i: 0));
7682 Ops.push_back(Elt: II->getArgOperand(i: 1));
7683 return nullptr;
7684 case Intrinsic::start_loop_iterations:
7685 case Intrinsic::annotation:
7686 case Intrinsic::ptr_annotation:
7687 Ops.push_back(Elt: II->getArgOperand(i: 0));
7688 return nullptr;
7689 default:
7690 break;
7691 }
7692 }
7693 break;
7694 }
7695
7696 return nullptr;
7697}
7698
7699const SCEV *ScalarEvolution::createSCEV(Value *V) {
7700 if (!isSCEVable(Ty: V->getType()))
7701 return getUnknown(V);
7702
7703 if (Instruction *I = dyn_cast<Instruction>(Val: V)) {
7704 // Don't attempt to analyze instructions in blocks that aren't
7705 // reachable. Such instructions don't matter, and they aren't required
7706 // to obey basic rules for definitions dominating uses which this
7707 // analysis depends on.
7708 if (!DT.isReachableFromEntry(A: I->getParent()))
7709 return getUnknown(V: PoisonValue::get(T: V->getType()));
7710 } else if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: V))
7711 return getConstant(V: CI);
7712 else if (isa<GlobalAlias>(Val: V))
7713 return getUnknown(V);
7714 else if (!isa<ConstantExpr>(Val: V))
7715 return getUnknown(V);
7716
7717 const SCEV *LHS;
7718 const SCEV *RHS;
7719
7720 Operator *U = cast<Operator>(Val: V);
7721 if (auto BO = MatchBinaryOp(V: U, DT)) {
7722 switch (BO->Opcode) {
7723 case Instruction::Add: {
7724 // The simple thing to do would be to just call getSCEV on both operands
7725 // and call getAddExpr with the result. However if we're looking at a
7726 // bunch of things all added together, this can be quite inefficient,
7727 // because it leads to N-1 getAddExpr calls for N ultimate operands.
7728 // Instead, gather up all the operands and make a single getAddExpr call.
7729 // LLVM IR canonical form means we need only traverse the left operands.
7730 SmallVector<SCEVUse, 4> AddOps;
7731 do {
7732 if (BO->Op) {
7733 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
7734 AddOps.push_back(Elt: OpSCEV);
7735 break;
7736 }
7737
7738 // If a NUW or NSW flag can be applied to the SCEV for this
7739 // addition, then compute the SCEV for this addition by itself
7740 // with a separate call to getAddExpr. We need to do that
7741 // instead of pushing the operands of the addition onto AddOps,
7742 // since the flags are only known to apply to this particular
7743 // addition - they may not apply to other additions that can be
7744 // formed with operands from AddOps.
7745 const SCEV *RHS = getSCEV(V: BO->RHS);
7746 SCEVFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
7747 if (Flags != SCEV::FlagNone) {
7748 const SCEV *LHS = getSCEV(V: BO->LHS);
7749 if (BO->Opcode == Instruction::Sub)
7750 AddOps.push_back(Elt: getMinusSCEV(LHS, RHS, Flags));
7751 else
7752 AddOps.push_back(Elt: getAddExpr(LHS, RHS, Flags));
7753 break;
7754 }
7755 }
7756
7757 if (BO->Opcode == Instruction::Sub)
7758 AddOps.push_back(Elt: getNegativeSCEV(V: getSCEV(V: BO->RHS)));
7759 else
7760 AddOps.push_back(Elt: getSCEV(V: BO->RHS));
7761
7762 auto NewBO = MatchBinaryOp(V: BO->LHS, DT);
7763 if (!NewBO || (NewBO->Opcode != Instruction::Add &&
7764 NewBO->Opcode != Instruction::Sub)) {
7765 AddOps.push_back(Elt: getSCEV(V: BO->LHS));
7766 break;
7767 }
7768 BO = NewBO;
7769 } while (true);
7770
7771 return getAddExpr(Ops&: AddOps);
7772 }
7773
7774 case Instruction::Mul: {
7775 SmallVector<SCEVUse, 4> MulOps;
7776 do {
7777 if (BO->Op) {
7778 if (auto *OpSCEV = getExistingSCEV(V: BO->Op)) {
7779 MulOps.push_back(Elt: OpSCEV);
7780 break;
7781 }
7782
7783 SCEVFlags Flags = getNoWrapFlagsFromUB(V: BO->Op);
7784 if (Flags != SCEV::FlagNone) {
7785 LHS = getSCEV(V: BO->LHS);
7786 RHS = getSCEV(V: BO->RHS);
7787 MulOps.push_back(Elt: getMulExpr(LHS, RHS, Flags));
7788 break;
7789 }
7790 }
7791
7792 MulOps.push_back(Elt: getSCEV(V: BO->RHS));
7793 auto NewBO = MatchBinaryOp(V: BO->LHS, DT);
7794 if (!NewBO || NewBO->Opcode != Instruction::Mul) {
7795 MulOps.push_back(Elt: getSCEV(V: BO->LHS));
7796 break;
7797 }
7798 BO = NewBO;
7799 } while (true);
7800
7801 return getMulExpr(Ops&: MulOps);
7802 }
7803 case Instruction::UDiv:
7804 LHS = getSCEV(V: BO->LHS);
7805 RHS = getSCEV(V: BO->RHS);
7806 return getUDivExpr(LHS, RHS);
7807 case Instruction::URem:
7808 LHS = getSCEV(V: BO->LHS);
7809 RHS = getSCEV(V: BO->RHS);
7810 return getURemExpr(LHS, RHS);
7811 case Instruction::Sub: {
7812 SCEVFlags Flags = SCEV::FlagNone;
7813 if (BO->Op)
7814 Flags = getNoWrapFlagsFromUB(V: BO->Op);
7815
7816 // Try to use ptrtoaddr for subtracts with at least one ptrtoint
7817 // operand. While we don't model ptrtoint directly in SCEV, the
7818 // difference between two pointer addresses is well-defined.
7819 Value *PtrLHS = nullptr, *PtrRHS = nullptr;
7820 bool HasPtrLHS = match(V: BO->LHS, P: m_PtrToInt(Op: m_Value(V&: PtrLHS)));
7821 bool HasPtrRHS = match(V: BO->RHS, P: m_PtrToInt(Op: m_Value(V&: PtrRHS)));
7822 if (HasPtrLHS || HasPtrRHS) {
7823 // Convert a ptrtoint operand (OrigOp) to ptrtoaddr of its pointer
7824 // PtrOp. When only one side is ptrtoint (BothPtr is false), skip
7825 // SCEVUnknown pointers since wrapping them in ptrtoaddr adds no
7826 // useful structure.
7827 auto GetOp = [&](bool HasPtr, Value *PtrOp, Value *OrigOp,
7828 bool BothPtr) -> const SCEV * {
7829 if (!HasPtr)
7830 return getSCEV(V: OrigOp);
7831 const SCEV *PtrSCEV = getSCEV(V: PtrOp);
7832 if (BothPtr || !isa<SCEVUnknown>(Val: PtrSCEV)) {
7833 const SCEV *Addr = getPtrToAddrExpr(Op: PtrSCEV);
7834 if (!isa<SCEVCouldNotCompute>(Val: Addr) &&
7835 getTypeSizeInBits(Ty: OrigOp->getType()) <=
7836 getTypeSizeInBits(Ty: Addr->getType()))
7837 return getTruncateOrNoop(V: Addr, Ty: OrigOp->getType());
7838 }
7839 return getSCEV(V: OrigOp);
7840 };
7841 const SCEV *L = GetOp(HasPtrLHS, PtrLHS, BO->LHS, HasPtrRHS);
7842 const SCEV *R = GetOp(HasPtrRHS, PtrRHS, BO->RHS, HasPtrLHS);
7843 return getMinusSCEV(LHS: L, RHS: R, Flags);
7844 }
7845
7846 LHS = getSCEV(V: BO->LHS);
7847 RHS = getSCEV(V: BO->RHS);
7848 return getMinusSCEV(LHS, RHS, Flags);
7849 }
7850 case Instruction::And:
7851 // For an expression like x&255 that merely masks off the high bits,
7852 // use zext(trunc(x)) as the SCEV expression.
7853 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
7854 if (CI->isZero())
7855 return getSCEV(V: BO->RHS);
7856 if (CI->isMinusOne())
7857 return getSCEV(V: BO->LHS);
7858 const APInt &A = CI->getValue();
7859
7860 // Instcombine's ShrinkDemandedConstant may strip bits out of
7861 // constants, obscuring what would otherwise be a low-bits mask.
7862 // Use computeKnownBits to compute what ShrinkDemandedConstant
7863 // knew about to reconstruct a low-bits mask value.
7864 unsigned LZ = A.countl_zero();
7865 unsigned TZ = A.countr_zero();
7866 unsigned BitWidth = A.getBitWidth();
7867 KnownBits Known(BitWidth);
7868 computeKnownBits(V: BO->LHS, Known, DL: getDataLayout(), AC: &AC, CtxI: nullptr, DT: &DT);
7869
7870 APInt EffectiveMask =
7871 APInt::getLowBitsSet(numBits: BitWidth, loBitsSet: BitWidth - LZ - TZ).shl(shiftAmt: TZ);
7872 if ((LZ != 0 || TZ != 0) && !((~A & ~Known.Zero) & EffectiveMask)) {
7873 const SCEV *MulCount = getConstant(Val: APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ));
7874 const SCEV *LHS = getSCEV(V: BO->LHS);
7875 const SCEV *ShiftedLHS = nullptr;
7876 if (auto *LHSMul = dyn_cast<SCEVMulExpr>(Val: LHS)) {
7877 if (auto *OpC = dyn_cast<SCEVConstant>(Val: LHSMul->getOperand(i: 0))) {
7878 // For an expression like (x * 8) & 8, simplify the multiply.
7879 unsigned MulZeros = OpC->getAPInt().countr_zero();
7880 unsigned GCD = std::min(a: MulZeros, b: TZ);
7881 APInt DivAmt = APInt::getOneBitSet(numBits: BitWidth, BitNo: TZ - GCD);
7882 SmallVector<SCEVUse, 4> MulOps;
7883 MulOps.push_back(Elt: getConstant(Val: OpC->getAPInt().ashr(ShiftAmt: GCD)));
7884 append_range(C&: MulOps, R: LHSMul->operands().drop_front());
7885 const SCEV *NewMul = getMulExpr(Ops&: MulOps, Flags: LHSMul->getNoWrapFlags());
7886 ShiftedLHS = getUDivExpr(LHS: NewMul, RHS: getConstant(Val: DivAmt));
7887 }
7888 }
7889 if (!ShiftedLHS)
7890 ShiftedLHS = getUDivExpr(LHS, RHS: MulCount);
7891 return getMulExpr(
7892 LHS: getZeroExtendExpr(
7893 Op: getTruncateExpr(Op: ShiftedLHS,
7894 Ty: IntegerType::get(C&: getContext(), NumBits: BitWidth - LZ - TZ)),
7895 Ty: BO->LHS->getType()),
7896 RHS: MulCount);
7897 }
7898 }
7899 // Binary `and` is a bit-wise `umin`.
7900 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
7901 LHS = getSCEV(V: BO->LHS);
7902 RHS = getSCEV(V: BO->RHS);
7903 return getUMinExpr(LHS, RHS);
7904 }
7905 break;
7906
7907 case Instruction::Or:
7908 // Binary `or` is a bit-wise `umax`.
7909 if (BO->LHS->getType()->isIntegerTy(BitWidth: 1)) {
7910 LHS = getSCEV(V: BO->LHS);
7911 RHS = getSCEV(V: BO->RHS);
7912 return getUMaxExpr(LHS, RHS);
7913 }
7914 break;
7915
7916 case Instruction::Xor:
7917 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS)) {
7918 // If the RHS of xor is -1, then this is a not operation.
7919 if (CI->isMinusOne())
7920 return getNotSCEV(V: getSCEV(V: BO->LHS));
7921
7922 // Model xor(and(x, C), C) as and(~x, C), if C is a low-bits mask.
7923 // This is a variant of the check for xor with -1, and it handles
7924 // the case where instcombine has trimmed non-demanded bits out
7925 // of an xor with -1.
7926 if (auto *LBO = dyn_cast<BinaryOperator>(Val: BO->LHS))
7927 if (ConstantInt *LCI = dyn_cast<ConstantInt>(Val: LBO->getOperand(i_nocapture: 1)))
7928 if (LBO->getOpcode() == Instruction::And &&
7929 LCI->getValue() == CI->getValue())
7930 if (const SCEVZeroExtendExpr *Z =
7931 dyn_cast<SCEVZeroExtendExpr>(Val: getSCEV(V: BO->LHS))) {
7932 Type *UTy = BO->LHS->getType();
7933 const SCEV *Z0 = Z->getOperand();
7934 Type *Z0Ty = Z0->getType();
7935 unsigned Z0TySize = getTypeSizeInBits(Ty: Z0Ty);
7936
7937 // If C is a low-bits mask, the zero extend is serving to
7938 // mask off the high bits. Complement the operand and
7939 // re-apply the zext.
7940 if (CI->getValue().isMask(numBits: Z0TySize))
7941 return getZeroExtendExpr(Op: getNotSCEV(V: Z0), Ty: UTy);
7942
7943 // If C is a single bit, it may be in the sign-bit position
7944 // before the zero-extend. In this case, represent the xor
7945 // using an add, which is equivalent, and re-apply the zext.
7946 APInt Trunc = CI->getValue().trunc(width: Z0TySize);
7947 if (Trunc.zext(width: getTypeSizeInBits(Ty: UTy)) == CI->getValue() &&
7948 Trunc.isSignMask())
7949 return getZeroExtendExpr(Op: getAddExpr(LHS: Z0, RHS: getConstant(Val: Trunc)),
7950 Ty: UTy);
7951 }
7952 }
7953 break;
7954
7955 case Instruction::Shl:
7956 // Turn shift left of a constant amount into a multiply.
7957 if (ConstantInt *SA = dyn_cast<ConstantInt>(Val: BO->RHS)) {
7958 uint32_t BitWidth = cast<IntegerType>(Val: SA->getType())->getBitWidth();
7959
7960 // If the shift count is not less than the bitwidth, the result of
7961 // the shift is undefined. Don't try to analyze it, because the
7962 // resolution chosen here may differ from the resolution chosen in
7963 // other parts of the compiler.
7964 if (SA->getValue().uge(RHS: BitWidth))
7965 break;
7966
7967 // We can safely preserve the nuw flag in all cases. It's also safe to
7968 // turn a nuw nsw shl into a nuw nsw mul. However, nsw in isolation
7969 // requires special handling. It can be preserved as long as we're not
7970 // left shifting by bitwidth - 1.
7971 auto Flags = SCEV::FlagNone;
7972 if (BO->Op) {
7973 auto MulFlags = getNoWrapFlagsFromUB(V: BO->Op);
7974 if (any(Val: MulFlags & SCEV::FlagNSW) &&
7975 (any(Val: MulFlags & SCEV::FlagNUW) ||
7976 SA->getValue().ult(RHS: BitWidth - 1)))
7977 Flags = Flags | SCEV::FlagNSW;
7978 if (any(Val: MulFlags & SCEV::FlagNUW))
7979 Flags = Flags | SCEV::FlagNUW;
7980 }
7981
7982 ConstantInt *X = ConstantInt::get(
7983 Context&: getContext(), V: APInt::getOneBitSet(numBits: BitWidth, BitNo: SA->getZExtValue()));
7984 return getMulExpr(LHS: getSCEV(V: BO->LHS), RHS: getConstant(V: X), Flags);
7985 }
7986 break;
7987
7988 case Instruction::AShr:
7989 // AShr X, C, where C is a constant.
7990 ConstantInt *CI = dyn_cast<ConstantInt>(Val: BO->RHS);
7991 if (!CI)
7992 break;
7993
7994 Type *OuterTy = BO->LHS->getType();
7995 uint64_t BitWidth = getTypeSizeInBits(Ty: OuterTy);
7996 // If the shift count is not less than the bitwidth, the result of
7997 // the shift is undefined. Don't try to analyze it, because the
7998 // resolution chosen here may differ from the resolution chosen in
7999 // other parts of the compiler.
8000 if (CI->getValue().uge(RHS: BitWidth))
8001 break;
8002
8003 if (CI->isZero())
8004 return getSCEV(V: BO->LHS); // shift by zero --> noop
8005
8006 uint64_t AShrAmt = CI->getZExtValue();
8007 Type *TruncTy = IntegerType::get(C&: getContext(), NumBits: BitWidth - AShrAmt);
8008
8009 Operator *L = dyn_cast<Operator>(Val: BO->LHS);
8010 const SCEV *AddTruncateExpr = nullptr;
8011 ConstantInt *ShlAmtCI = nullptr;
8012 const SCEV *AddConstant = nullptr;
8013
8014 if (L && L->getOpcode() == Instruction::Add) {
8015 // X = Shl A, n
8016 // Y = Add X, c
8017 // Z = AShr Y, m
8018 // n, c and m are constants.
8019
8020 Operator *LShift = dyn_cast<Operator>(Val: L->getOperand(i: 0));
8021 ConstantInt *AddOperandCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8022 if (LShift && LShift->getOpcode() == Instruction::Shl) {
8023 if (AddOperandCI) {
8024 const SCEV *ShlOp0SCEV = getSCEV(V: LShift->getOperand(i: 0));
8025 ShlAmtCI = dyn_cast<ConstantInt>(Val: LShift->getOperand(i: 1));
8026 // since we truncate to TruncTy, the AddConstant should be of the
8027 // same type, so create a new Constant with type same as TruncTy.
8028 // Also, the Add constant should be shifted right by AShr amount.
8029 APInt AddOperand = AddOperandCI->getValue().ashr(ShiftAmt: AShrAmt);
8030 AddConstant = getConstant(Val: AddOperand.trunc(width: BitWidth - AShrAmt));
8031 // we model the expression as sext(add(trunc(A), c << n)), since the
8032 // sext(trunc) part is already handled below, we create a
8033 // AddExpr(TruncExp) which will be used later.
8034 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8035 }
8036 }
8037 } else if (L && L->getOpcode() == Instruction::Shl) {
8038 // X = Shl A, n
8039 // Y = AShr X, m
8040 // Both n and m are constant.
8041
8042 const SCEV *ShlOp0SCEV = getSCEV(V: L->getOperand(i: 0));
8043 ShlAmtCI = dyn_cast<ConstantInt>(Val: L->getOperand(i: 1));
8044 AddTruncateExpr = getTruncateExpr(Op: ShlOp0SCEV, Ty: TruncTy);
8045 }
8046
8047 if (AddTruncateExpr && ShlAmtCI) {
8048 // We can merge the two given cases into a single SCEV statement,
8049 // incase n = m, the mul expression will be 2^0, so it gets resolved to
8050 // a simpler case. The following code handles the two cases:
8051 //
8052 // 1) For a two-shift sext-inreg, i.e. n = m,
8053 // use sext(trunc(x)) as the SCEV expression.
8054 //
8055 // 2) When n > m, use sext(mul(trunc(x), 2^(n-m)))) as the SCEV
8056 // expression. We already checked that ShlAmt < BitWidth, so
8057 // the multiplier, 1 << (ShlAmt - AShrAmt), fits into TruncTy as
8058 // ShlAmt - AShrAmt < Amt.
8059 const APInt &ShlAmt = ShlAmtCI->getValue();
8060 if (ShlAmt.ult(RHS: BitWidth) && ShlAmt.uge(RHS: AShrAmt)) {
8061 APInt Mul = APInt::getOneBitSet(numBits: BitWidth - AShrAmt,
8062 BitNo: ShlAmtCI->getZExtValue() - AShrAmt);
8063 const SCEV *CompositeExpr =
8064 getMulExpr(LHS: AddTruncateExpr, RHS: getConstant(Val: Mul));
8065 if (L->getOpcode() != Instruction::Shl)
8066 CompositeExpr = getAddExpr(LHS: CompositeExpr, RHS: AddConstant);
8067
8068 return getSignExtendExpr(Op: CompositeExpr, Ty: OuterTy);
8069 }
8070 }
8071 break;
8072 }
8073 }
8074
8075 switch (U->getOpcode()) {
8076 case Instruction::Trunc:
8077 return getTruncateExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8078
8079 case Instruction::ZExt:
8080 return getZeroExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8081
8082 case Instruction::SExt:
8083 if (auto BO = MatchBinaryOp(V: U->getOperand(i: 0), DT)) {
8084 // The NSW flag of a subtract does not always survive the conversion to
8085 // A + (-1)*B. By pushing sign extension onto its operands we are much
8086 // more likely to preserve NSW and allow later AddRec optimisations.
8087 //
8088 // NOTE: This is effectively duplicating this logic from getSignExtend:
8089 // sext((A + B + ...)<nsw>) --> (sext(A) + sext(B) + ...)<nsw>
8090 // but by that point the NSW information has potentially been lost.
8091 if (BO->Opcode == Instruction::Sub && BO->IsNSW) {
8092 Type *Ty = U->getType();
8093 auto *V1 = getSignExtendExpr(Op: getSCEV(V: BO->LHS), Ty);
8094 auto *V2 = getSignExtendExpr(Op: getSCEV(V: BO->RHS), Ty);
8095 return getMinusSCEV(LHS: V1, RHS: V2, Flags: SCEV::FlagNSW);
8096 }
8097 }
8098 return getSignExtendExpr(Op: getSCEV(V: U->getOperand(i: 0)), Ty: U->getType());
8099
8100 case Instruction::BitCast:
8101 // BitCasts are no-op casts so we just eliminate the cast.
8102 if (isSCEVable(Ty: U->getType()) && isSCEVable(Ty: U->getOperand(i: 0)->getType()))
8103 return getSCEV(V: U->getOperand(i: 0));
8104 break;
8105
8106 case Instruction::PtrToAddr: {
8107 const SCEV *IntOp = getPtrToAddrExpr(Op: getSCEV(V: U->getOperand(i: 0)));
8108 if (isa<SCEVCouldNotCompute>(Val: IntOp))
8109 return getUnknown(V);
8110 return IntOp;
8111 }
8112
8113 case Instruction::PtrToInt:
8114 // SCEV only models ptrtoaddr.
8115 return getUnknown(V);
8116
8117 case Instruction::IntToPtr:
8118 // Just don't deal with inttoptr casts.
8119 return getUnknown(V);
8120
8121 case Instruction::SDiv:
8122 // If both operands are non-negative, this is just an udiv.
8123 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8124 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8125 return getUDivExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8126 break;
8127
8128 case Instruction::SRem:
8129 // If both operands are non-negative, this is just an urem.
8130 if (isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 0))) &&
8131 isKnownNonNegative(S: getSCEV(V: U->getOperand(i: 1))))
8132 return getURemExpr(LHS: getSCEV(V: U->getOperand(i: 0)), RHS: getSCEV(V: U->getOperand(i: 1)));
8133 break;
8134
8135 case Instruction::GetElementPtr:
8136 return createNodeForGEP(GEP: cast<GEPOperator>(Val: U));
8137
8138 case Instruction::PHI:
8139 return createNodeForPHI(PN: cast<PHINode>(Val: U));
8140
8141 case Instruction::Select:
8142 return createNodeForSelectOrPHI(V: U, Cond: U->getOperand(i: 0), TrueVal: U->getOperand(i: 1),
8143 FalseVal: U->getOperand(i: 2));
8144
8145 case Instruction::Call:
8146 case Instruction::Invoke:
8147 if (Value *RV = cast<CallBase>(Val: U)->getReturnedArgOperand())
8148 return getSCEV(V: RV);
8149
8150 if (auto *II = dyn_cast<IntrinsicInst>(Val: U)) {
8151 switch (II->getIntrinsicID()) {
8152 case Intrinsic::abs:
8153 return getAbsExpr(
8154 Op: getSCEV(V: II->getArgOperand(i: 0)),
8155 /*IsNSW=*/cast<ConstantInt>(Val: II->getArgOperand(i: 1))->isOne());
8156 case Intrinsic::umax:
8157 LHS = getSCEV(V: II->getArgOperand(i: 0));
8158 RHS = getSCEV(V: II->getArgOperand(i: 1));
8159 return getUMaxExpr(LHS, RHS);
8160 case Intrinsic::umin:
8161 LHS = getSCEV(V: II->getArgOperand(i: 0));
8162 RHS = getSCEV(V: II->getArgOperand(i: 1));
8163 return getUMinExpr(LHS, RHS);
8164 case Intrinsic::smax:
8165 LHS = getSCEV(V: II->getArgOperand(i: 0));
8166 RHS = getSCEV(V: II->getArgOperand(i: 1));
8167 return getSMaxExpr(LHS, RHS);
8168 case Intrinsic::smin:
8169 LHS = getSCEV(V: II->getArgOperand(i: 0));
8170 RHS = getSCEV(V: II->getArgOperand(i: 1));
8171 return getSMinExpr(LHS, RHS);
8172 case Intrinsic::usub_sat: {
8173 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8174 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8175 const SCEV *ClampedY = getUMinExpr(LHS: X, RHS: Y);
8176 return getMinusSCEV(LHS: X, RHS: ClampedY, Flags: SCEV::FlagNUW);
8177 }
8178 case Intrinsic::uadd_sat: {
8179 const SCEV *X = getSCEV(V: II->getArgOperand(i: 0));
8180 const SCEV *Y = getSCEV(V: II->getArgOperand(i: 1));
8181 const SCEV *ClampedX = getUMinExpr(LHS: X, RHS: getNotSCEV(V: Y));
8182 return getAddExpr(LHS: ClampedX, RHS: Y, Flags: SCEV::FlagNUW);
8183 }
8184 case Intrinsic::start_loop_iterations:
8185 case Intrinsic::annotation:
8186 case Intrinsic::ptr_annotation:
8187 // A start_loop_iterations or llvm.annotation or llvm.prt.annotation is
8188 // just eqivalent to the first operand for SCEV purposes.
8189 return getSCEV(V: II->getArgOperand(i: 0));
8190 case Intrinsic::vscale:
8191 return getVScale(Ty: II->getType());
8192 default:
8193 break;
8194 }
8195 }
8196 break;
8197 }
8198
8199 return getUnknown(V);
8200}
8201
8202//===----------------------------------------------------------------------===//
8203// Iteration Count Computation Code
8204//
8205
8206const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount) {
8207 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8208 return getCouldNotCompute();
8209
8210 auto *ExitCountType = ExitCount->getType();
8211 assert(ExitCountType->isIntegerTy());
8212 auto *EvalTy = Type::getIntNTy(C&: ExitCountType->getContext(),
8213 N: 1 + ExitCountType->getScalarSizeInBits());
8214 return getTripCountFromExitCount(ExitCount, EvalTy, L: nullptr);
8215}
8216
8217const SCEV *ScalarEvolution::getTripCountFromExitCount(const SCEV *ExitCount,
8218 Type *EvalTy,
8219 const Loop *L) {
8220 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8221 return getCouldNotCompute();
8222
8223 unsigned ExitCountSize = getTypeSizeInBits(Ty: ExitCount->getType());
8224 unsigned EvalSize = EvalTy->getPrimitiveSizeInBits();
8225
8226 auto CanAddOneWithoutOverflow = [&]() {
8227 ConstantRange ExitCountRange =
8228 getRangeRef(S: ExitCount, SignHint: RangeSignHint::HINT_RANGE_UNSIGNED);
8229 if (!ExitCountRange.contains(Val: APInt::getMaxValue(numBits: ExitCountSize)))
8230 return true;
8231
8232 return L && isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: ExitCount,
8233 RHS: getMinusOne(Ty: ExitCount->getType()));
8234 };
8235
8236 // If we need to zero extend the backedge count, check if we can add one to
8237 // it prior to zero extending without overflow. Provided this is safe, it
8238 // allows better simplification of the +1.
8239 if (EvalSize > ExitCountSize && CanAddOneWithoutOverflow())
8240 return getZeroExtendExpr(
8241 Op: getAddExpr(LHS: ExitCount, RHS: getOne(Ty: ExitCount->getType())), Ty: EvalTy);
8242
8243 // Get the total trip count from the count by adding 1. This may wrap.
8244 return getAddExpr(LHS: getTruncateOrZeroExtend(V: ExitCount, Ty: EvalTy), RHS: getOne(Ty: EvalTy));
8245}
8246
8247static unsigned getConstantTripCount(const SCEVConstant *ExitCount) {
8248 if (!ExitCount)
8249 return 0;
8250
8251 ConstantInt *ExitConst = ExitCount->getValue();
8252
8253 // Guard against huge trip counts.
8254 if (ExitConst->getValue().getActiveBits() > 32)
8255 return 0;
8256
8257 // In case of integer overflow, this returns 0, which is correct.
8258 return ((unsigned)ExitConst->getZExtValue()) + 1;
8259}
8260
8261unsigned ScalarEvolution::getSmallConstantTripCount(const Loop *L) {
8262 auto *ExitCount = dyn_cast<SCEVConstant>(Val: getBackedgeTakenCount(L, Kind: Exact));
8263 return getConstantTripCount(ExitCount);
8264}
8265
8266unsigned
8267ScalarEvolution::getSmallConstantTripCount(const Loop *L,
8268 const BasicBlock *ExitingBlock) {
8269 assert(ExitingBlock && "Must pass a non-null exiting block!");
8270 assert(L->isLoopExiting(ExitingBlock) &&
8271 "Exiting block must actually branch out of the loop!");
8272 const SCEVConstant *ExitCount =
8273 dyn_cast<SCEVConstant>(Val: getExitCount(L, ExitingBlock));
8274 return getConstantTripCount(ExitCount);
8275}
8276
8277unsigned ScalarEvolution::getSmallConstantMaxTripCount(
8278 const Loop *L, SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8279
8280 const auto *MaxExitCount =
8281 Predicates ? getPredicatedConstantMaxBackedgeTakenCount(L, Predicates&: *Predicates)
8282 : getConstantMaxBackedgeTakenCount(L);
8283 return getConstantTripCount(ExitCount: dyn_cast<SCEVConstant>(Val: MaxExitCount));
8284}
8285
8286unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L) {
8287 SmallVector<BasicBlock *, 8> ExitingBlocks;
8288 L->getExitingBlocks(ExitingBlocks);
8289
8290 // An exit with an uncomputable exit count makes the result 1.
8291 if (ExitingBlocks.empty() ||
8292 any_of(Range&: ExitingBlocks, P: [this, L](BasicBlock *ExitingBB) {
8293 return isa<SCEVCouldNotCompute>(Val: getExitCount(L, ExitingBlock: ExitingBB));
8294 }))
8295 return 1;
8296
8297 LoopGuards Guards = LoopGuards::collectOnDemand(L, SE&: *this);
8298 unsigned Res = 0;
8299 for (BasicBlock *ExitingBB : ExitingBlocks)
8300 Res = std::gcd(
8301 m: Res, n: getSmallConstantTripMultiple(ExitCount: getExitCount(L, ExitingBlock: ExitingBB), Guards));
8302 return Res;
8303}
8304
8305unsigned ScalarEvolution::getSmallConstantTripMultiple(const SCEV *ExitCount,
8306 LoopGuards &Guards) {
8307 assert(!isa<SCEVCouldNotCompute>(ExitCount) && "Must be computable!");
8308
8309 // Get the trip count
8310 const SCEV *TCExpr =
8311 getTripCountFromExitCount(ExitCount: applyLoopGuards(Expr: ExitCount, Guards));
8312
8313 APInt Multiple = getNonZeroConstantMultiple(S: TCExpr);
8314 // If a trip multiple is huge (>=2^32), the trip count is still divisible by
8315 // the greatest power of 2 divisor less than 2^32.
8316 return Multiple.getActiveBits() > 32
8317 ? 1U << std::min(a: 31U, b: Multiple.countTrailingZeros())
8318 : (unsigned)Multiple.getZExtValue();
8319}
8320
8321unsigned ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8322 const SCEV *ExitCount) {
8323 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
8324 return 1;
8325
8326 LoopGuards Guards = LoopGuards::collectOnDemand(L, SE&: *this);
8327 return getSmallConstantTripMultiple(ExitCount, Guards);
8328}
8329
8330/// Returns the largest constant divisor of the trip count of this loop as a
8331/// normal unsigned value, if possible. This means that the actual trip count is
8332/// always a multiple of the returned value (don't forget the trip count could
8333/// very well be zero as well!).
8334///
8335/// Returns 1 if the trip count is unknown or not guaranteed to be the
8336/// multiple of a constant (which is also the case if the trip count is simply
8337/// constant, use getSmallConstantTripCount for that case), Will also return 1
8338/// if the trip count is very large (>= 2^32).
8339///
8340/// As explained in the comments for getSmallConstantTripCount, this assumes
8341/// that control exits the loop via ExitingBlock.
8342unsigned
8343ScalarEvolution::getSmallConstantTripMultiple(const Loop *L,
8344 const BasicBlock *ExitingBlock) {
8345 assert(ExitingBlock && "Must pass a non-null exiting block!");
8346 assert(L->isLoopExiting(ExitingBlock) &&
8347 "Exiting block must actually branch out of the loop!");
8348 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
8349 return getSmallConstantTripMultiple(L, ExitCount);
8350}
8351
8352const SCEV *ScalarEvolution::getExitCount(const Loop *L,
8353 const BasicBlock *ExitingBlock,
8354 ExitCountKind Kind) {
8355 switch (Kind) {
8356 case Exact:
8357 return getBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this);
8358 case SymbolicMaximum:
8359 return getBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this);
8360 case ConstantMaximum:
8361 return getBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this);
8362 };
8363 llvm_unreachable("Invalid ExitCountKind!");
8364}
8365
8366const SCEV *ScalarEvolution::getPredicatedExitCount(
8367 const Loop *L, const BasicBlock *ExitingBlock,
8368 SmallVectorImpl<const SCEVPredicate *> *Predicates, ExitCountKind Kind) {
8369 switch (Kind) {
8370 case Exact:
8371 return getPredicatedBackedgeTakenInfo(L).getExact(ExitingBlock, SE: this,
8372 Predicates);
8373 case SymbolicMaximum:
8374 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(ExitingBlock, SE: this,
8375 Predicates);
8376 case ConstantMaximum:
8377 return getPredicatedBackedgeTakenInfo(L).getConstantMax(ExitingBlock, SE: this,
8378 Predicates);
8379 };
8380 llvm_unreachable("Invalid ExitCountKind!");
8381}
8382
8383const SCEV *ScalarEvolution::getPredicatedBackedgeTakenCount(
8384 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8385 return getPredicatedBackedgeTakenInfo(L).getExact(L, SE: this, Predicates: &Preds);
8386}
8387
8388const SCEV *ScalarEvolution::getBackedgeTakenCount(const Loop *L,
8389 ExitCountKind Kind) {
8390 switch (Kind) {
8391 case Exact:
8392 return getBackedgeTakenInfo(L).getExact(L, SE: this);
8393 case ConstantMaximum:
8394 return getBackedgeTakenInfo(L).getConstantMax(SE: this);
8395 case SymbolicMaximum:
8396 return getBackedgeTakenInfo(L).getSymbolicMax(L, SE: this);
8397 };
8398 llvm_unreachable("Invalid ExitCountKind!");
8399}
8400
8401const SCEV *ScalarEvolution::getPredicatedSymbolicMaxBackedgeTakenCount(
8402 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8403 return getPredicatedBackedgeTakenInfo(L).getSymbolicMax(L, SE: this, Predicates: &Preds);
8404}
8405
8406const SCEV *ScalarEvolution::getPredicatedConstantMaxBackedgeTakenCount(
8407 const Loop *L, SmallVectorImpl<const SCEVPredicate *> &Preds) {
8408 return getPredicatedBackedgeTakenInfo(L).getConstantMax(SE: this, Predicates: &Preds);
8409}
8410
8411bool ScalarEvolution::isBackedgeTakenCountMaxOrZero(const Loop *L) {
8412 return getBackedgeTakenInfo(L).isConstantMaxOrZero(SE: this);
8413}
8414
8415/// Push PHI nodes in the header of the given loop onto the given Worklist.
8416static void PushLoopPHIs(const Loop *L,
8417 SmallVectorImpl<Instruction *> &Worklist,
8418 SmallPtrSetImpl<Instruction *> &Visited) {
8419 BasicBlock *Header = L->getHeader();
8420
8421 // Push all Loop-header PHIs onto the Worklist stack.
8422 for (PHINode &PN : Header->phis())
8423 if (Visited.insert(Ptr: &PN).second)
8424 Worklist.push_back(Elt: &PN);
8425}
8426
8427ScalarEvolution::BackedgeTakenInfo &
8428ScalarEvolution::getPredicatedBackedgeTakenInfo(const Loop *L) {
8429 auto &BTI = getBackedgeTakenInfo(L);
8430 if (BTI.hasFullInfo())
8431 return BTI;
8432
8433 auto Pair = PredicatedBackedgeTakenCounts.try_emplace(Key: L);
8434
8435 if (!Pair.second)
8436 return Pair.first->second;
8437
8438 BackedgeTakenInfo Result =
8439 computeBackedgeTakenCount(L, /*AllowPredicates=*/true);
8440
8441 return PredicatedBackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8442}
8443
8444ScalarEvolution::BackedgeTakenInfo &
8445ScalarEvolution::getBackedgeTakenInfo(const Loop *L) {
8446 // Initially insert an invalid entry for this loop. If the insertion
8447 // succeeds, proceed to actually compute a backedge-taken count and
8448 // update the value. The temporary CouldNotCompute value tells SCEV
8449 // code elsewhere that it shouldn't attempt to request a new
8450 // backedge-taken count, which could result in infinite recursion.
8451 std::pair<DenseMap<const Loop *, BackedgeTakenInfo>::iterator, bool> Pair =
8452 BackedgeTakenCounts.try_emplace(Key: L);
8453 if (!Pair.second)
8454 return Pair.first->second;
8455
8456 // computeBackedgeTakenCount may allocate memory for its result. Inserting it
8457 // into the BackedgeTakenCounts map transfers ownership. Otherwise, the result
8458 // must be cleared in this scope.
8459 BackedgeTakenInfo Result = computeBackedgeTakenCount(L);
8460
8461 // Now that we know more about the trip count for this loop, forget any
8462 // existing SCEV values for PHI nodes in this loop since they are only
8463 // conservative estimates made without the benefit of trip count
8464 // information. This invalidation is not necessary for correctness, and is
8465 // only done to produce more precise results.
8466 if (Result.hasAnyInfo()) {
8467 // Invalidate any expression using an addrec in this loop.
8468 SmallVector<SCEVUse, 8> ToForget;
8469 auto LoopUsersIt = LoopUsers.find(Val: L);
8470 if (LoopUsersIt != LoopUsers.end())
8471 append_range(C&: ToForget, R&: LoopUsersIt->second);
8472 forgetMemoizedResults(SCEVs: ToForget);
8473
8474 // Invalidate constant-evolved loop header phis.
8475 for (PHINode &PN : L->getHeader()->phis())
8476 ConstantEvolutionLoopExitValue.erase(Val: &PN);
8477 }
8478
8479 // Re-lookup the insert position, since the call to
8480 // computeBackedgeTakenCount above could result in a
8481 // recusive call to getBackedgeTakenInfo (on a different
8482 // loop), which would invalidate the iterator computed
8483 // earlier.
8484 return BackedgeTakenCounts.find(Val: L)->second = std::move(Result);
8485}
8486
8487void ScalarEvolution::forgetAllLoops() {
8488 // This method is intended to forget all info about loops. It should
8489 // invalidate caches as if the following happened:
8490 // - The trip counts of all loops have changed arbitrarily
8491 // - Every llvm::Value has been updated in place to produce a different
8492 // result.
8493 BackedgeTakenCounts.clear();
8494 PredicatedBackedgeTakenCounts.clear();
8495 BECountUsers.clear();
8496 LoopPropertiesCache.clear();
8497 ConstantEvolutionLoopExitValue.clear();
8498 ValueExprMap.clear();
8499 ValuesAtScopes.clear();
8500 ValuesAtScopesUsers.clear();
8501 LoopDispositions.clear();
8502 BlockDispositions.clear();
8503 UnsignedRanges.clear();
8504 SignedRanges.clear();
8505 ExprValueMap.clear();
8506 HasRecMap.clear();
8507 ConstantMultipleCache.clear();
8508 PredicatedSCEVRewrites.clear();
8509 FoldCache.clear();
8510 FoldCacheUser.clear();
8511}
8512void ScalarEvolution::visitAndClearUsers(
8513 SmallVectorImpl<Instruction *> &Worklist,
8514 SmallPtrSetImpl<Instruction *> &Visited,
8515 SmallVectorImpl<SCEVUse> &ToForget) {
8516 // Nothing can be invalidated if no value has a SCEV yet.
8517 if (ValueExprMap.empty()) {
8518 Worklist.clear();
8519 return;
8520 }
8521 while (!Worklist.empty()) {
8522 Instruction *I = Worklist.pop_back_val();
8523 if (!isSCEVable(Ty: I->getType()) && !isa<WithOverflowInst>(Val: I))
8524 continue;
8525
8526 ValueExprMapType::iterator It =
8527 ValueExprMap.find_as(Val: static_cast<Value *>(I));
8528 if (It != ValueExprMap.end()) {
8529 ToForget.push_back(Elt: It->second);
8530 eraseValueFromMap(V: It->first);
8531 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
8532 ConstantEvolutionLoopExitValue.erase(Val: PN);
8533 }
8534
8535 PushDefUseChildren(I, Worklist, Visited);
8536 }
8537}
8538
8539void ScalarEvolution::forgetLoop(const Loop *L) {
8540 SmallVector<const Loop *, 16> LoopWorklist(1, L);
8541 SmallVector<Instruction *, 32> Worklist;
8542 SmallPtrSet<Instruction *, 16> Visited;
8543 SmallVector<SCEVUse, 16> ToForget;
8544
8545 // Iterate over all the loops and sub-loops to drop SCEV information.
8546 while (!LoopWorklist.empty()) {
8547 auto *CurrL = LoopWorklist.pop_back_val();
8548
8549 // Drop any stored trip count value.
8550 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ false);
8551 forgetBackedgeTakenCounts(L: CurrL, /* Predicated */ true);
8552
8553 // Drop information about predicated SCEV rewrites for this loop.
8554 PredicatedSCEVRewrites.remove_if(
8555 Pred: [&](const auto &Entry) { return Entry.first.second == CurrL; });
8556
8557 auto LoopUsersItr = LoopUsers.find(Val: CurrL);
8558 if (LoopUsersItr != LoopUsers.end())
8559 llvm::append_range(C&: ToForget, R&: LoopUsersItr->second);
8560
8561 // Drop information about expressions based on loop-header PHIs.
8562 PushLoopPHIs(L: CurrL, Worklist, Visited);
8563 visitAndClearUsers(Worklist, Visited, ToForget);
8564
8565 LoopPropertiesCache.erase(Val: CurrL);
8566 // Forget all contained loops too, to avoid dangling entries in the
8567 // ValuesAtScopes map.
8568 LoopWorklist.append(in_start: CurrL->begin(), in_end: CurrL->end());
8569 }
8570 forgetMemoizedResults(SCEVs: ToForget);
8571}
8572
8573void ScalarEvolution::forgetTopmostLoop(const Loop *L) {
8574 forgetLoop(L: L->getOutermostLoop());
8575}
8576
8577void ScalarEvolution::forgetValue(Value *V) {
8578 Instruction *I = dyn_cast<Instruction>(Val: V);
8579 if (!I) return;
8580
8581 // Drop information about expressions based on loop-header PHIs.
8582 SmallVector<Instruction *, 16> Worklist;
8583 SmallPtrSet<Instruction *, 8> Visited;
8584 SmallVector<SCEVUse, 8> ToForget;
8585 Worklist.push_back(Elt: I);
8586 Visited.insert(Ptr: I);
8587 visitAndClearUsers(Worklist, Visited, ToForget);
8588
8589 forgetMemoizedResults(SCEVs: ToForget);
8590}
8591
8592void ScalarEvolution::forgetValues(ArrayRef<Value *> Values) {
8593 SmallVector<Instruction *, 16> Worklist;
8594 SmallPtrSet<Instruction *, 8> Visited;
8595 SmallVector<SCEVUse, 8> ToForget;
8596 for (Value *V : Values)
8597 if (auto *I = dyn_cast<Instruction>(Val: V))
8598 if (Visited.insert(Ptr: I).second)
8599 Worklist.push_back(Elt: I);
8600 visitAndClearUsers(Worklist, Visited, ToForget);
8601
8602 forgetMemoizedResults(SCEVs: ToForget);
8603}
8604
8605void ScalarEvolution::forgetLcssaPhiWithNewPredecessor(Loop *L, PHINode *V) {
8606 // If SCEV looked through a trivial LCSSA phi node, we might have SCEV's
8607 // directly using a SCEVUnknown/SCEVAddRec defined in the loop. After an
8608 // extra predecessor is added, this is no longer valid. Find all Unknowns and
8609 // AddRecs defined in the loop and invalidate any SCEV's making use of them.
8610 auto InvalidateValue = [&](Value *Val) {
8611 if (!isSCEVable(Ty: Val->getType()))
8612 return;
8613 if (const SCEV *S = getExistingSCEV(V: Val)) {
8614 struct InvalidationRootCollector {
8615 Loop *L;
8616 SmallVector<SCEVUse, 8> Roots;
8617
8618 InvalidationRootCollector(Loop *L) : L(L) {}
8619
8620 bool follow(const SCEV *S) {
8621 if (auto *SU = dyn_cast<SCEVUnknown>(Val: S)) {
8622 if (auto *I = dyn_cast<Instruction>(Val: SU->getValue()))
8623 if (L->contains(Inst: I))
8624 Roots.push_back(Elt: S);
8625 } else if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S)) {
8626 if (L->contains(L: AddRec->getLoop()))
8627 Roots.push_back(Elt: S);
8628 }
8629 return true;
8630 }
8631 bool isDone() const { return false; }
8632 };
8633
8634 InvalidationRootCollector C(L);
8635 visitAll(Root: S, Visitor&: C);
8636 forgetMemoizedResults(SCEVs: C.Roots);
8637 }
8638 };
8639
8640 InvalidateValue(V);
8641
8642 // If V has a non-SCEV-able type (e.g. {i64, i1} from a with.overflow
8643 // intrinsic), its users (e.g. extractvalue) may have stale SCEV
8644 // expressions referencing loop-internal values.
8645 if (!isSCEVable(Ty: V->getType()) &&
8646 any_of(Range: V->incoming_values(), P: IsaPred<WithOverflowInst>))
8647 for (User *U : V->users())
8648 InvalidateValue(U);
8649 // Also perform the normal invalidation.
8650 forgetValue(V);
8651}
8652
8653void ScalarEvolution::forgetLoopDispositions() { LoopDispositions.clear(); }
8654
8655void ScalarEvolution::forgetBlockAndLoopDispositions(Value *V) {
8656 // Unless a specific value is passed to invalidation, completely clear both
8657 // caches.
8658 if (!V) {
8659 BlockDispositions.clear();
8660 LoopDispositions.clear();
8661 return;
8662 }
8663
8664 if (!isSCEVable(Ty: V->getType()))
8665 return;
8666
8667 const SCEV *S = getExistingSCEV(V);
8668 if (!S)
8669 return;
8670
8671 // Invalidate the block and loop dispositions cached for S. Dispositions of
8672 // S's users may change if S's disposition changes (i.e. a user may change to
8673 // loop-invariant, if S changes to loop invariant), so also invalidate
8674 // dispositions of S's users recursively.
8675 SmallVector<SCEVUse, 8> Worklist = {S};
8676 SmallPtrSet<const SCEV *, 8> Seen = {S};
8677 while (!Worklist.empty()) {
8678 const SCEV *Curr = Worklist.pop_back_val();
8679 bool LoopDispoRemoved = LoopDispositions.erase(Val: Curr);
8680 bool BlockDispoRemoved = BlockDispositions.erase(Val: Curr);
8681 if (!LoopDispoRemoved && !BlockDispoRemoved)
8682 continue;
8683 auto Users = SCEVUsers.find(Val: Curr);
8684 if (Users != SCEVUsers.end())
8685 for (const auto *User : Users->second)
8686 if (Seen.insert(Ptr: User).second)
8687 Worklist.push_back(Elt: User);
8688 }
8689}
8690
8691/// Get the exact loop backedge taken count considering all loop exits. A
8692/// computable result can only be returned for loops with all exiting blocks
8693/// dominating the latch. howFarToZero assumes that the limit of each loop test
8694/// is never skipped. This is a valid assumption as long as the loop exits via
8695/// that test. For precise results, it is the caller's responsibility to specify
8696/// the relevant loop exiting block using getExact(ExitingBlock, SE).
8697const SCEV *ScalarEvolution::BackedgeTakenInfo::getExact(
8698 const Loop *L, ScalarEvolution *SE,
8699 SmallVectorImpl<const SCEVPredicate *> *Preds) const {
8700 // If any exits were not computable, the loop is not computable.
8701 if (!isComplete() || ExitNotTaken.empty())
8702 return SE->getCouldNotCompute();
8703
8704 const BasicBlock *Latch = L->getLoopLatch();
8705 // All exiting blocks we have collected must dominate the only backedge.
8706 if (!Latch)
8707 return SE->getCouldNotCompute();
8708
8709 // All exiting blocks we have gathered dominate loop's latch, so exact trip
8710 // count is simply a minimum out of all these calculated exit counts.
8711 SmallVector<SCEVUse, 2> Ops;
8712 for (const auto &ENT : ExitNotTaken) {
8713 const SCEV *BECount = ENT.ExactNotTaken;
8714 assert(BECount != SE->getCouldNotCompute() && "Bad exit SCEV!");
8715 assert(SE->DT.dominates(ENT.ExitingBlock, Latch) &&
8716 "We should only have known counts for exiting blocks that dominate "
8717 "latch!");
8718
8719 Ops.push_back(Elt: BECount);
8720
8721 if (Preds)
8722 append_range(C&: *Preds, R: ENT.Predicates);
8723
8724 assert((Preds || ENT.hasAlwaysTruePredicate()) &&
8725 "Predicate should be always true!");
8726 }
8727
8728 // If an earlier exit exits on the first iteration (exit count zero), then
8729 // a later poison exit count should not propagate into the result. This are
8730 // exactly the semantics provided by umin_seq.
8731 return SE->getUMinFromMismatchedTypes(Ops, /* Sequential */ true);
8732}
8733
8734const ScalarEvolution::ExitNotTakenInfo *
8735ScalarEvolution::BackedgeTakenInfo::getExitNotTaken(
8736 const BasicBlock *ExitingBlock,
8737 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8738 for (const auto &ENT : ExitNotTaken)
8739 if (ENT.ExitingBlock == ExitingBlock) {
8740 if (ENT.hasAlwaysTruePredicate())
8741 return &ENT;
8742 else if (Predicates) {
8743 append_range(C&: *Predicates, R: ENT.Predicates);
8744 return &ENT;
8745 }
8746 }
8747
8748 return nullptr;
8749}
8750
8751/// getConstantMax - Get the constant max backedge taken count for the loop.
8752const SCEV *ScalarEvolution::BackedgeTakenInfo::getConstantMax(
8753 ScalarEvolution *SE,
8754 SmallVectorImpl<const SCEVPredicate *> *Predicates) const {
8755 if (!getConstantMax())
8756 return SE->getCouldNotCompute();
8757
8758 for (const auto &ENT : ExitNotTaken)
8759 if (!ENT.hasAlwaysTruePredicate()) {
8760 if (!Predicates)
8761 return SE->getCouldNotCompute();
8762 append_range(C&: *Predicates, R: ENT.Predicates);
8763 }
8764
8765 assert((isa<SCEVCouldNotCompute>(getConstantMax()) ||
8766 isa<SCEVConstant>(getConstantMax())) &&
8767 "No point in having a non-constant max backedge taken count!");
8768 return getConstantMax();
8769}
8770
8771const SCEV *ScalarEvolution::BackedgeTakenInfo::getSymbolicMax(
8772 const Loop *L, ScalarEvolution *SE,
8773 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
8774 if (!SymbolicMax) {
8775 // Form an expression for the maximum exit count possible for this loop. We
8776 // merge the max and exact information to approximate a version of
8777 // getConstantMaxBackedgeTakenCount which isn't restricted to just
8778 // constants.
8779 SmallVector<SCEVUse, 4> ExitCounts;
8780
8781 for (const auto &ENT : ExitNotTaken) {
8782 const SCEV *ExitCount = ENT.SymbolicMaxNotTaken;
8783 if (!isa<SCEVCouldNotCompute>(Val: ExitCount)) {
8784 assert(SE->DT.dominates(ENT.ExitingBlock, L->getLoopLatch()) &&
8785 "We should only have known counts for exiting blocks that "
8786 "dominate latch!");
8787 ExitCounts.push_back(Elt: ExitCount);
8788 if (Predicates)
8789 append_range(C&: *Predicates, R: ENT.Predicates);
8790
8791 assert((Predicates || ENT.hasAlwaysTruePredicate()) &&
8792 "Predicate should be always true!");
8793 }
8794 }
8795 if (ExitCounts.empty())
8796 SymbolicMax = SE->getCouldNotCompute();
8797 else
8798 SymbolicMax =
8799 SE->getUMinFromMismatchedTypes(Ops&: ExitCounts, /*Sequential*/ true);
8800 }
8801 return SymbolicMax;
8802}
8803
8804bool ScalarEvolution::BackedgeTakenInfo::isConstantMaxOrZero(
8805 ScalarEvolution *SE) const {
8806 auto PredicateNotAlwaysTrue = [](const ExitNotTakenInfo &ENT) {
8807 return !ENT.hasAlwaysTruePredicate();
8808 };
8809 return MaxOrZero && !any_of(Range: ExitNotTaken, P: PredicateNotAlwaysTrue);
8810}
8811
8812ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E)
8813 : ExitLimit(E, E, E, false) {}
8814
8815ScalarEvolution::ExitLimit::ExitLimit(
8816 const SCEV *E, const SCEV *ConstantMaxNotTaken,
8817 const SCEV *SymbolicMaxNotTaken, bool MaxOrZero,
8818 ArrayRef<ArrayRef<const SCEVPredicate *>> PredLists)
8819 : ExactNotTaken(E), ConstantMaxNotTaken(ConstantMaxNotTaken),
8820 SymbolicMaxNotTaken(SymbolicMaxNotTaken), MaxOrZero(MaxOrZero) {
8821 // If we prove the max count is zero, so is the symbolic bound. This happens
8822 // in practice due to differences in a) how context sensitive we've chosen
8823 // to be and b) how we reason about bounds implied by UB.
8824 if (ConstantMaxNotTaken->isZero()) {
8825 this->ExactNotTaken = E = ConstantMaxNotTaken;
8826 this->SymbolicMaxNotTaken = SymbolicMaxNotTaken = ConstantMaxNotTaken;
8827 }
8828
8829 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
8830 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
8831 "Exact is not allowed to be less precise than Constant Max");
8832 assert((isa<SCEVCouldNotCompute>(ExactNotTaken) ||
8833 !isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken)) &&
8834 "Exact is not allowed to be less precise than Symbolic Max");
8835 assert((isa<SCEVCouldNotCompute>(SymbolicMaxNotTaken) ||
8836 !isa<SCEVCouldNotCompute>(ConstantMaxNotTaken)) &&
8837 "Symbolic Max is not allowed to be less precise than Constant Max");
8838 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
8839 isa<SCEVConstant>(ConstantMaxNotTaken)) &&
8840 "No point in having a non-constant max backedge taken count!");
8841 SmallPtrSet<const SCEVPredicate *, 4> SeenPreds;
8842 for (const auto PredList : PredLists)
8843 for (const auto *P : PredList) {
8844 if (SeenPreds.contains(Ptr: P))
8845 continue;
8846 assert(!isa<SCEVUnionPredicate>(P) && "Only add leaf predicates here!");
8847 SeenPreds.insert(Ptr: P);
8848 Predicates.push_back(Elt: P);
8849 }
8850 assert((isa<SCEVCouldNotCompute>(E) || !E->getType()->isPointerTy()) &&
8851 "Backedge count should be int");
8852 assert((isa<SCEVCouldNotCompute>(ConstantMaxNotTaken) ||
8853 !ConstantMaxNotTaken->getType()->isPointerTy()) &&
8854 "Max backedge count should be int");
8855}
8856
8857ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E,
8858 const SCEV *ConstantMaxNotTaken,
8859 const SCEV *SymbolicMaxNotTaken,
8860 bool MaxOrZero,
8861 ArrayRef<const SCEVPredicate *> PredList)
8862 : ExitLimit(E, ConstantMaxNotTaken, SymbolicMaxNotTaken, MaxOrZero,
8863 ArrayRef({PredList})) {}
8864
8865/// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each
8866/// computable exit into a persistent ExitNotTakenInfo array.
8867ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo(
8868 ArrayRef<ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo> ExitCounts,
8869 bool IsComplete, const SCEV *ConstantMax, bool MaxOrZero)
8870 : ConstantMax(ConstantMax), IsComplete(IsComplete), MaxOrZero(MaxOrZero) {
8871 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8872
8873 ExitNotTaken.reserve(N: ExitCounts.size());
8874 std::transform(first: ExitCounts.begin(), last: ExitCounts.end(),
8875 result: std::back_inserter(x&: ExitNotTaken),
8876 unary_op: [&](const EdgeExitInfo &EEI) {
8877 BasicBlock *ExitBB = EEI.first;
8878 const ExitLimit &EL = EEI.second;
8879 return ExitNotTakenInfo(ExitBB, EL.ExactNotTaken,
8880 EL.ConstantMaxNotTaken, EL.SymbolicMaxNotTaken,
8881 EL.Predicates);
8882 });
8883 assert((isa<SCEVCouldNotCompute>(ConstantMax) ||
8884 isa<SCEVConstant>(ConstantMax)) &&
8885 "No point in having a non-constant max backedge taken count!");
8886}
8887
8888/// Compute the number of times the backedge of the specified loop will execute.
8889ScalarEvolution::BackedgeTakenInfo
8890ScalarEvolution::computeBackedgeTakenCount(const Loop *L,
8891 bool AllowPredicates) {
8892 SmallVector<BasicBlock *, 8> ExitingBlocks;
8893 L->getExitingBlocks(ExitingBlocks);
8894
8895 using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo;
8896
8897 SmallVector<EdgeExitInfo, 4> ExitCounts;
8898 bool CouldComputeBECount = true;
8899 BasicBlock *Latch = L->getLoopLatch(); // may be NULL.
8900 const SCEV *MustExitMaxBECount = nullptr;
8901 const SCEV *MayExitMaxBECount = nullptr;
8902 bool MustExitMaxOrZero = false;
8903 bool IsOnlyExit = ExitingBlocks.size() == 1;
8904
8905 // Compute the ExitLimit for each loop exit. Use this to populate ExitCounts
8906 // and compute maxBECount.
8907 // Do a union of all the predicates here.
8908 for (BasicBlock *ExitBB : ExitingBlocks) {
8909 // We canonicalize untaken exits to br (constant), ignore them so that
8910 // proving an exit untaken doesn't negatively impact our ability to reason
8911 // about the loop as whole.
8912 if (auto *BI = dyn_cast<CondBrInst>(Val: ExitBB->getTerminator()))
8913 if (auto *CI = dyn_cast<ConstantInt>(Val: BI->getCondition())) {
8914 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
8915 if (ExitIfTrue == CI->isZero())
8916 continue;
8917 }
8918
8919 ExitLimit EL = computeExitLimit(L, ExitingBlock: ExitBB, IsOnlyExit, AllowPredicates);
8920
8921 assert((AllowPredicates || EL.Predicates.empty()) &&
8922 "Predicated exit limit when predicates are not allowed!");
8923
8924 // 1. For each exit that can be computed, add an entry to ExitCounts.
8925 // CouldComputeBECount is true only if all exits can be computed.
8926 if (EL.ExactNotTaken != getCouldNotCompute())
8927 ++NumExitCountsComputed;
8928 else
8929 // We couldn't compute an exact value for this exit, so
8930 // we won't be able to compute an exact value for the loop.
8931 CouldComputeBECount = false;
8932 // Remember exit count if either exact or symbolic is known. Because
8933 // Exact always implies symbolic, only check symbolic.
8934 if (EL.SymbolicMaxNotTaken != getCouldNotCompute())
8935 ExitCounts.emplace_back(Args&: ExitBB, Args&: EL);
8936 else {
8937 assert(EL.ExactNotTaken == getCouldNotCompute() &&
8938 "Exact is known but symbolic isn't?");
8939 ++NumExitCountsNotComputed;
8940 }
8941
8942 // 2. Derive the loop's MaxBECount from each exit's max number of
8943 // non-exiting iterations. Partition the loop exits into two kinds:
8944 // LoopMustExits and LoopMayExits.
8945 //
8946 // If the exit dominates the loop latch, it is a LoopMustExit otherwise it
8947 // is a LoopMayExit. If any computable LoopMustExit is found, then
8948 // MaxBECount is the minimum EL.ConstantMaxNotTaken of computable
8949 // LoopMustExits. Otherwise, MaxBECount is conservatively the maximum
8950 // EL.ConstantMaxNotTaken, where CouldNotCompute is considered greater than
8951 // any
8952 // computable EL.ConstantMaxNotTaken.
8953 if (EL.ConstantMaxNotTaken != getCouldNotCompute() && Latch &&
8954 DT.dominates(A: ExitBB, B: Latch)) {
8955 if (!MustExitMaxBECount) {
8956 MustExitMaxBECount = EL.ConstantMaxNotTaken;
8957 MustExitMaxOrZero = EL.MaxOrZero;
8958 } else {
8959 MustExitMaxBECount = getUMinFromMismatchedTypes(LHS: MustExitMaxBECount,
8960 RHS: EL.ConstantMaxNotTaken);
8961 }
8962 } else if (MayExitMaxBECount != getCouldNotCompute()) {
8963 if (!MayExitMaxBECount || EL.ConstantMaxNotTaken == getCouldNotCompute())
8964 MayExitMaxBECount = EL.ConstantMaxNotTaken;
8965 else {
8966 MayExitMaxBECount = getUMaxFromMismatchedTypes(LHS: MayExitMaxBECount,
8967 RHS: EL.ConstantMaxNotTaken);
8968 }
8969 }
8970 }
8971 const SCEV *MaxBECount = MustExitMaxBECount ? MustExitMaxBECount :
8972 (MayExitMaxBECount ? MayExitMaxBECount : getCouldNotCompute());
8973 // The loop backedge will be taken the maximum or zero times if there's
8974 // a single exit that must be taken the maximum or zero times.
8975 bool MaxOrZero = (MustExitMaxOrZero && ExitingBlocks.size() == 1);
8976
8977 // Remember which SCEVs are used in exit limits for invalidation purposes.
8978 // We only care about non-constant SCEVs here, so we can ignore
8979 // EL.ConstantMaxNotTaken
8980 // and MaxBECount, which must be SCEVConstant.
8981 for (const auto &Pair : ExitCounts) {
8982 if (!isa<SCEVConstant>(Val: Pair.second.ExactNotTaken))
8983 BECountUsers[Pair.second.ExactNotTaken].insert(Ptr: {L, AllowPredicates});
8984 if (!isa<SCEVConstant>(Val: Pair.second.SymbolicMaxNotTaken))
8985 BECountUsers[Pair.second.SymbolicMaxNotTaken].insert(
8986 Ptr: {L, AllowPredicates});
8987 }
8988 return BackedgeTakenInfo(std::move(ExitCounts), CouldComputeBECount,
8989 MaxBECount, MaxOrZero);
8990}
8991
8992ScalarEvolution::ExitLimit
8993ScalarEvolution::computeExitLimit(const Loop *L, BasicBlock *ExitingBlock,
8994 bool IsOnlyExit, bool AllowPredicates) {
8995 assert(L->contains(ExitingBlock) && "Exit count for non-loop block?");
8996 // If our exiting block does not dominate the latch, then its connection with
8997 // loop's exit limit may be far from trivial.
8998 const BasicBlock *Latch = L->getLoopLatch();
8999 if (!Latch || !DT.dominates(A: ExitingBlock, B: Latch))
9000 return getCouldNotCompute();
9001
9002 Instruction *Term = ExitingBlock->getTerminator();
9003 if (CondBrInst *BI = dyn_cast<CondBrInst>(Val: Term)) {
9004 bool ExitIfTrue = !L->contains(BB: BI->getSuccessor(i: 0));
9005 assert(ExitIfTrue == L->contains(BI->getSuccessor(1)) &&
9006 "It should have one successor in loop and one exit block!");
9007 // Proceed to the next level to examine the exit condition expression.
9008 return computeExitLimitFromCond(L, ExitCond: BI->getCondition(), ExitIfTrue,
9009 /*ControlsOnlyExit=*/IsOnlyExit,
9010 AllowPredicates);
9011 }
9012
9013 if (SwitchInst *SI = dyn_cast<SwitchInst>(Val: Term)) {
9014 // For switch, make sure that there is a single exit from the loop.
9015 BasicBlock *Exit = nullptr;
9016 for (auto *SBB : successors(BB: ExitingBlock))
9017 if (!L->contains(BB: SBB)) {
9018 if (Exit) // Multiple exit successors.
9019 return getCouldNotCompute();
9020 Exit = SBB;
9021 }
9022 assert(Exit && "Exiting block must have at least one exit");
9023 return computeExitLimitFromSingleExitSwitch(
9024 L, Switch: SI, ExitingBB: Exit, /*ControlsOnlyExit=*/IsSubExpr: IsOnlyExit);
9025 }
9026
9027 return getCouldNotCompute();
9028}
9029
9030ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCond(
9031 const Loop *L, Value *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9032 bool AllowPredicates) {
9033 ScalarEvolution::ExitLimitCacheTy Cache(L, ExitIfTrue, AllowPredicates);
9034 return computeExitLimitFromCondCached(Cache, L, ExitCond, ExitIfTrue,
9035 ControlsOnlyExit, AllowPredicates);
9036}
9037
9038std::optional<ScalarEvolution::ExitLimit>
9039ScalarEvolution::ExitLimitCache::find(const Loop *L, Value *ExitCond,
9040 bool ExitIfTrue, bool ControlsOnlyExit,
9041 bool AllowPredicates) {
9042 (void)this->L;
9043 (void)this->ExitIfTrue;
9044 (void)this->AllowPredicates;
9045
9046 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9047 this->AllowPredicates == AllowPredicates &&
9048 "Variance in assumed invariant key components!");
9049 auto Itr = TripCountMap.find(Val: {ExitCond, ControlsOnlyExit});
9050 if (Itr == TripCountMap.end())
9051 return std::nullopt;
9052 return Itr->second;
9053}
9054
9055void ScalarEvolution::ExitLimitCache::insert(const Loop *L, Value *ExitCond,
9056 bool ExitIfTrue,
9057 bool ControlsOnlyExit,
9058 bool AllowPredicates,
9059 const ExitLimit &EL) {
9060 assert(this->L == L && this->ExitIfTrue == ExitIfTrue &&
9061 this->AllowPredicates == AllowPredicates &&
9062 "Variance in assumed invariant key components!");
9063
9064 auto InsertResult = TripCountMap.insert(KV: {{ExitCond, ControlsOnlyExit}, EL});
9065 assert(InsertResult.second && "Expected successful insertion!");
9066 (void)InsertResult;
9067 (void)ExitIfTrue;
9068}
9069
9070ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondCached(
9071 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9072 bool ControlsOnlyExit, bool AllowPredicates) {
9073
9074 if (auto MaybeEL = Cache.find(L, ExitCond, ExitIfTrue, ControlsOnlyExit,
9075 AllowPredicates))
9076 return *MaybeEL;
9077
9078 ExitLimit EL = computeExitLimitFromCondImpl(
9079 Cache, L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates);
9080 Cache.insert(L, ExitCond, ExitIfTrue, ControlsOnlyExit, AllowPredicates, EL);
9081 return EL;
9082}
9083
9084ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl(
9085 ExitLimitCacheTy &Cache, const Loop *L, Value *ExitCond, bool ExitIfTrue,
9086 bool ControlsOnlyExit, bool AllowPredicates) {
9087 // Handle BinOp conditions (And, Or).
9088 if (auto LimitFromBinOp = computeExitLimitFromCondFromBinOp(
9089 Cache, L, ExitCond, ExitIfTrue, AllowPredicates))
9090 return *LimitFromBinOp;
9091
9092 // With an icmp, it may be feasible to compute an exact backedge-taken count.
9093 // Proceed to the next level to examine the icmp.
9094 if (ICmpInst *ExitCondICmp = dyn_cast<ICmpInst>(Val: ExitCond)) {
9095 ExitLimit EL =
9096 computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue, IsSubExpr: ControlsOnlyExit);
9097 if (EL.hasFullInfo() || !AllowPredicates)
9098 return EL;
9099
9100 // Try again, but use SCEV predicates this time.
9101 return computeExitLimitFromICmp(L, ExitCond: ExitCondICmp, ExitIfTrue,
9102 IsSubExpr: ControlsOnlyExit,
9103 /*AllowPredicates=*/true);
9104 }
9105
9106 // Check for a constant condition. These are normally stripped out by
9107 // SimplifyCFG, but ScalarEvolution may be used by a pass which wishes to
9108 // preserve the CFG and is temporarily leaving constant conditions
9109 // in place.
9110 if (ConstantInt *CI = dyn_cast<ConstantInt>(Val: ExitCond)) {
9111 if (ExitIfTrue == !CI->getZExtValue())
9112 // The backedge is always taken.
9113 return getCouldNotCompute();
9114 // The backedge is never taken.
9115 return getZero(Ty: CI->getType());
9116 }
9117
9118 // If we're exiting based on the overflow flag of an x.with.overflow intrinsic
9119 // with a constant step, we can form an equivalent icmp predicate and figure
9120 // out how many iterations will be taken before we exit.
9121 const WithOverflowInst *WO;
9122 const APInt *C;
9123 if (match(V: ExitCond, P: m_ExtractValue<1>(V: m_WithOverflowInst(I&: WO))) &&
9124 match(V: WO->getRHS(), P: m_APInt(Res&: C))) {
9125 ConstantRange NWR =
9126 ConstantRange::makeExactNoWrapRegion(BinOp: WO->getBinaryOp(), Other: *C,
9127 NoWrapKind: WO->getNoWrapKind());
9128 CmpInst::Predicate Pred;
9129 APInt NewRHSC, Offset;
9130 NWR.getEquivalentICmp(Pred, RHS&: NewRHSC, Offset);
9131 if (!ExitIfTrue)
9132 Pred = ICmpInst::getInversePredicate(pred: Pred);
9133 auto *LHS = getSCEV(V: WO->getLHS());
9134 if (Offset != 0)
9135 LHS = getAddExpr(LHS, RHS: getConstant(Val: Offset));
9136 auto EL = computeExitLimitFromICmp(L, Pred, LHS, RHS: getConstant(Val: NewRHSC),
9137 IsSubExpr: ControlsOnlyExit, AllowPredicates);
9138 if (EL.hasAnyInfo())
9139 return EL;
9140 }
9141
9142 // If it's not an integer or pointer comparison then compute it the hard way.
9143 return computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9144}
9145
9146std::optional<ScalarEvolution::ExitLimit>
9147ScalarEvolution::computeExitLimitFromCondFromBinOp(ExitLimitCacheTy &Cache,
9148 const Loop *L,
9149 Value *ExitCond,
9150 bool ExitIfTrue,
9151 bool AllowPredicates) {
9152 // Check if the controlling expression for this loop is an And or Or.
9153 Value *Op0, *Op1;
9154 bool IsAnd;
9155 if (match(V: ExitCond, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9156 IsAnd = true;
9157 else if (match(V: ExitCond, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))
9158 IsAnd = false;
9159 else
9160 return std::nullopt;
9161
9162 // A sub-condition of a non-trivial binop never solely controls the exit,
9163 // whether we exit always depends on both conditions.
9164 ExitLimit EL0 = computeExitLimitFromCondCached(
9165 Cache, L, ExitCond: Op0, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9166 ExitLimit EL1 = computeExitLimitFromCondCached(
9167 Cache, L, ExitCond: Op1, ExitIfTrue, /*ControlsOnlyExit=*/false, AllowPredicates);
9168
9169 // EitherMayExit is true in these two cases:
9170 // br (and Op0 Op1), loop, exit
9171 // br (or Op0 Op1), exit, loop
9172 bool EitherMayExit = IsAnd ^ ExitIfTrue;
9173
9174 const SCEV *BECount = getCouldNotCompute();
9175 const SCEV *ConstantMaxBECount = getCouldNotCompute();
9176 const SCEV *SymbolicMaxBECount = getCouldNotCompute();
9177 if (EitherMayExit) {
9178 bool UseSequentialUMin = !isa<BinaryOperator>(Val: ExitCond);
9179 // Both conditions must be same for the loop to continue executing.
9180 // Choose the less conservative count.
9181 if (EL0.ExactNotTaken != getCouldNotCompute() &&
9182 EL1.ExactNotTaken != getCouldNotCompute()) {
9183 BECount = getUMinFromMismatchedTypes(LHS: EL0.ExactNotTaken, RHS: EL1.ExactNotTaken,
9184 Sequential: UseSequentialUMin);
9185 }
9186 if (EL0.ConstantMaxNotTaken == getCouldNotCompute())
9187 ConstantMaxBECount = EL1.ConstantMaxNotTaken;
9188 else if (EL1.ConstantMaxNotTaken == getCouldNotCompute())
9189 ConstantMaxBECount = EL0.ConstantMaxNotTaken;
9190 else
9191 ConstantMaxBECount = getUMinFromMismatchedTypes(LHS: EL0.ConstantMaxNotTaken,
9192 RHS: EL1.ConstantMaxNotTaken);
9193 if (EL0.SymbolicMaxNotTaken == getCouldNotCompute())
9194 SymbolicMaxBECount = EL1.SymbolicMaxNotTaken;
9195 else if (EL1.SymbolicMaxNotTaken == getCouldNotCompute())
9196 SymbolicMaxBECount = EL0.SymbolicMaxNotTaken;
9197 else
9198 SymbolicMaxBECount = getUMinFromMismatchedTypes(
9199 LHS: EL0.SymbolicMaxNotTaken, RHS: EL1.SymbolicMaxNotTaken, Sequential: UseSequentialUMin);
9200 } else {
9201 // Both conditions must be same at the same time for the loop to exit.
9202 // For now, be conservative.
9203 if (EL0.ExactNotTaken == EL1.ExactNotTaken)
9204 BECount = EL0.ExactNotTaken;
9205 }
9206
9207 // There are cases (e.g. PR26207) where computeExitLimitFromCond is able
9208 // to be more aggressive when computing BECount than when computing
9209 // ConstantMaxBECount. In these cases it is possible for EL0.ExactNotTaken
9210 // and
9211 // EL1.ExactNotTaken to match, but for EL0.ConstantMaxNotTaken and
9212 // EL1.ConstantMaxNotTaken to not.
9213 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
9214 !isa<SCEVCouldNotCompute>(Val: BECount))
9215 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
9216 if (isa<SCEVCouldNotCompute>(Val: SymbolicMaxBECount))
9217 SymbolicMaxBECount =
9218 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
9219 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, false,
9220 {ArrayRef(EL0.Predicates), ArrayRef(EL1.Predicates)});
9221}
9222
9223ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9224 const Loop *L, ICmpInst *ExitCond, bool ExitIfTrue, bool ControlsOnlyExit,
9225 bool AllowPredicates) {
9226 // If the condition was exit on true, convert the condition to exit on false
9227 CmpPredicate Pred;
9228 if (!ExitIfTrue)
9229 Pred = ExitCond->getCmpPredicate();
9230 else
9231 Pred = ExitCond->getInverseCmpPredicate();
9232 const ICmpInst::Predicate OriginalPred = Pred;
9233
9234 const SCEV *LHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 0));
9235 const SCEV *RHS = getSCEV(V: ExitCond->getOperand(i_nocapture: 1));
9236
9237 ExitLimit EL = computeExitLimitFromICmp(L, Pred, LHS, RHS, IsSubExpr: ControlsOnlyExit,
9238 AllowPredicates);
9239 if (EL.hasAnyInfo())
9240 return EL;
9241
9242 auto *ExhaustiveCount =
9243 computeExitCountExhaustively(L, Cond: ExitCond, ExitWhen: ExitIfTrue);
9244
9245 if (!isa<SCEVCouldNotCompute>(Val: ExhaustiveCount))
9246 return ExhaustiveCount;
9247
9248 return computeShiftCompareExitLimit(LHS: ExitCond->getOperand(i_nocapture: 0),
9249 RHS: ExitCond->getOperand(i_nocapture: 1), L, Pred: OriginalPred);
9250}
9251ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromICmp(
9252 const Loop *L, CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS,
9253 bool ControlsOnlyExit, bool AllowPredicates) {
9254
9255 // Try to evaluate any dependencies out of the loop.
9256 LHS = getSCEVAtScope(S: LHS, L);
9257 RHS = getSCEVAtScope(S: RHS, L);
9258
9259 // At this point, we would like to compute how many iterations of the
9260 // loop the predicate will return true for these inputs.
9261 if (isLoopInvariant(S: LHS, L) && !isLoopInvariant(S: RHS, L)) {
9262 // If there is a loop-invariant, force it into the RHS.
9263 std::swap(a&: LHS, b&: RHS);
9264 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
9265 }
9266
9267 bool ControllingFiniteLoop = ControlsOnlyExit && loopHasNoAbnormalExits(L) &&
9268 loopIsFiniteByAssumption(L);
9269 // Simplify the operands before analyzing them.
9270 (void)SimplifyICmpOperands(Pred, LHS, RHS, /*Depth=*/0);
9271
9272 // If we have a comparison of a chrec against a constant, try to use value
9273 // ranges to answer this query.
9274 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS))
9275 if (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Val&: LHS))
9276 if (AddRec->getLoop() == L) {
9277 // Form the constant range.
9278 ConstantRange CompRange =
9279 ConstantRange::makeExactICmpRegion(Pred, Other: RHSC->getAPInt());
9280
9281 const SCEV *Ret = AddRec->getNumIterationsInRange(Range: CompRange, SE&: *this);
9282 if (!isa<SCEVCouldNotCompute>(Val: Ret)) return Ret;
9283 }
9284
9285 // If this loop must exit based on this condition (or execute undefined
9286 // behaviour), see if we can improve wrap flags. This is essentially
9287 // a must execute style proof.
9288 if (ControllingFiniteLoop && isLoopInvariant(S: RHS, L)) {
9289 // If we can prove the test sequence produced must repeat the same values
9290 // on self-wrap of the IV, then we can infer that IV doesn't self wrap
9291 // because if it did, we'd have an infinite (undefined) loop.
9292 // TODO: We can peel off any functions which are invertible *in L*. Loop
9293 // invariant terms are effectively constants for our purposes here.
9294 SCEVUse InnerLHS = LHS;
9295 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val&: LHS))
9296 InnerLHS = ZExt->getOperand();
9297 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: InnerLHS);
9298 AR && !AR->hasNoSelfWrap() && AR->getLoop() == L && AR->isAffine() &&
9299 isKnownToBeAPowerOfTwo(S: AR->getStepRecurrence(SE&: *this), /*OrZero=*/true,
9300 /*OrNegative=*/true)) {
9301 auto Flags = AR->getNoWrapFlags();
9302 Flags = setFlags(Flags, OnFlags: SCEV::FlagNW);
9303 SmallVector<SCEVUse> Operands{AR->operands()};
9304 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9305 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9306 }
9307
9308 // For a slt/ult condition with a positive step, can we prove nsw/nuw?
9309 // From no-self-wrap, this follows trivially from the fact that every
9310 // (un)signed-wrapped, but not self-wrapped value must be LT than the
9311 // last value before (un)signed wrap. Since we know that last value
9312 // didn't exit, nor will any smaller one.
9313 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_ULT) {
9314 auto WrapType = Pred == ICmpInst::ICMP_SLT ? SCEV::FlagNSW : SCEV::FlagNUW;
9315 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: LHS);
9316 AR && AR->getLoop() == L && AR->isAffine() &&
9317 !AR->getNoWrapFlags(Mask: WrapType) && AR->hasNoSelfWrap() &&
9318 isKnownPositive(S: AR->getStepRecurrence(SE&: *this))) {
9319 auto Flags = AR->getNoWrapFlags();
9320 Flags = setFlags(Flags, OnFlags: WrapType);
9321 SmallVector<SCEVUse> Operands{AR->operands()};
9322 Flags = StrengthenNoWrapFlags(SE: this, Type: scAddRecExpr, Ops: Operands, Flags);
9323 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
9324 }
9325 }
9326 }
9327
9328 switch (Pred) {
9329 case ICmpInst::ICMP_NE: { // while (X != Y)
9330 // Convert to: while (X-Y != 0)
9331 if (LHS->getType()->isPointerTy()) {
9332 LHS = getPtrToAddrExpr(Op: LHS);
9333 if (isa<SCEVCouldNotCompute>(Val: LHS))
9334 return LHS;
9335 }
9336 if (RHS->getType()->isPointerTy()) {
9337 RHS = getPtrToAddrExpr(Op: RHS);
9338 if (isa<SCEVCouldNotCompute>(Val: RHS))
9339 return RHS;
9340 }
9341 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit,
9342 AllowPredicates);
9343 if (EL.hasAnyInfo())
9344 return EL;
9345 break;
9346 }
9347 case ICmpInst::ICMP_EQ: { // while (X == Y)
9348 // Convert to: while (X-Y == 0)
9349 if (LHS->getType()->isPointerTy()) {
9350 LHS = getPtrToAddrExpr(Op: LHS);
9351 if (isa<SCEVCouldNotCompute>(Val: LHS))
9352 return LHS;
9353 }
9354 if (RHS->getType()->isPointerTy()) {
9355 RHS = getPtrToAddrExpr(Op: RHS);
9356 if (isa<SCEVCouldNotCompute>(Val: RHS))
9357 return RHS;
9358 }
9359 ExitLimit EL = howFarToNonZero(V: getMinusSCEV(LHS, RHS), L);
9360 if (EL.hasAnyInfo()) return EL;
9361 break;
9362 }
9363 case ICmpInst::ICMP_SLE:
9364 case ICmpInst::ICMP_ULE:
9365 // Since the loop is finite, an invariant RHS cannot include the boundary
9366 // value, otherwise it would loop forever.
9367 if (!Opts.scalar_evolution_finite_loop || !ControllingFiniteLoop ||
9368 !isLoopInvariant(S: RHS, L)) {
9369 // Otherwise, perform the addition in a wider type, to avoid overflow.
9370 // If the LHS is an addrec with the appropriate nowrap flag, the
9371 // extension will be sunk into it and the exit count can be analyzed.
9372 auto *OldType = dyn_cast<IntegerType>(Val: LHS->getType());
9373 if (!OldType)
9374 break;
9375 // Prefer doubling the bitwidth over adding a single bit to make it more
9376 // likely that we use a legal type.
9377 auto *NewType =
9378 Type::getIntNTy(C&: OldType->getContext(), N: OldType->getBitWidth() * 2);
9379 if (ICmpInst::isSigned(Pred)) {
9380 LHS = getSignExtendExpr(Op: LHS, Ty: NewType);
9381 RHS = getSignExtendExpr(Op: RHS, Ty: NewType);
9382 } else {
9383 LHS = getZeroExtendExpr(Op: LHS, Ty: NewType);
9384 RHS = getZeroExtendExpr(Op: RHS, Ty: NewType);
9385 }
9386 }
9387 RHS = getAddExpr(LHS: getOne(Ty: RHS->getType()), RHS);
9388 [[fallthrough]];
9389 case ICmpInst::ICMP_SLT:
9390 case ICmpInst::ICMP_ULT: { // while (X < Y)
9391 bool IsSigned = ICmpInst::isSigned(Pred);
9392 ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, /*Invert=*/false,
9393 ControlsOnlyExit, AllowPredicates);
9394 if (EL.hasAnyInfo())
9395 return EL;
9396 break;
9397 }
9398 case ICmpInst::ICMP_SGE:
9399 case ICmpInst::ICMP_UGE:
9400 // Since the loop is finite, an invariant RHS cannot include the boundary
9401 // value, otherwise it would loop forever.
9402 if (!Opts.scalar_evolution_finite_loop || !ControllingFiniteLoop ||
9403 !isLoopInvariant(S: RHS, L))
9404 break;
9405 RHS = getAddExpr(LHS: getMinusOne(Ty: RHS->getType()), RHS);
9406 [[fallthrough]];
9407 case ICmpInst::ICMP_SGT:
9408 case ICmpInst::ICMP_UGT: { // while (X > Y)
9409 // "X > Y" is analyzed as the equivalent "~X < ~Y".
9410 bool IsSigned = ICmpInst::isSigned(Pred);
9411 ExitLimit EL = howManyLessThans(LHS, RHS, L, IsSigned, /*Invert=*/true,
9412 ControlsOnlyExit, AllowPredicates);
9413 if (EL.hasAnyInfo())
9414 return EL;
9415 break;
9416 }
9417 default:
9418 break;
9419 }
9420
9421 return getCouldNotCompute();
9422}
9423
9424ScalarEvolution::ExitLimit
9425ScalarEvolution::computeExitLimitFromSingleExitSwitch(const Loop *L,
9426 SwitchInst *Switch,
9427 BasicBlock *ExitingBlock,
9428 bool ControlsOnlyExit) {
9429 assert(!L->contains(ExitingBlock) && "Not an exiting block!");
9430
9431 // Give up if the exit is the default dest of a switch.
9432 if (Switch->getDefaultDest() == ExitingBlock)
9433 return getCouldNotCompute();
9434
9435 assert(L->contains(Switch->getDefaultDest()) &&
9436 "Default case must not exit the loop!");
9437 const SCEV *LHS = getSCEVAtScope(V: Switch->getCondition(), L);
9438 const SCEV *RHS = getConstant(V: Switch->findCaseDest(BB: ExitingBlock));
9439
9440 // while (X != Y) --> while (X-Y != 0)
9441 ExitLimit EL = howFarToZero(V: getMinusSCEV(LHS, RHS), L, IsSubExpr: ControlsOnlyExit);
9442 if (EL.hasAnyInfo())
9443 return EL;
9444
9445 return getCouldNotCompute();
9446}
9447
9448static ConstantInt *
9449EvaluateConstantChrecAtConstant(const SCEVAddRecExpr *AddRec, ConstantInt *C,
9450 ScalarEvolution &SE) {
9451 const SCEV *InVal = SE.getConstant(V: C);
9452 const SCEV *Val = AddRec->evaluateAtIteration(It: InVal, SE);
9453 assert(isa<SCEVConstant>(Val) &&
9454 "Evaluation of SCEV at constant didn't fold correctly?");
9455 return cast<SCEVConstant>(Val)->getValue();
9456}
9457
9458ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit(
9459 Value *LHS, Value *RHSV, const Loop *L, ICmpInst::Predicate Pred) {
9460 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: RHSV);
9461 if (!RHS)
9462 return getCouldNotCompute();
9463
9464 const BasicBlock *Latch = L->getLoopLatch();
9465 if (!Latch)
9466 return getCouldNotCompute();
9467
9468 const BasicBlock *Predecessor = L->getLoopPredecessor();
9469 if (!Predecessor)
9470 return getCouldNotCompute();
9471
9472 // Return true if V is of the form "LHS `shift_op` <positive constant>".
9473 // Return LHS in OutLHS, shift_op in OutOpCode, and the shift amount in
9474 // OutShiftAmt.
9475 auto MatchPositiveShift = [](Value *V, Value *&OutLHS,
9476 Instruction::BinaryOps &OutOpCode,
9477 unsigned &OutShiftAmt) {
9478 using namespace PatternMatch;
9479
9480 ConstantInt *ShiftAmt;
9481 if (match(V, P: m_LShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9482 OutOpCode = Instruction::LShr;
9483 else if (match(V, P: m_AShr(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9484 OutOpCode = Instruction::AShr;
9485 else if (match(V, P: m_Shl(L: m_Value(V&: OutLHS), R: m_ConstantInt(CI&: ShiftAmt))))
9486 OutOpCode = Instruction::Shl;
9487 else
9488 return false;
9489
9490 uint64_t Amt = ShiftAmt->getValue().getLimitedValue();
9491 if (Amt == 0 || Amt >= OutLHS->getType()->getScalarSizeInBits())
9492 return false;
9493 OutShiftAmt = Amt;
9494 return true;
9495 };
9496
9497 // Recognize a "shift recurrence" either of the form %iv or of %iv.shifted in
9498 //
9499 // loop:
9500 // %iv = phi i32 [ %iv.shifted, %loop ], [ %val, %preheader ]
9501 // %iv.shifted = lshr i32 %iv, <positive constant>
9502 //
9503 // Return true on a successful match. Return the corresponding PHI node (%iv
9504 // above) in PNOut, the opcode of the shift operation in OpCodeOut, and the
9505 // shift amount in ShiftAmtOut.
9506 auto MatchShiftRecurrence = [&](Value *V, PHINode *&PNOut,
9507 Instruction::BinaryOps &OpCodeOut,
9508 unsigned &ShiftAmtOut) {
9509 std::optional<Instruction::BinaryOps> PostShiftOpCode;
9510
9511 {
9512 Instruction::BinaryOps OpC;
9513 Value *V;
9514 unsigned Amt;
9515
9516 // If we encounter a shift instruction, "peel off" the shift operation,
9517 // and remember that we did so. Later when we inspect %iv's backedge
9518 // value, we will make sure that the backedge value uses the same
9519 // operation.
9520 //
9521 // Note: the peeled shift operation does not have to be the same
9522 // instruction as the one feeding into the PHI's backedge value. We only
9523 // really care about it being the same *kind* of shift instruction --
9524 // that's all that is required for our later inferences to hold.
9525 if (MatchPositiveShift(LHS, V, OpC, Amt)) {
9526 PostShiftOpCode = OpC;
9527 LHS = V;
9528 }
9529 }
9530
9531 PNOut = dyn_cast<PHINode>(Val: LHS);
9532 if (!PNOut || PNOut->getParent() != L->getHeader())
9533 return false;
9534
9535 Value *BEValue = PNOut->getIncomingValueForBlock(BB: Latch);
9536 Value *OpLHS;
9537
9538 return
9539 // The backedge value for the PHI node must be a shift by a positive
9540 // amount
9541 MatchPositiveShift(BEValue, OpLHS, OpCodeOut, ShiftAmtOut) &&
9542
9543 // of the PHI node itself
9544 OpLHS == PNOut &&
9545
9546 // and the kind of shift should be match the kind of shift we peeled
9547 // off, if any.
9548 (!PostShiftOpCode || *PostShiftOpCode == OpCodeOut);
9549 };
9550
9551 PHINode *PN;
9552 Instruction::BinaryOps OpCode;
9553 unsigned ShiftAmt;
9554 if (!MatchShiftRecurrence(LHS, PN, OpCode, ShiftAmt))
9555 return getCouldNotCompute();
9556
9557 const DataLayout &DL = getDataLayout();
9558
9559 // The key rationale for this optimization is that for some kinds of shift
9560 // recurrences, the value of the recurrence "stabilizes" to either 0 or -1
9561 // within a finite number of iterations. If the condition guarding the
9562 // backedge (in the sense that the backedge is taken if the condition is true)
9563 // is false for the value the shift recurrence stabilizes to, then we know
9564 // that the backedge is taken only a finite number of times.
9565
9566 ConstantInt *StableValue = nullptr;
9567 switch (OpCode) {
9568 default:
9569 llvm_unreachable("Impossible case!");
9570
9571 case Instruction::AShr: {
9572 // {K,ashr,<positive-constant>} stabilizes to signum(K) in at most
9573 // bitwidth(K) iterations.
9574 Value *FirstValue = PN->getIncomingValueForBlock(BB: Predecessor);
9575 KnownBits Known = computeKnownBits(V: FirstValue, DL, AC: &AC,
9576 CtxI: Predecessor->getTerminator(), DT: &DT);
9577 auto *Ty = cast<IntegerType>(Val: RHS->getType());
9578 if (Known.isNonNegative())
9579 StableValue = ConstantInt::get(Ty, V: 0);
9580 else if (Known.isNegative())
9581 StableValue = ConstantInt::get(Ty, V: -1, IsSigned: true);
9582 else
9583 return getCouldNotCompute();
9584
9585 break;
9586 }
9587 case Instruction::LShr:
9588 case Instruction::Shl:
9589 // Both {K,lshr,<positive-constant>} and {K,shl,<positive-constant>}
9590 // stabilize to 0 in at most bitwidth(K) iterations.
9591 StableValue = ConstantInt::get(Ty: cast<IntegerType>(Val: RHS->getType()), V: 0);
9592 break;
9593 }
9594
9595 auto *Result =
9596 ConstantFoldCompareInstOperands(Predicate: Pred, LHS: StableValue, RHS, DL, TLI: &TLI);
9597 assert(Result->getType()->isIntegerTy(1) &&
9598 "Otherwise cannot be an operand to a branch instruction");
9599
9600 if (Result->isNullValue()) {
9601 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
9602 unsigned MaxBTC = BitWidth;
9603
9604 // For right-shift recurrences (lshr/ashr with non-negative start), we can
9605 // compute a tighter max backedge-taken count from the range of the start
9606 // value. After k shifts of ShiftAmt, value = start >> (k * ShiftAmt).
9607 // The value reaches 0 (the stable value) when k * ShiftAmt >=
9608 // activeBits(start), so max BTC = ceil(activeBits(maxStart) / ShiftAmt).
9609 if (OpCode == Instruction::LShr || OpCode == Instruction::AShr) {
9610 Value *StartValue = PN->getIncomingValueForBlock(BB: Predecessor);
9611 const SCEV *StartSCEV = getSCEV(V: StartValue);
9612 APInt MaxStart = getUnsignedRangeMax(S: StartSCEV);
9613 if (MaxStart.isStrictlyPositive()) {
9614 unsigned ActiveBits = MaxStart.getActiveBits();
9615 unsigned RangeBTC = divideCeil(Numerator: ActiveBits, Denominator: ShiftAmt);
9616 MaxBTC = std::min(a: MaxBTC, b: RangeBTC);
9617 }
9618 }
9619
9620 const SCEV *UpperBound =
9621 getConstant(Ty: getEffectiveSCEVType(Ty: RHS->getType()), V: MaxBTC);
9622 return ExitLimit(getCouldNotCompute(), UpperBound, UpperBound, false);
9623 }
9624
9625 return getCouldNotCompute();
9626}
9627
9628/// Return true if we can constant fold an instruction of the specified type,
9629/// assuming that all operands were constants.
9630static bool canConstantFold(const Instruction *I,
9631 const TargetLibraryInfo *TLI) {
9632 if (isa<BinaryOperator, UnaryOperator, GEPOperator, FreezeInst, CmpInst,
9633 SelectInst, CastInst, LoadInst, ExtractElementInst, InsertElementInst,
9634 ExtractValueInst, InsertValueInst>(Val: I))
9635 return true;
9636
9637 if (const CallInst *CI = dyn_cast<CallInst>(Val: I))
9638 if (const Function *F = CI->getCalledFunction())
9639 return canConstantFoldCallTo(Call: CI, F, TLI);
9640 return false;
9641}
9642
9643/// Determine whether this instruction can constant evolve within this loop
9644/// assuming its operands can all constant evolve.
9645static bool canConstantEvolve(Instruction *I, const Loop *L,
9646 const TargetLibraryInfo *TLI) {
9647 // An instruction outside of the loop can't be derived from a loop PHI.
9648 if (!L->contains(Inst: I)) return false;
9649
9650 if (isa<PHINode>(Val: I)) {
9651 // We don't currently keep track of the control flow needed to evaluate
9652 // PHIs, so we cannot handle PHIs inside of loops.
9653 return L->getHeader() == I->getParent();
9654 }
9655
9656 // If we won't be able to constant fold this expression even if the operands
9657 // are constants, bail early.
9658 return canConstantFold(I, TLI);
9659}
9660
9661/// getConstantEvolvingPHIOperands - Implement getConstantEvolvingPHI by
9662/// recursing through each instruction operand until reaching a loop header phi.
9663static PHINode *
9664getConstantEvolvingPHIOperands(const AnalysisOptions &Opts,
9665 Instruction *UseInst, const Loop *L,
9666 DenseMap<Instruction *, PHINode *> &PHIMap,
9667 const TargetLibraryInfo *TLI, unsigned Depth) {
9668 if (Depth > Opts.scalar_evolution_max_constant_evolving_depth)
9669 return nullptr;
9670
9671 // Otherwise, we can evaluate this instruction if all of its operands are
9672 // constant or derived from a PHI node themselves.
9673 PHINode *PHI = nullptr;
9674 for (Value *Op : UseInst->operands()) {
9675 if (isa<Constant>(Val: Op)) continue;
9676
9677 Instruction *OpInst = dyn_cast<Instruction>(Val: Op);
9678 if (!OpInst || !canConstantEvolve(I: OpInst, L, TLI))
9679 return nullptr;
9680
9681 PHINode *P = dyn_cast<PHINode>(Val: OpInst);
9682 if (!P)
9683 // If this operand is already visited, reuse the prior result.
9684 // We may have P != PHI if this is the deepest point at which the
9685 // inconsistent paths meet.
9686 P = PHIMap.lookup(Val: OpInst);
9687 if (!P) {
9688 // Recurse and memoize the results, whether a phi is found or not.
9689 // This recursive call invalidates pointers into PHIMap.
9690 P = getConstantEvolvingPHIOperands(Opts, UseInst: OpInst, L, PHIMap, TLI,
9691 Depth: Depth + 1);
9692 PHIMap[OpInst] = P;
9693 }
9694 if (!P)
9695 return nullptr; // Not evolving from PHI
9696 if (PHI && PHI != P)
9697 return nullptr; // Evolving from multiple different PHIs.
9698 PHI = P;
9699 }
9700 // This is a expression evolving from a constant PHI!
9701 return PHI;
9702}
9703
9704/// getConstantEvolvingPHI - Given an LLVM value and a loop, return a PHI node
9705/// in the loop that V is derived from. We allow arbitrary operations along the
9706/// way, but the operands of an operation must either be constants or a value
9707/// derived from a constant PHI. If this expression does not fit with these
9708/// constraints, return null.
9709static PHINode *getConstantEvolvingPHI(const AnalysisOptions &Opts, Value *V,
9710 const Loop *L,
9711 const TargetLibraryInfo *TLI) {
9712 Instruction *I = dyn_cast<Instruction>(Val: V);
9713 if (!I || !canConstantEvolve(I, L, TLI))
9714 return nullptr;
9715
9716 if (PHINode *PN = dyn_cast<PHINode>(Val: I))
9717 return PN;
9718
9719 // Record non-constant instructions contained by the loop.
9720 DenseMap<Instruction *, PHINode *> PHIMap;
9721 return getConstantEvolvingPHIOperands(Opts, UseInst: I, L, PHIMap, TLI, Depth: 0);
9722}
9723
9724/// EvaluateExpression - Given an expression that passes the
9725/// getConstantEvolvingPHI predicate, evaluate its value assuming the PHI node
9726/// in the loop has the value PHIVal. If we can't fold this expression for some
9727/// reason, return null.
9728static Constant *EvaluateExpression(Value *V, const Loop *L,
9729 DenseMap<Instruction *, Constant *> &Vals,
9730 const DataLayout &DL,
9731 const TargetLibraryInfo *TLI) {
9732 // Convenient constant check, but redundant for recursive calls.
9733 if (Constant *C = dyn_cast<Constant>(Val: V)) return C;
9734 Instruction *I = dyn_cast<Instruction>(Val: V);
9735 if (!I) return nullptr;
9736
9737 if (Constant *C = Vals.lookup(Val: I)) return C;
9738
9739 // An instruction inside the loop depends on a value outside the loop that we
9740 // weren't given a mapping for, or a value such as a call inside the loop.
9741 if (!canConstantEvolve(I, L, TLI))
9742 return nullptr;
9743
9744 // An unmapped PHI can be due to a branch or another loop inside this loop,
9745 // or due to this not being the initial iteration through a loop where we
9746 // couldn't compute the evolution of this particular PHI last time.
9747 if (isa<PHINode>(Val: I)) return nullptr;
9748
9749 std::vector<Constant*> Operands(I->getNumOperands());
9750
9751 for (unsigned i = 0, e = I->getNumOperands(); i != e; ++i) {
9752 Instruction *Operand = dyn_cast<Instruction>(Val: I->getOperand(i));
9753 if (!Operand) {
9754 Operands[i] = dyn_cast<Constant>(Val: I->getOperand(i));
9755 if (!Operands[i]) return nullptr;
9756 continue;
9757 }
9758 Constant *C = EvaluateExpression(V: Operand, L, Vals, DL, TLI);
9759 Vals[Operand] = C;
9760 if (!C) return nullptr;
9761 Operands[i] = C;
9762 }
9763
9764 return ConstantFoldInstOperands(I, Ops: Operands, DL, TLI,
9765 /*AllowNonDeterministic=*/false);
9766}
9767
9768
9769// If every incoming value to PN except the one for BB is a specific Constant,
9770// return that, else return nullptr.
9771static Constant *getOtherIncomingValue(PHINode *PN, BasicBlock *BB) {
9772 Constant *IncomingVal = nullptr;
9773
9774 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
9775 if (PN->getIncomingBlock(i) == BB)
9776 continue;
9777
9778 auto *CurrentVal = dyn_cast<Constant>(Val: PN->getIncomingValue(i));
9779 if (!CurrentVal)
9780 return nullptr;
9781
9782 if (IncomingVal != CurrentVal) {
9783 if (IncomingVal)
9784 return nullptr;
9785 IncomingVal = CurrentVal;
9786 }
9787 }
9788
9789 return IncomingVal;
9790}
9791
9792/// getConstantEvolutionLoopExitValue - If we know that the specified Phi is
9793/// in the header of its containing loop, we know the loop executes a
9794/// constant number of times, and the PHI node is just a recurrence
9795/// involving constants, fold it.
9796Constant *
9797ScalarEvolution::getConstantEvolutionLoopExitValue(PHINode *PN,
9798 const APInt &BEs,
9799 const Loop *L) {
9800 auto [I, Inserted] = ConstantEvolutionLoopExitValue.try_emplace(Key: PN);
9801 if (!Inserted)
9802 return I->second;
9803
9804 if (BEs.ugt(RHS: Opts.scalar_evolution_max_iterations))
9805 return nullptr; // Not going to evaluate it.
9806
9807 Constant *&RetVal = I->second;
9808
9809 DenseMap<Instruction *, Constant *> CurrentIterVals;
9810 BasicBlock *Header = L->getHeader();
9811 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9812
9813 BasicBlock *Latch = L->getLoopLatch();
9814 if (!Latch)
9815 return nullptr;
9816
9817 for (PHINode &PHI : Header->phis()) {
9818 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
9819 CurrentIterVals[&PHI] = StartCST;
9820 }
9821 if (!CurrentIterVals.count(Val: PN))
9822 return RetVal = nullptr;
9823
9824 Value *BEValue = PN->getIncomingValueForBlock(BB: Latch);
9825
9826 // Execute the loop symbolically to determine the exit value.
9827 assert(BEs.getActiveBits() < CHAR_BIT * sizeof(unsigned) &&
9828 "BEs is <= -scalar-evolution-max-iterations which is an 'unsigned'!");
9829
9830 unsigned NumIterations = BEs.getZExtValue(); // must be in range
9831 unsigned IterationNum = 0;
9832 const DataLayout &DL = getDataLayout();
9833 for (; ; ++IterationNum) {
9834 if (IterationNum == NumIterations)
9835 return RetVal = CurrentIterVals[PN]; // Got exit value!
9836
9837 // Compute the value of the PHIs for the next iteration.
9838 // EvaluateExpression adds non-phi values to the CurrentIterVals map.
9839 DenseMap<Instruction *, Constant *> NextIterVals;
9840 Constant *NextPHI =
9841 EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9842 if (!NextPHI)
9843 return nullptr; // Couldn't evaluate!
9844 NextIterVals[PN] = NextPHI;
9845
9846 bool StoppedEvolving = NextPHI == CurrentIterVals[PN];
9847
9848 // Also evaluate the other PHI nodes. However, we don't get to stop if we
9849 // cease to be able to evaluate one of them or if they stop evolving,
9850 // because that doesn't necessarily prevent us from computing PN.
9851 SmallVector<std::pair<PHINode *, Constant *>, 8> PHIsToCompute;
9852 for (const auto &I : CurrentIterVals) {
9853 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
9854 if (!PHI || PHI == PN || PHI->getParent() != Header) continue;
9855 PHIsToCompute.emplace_back(Args&: PHI, Args: I.second);
9856 }
9857 // We use two distinct loops because EvaluateExpression may invalidate any
9858 // iterators into CurrentIterVals.
9859 for (const auto &I : PHIsToCompute) {
9860 PHINode *PHI = I.first;
9861 Constant *&NextPHI = NextIterVals[PHI];
9862 if (!NextPHI) { // Not already computed.
9863 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
9864 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9865 }
9866 if (NextPHI != I.second)
9867 StoppedEvolving = false;
9868 }
9869
9870 // If all entries in CurrentIterVals == NextIterVals then we can stop
9871 // iterating, the loop can't continue to change.
9872 if (StoppedEvolving)
9873 return RetVal = CurrentIterVals[PN];
9874
9875 CurrentIterVals.swap(RHS&: NextIterVals);
9876 }
9877}
9878
9879const SCEV *ScalarEvolution::computeExitCountExhaustively(const Loop *L,
9880 Value *Cond,
9881 bool ExitWhen) {
9882 PHINode *PN = getConstantEvolvingPHI(Opts, V: Cond, L, TLI: &TLI);
9883 if (!PN) return getCouldNotCompute();
9884
9885 // If the loop is canonicalized, the PHI will have exactly two entries.
9886 // That's the only form we support here.
9887 if (PN->getNumIncomingValues() != 2) return getCouldNotCompute();
9888
9889 DenseMap<Instruction *, Constant *> CurrentIterVals;
9890 BasicBlock *Header = L->getHeader();
9891 assert(PN->getParent() == Header && "Can't evaluate PHI not in loop header!");
9892
9893 BasicBlock *Latch = L->getLoopLatch();
9894 assert(Latch && "Should follow from NumIncomingValues == 2!");
9895
9896 for (PHINode &PHI : Header->phis()) {
9897 if (auto *StartCST = getOtherIncomingValue(PN: &PHI, BB: Latch))
9898 CurrentIterVals[&PHI] = StartCST;
9899 }
9900 if (!CurrentIterVals.count(Val: PN))
9901 return getCouldNotCompute();
9902
9903 // Okay, we find a PHI node that defines the trip count of this loop. Execute
9904 // the loop symbolically to determine when the condition gets a value of
9905 // "ExitWhen".
9906 unsigned MaxIterations =
9907 Opts.scalar_evolution_max_iterations; // Limit analysis.
9908 const DataLayout &DL = getDataLayout();
9909 for (unsigned IterationNum = 0; IterationNum != MaxIterations;++IterationNum){
9910 auto *CondVal = dyn_cast_or_null<ConstantInt>(
9911 Val: EvaluateExpression(V: Cond, L, Vals&: CurrentIterVals, DL, TLI: &TLI));
9912
9913 // Couldn't symbolically evaluate.
9914 if (!CondVal) return getCouldNotCompute();
9915
9916 if (CondVal->getValue() == uint64_t(ExitWhen)) {
9917 ++NumBruteForceTripCountsComputed;
9918 return getConstant(Ty: Type::getInt32Ty(C&: getContext()), V: IterationNum);
9919 }
9920
9921 // Update all the PHI nodes for the next iteration.
9922 DenseMap<Instruction *, Constant *> NextIterVals;
9923
9924 // Create a list of which PHIs we need to compute. We want to do this before
9925 // calling EvaluateExpression on them because that may invalidate iterators
9926 // into CurrentIterVals.
9927 SmallVector<PHINode *, 8> PHIsToCompute;
9928 for (const auto &I : CurrentIterVals) {
9929 PHINode *PHI = dyn_cast<PHINode>(Val: I.first);
9930 if (!PHI || PHI->getParent() != Header) continue;
9931 PHIsToCompute.push_back(Elt: PHI);
9932 }
9933 for (PHINode *PHI : PHIsToCompute) {
9934 Constant *&NextPHI = NextIterVals[PHI];
9935 if (NextPHI) continue; // Already computed!
9936
9937 Value *BEValue = PHI->getIncomingValueForBlock(BB: Latch);
9938 NextPHI = EvaluateExpression(V: BEValue, L, Vals&: CurrentIterVals, DL, TLI: &TLI);
9939 }
9940 CurrentIterVals.swap(RHS&: NextIterVals);
9941 }
9942
9943 // Too many iterations were needed to evaluate.
9944 return getCouldNotCompute();
9945}
9946
9947SCEVUse ScalarEvolution::getSCEVAtScope(const SCEV *V, const Loop *L) {
9948 if (isa<SCEVConstant>(Val: V))
9949 return V;
9950
9951 auto &Values = ValuesAtScopes[V];
9952 // Check to see if we've folded this expression at this loop before.
9953 for (auto &LS : Values)
9954 if (LS.first == L)
9955 return LS.second ? LS.second : SCEVUse(V);
9956
9957 Values.emplace_back(Args&: L, Args: nullptr);
9958
9959 // Otherwise compute it.
9960 SCEVUse C = computeSCEVAtScope(S: V, L);
9961 for (auto &LS : reverse(C&: ValuesAtScopes[V]))
9962 if (LS.first == L) {
9963 LS.second = C;
9964 // Record the dependency under the bare expression: invalidation walks
9965 // expressions, and any use flags on C do not change which expression
9966 // this is the value at scope of.
9967 if (!isa<SCEVConstant>(Val: C))
9968 ValuesAtScopesUsers[C.getPointer()].push_back(Elt: {L, V});
9969 break;
9970 }
9971 return C;
9972}
9973
9974SCEVUse ScalarEvolution::getSCEVAtExit(const SCEV *V, const Loop *L,
9975 const BasicBlock *ExitingBlock) {
9976 SCEVUse ExitValue = getSCEVAtScope(V, L: L->getParentLoop());
9977 if (!isLoopInvariant(S: ExitValue, L)) {
9978 // If we failed to evaluate it in the outer scope, try to evaluate an
9979 // addrec for the specific exit.
9980 // TODO: Generalize this to other expressions.
9981 const SCEV *ExitCount = getExitCount(L, ExitingBlock);
9982 if (!isa<SCEVCouldNotCompute>(Val: ExitCount))
9983 if (auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: V))
9984 if (AddRec->getLoop() == L)
9985 ExitValue = AddRec->evaluateAtIteration(It: ExitCount, SE&: *this);
9986 }
9987 return ExitValue;
9988}
9989
9990/// This builds up a Constant using the ConstantExpr interface. That way, we
9991/// will return Constants for objects which aren't represented by a
9992/// SCEVConstant, because SCEVConstant is restricted to ConstantInt.
9993/// Returns NULL if the SCEV isn't representable as a Constant.
9994static Constant *BuildConstantFromSCEV(const SCEV *V) {
9995 switch (V->getSCEVType()) {
9996 case scCouldNotCompute:
9997 case scAddRecExpr:
9998 case scVScale:
9999 return nullptr;
10000 case scConstant:
10001 return cast<SCEVConstant>(Val: V)->getValue();
10002 case scUnknown:
10003 return dyn_cast<Constant>(Val: cast<SCEVUnknown>(Val: V)->getValue());
10004 case scPtrToAddr: {
10005 const SCEVPtrToAddrExpr *P2I = cast<SCEVPtrToAddrExpr>(Val: V);
10006 if (Constant *CastOp = BuildConstantFromSCEV(V: P2I->getOperand()))
10007 return ConstantExpr::getPtrToAddr(C: CastOp, Ty: P2I->getType());
10008
10009 return nullptr;
10010 }
10011 case scTruncate: {
10012 const SCEVTruncateExpr *ST = cast<SCEVTruncateExpr>(Val: V);
10013 if (Constant *CastOp = BuildConstantFromSCEV(V: ST->getOperand()))
10014 return ConstantExpr::getTrunc(C: CastOp, Ty: ST->getType());
10015 return nullptr;
10016 }
10017 case scAddExpr: {
10018 const SCEVAddExpr *SA = cast<SCEVAddExpr>(Val: V);
10019 Constant *C = nullptr;
10020 for (const SCEV *Op : SA->operands()) {
10021 Constant *OpC = BuildConstantFromSCEV(V: Op);
10022 if (!OpC)
10023 return nullptr;
10024 if (!C) {
10025 C = OpC;
10026 continue;
10027 }
10028 assert(!C->getType()->isPointerTy() &&
10029 "Can only have one pointer, and it must be last");
10030 if (OpC->getType()->isPointerTy()) {
10031 // The offsets have been converted to bytes. We can add bytes using
10032 // an i8 GEP.
10033 C = ConstantExpr::getPtrAdd(Ptr: OpC, Offset: C);
10034 } else {
10035 C = ConstantExpr::getAdd(C1: C, C2: OpC);
10036 }
10037 }
10038 return C;
10039 }
10040 case scMulExpr:
10041 case scSignExtend:
10042 case scZeroExtend:
10043 case scUDivExpr:
10044 case scSMaxExpr:
10045 case scUMaxExpr:
10046 case scSMinExpr:
10047 case scUMinExpr:
10048 case scSequentialUMinExpr:
10049 return nullptr;
10050 }
10051 llvm_unreachable("Unknown SCEV kind!");
10052}
10053
10054const SCEV *ScalarEvolution::getWithOperands(const SCEV *S,
10055 SmallVectorImpl<SCEVUse> &NewOps) {
10056 switch (S->getSCEVType()) {
10057 case scTruncate:
10058 case scZeroExtend:
10059 case scSignExtend:
10060 case scPtrToAddr:
10061 return getCastExpr(Kind: S->getSCEVType(), Op: NewOps[0], Ty: S->getType());
10062 case scAddRecExpr: {
10063 auto *AddRec = cast<SCEVAddRecExpr>(Val: S);
10064 return getAddRecExpr(Operands&: NewOps, L: AddRec->getLoop(), NWFlags: AddRec->getNoWrapFlags());
10065 }
10066 case scAddExpr:
10067 return getAddExpr(Ops&: NewOps, Flags: cast<SCEVAddExpr>(Val: S)->getNoWrapFlags());
10068 case scMulExpr:
10069 return getMulExpr(Ops&: NewOps, Flags: cast<SCEVMulExpr>(Val: S)->getNoWrapFlags());
10070 case scUDivExpr:
10071 return getUDivExpr(LHS: NewOps[0], RHS: NewOps[1]);
10072 case scUMaxExpr:
10073 case scSMaxExpr:
10074 case scUMinExpr:
10075 case scSMinExpr:
10076 return getMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10077 case scSequentialUMinExpr:
10078 return getSequentialMinMaxExpr(Kind: S->getSCEVType(), Ops&: NewOps);
10079 case scConstant:
10080 case scVScale:
10081 case scUnknown:
10082 return S;
10083 case scCouldNotCompute:
10084 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10085 }
10086 llvm_unreachable("Unknown SCEV kind!");
10087}
10088
10089SCEVUse ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) {
10090 switch (V->getSCEVType()) {
10091 case scConstant:
10092 case scVScale:
10093 return V;
10094 case scAddRecExpr: {
10095 // If this is a loop recurrence for a loop that does not contain L, then we
10096 // are dealing with the final value computed by the loop.
10097 const SCEVAddRecExpr *AddRec = cast<SCEVAddRecExpr>(Val: V);
10098 // First, attempt to evaluate each operand.
10099 // Avoid performing the look-up in the common case where the specified
10100 // expression has no loop-variant portions.
10101 for (unsigned i = 0, e = AddRec->getNumOperands(); i != e; ++i) {
10102 SCEVUse OpAtScope = getSCEVAtScope(V: AddRec->getOperand(i), L);
10103 if (OpAtScope == AddRec->getOperand(i))
10104 continue;
10105
10106 // Okay, at least one of these operands is loop variant but might be
10107 // foldable. Build a new instance of the folded commutative expression.
10108 SmallVector<SCEVUse, 8> NewOps;
10109 NewOps.reserve(N: AddRec->getNumOperands());
10110 append_range(C&: NewOps, R: AddRec->operands().take_front(N: i));
10111 NewOps.push_back(Elt: OpAtScope);
10112 for (++i; i != e; ++i)
10113 NewOps.push_back(Elt: getSCEVAtScope(V: AddRec->getOperand(i), L));
10114
10115 const SCEV *FoldedRec = getAddRecExpr(
10116 Operands&: NewOps, L: AddRec->getLoop(), NWFlags: AddRec->getNoWrapFlags(Mask: SCEV::FlagNW));
10117 AddRec = dyn_cast<SCEVAddRecExpr>(Val: FoldedRec);
10118 // The addrec may be folded to a nonrecurrence, for example, if the
10119 // induction variable is multiplied by zero after constant folding. Go
10120 // ahead and return the folded value.
10121 if (!AddRec)
10122 return FoldedRec;
10123 break;
10124 }
10125
10126 // If the scope is outside the addrec's loop, evaluate it by using the
10127 // loop exit value of the addrec.
10128 if (!AddRec->getLoop()->contains(L)) {
10129 SCEVUse ExitValue = AddRec->getExitValue(SE&: *this);
10130 if (isa<SCEVCouldNotCompute>(Val: ExitValue))
10131 return AddRec;
10132 return ExitValue;
10133 }
10134
10135 return AddRec;
10136 }
10137 case scTruncate:
10138 case scZeroExtend:
10139 case scSignExtend:
10140 case scPtrToAddr:
10141 case scAddExpr:
10142 case scMulExpr:
10143 case scUDivExpr:
10144 case scUMaxExpr:
10145 case scSMaxExpr:
10146 case scUMinExpr:
10147 case scSMinExpr:
10148 case scSequentialUMinExpr: {
10149 ArrayRef<SCEVUse> Ops = V->operands();
10150 // Avoid performing the look-up in the common case where the specified
10151 // expression has no loop-variant portions.
10152 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
10153 SCEVUse OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10154 if (OpAtScope != Ops[i].getPointer()) {
10155 // Okay, at least one of these operands is loop variant but might be
10156 // foldable. Build a new instance of the folded commutative expression.
10157 SmallVector<SCEVUse, 8> NewOps;
10158 NewOps.reserve(N: Ops.size());
10159 append_range(C&: NewOps, R: Ops.take_front(N: i));
10160 NewOps.push_back(Elt: OpAtScope);
10161
10162 for (++i; i != e; ++i) {
10163 OpAtScope = getSCEVAtScope(V: Ops[i].getPointer(), L);
10164 NewOps.push_back(Elt: OpAtScope);
10165 }
10166
10167 return getWithOperands(S: V, NewOps);
10168 }
10169 }
10170 // If we got here, all operands are loop invariant.
10171 return V;
10172 }
10173 case scUnknown: {
10174 // If this instruction is evolved from a constant-evolving PHI, compute the
10175 // exit value from the loop without using SCEVs.
10176 const SCEVUnknown *SU = cast<SCEVUnknown>(Val: V);
10177 Instruction *I = dyn_cast<Instruction>(Val: SU->getValue());
10178 if (!I)
10179 return V; // This is some other type of SCEVUnknown, just return it.
10180
10181 if (PHINode *PN = dyn_cast<PHINode>(Val: I)) {
10182 const Loop *CurrLoop = this->LI[I->getParent()];
10183 // Looking for loop exit value.
10184 if (CurrLoop && CurrLoop->getParentLoop() == L &&
10185 PN->getParent() == CurrLoop->getHeader()) {
10186 // Okay, there is no closed form solution for the PHI node. Check
10187 // to see if the loop that contains it has a known backedge-taken
10188 // count. If so, we may be able to force computation of the exit
10189 // value.
10190 const SCEV *BackedgeTakenCount = getBackedgeTakenCount(L: CurrLoop);
10191 // This trivial case can show up in some degenerate cases where
10192 // the incoming IR has not yet been fully simplified.
10193 if (BackedgeTakenCount->isZero()) {
10194 Value *InitValue = nullptr;
10195 bool MultipleInitValues = false;
10196 for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) {
10197 if (!CurrLoop->contains(BB: PN->getIncomingBlock(i))) {
10198 if (!InitValue)
10199 InitValue = PN->getIncomingValue(i);
10200 else if (InitValue != PN->getIncomingValue(i)) {
10201 MultipleInitValues = true;
10202 break;
10203 }
10204 }
10205 }
10206 if (!MultipleInitValues && InitValue)
10207 return getSCEV(V: InitValue);
10208 }
10209 // Do we have a loop invariant value flowing around the backedge
10210 // for a loop which must execute the backedge?
10211 if (!isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount) &&
10212 isKnownNonZero(S: BackedgeTakenCount) &&
10213 PN->getNumIncomingValues() == 2) {
10214
10215 unsigned InLoopPred =
10216 CurrLoop->contains(BB: PN->getIncomingBlock(i: 0)) ? 0 : 1;
10217 Value *BackedgeVal = PN->getIncomingValue(i: InLoopPred);
10218 if (CurrLoop->isLoopInvariant(V: BackedgeVal))
10219 return getSCEV(V: BackedgeVal);
10220 }
10221 if (auto *BTCC = dyn_cast<SCEVConstant>(Val: BackedgeTakenCount)) {
10222 // Okay, we know how many times the containing loop executes. If
10223 // this is a constant evolving PHI node, get the final value at
10224 // the specified iteration number.
10225 Constant *RV =
10226 getConstantEvolutionLoopExitValue(PN, BEs: BTCC->getAPInt(), L: CurrLoop);
10227 if (RV)
10228 return getSCEV(V: RV);
10229 }
10230 }
10231 }
10232
10233 // Okay, this is an expression that we cannot symbolically evaluate
10234 // into a SCEV. Check to see if it's possible to symbolically evaluate
10235 // the arguments into constants, and if so, try to constant propagate the
10236 // result. This is particularly useful for computing loop exit values.
10237 if (!canConstantFold(I, TLI: &TLI))
10238 return V; // This is some other type of SCEVUnknown, just return it.
10239
10240 SmallVector<Constant *, 4> Operands;
10241 Operands.reserve(N: I->getNumOperands());
10242 bool MadeImprovement = false;
10243 for (Value *Op : I->operands()) {
10244 if (Constant *C = dyn_cast<Constant>(Val: Op)) {
10245 Operands.push_back(Elt: C);
10246 continue;
10247 }
10248
10249 // If any of the operands is non-constant and if they are
10250 // non-integer and non-pointer, don't even try to analyze them
10251 // with scev techniques.
10252 if (!isSCEVable(Ty: Op->getType()))
10253 return V;
10254
10255 const SCEV *OrigV = getSCEV(V: Op);
10256 const SCEV *OpV = getSCEVAtScope(V: OrigV, L);
10257 MadeImprovement |= OrigV != OpV;
10258
10259 Constant *C = BuildConstantFromSCEV(V: OpV);
10260 if (!C)
10261 return V;
10262 assert(C->getType() == Op->getType() && "Type mismatch");
10263 Operands.push_back(Elt: C);
10264 }
10265
10266 // Check to see if getSCEVAtScope actually made an improvement.
10267 if (!MadeImprovement)
10268 return V; // This is some other type of SCEVUnknown, just return it.
10269
10270 Constant *C = nullptr;
10271 const DataLayout &DL = getDataLayout();
10272 C = ConstantFoldInstOperands(I, Ops: Operands, DL, TLI: &TLI,
10273 /*AllowNonDeterministic=*/false);
10274 if (!C)
10275 return V;
10276 return getSCEV(V: C);
10277 }
10278 case scCouldNotCompute:
10279 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
10280 }
10281 llvm_unreachable("Unknown SCEV type!");
10282}
10283
10284SCEVUse ScalarEvolution::getSCEVAtScope(Value *V, const Loop *L) {
10285 return getSCEVAtScope(V: getSCEV(V), L);
10286}
10287
10288const SCEV *ScalarEvolution::stripInjectiveFunctions(const SCEV *S) const {
10289 if (const SCEVZeroExtendExpr *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: S))
10290 return stripInjectiveFunctions(S: ZExt->getOperand());
10291 if (const SCEVSignExtendExpr *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
10292 return stripInjectiveFunctions(S: SExt->getOperand());
10293 return S;
10294}
10295
10296/// Finds the minimum unsigned root of the following equation:
10297///
10298/// A * X = B (mod N)
10299///
10300/// where N = 2^BW and BW is the common bit width of A and B. The signedness of
10301/// A and B isn't important.
10302///
10303/// If the equation does not have a solution, SCEVCouldNotCompute is returned.
10304static const SCEV *
10305SolveLinEquationWithOverflow(const APInt &A, const SCEV *B,
10306 SmallVectorImpl<const SCEVPredicate *> *Predicates,
10307 ScalarEvolution &SE, const Loop *L) {
10308 uint32_t BW = A.getBitWidth();
10309 assert(BW == SE.getTypeSizeInBits(B->getType()));
10310 assert(A != 0 && "A must be non-zero.");
10311
10312 // 1. D = gcd(A, N)
10313 //
10314 // The gcd of A and N may have only one prime factor: 2. The number of
10315 // trailing zeros in A is its multiplicity
10316 uint32_t Mult2 = A.countr_zero();
10317 // D = 2^Mult2
10318
10319 // 2. Check if B is divisible by D.
10320 //
10321 // B is divisible by D if and only if the multiplicity of prime factor 2 for B
10322 // is not less than multiplicity of this prime factor for D.
10323 unsigned MinTZ = SE.getMinTrailingZeros(S: B);
10324 // Try again with the terminator of the loop predecessor for context-specific
10325 // result, if MinTZ s too small.
10326 if (MinTZ < Mult2 && L->getLoopPredecessor())
10327 MinTZ = SE.getMinTrailingZeros(S: B, CtxI: L->getLoopPredecessor()->getTerminator());
10328 if (MinTZ < Mult2) {
10329 // Check if we can prove there's no remainder using URem.
10330 const SCEV *URem =
10331 SE.getURemExpr(LHS: B, RHS: SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2)));
10332 const SCEV *Zero = SE.getZero(Ty: B->getType());
10333 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: URem, RHS: Zero)) {
10334 // Try to add a predicate ensuring B is a multiple of 1 << Mult2.
10335 if (!Predicates)
10336 return SE.getCouldNotCompute();
10337
10338 // Avoid adding a predicate that is known to be false.
10339 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_NE, LHS: URem, RHS: Zero))
10340 return SE.getCouldNotCompute();
10341 Predicates->push_back(Elt: SE.getEqualPredicate(LHS: URem, RHS: Zero));
10342 }
10343 }
10344
10345 // 3. Compute I: the multiplicative inverse of (A / D) in arithmetic
10346 // modulo (N / D).
10347 //
10348 // If D == 1, (N / D) == N == 2^BW, so we need one extra bit to represent
10349 // (N / D) in general. The inverse itself always fits into BW bits, though,
10350 // so we immediately truncate it.
10351 APInt AD = A.lshr(shiftAmt: Mult2).trunc(width: BW - Mult2); // AD = A / D
10352 APInt I = AD.multiplicativeInverse().zext(width: BW);
10353
10354 // 4. Compute the minimum unsigned root of the equation:
10355 // I * (B / D) mod (N / D)
10356 // To simplify the computation, we factor out the divide by D:
10357 // (I * B mod N) / D
10358 const SCEV *D = SE.getConstant(Val: APInt::getOneBitSet(numBits: BW, BitNo: Mult2));
10359 return SE.getUDivExactExpr(LHS: SE.getMulExpr(LHS: B, RHS: SE.getConstant(Val: I)), RHS: D);
10360}
10361
10362/// For a given quadratic addrec, generate coefficients of the corresponding
10363/// quadratic equation, multiplied by a common value to ensure that they are
10364/// integers.
10365/// The returned value is a tuple { A, B, C, M, BitWidth }, where
10366/// Ax^2 + Bx + C is the quadratic function, M is the value that A, B and C
10367/// were multiplied by, and BitWidth is the bit width of the original addrec
10368/// coefficients.
10369/// This function returns std::nullopt if the addrec coefficients are not
10370/// compile- time constants.
10371static std::optional<std::tuple<APInt, APInt, APInt, APInt, unsigned>>
10372GetQuadraticEquation(const SCEVAddRecExpr *AddRec) {
10373 assert(AddRec->getNumOperands() == 3 && "This is not a quadratic chrec!");
10374 const SCEVConstant *LC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 0));
10375 const SCEVConstant *MC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 1));
10376 const SCEVConstant *NC = dyn_cast<SCEVConstant>(Val: AddRec->getOperand(i: 2));
10377 LLVM_DEBUG(dbgs() << __func__ << ": analyzing quadratic addrec: "
10378 << *AddRec << '\n');
10379
10380 // We currently can only solve this if the coefficients are constants.
10381 if (!LC || !MC || !NC) {
10382 LLVM_DEBUG(dbgs() << __func__ << ": coefficients are not constant\n");
10383 return std::nullopt;
10384 }
10385
10386 APInt L = LC->getAPInt();
10387 APInt M = MC->getAPInt();
10388 APInt N = NC->getAPInt();
10389 assert(!N.isZero() && "This is not a quadratic addrec");
10390
10391 unsigned BitWidth = LC->getAPInt().getBitWidth();
10392 unsigned NewWidth = BitWidth + 1;
10393 LLVM_DEBUG(dbgs() << __func__ << ": addrec coeff bw: "
10394 << BitWidth << '\n');
10395 // The sign-extension (as opposed to a zero-extension) here matches the
10396 // extension used in SolveQuadraticEquationWrap (with the same motivation).
10397 N = N.sext(width: NewWidth);
10398 M = M.sext(width: NewWidth);
10399 L = L.sext(width: NewWidth);
10400
10401 // The increments are M, M+N, M+2N, ..., so the accumulated values are
10402 // L+M, (L+M)+(M+N), (L+M)+(M+N)+(M+2N), ..., that is,
10403 // L+M, L+2M+N, L+3M+3N, ...
10404 // After n iterations the accumulated value Acc is L + nM + n(n-1)/2 N.
10405 //
10406 // The equation Acc = 0 is then
10407 // L + nM + n(n-1)/2 N = 0, or 2L + 2M n + n(n-1) N = 0.
10408 // In a quadratic form it becomes:
10409 // N n^2 + (2M-N) n + 2L = 0.
10410
10411 APInt A = N;
10412 APInt B = 2 * M - A;
10413 APInt C = 2 * L;
10414 APInt T = APInt(NewWidth, 2);
10415 LLVM_DEBUG(dbgs() << __func__ << ": equation " << A << "x^2 + " << B
10416 << "x + " << C << ", coeff bw: " << NewWidth
10417 << ", multiplied by " << T << '\n');
10418 return std::make_tuple(args&: A, args&: B, args&: C, args&: T, args&: BitWidth);
10419}
10420
10421/// Helper function to compare optional APInts:
10422/// (a) if X and Y both exist, return min(X, Y),
10423/// (b) if neither X nor Y exist, return std::nullopt,
10424/// (c) if exactly one of X and Y exists, return that value.
10425static std::optional<APInt> MinOptional(std::optional<APInt> X,
10426 std::optional<APInt> Y) {
10427 if (X && Y) {
10428 unsigned W = std::max(a: X->getBitWidth(), b: Y->getBitWidth());
10429 APInt XW = X->sext(width: W);
10430 APInt YW = Y->sext(width: W);
10431 return XW.slt(RHS: YW) ? *X : *Y;
10432 }
10433 if (!X && !Y)
10434 return std::nullopt;
10435 return X ? *X : *Y;
10436}
10437
10438/// Helper function to truncate an optional APInt to a given BitWidth.
10439/// When solving addrec-related equations, it is preferable to return a value
10440/// that has the same bit width as the original addrec's coefficients. If the
10441/// solution fits in the original bit width, truncate it (except for i1).
10442/// Returning a value of a different bit width may inhibit some optimizations.
10443///
10444/// In general, a solution to a quadratic equation generated from an addrec
10445/// may require BW+1 bits, where BW is the bit width of the addrec's
10446/// coefficients. The reason is that the coefficients of the quadratic
10447/// equation are BW+1 bits wide (to avoid truncation when converting from
10448/// the addrec to the equation).
10449static std::optional<APInt> TruncIfPossible(std::optional<APInt> X,
10450 unsigned BitWidth) {
10451 if (!X)
10452 return std::nullopt;
10453 unsigned W = X->getBitWidth();
10454 if (BitWidth > 1 && BitWidth < W && X->isIntN(N: BitWidth))
10455 return X->trunc(width: BitWidth);
10456 return X;
10457}
10458
10459/// Let c(n) be the value of the quadratic chrec {L,+,M,+,N} after n
10460/// iterations. The values L, M, N are assumed to be signed, and they
10461/// should all have the same bit widths.
10462/// Find the least n >= 0 such that c(n) = 0 in the arithmetic modulo 2^BW,
10463/// where BW is the bit width of the addrec's coefficients.
10464/// If the calculated value is a BW-bit integer (for BW > 1), it will be
10465/// returned as such, otherwise the bit width of the returned value may
10466/// be greater than BW.
10467///
10468/// This function returns std::nullopt if
10469/// (a) the addrec coefficients are not constant, or
10470/// (b) SolveQuadraticEquationWrap was unable to find a solution. For cases
10471/// like x^2 = 5, no integer solutions exist, in other cases an integer
10472/// solution may exist, but SolveQuadraticEquationWrap may fail to find it.
10473static std::optional<APInt>
10474SolveQuadraticAddRecExact(const SCEVAddRecExpr *AddRec, ScalarEvolution &SE) {
10475 APInt A, B, C, M;
10476 unsigned BitWidth;
10477 auto T = GetQuadraticEquation(AddRec);
10478 if (!T)
10479 return std::nullopt;
10480
10481 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10482 LLVM_DEBUG(dbgs() << __func__ << ": solving for unsigned overflow\n");
10483 std::optional<APInt> X =
10484 APIntOps::SolveQuadraticEquationWrap(A, B, C, RangeWidth: BitWidth + 1);
10485 if (!X)
10486 return std::nullopt;
10487
10488 ConstantInt *CX = ConstantInt::get(Context&: SE.getContext(), V: *X);
10489 ConstantInt *V = EvaluateConstantChrecAtConstant(AddRec, C: CX, SE);
10490 if (!V->isZero())
10491 return std::nullopt;
10492
10493 return TruncIfPossible(X, BitWidth);
10494}
10495
10496/// Let c(n) be the value of the quadratic chrec {0,+,M,+,N} after n
10497/// iterations. The values M, N are assumed to be signed, and they
10498/// should all have the same bit widths.
10499/// Find the least n such that c(n) does not belong to the given range,
10500/// while c(n-1) does.
10501///
10502/// This function returns std::nullopt if
10503/// (a) the addrec coefficients are not constant, or
10504/// (b) SolveQuadraticEquationWrap was unable to find a solution for the
10505/// bounds of the range.
10506static std::optional<APInt>
10507SolveQuadraticAddRecRange(const SCEVAddRecExpr *AddRec,
10508 const ConstantRange &Range, ScalarEvolution &SE) {
10509 assert(AddRec->getOperand(0)->isZero() &&
10510 "Starting value of addrec should be 0");
10511 LLVM_DEBUG(dbgs() << __func__ << ": solving boundary crossing for range "
10512 << Range << ", addrec " << *AddRec << '\n');
10513 // This case is handled in getNumIterationsInRange. Here we can assume that
10514 // we start in the range.
10515 assert(Range.contains(APInt(SE.getTypeSizeInBits(AddRec->getType()), 0)) &&
10516 "Addrec's initial value should be in range");
10517
10518 APInt A, B, C, M;
10519 unsigned BitWidth;
10520 auto T = GetQuadraticEquation(AddRec);
10521 if (!T)
10522 return std::nullopt;
10523
10524 // Be careful about the return value: there can be two reasons for not
10525 // returning an actual number. First, if no solutions to the equations
10526 // were found, and second, if the solutions don't leave the given range.
10527 // The first case means that the actual solution is "unknown", the second
10528 // means that it's known, but not valid. If the solution is unknown, we
10529 // cannot make any conclusions.
10530 // Return a pair: the optional solution and a flag indicating if the
10531 // solution was found.
10532 auto SolveForBoundary =
10533 [&](APInt Bound) -> std::pair<std::optional<APInt>, bool> {
10534 // Solve for signed overflow and unsigned overflow, pick the lower
10535 // solution.
10536 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: checking boundary "
10537 << Bound << " (before multiplying by " << M << ")\n");
10538 Bound *= M; // The quadratic equation multiplier.
10539
10540 std::optional<APInt> SO;
10541 if (BitWidth > 1) {
10542 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10543 "signed overflow\n");
10544 SO = APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth);
10545 }
10546 LLVM_DEBUG(dbgs() << "SolveQuadraticAddRecRange: solving for "
10547 "unsigned overflow\n");
10548 std::optional<APInt> UO =
10549 APIntOps::SolveQuadraticEquationWrap(A, B, C: -Bound, RangeWidth: BitWidth + 1);
10550
10551 auto LeavesRange = [&] (const APInt &X) {
10552 ConstantInt *C0 = ConstantInt::get(Context&: SE.getContext(), V: X);
10553 ConstantInt *V0 = EvaluateConstantChrecAtConstant(AddRec, C: C0, SE);
10554 if (Range.contains(Val: V0->getValue()))
10555 return false;
10556 // X should be at least 1, so X-1 is non-negative.
10557 ConstantInt *C1 = ConstantInt::get(Context&: SE.getContext(), V: X-1);
10558 ConstantInt *V1 = EvaluateConstantChrecAtConstant(AddRec, C: C1, SE);
10559 if (Range.contains(Val: V1->getValue()))
10560 return true;
10561 return false;
10562 };
10563
10564 // If SolveQuadraticEquationWrap returns std::nullopt, it means that there
10565 // can be a solution, but the function failed to find it. We cannot treat it
10566 // as "no solution".
10567 if (!SO || !UO)
10568 return {std::nullopt, false};
10569
10570 // Check the smaller value first to see if it leaves the range.
10571 // At this point, both SO and UO must have values.
10572 std::optional<APInt> Min = MinOptional(X: SO, Y: UO);
10573 if (LeavesRange(*Min))
10574 return { Min, true };
10575 std::optional<APInt> Max = Min == SO ? UO : SO;
10576 if (LeavesRange(*Max))
10577 return { Max, true };
10578
10579 // Solutions were found, but were eliminated, hence the "true".
10580 return {std::nullopt, true};
10581 };
10582
10583 std::tie(args&: A, args&: B, args&: C, args&: M, args&: BitWidth) = *T;
10584 // Lower bound is inclusive, subtract 1 to represent the exiting value.
10585 APInt Lower = Range.getLower().sext(width: A.getBitWidth()) - 1;
10586 APInt Upper = Range.getUpper().sext(width: A.getBitWidth());
10587 auto SL = SolveForBoundary(Lower);
10588 auto SU = SolveForBoundary(Upper);
10589 // If any of the solutions was unknown, no meaninigful conclusions can
10590 // be made.
10591 if (!SL.second || !SU.second)
10592 return std::nullopt;
10593
10594 // Claim: The correct solution is not some value between Min and Max.
10595 //
10596 // Justification: Assuming that Min and Max are different values, one of
10597 // them is when the first signed overflow happens, the other is when the
10598 // first unsigned overflow happens. Crossing the range boundary is only
10599 // possible via an overflow (treating 0 as a special case of it, modeling
10600 // an overflow as crossing k*2^W for some k).
10601 //
10602 // The interesting case here is when Min was eliminated as an invalid
10603 // solution, but Max was not. The argument is that if there was another
10604 // overflow between Min and Max, it would also have been eliminated if
10605 // it was considered.
10606 //
10607 // For a given boundary, it is possible to have two overflows of the same
10608 // type (signed/unsigned) without having the other type in between: this
10609 // can happen when the vertex of the parabola is between the iterations
10610 // corresponding to the overflows. This is only possible when the two
10611 // overflows cross k*2^W for the same k. In such case, if the second one
10612 // left the range (and was the first one to do so), the first overflow
10613 // would have to enter the range, which would mean that either we had left
10614 // the range before or that we started outside of it. Both of these cases
10615 // are contradictions.
10616 //
10617 // Claim: In the case where SolveForBoundary returns std::nullopt, the correct
10618 // solution is not some value between the Max for this boundary and the
10619 // Min of the other boundary.
10620 //
10621 // Justification: Assume that we had such Max_A and Min_B corresponding
10622 // to range boundaries A and B and such that Max_A < Min_B. If there was
10623 // a solution between Max_A and Min_B, it would have to be caused by an
10624 // overflow corresponding to either A or B. It cannot correspond to B,
10625 // since Min_B is the first occurrence of such an overflow. If it
10626 // corresponded to A, it would have to be either a signed or an unsigned
10627 // overflow that is larger than both eliminated overflows for A. But
10628 // between the eliminated overflows and this overflow, the values would
10629 // cover the entire value space, thus crossing the other boundary, which
10630 // is a contradiction.
10631
10632 return TruncIfPossible(X: MinOptional(X: SL.first, Y: SU.first), BitWidth);
10633}
10634
10635ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V,
10636 const Loop *L,
10637 bool ControlsOnlyExit,
10638 bool AllowPredicates) {
10639
10640 // This is only used for loops with a "x != y" exit test. The exit condition
10641 // is now expressed as a single expression, V = x-y. So the exit test is
10642 // effectively V != 0. We know and take advantage of the fact that this
10643 // expression only being used in a comparison by zero context.
10644
10645 SmallVector<const SCEVPredicate *> Predicates;
10646 // If the value is a constant
10647 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10648 // If the value is already zero, the branch will execute zero times.
10649 if (C->getValue()->isZero()) return C;
10650 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10651 }
10652
10653 const SCEVAddRecExpr *AddRec =
10654 dyn_cast<SCEVAddRecExpr>(Val: stripInjectiveFunctions(S: V));
10655
10656 if (!AddRec && AllowPredicates)
10657 // Try to make this an AddRec using runtime tests, in the first X
10658 // iterations of this loop, where X is the SCEV expression found by the
10659 // algorithm below.
10660 AddRec = convertSCEVToAddRecWithPredicates(S: V, L, Preds&: Predicates);
10661
10662 if (!AddRec || AddRec->getLoop() != L)
10663 return getCouldNotCompute();
10664
10665 // If this is a quadratic (3-term) AddRec {L,+,M,+,N}, find the roots of
10666 // the quadratic equation to solve it.
10667 if (AddRec->isQuadratic() && AddRec->getType()->isIntegerTy()) {
10668 // We can only use this value if the chrec ends up with an exact zero
10669 // value at this index. When solving for "X*X != 5", for example, we
10670 // should not accept a root of 2.
10671 if (auto S = SolveQuadraticAddRecExact(AddRec, SE&: *this)) {
10672 const auto *R = cast<SCEVConstant>(Val: getConstant(Val: *S));
10673 return ExitLimit(R, R, R, false, Predicates);
10674 }
10675 return getCouldNotCompute();
10676 }
10677
10678 // Otherwise we can only handle this if it is affine.
10679 if (!AddRec->isAffine())
10680 return getCouldNotCompute();
10681
10682 // If this is an affine expression, the execution count of this branch is
10683 // the minimum unsigned root of the following equation:
10684 //
10685 // Start + Step*N = 0 (mod 2^BW)
10686 //
10687 // equivalent to:
10688 //
10689 // Step*N = -Start (mod 2^BW)
10690 //
10691 // where BW is the common bit width of Start and Step.
10692
10693 // Get the initial value for the loop.
10694 const SCEV *Start = getSCEVAtScope(V: AddRec->getStart(), L: L->getParentLoop());
10695 const SCEV *Step = getSCEVAtScope(V: AddRec->getOperand(i: 1), L: L->getParentLoop());
10696
10697 if (!isLoopInvariant(S: Step, L))
10698 return getCouldNotCompute();
10699
10700 LoopGuards Guards = LoopGuards::collectOnDemand(L, SE&: *this);
10701 // Specialize step for this loop so we get context sensitive facts below.
10702 const SCEV *StepWLG = applyLoopGuards(Expr: Step, Guards);
10703
10704 // For positive steps (counting up until unsigned overflow):
10705 // N = -Start/Step (as unsigned)
10706 // For negative steps (counting down to zero):
10707 // N = Start/-Step
10708 // First compute the unsigned distance from zero in the direction of Step.
10709 bool CountDown = isKnownNegative(S: StepWLG);
10710 if (!CountDown && !isKnownNonNegative(S: StepWLG))
10711 return getCouldNotCompute();
10712
10713 const SCEV *Distance = CountDown ? Start : getNegativeSCEV(V: Start);
10714 // Handle unitary steps, which cannot wraparound.
10715 // 1*N = -Start; -1*N = Start (mod 2^BW), so:
10716 // N = Distance (as unsigned)
10717
10718 if (match(S: Step, P: m_CombineOr(Ps: m_scev_One(), Ps: m_scev_AllOnes()))) {
10719 APInt MaxBECount = getUnsignedRangeMax(S: applyLoopGuards(Expr: Distance, Guards));
10720 MaxBECount = APIntOps::umin(A: MaxBECount, B: getUnsignedRangeMax(S: Distance));
10721
10722 // When a loop like "for (int i = 0; i != n; ++i) { /* body */ }" is rotated,
10723 // we end up with a loop whose backedge-taken count is n - 1. Detect this
10724 // case, and see if we can improve the bound.
10725 //
10726 // Explicitly handling this here is necessary because getUnsignedRange
10727 // isn't context-sensitive; it doesn't know that we only care about the
10728 // range inside the loop.
10729 const SCEV *Zero = getZero(Ty: Distance->getType());
10730 const SCEV *One = getOne(Ty: Distance->getType());
10731 const SCEV *DistancePlusOne = getAddExpr(LHS: Distance, RHS: One);
10732 if (isLoopEntryGuardedByCond(L, Pred: ICmpInst::ICMP_NE, LHS: DistancePlusOne, RHS: Zero)) {
10733 // If Distance + 1 doesn't overflow, we can compute the maximum distance
10734 // as "unsigned_max(Distance + 1) - 1". Also apply the loop guards to
10735 // Distance + 1; the range of Distance itself may be a wrapped set even
10736 // when the guards bound Distance + 1 tightly.
10737 APInt Max = APIntOps::umin(
10738 A: getUnsignedRangeMax(S: applyLoopGuards(Expr: DistancePlusOne, Guards)),
10739 B: getUnsignedRangeMax(S: DistancePlusOne));
10740 MaxBECount = APIntOps::umin(A: MaxBECount, B: Max - 1);
10741 }
10742 return ExitLimit(Distance, getConstant(Val: MaxBECount), Distance, false,
10743 Predicates);
10744 }
10745
10746 // If the condition controls loop exit (the loop exits only if the expression
10747 // is true) and the addition is no-wrap we can use unsigned divide to
10748 // compute the backedge count. In this case, the step may not divide the
10749 // distance, but we don't care because if the condition is "missed" the loop
10750 // will have undefined behavior due to wrapping.
10751 if (ControlsOnlyExit && AddRec->hasNoSelfWrap() &&
10752 loopHasNoAbnormalExits(L: AddRec->getLoop())) {
10753
10754 // If the stride is zero and the start is non-zero, the loop must be
10755 // infinite. In C++, most loops are finite by assumption, in which case the
10756 // step being zero implies UB must execute if the loop is entered.
10757 if (!(loopIsFiniteByAssumption(L) && isKnownNonZero(S: Start)) &&
10758 !isKnownNonZero(S: StepWLG))
10759 return getCouldNotCompute();
10760
10761 const SCEV *Exact =
10762 getUDivExpr(LHS: Distance, RHS: CountDown ? getNegativeSCEV(V: Step) : Step);
10763 const SCEV *ConstantMax = getCouldNotCompute();
10764 if (Exact != getCouldNotCompute()) {
10765 APInt MaxInt = getUnsignedRangeMax(S: applyLoopGuards(Expr: Exact, Guards));
10766 ConstantMax =
10767 getConstant(Val: APIntOps::umin(A: MaxInt, B: getUnsignedRangeMax(S: Exact)));
10768 }
10769 const SCEV *SymbolicMax =
10770 isa<SCEVCouldNotCompute>(Val: Exact) ? ConstantMax : Exact;
10771 return ExitLimit(Exact, ConstantMax, SymbolicMax, false, Predicates);
10772 }
10773
10774 // Solve the general equation.
10775 const SCEVConstant *StepC = dyn_cast<SCEVConstant>(Val: Step);
10776 if (!StepC || StepC->getValue()->isZero())
10777 return getCouldNotCompute();
10778 const SCEV *E = SolveLinEquationWithOverflow(
10779 A: StepC->getAPInt(), B: getNegativeSCEV(V: Start),
10780 Predicates: AllowPredicates ? &Predicates : nullptr, SE&: *this, L);
10781
10782 const SCEV *M = E;
10783 if (E != getCouldNotCompute()) {
10784 APInt MaxWithGuards = getUnsignedRangeMax(S: applyLoopGuards(Expr: E, Guards));
10785 M = getConstant(Val: APIntOps::umin(A: MaxWithGuards, B: getUnsignedRangeMax(S: E)));
10786 }
10787 auto *S = isa<SCEVCouldNotCompute>(Val: E) ? M : E;
10788 return ExitLimit(E, M, S, false, Predicates);
10789}
10790
10791ScalarEvolution::ExitLimit
10792ScalarEvolution::howFarToNonZero(const SCEV *V, const Loop *L) {
10793 // Loops that look like: while (X == 0) are very strange indeed. We don't
10794 // handle them yet except for the trivial case. This could be expanded in the
10795 // future as needed.
10796
10797 // If the value is a constant, check to see if it is known to be non-zero
10798 // already. If so, the backedge will execute zero times.
10799 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: V)) {
10800 if (!C->getValue()->isZero())
10801 return getZero(Ty: C->getType());
10802 return getCouldNotCompute(); // Otherwise it will loop infinitely.
10803 }
10804
10805 // We could implement others, but I really doubt anyone writes loops like
10806 // this, and if they did, they would already be constant folded.
10807 return getCouldNotCompute();
10808}
10809
10810std::pair<const BasicBlock *, const BasicBlock *>
10811ScalarEvolution::getPredecessorWithUniqueSuccessorForBB(const BasicBlock *BB)
10812 const {
10813 // If the block has a unique predecessor, then there is no path from the
10814 // predecessor to the block that does not go through the direct edge
10815 // from the predecessor to the block.
10816 if (const BasicBlock *Pred = BB->getSinglePredecessor())
10817 return {Pred, BB};
10818
10819 // A loop's header is defined to be a block that dominates the loop.
10820 // If the header has a unique predecessor outside the loop, it must be
10821 // a block that has exactly one successor that can reach the loop.
10822 if (const Loop *L = LI.getLoopFor(BB))
10823 return {L->getLoopPredecessor(), L->getHeader()};
10824
10825 return {nullptr, BB};
10826}
10827
10828/// SCEV structural equivalence is usually sufficient for testing whether two
10829/// expressions are equal, however for the purposes of looking for a condition
10830/// guarding a loop, it can be useful to be a little more general, since a
10831/// front-end may have replicated the controlling expression.
10832static bool HasSameValue(const SCEV *A, const SCEV *B) {
10833 // Quick check to see if they are the same SCEV.
10834 if (A == B) return true;
10835
10836 auto ComputesEqualValues = [](const Instruction *A, const Instruction *B) {
10837 // Not all instructions that are "identical" compute the same value. For
10838 // instance, two distinct alloca instructions allocating the same type are
10839 // identical and do not read memory; but compute distinct values.
10840 return A->isIdenticalTo(I: B) && (isa<BinaryOperator>(Val: A) || isa<GetElementPtrInst>(Val: A));
10841 };
10842
10843 // Otherwise, if they're both SCEVUnknown, it's possible that they hold
10844 // two different instructions with the same value. Check for this case.
10845 if (const SCEVUnknown *AU = dyn_cast<SCEVUnknown>(Val: A))
10846 if (const SCEVUnknown *BU = dyn_cast<SCEVUnknown>(Val: B))
10847 if (const Instruction *AI = dyn_cast<Instruction>(Val: AU->getValue()))
10848 if (const Instruction *BI = dyn_cast<Instruction>(Val: BU->getValue()))
10849 if (ComputesEqualValues(AI, BI))
10850 return true;
10851
10852 // Otherwise assume they may have a different value.
10853 return false;
10854}
10855
10856static bool MatchBinarySub(const SCEV *S, SCEVUse &LHS, SCEVUse &RHS) {
10857 const SCEV *Op0, *Op1;
10858 if (!match(S, P: m_scev_Add(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1))))
10859 return false;
10860 if (match(S: Op0, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
10861 LHS = Op1;
10862 return true;
10863 }
10864 if (match(S: Op1, P: m_scev_Mul(Op0: m_scev_AllOnes(), Op1: m_SCEV(V&: RHS)))) {
10865 LHS = Op0;
10866 return true;
10867 }
10868 return false;
10869}
10870
10871bool ScalarEvolution::SimplifyICmpOperands(CmpPredicate &Pred, SCEVUse &LHS,
10872 SCEVUse &RHS, unsigned Depth) {
10873 bool Changed = false;
10874 // Simplifies ICMP to trivial true or false by turning it into '0 == 0' or
10875 // '0 != 0'.
10876 auto TrivialCase = [&](bool TriviallyTrue) {
10877 LHS = RHS = getConstant(V: ConstantInt::getFalse(Context&: getContext()));
10878 Pred = TriviallyTrue ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
10879 return true;
10880 };
10881 // If we hit the max recursion limit bail out.
10882 if (Depth >= 3)
10883 return false;
10884
10885 const SCEV *NewLHS, *NewRHS;
10886 if (match(U: LHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewLHS), Op1: m_SCEVVScale())) &&
10887 match(U: RHS, P: m_scev_c_Mul(Op0: m_SCEV(V&: NewRHS), Op1: m_SCEVVScale()))) {
10888 const SCEVMulExpr *LMul = cast<SCEVMulExpr>(Val&: LHS);
10889 const SCEVMulExpr *RMul = cast<SCEVMulExpr>(Val&: RHS);
10890
10891 // (X * vscale) pred (Y * vscale) ==> X pred Y
10892 // when both multiples are NSW.
10893 // (X * vscale) uicmp/eq/ne (Y * vscale) ==> X uicmp/eq/ne Y
10894 // when both multiples are NUW.
10895 if ((LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap()) ||
10896 (LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap() &&
10897 !ICmpInst::isSigned(Pred))) {
10898 LHS = NewLHS;
10899 RHS = NewRHS;
10900 Changed = true;
10901 }
10902 }
10903
10904 // Canonicalize a constant to the right side.
10905 if (const SCEVConstant *LHSC = dyn_cast<SCEVConstant>(Val&: LHS)) {
10906 // Check for both operands constant.
10907 if (const SCEVConstant *RHSC = dyn_cast<SCEVConstant>(Val&: RHS)) {
10908 if (!ICmpInst::compare(LHS: LHSC->getAPInt(), RHS: RHSC->getAPInt(), Pred))
10909 return TrivialCase(false);
10910 return TrivialCase(true);
10911 }
10912 // Otherwise swap the operands to put the constant on the right.
10913 std::swap(a&: LHS, b&: RHS);
10914 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
10915 Changed = true;
10916 }
10917
10918 // (K + A) pred (K + B) --> A pred B
10919 // For equality, no flags are needed.
10920 // For signed, both adds must be NSW. For unsigned, both must be NUW.
10921 {
10922 const SCEVConstant *C = nullptr;
10923 if (match(U: LHS, P: m_scev_Add(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: NewLHS))) &&
10924 match(U: RHS, P: m_scev_Add(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: NewRHS)))) {
10925 const auto *LAdd = cast<SCEVAddExpr>(Val&: LHS);
10926 const auto *RAdd = cast<SCEVAddExpr>(Val&: RHS);
10927 bool BothNUW = LAdd->hasNoUnsignedWrap() && RAdd->hasNoUnsignedWrap();
10928 bool BothNSW = LAdd->hasNoSignedWrap() && RAdd->hasNoSignedWrap();
10929 if (ICmpInst::isEquality(P: Pred) || (ICmpInst::isSigned(Pred) && BothNSW) ||
10930 (ICmpInst::isUnsigned(Pred) && BothNUW)) {
10931 LHS = NewLHS;
10932 RHS = NewRHS;
10933 // Nothing follows about the signs of A and B themselves, as adding K
10934 // may move either A or B across the sign boundary, unless both adds are
10935 // guaranteed not to wrap in either domain.
10936 if (!(BothNUW && BothNSW))
10937 Pred = Pred.dropSameSign();
10938 Changed = true;
10939 }
10940 }
10941 }
10942
10943 // (C * A) pred (C * B) --> A pred B
10944 // For equality predicates, both muls must be NUW or both must be NSW
10945 // (either suffices to make multiplication by C injective; C == 0 is
10946 // impossible because SCEV folds 0 * X to 0).
10947 // For signed ordering, C must be positive and both muls must be NSW.
10948 // For unsigned ordering, both muls must be NUW.
10949 {
10950 const SCEVConstant *C = nullptr;
10951 if (match(U: LHS, P: m_scev_Mul(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: NewLHS))) &&
10952 match(U: RHS, P: m_scev_Mul(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: NewRHS)))) {
10953 const auto *LMul = cast<SCEVMulExpr>(Val&: LHS);
10954 const auto *RMul = cast<SCEVMulExpr>(Val&: RHS);
10955 bool BothNUW = LMul->hasNoUnsignedWrap() && RMul->hasNoUnsignedWrap();
10956 bool BothNSW = LMul->hasNoSignedWrap() && RMul->hasNoSignedWrap();
10957 if ((ICmpInst::isEquality(P: Pred) && (BothNUW || BothNSW)) ||
10958 (ICmpInst::isSigned(Pred) && BothNSW &&
10959 C->getAPInt().isStrictlyPositive()) ||
10960 (ICmpInst::isUnsigned(Pred) && BothNUW)) {
10961 LHS = NewLHS;
10962 RHS = NewRHS;
10963 Changed = true;
10964 }
10965 }
10966 }
10967
10968 // If we're comparing an addrec with a value which is loop-invariant in the
10969 // addrec's loop, put the addrec on the left. Also make a dominance check,
10970 // as both operands could be addrecs loop-invariant in each other's loop.
10971 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: RHS)) {
10972 const Loop *L = AR->getLoop();
10973 if (isLoopInvariant(S: LHS, L) && properlyDominates(S: LHS, BB: L->getHeader())) {
10974 std::swap(a&: LHS, b&: RHS);
10975 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
10976 Changed = true;
10977 }
10978 }
10979
10980 // If there's a constant operand, canonicalize comparisons with boundary
10981 // cases, and canonicalize *-or-equal comparisons to regular comparisons.
10982 if (const SCEVConstant *RC = dyn_cast<SCEVConstant>(Val&: RHS)) {
10983 const APInt &RA = RC->getAPInt();
10984
10985 bool SimplifiedByConstantRange = false;
10986
10987 if (!ICmpInst::isEquality(P: Pred)) {
10988 ConstantRange ExactCR = ConstantRange::makeExactICmpRegion(Pred, Other: RA);
10989 if (ExactCR.isFullSet())
10990 return TrivialCase(true);
10991 if (ExactCR.isEmptySet())
10992 return TrivialCase(false);
10993
10994 APInt NewRHS;
10995 CmpInst::Predicate NewPred;
10996 if (ExactCR.getEquivalentICmp(Pred&: NewPred, RHS&: NewRHS) &&
10997 ICmpInst::isEquality(P: NewPred)) {
10998 // We were able to convert an inequality to an equality.
10999 Pred = NewPred;
11000 RHS = getConstant(Val: NewRHS);
11001 Changed = SimplifiedByConstantRange = true;
11002 }
11003 }
11004
11005 if (!SimplifiedByConstantRange) {
11006 switch (Pred) {
11007 default:
11008 break;
11009 case ICmpInst::ICMP_EQ:
11010 case ICmpInst::ICMP_NE:
11011 // Fold ((-1) * %a) + %b == 0 (equivalent to %b-%a == 0) into %a == %b.
11012 if (RA.isZero() && MatchBinarySub(S: LHS, LHS, RHS))
11013 Changed = true;
11014 break;
11015
11016 // The "Should have been caught earlier!" messages refer to the fact
11017 // that the ExactCR.isFullSet() or ExactCR.isEmptySet() check above
11018 // should have fired on the corresponding cases, and canonicalized the
11019 // check to trivial case.
11020
11021 case ICmpInst::ICMP_UGE:
11022 assert(!RA.isMinValue() && "Should have been caught earlier!");
11023 Pred = ICmpInst::ICMP_UGT;
11024 RHS = getConstant(Val: RA - 1);
11025 Changed = true;
11026 break;
11027 case ICmpInst::ICMP_ULE:
11028 assert(!RA.isMaxValue() && "Should have been caught earlier!");
11029 Pred = ICmpInst::ICMP_ULT;
11030 RHS = getConstant(Val: RA + 1);
11031 Changed = true;
11032 break;
11033 case ICmpInst::ICMP_SGE:
11034 assert(!RA.isMinSignedValue() && "Should have been caught earlier!");
11035 Pred = ICmpInst::ICMP_SGT;
11036 RHS = getConstant(Val: RA - 1);
11037 Changed = true;
11038 break;
11039 case ICmpInst::ICMP_SLE:
11040 assert(!RA.isMaxSignedValue() && "Should have been caught earlier!");
11041 Pred = ICmpInst::ICMP_SLT;
11042 RHS = getConstant(Val: RA + 1);
11043 Changed = true;
11044 break;
11045 }
11046 }
11047 }
11048
11049 // a /u b == 0 => a < b
11050 // a /u b != 0 => a >= b
11051 if (ICmpInst::isEquality(P: Pred) && RHS->isZero() &&
11052 match(U: LHS, P: m_scev_UDiv(Op0: m_SCEV(V&: LHS), Op1: m_SCEV(V&: RHS)))) {
11053 Pred = Pred == ICmpInst::ICMP_EQ ? ICmpInst::ICMP_ULT : ICmpInst::ICMP_UGE;
11054 Changed = true;
11055 }
11056
11057 // Check for obvious equality.
11058 if (HasSameValue(A: LHS, B: RHS)) {
11059 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
11060 return TrivialCase(true);
11061 if (ICmpInst::isFalseWhenEqual(predicate: Pred))
11062 return TrivialCase(false);
11063 }
11064
11065 // If possible, canonicalize GE/LE comparisons to GT/LT comparisons, by
11066 // adding or subtracting 1 from one of the operands.
11067 switch (Pred) {
11068 case ICmpInst::ICMP_SLE:
11069 if (!getSignedRangeMax(S: RHS).isMaxSignedValue()) {
11070 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11071 Flags: SCEV::FlagNSW);
11072 Pred = ICmpInst::ICMP_SLT;
11073 Changed = true;
11074 } else if (!getSignedRangeMin(S: LHS).isMinSignedValue()) {
11075 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS,
11076 Flags: SCEV::FlagNSW);
11077 Pred = ICmpInst::ICMP_SLT;
11078 Changed = true;
11079 }
11080 break;
11081 case ICmpInst::ICMP_SGE:
11082 if (!getSignedRangeMin(S: RHS).isMinSignedValue()) {
11083 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS,
11084 Flags: SCEV::FlagNSW);
11085 Pred = ICmpInst::ICMP_SGT;
11086 Changed = true;
11087 } else if (!getSignedRangeMax(S: LHS).isMaxSignedValue()) {
11088 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11089 Flags: SCEV::FlagNSW);
11090 Pred = ICmpInst::ICMP_SGT;
11091 Changed = true;
11092 }
11093 break;
11094 case ICmpInst::ICMP_ULE:
11095 if (!getUnsignedRangeMax(S: RHS).isMaxValue()) {
11096 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS,
11097 Flags: SCEV::FlagNUW);
11098 Pred = ICmpInst::ICMP_ULT;
11099 Changed = true;
11100 } else if (!getUnsignedRangeMin(S: LHS).isMinValue()) {
11101 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS: LHS);
11102 Pred = ICmpInst::ICMP_ULT;
11103 Changed = true;
11104 }
11105 break;
11106 case ICmpInst::ICMP_UGE:
11107 // If RHS is an op we can fold the -1, try that first.
11108 // Otherwise prefer LHS to preserve the nuw flag.
11109 if ((isa<SCEVConstant>(Val: RHS) ||
11110 (isa<SCEVAddExpr, SCEVAddRecExpr>(Val: RHS) &&
11111 isa<SCEVConstant>(Val: cast<SCEVNAryExpr>(Val&: RHS)->getOperand(i: 0)))) &&
11112 !getUnsignedRangeMin(S: RHS).isMinValue()) {
11113 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11114 Pred = ICmpInst::ICMP_UGT;
11115 Changed = true;
11116 } else if (!getUnsignedRangeMax(S: LHS).isMaxValue()) {
11117 LHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: 1, isSigned: true), RHS: LHS,
11118 Flags: SCEV::FlagNUW);
11119 Pred = ICmpInst::ICMP_UGT;
11120 Changed = true;
11121 } else if (!getUnsignedRangeMin(S: RHS).isMinValue()) {
11122 RHS = getAddExpr(LHS: getConstant(Ty: RHS->getType(), V: (uint64_t)-1, isSigned: true), RHS);
11123 Pred = ICmpInst::ICMP_UGT;
11124 Changed = true;
11125 }
11126 break;
11127 default:
11128 break;
11129 }
11130
11131 // TODO: More simplifications are possible here.
11132
11133 // Recursively simplify until we either hit a recursion limit or nothing
11134 // changes.
11135 if (Changed)
11136 (void)SimplifyICmpOperands(Pred, LHS, RHS, Depth: Depth + 1);
11137
11138 return Changed;
11139}
11140
11141bool ScalarEvolution::isKnownNegative(const SCEV *S) {
11142 return getSignedRangeMax(S).isNegative();
11143}
11144
11145bool ScalarEvolution::isKnownPositive(const SCEV *S) {
11146 return getSignedRangeMin(S).isStrictlyPositive();
11147}
11148
11149bool ScalarEvolution::isKnownNonNegative(const SCEV *S) {
11150 return !getSignedRangeMin(S).isNegative();
11151}
11152
11153bool ScalarEvolution::isKnownNonPositive(const SCEV *S) {
11154 return !getSignedRangeMax(S).isStrictlyPositive();
11155}
11156
11157bool ScalarEvolution::isKnownNonZero(const SCEV *S) {
11158 // Query push down for cases where the unsigned range is
11159 // less than sufficient.
11160 if (const auto *SExt = dyn_cast<SCEVSignExtendExpr>(Val: S))
11161 return isKnownNonZero(S: SExt->getOperand(i: 0));
11162 return getUnsignedRangeMin(S) != 0;
11163}
11164
11165bool ScalarEvolution::isKnownToBeAPowerOfTwo(const SCEV *S, bool OrZero,
11166 bool OrNegative) {
11167 auto NonRecursive = [OrNegative](const SCEV *S) {
11168 if (auto *C = dyn_cast<SCEVConstant>(Val: S))
11169 return C->getAPInt().isPowerOf2() ||
11170 (OrNegative && C->getAPInt().isNegatedPowerOf2());
11171
11172 // vscale is a power-of-two.
11173 return isa<SCEVVScale>(Val: S);
11174 };
11175
11176 if (NonRecursive(S))
11177 return true;
11178
11179 auto *Mul = dyn_cast<SCEVMulExpr>(Val: S);
11180 if (!Mul)
11181 return false;
11182 return all_of(Range: Mul->operands(), P: NonRecursive) && (OrZero || isKnownNonZero(S));
11183}
11184
11185bool ScalarEvolution::isKnownMultipleOf(
11186 const SCEV *S, uint64_t M,
11187 SmallVectorImpl<const SCEVPredicate *> *Predicates) {
11188 if (M == 0)
11189 return false;
11190 if (M == 1)
11191 return true;
11192
11193 // For a constant, check that "S % M == 0".
11194 if (auto *Cst = dyn_cast<SCEVConstant>(Val: S)) {
11195 APInt C = Cst->getAPInt();
11196 return C.urem(RHS: M) == 0;
11197 }
11198
11199 // Basic tests have failed.
11200 // Check "S % M == 0" at compile time and record runtime Assumptions.
11201 auto *STy = dyn_cast<IntegerType>(Val: S->getType());
11202 const SCEV *SmodM =
11203 getURemExpr(LHS: S, RHS: getConstant(V: ConstantInt::get(Ty: STy, V: M, IsSigned: false)));
11204 const SCEV *Zero = getZero(Ty: STy);
11205
11206 // Check whether "S % M == 0" is known at compile time.
11207 if (isKnownPredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero))
11208 return true;
11209
11210 // Check whether "S % M != 0" is known at compile time.
11211 if (isKnownPredicate(Pred: ICmpInst::ICMP_NE, LHS: SmodM, RHS: Zero))
11212 return false;
11213
11214 if (!Predicates)
11215 return false;
11216
11217 // Look through Add and AddRec expressions with nuw to improve the
11218 // precision of added predicates. S is a multiple of M if S starts with a
11219 // multiple of M and at every iteration step S only adds multiples of M.
11220 if (isa<SCEVAddExpr, SCEVAddRecExpr>(Val: S) &&
11221 cast<SCEVNAryExpr>(Val: S)->hasNoUnsignedWrap() &&
11222 all_of(Range: S->operands(),
11223 P: [&](SCEVUse Op) { return isKnownMultipleOf(S: Op, M, Predicates); }))
11224 return true;
11225
11226 // Similarly, look through Mul with nuw, where any operand being a
11227 // known-multiple is sufficient.
11228 if (auto *Mul = dyn_cast<SCEVMulExpr>(Val: S))
11229 if (Mul->hasNoUnsignedWrap() && any_of(Range: S->operands(), P: [&](SCEVUse Op) {
11230 return isKnownMultipleOf(S: Op, M, Predicates);
11231 }))
11232 return true;
11233
11234 // Similarly, look through MinMax, with no wrapping arithmetic to consider.
11235 if (isa<SCEVMinMaxExpr>(Val: S) && all_of(Range: S->operands(), P: [&](SCEVUse Op) {
11236 return isKnownMultipleOf(S: Op, M, Predicates);
11237 }))
11238 return true;
11239
11240 const SCEVPredicate *P = getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS: SmodM, RHS: Zero);
11241
11242 // Detect redundant predicates.
11243 for (auto *A : *Predicates)
11244 if (A->implies(N: P, SE&: *this))
11245 return true;
11246
11247 // Only record non-redundant predicates.
11248 Predicates->push_back(Elt: P);
11249 return true;
11250}
11251
11252bool ScalarEvolution::haveSameSign(const SCEV *S1, const SCEV *S2) {
11253 return ((isKnownNonNegative(S: S1) && isKnownNonNegative(S: S2)) ||
11254 (isKnownNegative(S: S1) && isKnownNegative(S: S2)));
11255}
11256
11257std::pair<const SCEV *, const SCEV *>
11258ScalarEvolution::SplitIntoInitAndPostInc(const Loop *L, const SCEV *S) {
11259 // Compute SCEV on entry of loop L.
11260 const SCEV *Start = SCEVInitRewriter::rewrite(S, L, SE&: *this);
11261 if (Start == getCouldNotCompute())
11262 return { Start, Start };
11263 // Compute post increment SCEV for loop L.
11264 const SCEV *PostInc = SCEVPostIncRewriter::rewrite(S, L, SE&: *this);
11265 assert(PostInc != getCouldNotCompute() && "Unexpected could not compute");
11266 return { Start, PostInc };
11267}
11268
11269bool ScalarEvolution::isKnownViaInduction(CmpPredicate Pred, SCEVUse LHS,
11270 SCEVUse RHS) {
11271 // First collect all loops.
11272 SmallPtrSet<const Loop *, 8> LoopsUsed;
11273 getUsedLoops(S: LHS, LoopsUsed);
11274 getUsedLoops(S: RHS, LoopsUsed);
11275
11276 if (LoopsUsed.empty())
11277 return false;
11278
11279 // Domination relationship must be a linear order on collected loops.
11280#ifndef NDEBUG
11281 for (const auto *L1 : LoopsUsed)
11282 for (const auto *L2 : LoopsUsed)
11283 assert((DT.dominates(L1->getHeader(), L2->getHeader()) ||
11284 DT.dominates(L2->getHeader(), L1->getHeader())) &&
11285 "Domination relationship is not a linear order");
11286#endif
11287
11288 const Loop *MDL =
11289 *llvm::max_element(Range&: LoopsUsed, C: [&](const Loop *L1, const Loop *L2) {
11290 return DT.properlyDominates(A: L1->getHeader(), B: L2->getHeader());
11291 });
11292
11293 // Get init and post increment value for LHS.
11294 auto SplitLHS = SplitIntoInitAndPostInc(L: MDL, S: LHS);
11295 // if LHS contains unknown non-invariant SCEV then bail out.
11296 if (SplitLHS.first == getCouldNotCompute())
11297 return false;
11298 assert (SplitLHS.second != getCouldNotCompute() && "Unexpected CNC");
11299 // Get init and post increment value for RHS.
11300 auto SplitRHS = SplitIntoInitAndPostInc(L: MDL, S: RHS);
11301 // if RHS contains unknown non-invariant SCEV then bail out.
11302 if (SplitRHS.first == getCouldNotCompute())
11303 return false;
11304 assert (SplitRHS.second != getCouldNotCompute() && "Unexpected CNC");
11305 // It is possible that init SCEV contains an invariant load but it does
11306 // not dominate MDL and is not available at MDL loop entry, so we should
11307 // check it here.
11308 if (!isAvailableAtLoopEntry(S: SplitLHS.first, L: MDL) ||
11309 !isAvailableAtLoopEntry(S: SplitRHS.first, L: MDL))
11310 return false;
11311
11312 // It seems backedge guard check is faster than entry one so in some cases
11313 // it can speed up whole estimation by short circuit
11314 return isLoopBackedgeGuardedByCond(L: MDL, Pred, LHS: SplitLHS.second,
11315 RHS: SplitRHS.second) &&
11316 isLoopEntryGuardedByCond(L: MDL, Pred, LHS: SplitLHS.first, RHS: SplitRHS.first);
11317}
11318
11319bool ScalarEvolution::isKnownPredicate(CmpPredicate Pred, SCEVUse LHS,
11320 SCEVUse RHS) {
11321 // Canonicalize the inputs first.
11322 (void)SimplifyICmpOperands(Pred, LHS, RHS);
11323
11324 return isKnownViaInduction(Pred, LHS, RHS) ||
11325 isKnownPredicateViaSplitting(Pred, LHS, RHS) ||
11326 isKnownViaNonRecursiveReasoning(Pred, LHS, RHS);
11327}
11328
11329std::optional<bool> ScalarEvolution::evaluatePredicate(CmpPredicate Pred,
11330 const SCEV *LHS,
11331 const SCEV *RHS) {
11332 if (isKnownPredicate(Pred, LHS, RHS))
11333 return true;
11334 if (isKnownPredicate(Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11335 return false;
11336 return std::nullopt;
11337}
11338
11339bool ScalarEvolution::isKnownPredicateAt(CmpPredicate Pred, const SCEV *LHS,
11340 const SCEV *RHS,
11341 const Instruction *CtxI) {
11342 // TODO: Analyze guards and assumes from Context's block.
11343 return isKnownPredicate(Pred, LHS, RHS) ||
11344 isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS);
11345}
11346
11347std::optional<bool>
11348ScalarEvolution::evaluatePredicateAt(CmpPredicate Pred, const SCEV *LHS,
11349 const SCEV *RHS, const Instruction *CtxI) {
11350 std::optional<bool> KnownWithoutContext = evaluatePredicate(Pred, LHS, RHS);
11351 if (KnownWithoutContext)
11352 return KnownWithoutContext;
11353
11354 if (isBasicBlockEntryGuardedByCond(BB: CtxI->getParent(), Pred, LHS, RHS))
11355 return true;
11356 if (isBasicBlockEntryGuardedByCond(
11357 BB: CtxI->getParent(), Pred: ICmpInst::getInverseCmpPredicate(Pred), LHS, RHS))
11358 return false;
11359 return std::nullopt;
11360}
11361
11362bool ScalarEvolution::isKnownOnEveryIteration(CmpPredicate Pred,
11363 const SCEVAddRecExpr *LHS,
11364 const SCEV *RHS) {
11365 const Loop *L = LHS->getLoop();
11366 return isLoopEntryGuardedByCond(L, Pred, LHS: LHS->getStart(), RHS) &&
11367 isLoopBackedgeGuardedByCond(L, Pred, LHS: LHS->getPostIncExpr(SE&: *this), RHS);
11368}
11369
11370std::optional<ScalarEvolution::MonotonicPredicateType>
11371ScalarEvolution::getMonotonicPredicateType(const SCEVAddRecExpr *LHS,
11372 ICmpInst::Predicate Pred) {
11373 auto Result = getMonotonicPredicateTypeImpl(LHS, Pred);
11374
11375#ifndef NDEBUG
11376 // Verify an invariant: inverting the predicate should turn a monotonically
11377 // increasing change to a monotonically decreasing one, and vice versa.
11378 if (Result) {
11379 auto ResultSwapped =
11380 getMonotonicPredicateTypeImpl(LHS, ICmpInst::getSwappedPredicate(Pred));
11381
11382 assert(*ResultSwapped != *Result &&
11383 "monotonicity should flip as we flip the predicate");
11384 }
11385#endif
11386
11387 return Result;
11388}
11389
11390std::optional<ScalarEvolution::MonotonicPredicateType>
11391ScalarEvolution::getMonotonicPredicateTypeImpl(const SCEVAddRecExpr *LHS,
11392 ICmpInst::Predicate Pred) {
11393 // A zero step value for LHS means the induction variable is essentially a
11394 // loop invariant value. We don't really depend on the predicate actually
11395 // flipping from false to true (for increasing predicates, and the other way
11396 // around for decreasing predicates), all we care about is that *if* the
11397 // predicate changes then it only changes from false to true.
11398 //
11399 // A zero step value in itself is not very useful, but there may be places
11400 // where SCEV can prove X >= 0 but not prove X > 0, so it is helpful to be
11401 // as general as possible.
11402
11403 // Only handle LE/LT/GE/GT predicates.
11404 if (!ICmpInst::isRelational(P: Pred))
11405 return std::nullopt;
11406
11407 bool IsGreater = ICmpInst::isGE(P: Pred) || ICmpInst::isGT(P: Pred);
11408 assert((IsGreater || ICmpInst::isLE(Pred) || ICmpInst::isLT(Pred)) &&
11409 "Should be greater or less!");
11410
11411 // Check that AR does not wrap.
11412 if (ICmpInst::isUnsigned(Pred)) {
11413 if (!LHS->hasNoUnsignedWrap())
11414 return std::nullopt;
11415 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11416 }
11417 assert(ICmpInst::isSigned(Pred) &&
11418 "Relational predicate is either signed or unsigned!");
11419 if (!LHS->hasNoSignedWrap())
11420 return std::nullopt;
11421
11422 const SCEV *Step = LHS->getStepRecurrence(SE&: *this);
11423
11424 if (isKnownNonNegative(S: Step))
11425 return IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11426
11427 if (isKnownNonPositive(S: Step))
11428 return !IsGreater ? MonotonicallyIncreasing : MonotonicallyDecreasing;
11429
11430 return std::nullopt;
11431}
11432
11433std::optional<ScalarEvolution::LoopInvariantPredicate>
11434ScalarEvolution::getLoopInvariantPredicate(CmpPredicate Pred, const SCEV *LHS,
11435 const SCEV *RHS, const Loop *L,
11436 const Instruction *CtxI) {
11437 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11438 if (!isLoopInvariant(S: RHS, L)) {
11439 if (!isLoopInvariant(S: LHS, L))
11440 return std::nullopt;
11441
11442 std::swap(a&: LHS, b&: RHS);
11443 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11444 }
11445
11446 const SCEVAddRecExpr *ArLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11447 if (!ArLHS || ArLHS->getLoop() != L)
11448 return std::nullopt;
11449
11450 // Even if both are valid, we need to consistently choose the unsigned or the
11451 // signed predicate for the monotonicity and backedge checks below, not
11452 // mixtures of both. For now, prefer the unsigned predicate.
11453 Pred = Pred.dropSameSign();
11454 auto MonotonicType = getMonotonicPredicateType(LHS: ArLHS, Pred);
11455 if (!MonotonicType)
11456 return std::nullopt;
11457 // If the predicate "ArLHS `Pred` RHS" monotonically increases from false to
11458 // true as the loop iterates, and the backedge is control dependent on
11459 // "ArLHS `Pred` RHS" == true then we can reason as follows:
11460 //
11461 // * if the predicate was false in the first iteration then the predicate
11462 // is never evaluated again, since the loop exits without taking the
11463 // backedge.
11464 // * if the predicate was true in the first iteration then it will
11465 // continue to be true for all future iterations since it is
11466 // monotonically increasing.
11467 //
11468 // For both the above possibilities, we can replace the loop varying
11469 // predicate with its value on the first iteration of the loop (which is
11470 // loop invariant).
11471 //
11472 // A similar reasoning applies for a monotonically decreasing predicate, by
11473 // replacing true with false and false with true in the above two bullets.
11474 bool Increasing = *MonotonicType == ScalarEvolution::MonotonicallyIncreasing;
11475 auto P = Increasing ? Pred : ICmpInst::getInverseCmpPredicate(Pred);
11476
11477 if (isLoopBackedgeGuardedByCond(L, Pred: P, LHS, RHS))
11478 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11479 RHS);
11480
11481 if (!CtxI)
11482 return std::nullopt;
11483 // Try to prove via context.
11484 // TODO: Support other cases.
11485 switch (Pred) {
11486 default:
11487 break;
11488 case ICmpInst::ICMP_ULE:
11489 case ICmpInst::ICMP_ULT: {
11490 assert(ArLHS->hasNoUnsignedWrap() && "Is a requirement of monotonicity!");
11491 // Given preconditions
11492 // (1) ArLHS does not cross the border of positive and negative parts of
11493 // range because of:
11494 // - Positive step; (TODO: lift this limitation)
11495 // - nuw - does not cross zero boundary;
11496 // - nsw - does not cross SINT_MAX boundary;
11497 // (2) ArLHS <s RHS
11498 // (3) RHS >=s 0
11499 // we can replace the loop variant ArLHS <u RHS condition with loop
11500 // invariant Start(ArLHS) <u RHS.
11501 //
11502 // Because of (1) there are two options:
11503 // - ArLHS is always negative. It means that ArLHS <u RHS is always false;
11504 // - ArLHS is always non-negative. Because of (3) RHS is also non-negative.
11505 // It means that ArLHS <s RHS <=> ArLHS <u RHS.
11506 // Because of (2) ArLHS <u RHS is trivially true.
11507 // All together it means that ArLHS <u RHS <=> Start(ArLHS) >=s 0.
11508 // We can strengthen this to Start(ArLHS) <u RHS.
11509 auto SignFlippedPred = ICmpInst::getFlippedSignednessPredicate(Pred);
11510 if (ArLHS->hasNoSignedWrap() && ArLHS->isAffine() &&
11511 isKnownPositive(S: ArLHS->getStepRecurrence(SE&: *this)) &&
11512 isKnownNonNegative(S: RHS) &&
11513 isKnownPredicateAt(Pred: SignFlippedPred, LHS: ArLHS, RHS, CtxI))
11514 return ScalarEvolution::LoopInvariantPredicate(Pred, ArLHS->getStart(),
11515 RHS);
11516 }
11517 }
11518
11519 return std::nullopt;
11520}
11521
11522std::optional<ScalarEvolution::LoopInvariantPredicate>
11523ScalarEvolution::getLoopInvariantExitCondDuringFirstIterations(
11524 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11525 const Instruction *CtxI, const SCEV *MaxIter) {
11526 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11527 Pred, LHS, RHS, L, CtxI, MaxIter))
11528 return LIP;
11529 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: MaxIter))
11530 // Number of iterations expressed as UMIN isn't always great for expressing
11531 // the value on the last iteration. If the straightforward approach didn't
11532 // work, try the following trick: if the a predicate is invariant for X, it
11533 // is also invariant for umin(X, ...). So try to find something that works
11534 // among subexpressions of MaxIter expressed as umin.
11535 for (SCEVUse Op : UMin->operands())
11536 if (auto LIP = getLoopInvariantExitCondDuringFirstIterationsImpl(
11537 Pred, LHS, RHS, L, CtxI, MaxIter: Op))
11538 return LIP;
11539 return std::nullopt;
11540}
11541
11542std::optional<ScalarEvolution::LoopInvariantPredicate>
11543ScalarEvolution::getLoopInvariantExitCondDuringFirstIterationsImpl(
11544 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const Loop *L,
11545 const Instruction *CtxI, const SCEV *MaxIter) {
11546 // Try to prove the following set of facts:
11547 // - The predicate is monotonic in the iteration space.
11548 // - If the check does not fail on the 1st iteration:
11549 // - No overflow will happen during first MaxIter iterations;
11550 // - It will not fail on the MaxIter'th iteration.
11551 // If the check does fail on the 1st iteration, we leave the loop and no
11552 // other checks matter.
11553
11554 // If there is a loop-invariant, force it into the RHS, otherwise bail out.
11555 if (!isLoopInvariant(S: RHS, L)) {
11556 if (!isLoopInvariant(S: LHS, L))
11557 return std::nullopt;
11558
11559 std::swap(a&: LHS, b&: RHS);
11560 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
11561 }
11562
11563 auto *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS);
11564 if (!AR || AR->getLoop() != L)
11565 return std::nullopt;
11566
11567 // Even if both are valid, we need to consistently chose the unsigned or the
11568 // signed predicate below, not mixtures of both. For now, prefer the unsigned
11569 // predicate.
11570 Pred = Pred.dropSameSign();
11571
11572 // The predicate must be relational (i.e. <, <=, >=, >).
11573 if (!ICmpInst::isRelational(P: Pred))
11574 return std::nullopt;
11575
11576 // TODO: Support steps other than +/- 1.
11577 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
11578 auto *One = getOne(Ty: Step->getType());
11579 auto *MinusOne = getNegativeSCEV(V: One);
11580 if (Step != One && Step != MinusOne)
11581 return std::nullopt;
11582
11583 // Type mismatch here means that MaxIter is potentially larger than max
11584 // unsigned value in start type, which mean we cannot prove no wrap for the
11585 // indvar.
11586 if (AR->getType() != MaxIter->getType())
11587 return std::nullopt;
11588
11589 // Value of IV on suggested last iteration.
11590 const SCEV *Last = AR->evaluateAtIteration(It: MaxIter, SE&: *this);
11591 // Does it still meet the requirement?
11592 if (!isLoopBackedgeGuardedByCond(L, Pred, LHS: Last, RHS))
11593 return std::nullopt;
11594 // Because step is +/- 1 and MaxIter has same type as Start (i.e. it does
11595 // not exceed max unsigned value of this type), this effectively proves
11596 // that there is no wrap during the iteration. To prove that there is no
11597 // signed/unsigned wrap, we need to check that
11598 // Start <= Last for step = 1 or Start >= Last for step = -1.
11599 ICmpInst::Predicate NoOverflowPred =
11600 CmpInst::isSigned(Pred) ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE;
11601 if (Step == MinusOne)
11602 NoOverflowPred = ICmpInst::getSwappedPredicate(pred: NoOverflowPred);
11603 const SCEV *Start = AR->getStart();
11604 if (!isKnownPredicateAt(Pred: NoOverflowPred, LHS: Start, RHS: Last, CtxI))
11605 return std::nullopt;
11606
11607 // Everything is fine.
11608 return ScalarEvolution::LoopInvariantPredicate(Pred, Start, RHS);
11609}
11610
11611bool ScalarEvolution::isKnownPredicateViaConstantRanges(CmpPredicate Pred,
11612 SCEVUse LHS,
11613 SCEVUse RHS) {
11614 if (HasSameValue(A: LHS, B: RHS))
11615 return ICmpInst::isTrueWhenEqual(predicate: Pred);
11616
11617 auto CheckRange = [&](bool IsSigned) {
11618 auto RangeLHS = IsSigned ? getSignedRange(S: LHS) : getUnsignedRange(S: LHS);
11619 auto RangeRHS = IsSigned ? getSignedRange(S: RHS) : getUnsignedRange(S: RHS);
11620 return RangeLHS.icmp(Pred, Other: RangeRHS);
11621 };
11622
11623 // The check at the top of the function catches the case where the values are
11624 // known to be equal.
11625 if (Pred == CmpInst::ICMP_EQ)
11626 return false;
11627
11628 if (Pred == CmpInst::ICMP_NE) {
11629 if (CheckRange(true) || CheckRange(false))
11630 return true;
11631 auto *Diff = getMinusSCEV(LHS, RHS);
11632 return !isa<SCEVCouldNotCompute>(Val: Diff) && isKnownNonZero(S: Diff);
11633 }
11634
11635 return CheckRange(CmpInst::isSigned(Pred));
11636}
11637
11638bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred,
11639 SCEVUse LHS, SCEVUse RHS) {
11640 // Match X to (A + C1)<ExpectedFlags> and Y to (A + C2)<ExpectedFlags>, where
11641 // C1 and C2 are constant integers. If either X or Y are not add expressions,
11642 // consider them as X + 0 and Y + 0 respectively. C1 and C2 are returned via
11643 // OutC1 and OutC2.
11644 auto MatchBinaryAddToConst = [this](SCEVUse X, SCEVUse Y, APInt &OutC1,
11645 APInt &OutC2, SCEVFlags ExpectedFlags) {
11646 SCEVUse XNonConstOp, XConstOp;
11647 SCEVUse YNonConstOp, YConstOp;
11648 SCEVFlags XFlagsPresent;
11649 SCEVFlags YFlagsPresent;
11650
11651 // Use nullptr for 0, to avoid creating a zero SCEV.
11652 if (!splitBinaryAdd(Expr: X, L&: XConstOp, R&: XNonConstOp, Flags&: XFlagsPresent)) {
11653 XConstOp = nullptr;
11654 XNonConstOp = X;
11655 XFlagsPresent = ExpectedFlags;
11656 }
11657 if (XConstOp && !isa<SCEVConstant>(Val: XConstOp))
11658 return false;
11659
11660 if (!splitBinaryAdd(Expr: Y, L&: YConstOp, R&: YNonConstOp, Flags&: YFlagsPresent)) {
11661 YConstOp = nullptr;
11662 YNonConstOp = Y;
11663 YFlagsPresent = ExpectedFlags;
11664 }
11665
11666 if (YNonConstOp != XNonConstOp)
11667 return false;
11668
11669 if (YConstOp && !isa<SCEVConstant>(Val: YConstOp))
11670 return false;
11671
11672 // When matching ADDs with NUW flags (and unsigned predicates), only the
11673 // second ADD (with the larger constant) requires NUW.
11674 if ((YFlagsPresent & ExpectedFlags) != ExpectedFlags)
11675 return false;
11676 if (ExpectedFlags != SCEV::FlagNUW &&
11677 (XFlagsPresent & ExpectedFlags) != ExpectedFlags) {
11678 return false;
11679 }
11680
11681 OutC1 = XConstOp ? cast<SCEVConstant>(Val&: XConstOp)->getAPInt()
11682 : APInt::getZero(numBits: getTypeSizeInBits(Ty: X->getType()));
11683 OutC2 = YConstOp ? cast<SCEVConstant>(Val&: YConstOp)->getAPInt()
11684 : APInt::getZero(numBits: getTypeSizeInBits(Ty: Y->getType()));
11685
11686 return true;
11687 };
11688
11689 APInt C1;
11690 APInt C2;
11691
11692 switch (Pred) {
11693 default:
11694 break;
11695
11696 case ICmpInst::ICMP_SGE:
11697 std::swap(a&: LHS, b&: RHS);
11698 [[fallthrough]];
11699 case ICmpInst::ICMP_SLE:
11700 // (X + C1)<nsw> s<= (X + C2)<nsw> if C1 s<= C2.
11701 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.sle(RHS: C2))
11702 return true;
11703
11704 break;
11705
11706 case ICmpInst::ICMP_SGT:
11707 std::swap(a&: LHS, b&: RHS);
11708 [[fallthrough]];
11709 case ICmpInst::ICMP_SLT:
11710 // (X + C1)<nsw> s< (X + C2)<nsw> if C1 s< C2.
11711 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNSW) && C1.slt(RHS: C2))
11712 return true;
11713
11714 break;
11715
11716 case ICmpInst::ICMP_UGE:
11717 std::swap(a&: LHS, b&: RHS);
11718 [[fallthrough]];
11719 case ICmpInst::ICMP_ULE:
11720 // (X + C1) u<= (X + C2)<nuw> for C1 u<= C2.
11721 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ule(RHS: C2))
11722 return true;
11723
11724 break;
11725
11726 case ICmpInst::ICMP_UGT:
11727 std::swap(a&: LHS, b&: RHS);
11728 [[fallthrough]];
11729 case ICmpInst::ICMP_ULT:
11730 // (X + C1) u< (X + C2)<nuw> if C1 u< C2.
11731 if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ult(RHS: C2))
11732 return true;
11733 break;
11734 }
11735
11736 return false;
11737}
11738
11739bool ScalarEvolution::isKnownPredicateViaSplitting(CmpPredicate Pred,
11740 SCEVUse LHS, SCEVUse RHS) {
11741 if (Pred != ICmpInst::ICMP_ULT || ProvingSplitPredicate)
11742 return false;
11743
11744 // Allowing arbitrary number of activations of isKnownPredicateViaSplitting on
11745 // the stack can result in exponential time complexity.
11746 SaveAndRestore Restore(ProvingSplitPredicate, true);
11747
11748 // If L >= 0 then I `ult` L <=> I >= 0 && I `slt` L
11749 //
11750 // To prove L >= 0 we use isKnownNonNegative whereas to prove I >= 0 we use
11751 // isKnownPredicate. isKnownPredicate is more powerful, but also more
11752 // expensive; and using isKnownNonNegative(RHS) is sufficient for most of the
11753 // interesting cases seen in practice. We can consider "upgrading" L >= 0 to
11754 // use isKnownPredicate later if needed.
11755 return isKnownNonNegative(S: RHS) &&
11756 isKnownPredicate(Pred: CmpInst::ICMP_SGE, LHS, RHS: getZero(Ty: LHS->getType())) &&
11757 isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS, RHS);
11758}
11759
11760bool ScalarEvolution::isImpliedViaGuard(const BasicBlock *BB, CmpPredicate Pred,
11761 const SCEV *LHS, const SCEV *RHS) {
11762 // No need to even try if we know the module has no guards.
11763 if (!HasGuards)
11764 return false;
11765
11766 return any_of(Range: *BB, P: [&](const Instruction &I) {
11767 using namespace llvm::PatternMatch;
11768
11769 Value *Condition;
11770 return match(V: &I, P: m_Intrinsic<Intrinsic::experimental_guard>(
11771 Ops: m_Value(V&: Condition))) &&
11772 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse: false);
11773 });
11774}
11775
11776/// isLoopBackedgeGuardedByCond - Test whether the backedge of the loop is
11777/// protected by a conditional between LHS and RHS. This is used to
11778/// to eliminate casts.
11779bool ScalarEvolution::isLoopBackedgeGuardedByCond(const Loop *L,
11780 CmpPredicate Pred,
11781 const SCEV *LHS,
11782 const SCEV *RHS) {
11783 // Interpret a null as meaning no loop, where there is obviously no guard
11784 // (interprocedural conditions notwithstanding). Do not bother about
11785 // unreachable loops.
11786 if (!L || !DT.isReachableFromEntry(A: L->getHeader()))
11787 return true;
11788
11789 if (Opts.scev_verify_ir)
11790 assert(!verifyFunction(*L->getHeader()->getParent(), &dbgs()) &&
11791 "This cannot be done on broken IR!");
11792
11793
11794 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11795 return true;
11796
11797 BasicBlock *Latch = L->getLoopLatch();
11798 if (!Latch)
11799 return false;
11800
11801 CondBrInst *LoopContinuePredicate =
11802 dyn_cast<CondBrInst>(Val: Latch->getTerminator());
11803 if (LoopContinuePredicate &&
11804 isImpliedCond(Pred, LHS, RHS, FoundCondValue: LoopContinuePredicate->getCondition(),
11805 Inverse: LoopContinuePredicate->getSuccessor(i: 0) != L->getHeader()))
11806 return true;
11807
11808 // We don't want more than one activation of the following loops on the stack
11809 // -- that can lead to O(n!) time complexity.
11810 if (WalkingBEDominatingConds)
11811 return false;
11812
11813 SaveAndRestore ClearOnExit(WalkingBEDominatingConds, true);
11814
11815 // See if we can exploit a trip count to prove the predicate.
11816 const auto &BETakenInfo = getBackedgeTakenInfo(L);
11817 const SCEV *LatchBECount = BETakenInfo.getExact(ExitingBlock: Latch, SE: this);
11818 if (LatchBECount != getCouldNotCompute()) {
11819 // We know that Latch branches back to the loop header exactly
11820 // LatchBECount times. This means the backdege condition at Latch is
11821 // equivalent to "{0,+,1} u< LatchBECount".
11822 Type *Ty = LatchBECount->getType();
11823 auto NoWrapFlags = SCEVFlags(SCEV::FlagNUW | SCEV::FlagNW);
11824 const SCEV *LoopCounter =
11825 getAddRecExpr(Start: getZero(Ty), Step: getOne(Ty), L, Flags: NoWrapFlags);
11826 if (isImpliedCond(Pred, LHS, RHS, FoundPred: ICmpInst::ICMP_ULT, FoundLHS: LoopCounter,
11827 FoundRHS: LatchBECount))
11828 return true;
11829 }
11830
11831 // Check conditions due to any @llvm.assume intrinsics.
11832 for (auto &AssumeVH : AC.assumptions()) {
11833 if (!AssumeVH)
11834 continue;
11835 auto *CI = cast<CallInst>(Val&: AssumeVH);
11836 if (!DT.dominates(Def: CI, User: Latch->getTerminator()))
11837 continue;
11838
11839 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: CI->getArgOperand(i: 0), Inverse: false))
11840 return true;
11841 }
11842
11843 if (isImpliedViaGuard(BB: Latch, Pred, LHS, RHS))
11844 return true;
11845
11846 for (DomTreeNode *DTN = DT[Latch], *HeaderDTN = DT[L->getHeader()];
11847 DTN != HeaderDTN; DTN = DTN->getIDom()) {
11848 assert(DTN && "should reach the loop header before reaching the root!");
11849
11850 BasicBlock *BB = DTN->getBlock();
11851 if (isImpliedViaGuard(BB, Pred, LHS, RHS))
11852 return true;
11853
11854 BasicBlock *PBB = BB->getSinglePredecessor();
11855 if (!PBB)
11856 continue;
11857
11858 CondBrInst *ContBr = dyn_cast<CondBrInst>(Val: PBB->getTerminator());
11859 if (!ContBr || ContBr->getSuccessor(i: 0) == ContBr->getSuccessor(i: 1))
11860 continue;
11861
11862 // If we have an edge `E` within the loop body that dominates the only
11863 // latch, the condition guarding `E` also guards the backedge. This
11864 // reasoning works only for loops with a single latch.
11865 // We're constructively (and conservatively) enumerating edges within the
11866 // loop body that dominate the latch. The dominator tree better agree
11867 // with us on this:
11868 assert(DT.dominates(BasicBlockEdge(PBB, BB), Latch) && "should be!");
11869 if (isImpliedCond(Pred, LHS, RHS, FoundCondValue: ContBr->getCondition(),
11870 Inverse: BB != ContBr->getSuccessor(i: 0)))
11871 return true;
11872 }
11873
11874 return false;
11875}
11876
11877bool ScalarEvolution::isBasicBlockEntryGuardedByCond(const BasicBlock *BB,
11878 CmpPredicate Pred,
11879 const SCEV *LHS,
11880 const SCEV *RHS) {
11881 // Do not bother proving facts for unreachable code.
11882 if (!DT.isReachableFromEntry(A: BB))
11883 return true;
11884 if (Opts.scev_verify_ir)
11885 assert(!verifyFunction(*BB->getParent(), &dbgs()) &&
11886 "This cannot be done on broken IR!");
11887
11888 // If we cannot prove strict comparison (e.g. a > b), maybe we can prove
11889 // the facts (a >= b && a != b) separately. A typical situation is when the
11890 // non-strict comparison is known from ranges and non-equality is known from
11891 // dominating predicates. If we are proving strict comparison, we always try
11892 // to prove non-equality and non-strict comparison separately.
11893 CmpPredicate NonStrictPredicate = ICmpInst::getNonStrictCmpPredicate(Pred);
11894 const bool ProvingStrictComparison =
11895 Pred != NonStrictPredicate.dropSameSign();
11896 const Instruction *CtxI = &BB->front();
11897 SmallVector<std::pair<const Value *, bool>> Conds;
11898
11899 // Try to prove (Pred, LHS, RHS) using isImpliedCond.
11900 auto ProveViaCond = [&](const Value *Condition, bool Inverse) {
11901 Conds.emplace_back(Args&: Condition, Args&: Inverse);
11902 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Condition, Inverse, Context: CtxI);
11903 };
11904
11905 // Starting at the block's predecessor, climb up the predecessor chain, as long
11906 // as there are predecessors that can be found that have unique successors
11907 // leading to the original block.
11908 const Loop *ContainingLoop = LI.getLoopFor(BB);
11909 const BasicBlock *PredBB;
11910 if (ContainingLoop && ContainingLoop->getHeader() == BB)
11911 PredBB = ContainingLoop->getLoopPredecessor();
11912 else
11913 PredBB = BB->getSinglePredecessor();
11914 for (std::pair<const BasicBlock *, const BasicBlock *> Pair(PredBB, BB);
11915 Pair.first; Pair = getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
11916 const CondBrInst *BlockEntryPredicate =
11917 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
11918 if (!BlockEntryPredicate)
11919 continue;
11920
11921 if (ProveViaCond(BlockEntryPredicate->getCondition(),
11922 BlockEntryPredicate->getSuccessor(i: 0) != Pair.second))
11923 return true;
11924 }
11925
11926 // Check conditions due to any @llvm.assume intrinsics.
11927 for (auto &AssumeVH : AC.assumptions()) {
11928 if (!AssumeVH)
11929 continue;
11930 auto *CI = cast<CallInst>(Val&: AssumeVH);
11931 if (!DT.dominates(Def: CI, BB))
11932 continue;
11933
11934 if (ProveViaCond(CI->getArgOperand(i: 0), false))
11935 return true;
11936 }
11937
11938 // Check conditions due to any @llvm.experimental.guard intrinsics.
11939 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
11940 M: F.getParent(), id: Intrinsic::experimental_guard);
11941 if (GuardDecl)
11942 for (const auto *GU : GuardDecl->users())
11943 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
11944 if (Guard->getFunction() == BB->getParent() && DT.dominates(Def: Guard, BB))
11945 if (ProveViaCond(Guard->getArgOperand(i: 0), false))
11946 return true;
11947
11948 // First try to prove conditions directly, then fall back to proving via the
11949 // non-strict predicate and not-equal.
11950 if (!ProvingStrictComparison)
11951 return false;
11952 auto ProveViaAll = [&](CmpPredicate P) {
11953 return isKnownViaNonRecursiveReasoning(Pred: P, LHS, RHS) ||
11954 any_of(Range&: Conds, P: [&](const auto &C) {
11955 return isImpliedCond(P, LHS, RHS, C.first, C.second, CtxI);
11956 });
11957 };
11958 return ProveViaAll(NonStrictPredicate) && ProveViaAll(ICmpInst::ICMP_NE);
11959}
11960
11961bool ScalarEvolution::isLoopEntryGuardedByCond(const Loop *L, CmpPredicate Pred,
11962 const SCEV *LHS,
11963 const SCEV *RHS) {
11964 // Interpret a null as meaning no loop, where there is obviously no guard
11965 // (interprocedural conditions notwithstanding).
11966 if (!L)
11967 return false;
11968
11969 // Both LHS and RHS must be available at loop entry.
11970 assert(isAvailableAtLoopEntry(LHS, L) &&
11971 "LHS is not available at Loop Entry");
11972 assert(isAvailableAtLoopEntry(RHS, L) &&
11973 "RHS is not available at Loop Entry");
11974
11975 if (isKnownViaNonRecursiveReasoning(Pred, LHS, RHS))
11976 return true;
11977
11978 return isBasicBlockEntryGuardedByCond(BB: L->getHeader(), Pred, LHS, RHS);
11979}
11980
11981bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
11982 const SCEV *RHS,
11983 const Value *FoundCondValue, bool Inverse,
11984 const Instruction *CtxI) {
11985 // False conditions implies anything. Do not bother analyzing it further.
11986 if (FoundCondValue ==
11987 ConstantInt::getBool(Context&: FoundCondValue->getContext(), V: Inverse))
11988 return true;
11989
11990 if (!PendingLoopPredicates.insert(Ptr: FoundCondValue).second)
11991 return false;
11992
11993 llvm::scope_exit ClearOnExit(
11994 [&]() { PendingLoopPredicates.erase(Ptr: FoundCondValue); });
11995
11996 // Recursively handle And and Or conditions.
11997 const Value *Op0, *Op1;
11998 if (match(V: FoundCondValue, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
11999 if (!Inverse)
12000 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12001 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12002 } else if (match(V: FoundCondValue, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1)))) {
12003 if (Inverse)
12004 return isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op0, Inverse, CtxI) ||
12005 isImpliedCond(Pred, LHS, RHS, FoundCondValue: Op1, Inverse, CtxI);
12006 }
12007
12008 const ICmpInst *ICI = dyn_cast<ICmpInst>(Val: FoundCondValue);
12009 if (!ICI) return false;
12010
12011 // Now that we found a conditional branch that dominates the loop or controls
12012 // the loop latch. Check to see if it is the comparison we are looking for.
12013 CmpPredicate FoundPred;
12014 if (Inverse)
12015 FoundPred = ICI->getInverseCmpPredicate();
12016 else
12017 FoundPred = ICI->getCmpPredicate();
12018
12019 const SCEV *FoundLHS = getSCEV(V: ICI->getOperand(i_nocapture: 0));
12020 const SCEV *FoundRHS = getSCEV(V: ICI->getOperand(i_nocapture: 1));
12021
12022 return isImpliedCond(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS, Context: CtxI);
12023}
12024
12025bool ScalarEvolution::isImpliedCond(CmpPredicate Pred, const SCEV *LHS,
12026 const SCEV *RHS, CmpPredicate FoundPred,
12027 const SCEV *FoundLHS, const SCEV *FoundRHS,
12028 const Instruction *CtxI) {
12029 // Balance the types.
12030 if (getTypeSizeInBits(Ty: LHS->getType()) <
12031 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12032 // For unsigned and equality predicates, try to prove that both found
12033 // operands fit into narrow unsigned range. If so, try to prove facts in
12034 // narrow types.
12035 if (!CmpInst::isSigned(Pred: FoundPred) && !FoundLHS->getType()->isPointerTy() &&
12036 !FoundRHS->getType()->isPointerTy()) {
12037 auto *NarrowType = LHS->getType();
12038 auto *WideType = FoundLHS->getType();
12039 auto BitWidth = getTypeSizeInBits(Ty: NarrowType);
12040 const SCEV *MaxValue = getZeroExtendExpr(
12041 Op: getConstant(Val: APInt::getMaxValue(numBits: BitWidth)), Ty: WideType);
12042 if (isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundLHS,
12043 RHS: MaxValue) &&
12044 isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_ULE, LHS: FoundRHS,
12045 RHS: MaxValue)) {
12046 const SCEV *TruncFoundLHS = getTruncateExpr(Op: FoundLHS, Ty: NarrowType);
12047 const SCEV *TruncFoundRHS = getTruncateExpr(Op: FoundRHS, Ty: NarrowType);
12048 // We cannot preserve samesign after truncation.
12049 if (isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred: FoundPred.dropSameSign(),
12050 FoundLHS: TruncFoundLHS, FoundRHS: TruncFoundRHS, CtxI))
12051 return true;
12052 }
12053 }
12054
12055 if (LHS->getType()->isPointerTy() || RHS->getType()->isPointerTy())
12056 return false;
12057 if (CmpInst::isSigned(Pred)) {
12058 LHS = getSignExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12059 RHS = getSignExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12060 } else {
12061 LHS = getZeroExtendExpr(Op: LHS, Ty: FoundLHS->getType());
12062 RHS = getZeroExtendExpr(Op: RHS, Ty: FoundLHS->getType());
12063 }
12064 } else if (getTypeSizeInBits(Ty: LHS->getType()) >
12065 getTypeSizeInBits(Ty: FoundLHS->getType())) {
12066 if (FoundLHS->getType()->isPointerTy() || FoundRHS->getType()->isPointerTy())
12067 return false;
12068 if (CmpInst::isSigned(Pred: FoundPred)) {
12069 FoundLHS = getSignExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12070 FoundRHS = getSignExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12071 } else {
12072 FoundLHS = getZeroExtendExpr(Op: FoundLHS, Ty: LHS->getType());
12073 FoundRHS = getZeroExtendExpr(Op: FoundRHS, Ty: LHS->getType());
12074 }
12075 }
12076 return isImpliedCondBalancedTypes(Pred, LHS, RHS, FoundPred, FoundLHS,
12077 FoundRHS, CtxI);
12078}
12079
12080bool ScalarEvolution::isImpliedCondBalancedTypes(
12081 CmpPredicate Pred, SCEVUse LHS, SCEVUse RHS, CmpPredicate FoundPred,
12082 SCEVUse FoundLHS, SCEVUse FoundRHS, const Instruction *CtxI) {
12083 assert(getTypeSizeInBits(LHS->getType()) ==
12084 getTypeSizeInBits(FoundLHS->getType()) &&
12085 "Types should be balanced!");
12086 // Canonicalize the query to match the way instcombine will have
12087 // canonicalized the comparison.
12088 if (SimplifyICmpOperands(Pred, LHS, RHS))
12089 if (LHS == RHS)
12090 return CmpInst::isTrueWhenEqual(predicate: Pred);
12091 if (SimplifyICmpOperands(Pred&: FoundPred, LHS&: FoundLHS, RHS&: FoundRHS))
12092 if (FoundLHS == FoundRHS)
12093 return CmpInst::isFalseWhenEqual(predicate: FoundPred);
12094
12095 // Check to see if we can make the LHS or RHS match.
12096 if (LHS == FoundRHS || RHS == FoundLHS) {
12097 if (isa<SCEVConstant>(Val: RHS)) {
12098 std::swap(a&: FoundLHS, b&: FoundRHS);
12099 FoundPred = ICmpInst::getSwappedCmpPredicate(Pred: FoundPred);
12100 } else {
12101 std::swap(a&: LHS, b&: RHS);
12102 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12103 }
12104 }
12105
12106 // Check whether the found predicate is the same as the desired predicate.
12107 if (auto P = CmpPredicate::getMatching(A: FoundPred, B: Pred))
12108 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12109
12110 // Check whether swapping the found predicate makes it the same as the
12111 // desired predicate.
12112 if (auto P = CmpPredicate::getMatching(
12113 A: ICmpInst::getSwappedCmpPredicate(Pred: FoundPred), B: Pred)) {
12114 // We can write the implication
12115 // 0. LHS Pred RHS <- FoundLHS SwapPred FoundRHS
12116 // using one of the following ways:
12117 // 1. LHS Pred RHS <- FoundRHS Pred FoundLHS
12118 // 2. RHS SwapPred LHS <- FoundLHS SwapPred FoundRHS
12119 // Both require swapping the operands of one condition. Don't do this if it
12120 // would break canonical constant/addrec ordering.
12121 if (!isa<SCEVConstant>(Val: RHS) && !isa<SCEVAddRecExpr>(Val: LHS))
12122 return isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred: *P), LHS: RHS,
12123 RHS: LHS, FoundLHS, FoundRHS, Context: CtxI);
12124 if (!isa<SCEVConstant>(Val: FoundRHS) && !isa<SCEVAddRecExpr>(Val: FoundLHS))
12125 return isImpliedCondOperands(Pred: *P, LHS, RHS, FoundLHS: FoundRHS, FoundRHS: FoundLHS, Context: CtxI);
12126
12127 return false;
12128 }
12129
12130 // Try the query as is and with its operands swapped. E.g. "x >s -1" is
12131 // tried as "-1 <s x" too, to match the found condition "x <u y".
12132 auto IsSignFlippedPredicate = [](CmpInst::Predicate P1,
12133 CmpInst::Predicate P2) {
12134 return CmpInst::isRelational(P: P2) &&
12135 P1 == ICmpInst::getFlippedSignednessPredicate(Pred: P2);
12136 };
12137 if (IsSignFlippedPredicate(Pred, FoundPred) &&
12138 isImpliedCondViaSignFlippedPred(Pred, LHS, RHS, FoundLHS, FoundRHS, CtxI))
12139 return true;
12140 CmpPredicate SwappedPred = ICmpInst::getSwappedCmpPredicate(Pred);
12141 if (IsSignFlippedPredicate(SwappedPred, FoundPred) &&
12142 isImpliedCondViaSignFlippedPred(Pred: SwappedPred, LHS: RHS, RHS: LHS, FoundLHS, FoundRHS,
12143 CtxI))
12144 return true;
12145
12146 // Check if we can make progress by sharpening ranges.
12147 if (FoundPred == ICmpInst::ICMP_NE &&
12148 (isa<SCEVConstant>(Val: FoundLHS) || isa<SCEVConstant>(Val: FoundRHS))) {
12149
12150 const SCEVConstant *C = nullptr;
12151 const SCEV *V = nullptr;
12152
12153 if (isa<SCEVConstant>(Val: FoundLHS)) {
12154 C = cast<SCEVConstant>(Val&: FoundLHS);
12155 V = FoundRHS;
12156 } else {
12157 C = cast<SCEVConstant>(Val&: FoundRHS);
12158 V = FoundLHS;
12159 }
12160
12161 // The guarding predicate tells us that C != V. If the known range
12162 // of V is [C, t), we can sharpen the range to [C + 1, t). The
12163 // range we consider has to correspond to same signedness as the
12164 // predicate we're interested in folding.
12165
12166 APInt Min = ICmpInst::isSigned(Pred) ?
12167 getSignedRangeMin(S: V) : getUnsignedRangeMin(S: V);
12168
12169 if (Min == C->getAPInt()) {
12170 // Given (V >= Min && V != Min) we conclude V >= (Min + 1).
12171 // This is true even if (Min + 1) wraps around -- in case of
12172 // wraparound, (Min + 1) < Min, so (V >= Min => V >= (Min + 1)).
12173
12174 APInt SharperMin = Min + 1;
12175
12176 switch (Pred) {
12177 case ICmpInst::ICMP_SGE:
12178 case ICmpInst::ICMP_UGE:
12179 // We know V `Pred` SharperMin. If this implies LHS `Pred`
12180 // RHS, we're done.
12181 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin),
12182 Context: CtxI))
12183 return true;
12184 [[fallthrough]];
12185
12186 case ICmpInst::ICMP_SGT:
12187 case ICmpInst::ICMP_UGT:
12188 // We know from the range information that (V `Pred` Min ||
12189 // V == Min). We know from the guarding condition that !(V
12190 // == Min). This gives us
12191 //
12192 // V `Pred` Min || V == Min && !(V == Min)
12193 // => V `Pred` Min
12194 //
12195 // If V `Pred` Min implies LHS `Pred` RHS, we're done.
12196
12197 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12198 return true;
12199 break;
12200
12201 // `LHS < RHS` and `LHS <= RHS` are handled in the same way as `RHS > LHS` and `RHS >= LHS` respectively.
12202 case ICmpInst::ICMP_SLE:
12203 case ICmpInst::ICMP_ULE:
12204 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12205 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: SharperMin), Context: CtxI))
12206 return true;
12207 [[fallthrough]];
12208
12209 case ICmpInst::ICMP_SLT:
12210 case ICmpInst::ICMP_ULT:
12211 if (isImpliedCondOperands(Pred: ICmpInst::getSwappedCmpPredicate(Pred), LHS: RHS,
12212 RHS: LHS, FoundLHS: V, FoundRHS: getConstant(Val: Min), Context: CtxI))
12213 return true;
12214 break;
12215
12216 default:
12217 // No change
12218 break;
12219 }
12220 }
12221 }
12222
12223 // Check whether the actual condition is beyond sufficient.
12224 if (FoundPred == ICmpInst::ICMP_EQ)
12225 if (ICmpInst::isTrueWhenEqual(predicate: Pred))
12226 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12227 return true;
12228 if (Pred == ICmpInst::ICMP_NE)
12229 if (!ICmpInst::isTrueWhenEqual(predicate: FoundPred))
12230 if (isImpliedCondOperands(Pred: FoundPred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI))
12231 return true;
12232
12233 if (isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred, FoundLHS, FoundRHS))
12234 return true;
12235
12236 // Otherwise assume the worst.
12237 return false;
12238}
12239
12240bool ScalarEvolution::isImpliedCondViaSignFlippedPred(
12241 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12242 const SCEV *FoundRHS, const Instruction *CtxI) {
12243 CmpPredicate FoundPred = ICmpInst::getFlippedSignednessPredicate(Pred);
12244 // Unsigned comparison is the same as signed comparison when both the
12245 // operands are non-negative or negative.
12246 if (haveSameSign(S1: FoundLHS, S2: FoundRHS))
12247 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS, Context: CtxI);
12248 // Canonicalize both conditions to "le/lt".
12249 if (ICmpInst::isGT(P: Pred) || ICmpInst::isGE(P: Pred)) {
12250 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12251 FoundPred = ICmpInst::getSwappedCmpPredicate(Pred: FoundPred);
12252 std::swap(a&: LHS, b&: RHS);
12253 std::swap(a&: FoundLHS, b&: FoundRHS);
12254 }
12255 assert((ICmpInst::isLT(Pred) || ICmpInst::isLE(Pred)) && "Must be!");
12256 assert((ICmpInst::isLT(FoundPred) || ICmpInst::isLE(FoundPred)) &&
12257 "Must be!");
12258 if (ICmpInst::isSigned(Pred)) {
12259 // Use implication:
12260 // x <u y && y <=u 2^(n-1) --> x >=s 0,
12261 // x <=u y && y <u 2^(n-1) --> x >=s 0,
12262 // where n is the bit width of x and y.
12263 // In both cases x <u 2^(n-1), so x >=s 0.
12264 // y <=u 2^(n-1) means y >=s 0 or y == INT_MIN.
12265 // If the implication's left part holds, try proving the query from x >=s
12266 // 0.
12267 Type *Ty = FoundLHS->getType();
12268 unsigned BitWidth = getTypeSizeInBits(Ty);
12269 const SCEV *SignedMin = getConstant(Val: APInt::getSignedMinValue(numBits: BitWidth));
12270 CmpPredicate BoundPred =
12271 ICmpInst::isLT(P: FoundPred) ? ICmpInst::ICMP_ULE : ICmpInst::ICMP_ULT;
12272 if (isKnownViaNonRecursiveReasoning(Pred: BoundPred, LHS: FoundRHS, RHS: SignedMin)) {
12273 // x >=s 0, written as 0 <=s x, or as -1 <s x to match a strict query.
12274 const SCEV *Bound = ICmpInst::isLE(P: Pred) ? getZero(Ty) : getMinusOne(Ty);
12275 if (isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: Bound, FoundRHS: FoundLHS, Context: CtxI))
12276 return true;
12277 }
12278 }
12279 if (ICmpInst::isSigned(Pred) && isKnownNonNegative(S: RHS)) {
12280 // Use implication:
12281 // x <u y && y >=s 0 --> x <s y.
12282 // If we can prove the left part, the right part is also proven.
12283 return isImpliedCondOperands(Pred: FoundPred, LHS, RHS, FoundLHS, FoundRHS);
12284 }
12285 if (ICmpInst::isUnsigned(Pred) && isKnownNegative(S: RHS)) {
12286 // Use implication:
12287 // x <s y && y <s 0 --> x <u y.
12288 // If we can prove the left part, the right part is also proven.
12289 return isImpliedCondOperands(Pred: FoundPred, LHS, RHS, FoundLHS, FoundRHS);
12290 }
12291 return false;
12292}
12293
12294bool ScalarEvolution::splitBinaryAdd(SCEVUse Expr, SCEVUse &L, SCEVUse &R,
12295 SCEVFlags &Flags) {
12296 if (!match(U: Expr, P: m_scev_Add(Op0: m_SCEV(V&: L), Op1: m_SCEV(V&: R))))
12297 return false;
12298
12299 Flags = cast<SCEVAddExpr>(Val&: Expr)->getNoWrapFlags();
12300 return true;
12301}
12302
12303std::optional<APInt>
12304ScalarEvolution::computeConstantDifference(const SCEV *More, const SCEV *Less) {
12305 // We avoid subtracting expressions here because this function is usually
12306 // fairly deep in the call stack (i.e. is called many times).
12307
12308 unsigned BW = getTypeSizeInBits(Ty: More->getType());
12309 APInt Diff(BW, 0);
12310 APInt DiffMul(BW, 1);
12311 // Try various simplifications to reduce the difference to a constant. Limit
12312 // the number of allowed simplifications to keep compile-time low.
12313 for (unsigned I = 0; I < 8; ++I) {
12314 if (More == Less)
12315 return Diff;
12316
12317 // Reduce addrecs with identical steps to their start value.
12318 if (isa<SCEVAddRecExpr>(Val: Less) && isa<SCEVAddRecExpr>(Val: More)) {
12319 const auto *LAR = cast<SCEVAddRecExpr>(Val: Less);
12320 const auto *MAR = cast<SCEVAddRecExpr>(Val: More);
12321
12322 if (LAR->getLoop() != MAR->getLoop())
12323 return std::nullopt;
12324
12325 // We look at affine expressions only; not for correctness but to keep
12326 // getStepRecurrence cheap.
12327 if (!LAR->isAffine() || !MAR->isAffine())
12328 return std::nullopt;
12329
12330 if (LAR->getStepRecurrence(SE&: *this) != MAR->getStepRecurrence(SE&: *this))
12331 return std::nullopt;
12332
12333 Less = LAR->getStart();
12334 More = MAR->getStart();
12335 continue;
12336 }
12337
12338 // Try to match a common constant multiply.
12339 auto MatchConstMul =
12340 [](const SCEV *S) -> std::optional<std::pair<const SCEV *, APInt>> {
12341 const APInt *C;
12342 const SCEV *Op;
12343 if (match(S, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEV(V&: Op))))
12344 return {{Op, *C}};
12345 return std::nullopt;
12346 };
12347 if (auto MatchedMore = MatchConstMul(More)) {
12348 if (auto MatchedLess = MatchConstMul(Less)) {
12349 if (MatchedMore->second == MatchedLess->second) {
12350 More = MatchedMore->first;
12351 Less = MatchedLess->first;
12352 DiffMul *= MatchedMore->second;
12353 continue;
12354 }
12355 }
12356 }
12357
12358 // Try to cancel out common factors in two add expressions.
12359 SmallDenseMap<const SCEV *, int, 8> Multiplicity;
12360 auto Add = [&](const SCEV *S, int Mul) {
12361 if (auto *C = dyn_cast<SCEVConstant>(Val: S)) {
12362 if (Mul == 1) {
12363 Diff += C->getAPInt() * DiffMul;
12364 } else {
12365 assert(Mul == -1);
12366 Diff -= C->getAPInt() * DiffMul;
12367 }
12368 } else
12369 Multiplicity[S] += Mul;
12370 };
12371 auto Decompose = [&](const SCEV *S, int Mul) {
12372 if (isa<SCEVAddExpr>(Val: S)) {
12373 for (const SCEV *Op : S->operands())
12374 Add(Op, Mul);
12375 } else
12376 Add(S, Mul);
12377 };
12378 Decompose(More, 1);
12379 Decompose(Less, -1);
12380
12381 // Check whether all the non-constants cancel out, or reduce to new
12382 // More/Less values.
12383 const SCEV *NewMore = nullptr, *NewLess = nullptr;
12384 for (const auto &[S, Mul] : Multiplicity) {
12385 if (Mul == 0)
12386 continue;
12387 if (Mul == 1) {
12388 if (NewMore)
12389 return std::nullopt;
12390 NewMore = S;
12391 } else if (Mul == -1) {
12392 if (NewLess)
12393 return std::nullopt;
12394 NewLess = S;
12395 } else
12396 return std::nullopt;
12397 }
12398
12399 // Values stayed the same, no point in trying further.
12400 if (NewMore == More || NewLess == Less)
12401 return std::nullopt;
12402
12403 More = NewMore;
12404 Less = NewLess;
12405
12406 // Reduced to constant.
12407 if (!More && !Less)
12408 return Diff;
12409
12410 // Left with variable on only one side, bail out.
12411 if (!More || !Less)
12412 return std::nullopt;
12413 }
12414
12415 // Did not reduce to constant.
12416 return std::nullopt;
12417}
12418
12419bool ScalarEvolution::isImpliedCondOperandsViaAddRecStart(
12420 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12421 const SCEV *FoundRHS, const Instruction *CtxI) {
12422 // Try to recognize the following pattern:
12423 //
12424 // FoundRHS = ...
12425 // ...
12426 // loop:
12427 // FoundLHS = {Start,+,W}
12428 // context_bb: // Basic block from the same loop
12429 // known(Pred, FoundLHS, FoundRHS)
12430 //
12431 // If some predicate is known in the context of a loop, it is also known on
12432 // each iteration of this loop, including the first iteration. Therefore, in
12433 // this case, `FoundLHS Pred FoundRHS` implies `Start Pred FoundRHS`. Try to
12434 // prove the original pred using this fact.
12435 if (!CtxI)
12436 return false;
12437 const BasicBlock *ContextBB = CtxI->getParent();
12438 // Make sure AR varies in the context block.
12439 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS)) {
12440 const Loop *L = AR->getLoop();
12441 const auto *Latch = L->getLoopLatch();
12442 // Make sure that context belongs to the loop and executes on 1st iteration
12443 // (if it ever executes at all).
12444 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12445 return false;
12446 if (!isAvailableAtLoopEntry(S: FoundRHS, L: AR->getLoop()))
12447 return false;
12448 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS: AR->getStart(), FoundRHS);
12449 }
12450
12451 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: FoundRHS)) {
12452 const Loop *L = AR->getLoop();
12453 const auto *Latch = L->getLoopLatch();
12454 // Make sure that context belongs to the loop and executes on 1st iteration
12455 // (if it ever executes at all).
12456 if (!L->contains(BB: ContextBB) || !Latch || !DT.dominates(A: ContextBB, B: Latch))
12457 return false;
12458 if (!isAvailableAtLoopEntry(S: FoundLHS, L: AR->getLoop()))
12459 return false;
12460 return isImpliedCondOperands(Pred, LHS, RHS, FoundLHS, FoundRHS: AR->getStart());
12461 }
12462
12463 return false;
12464}
12465
12466bool ScalarEvolution::isImpliedCondOperandsViaNoOverflow(CmpPredicate Pred,
12467 const SCEV *LHS,
12468 const SCEV *RHS,
12469 const SCEV *FoundLHS,
12470 const SCEV *FoundRHS) {
12471 if (Pred != CmpInst::ICMP_SLT && Pred != CmpInst::ICMP_ULT)
12472 return false;
12473
12474 const auto *AddRecLHS = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12475 if (!AddRecLHS)
12476 return false;
12477
12478 const auto *AddRecFoundLHS = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12479 if (!AddRecFoundLHS)
12480 return false;
12481
12482 // We'd like to let SCEV reason about control dependencies, so we constrain
12483 // both the inequalities to be about add recurrences on the same loop. This
12484 // way we can use isLoopEntryGuardedByCond later.
12485
12486 const Loop *L = AddRecFoundLHS->getLoop();
12487 if (L != AddRecLHS->getLoop())
12488 return false;
12489
12490 // FoundLHS u< FoundRHS u< -C => (FoundLHS + C) u< (FoundRHS + C) ... (1)
12491 //
12492 // FoundLHS s< FoundRHS s< INT_MIN - C => (FoundLHS + C) s< (FoundRHS + C)
12493 // ... (2)
12494 //
12495 // Informal proof for (2), assuming (1) [*]:
12496 //
12497 // We'll also assume (A s< B) <=> ((A + INT_MIN) u< (B + INT_MIN)) ... (3)[**]
12498 //
12499 // Then
12500 //
12501 // FoundLHS s< FoundRHS s< INT_MIN - C
12502 // <=> (FoundLHS + INT_MIN) u< (FoundRHS + INT_MIN) u< -C [ using (3) ]
12503 // <=> (FoundLHS + INT_MIN + C) u< (FoundRHS + INT_MIN + C) [ using (1) ]
12504 // <=> (FoundLHS + INT_MIN + C + INT_MIN) s<
12505 // (FoundRHS + INT_MIN + C + INT_MIN) [ using (3) ]
12506 // <=> FoundLHS + C s< FoundRHS + C
12507 //
12508 // [*]: (1) can be proved by ruling out overflow.
12509 //
12510 // [**]: This can be proved by analyzing all the four possibilities:
12511 // (A s< 0, B s< 0), (A s< 0, B s>= 0), (A s>= 0, B s< 0) and
12512 // (A s>= 0, B s>= 0).
12513 //
12514 // Note:
12515 // Despite (2), "FoundRHS s< INT_MIN - C" does not mean that "FoundRHS + C"
12516 // will not sign underflow. For instance, say FoundLHS = (i8 -128), FoundRHS
12517 // = (i8 -127) and C = (i8 -100). Then INT_MIN - C = (i8 -28), and FoundRHS
12518 // s< (INT_MIN - C). Lack of sign overflow / underflow in "FoundRHS + C" is
12519 // neither necessary nor sufficient to prove "(FoundLHS + C) s< (FoundRHS +
12520 // C)".
12521
12522 std::optional<APInt> LDiff = computeConstantDifference(More: LHS, Less: FoundLHS);
12523 if (!LDiff)
12524 return false;
12525 std::optional<APInt> RDiff = computeConstantDifference(More: RHS, Less: FoundRHS);
12526 if (!RDiff || *LDiff != *RDiff)
12527 return false;
12528
12529 if (LDiff->isMinValue())
12530 return true;
12531
12532 APInt FoundRHSLimit;
12533
12534 if (Pred == CmpInst::ICMP_ULT) {
12535 FoundRHSLimit = -(*RDiff);
12536 } else {
12537 assert(Pred == CmpInst::ICMP_SLT && "Checked above!");
12538 FoundRHSLimit = APInt::getSignedMinValue(numBits: getTypeSizeInBits(Ty: RHS->getType())) - *RDiff;
12539 }
12540
12541 // Try to prove (1) or (2), as needed.
12542 return isAvailableAtLoopEntry(S: FoundRHS, L) &&
12543 isLoopEntryGuardedByCond(L, Pred, LHS: FoundRHS,
12544 RHS: getConstant(Val: FoundRHSLimit));
12545}
12546
12547bool ScalarEvolution::isImpliedViaMerge(CmpPredicate Pred, const SCEV *LHS,
12548 const SCEV *RHS, const SCEV *FoundLHS,
12549 const SCEV *FoundRHS, unsigned Depth) {
12550 const PHINode *LPhi = nullptr, *RPhi = nullptr;
12551
12552 llvm::scope_exit ClearOnExit([&]() {
12553 if (LPhi) {
12554 bool Erased = PendingMerges.erase(Ptr: LPhi);
12555 assert(Erased && "Failed to erase LPhi!");
12556 (void)Erased;
12557 }
12558 if (RPhi) {
12559 bool Erased = PendingMerges.erase(Ptr: RPhi);
12560 assert(Erased && "Failed to erase RPhi!");
12561 (void)Erased;
12562 }
12563 });
12564
12565 // Find respective Phis and check that they are not being pending.
12566 if (const SCEVUnknown *LU = dyn_cast<SCEVUnknown>(Val: LHS))
12567 if (auto *Phi = dyn_cast<PHINode>(Val: LU->getValue())) {
12568 if (!PendingMerges.insert(Ptr: Phi).second)
12569 return false;
12570 LPhi = Phi;
12571 }
12572 if (const SCEVUnknown *RU = dyn_cast<SCEVUnknown>(Val: RHS))
12573 if (auto *Phi = dyn_cast<PHINode>(Val: RU->getValue())) {
12574 // If we detect a loop of Phi nodes being processed by this method, for
12575 // example:
12576 //
12577 // %a = phi i32 [ %some1, %preheader ], [ %b, %latch ]
12578 // %b = phi i32 [ %some2, %preheader ], [ %a, %latch ]
12579 //
12580 // we don't want to deal with a case that complex, so return conservative
12581 // answer false.
12582 if (!PendingMerges.insert(Ptr: Phi).second)
12583 return false;
12584 RPhi = Phi;
12585 }
12586
12587 // If none of LHS, RHS is a Phi, nothing to do here.
12588 if (!LPhi && !RPhi)
12589 return false;
12590
12591 // If there is a SCEVUnknown Phi we are interested in, make it left.
12592 if (!LPhi) {
12593 std::swap(a&: LHS, b&: RHS);
12594 std::swap(a&: FoundLHS, b&: FoundRHS);
12595 std::swap(a&: LPhi, b&: RPhi);
12596 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12597 }
12598
12599 assert(LPhi && "LPhi should definitely be a SCEVUnknown Phi!");
12600 const BasicBlock *LBB = LPhi->getParent();
12601 const SCEVAddRecExpr *RAR = dyn_cast<SCEVAddRecExpr>(Val: RHS);
12602
12603 auto ProvedEasily = [&](const SCEV *S1, const SCEV *S2) {
12604 return isKnownViaNonRecursiveReasoning(Pred, LHS: S1, RHS: S2) ||
12605 isImpliedCondOperandsViaRanges(Pred, LHS: S1, RHS: S2, FoundPred: Pred, FoundLHS, FoundRHS) ||
12606 isImpliedViaOperations(Pred, LHS: S1, RHS: S2, FoundLHS, FoundRHS, Depth);
12607 };
12608
12609 if (RPhi && RPhi->getParent() == LBB) {
12610 // Case one: RHS is also a SCEVUnknown Phi from the same basic block.
12611 // If we compare two Phis from the same block, and for each entry block
12612 // the predicate is true for incoming values from this block, then the
12613 // predicate is also true for the Phis.
12614 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12615 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12616 const SCEV *R = getSCEV(V: RPhi->getIncomingValueForBlock(BB: IncBB));
12617 if (!ProvedEasily(L, R))
12618 return false;
12619 }
12620 } else if (RAR && RAR->getLoop()->getHeader() == LBB) {
12621 // Case two: RHS is also a Phi from the same basic block, and it is an
12622 // AddRec. It means that there is a loop which has both AddRec and Unknown
12623 // PHIs, for it we can compare incoming values of AddRec from above the loop
12624 // and latch with their respective incoming values of LPhi.
12625 // TODO: Generalize to handle loops with many inputs in a header.
12626 if (LPhi->getNumIncomingValues() != 2) return false;
12627
12628 auto *RLoop = RAR->getLoop();
12629 auto *Predecessor = RLoop->getLoopPredecessor();
12630 assert(Predecessor && "Loop with AddRec with no predecessor?");
12631 const SCEV *L1 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Predecessor));
12632 if (!ProvedEasily(L1, RAR->getStart()))
12633 return false;
12634 auto *Latch = RLoop->getLoopLatch();
12635 assert(Latch && "Loop with AddRec with no latch?");
12636 const SCEV *L2 = getSCEV(V: LPhi->getIncomingValueForBlock(BB: Latch));
12637 if (!ProvedEasily(L2, RAR->getPostIncExpr(SE&: *this)))
12638 return false;
12639 } else {
12640 // In all other cases go over inputs of LHS and compare each of them to RHS,
12641 // the predicate is true for (LHS, RHS) if it is true for all such pairs.
12642 // At this point RHS is either a non-Phi, or it is a Phi from some block
12643 // different from LBB.
12644 for (const BasicBlock *IncBB : predecessors(BB: LBB)) {
12645 // Check that RHS is available in this block.
12646 if (!dominates(S: RHS, BB: IncBB))
12647 return false;
12648 const SCEV *L = getSCEV(V: LPhi->getIncomingValueForBlock(BB: IncBB));
12649 // Make sure L does not refer to a value from a potentially previous
12650 // iteration of a loop.
12651 if (!properlyDominates(S: L, BB: LBB))
12652 return false;
12653 // Addrecs are considered to properly dominate their loop, so are missed
12654 // by the previous check. Discard any values that have computable
12655 // evolution in this loop.
12656 if (auto *Loop = LI.getLoopFor(BB: LBB))
12657 if (hasComputableLoopEvolution(S: L, L: Loop))
12658 return false;
12659 if (!ProvedEasily(L, RHS))
12660 return false;
12661 }
12662 }
12663 return true;
12664}
12665
12666bool ScalarEvolution::isImpliedCondOperandsViaShift(CmpPredicate Pred,
12667 const SCEV *LHS,
12668 const SCEV *RHS,
12669 const SCEV *FoundLHS,
12670 const SCEV *FoundRHS) {
12671 // We want to imply LHS < RHS from LHS < (RHS >> shiftvalue). First, make
12672 // sure that we are dealing with same LHS.
12673 if (RHS == FoundRHS) {
12674 std::swap(a&: LHS, b&: RHS);
12675 std::swap(a&: FoundLHS, b&: FoundRHS);
12676 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12677 }
12678 if (LHS != FoundLHS)
12679 return false;
12680
12681 auto *SUFoundRHS = dyn_cast<SCEVUnknown>(Val: FoundRHS);
12682 if (!SUFoundRHS)
12683 return false;
12684
12685 Value *Shiftee, *ShiftValue;
12686
12687 using namespace PatternMatch;
12688 if (match(V: SUFoundRHS->getValue(),
12689 P: m_LShr(L: m_Value(V&: Shiftee), R: m_Value(V&: ShiftValue)))) {
12690 auto *ShifteeS = getSCEV(V: Shiftee);
12691 // Prove one of the following:
12692 // LHS <u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <u RHS
12693 // LHS <=u (shiftee >> shiftvalue) && shiftee <=u RHS ---> LHS <=u RHS
12694 // LHS <s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12695 // ---> LHS <s RHS
12696 // LHS <=s (shiftee >> shiftvalue) && shiftee <=s RHS && shiftee >=s 0
12697 // ---> LHS <=s RHS
12698 if (Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_ULE)
12699 return isKnownPredicate(Pred: ICmpInst::ICMP_ULE, LHS: ShifteeS, RHS);
12700 if (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SLE)
12701 if (isKnownNonNegative(S: ShifteeS))
12702 return isKnownPredicate(Pred: ICmpInst::ICMP_SLE, LHS: ShifteeS, RHS);
12703 }
12704
12705 return false;
12706}
12707
12708bool ScalarEvolution::isImpliedCondOperandsViaMatchingDiff(
12709 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, const SCEV *FoundLHS,
12710 const SCEV *FoundRHS) {
12711 // Only valid for equality predicates: (A == B) implies (C == D) when
12712 // the SCEV difference A - B equals C - D (they check the same
12713 // underlying relationship at every iteration).
12714 if (!ICmpInst::isEquality(P: Pred))
12715 return false;
12716
12717 // Restrict to cases involving loop recurrences - that's where this
12718 // pattern arises (correlated IV comparisons). This avoids calling
12719 // getMinusSCEV on arbitrary non-loop expressions.
12720 if ((!isa<SCEVAddRecExpr>(Val: LHS) && !isa<SCEVAddRecExpr>(Val: RHS)) ||
12721 (!isa<SCEVAddRecExpr>(Val: FoundLHS) && !isa<SCEVAddRecExpr>(Val: FoundRHS)))
12722 return false;
12723
12724 // AddRecs from different loops can never produce matching differences.
12725 const SCEVAddRecExpr *QueryAddRec = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12726 if (!QueryAddRec)
12727 QueryAddRec = cast<SCEVAddRecExpr>(Val: RHS);
12728 const SCEVAddRecExpr *FoundAddRec = dyn_cast<SCEVAddRecExpr>(Val: FoundLHS);
12729 if (!FoundAddRec)
12730 FoundAddRec = cast<SCEVAddRecExpr>(Val: FoundRHS);
12731 if (QueryAddRec->getLoop() != FoundAddRec->getLoop())
12732 return false;
12733
12734 // If the strides differ, the differences can never match.
12735 if (QueryAddRec->getStepRecurrence(SE&: *this) !=
12736 FoundAddRec->getStepRecurrence(SE&: *this))
12737 return false;
12738
12739 // Compute differences. For pointer-typed operands sharing the same base,
12740 // getMinusSCEV strips the common base and returns an integer SCEV.
12741 // For example, {base,+,8} - (base+8*n) = {-8n,+,8}
12742 const SCEV *FoundDiff = getMinusSCEV(LHS: FoundLHS, RHS: FoundRHS);
12743 if (isa<SCEVCouldNotCompute>(Val: FoundDiff))
12744 return false;
12745
12746 const SCEV *Diff = getMinusSCEV(LHS, RHS);
12747 if (isa<SCEVCouldNotCompute>(Val: Diff))
12748 return false;
12749
12750 return Diff == FoundDiff;
12751}
12752
12753bool ScalarEvolution::isImpliedCondOperands(CmpPredicate Pred, const SCEV *LHS,
12754 const SCEV *RHS,
12755 const SCEV *FoundLHS,
12756 const SCEV *FoundRHS,
12757 const Instruction *CtxI) {
12758 return isImpliedCondOperandsViaRanges(Pred, LHS, RHS, FoundPred: Pred, FoundLHS,
12759 FoundRHS) ||
12760 isImpliedCondOperandsViaNoOverflow(Pred, LHS, RHS, FoundLHS,
12761 FoundRHS) ||
12762 isImpliedCondOperandsViaShift(Pred, LHS, RHS, FoundLHS, FoundRHS) ||
12763 isImpliedCondOperandsViaAddRecStart(Pred, LHS, RHS, FoundLHS, FoundRHS,
12764 CtxI) ||
12765 isImpliedCondOperandsViaMatchingDiff(Pred, LHS, RHS, FoundLHS,
12766 FoundRHS) ||
12767 isImpliedCondOperandsHelper(Pred, LHS, RHS, FoundLHS, FoundRHS);
12768}
12769
12770/// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values?
12771template <typename MinMaxExprType>
12772static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr,
12773 const SCEV *Candidate) {
12774 const MinMaxExprType *MinMaxExpr = dyn_cast<MinMaxExprType>(MaybeMinMaxExpr);
12775 if (!MinMaxExpr)
12776 return false;
12777
12778 return is_contained(MinMaxExpr->operands(), Candidate);
12779}
12780
12781static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE,
12782 CmpPredicate Pred, const SCEV *LHS,
12783 const SCEV *RHS) {
12784 // If both sides are affine addrecs for the same loop, with equal
12785 // steps, and we know the recurrences don't wrap, then we only
12786 // need to check the predicate on the starting values.
12787
12788 if (!ICmpInst::isRelational(P: Pred))
12789 return false;
12790
12791 const SCEV *LStart, *RStart, *Step;
12792 const Loop *L;
12793 if (!match(S: LHS,
12794 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: LStart), Op1: m_SCEV(V&: Step), L: m_Loop(L))) ||
12795 !match(S: RHS, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: RStart), Op1: m_scev_Specific(S: Step),
12796 L: m_SpecificLoop(L))))
12797 return false;
12798 const SCEVAddRecExpr *LAR = cast<SCEVAddRecExpr>(Val: LHS);
12799 const SCEVAddRecExpr *RAR = cast<SCEVAddRecExpr>(Val: RHS);
12800 SCEVFlags NW = ICmpInst::isSigned(Pred) ? SCEV::FlagNSW : SCEV::FlagNUW;
12801 if (!LAR->getNoWrapFlags(Mask: NW) || !RAR->getNoWrapFlags(Mask: NW))
12802 return false;
12803
12804 return SE.isKnownPredicate(Pred, LHS: LStart, RHS: RStart);
12805}
12806
12807/// Is LHS `Pred` RHS true because one of them is an AddRec that is known not to
12808/// go below its own start value?
12809static bool IsKnownPredicateViaAddRecMonotonicity(ScalarEvolution &SE,
12810 CmpPredicate Pred,
12811 const SCEV *LHS,
12812 const SCEV *RHS) {
12813 // Normalize to (AddRec Pred Start).
12814 if (!isa<SCEVAddRecExpr>(Val: LHS) && isa<SCEVAddRecExpr>(Val: RHS)) {
12815 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12816 std::swap(a&: LHS, b&: RHS);
12817 }
12818
12819 // The recurrence is equal to Start in the first iteration, so only the
12820 // non-strict predicate holds.
12821 if (Pred != ICmpInst::ICMP_UGE && Pred != ICmpInst::ICMP_SGE)
12822 return false;
12823
12824 const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS);
12825 if (!AR || AR->getStart() != RHS)
12826 return false;
12827
12828 return SE.getMonotonicPredicateType(LHS: AR, Pred) ==
12829 ScalarEvolution::MonotonicallyIncreasing;
12830}
12831
12832/// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max
12833/// expression?
12834static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, CmpPredicate Pred,
12835 const SCEV *LHS, const SCEV *RHS) {
12836 switch (Pred) {
12837 default:
12838 return false;
12839
12840 case ICmpInst::ICMP_SGE:
12841 std::swap(a&: LHS, b&: RHS);
12842 [[fallthrough]];
12843 case ICmpInst::ICMP_SLE:
12844 return
12845 // min(A, ...) <= A
12846 IsMinMaxConsistingOf<SCEVSMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12847 // A <= max(A, ...)
12848 IsMinMaxConsistingOf<SCEVSMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12849
12850 case ICmpInst::ICMP_UGE:
12851 std::swap(a&: LHS, b&: RHS);
12852 [[fallthrough]];
12853 case ICmpInst::ICMP_ULE:
12854 return
12855 // min(A, ...) <= A
12856 // FIXME: what about umin_seq?
12857 IsMinMaxConsistingOf<SCEVUMinExpr>(MaybeMinMaxExpr: LHS, Candidate: RHS) ||
12858 // A <= max(A, ...)
12859 IsMinMaxConsistingOf<SCEVUMaxExpr>(MaybeMinMaxExpr: RHS, Candidate: LHS);
12860
12861 case ICmpInst::ICMP_UGT:
12862 std::swap(a&: LHS, b&: RHS);
12863 [[fallthrough]];
12864 case ICmpInst::ICMP_ULT:
12865 // umin(Ops) u<= each Op, so proving Op u< RHS for any Op proves
12866 // umin(Ops) u< RHS.
12867 //
12868 // Use computeConstantDifference instead of the more powerful
12869 // isKnownPredicate to keep this check cheap: isKnownPredicateViaMinOrMax
12870 // is called from isKnownViaNonRecursiveReasoning, so recursing into
12871 // the full predicate prover would be expensive.
12872 if (const auto *Min = dyn_cast<SCEVUMinExpr>(Val: LHS)) {
12873 for (SCEVUse Op : Min->operands()) {
12874 std::optional<APInt> Diff = SE.computeConstantDifference(More: RHS, Less: Op);
12875 // When Op and RHS share a common base differing by a
12876 // constant offset D (RHS - Op = D), Op u< RHS holds iff D != 0 and
12877 // RHS >= D (unsigned), i.e. the subtraction doesn't underflow.
12878 if (Diff && !Diff->isZero() && SE.getUnsignedRangeMin(S: RHS).uge(RHS: *Diff))
12879 return true;
12880 }
12881 }
12882 return false;
12883 }
12884
12885 llvm_unreachable("covered switch fell through?!");
12886}
12887
12888bool ScalarEvolution::isImpliedViaOperations(CmpPredicate Pred, const SCEV *LHS,
12889 const SCEV *RHS,
12890 const SCEV *FoundLHS,
12891 const SCEV *FoundRHS,
12892 unsigned Depth) {
12893 assert(getTypeSizeInBits(LHS->getType()) ==
12894 getTypeSizeInBits(RHS->getType()) &&
12895 "LHS and RHS have different sizes?");
12896 assert(getTypeSizeInBits(FoundLHS->getType()) ==
12897 getTypeSizeInBits(FoundRHS->getType()) &&
12898 "FoundLHS and FoundRHS have different sizes?");
12899 // We want to avoid hurting the compile time with analysis of too big trees.
12900 if (Depth > Opts.scalar_evolution_max_scev_operations_implication_depth)
12901 return false;
12902
12903 // We only want to work with GT comparison so far.
12904 if (ICmpInst::isLT(P: Pred)) {
12905 Pred = ICmpInst::getSwappedCmpPredicate(Pred);
12906 std::swap(a&: LHS, b&: RHS);
12907 std::swap(a&: FoundLHS, b&: FoundRHS);
12908 }
12909
12910 CmpInst::Predicate P = Pred.getPreferredSignedPredicate();
12911
12912 // For unsigned, try to reduce it to corresponding signed comparison.
12913 if (P == ICmpInst::ICMP_UGT)
12914 // We can replace unsigned predicate with its signed counterpart if all
12915 // involved values are non-negative.
12916 // TODO: We could have better support for unsigned.
12917 if (isKnownNonNegative(S: FoundLHS) && isKnownNonNegative(S: FoundRHS)) {
12918 // Knowing that both FoundLHS and FoundRHS are non-negative, and knowing
12919 // FoundLHS >u FoundRHS, we also know that FoundLHS >s FoundRHS. Let us
12920 // use this fact to prove that LHS and RHS are non-negative.
12921 const SCEV *MinusOne = getMinusOne(Ty: LHS->getType());
12922 if (isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS, RHS: MinusOne, FoundLHS,
12923 FoundRHS) &&
12924 isImpliedCondOperands(Pred: ICmpInst::ICMP_SGT, LHS: RHS, RHS: MinusOne, FoundLHS,
12925 FoundRHS))
12926 P = ICmpInst::ICMP_SGT;
12927 }
12928
12929 if (P != ICmpInst::ICMP_SGT)
12930 return false;
12931
12932 auto GetOpFromSExt = [&](const SCEV *S) -> const SCEV * {
12933 if (auto *Ext = dyn_cast<SCEVSignExtendExpr>(Val: S))
12934 return Ext->getOperand();
12935 // TODO: If S is a SCEVConstant then you can cheaply "strip" the sext off
12936 // the constant in some cases.
12937 return S;
12938 };
12939
12940 // Acquire values from extensions.
12941 auto *OrigLHS = LHS;
12942 auto *OrigFoundLHS = FoundLHS;
12943 LHS = GetOpFromSExt(LHS);
12944 FoundLHS = GetOpFromSExt(FoundLHS);
12945
12946 // Is the SGT predicate can be proved trivially or using the found context.
12947 auto IsSGTViaContext = [&](const SCEV *S1, const SCEV *S2) {
12948 return isKnownViaNonRecursiveReasoning(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2) ||
12949 isImpliedViaOperations(Pred: ICmpInst::ICMP_SGT, LHS: S1, RHS: S2, FoundLHS: OrigFoundLHS,
12950 FoundRHS, Depth: Depth + 1);
12951 };
12952
12953 SCEVUse LL;
12954 SCEVUse LR;
12955 if (match(S: LHS, P: m_scev_NSWAdd(Op0: m_SCEV(V&: LL), Op1: m_SCEV(V&: LR)))) {
12956 // We want to avoid creation of any new non-constant SCEV. Since we are
12957 // going to compare the operands to RHS, we should be certain that we don't
12958 // need any size extensions for this. So let's decline all cases when the
12959 // sizes of types of LHS and RHS do not match.
12960 // TODO: Maybe try to get RHS from sext to catch more cases?
12961 if (getTypeSizeInBits(Ty: LHS->getType()) != getTypeSizeInBits(Ty: RHS->getType()))
12962 return false;
12963
12964 auto *MinusOne = getMinusOne(Ty: RHS->getType());
12965
12966 // Checks that S1 >= 0 && S2 > RHS, trivially or using the found context.
12967 auto IsSumGreaterThanRHS = [&](const SCEV *S1, const SCEV *S2) {
12968 return IsSGTViaContext(S1, MinusOne) && IsSGTViaContext(S2, RHS);
12969 };
12970 // Try to prove the following rule:
12971 // (LHS = LL + LR) && (LL >= 0) && (LR > RHS) => (LHS > RHS).
12972 // (LHS = LL + LR) && (LR >= 0) && (LL > RHS) => (LHS > RHS).
12973 if (IsSumGreaterThanRHS(LL, LR) || IsSumGreaterThanRHS(LR, LL))
12974 return true;
12975 } else if (auto *LHSUnknownExpr = dyn_cast<SCEVUnknown>(Val: LHS)) {
12976 Value *LL, *LR;
12977 // FIXME: Once we have SDiv implemented, we can get rid of this matching.
12978
12979 using namespace llvm::PatternMatch;
12980
12981 if (match(V: LHSUnknownExpr->getValue(), P: m_SDiv(L: m_Value(V&: LL), R: m_Value(V&: LR)))) {
12982 // Rules for division.
12983 // We are going to perform some comparisons with Denominator and its
12984 // derivative expressions. In general case, creating a SCEV for it may
12985 // lead to a complex analysis of the entire graph, and in particular it
12986 // can request trip count recalculation for the same loop. This would
12987 // cache as SCEVCouldNotCompute to avoid the infinite recursion. To avoid
12988 // this, we only want to create SCEVs that are constants in this section.
12989 // So we bail if Denominator is not a constant.
12990 if (!isa<ConstantInt>(Val: LR))
12991 return false;
12992
12993 auto *Denominator = cast<SCEVConstant>(Val: getSCEV(V: LR));
12994
12995 // We want to make sure that LHS = FoundLHS / Denominator. If it is so,
12996 // then a SCEV for the numerator already exists and matches with FoundLHS.
12997 auto *Numerator = getExistingSCEV(V: LL);
12998 if (!Numerator || Numerator->getType() != FoundLHS->getType())
12999 return false;
13000
13001 // Make sure that the numerator matches with FoundLHS and the denominator
13002 // is positive.
13003 if (!HasSameValue(A: Numerator, B: FoundLHS) || !isKnownPositive(S: Denominator))
13004 return false;
13005
13006 auto *DTy = Denominator->getType();
13007 auto *FRHSTy = FoundRHS->getType();
13008 if (DTy->isPointerTy() != FRHSTy->isPointerTy())
13009 // One of types is a pointer and another one is not. We cannot extend
13010 // them properly to a wider type, so let us just reject this case.
13011 // TODO: Usage of getEffectiveSCEVType for DTy, FRHSTy etc should help
13012 // to avoid this check.
13013 return false;
13014
13015 // Given that:
13016 // FoundLHS > FoundRHS, LHS = FoundLHS / Denominator, Denominator > 0.
13017 auto *WTy = getWiderType(T1: DTy, T2: FRHSTy);
13018 auto *DenominatorExt = getNoopOrSignExtend(V: Denominator, Ty: WTy);
13019 auto *FoundRHSExt = getNoopOrSignExtend(V: FoundRHS, Ty: WTy);
13020
13021 // Try to prove the following rule:
13022 // (FoundRHS > Denominator - 2) && (RHS <= 0) => (LHS > RHS).
13023 // For example, given that FoundLHS > 2. It means that FoundLHS is at
13024 // least 3. If we divide it by Denominator < 4, we will have at least 1.
13025 auto *DenomMinusTwo = getMinusSCEV(LHS: DenominatorExt, RHS: getConstant(Ty: WTy, V: 2));
13026 if (isKnownNonPositive(S: RHS) &&
13027 IsSGTViaContext(FoundRHSExt, DenomMinusTwo))
13028 return true;
13029
13030 // Try to prove the following rule:
13031 // (FoundRHS > -1 - Denominator) && (RHS < 0) => (LHS > RHS).
13032 // For example, given that FoundLHS > -3. Then FoundLHS is at least -2.
13033 // If we divide it by Denominator > 2, then:
13034 // 1. If FoundLHS is negative, then the result is 0.
13035 // 2. If FoundLHS is non-negative, then the result is non-negative.
13036 // Anyways, the result is non-negative.
13037 auto *MinusOne = getMinusOne(Ty: WTy);
13038 auto *NegDenomMinusOne = getMinusSCEV(LHS: MinusOne, RHS: DenominatorExt);
13039 if (isKnownNegative(S: RHS) &&
13040 IsSGTViaContext(FoundRHSExt, NegDenomMinusOne))
13041 return true;
13042 }
13043 }
13044
13045 // If our expression contained SCEVUnknown Phis, and we split it down and now
13046 // need to prove something for them, try to prove the predicate for every
13047 // possible incoming values of those Phis.
13048 if (isImpliedViaMerge(Pred, LHS: OrigLHS, RHS, FoundLHS: OrigFoundLHS, FoundRHS, Depth: Depth + 1))
13049 return true;
13050
13051 return false;
13052}
13053
13054static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS,
13055 const SCEV *RHS) {
13056 // zext x u<= sext x, sext x s<= zext x
13057 const SCEV *Op;
13058 switch (Pred) {
13059 case ICmpInst::ICMP_SGE:
13060 std::swap(a&: LHS, b&: RHS);
13061 [[fallthrough]];
13062 case ICmpInst::ICMP_SLE: {
13063 // If operand >=s 0 then ZExt == SExt. If operand <s 0 then SExt <s ZExt.
13064 return match(S: LHS, P: m_scev_SExt(Op0: m_SCEV(V&: Op))) &&
13065 match(S: RHS, P: m_scev_ZExt(Op0: m_scev_Specific(S: Op)));
13066 }
13067 case ICmpInst::ICMP_UGE:
13068 std::swap(a&: LHS, b&: RHS);
13069 [[fallthrough]];
13070 case ICmpInst::ICMP_ULE: {
13071 // If operand >=u 0 then ZExt == SExt. If operand <u 0 then ZExt <u SExt.
13072 return match(S: LHS, P: m_scev_ZExt(Op0: m_SCEV(V&: Op))) &&
13073 match(S: RHS, P: m_scev_SExt(Op0: m_scev_Specific(S: Op)));
13074 }
13075 default:
13076 return false;
13077 };
13078 llvm_unreachable("unhandled case");
13079}
13080
13081bool ScalarEvolution::isKnownViaNonRecursiveReasoning(CmpPredicate Pred,
13082 SCEVUse LHS,
13083 SCEVUse RHS) {
13084 return isKnownPredicateExtendIdiom(Pred, LHS, RHS) ||
13085 isKnownPredicateViaConstantRanges(Pred, LHS, RHS) ||
13086 IsKnownPredicateViaMinOrMax(SE&: *this, Pred, LHS, RHS) ||
13087 IsKnownPredicateViaAddRecStart(SE&: *this, Pred, LHS, RHS) ||
13088 IsKnownPredicateViaAddRecMonotonicity(SE&: *this, Pred, LHS, RHS) ||
13089 isKnownPredicateViaNoOverflow(Pred, LHS, RHS);
13090}
13091
13092bool ScalarEvolution::isImpliedCondOperandsHelper(CmpPredicate Pred,
13093 const SCEV *LHS,
13094 const SCEV *RHS,
13095 const SCEV *FoundLHS,
13096 const SCEV *FoundRHS) {
13097 if (ICmpInst::isEquality(P: Pred)) {
13098 if (HasSameValue(A: LHS, B: FoundLHS) && HasSameValue(A: RHS, B: FoundRHS))
13099 return true;
13100 } else {
13101 ICmpInst::Predicate NonStrictPred = ICmpInst::getNonStrictPredicate(pred: Pred);
13102 if (isKnownViaNonRecursiveReasoning(Pred: NonStrictPred, LHS, RHS: FoundLHS) &&
13103 isKnownViaNonRecursiveReasoning(
13104 Pred: ICmpInst::getSwappedPredicate(pred: NonStrictPred), LHS: RHS, RHS: FoundRHS))
13105 return true;
13106 }
13107
13108 // Maybe it can be proved via operations?
13109 return isImpliedViaOperations(Pred, LHS, RHS, FoundLHS, FoundRHS);
13110}
13111
13112bool ScalarEvolution::isImpliedCondOperandsViaRanges(
13113 CmpPredicate Pred, const SCEV *LHS, const SCEV *RHS, CmpPredicate FoundPred,
13114 const SCEV *FoundLHS, const SCEV *FoundRHS) {
13115 if (!isa<SCEVConstant>(Val: RHS) || !isa<SCEVConstant>(Val: FoundRHS))
13116 // The restriction on `FoundRHS` be lifted easily -- it exists only to
13117 // reduce the compile time impact of this optimization.
13118 return false;
13119
13120 std::optional<APInt> Addend = computeConstantDifference(More: LHS, Less: FoundLHS);
13121 if (!Addend)
13122 return false;
13123
13124 const APInt &ConstFoundRHS = cast<SCEVConstant>(Val: FoundRHS)->getAPInt();
13125
13126 // `FoundLHSRange` is the range we know `FoundLHS` to be in by virtue of the
13127 // antecedent "`FoundLHS` `FoundPred` `FoundRHS`".
13128 ConstantRange FoundLHSRange =
13129 ConstantRange::makeExactICmpRegion(Pred: FoundPred, Other: ConstFoundRHS);
13130
13131 // Since `LHS` is `FoundLHS` + `Addend`, we can compute a range for `LHS`:
13132 ConstantRange LHSRange = FoundLHSRange.add(Other: ConstantRange(*Addend));
13133
13134 // We can also compute the range of values for `LHS` that satisfy the
13135 // consequent, "`LHS` `Pred` `RHS`":
13136 const APInt &ConstRHS = cast<SCEVConstant>(Val: RHS)->getAPInt();
13137 // The antecedent implies the consequent if every value of `LHS` that
13138 // satisfies the antecedent also satisfies the consequent.
13139 return LHSRange.icmp(Pred, Other: ConstRHS);
13140}
13141
13142bool ScalarEvolution::canIVOverflowOnLT(const SCEV *RHS, const SCEV *Stride,
13143 bool IsSigned, bool Invert) {
13144 assert(isKnownPositive(Stride) && "Positive stride expected!");
13145
13146 unsigned BitWidth = getTypeSizeInBits(Ty: RHS->getType());
13147 const SCEV *One = getOne(Ty: Stride->getType());
13148
13149 if (IsSigned) {
13150 APInt MaxRHS = getRangeMax(S: RHS, /*IsSigned=*/true, Invert);
13151 APInt MaxValue = APInt::getSignedMaxValue(numBits: BitWidth);
13152 APInt MaxStrideMinusOne = getSignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13153
13154 // SMaxRHS + SMaxStrideMinusOne > SMaxValue => overflow!
13155 return (std::move(MaxValue) - MaxStrideMinusOne).slt(RHS: MaxRHS);
13156 }
13157
13158 APInt MaxRHS = getRangeMax(S: RHS, /*IsSigned=*/false, Invert);
13159 APInt MaxValue = APInt::getMaxValue(numBits: BitWidth);
13160 APInt MaxStrideMinusOne = getUnsignedRangeMax(S: getMinusSCEV(LHS: Stride, RHS: One));
13161
13162 // UMaxRHS + UMaxStrideMinusOne > UMaxValue => overflow!
13163 return (std::move(MaxValue) - MaxStrideMinusOne).ult(RHS: MaxRHS);
13164}
13165
13166const SCEV *ScalarEvolution::getUDivCeilSCEV(const SCEV *N, const SCEV *D) {
13167 // umin(N, 1) + floor((N - umin(N, 1)) / D)
13168 // This is equivalent to "1 + floor((N - 1) / D)" for N != 0. The umin
13169 // expression fixes the case of N=0.
13170 const SCEV *MinNOne = getUMinExpr(LHS: N, RHS: getOne(Ty: N->getType()));
13171 const SCEV *NMinusOne = getMinusSCEV(LHS: N, RHS: MinNOne);
13172 return getAddExpr(LHS: MinNOne, RHS: getUDivExpr(LHS: NMinusOne, RHS: D));
13173}
13174
13175const SCEV *
13176ScalarEvolution::computeMaxBECountForLT(const SCEV *Start, const SCEV *Stride,
13177 const SCEV *End, unsigned BitWidth,
13178 bool IsSigned, bool Invert) {
13179 // The logic in this function assumes we can represent a positive stride.
13180 // If we can't, the backedge-taken count must be zero.
13181 if (IsSigned && BitWidth == 1)
13182 return getZero(Ty: Stride->getType());
13183
13184 // This code below only been closely audited for negative strides in the
13185 // unsigned comparison case, it may be correct for signed comparison, but
13186 // that needs to be established.
13187 if (IsSigned && isKnownNegative(S: Stride))
13188 return getCouldNotCompute();
13189
13190 // Calculate the maximum backedge count based on the range of values
13191 // permitted by Start, End, and Stride. If Invert is true, both Start and End
13192 // need inverting. Stride was already negated by the caller.
13193 APInt MinStart = getRangeMin(S: Start, IsSigned, Invert);
13194
13195 APInt MinStride =
13196 IsSigned ? getSignedRangeMin(S: Stride) : getUnsignedRangeMin(S: Stride);
13197
13198 // We assume either the stride is positive, or the backedge-taken count
13199 // is zero. So force StrideForMaxBECount to be at least one.
13200 APInt One(BitWidth, 1);
13201 APInt StrideForMaxBECount = IsSigned ? APIntOps::smax(A: One, B: MinStride)
13202 : APIntOps::umax(A: One, B: MinStride);
13203
13204 APInt MaxValue = IsSigned ? APInt::getSignedMaxValue(numBits: BitWidth)
13205 : APInt::getMaxValue(numBits: BitWidth);
13206 APInt Limit = MaxValue - (StrideForMaxBECount - 1);
13207
13208 // Although End can be a MAX expression we estimate MaxEnd considering only
13209 // the case End = RHS of the loop termination condition. This is safe because
13210 // in the other case (End - Start) is zero, leading to a zero maximum backedge
13211 // taken count.
13212 APInt MaxEnd = getRangeMax(S: End, IsSigned, Invert);
13213 MaxEnd =
13214 IsSigned ? APIntOps::smin(A: MaxEnd, B: Limit) : APIntOps::umin(A: MaxEnd, B: Limit);
13215
13216 // MaxBECount = ceil((max(MaxEnd, MinStart) - MinStart) / Stride)
13217 MaxEnd = IsSigned ? APIntOps::smax(A: MaxEnd, B: MinStart)
13218 : APIntOps::umax(A: MaxEnd, B: MinStart);
13219
13220 APInt Delta = MaxEnd - MinStart;
13221
13222 // Try to refine Delta in case End - Start (or Start - End if Invert) gives a
13223 // tighter bound after folding.
13224 const SCEV *DeltaExpr =
13225 Invert ? getMinusSCEV(LHS: Start, RHS: End) : getMinusSCEV(LHS: End, RHS: Start);
13226 Delta = APIntOps::umin(A: Delta, B: getUnsignedRangeMax(S: DeltaExpr));
13227
13228 return getUDivCeilSCEV(N: getConstant(Val: Delta), D: getConstant(Val: StrideForMaxBECount));
13229}
13230
13231ScalarEvolution::ExitLimit
13232ScalarEvolution::howManyLessThans(const SCEV *LHS, const SCEV *RHS,
13233 const Loop *L, bool IsSigned, bool Invert,
13234 bool ControlsOnlyExit, bool AllowPredicates) {
13235 SmallVector<const SCEVPredicate *> Predicates;
13236
13237 LoopGuards Guards = LoopGuards::collectOnDemand(L, SE&: *this);
13238
13239 // FIXME: Extend the non-invariant RHS analysis to greater-than comparisons.
13240 if (Invert && !isLoopInvariant(S: RHS, L))
13241 return getCouldNotCompute();
13242
13243 const SCEVAddRecExpr *IV = dyn_cast<SCEVAddRecExpr>(Val: LHS);
13244 bool PredicatedIV = false;
13245 // FIXME: Generalize the NUW inference below to decreasing IVs.
13246 if (!IV && !Invert) {
13247 if (auto *ZExt = dyn_cast<SCEVZeroExtendExpr>(Val: LHS)) {
13248 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: ZExt->getOperand());
13249 if (AR && AR->getLoop() == L && AR->isAffine()) {
13250 auto canProveNUW = [&]() {
13251 // We can use the comparison to infer no-wrap flags only if it fully
13252 // controls the loop exit.
13253 if (!ControlsOnlyExit)
13254 return false;
13255
13256 if (!isLoopInvariant(S: RHS, L))
13257 return false;
13258
13259 if (!isKnownNonZero(S: AR->getStepRecurrence(SE&: *this)))
13260 // We need the sequence defined by AR to strictly increase in the
13261 // unsigned integer domain for the logic below to hold.
13262 return false;
13263
13264 const unsigned InnerBitWidth = getTypeSizeInBits(Ty: AR->getType());
13265 const unsigned OuterBitWidth = getTypeSizeInBits(Ty: RHS->getType());
13266 // If RHS <=u Limit, then there must exist a value V in the sequence
13267 // defined by AR (e.g. {Start,+,Step}) such that V >u RHS, and
13268 // V <=u UINT_MAX. Thus, we must exit the loop before unsigned
13269 // overflow occurs. This limit also implies that a signed comparison
13270 // (in the wide bitwidth) is equivalent to an unsigned comparison as
13271 // the high bits on both sides must be zero.
13272 APInt StrideMax = getUnsignedRangeMax(S: AR->getStepRecurrence(SE&: *this));
13273 APInt Limit = APInt::getMaxValue(numBits: InnerBitWidth) - (StrideMax - 1);
13274 Limit = Limit.zext(width: OuterBitWidth);
13275 return getUnsignedRangeMax(S: applyLoopGuards(Expr: RHS, Guards)).ule(RHS: Limit);
13276 };
13277 auto Flags = AR->getNoWrapFlags();
13278 if (!hasFlags(Flags, TestFlags: SCEV::FlagNUW) && canProveNUW())
13279 Flags = setFlags(Flags, OnFlags: SCEV::FlagNUW);
13280
13281 setNoWrapFlags(AddRec: const_cast<SCEVAddRecExpr *>(AR), Flags);
13282 if (AR->hasNoUnsignedWrap()) {
13283 // Emulate what getZeroExtendExpr would have done during construction
13284 // if we'd been able to infer the fact just above at that time.
13285 const SCEV *Step = AR->getStepRecurrence(SE&: *this);
13286 Type *Ty = ZExt->getType();
13287 const SCEV *S = getAddRecExpr(
13288 Start: getExtendAddRecStart<SCEVZeroExtendExpr>(AR, Ty, SE: this, Depth: 0),
13289 Step: getZeroExtendExpr(Op: Step, Ty, Depth: 0), L, Flags: AR->getNoWrapFlags());
13290 IV = dyn_cast<SCEVAddRecExpr>(Val: S);
13291 }
13292 }
13293 }
13294 }
13295
13296 if (!IV && AllowPredicates) {
13297 // Try to make this an AddRec using runtime tests, in the first X
13298 // iterations of this loop, where X is the SCEV expression found by the
13299 // algorithm below.
13300 IV = convertSCEVToAddRecWithPredicates(S: LHS, L, Preds&: Predicates);
13301 PredicatedIV = true;
13302 }
13303
13304 // Avoid weird loops
13305 if (!IV || IV->getLoop() != L || !IV->isAffine())
13306 return getCouldNotCompute();
13307
13308 // A precondition of this method is that the condition being analyzed
13309 // reaches an exiting branch which dominates the latch. Given that, we can
13310 // assume that an increment which violates the nowrap specification and
13311 // produces poison must cause undefined behavior when the resulting poison
13312 // value is branched upon and thus we can conclude that the backedge is
13313 // taken no more often than would be required to produce that poison value.
13314 // Note that a well defined loop can exit on the iteration which violates
13315 // the nowrap specification if there is another exit (either explicit or
13316 // implicit/exceptional) which causes the loop to execute before the
13317 // exiting instruction we're analyzing would trigger UB.
13318 auto WrapType = IsSigned ? SCEV::FlagNSW : SCEV::FlagNUW;
13319 bool NoWrap = ControlsOnlyExit && any(Val: IV->getNoWrapFlags(Mask: WrapType));
13320 // Reverse the ordering for greater-than comparisons.
13321 ICmpInst::Predicate Cond = IsSigned ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
13322 if (Invert)
13323 Cond = ICmpInst::getSwappedPredicate(pred: Cond);
13324
13325 // The step of ~IV is the negated step of IV.
13326 const SCEV *Stride = IV->getStepRecurrence(SE&: *this);
13327 if (Invert)
13328 Stride = getNegativeSCEV(V: Stride);
13329 const SCEV *GuardedStride = Stride;
13330
13331 // Whether the IV may reach the maximum (or minimum if inverted) value
13332 // before the exit is taken.
13333 bool IVMayOverflow = true;
13334
13335 bool PositiveStride = isKnownPositive(S: Stride);
13336 // A dominating guard may prove the stride positive.
13337 if (!PositiveStride) {
13338 const SCEV *LoopGuardedStride = applyLoopGuards(Expr: Stride, Guards);
13339 if (isKnownPositive(S: LoopGuardedStride)) {
13340 GuardedStride = LoopGuardedStride;
13341 PositiveStride = true;
13342 // Encode the context-sensitive stride > 0 fact into the expression
13343 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13344 }
13345 }
13346
13347 // Avoid negative or zero stride values.
13348 if (!PositiveStride) {
13349 // FIXME: Generalize the unknown-stride analysis to decreasing IVs.
13350 if (Invert)
13351 return getCouldNotCompute();
13352
13353 // We can compute the correct backedge taken count for loops with unknown
13354 // strides if we can prove that the loop is not an infinite loop with side
13355 // effects. Here's the loop structure we are trying to handle -
13356 //
13357 // i = start
13358 // do {
13359 // A[i] = i;
13360 // i += s;
13361 // } while (i < end);
13362 //
13363 // The backedge taken count for such loops is evaluated as -
13364 // (max(end, start + stride) - start - 1) /u stride
13365 //
13366 // The additional preconditions that we need to check to prove correctness
13367 // of the above formula is as follows -
13368 //
13369 // a) IV is either nuw or nsw depending upon signedness (indicated by the
13370 // NoWrap flag).
13371 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13372 // b) the loop is guaranteed to be finite (e.g. is mustprogress and has
13373 // no side effects within the loop) or a predicate is added to ensure
13374 // stride is positive.
13375 // c) loop has a single static exit (with no abnormal exits)
13376 //
13377 // Precondition a) implies that if the stride is negative, this is a single
13378 // trip loop. The backedge taken count formula reduces to zero in this case.
13379 //
13380 // Precondition b) and c) combine to imply that if rhs is invariant in L,
13381 // then a zero stride means the backedge can't be taken without executing
13382 // undefined behavior.
13383 //
13384 // The positive stride case is the same as isKnownPositive(Stride) returning
13385 // true (original behavior of the function).
13386 //
13387 if (PredicatedIV || !NoWrap || !loopHasNoAbnormalExits(L))
13388 return getCouldNotCompute();
13389
13390 if (!loopIsFiniteByAssumption(L)) {
13391 // If the loop may be infinite, add a predicate ensuring Stride is
13392 // positive, to guarantee forward progress.
13393 if (!AllowPredicates || !isLoopInvariant(S: Stride, L))
13394 return getCouldNotCompute();
13395
13396 const SCEV *Zero = getZero(Ty: Stride->getType());
13397 const SCEVPredicate *P =
13398 getComparePredicate(Pred: ICmpInst::ICMP_SGT, LHS: Stride, RHS: Zero);
13399 Predicates.push_back(Elt: P);
13400 // When the predicate holds (Stride > 0), umax(Stride, 1) == Stride,
13401 // so the result is unchanged. To prevent div by zero.
13402 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13403 } else if (!isKnownNonZero(S: Stride)) {
13404 // If we have a step of zero, and RHS isn't invariant in L, we don't know
13405 // if it might eventually be greater than start and if so, on which
13406 // iteration. We can't even produce a useful upper bound.
13407 if (!isLoopInvariant(S: RHS, L))
13408 return getCouldNotCompute();
13409
13410 // We allow a potentially zero stride, but we need to divide by stride
13411 // below. Since the loop can't be infinite and this check must control
13412 // the sole exit, we can infer the exit must be taken on the first
13413 // iteration (e.g. backedge count = 0) if the stride is zero. Given that,
13414 // we know the numerator in the divides below must be zero, so we can
13415 // pick an arbitrary non-zero value for the denominator (e.g. stride)
13416 // and produce the right result.
13417 // FIXME: Handle the case where Stride is poison?
13418 auto wouldZeroStrideBeUB = [&]() {
13419 // Proof by contradiction. Suppose the stride were zero. If we can
13420 // prove that the backedge *is* taken on the first iteration, then since
13421 // we know this condition controls the sole exit, we must have an
13422 // infinite loop. We can't have a (well defined) infinite loop per
13423 // check just above.
13424 // Note: The (Start - Stride) term is used to get the start' term from
13425 // (start' + stride,+,stride). Remember that we only care about the
13426 // result of this expression when stride == 0 at runtime.
13427 auto *StartIfZero = getMinusSCEV(LHS: IV->getStart(), RHS: Stride);
13428 return isLoopEntryGuardedByCond(L, Pred: Cond, LHS: StartIfZero, RHS);
13429 };
13430 if (!wouldZeroStrideBeUB()) {
13431 Stride = getUMaxExpr(LHS: Stride, RHS: getOne(Ty: Stride->getType()));
13432 }
13433 }
13434 } else {
13435 // Avoid proven overflow cases: this will ensure that the backedge taken
13436 // count will not generate any unsigned overflow.
13437 IVMayOverflow = canIVOverflowOnLT(RHS, Stride: GuardedStride, IsSigned, Invert) &&
13438 canIVOverflowOnLT(RHS: applyLoopGuards(Expr: RHS, Guards),
13439 Stride: GuardedStride, IsSigned, Invert);
13440 if (IVMayOverflow && !NoWrap)
13441 return getCouldNotCompute();
13442 }
13443
13444 // On all paths just preceeding, we established the following invariant:
13445 // IV can be assumed not to overflow up to and including the exiting
13446 // iteration. We proved this in one of two ways:
13447 // 1) We can show overflow doesn't occur before the exiting iteration
13448 // 1a) canIVOverflowOnLT, and b) step of one
13449 // 2) We can show that if overflow occurs, the loop must execute UB
13450 // before any possible exit.
13451 // Note that we have not yet proved RHS invariant (in general).
13452
13453 const SCEV *Start = IV->getStart();
13454
13455 // Preserve pointer-typed Start/RHS to pass to isLoopEntryGuardedByCond.
13456 // If we convert to integers, isLoopEntryGuardedByCond will miss some cases.
13457 // Use integer-typed versions for actual computation; we can't subtract
13458 // pointers in general.
13459 const SCEV *OrigStart = Start;
13460 const SCEV *OrigRHS = RHS;
13461 if (Start->getType()->isPointerTy()) {
13462 Start = getPtrToAddrExpr(Op: Start);
13463 if (isa<SCEVCouldNotCompute>(Val: Start))
13464 return Start;
13465 }
13466 if (RHS->getType()->isPointerTy()) {
13467 RHS = getPtrToAddrExpr(Op: RHS);
13468 if (isa<SCEVCouldNotCompute>(Val: RHS))
13469 return RHS;
13470 }
13471
13472 const SCEV *End = nullptr, *BECount = getCouldNotCompute(),
13473 *BECountIfBackedgeTaken = getCouldNotCompute();
13474 if (!isLoopInvariant(S: RHS, L)) {
13475 assert(!Invert && "RHS must be loop-invariant for Invert");
13476 const auto *RHSAddRec = dyn_cast<SCEVAddRecExpr>(Val: RHS);
13477 if (PositiveStride && RHSAddRec != nullptr && RHSAddRec->getLoop() == L &&
13478 any(Val: RHSAddRec->getNoWrapFlags())) {
13479 // The structure of loop we are trying to calculate backedge count of:
13480 //
13481 // left = left_start
13482 // right = right_start
13483 //
13484 // while(left < right){
13485 // ... do something here ...
13486 // left += s1; // stride of left is s1 (s1 > 0)
13487 // right += s2; // stride of right is s2 (s2 < 0)
13488 // }
13489 //
13490
13491 const SCEV *RHSStart = RHSAddRec->getStart();
13492 const SCEV *RHSStride = RHSAddRec->getStepRecurrence(SE&: *this);
13493
13494 // If Stride - RHSStride is positive and does not overflow, we can write
13495 // backedge count as ->
13496 // ceil((End - Start) /u (Stride - RHSStride))
13497 // Where, End = max(RHSStart, Start)
13498
13499 // Check if RHSStride < 0 and Stride - RHSStride will not overflow.
13500 if (isKnownNegative(S: RHSStride) &&
13501 willNotOverflow(BinOp: Instruction::Sub, /*Signed=*/true, LHS: Stride,
13502 RHS: RHSStride)) {
13503
13504 const SCEV *Denominator = getMinusSCEV(LHS: Stride, RHS: RHSStride);
13505 if (isKnownPositive(S: Denominator)) {
13506 End = IsSigned ? getSMaxExpr(LHS: RHSStart, RHS: Start)
13507 : getUMaxExpr(LHS: RHSStart, RHS: Start);
13508
13509 // We can do this because End >= Start, as End = max(RHSStart, Start)
13510 const SCEV *Delta = getMinusSCEV(LHS: End, RHS: Start);
13511
13512 BECount = getUDivCeilSCEV(N: Delta, D: Denominator);
13513 BECountIfBackedgeTaken =
13514 getUDivCeilSCEV(N: getMinusSCEV(LHS: RHSStart, RHS: Start), D: Denominator);
13515 }
13516 }
13517 }
13518 } else {
13519 // Let End = max(RHS,Start). We use the expression (End-Start)/Stride to
13520 // describe the backedge count: if the backedge is taken at least once then
13521 // End is RHS, and if not End is Start so we get a backedge count of zero.
13522 // Inverted, End is min(RHS, Start).
13523 //
13524 // AddingStrideMinusOneMayOverflow has the following preconditions:
13525 //
13526 // 1. Start <= End, signed if IsSigned (inverted: End <= Start)
13527 // 2. The index variable doesn't overflow.
13528 //
13529 // Therefore, we know N exists such that
13530 // (Start + Stride * N) >= End, and computing "(Start + Stride * N)"
13531 // doesn't overflow.
13532 //
13533 // Using this information, try to prove whether the addition in
13534 // "(End - Start) + (Stride - 1)" has unsigned overflow.
13535 //
13536 // If the IV cannot overflow, RHS is at least Stride - 1 below the maximum
13537 // value, so the distance End - Start is at most UMAX - (Stride - 1) and
13538 // the (Stride - 1) addition below cannot overflow.
13539 const SCEV *One = getOne(Ty: Stride->getType());
13540 bool AddingStrideMinusOneMayOverflow = IVMayOverflow && [&] {
13541 if (isKnownToBeAPowerOfTwo(S: Stride)) {
13542 // Suppose Stride is a power of two, and Start/End are unsigned
13543 // integers. Let UMAX be the largest representable unsigned
13544 // integer.
13545 //
13546 // By the preconditions of this function, we know
13547 // "(Start + Stride * N) >= End", and this doesn't overflow.
13548 // As a formula:
13549 //
13550 // End <= (Start + Stride * N) <= UMAX
13551 //
13552 // Subtracting Start from all the terms:
13553 //
13554 // End - Start <= Stride * N <= UMAX - Start
13555 //
13556 // Since Start is unsigned, UMAX - Start <= UMAX. Therefore:
13557 //
13558 // End - Start <= Stride * N <= UMAX
13559 //
13560 // Stride * N is a multiple of Stride. Therefore,
13561 //
13562 // End - Start <= Stride * N <= UMAX - (UMAX mod Stride)
13563 //
13564 // Since Stride is a power of two, UMAX + 1 is divisible by
13565 // Stride. Therefore, UMAX mod Stride == Stride - 1. So we can
13566 // write:
13567 //
13568 // End - Start <= Stride * N <= UMAX - Stride - 1
13569 //
13570 // Dropping the middle term:
13571 //
13572 // End - Start <= UMAX - Stride - 1
13573 //
13574 // Adding Stride - 1 to both sides:
13575 //
13576 // (End - Start) + (Stride - 1) <= UMAX
13577 //
13578 // In other words, the addition doesn't have unsigned overflow.
13579 //
13580 // A similar proof works if we treat Start/End as signed values.
13581 // Just rewrite steps before "End - Start <= Stride * N <= UMAX"
13582 // to use signed max instead of unsigned max. Note that we're
13583 // trying to prove a lack of unsigned overflow in either case.
13584 // Inverted: "Start - End <= Stride * N <= Start - MIN <= UMAX", same.
13585 return false;
13586 }
13587 if (!Invert && (Start == Stride || Start == getMinusSCEV(LHS: Stride, RHS: One))) {
13588 // If Start is equal to Stride, (End - Start) + (Stride - 1) == End
13589 // - 1. If !IsSigned, 0 <u Stride == Start <=u End; so 0 <u End - 1
13590 // <u End. If IsSigned, 0 <s Stride == Start <=s End; so 0 <s End -
13591 // 1 <s End.
13592 //
13593 // If Start is equal to Stride - 1, (End - Start) + Stride - 1 ==
13594 // End.
13595 //
13596 // Both need Start to be the smaller value, so neither applies inverted.
13597 return false;
13598 }
13599 return true;
13600 }();
13601
13602 // If inverted, the analyzed values are complements: "~V - Offset" is "~(V +
13603 // Offset)" and "~To - ~From" is "From - To".
13604 auto StepBack = [&](const SCEV *V, const SCEV *Offset) -> const SCEV * {
13605 if (Invert)
13606 return getAddExpr(LHS: V, RHS: Offset);
13607 return getMinusSCEV(LHS: V, RHS: Offset);
13608 };
13609 auto Distance = [&](const SCEV *From, const SCEV *To) {
13610 return Invert ? getMinusSCEV(LHS: From, RHS: To) : getMinusSCEV(LHS: To, RHS: From);
13611 };
13612
13613 const SCEV *OrigPrevStart = StepBack(OrigStart, Stride);
13614 assert(isAvailableAtLoopEntry(OrigPrevStart, L) && "Must be!");
13615 assert(isAvailableAtLoopEntry(OrigStart, L) && "Must be!");
13616 assert(isAvailableAtLoopEntry(OrigRHS, L) && "Must be!");
13617 // Can we prove Start - Stride < RHS, and either Start - Stride < Start or
13618 // (via !AddingStrideMinusOneMayOverflow) that (RHS - Start) + (Stride - 1)
13619 // does not overflow?
13620 if ((!AddingStrideMinusOneMayOverflow ||
13621 isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigPrevStart, RHS: OrigStart)) &&
13622 isLoopEntryGuardedByCond(L, Pred: Cond, LHS: OrigPrevStart, RHS: OrigRHS)) {
13623 // In this case, we can use a refined formula for computing backedge
13624 // taken count. The general formula remains:
13625 // "End-Start /uceiling Stride"
13626 // We want to use the alternate formula:
13627 // "((RHS - 1) - (Start - Stride)) /u Stride"
13628 // Let's do a quick case analysis to show these are equivalent under
13629 // our preconditions. When inverted, the proof uses complemented Start,
13630 // RHS and End; Stride remains positive.
13631 // * For RHS <= Start (End is Start), the backedge-taken count must be
13632 // zero. Together with the precondition "Start - Stride < RHS", we have
13633 // "Start - Stride < RHS <= Start". Subtracting Start - Stride from
13634 // all sides we get "0 < RHS - (Start - Stride) <= Stride".
13635 // Subtracting 1 we get "0 <= (RHS - 1) - (Start - Stride) < Stride".
13636 // So dividing that by Stride gives zero.
13637 //
13638 // * For RHS > Start (End is RHS), the backedge count must be
13639 // "RHS-Start /uceil Stride", so it is sufficient to show that the
13640 // numerator "((RHS - 1) - (Start - Stride))" does not overflow.
13641 //
13642 // If "Start - Stride < Start" holds, we have
13643 // "RHS > Start > Start - Stride". As such
13644 // "RHS - (Start - Stride) - 1" does not overflow, which is the
13645 // reassociated numerator.
13646 //
13647 // Otherwise !AddingStrideMinusOneMayOverflow guarantees that
13648 // "(End - Start) + (Stride - 1)" does not overflow unsigned. Here
13649 // "End" is "RHS", as "RHS > Start", so this is the reassociated
13650 // numerator. Neither sub-term wraps unsigned: "RHS - Start"
13651 // due to "RHS > Start", and "Stride - 1", as Stride is non-zero.
13652 const SCEV *Numerator =
13653 getMinusSCEV(LHS: Distance(StepBack(Start, Stride), RHS), RHS: One);
13654 BECount = getUDivExpr(LHS: Numerator, RHS: Stride);
13655 }
13656
13657 if (isa<SCEVCouldNotCompute>(Val: BECount)) {
13658 auto canProveRHSIsAtOrBeyondStart = [&]() {
13659 // Inverted, the claim is "Start >= RHS". Reverse the comparisons below
13660 // by swapping their operands rather than their predicates:
13661 // isLoopEntryGuardedByCond is sensitive to operand order and loses the
13662 // proof if the IV bound moves to the other side.
13663 auto SwapIfInverted = [&](const SCEV *A, const SCEV *B) {
13664 return Invert ? std::pair(B, A) : std::pair(A, B);
13665 };
13666
13667 auto CondGE = IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE;
13668 const SCEV *GuardedRHS = applyLoopGuards(Expr: OrigRHS, Guards);
13669 const SCEV *GuardedStart = applyLoopGuards(Expr: OrigStart, Guards);
13670 if (Invert)
13671 std::swap(a&: GuardedRHS, b&: GuardedStart);
13672
13673 auto [GELHS, GERHS] = SwapIfInverted(OrigRHS, OrigStart);
13674 if (isLoopEntryGuardedByCond(L, Pred: CondGE, LHS: GELHS, RHS: GERHS) ||
13675 isKnownPredicate(Pred: CondGE, LHS: GuardedRHS, RHS: GuardedStart))
13676 return true;
13677
13678 if ((GuardedRHS != GELHS || GuardedStart != GERHS) &&
13679 isKnownPredicate(Pred: CondGE, LHS: GELHS, RHS: GERHS))
13680 return true;
13681
13682 // (RHS > Start - 1) implies RHS >= Start.
13683 // * "RHS >= Start" is trivially equivalent to "RHS > Start - 1" if
13684 // "Start - 1" doesn't overflow.
13685 // * For signed comparison, if Start - 1 does overflow, it's equal
13686 // to INT_MAX, and "RHS >s INT_MAX" is trivially false.
13687 // * For unsigned comparison, if Start - 1 does overflow, it's equal
13688 // to UINT_MAX, and "RHS >u UINT_MAX" is trivially false.
13689 //
13690 // FIXME: Should isLoopEntryGuardedByCond do this for us?
13691 auto CondGT = IsSigned ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT;
13692 auto [GTLHS, GTRHS] = SwapIfInverted(OrigRHS, StepBack(OrigStart, One));
13693 return isLoopEntryGuardedByCond(L, Pred: CondGT, LHS: GTLHS, RHS: GTRHS);
13694 };
13695
13696 // If we know that RHS >= Start in the context of loop, then we know
13697 // that max(RHS, Start) = RHS at this point.
13698 if (canProveRHSIsAtOrBeyondStart()) {
13699 End = RHS;
13700 } else {
13701 // If RHS < Start, the backedge will be taken zero times. So in
13702 // general, we can write the backedge-taken count as:
13703 //
13704 // RHS >= Start ? ceil(RHS - Start) / Stride : 0
13705 //
13706 // We convert it to the following to make it more convenient for SCEV:
13707 //
13708 // ceil(max(RHS, Start) - Start) / Stride
13709 //
13710 // Inverted, this is ceil(Start - min(RHS, Start)) / Stride.
13711 if (Invert)
13712 End = IsSigned ? getSMinExpr(LHS: RHS, RHS: Start) : getUMinExpr(LHS: RHS, RHS: Start);
13713 else
13714 End = IsSigned ? getSMaxExpr(LHS: RHS, RHS: Start) : getUMaxExpr(LHS: RHS, RHS: Start);
13715
13716 // See what would happen if we assume the backedge is taken. This is
13717 // used to compute MaxBECount.
13718 BECountIfBackedgeTaken = getUDivCeilSCEV(N: Distance(Start, RHS), D: Stride);
13719 }
13720
13721 const SCEV *Delta = Distance(Start, End);
13722 if (!AddingStrideMinusOneMayOverflow) {
13723 // floor((D + (S - 1)) / S)
13724 // We prefer this formulation if it's legal because it's fewer
13725 // operations.
13726 BECount =
13727 getUDivExpr(LHS: getAddExpr(LHS: Delta, RHS: getMinusSCEV(LHS: Stride, RHS: One)), RHS: Stride);
13728 } else {
13729 BECount = getUDivCeilSCEV(N: Delta, D: Stride);
13730 }
13731 }
13732 }
13733
13734 const SCEV *ConstantMaxBECount;
13735 bool MaxOrZero = false;
13736 if (isa<SCEVConstant>(Val: BECount)) {
13737 ConstantMaxBECount = BECount;
13738 } else {
13739 ConstantMaxBECount = computeMaxBECountForLT(
13740 Start, Stride, End: RHS, BitWidth: getTypeSizeInBits(Ty: LHS->getType()), IsSigned,
13741 Invert);
13742 // If we know exactly how many times the backedge will be taken if it's
13743 // taken at least once, then the backedge count will either be that or
13744 // zero. If that count exceeds the range-based bound, the backedge can
13745 // never be taken.
13746 const APInt *IfTaken, *RangeMax;
13747 if (match(S: BECountIfBackedgeTaken, P: m_scev_APInt(C&: IfTaken))) {
13748 if (match(S: ConstantMaxBECount, P: m_scev_APInt(C&: RangeMax)) &&
13749 IfTaken->ugt(RHS: *RangeMax)) {
13750 ConstantMaxBECount = getZero(Ty: BECountIfBackedgeTaken->getType());
13751 } else {
13752 ConstantMaxBECount = BECountIfBackedgeTaken;
13753 MaxOrZero = true;
13754 }
13755 }
13756 }
13757
13758 if (isa<SCEVCouldNotCompute>(Val: ConstantMaxBECount) &&
13759 !isa<SCEVCouldNotCompute>(Val: BECount))
13760 ConstantMaxBECount = getConstant(Val: getUnsignedRangeMax(S: BECount));
13761
13762 const SCEV *SymbolicMaxBECount =
13763 isa<SCEVCouldNotCompute>(Val: BECount) ? ConstantMaxBECount : BECount;
13764 return ExitLimit(BECount, ConstantMaxBECount, SymbolicMaxBECount, MaxOrZero,
13765 Predicates);
13766}
13767
13768const SCEV *SCEVAddRecExpr::getNumIterationsInRange(const ConstantRange &Range,
13769 ScalarEvolution &SE) const {
13770 if (Range.isFullSet()) // Infinite loop.
13771 return SE.getCouldNotCompute();
13772
13773 // If the start is a non-zero constant, shift the range to simplify things.
13774 if (const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: getStart()))
13775 if (!SC->getValue()->isZero()) {
13776 SmallVector<SCEVUse, 4> Operands(operands());
13777 Operands[0] = SE.getZero(Ty: SC->getType());
13778 const SCEV *Shifted = SE.getAddRecExpr(Operands, L: getLoop(),
13779 NWFlags: getNoWrapFlags(Mask: FlagNW));
13780 if (const auto *ShiftedAddRec = dyn_cast<SCEVAddRecExpr>(Val: Shifted))
13781 return ShiftedAddRec->getNumIterationsInRange(
13782 Range: Range.subtract(CI: SC->getAPInt()), SE);
13783 // This is strange and shouldn't happen.
13784 return SE.getCouldNotCompute();
13785 }
13786
13787 // The only time we can solve this is when we have all constant indices.
13788 // Otherwise, we cannot determine the overflow conditions.
13789 if (!all_of(Range: operands(), P: IsaPred<SCEVConstant>))
13790 return SE.getCouldNotCompute();
13791
13792 // Okay at this point we know that all elements of the chrec are constants and
13793 // that the start element is zero.
13794
13795 // First check to see if the range contains zero. If not, the first
13796 // iteration exits.
13797 unsigned BitWidth = SE.getTypeSizeInBits(Ty: getType());
13798 if (!Range.contains(Val: APInt(BitWidth, 0)))
13799 return SE.getZero(Ty: getType());
13800
13801 if (isAffine()) {
13802 // If this is an affine expression then we have this situation:
13803 // Solve {0,+,A} in Range === Ax in Range
13804
13805 // We know that zero is in the range. If A is positive then we know that
13806 // the upper value of the range must be the first possible exit value.
13807 // If A is negative then the lower of the range is the last possible loop
13808 // value. Also note that we already checked for a full range.
13809 APInt A = cast<SCEVConstant>(Val: getOperand(i: 1))->getAPInt();
13810 APInt End = A.sge(RHS: 1) ? (Range.getUpper() - 1) : Range.getLower();
13811
13812 // The exit value should be (End+A)/A.
13813 APInt ExitVal = (End + A).udiv(RHS: A);
13814 ConstantInt *ExitValue = ConstantInt::get(Context&: SE.getContext(), V: ExitVal);
13815
13816 // Evaluate at the exit value. If we really did fall out of the valid
13817 // range, then we computed our trip count, otherwise wrap around or other
13818 // things must have happened.
13819 ConstantInt *Val = EvaluateConstantChrecAtConstant(AddRec: this, C: ExitValue, SE);
13820 if (Range.contains(Val: Val->getValue()))
13821 return SE.getCouldNotCompute(); // Something strange happened
13822
13823 // Ensure that the previous value is in the range.
13824 assert(Range.contains(
13825 EvaluateConstantChrecAtConstant(this,
13826 ConstantInt::get(SE.getContext(), ExitVal - 1), SE)->getValue()) &&
13827 "Linear scev computation is off in a bad way!");
13828 return SE.getConstant(V: ExitValue);
13829 }
13830
13831 if (isQuadratic()) {
13832 if (auto S = SolveQuadraticAddRecRange(AddRec: this, Range, SE))
13833 return SE.getConstant(Val: *S);
13834 }
13835
13836 return SE.getCouldNotCompute();
13837}
13838
13839const SCEVAddRecExpr *
13840SCEVAddRecExpr::getPostIncExpr(ScalarEvolution &SE) const {
13841 assert(getNumOperands() > 1 && "AddRec with zero step?");
13842 // There is a temptation to just call getAddExpr(this, getStepRecurrence(SE)),
13843 // but in this case we cannot guarantee that the value returned will be an
13844 // AddRec because SCEV does not have a fixed point where it stops
13845 // simplification: it is legal to return ({rec1} + {rec2}). For example, it
13846 // may happen if we reach arithmetic depth limit while simplifying. So we
13847 // construct the returned value explicitly.
13848 SmallVector<SCEVUse, 3> Ops;
13849 // If this is {A,+,B,+,C,...,+,N}, then its step is {B,+,C,+,...,+,N}, and
13850 // (this + Step) is {A+B,+,B+C,+...,+,N}.
13851 for (unsigned i = 0, e = getNumOperands() - 1; i < e; ++i)
13852 Ops.push_back(Elt: SE.getAddExpr(LHS: getOperand(i), RHS: getOperand(i: i + 1)));
13853 // We know that the last operand is not a constant zero (otherwise it would
13854 // have been popped out earlier). This guarantees us that if the result has
13855 // the same last operand, then it will also not be popped out, meaning that
13856 // the returned value will be an AddRec.
13857 const SCEV *Last = getOperand(i: getNumOperands() - 1);
13858 assert(!Last->isZero() && "Recurrency with zero step?");
13859 Ops.push_back(Elt: Last);
13860 return cast<SCEVAddRecExpr>(Val: SE.getAddRecExpr(Operands&: Ops, L: getLoop(), NWFlags: SCEV::FlagNone));
13861}
13862
13863// Return true when S contains at least an undef value.
13864bool ScalarEvolution::containsUndefs(const SCEV *S) const {
13865 return SCEVExprContains(
13866 Root: S, Pred: [](const SCEV *S) { return match(S, P: m_scev_UndefOrPoison()); });
13867}
13868
13869// Return true when S contains a value that is a nullptr.
13870bool ScalarEvolution::containsErasedValue(const SCEV *S) const {
13871 return SCEVExprContains(Root: S, Pred: [](const SCEV *S) {
13872 if (const auto *SU = dyn_cast<SCEVUnknown>(Val: S))
13873 return SU->getValue() == nullptr;
13874 return false;
13875 });
13876}
13877
13878/// Return the size of an element read or written by Inst.
13879const SCEV *ScalarEvolution::getElementSize(Instruction *Inst) {
13880 if (!isa<LoadInst, StoreInst>(Val: Inst))
13881 return nullptr;
13882 Type *ETy = getEffectiveSCEVType(Ty: getLoadStorePointerOperand(V: Inst)->getType());
13883 return getSizeOfExpr(IntTy: ETy, AllocTy: getLoadStoreType(I: Inst));
13884}
13885
13886//===----------------------------------------------------------------------===//
13887// SCEVCallbackVH Class Implementation
13888//===----------------------------------------------------------------------===//
13889
13890void ScalarEvolution::SCEVCallbackVH::deleted() {
13891 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
13892 if (PHINode *PN = dyn_cast<PHINode>(Val: getValPtr()))
13893 SE->ConstantEvolutionLoopExitValue.erase(Val: PN);
13894 SE->eraseValueFromMap(V: getValPtr());
13895 // this now dangles!
13896}
13897
13898void ScalarEvolution::SCEVCallbackVH::allUsesReplacedWith(Value *V) {
13899 assert(SE && "SCEVCallbackVH called with a null ScalarEvolution!");
13900
13901 // Forget all the expressions associated with users of the old value,
13902 // so that future queries will recompute the expressions using the new
13903 // value.
13904 SE->forgetValue(V: getValPtr());
13905 // this now dangles!
13906}
13907
13908ScalarEvolution::SCEVCallbackVH::SCEVCallbackVH(Value *V, ScalarEvolution *se)
13909 : CallbackVH(V), SE(se) {}
13910
13911//===----------------------------------------------------------------------===//
13912// ScalarEvolution Class Implementation
13913//===----------------------------------------------------------------------===//
13914
13915ScalarEvolution::ScalarEvolution(Function &F, TargetLibraryInfo &TLI,
13916 AssumptionCache &AC, DominatorTree &DT,
13917 LoopInfo &LI)
13918 : Opts(getAnalysisOptions(F)), F(F), DL(F.getDataLayout()), TLI(TLI),
13919 AC(AC), DT(DT), LI(LI), CouldNotCompute(new SCEVCouldNotCompute()),
13920 ValuesAtScopes(64), LoopDispositions(64), BlockDispositions(64) {
13921 // To use guards for proving predicates, we need to scan every instruction in
13922 // relevant basic blocks, and not just terminators. Doing this is a waste of
13923 // time if the IR does not actually contain any calls to
13924 // @llvm.experimental.guard, so do a quick check and remember this beforehand.
13925 //
13926 // This pessimizes the case where a pass that preserves ScalarEvolution wants
13927 // to _add_ guards to the module when there weren't any before, and wants
13928 // ScalarEvolution to optimize based on those guards. For now we prefer to be
13929 // efficient in lieu of being smart in that rather obscure case.
13930
13931 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
13932 M: F.getParent(), id: Intrinsic::experimental_guard);
13933 HasGuards = GuardDecl && !GuardDecl->use_empty();
13934}
13935
13936ScalarEvolution::ScalarEvolution(ScalarEvolution &&Arg)
13937 : Opts(Arg.Opts), F(Arg.F), DL(Arg.DL), HasGuards(Arg.HasGuards),
13938 TLI(Arg.TLI), AC(Arg.AC), DT(Arg.DT), LI(Arg.LI),
13939 CouldNotCompute(std::move(Arg.CouldNotCompute)),
13940 ValueExprMap(std::move(Arg.ValueExprMap)),
13941 PendingLoopPredicates(std::move(Arg.PendingLoopPredicates)),
13942 PendingMerges(std::move(Arg.PendingMerges)),
13943 ConstantMultipleCache(std::move(Arg.ConstantMultipleCache)),
13944 BackedgeTakenCounts(std::move(Arg.BackedgeTakenCounts)),
13945 PredicatedBackedgeTakenCounts(
13946 std::move(Arg.PredicatedBackedgeTakenCounts)),
13947 BECountUsers(std::move(Arg.BECountUsers)),
13948 ConstantEvolutionLoopExitValue(
13949 std::move(Arg.ConstantEvolutionLoopExitValue)),
13950 ValuesAtScopes(std::move(Arg.ValuesAtScopes)),
13951 ValuesAtScopesUsers(std::move(Arg.ValuesAtScopesUsers)),
13952 LoopDispositions(std::move(Arg.LoopDispositions)),
13953 LoopPropertiesCache(std::move(Arg.LoopPropertiesCache)),
13954 BlockDispositions(std::move(Arg.BlockDispositions)),
13955 SCEVUsers(std::move(Arg.SCEVUsers)),
13956 UnsignedRanges(std::move(Arg.UnsignedRanges)),
13957 SignedRanges(std::move(Arg.SignedRanges)),
13958 UniqueSCEVs(std::move(Arg.UniqueSCEVs)),
13959 UniquePreds(std::move(Arg.UniquePreds)),
13960 SCEVAllocator(std::move(Arg.SCEVAllocator)),
13961 ConstantSCEVs(std::move(Arg.ConstantSCEVs)),
13962 LoopUsers(std::move(Arg.LoopUsers)),
13963 PredicatedSCEVRewrites(std::move(Arg.PredicatedSCEVRewrites)),
13964 FirstUnknown(Arg.FirstUnknown),
13965 NextUnknownOrderID(Arg.NextUnknownOrderID) {
13966 Arg.FirstUnknown = nullptr;
13967}
13968
13969ScalarEvolution::~ScalarEvolution() {
13970 // Iterate through all the SCEVUnknown instances and call their
13971 // destructors, so that they release their references to their values.
13972 for (SCEVUnknown *U = FirstUnknown; U;) {
13973 SCEVUnknown *Tmp = U;
13974 U = U->Next;
13975 Tmp->~SCEVUnknown();
13976 }
13977 FirstUnknown = nullptr;
13978
13979 ExprValueMap.clear();
13980 ValueExprMap.clear();
13981 HasRecMap.clear();
13982 BackedgeTakenCounts.clear();
13983 PredicatedBackedgeTakenCounts.clear();
13984
13985 assert(PendingLoopPredicates.empty() && "isImpliedCond garbage");
13986 assert(PendingMerges.empty() && "isImpliedViaMerge garbage");
13987 assert(!WalkingBEDominatingConds && "isLoopBackedgeGuardedByCond garbage!");
13988 assert(!ProvingSplitPredicate && "ProvingSplitPredicate garbage!");
13989}
13990
13991bool ScalarEvolution::hasLoopInvariantBackedgeTakenCount(const Loop *L) {
13992 return !isa<SCEVCouldNotCompute>(Val: getBackedgeTakenCount(L));
13993}
13994
13995/// When printing a top-level SCEV for trip counts, it's helpful to include
13996/// a type for constants which are otherwise hard to disambiguate.
13997static void PrintSCEVWithTypeHint(raw_ostream &OS, const SCEV* S) {
13998 if (isa<SCEVConstant>(Val: S))
13999 OS << *S->getType() << " ";
14000 OS << *S;
14001}
14002
14003static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE,
14004 const Loop *L) {
14005 // Print all inner loops first
14006 for (Loop *I : *L)
14007 PrintLoopInfo(OS, SE, L: I);
14008
14009 OS << "Loop ";
14010 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14011 OS << ": ";
14012
14013 SmallVector<BasicBlock *, 8> ExitingBlocks;
14014 L->getExitingBlocks(ExitingBlocks);
14015 if (ExitingBlocks.size() != 1)
14016 OS << "<multiple exits> ";
14017
14018 auto *BTC = SE->getBackedgeTakenCount(L);
14019 if (!isa<SCEVCouldNotCompute>(Val: BTC)) {
14020 OS << "backedge-taken count is ";
14021 PrintSCEVWithTypeHint(OS, S: BTC);
14022 } else
14023 OS << "Unpredictable backedge-taken count.";
14024 OS << "\n";
14025
14026 if (ExitingBlocks.size() > 1)
14027 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14028 OS << " exit count for " << ExitingBlock->getName() << ": ";
14029 const SCEV *EC = SE->getExitCount(L, ExitingBlock);
14030 PrintSCEVWithTypeHint(OS, S: EC);
14031 if (isa<SCEVCouldNotCompute>(Val: EC)) {
14032 // Retry with predicates.
14033 SmallVector<const SCEVPredicate *> Predicates;
14034 EC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates);
14035 if (!isa<SCEVCouldNotCompute>(Val: EC)) {
14036 OS << "\n predicated exit count for " << ExitingBlock->getName()
14037 << ": ";
14038 PrintSCEVWithTypeHint(OS, S: EC);
14039 OS << "\n Predicates:\n";
14040 for (const auto *P : Predicates)
14041 P->print(OS, Depth: 4);
14042 }
14043 }
14044 OS << "\n";
14045 }
14046
14047 OS << "Loop ";
14048 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14049 OS << ": ";
14050
14051 auto *ConstantBTC = SE->getConstantMaxBackedgeTakenCount(L);
14052 if (!isa<SCEVCouldNotCompute>(Val: ConstantBTC)) {
14053 OS << "constant max backedge-taken count is ";
14054 PrintSCEVWithTypeHint(OS, S: ConstantBTC);
14055 if (SE->isBackedgeTakenCountMaxOrZero(L))
14056 OS << ", actual taken count either this or zero.";
14057 } else {
14058 OS << "Unpredictable constant max backedge-taken count. ";
14059 }
14060
14061 OS << "\n"
14062 "Loop ";
14063 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14064 OS << ": ";
14065
14066 auto *SymbolicBTC = SE->getSymbolicMaxBackedgeTakenCount(L);
14067 if (!isa<SCEVCouldNotCompute>(Val: SymbolicBTC)) {
14068 OS << "symbolic max backedge-taken count is ";
14069 PrintSCEVWithTypeHint(OS, S: SymbolicBTC);
14070 if (SE->isBackedgeTakenCountMaxOrZero(L))
14071 OS << ", actual taken count either this or zero.";
14072 } else {
14073 OS << "Unpredictable symbolic max backedge-taken count. ";
14074 }
14075 OS << "\n";
14076
14077 if (ExitingBlocks.size() > 1)
14078 for (BasicBlock *ExitingBlock : ExitingBlocks) {
14079 OS << " symbolic max exit count for " << ExitingBlock->getName() << ": ";
14080 auto *ExitBTC = SE->getExitCount(L, ExitingBlock,
14081 Kind: ScalarEvolution::SymbolicMaximum);
14082 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14083 if (isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14084 // Retry with predicates.
14085 SmallVector<const SCEVPredicate *> Predicates;
14086 ExitBTC = SE->getPredicatedExitCount(L, ExitingBlock, Predicates: &Predicates,
14087 Kind: ScalarEvolution::SymbolicMaximum);
14088 if (!isa<SCEVCouldNotCompute>(Val: ExitBTC)) {
14089 OS << "\n predicated symbolic max exit count for "
14090 << ExitingBlock->getName() << ": ";
14091 PrintSCEVWithTypeHint(OS, S: ExitBTC);
14092 OS << "\n Predicates:\n";
14093 for (const auto *P : Predicates)
14094 P->print(OS, Depth: 4);
14095 }
14096 }
14097 OS << "\n";
14098 }
14099
14100 SmallVector<const SCEVPredicate *, 4> Preds;
14101 auto *PBT = SE->getPredicatedBackedgeTakenCount(L, Preds);
14102 if (PBT != BTC) {
14103 OS << "Loop ";
14104 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14105 OS << ": ";
14106 if (!isa<SCEVCouldNotCompute>(Val: PBT)) {
14107 OS << "Predicated backedge-taken count is ";
14108 PrintSCEVWithTypeHint(OS, S: PBT);
14109 } else
14110 OS << "Unpredictable predicated backedge-taken count.";
14111 OS << "\n";
14112 OS << " Predicates:\n";
14113 for (const auto *P : Preds)
14114 P->print(OS, Depth: 4);
14115 }
14116 Preds.clear();
14117
14118 auto *PredConstantMax =
14119 SE->getPredicatedConstantMaxBackedgeTakenCount(L, Preds);
14120 if (PredConstantMax != ConstantBTC) {
14121 OS << "Loop ";
14122 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14123 OS << ": ";
14124 if (!isa<SCEVCouldNotCompute>(Val: PredConstantMax)) {
14125 OS << "Predicated constant max backedge-taken count is ";
14126 PrintSCEVWithTypeHint(OS, S: PredConstantMax);
14127 } else
14128 OS << "Unpredictable predicated constant max backedge-taken count.";
14129 OS << "\n";
14130 OS << " Predicates:\n";
14131 for (const auto *P : Preds)
14132 P->print(OS, Depth: 4);
14133 }
14134 Preds.clear();
14135
14136 auto *PredSymbolicMax =
14137 SE->getPredicatedSymbolicMaxBackedgeTakenCount(L, Preds);
14138 if (SymbolicBTC != PredSymbolicMax) {
14139 OS << "Loop ";
14140 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14141 OS << ": ";
14142 if (!isa<SCEVCouldNotCompute>(Val: PredSymbolicMax)) {
14143 OS << "Predicated symbolic max backedge-taken count is ";
14144 PrintSCEVWithTypeHint(OS, S: PredSymbolicMax);
14145 } else
14146 OS << "Unpredictable predicated symbolic max backedge-taken count.";
14147 OS << "\n";
14148 OS << " Predicates:\n";
14149 for (const auto *P : Preds)
14150 P->print(OS, Depth: 4);
14151 }
14152
14153 if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
14154 OS << "Loop ";
14155 L->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14156 OS << ": ";
14157 OS << "Trip multiple is " << SE->getSmallConstantTripMultiple(L) << "\n";
14158 }
14159}
14160
14161namespace llvm {
14162// Note: these overloaded operators need to be in the llvm namespace for them
14163// to be resolved correctly. If we put them outside the llvm namespace, the
14164//
14165// OS << ": " << SE.getLoopDisposition(SV, InnerL);
14166//
14167// code below "breaks" and start printing raw enum values as opposed to the
14168// string values.
14169static raw_ostream &operator<<(raw_ostream &OS,
14170 ScalarEvolution::LoopDisposition LD) {
14171 switch (LD) {
14172 case ScalarEvolution::LoopVariant:
14173 OS << "Variant";
14174 break;
14175 case ScalarEvolution::LoopInvariant:
14176 OS << "Invariant";
14177 break;
14178 case ScalarEvolution::LoopUniform:
14179 OS << "Uniform";
14180 break;
14181 case ScalarEvolution::LoopComputable:
14182 OS << "Computable";
14183 break;
14184 }
14185 return OS;
14186}
14187
14188static raw_ostream &operator<<(raw_ostream &OS,
14189 llvm::ScalarEvolution::BlockDisposition BD) {
14190 switch (BD) {
14191 case ScalarEvolution::DoesNotDominateBlock:
14192 OS << "DoesNotDominate";
14193 break;
14194 case ScalarEvolution::DominatesBlock:
14195 OS << "Dominates";
14196 break;
14197 case ScalarEvolution::ProperlyDominatesBlock:
14198 OS << "ProperlyDominates";
14199 break;
14200 }
14201 return OS;
14202}
14203} // namespace llvm
14204
14205void ScalarEvolution::print(raw_ostream &OS) const {
14206 // ScalarEvolution's implementation of the print method is to print
14207 // out SCEV values of all instructions that are interesting. Doing
14208 // this potentially causes it to create new SCEV objects though,
14209 // which technically conflicts with the const qualifier. This isn't
14210 // observable from outside the class though, so casting away the
14211 // const isn't dangerous.
14212 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14213
14214 if (Opts.scalar_evolution_classify_expressions) {
14215 OS << "Classifying expressions for: ";
14216 F.printAsOperand(O&: OS, /*PrintType=*/false);
14217 OS << "\n";
14218 for (Instruction &I : instructions(F))
14219 if (isSCEVable(Ty: I.getType()) && !isa<CmpInst>(Val: I)) {
14220 OS << I << '\n';
14221 OS << " --> ";
14222 const SCEV *SV = SE.getSCEV(V: &I);
14223 SV->print(OS);
14224 if (!isa<SCEVCouldNotCompute>(Val: SV)) {
14225 OS << " U: ";
14226 SE.getUnsignedRange(S: SV).print(OS);
14227 OS << " S: ";
14228 SE.getSignedRange(S: SV).print(OS);
14229 }
14230
14231 const Loop *L = LI.getLoopFor(BB: I.getParent());
14232
14233 SCEVUse AtUse = SE.getSCEVAtScope(V: SV, L);
14234 if (AtUse != SV) {
14235 OS << " --> ";
14236 OS << AtUse;
14237 if (!isa<SCEVCouldNotCompute>(Val: AtUse)) {
14238 OS << " U: ";
14239 SE.getUnsignedRange(S: AtUse).print(OS);
14240 OS << " S: ";
14241 SE.getSignedRange(S: AtUse).print(OS);
14242 }
14243 }
14244
14245 if (L) {
14246 OS << "\t\t" "Exits: ";
14247 SCEVUse ExitValue = SE.getSCEVAtScope(V: SV, L: L->getParentLoop());
14248 if (!SE.isLoopInvariant(S: ExitValue, L)) {
14249 OS << "<<Unknown>>";
14250 } else {
14251 OS << ExitValue;
14252 }
14253
14254 ListSeparator LS(", ", "\t\tLoopDispositions: { ");
14255 for (const auto *Iter = L; Iter; Iter = Iter->getParentLoop()) {
14256 OS << LS;
14257 Iter->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14258 OS << ": " << SE.getLoopDisposition(S: SV, L: Iter);
14259 }
14260
14261 for (const auto *InnerL : depth_first(G: L)) {
14262 if (InnerL == L)
14263 continue;
14264 OS << LS;
14265 InnerL->getHeader()->printAsOperand(O&: OS, /*PrintType=*/false);
14266 OS << ": " << SE.getLoopDisposition(S: SV, L: InnerL);
14267 }
14268
14269 OS << " }";
14270 }
14271
14272 OS << "\n";
14273 }
14274 }
14275
14276 OS << "Determining loop execution counts for: ";
14277 F.printAsOperand(O&: OS, /*PrintType=*/false);
14278 OS << "\n";
14279 for (Loop *I : LI)
14280 PrintLoopInfo(OS, SE: &SE, L: I);
14281}
14282
14283ScalarEvolution::LoopDisposition
14284ScalarEvolution::getLoopDisposition(const SCEV *S, const Loop *L) {
14285 if (isa<SCEVConstant>(Val: S))
14286 return LoopInvariant;
14287
14288 auto &Values = LoopDispositions[S];
14289 for (auto &V : Values) {
14290 if (V.getPointer() == L)
14291 return V.getInt();
14292 }
14293 Values.emplace_back(Args&: L, Args: LoopVariant);
14294 LoopDisposition D = computeLoopDisposition(S, L);
14295 auto &Values2 = LoopDispositions[S];
14296 for (auto &V : llvm::reverse(C&: Values2)) {
14297 if (V.getPointer() == L) {
14298 V.setInt(D);
14299 break;
14300 }
14301 }
14302 return D;
14303}
14304
14305ScalarEvolution::LoopDisposition
14306ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) {
14307 switch (S->getSCEVType()) {
14308 case scConstant:
14309 case scVScale:
14310 return LoopInvariant;
14311 case scAddRecExpr: {
14312 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14313
14314 // If L is the addrec's loop, it's computable.
14315 if (AR->getLoop() == L)
14316 return LoopComputable;
14317
14318 // Add recurrences are never invariant in the function-body (null loop).
14319 if (!L)
14320 return LoopVariant;
14321
14322 // Everything that is not defined at loop entry is variant.
14323 if (DT.dominates(A: L->getHeader(), B: AR->getLoop()->getHeader())) {
14324 if (L->contains(L: AR->getLoop()) &&
14325 llvm::all_of(Range: AR->operands(),
14326 P: [&](const SCEV *Op) { return isLoopUniform(S: Op, L); }))
14327 return LoopUniform;
14328
14329 return LoopVariant;
14330 }
14331 assert(!L->contains(AR->getLoop()) && "Containing loop's header does not"
14332 " dominate the contained loop's header?");
14333
14334 // This recurrence is invariant w.r.t. L if AR's loop contains L.
14335 if (AR->getLoop()->contains(L))
14336 return LoopInvariant;
14337
14338 // This recurrence is variant w.r.t. L if any of its operands
14339 // are variant.
14340 for (SCEVUse Op : AR->operands())
14341 if (!isLoopInvariant(S: Op, L))
14342 return LoopVariant;
14343
14344 // Otherwise it's loop-invariant.
14345 return LoopInvariant;
14346 }
14347 case scTruncate:
14348 case scZeroExtend:
14349 case scSignExtend:
14350 case scPtrToAddr:
14351 case scAddExpr:
14352 case scMulExpr:
14353 case scUDivExpr:
14354 case scUMaxExpr:
14355 case scSMaxExpr:
14356 case scUMinExpr:
14357 case scSMinExpr:
14358 case scSequentialUMinExpr: {
14359 bool HasVarying = false;
14360 bool HasUniform = false;
14361 for (SCEVUse Op : S->operands()) {
14362 LoopDisposition D = getLoopDisposition(S: Op, L);
14363 if (D == LoopVariant)
14364 return LoopVariant;
14365 if (D == LoopComputable)
14366 HasVarying = true;
14367 if (D == LoopUniform)
14368 HasUniform = true;
14369 }
14370 return HasVarying ? (HasUniform ? LoopVariant : LoopComputable)
14371 : (HasUniform ? LoopUniform : LoopInvariant);
14372 }
14373 case scUnknown:
14374 // All non-instruction values are loop invariant. All instructions are loop
14375 // invariant if they are not contained in the specified loop.
14376 // Instructions are never considered invariant in the function body
14377 // (null loop) because they are defined within the "loop".
14378 if (auto *I = dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue()))
14379 return (L && !L->contains(Inst: I)) ? LoopInvariant : LoopVariant;
14380 return LoopInvariant;
14381 case scCouldNotCompute:
14382 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14383 }
14384 llvm_unreachable("Unknown SCEV kind!");
14385}
14386
14387bool ScalarEvolution::isLoopUniform(const SCEV *S, const Loop *L) {
14388 LoopDisposition D = getLoopDisposition(S, L);
14389 return D == LoopUniform || D == LoopInvariant;
14390}
14391
14392bool ScalarEvolution::isLoopInvariant(const SCEV *S, const Loop *L) {
14393 return getLoopDisposition(S, L) == LoopInvariant;
14394}
14395
14396bool ScalarEvolution::hasComputableLoopEvolution(const SCEV *S, const Loop *L) {
14397 return getLoopDisposition(S, L) == LoopComputable;
14398}
14399
14400ScalarEvolution::BlockDisposition
14401ScalarEvolution::getBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14402 if (isa<SCEVConstant>(Val: S))
14403 return ProperlyDominatesBlock;
14404
14405 auto &Values = BlockDispositions[S];
14406 for (auto &V : Values) {
14407 if (V.getPointer() == BB)
14408 return V.getInt();
14409 }
14410 Values.emplace_back(Args&: BB, Args: DoesNotDominateBlock);
14411 BlockDisposition D = computeBlockDisposition(S, BB);
14412 auto &Values2 = BlockDispositions[S];
14413 for (auto &V : llvm::reverse(C&: Values2)) {
14414 if (V.getPointer() == BB) {
14415 V.setInt(D);
14416 break;
14417 }
14418 }
14419 return D;
14420}
14421
14422ScalarEvolution::BlockDisposition
14423ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) {
14424 switch (S->getSCEVType()) {
14425 case scConstant:
14426 case scVScale:
14427 return ProperlyDominatesBlock;
14428 case scAddRecExpr: {
14429 // This uses a "dominates" query instead of "properly dominates" query
14430 // to test for proper dominance too, because the instruction which
14431 // produces the addrec's value is a PHI, and a PHI effectively properly
14432 // dominates its entire containing block.
14433 const SCEVAddRecExpr *AR = cast<SCEVAddRecExpr>(Val: S);
14434 if (!DT.dominates(A: AR->getLoop()->getHeader(), B: BB))
14435 return DoesNotDominateBlock;
14436
14437 // Fall through into SCEVNAryExpr handling.
14438 [[fallthrough]];
14439 }
14440 case scTruncate:
14441 case scZeroExtend:
14442 case scSignExtend:
14443 case scPtrToAddr:
14444 case scAddExpr:
14445 case scMulExpr:
14446 case scUDivExpr:
14447 case scUMaxExpr:
14448 case scSMaxExpr:
14449 case scUMinExpr:
14450 case scSMinExpr:
14451 case scSequentialUMinExpr: {
14452 bool Proper = true;
14453 for (const SCEV *NAryOp : S->operands()) {
14454 BlockDisposition D = getBlockDisposition(S: NAryOp, BB);
14455 if (D == DoesNotDominateBlock)
14456 return DoesNotDominateBlock;
14457 if (D == DominatesBlock)
14458 Proper = false;
14459 }
14460 return Proper ? ProperlyDominatesBlock : DominatesBlock;
14461 }
14462 case scUnknown:
14463 if (Instruction *I =
14464 dyn_cast<Instruction>(Val: cast<SCEVUnknown>(Val: S)->getValue())) {
14465 if (I->getParent() == BB)
14466 return DominatesBlock;
14467 if (DT.properlyDominates(A: I->getParent(), B: BB))
14468 return ProperlyDominatesBlock;
14469 return DoesNotDominateBlock;
14470 }
14471 return ProperlyDominatesBlock;
14472 case scCouldNotCompute:
14473 llvm_unreachable("Attempt to use a SCEVCouldNotCompute object!");
14474 }
14475 llvm_unreachable("Unknown SCEV kind!");
14476}
14477
14478bool ScalarEvolution::dominates(const SCEV *S, const BasicBlock *BB) {
14479 return getBlockDisposition(S, BB) >= DominatesBlock;
14480}
14481
14482bool ScalarEvolution::properlyDominates(const SCEV *S, const BasicBlock *BB) {
14483 return getBlockDisposition(S, BB) == ProperlyDominatesBlock;
14484}
14485
14486void ScalarEvolution::forgetBackedgeTakenCounts(const Loop *L,
14487 bool Predicated) {
14488 auto &BECounts =
14489 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14490 auto It = BECounts.find(Val: L);
14491 if (It != BECounts.end()) {
14492 for (const ExitNotTakenInfo &ENT : It->second.ExitNotTaken) {
14493 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14494 if (!isa<SCEVConstant>(Val: S)) {
14495 auto UserIt = BECountUsers.find(Val: S);
14496 assert(UserIt != BECountUsers.end());
14497 UserIt->second.erase(Ptr: {L, Predicated});
14498 }
14499 }
14500 }
14501 BECounts.erase(I: It);
14502 }
14503}
14504
14505void ScalarEvolution::forgetMemoizedResults(ArrayRef<SCEVUse> SCEVs) {
14506 SmallPtrSet<const SCEV *, 8> ToForget(llvm::from_range, SCEVs);
14507 SmallVector<SCEVUse, 8> Worklist(ToForget.begin(), ToForget.end());
14508
14509 while (!Worklist.empty()) {
14510 const SCEV *Curr = Worklist.pop_back_val();
14511 auto Users = SCEVUsers.find(Val: Curr);
14512 if (Users != SCEVUsers.end())
14513 for (const auto *User : Users->second)
14514 if (ToForget.insert(Ptr: User).second)
14515 Worklist.push_back(Elt: User);
14516 }
14517
14518 for (const auto *S : ToForget)
14519 forgetMemoizedResultsImpl(S);
14520
14521 PredicatedSCEVRewrites.remove_if(
14522 Pred: [&](const auto &Entry) { return ToForget.count(Ptr: Entry.first.first); });
14523}
14524
14525void ScalarEvolution::forgetMemoizedResultsImpl(const SCEV *S) {
14526 LoopDispositions.erase(Val: S);
14527 BlockDispositions.erase(Val: S);
14528 UnsignedRanges.erase(Val: S);
14529 SignedRanges.erase(Val: S);
14530 HasRecMap.erase(Val: S);
14531 ConstantMultipleCache.erase(Val: S);
14532
14533 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
14534 UnsignedWrapViaInductionTried.erase(Ptr: AR);
14535 SignedWrapViaInductionTried.erase(Ptr: AR);
14536 }
14537
14538 auto ExprIt = ExprValueMap.find(Val: S);
14539 if (ExprIt != ExprValueMap.end()) {
14540 for (Value *V : ExprIt->second) {
14541 auto ValueIt = ValueExprMap.find_as(Val: V);
14542 if (ValueIt != ValueExprMap.end())
14543 ValueExprMap.erase(I: ValueIt);
14544 }
14545 ExprValueMap.erase(I: ExprIt);
14546 }
14547
14548 auto ScopeIt = ValuesAtScopes.find(Val: S);
14549 if (ScopeIt != ValuesAtScopes.end()) {
14550 for (const auto &Pair : ScopeIt->second)
14551 if (!isa_and_nonnull<SCEVConstant>(Val: Pair.second))
14552 llvm::erase(C&: ValuesAtScopesUsers[Pair.second.getPointer()],
14553 V: std::make_pair(x: Pair.first, y&: S));
14554 ValuesAtScopes.erase(I: ScopeIt);
14555 }
14556
14557 auto ScopeUserIt = ValuesAtScopesUsers.find(Val: S);
14558 if (ScopeUserIt != ValuesAtScopesUsers.end()) {
14559 for (const auto &Pair : ScopeUserIt->second)
14560 // The recorded value at scope is a use of S, which may carry no-wrap
14561 // flags that are not part of this key.
14562 llvm::erase_if(C&: ValuesAtScopes[Pair.second], P: [&](const auto &LS) {
14563 return LS.first == Pair.first && LS.second.getPointer() == S;
14564 });
14565 ValuesAtScopesUsers.erase(I: ScopeUserIt);
14566 }
14567
14568 auto BEUsersIt = BECountUsers.find(Val: S);
14569 if (BEUsersIt != BECountUsers.end()) {
14570 // Work on a copy, as forgetBackedgeTakenCounts() will modify the original.
14571 auto Copy = BEUsersIt->second;
14572 for (const auto &Pair : Copy)
14573 forgetBackedgeTakenCounts(L: Pair.getPointer(), Predicated: Pair.getInt());
14574 BECountUsers.erase(I: BEUsersIt);
14575 }
14576
14577 auto FoldUser = FoldCacheUser.find(Val: S);
14578 if (FoldUser != FoldCacheUser.end()) {
14579 for (auto &KV : FoldUser->second)
14580 FoldCache.erase(Val: KV);
14581 FoldCacheUser.erase(I: FoldUser);
14582 }
14583}
14584
14585void
14586ScalarEvolution::getUsedLoops(const SCEV *S,
14587 SmallPtrSetImpl<const Loop *> &LoopsUsed) {
14588 struct FindUsedLoops {
14589 FindUsedLoops(SmallPtrSetImpl<const Loop *> &LoopsUsed)
14590 : LoopsUsed(LoopsUsed) {}
14591 SmallPtrSetImpl<const Loop *> &LoopsUsed;
14592 bool follow(const SCEV *S) {
14593 if (auto *AR = dyn_cast<SCEVAddRecExpr>(Val: S))
14594 LoopsUsed.insert(Ptr: AR->getLoop());
14595 return true;
14596 }
14597
14598 bool isDone() const { return false; }
14599 };
14600
14601 FindUsedLoops F(LoopsUsed);
14602 SCEVTraversal<FindUsedLoops>(F).visitAll(Root: S);
14603}
14604
14605void ScalarEvolution::getReachableBlocks(
14606 SmallPtrSetImpl<BasicBlock *> &Reachable, Function &F) {
14607 SmallVector<BasicBlock *> Worklist;
14608 Worklist.push_back(Elt: &F.getEntryBlock());
14609 while (!Worklist.empty()) {
14610 BasicBlock *BB = Worklist.pop_back_val();
14611 if (!Reachable.insert(Ptr: BB).second)
14612 continue;
14613
14614 Value *Cond;
14615 BasicBlock *TrueBB, *FalseBB;
14616 if (match(V: BB->getTerminator(), P: m_Br(C: m_Value(V&: Cond), T: m_BasicBlock(V&: TrueBB),
14617 F: m_BasicBlock(V&: FalseBB)))) {
14618 if (auto *C = dyn_cast<ConstantInt>(Val: Cond)) {
14619 Worklist.push_back(Elt: C->isOne() ? TrueBB : FalseBB);
14620 continue;
14621 }
14622
14623 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
14624 const SCEV *L = getSCEV(V: Cmp->getOperand(i_nocapture: 0));
14625 const SCEV *R = getSCEV(V: Cmp->getOperand(i_nocapture: 1));
14626 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getCmpPredicate(), LHS: L, RHS: R)) {
14627 Worklist.push_back(Elt: TrueBB);
14628 continue;
14629 }
14630 if (isKnownPredicateViaConstantRanges(Pred: Cmp->getInverseCmpPredicate(), LHS: L,
14631 RHS: R)) {
14632 Worklist.push_back(Elt: FalseBB);
14633 continue;
14634 }
14635 }
14636 }
14637
14638 append_range(C&: Worklist, R: successors(BB));
14639 }
14640}
14641
14642void ScalarEvolution::verify() const {
14643 ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
14644 ScalarEvolution SE2(F, TLI, AC, DT, LI);
14645
14646 SmallVector<Loop *, 8> LoopStack(LI.begin(), LI.end());
14647
14648 // Map's SCEV expressions from one ScalarEvolution "universe" to another.
14649 struct SCEVMapper : public SCEVRewriteVisitor<SCEVMapper> {
14650 SCEVMapper(ScalarEvolution &SE) : SCEVRewriteVisitor<SCEVMapper>(SE) {}
14651
14652 const SCEV *visitConstant(const SCEVConstant *Constant) {
14653 return SE.getConstant(Val: Constant->getAPInt());
14654 }
14655
14656 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
14657 return SE.getUnknown(V: Expr->getValue());
14658 }
14659
14660 const SCEV *visitCouldNotCompute(const SCEVCouldNotCompute *Expr) {
14661 return SE.getCouldNotCompute();
14662 }
14663 };
14664
14665 SCEVMapper SCM(SE2);
14666 SmallPtrSet<BasicBlock *, 16> ReachableBlocks;
14667 SE2.getReachableBlocks(Reachable&: ReachableBlocks, F);
14668
14669 auto GetDelta = [&](const SCEV *Old, const SCEV *New) -> const SCEV * {
14670 if (containsUndefs(S: Old) || containsUndefs(S: New)) {
14671 // SCEV treats "undef" as an unknown but consistent value (i.e. it does
14672 // not propagate undef aggressively). This means we can (and do) fail
14673 // verification in cases where a transform makes a value go from "undef"
14674 // to "undef+1" (say). The transform is fine, since in both cases the
14675 // result is "undef", but SCEV thinks the value increased by 1.
14676 return nullptr;
14677 }
14678
14679 // Unless -verify-scev-strict is set, we only compare constant deltas.
14680 const SCEV *Delta = SE2.getMinusSCEV(LHS: Old, RHS: New);
14681 if (!Opts.verify_scev_strict && !isa<SCEVConstant>(Val: Delta))
14682 return nullptr;
14683
14684 return Delta;
14685 };
14686
14687 while (!LoopStack.empty()) {
14688 auto *L = LoopStack.pop_back_val();
14689 llvm::append_range(C&: LoopStack, R&: *L);
14690
14691 // Only verify BECounts in reachable loops. For an unreachable loop,
14692 // any BECount is legal.
14693 if (!ReachableBlocks.contains(Ptr: L->getHeader()))
14694 continue;
14695
14696 // Only verify cached BECounts. Computing new BECounts may change the
14697 // results of subsequent SCEV uses.
14698 auto It = BackedgeTakenCounts.find(Val: L);
14699 if (It == BackedgeTakenCounts.end())
14700 continue;
14701
14702 auto *CurBECount =
14703 SCM.visit(S: It->second.getExact(L, SE: const_cast<ScalarEvolution *>(this)));
14704 auto *NewBECount = SE2.getBackedgeTakenCount(L);
14705
14706 if (CurBECount == SE2.getCouldNotCompute() ||
14707 NewBECount == SE2.getCouldNotCompute()) {
14708 // NB! This situation is legal, but is very suspicious -- whatever pass
14709 // change the loop to make a trip count go from could not compute to
14710 // computable or vice-versa *should have* invalidated SCEV. However, we
14711 // choose not to assert here (for now) since we don't want false
14712 // positives.
14713 continue;
14714 }
14715
14716 if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) >
14717 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14718 NewBECount = SE2.getZeroExtendExpr(Op: NewBECount, Ty: CurBECount->getType());
14719 else if (SE.getTypeSizeInBits(Ty: CurBECount->getType()) <
14720 SE.getTypeSizeInBits(Ty: NewBECount->getType()))
14721 CurBECount = SE2.getZeroExtendExpr(Op: CurBECount, Ty: NewBECount->getType());
14722
14723 const SCEV *Delta = GetDelta(CurBECount, NewBECount);
14724 if (Delta && !Delta->isZero()) {
14725 dbgs() << "Trip Count for " << *L << " Changed!\n";
14726 dbgs() << "Old: " << *CurBECount << "\n";
14727 dbgs() << "New: " << *NewBECount << "\n";
14728 dbgs() << "Delta: " << *Delta << "\n";
14729 std::abort();
14730 }
14731 }
14732
14733 // Collect all valid loops currently in LoopInfo.
14734 SmallPtrSet<Loop *, 32> ValidLoops;
14735 SmallVector<Loop *, 32> Worklist(LI.begin(), LI.end());
14736 while (!Worklist.empty()) {
14737 Loop *L = Worklist.pop_back_val();
14738 if (ValidLoops.insert(Ptr: L).second)
14739 Worklist.append(in_start: L->begin(), in_end: L->end());
14740 }
14741 for (const auto &KV : ValueExprMap) {
14742#ifndef NDEBUG
14743 // Check for SCEV expressions referencing invalid/deleted loops.
14744 if (auto *AR = dyn_cast<SCEVAddRecExpr>(KV.second)) {
14745 assert(ValidLoops.contains(AR->getLoop()) &&
14746 "AddRec references invalid loop");
14747 }
14748#endif
14749
14750 // Check that the value is also part of the reverse map.
14751 auto It = ExprValueMap.find(Val: KV.second);
14752 if (It == ExprValueMap.end() || !is_contained(Range: It->second, Element: KV.first)) {
14753 dbgs() << "Value " << *KV.first
14754 << " is in ValueExprMap but not in ExprValueMap\n";
14755 std::abort();
14756 }
14757
14758 if (auto *I = dyn_cast<Instruction>(Val: &*KV.first)) {
14759 if (!ReachableBlocks.contains(Ptr: I->getParent()))
14760 continue;
14761 const SCEV *OldSCEV = SCM.visit(S: KV.second);
14762 const SCEV *NewSCEV = SE2.getSCEV(V: I);
14763 const SCEV *Delta = GetDelta(OldSCEV, NewSCEV);
14764 if (Delta && !Delta->isZero()) {
14765 dbgs() << "SCEV for value " << *I << " changed!\n"
14766 << "Old: " << *OldSCEV << "\n"
14767 << "New: " << *NewSCEV << "\n"
14768 << "Delta: " << *Delta << "\n";
14769 std::abort();
14770 }
14771 }
14772 }
14773
14774 for (const auto &KV : ExprValueMap) {
14775 for (Value *V : KV.second) {
14776 const SCEV *S = ValueExprMap.lookup(Val: V);
14777 if (!S) {
14778 dbgs() << "Value " << *V
14779 << " is in ExprValueMap but not in ValueExprMap\n";
14780 std::abort();
14781 }
14782 if (S != KV.first) {
14783 dbgs() << "Value " << *V << " mapped to " << *S << " rather than "
14784 << *KV.first << "\n";
14785 std::abort();
14786 }
14787 }
14788 }
14789
14790 // Verify integrity of SCEV users.
14791 for (const auto &S : UniqueSCEVs) {
14792 for (SCEVUse Op : S.operands()) {
14793 // We do not store dependencies of constants.
14794 if (isa<SCEVConstant>(Val: Op))
14795 continue;
14796 auto It = SCEVUsers.find(Val: Op);
14797 if (It != SCEVUsers.end() && It->second.count(Ptr: &S))
14798 continue;
14799 dbgs() << "Use of operand " << *Op << " by user " << S
14800 << " is not being tracked!\n";
14801 std::abort();
14802 }
14803 }
14804
14805 // Verify integrity of ValuesAtScopes users.
14806 for (const auto &ValueAndVec : ValuesAtScopes) {
14807 const SCEV *Value = ValueAndVec.first;
14808 for (const auto &LoopAndValueAtScope : ValueAndVec.second) {
14809 const Loop *L = LoopAndValueAtScope.first;
14810 SCEVUse ValueAtScope = LoopAndValueAtScope.second;
14811 if (!isa<SCEVConstant>(Val: ValueAtScope)) {
14812 auto It = ValuesAtScopesUsers.find(Val: ValueAtScope.getPointer());
14813 if (It != ValuesAtScopesUsers.end() &&
14814 is_contained(Range: It->second, Element: std::make_pair(x&: L, y&: Value)))
14815 continue;
14816 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14817 << *ValueAtScope << " missing in ValuesAtScopesUsers\n";
14818 std::abort();
14819 }
14820 }
14821 }
14822
14823 for (const auto &ValueAtScopeAndVec : ValuesAtScopesUsers) {
14824 const SCEV *ValueAtScope = ValueAtScopeAndVec.first;
14825 for (const auto &LoopAndValue : ValueAtScopeAndVec.second) {
14826 const Loop *L = LoopAndValue.first;
14827 const SCEV *Value = LoopAndValue.second;
14828 assert(!isa<SCEVConstant>(Value));
14829 auto It = ValuesAtScopes.find(Val: Value);
14830 // The recorded value at scope may carry no-wrap flags that are not part
14831 // of the key it is recorded under.
14832 if (It != ValuesAtScopes.end() && any_of(Range: It->second, P: [&](const auto &LS) {
14833 return LS.first == L && LS.second.getPointer() == ValueAtScope;
14834 }))
14835 continue;
14836 dbgs() << "Value: " << *Value << ", Loop: " << *L << ", ValueAtScope: "
14837 << *ValueAtScope << " missing in ValuesAtScopes\n";
14838 std::abort();
14839 }
14840 }
14841
14842 // Verify integrity of BECountUsers.
14843 auto VerifyBECountUsers = [&](bool Predicated) {
14844 auto &BECounts =
14845 Predicated ? PredicatedBackedgeTakenCounts : BackedgeTakenCounts;
14846 for (const auto &LoopAndBEInfo : BECounts) {
14847 for (const ExitNotTakenInfo &ENT : LoopAndBEInfo.second.ExitNotTaken) {
14848 for (const SCEV *S : {ENT.ExactNotTaken, ENT.SymbolicMaxNotTaken}) {
14849 if (!isa<SCEVConstant>(Val: S)) {
14850 auto UserIt = BECountUsers.find(Val: S);
14851 if (UserIt != BECountUsers.end() &&
14852 UserIt->second.contains(Ptr: { LoopAndBEInfo.first, Predicated }))
14853 continue;
14854 dbgs() << "Value " << *S << " for loop " << *LoopAndBEInfo.first
14855 << " missing from BECountUsers\n";
14856 std::abort();
14857 }
14858 }
14859 }
14860 }
14861 };
14862 VerifyBECountUsers(/* Predicated */ false);
14863 VerifyBECountUsers(/* Predicated */ true);
14864
14865 // Verify intergity of loop disposition cache.
14866 for (auto &[S, Values] : LoopDispositions) {
14867 for (auto [Loop, CachedDisposition] : Values) {
14868 const auto RecomputedDisposition = SE2.getLoopDisposition(S, L: Loop);
14869 if (CachedDisposition != RecomputedDisposition) {
14870 dbgs() << "Cached disposition of " << *S << " for loop " << *Loop
14871 << " is incorrect: cached " << CachedDisposition << ", actual "
14872 << RecomputedDisposition << "\n";
14873 std::abort();
14874 }
14875 }
14876 }
14877
14878 // Verify integrity of the block disposition cache.
14879 for (auto &[S, Values] : BlockDispositions) {
14880 for (auto [BB, CachedDisposition] : Values) {
14881 const auto RecomputedDisposition = SE2.getBlockDisposition(S, BB);
14882 if (CachedDisposition != RecomputedDisposition) {
14883 dbgs() << "Cached disposition of " << *S << " for block %"
14884 << BB->getName() << " is incorrect: cached " << CachedDisposition
14885 << ", actual " << RecomputedDisposition << "\n";
14886 std::abort();
14887 }
14888 }
14889 }
14890
14891 // Verify FoldCache/FoldCacheUser caches.
14892 for (auto [FoldID, Expr] : FoldCache) {
14893 auto I = FoldCacheUser.find(Val: Expr);
14894 if (I == FoldCacheUser.end()) {
14895 dbgs() << "Missing entry in FoldCacheUser for cached expression " << *Expr
14896 << "!\n";
14897 std::abort();
14898 }
14899 if (!is_contained(Range: I->second, Element: FoldID)) {
14900 dbgs() << "Missing FoldID in cached users of " << *Expr << "!\n";
14901 std::abort();
14902 }
14903 }
14904 for (auto [Expr, IDs] : FoldCacheUser) {
14905 for (auto &FoldID : IDs) {
14906 const SCEV *S = FoldCache.lookup(Val: FoldID);
14907 if (!S) {
14908 dbgs() << "Missing entry in FoldCache for expression " << *Expr
14909 << "!\n";
14910 std::abort();
14911 }
14912 if (S != Expr) {
14913 dbgs() << "Entry in FoldCache doesn't match FoldCacheUser: " << *S
14914 << " != " << *Expr << "!\n";
14915 std::abort();
14916 }
14917 }
14918 }
14919
14920 // Verify that ConstantMultipleCache computations are correct. We check that
14921 // cached multiples and recomputed multiples are multiples of each other to
14922 // verify correctness. It is possible that a recomputed multiple is different
14923 // from the cached multiple due to strengthened no wrap flags or changes in
14924 // KnownBits computations.
14925 for (auto [S, Multiple] : ConstantMultipleCache) {
14926 APInt RecomputedMultiple = SE2.getConstantMultiple(S);
14927 if ((Multiple != 0 && RecomputedMultiple != 0 &&
14928 Multiple.urem(RHS: RecomputedMultiple) != 0 &&
14929 RecomputedMultiple.urem(RHS: Multiple) != 0)) {
14930 dbgs() << "Incorrect cached computation in ConstantMultipleCache for "
14931 << *S << " : Computed " << RecomputedMultiple
14932 << " but cache contains " << Multiple << "!\n";
14933 std::abort();
14934 }
14935 }
14936}
14937
14938bool ScalarEvolution::invalidate(
14939 Function &F, const PreservedAnalyses &PA,
14940 FunctionAnalysisManager::Invalidator &Inv) {
14941 // Invalidate the ScalarEvolution object whenever it isn't preserved or one
14942 // of its dependencies is invalidated.
14943 auto PAC = PA.getChecker<ScalarEvolutionAnalysis>();
14944 return !(PAC.preserved() || PAC.preservedSet<AllAnalysesOn<Function>>()) ||
14945 Inv.invalidate<AssumptionAnalysis>(IR&: F, PA) ||
14946 Inv.invalidate<DominatorTreeAnalysis>(IR&: F, PA) ||
14947 Inv.invalidate<LoopAnalysis>(IR&: F, PA);
14948}
14949
14950AnalysisKey ScalarEvolutionAnalysis::Key;
14951
14952ScalarEvolution ScalarEvolutionAnalysis::run(Function &F,
14953 FunctionAnalysisManager &AM) {
14954 auto &TLI = AM.getResult<TargetLibraryAnalysis>(IR&: F);
14955 auto &AC = AM.getResult<AssumptionAnalysis>(IR&: F);
14956 auto &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
14957 auto &LI = AM.getResult<LoopAnalysis>(IR&: F);
14958 return ScalarEvolution(F, TLI, AC, DT, LI);
14959}
14960
14961PreservedAnalyses
14962ScalarEvolutionVerifierPass::run(Function &F, FunctionAnalysisManager &AM) {
14963 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).verify();
14964 return PreservedAnalyses::all();
14965}
14966
14967PreservedAnalyses
14968ScalarEvolutionPrinterPass::run(Function &F, FunctionAnalysisManager &AM) {
14969 // For compatibility with opt's -analyze feature under legacy pass manager
14970 // which was not ported to NPM. This keeps tests using
14971 // update_analyze_test_checks.py working.
14972 OS << "Printing analysis 'Scalar Evolution Analysis' for function '"
14973 << F.getName() << "':\n";
14974 AM.getResult<ScalarEvolutionAnalysis>(IR&: F).print(OS);
14975 return PreservedAnalyses::all();
14976}
14977
14978INITIALIZE_PASS_BEGIN(ScalarEvolutionWrapperPass, "scalar-evolution",
14979 "Scalar Evolution Analysis", false, true)
14980INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
14981INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
14982INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
14983INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
14984INITIALIZE_PASS_END(ScalarEvolutionWrapperPass, "scalar-evolution",
14985 "Scalar Evolution Analysis", false, true)
14986
14987char ScalarEvolutionWrapperPass::ID = 0;
14988
14989ScalarEvolutionWrapperPass::ScalarEvolutionWrapperPass() : FunctionPass(ID) {}
14990
14991bool ScalarEvolutionWrapperPass::runOnFunction(Function &F) {
14992 SE.reset(p: new ScalarEvolution(
14993 F, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F),
14994 getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
14995 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
14996 getAnalysis<LoopInfoWrapperPass>().getLoopInfo()));
14997 return false;
14998}
14999
15000void ScalarEvolutionWrapperPass::releaseMemory() { SE.reset(); }
15001
15002void ScalarEvolutionWrapperPass::print(raw_ostream &OS, const Module *) const {
15003 SE->print(OS);
15004}
15005
15006void ScalarEvolutionWrapperPass::verifyAnalysis() const {
15007 if (!VerifySCEV)
15008 return;
15009
15010 SE->verify();
15011}
15012
15013void ScalarEvolutionWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const {
15014 AU.setPreservesAll();
15015 AU.addRequiredTransitive<AssumptionCacheTracker>();
15016 AU.addRequiredTransitive<LoopInfoWrapperPass>();
15017 AU.addRequiredTransitive<DominatorTreeWrapperPass>();
15018 AU.addRequiredTransitive<TargetLibraryInfoWrapperPass>();
15019}
15020
15021const SCEVPredicate *ScalarEvolution::getEqualPredicate(const SCEV *LHS,
15022 const SCEV *RHS) {
15023 return getComparePredicate(Pred: ICmpInst::ICMP_EQ, LHS, RHS);
15024}
15025
15026const SCEVPredicate *
15027ScalarEvolution::getComparePredicate(const ICmpInst::Predicate Pred,
15028 const SCEV *LHS, const SCEV *RHS) {
15029 FoldingSetNodeID ID;
15030 assert(LHS->getType() == RHS->getType() &&
15031 "Type mismatch between LHS and RHS");
15032 // Unique this node based on the arguments
15033 ID.AddInteger(I: SCEVPredicate::P_Compare);
15034 ID.AddInteger(I: Pred);
15035 ID.AddPointer(Ptr: LHS);
15036 ID.AddPointer(Ptr: RHS);
15037 FoldingSetInsertToken Token;
15038 if (const auto *S = UniquePreds.lookup(ID, Token))
15039 return S;
15040 SCEVComparePredicate *Eq = new (SCEVAllocator)
15041 SCEVComparePredicate(ID.Intern(Allocator&: SCEVAllocator), Pred, LHS, RHS);
15042 UniquePreds.insert(N: Eq, Token);
15043 return Eq;
15044}
15045
15046const SCEVPredicate *ScalarEvolution::getWrapPredicate(
15047 const SCEVAddRecExpr *AR,
15048 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15049 FoldingSetNodeID ID;
15050 // Unique this node based on the arguments
15051 ID.AddInteger(I: SCEVPredicate::P_Wrap);
15052 ID.AddPointer(Ptr: AR);
15053 ID.AddInteger(I: AddedFlags);
15054 FoldingSetInsertToken Token;
15055 if (const auto *S = UniquePreds.lookup(ID, Token))
15056 return S;
15057 auto *OF = new (SCEVAllocator)
15058 SCEVWrapPredicate(ID.Intern(Allocator&: SCEVAllocator), AR, AddedFlags);
15059 UniquePreds.insert(N: OF, Token);
15060 return OF;
15061}
15062
15063namespace {
15064
15065class SCEVPredicateRewriter : public SCEVRewriteVisitor<SCEVPredicateRewriter> {
15066public:
15067
15068 /// Rewrites \p S in the context of a loop L and the SCEV predication
15069 /// infrastructure.
15070 ///
15071 /// If \p Pred is non-null, the SCEV expression is rewritten to respect the
15072 /// equivalences present in \p Pred.
15073 ///
15074 /// If \p NewPreds is non-null, rewrite is free to add further predicates to
15075 /// \p NewPreds such that the result will be an AddRecExpr.
15076 static const SCEV *rewrite(const SCEV *S, const Loop *L, ScalarEvolution &SE,
15077 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15078 const SCEVPredicate *Pred) {
15079 SCEVPredicateRewriter Rewriter(L, SE, NewPreds, Pred);
15080 return Rewriter.visit(S);
15081 }
15082
15083 const SCEV *visitUnknown(const SCEVUnknown *Expr) {
15084 if (Pred) {
15085 if (auto *U = dyn_cast<SCEVUnionPredicate>(Val: Pred)) {
15086 for (const auto *Pred : U->getPredicates())
15087 if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred))
15088 if (IPred->getLHS() == Expr &&
15089 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15090 return IPred->getRHS();
15091 } else if (const auto *IPred = dyn_cast<SCEVComparePredicate>(Val: Pred)) {
15092 if (IPred->getLHS() == Expr &&
15093 IPred->getPredicate() == ICmpInst::ICMP_EQ)
15094 return IPred->getRHS();
15095 }
15096 }
15097 return convertToAddRecWithPreds(Expr);
15098 }
15099
15100 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
15101 const SCEV *Operand = visit(S: Expr->getOperand());
15102 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15103 if (AR && AR->getLoop() == L && AR->isAffine()) {
15104 // This couldn't be folded because the operand didn't have the nuw
15105 // flag. Add the nusw flag as an assumption that we could make.
15106 const SCEV *Step = AR->getStepRecurrence(SE);
15107 Type *Ty = Expr->getType();
15108 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNUSW))
15109 return SE.getAddRecExpr(Start: SE.getZeroExtendExpr(Op: AR->getStart(), Ty),
15110 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15111 Flags: AR->getNoWrapFlags());
15112 }
15113 return SE.getZeroExtendExpr(Op: Operand, Ty: Expr->getType());
15114 }
15115
15116 const SCEV *visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
15117 const SCEV *Operand = visit(S: Expr->getOperand());
15118 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Operand);
15119 if (AR && AR->getLoop() == L && AR->isAffine()) {
15120 // This couldn't be folded because the operand didn't have the nsw
15121 // flag. Add the nssw flag as an assumption that we could make.
15122 const SCEV *Step = AR->getStepRecurrence(SE);
15123 Type *Ty = Expr->getType();
15124 if (addOverflowAssumption(AR, AddedFlags: SCEVWrapPredicate::IncrementNSSW))
15125 return SE.getAddRecExpr(Start: SE.getSignExtendExpr(Op: AR->getStart(), Ty),
15126 Step: SE.getSignExtendExpr(Op: Step, Ty), L,
15127 Flags: AR->getNoWrapFlags());
15128 }
15129 return SE.getSignExtendExpr(Op: Operand, Ty: Expr->getType());
15130 }
15131
15132private:
15133 explicit SCEVPredicateRewriter(
15134 const Loop *L, ScalarEvolution &SE,
15135 SmallVectorImpl<const SCEVPredicate *> *NewPreds,
15136 const SCEVPredicate *Pred)
15137 : SCEVRewriteVisitor(SE), NewPreds(NewPreds), Pred(Pred), L(L) {}
15138
15139 bool addOverflowAssumption(const SCEVPredicate *P) {
15140 if (!NewPreds) {
15141 // Check if we've already made this assumption.
15142 return Pred && Pred->implies(N: P, SE);
15143 }
15144 NewPreds->push_back(Elt: P);
15145 return true;
15146 }
15147
15148 bool addOverflowAssumption(const SCEVAddRecExpr *AR,
15149 SCEVWrapPredicate::IncrementWrapFlags AddedFlags) {
15150 auto *A = SE.getWrapPredicate(AR, AddedFlags);
15151 return addOverflowAssumption(P: A);
15152 }
15153
15154 // If \p Expr represents a PHINode, we try to see if it can be represented
15155 // as an AddRec, possibly under a predicate (PHISCEVPred). If it is possible
15156 // to add this predicate as a runtime overflow check, we return the AddRec.
15157 // If \p Expr does not meet these conditions (is not a PHI node, or we
15158 // couldn't create an AddRec for it, or couldn't add the predicate), we just
15159 // return \p Expr.
15160 const SCEV *convertToAddRecWithPreds(const SCEVUnknown *Expr) {
15161 if (!isa<PHINode>(Val: Expr->getValue()))
15162 return Expr;
15163 std::optional<
15164 std::pair<const SCEV *, SmallVector<const SCEVPredicate *, 3>>>
15165 PredicatedRewrite = SE.createAddRecFromPHIWithCasts(SymbolicPHI: Expr);
15166 if (!PredicatedRewrite)
15167 return Expr;
15168 for (const auto *P : PredicatedRewrite->second){
15169 // Wrap predicates from outer loops are not supported.
15170 if (auto *WP = dyn_cast<const SCEVWrapPredicate>(Val: P)) {
15171 if (L != WP->getExpr()->getLoop())
15172 return Expr;
15173 }
15174 if (!addOverflowAssumption(P))
15175 return Expr;
15176 }
15177 return PredicatedRewrite->first;
15178 }
15179
15180 SmallVectorImpl<const SCEVPredicate *> *NewPreds;
15181 const SCEVPredicate *Pred;
15182 const Loop *L;
15183};
15184
15185} // end anonymous namespace
15186
15187const SCEV *
15188ScalarEvolution::rewriteUsingPredicate(const SCEV *S, const Loop *L,
15189 const SCEVPredicate &Preds) {
15190 return SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: nullptr, Pred: &Preds);
15191}
15192
15193const SCEVAddRecExpr *ScalarEvolution::convertSCEVToAddRecWithPredicates(
15194 const SCEV *S, const Loop *L,
15195 SmallVectorImpl<const SCEVPredicate *> &Preds) {
15196 SmallVector<const SCEVPredicate *> TransformPreds;
15197 S = SCEVPredicateRewriter::rewrite(S, L, SE&: *this, NewPreds: &TransformPreds, Pred: nullptr);
15198 auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: S);
15199
15200 if (!AddRec)
15201 return nullptr;
15202
15203 // Check if any of the transformed predicates is known to be false. In that
15204 // case, it doesn't make sense to convert to a predicated AddRec, as the
15205 // versioned loop will never execute.
15206 for (const SCEVPredicate *Pred : TransformPreds) {
15207 auto *WrapPred = dyn_cast<SCEVWrapPredicate>(Val: Pred);
15208 if (!WrapPred || WrapPred->getFlags() != SCEVWrapPredicate::IncrementNSSW)
15209 continue;
15210
15211 const SCEVAddRecExpr *AddRecToCheck = WrapPred->getExpr();
15212 const SCEV *ExitCount = getBackedgeTakenCount(L: AddRecToCheck->getLoop());
15213 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
15214 continue;
15215
15216 const SCEV *Step = AddRecToCheck->getStepRecurrence(SE&: *this);
15217 if (!Step->isOne())
15218 continue;
15219
15220 ExitCount = getTruncateOrSignExtend(V: ExitCount, Ty: Step->getType());
15221 const SCEV *Add = getAddExpr(LHS: AddRecToCheck->getStart(), RHS: ExitCount);
15222 if (isKnownPredicate(Pred: CmpInst::ICMP_SLT, LHS: Add, RHS: AddRecToCheck->getStart()))
15223 return nullptr;
15224 }
15225
15226 // Since the transformation was successful, we can now transfer the SCEV
15227 // predicates.
15228 Preds.append(in_start: TransformPreds.begin(), in_end: TransformPreds.end());
15229
15230 return AddRec;
15231}
15232
15233/// SCEV predicates
15234SCEVPredicate::SCEVPredicate(const FoldingSetNodeIDRef ID,
15235 SCEVPredicateKind Kind)
15236 : FastID(ID), Kind(Kind) {}
15237
15238SCEVComparePredicate::SCEVComparePredicate(const FoldingSetNodeIDRef ID,
15239 const ICmpInst::Predicate Pred,
15240 const SCEV *LHS, const SCEV *RHS)
15241 : SCEVPredicate(ID, P_Compare), Pred(Pred), LHS(LHS), RHS(RHS) {
15242 assert(LHS->getType() == RHS->getType() && "LHS and RHS types don't match");
15243 assert(LHS != RHS && "LHS and RHS are the same SCEV");
15244}
15245
15246bool SCEVComparePredicate::implies(const SCEVPredicate *N,
15247 ScalarEvolution &SE) const {
15248 const auto *Op = dyn_cast<SCEVComparePredicate>(Val: N);
15249
15250 if (!Op)
15251 return false;
15252
15253 if (Op->LHS != LHS || Op->RHS != RHS)
15254 return false;
15255
15256 return ICmpInst::isImpliedByMatchingCmp(Pred1: Pred, Pred2: Op->Pred) == true;
15257}
15258
15259bool SCEVComparePredicate::isAlwaysTrue() const { return false; }
15260
15261void SCEVComparePredicate::print(raw_ostream &OS, unsigned Depth) const {
15262 if (Pred == ICmpInst::ICMP_EQ)
15263 OS.indent(NumSpaces: Depth) << "Equal predicate: " << *LHS << " == " << *RHS << "\n";
15264 else
15265 OS.indent(NumSpaces: Depth) << "Compare predicate: " << *LHS << " " << Pred << ") "
15266 << *RHS << "\n";
15267
15268}
15269
15270SCEVWrapPredicate::SCEVWrapPredicate(const FoldingSetNodeIDRef ID,
15271 const SCEVAddRecExpr *AR,
15272 IncrementWrapFlags Flags)
15273 : SCEVPredicate(ID, P_Wrap), AR(AR), Flags(Flags) {}
15274
15275const SCEVAddRecExpr *SCEVWrapPredicate::getExpr() const { return AR; }
15276
15277bool SCEVWrapPredicate::implies(const SCEVPredicate *N,
15278 ScalarEvolution &SE) const {
15279 const auto *Op = dyn_cast<SCEVWrapPredicate>(Val: N);
15280 if (!Op || setFlags(Flags, OnFlags: Op->Flags) != Flags)
15281 return false;
15282
15283 if (Op->AR == AR)
15284 return true;
15285
15286 if (Flags != SCEVWrapPredicate::IncrementNSSW &&
15287 Flags != SCEVWrapPredicate::IncrementNUSW)
15288 return false;
15289
15290 const SCEV *Start = AR->getStart();
15291 const SCEV *OpStart = Op->AR->getStart();
15292 if (Start->getType()->isPointerTy() != OpStart->getType()->isPointerTy())
15293 return false;
15294
15295 // Reject pointers to different address spaces.
15296 if (Start->getType()->isPointerTy() && Start->getType() != OpStart->getType())
15297 return false;
15298
15299 // NUSW/NSSW on a wider-type AddRec does not imply the same on a
15300 // narrower-type AddRec.
15301 if (SE.getTypeSizeInBits(Ty: AR->getType()) >
15302 SE.getTypeSizeInBits(Ty: Op->AR->getType()))
15303 return false;
15304
15305 const SCEV *Step = AR->getStepRecurrence(SE);
15306 const SCEV *OpStep = Op->AR->getStepRecurrence(SE);
15307 if (!SE.isKnownPositive(S: Step) || !SE.isKnownPositive(S: OpStep))
15308 return false;
15309
15310 // If both steps are positive, this implies N, if N's start and step are
15311 // ULE/SLE (for NSUW/NSSW) than this'.
15312 Type *WiderTy = SE.getWiderType(T1: Step->getType(), T2: OpStep->getType());
15313 Step = SE.getNoopOrZeroExtend(V: Step, Ty: WiderTy);
15314 OpStep = SE.getNoopOrZeroExtend(V: OpStep, Ty: WiderTy);
15315
15316 bool IsNUW = Flags == SCEVWrapPredicate::IncrementNUSW;
15317 OpStart = IsNUW ? SE.getNoopOrZeroExtend(V: OpStart, Ty: WiderTy)
15318 : SE.getNoopOrSignExtend(V: OpStart, Ty: WiderTy);
15319 Start = IsNUW ? SE.getNoopOrZeroExtend(V: Start, Ty: WiderTy)
15320 : SE.getNoopOrSignExtend(V: Start, Ty: WiderTy);
15321 CmpInst::Predicate Pred = IsNUW ? CmpInst::ICMP_ULE : CmpInst::ICMP_SLE;
15322 return SE.isKnownPredicate(Pred, LHS: OpStep, RHS: Step) &&
15323 SE.isKnownPredicate(Pred, LHS: OpStart, RHS: Start);
15324}
15325
15326bool SCEVWrapPredicate::isAlwaysTrue() const {
15327 SCEVFlags ScevFlags = AR->getNoWrapFlags();
15328 IncrementWrapFlags IFlags = Flags;
15329
15330 if (ScalarEvolution::setFlags(Flags: ScevFlags, OnFlags: SCEV::FlagNSW) == ScevFlags)
15331 IFlags = clearFlags(Flags: IFlags, OffFlags: IncrementNSSW);
15332
15333 return IFlags == IncrementAnyWrap;
15334}
15335
15336void SCEVWrapPredicate::print(raw_ostream &OS, unsigned Depth) const {
15337 OS.indent(NumSpaces: Depth) << *getExpr() << " Added Flags: ";
15338 if (SCEVWrapPredicate::IncrementNUSW & getFlags())
15339 OS << "<nusw>";
15340 if (SCEVWrapPredicate::IncrementNSSW & getFlags())
15341 OS << "<nssw>";
15342 OS << "\n";
15343}
15344
15345/// Union predicates don't get cached so create a dummy set ID for it.
15346SCEVUnionPredicate::SCEVUnionPredicate(ArrayRef<const SCEVPredicate *> Preds,
15347 ScalarEvolution &SE)
15348 : SCEVPredicate(FoldingSetNodeIDRef(), P_Union) {
15349 for (const auto *P : Preds)
15350 add(N: P, SE);
15351}
15352
15353bool SCEVUnionPredicate::isAlwaysTrue() const {
15354 return all_of(Range: Preds,
15355 P: [](const SCEVPredicate *I) { return I->isAlwaysTrue(); });
15356}
15357
15358bool SCEVUnionPredicate::implies(const SCEVPredicate *N,
15359 ScalarEvolution &SE) const {
15360 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N))
15361 return all_of(Range: Set->Preds, P: [this, &SE](const SCEVPredicate *I) {
15362 return this->implies(N: I, SE);
15363 });
15364
15365 if (any_of(Range: Preds,
15366 P: [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); }))
15367 return true;
15368
15369 // A wrap predicate may be implied by a wrap predicate in Preds after applying
15370 // equal predicates.
15371 const auto *NWrap = dyn_cast<SCEVWrapPredicate>(Val: N);
15372 if (!NWrap)
15373 return false;
15374 const Loop *L = NWrap->getExpr()->getLoop();
15375 return any_of(Range: Preds, P: [&](const SCEVPredicate *I) {
15376 const auto *IWrap = dyn_cast<SCEVWrapPredicate>(Val: I);
15377 if (!IWrap)
15378 return false;
15379 const auto *RewrittenAR = dyn_cast<SCEVAddRecExpr>(
15380 Val: SE.rewriteUsingPredicate(S: IWrap->getExpr(), L, Preds: *this));
15381 return RewrittenAR &&
15382 SE.getWrapPredicate(AR: RewrittenAR, AddedFlags: IWrap->getFlags())->implies(N, SE);
15383 });
15384}
15385
15386void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const {
15387 for (const auto *Pred : Preds)
15388 Pred->print(OS, Depth);
15389}
15390
15391void SCEVUnionPredicate::add(const SCEVPredicate *N, ScalarEvolution &SE) {
15392 if (const auto *Set = dyn_cast<SCEVUnionPredicate>(Val: N)) {
15393 for (const auto *Pred : Set->Preds)
15394 add(N: Pred, SE);
15395 return;
15396 }
15397
15398 // Implication checks are quadratic in the number of predicates. Stop doing
15399 // them if there are many predicates, as they should be too expensive to use
15400 // anyway at that point.
15401 bool CheckImplies = Preds.size() < 16;
15402
15403 // Only add predicate if it is not already implied by this union predicate.
15404 if (CheckImplies && implies(N, SE))
15405 return;
15406
15407 // Build a new vector containing the current predicates, except the ones that
15408 // are implied by the new predicate N.
15409 SmallVector<const SCEVPredicate *> PrunedPreds;
15410 for (auto *P : Preds) {
15411 if (CheckImplies && N->implies(N: P, SE))
15412 continue;
15413 PrunedPreds.push_back(Elt: P);
15414 }
15415 Preds = std::move(PrunedPreds);
15416 Preds.push_back(Elt: N);
15417}
15418
15419PredicatedScalarEvolution::PredicatedScalarEvolution(ScalarEvolution &SE,
15420 Loop &L)
15421 : SE(SE), L(L) {
15422 SmallVector<const SCEVPredicate*, 4> Empty;
15423 Preds = std::make_unique<SCEVUnionPredicate>(args&: Empty, args&: SE);
15424}
15425
15426void ScalarEvolution::registerUser(const SCEV *User, ArrayRef<SCEVUse> Ops) {
15427 for (const SCEV *Op : Ops)
15428 // We do not expect that forgetting cached data for SCEVConstants will ever
15429 // open any prospects for sharpening or introduce any correctness issues,
15430 // so we don't bother storing their dependencies.
15431 if (!isa<SCEVConstant>(Val: Op))
15432 SCEVUsers[Op].insert(Ptr: User);
15433}
15434
15435const SCEV *PredicatedScalarEvolution::getSCEV(Value *V) {
15436 const SCEV *Expr = SE.getSCEV(V);
15437 return getPredicatedSCEV(Expr);
15438}
15439
15440const SCEV *PredicatedScalarEvolution::getPredicatedSCEV(const SCEV *Expr) {
15441 RewriteEntry &Entry = RewriteMap[Expr];
15442
15443 // If we already have an entry and the version matches, return it.
15444 if (Entry.second && Generation == Entry.first)
15445 return Entry.second;
15446
15447 // We found an entry but it's stale. Rewrite the stale entry
15448 // according to the current predicate.
15449 if (Entry.second)
15450 Expr = Entry.second;
15451
15452 const SCEV *NewSCEV = SE.rewriteUsingPredicate(S: Expr, L: &L, Preds: *Preds);
15453 Entry = {Generation, NewSCEV};
15454
15455 return NewSCEV;
15456}
15457
15458const SCEV *PredicatedScalarEvolution::getBackedgeTakenCount() {
15459 if (!BackedgeCount) {
15460 SmallVector<const SCEVPredicate *, 4> Preds;
15461 BackedgeCount = SE.getPredicatedBackedgeTakenCount(L: &L, Preds);
15462 for (const auto *P : Preds)
15463 addPredicate(Pred: *P);
15464 }
15465 return BackedgeCount;
15466}
15467
15468const SCEV *PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount() {
15469 if (!SymbolicMaxBackedgeCount) {
15470 SmallVector<const SCEVPredicate *, 4> Preds;
15471 SymbolicMaxBackedgeCount =
15472 SE.getPredicatedSymbolicMaxBackedgeTakenCount(L: &L, Preds);
15473 for (const auto *P : Preds)
15474 addPredicate(Pred: *P);
15475 }
15476 return SymbolicMaxBackedgeCount;
15477}
15478
15479unsigned PredicatedScalarEvolution::getSmallConstantMaxTripCount() {
15480 if (!SmallConstantMaxTripCount) {
15481 SmallVector<const SCEVPredicate *, 4> Preds;
15482 SmallConstantMaxTripCount = SE.getSmallConstantMaxTripCount(L: &L, Predicates: &Preds);
15483 for (const auto *P : Preds)
15484 addPredicate(Pred: *P);
15485 }
15486 return *SmallConstantMaxTripCount;
15487}
15488
15489void PredicatedScalarEvolution::addPredicate(const SCEVPredicate &Pred) {
15490 if (Preds->implies(N: &Pred, SE))
15491 return;
15492
15493 Preds = std::make_unique<SCEVUnionPredicate>(args: Preds->getUnionWith(N: &Pred, SE));
15494 updateGeneration();
15495}
15496
15497void PredicatedScalarEvolution::addPredicates(
15498 ArrayRef<const SCEVPredicate *> Preds) {
15499 for (const SCEVPredicate *P : Preds)
15500 addPredicate(Pred: *P);
15501}
15502
15503const SCEVPredicate &PredicatedScalarEvolution::getPredicate() const {
15504 return *Preds;
15505}
15506
15507void PredicatedScalarEvolution::updateGeneration() {
15508 // If the generation number wrapped recompute everything.
15509 if (++Generation == 0) {
15510 for (auto &II : RewriteMap) {
15511 const SCEV *Rewritten = II.second.second;
15512 II.second = {Generation, SE.rewriteUsingPredicate(S: Rewritten, L: &L, Preds: *Preds)};
15513 }
15514 }
15515}
15516
15517const SCEVAddRecExpr *PredicatedScalarEvolution::getAsAddRec(
15518 Value *V, SmallVectorImpl<const SCEVPredicate *> *ExtraPreds) {
15519 const SCEV *Expr = this->getSCEV(V);
15520 SmallVector<const SCEVPredicate *, 4> NewPreds;
15521 auto *New = SE.convertSCEVToAddRecWithPredicates(S: Expr, L: &L, Preds&: NewPreds);
15522
15523 if (!New)
15524 return nullptr;
15525
15526 if (ExtraPreds) {
15527 ExtraPreds->append(RHS: NewPreds);
15528 return New;
15529 }
15530
15531 addPredicates(Preds: NewPreds);
15532
15533 RewriteMap[SE.getSCEV(V)] = {Generation, New};
15534 return New;
15535}
15536
15537PredicatedScalarEvolution::PredicatedScalarEvolution(
15538 const PredicatedScalarEvolution &Init)
15539 : RewriteMap(Init.RewriteMap), SE(Init.SE), L(Init.L),
15540 Preds(std::make_unique<SCEVUnionPredicate>(args&: *Init.Preds)),
15541 Generation(Init.Generation), BackedgeCount(Init.BackedgeCount) {}
15542
15543void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const {
15544 // For each block.
15545 for (auto *BB : L.getBlocks())
15546 for (auto &I : *BB) {
15547 if (!SE.isSCEVable(Ty: I.getType()))
15548 continue;
15549
15550 auto *Expr = SE.getSCEV(V: &I);
15551 auto II = RewriteMap.find(Val: Expr);
15552
15553 if (II == RewriteMap.end())
15554 continue;
15555
15556 // Don't print things that are not interesting.
15557 if (II->second.second == Expr)
15558 continue;
15559
15560 OS.indent(NumSpaces: Depth) << "[PSE]" << I << ":\n";
15561 OS.indent(NumSpaces: Depth + 2) << *Expr << "\n";
15562 OS.indent(NumSpaces: Depth + 2) << "--> " << *II->second.second << "\n";
15563 }
15564}
15565
15566void ScalarEvolution::LoopGuards::collectPending() {
15567 const Loop *L = PendingLoop;
15568 if (!L)
15569 return;
15570 PendingLoop = nullptr;
15571 BasicBlock *Pred = L->getLoopPredecessor();
15572 if (!Pred)
15573 return;
15574 SmallPtrSet<const BasicBlock *, 8> VisitedBlocks;
15575 collectFromBlock(SE, Guards&: *this, Block: L->getHeader(), Pred, VisitedBlocks);
15576}
15577
15578void ScalarEvolution::LoopGuards::collectFromPHI(
15579 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15580 const PHINode &Phi, SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks,
15581 SmallDenseMap<const BasicBlock *, LoopGuards> &IncomingGuards,
15582 unsigned Depth) {
15583 if (!SE.isSCEVable(Ty: Phi.getType()))
15584 return;
15585
15586 using MinMaxPattern = std::pair<const SCEVConstant *, SCEVTypes>;
15587 auto GetMinMaxConst = [&](unsigned IncomingIdx) -> MinMaxPattern {
15588 const BasicBlock *InBlock = Phi.getIncomingBlock(i: IncomingIdx);
15589 if (!VisitedBlocks.insert(Ptr: InBlock).second)
15590 return {nullptr, scCouldNotCompute};
15591
15592 // Avoid analyzing unreachable blocks so that we don't get trapped
15593 // traversing cycles with ill-formed dominance or infinite cycles
15594 if (!SE.DT.isReachableFromEntry(A: InBlock))
15595 return {nullptr, scCouldNotCompute};
15596
15597 auto [G, Inserted] = IncomingGuards.try_emplace(Key: InBlock, Args: LoopGuards(SE));
15598 if (Inserted)
15599 collectFromBlock(SE, Guards&: G->second, Block: Phi.getParent(), Pred: InBlock, VisitedBlocks,
15600 Depth: Depth + 1);
15601 auto &RewriteMap = G->second.RewriteMap;
15602 if (RewriteMap.empty())
15603 return {nullptr, scCouldNotCompute};
15604 auto S = RewriteMap.find(Val: SE.getSCEV(V: Phi.getIncomingValue(i: IncomingIdx)));
15605 if (S == RewriteMap.end())
15606 return {nullptr, scCouldNotCompute};
15607 auto *SM = dyn_cast_if_present<SCEVMinMaxExpr>(Val: S->second);
15608 if (!SM)
15609 return {nullptr, scCouldNotCompute};
15610 if (const SCEVConstant *C0 = dyn_cast<SCEVConstant>(Val: SM->getOperand(i: 0)))
15611 return {C0, SM->getSCEVType()};
15612 return {nullptr, scCouldNotCompute};
15613 };
15614 auto MergeMinMaxConst = [](MinMaxPattern P1,
15615 MinMaxPattern P2) -> MinMaxPattern {
15616 auto [C1, T1] = P1;
15617 auto [C2, T2] = P2;
15618 if (!C1 || !C2 || T1 != T2)
15619 return {nullptr, scCouldNotCompute};
15620 switch (T1) {
15621 case scUMaxExpr:
15622 return {C1->getAPInt().ult(RHS: C2->getAPInt()) ? C1 : C2, T1};
15623 case scSMaxExpr:
15624 return {C1->getAPInt().slt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15625 case scUMinExpr:
15626 return {C1->getAPInt().ugt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15627 case scSMinExpr:
15628 return {C1->getAPInt().sgt(RHS: C2->getAPInt()) ? C1 : C2, T1};
15629 default:
15630 llvm_unreachable("Trying to merge non-MinMaxExpr SCEVs.");
15631 }
15632 };
15633 auto P = GetMinMaxConst(0);
15634 for (unsigned int In = 1; In < Phi.getNumIncomingValues(); In++) {
15635 if (!P.first)
15636 break;
15637 P = MergeMinMaxConst(P, GetMinMaxConst(In));
15638 }
15639 if (P.first) {
15640 const SCEV *LHS = SE.getSCEV(V: const_cast<PHINode *>(&Phi));
15641 SmallVector<SCEVUse, 2> Ops({P.first, LHS});
15642 const SCEV *RHS = SE.getMinMaxExpr(Kind: P.second, Ops);
15643 Guards.RewriteMap.insert(KV: {LHS, RHS});
15644 }
15645}
15646
15647// Return a new SCEV that modifies \p Expr to the closest number divides by
15648// \p Divisor and less or equal than Expr. For now, only handle constant
15649// Expr.
15650static const SCEV *getPreviousSCEVDivisibleByDivisor(const SCEV *Expr,
15651 const APInt &DivisorVal,
15652 ScalarEvolution &SE) {
15653 const APInt *ExprVal;
15654 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15655 DivisorVal.isNonPositive())
15656 return Expr;
15657 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15658 // return the SCEV: Expr - Expr % Divisor
15659 return SE.getConstant(Val: *ExprVal - Rem);
15660}
15661
15662// Return a new SCEV that modifies \p Expr to the closest number divides by
15663// \p Divisor and greater or equal than Expr. For now, only handle constant
15664// Expr.
15665static const SCEV *getNextSCEVDivisibleByDivisor(const SCEV *Expr,
15666 const APInt &DivisorVal,
15667 ScalarEvolution &SE) {
15668 const APInt *ExprVal;
15669 if (!match(S: Expr, P: m_scev_APInt(C&: ExprVal)) || ExprVal->isNegative() ||
15670 DivisorVal.isNonPositive())
15671 return Expr;
15672 APInt Rem = ExprVal->urem(RHS: DivisorVal);
15673 if (Rem.isZero())
15674 return Expr;
15675 // return the SCEV: Expr + Divisor - Expr % Divisor
15676 return SE.getConstant(Val: *ExprVal + DivisorVal - Rem);
15677}
15678
15679static bool collectDivisibilityInformation(
15680 ICmpInst::Predicate Predicate, const SCEV *LHS, const SCEV *RHS,
15681 DenseMap<const SCEV *, const SCEV *> &DivInfo,
15682 DenseMap<const SCEV *, APInt> &Multiples, ScalarEvolution &SE) {
15683 // If we have LHS == 0, check if LHS is computing a property of some unknown
15684 // SCEV %v which we can rewrite %v to express explicitly.
15685 if (Predicate != CmpInst::ICMP_EQ || !match(S: RHS, P: m_scev_Zero()))
15686 return false;
15687 // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to
15688 // explicitly express that.
15689 const SCEVUnknown *URemLHS = nullptr;
15690 const SCEV *URemRHS = nullptr;
15691 if (!match(S: LHS, P: m_scev_URem(LHS: m_SCEVUnknown(V&: URemLHS), RHS: m_SCEV(V&: URemRHS), SE)))
15692 return false;
15693
15694 const SCEV *Multiple =
15695 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: URemLHS, RHS: URemRHS), RHS: URemRHS);
15696 DivInfo[URemLHS] = Multiple;
15697 if (auto *C = dyn_cast<SCEVConstant>(Val: URemRHS))
15698 Multiples[URemLHS] = C->getAPInt();
15699 return true;
15700}
15701
15702// Check if the condition is a divisibility guard (A % B == 0).
15703static bool isDivisibilityGuard(const SCEV *LHS, const SCEV *RHS,
15704 ScalarEvolution &SE) {
15705 const SCEV *X, *Y;
15706 return match(S: LHS, P: m_scev_URem(LHS: m_SCEV(V&: X), RHS: m_SCEV(V&: Y), SE)) && RHS->isZero();
15707}
15708
15709// Apply divisibility by \p Divisor on MinMaxExpr with constant values,
15710// recursively. This is done by aligning up/down the constant value to the
15711// Divisor.
15712static const SCEV *applyDivisibilityOnMinMaxExpr(const SCEV *MinMaxExpr,
15713 const APInt &Divisor,
15714 ScalarEvolution &SE) {
15715 // Only handle min/max expressions of a non-negative constant and another
15716 // operand.
15717 const SCEVConstant *C;
15718 const SCEV *Op;
15719 if (!match(S: MinMaxExpr, P: m_scev_MinMax(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: Op))) ||
15720 C->getAPInt().isNegative())
15721 return MinMaxExpr;
15722 auto *DivisibleExpr = isa<SCEVSMinExpr, SCEVUMinExpr>(Val: MinMaxExpr)
15723 ? getPreviousSCEVDivisibleByDivisor(Expr: C, DivisorVal: Divisor, SE)
15724 : getNextSCEVDivisibleByDivisor(Expr: C, DivisorVal: Divisor, SE);
15725 SmallVector<SCEVUse> Ops = {applyDivisibilityOnMinMaxExpr(MinMaxExpr: Op, Divisor, SE),
15726 DivisibleExpr};
15727 return SE.getMinMaxExpr(Kind: MinMaxExpr->getSCEVType(), Ops);
15728}
15729
15730void ScalarEvolution::LoopGuards::collectFromBlock(
15731 ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards,
15732 const BasicBlock *Block, const BasicBlock *Pred,
15733 SmallPtrSetImpl<const BasicBlock *> &VisitedBlocks, unsigned Depth) {
15734
15735 assert(SE.DT.isReachableFromEntry(Block) && SE.DT.isReachableFromEntry(Pred));
15736
15737 SmallVector<SCEVUse> ExprsToRewrite;
15738 auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS,
15739 const SCEV *RHS,
15740 DenseMap<const SCEV *, const SCEV *> &RewriteMap,
15741 const LoopGuards &DivGuards) {
15742 // WARNING: It is generally unsound to apply any wrap flags to the proposed
15743 // replacement SCEV which isn't directly implied by the structure of that
15744 // SCEV. In particular, using contextual facts to imply flags is *NOT*
15745 // legal. See the scoping rules for flags in the header to understand why.
15746
15747 // Puts rewrite rule \p From -> \p To into the rewrite map. Also if \p From
15748 // and \p FromRewritten are the same (i.e. there has been no rewrite
15749 // registered for \p From), then puts this value in the list of rewritten
15750 // expressions.
15751 auto AddRewrite = [&](const SCEV *From, const SCEV *FromRewritten,
15752 const SCEV *To) {
15753 if (From == FromRewritten)
15754 ExprsToRewrite.push_back(Elt: From);
15755 RewriteMap[From] = To;
15756 };
15757
15758 // Checks whether \p S has already been rewritten. In that case returns the
15759 // existing rewrite because we want to chain further rewrites onto the
15760 // already rewritten value. Otherwise returns \p S.
15761 auto GetMaybeRewritten = [&](const SCEV *S) {
15762 return RewriteMap.lookup_or(Val: S, Default&: S);
15763 };
15764
15765 // Check for a condition of the form (-C1 + X < C2). InstCombine will
15766 // create this form when combining two checks of the form (X u< C2 + C1) and
15767 // (X >=u C1).
15768 auto MatchRangeCheckIdiom = [&](ICmpInst::Predicate Pred,
15769 const SCEV *MatchLHS,
15770 const SCEV *MatchRHS) {
15771 const SCEVConstant *C1;
15772 const SCEVUnknown *LHSUnknown;
15773 auto *C2 = dyn_cast<SCEVConstant>(Val: MatchRHS);
15774 if (!match(S: MatchLHS,
15775 P: m_scev_Add(Op0: m_SCEVConstant(V&: C1), Op1: m_SCEVUnknown(V&: LHSUnknown))) ||
15776 !C2)
15777 return false;
15778
15779 auto ExactRegion =
15780 ConstantRange::makeExactICmpRegion(Pred, Other: C2->getAPInt())
15781 .sub(Other: C1->getAPInt());
15782
15783 // Tighten the raw range with what we already know about LHSUnknown
15784 // from prior guards recorded in RewriteMap, or from SCEV's own range
15785 // analysis.
15786 const SCEV *RewrittenLHS = GetMaybeRewritten(LHSUnknown);
15787 ExactRegion = ExactRegion.intersectWith(CR: SE.getUnsignedRange(S: RewrittenLHS),
15788 Type: ConstantRange::Unsigned);
15789
15790 // Bail if the guard is inconsistent with prior facts, or if the range
15791 // is still not a monotonic non-wrapping interval after tightening.
15792 if (ExactRegion.isEmptySet() || ExactRegion.isWrappedSet() ||
15793 ExactRegion.isFullSet())
15794 return false;
15795
15796 const SCEV *RegionMin = SE.getConstant(Val: ExactRegion.getUnsignedMin());
15797 const SCEV *RegionMax = SE.getConstant(Val: ExactRegion.getUnsignedMax());
15798 const SCEV *ClampedLHS =
15799 SE.getUMaxExpr(LHS: RegionMin, RHS: SE.getUMinExpr(LHS: RewrittenLHS, RHS: RegionMax));
15800 AddRewrite(LHSUnknown, RewrittenLHS, ClampedLHS);
15801 return true;
15802 };
15803 if (MatchRangeCheckIdiom(Predicate, LHS, RHS))
15804 return;
15805
15806 // Do not apply information for constants or if RHS contains an AddRec.
15807 if (isa<SCEVConstant>(Val: LHS) || SE.containsAddRecurrence(S: RHS))
15808 return;
15809
15810 // If RHS is SCEVUnknown, make sure the information is applied to it.
15811 if (!isa<SCEVUnknown>(Val: LHS) && isa<SCEVUnknown>(Val: RHS)) {
15812 std::swap(a&: LHS, b&: RHS);
15813 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
15814 }
15815
15816 const SCEV *RewrittenLHS = GetMaybeRewritten(LHS);
15817 // Apply divisibility information when computing the constant multiple.
15818 const APInt &DividesBy =
15819 SE.getConstantMultiple(S: DivGuards.rewrite(Expr: RewrittenLHS));
15820
15821 // Collect rewrites for LHS and its transitive operands based on the
15822 // condition.
15823 // For min/max expressions, also apply the guard to its operands:
15824 // 'min(a, b) >= c' -> '(a >= c) and (b >= c)',
15825 // 'min(a, b) > c' -> '(a > c) and (b > c)',
15826 // 'max(a, b) <= c' -> '(a <= c) and (b <= c)',
15827 // 'max(a, b) < c' -> '(a < c) and (b < c)'.
15828
15829 // We cannot express strict predicates in SCEV, so instead we replace them
15830 // with non-strict ones against plus or minus one of RHS depending on the
15831 // predicate.
15832 const SCEV *One = SE.getOne(Ty: RHS->getType());
15833 switch (Predicate) {
15834 case CmpInst::ICMP_ULT:
15835 if (RHS->getType()->isPointerTy())
15836 return;
15837 RHS = SE.getUMaxExpr(LHS: RHS, RHS: One);
15838 [[fallthrough]];
15839 case CmpInst::ICMP_SLT: {
15840 RHS = SE.getMinusSCEV(LHS: RHS, RHS: One);
15841 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
15842 break;
15843 }
15844 case CmpInst::ICMP_UGT:
15845 case CmpInst::ICMP_SGT:
15846 RHS = SE.getAddExpr(LHS: RHS, RHS: One);
15847 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
15848 break;
15849 case CmpInst::ICMP_ULE:
15850 case CmpInst::ICMP_SLE:
15851 RHS = getPreviousSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
15852 break;
15853 case CmpInst::ICMP_UGE:
15854 case CmpInst::ICMP_SGE:
15855 RHS = getNextSCEVDivisibleByDivisor(Expr: RHS, DivisorVal: DividesBy, SE);
15856 break;
15857 default:
15858 break;
15859 }
15860
15861 SmallVector<SCEVUse, 16> Worklist(1, LHS);
15862 SmallPtrSet<const SCEV *, 16> Visited;
15863
15864 auto EnqueueOperands = [&Worklist](const SCEVNAryExpr *S) {
15865 append_range(C&: Worklist, R: S->operands());
15866 };
15867
15868 while (!Worklist.empty()) {
15869 const SCEV *From = Worklist.pop_back_val();
15870 if (isa<SCEVConstant>(Val: From))
15871 continue;
15872 if (!Visited.insert(Ptr: From).second)
15873 continue;
15874 const SCEV *FromRewritten = GetMaybeRewritten(From);
15875 const SCEV *To = nullptr;
15876
15877 switch (Predicate) {
15878 case CmpInst::ICMP_ULT:
15879 case CmpInst::ICMP_ULE:
15880 To = SE.getUMinExpr(LHS: FromRewritten, RHS);
15881 if (auto *UMax = dyn_cast<SCEVUMaxExpr>(Val: FromRewritten))
15882 EnqueueOperands(UMax);
15883 break;
15884 case CmpInst::ICMP_SLT:
15885 case CmpInst::ICMP_SLE:
15886 To = SE.getSMinExpr(LHS: FromRewritten, RHS);
15887 if (auto *SMax = dyn_cast<SCEVSMaxExpr>(Val: FromRewritten))
15888 EnqueueOperands(SMax);
15889 break;
15890 case CmpInst::ICMP_UGT:
15891 case CmpInst::ICMP_UGE:
15892 To = SE.getUMaxExpr(LHS: FromRewritten, RHS);
15893 if (auto *UMin = dyn_cast<SCEVUMinExpr>(Val: FromRewritten))
15894 EnqueueOperands(UMin);
15895 break;
15896 case CmpInst::ICMP_SGT:
15897 case CmpInst::ICMP_SGE: {
15898 // From s>= RHS s>= 0 implies From u>= RHS, so an earlier
15899 // umax(C, From) with C u<= RHS is redundant. Drop it, as the range of
15900 // the umax may be much less precise than the one of From.
15901 const APInt *C, *RHSC;
15902 if (match(S: RHS, P: m_scev_APInt(C&: RHSC)) && RHSC->isNonNegative() &&
15903 match(S: FromRewritten,
15904 P: m_scev_UMax(Op0: m_scev_APInt(C), Op1: m_scev_Specific(S: From))) &&
15905 C->ule(RHS: *RHSC))
15906 FromRewritten = From;
15907 To = SE.getSMaxExpr(LHS: FromRewritten, RHS);
15908 if (auto *SMin = dyn_cast<SCEVSMinExpr>(Val: FromRewritten))
15909 EnqueueOperands(SMin);
15910 break;
15911 }
15912 case CmpInst::ICMP_EQ:
15913 if (isa<SCEVConstant>(Val: RHS))
15914 To = RHS;
15915 break;
15916 case CmpInst::ICMP_NE:
15917 if (match(S: RHS, P: m_scev_Zero())) {
15918 const SCEV *OneAlignedUp =
15919 getNextSCEVDivisibleByDivisor(Expr: One, DivisorVal: DividesBy, SE);
15920 To = SE.getUMaxExpr(LHS: FromRewritten, RHS: OneAlignedUp);
15921 } else {
15922 // LHS != RHS can be rewritten as (LHS - RHS) = UMax(1, LHS - RHS),
15923 // but creating the subtraction eagerly is expensive. Track the
15924 // inequalities in a separate map, and materialize the rewrite lazily
15925 // when encountering a suitable subtraction while re-writing.
15926 if (LHS->getType()->isPointerTy()) {
15927 LHS = SE.getPtrToAddrExpr(Op: LHS);
15928 RHS = SE.getPtrToAddrExpr(Op: RHS);
15929 if (isa<SCEVCouldNotCompute>(Val: LHS) || isa<SCEVCouldNotCompute>(Val: RHS))
15930 break;
15931 }
15932 const SCEVConstant *C;
15933 const SCEV *A, *B;
15934 if (match(S: RHS, P: m_scev_Add(Op0: m_SCEVConstant(V&: C), Op1: m_SCEV(V&: A))) &&
15935 match(S: LHS, P: m_scev_Add(Op0: m_scev_Specific(S: C), Op1: m_SCEV(V&: B)))) {
15936 RHS = A;
15937 LHS = B;
15938 }
15939 if (LHS > RHS)
15940 std::swap(a&: LHS, b&: RHS);
15941 Guards.NotEqual.insert(V: {LHS, RHS});
15942 continue;
15943 }
15944 break;
15945 default:
15946 break;
15947 }
15948
15949 if (To)
15950 AddRewrite(From, FromRewritten, To);
15951 }
15952 };
15953
15954 SmallVector<PointerIntPair<Value *, 1, bool>> Terms;
15955 // First, collect information from assumptions dominating the loop.
15956 for (auto &AssumeVH : SE.AC.assumptions()) {
15957 if (!AssumeVH)
15958 continue;
15959 auto *AssumeI = cast<CallInst>(Val&: AssumeVH);
15960 if (!SE.DT.dominates(Def: AssumeI, BB: Block))
15961 continue;
15962 Terms.emplace_back(Args: AssumeI->getOperand(i_nocapture: 0), Args: true);
15963 }
15964
15965 // Second, collect information from llvm.experimental.guards dominating the loop.
15966 auto *GuardDecl = Intrinsic::getDeclarationIfExists(
15967 M: SE.F.getParent(), id: Intrinsic::experimental_guard);
15968 if (GuardDecl)
15969 for (const auto *GU : GuardDecl->users())
15970 if (const auto *Guard = dyn_cast<IntrinsicInst>(Val: GU))
15971 if (Guard->getFunction() == Block->getParent() &&
15972 SE.DT.dominates(Def: Guard, BB: Block))
15973 Terms.emplace_back(Args: Guard->getArgOperand(i: 0), Args: true);
15974
15975 // Third, collect conditions from dominating branches. Starting at the loop
15976 // predecessor, climb up the predecessor chain, as long as there are
15977 // predecessors that can be found that have unique successors leading to the
15978 // original header. If there is no such predecessor, continue at the
15979 // immediate dominator.
15980 // TODO: share this logic with isLoopEntryGuardedByCond.
15981 unsigned NumCollectedConditions = 0;
15982 unsigned NumDomTreeSteps = 0;
15983 VisitedBlocks.insert(Ptr: Block);
15984 std::pair<const BasicBlock *, const BasicBlock *> Pair(Pred, Block);
15985 for (;; Pair = SE.getPredecessorWithUniqueSuccessorForBB(BB: Pair.first)) {
15986 if (!Pair.first) {
15987 // At the first block without a unique predecessor, try to build
15988 // conditions for its phis by merging those of their incoming blocks.
15989 if (!NumDomTreeSteps &&
15990 Depth < SE.Opts.scalar_evolution_max_loop_guard_collection_depth) {
15991 SmallDenseMap<const BasicBlock *, LoopGuards> IncomingGuards;
15992 for (auto &Phi : Pair.second->phis())
15993 collectFromPHI(SE, Guards, Phi, VisitedBlocks, IncomingGuards, Depth);
15994 }
15995 const DomTreeNode *IDom = SE.DT.getNode(BB: Pair.second)->getIDom();
15996 if (!IDom ||
15997 ++NumDomTreeSteps > SE.Opts.scalar_evolution_max_guard_dom_tree_steps)
15998 break;
15999 Pair.first = IDom->getBlock();
16000 continue;
16001 }
16002 VisitedBlocks.insert(Ptr: Pair.second);
16003 const CondBrInst *LoopEntryPredicate =
16004 dyn_cast<CondBrInst>(Val: Pair.first->getTerminator());
16005 if (!LoopEntryPredicate)
16006 continue;
16007
16008 Terms.emplace_back(Args: LoopEntryPredicate->getCondition(),
16009 Args: LoopEntryPredicate->getSuccessor(i: 0) == Pair.second);
16010 NumCollectedConditions++;
16011
16012 // If we are recursively collecting guards stop after 2
16013 // conditions to limit compile-time impact for now.
16014 if (Depth > 0 && NumCollectedConditions == 2)
16015 break;
16016 }
16017
16018 // Now apply the information from the collected conditions to
16019 // Guards.RewriteMap. Conditions are processed in reverse order, so the
16020 // earliest conditions is processed first, except guards with divisibility
16021 // information, which are moved to the back. This ensures the SCEVs with the
16022 // shortest dependency chains are constructed first.
16023 SmallVector<std::tuple<CmpInst::Predicate, const SCEV *, const SCEV *>>
16024 GuardsToProcess;
16025 for (auto [Term, EnterIfTrue] : reverse(C&: Terms)) {
16026 SmallVector<Value *, 8> Worklist;
16027 SmallPtrSet<Value *, 8> Visited;
16028 Worklist.push_back(Elt: Term);
16029 while (!Worklist.empty()) {
16030 Value *Cond = Worklist.pop_back_val();
16031 if (!Visited.insert(Ptr: Cond).second)
16032 continue;
16033
16034 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond)) {
16035 auto Predicate =
16036 EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate();
16037 const auto *LHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 0));
16038 const auto *RHS = SE.getSCEV(V: Cmp->getOperand(i_nocapture: 1));
16039 // If LHS is a constant, apply information to the other expression.
16040 // TODO: If LHS is not a constant, check if using compareSCEVComplexity
16041 // can improve results.
16042 if (isa<SCEVConstant>(Val: LHS)) {
16043 std::swap(a&: LHS, b&: RHS);
16044 Predicate = CmpInst::getSwappedPredicate(pred: Predicate);
16045 }
16046 GuardsToProcess.emplace_back(Args&: Predicate, Args&: LHS, Args&: RHS);
16047 continue;
16048 }
16049
16050 Value *L, *R;
16051 if (EnterIfTrue ? match(V: Cond, P: m_LogicalAnd(L: m_Value(V&: L), R: m_Value(V&: R)))
16052 : match(V: Cond, P: m_LogicalOr(L: m_Value(V&: L), R: m_Value(V&: R)))) {
16053 Worklist.push_back(Elt: L);
16054 Worklist.push_back(Elt: R);
16055 }
16056 }
16057 }
16058
16059 // Process divisibility guards in reverse order to populate DivGuards early.
16060 DenseMap<const SCEV *, APInt> Multiples;
16061 LoopGuards DivGuards(SE);
16062 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess) {
16063 if (!isDivisibilityGuard(LHS, RHS, SE))
16064 continue;
16065 collectDivisibilityInformation(Predicate, LHS, RHS, DivInfo&: DivGuards.RewriteMap,
16066 Multiples, SE);
16067 }
16068
16069 for (const auto &[Predicate, LHS, RHS] : GuardsToProcess)
16070 CollectCondition(Predicate, LHS, RHS, Guards.RewriteMap, DivGuards);
16071
16072 // Apply divisibility information last. This ensures it is applied to the
16073 // outermost expression after other rewrites for the given value.
16074 for (const auto &[K, Divisor] : Multiples) {
16075 const SCEV *DivisorSCEV = SE.getConstant(Val: Divisor);
16076 Guards.RewriteMap[K] =
16077 SE.getMulExpr(LHS: SE.getUDivExpr(LHS: applyDivisibilityOnMinMaxExpr(
16078 MinMaxExpr: Guards.rewrite(Expr: K), Divisor, SE),
16079 RHS: DivisorSCEV),
16080 RHS: DivisorSCEV);
16081 ExprsToRewrite.push_back(Elt: K);
16082 }
16083
16084 // Let the rewriter preserve NUW/NSW flags if the unsigned/signed ranges of
16085 // the replacement expressions are contained in the ranges of the replaced
16086 // expressions.
16087 Guards.PreserveNUW = true;
16088 Guards.PreserveNSW = true;
16089 for (const SCEV *Expr : ExprsToRewrite) {
16090 const SCEV *RewriteTo = Guards.RewriteMap[Expr];
16091 Guards.PreserveNUW &=
16092 SE.getUnsignedRange(S: Expr).contains(CR: SE.getUnsignedRange(S: RewriteTo));
16093 Guards.PreserveNSW &=
16094 SE.getSignedRange(S: Expr).contains(CR: SE.getSignedRange(S: RewriteTo));
16095 }
16096
16097 // Now that all rewrite information is collect, rewrite the collected
16098 // expressions with the information in the map. This applies information to
16099 // sub-expressions. A collected expression is rewritten when it is first
16100 // looked up, so the rewrites of the expressions it contains are used once
16101 // they are final.
16102 if (ExprsToRewrite.size() > 1) {
16103 constexpr unsigned MaxDepth = 16;
16104 unsigned Depth = 0;
16105 SmallPtrSet<const SCEV *, 16> ToRewrite(llvm::from_range, ExprsToRewrite);
16106 std::function<const SCEV *(const SCEV *)> Lookup = [&](const SCEV *Expr) {
16107 if (Depth < MaxDepth && ToRewrite.erase(Ptr: Expr)) {
16108 auto It = Guards.RewriteMap.find(Val: Expr);
16109 assert(It != Guards.RewriteMap.end() && "Expected rewrite entry");
16110 const SCEV *RewriteTo = It->second;
16111 Guards.RewriteMap.erase(I: It);
16112 SaveAndRestore Nest(Depth, Depth + 1);
16113 const SCEV *Rewritten = Guards.rewrite(Expr: RewriteTo, Lookup);
16114 Guards.RewriteMap[Expr] = Rewritten;
16115 return Rewritten;
16116 }
16117 return Guards.RewriteMap.lookup(Val: Expr);
16118 };
16119 for (const SCEV *Expr : ExprsToRewrite)
16120 Lookup(Expr);
16121 }
16122}
16123
16124const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const {
16125 return rewrite(Expr, Lookup: nullptr);
16126}
16127
16128const SCEV *ScalarEvolution::LoopGuards::rewrite(
16129 const SCEV *Expr, function_ref<const SCEV *(const SCEV *)> Lookup) const {
16130 /// A rewriter to replace SCEV expressions in Map with the corresponding entry
16131 /// in the map. It skips AddRecExpr because we cannot guarantee that the
16132 /// replacement is loop invariant in the loop of the AddRec.
16133 class SCEVLoopGuardRewriter
16134 : public SCEVRewriteVisitor<SCEVLoopGuardRewriter> {
16135 const DenseMap<const SCEV *, const SCEV *> &Map;
16136 const SmallDenseSet<std::pair<const SCEV *, const SCEV *>> &NotEqual;
16137 function_ref<const SCEV *(const SCEV *)> Lookup;
16138
16139 SCEVFlags FlagMask = SCEV::FlagNone;
16140
16141 const SCEV *lookup(const SCEV *Key) {
16142 return Lookup ? Lookup(Key) : Map.lookup(Val: Key);
16143 }
16144
16145 public:
16146 SCEVLoopGuardRewriter(ScalarEvolution &SE,
16147 const ScalarEvolution::LoopGuards &Guards,
16148 function_ref<const SCEV *(const SCEV *)> Lookup)
16149 : SCEVRewriteVisitor(SE), Map(Guards.RewriteMap),
16150 NotEqual(Guards.NotEqual), Lookup(Lookup) {
16151 if (Guards.PreserveNUW)
16152 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNUW);
16153 if (Guards.PreserveNSW)
16154 FlagMask = ScalarEvolution::setFlags(Flags: FlagMask, OnFlags: SCEV::FlagNSW);
16155 }
16156
16157 const SCEV *visit(const SCEV *Expr) {
16158 if (isa<SCEVUnknown, SCEVPtrToAddrExpr, SCEVZeroExtendExpr,
16159 SCEVSignExtendExpr, SCEVUMinExpr, SCEVSMinExpr, SCEVAddExpr,
16160 SCEVMulExpr>(Val: Expr))
16161 if (const SCEV *S = lookup(Key: Expr))
16162 return S;
16163 return SCEVRewriteVisitor::visit(S: Expr);
16164 }
16165
16166 const SCEV *visitAddRecExpr(const SCEVAddRecExpr *Expr) { return Expr; }
16167
16168 const SCEV *visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
16169 // If there is no replacement for the exact ZExt expr, check if there's
16170 // one for a smaller ZExt we can use instead.
16171 Type *Ty = Expr->getType();
16172 const SCEV *Op = Expr->getOperand(i: 0);
16173 unsigned Bitwidth = Ty->getScalarSizeInBits() / 2;
16174 while (Bitwidth % 8 == 0 && Bitwidth >= 8 &&
16175 Bitwidth > Op->getType()->getScalarSizeInBits()) {
16176 Type *NarrowTy = IntegerType::get(C&: SE.getContext(), NumBits: Bitwidth);
16177 auto *NarrowExt = SE.getZeroExtendExpr(Op, Ty: NarrowTy);
16178 if (const SCEV *S = lookup(Key: NarrowExt))
16179 return SE.getZeroExtendExpr(Op: S, Ty);
16180 Bitwidth = Bitwidth / 2;
16181 }
16182
16183 return SCEVRewriteVisitor::visitZeroExtendExpr(Expr);
16184 }
16185
16186 const SCEV *visitAddExpr(const SCEVAddExpr *Expr) {
16187 // Helper to check if S is a subtraction (A - B) where A != B, and if so,
16188 // return UMax(S, 1).
16189 auto RewriteSubtraction = [&](const SCEV *S) -> const SCEV * {
16190 SCEVUse LHS, RHS;
16191 if (MatchBinarySub(S, LHS, RHS)) {
16192 if (LHS > RHS)
16193 std::swap(a&: LHS, b&: RHS);
16194 if (NotEqual.contains(V: {LHS, RHS})) {
16195 const SCEV *OneAlignedUp = getNextSCEVDivisibleByDivisor(
16196 Expr: SE.getOne(Ty: S->getType()), DivisorVal: SE.getConstantMultiple(S), SE);
16197 return SE.getUMaxExpr(LHS: OneAlignedUp, RHS: S);
16198 }
16199 }
16200 return nullptr;
16201 };
16202
16203 // Check if Expr itself is a subtraction pattern with guard info.
16204 if (const SCEV *Rewritten = RewriteSubtraction(Expr))
16205 return Rewritten;
16206
16207 // Trip count expressions sometimes consist of adding 3 operands, i.e.
16208 // (Const + A + B). There may be guard info for A + B, and if so, apply
16209 // it.
16210 // TODO: Could more generally apply guards to Add sub-expressions.
16211 if (isa<SCEVConstant>(Val: Expr->getOperand(i: 0))) {
16212 if (Expr->getNumOperands() == 3) {
16213 const SCEV *Add =
16214 SE.getAddExpr(LHS: Expr->getOperand(i: 1), RHS: Expr->getOperand(i: 2));
16215 if (const SCEV *Rewritten = RewriteSubtraction(Add))
16216 return SE.getAddExpr(
16217 LHS: Expr->getOperand(i: 0), RHS: Rewritten,
16218 Flags: ScalarEvolution::maskFlags(Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16219 if (const SCEV *S = lookup(Key: Add))
16220 return SE.getAddExpr(LHS: Expr->getOperand(i: 0), RHS: S);
16221 }
16222
16223 // For expressions of the form (Const + A), check if we have guard info
16224 // for (Const + 1 + A), and rewrite to ((Const + 1 + A) - 1). This makes
16225 // sure we don't lose information when rewriting expressions based on
16226 // back-edge taken counts in some cases.
16227 if (Expr->getNumOperands() == 2) {
16228 const SCEV *S = nullptr;
16229 // Handle (-1 + 1 + A) without constructing SCEVs.
16230 if (match(U: Expr->getOperand(i: 0), P: m_scev_AllOnes())) {
16231 S = lookup(Key: Expr->getOperand(i: 1));
16232 } else {
16233 const SCEV *NewC =
16234 SE.getAddExpr(LHS: Expr->getOperand(i: 0), RHS: SE.getOne(Ty: Expr->getType()));
16235 S = lookup(Key: SE.getAddExpr(LHS: NewC, RHS: Expr->getOperand(i: 1)));
16236 }
16237 if (S)
16238 return SE.getAddExpr(LHS: S, RHS: SE.getMinusOne(Ty: Expr->getType()));
16239 }
16240 }
16241 SmallVector<SCEVUse, 2> Operands;
16242 bool Changed = false;
16243 for (SCEVUse Op : Expr->operands()) {
16244 Operands.push_back(Elt: visit(Expr: Op));
16245 Changed |= Op != Operands.back();
16246 }
16247 // We are only replacing operands with equivalent values, so transfer the
16248 // flags from the original expression.
16249 return !Changed ? Expr
16250 : SE.getAddExpr(Ops&: Operands,
16251 Flags: ScalarEvolution::maskFlags(
16252 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16253 }
16254
16255 const SCEV *visitMulExpr(const SCEVMulExpr *Expr) {
16256 SmallVector<SCEVUse, 2> Operands;
16257 bool Changed = false;
16258 for (SCEVUse Op : Expr->operands()) {
16259 Operands.push_back(Elt: visit(Expr: Op));
16260 Changed |= Op != Operands.back();
16261 }
16262 // We are only replacing operands with equivalent values, so transfer the
16263 // flags from the original expression.
16264 return !Changed ? Expr
16265 : SE.getMulExpr(Ops&: Operands,
16266 Flags: ScalarEvolution::maskFlags(
16267 Flags: Expr->getNoWrapFlags(), Mask: FlagMask));
16268 }
16269 };
16270
16271 if (RewriteMap.empty() && NotEqual.empty())
16272 return Expr;
16273
16274 SCEVLoopGuardRewriter Rewriter(SE, *this, Lookup);
16275 return Rewriter.visit(Expr);
16276}
16277
16278const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr, const Loop *L) {
16279 LoopGuards Guards = LoopGuards::collectOnDemand(L, SE&: *this);
16280 return applyLoopGuards(Expr, Guards);
16281}
16282
16283const SCEV *ScalarEvolution::applyLoopGuards(const SCEV *Expr,
16284 LoopGuards &Guards) {
16285 // Loop guards never rewrite constants and AddRecs, so there is no need to
16286 // collect pending guards for them.
16287 if (isa<SCEVConstant, SCEVAddRecExpr>(Val: Expr))
16288 return Expr;
16289 if (Guards.PendingLoop)
16290 Guards.collectPending();
16291 return Guards.rewrite(Expr);
16292}
16293