LoopStrengthReduce.cpp source code [llvm_projects/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp]

1	//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This transformation analyzes and transforms the induction variables (and
10	// computations derived from them) into forms suitable for efficient execution
11	// on the target.
12	//
13	// This pass performs a strength reduction on array references inside loops that
14	// have as one or more of their components the loop induction variable, it
15	// rewrites expressions to take advantage of scaled-index addressing modes
16	// available on the target, and it performs a variety of other optimizations
17	// related to loop induction variables.
18	//
19	// Terminology note: this code has a lot of handling for "post-increment" or
20	// "post-inc" users. This is not talking about post-increment addressing modes;
21	// it is instead talking about code like this:
22	//
23	// %i = phi [ 0, %entry ], [ %i.next, %latch ]
24	// ...
25	// %i.next = add %i, 1
26	// %c = icmp eq %i.next, %n
27	//
28	// The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however
29	// it's useful to think about these as the same register, with some uses using
30	// the value of the register before the add and some using it after. In this
31	// example, the icmp is a post-increment user, since it uses %i.next, which is
32	// the value of the induction variable after the increment. The other common
33	// case of post-increment users is users outside the loop.
34	//
35	// TODO: More sophistication in the way Formulae are generated and filtered.
36	//
37	// TODO: Handle multiple loops at a time.
38	//
39	// TODO: Should the addressing mode BaseGV be changed to a ConstantExpr instead
40	// of a GlobalValue?
41	//
42	// TODO: When truncation is free, truncate ICmp users' operands to make it a
43	// smaller encoding (on x86 at least).
44	//
45	// TODO: When a negated register is used by an add (such as in a list of
46	// multiple base registers, or as the increment expression in an addrec),
47	// we may not actually need both reg and (-1 reg) in registers; the*
48	// negation can be implemented by using a sub instead of an add. The
49	// lack of support for taking this into consideration when making
50	// register pressure decisions is partly worked around by the "Special"
51	// use kind.
52	//
53	//===----------------------------------------------------------------------===//
54
55	#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
56	#include "llvm/ADT/APInt.h"
57	#include "llvm/ADT/DenseMap.h"
58	#include "llvm/ADT/DenseSet.h"
59	#include "llvm/ADT/PointerIntPair.h"
60	#include "llvm/ADT/STLExtras.h"
61	#include "llvm/ADT/SetVector.h"
62	#include "llvm/ADT/SmallBitVector.h"
63	#include "llvm/ADT/SmallPtrSet.h"
64	#include "llvm/ADT/SmallSet.h"
65	#include "llvm/ADT/SmallVector.h"
66	#include "llvm/ADT/Statistic.h"
67	#include "llvm/ADT/iterator_range.h"
68	#include "llvm/Analysis/AssumptionCache.h"
69	#include "llvm/Analysis/DomTreeUpdater.h"
70	#include "llvm/Analysis/IVUsers.h"
71	#include "llvm/Analysis/LoopAnalysisManager.h"
72	#include "llvm/Analysis/LoopInfo.h"
73	#include "llvm/Analysis/LoopPass.h"
74	#include "llvm/Analysis/MemorySSA.h"
75	#include "llvm/Analysis/MemorySSAUpdater.h"
76	#include "llvm/Analysis/ScalarEvolution.h"
77	#include "llvm/Analysis/ScalarEvolutionExpressions.h"
78	#include "llvm/Analysis/ScalarEvolutionNormalization.h"
79	#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
80	#include "llvm/Analysis/TargetLibraryInfo.h"
81	#include "llvm/Analysis/TargetTransformInfo.h"
82	#include "llvm/Analysis/ValueTracking.h"
83	#include "llvm/BinaryFormat/Dwarf.h"
84	#include "llvm/IR/BasicBlock.h"
85	#include "llvm/IR/Constant.h"
86	#include "llvm/IR/Constants.h"
87	#include "llvm/IR/DebugInfoMetadata.h"
88	#include "llvm/IR/DerivedTypes.h"
89	#include "llvm/IR/Dominators.h"
90	#include "llvm/IR/GlobalValue.h"
91	#include "llvm/IR/IRBuilder.h"
92	#include "llvm/IR/InstrTypes.h"
93	#include "llvm/IR/Instruction.h"
94	#include "llvm/IR/Instructions.h"
95	#include "llvm/IR/IntrinsicInst.h"
96	#include "llvm/IR/Module.h"
97	#include "llvm/IR/Operator.h"
98	#include "llvm/IR/Type.h"
99	#include "llvm/IR/Use.h"
100	#include "llvm/IR/User.h"
101	#include "llvm/IR/Value.h"
102	#include "llvm/IR/ValueHandle.h"
103	#include "llvm/InitializePasses.h"
104	#include "llvm/Pass.h"
105	#include "llvm/Support/Casting.h"
106	#include "llvm/Support/CommandLine.h"
107	#include "llvm/Support/Compiler.h"
108	#include "llvm/Support/Debug.h"
109	#include "llvm/Support/ErrorHandling.h"
110	#include "llvm/Support/MathExtras.h"
111	#include "llvm/Support/raw_ostream.h"
112	#include "llvm/Transforms/Scalar.h"
113	#include "llvm/Transforms/Utils.h"
114	#include "llvm/Transforms/Utils/BasicBlockUtils.h"
115	#include "llvm/Transforms/Utils/Local.h"
116	#include "llvm/Transforms/Utils/LoopUtils.h"
117	#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
118	#include <algorithm>
119	#include <cassert>
120	#include <cstddef>
121	#include <cstdint>
122	#include <iterator>
123	#include <limits>
124	#include <map>
125	#include <numeric>
126	#include <optional>
127	#include <utility>
128
129	using namespace llvm;
130	using namespace SCEVPatternMatch;
131
132	#define DEBUG_TYPE "loop-reduce"
133
134	/// MaxIVUsers is an arbitrary threshold that provides an early opportunity for
135	/// bail out. This threshold is far beyond the number of users that LSR can
136	/// conceivably solve, so it should not affect generated code, but catches the
137	/// worst cases before LSR burns too much compile time and stack space.
138	static const unsigned MaxIVUsers = `200`;
139
140	/// Limit the size of expression that SCEV-based salvaging will attempt to
141	/// translate into a DIExpression.
142	/// Choose a maximum size such that debuginfo is not excessively increased and
143	/// the salvaging is not too expensive for the compiler.
144	static const unsigned MaxSCEVSalvageExpressionSize = `64`;
145
146	// Cleanup congruent phis after LSR phi expansion.
147	static cl::opt<bool> EnablePhiElim(
148	"enable-lsr-phielim", cl::Hidden, cl::init(Val: true),
149	cl::desc("Enable LSR phi elimination"));
150
151	// The flag adds instruction count to solutions cost comparison.
152	static cl::opt<bool> InsnsCost(
153	"lsr-insns-cost", cl::Hidden, cl::init(Val: true),
154	cl::desc("Add instruction count to a LSR cost model"));
155
156	// Flag to choose how to narrow complex lsr solution
157	static cl::opt<bool> LSRExpNarrow(
158	"lsr-exp-narrow", cl::Hidden, cl::init(Val: false),
159	cl::desc("Narrow LSR complex solution using"
160	" expectation of registers number"));
161
162	// Flag to narrow search space by filtering non-optimal formulae with
163	// the same ScaledReg and Scale.
164	static cl::opt<bool> FilterSameScaledReg(
165	"lsr-filter-same-scaled-reg", cl::Hidden, cl::init(Val: true),
166	cl::desc("Narrow LSR search space by filtering non-optimal formulae"
167	" with the same ScaledReg and Scale"));
168
169	static cl::opt<TTI::AddressingModeKind> PreferredAddresingMode(
170	"lsr-preferred-addressing-mode", cl::Hidden, cl::init(Val: TTI::AMK_None),
171	cl::desc("A flag that overrides the target's preferred addressing mode."),
172	cl::values(
173	clEnumValN(TTI::AMK_None, "none", "Don't prefer any addressing mode"),
174	clEnumValN(TTI::AMK_PreIndexed, "preindexed",
175	"Prefer pre-indexed addressing mode"),
176	clEnumValN(TTI::AMK_PostIndexed, "postindexed",
177	"Prefer post-indexed addressing mode"),
178	clEnumValN(TTI::AMK_All, "all", "Consider all addressing modes")));
179
180	static cl::opt<unsigned> ComplexityLimit(
181	"lsr-complexity-limit", cl::Hidden,
182	cl::init(Val: std::numeric_limits<uint16_t>::max()),
183	cl::desc("LSR search space complexity limit"));
184
185	static cl::opt<unsigned> SetupCostDepthLimit(
186	"lsr-setupcost-depth-limit", cl::Hidden, cl::init(Val: `7`),
187	cl::desc("The limit on recursion depth for LSRs setup cost"));
188
189	static cl::opt<cl::boolOrDefault> AllowDropSolutionIfLessProfitable(
190	"lsr-drop-solution", cl::Hidden,
191	cl::desc("Attempt to drop solution if it is less profitable"));
192
193	static cl::opt<bool> EnableVScaleImmediates(
194	"lsr-enable-vscale-immediates", cl::Hidden, cl::init(Val: true),
195	cl::desc("Enable analysis of vscale-relative immediates in LSR"));
196
197	static cl::opt<bool> DropScaledForVScale(
198	"lsr-drop-scaled-reg-for-vscale", cl::Hidden, cl::init(Val: true),
199	cl::desc("Avoid using scaled registers with vscale-relative addressing"));
200
201	#ifndef NDEBUG
202	// Stress test IV chain generation.
203	static cl::opt<bool> StressIVChain(
204	"stress-ivchain", cl::Hidden, cl::init(false),
205	cl::desc("Stress test LSR IV chains"));
206	#else
207	static bool StressIVChain = false;
208	#endif
209
210	namespace {
211
212	struct MemAccessTy {
213	/// Used in situations where the accessed memory type is unknown.
214	static const unsigned UnknownAddressSpace =
215	std::numeric_limits<unsigned>::max();
216
217	Type MemTy = nullptr*;
218	unsigned AddrSpace = UnknownAddressSpace;
219
220	MemAccessTy() = default;
221	MemAccessTy(Type Ty, unsigned* AS) : MemTy(Ty), AddrSpace(AS) {}
222
223	bool operator==(MemAccessTy Other) const {
224	return MemTy == Other.MemTy && AddrSpace == Other.AddrSpace;
225	}
226
227	bool operator!=(MemAccessTy Other) const { return !(*this == Other); }
228
229	static MemAccessTy getUnknown(LLVMContext &Ctx,
230	unsigned AS = UnknownAddressSpace) {
231	return MemAccessTy (Type::getVoidTy(C&: Ctx), AS);
232	}
233
234	Type getType() { return* MemTy; }
235	};
236
237	/// This class holds data which is used to order reuse candidates.
238	class RegSortData {
239	public:
240	/// This represents the set of LSRUse indices which reference
241	/// a particular register.
242	SmallBitVector UsedByIndices;
243
244	void print(raw_ostream &OS) const;
245	void dump() const;
246	};
247
248	// An offset from an address that is either scalable or fixed. Used for
249	// per-target optimizations of addressing modes.
250	class Immediate : public details::FixedOrScalableQuantity<Immediate, int64_t> {
251	constexpr Immediate(ScalarTy MinVal, bool Scalable)
252	: FixedOrScalableQuantity (MinVal, Scalable) {}
253
254	constexpr Immediate(const FixedOrScalableQuantity<Immediate, int64_t> &V)
255	: FixedOrScalableQuantity (V) {}
256
257	public:
258	constexpr Immediate() = delete;
259
260	static constexpr Immediate getFixed(ScalarTy MinVal) {
261	return {MinVal, false};
262	}
263	static constexpr Immediate getScalable(ScalarTy MinVal) {
264	return {MinVal, true};
265	}
266	static constexpr Immediate get(ScalarTy MinVal, bool Scalable) {
267	return {MinVal, Scalable};
268	}
269	static constexpr Immediate getZero() { return {`0`, false}; }
270	static constexpr Immediate getFixedMin() {
271	return {std::numeric_limits<int64_t>::min(), false};
272	}
273	static constexpr Immediate getFixedMax() {
274	return {std::numeric_limits<int64_t>::max(), false};
275	}
276	static constexpr Immediate getScalableMin() {
277	return {std::numeric_limits<int64_t>::min(), true};
278	}
279	static constexpr Immediate getScalableMax() {
280	return {std::numeric_limits<int64_t>::max(), true};
281	}
282
283	constexpr bool isLessThanZero() const { return Quantity < `0`; }
284
285	constexpr bool isGreaterThanZero() const { return Quantity > `0`; }
286
287	constexpr bool isCompatibleImmediate(const Immediate &Imm) const {
288	return isZero() \|\| Imm.isZero() \|\| Imm.Scalable == Scalable;
289	}
290
291	constexpr bool isMin() const {
292	return Quantity == std::numeric_limits<ScalarTy>::min();
293	}
294
295	constexpr bool isMax() const {
296	return Quantity == std::numeric_limits<ScalarTy>::max();
297	}
298
299	// Arithmetic 'operators' that cast to unsigned types first.
300	constexpr Immediate addUnsigned(const Immediate &RHS) const {
301	assert(isCompatibleImmediate(RHS) && "Incompatible Immediates");
302	ScalarTy Value = (uint64_t)Quantity + RHS.getKnownMinValue();
303	return {Value, Scalable \|\| RHS.isScalable()};
304	}
305
306	constexpr Immediate subUnsigned(const Immediate &RHS) const {
307	assert(isCompatibleImmediate(RHS) && "Incompatible Immediates");
308	ScalarTy Value = (uint64_t)Quantity - RHS.getKnownMinValue();
309	return {Value, Scalable \|\| RHS.isScalable()};
310	}
311
312	// Scale the quantity by a constant without caring about runtime scalability.
313	constexpr Immediate mulUnsigned(const ScalarTy RHS) const {
314	ScalarTy Value = (uint64_t)Quantity * RHS;
315	return {Value, Scalable};
316	}
317
318	// Helpers for generating SCEVs with vscale terms where needed.
319	const SCEV getSCEV(ScalarEvolution &SE, Type Ty) const {
320	const SCEV *S = SE.getConstant(Ty, V: Quantity);
321	if (Scalable)
322	S = SE.getMulExpr(LHS: S, RHS: SE.getVScale(Ty: S->getType()));
323	return S;
324	}
325
326	const SCEV getNegativeSCEV(ScalarEvolution &SE, Type Ty) const {
327	const SCEV *NegS = SE.getConstant(Ty, V: -(uint64_t)Quantity);
328	if (Scalable)
329	NegS = SE.getMulExpr(LHS: NegS, RHS: SE.getVScale(Ty: NegS->getType()));
330	return NegS;
331	}
332
333	const SCEV getUnknownSCEV(ScalarEvolution &SE, Type Ty) const {
334	// TODO: Avoid implicit trunc?
335	// See https://github.com/llvm/llvm-project/issues/112510.
336	const SCEV *SU = SE.getUnknown(
337	V: ConstantInt::getSigned(Ty, V: Quantity, /ImplicitTrunc=/true));
338	if (Scalable)
339	SU = SE.getMulExpr(LHS: SU, RHS: SE.getVScale(Ty: SU->getType()));
340	return SU;
341	}
342	};
343
344	// This is needed for the Compare type of std::map when Immediate is used
345	// as a key. We don't need it to be fully correct against any value of vscale,
346	// just to make sure that vscale-related terms in the map are considered against
347	// each other rather than being mixed up and potentially missing opportunities.
348	struct KeyOrderTargetImmediate {
349	bool operator()(const Immediate &LHS, const Immediate &RHS) const {
350	if (LHS.isScalable() && !RHS.isScalable())
351	return false;
352	if (!LHS.isScalable() && RHS.isScalable())
353	return true;
354	return LHS.getKnownMinValue() < RHS.getKnownMinValue();
355	}
356	};
357
358	// This would be nicer if we could be generic instead of directly using size_t,
359	// but there doesn't seem to be a type trait for is_orderable or
360	// is_lessthan_comparable or similar.
361	struct KeyOrderSizeTAndImmediate {
362	bool operator()(const std::pair<size_t, Immediate> &LHS,
363	const std::pair<size_t, Immediate> &RHS) const {
364	size_t LSize = LHS.first;
365	size_t RSize = RHS.first;
366	if (LSize != RSize)
367	return LSize < RSize;
368	return KeyOrderTargetImmediate ()(LHS.second, RHS.second);
369	}
370	};
371	} // end anonymous namespace
372
373	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
374	void RegSortData::print(raw_ostream &OS) const {
375	OS << "[NumUses=" << UsedByIndices.count() << `']'`;
376	}
377
378	LLVM_DUMP_METHOD void RegSortData::dump() const {
379	print(errs()); errs() << `'\n'`;
380	}
381	#endif
382
383	namespace {
384
385	/// Map register candidates to information about how they are used.
386	class RegUseTracker {
387	using RegUsesTy = DenseMap<const SCEV *, RegSortData>;
388
389	RegUsesTy RegUsesMap;
390	SmallVector<const SCEV *, `16`> RegSequence;
391
392	public:
393	void countRegister(const SCEV *Reg, size_t LUIdx);
394	void dropRegister(const SCEV *Reg, size_t LUIdx);
395	void swapAndDropUse(size_t LUIdx, size_t LastLUIdx);
396
397	bool isRegUsedByUsesOtherThan(const SCEV Reg, size_t LUIdx) const*;
398
399	const SmallBitVector &getUsedByIndices(const SCEV Reg) const*;
400
401	void clear();
402
403	using iterator = SmallVectorImpl<const SCEV *>::iterator;
404	using const_iterator = SmallVectorImpl<const SCEV *>::const_iterator;
405
406	iterator begin() { return RegSequence.begin(); }
407	iterator end() { return RegSequence.end(); }
408	const_iterator begin() const { return RegSequence.begin(); }
409	const_iterator end() const { return RegSequence.end(); }
410	};
411
412	} // end anonymous namespace
413
414	void
415	RegUseTracker::countRegister(const SCEV *Reg, size_t LUIdx) {
416	std::pair<RegUsesTy::iterator, bool> Pair = RegUsesMap.try_emplace(Key: Reg);
417	RegSortData &RSD = Pair.first ->second;
418	if (Pair.second)
419	RegSequence.push_back(Elt: Reg);
420	RSD.UsedByIndices.resize(N: std::max(a: RSD.UsedByIndices.size(), b: LUIdx + `1`));
421	RSD.UsedByIndices.set(LUIdx);
422	}
423
424	void
425	RegUseTracker::dropRegister(const SCEV *Reg, size_t LUIdx) {
426	RegUsesTy::iterator It = RegUsesMap.find(Val: Reg);
427	assert(It != RegUsesMap.end());
428	RegSortData &RSD = It ->second;
429	assert(RSD.UsedByIndices.size() > LUIdx);
430	RSD.UsedByIndices.reset(Idx: LUIdx);
431	}
432
433	void
434	RegUseTracker::swapAndDropUse(size_t LUIdx, size_t LastLUIdx) {
435	assert(LUIdx <= LastLUIdx);
436
437	// Update RegUses. The data structure is not optimized for this purpose;
438	// we must iterate through it and update each of the bit vectors.
439	for (auto &Pair : RegUsesMap) {
440	SmallBitVector &UsedByIndices = Pair.second.UsedByIndices;
441	if (LUIdx < UsedByIndices.size())
442	UsedByIndices [LUIdx] =
443	LastLUIdx < UsedByIndices.size() ? UsedByIndices [LastLUIdx] : false;
444	UsedByIndices.resize(N: std::min(a: UsedByIndices.size(), b: LastLUIdx));
445	}
446	}
447
448	bool
449	RegUseTracker::isRegUsedByUsesOtherThan(const SCEV Reg, size_t LUIdx) const* {
450	RegUsesTy::const_iterator I = RegUsesMap.find(Val: Reg);
451	if (I == RegUsesMap.end())
452	return false;
453	const SmallBitVector &UsedByIndices = I ->second.UsedByIndices;
454	int i = UsedByIndices.find_first();
455	if (i == -`1`) return false;
456	if ((size_t)i != LUIdx) return true;
457	return UsedByIndices.find_next(Prev: i) != -`1`;
458	}
459
460	const SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV Reg) const* {
461	RegUsesTy::const_iterator I = RegUsesMap.find(Val: Reg);
462	assert(I != RegUsesMap.end() && "Unknown register!");
463	return I ->second.UsedByIndices;
464	}
465
466	void RegUseTracker::clear() {
467	RegUsesMap.clear();
468	RegSequence.clear();
469	}
470
471	namespace {
472
473	/// This class holds information that describes a formula for computing
474	/// satisfying a use. It may include broken-out immediates and scaled registers.
475	struct Formula {
476	/// Global base address used for complex addressing.
477	GlobalValue BaseGV = nullptr*;
478
479	/// Base offset for complex addressing.
480	Immediate BaseOffset = Immediate::getZero();
481
482	/// Whether any complex addressing has a base register.
483	bool HasBaseReg = false;
484
485	/// The scale of any complex addressing.
486	int64_t Scale = `0`;
487
488	/// The list of "base" registers for this use. When this is non-empty. The
489	/// canonical representation of a formula is
490	/// 1. BaseRegs.size > 1 implies ScaledReg != NULL and
491	/// 2. ScaledReg != NULL implies Scale != 1 \|\| !BaseRegs.empty().
492	/// 3. The reg containing recurrent expr related with currect loop in the
493	/// formula should be put in the ScaledReg.
494	/// #1 enforces that the scaled register is always used when at least two
495	/// registers are needed by the formula: e.g., reg1 + reg2 is reg1 + 1 reg2.*
496	/// #2 enforces that 1 reg is reg.*
497	/// #3 ensures invariant regs with respect to current loop can be combined
498	/// together in LSR codegen.
499	/// This invariant can be temporarily broken while building a formula.
500	/// However, every formula inserted into the LSRInstance must be in canonical
501	/// form.
502	SmallVector<const SCEV *, `4`> BaseRegs;
503
504	/// The 'scaled' register for this use. This should be non-null when Scale is
505	/// not zero.
506	const SCEV ScaledReg = nullptr*;
507
508	/// An additional constant offset which added near the use. This requires a
509	/// temporary register, but the offset itself can live in an add immediate
510	/// field rather than a register.
511	Immediate UnfoldedOffset = Immediate::getZero();
512
513	Formula() = default;
514
515	void initialMatch(const SCEV S, Loop L, ScalarEvolution &SE);
516
517	bool isCanonical(const Loop &L) const;
518
519	void canonicalize(const Loop &L);
520
521	bool unscale();
522
523	bool hasZeroEnd() const;
524
525	bool countsDownToZero() const;
526
527	size_t getNumRegs() const;
528	Type getType() const*;
529
530	void deleteBaseReg(const SCEV *&S);
531
532	bool referencesReg(const SCEV S) const*;
533	bool hasRegsUsedByUsesOtherThan(size_t LUIdx,
534	const RegUseTracker &RegUses) const;
535
536	void print(raw_ostream &OS) const;
537	void dump() const;
538	};
539
540	} // end anonymous namespace
541
542	/// Recursion helper for initialMatch.
543	static void DoInitialMatch(const SCEV S, Loop L,
544	SmallVectorImpl<SCEVUse> &Good,
545	SmallVectorImpl<SCEVUse> &Bad, ScalarEvolution &SE) {
546	// Collect expressions which properly dominate the loop header.
547	if (SE.properlyDominates(S, BB: L->getHeader())) {
548	Good.push_back(Elt: S);
549	return;
550	}
551
552	// Look at add operands.
553	if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
554	for (const SCEV *S : Add->operands())
555	DoInitialMatch(S, L, Good, Bad, SE);
556	return;
557	}
558
559	// Look at addrec operands.
560	const SCEV Start, Step;
561	const Loop *ARLoop;
562	if (match(S,
563	P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L&: ARLoop))) &&
564	!Start->isZero()) {
565	DoInitialMatch(S: Start, L, Good, Bad, SE);
566	DoInitialMatch(S: SE.getAddRecExpr(Start: SE.getConstant(Ty: S->getType(), V: `0`), Step,
567	// FIXME: AR->getNoWrapFlags()
568	L: ARLoop, Flags: SCEV::FlagAnyWrap),
569	L, Good, Bad, SE);
570	return;
571	}
572
573	// Handle a multiplication by -1 (negation) if it didn't fold.
574	if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: S))
575	if (Mul->getOperand(i: `0`)->isAllOnesValue()) {
576	SmallVector<SCEVUse, `4`> Ops(drop_begin(RangeOrContainer: Mul->operands()));
577	const SCEV *NewMul = SE.getMulExpr(Ops);
578
579	SmallVector<SCEVUse, `4`> MyGood;
580	SmallVector<SCEVUse, `4`> MyBad;
581	DoInitialMatch(S: NewMul, L, Good&: MyGood, Bad&: MyBad, SE);
582	const SCEV *NegOne = SE.getSCEV(V: ConstantInt::getAllOnesValue(
583	Ty: SE.getEffectiveSCEVType(Ty: NewMul->getType())));
584	for (const SCEV *S : MyGood)
585	Good.push_back(Elt: SE.getMulExpr(LHS: NegOne, RHS: S));
586	for (const SCEV *S : MyBad)
587	Bad.push_back(Elt: SE.getMulExpr(LHS: NegOne, RHS: S));
588	return;
589	}
590
591	// Ok, we can't do anything interesting. Just stuff the whole thing into a
592	// register and hope for the best.
593	Bad.push_back(Elt: S);
594	}
595
596	/// Incorporate loop-variant parts of S into this Formula, attempting to keep
597	/// all loop-invariant and loop-computable values in a single base register.
598	void Formula::initialMatch(const SCEV S, Loop L, ScalarEvolution &SE) {
599	SmallVector<SCEVUse, `4`> Good;
600	SmallVector<SCEVUse, `4`> Bad;
601	DoInitialMatch(S, L, Good, Bad, SE);
602	if (!Good.empty()) {
603	const SCEV *Sum = SE.getAddExpr(Ops&: Good);
604	if (!Sum->isZero())
605	BaseRegs.push_back(Elt: Sum);
606	HasBaseReg = true;
607	}
608	if (!Bad.empty()) {
609	const SCEV *Sum = SE.getAddExpr(Ops&: Bad);
610	if (!Sum->isZero())
611	BaseRegs.push_back(Elt: Sum);
612	HasBaseReg = true;
613	}
614	canonicalize(L: *L);
615	}
616
617	static bool containsAddRecDependentOnLoop(const SCEV S, const* Loop &L) {
618	return SCEVExprContains(Root: S, Pred: [&L](const SCEV *S) {
619	return isa<SCEVAddRecExpr>(Val: S) && (cast<SCEVAddRecExpr>(Val: S)->getLoop() == &L);
620	});
621	}
622
623	/// Check whether or not this formula satisfies the canonical
624	/// representation.
625	/// \see Formula::BaseRegs.
626	bool Formula::isCanonical(const Loop &L) const {
627	assert((Scale == `0` \|\| ScaledReg) &&
628	"ScaledReg must be non-null if Scale is non-zero");
629
630	if (!ScaledReg)
631	return BaseRegs.size() <= `1`;
632
633	if (Scale != `1`)
634	return true;
635
636	if (Scale == `1` && BaseRegs.empty())
637	return false;
638
639	if (containsAddRecDependentOnLoop(S: ScaledReg, L))
640	return true;
641
642	// If ScaledReg is not a recurrent expr, or it is but its loop is not current
643	// loop, meanwhile BaseRegs contains a recurrent expr reg related with current
644	// loop, we want to swap the reg in BaseRegs with ScaledReg.
645	return none_of(Range: BaseRegs, P: [&L](const SCEV *S) {
646	return containsAddRecDependentOnLoop(S, L);
647	});
648	}
649
650	/// Helper method to morph a formula into its canonical representation.
651	/// \see Formula::BaseRegs.
652	/// Every formula having more than one base register, must use the ScaledReg
653	/// field. Otherwise, we would have to do special cases everywhere in LSR
654	/// to treat reg1 + reg2 + ... the same way as reg1 + 1reg2 + ...*
655	/// On the other hand, 1reg should be canonicalized into reg.*
656	void Formula::canonicalize(const Loop &L) {
657	if (isCanonical(L))
658	return;
659
660	if (BaseRegs.empty()) {
661	// No base reg? Use scale reg with scale = 1 as such.
662	assert(ScaledReg && "Expected 1*reg => reg");
663	assert(Scale == `1` && "Expected 1*reg => reg");
664	BaseRegs.push_back(Elt: ScaledReg);
665	Scale = `0`;
666	ScaledReg = nullptr;
667	return;
668	}
669
670	// Keep the invariant sum in BaseRegs and one of the variant sum in ScaledReg.
671	if (!ScaledReg) {
672	ScaledReg = BaseRegs.pop_back_val();
673	Scale = `1`;
674	}
675
676	// If ScaledReg is an invariant with respect to L, find the reg from
677	// BaseRegs containing the recurrent expr related with Loop L. Swap the
678	// reg with ScaledReg.
679	if (!containsAddRecDependentOnLoop(S: ScaledReg, L)) {
680	auto I = find_if(Range&: BaseRegs, P: [&L](const SCEV *S) {
681	return containsAddRecDependentOnLoop(S, L);
682	});
683	if (I != BaseRegs.end())
684	std::swap(a&: ScaledReg, b&: *I);
685	}
686	assert(isCanonical(L) && "Failed to canonicalize?");
687	}
688
689	/// Get rid of the scale in the formula.
690	/// In other words, this method morphes reg1 + 1reg2 into reg1 + reg2.*
691	/// \return true if it was possible to get rid of the scale, false otherwise.
692	/// \note After this operation the formula may not be in the canonical form.
693	bool Formula::unscale() {
694	if (Scale != `1`)
695	return false;
696	Scale = `0`;
697	BaseRegs.push_back(Elt: ScaledReg);
698	ScaledReg = nullptr;
699	return true;
700	}
701
702	bool Formula::hasZeroEnd() const {
703	if (UnfoldedOffset \|\| BaseOffset)
704	return false;
705	if (BaseRegs.size() != `1` \|\| ScaledReg)
706	return false;
707	return true;
708	}
709
710	bool Formula::countsDownToZero() const {
711	if (!hasZeroEnd())
712	return false;
713	assert(BaseRegs.size() == `1` && "hasZeroEnd should mean one BaseReg");
714	const APInt *StepInt;
715	if (!match(S: BaseRegs [`0`], P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_scev_APInt(C&: StepInt))))
716	return false;
717	return StepInt->isNegative();
718	}
719
720	/// Return the total number of register operands used by this formula. This does
721	/// not include register uses implied by non-constant addrec strides.
722	size_t Formula::getNumRegs() const {
723	return !!ScaledReg + BaseRegs.size();
724	}
725
726	/// Return the type of this formula, if it has one, or null otherwise. This type
727	/// is meaningless except for the bit size.
728	Type Formula::getType() const* {
729	return !BaseRegs.empty() ? BaseRegs.front()->getType() :
730	ScaledReg ? ScaledReg->getType() :
731	BaseGV ? BaseGV->getType() :
732	nullptr;
733	}
734
735	/// Delete the given base reg from the BaseRegs list.
736	void Formula::deleteBaseReg(const SCEV *&S) {
737	if (&S != &BaseRegs.back())
738	std::swap(a&: S, b&: BaseRegs.back());
739	BaseRegs.pop_back();
740	}
741
742	/// Test if this formula references the given register.
743	bool Formula::referencesReg(const SCEV S) const* {
744	return S == ScaledReg \|\| is_contained(Range: BaseRegs, Element: S);
745	}
746
747	/// Test whether this formula uses registers which are used by uses other than
748	/// the use with the given index.
749	bool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx,
750	const RegUseTracker &RegUses) const {
751	if (ScaledReg)
752	if (RegUses.isRegUsedByUsesOtherThan(Reg: ScaledReg, LUIdx))
753	return true;
754	for (const SCEV *BaseReg : BaseRegs)
755	if (RegUses.isRegUsedByUsesOtherThan(Reg: BaseReg, LUIdx))
756	return true;
757	return false;
758	}
759
760	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
761	void Formula::print(raw_ostream &OS) const {
762	ListSeparator Plus(" + ");
763	if (BaseGV) {
764	OS << Plus;
765	BaseGV->printAsOperand(OS, /PrintType=/false);
766	}
767	if (BaseOffset.isNonZero())
768	OS << Plus << BaseOffset;
769
770	for (const SCEV *BaseReg : BaseRegs)
771	OS << Plus << "reg(" << *BaseReg << `')'`;
772
773	if (HasBaseReg && BaseRegs.empty())
774	OS << Plus << "error: HasBaseReg";
775	else if (!HasBaseReg && !BaseRegs.empty())
776	OS << Plus << "error: !HasBaseReg";
777
778	if (Scale != `0`) {
779	OS << Plus << Scale << "*reg(";
780	if (ScaledReg)
781	OS << *ScaledReg;
782	else
783	OS << "<unknown>";
784	OS << `')'`;
785	}
786	if (UnfoldedOffset.isNonZero())
787	OS << Plus << "imm(" << UnfoldedOffset << `')'`;
788	}
789
790	LLVM_DUMP_METHOD void Formula::dump() const {
791	print(errs()); errs() << `'\n'`;
792	}
793	#endif
794
795	/// Return true if the given addrec can be sign-extended without changing its
796	/// value.
797	static bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
798	Type *WideTy =
799	IntegerType::get(C&: SE.getContext(), NumBits: SE.getTypeSizeInBits(Ty: AR->getType()) + `1`);
800	return isa<SCEVAddRecExpr>(Val: SE.getSignExtendExpr(Op: AR, Ty: WideTy));
801	}
802
803	/// Return true if the given add can be sign-extended without changing its
804	/// value.
805	static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) {
806	Type *WideTy =
807	IntegerType::get(C&: SE.getContext(), NumBits: SE.getTypeSizeInBits(Ty: A->getType()) + `1`);
808	return isa<SCEVAddExpr>(Val: SE.getSignExtendExpr(Op: A, Ty: WideTy));
809	}
810
811	/// Return true if the given mul can be sign-extended without changing its
812	/// value.
813	static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) {
814	Type *WideTy =
815	IntegerType::get(C&: SE.getContext(),
816	NumBits: SE.getTypeSizeInBits(Ty: M->getType()) * M->getNumOperands());
817	return isa<SCEVMulExpr>(Val: SE.getSignExtendExpr(Op: M, Ty: WideTy));
818	}
819
820	/// Return an expression for LHS /s RHS, if it can be determined and if the
821	/// remainder is known to be zero, or null otherwise. If IgnoreSignificantBits
822	/// is true, expressions like (X Y) /s Y are simplified to X, ignoring that*
823	/// the multiplication may overflow, which is useful when the result will be
824	/// used in a context where the most significant bits are ignored.
825	static const SCEV getExactSDiv(const* SCEV LHS, const* SCEV *RHS,
826	ScalarEvolution &SE,
827	bool IgnoreSignificantBits = false) {
828	// Handle the trivial case, which works for any SCEV type.
829	if (LHS == RHS)
830	return SE.getConstant(Ty: LHS->getType(), V: `1`);
831
832	// Handle a few RHS special cases.
833	const SCEVConstant *RC = dyn_cast<SCEVConstant>(Val: RHS);
834	if (RC) {
835	const APInt &RA = RC->getAPInt();
836	// Handle x /s -1 as x -1, to give ScalarEvolution a chance to do*
837	// some folding.
838	if (RA.isAllOnes()) {
839	if (LHS->getType()->isPointerTy())
840	return nullptr;
841	return SE.getMulExpr(LHS, RHS: RC);
842	}
843	// Handle x /s 1 as x.
844	if (RA == `1`)
845	return LHS;
846	}
847
848	// Check for a division of a constant by a constant.
849	if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: LHS)) {
850	if (!RC)
851	return nullptr;
852	const APInt &LA = C->getAPInt();
853	const APInt &RA = RC->getAPInt();
854	if (LA.srem(RHS: RA) != `0`)
855	return nullptr;
856	return SE.getConstant(Val: LA.sdiv(RHS: RA));
857	}
858
859	// Distribute the sdiv over addrec operands, if the addrec doesn't overflow.
860	if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS)) {
861	if ((IgnoreSignificantBits \|\| isAddRecSExtable(AR, SE)) && AR->isAffine()) {
862	const SCEV *Step = getExactSDiv(LHS: AR->getStepRecurrence(SE), RHS, SE,
863	IgnoreSignificantBits);
864	if (!Step) return nullptr;
865	const SCEV *Start = getExactSDiv(LHS: AR->getStart(), RHS, SE,
866	IgnoreSignificantBits);
867	if (!Start) return nullptr;
868	// FlagNW is independent of the start value, step direction, and is
869	// preserved with smaller magnitude steps.
870	// FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
871	return SE.getAddRecExpr(Start, Step, L: AR->getLoop(), Flags: SCEV::FlagAnyWrap);
872	}
873	return nullptr;
874	}
875
876	// Distribute the sdiv over add operands, if the add doesn't overflow.
877	if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: LHS)) {
878	if (IgnoreSignificantBits \|\| isAddSExtable(A: Add, SE)) {
879	SmallVector<SCEVUse, `8`> Ops;
880	for (const SCEV *S : Add->operands()) {
881	const SCEV *Op = getExactSDiv(LHS: S, RHS, SE, IgnoreSignificantBits);
882	if (!Op) return nullptr;
883	Ops.push_back(Elt: Op);
884	}
885	return SE.getAddExpr(Ops);
886	}
887	return nullptr;
888	}
889
890	// Check for a multiply operand that we can pull RHS out of.
891	if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: LHS)) {
892	if (IgnoreSignificantBits \|\| isMulSExtable(M: Mul, SE)) {
893	// Handle special case C1XY /s C2XY.
894	if (const SCEVMulExpr *MulRHS = dyn_cast<SCEVMulExpr>(Val: RHS)) {
895	if (IgnoreSignificantBits \|\| isMulSExtable(M: MulRHS, SE)) {
896	const SCEVConstant *LC = dyn_cast<SCEVConstant>(Val: Mul->getOperand(i: `0`));
897	const SCEVConstant *RC =
898	dyn_cast<SCEVConstant>(Val: MulRHS->getOperand(i: `0`));
899	if (LC && RC) {
900	SmallVector<const SCEV *, `4`> LOps(drop_begin(RangeOrContainer: Mul->operands()));
901	SmallVector<const SCEV *, `4`> ROps(drop_begin(RangeOrContainer: MulRHS->operands()));
902	if (LOps == ROps)
903	return getExactSDiv(LHS: LC, RHS: RC, SE, IgnoreSignificantBits);
904	}
905	}
906	}
907
908	SmallVector<SCEVUse, `4`> Ops;
909	bool Found = false;
910	for (const SCEV *S : Mul->operands()) {
911	if (!Found)
912	if (const SCEV *Q = getExactSDiv(LHS: S, RHS, SE,
913	IgnoreSignificantBits)) {
914	S = Q;
915	Found = true;
916	}
917	Ops.push_back(Elt: S);
918	}
919	return Found ? SE.getMulExpr(Ops) : nullptr;
920	}
921	return nullptr;
922	}
923
924	// Otherwise we don't know.
925	return nullptr;
926	}
927
928	/// Extracts an immediate operand from \p Ops and replaces the operand with
929	/// zero. If \p PreferScalable is true and \p Ops contains both a scalable and
930	/// non-scalable offsets, the scalable offset will be extracted.
931	static Immediate ExtractImmediateOperand(MutableArrayRef<SCEVUse> Ops,
932	ScalarEvolution &SE,
933	bool PreferScalable) {
934	const APInt *C;
935	SCEVUse Op = nullptr*;
936	Immediate Result = Immediate::getZero();
937
938	// Ops are sorted by their SCEVType (the order of SCEVTypes enum). So, for an
939	// AddExpr the possible order of operands is:
940	// Constant < VScale < Truncate < ZeroExtend < SignExtend < MulExpr < ...
941
942	// This means fixed-size immediates will always appear on the LHS:
943	SCEVUse &S = Ops.front();
944	if (match(U: S, P: m_scev_APInt(C)) && !C->isZero() &&
945	C->getSignificantBits() <= `64`) {
946	Op = &S;
947	Result = Immediate::getFixed(MinVal: C->getSExtValue());
948	}
949
950	// But scalable immediates, which are MulExpr(Vscale, Constant), can appear
951	// later in the operand list:
952	if (EnableVScaleImmediates && (Result.isZero() \|\| PreferScalable)) {
953	for (SCEVUse &S : Ops) {
954	// We know anything past scMulExpr will not be a vscale immediate.
955	if (S ->getSCEVType() > scMulExpr)
956	break;
957	if (match(U: S, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEVVScale()))) {
958	Op = &S;
959	Result = Immediate::getScalable(MinVal: C->getSExtValue());
960	break;
961	}
962	}
963	}
964
965	if (Result.isNonZero()) {
966	SCEVUse &S = *Op;
967	S = SE.getConstant(Ty: S ->getType(), V: `0`);
968	}
969
970	return Result;
971	}
972
973	/// If S involves the addition of a constant integer value, return that integer
974	/// value, and mutate S to point to a new SCEV with that value excluded.
975	static Immediate ExtractImmediate(SCEVUse &S, ScalarEvolution &SE,
976	bool PreferScalable = false) {
977	if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: S)) {
978	SmallVector<SCEVUse, `8`> NewOps(Add->operands());
979	Immediate Result = ExtractImmediateOperand(Ops: NewOps, SE, PreferScalable);
980	if (Result.isZero())
981	Result = ExtractImmediate(S&: NewOps.front(), SE, PreferScalable);
982	if (Result.isNonZero())
983	S = SE.getAddExpr(Ops&: NewOps);
984	return Result;
985	} else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: S)) {
986	SmallVector<SCEVUse, `8`> NewOps(AR->operands());
987	Immediate Result = ExtractImmediate(S&: NewOps.front(), SE, PreferScalable);
988	if (Result.isNonZero())
989	S = SE.getAddRecExpr(Operands&: NewOps, L: AR->getLoop(),
990	// FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
991	Flags: SCEV::FlagAnyWrap);
992	return Result;
993	}
994	return ExtractImmediateOperand(Ops: {S}, SE, PreferScalable);
995	}
996
997	/// If S involves the addition of a GlobalValue address, return that symbol, and
998	/// mutate S to point to a new SCEV with that value excluded.
999	static GlobalValue *ExtractSymbol(SCEVUse &S, ScalarEvolution &SE) {
1000	if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Val&: S)) {
1001	if (GlobalValue *GV = dyn_cast<GlobalValue>(Val: U->getValue())) {
1002	S = SE.getConstant(Ty: GV->getType(), V: `0`);
1003	return GV;
1004	}
1005	} else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: S)) {
1006	SmallVector<SCEVUse, `8`> NewOps(Add->operands());
1007	GlobalValue *Result = ExtractSymbol(S&: NewOps.back(), SE);
1008	if (Result)
1009	S = SE.getAddExpr(Ops&: NewOps);
1010	return Result;
1011	} else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: S)) {
1012	SmallVector<SCEVUse, `8`> NewOps(AR->operands());
1013	GlobalValue *Result = ExtractSymbol(S&: NewOps.front(), SE);
1014	if (Result)
1015	S = SE.getAddRecExpr(Operands&: NewOps, L: AR->getLoop(),
1016	// FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
1017	Flags: SCEV::FlagAnyWrap);
1018	return Result;
1019	}
1020	return nullptr;
1021	}
1022
1023	/// Returns true if the specified instruction is using the specified value as an
1024	/// address.
1025	static bool isAddressUse(const TargetTransformInfo &TTI,
1026	Instruction Inst, Value OperandVal) {
1027	bool isAddress = isa<LoadInst>(Val: Inst);
1028	if (StoreInst *SI = dyn_cast<StoreInst>(Val: Inst)) {
1029	if (SI->getPointerOperand() == OperandVal)
1030	isAddress = true;
1031	} else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Inst)) {
1032	// Addressing modes can also be folded into prefetches and a variety
1033	// of intrinsics.
1034	switch (II->getIntrinsicID()) {
1035	case Intrinsic::memset:
1036	case Intrinsic::prefetch:
1037	case Intrinsic::masked_load:
1038	if (II->getArgOperand(i: `0`) == OperandVal)
1039	isAddress = true;
1040	break;
1041	case Intrinsic::masked_store:
1042	if (II->getArgOperand(i: `1`) == OperandVal)
1043	isAddress = true;
1044	break;
1045	case Intrinsic::memmove:
1046	case Intrinsic::memcpy:
1047	if (II->getArgOperand(i: `0`) == OperandVal \|\|
1048	II->getArgOperand(i: `1`) == OperandVal)
1049	isAddress = true;
1050	break;
1051	default: {
1052	MemIntrinsicInfo IntrInfo;
1053	if (TTI.getTgtMemIntrinsic(Inst: II, Info&: IntrInfo)) {
1054	if (IntrInfo.PtrVal == OperandVal)
1055	isAddress = true;
1056	}
1057	}
1058	}
1059	} else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
1060	if (RMW->getPointerOperand() == OperandVal)
1061	isAddress = true;
1062	} else if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
1063	if (CmpX->getPointerOperand() == OperandVal)
1064	isAddress = true;
1065	}
1066	return isAddress;
1067	}
1068
1069	/// Return the type of the memory being accessed.
1070	static MemAccessTy getAccessType(const TargetTransformInfo &TTI,
1071	Instruction Inst, Value OperandVal) {
1072	MemAccessTy AccessTy = MemAccessTy::getUnknown(Ctx&: Inst->getContext());
1073
1074	// First get the type of memory being accessed.
1075	if (Type *Ty = Inst->getAccessType())
1076	AccessTy.MemTy = Ty;
1077
1078	// Then get the pointer address space.
1079	if (const StoreInst *SI = dyn_cast<StoreInst>(Val: Inst)) {
1080	AccessTy.AddrSpace = SI->getPointerAddressSpace();
1081	} else if (const LoadInst *LI = dyn_cast<LoadInst>(Val: Inst)) {
1082	AccessTy.AddrSpace = LI->getPointerAddressSpace();
1083	} else if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
1084	AccessTy.AddrSpace = RMW->getPointerAddressSpace();
1085	} else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
1086	AccessTy.AddrSpace = CmpX->getPointerAddressSpace();
1087	} else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Inst)) {
1088	switch (II->getIntrinsicID()) {
1089	case Intrinsic::prefetch:
1090	case Intrinsic::memset:
1091	AccessTy.AddrSpace = II->getArgOperand(i: `0`)->getType()->getPointerAddressSpace();
1092	AccessTy.MemTy = OperandVal->getType();
1093	break;
1094	case Intrinsic::memmove:
1095	case Intrinsic::memcpy:
1096	AccessTy.AddrSpace = OperandVal->getType()->getPointerAddressSpace();
1097	AccessTy.MemTy = OperandVal->getType();
1098	break;
1099	case Intrinsic::masked_load:
1100	AccessTy.AddrSpace =
1101	II->getArgOperand(i: `0`)->getType()->getPointerAddressSpace();
1102	break;
1103	case Intrinsic::masked_store:
1104	AccessTy.AddrSpace =
1105	II->getArgOperand(i: `1`)->getType()->getPointerAddressSpace();
1106	break;
1107	default: {
1108	MemIntrinsicInfo IntrInfo;
1109	if (TTI.getTgtMemIntrinsic(Inst: II, Info&: IntrInfo) && IntrInfo.PtrVal) {
1110	AccessTy.AddrSpace
1111	= IntrInfo.PtrVal->getType()->getPointerAddressSpace();
1112	}
1113
1114	break;
1115	}
1116	}
1117	}
1118
1119	return AccessTy;
1120	}
1121
1122	/// Return true if this AddRec is already a phi in its loop.
1123	static bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
1124	for (PHINode &PN : AR->getLoop()->getHeader()->phis()) {
1125	if (SE.isSCEVable(Ty: PN.getType()) &&
1126	(SE.getEffectiveSCEVType(Ty: PN.getType()) ==
1127	SE.getEffectiveSCEVType(Ty: AR->getType())) &&
1128	SE.getSCEV(V: &PN) == AR)
1129	return true;
1130	}
1131	return false;
1132	}
1133
1134	/// Check if expanding this expression is likely to incur significant cost. This
1135	/// is tricky because SCEV doesn't track which expressions are actually computed
1136	/// by the current IR.
1137	///
1138	/// We currently allow expansion of IV increments that involve adds,
1139	/// multiplication by constants, and AddRecs from existing phis.
1140	///
1141	/// TODO: Allow UDivExpr if we can find an existing IV increment that is an
1142	/// obvious multiple of the UDivExpr.
1143	static bool isHighCostExpansion(const SCEV *S,
1144	SmallPtrSetImpl<const SCEV*> &Processed,
1145	ScalarEvolution &SE) {
1146	// Zero/One operand expressions
1147	switch (S->getSCEVType()) {
1148	case scUnknown:
1149	case scConstant:
1150	case scVScale:
1151	return false;
1152	case scTruncate:
1153	return isHighCostExpansion(S: cast<SCEVTruncateExpr>(Val: S)->getOperand(),
1154	Processed, SE);
1155	case scZeroExtend:
1156	return isHighCostExpansion(S: cast<SCEVZeroExtendExpr>(Val: S)->getOperand(),
1157	Processed, SE);
1158	case scSignExtend:
1159	return isHighCostExpansion(S: cast<SCEVSignExtendExpr>(Val: S)->getOperand(),
1160	Processed, SE);
1161	default:
1162	break;
1163	}
1164
1165	if (!Processed.insert(Ptr: S).second)
1166	return false;
1167
1168	if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
1169	for (const SCEV *S : Add->operands()) {
1170	if (isHighCostExpansion(S, Processed, SE))
1171	return true;
1172	}
1173	return false;
1174	}
1175
1176	const SCEV Op0, Op1;
1177	if (match(S, P: m_scev_Mul(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1)))) {
1178	// Multiplication by a constant is ok
1179	if (isa<SCEVConstant>(Val: Op0))
1180	return isHighCostExpansion(S: Op1, Processed, SE);
1181
1182	// If we have the value of one operand, check if an existing
1183	// multiplication already generates this expression.
1184	if (const auto *U = dyn_cast<SCEVUnknown>(Val: Op1)) {
1185	Value *UVal = U->getValue();
1186	for (User *UR : UVal->users()) {
1187	// If U is a constant, it may be used by a ConstantExpr.
1188	Instruction *UI = dyn_cast<Instruction>(Val: UR);
1189	if (UI && UI->getOpcode() == Instruction::Mul &&
1190	SE.isSCEVable(Ty: UI->getType())) {
1191	return SE.getSCEV(V: UI) == S;
1192	}
1193	}
1194	}
1195	}
1196
1197	if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
1198	if (isExistingPhi(AR, SE))
1199	return false;
1200	}
1201
1202	// Fow now, consider any other type of expression (div/mul/min/max) high cost.
1203	return true;
1204	}
1205
1206	namespace {
1207
1208	class LSRUse;
1209
1210	} // end anonymous namespace
1211
1212	/// Check if the addressing mode defined by \p F is completely
1213	/// folded in \p LU at isel time.
1214	/// This includes address-mode folding and special icmp tricks.
1215	/// This function returns true if \p LU can accommodate what \p F
1216	/// defines and up to 1 base + 1 scaled + offset.
1217	/// In other words, if \p F has several base registers, this function may
1218	/// still return true. Therefore, users still need to account for
1219	/// additional base registers and/or unfolded offsets to derive an
1220	/// accurate cost model.
1221	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1222	const LSRUse &LU, const Formula &F);
1223
1224	// Get the cost of the scaling factor used in F for LU.
1225	static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI,
1226	const LSRUse &LU, const Formula &F,
1227	const Loop &L);
1228
1229	namespace {
1230
1231	/// This class is used to measure and compare candidate formulae.
1232	class Cost {
1233	const Loop L = nullptr*;
1234	ScalarEvolution SE = nullptr*;
1235	const TargetTransformInfo TTI = nullptr*;
1236	TargetTransformInfo::LSRCost C;
1237	TTI::AddressingModeKind AMK = TTI::AMK_None;
1238
1239	public:
1240	Cost() = delete;
1241	Cost(const Loop L, ScalarEvolution &SE, const* TargetTransformInfo &TTI,
1242	TTI::AddressingModeKind AMK) :
1243	L(L), SE(&SE), TTI(&TTI), AMK(AMK) {
1244	C.Insns = `0`;
1245	C.NumRegs = `0`;
1246	C.AddRecCost = `0`;
1247	C.NumIVMuls = `0`;
1248	C.NumBaseAdds = `0`;
1249	C.ImmCost = `0`;
1250	C.SetupCost = `0`;
1251	C.ScaleCost = `0`;
1252	}
1253
1254	bool isLess(const Cost &Other) const;
1255
1256	void Lose();
1257
1258	#ifndef NDEBUG
1259	// Once any of the metrics loses, they must all remain losers.
1260	bool isValid() {
1261	return ((C.Insns \| C.NumRegs \| C.AddRecCost \| C.NumIVMuls \| C.NumBaseAdds
1262	\| C.ImmCost \| C.SetupCost \| C.ScaleCost) != ~`0u`)
1263	\|\| ((C.Insns & C.NumRegs & C.AddRecCost & C.NumIVMuls & C.NumBaseAdds
1264	& C.ImmCost & C.SetupCost & C.ScaleCost) == ~`0u`);
1265	}
1266	#endif
1267
1268	bool isLoser() {
1269	assert(isValid() && "invalid cost");
1270	return C.NumRegs == ~`0u`;
1271	}
1272
1273	void RateFormula(const Formula &F, SmallPtrSetImpl<const SCEV *> &Regs,
1274	const DenseSet<const SCEV > &VisitedRegs, const* LSRUse &LU,
1275	bool HardwareLoopProfitable,
1276	SmallPtrSetImpl<const SCEV > LoserRegs = nullptr);
1277
1278	void print(raw_ostream &OS) const;
1279	void dump() const;
1280
1281	private:
1282	void RateRegister(const Formula &F, const SCEV *Reg,
1283	SmallPtrSetImpl<const SCEV > &Regs, const* LSRUse &LU,
1284	bool HardwareLoopProfitable);
1285	void RatePrimaryRegister(const Formula &F, const SCEV *Reg,
1286	SmallPtrSetImpl<const SCEV *> &Regs,
1287	const LSRUse &LU, bool HardwareLoopProfitable,
1288	SmallPtrSetImpl<const SCEV > LoserRegs);
1289	};
1290
1291	/// An operand value in an instruction which is to be replaced with some
1292	/// equivalent, possibly strength-reduced, replacement.
1293	struct LSRFixup {
1294	/// The instruction which will be updated.
1295	Instruction UserInst = nullptr*;
1296
1297	/// The operand of the instruction which will be replaced. The operand may be
1298	/// used more than once; every instance will be replaced.
1299	Value OperandValToReplace = nullptr*;
1300
1301	/// If this user is to use the post-incremented value of an induction
1302	/// variable, this set is non-empty and holds the loops associated with the
1303	/// induction variable.
1304	PostIncLoopSet PostIncLoops;
1305
1306	/// A constant offset to be added to the LSRUse expression. This allows
1307	/// multiple fixups to share the same LSRUse with different offsets, for
1308	/// example in an unrolled loop.
1309	Immediate Offset = Immediate::getZero();
1310
1311	LSRFixup() = default;
1312
1313	bool isUseFullyOutsideLoop(const Loop L) const*;
1314
1315	void print(raw_ostream &OS) const;
1316	void dump() const;
1317	};
1318
1319	/// This class holds the state that LSR keeps for each use in IVUsers, as well
1320	/// as uses invented by LSR itself. It includes information about what kinds of
1321	/// things can be folded into the user, information about the user itself, and
1322	/// information about how the use may be satisfied. TODO: Represent multiple
1323	/// users of the same expression in common?
1324	class LSRUse {
1325	DenseSet<SmallVector<const SCEV *, `4`>> Uniquifier;
1326
1327	public:
1328	/// An enum for a kind of use, indicating what types of scaled and immediate
1329	/// operands it might support.
1330	enum KindType {
1331	Basic, ///< A normal use, with no folding.
1332	Special, ///< A special case of basic, allowing -1 scales.
1333	Address, ///< An address use; folding according to TargetLowering
1334	ICmpZero ///< An equality icmp with both operands folded into one.
1335	// TODO: Add a generic icmp too?
1336	};
1337
1338	using SCEVUseKindPair = PointerIntPair<const SCEV *, `2`, KindType>;
1339
1340	KindType Kind;
1341	MemAccessTy AccessTy;
1342
1343	/// The list of operands which are to be replaced.
1344	SmallVector<LSRFixup, `8`> Fixups;
1345
1346	/// Keep track of the min and max offsets of the fixups.
1347	Immediate MinOffset = Immediate::getFixedMax();
1348	Immediate MaxOffset = Immediate::getFixedMin();
1349
1350	/// This records whether all of the fixups using this LSRUse are outside of
1351	/// the loop, in which case some special-case heuristics may be used.
1352	bool AllFixupsOutsideLoop = true;
1353
1354	/// This records whether all of the fixups using this LSRUse are unconditional
1355	/// within the loop, meaning they will be executed on every path to the loop
1356	/// latch. This includes fixups before early exits.
1357	bool AllFixupsUnconditional = true;
1358
1359	/// RigidFormula is set to true to guarantee that this use will be associated
1360	/// with a single formula--the one that initially matched. Some SCEV
1361	/// expressions cannot be expanded. This allows LSR to consider the registers
1362	/// used by those expressions without the need to expand them later after
1363	/// changing the formula.
1364	bool RigidFormula = false;
1365
1366	/// A list of ways to build a value that can satisfy this user. After the
1367	/// list is populated, one of these is selected heuristically and used to
1368	/// formulate a replacement for OperandValToReplace in UserInst.
1369	SmallVector<Formula, `12`> Formulae;
1370
1371	/// The set of register candidates used by all formulae in this LSRUse.
1372	SmallPtrSet<const SCEV *, `4`> Regs;
1373
1374	LSRUse(KindType K, MemAccessTy AT) : Kind(K), AccessTy (AT) {}
1375
1376	LSRFixup &getNewFixup() {
1377	Fixups.push_back(Elt: LSRFixup ());
1378	return Fixups.back();
1379	}
1380
1381	void pushFixup(LSRFixup &f) {
1382	Fixups.push_back(Elt: f);
1383	if (Immediate::isKnownGT(LHS: f.Offset, RHS: MaxOffset))
1384	MaxOffset = f.Offset;
1385	if (Immediate::isKnownLT(LHS: f.Offset, RHS: MinOffset))
1386	MinOffset = f.Offset;
1387	}
1388
1389	bool HasFormulaWithSameRegs(const Formula &F) const;
1390	float getNotSelectedProbability(const SCEV Reg) const*;
1391	bool InsertFormula(const Formula &F, const Loop &L);
1392	void DeleteFormula(Formula &F);
1393	void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses);
1394
1395	void print(raw_ostream &OS) const;
1396	void dump() const;
1397	};
1398
1399	} // end anonymous namespace
1400
1401	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1402	LSRUse::KindType Kind, MemAccessTy AccessTy,
1403	GlobalValue *BaseGV, Immediate BaseOffset,
1404	bool HasBaseReg, int64_t Scale,
1405	Instruction Fixup = nullptr*);
1406
1407	static unsigned getSetupCost(const SCEV Reg, unsigned* Depth,
1408	const TargetTransformInfo &TTI) {
1409	if (isa<SCEVUnknown>(Val: Reg))
1410	return `1`;
1411	if (const auto *C = dyn_cast<SCEVConstant>(Val: Reg)) {
1412	if (TTI.getIntImmCost(Imm: C->getAPInt(), Ty: C->getType(),
1413	CostKind: TargetTransformInfo::TCK_RecipThroughput) ==
1414	TargetTransformInfo::TCC_Free)
1415	return `0`;
1416	return `1`;
1417	}
1418	if (Depth == `0`)
1419	return `0`;
1420	if (const auto *S = dyn_cast<SCEVAddRecExpr>(Val: Reg))
1421	return getSetupCost(Reg: S->getStart(), Depth: Depth - `1`, TTI);
1422	if (auto S = dyn_cast<SCEVIntegralCastExpr>(Val: Reg))
1423	return getSetupCost(Reg: S->getOperand(), Depth: Depth - `1`, TTI);
1424	if (auto S = dyn_cast<SCEVNAryExpr>(Val: Reg))
1425	return std::accumulate(first: S->operands().begin(), last: S->operands().end(), init: `0`,
1426	binary_op: [&](unsigned i, const SCEV *Reg) {
1427	return i + getSetupCost(Reg, Depth: Depth - `1`, TTI);
1428	});
1429	if (auto S = dyn_cast<SCEVUDivExpr>(Val: Reg))
1430	return getSetupCost(Reg: S->getLHS(), Depth: Depth - `1`, TTI) +
1431	getSetupCost(Reg: S->getRHS(), Depth: Depth - `1`, TTI);
1432	return `0`;
1433	}
1434
1435	/// Tally up interesting quantities from the given register.
1436	void Cost::RateRegister(const Formula &F, const SCEV *Reg,
1437	SmallPtrSetImpl<const SCEV > &Regs, const* LSRUse &LU,
1438	bool HardwareLoopProfitable) {
1439	if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Reg)) {
1440	// If this is an addrec for another loop, it should be an invariant
1441	// with respect to L since L is the innermost loop (at least
1442	// for now LSR only handles innermost loops).
1443	if (AR->getLoop() != L) {
1444	// If the AddRec exists, consider it's register free and leave it alone.
1445	if (isExistingPhi(AR, SE&: *SE) && !(AMK & TTI::AMK_PostIndexed))
1446	return;
1447
1448	// It is bad to allow LSR for current loop to add induction variables
1449	// for its sibling loops.
1450	if (!AR->getLoop()->contains(L)) {
1451	Lose();
1452	return;
1453	}
1454
1455	// Otherwise, it will be an invariant with respect to Loop L.
1456	++C.NumRegs;
1457	return;
1458	}
1459
1460	unsigned LoopCost = `1`;
1461	if (TTI->isIndexedLoadLegal(Mode: TTI->MIM_PostInc, Ty: AR->getType()) \|\|
1462	TTI->isIndexedStoreLegal(Mode: TTI->MIM_PostInc, Ty: AR->getType())) {
1463	const SCEV *Start;
1464	const APInt *Step;
1465	if (match(S: AR, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_scev_APInt(C&: Step)))) {
1466	// If the step size matches the base offset, we could use pre-indexed
1467	// addressing.
1468	bool CanPreIndex = (AMK & TTI::AMK_PreIndexed) &&
1469	F.BaseOffset.isFixed() &&
1470	*Step == F.BaseOffset.getFixedValue();
1471	bool CanPostIndex = (AMK & TTI::AMK_PostIndexed) &&
1472	!isa<SCEVConstant>(Val: Start) &&
1473	SE->isLoopInvariant(S: Start, L);
1474	// We can only pre or post index when the load/store is unconditional.
1475	if ((CanPreIndex \|\| CanPostIndex) && LU.AllFixupsUnconditional)
1476	LoopCost = `0`;
1477	}
1478	}
1479
1480	// If the loop counts down to zero and we'll be using a hardware loop then
1481	// the addrec will be combined into the hardware loop instruction.
1482	if (LU.Kind == LSRUse::ICmpZero && F.countsDownToZero() &&
1483	HardwareLoopProfitable)
1484	LoopCost = `0`;
1485	C.AddRecCost += LoopCost;
1486
1487	// Add the step value register, if it needs one.
1488	// TODO: The non-affine case isn't precisely modeled here.
1489	const SCEV *StepReg = AR->getOperand(i: `1`);
1490	if (!AR->isAffine() \|\| !isa<SCEVConstant>(Val: StepReg)) {
1491	// If the step amount is a constant multiplied by vscale then it can form
1492	// the immediate value of an add and doesn't use a register, so long as
1493	// the immediate value is legal.
1494	auto IsVScaleStep = [](const SCEV Reg, const* TargetTransformInfo *TTI) {
1495	const APInt *X;
1496	if (!match(S: Reg, P: m_scev_Mul(Op0: m_scev_APInt(C&: X), Op1: m_SCEVVScale())))
1497	return false;
1498	return TTI->isLegalAddScalableImmediate(Imm: X->getLimitedValue());
1499	};
1500	if (!Regs.count(Ptr: StepReg) && !IsVScaleStep (StepReg, TTI)) {
1501	RateRegister(F, Reg: StepReg, Regs, LU, HardwareLoopProfitable);
1502	if (isLoser())
1503	return;
1504	}
1505	}
1506	}
1507	++C.NumRegs;
1508
1509	// Rough heuristic; favor registers which don't require extra setup
1510	// instructions in the preheader.
1511	C.SetupCost += getSetupCost(Reg, Depth: SetupCostDepthLimit, TTI: *TTI);
1512	// Ensure we don't, even with the recusion limit, produce invalid costs.
1513	C.SetupCost = std::min<unsigned>(a: C.SetupCost, b: `1` << `16`);
1514
1515	C.NumIVMuls += isa<SCEVMulExpr>(Val: Reg) &&
1516	SE->hasComputableLoopEvolution(S: Reg, L);
1517	}
1518
1519	/// Record this register in the set. If we haven't seen it before, rate
1520	/// it. Optional LoserRegs provides a way to declare any formula that refers to
1521	/// one of those regs an instant loser.
1522	void Cost::RatePrimaryRegister(const Formula &F, const SCEV *Reg,
1523	SmallPtrSetImpl<const SCEV *> &Regs,
1524	const LSRUse &LU, bool HardwareLoopProfitable,
1525	SmallPtrSetImpl<const SCEV > LoserRegs) {
1526	if (LoserRegs && LoserRegs->count(Ptr: Reg)) {
1527	Lose();
1528	return;
1529	}
1530	if (Regs.insert(Ptr: Reg).second) {
1531	RateRegister(F, Reg, Regs, LU, HardwareLoopProfitable);
1532	if (LoserRegs && isLoser())
1533	LoserRegs->insert(Ptr: Reg);
1534	}
1535	}
1536
1537	void Cost::RateFormula(const Formula &F, SmallPtrSetImpl<const SCEV *> &Regs,
1538	const DenseSet<const SCEV *> &VisitedRegs,
1539	const LSRUse &LU, bool HardwareLoopProfitable,
1540	SmallPtrSetImpl<const SCEV > LoserRegs) {
1541	if (isLoser())
1542	return;
1543	assert(F.isCanonical(*L) && "Cost is accurate only for canonical formula");
1544	// Tally up the registers.
1545	unsigned PrevAddRecCost = C.AddRecCost;
1546	unsigned PrevNumRegs = C.NumRegs;
1547	unsigned PrevNumBaseAdds = C.NumBaseAdds;
1548	if (const SCEV *ScaledReg = F.ScaledReg) {
1549	if (VisitedRegs.count(V: ScaledReg)) {
1550	Lose();
1551	return;
1552	}
1553	RatePrimaryRegister(F, Reg: ScaledReg, Regs, LU, HardwareLoopProfitable,
1554	LoserRegs);
1555	if (isLoser())
1556	return;
1557	}
1558	for (const SCEV *BaseReg : F.BaseRegs) {
1559	if (VisitedRegs.count(V: BaseReg)) {
1560	Lose();
1561	return;
1562	}
1563	RatePrimaryRegister(F, Reg: BaseReg, Regs, LU, HardwareLoopProfitable,
1564	LoserRegs);
1565	if (isLoser())
1566	return;
1567	}
1568
1569	// Determine how many (unfolded) adds we'll need inside the loop.
1570	size_t NumBaseParts = F.getNumRegs();
1571	if (NumBaseParts > `1`)
1572	// Do not count the base and a possible second register if the target
1573	// allows to fold 2 registers.
1574	C.NumBaseAdds +=
1575	NumBaseParts - (`1` + (F.Scale && isAMCompletelyFolded(TTI: *TTI, LU, F)));
1576	C.NumBaseAdds += (F.UnfoldedOffset.isNonZero());
1577
1578	// Accumulate non-free scaling amounts.
1579	C.ScaleCost += getScalingFactorCost(TTI: TTI, LU, F, L: L).getValue();
1580
1581	// Tally up the non-zero immediates.
1582	for (const LSRFixup &Fixup : LU.Fixups) {
1583	if (Fixup.Offset.isCompatibleImmediate(Imm: F.BaseOffset)) {
1584	Immediate Offset = Fixup.Offset.addUnsigned(RHS: F.BaseOffset);
1585	if (F.BaseGV)
1586	C.ImmCost += `64`; // Handle symbolic values conservatively.
1587	// TODO: This should probably be the pointer size.
1588	else if (Offset.isNonZero())
1589	C.ImmCost +=
1590	APInt (`64`, Offset.getKnownMinValue(), true).getSignificantBits();
1591
1592	// Check with target if this offset with this instruction is
1593	// specifically not supported.
1594	if (LU.Kind == LSRUse::Address && Offset.isNonZero() &&
1595	!isAMCompletelyFolded(TTI: *TTI, Kind: LSRUse::Address, AccessTy: LU.AccessTy, BaseGV: F.BaseGV,
1596	BaseOffset: Offset, HasBaseReg: F.HasBaseReg, Scale: F.Scale, Fixup: Fixup.UserInst))
1597	C.NumBaseAdds++;
1598	} else {
1599	// Incompatible immediate type, increase cost to avoid using
1600	C.ImmCost += `2048`;
1601	}
1602	}
1603
1604	// If we don't count instruction cost exit here.
1605	if (!InsnsCost) {
1606	assert(isValid() && "invalid cost");
1607	return;
1608	}
1609
1610	// Treat every new register that exceeds TTI.getNumberOfRegisters() - 1 as
1611	// additional instruction (at least fill).
1612	// TODO: Need distinguish register class?
1613	unsigned TTIRegNum = TTI->getNumberOfRegisters(
1614	ClassID: TTI->getRegisterClassForType(Vector: false, Ty: F.getType())) - `1`;
1615	if (C.NumRegs > TTIRegNum) {
1616	// Cost already exceeded TTIRegNum, then only newly added register can add
1617	// new instructions.
1618	if (PrevNumRegs > TTIRegNum)
1619	C.Insns += (C.NumRegs - PrevNumRegs);
1620	else
1621	C.Insns += (C.NumRegs - TTIRegNum);
1622	}
1623
1624	// If ICmpZero formula ends with not 0, it could not be replaced by
1625	// just add or sub. We'll need to compare final result of AddRec.
1626	// That means we'll need an additional instruction. But if the target can
1627	// macro-fuse a compare with a branch, don't count this extra instruction.
1628	// For -10 + {0, +, 1}:
1629	// i = i + 1;
1630	// cmp i, 10
1631	//
1632	// For {-10, +, 1}:
1633	// i = i + 1;
1634	if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd() &&
1635	!TTI->canMacroFuseCmp())
1636	C.Insns++;
1637	// Each new AddRec adds 1 instruction to calculation.
1638	C.Insns += (C.AddRecCost - PrevAddRecCost);
1639
1640	// BaseAdds adds instructions for unfolded registers.
1641	if (LU.Kind != LSRUse::ICmpZero)
1642	C.Insns += C.NumBaseAdds - PrevNumBaseAdds;
1643	assert(isValid() && "invalid cost");
1644	}
1645
1646	/// Set this cost to a losing value.
1647	void Cost::Lose() {
1648	C.Insns = std::numeric_limits<unsigned>::max();
1649	C.NumRegs = std::numeric_limits<unsigned>::max();
1650	C.AddRecCost = std::numeric_limits<unsigned>::max();
1651	C.NumIVMuls = std::numeric_limits<unsigned>::max();
1652	C.NumBaseAdds = std::numeric_limits<unsigned>::max();
1653	C.ImmCost = std::numeric_limits<unsigned>::max();
1654	C.SetupCost = std::numeric_limits<unsigned>::max();
1655	C.ScaleCost = std::numeric_limits<unsigned>::max();
1656	}
1657
1658	/// Choose the lower cost.
1659	bool Cost::isLess(const Cost &Other) const {
1660	if (InsnsCost.getNumOccurrences() > `0` && InsnsCost &&
1661	C.Insns != Other.C.Insns)
1662	return C.Insns < Other.C.Insns;
1663	return TTI->isLSRCostLess(C1: C, C2: Other.C);
1664	}
1665
1666	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1667	void Cost::print(raw_ostream &OS) const {
1668	if (InsnsCost)
1669	OS << C.Insns << " instruction" << (C.Insns == `1` ? " " : "s ");
1670	OS << C.NumRegs << " reg" << (C.NumRegs == `1` ? "" : "s");
1671	if (C.AddRecCost != `0`)
1672	OS << ", with addrec cost " << C.AddRecCost;
1673	if (C.NumIVMuls != `0`)
1674	OS << ", plus " << C.NumIVMuls << " IV mul"
1675	<< (C.NumIVMuls == `1` ? "" : "s");
1676	if (C.NumBaseAdds != `0`)
1677	OS << ", plus " << C.NumBaseAdds << " base add"
1678	<< (C.NumBaseAdds == `1` ? "" : "s");
1679	if (C.ScaleCost != `0`)
1680	OS << ", plus " << C.ScaleCost << " scale cost";
1681	if (C.ImmCost != `0`)
1682	OS << ", plus " << C.ImmCost << " imm cost";
1683	if (C.SetupCost != `0`)
1684	OS << ", plus " << C.SetupCost << " setup cost";
1685	}
1686
1687	LLVM_DUMP_METHOD void Cost::dump() const {
1688	print(errs()); errs() << `'\n'`;
1689	}
1690	#endif
1691
1692	/// Test whether this fixup always uses its value outside of the given loop.
1693	bool LSRFixup::isUseFullyOutsideLoop(const Loop L) const* {
1694	// PHI nodes use their value in their incoming blocks.
1695	if (const PHINode *PN = dyn_cast<PHINode>(Val: UserInst)) {
1696	for (unsigned i = `0`, e = PN->getNumIncomingValues(); i != e; ++i)
1697	if (PN->getIncomingValue(i) == OperandValToReplace &&
1698	L->contains(BB: PN->getIncomingBlock(i)))
1699	return false;
1700	return true;
1701	}
1702
1703	return !L->contains(Inst: UserInst);
1704	}
1705
1706	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1707	void LSRFixup::print(raw_ostream &OS) const {
1708	OS << "UserInst=";
1709	// Store is common and interesting enough to be worth special-casing.
1710	if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) {
1711	OS << "store ";
1712	Store->getOperand(`0`)->printAsOperand(OS, /PrintType=/false);
1713	} else if (UserInst->getType()->isVoidTy())
1714	OS << UserInst->getOpcodeName();
1715	else
1716	UserInst->printAsOperand(OS, /PrintType=/false);
1717
1718	OS << ", OperandValToReplace=";
1719	OperandValToReplace->printAsOperand(OS, /PrintType=/false);
1720
1721	for (const Loop *PIL : PostIncLoops) {
1722	OS << ", PostIncLoop=";
1723	PIL->getHeader()->printAsOperand(OS, /PrintType=/false);
1724	}
1725
1726	if (Offset.isNonZero())
1727	OS << ", Offset=" << Offset;
1728	}
1729
1730	LLVM_DUMP_METHOD void LSRFixup::dump() const {
1731	print(errs()); errs() << `'\n'`;
1732	}
1733	#endif
1734
1735	/// Test whether this use as a formula which has the same registers as the given
1736	/// formula.
1737	bool LSRUse::HasFormulaWithSameRegs(const Formula &F) const {
1738	SmallVector<const SCEV *, `4`> Key = F.BaseRegs;
1739	if (F.ScaledReg) Key.push_back(Elt: F.ScaledReg);
1740	// Unstable sort by host order ok, because this is only used for uniquifying.
1741	llvm::sort(C&: Key);
1742	return Uniquifier.count(V: Key);
1743	}
1744
1745	/// The function returns a probability of selecting formula without Reg.
1746	float LSRUse::getNotSelectedProbability(const SCEV Reg) const* {
1747	unsigned FNum = `0`;
1748	for (const Formula &F : Formulae)
1749	if (F.referencesReg(S: Reg))
1750	FNum++;
1751	return ((float)(Formulae.size() - FNum)) / Formulae.size();
1752	}
1753
1754	/// If the given formula has not yet been inserted, add it to the list, and
1755	/// return true. Return false otherwise. The formula must be in canonical form.
1756	bool LSRUse::InsertFormula(const Formula &F, const Loop &L) {
1757	assert(F.isCanonical(L) && "Invalid canonical representation");
1758
1759	if (!Formulae.empty() && RigidFormula)
1760	return false;
1761
1762	SmallVector<const SCEV *, `4`> Key = F.BaseRegs;
1763	if (F.ScaledReg) Key.push_back(Elt: F.ScaledReg);
1764	// Unstable sort by host order ok, because this is only used for uniquifying.
1765	llvm::sort(C&: Key);
1766
1767	if (!Uniquifier.insert(V: Key).second)
1768	return false;
1769
1770	// Using a register to hold the value of 0 is not profitable.
1771	assert((!F.ScaledReg \|\| !F.ScaledReg->isZero()) &&
1772	"Zero allocated in a scaled register!");
1773	#ifndef NDEBUG
1774	for (const SCEV *BaseReg : F.BaseRegs)
1775	assert(!BaseReg->isZero() && "Zero allocated in a base register!");
1776	#endif
1777
1778	// Add the formula to the list.
1779	Formulae.push_back(Elt: F);
1780
1781	// Record registers now being used by this use.
1782	Regs.insert_range(R: F.BaseRegs);
1783	if (F.ScaledReg)
1784	Regs.insert(Ptr: F.ScaledReg);
1785
1786	return true;
1787	}
1788
1789	/// Remove the given formula from this use's list.
1790	void LSRUse::DeleteFormula(Formula &F) {
1791	if (&F != &Formulae.back())
1792	std::swap(a&: F, b&: Formulae.back());
1793	Formulae.pop_back();
1794	}
1795
1796	/// Recompute the Regs field, and update RegUses.
1797	void LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) {
1798	// Now that we've filtered out some formulae, recompute the Regs set.
1799	SmallPtrSet<const SCEV *, `4`> OldRegs = std::move(Regs);
1800	Regs.clear();
1801	for (const Formula &F : Formulae) {
1802	if (F.ScaledReg) Regs.insert(Ptr: F.ScaledReg);
1803	Regs.insert_range(R: F.BaseRegs);
1804	}
1805
1806	// Update the RegTracker.
1807	for (const SCEV *S : OldRegs)
1808	if (!Regs.count(Ptr: S))
1809	RegUses.dropRegister(Reg: S, LUIdx);
1810	}
1811
1812	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1813	void LSRUse::print(raw_ostream &OS) const {
1814	OS << "LSR Use: Kind=";
1815	switch (Kind) {
1816	case Basic: OS << "Basic"; break;
1817	case Special: OS << "Special"; break;
1818	case ICmpZero: OS << "ICmpZero"; break;
1819	case Address:
1820	OS << "Address of ";
1821	if (AccessTy.MemTy->isPointerTy())
1822	OS << "pointer"; // the full pointer type could be really verbose
1823	else {
1824	OS << *AccessTy.MemTy;
1825	}
1826
1827	OS << " in addrspace(" << AccessTy.AddrSpace << `')'`;
1828	}
1829
1830	OS << ", Offsets={";
1831	bool NeedComma = false;
1832	for (const LSRFixup &Fixup : Fixups) {
1833	if (NeedComma) OS << `','`;
1834	OS << Fixup.Offset;
1835	NeedComma = true;
1836	}
1837	OS << `'}'`;
1838
1839	if (AllFixupsOutsideLoop)
1840	OS << ", all-fixups-outside-loop";
1841
1842	if (AllFixupsUnconditional)
1843	OS << ", all-fixups-unconditional";
1844	}
1845
1846	LLVM_DUMP_METHOD void LSRUse::dump() const {
1847	print(errs()); errs() << `'\n'`;
1848	}
1849	#endif
1850
1851	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1852	LSRUse::KindType Kind, MemAccessTy AccessTy,
1853	GlobalValue *BaseGV, Immediate BaseOffset,
1854	bool HasBaseReg, int64_t Scale,
1855	Instruction Fixup /* = nullptr /) {
1856	switch (Kind) {
1857	case LSRUse::Address: {
1858	int64_t FixedOffset =
1859	BaseOffset.isScalable() ? `0` : BaseOffset.getFixedValue();
1860	int64_t ScalableOffset =
1861	BaseOffset.isScalable() ? BaseOffset.getKnownMinValue() : `0`;
1862	return TTI.isLegalAddressingMode(Ty: AccessTy.MemTy, BaseGV, BaseOffset: FixedOffset,
1863	HasBaseReg, Scale, AddrSpace: AccessTy.AddrSpace,
1864	I: Fixup, ScalableOffset);
1865	}
1866	case LSRUse::ICmpZero:
1867	// There's not even a target hook for querying whether it would be legal to
1868	// fold a GV into an ICmp.
1869	if (BaseGV)
1870	return false;
1871
1872	// ICmp only has two operands; don't allow more than two non-trivial parts.
1873	if (Scale != `0` && HasBaseReg && BaseOffset.isNonZero())
1874	return false;
1875
1876	// ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by
1877	// putting the scaled register in the other operand of the icmp.
1878	if (Scale != `0` && Scale != -`1`)
1879	return false;
1880
1881	// If we have low-level target information, ask the target if it can fold an
1882	// integer immediate on an icmp.
1883	if (BaseOffset.isNonZero()) {
1884	// We don't have an interface to query whether the target supports
1885	// icmpzero against scalable quantities yet.
1886	if (BaseOffset.isScalable())
1887	return false;
1888
1889	// We have one of:
1890	// ICmpZero BaseReg + BaseOffset => ICmp BaseReg, -BaseOffset
1891	// ICmpZero -1ScaleReg + BaseOffset => ICmp ScaleReg, BaseOffset*
1892	// Offs is the ICmp immediate.
1893	if (Scale == `0`)
1894	// The cast does the right thing with
1895	// std::numeric_limits<int64_t>::min().
1896	BaseOffset = BaseOffset.getFixed(MinVal: -(uint64_t)BaseOffset.getFixedValue());
1897	return TTI.isLegalICmpImmediate(Imm: BaseOffset.getFixedValue());
1898	}
1899
1900	// ICmpZero BaseReg + -1ScaleReg => ICmp BaseReg, ScaleReg*
1901	return true;
1902
1903	case LSRUse::Basic:
1904	// Only handle single-register values.
1905	return !BaseGV && Scale == `0` && BaseOffset.isZero();
1906
1907	case LSRUse::Special:
1908	// Special case Basic to handle -1 scales.
1909	return !BaseGV && (Scale == `0` \|\| Scale == -`1`) && BaseOffset.isZero();
1910	}
1911
1912	llvm_unreachable("Invalid LSRUse Kind!");
1913	}
1914
1915	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1916	Immediate MinOffset, Immediate MaxOffset,
1917	LSRUse::KindType Kind, MemAccessTy AccessTy,
1918	GlobalValue *BaseGV, Immediate BaseOffset,
1919	bool HasBaseReg, int64_t Scale) {
1920	if (BaseOffset.isNonZero() &&
1921	(BaseOffset.isScalable() != MinOffset.isScalable() \|\|
1922	BaseOffset.isScalable() != MaxOffset.isScalable()))
1923	return false;
1924	// Check for overflow.
1925	int64_t Base = BaseOffset.getKnownMinValue();
1926	int64_t Min = MinOffset.getKnownMinValue();
1927	int64_t Max = MaxOffset.getKnownMinValue();
1928	if (((int64_t)((uint64_t)Base + Min) > Base) != (Min > `0`))
1929	return false;
1930	MinOffset = Immediate::get(MinVal: (uint64_t)Base + Min, Scalable: MinOffset.isScalable());
1931	if (((int64_t)((uint64_t)Base + Max) > Base) != (Max > `0`))
1932	return false;
1933	MaxOffset = Immediate::get(MinVal: (uint64_t)Base + Max, Scalable: MaxOffset.isScalable());
1934
1935	return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset: MinOffset,
1936	HasBaseReg, Scale) &&
1937	isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset: MaxOffset,
1938	HasBaseReg, Scale);
1939	}
1940
1941	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1942	Immediate MinOffset, Immediate MaxOffset,
1943	LSRUse::KindType Kind, MemAccessTy AccessTy,
1944	const Formula &F, const Loop &L) {
1945	// For the purpose of isAMCompletelyFolded either having a canonical formula
1946	// or a scale not equal to zero is correct.
1947	// Problems may arise from non canonical formulae having a scale == 0.
1948	// Strictly speaking it would best to just rely on canonical formulae.
1949	// However, when we generate the scaled formulae, we first check that the
1950	// scaling factor is profitable before computing the actual ScaledReg for
1951	// compile time sake.
1952	assert((F.isCanonical(L) \|\| F.Scale != `0`));
1953	return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1954	BaseGV: F.BaseGV, BaseOffset: F.BaseOffset, HasBaseReg: F.HasBaseReg, Scale: F.Scale);
1955	}
1956
1957	/// Test whether we know how to expand the current formula.
1958	static bool isLegalUse(const TargetTransformInfo &TTI, Immediate MinOffset,
1959	Immediate MaxOffset, LSRUse::KindType Kind,
1960	MemAccessTy AccessTy, GlobalValue *BaseGV,
1961	Immediate BaseOffset, bool HasBaseReg, int64_t Scale) {
1962	// We know how to expand completely foldable formulae.
1963	return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
1964	BaseOffset, HasBaseReg, Scale) \|\|
1965	// Or formulae that use a base register produced by a sum of base
1966	// registers.
1967	(Scale == `1` &&
1968	isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1969	BaseGV, BaseOffset, HasBaseReg: true, Scale: `0`));
1970	}
1971
1972	static bool isLegalUse(const TargetTransformInfo &TTI, Immediate MinOffset,
1973	Immediate MaxOffset, LSRUse::KindType Kind,
1974	MemAccessTy AccessTy, const Formula &F) {
1975	return isLegalUse(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV: F.BaseGV,
1976	BaseOffset: F.BaseOffset, HasBaseReg: F.HasBaseReg, Scale: F.Scale);
1977	}
1978
1979	static bool isLegalAddImmediate(const TargetTransformInfo &TTI,
1980	Immediate Offset) {
1981	if (Offset.isScalable())
1982	return TTI.isLegalAddScalableImmediate(Imm: Offset.getKnownMinValue());
1983
1984	return TTI.isLegalAddImmediate(Imm: Offset.getFixedValue());
1985	}
1986
1987	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1988	const LSRUse &LU, const Formula &F) {
1989	// Target may want to look at the user instructions.
1990	if (LU.Kind == LSRUse::Address && TTI.LSRWithInstrQueries()) {
1991	for (const LSRFixup &Fixup : LU.Fixups)
1992	if (!isAMCompletelyFolded(TTI, Kind: LSRUse::Address, AccessTy: LU.AccessTy, BaseGV: F.BaseGV,
1993	BaseOffset: (F.BaseOffset + Fixup.Offset), HasBaseReg: F.HasBaseReg,
1994	Scale: F.Scale, Fixup: Fixup.UserInst))
1995	return false;
1996	return true;
1997	}
1998
1999	return isAMCompletelyFolded(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
2000	AccessTy: LU.AccessTy, BaseGV: F.BaseGV, BaseOffset: F.BaseOffset, HasBaseReg: F.HasBaseReg,
2001	Scale: F.Scale);
2002	}
2003
2004	static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI,
2005	const LSRUse &LU, const Formula &F,
2006	const Loop &L) {
2007	if (!F.Scale)
2008	return `0`;
2009
2010	// If the use is not completely folded in that instruction, we will have to
2011	// pay an extra cost only for scale != 1.
2012	if (!isAMCompletelyFolded(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
2013	AccessTy: LU.AccessTy, F, L))
2014	return F.Scale != `1`;
2015
2016	switch (LU.Kind) {
2017	case LSRUse::Address: {
2018	// Check the scaling factor cost with both the min and max offsets.
2019	int64_t ScalableMin = `0`, ScalableMax = `0`, FixedMin = `0`, FixedMax = `0`;
2020	if (F.BaseOffset.isScalable()) {
2021	ScalableMin = (F.BaseOffset + LU.MinOffset).getKnownMinValue();
2022	ScalableMax = (F.BaseOffset + LU.MaxOffset).getKnownMinValue();
2023	} else {
2024	FixedMin = (F.BaseOffset + LU.MinOffset).getFixedValue();
2025	FixedMax = (F.BaseOffset + LU.MaxOffset).getFixedValue();
2026	}
2027	InstructionCost ScaleCostMinOffset = TTI.getScalingFactorCost(
2028	Ty: LU.AccessTy.MemTy, BaseGV: F.BaseGV, BaseOffset: StackOffset::get(Fixed: FixedMin, Scalable: ScalableMin),
2029	HasBaseReg: F.HasBaseReg, Scale: F.Scale, AddrSpace: LU.AccessTy.AddrSpace);
2030	InstructionCost ScaleCostMaxOffset = TTI.getScalingFactorCost(
2031	Ty: LU.AccessTy.MemTy, BaseGV: F.BaseGV, BaseOffset: StackOffset::get(Fixed: FixedMax, Scalable: ScalableMax),
2032	HasBaseReg: F.HasBaseReg, Scale: F.Scale, AddrSpace: LU.AccessTy.AddrSpace);
2033
2034	assert(ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() &&
2035	"Legal addressing mode has an illegal cost!");
2036	return std::max(a: ScaleCostMinOffset, b: ScaleCostMaxOffset);
2037	}
2038	case LSRUse::ICmpZero:
2039	case LSRUse::Basic:
2040	case LSRUse::Special:
2041	// The use is completely folded, i.e., everything is folded into the
2042	// instruction.
2043	return `0`;
2044	}
2045
2046	llvm_unreachable("Invalid LSRUse Kind!");
2047	}
2048
2049	static bool isAlwaysFoldable(const TargetTransformInfo &TTI,
2050	LSRUse::KindType Kind, MemAccessTy AccessTy,
2051	GlobalValue *BaseGV, Immediate BaseOffset,
2052	bool HasBaseReg) {
2053	// Fast-path: zero is always foldable.
2054	if (BaseOffset.isZero() && !BaseGV)
2055	return true;
2056
2057	// Conservatively, create an address with an immediate and a
2058	// base and a scale.
2059	int64_t Scale = Kind == LSRUse::ICmpZero ? -`1` : `1`;
2060
2061	// Canonicalize a scale of 1 to a base register if the formula doesn't
2062	// already have a base register.
2063	if (!HasBaseReg && Scale == `1`) {
2064	Scale = `0`;
2065	HasBaseReg = true;
2066	}
2067
2068	// FIXME: Try with + without a scale? Maybe based on TTI?
2069	// I think basereg + scaledreg + immediateoffset isn't a good 'conservative'
2070	// default for many architectures, not just AArch64 SVE. More investigation
2071	// needed later to determine if this should be used more widely than just
2072	// on scalable types.
2073	if (HasBaseReg && BaseOffset.isNonZero() && Kind != LSRUse::ICmpZero &&
2074	AccessTy.MemTy && AccessTy.MemTy->isScalableTy() && DropScaledForVScale)
2075	Scale = `0`;
2076
2077	return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset,
2078	HasBaseReg, Scale);
2079	}
2080
2081	static bool isAlwaysFoldable(const TargetTransformInfo &TTI,
2082	ScalarEvolution &SE, Immediate MinOffset,
2083	Immediate MaxOffset, LSRUse::KindType Kind,
2084	MemAccessTy AccessTy, const SCEV *S,
2085	bool HasBaseReg) {
2086	// Fast-path: zero is always foldable.
2087	if (S->isZero()) return true;
2088
2089	// Conservatively, create an address with an immediate and a
2090	// base and a scale.
2091	SCEVUse SCopy = S;
2092	Immediate BaseOffset = ExtractImmediate(S&: SCopy, SE);
2093	GlobalValue *BaseGV = ExtractSymbol(S&: SCopy, SE);
2094
2095	// If there's anything else involved, it's not foldable.
2096	if (!SCopy ->isZero())
2097	return false;
2098
2099	// Fast-path: zero is always foldable.
2100	if (BaseOffset.isZero() && !BaseGV)
2101	return true;
2102
2103	if (BaseOffset.isScalable())
2104	return false;
2105
2106	// Conservatively, create an address with an immediate and a
2107	// base and a scale.
2108	int64_t Scale = Kind == LSRUse::ICmpZero ? -`1` : `1`;
2109
2110	return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
2111	BaseOffset, HasBaseReg, Scale);
2112	}
2113
2114	namespace {
2115
2116	/// An individual increment in a Chain of IV increments. Relate an IV user to
2117	/// an expression that computes the IV it uses from the IV used by the previous
2118	/// link in the Chain.
2119	///
2120	/// For the head of a chain, IncExpr holds the absolute SCEV expression for the
2121	/// original IVOperand. The head of the chain's IVOperand is only valid during
2122	/// chain collection, before LSR replaces IV users. During chain generation,
2123	/// IncExpr can be used to find the new IVOperand that computes the same
2124	/// expression.
2125	struct IVInc {
2126	Instruction *UserInst;
2127	Value* IVOperand;
2128	const SCEV *IncExpr;
2129
2130	IVInc(Instruction U, Value O, const SCEV *E)
2131	: UserInst(U), IVOperand(O), IncExpr(E) {}
2132	};
2133
2134	// The list of IV increments in program order. We typically add the head of a
2135	// chain without finding subsequent links.
2136	struct IVChain {
2137	SmallVector<IVInc, `1`> Incs;
2138	const SCEV ExprBase = nullptr*;
2139
2140	IVChain() = default;
2141	IVChain(const IVInc &Head, const SCEV *Base)
2142	: Incs (`1`, Head), ExprBase(Base) {}
2143
2144	using const_iterator = SmallVectorImpl<IVInc>::const_iterator;
2145
2146	// Return the first increment in the chain.
2147	const_iterator begin() const {
2148	assert(!Incs.empty());
2149	return std::next(x: Incs.begin());
2150	}
2151	const_iterator end() const {
2152	return Incs.end();
2153	}
2154
2155	// Returns true if this chain contains any increments.
2156	bool hasIncs() const { return Incs.size() >= `2`; }
2157
2158	// Add an IVInc to the end of this chain.
2159	void add(const IVInc &X) { Incs.push_back(Elt: X); }
2160
2161	// Returns the last UserInst in the chain.
2162	Instruction tailUserInst() const* { return Incs.back().UserInst; }
2163
2164	// Returns true if IncExpr can be profitably added to this chain.
2165	bool isProfitableIncrement(const SCEV *OperExpr,
2166	const SCEV *IncExpr,
2167	ScalarEvolution&);
2168	};
2169
2170	/// Helper for CollectChains to track multiple IV increment uses. Distinguish
2171	/// between FarUsers that definitely cross IV increments and NearUsers that may
2172	/// be used between IV increments.
2173	struct ChainUsers {
2174	SmallPtrSet<Instruction*, `4`> FarUsers;
2175	SmallPtrSet<Instruction*, `4`> NearUsers;
2176	};
2177
2178	/// This class holds state for the main loop strength reduction logic.
2179	class LSRInstance {
2180	IVUsers &IU;
2181	ScalarEvolution &SE;
2182	DominatorTree &DT;
2183	LoopInfo &LI;
2184	AssumptionCache &AC;
2185	TargetLibraryInfo &TLI;
2186	const TargetTransformInfo &TTI;
2187	Loop *const L;
2188	MemorySSAUpdater *MSSAU;
2189	TTI::AddressingModeKind AMK;
2190	mutable SCEVExpander Rewriter;
2191	bool Changed = false;
2192	bool HardwareLoopProfitable = false;
2193	bool ShouldPreserveLCSSA = false;
2194
2195	/// This is the insert position that the current loop's induction variable
2196	/// increment should be placed. In simple loops, this is the latch block's
2197	/// terminator. But in more complicated cases, this is a position which will
2198	/// dominate all the in-loop post-increment users.
2199	Instruction IVIncInsertPos = nullptr*;
2200
2201	/// Interesting factors between use strides.
2202	///
2203	/// We explicitly use a SetVector which contains a SmallSet, instead of the
2204	/// default, a SmallDenseSet, because we need to use the full range of
2205	/// int64_ts, and there's currently no good way of doing that with
2206	/// SmallDenseSet.
2207	SetVector<int64_t, SmallVector<int64_t, `8`>, SmallSet<int64_t, `8`>> Factors;
2208
2209	/// The cost of the current SCEV, the best solution by LSR will be dropped if
2210	/// the solution is not profitable.
2211	Cost BaselineCost;
2212
2213	/// Interesting use types, to facilitate truncation reuse.
2214	SmallSetVector<Type *, `4`> Types;
2215
2216	/// The list of interesting uses.
2217	mutable SmallVector<LSRUse, `16`> Uses;
2218
2219	/// Track which uses use which register candidates.
2220	RegUseTracker RegUses;
2221
2222	// Limit the number of chains to avoid quadratic behavior. We don't expect to
2223	// have more than a few IV increment chains in a loop. Missing a Chain falls
2224	// back to normal LSR behavior for those uses.
2225	static const unsigned MaxChains = `8`;
2226
2227	/// IV users can form a chain of IV increments.
2228	SmallVector<IVChain, MaxChains> IVChainVec;
2229
2230	/// IV users that belong to profitable IVChains.
2231	SmallPtrSet<Use*, MaxChains> IVIncSet;
2232
2233	/// Induction variables that were generated and inserted by the SCEV Expander.
2234	SmallVector<llvm::WeakVH, `2`> ScalarEvolutionIVs;
2235
2236	// Inserting instructions in the loop and using them as PHI's input could
2237	// break LCSSA in case if PHI's parent block is not a loop exit (i.e. the
2238	// corresponding incoming block is not loop exiting). So collect all such
2239	// instructions to form LCSSA for them later.
2240	SmallSetVector<Instruction *, `4`> InsertedNonLCSSAInsts;
2241
2242	void OptimizeShadowIV();
2243	bool FindIVUserForCond(Instruction Cond, IVStrideUse &CondUse);
2244	Instruction OptimizeMax(ICmpInst Cond, IVStrideUse *&CondUse);
2245	void OptimizeLoopTermCond();
2246
2247	void ChainInstruction(Instruction UserInst, Instruction IVOper,
2248	SmallVectorImpl<ChainUsers> &ChainUsersVec);
2249	void FinalizeChain(IVChain &Chain);
2250	void CollectChains();
2251	void GenerateIVChain(const IVChain &Chain,
2252	SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2253
2254	void CollectInterestingTypesAndFactors();
2255	void CollectFixupsAndInitialFormulae();
2256
2257	// Support for sharing of LSRUses between LSRFixups.
2258	using UseMapTy = DenseMap<LSRUse::SCEVUseKindPair, size_t>;
2259	UseMapTy UseMap;
2260
2261	bool reconcileNewOffset(LSRUse &LU, Immediate NewOffset, bool HasBaseReg,
2262	LSRUse::KindType Kind, MemAccessTy AccessTy);
2263
2264	std::pair<size_t, Immediate> getUse(const SCEV *&Expr, LSRUse::KindType Kind,
2265	MemAccessTy AccessTy);
2266
2267	void DeleteUse(LSRUse &LU, size_t LUIdx);
2268
2269	LSRUse FindUseWithSimilarFormula(const* Formula &F, const LSRUse &OrigLU);
2270
2271	void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2272	void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2273	void CountRegisters(const Formula &F, size_t LUIdx);
2274	bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
2275	bool IsFixupExecutedEachIncrement(const LSRFixup &LF) const;
2276
2277	void CollectLoopInvariantFixupsAndFormulae();
2278
2279	void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
2280	unsigned Depth = `0`);
2281
2282	void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
2283	const Formula &Base, unsigned Depth,
2284	size_t Idx, bool IsScaledReg = false);
2285	void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
2286	void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2287	const Formula &Base, size_t Idx,
2288	bool IsScaledReg = false);
2289	void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2290	void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2291	const Formula &Base,
2292	const SmallVectorImpl<Immediate> &Worklist,
2293	size_t Idx, bool IsScaledReg = false);
2294	void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2295	void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2296	void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2297	void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
2298	void GenerateCrossUseConstantOffsets();
2299	void GenerateAllReuseFormulae();
2300
2301	void FilterOutUndesirableDedicatedRegisters();
2302
2303	size_t EstimateSearchSpaceComplexity() const;
2304	void NarrowSearchSpaceByDetectingSupersets();
2305	void NarrowSearchSpaceByCollapsingUnrolledCode();
2306	void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
2307	void NarrowSearchSpaceByFilterFormulaWithSameScaledReg();
2308	void NarrowSearchSpaceByFilterPostInc();
2309	void NarrowSearchSpaceByMergingUsesOutsideLoop();
2310	void NarrowSearchSpaceByDeletingCostlyFormulas();
2311	void NarrowSearchSpaceByPickingWinnerRegs();
2312	void NarrowSearchSpaceUsingHeuristics();
2313
2314	void SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
2315	Cost &SolutionCost,
2316	SmallVectorImpl<const Formula *> &Workspace,
2317	const Cost &CurCost,
2318	const SmallPtrSet<const SCEV *, `16`> &CurRegs,
2319	DenseSet<const SCEV > &VisitedRegs) const*;
2320	void Solve(SmallVectorImpl<const Formula > &Solution) const*;
2321
2322	BasicBlock::iterator
2323	HoistInsertPosition(BasicBlock::iterator IP,
2324	const SmallVectorImpl<Instruction > &Inputs) const*;
2325	BasicBlock::iterator AdjustInsertPositionForExpand(BasicBlock::iterator IP,
2326	const LSRFixup &LF,
2327	const LSRUse &LU) const;
2328
2329	Value Expand(const* LSRUse &LU, const LSRFixup &LF, const Formula &F,
2330	BasicBlock::iterator IP,
2331	SmallVectorImpl<WeakTrackingVH> &DeadInsts) const;
2332	void RewriteForPHI(PHINode PN, const* LSRUse &LU, const LSRFixup &LF,
2333	const Formula &F,
2334	SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2335	void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2336	SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2337	void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution);
2338
2339	public:
2340	// TODO(boomanaiden154): The PreserveLCSSA flag is a hack to allow
2341	// experimentation with the NewPM which requires LCSSA preservation while
2342	// some of the details are worked out in LSR. Eventually it should be set
2343	// to true and removed.
2344	LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT,
2345	LoopInfo &LI, const TargetTransformInfo &TTI, AssumptionCache &AC,
2346	TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU,
2347	bool PreserveLCSSA);
2348
2349	bool getChanged() const { return Changed; }
2350	const SmallVectorImpl<WeakVH> &getScalarEvolutionIVs() const {
2351	return ScalarEvolutionIVs;
2352	}
2353
2354	void print_factors_and_types(raw_ostream &OS) const;
2355	void print_fixups(raw_ostream &OS) const;
2356	void print_uses(raw_ostream &OS) const;
2357	void print(raw_ostream &OS) const;
2358	void dump() const;
2359	};
2360
2361	} // end anonymous namespace
2362
2363	/// If IV is used in a int-to-float cast inside the loop then try to eliminate
2364	/// the cast operation.
2365	void LSRInstance::OptimizeShadowIV() {
2366	const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2367	if (isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount))
2368	return;
2369
2370	for (IVUsers::const_iterator UI = IU.begin(), E = IU.end();
2371	UI != E; / empty /) {
2372	IVUsers::const_iterator CandidateUI = UI;
2373	++UI;
2374	Instruction *ShadowUse = CandidateUI ->getUser();
2375	Type DestTy = nullptr*;
2376	bool IsSigned = false;
2377
2378	/ If shadow use is a int->float cast then insert a second IV*
2379	to eliminate this cast.
2380
2381	for (unsigned i = 0; i < n; ++i)
2382	foo((double)i);
2383
2384	is transformed into
2385
2386	double d = 0.0;
2387	for (unsigned i = 0; i < n; ++i, ++d)
2388	foo(d);
2389	*/
2390	if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(Val: CandidateUI ->getUser())) {
2391	IsSigned = false;
2392	DestTy = UCast->getDestTy();
2393	}
2394	else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(Val: CandidateUI ->getUser())) {
2395	IsSigned = true;
2396	DestTy = SCast->getDestTy();
2397	}
2398	if (!DestTy) continue;
2399
2400	// If target does not support DestTy natively then do not apply
2401	// this transformation.
2402	if (!TTI.isTypeLegal(Ty: DestTy)) continue;
2403
2404	PHINode *PH = dyn_cast<PHINode>(Val: ShadowUse->getOperand(i: `0`));
2405	if (!PH) continue;
2406	if (PH->getNumIncomingValues() != `2`) continue;
2407
2408	// If the calculation in integers overflows, the result in FP type will
2409	// differ. So we only can do this transformation if we are guaranteed to not
2410	// deal with overflowing values
2411	const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: SE.getSCEV(V: PH));
2412	if (!AR) continue;
2413	if (IsSigned && !AR->hasNoSignedWrap()) continue;
2414	if (!IsSigned && !AR->hasNoUnsignedWrap()) continue;
2415
2416	Type *SrcTy = PH->getType();
2417	int Mantissa = DestTy->getFPMantissaWidth();
2418	if (Mantissa == -`1`) continue;
2419	if ((int)SE.getTypeSizeInBits(Ty: SrcTy) > Mantissa)
2420	continue;
2421
2422	unsigned Entry, Latch;
2423	if (PH->getIncomingBlock(i: `0`) == L->getLoopPreheader()) {
2424	Entry = `0`;
2425	Latch = `1`;
2426	} else {
2427	Entry = `1`;
2428	Latch = `0`;
2429	}
2430
2431	ConstantInt *Init = dyn_cast<ConstantInt>(Val: PH->getIncomingValue(i: Entry));
2432	if (!Init) continue;
2433	Constant *NewInit = ConstantFP::get(Ty: DestTy, V: IsSigned ?
2434	(double)Init->getSExtValue() :
2435	(double)Init->getZExtValue());
2436
2437	BinaryOperator *Incr =
2438	dyn_cast<BinaryOperator>(Val: PH->getIncomingValue(i: Latch));
2439	if (!Incr) continue;
2440	if (Incr->getOpcode() != Instruction::Add
2441	&& Incr->getOpcode() != Instruction::Sub)
2442	continue;
2443
2444	/ Initialize new IV, double d = 0.0 in above example. /
2445	ConstantInt C = nullptr*;
2446	if (Incr->getOperand(i_nocapture: `0`) == PH)
2447	C = dyn_cast<ConstantInt>(Val: Incr->getOperand(i_nocapture: `1`));
2448	else if (Incr->getOperand(i_nocapture: `1`) == PH)
2449	C = dyn_cast<ConstantInt>(Val: Incr->getOperand(i_nocapture: `0`));
2450	else
2451	continue;
2452
2453	if (!C) continue;
2454
2455	// Ignore negative constants, as the code below doesn't handle them
2456	// correctly. TODO: Remove this restriction.
2457	if (!C->getValue().isStrictlyPositive())
2458	continue;
2459
2460	/ Add new PHINode. /
2461	PHINode *NewPH = PHINode::Create(Ty: DestTy, NumReservedValues: `2`, NameStr: "IV.S.", InsertBefore: PH->getIterator());
2462	NewPH->setDebugLoc(PH->getDebugLoc());
2463
2464	/ create new increment. '++d' in above example. /
2465	Constant *CFP = ConstantFP::get(Ty: DestTy, V: C->getZExtValue());
2466	BinaryOperator *NewIncr = BinaryOperator::Create(
2467	Op: Incr->getOpcode() == Instruction::Add ? Instruction::FAdd
2468	: Instruction::FSub,
2469	S1: NewPH, S2: CFP, Name: "IV.S.next.", InsertBefore: Incr->getIterator());
2470	NewIncr->setDebugLoc(Incr->getDebugLoc());
2471
2472	NewPH->addIncoming(V: NewInit, BB: PH->getIncomingBlock(i: Entry));
2473	NewPH->addIncoming(V: NewIncr, BB: PH->getIncomingBlock(i: Latch));
2474
2475	/ Remove cast operation /
2476	ShadowUse->replaceAllUsesWith(V: NewPH);
2477	ShadowUse->eraseFromParent();
2478	Changed = true;
2479	break;
2480	}
2481	}
2482
2483	/// If Cond has an operand that is an expression of an IV, set the IV user and
2484	/// stride information and return true, otherwise return false.
2485	bool LSRInstance::FindIVUserForCond(Instruction Cond, IVStrideUse &CondUse) {
2486	for (IVStrideUse &U : IU)
2487	if (U.getUser() == Cond) {
2488	// NOTE: we could handle setcc instructions with multiple uses here, but
2489	// InstCombine does it as well for simple uses, it's not clear that it
2490	// occurs enough in real life to handle.
2491	CondUse = &U;
2492	return true;
2493	}
2494	return false;
2495	}
2496
2497	/// Rewrite the loop's terminating condition if it uses a max computation.
2498	///
2499	/// This is a narrow solution to a specific, but acute, problem. For loops
2500	/// like this:
2501	///
2502	/// i = 0;
2503	/// do {
2504	/// p[i] = 0.0;
2505	/// } while (++i < n);
2506	///
2507	/// the trip count isn't just 'n', because 'n' might not be positive. And
2508	/// unfortunately this can come up even for loops where the user didn't use
2509	/// a C do-while loop. For example, seemingly well-behaved top-test loops
2510	/// will commonly be lowered like this:
2511	///
2512	/// if (n > 0) {
2513	/// i = 0;
2514	/// do {
2515	/// p[i] = 0.0;
2516	/// } while (++i < n);
2517	/// }
2518	///
2519	/// and then it's possible for subsequent optimization to obscure the if
2520	/// test in such a way that indvars can't find it.
2521	///
2522	/// When indvars can't find the if test in loops like this, it creates a
2523	/// max expression, which allows it to give the loop a canonical
2524	/// induction variable:
2525	///
2526	/// i = 0;
2527	/// max = n < 1 ? 1 : n;
2528	/// do {
2529	/// p[i] = 0.0;
2530	/// } while (++i != max);
2531	///
2532	/// Canonical induction variables are necessary because the loop passes
2533	/// are designed around them. The most obvious example of this is the
2534	/// LoopInfo analysis, which doesn't remember trip count values. It
2535	/// expects to be able to rediscover the trip count each time it is
2536	/// needed, and it does this using a simple analysis that only succeeds if
2537	/// the loop has a canonical induction variable.
2538	///
2539	/// However, when it comes time to generate code, the maximum operation
2540	/// can be quite costly, especially if it's inside of an outer loop.
2541	///
2542	/// This function solves this problem by detecting this type of loop and
2543	/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting
2544	/// the instructions for the maximum computation.
2545	Instruction LSRInstance::OptimizeMax(ICmpInst Cond, IVStrideUse *&CondUse) {
2546	// Check that the loop matches the pattern we're looking for.
2547	if (Cond->getPredicate() != CmpInst::ICMP_EQ &&
2548	Cond->getPredicate() != CmpInst::ICMP_NE)
2549	return Cond;
2550
2551	SelectInst *Sel = dyn_cast<SelectInst>(Val: Cond->getOperand(i_nocapture: `1`));
2552	if (!Sel \|\| !Sel->hasOneUse()) return Cond;
2553
2554	const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2555	if (isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount))
2556	return Cond;
2557	const SCEV *One = SE.getConstant(Ty: BackedgeTakenCount->getType(), V: `1`);
2558
2559	// Add one to the backedge-taken count to get the trip count.
2560	const SCEV *IterationCount = SE.getAddExpr(LHS: One, RHS: BackedgeTakenCount);
2561	if (IterationCount != SE.getSCEV(V: Sel)) return Cond;
2562
2563	// Check for a max calculation that matches the pattern. There's no check
2564	// for ICMP_ULE here because the comparison would be with zero, which
2565	// isn't interesting.
2566	CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
2567	const SCEVNAryExpr Max = nullptr*;
2568	if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(Val: BackedgeTakenCount)) {
2569	Pred = ICmpInst::ICMP_SLE;
2570	Max = S;
2571	} else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(Val: IterationCount)) {
2572	Pred = ICmpInst::ICMP_SLT;
2573	Max = S;
2574	} else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(Val: IterationCount)) {
2575	Pred = ICmpInst::ICMP_ULT;
2576	Max = U;
2577	} else {
2578	// No match; bail.
2579	return Cond;
2580	}
2581
2582	// To handle a max with more than two operands, this optimization would
2583	// require additional checking and setup.
2584	if (Max->getNumOperands() != `2`)
2585	return Cond;
2586
2587	const SCEV *MaxLHS = Max->getOperand(i: `0`);
2588	const SCEV *MaxRHS = Max->getOperand(i: `1`);
2589
2590	// ScalarEvolution canonicalizes constants to the left. For < and >, look
2591	// for a comparison with 1. For <= and >=, a comparison with zero.
2592	if (!MaxLHS \|\|
2593	(ICmpInst::isTrueWhenEqual(predicate: Pred) ? !MaxLHS->isZero() : (MaxLHS != One)))
2594	return Cond;
2595
2596	// Check the relevant induction variable for conformance to
2597	// the pattern.
2598	const SCEV *IV = SE.getSCEV(V: Cond->getOperand(i_nocapture: `0`));
2599	if (!match(S: IV,
2600	P: m_scev_AffineAddRec(Op0: m_scev_SpecificInt(V: `1`), Op1: m_scev_SpecificInt(V: `1`))))
2601	return Cond;
2602
2603	assert(cast<SCEVAddRecExpr>(IV)->getLoop() == L &&
2604	"Loop condition operand is an addrec in a different loop!");
2605
2606	// Check the right operand of the select, and remember it, as it will
2607	// be used in the new comparison instruction.
2608	Value NewRHS = nullptr*;
2609	if (ICmpInst::isTrueWhenEqual(predicate: Pred)) {
2610	// Look for n+1, and grab n.
2611	if (AddOperator *BO = dyn_cast<AddOperator>(Val: Sel->getOperand(i_nocapture: `1`)))
2612	if (ConstantInt *BO1 = dyn_cast<ConstantInt>(Val: BO->getOperand(i_nocapture: `1`)))
2613	if (BO1->isOne() && SE.getSCEV(V: BO->getOperand(i_nocapture: `0`)) == MaxRHS)
2614	NewRHS = BO->getOperand(i_nocapture: `0`);
2615	if (AddOperator *BO = dyn_cast<AddOperator>(Val: Sel->getOperand(i_nocapture: `2`)))
2616	if (ConstantInt *BO1 = dyn_cast<ConstantInt>(Val: BO->getOperand(i_nocapture: `1`)))
2617	if (BO1->isOne() && SE.getSCEV(V: BO->getOperand(i_nocapture: `0`)) == MaxRHS)
2618	NewRHS = BO->getOperand(i_nocapture: `0`);
2619	if (!NewRHS)
2620	return Cond;
2621	} else if (SE.getSCEV(V: Sel->getOperand(i_nocapture: `1`)) == MaxRHS)
2622	NewRHS = Sel->getOperand(i_nocapture: `1`);
2623	else if (SE.getSCEV(V: Sel->getOperand(i_nocapture: `2`)) == MaxRHS)
2624	NewRHS = Sel->getOperand(i_nocapture: `2`);
2625	else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(Val: MaxRHS))
2626	NewRHS = SU->getValue();
2627	else
2628	// Max doesn't match expected pattern.
2629	return Cond;
2630
2631	// Determine the new comparison opcode. It may be signed or unsigned,
2632	// and the original comparison may be either equality or inequality.
2633	if (Cond->getPredicate() == CmpInst::ICMP_EQ)
2634	Pred = CmpInst::getInversePredicate(pred: Pred);
2635
2636	// Ok, everything looks ok to change the condition into an SLT or SGE and
2637	// delete the max calculation.
2638	ICmpInst NewCond = new* ICmpInst (Cond->getIterator(), Pred,
2639	Cond->getOperand(i_nocapture: `0`), NewRHS, "scmp");
2640
2641	// Delete the max calculation instructions.
2642	NewCond->setDebugLoc(Cond->getDebugLoc());
2643	Cond->replaceAllUsesWith(V: NewCond);
2644	CondUse->setUser(NewCond);
2645	Instruction *Cmp = cast<Instruction>(Val: Sel->getOperand(i_nocapture: `0`));
2646	Cond->eraseFromParent();
2647	Sel->eraseFromParent();
2648	if (Cmp->use_empty()) {
2649	salvageDebugInfo(I&: *Cmp);
2650	Cmp->eraseFromParent();
2651	}
2652	return NewCond;
2653	}
2654
2655	/// Change loop terminating condition to use the postinc iv when possible.
2656	void
2657	LSRInstance::OptimizeLoopTermCond() {
2658	SmallPtrSet<Instruction *, `4`> PostIncs;
2659
2660	// We need a different set of heuristics for rotated and non-rotated loops.
2661	// If a loop is rotated then the latch is also the backedge, so inserting
2662	// post-inc expressions just before the latch is ideal. To reduce live ranges
2663	// it also makes sense to rewrite terminating conditions to use post-inc
2664	// expressions.
2665	//
2666	// If the loop is not rotated then the latch is not a backedge; the latch
2667	// check is done in the loop head. Adding post-inc expressions before the
2668	// latch will cause overlapping live-ranges of pre-inc and post-inc expressions
2669	// in the loop body. In this case we do not* want to use post-inc expressions*
2670	// in the latch check, and we want to insert post-inc expressions before
2671	// the backedge.
2672	BasicBlock *LatchBlock = L->getLoopLatch();
2673	SmallVector<BasicBlock*, `8`> ExitingBlocks;
2674	L->getExitingBlocks(ExitingBlocks);
2675	if (!llvm::is_contained(Range&: ExitingBlocks, Element: LatchBlock)) {
2676	// The backedge doesn't exit the loop; treat this as a head-tested loop.
2677	IVIncInsertPos = LatchBlock->getTerminator();
2678	return;
2679	}
2680
2681	// Otherwise treat this as a rotated loop.
2682	for (BasicBlock *ExitingBlock : ExitingBlocks) {
2683	// Get the terminating condition for the loop if possible. If we
2684	// can, we want to change it to use a post-incremented version of its
2685	// induction variable, to allow coalescing the live ranges for the IV into
2686	// one register value.
2687
2688	CondBrInst *TermBr = dyn_cast<CondBrInst>(Val: ExitingBlock->getTerminator());
2689	if (!TermBr)
2690	continue;
2691
2692	Instruction *Cond = dyn_cast<Instruction>(Val: TermBr->getCondition());
2693	// If the argument to TermBr is an extractelement, then the source of that
2694	// instruction is what's generated the condition.
2695	auto *Extract = dyn_cast_or_null<ExtractElementInst>(Val: Cond);
2696	if (Extract)
2697	Cond = dyn_cast<Instruction>(Val: Extract->getVectorOperand());
2698	// FIXME: We could do more here, like handling logical operations where one
2699	// side is a cmp that uses an induction variable.
2700	if (!Cond)
2701	continue;
2702
2703	// Search IVUsesByStride to find Cond's IVUse if there is one.
2704	IVStrideUse CondUse = nullptr*;
2705	if (!FindIVUserForCond(Cond, CondUse))
2706	continue;
2707
2708	// If the trip count is computed in terms of a max (due to ScalarEvolution
2709	// being unable to find a sufficient guard, for example), change the loop
2710	// comparison to use SLT or ULT instead of NE.
2711	// One consequence of doing this now is that it disrupts the count-down
2712	// optimization. That's not always a bad thing though, because in such
2713	// cases it may still be worthwhile to avoid a max.
2714	if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond))
2715	Cond = OptimizeMax(Cond: Cmp, CondUse);
2716
2717	// If this exiting block dominates the latch block, it may also use
2718	// the post-inc value if it won't be shared with other uses.
2719	// Check for dominance.
2720	if (!DT.dominates(A: ExitingBlock, B: LatchBlock))
2721	continue;
2722
2723	// Conservatively avoid trying to use the post-inc value in non-latch
2724	// exits if there may be pre-inc users in intervening blocks.
2725	if (LatchBlock != ExitingBlock)
2726	for (const IVStrideUse &UI : IU)
2727	// Test if the use is reachable from the exiting block. This dominator
2728	// query is a conservative approximation of reachability.
2729	if (&UI != CondUse &&
2730	!DT.properlyDominates(A: UI.getUser()->getParent(), B: ExitingBlock)) {
2731	// Conservatively assume there may be reuse if the quotient of their
2732	// strides could be a legal scale.
2733	const SCEV A = IU.getStride(IU: CondUse, L);
2734	const SCEV *B = IU.getStride(IU: UI, L);
2735	if (!A \|\| !B) continue;
2736	if (SE.getTypeSizeInBits(Ty: A->getType()) !=
2737	SE.getTypeSizeInBits(Ty: B->getType())) {
2738	if (SE.getTypeSizeInBits(Ty: A->getType()) >
2739	SE.getTypeSizeInBits(Ty: B->getType()))
2740	B = SE.getSignExtendExpr(Op: B, Ty: A->getType());
2741	else
2742	A = SE.getSignExtendExpr(Op: A, Ty: B->getType());
2743	}
2744	if (const SCEVConstant *D =
2745	dyn_cast_or_null<SCEVConstant>(Val: getExactSDiv(LHS: B, RHS: A, SE))) {
2746	const ConstantInt *C = D->getValue();
2747	// Stride of one or negative one can have reuse with non-addresses.
2748	if (C->isOne() \|\| C->isMinusOne())
2749	goto decline_post_inc;
2750	// Avoid weird situations.
2751	if (C->getValue().getSignificantBits() >= `64` \|\|
2752	C->getValue().isMinSignedValue())
2753	goto decline_post_inc;
2754	// Check for possible scaled-address reuse.
2755	if (isAddressUse(TTI, Inst: UI.getUser(), OperandVal: UI.getOperandValToReplace())) {
2756	MemAccessTy AccessTy =
2757	getAccessType(TTI, Inst: UI.getUser(), OperandVal: UI.getOperandValToReplace());
2758	int64_t Scale = C->getSExtValue();
2759	if (TTI.isLegalAddressingMode(Ty: AccessTy.MemTy, /BaseGV=/nullptr,
2760	/BaseOffset=/`0`,
2761	/HasBaseReg=/true, Scale,
2762	AddrSpace: AccessTy.AddrSpace))
2763	goto decline_post_inc;
2764	Scale = -Scale;
2765	if (TTI.isLegalAddressingMode(Ty: AccessTy.MemTy, /BaseGV=/nullptr,
2766	/BaseOffset=/`0`,
2767	/HasBaseReg=/true, Scale,
2768	AddrSpace: AccessTy.AddrSpace))
2769	goto decline_post_inc;
2770	}
2771	}
2772	}
2773
2774	LLVM_DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: "
2775	<< *Cond << `'\n'`);
2776
2777	// It's possible for the setcc instruction to be anywhere in the loop, and
2778	// possible for it to have multiple users. If it is not immediately before
2779	// the exiting block branch, move it.
2780	if (isa_and_nonnull<CmpInst>(Val: Cond) && Cond->getNextNode() != TermBr &&
2781	!Extract) {
2782	if (Cond->hasOneUse()) {
2783	Cond->moveBefore(InsertPos: TermBr->getIterator());
2784	} else {
2785	// Clone the terminating condition and insert into the loopend.
2786	Instruction *OldCond = Cond;
2787	Cond = Cond->clone();
2788	Cond->setName(L->getHeader()->getName() + ".termcond");
2789	Cond->insertInto(ParentBB: ExitingBlock, It: TermBr->getIterator());
2790
2791	// Clone the IVUse, as the old use still exists!
2792	CondUse = &IU.AddUser(User: Cond, Operand: CondUse->getOperandValToReplace());
2793	TermBr->replaceUsesOfWith(From: OldCond, To: Cond);
2794	}
2795	}
2796
2797	// If we get to here, we know that we can transform the setcc instruction to
2798	// use the post-incremented version of the IV, allowing us to coalesce the
2799	// live ranges for the IV correctly.
2800	CondUse->transformToPostInc(L);
2801	Changed = true;
2802
2803	PostIncs.insert(Ptr: Cond);
2804	decline_post_inc:;
2805	}
2806
2807	// Determine an insertion point for the loop induction variable increment. It
2808	// must dominate all the post-inc comparisons we just set up, and it must
2809	// dominate the loop latch edge.
2810	IVIncInsertPos = L->getLoopLatch()->getTerminator();
2811	for (Instruction *Inst : PostIncs)
2812	IVIncInsertPos = DT.findNearestCommonDominator(I1: IVIncInsertPos, I2: Inst);
2813	}
2814
2815	/// Determine if the given use can accommodate a fixup at the given offset and
2816	/// other details. If so, update the use and return true.
2817	bool LSRInstance::reconcileNewOffset(LSRUse &LU, Immediate NewOffset,
2818	bool HasBaseReg, LSRUse::KindType Kind,
2819	MemAccessTy AccessTy) {
2820	Immediate NewMinOffset = LU.MinOffset;
2821	Immediate NewMaxOffset = LU.MaxOffset;
2822	MemAccessTy NewAccessTy = AccessTy;
2823
2824	// Check for a mismatched kind. It's tempting to collapse mismatched kinds to
2825	// something conservative, however this can pessimize in the case that one of
2826	// the uses will have all its uses outside the loop, for example.
2827	if (LU.Kind != Kind)
2828	return false;
2829
2830	// Check for a mismatched access type, and fall back conservatively as needed.
2831	// TODO: Be less conservative when the type is similar and can use the same
2832	// addressing modes.
2833	if (Kind == LSRUse::Address) {
2834	if (AccessTy.MemTy != LU.AccessTy.MemTy) {
2835	NewAccessTy = MemAccessTy::getUnknown(Ctx&: AccessTy.MemTy->getContext(),
2836	AS: AccessTy.AddrSpace);
2837	}
2838	}
2839
2840	// Conservatively assume HasBaseReg is true for now.
2841	if (Immediate::isKnownLT(LHS: NewOffset, RHS: LU.MinOffset)) {
2842	if (!isAlwaysFoldable(TTI, Kind, AccessTy: NewAccessTy, /BaseGV=/nullptr,
2843	BaseOffset: LU.MaxOffset - NewOffset, HasBaseReg))
2844	return false;
2845	NewMinOffset = NewOffset;
2846	} else if (Immediate::isKnownGT(LHS: NewOffset, RHS: LU.MaxOffset)) {
2847	if (!isAlwaysFoldable(TTI, Kind, AccessTy: NewAccessTy, /BaseGV=/nullptr,
2848	BaseOffset: NewOffset - LU.MinOffset, HasBaseReg))
2849	return false;
2850	NewMaxOffset = NewOffset;
2851	}
2852
2853	// FIXME: We should be able to handle some level of scalable offset support
2854	// for 'void', but in order to get basic support up and running this is
2855	// being left out.
2856	if (NewAccessTy.MemTy && NewAccessTy.MemTy->isVoidTy() &&
2857	(NewMinOffset.isScalable() \|\| NewMaxOffset.isScalable()))
2858	return false;
2859
2860	// Update the use.
2861	LU.MinOffset = NewMinOffset;
2862	LU.MaxOffset = NewMaxOffset;
2863	LU.AccessTy = NewAccessTy;
2864	return true;
2865	}
2866
2867	/// Return an LSRUse index and an offset value for a fixup which needs the given
2868	/// expression, with the given kind and optional access type. Either reuse an
2869	/// existing use or create a new one, as needed.
2870	std::pair<size_t, Immediate> LSRInstance::getUse(const SCEV *&Expr,
2871	LSRUse::KindType Kind,
2872	MemAccessTy AccessTy) {
2873	const SCEV *Copy = Expr;
2874	SCEVUse ExprUse = Expr;
2875	Immediate Offset = ExtractImmediate(
2876	S&: ExprUse, SE, PreferScalable: AccessTy.MemTy && AccessTy.MemTy->isScalableTy());
2877	Expr = ExprUse;
2878
2879	// Basic uses can't accept any offset, for example.
2880	if (!isAlwaysFoldable(TTI, Kind, AccessTy, /BaseGV=/ nullptr,
2881	BaseOffset: Offset, /HasBaseReg=/ true)) {
2882	Expr = Copy;
2883	Offset = Immediate::getFixed(MinVal: `0`);
2884	}
2885
2886	std::pair<UseMapTy::iterator, bool> P =
2887	UseMap.try_emplace(Key: LSRUse::SCEVUseKindPair(Expr, Kind));
2888	if (!P.second) {
2889	// A use already existed with this base.
2890	size_t LUIdx = P.first ->second;
2891	LSRUse &LU = Uses [LUIdx];
2892	if (reconcileNewOffset(LU, NewOffset: Offset, /HasBaseReg=/true, Kind, AccessTy))
2893	// Reuse this use.
2894	return std::make_pair(x&: LUIdx, y&: Offset);
2895	}
2896
2897	// Create a new use.
2898	size_t LUIdx = Uses.size();
2899	P.first ->second = LUIdx;
2900	Uses.push_back(Elt: LSRUse (Kind, AccessTy));
2901	LSRUse &LU = Uses [LUIdx];
2902
2903	LU.MinOffset = Offset;
2904	LU.MaxOffset = Offset;
2905	return std::make_pair(x&: LUIdx, y&: Offset);
2906	}
2907
2908	/// Delete the given use from the Uses list.
2909	void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
2910	if (&LU != &Uses.back())
2911	std::swap(a&: LU, b&: Uses.back());
2912	Uses.pop_back();
2913
2914	// Update RegUses.
2915	RegUses.swapAndDropUse(LUIdx, LastLUIdx: Uses.size());
2916	}
2917
2918	/// Look for a use distinct from OrigLU which is has a formula that has the same
2919	/// registers as the given formula.
2920	LSRUse *
2921	LSRInstance::FindUseWithSimilarFormula(const Formula &OrigF,
2922	const LSRUse &OrigLU) {
2923	// Search all uses for the formula. This could be more clever.
2924	for (LSRUse &LU : Uses) {
2925	// Check whether this use is close enough to OrigLU, to see whether it's
2926	// worthwhile looking through its formulae.
2927	// Ignore ICmpZero uses because they may contain formulae generated by
2928	// GenerateICmpZeroScales, in which case adding fixup offsets may
2929	// be invalid.
2930	if (&LU != &OrigLU && LU.Kind != LSRUse::ICmpZero &&
2931	LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
2932	LU.HasFormulaWithSameRegs(F: OrigF)) {
2933	// Scan through this use's formulae.
2934	for (const Formula &F : LU.Formulae) {
2935	// Check to see if this formula has the same registers and symbols
2936	// as OrigF.
2937	if (F.BaseRegs == OrigF.BaseRegs &&
2938	F.ScaledReg == OrigF.ScaledReg &&
2939	F.BaseGV == OrigF.BaseGV &&
2940	F.Scale == OrigF.Scale &&
2941	F.UnfoldedOffset == OrigF.UnfoldedOffset) {
2942	if (F.BaseOffset.isZero())
2943	return &LU;
2944	// This is the formula where all the registers and symbols matched;
2945	// there aren't going to be any others. Since we declined it, we
2946	// can skip the rest of the formulae and proceed to the next LSRUse.
2947	break;
2948	}
2949	}
2950	}
2951	}
2952
2953	// Nothing looked good.
2954	return nullptr;
2955	}
2956
2957	void LSRInstance::CollectInterestingTypesAndFactors() {
2958	SmallSetVector<const SCEV *, `4`> Strides;
2959
2960	// Collect interesting types and strides.
2961	SmallVector<const SCEV *, `4`> Worklist;
2962	for (const IVStrideUse &U : IU) {
2963	const SCEV *Expr = IU.getExpr(IU: U);
2964	if (!Expr)
2965	continue;
2966
2967	// Collect interesting types.
2968	Types.insert(X: SE.getEffectiveSCEVType(Ty: Expr->getType()));
2969
2970	// Add strides for mentioned loops.
2971	Worklist.push_back(Elt: Expr);
2972	do {
2973	const SCEV *S = Worklist.pop_back_val();
2974	if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
2975	if (AR->getLoop() == L)
2976	Strides.insert(X: AR->getStepRecurrence(SE));
2977	Worklist.push_back(Elt: AR->getStart());
2978	} else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
2979	append_range(C&: Worklist, R: Add->operands());
2980	}
2981	} while (!Worklist.empty());
2982	}
2983
2984	// Compute interesting factors from the set of interesting strides.
2985	for (SmallSetVector<const SCEV *, `4`>::const_iterator
2986	I = Strides.begin(), E = Strides.end(); I != E; ++I)
2987	for (SmallSetVector<const SCEV *, `4`>::const_iterator NewStrideIter =
2988	std::next(x: I); NewStrideIter != E; ++NewStrideIter) {
2989	const SCEV OldStride = I;
2990	const SCEV NewStride = NewStrideIter;
2991
2992	if (SE.getTypeSizeInBits(Ty: OldStride->getType()) !=
2993	SE.getTypeSizeInBits(Ty: NewStride->getType())) {
2994	if (SE.getTypeSizeInBits(Ty: OldStride->getType()) >
2995	SE.getTypeSizeInBits(Ty: NewStride->getType()))
2996	NewStride = SE.getSignExtendExpr(Op: NewStride, Ty: OldStride->getType());
2997	else
2998	OldStride = SE.getSignExtendExpr(Op: OldStride, Ty: NewStride->getType());
2999	}
3000	if (const SCEVConstant *Factor =
3001	dyn_cast_or_null<SCEVConstant>(Val: getExactSDiv(LHS: NewStride, RHS: OldStride,
3002	SE, IgnoreSignificantBits: true))) {
3003	if (Factor->getAPInt().getSignificantBits() <= `64` && !Factor->isZero())
3004	Factors.insert(X: Factor->getAPInt().getSExtValue());
3005	} else if (const SCEVConstant *Factor =
3006	dyn_cast_or_null<SCEVConstant>(Val: getExactSDiv(LHS: OldStride,
3007	RHS: NewStride,
3008	SE, IgnoreSignificantBits: true))) {
3009	if (Factor->getAPInt().getSignificantBits() <= `64` && !Factor->isZero())
3010	Factors.insert(X: Factor->getAPInt().getSExtValue());
3011	}
3012	}
3013
3014	// If all uses use the same type, don't bother looking for truncation-based
3015	// reuse.
3016	if (Types.size() == `1`)
3017	Types.clear();
3018
3019	LLVM_DEBUG(print_factors_and_types(dbgs()));
3020	}
3021
3022	/// Helper for CollectChains that finds an IV operand (computed by an AddRec in
3023	/// this loop) within [OI,OE) or returns OE. If IVUsers mapped Instructions to
3024	/// IVStrideUses, we could partially skip this.
3025	static User::op_iterator
3026	findIVOperand(User::op_iterator OI, User::op_iterator OE,
3027	Loop *L, ScalarEvolution &SE) {
3028	for(; OI != OE; ++OI) {
3029	if (Instruction Oper = dyn_cast<Instruction>(Val&: OI)) {
3030	if (!SE.isSCEVable(Ty: Oper->getType()))
3031	continue;
3032
3033	if (const SCEVAddRecExpr *AR =
3034	dyn_cast<SCEVAddRecExpr>(Val: SE.getSCEV(V: Oper))) {
3035	if (AR->getLoop() == L)
3036	break;
3037	}
3038	}
3039	}
3040	return OI;
3041	}
3042
3043	/// IVChain logic must consistently peek base TruncInst operands, so wrap it in
3044	/// a convenient helper.
3045	static Value getWideOperand(Value Oper) {
3046	if (TruncInst *Trunc = dyn_cast<TruncInst>(Val: Oper))
3047	return Trunc->getOperand(i_nocapture: `0`);
3048	return Oper;
3049	}
3050
3051	/// Return an approximation of this SCEV expression's "base", or NULL for any
3052	/// constant. Returning the expression itself is conservative. Returning a
3053	/// deeper subexpression is more precise and valid as long as it isn't less
3054	/// complex than another subexpression. For expressions involving multiple
3055	/// unscaled values, we need to return the pointer-type SCEVUnknown. This avoids
3056	/// forming chains across objects, such as: PrevOper==a[i], IVOper==b[i],
3057	/// IVInc==b-a.
3058	///
3059	/// Since SCEVUnknown is the rightmost type, and pointers are the rightmost
3060	/// SCEVUnknown, we simply return the rightmost SCEV operand.
3061	static const SCEV getExprBase(const* SCEV *S) {
3062	switch (S->getSCEVType()) {
3063	default: // including scUnknown.
3064	return S;
3065	case scConstant:
3066	case scVScale:
3067	return nullptr;
3068	case scTruncate:
3069	return getExprBase(S: cast<SCEVTruncateExpr>(Val: S)->getOperand());
3070	case scZeroExtend:
3071	return getExprBase(S: cast<SCEVZeroExtendExpr>(Val: S)->getOperand());
3072	case scSignExtend:
3073	return getExprBase(S: cast<SCEVSignExtendExpr>(Val: S)->getOperand());
3074	case scAddExpr: {
3075	// Skip over scaled operands (scMulExpr) to follow add operands as long as
3076	// there's nothing more complex.
3077	// FIXME: not sure if we want to recognize negation.
3078	const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: S);
3079	for (const SCEV *SubExpr : reverse(C: Add->operands())) {
3080	if (SubExpr->getSCEVType() == scAddExpr)
3081	return getExprBase(S: SubExpr);
3082
3083	if (SubExpr->getSCEVType() != scMulExpr)
3084	return SubExpr;
3085	}
3086	return S; // all operands are scaled, be conservative.
3087	}
3088	case scAddRecExpr:
3089	return getExprBase(S: cast<SCEVAddRecExpr>(Val: S)->getStart());
3090	}
3091	llvm_unreachable("Unknown SCEV kind!");
3092	}
3093
3094	/// Return true if the chain increment is profitable to expand into a loop
3095	/// invariant value, which may require its own register. A profitable chain
3096	/// increment will be an offset relative to the same base. We allow such offsets
3097	/// to potentially be used as chain increment as long as it's not obviously
3098	/// expensive to expand using real instructions.
3099	bool IVChain::isProfitableIncrement(const SCEV *OperExpr,
3100	const SCEV *IncExpr,
3101	ScalarEvolution &SE) {
3102	// Aggressively form chains when -stress-ivchain.
3103	if (StressIVChain)
3104	return true;
3105
3106	// Do not replace a constant offset from IV head with a nonconstant IV
3107	// increment.
3108	if (!isa<SCEVConstant>(Val: IncExpr)) {
3109	const SCEV *HeadExpr = SE.getSCEV(V: getWideOperand(Oper: Incs [`0`].IVOperand));
3110	if (isa<SCEVConstant>(Val: SE.getMinusSCEV(LHS: OperExpr, RHS: HeadExpr)))
3111	return false;
3112	}
3113
3114	SmallPtrSet<const SCEV*, `8`> Processed;
3115	return !isHighCostExpansion(S: IncExpr, Processed, SE);
3116	}
3117
3118	/// Return true if the number of registers needed for the chain is estimated to
3119	/// be less than the number required for the individual IV users. First prohibit
3120	/// any IV users that keep the IV live across increments (the Users set should
3121	/// be empty). Next count the number and type of increments in the chain.
3122	///
3123	/// Chaining IVs can lead to considerable code bloat if ISEL doesn't
3124	/// effectively use postinc addressing modes. Only consider it profitable it the
3125	/// increments can be computed in fewer registers when chained.
3126	///
3127	/// TODO: Consider IVInc free if it's already used in another chains.
3128	static bool isProfitableChain(IVChain &Chain,
3129	SmallPtrSetImpl<Instruction *> &Users,
3130	ScalarEvolution &SE,
3131	const TargetTransformInfo &TTI) {
3132	if (StressIVChain)
3133	return true;
3134
3135	if (!Chain.hasIncs())
3136	return false;
3137
3138	if (!Users.empty()) {
3139	LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[`0`].UserInst << " users:\n";
3140	for (Instruction *Inst
3141	: Users) { dbgs() << " " << *Inst << "\n"; });
3142	return false;
3143	}
3144	assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
3145
3146	// The chain itself may require a register, so initialize cost to 1.
3147	int cost = `1`;
3148
3149	// A complete chain likely eliminates the need for keeping the original IV in
3150	// a register. LSR does not currently know how to form a complete chain unless
3151	// the header phi already exists.
3152	if (isa<PHINode>(Val: Chain.tailUserInst())
3153	&& SE.getSCEV(V: Chain.tailUserInst()) == Chain.Incs [`0`].IncExpr) {
3154	--cost;
3155	}
3156	const SCEV LastIncExpr = nullptr*;
3157	unsigned NumConstIncrements = `0`;
3158	unsigned NumVarIncrements = `0`;
3159	unsigned NumReusedIncrements = `0`;
3160
3161	if (TTI.isProfitableLSRChainElement(I: Chain.Incs [`0`].UserInst))
3162	return true;
3163
3164	for (const IVInc &Inc : Chain) {
3165	if (TTI.isProfitableLSRChainElement(I: Inc.UserInst))
3166	return true;
3167	if (Inc.IncExpr->isZero())
3168	continue;
3169
3170	// Incrementing by zero or some constant is neutral. We assume constants can
3171	// be folded into an addressing mode or an add's immediate operand.
3172	if (isa<SCEVConstant>(Val: Inc.IncExpr)) {
3173	++NumConstIncrements;
3174	continue;
3175	}
3176
3177	if (Inc.IncExpr == LastIncExpr)
3178	++NumReusedIncrements;
3179	else
3180	++NumVarIncrements;
3181
3182	LastIncExpr = Inc.IncExpr;
3183	}
3184	// An IV chain with a single increment is handled by LSR's postinc
3185	// uses. However, a chain with multiple increments requires keeping the IV's
3186	// value live longer than it needs to be if chained.
3187	if (NumConstIncrements > `1`)
3188	--cost;
3189
3190	// Materializing increment expressions in the preheader that didn't exist in
3191	// the original code may cost a register. For example, sign-extended array
3192	// indices can produce ridiculous increments like this:
3193	// IV + ((sext i32 (2 %s) to i64) + (-1 * (sext i32 %s to i64)))*
3194	cost += NumVarIncrements;
3195
3196	// Reusing variable increments likely saves a register to hold the multiple of
3197	// the stride.
3198	cost -= NumReusedIncrements;
3199
3200	LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[`0`].UserInst << " Cost: " << cost
3201	<< "\n");
3202
3203	return cost < `0`;
3204	}
3205
3206	/// Add this IV user to an existing chain or make it the head of a new chain.
3207	void LSRInstance::ChainInstruction(Instruction UserInst, Instruction IVOper,
3208	SmallVectorImpl<ChainUsers> &ChainUsersVec) {
3209	// When IVs are used as types of varying widths, they are generally converted
3210	// to a wider type with some uses remaining narrow under a (free) trunc.
3211	Value *const NextIV = getWideOperand(Oper: IVOper);
3212	const SCEV *const OperExpr = SE.getSCEV(V: NextIV);
3213	const SCEV *const OperExprBase = getExprBase(S: OperExpr);
3214
3215	// Visit all existing chains. Check if its IVOper can be computed as a
3216	// profitable loop invariant increment from the last link in the Chain.
3217	unsigned ChainIdx = `0`, NChains = IVChainVec.size();
3218	const SCEV LastIncExpr = nullptr*;
3219	for (; ChainIdx < NChains; ++ChainIdx) {
3220	IVChain &Chain = IVChainVec [ChainIdx];
3221
3222	// Prune the solution space aggressively by checking that both IV operands
3223	// are expressions that operate on the same unscaled SCEVUnknown. This
3224	// "base" will be canceled by the subsequent getMinusSCEV call. Checking
3225	// first avoids creating extra SCEV expressions.
3226	if (!StressIVChain && Chain.ExprBase != OperExprBase)
3227	continue;
3228
3229	Value *PrevIV = getWideOperand(Oper: Chain.Incs.back().IVOperand);
3230	if (PrevIV->getType() != NextIV->getType())
3231	continue;
3232
3233	// A phi node terminates a chain.
3234	if (isa<PHINode>(Val: UserInst) && isa<PHINode>(Val: Chain.tailUserInst()))
3235	continue;
3236
3237	// The increment must be loop-invariant so it can be kept in a register.
3238	const SCEV *PrevExpr = SE.getSCEV(V: PrevIV);
3239	const SCEV *IncExpr = SE.getMinusSCEV(LHS: OperExpr, RHS: PrevExpr);
3240	if (isa<SCEVCouldNotCompute>(Val: IncExpr) \|\| !SE.isLoopInvariant(S: IncExpr, L))
3241	continue;
3242
3243	if (Chain.isProfitableIncrement(OperExpr, IncExpr, SE)) {
3244	LastIncExpr = IncExpr;
3245	break;
3246	}
3247	}
3248	// If we haven't found a chain, create a new one, unless we hit the max. Don't
3249	// bother for phi nodes, because they must be last in the chain.
3250	if (ChainIdx == NChains) {
3251	if (isa<PHINode>(Val: UserInst))
3252	return;
3253	if (NChains >= MaxChains && !StressIVChain) {
3254	LLVM_DEBUG(dbgs() << "IV Chain Limit\n");
3255	return;
3256	}
3257	LastIncExpr = OperExpr;
3258	// IVUsers may have skipped over sign/zero extensions. We don't currently
3259	// attempt to form chains involving extensions unless they can be hoisted
3260	// into this loop's AddRec.
3261	if (!isa<SCEVAddRecExpr>(Val: LastIncExpr))
3262	return;
3263	++NChains;
3264	IVChainVec.push_back(Elt: IVChain (IVInc (UserInst, IVOper, LastIncExpr),
3265	OperExprBase));
3266	ChainUsersVec.resize(N: NChains);
3267	LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Head: (" << *UserInst
3268	<< ") IV=" << *LastIncExpr << "\n");
3269	} else {
3270	LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Inc: (" << *UserInst
3271	<< ") IV+" << *LastIncExpr << "\n");
3272	// Add this IV user to the end of the chain.
3273	IVChainVec [ChainIdx].add(X: IVInc (UserInst, IVOper, LastIncExpr));
3274	}
3275	IVChain &Chain = IVChainVec [ChainIdx];
3276
3277	SmallPtrSet<Instruction*,`4`> &NearUsers = ChainUsersVec [ChainIdx].NearUsers;
3278	// This chain's NearUsers become FarUsers.
3279	if (!LastIncExpr->isZero()) {
3280	ChainUsersVec [ChainIdx].FarUsers.insert_range(R&: NearUsers);
3281	NearUsers.clear();
3282	}
3283
3284	// All other uses of IVOperand become near uses of the chain.
3285	// We currently ignore intermediate values within SCEV expressions, assuming
3286	// they will eventually be used be the current chain, or can be computed
3287	// from one of the chain increments. To be more precise we could
3288	// transitively follow its user and only add leaf IV users to the set.
3289	for (User *U : IVOper->users()) {
3290	Instruction *OtherUse = dyn_cast<Instruction>(Val: U);
3291	if (!OtherUse)
3292	continue;
3293	// Uses in the chain will no longer be uses if the chain is formed.
3294	// Include the head of the chain in this iteration (not Chain.begin()).
3295	IVChain::const_iterator IncIter = Chain.Incs.begin();
3296	IVChain::const_iterator IncEnd = Chain.Incs.end();
3297	for( ; IncIter != IncEnd; ++IncIter) {
3298	if (IncIter->UserInst == OtherUse)
3299	break;
3300	}
3301	if (IncIter != IncEnd)
3302	continue;
3303
3304	if (SE.isSCEVable(Ty: OtherUse->getType())
3305	&& !isa<SCEVUnknown>(Val: SE.getSCEV(V: OtherUse))
3306	&& IU.isIVUserOrOperand(Inst: OtherUse)) {
3307	continue;
3308	}
3309	NearUsers.insert(Ptr: OtherUse);
3310	}
3311
3312	// Since this user is part of the chain, it's no longer considered a use
3313	// of the chain.
3314	ChainUsersVec [ChainIdx].FarUsers.erase(Ptr: UserInst);
3315	}
3316
3317	/// Populate the vector of Chains.
3318	///
3319	/// This decreases ILP at the architecture level. Targets with ample registers,
3320	/// multiple memory ports, and no register renaming probably don't want
3321	/// this. However, such targets should probably disable LSR altogether.
3322	///
3323	/// The job of LSR is to make a reasonable choice of induction variables across
3324	/// the loop. Subsequent passes can easily "unchain" computation exposing more
3325	/// ILP within the loop* if the target wants it.*
3326	///
3327	/// Finding the best IV chain is potentially a scheduling problem. Since LSR
3328	/// will not reorder memory operations, it will recognize this as a chain, but
3329	/// will generate redundant IV increments. Ideally this would be corrected later
3330	/// by a smart scheduler:
3331	/// = A[i]
3332	/// = A[i+x]
3333	/// A[i] =
3334	/// A[i+x] =
3335	///
3336	/// TODO: Walk the entire domtree within this loop, not just the path to the
3337	/// loop latch. This will discover chains on side paths, but requires
3338	/// maintaining multiple copies of the Chains state.
3339	void LSRInstance::CollectChains() {
3340	LLVM_DEBUG(dbgs() << "Collecting IV Chains.\n");
3341	SmallVector<ChainUsers, `8`> ChainUsersVec;
3342
3343	SmallVector<BasicBlock *,`8`> LatchPath;
3344	BasicBlock *LoopHeader = L->getHeader();
3345	for (DomTreeNode *Rung = DT.getNode(BB: L->getLoopLatch());
3346	Rung->getBlock() != LoopHeader; Rung = Rung->getIDom()) {
3347	LatchPath.push_back(Elt: Rung->getBlock());
3348	}
3349	LatchPath.push_back(Elt: LoopHeader);
3350
3351	// Walk the instruction stream from the loop header to the loop latch.
3352	for (BasicBlock *BB : reverse(C&: LatchPath)) {
3353	for (Instruction &I : *BB) {
3354	// Skip instructions that weren't seen by IVUsers analysis.
3355	if (isa<PHINode>(Val: I) \|\| !IU.isIVUserOrOperand(Inst: &I))
3356	continue;
3357
3358	// Skip ephemeral values, as they don't produce real code.
3359	if (IU.isEphemeral(V: &I))
3360	continue;
3361
3362	// Ignore users that are part of a SCEV expression. This way we only
3363	// consider leaf IV Users. This effectively rediscovers a portion of
3364	// IVUsers analysis but in program order this time.
3365	if (SE.isSCEVable(Ty: I.getType()) && !isa<SCEVUnknown>(Val: SE.getSCEV(V: &I)))
3366	continue;
3367
3368	// Remove this instruction from any NearUsers set it may be in.
3369	for (unsigned ChainIdx = `0`, NChains = IVChainVec.size();
3370	ChainIdx < NChains; ++ChainIdx) {
3371	ChainUsersVec [ChainIdx].NearUsers.erase(Ptr: &I);
3372	}
3373	// Search for operands that can be chained.
3374	SmallPtrSet<Instruction*, `4`> UniqueOperands;
3375	User::op_iterator IVOpEnd = I.op_end();
3376	User::op_iterator IVOpIter = findIVOperand(OI: I.op_begin(), OE: IVOpEnd, L, SE);
3377	while (IVOpIter != IVOpEnd) {
3378	Instruction IVOpInst = cast<Instruction>(Val&: IVOpIter);
3379	if (UniqueOperands.insert(Ptr: IVOpInst).second)
3380	ChainInstruction(UserInst: &I, IVOper: IVOpInst, ChainUsersVec);
3381	IVOpIter = findIVOperand(OI: std::next(x: IVOpIter), OE: IVOpEnd, L, SE);
3382	}
3383	} // Continue walking down the instructions.
3384	} // Continue walking down the domtree.
3385	// Visit phi backedges to determine if the chain can generate the IV postinc.
3386	for (PHINode &PN : L->getHeader()->phis()) {
3387	if (!SE.isSCEVable(Ty: PN.getType()))
3388	continue;
3389
3390	Instruction *IncV =
3391	dyn_cast<Instruction>(Val: PN.getIncomingValueForBlock(BB: L->getLoopLatch()));
3392	if (IncV)
3393	ChainInstruction(UserInst: &PN, IVOper: IncV, ChainUsersVec);
3394	}
3395	// Remove any unprofitable chains.
3396	unsigned ChainIdx = `0`;
3397	for (unsigned UsersIdx = `0`, NChains = IVChainVec.size();
3398	UsersIdx < NChains; ++UsersIdx) {
3399	if (!isProfitableChain(Chain&: IVChainVec [UsersIdx],
3400	Users&: ChainUsersVec [UsersIdx].FarUsers, SE, TTI))
3401	continue;
3402	// Preserve the chain at UsesIdx.
3403	if (ChainIdx != UsersIdx)
3404	IVChainVec [ChainIdx] = IVChainVec [UsersIdx];
3405	FinalizeChain(Chain&: IVChainVec [ChainIdx]);
3406	++ChainIdx;
3407	}
3408	IVChainVec.resize(N: ChainIdx);
3409	}
3410
3411	void LSRInstance::FinalizeChain(IVChain &Chain) {
3412	assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
3413	LLVM_DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[`0`].UserInst << "\n");
3414
3415	for (const IVInc &Inc : Chain) {
3416	LLVM_DEBUG(dbgs() << " Inc: " << *Inc.UserInst << "\n");
3417	auto UseI = find(Range: Inc.UserInst->operands(), Val: Inc.IVOperand);
3418	assert(UseI != Inc.UserInst->op_end() && "cannot find IV operand");
3419	IVIncSet.insert(Ptr: UseI);
3420	}
3421	}
3422
3423	/// Return true if the IVInc can be folded into an addressing mode.
3424	static bool canFoldIVIncExpr(const SCEV IncExpr, Instruction UserInst,
3425	Value Operand, const* TargetTransformInfo &TTI) {
3426	const SCEVConstant *IncConst = dyn_cast<SCEVConstant>(Val: IncExpr);
3427	Immediate IncOffset = Immediate::getZero();
3428	if (IncConst) {
3429	if (IncConst && IncConst->getAPInt().getSignificantBits() > `64`)
3430	return false;
3431	IncOffset = Immediate::getFixed(MinVal: IncConst->getValue()->getSExtValue());
3432	} else {
3433	// Look for mul(vscale, constant), to detect a scalable offset.
3434	const APInt *C;
3435	if (!match(S: IncExpr, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEVVScale())) \|\|
3436	C->getSignificantBits() > `64`)
3437	return false;
3438	IncOffset = Immediate::getScalable(MinVal: C->getSExtValue());
3439	}
3440
3441	if (!isAddressUse(TTI, Inst: UserInst, OperandVal: Operand))
3442	return false;
3443
3444	MemAccessTy AccessTy = getAccessType(TTI, Inst: UserInst, OperandVal: Operand);
3445	if (!isAlwaysFoldable(TTI, Kind: LSRUse::Address, AccessTy, /BaseGV=/nullptr,
3446	BaseOffset: IncOffset, /HasBaseReg=/false))
3447	return false;
3448
3449	return true;
3450	}
3451
3452	/// Generate an add or subtract for each IVInc in a chain to materialize the IV
3453	/// user's operand from the previous IV user's operand.
3454	void LSRInstance::GenerateIVChain(const IVChain &Chain,
3455	SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
3456	// Find the new IVOperand for the head of the chain. It may have been replaced
3457	// by LSR.
3458	const IVInc &Head = Chain.Incs [`0`];
3459	User::op_iterator IVOpEnd = Head.UserInst->op_end();
3460	// findIVOperand returns IVOpEnd if it can no longer find a valid IV user.
3461	User::op_iterator IVOpIter = findIVOperand(OI: Head.UserInst->op_begin(),
3462	OE: IVOpEnd, L, SE);
3463	Value IVSrc = nullptr*;
3464	while (IVOpIter != IVOpEnd) {
3465	IVSrc = getWideOperand(Oper: *IVOpIter);
3466
3467	// If this operand computes the expression that the chain needs, we may use
3468	// it. (Check this after setting IVSrc which is used below.)
3469	//
3470	// Note that if Head.IncExpr is wider than IVSrc, then this phi is too
3471	// narrow for the chain, so we can no longer use it. We do allow using a
3472	// wider phi, assuming the LSR checked for free truncation. In that case we
3473	// should already have a truncate on this operand such that
3474	// getSCEV(IVSrc) == IncExpr.
3475	if (SE.getSCEV(V: *IVOpIter) == Head.IncExpr
3476	\|\| SE.getSCEV(V: IVSrc) == Head.IncExpr) {
3477	break;
3478	}
3479	IVOpIter = findIVOperand(OI: std::next(x: IVOpIter), OE: IVOpEnd, L, SE);
3480	}
3481	if (IVOpIter == IVOpEnd) {
3482	// Gracefully give up on this chain.
3483	LLVM_DEBUG(dbgs() << "Concealed chain head: " << *Head.UserInst << "\n");
3484	return;
3485	}
3486	assert(IVSrc && "Failed to find IV chain source");
3487
3488	LLVM_DEBUG(dbgs() << "Generate chain at: " << *IVSrc << "\n");
3489	Type *IVTy = IVSrc->getType();
3490	Type *IntTy = SE.getEffectiveSCEVType(Ty: IVTy);
3491	const SCEV LeftOverExpr = nullptr*;
3492	const SCEV *Accum = SE.getZero(Ty: IntTy);
3493	SmallVector<std::pair<const SCEV , Value >> Bases;
3494	Bases.emplace_back(Args&: Accum, Args&: IVSrc);
3495
3496	for (const IVInc &Inc : Chain) {
3497	Instruction *InsertPt = Inc.UserInst;
3498	if (isa<PHINode>(Val: InsertPt))
3499	InsertPt = L->getLoopLatch()->getTerminator();
3500
3501	// IVOper will replace the current IV User's operand. IVSrc is the IV
3502	// value currently held in a register.
3503	Value *IVOper = IVSrc;
3504	if (!Inc.IncExpr->isZero()) {
3505	// IncExpr was the result of subtraction of two narrow values, so must
3506	// be signed.
3507	const SCEV *IncExpr = SE.getNoopOrSignExtend(V: Inc.IncExpr, Ty: IntTy);
3508	Accum = SE.getAddExpr(LHS: Accum, RHS: IncExpr);
3509	LeftOverExpr = LeftOverExpr ?
3510	SE.getAddExpr(LHS: LeftOverExpr, RHS: IncExpr) : IncExpr;
3511	}
3512
3513	// Look through each base to see if any can produce a nice addressing mode.
3514	bool FoundBase = false;
3515	for (auto [MapScev, MapIVOper] : reverse(C&: Bases)) {
3516	const SCEV *Remainder = SE.getMinusSCEV(LHS: Accum, RHS: MapScev);
3517	if (canFoldIVIncExpr(IncExpr: Remainder, UserInst: Inc.UserInst, Operand: Inc.IVOperand, TTI)) {
3518	if (!Remainder->isZero()) {
3519	Rewriter.clearPostInc();
3520	Value *IncV = Rewriter.expandCodeFor(SH: Remainder, Ty: IntTy, I: InsertPt);
3521	const SCEV *IVOperExpr =
3522	SE.getAddExpr(LHS: SE.getUnknown(V: MapIVOper), RHS: SE.getUnknown(V: IncV));
3523	IVOper = Rewriter.expandCodeFor(SH: IVOperExpr, Ty: IVTy, I: InsertPt);
3524	} else {
3525	IVOper = MapIVOper;
3526	}
3527
3528	FoundBase = true;
3529	break;
3530	}
3531	}
3532	if (!FoundBase && LeftOverExpr && !LeftOverExpr->isZero()) {
3533	// Expand the IV increment.
3534	Rewriter.clearPostInc();
3535	Value *IncV = Rewriter.expandCodeFor(SH: LeftOverExpr, Ty: IntTy, I: InsertPt);
3536	const SCEV *IVOperExpr = SE.getAddExpr(LHS: SE.getUnknown(V: IVSrc),
3537	RHS: SE.getUnknown(V: IncV));
3538	IVOper = Rewriter.expandCodeFor(SH: IVOperExpr, Ty: IVTy, I: InsertPt);
3539
3540	// If an IV increment can't be folded, use it as the next IV value.
3541	if (!canFoldIVIncExpr(IncExpr: LeftOverExpr, UserInst: Inc.UserInst, Operand: Inc.IVOperand, TTI)) {
3542	assert(IVTy == IVOper->getType() && "inconsistent IV increment type");
3543	Bases.emplace_back(Args&: Accum, Args&: IVOper);
3544	IVSrc = IVOper;
3545	LeftOverExpr = nullptr;
3546	}
3547	}
3548	Type *OperTy = Inc.IVOperand->getType();
3549	if (IVTy != OperTy) {
3550	assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) &&
3551	"cannot extend a chained IV");
3552	IRBuilder<> Builder(InsertPt);
3553	IVOper = Builder.CreateTruncOrBitCast(V: IVOper, DestTy: OperTy, Name: "lsr.chain");
3554	}
3555	Inc.UserInst->replaceUsesOfWith(From: Inc.IVOperand, To: IVOper);
3556	if (auto *OperandIsInstr = dyn_cast<Instruction>(Val: Inc.IVOperand))
3557	DeadInsts.emplace_back(Args&: OperandIsInstr);
3558	}
3559	// If LSR created a new, wider phi, we may also replace its postinc. We only
3560	// do this if we also found a wide value for the head of the chain.
3561	if (isa<PHINode>(Val: Chain.tailUserInst())) {
3562	for (PHINode &Phi : L->getHeader()->phis()) {
3563	if (Phi.getType() != IVSrc->getType())
3564	continue;
3565	Instruction *PostIncV = dyn_cast<Instruction>(
3566	Val: Phi.getIncomingValueForBlock(BB: L->getLoopLatch()));
3567	if (!PostIncV \|\| (SE.getSCEV(V: PostIncV) != SE.getSCEV(V: IVSrc)))
3568	continue;
3569	Value *IVOper = IVSrc;
3570	Type *PostIncTy = PostIncV->getType();
3571	if (IVTy != PostIncTy) {
3572	assert(PostIncTy->isPointerTy() && "mixing int/ptr IV types");
3573	IRBuilder<> Builder(L->getLoopLatch()->getTerminator());
3574	Builder.SetCurrentDebugLocation(PostIncV->getDebugLoc());
3575	IVOper = Builder.CreatePointerCast(V: IVSrc, DestTy: PostIncTy, Name: "lsr.chain");
3576	}
3577	Phi.replaceUsesOfWith(From: PostIncV, To: IVOper);
3578	DeadInsts.emplace_back(Args&: PostIncV);
3579	}
3580	}
3581	}
3582
3583	void LSRInstance::CollectFixupsAndInitialFormulae() {
3584	CondBrInst ExitBranch = nullptr*;
3585	bool SaveCmp = TTI.canSaveCmp(L, BI: &ExitBranch, SE: &SE, LI: &LI, DT: &DT, AC: &AC, LibInfo: &TLI);
3586
3587	// For calculating baseline cost
3588	SmallPtrSet<const SCEV *, `16`> Regs;
3589	DenseSet<const SCEV *> VisitedRegs;
3590	DenseSet<size_t> VisitedLSRUse;
3591
3592	for (const IVStrideUse &U : IU) {
3593	Instruction *UserInst = U.getUser();
3594	// Skip IV users that are part of profitable IV Chains.
3595	User::op_iterator UseI =
3596	find(Range: UserInst->operands(), Val: U.getOperandValToReplace());
3597	assert(UseI != UserInst->op_end() && "cannot find IV operand");
3598	if (IVIncSet.count(Ptr: UseI)) {
3599	LLVM_DEBUG(dbgs() << "Use is in profitable chain: " << **UseI << `'\n'`);
3600	continue;
3601	}
3602
3603	LSRUse::KindType Kind = LSRUse::Basic;
3604	MemAccessTy AccessTy;
3605	if (isAddressUse(TTI, Inst: UserInst, OperandVal: U.getOperandValToReplace())) {
3606	Kind = LSRUse::Address;
3607	AccessTy = getAccessType(TTI, Inst: UserInst, OperandVal: U.getOperandValToReplace());
3608	}
3609
3610	const SCEV *S = IU.getExpr(IU: U);
3611	if (!S)
3612	continue;
3613	PostIncLoopSet TmpPostIncLoops = U.getPostIncLoops();
3614
3615	// Equality (== and !=) ICmps are special. We can rewrite (i == N) as
3616	// (N - i == 0), and this allows (N - i) to be the expression that we work
3617	// with rather than just N or i, so we can consider the register
3618	// requirements for both N and i at the same time. Limiting this code to
3619	// equality icmps is not a problem because all interesting loops use
3620	// equality icmps, thanks to IndVarSimplify.
3621	if (ICmpInst *CI = dyn_cast<ICmpInst>(Val: UserInst)) {
3622	// If CI can be saved in some target, like replaced inside hardware loop
3623	// in PowerPC, no need to generate initial formulae for it.
3624	if (SaveCmp && CI == dyn_cast<ICmpInst>(Val: ExitBranch->getCondition()))
3625	continue;
3626	if (CI->isEquality()) {
3627	// Swap the operands if needed to put the OperandValToReplace on the
3628	// left, for consistency.
3629	Value *NV = CI->getOperand(i_nocapture: `1`);
3630	if (NV == U.getOperandValToReplace()) {
3631	CI->setOperand(i_nocapture: `1`, Val_nocapture: CI->getOperand(i_nocapture: `0`));
3632	CI->setOperand(i_nocapture: `0`, Val_nocapture: NV);
3633	NV = CI->getOperand(i_nocapture: `1`);
3634	Changed = true;
3635	}
3636
3637	// x == y --> x - y == 0
3638	const SCEV *N = SE.getSCEV(V: NV);
3639	if (SE.isLoopInvariant(S: N, L) && Rewriter.isSafeToExpand(S: N) &&
3640	(!NV->getType()->isPointerTy() \|\|
3641	SE.getPointerBase(V: N) == SE.getPointerBase(V: S))) {
3642	// S is normalized, so normalize N before folding it into S
3643	// to keep the result normalized.
3644	N = normalizeForPostIncUse(S: N, Loops: TmpPostIncLoops, SE);
3645	if (!N)
3646	continue;
3647	Kind = LSRUse::ICmpZero;
3648	S = SE.getMinusSCEV(LHS: N, RHS: S);
3649	} else if (L->isLoopInvariant(V: NV) &&
3650	(!isa<Instruction>(Val: NV) \|\|
3651	DT.dominates(Def: cast<Instruction>(Val: NV), BB: L->getHeader())) &&
3652	!NV->getType()->isPointerTy()) {
3653	// If we can't generally expand the expression (e.g. it contains
3654	// a divide), but it is already at a loop invariant point before the
3655	// loop, wrap it in an unknown (to prevent the expander from trying
3656	// to re-expand in a potentially unsafe way.) The restriction to
3657	// integer types is required because the unknown hides the base, and
3658	// SCEV can't compute the difference of two unknown pointers.
3659	N = SE.getUnknown(V: NV);
3660	N = normalizeForPostIncUse(S: N, Loops: TmpPostIncLoops, SE);
3661	if (!N)
3662	continue;
3663	Kind = LSRUse::ICmpZero;
3664	S = SE.getMinusSCEV(LHS: N, RHS: S);
3665	assert(!isa<SCEVCouldNotCompute>(S));
3666	}
3667
3668	// -1 and the negations of all interesting strides (except the negation
3669	// of -1) are now also interesting.
3670	for (size_t i = `0`, e = Factors.size(); i != e; ++i)
3671	if (Factors [i] != -`1`)
3672	Factors.insert(X: -(uint64_t)Factors [i]);
3673	Factors.insert(X: -`1`);
3674	}
3675	}
3676
3677	// Get or create an LSRUse.
3678	std::pair<size_t, Immediate> P = getUse(Expr&: S, Kind, AccessTy);
3679	size_t LUIdx = P.first;
3680	Immediate Offset = P.second;
3681	LSRUse &LU = Uses [LUIdx];
3682
3683	// Record the fixup.
3684	LSRFixup &LF = LU.getNewFixup();
3685	LF.UserInst = UserInst;
3686	LF.OperandValToReplace = U.getOperandValToReplace();
3687	LF.PostIncLoops = TmpPostIncLoops;
3688	LF.Offset = Offset;
3689	LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3690	LU.AllFixupsUnconditional &= IsFixupExecutedEachIncrement(LF);
3691
3692	// Create SCEV as Formula for calculating baseline cost
3693	if (!VisitedLSRUse.count(V: LUIdx) && !LF.isUseFullyOutsideLoop(L)) {
3694	Formula F;
3695	F.initialMatch(S, L, SE);
3696	BaselineCost.RateFormula(F, Regs, VisitedRegs, LU,
3697	HardwareLoopProfitable);
3698	VisitedLSRUse.insert(V: LUIdx);
3699	}
3700
3701	// If this is the first use of this LSRUse, give it a formula.
3702	if (LU.Formulae.empty()) {
3703	InsertInitialFormula(S, LU, LUIdx);
3704	CountRegisters(F: LU.Formulae.back(), LUIdx);
3705	}
3706	}
3707
3708	LLVM_DEBUG(print_fixups(dbgs()));
3709	}
3710
3711	/// Insert a formula for the given expression into the given use, separating out
3712	/// loop-variant portions from loop-invariant and loop-computable portions.
3713	void LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU,
3714	size_t LUIdx) {
3715	// Mark uses whose expressions cannot be expanded.
3716	if (!Rewriter.isSafeToExpand(S))
3717	LU.RigidFormula = true;
3718
3719	Formula F;
3720	F.initialMatch(S, L, SE);
3721	bool Inserted = InsertFormula(LU, LUIdx, F);
3722	assert(Inserted && "Initial formula already exists!"); (void)Inserted;
3723	}
3724
3725	/// Insert a simple single-register formula for the given expression into the
3726	/// given use.
3727	void
3728	LSRInstance::InsertSupplementalFormula(const SCEV *S,
3729	LSRUse &LU, size_t LUIdx) {
3730	Formula F;
3731	F.BaseRegs.push_back(Elt: S);
3732	F.HasBaseReg = true;
3733	bool Inserted = InsertFormula(LU, LUIdx, F);
3734	assert(Inserted && "Supplemental formula already exists!"); (void)Inserted;
3735	}
3736
3737	/// Note which registers are used by the given formula, updating RegUses.
3738	void LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) {
3739	if (F.ScaledReg)
3740	RegUses.countRegister(Reg: F.ScaledReg, LUIdx);
3741	for (const SCEV *BaseReg : F.BaseRegs)
3742	RegUses.countRegister(Reg: BaseReg, LUIdx);
3743	}
3744
3745	/// If the given formula has not yet been inserted, add it to the list, and
3746	/// return true. Return false otherwise.
3747	bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
3748	// Do not insert formula that we will not be able to expand.
3749	assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
3750	"Formula is illegal");
3751
3752	if (!LU.InsertFormula(F, L: *L))
3753	return false;
3754
3755	CountRegisters(F, LUIdx);
3756	return true;
3757	}
3758
3759	/// Test whether this fixup will be executed each time the corresponding IV
3760	/// increment instruction is executed.
3761	bool LSRInstance::IsFixupExecutedEachIncrement(const LSRFixup &LF) const {
3762	// If the fixup block dominates the IV increment block then there is no path
3763	// through the loop to the increment that doesn't pass through the fixup.
3764	return DT.dominates(A: LF.UserInst->getParent(), B: IVIncInsertPos->getParent());
3765	}
3766
3767	/// Check for other uses of loop-invariant values which we're tracking. These
3768	/// other uses will pin these values in registers, making them less profitable
3769	/// for elimination.
3770	/// TODO: This currently misses non-constant addrec step registers.
3771	/// TODO: Should this give more weight to users inside the loop?
3772	void
3773	LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
3774	SmallVector<const SCEV *, `8`> Worklist(RegUses.begin(), RegUses.end());
3775	SmallPtrSet<const SCEV *, `32`> Visited;
3776
3777	// Don't collect outside uses if we are favoring postinc - the instructions in
3778	// the loop are more important than the ones outside of it.
3779	if (AMK == TTI::AMK_PostIndexed)
3780	return;
3781
3782	while (!Worklist.empty()) {
3783	const SCEV *S = Worklist.pop_back_val();
3784
3785	// Don't process the same SCEV twice
3786	if (!Visited.insert(Ptr: S).second)
3787	continue;
3788
3789	if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(Val: S))
3790	append_range(C&: Worklist, R: N->operands());
3791	else if (const SCEVIntegralCastExpr *C = dyn_cast<SCEVIntegralCastExpr>(Val: S))
3792	Worklist.push_back(Elt: C->getOperand());
3793	else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(Val: S)) {
3794	Worklist.push_back(Elt: D->getLHS());
3795	Worklist.push_back(Elt: D->getRHS());
3796	} else if (const SCEVUnknown *US = dyn_cast<SCEVUnknown>(Val: S)) {
3797	const Value *V = US->getValue();
3798	if (const Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
3799	// Look for instructions defined outside the loop.
3800	if (L->contains(Inst)) continue;
3801	} else if (isa<Constant>(Val: V))
3802	// Constants can be re-materialized.
3803	continue;
3804	for (const Use &U : V->uses()) {
3805	const Instruction *UserInst = dyn_cast<Instruction>(Val: U.getUser());
3806	// Ignore non-instructions.
3807	if (!UserInst)
3808	continue;
3809	// Don't bother if the instruction is an EHPad.
3810	if (UserInst->isEHPad())
3811	continue;
3812	// Ignore instructions in other functions (as can happen with
3813	// Constants).
3814	if (UserInst->getParent()->getParent() != L->getHeader()->getParent())
3815	continue;
3816	// Ignore instructions not dominated by the loop.
3817	const BasicBlock *UseBB = !isa<PHINode>(Val: UserInst) ?
3818	UserInst->getParent() :
3819	cast<PHINode>(Val: UserInst)->getIncomingBlock(
3820	i: PHINode::getIncomingValueNumForOperand(i: U.getOperandNo()));
3821	if (!DT.dominates(A: L->getHeader(), B: UseBB))
3822	continue;
3823	// Don't bother if the instruction is in a BB which ends in an EHPad.
3824	if (UseBB->getTerminator()->isEHPad())
3825	continue;
3826
3827	// Ignore cases in which the currently-examined value could come from
3828	// a basic block terminated with an EHPad. This checks all incoming
3829	// blocks of the phi node since it is possible that the same incoming
3830	// value comes from multiple basic blocks, only some of which may end
3831	// in an EHPad. If any of them do, a subsequent rewrite attempt by this
3832	// pass would try to insert instructions into an EHPad, hitting an
3833	// assertion.
3834	if (isa<PHINode>(Val: UserInst)) {
3835	const auto *PhiNode = cast<PHINode>(Val: UserInst);
3836	bool HasIncompatibleEHPTerminatedBlock = false;
3837	llvm::Value *ExpectedValue = U;
3838	for (unsigned int I = `0`; I < PhiNode->getNumIncomingValues(); I++) {
3839	if (PhiNode->getIncomingValue(i: I) == ExpectedValue) {
3840	if (PhiNode->getIncomingBlock(i: I)->getTerminator()->isEHPad()) {
3841	HasIncompatibleEHPTerminatedBlock = true;
3842	break;
3843	}
3844	}
3845	}
3846	if (HasIncompatibleEHPTerminatedBlock) {
3847	continue;
3848	}
3849	}
3850
3851	// Don't bother rewriting PHIs in catchswitch blocks.
3852	if (isa<CatchSwitchInst>(Val: UserInst->getParent()->getTerminator()))
3853	continue;
3854	// Ignore uses which are part of other SCEV expressions, to avoid
3855	// analyzing them multiple times.
3856	if (SE.isSCEVable(Ty: UserInst->getType())) {
3857	const SCEV UserS = SE.getSCEV(V: const_cast<Instruction >(UserInst));
3858	// If the user is a no-op, look through to its uses.
3859	if (!isa<SCEVUnknown>(Val: UserS))
3860	continue;
3861	if (UserS == US) {
3862	Worklist.push_back(
3863	Elt: SE.getUnknown(V: const_cast<Instruction *>(UserInst)));
3864	continue;
3865	}
3866	}
3867	// Ignore icmp instructions which are already being analyzed.
3868	if (const ICmpInst *ICI = dyn_cast<ICmpInst>(Val: UserInst)) {
3869	unsigned OtherIdx = !U.getOperandNo();
3870	Value *OtherOp = ICI->getOperand(i_nocapture: OtherIdx);
3871	if (SE.hasComputableLoopEvolution(S: SE.getSCEV(V: OtherOp), L))
3872	continue;
3873	}
3874
3875	// Do not consider uses inside lifetime intrinsics. These are not
3876	// actually materialized.
3877	if (UserInst->isLifetimeStartOrEnd())
3878	continue;
3879
3880	std::pair<size_t, Immediate> P =
3881	getUse(Expr&: S, Kind: LSRUse::Basic, AccessTy: MemAccessTy ());
3882	size_t LUIdx = P.first;
3883	Immediate Offset = P.second;
3884	LSRUse &LU = Uses [LUIdx];
3885	LSRFixup &LF = LU.getNewFixup();
3886	LF.UserInst = const_cast<Instruction *>(UserInst);
3887	LF.OperandValToReplace = U;
3888	LF.Offset = Offset;
3889	LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3890	LU.AllFixupsUnconditional &= IsFixupExecutedEachIncrement(LF);
3891	InsertSupplementalFormula(S: US, LU, LUIdx);
3892	CountRegisters(F: LU.Formulae.back(), LUIdx: Uses.size() - `1`);
3893	break;
3894	}
3895	}
3896	}
3897	}
3898
3899	/// Split S into subexpressions which can be pulled out into separate
3900	/// registers. If C is non-null, multiply each subexpression by C.
3901	///
3902	/// Return remainder expression after factoring the subexpressions captured by
3903	/// Ops. If Ops is complete, return NULL.
3904	static const SCEV CollectSubexprs(const* SCEV S, const* SCEVConstant *C,
3905	SmallVectorImpl<const SCEV *> &Ops,
3906	const Loop *L,
3907	ScalarEvolution &SE,
3908	unsigned Depth = `0`) {
3909	// Arbitrarily cap recursion to protect compile time.
3910	if (Depth >= `3`)
3911	return S;
3912
3913	if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
3914	// Break out add operands.
3915	for (const SCEV *S : Add->operands()) {
3916	const SCEV *Remainder = CollectSubexprs(S, C, Ops, L, SE, Depth: Depth+`1`);
3917	if (Remainder)
3918	Ops.push_back(Elt: C ? SE.getMulExpr(LHS: C, RHS: Remainder) : Remainder);
3919	}
3920	return nullptr;
3921	}
3922	const SCEV Start, Step;
3923	const SCEVConstant *Op0;
3924	const SCEV *Op1;
3925	if (match(S, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step)))) {
3926	// Split a non-zero base out of an addrec.
3927	if (Start->isZero())
3928	return S;
3929
3930	const SCEV *Remainder = CollectSubexprs(S: Start, C, Ops, L, SE, Depth: Depth + `1`);
3931	// Split the non-zero AddRec unless it is part of a nested recurrence that
3932	// does not pertain to this loop.
3933	if (Remainder && (cast<SCEVAddRecExpr>(Val: S)->getLoop() == L \|\|
3934	!isa<SCEVAddRecExpr>(Val: Remainder))) {
3935	Ops.push_back(Elt: C ? SE.getMulExpr(LHS: C, RHS: Remainder) : Remainder);
3936	Remainder = nullptr;
3937	}
3938	if (Remainder != Start) {
3939	if (!Remainder)
3940	Remainder = SE.getConstant(Ty: S->getType(), V: `0`);
3941	return SE.getAddRecExpr(Start: Remainder, Step,
3942	L: cast<SCEVAddRecExpr>(Val: S)->getLoop(),
3943	// FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
3944	Flags: SCEV::FlagAnyWrap);
3945	}
3946	} else if (match(S, P: m_scev_Mul(Op0: m_SCEVConstant(V&: Op0), Op1: m_SCEV(V&: Op1)))) {
3947	// Break (C (a + b + c)) into Ca + Cb + Cc.
3948	C = C ? cast<SCEVConstant>(Val: SE.getMulExpr(LHS: C, RHS: Op0)) : Op0;
3949	const SCEV *Remainder = CollectSubexprs(S: Op1, C, Ops, L, SE, Depth: Depth + `1`);
3950	if (Remainder)
3951	Ops.push_back(Elt: SE.getMulExpr(LHS: C, RHS: Remainder));
3952	return nullptr;
3953	}
3954	return S;
3955	}
3956
3957	/// Return true if the SCEV represents a value that may end up as a
3958	/// post-increment operation.
3959	static bool mayUsePostIncMode(const TargetTransformInfo &TTI,
3960	LSRUse &LU, const SCEV S, const* Loop *L,
3961	ScalarEvolution &SE) {
3962	if (LU.Kind != LSRUse::Address \|\|
3963	!LU.AccessTy.getType()->isIntOrIntVectorTy())
3964	return false;
3965	const SCEV *Start;
3966	if (!match(S, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEVConstant())))
3967	return false;
3968	// Check if a post-indexed load/store can be used.
3969	if (TTI.isIndexedLoadLegal(Mode: TTI.MIM_PostInc, Ty: S->getType()) \|\|
3970	TTI.isIndexedStoreLegal(Mode: TTI.MIM_PostInc, Ty: S->getType())) {
3971	if (!isa<SCEVConstant>(Val: Start) && SE.isLoopInvariant(S: Start, L))
3972	return true;
3973	}
3974	return false;
3975	}
3976
3977	/// Helper function for LSRInstance::GenerateReassociations.
3978	void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
3979	const Formula &Base,
3980	unsigned Depth, size_t Idx,
3981	bool IsScaledReg) {
3982	const SCEV *BaseReg = IsScaledReg ? Base.ScaledReg : Base.BaseRegs [Idx];
3983	// Don't generate reassociations for the base register of a value that
3984	// may generate a post-increment operator. The reason is that the
3985	// reassociations cause extra base+register formula to be created,
3986	// and possibly chosen, but the post-increment is more efficient.
3987	if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, S: BaseReg, L, SE))
3988	return;
3989	SmallVector<const SCEV *, `8`> AddOps;
3990	const SCEV Remainder = CollectSubexprs(S: BaseReg, C: nullptr*, Ops&: AddOps, L, SE);
3991	if (Remainder)
3992	AddOps.push_back(Elt: Remainder);
3993
3994	if (AddOps.size() == `1`)
3995	return;
3996
3997	for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(),
3998	JE = AddOps.end();
3999	J != JE; ++J) {
4000	// Loop-variant "unknown" values are uninteresting; we won't be able to
4001	// do anything meaningful with them.
4002	if (isa<SCEVUnknown>(Val: J) && !SE.isLoopInvariant(S: J, L))
4003	continue;
4004
4005	// Don't pull a constant into a register if the constant could be folded
4006	// into an immediate field.
4007	if (isAlwaysFoldable(TTI, SE, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
4008	AccessTy: LU.AccessTy, S: *J, HasBaseReg: Base.getNumRegs() > `1`))
4009	continue;
4010
4011	// Collect all operands except J.*
4012	SmallVector<SCEVUse, `8`> InnerAddOps(std::as_const(t&: AddOps).begin(), J);
4013	InnerAddOps.append(in_start: std::next(x: J), in_end: std::as_const(t&: AddOps).end());
4014
4015	// Don't leave just a constant behind in a register if the constant could
4016	// be folded into an immediate field.
4017	if (InnerAddOps.size() == `1` &&
4018	isAlwaysFoldable(TTI, SE, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
4019	AccessTy: LU.AccessTy, S: InnerAddOps [`0`], HasBaseReg: Base.getNumRegs() > `1`))
4020	continue;
4021
4022	const SCEV *InnerSum = SE.getAddExpr(Ops&: InnerAddOps);
4023	if (InnerSum->isZero())
4024	continue;
4025	Formula F = Base;
4026
4027	if (F.UnfoldedOffset.isNonZero() && F.UnfoldedOffset.isScalable())
4028	continue;
4029
4030	// Add the remaining pieces of the add back into the new formula.
4031	const SCEVConstant *InnerSumSC = dyn_cast<SCEVConstant>(Val: InnerSum);
4032	if (InnerSumSC && SE.getTypeSizeInBits(Ty: InnerSumSC->getType()) <= `64` &&
4033	TTI.isLegalAddImmediate(Imm: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4034	InnerSumSC->getValue()->getZExtValue())) {
4035	F.UnfoldedOffset =
4036	Immediate::getFixed(MinVal: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4037	InnerSumSC->getValue()->getZExtValue());
4038	if (IsScaledReg) {
4039	F.ScaledReg = nullptr;
4040	F.Scale = `0`;
4041	} else
4042	F.BaseRegs.erase(CI: F.BaseRegs.begin() + Idx);
4043	} else if (IsScaledReg)
4044	F.ScaledReg = InnerSum;
4045	else
4046	F.BaseRegs [Idx] = InnerSum;
4047
4048	// Add J as its own register, or an unfolded immediate.
4049	const SCEVConstant SC = dyn_cast<SCEVConstant>(Val: J);
4050	if (SC && SE.getTypeSizeInBits(Ty: SC->getType()) <= `64` &&
4051	TTI.isLegalAddImmediate(Imm: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4052	SC->getValue()->getZExtValue()))
4053	F.UnfoldedOffset =
4054	Immediate::getFixed(MinVal: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4055	SC->getValue()->getZExtValue());
4056	else
4057	F.BaseRegs.push_back(Elt: *J);
4058	// We may have changed the number of register in base regs, adjust the
4059	// formula accordingly.
4060	F.canonicalize(L: *L);
4061
4062	if (InsertFormula(LU, LUIdx, F))
4063	// If that formula hadn't been seen before, recurse to find more like
4064	// it.
4065	// Add check on Log16(AddOps.size()) - same as Log2_32(AddOps.size()) >> 2)
4066	// Because just Depth is not enough to bound compile time.
4067	// This means that every time AddOps.size() is greater 16^x we will add
4068	// x to Depth.
4069	GenerateReassociations(LU, LUIdx, Base: LU.Formulae.back(),
4070	Depth: Depth + `1` + (Log2_32(Value: AddOps.size()) >> `2`));
4071	}
4072	}
4073
4074	/// Split out subexpressions from adds and the bases of addrecs.
4075	void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
4076	Formula Base, unsigned Depth) {
4077	assert(Base.isCanonical(*L) && "Input must be in the canonical form");
4078	// Arbitrarily cap recursion to protect compile time.
4079	if (Depth >= `3`)
4080	return;
4081
4082	for (size_t i = `0`, e = Base.BaseRegs.size(); i != e; ++i)
4083	GenerateReassociationsImpl(LU, LUIdx, Base, Depth, Idx: i);
4084
4085	if (Base.Scale == `1`)
4086	GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
4087	/ Idx / -`1`, / IsScaledReg / true);
4088	}
4089
4090	/// Generate a formula consisting of all of the loop-dominating registers added
4091	/// into a single register.
4092	void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
4093	Formula Base) {
4094	// This method is only interesting on a plurality of registers.
4095	if (Base.BaseRegs.size() + (Base.Scale == `1`) +
4096	(Base.UnfoldedOffset.isNonZero()) <=
4097	`1`)
4098	return;
4099
4100	// Flatten the representation, i.e., reg1 + 1reg2 => reg1 + reg2, before*
4101	// processing the formula.
4102	Base.unscale();
4103	SmallVector<SCEVUse, `4`> Ops;
4104	Formula NewBase = Base;
4105	NewBase.BaseRegs.clear();
4106	Type CombinedIntegerType = nullptr*;
4107	for (const SCEV *BaseReg : Base.BaseRegs) {
4108	if (SE.properlyDominates(S: BaseReg, BB: L->getHeader()) &&
4109	!SE.hasComputableLoopEvolution(S: BaseReg, L)) {
4110	if (!CombinedIntegerType)
4111	CombinedIntegerType = SE.getEffectiveSCEVType(Ty: BaseReg->getType());
4112	Ops.push_back(Elt: BaseReg);
4113	}
4114	else
4115	NewBase.BaseRegs.push_back(Elt: BaseReg);
4116	}
4117
4118	// If no register is relevant, we're done.
4119	if (Ops.size() == `0`)
4120	return;
4121
4122	// Utility function for generating the required variants of the combined
4123	// registers.
4124	auto GenerateFormula = [&](const SCEV *Sum) {
4125	Formula F = NewBase;
4126
4127	// TODO: If Sum is zero, it probably means ScalarEvolution missed an
4128	// opportunity to fold something. For now, just ignore such cases
4129	// rather than proceed with zero in a register.
4130	if (Sum->isZero())
4131	return;
4132
4133	F.BaseRegs.push_back(Elt: Sum);
4134	F.canonicalize(L: *L);
4135	(void)InsertFormula(LU, LUIdx, F);
4136	};
4137
4138	// If we collected at least two registers, generate a formula combining them.
4139	if (Ops.size() > `1`) {
4140	SmallVector<SCEVUse, `4`> OpsCopy(Ops); // Don't let SE modify Ops.
4141	GenerateFormula (SE.getAddExpr(Ops&: OpsCopy));
4142	}
4143
4144	// If we have an unfolded offset, generate a formula combining it with the
4145	// registers collected.
4146	if (NewBase.UnfoldedOffset.isNonZero() && NewBase.UnfoldedOffset.isFixed()) {
4147	assert(CombinedIntegerType && "Missing a type for the unfolded offset");
4148	Ops.push_back(Elt: SE.getConstant(Ty: CombinedIntegerType,
4149	V: NewBase.UnfoldedOffset.getFixedValue(), isSigned: true));
4150	NewBase.UnfoldedOffset = Immediate::getFixed(MinVal: `0`);
4151	GenerateFormula (SE.getAddExpr(Ops));
4152	}
4153	}
4154
4155	/// Helper function for LSRInstance::GenerateSymbolicOffsets.
4156	void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
4157	const Formula &Base, size_t Idx,
4158	bool IsScaledReg) {
4159	SCEVUse G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs [Idx];
4160	GlobalValue *GV = ExtractSymbol(S&: G, SE);
4161	if (G ->isZero() \|\| !GV)
4162	return;
4163	Formula F = Base;
4164	F.BaseGV = GV;
4165	if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy, F))
4166	return;
4167	if (IsScaledReg)
4168	F.ScaledReg = G;
4169	else
4170	F.BaseRegs [Idx] = G;
4171	(void)InsertFormula(LU, LUIdx, F);
4172	}
4173
4174	/// Generate reuse formulae using symbolic offsets.
4175	void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
4176	Formula Base) {
4177	// We can't add a symbolic offset if the address already contains one.
4178	if (Base.BaseGV) return;
4179
4180	for (size_t i = `0`, e = Base.BaseRegs.size(); i != e; ++i)
4181	GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, Idx: i);
4182	if (Base.Scale == `1`)
4183	GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, / Idx / -`1`,
4184	/ IsScaledReg / true);
4185	}
4186
4187	/// Helper function for LSRInstance::GenerateConstantOffsets.
4188	void LSRInstance::GenerateConstantOffsetsImpl(
4189	LSRUse &LU, unsigned LUIdx, const Formula &Base,
4190	const SmallVectorImpl<Immediate> &Worklist, size_t Idx, bool IsScaledReg) {
4191
4192	auto GenerateOffset = [&](const SCEV *G, Immediate Offset) {
4193	Formula F = Base;
4194	if (!Base.BaseOffset.isCompatibleImmediate(Imm: Offset))
4195	return;
4196	F.BaseOffset = Base.BaseOffset.subUnsigned(RHS: Offset);
4197
4198	if (isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy, F)) {
4199	// Add the offset to the base register.
4200	const SCEV *NewOffset = Offset.getSCEV(SE, Ty: G->getType());
4201	const SCEV *NewG = SE.getAddExpr(LHS: NewOffset, RHS: G);
4202	// If it cancelled out, drop the base register, otherwise update it.
4203	if (NewG->isZero()) {
4204	if (IsScaledReg) {
4205	F.Scale = `0`;
4206	F.ScaledReg = nullptr;
4207	} else
4208	F.deleteBaseReg(S&: F.BaseRegs [Idx]);
4209	F.canonicalize(L: *L);
4210	} else if (IsScaledReg)
4211	F.ScaledReg = NewG;
4212	else
4213	F.BaseRegs [Idx] = NewG;
4214
4215	(void)InsertFormula(LU, LUIdx, F);
4216	}
4217	};
4218
4219	SCEVUse G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs [Idx];
4220
4221	// With constant offsets and constant steps, we can generate pre-inc
4222	// accesses by having the offset equal the step. So, for access #0 with a
4223	// step of 8, we generate a G - 8 base which would require the first access
4224	// to be ((G - 8) + 8),+,8. The pre-indexed access then updates the pointer
4225	// for itself and hopefully becomes the base for other accesses. This means
4226	// means that a single pre-indexed access can be generated to become the new
4227	// base pointer for each iteration of the loop, resulting in no extra add/sub
4228	// instructions for pointer updating.
4229	if ((AMK & TTI::AMK_PreIndexed) && LU.Kind == LSRUse::Address) {
4230	const APInt *StepInt;
4231	if (match(U: G, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_scev_APInt(C&: StepInt)))) {
4232	int64_t Step = StepInt->isNegative() ? StepInt->getSExtValue()
4233	: StepInt->getZExtValue();
4234
4235	for (Immediate Offset : Worklist) {
4236	if (Offset.isFixed()) {
4237	Offset = Immediate::getFixed(MinVal: Offset.getFixedValue() - Step);
4238	GenerateOffset (G, Offset);
4239	}
4240	}
4241	}
4242	}
4243	for (Immediate Offset : Worklist)
4244	GenerateOffset (G, Offset);
4245
4246	// TODO: It likely makes sense to extract the immediate corresponding to the
4247	// access type (i.e., set PreferScalable to AccessTy.MemTy &&
4248	// AccessTy.MemTy->isScalableTy()).
4249	Immediate Imm = ExtractImmediate(S&: G, SE, /PreferScalable=/false);
4250	if (G ->isZero() \|\| Imm.isZero() \|\|
4251	!Base.BaseOffset.isCompatibleImmediate(Imm))
4252	return;
4253	Formula F = Base;
4254	F.BaseOffset = F.BaseOffset.addUnsigned(RHS: Imm);
4255	if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy, F))
4256	return;
4257	if (IsScaledReg) {
4258	F.ScaledReg = G;
4259	} else {
4260	F.BaseRegs [Idx] = G;
4261	// We may generate non canonical Formula if G is a recurrent expr reg
4262	// related with current loop while F.ScaledReg is not.
4263	F.canonicalize(L: *L);
4264	}
4265	(void)InsertFormula(LU, LUIdx, F);
4266	}
4267
4268	/// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets.
4269	void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
4270	Formula Base) {
4271	// TODO: For now, just add the min and max offset, because it usually isn't
4272	// worthwhile looking at everything inbetween.
4273	SmallVector<Immediate, `2`> Worklist;
4274	Worklist.push_back(Elt: LU.MinOffset);
4275	if (LU.MaxOffset != LU.MinOffset)
4276	Worklist.push_back(Elt: LU.MaxOffset);
4277
4278	for (size_t i = `0`, e = Base.BaseRegs.size(); i != e; ++i)
4279	GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, Idx: i);
4280	if (Base.Scale == `1`)
4281	GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, / Idx / -`1`,
4282	/ IsScaledReg / true);
4283	}
4284
4285	/// For ICmpZero, check to see if we can scale up the comparison. For example, x
4286	/// == y -> xc == yc.
4287	void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
4288	Formula Base) {
4289	if (LU.Kind != LSRUse::ICmpZero) return;
4290
4291	// Determine the integer type for the base formula.
4292	Type *IntTy = Base.getType();
4293	if (!IntTy) return;
4294	if (SE.getTypeSizeInBits(Ty: IntTy) > `64`) return;
4295
4296	// Don't do this if there is more than one offset.
4297	if (LU.MinOffset != LU.MaxOffset) return;
4298
4299	// Check if transformation is valid. It is illegal to multiply pointer.
4300	if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy())
4301	return;
4302	for (const SCEV *BaseReg : Base.BaseRegs)
4303	if (BaseReg->getType()->isPointerTy())
4304	return;
4305	assert(!Base.BaseGV && "ICmpZero use is not legal!");
4306
4307	// Check each interesting stride.
4308	for (int64_t Factor : Factors) {
4309	// Check that Factor can be represented by IntTy
4310	if (!ConstantInt::isValueValidForType(Ty: IntTy, V: Factor))
4311	continue;
4312	// Check that the multiplication doesn't overflow.
4313	if (Base.BaseOffset.isMin() && Factor == -`1`)
4314	continue;
4315	// Not supporting scalable immediates.
4316	if (Base.BaseOffset.isNonZero() && Base.BaseOffset.isScalable())
4317	continue;
4318	Immediate NewBaseOffset = Base.BaseOffset.mulUnsigned(RHS: Factor);
4319	assert(Factor != `0` && "Zero factor not expected!");
4320	if (NewBaseOffset.getFixedValue() / Factor !=
4321	Base.BaseOffset.getFixedValue())
4322	continue;
4323	// If the offset will be truncated at this use, check that it is in bounds.
4324	if (!IntTy->isPointerTy() &&
4325	!ConstantInt::isValueValidForType(Ty: IntTy, V: NewBaseOffset.getFixedValue()))
4326	continue;
4327
4328	// Check that multiplying with the use offset doesn't overflow.
4329	Immediate Offset = LU.MinOffset;
4330	if (Offset.isMin() && Factor == -`1`)
4331	continue;
4332	Offset = Offset.mulUnsigned(RHS: Factor);
4333	if (Offset.getFixedValue() / Factor != LU.MinOffset.getFixedValue())
4334	continue;
4335	// If the offset will be truncated at this use, check that it is in bounds.
4336	if (!IntTy->isPointerTy() &&
4337	!ConstantInt::isValueValidForType(Ty: IntTy, V: Offset.getFixedValue()))
4338	continue;
4339
4340	Formula F = Base;
4341	F.BaseOffset = NewBaseOffset;
4342
4343	// Check that this scale is legal.
4344	if (!isLegalUse(TTI, MinOffset: Offset, MaxOffset: Offset, Kind: LU.Kind, AccessTy: LU.AccessTy, F))
4345	continue;
4346
4347	// Compensate for the use having MinOffset built into it.
4348	F.BaseOffset = F.BaseOffset.addUnsigned(RHS: Offset).subUnsigned(RHS: LU.MinOffset);
4349
4350	const SCEV *FactorS = SE.getConstant(Ty: IntTy, V: Factor);
4351
4352	// Check that multiplying with each base register doesn't overflow.
4353	for (size_t i = `0`, e = F.BaseRegs.size(); i != e; ++i) {
4354	F.BaseRegs [i] = SE.getMulExpr(LHS: F.BaseRegs [i], RHS: FactorS);
4355	if (getExactSDiv(LHS: F.BaseRegs [i], RHS: FactorS, SE) != Base.BaseRegs [i])
4356	goto next;
4357	}
4358
4359	// Check that multiplying with the scaled register doesn't overflow.
4360	if (F.ScaledReg) {
4361	F.ScaledReg = SE.getMulExpr(LHS: F.ScaledReg, RHS: FactorS);
4362	if (getExactSDiv(LHS: F.ScaledReg, RHS: FactorS, SE) != Base.ScaledReg)
4363	continue;
4364	}
4365
4366	// Check that multiplying with the unfolded offset doesn't overflow.
4367	if (F.UnfoldedOffset.isNonZero()) {
4368	if (F.UnfoldedOffset.isMin() && Factor == -`1`)
4369	continue;
4370	F.UnfoldedOffset = F.UnfoldedOffset.mulUnsigned(RHS: Factor);
4371	if (F.UnfoldedOffset.getFixedValue() / Factor !=
4372	Base.UnfoldedOffset.getFixedValue())
4373	continue;
4374	// If the offset will be truncated, check that it is in bounds.
4375	if (!IntTy->isPointerTy() && !ConstantInt::isValueValidForType(
4376	Ty: IntTy, V: F.UnfoldedOffset.getFixedValue()))
4377	continue;
4378	}
4379
4380	// If we make it here and it's legal, add it.
4381	(void)InsertFormula(LU, LUIdx, F);
4382	next:;
4383	}
4384	}
4385
4386	/// Generate stride factor reuse formulae by making use of scaled-offset address
4387	/// modes, for example.
4388	void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
4389	// Determine the integer type for the base formula.
4390	Type *IntTy = Base.getType();
4391	if (!IntTy) return;
4392
4393	// If this Formula already has a scaled register, we can't add another one.
4394	// Try to unscale the formula to generate a better scale.
4395	if (Base.Scale != `0` && !Base.unscale())
4396	return;
4397
4398	assert(Base.Scale == `0` && "unscale did not did its job!");
4399
4400	// Check each interesting stride.
4401	for (int64_t Factor : Factors) {
4402	Base.Scale = Factor;
4403	Base.HasBaseReg = Base.BaseRegs.size() > `1`;
4404	// Check whether this scale is going to be legal.
4405	if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy,
4406	F: Base)) {
4407	// As a special-case, handle special out-of-loop Basic users specially.
4408	// TODO: Reconsider this special case.
4409	if (LU.Kind == LSRUse::Basic &&
4410	isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LSRUse::Special,
4411	AccessTy: LU.AccessTy, F: Base) &&
4412	LU.AllFixupsOutsideLoop)
4413	LU.Kind = LSRUse::Special;
4414	else
4415	continue;
4416	}
4417	// For an ICmpZero, negating a solitary base register won't lead to
4418	// new solutions.
4419	if (LU.Kind == LSRUse::ICmpZero && !Base.HasBaseReg &&
4420	Base.BaseOffset.isZero() && !Base.BaseGV)
4421	continue;
4422	// For each addrec base reg, if its loop is current loop, apply the scale.
4423	for (size_t i = `0`, e = Base.BaseRegs.size(); i != e; ++i) {
4424	const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Base.BaseRegs [i]);
4425	if (AR && (AR->getLoop() == L \|\| LU.AllFixupsOutsideLoop)) {
4426	const SCEV *FactorS = SE.getConstant(Ty: IntTy, V: Factor);
4427	if (FactorS->isZero())
4428	continue;
4429	// Divide out the factor, ignoring high bits, since we'll be
4430	// scaling the value back up in the end.
4431	if (const SCEV Quotient = getExactSDiv(LHS: AR, RHS: FactorS, SE, IgnoreSignificantBits: true*))
4432	if (!Quotient->isZero()) {
4433	// TODO: This could be optimized to avoid all the copying.
4434	Formula F = Base;
4435	F.ScaledReg = Quotient;
4436	F.deleteBaseReg(S&: F.BaseRegs [i]);
4437	// The canonical representation of 1reg is reg, which is already in*
4438	// Base. In that case, do not try to insert the formula, it will be
4439	// rejected anyway.
4440	if (F.Scale == `1` && (F.BaseRegs.empty() \|\|
4441	(AR->getLoop() != L && LU.AllFixupsOutsideLoop)))
4442	continue;
4443	// If AllFixupsOutsideLoop is true and F.Scale is 1, we may generate
4444	// non canonical Formula with ScaledReg's loop not being L.
4445	if (F.Scale == `1` && LU.AllFixupsOutsideLoop)
4446	F.canonicalize(L: *L);
4447	(void)InsertFormula(LU, LUIdx, F);
4448	}
4449	}
4450	}
4451	}
4452	}
4453
4454	/// Extend/Truncate \p Expr to \p ToTy considering post-inc uses in \p Loops.
4455	/// For all PostIncLoopSets in \p Loops, first de-normalize \p Expr, then
4456	/// perform the extension/truncate and normalize again, as the normalized form
4457	/// can result in folds that are not valid in the post-inc use contexts. The
4458	/// expressions for all PostIncLoopSets must match, otherwise return nullptr.
4459	static const SCEV *
4460	getAnyExtendConsideringPostIncUses(ArrayRef<PostIncLoopSet> Loops,
4461	const SCEV Expr, Type ToTy,
4462	ScalarEvolution &SE) {
4463	const SCEV Result = nullptr*;
4464	for (auto &L : Loops) {
4465	auto *DenormExpr = denormalizeForPostIncUse(S: Expr, Loops: L, SE);
4466	const SCEV *NewDenormExpr = SE.getAnyExtendExpr(Op: DenormExpr, Ty: ToTy);
4467	const SCEV *New = normalizeForPostIncUse(S: NewDenormExpr, Loops: L, SE);
4468	if (!New \|\| (Result && New != Result))
4469	return nullptr;
4470	Result = New;
4471	}
4472
4473	assert(Result && "failed to create expression");
4474	return Result;
4475	}
4476
4477	/// Generate reuse formulae from different IV types.
4478	void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
4479	// Don't bother truncating symbolic values.
4480	if (Base.BaseGV) return;
4481
4482	// Determine the integer type for the base formula.
4483	Type *DstTy = Base.getType();
4484	if (!DstTy) return;
4485	if (DstTy->isPointerTy())
4486	return;
4487
4488	// It is invalid to extend a pointer type so exit early if ScaledReg or
4489	// any of the BaseRegs are pointers.
4490	if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy())
4491	return;
4492	if (any_of(Range&: Base.BaseRegs,
4493	P: [](const SCEV S) { return* S->getType()->isPointerTy(); }))
4494	return;
4495
4496	SmallVector<PostIncLoopSet> Loops;
4497	for (auto &LF : LU.Fixups)
4498	Loops.push_back(Elt: LF.PostIncLoops);
4499
4500	for (Type *SrcTy : Types) {
4501	if (SrcTy != DstTy && TTI.isTruncateFree(Ty1: SrcTy, Ty2: DstTy)) {
4502	Formula F = Base;
4503
4504	// Sometimes SCEV is able to prove zero during ext transform. It may
4505	// happen if SCEV did not do all possible transforms while creating the
4506	// initial node (maybe due to depth limitations), but it can do them while
4507	// taking ext.
4508	if (F.ScaledReg) {
4509	const SCEV *NewScaledReg =
4510	getAnyExtendConsideringPostIncUses(Loops, Expr: F.ScaledReg, ToTy: SrcTy, SE);
4511	if (!NewScaledReg \|\| NewScaledReg->isZero())
4512	continue;
4513	F.ScaledReg = NewScaledReg;
4514	}
4515	bool HasZeroBaseReg = false;
4516	for (const SCEV *&BaseReg : F.BaseRegs) {
4517	const SCEV *NewBaseReg =
4518	getAnyExtendConsideringPostIncUses(Loops, Expr: BaseReg, ToTy: SrcTy, SE);
4519	if (!NewBaseReg \|\| NewBaseReg->isZero()) {
4520	HasZeroBaseReg = true;
4521	break;
4522	}
4523	BaseReg = NewBaseReg;
4524	}
4525	if (HasZeroBaseReg)
4526	continue;
4527
4528	// TODO: This assumes we've done basic processing on all uses and
4529	// have an idea what the register usage is.
4530	if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses))
4531	continue;
4532
4533	F.canonicalize(L: *L);
4534	(void)InsertFormula(LU, LUIdx, F);
4535	}
4536	}
4537	}
4538
4539	namespace {
4540
4541	/// Helper class for GenerateCrossUseConstantOffsets. It's used to defer
4542	/// modifications so that the search phase doesn't have to worry about the data
4543	/// structures moving underneath it.
4544	struct WorkItem {
4545	size_t LUIdx;
4546	Immediate Imm;
4547	const SCEV *OrigReg;
4548
4549	WorkItem(size_t LI, Immediate I, const SCEV *R)
4550	: LUIdx(LI), Imm (I), OrigReg(R) {}
4551
4552	void print(raw_ostream &OS) const;
4553	void dump() const;
4554	};
4555
4556	} // end anonymous namespace
4557
4558	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
4559	void WorkItem::print(raw_ostream &OS) const {
4560	OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx
4561	<< " , add offset " << Imm;
4562	}
4563
4564	LLVM_DUMP_METHOD void WorkItem::dump() const {
4565	print(errs()); errs() << `'\n'`;
4566	}
4567	#endif
4568
4569	/// Look for registers which are a constant distance apart and try to form reuse
4570	/// opportunities between them.
4571	void LSRInstance::GenerateCrossUseConstantOffsets() {
4572	// Group the registers by their value without any added constant offset.
4573	using ImmMapTy = std::map<Immediate, const SCEV *, KeyOrderTargetImmediate>;
4574
4575	DenseMap<const SCEV *, ImmMapTy> Map;
4576	DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap;
4577	SmallVector<const SCEV *, `8`> Sequence;
4578	for (const SCEV *Use : RegUses) {
4579	SCEVUse Reg = Use; // Make a copy for ExtractImmediate to modify.
4580	// TODO: Extract both scalable and fixed immediates (if present)?
4581	Immediate Imm = ExtractImmediate(S&: Reg, SE);
4582	auto Pair = Map.try_emplace(Key: Reg);
4583	if (Pair.second)
4584	Sequence.push_back(Elt: Reg);
4585	Pair.first ->second.insert(x: std::make_pair(x&: Imm, y&: Use));
4586	UsedByIndicesMap [Reg] \|= RegUses.getUsedByIndices(Reg: Use);
4587	}
4588
4589	// Now examine each set of registers with the same base value. Build up
4590	// a list of work to do and do the work in a separate step so that we're
4591	// not adding formulae and register counts while we're searching.
4592	SmallVector<WorkItem, `32`> WorkItems;
4593	SmallSet<std::pair<size_t, Immediate>, `32`, KeyOrderSizeTAndImmediate>
4594	UniqueItems;
4595	for (const SCEV *Reg : Sequence) {
4596	const ImmMapTy &Imms = Map.find(Val: Reg)->second;
4597
4598	// It's not worthwhile looking for reuse if there's only one offset.
4599	if (Imms.size() == `1`)
4600	continue;
4601
4602	LLVM_DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << `':'`;
4603	for (const auto &Entry
4604	: Imms) dbgs()
4605	<< `' '` << Entry.first;
4606	dbgs() << `'\n'`);
4607
4608	// Examine each offset.
4609	for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end();
4610	J != JE; ++J) {
4611	const SCEV *OrigReg = J ->second;
4612
4613	Immediate JImm = J ->first;
4614	const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg: OrigReg);
4615
4616	if (!isa<SCEVConstant>(Val: OrigReg) &&
4617	UsedByIndicesMap [Reg].count() == `1`) {
4618	LLVM_DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg
4619	<< `'\n'`);
4620	continue;
4621	}
4622
4623	// Conservatively examine offsets between this orig reg a few selected
4624	// other orig regs.
4625	Immediate First = Imms.begin()->first;
4626	Immediate Last = std::prev(x: Imms.end())->first;
4627	if (!First.isCompatibleImmediate(Imm: Last)) {
4628	LLVM_DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg
4629	<< "\n");
4630	continue;
4631	}
4632	// Only scalable if both terms are scalable, or if one is scalable and
4633	// the other is 0.
4634	bool Scalable = First.isScalable() \|\| Last.isScalable();
4635	int64_t FI = First.getKnownMinValue();
4636	int64_t LI = Last.getKnownMinValue();
4637	// Compute (First + Last) / 2 without overflow using the fact that
4638	// First + Last = 2 (First + Last) + (First ^ Last).*
4639	int64_t Avg = (FI & LI) + ((FI ^ LI) >> `1`);
4640	// If the result is negative and FI is odd and LI even (or vice versa),
4641	// we rounded towards -inf. Add 1 in that case, to round towards 0.
4642	Avg = Avg + ((FI ^ LI) & ((uint64_t)Avg >> `63`));
4643	ImmMapTy::const_iterator OtherImms[] = {
4644	Imms.begin(), std::prev(x: Imms.end()),
4645	Imms.lower_bound(x: Immediate::get(MinVal: Avg, Scalable))};
4646	for (const auto &M : OtherImms) {
4647	if (M == J \|\| M == JE) continue;
4648	if (!JImm.isCompatibleImmediate(Imm: M ->first))
4649	continue;
4650
4651	// Compute the difference between the two.
4652	Immediate Imm = JImm.subUnsigned(RHS: M ->first);
4653	for (unsigned LUIdx : UsedByIndices.set_bits())
4654	// Make a memo of this use, offset, and register tuple.
4655	if (UniqueItems.insert(V: std::make_pair(x&: LUIdx, y&: Imm)).second)
4656	WorkItems.push_back(Elt: WorkItem (LUIdx, Imm, OrigReg));
4657	}
4658	}
4659	}
4660
4661	Map.clear();
4662	Sequence.clear();
4663	UsedByIndicesMap.clear();
4664	UniqueItems.clear();
4665
4666	// Now iterate through the worklist and add new formulae.
4667	for (const WorkItem &WI : WorkItems) {
4668	size_t LUIdx = WI.LUIdx;
4669	LSRUse &LU = Uses [LUIdx];
4670	Immediate Imm = WI.Imm;
4671	const SCEV *OrigReg = WI.OrigReg;
4672
4673	Type *IntTy = SE.getEffectiveSCEVType(Ty: OrigReg->getType());
4674	const SCEV *NegImmS = Imm.getNegativeSCEV(SE, Ty: IntTy);
4675	unsigned BitWidth = SE.getTypeSizeInBits(Ty: IntTy);
4676
4677	// TODO: Use a more targeted data structure.
4678	for (size_t L = `0`, LE = LU.Formulae.size(); L != LE; ++L) {
4679	Formula F = LU.Formulae [L];
4680	// FIXME: The code for the scaled and unscaled registers looks
4681	// very similar but slightly different. Investigate if they
4682	// could be merged. That way, we would not have to unscale the
4683	// Formula.
4684	F.unscale();
4685	// Use the immediate in the scaled register.
4686	if (F.ScaledReg == OrigReg) {
4687	if (!F.BaseOffset.isCompatibleImmediate(Imm))
4688	continue;
4689	Immediate Offset = F.BaseOffset.addUnsigned(RHS: Imm.mulUnsigned(RHS: F.Scale));
4690	// Don't create 50 + reg(-50).
4691	const SCEV *S = Offset.getNegativeSCEV(SE, Ty: IntTy);
4692	if (F.referencesReg(S))
4693	continue;
4694	Formula NewF = F;
4695	NewF.BaseOffset = Offset;
4696	if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy,
4697	F: NewF))
4698	continue;
4699	NewF.ScaledReg = SE.getAddExpr(LHS: NegImmS, RHS: NewF.ScaledReg);
4700
4701	// If the new scale is a constant in a register, and adding the constant
4702	// value to the immediate would produce a value closer to zero than the
4703	// immediate itself, then the formula isn't worthwhile.
4704	if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: NewF.ScaledReg)) {
4705	// FIXME: Do we need to do something for scalable immediates here?
4706	// A scalable SCEV won't be constant, but we might still have
4707	// something in the offset? Bail out for now to be safe.
4708	if (NewF.BaseOffset.isNonZero() && NewF.BaseOffset.isScalable())
4709	continue;
4710	if (C->getValue()->isNegative() !=
4711	(NewF.BaseOffset.isLessThanZero()) &&
4712	(C->getAPInt().abs() * APInt (BitWidth, F.Scale))
4713	.ule(RHS: std::abs(i: NewF.BaseOffset.getFixedValue())))
4714	continue;
4715	}
4716
4717	// OK, looks good.
4718	NewF.canonicalize(L: *this->L);
4719	(void)InsertFormula(LU, LUIdx, F: NewF);
4720	} else {
4721	// Use the immediate in a base register.
4722	for (size_t N = `0`, NE = F.BaseRegs.size(); N != NE; ++N) {
4723	const SCEV *BaseReg = F.BaseRegs [N];
4724	if (BaseReg != OrigReg)
4725	continue;
4726	Formula NewF = F;
4727	if (!NewF.BaseOffset.isCompatibleImmediate(Imm) \|\|
4728	!NewF.UnfoldedOffset.isCompatibleImmediate(Imm) \|\|
4729	!NewF.BaseOffset.isCompatibleImmediate(Imm: NewF.UnfoldedOffset))
4730	continue;
4731	NewF.BaseOffset = NewF.BaseOffset.addUnsigned(RHS: Imm);
4732	if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset,
4733	Kind: LU.Kind, AccessTy: LU.AccessTy, F: NewF)) {
4734	if (AMK == TTI::AMK_PostIndexed &&
4735	mayUsePostIncMode(TTI, LU, S: OrigReg, L: this->L, SE))
4736	continue;
4737	Immediate NewUnfoldedOffset = NewF.UnfoldedOffset.addUnsigned(RHS: Imm);
4738	if (!isLegalAddImmediate(TTI, Offset: NewUnfoldedOffset))
4739	continue;
4740	NewF = F;
4741	NewF.UnfoldedOffset = NewUnfoldedOffset;
4742	}
4743	NewF.BaseRegs [N] = SE.getAddExpr(LHS: NegImmS, RHS: BaseReg);
4744
4745	// If the new formula has a constant in a register, and adding the
4746	// constant value to the immediate would produce a value closer to
4747	// zero than the immediate itself, then the formula isn't worthwhile.
4748	for (const SCEV *NewReg : NewF.BaseRegs)
4749	if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: NewReg)) {
4750	if (NewF.BaseOffset.isNonZero() && NewF.BaseOffset.isScalable())
4751	goto skip_formula;
4752	if ((C->getAPInt() + NewF.BaseOffset.getFixedValue())
4753	.abs()
4754	.slt(RHS: std::abs(i: NewF.BaseOffset.getFixedValue())) &&
4755	(C->getAPInt() + NewF.BaseOffset.getFixedValue())
4756	.countr_zero() >=
4757	(unsigned)llvm::countr_zero<uint64_t>(
4758	Val: NewF.BaseOffset.getFixedValue()))
4759	goto skip_formula;
4760	}
4761
4762	// Ok, looks good.
4763	NewF.canonicalize(L: *this->L);
4764	(void)InsertFormula(LU, LUIdx, F: NewF);
4765	break;
4766	skip_formula:;
4767	}
4768	}
4769	}
4770	}
4771	}
4772
4773	/// Generate formulae for each use.
4774	void
4775	LSRInstance::GenerateAllReuseFormulae() {
4776	// This is split into multiple loops so that hasRegsUsedByUsesOtherThan
4777	// queries are more precise.
4778	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4779	LSRUse &LU = Uses [LUIdx];
4780	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4781	GenerateReassociations(LU, LUIdx, Base: LU.Formulae [i]);
4782	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4783	GenerateCombinations(LU, LUIdx, Base: LU.Formulae [i]);
4784	}
4785	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4786	LSRUse &LU = Uses [LUIdx];
4787	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4788	GenerateSymbolicOffsets(LU, LUIdx, Base: LU.Formulae [i]);
4789	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4790	GenerateConstantOffsets(LU, LUIdx, Base: LU.Formulae [i]);
4791	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4792	GenerateICmpZeroScales(LU, LUIdx, Base: LU.Formulae [i]);
4793	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4794	GenerateScales(LU, LUIdx, Base: LU.Formulae [i]);
4795	}
4796	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4797	LSRUse &LU = Uses [LUIdx];
4798	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4799	GenerateTruncates(LU, LUIdx, Base: LU.Formulae [i]);
4800	}
4801
4802	GenerateCrossUseConstantOffsets();
4803
4804	LLVM_DEBUG(dbgs() << "\n"
4805	"After generating reuse formulae:\n";
4806	print_uses(dbgs()));
4807	}
4808
4809	/// If there are multiple formulae with the same set of registers used
4810	/// by other uses, pick the best one and delete the others.
4811	void LSRInstance::FilterOutUndesirableDedicatedRegisters() {
4812	DenseSet<const SCEV *> VisitedRegs;
4813	SmallPtrSet<const SCEV *, `16`> Regs;
4814	SmallPtrSet<const SCEV *, `16`> LoserRegs;
4815	#ifndef NDEBUG
4816	bool ChangedFormulae = false;
4817	#endif
4818
4819	// Collect the best formula for each unique set of shared registers. This
4820	// is reset for each use.
4821	using BestFormulaeTy = DenseMap<SmallVector<const SCEV *, `4`>, size_t>;
4822
4823	BestFormulaeTy BestFormulae;
4824
4825	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4826	LSRUse &LU = Uses [LUIdx];
4827	LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4828	dbgs() << `'\n'`);
4829
4830	bool Any = false;
4831	for (size_t FIdx = `0`, NumForms = LU.Formulae.size();
4832	FIdx != NumForms; ++FIdx) {
4833	Formula &F = LU.Formulae [FIdx];
4834
4835	// Some formulas are instant losers. For example, they may depend on
4836	// nonexistent AddRecs from other loops. These need to be filtered
4837	// immediately, otherwise heuristics could choose them over others leading
4838	// to an unsatisfactory solution. Passing LoserRegs into RateFormula here
4839	// avoids the need to recompute this information across formulae using the
4840	// same bad AddRec. Passing LoserRegs is also essential unless we remove
4841	// the corresponding bad register from the Regs set.
4842	Cost CostF(L, SE, TTI, AMK);
4843	Regs.clear();
4844	CostF.RateFormula(F, Regs, VisitedRegs, LU, HardwareLoopProfitable,
4845	LoserRegs: &LoserRegs);
4846	if (CostF.isLoser()) {
4847	// During initial formula generation, undesirable formulae are generated
4848	// by uses within other loops that have some non-trivial address mode or
4849	// use the postinc form of the IV. LSR needs to provide these formulae
4850	// as the basis of rediscovering the desired formula that uses an AddRec
4851	// corresponding to the existing phi. Once all formulae have been
4852	// generated, these initial losers may be pruned.
4853	LLVM_DEBUG(dbgs() << " Filtering loser "; F.print(dbgs());
4854	dbgs() << "\n");
4855	}
4856	else {
4857	SmallVector<const SCEV *, `4`> Key;
4858	for (const SCEV *Reg : F.BaseRegs) {
4859	if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx))
4860	Key.push_back(Elt: Reg);
4861	}
4862	if (F.ScaledReg &&
4863	RegUses.isRegUsedByUsesOtherThan(Reg: F.ScaledReg, LUIdx))
4864	Key.push_back(Elt: F.ScaledReg);
4865	// Unstable sort by host order ok, because this is only used for
4866	// uniquifying.
4867	llvm::sort(C&: Key);
4868
4869	std::pair<BestFormulaeTy::const_iterator, bool> P =
4870	BestFormulae.insert(KV: std::make_pair(x&: Key, y&: FIdx));
4871	if (P.second)
4872	continue;
4873
4874	Formula &Best = LU.Formulae [P.first ->second];
4875
4876	Cost CostBest(L, SE, TTI, AMK);
4877	Regs.clear();
4878	CostBest.RateFormula(F: Best, Regs, VisitedRegs, LU,
4879	HardwareLoopProfitable);
4880	if (CostF.isLess(Other: CostBest))
4881	std::swap(a&: F, b&: Best);
4882	LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
4883	dbgs() << "\n"
4884	" in favor of formula ";
4885	Best.print(dbgs()); dbgs() << `'\n'`);
4886	}
4887	#ifndef NDEBUG
4888	ChangedFormulae = true;
4889	#endif
4890	LU.DeleteFormula(F);
4891	--FIdx;
4892	--NumForms;
4893	Any = true;
4894	}
4895
4896	// Now that we've filtered out some formulae, recompute the Regs set.
4897	if (Any)
4898	LU.RecomputeRegs(LUIdx, RegUses);
4899
4900	// Reset this to prepare for the next use.
4901	BestFormulae.clear();
4902	}
4903
4904	LLVM_DEBUG(if (ChangedFormulae) {
4905	dbgs() << "\n"
4906	"After filtering out undesirable candidates:\n";
4907	print_uses(dbgs());
4908	});
4909	}
4910
4911	/// Estimate the worst-case number of solutions the solver might have to
4912	/// consider. It almost never considers this many solutions because it prune the
4913	/// search space, but the pruning isn't always sufficient.
4914	size_t LSRInstance::EstimateSearchSpaceComplexity() const {
4915	size_t Power = `1`;
4916	for (const LSRUse &LU : Uses) {
4917	size_t FSize = LU.Formulae.size();
4918	if (FSize >= ComplexityLimit) {
4919	Power = ComplexityLimit;
4920	break;
4921	}
4922	Power *= FSize;
4923	if (Power >= ComplexityLimit)
4924	break;
4925	}
4926	return Power;
4927	}
4928
4929	/// When one formula uses a superset of the registers of another formula, it
4930	/// won't help reduce register pressure (though it may not necessarily hurt
4931	/// register pressure); remove it to simplify the system.
4932	void LSRInstance::NarrowSearchSpaceByDetectingSupersets() {
4933	if (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
4934	LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
4935
4936	LLVM_DEBUG(dbgs() << "Narrowing the search space by eliminating formulae "
4937	"which use a superset of registers used by other "
4938	"formulae.\n");
4939
4940	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4941	LSRUse &LU = Uses [LUIdx];
4942	bool Any = false;
4943	for (size_t i = `0`, e = LU.Formulae.size(); i != e; ++i) {
4944	Formula &F = LU.Formulae [i];
4945	if (F.BaseOffset.isNonZero() && F.BaseOffset.isScalable())
4946	continue;
4947	// Look for a formula with a constant or GV in a register. If the use
4948	// also has a formula with that same value in an immediate field,
4949	// delete the one that uses a register.
4950	for (SmallVectorImpl<const SCEV *>::const_iterator
4951	I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) {
4952	if (const SCEVConstant C = dyn_cast<SCEVConstant>(Val: I)) {
4953	Formula NewF = F;
4954	//FIXME: Formulas should store bitwidth to do wrapping properly.
4955	// See PR41034.
4956	NewF.BaseOffset =
4957	Immediate::getFixed(MinVal: NewF.BaseOffset.getFixedValue() +
4958	(uint64_t)C->getValue()->getSExtValue());
4959	NewF.BaseRegs.erase(CI: NewF.BaseRegs.begin() +
4960	(I - F.BaseRegs.begin()));
4961	if (LU.HasFormulaWithSameRegs(F: NewF)) {
4962	LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());
4963	dbgs() << `'\n'`);
4964	LU.DeleteFormula(F);
4965	--i;
4966	--e;
4967	Any = true;
4968	break;
4969	}
4970	} else if (const SCEVUnknown U = dyn_cast<SCEVUnknown>(Val: I)) {
4971	if (GlobalValue *GV = dyn_cast<GlobalValue>(Val: U->getValue()))
4972	if (!F.BaseGV) {
4973	Formula NewF = F;
4974	NewF.BaseGV = GV;
4975	NewF.BaseRegs.erase(CI: NewF.BaseRegs.begin() +
4976	(I - F.BaseRegs.begin()));
4977	if (LU.HasFormulaWithSameRegs(F: NewF)) {
4978	LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());
4979	dbgs() << `'\n'`);
4980	LU.DeleteFormula(F);
4981	--i;
4982	--e;
4983	Any = true;
4984	break;
4985	}
4986	}
4987	}
4988	}
4989	}
4990	if (Any)
4991	LU.RecomputeRegs(LUIdx, RegUses);
4992	}
4993
4994	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
4995	}
4996	}
4997
4998	/// When there are many registers for expressions like A, A+1, A+2, etc.,
4999	/// allocate a single register for them.
5000	void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() {
5001	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5002	return;
5003
5004	LLVM_DEBUG(
5005	dbgs() << "The search space is too complex.\n"
5006	"Narrowing the search space by assuming that uses separated "
5007	"by a constant offset will use the same registers.\n");
5008
5009	// This is especially useful for unrolled loops.
5010
5011	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5012	LSRUse &LU = Uses [LUIdx];
5013	for (const Formula &F : LU.Formulae) {
5014	if (F.BaseOffset.isZero() \|\| (F.Scale != `0` && F.Scale != `1`))
5015	continue;
5016	assert((LU.Kind == LSRUse::Address \|\| LU.Kind == LSRUse::ICmpZero) &&
5017	"Only address and cmp uses expected to have nonzero BaseOffset");
5018
5019	LSRUse *LUThatHas = FindUseWithSimilarFormula(OrigF: F, OrigLU: LU);
5020	if (!LUThatHas)
5021	continue;
5022
5023	if (!reconcileNewOffset(LU&: LUThatHas, NewOffset: F.BaseOffset, /HasBaseReg=/* false,
5024	Kind: LU.Kind, AccessTy: LU.AccessTy))
5025	continue;
5026
5027	LLVM_DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << `'\n'`);
5028
5029	LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
5030	LUThatHas->AllFixupsUnconditional &= LU.AllFixupsUnconditional;
5031
5032	// Transfer the fixups of LU to LUThatHas.
5033	for (LSRFixup &Fixup : LU.Fixups) {
5034	Fixup.Offset += F.BaseOffset;
5035	LUThatHas->pushFixup(f&: Fixup);
5036	LLVM_DEBUG(dbgs() << "New fixup has offset " << Fixup.Offset << `'\n'`);
5037	}
5038
5039	#ifndef NDEBUG
5040	Type *FixupType = LUThatHas->Fixups[`0`].OperandValToReplace->getType();
5041	for (LSRFixup &Fixup : LUThatHas->Fixups)
5042	assert(Fixup.OperandValToReplace->getType() == FixupType &&
5043	"Expected all fixups to have the same type");
5044	#endif
5045
5046	// Delete formulae from the new use which are no longer legal.
5047	bool Any = false;
5048	for (size_t i = `0`, e = LUThatHas->Formulae.size(); i != e; ++i) {
5049	Formula &F = LUThatHas->Formulae [i];
5050	if (!isLegalUse(TTI, MinOffset: LUThatHas->MinOffset, MaxOffset: LUThatHas->MaxOffset,
5051	Kind: LUThatHas->Kind, AccessTy: LUThatHas->AccessTy, F)) {
5052	LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << `'\n'`);
5053	LUThatHas->DeleteFormula(F);
5054	--i;
5055	--e;
5056	Any = true;
5057	}
5058	}
5059
5060	if (Any)
5061	LUThatHas->RecomputeRegs(LUIdx: LUThatHas - &Uses.front(), RegUses);
5062
5063	// Delete the old use.
5064	DeleteUse(LU, LUIdx);
5065	--LUIdx;
5066	--NumUses;
5067	break;
5068	}
5069	}
5070
5071	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5072	}
5073
5074	/// Call FilterOutUndesirableDedicatedRegisters again, if necessary, now that
5075	/// we've done more filtering, as it may be able to find more formulae to
5076	/// eliminate.
5077	void LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){
5078	if (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
5079	LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5080
5081	LLVM_DEBUG(dbgs() << "Narrowing the search space by re-filtering out "
5082	"undesirable dedicated registers.\n");
5083
5084	FilterOutUndesirableDedicatedRegisters();
5085
5086	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5087	}
5088	}
5089
5090	/// If a LSRUse has multiple formulae with the same ScaledReg and Scale.
5091	/// Pick the best one and delete the others.
5092	/// This narrowing heuristic is to keep as many formulae with different
5093	/// Scale and ScaledReg pair as possible while narrowing the search space.
5094	/// The benefit is that it is more likely to find out a better solution
5095	/// from a formulae set with more Scale and ScaledReg variations than
5096	/// a formulae set with the same Scale and ScaledReg. The picking winner
5097	/// reg heuristic will often keep the formulae with the same Scale and
5098	/// ScaledReg and filter others, and we want to avoid that if possible.
5099	void LSRInstance::NarrowSearchSpaceByFilterFormulaWithSameScaledReg() {
5100	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5101	return;
5102
5103	LLVM_DEBUG(
5104	dbgs() << "The search space is too complex.\n"
5105	"Narrowing the search space by choosing the best Formula "
5106	"from the Formulae with the same Scale and ScaledReg.\n");
5107
5108	// Map the "Scale ScaledReg" pair to the best formula of current LSRUse.*
5109	using BestFormulaeTy = DenseMap<std::pair<const SCEV *, int64_t>, size_t>;
5110
5111	BestFormulaeTy BestFormulae;
5112	#ifndef NDEBUG
5113	bool ChangedFormulae = false;
5114	#endif
5115	DenseSet<const SCEV *> VisitedRegs;
5116	SmallPtrSet<const SCEV *, `16`> Regs;
5117
5118	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5119	LSRUse &LU = Uses [LUIdx];
5120	LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
5121	dbgs() << `'\n'`);
5122
5123	// Return true if Formula FA is better than Formula FB.
5124	auto IsBetterThan = [&](Formula &FA, Formula &FB) {
5125	// First we will try to choose the Formula with fewer new registers.
5126	// For a register used by current Formula, the more the register is
5127	// shared among LSRUses, the less we increase the register number
5128	// counter of the formula.
5129	size_t FARegNum = `0`;
5130	for (const SCEV *Reg : FA.BaseRegs) {
5131	const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg);
5132	FARegNum += (NumUses - UsedByIndices.count() + `1`);
5133	}
5134	size_t FBRegNum = `0`;
5135	for (const SCEV *Reg : FB.BaseRegs) {
5136	const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg);
5137	FBRegNum += (NumUses - UsedByIndices.count() + `1`);
5138	}
5139	if (FARegNum != FBRegNum)
5140	return FARegNum < FBRegNum;
5141
5142	// If the new register numbers are the same, choose the Formula with
5143	// less Cost.
5144	Cost CostFA(L, SE, TTI, AMK);
5145	Cost CostFB(L, SE, TTI, AMK);
5146	Regs.clear();
5147	CostFA.RateFormula(F: FA, Regs, VisitedRegs, LU, HardwareLoopProfitable);
5148	Regs.clear();
5149	CostFB.RateFormula(F: FB, Regs, VisitedRegs, LU, HardwareLoopProfitable);
5150	return CostFA.isLess(Other: CostFB);
5151	};
5152
5153	bool Any = false;
5154	for (size_t FIdx = `0`, NumForms = LU.Formulae.size(); FIdx != NumForms;
5155	++FIdx) {
5156	Formula &F = LU.Formulae [FIdx];
5157	if (!F.ScaledReg)
5158	continue;
5159	auto P = BestFormulae.insert(KV: {{F.ScaledReg, F.Scale}, FIdx});
5160	if (P.second)
5161	continue;
5162
5163	Formula &Best = LU.Formulae [P.first ->second];
5164	if (IsBetterThan (F, Best))
5165	std::swap(a&: F, b&: Best);
5166	LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
5167	dbgs() << "\n"
5168	" in favor of formula ";
5169	Best.print(dbgs()); dbgs() << `'\n'`);
5170	#ifndef NDEBUG
5171	ChangedFormulae = true;
5172	#endif
5173	LU.DeleteFormula(F);
5174	--FIdx;
5175	--NumForms;
5176	Any = true;
5177	}
5178	if (Any)
5179	LU.RecomputeRegs(LUIdx, RegUses);
5180
5181	// Reset this to prepare for the next use.
5182	BestFormulae.clear();
5183	}
5184
5185	LLVM_DEBUG(if (ChangedFormulae) {
5186	dbgs() << "\n"
5187	"After filtering out undesirable candidates:\n";
5188	print_uses(dbgs());
5189	});
5190	}
5191
5192	/// If we are over the complexity limit, filter out any post-inc prefering
5193	/// variables to only post-inc values.
5194	void LSRInstance::NarrowSearchSpaceByFilterPostInc() {
5195	if (AMK != TTI::AMK_PostIndexed)
5196	return;
5197	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5198	return;
5199
5200	LLVM_DEBUG(dbgs() << "The search space is too complex.\n"
5201	"Narrowing the search space by choosing the lowest "
5202	"register Formula for PostInc Uses.\n");
5203
5204	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5205	LSRUse &LU = Uses [LUIdx];
5206
5207	if (LU.Kind != LSRUse::Address)
5208	continue;
5209	if (!TTI.isIndexedLoadLegal(Mode: TTI.MIM_PostInc, Ty: LU.AccessTy.getType()) &&
5210	!TTI.isIndexedStoreLegal(Mode: TTI.MIM_PostInc, Ty: LU.AccessTy.getType()))
5211	continue;
5212
5213	size_t MinRegs = std::numeric_limits<size_t>::max();
5214	for (const Formula &F : LU.Formulae)
5215	MinRegs = std::min(a: F.getNumRegs(), b: MinRegs);
5216
5217	bool Any = false;
5218	for (size_t FIdx = `0`, NumForms = LU.Formulae.size(); FIdx != NumForms;
5219	++FIdx) {
5220	Formula &F = LU.Formulae [FIdx];
5221	if (F.getNumRegs() > MinRegs) {
5222	LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
5223	dbgs() << "\n");
5224	LU.DeleteFormula(F);
5225	--FIdx;
5226	--NumForms;
5227	Any = true;
5228	}
5229	}
5230	if (Any)
5231	LU.RecomputeRegs(LUIdx, RegUses);
5232
5233	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5234	break;
5235	}
5236
5237	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5238	}
5239
5240	void LSRInstance::NarrowSearchSpaceByMergingUsesOutsideLoop() {
5241	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5242	return;
5243
5244	LLVM_DEBUG(
5245	dbgs() << "The search space is too complex.\n"
5246	"Narrowing the search space by merging uses with fixups "
5247	"entirely outside the loop with uses inside the loop.\n");
5248
5249	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5250	LSRUse &LU = Uses [LUIdx];
5251	// Don't merge ICmpZero uses outside the loop, as ICmpZero needs to be
5252	// handled specially when expanding.
5253	if (!LU.AllFixupsOutsideLoop \|\| LU.Formulae.empty() \|\|
5254	LU.Kind == LSRUse::ICmpZero)
5255	continue;
5256
5257	LLVM_DEBUG(dbgs() << " Trying to eliminate use "; LU.print(dbgs());
5258	dbgs() << `'\n'`);
5259
5260	// Find a compatible LSRUse inside the loop that we could merge LU with
5261	LSRUse LUToMergeWith = nullptr*;
5262	const Formula &ThisF = LU.Formulae [`0`];
5263	for (LSRUse &OtherLU : Uses) {
5264	// Only merge with uses inside the loop
5265	if (OtherLU.AllFixupsOutsideLoop)
5266	continue;
5267	// Can't merge with ICmpZero uses as they're handled specially when
5268	// expanding
5269	if (OtherLU.Kind == LSRUse::ICmpZero)
5270	continue;
5271	// Can't merge with uses without any formulae
5272	if (OtherLU.Formulae.empty())
5273	continue;
5274	// Can't merge if LU's offsets aren't legal for all of OtherLU's formulae
5275	if (any_of(Range&: OtherLU.Formulae, P: [&](const Formula &F) {
5276	return !isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: OtherLU.Kind,
5277	AccessTy: OtherLU.AccessTy, F);
5278	}))
5279	continue;
5280	// We can merge with uses that have the same initial formula. We allow
5281	// merging of uses with different Kind and AccessTy which means that the
5282	// cost may end up being inaccurate, but it's also what we would have
5283	// gotten if we'd ignored uses outside the loop entirely.
5284	const Formula &OtherF = OtherLU.Formulae [`0`];
5285	if (ThisF.BaseRegs == OtherF.BaseRegs &&
5286	ThisF.ScaledReg == OtherF.ScaledReg &&
5287	ThisF.BaseGV == OtherF.BaseGV && ThisF.Scale == OtherF.Scale &&
5288	ThisF.UnfoldedOffset == OtherF.UnfoldedOffset &&
5289	ThisF.BaseOffset == OtherF.BaseOffset) {
5290	LUToMergeWith = &OtherLU;
5291	break;
5292	}
5293	}
5294	if (!LUToMergeWith)
5295	continue;
5296
5297	LLVM_DEBUG(dbgs() << " Merging with "; LUToMergeWith->print(dbgs());
5298	dbgs() << `'\n'`);
5299
5300	// Copy fixups
5301	for (LSRFixup &Fixup : LU.Fixups) {
5302	LUToMergeWith->pushFixup(f&: Fixup);
5303	}
5304
5305	// Delete the old use.
5306	DeleteUse(LU, LUIdx);
5307	--LUIdx;
5308	--NumUses;
5309	}
5310
5311	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5312	}
5313
5314	/// The function delete formulas with high registers number expectation.
5315	/// Assuming we don't know the value of each formula (already delete
5316	/// all inefficient), generate probability of not selecting for each
5317	/// register.
5318	/// For example,
5319	/// Use1:
5320	/// reg(a) + reg({0,+,1})
5321	/// reg(a) + reg({-1,+,1}) + 1
5322	/// reg({a,+,1})
5323	/// Use2:
5324	/// reg(b) + reg({0,+,1})
5325	/// reg(b) + reg({-1,+,1}) + 1
5326	/// reg({b,+,1})
5327	/// Use3:
5328	/// reg(c) + reg(b) + reg({0,+,1})
5329	/// reg(c) + reg({b,+,1})
5330	///
5331	/// Probability of not selecting
5332	/// Use1 Use2 Use3
5333	/// reg(a) (1/3) 1 * 1*
5334	/// reg(b) 1 (1/3) * (1/2)*
5335	/// reg({0,+,1}) (2/3) (2/3) * (1/2)*
5336	/// reg({-1,+,1}) (2/3) (2/3) * 1*
5337	/// reg({a,+,1}) (2/3) 1 * 1*
5338	/// reg({b,+,1}) 1 (2/3) * (2/3)*
5339	/// reg(c) 1 1 * 0*
5340	///
5341	/// Now count registers number mathematical expectation for each formula:
5342	/// Note that for each use we exclude probability if not selecting for the use.
5343	/// For example for Use1 probability for reg(a) would be just 1 1 (excluding*
5344	/// probabilty 1/3 of not selecting for Use1).
5345	/// Use1:
5346	/// reg(a) + reg({0,+,1}) 1 + 1/3 -- to be deleted
5347	/// reg(a) + reg({-1,+,1}) + 1 1 + 4/9 -- to be deleted
5348	/// reg({a,+,1}) 1
5349	/// Use2:
5350	/// reg(b) + reg({0,+,1}) 1/2 + 1/3 -- to be deleted
5351	/// reg(b) + reg({-1,+,1}) + 1 1/2 + 2/3 -- to be deleted
5352	/// reg({b,+,1}) 2/3
5353	/// Use3:
5354	/// reg(c) + reg(b) + reg({0,+,1}) 1 + 1/3 + 4/9 -- to be deleted
5355	/// reg(c) + reg({b,+,1}) 1 + 2/3
5356	void LSRInstance::NarrowSearchSpaceByDeletingCostlyFormulas() {
5357	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5358	return;
5359	// Ok, we have too many of formulae on our hands to conveniently handle.
5360	// Use a rough heuristic to thin out the list.
5361
5362	// Set of Regs wich will be 100% used in final solution.
5363	// Used in each formula of a solution (in example above this is reg(c)).
5364	// We can skip them in calculations.
5365	SmallPtrSet<const SCEV *, `4`> UniqRegs;
5366	LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5367
5368	// Map each register to probability of not selecting
5369	DenseMap <const SCEV , float*> RegNumMap;
5370	for (const SCEV *Reg : RegUses) {
5371	if (UniqRegs.count(Ptr: Reg))
5372	continue;
5373	float PNotSel = `1`;
5374	for (const LSRUse &LU : Uses) {
5375	if (!LU.Regs.count(Ptr: Reg))
5376	continue;
5377	float P = LU.getNotSelectedProbability(Reg);
5378	if (P != `0.0`)
5379	PNotSel *= P;
5380	else
5381	UniqRegs.insert(Ptr: Reg);
5382	}
5383	RegNumMap.insert(KV: std::make_pair(x&: Reg, y&: PNotSel));
5384	}
5385
5386	LLVM_DEBUG(
5387	dbgs() << "Narrowing the search space by deleting costly formulas\n");
5388
5389	// Delete formulas where registers number expectation is high.
5390	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5391	LSRUse &LU = Uses [LUIdx];
5392	// If nothing to delete - continue.
5393	if (LU.Formulae.size() < `2`)
5394	continue;
5395	// This is temporary solution to test performance. Float should be
5396	// replaced with round independent type (based on integers) to avoid
5397	// different results for different target builds.
5398	float FMinRegNum = LU.Formulae [`0`].getNumRegs();
5399	float FMinARegNum = LU.Formulae [`0`].getNumRegs();
5400	size_t MinIdx = `0`;
5401	for (size_t i = `0`, e = LU.Formulae.size(); i != e; ++i) {
5402	Formula &F = LU.Formulae [i];
5403	float FRegNum = `0`;
5404	float FARegNum = `0`;
5405	for (const SCEV *BaseReg : F.BaseRegs) {
5406	if (UniqRegs.count(Ptr: BaseReg))
5407	continue;
5408	FRegNum += RegNumMap [BaseReg] / LU.getNotSelectedProbability(Reg: BaseReg);
5409	if (isa<SCEVAddRecExpr>(Val: BaseReg))
5410	FARegNum +=
5411	RegNumMap [BaseReg] / LU.getNotSelectedProbability(Reg: BaseReg);
5412	}
5413	if (const SCEV *ScaledReg = F.ScaledReg) {
5414	if (!UniqRegs.count(Ptr: ScaledReg)) {
5415	FRegNum +=
5416	RegNumMap [ScaledReg] / LU.getNotSelectedProbability(Reg: ScaledReg);
5417	if (isa<SCEVAddRecExpr>(Val: ScaledReg))
5418	FARegNum +=
5419	RegNumMap [ScaledReg] / LU.getNotSelectedProbability(Reg: ScaledReg);
5420	}
5421	}
5422	if (FMinRegNum > FRegNum \|\|
5423	(FMinRegNum == FRegNum && FMinARegNum > FARegNum)) {
5424	FMinRegNum = FRegNum;
5425	FMinARegNum = FARegNum;
5426	MinIdx = i;
5427	}
5428	}
5429	LLVM_DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs());
5430	dbgs() << " with min reg num " << FMinRegNum << `'\n'`);
5431	if (MinIdx != `0`)
5432	std::swap(a&: LU.Formulae [MinIdx], b&: LU.Formulae [`0`]);
5433	while (LU.Formulae.size() != `1`) {
5434	LLVM_DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs());
5435	dbgs() << `'\n'`);
5436	LU.Formulae.pop_back();
5437	}
5438	LU.RecomputeRegs(LUIdx, RegUses);
5439	assert(LU.Formulae.size() == `1` && "Should be exactly 1 min regs formula");
5440	Formula &F = LU.Formulae [`0`];
5441	LLVM_DEBUG(dbgs() << " Leaving only "; F.print(dbgs()); dbgs() << `'\n'`);
5442	// When we choose the formula, the regs become unique.
5443	UniqRegs.insert_range(R&: F.BaseRegs);
5444	if (F.ScaledReg)
5445	UniqRegs.insert(Ptr: F.ScaledReg);
5446	}
5447	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5448	}
5449
5450	// Check if Best and Reg are SCEVs separated by a constant amount C, and if so
5451	// would the addressing offset +C would be legal where the negative offset -C is
5452	// not.
5453	static bool IsSimplerBaseSCEVForTarget(const TargetTransformInfo &TTI,
5454	ScalarEvolution &SE, const SCEV *Best,
5455	const SCEV *Reg,
5456	MemAccessTy AccessType) {
5457	if (Best->getType() != Reg->getType() \|\|
5458	(isa<SCEVAddRecExpr>(Val: Best) && isa<SCEVAddRecExpr>(Val: Reg) &&
5459	cast<SCEVAddRecExpr>(Val: Best)->getLoop() !=
5460	cast<SCEVAddRecExpr>(Val: Reg)->getLoop()))
5461	return false;
5462	std::optional<APInt> Diff = SE.computeConstantDifference(LHS: Best, RHS: Reg);
5463	if (!Diff)
5464	return false;
5465
5466	return TTI.isLegalAddressingMode(
5467	Ty: AccessType.MemTy, /BaseGV=/nullptr,
5468	/BaseOffset=/Diff ->getSExtValue(),
5469	/HasBaseReg=/true, /Scale=/`0`, AddrSpace: AccessType.AddrSpace) &&
5470	!TTI.isLegalAddressingMode(
5471	Ty: AccessType.MemTy, /BaseGV=/nullptr,
5472	/BaseOffset=/-Diff ->getSExtValue(),
5473	/HasBaseReg=/true, /Scale=/`0`, AddrSpace: AccessType.AddrSpace);
5474	}
5475
5476	/// Pick a register which seems likely to be profitable, and then in any use
5477	/// which has any reference to that register, delete all formulae which do not
5478	/// reference that register.
5479	void LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() {
5480	// With all other options exhausted, loop until the system is simple
5481	// enough to handle.
5482	SmallPtrSet<const SCEV *, `4`> Taken;
5483	while (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
5484	// Ok, we have too many of formulae on our hands to conveniently handle.
5485	// Use a rough heuristic to thin out the list.
5486	LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5487
5488	// Pick the register which is used by the most LSRUses, which is likely
5489	// to be a good reuse register candidate.
5490	const SCEV Best = nullptr*;
5491	unsigned BestNum = `0`;
5492	for (const SCEV *Reg : RegUses) {
5493	if (Taken.count(Ptr: Reg))
5494	continue;
5495	if (!Best) {
5496	Best = Reg;
5497	BestNum = RegUses.getUsedByIndices(Reg).count();
5498	} else {
5499	unsigned Count = RegUses.getUsedByIndices(Reg).count();
5500	if (Count > BestNum) {
5501	Best = Reg;
5502	BestNum = Count;
5503	}
5504
5505	// If the scores are the same, but the Reg is simpler for the target
5506	// (for example {x,+,1} as opposed to {x+C,+,1}, where the target can
5507	// handle +C but not -C), opt for the simpler formula.
5508	if (Count == BestNum) {
5509	int LUIdx = RegUses.getUsedByIndices(Reg).find_first();
5510	if (LUIdx >= `0` && Uses [LUIdx].Kind == LSRUse::Address &&
5511	IsSimplerBaseSCEVForTarget(TTI, SE, Best, Reg,
5512	AccessType: Uses [LUIdx].AccessTy)) {
5513	Best = Reg;
5514	BestNum = Count;
5515	}
5516	}
5517	}
5518	}
5519	assert(Best && "Failed to find best LSRUse candidate");
5520
5521	LLVM_DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best
5522	<< " will yield profitable reuse.\n");
5523	Taken.insert(Ptr: Best);
5524
5525	// In any use with formulae which references this register, delete formulae
5526	// which don't reference it.
5527	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5528	LSRUse &LU = Uses [LUIdx];
5529	if (!LU.Regs.count(Ptr: Best)) continue;
5530
5531	bool Any = false;
5532	for (size_t i = `0`, e = LU.Formulae.size(); i != e; ++i) {
5533	Formula &F = LU.Formulae [i];
5534	if (!F.referencesReg(S: Best)) {
5535	LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << `'\n'`);
5536	LU.DeleteFormula(F);
5537	--e;
5538	--i;
5539	Any = true;
5540	assert(e != `0` && "Use has no formulae left! Is Regs inconsistent?");
5541	continue;
5542	}
5543	}
5544
5545	if (Any)
5546	LU.RecomputeRegs(LUIdx, RegUses);
5547	}
5548
5549	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5550	}
5551	}
5552
5553	/// If there are an extraordinary number of formulae to choose from, use some
5554	/// rough heuristics to prune down the number of formulae. This keeps the main
5555	/// solver from taking an extraordinary amount of time in some worst-case
5556	/// scenarios.
5557	void LSRInstance::NarrowSearchSpaceUsingHeuristics() {
5558	NarrowSearchSpaceByDetectingSupersets();
5559	NarrowSearchSpaceByCollapsingUnrolledCode();
5560	NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
5561	if (FilterSameScaledReg)
5562	NarrowSearchSpaceByFilterFormulaWithSameScaledReg();
5563	NarrowSearchSpaceByFilterPostInc();
5564	NarrowSearchSpaceByMergingUsesOutsideLoop();
5565	if (LSRExpNarrow)
5566	NarrowSearchSpaceByDeletingCostlyFormulas();
5567	else
5568	NarrowSearchSpaceByPickingWinnerRegs();
5569	}
5570
5571	/// This is the recursive solver.
5572	void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
5573	Cost &SolutionCost,
5574	SmallVectorImpl<const Formula *> &Workspace,
5575	const Cost &CurCost,
5576	const SmallPtrSet<const SCEV *, `16`> &CurRegs,
5577	DenseSet<const SCEV > &VisitedRegs) const* {
5578	// Some ideas:
5579	// - prune more:
5580	// - use more aggressive filtering
5581	// - sort the formula so that the most profitable solutions are found first
5582	// - sort the uses too
5583	// - search faster:
5584	// - don't compute a cost, and then compare. compare while computing a cost
5585	// and bail early.
5586	// - track register sets with SmallBitVector
5587
5588	const LSRUse &LU = Uses [Workspace.size()];
5589
5590	// If this use references any register that's already a part of the
5591	// in-progress solution, consider it a requirement that a formula must
5592	// reference that register in order to be considered. This prunes out
5593	// unprofitable searching.
5594	SmallSetVector<const SCEV *, `4`> ReqRegs;
5595	for (const SCEV *S : CurRegs)
5596	if (LU.Regs.count(Ptr: S))
5597	ReqRegs.insert(X: S);
5598
5599	SmallPtrSet<const SCEV *, `16`> NewRegs;
5600	Cost NewCost(L, SE, TTI, AMK);
5601	for (const Formula &F : LU.Formulae) {
5602	// Ignore formulae which may not be ideal in terms of register reuse of
5603	// ReqRegs. The formula should use all required registers before
5604	// introducing new ones.
5605	// This can sometimes (notably when trying to favour postinc) lead to
5606	// sub-optimial decisions. There it is best left to the cost modelling to
5607	// get correct.
5608	if (!(AMK & TTI::AMK_PostIndexed) \|\| LU.Kind != LSRUse::Address) {
5609	int NumReqRegsToFind = std::min(a: F.getNumRegs(), b: ReqRegs.size());
5610	for (const SCEV *Reg : ReqRegs) {
5611	if ((F.ScaledReg && F.ScaledReg == Reg) \|\|
5612	is_contained(Range: F.BaseRegs, Element: Reg)) {
5613	--NumReqRegsToFind;
5614	if (NumReqRegsToFind == `0`)
5615	break;
5616	}
5617	}
5618	if (NumReqRegsToFind != `0`) {
5619	// If none of the formulae satisfied the required registers, then we could
5620	// clear ReqRegs and try again. Currently, we simply give up in this case.
5621	continue;
5622	}
5623	}
5624
5625	// Evaluate the cost of the current formula. If it's already worse than
5626	// the current best, prune the search at that point.
5627	NewCost = CurCost;
5628	NewRegs = CurRegs;
5629	NewCost.RateFormula(F, Regs&: NewRegs, VisitedRegs, LU, HardwareLoopProfitable);
5630	if (NewCost.isLess(Other: SolutionCost)) {
5631	Workspace.push_back(Elt: &F);
5632	if (Workspace.size() != Uses.size()) {
5633	SolveRecurse(Solution, SolutionCost, Workspace, CurCost: NewCost,
5634	CurRegs: NewRegs, VisitedRegs);
5635	if (F.getNumRegs() == `1` && Workspace.size() == `1`)
5636	VisitedRegs.insert(V: F.ScaledReg ? F.ScaledReg : F.BaseRegs [`0`]);
5637	} else {
5638	LLVM_DEBUG(dbgs() << "New best at "; NewCost.print(dbgs());
5639	dbgs() << ".\nRegs:\n";
5640	for (const SCEV *S : NewRegs) dbgs()
5641	<< "- " << *S << "\n";
5642	dbgs() << `'\n'`);
5643
5644	SolutionCost = NewCost;
5645	Solution = Workspace;
5646	}
5647	Workspace.pop_back();
5648	}
5649	}
5650	}
5651
5652	/// Choose one formula from each use. Return the results in the given Solution
5653	/// vector.
5654	void LSRInstance::Solve(SmallVectorImpl<const Formula > &Solution) const* {
5655	SmallVector<const Formula *, `8`> Workspace;
5656	Cost SolutionCost(L, SE, TTI, AMK);
5657	SolutionCost.Lose();
5658	Cost CurCost(L, SE, TTI, AMK);
5659	SmallPtrSet<const SCEV *, `16`> CurRegs;
5660	DenseSet<const SCEV *> VisitedRegs;
5661	Workspace.reserve(N: Uses.size());
5662
5663	// SolveRecurse does all the work.
5664	SolveRecurse(Solution, SolutionCost, Workspace, CurCost,
5665	CurRegs, VisitedRegs);
5666	if (Solution.empty()) {
5667	LLVM_DEBUG(dbgs() << "\nNo Satisfactory Solution\n");
5668	return;
5669	}
5670
5671	// Ok, we've now made all our decisions.
5672	LLVM_DEBUG(dbgs() << "\n"
5673	"The chosen solution requires ";
5674	SolutionCost.print(dbgs()); dbgs() << ":\n";
5675	for (size_t i = `0`, e = Uses.size(); i != e; ++i) {
5676	dbgs() << " ";
5677	Uses[i].print(dbgs());
5678	dbgs() << "\n"
5679	" ";
5680	Solution[i]->print(dbgs());
5681	dbgs() << `'\n'`;
5682	});
5683
5684	assert(Solution.size() == Uses.size() && "Malformed solution!");
5685
5686	const bool EnableDropUnprofitableSolution = [&] {
5687	switch (AllowDropSolutionIfLessProfitable) {
5688	case cl::boolOrDefault::BOU_TRUE:
5689	return true;
5690	case cl::boolOrDefault::BOU_FALSE:
5691	return false;
5692	case cl::boolOrDefault::BOU_UNSET:
5693	return TTI.shouldDropLSRSolutionIfLessProfitable();
5694	}
5695	llvm_unreachable("Unhandled cl::boolOrDefault enum");
5696	}();
5697
5698	if (BaselineCost.isLess(Other: SolutionCost)) {
5699	if (!EnableDropUnprofitableSolution)
5700	LLVM_DEBUG(
5701	dbgs() << "Baseline is more profitable than chosen solution, "
5702	"add option 'lsr-drop-solution' to drop LSR solution.\n");
5703	else {
5704	LLVM_DEBUG(dbgs() << "Baseline is more profitable than chosen "
5705	"solution, dropping LSR solution.\n";);
5706	Solution.clear();
5707	}
5708	}
5709	}
5710
5711	/// Helper for AdjustInsertPositionForExpand. Climb up the dominator tree far as
5712	/// we can go while still being dominated by the input positions. This helps
5713	/// canonicalize the insert position, which encourages sharing.
5714	BasicBlock::iterator
5715	LSRInstance::HoistInsertPosition(BasicBlock::iterator IP,
5716	const SmallVectorImpl<Instruction *> &Inputs)
5717	const {
5718	Instruction Tentative = &IP;
5719	while (true) {
5720	bool AllDominate = true;
5721	Instruction BetterPos = nullptr*;
5722	// Don't bother attempting to insert before a catchswitch, their basic block
5723	// cannot have other non-PHI instructions.
5724	if (isa<CatchSwitchInst>(Val: Tentative))
5725	return IP;
5726
5727	for (Instruction *Inst : Inputs) {
5728	if (Inst == Tentative \|\| !DT.dominates(Def: Inst, User: Tentative)) {
5729	AllDominate = false;
5730	break;
5731	}
5732	// Attempt to find an insert position in the middle of the block,
5733	// instead of at the end, so that it can be used for other expansions.
5734	if (Tentative->getParent() == Inst->getParent() &&
5735	(!BetterPos \|\| !DT.dominates(Def: Inst, User: BetterPos)))
5736	BetterPos = &*std::next(x: BasicBlock::iterator(Inst));
5737	}
5738	if (!AllDominate)
5739	break;
5740	if (BetterPos)
5741	IP = BetterPos->getIterator();
5742	else
5743	IP = Tentative->getIterator();
5744
5745	const Loop *IPLoop = LI.getLoopFor(BB: IP ->getParent());
5746	unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : `0`;
5747
5748	BasicBlock *IDom;
5749	for (DomTreeNode *Rung = DT.getNode(BB: IP ->getParent()); ; ) {
5750	if (!Rung) return IP;
5751	Rung = Rung->getIDom();
5752	if (!Rung) return IP;
5753	IDom = Rung->getBlock();
5754
5755	// Don't climb into a loop though.
5756	const Loop *IDomLoop = LI.getLoopFor(BB: IDom);
5757	unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : `0`;
5758	if (IDomDepth <= IPLoopDepth &&
5759	(IDomDepth != IPLoopDepth \|\| IDomLoop == IPLoop))
5760	break;
5761	}
5762
5763	Tentative = IDom->getTerminator();
5764	}
5765
5766	return IP;
5767	}
5768
5769	/// Determine an input position which will be dominated by the operands and
5770	/// which will dominate the result.
5771	BasicBlock::iterator LSRInstance::AdjustInsertPositionForExpand(
5772	BasicBlock::iterator LowestIP, const LSRFixup &LF, const LSRUse &LU) const {
5773	// Collect some instructions which must be dominated by the
5774	// expanding replacement. These must be dominated by any operands that
5775	// will be required in the expansion.
5776	SmallVector<Instruction *, `4`> Inputs;
5777	if (Instruction *I = dyn_cast<Instruction>(Val: LF.OperandValToReplace))
5778	Inputs.push_back(Elt: I);
5779	if (LU.Kind == LSRUse::ICmpZero)
5780	if (Instruction *I =
5781	dyn_cast<Instruction>(Val: cast<ICmpInst>(Val: LF.UserInst)->getOperand(i_nocapture: `1`)))
5782	Inputs.push_back(Elt: I);
5783	if (LF.PostIncLoops.count(Ptr: L)) {
5784	if (LF.isUseFullyOutsideLoop(L))
5785	Inputs.push_back(Elt: L->getLoopLatch()->getTerminator());
5786	else
5787	Inputs.push_back(Elt: IVIncInsertPos);
5788	}
5789	// The expansion must also be dominated by the increment positions of any
5790	// loops it for which it is using post-inc mode.
5791	for (const Loop *PIL : LF.PostIncLoops) {
5792	if (PIL == L) continue;
5793
5794	// Be dominated by the loop exit.
5795	SmallVector<BasicBlock *, `4`> ExitingBlocks;
5796	PIL->getExitingBlocks(ExitingBlocks);
5797	if (!ExitingBlocks.empty()) {
5798	BasicBlock *BB = ExitingBlocks [`0`];
5799	for (unsigned i = `1`, e = ExitingBlocks.size(); i != e; ++i)
5800	BB = DT.findNearestCommonDominator(A: BB, B: ExitingBlocks [i]);
5801	Inputs.push_back(Elt: BB->getTerminator());
5802	}
5803	}
5804
5805	assert(!isa<PHINode>(LowestIP) && !LowestIP->isEHPad() &&
5806	"Insertion point must be a normal instruction");
5807
5808	// Then, climb up the immediate dominator tree as far as we can go while
5809	// still being dominated by the input positions.
5810	BasicBlock::iterator IP = HoistInsertPosition(IP: LowestIP, Inputs);
5811
5812	// Don't insert instructions before PHI nodes.
5813	while (isa<PHINode>(Val: IP)) ++IP;
5814
5815	// Ignore landingpad instructions.
5816	while (IP ->isEHPad()) ++IP;
5817
5818	// Set IP below instructions recently inserted by SCEVExpander. This keeps the
5819	// IP consistent across expansions and allows the previously inserted
5820	// instructions to be reused by subsequent expansion.
5821	while (Rewriter.isInsertedInstruction(I: &*IP) && IP != LowestIP)
5822	++IP;
5823
5824	return IP;
5825	}
5826
5827	/// Emit instructions for the leading candidate expression for this LSRUse (this
5828	/// is called "expanding").
5829	Value LSRInstance::Expand(const* LSRUse &LU, const LSRFixup &LF,
5830	const Formula &F, BasicBlock::iterator IP,
5831	SmallVectorImpl<WeakTrackingVH> &DeadInsts) const {
5832	if (LU.RigidFormula)
5833	return LF.OperandValToReplace;
5834
5835	// Determine an input position which will be dominated by the operands and
5836	// which will dominate the result.
5837	IP = AdjustInsertPositionForExpand(LowestIP: IP, LF, LU);
5838	Rewriter.setInsertPoint(&*IP);
5839
5840	// Inform the Rewriter if we have a post-increment use, so that it can
5841	// perform an advantageous expansion.
5842	Rewriter.setPostInc(LF.PostIncLoops);
5843
5844	// This is the type that the user actually needs.
5845	Type *OpTy = LF.OperandValToReplace->getType();
5846	// This will be the type that we'll initially expand to.
5847	Type *Ty = F.getType();
5848	if (!Ty)
5849	// No type known; just expand directly to the ultimate type.
5850	Ty = OpTy;
5851	else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(Ty: OpTy))
5852	// Expand directly to the ultimate type if it's the right size.
5853	Ty = OpTy;
5854	// This is the type to do integer arithmetic in.
5855	Type *IntTy = SE.getEffectiveSCEVType(Ty);
5856	// For ICmpZero with pointer-typed operands, keep the comparison in the
5857	// integer domain to avoid generating inttoptr casts. Use IntTy (the
5858	// formula's arithmetic width) so that both icmp operands match even when
5859	// the IV is wider than the pointer.
5860	if (LU.Kind == LSRUse::ICmpZero && OpTy->isPointerTy()) {
5861	OpTy = IntTy;
5862	Ty = IntTy;
5863	}
5864
5865	// Build up a list of operands to add together to form the full base.
5866	SmallVector<SCEVUse, `8`> Ops;
5867
5868	// Expand the BaseRegs portion.
5869	for (const SCEV *Reg : F.BaseRegs) {
5870	assert(!Reg->isZero() && "Zero allocated in a base register!");
5871
5872	// If we're expanding for a post-inc user, make the post-inc adjustment.
5873	Reg = denormalizeForPostIncUse(S: Reg, Loops: LF.PostIncLoops, SE);
5874	Ops.push_back(Elt: SE.getUnknown(V: Rewriter.expandCodeFor(SH: Reg, Ty: nullptr)));
5875	}
5876
5877	// Expand the ScaledReg portion.
5878	Value ICmpScaledV = nullptr*;
5879	if (F.Scale != `0`) {
5880	const SCEV *ScaledS = F.ScaledReg;
5881
5882	// If we're expanding for a post-inc user, make the post-inc adjustment.
5883	PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
5884	ScaledS = denormalizeForPostIncUse(S: ScaledS, Loops, SE);
5885
5886	if (LU.Kind == LSRUse::ICmpZero) {
5887	// Expand ScaleReg as if it was part of the base regs.
5888	if (F.Scale == `1`)
5889	Ops.push_back(
5890	Elt: SE.getUnknown(V: Rewriter.expandCodeFor(SH: ScaledS, Ty: nullptr)));
5891	else {
5892	// An interesting way of "folding" with an icmp is to use a negated
5893	// scale, which we'll implement by inserting it into the other operand
5894	// of the icmp.
5895	assert(F.Scale == -`1` &&
5896	"The only scale supported by ICmpZero uses is -1!");
5897	ICmpScaledV = Rewriter.expandCodeFor(SH: ScaledS, Ty: nullptr);
5898	}
5899	} else {
5900	// Otherwise just expand the scaled register and an explicit scale,
5901	// which is expected to be matched as part of the address.
5902
5903	// Flush the operand list to suppress SCEVExpander hoisting address modes.
5904	// Unless the addressing mode will not be folded.
5905	if (!Ops.empty() && LU.Kind == LSRUse::Address &&
5906	isAMCompletelyFolded(TTI, LU, F)) {
5907	Value FullV = Rewriter.expandCodeFor(SH: SE.getAddExpr(Ops), Ty: nullptr*);
5908	Ops.clear();
5909	Ops.push_back(Elt: SE.getUnknown(V: FullV));
5910	}
5911	ScaledS = SE.getUnknown(V: Rewriter.expandCodeFor(SH: ScaledS, Ty: nullptr));
5912	if (F.Scale != `1`)
5913	ScaledS =
5914	SE.getMulExpr(LHS: ScaledS, RHS: SE.getConstant(Ty: ScaledS->getType(), V: F.Scale));
5915	Ops.push_back(Elt: ScaledS);
5916	}
5917	}
5918
5919	// Expand the GV portion.
5920	if (F.BaseGV) {
5921	// Flush the operand list to suppress SCEVExpander hoisting.
5922	if (!Ops.empty()) {
5923	Value *FullV = Rewriter.expandCodeFor(SH: SE.getAddExpr(Ops), Ty: IntTy);
5924	Ops.clear();
5925	Ops.push_back(Elt: SE.getUnknown(V: FullV));
5926	}
5927	Ops.push_back(Elt: SE.getUnknown(V: F.BaseGV));
5928	}
5929
5930	// Flush the operand list to suppress SCEVExpander hoisting of both folded and
5931	// unfolded offsets. LSR assumes they both live next to their uses.
5932	if (!Ops.empty()) {
5933	Value *FullV = Rewriter.expandCodeFor(SH: SE.getAddExpr(Ops), Ty);
5934	Ops.clear();
5935	Ops.push_back(Elt: SE.getUnknown(V: FullV));
5936	}
5937
5938	// FIXME: Are we sure we won't get a mismatch here? Is there a way to bail
5939	// out at this point, or should we generate a SCEV adding together mixed
5940	// offsets?
5941	assert(F.BaseOffset.isCompatibleImmediate(LF.Offset) &&
5942	"Expanding mismatched offsets\n");
5943	// Expand the immediate portion.
5944	Immediate Offset = F.BaseOffset.addUnsigned(RHS: LF.Offset);
5945	if (Offset.isNonZero()) {
5946	if (LU.Kind == LSRUse::ICmpZero) {
5947	// The other interesting way of "folding" with an ICmpZero is to use a
5948	// negated immediate.
5949	if (!ICmpScaledV) {
5950	// TODO: Avoid implicit trunc?
5951	// See https://github.com/llvm/llvm-project/issues/112510.
5952	ICmpScaledV = ConstantInt::getSigned(
5953	Ty: IntTy, V: -(uint64_t)Offset.getFixedValue(), /ImplicitTrunc=/true);
5954	} else {
5955	Ops.push_back(Elt: SE.getUnknown(V: ICmpScaledV));
5956	ICmpScaledV = ConstantInt::getSigned(Ty: IntTy, V: Offset.getFixedValue(),
5957	/ImplicitTrunc=/true);
5958	}
5959	} else {
5960	// Just add the immediate values. These again are expected to be matched
5961	// as part of the address.
5962	Ops.push_back(Elt: Offset.getUnknownSCEV(SE, Ty: IntTy));
5963	}
5964	}
5965
5966	// Expand the unfolded offset portion.
5967	Immediate UnfoldedOffset = F.UnfoldedOffset;
5968	if (UnfoldedOffset.isNonZero()) {
5969	// Just add the immediate values.
5970	Ops.push_back(Elt: UnfoldedOffset.getUnknownSCEV(SE, Ty: IntTy));
5971	}
5972
5973	// Emit instructions summing all the operands.
5974	const SCEV *FullS = Ops.empty() ?
5975	SE.getConstant(Ty: IntTy, V: `0`) :
5976	SE.getAddExpr(Ops);
5977	Value *FullV = Rewriter.expandCodeFor(SH: FullS, Ty);
5978
5979	// We're done expanding now, so reset the rewriter.
5980	Rewriter.clearPostInc();
5981
5982	// An ICmpZero Formula represents an ICmp which we're handling as a
5983	// comparison against zero. Now that we've expanded an expression for that
5984	// form, update the ICmp's other operand.
5985	if (LU.Kind == LSRUse::ICmpZero) {
5986	ICmpInst *CI = cast<ICmpInst>(Val: LF.UserInst);
5987	if (auto *OperandIsInstr = dyn_cast<Instruction>(Val: CI->getOperand(i_nocapture: `1`)))
5988	DeadInsts.emplace_back(Args&: OperandIsInstr);
5989	assert(!F.BaseGV && "ICmp does not support folding a global value and "
5990	"a scale at the same time!");
5991	if (F.Scale == -`1`) {
5992	if (ICmpScaledV->getType() != OpTy) {
5993	Instruction *Cast = CastInst::Create(
5994	CastInst::getCastOpcode(Val: ICmpScaledV, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false),
5995	S: ICmpScaledV, Ty: OpTy, Name: "tmp", InsertBefore: CI->getIterator());
5996	ICmpScaledV = Cast;
5997	}
5998	CI->setOperand(i_nocapture: `1`, Val_nocapture: ICmpScaledV);
5999	} else {
6000	// A scale of 1 means that the scale has been expanded as part of the
6001	// base regs.
6002	assert((F.Scale == `0` \|\| F.Scale == `1`) &&
6003	"ICmp does not support folding a global value and "
6004	"a scale at the same time!");
6005	// TODO: Avoid implicit trunc?
6006	// See https://github.com/llvm/llvm-project/issues/112510.
6007	Constant *C = ConstantInt::getSigned(Ty: SE.getEffectiveSCEVType(Ty: OpTy),
6008	V: -(uint64_t)Offset.getFixedValue(),
6009	/ImplicitTrunc=/true);
6010	if (C->getType() != OpTy) {
6011	C = ConstantFoldCastOperand(
6012	Opcode: CastInst::getCastOpcode(Val: C, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false), C, DestTy: OpTy,
6013	DL: CI->getDataLayout());
6014	assert(C && "Cast of ConstantInt should have folded");
6015	}
6016
6017	CI->setOperand(i_nocapture: `1`, Val_nocapture: C);
6018	}
6019	}
6020
6021	return FullV;
6022	}
6023
6024	/// Helper for Rewrite. PHI nodes are special because the use of their operands
6025	/// effectively happens in their predecessor blocks, so the expression may need
6026	/// to be expanded in multiple places.
6027	void LSRInstance::RewriteForPHI(PHINode PN, const* LSRUse &LU,
6028	const LSRFixup &LF, const Formula &F,
6029	SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
6030	DenseMap<BasicBlock , Value > Inserted;
6031
6032	for (unsigned i = `0`, e = PN->getNumIncomingValues(); i != e; ++i)
6033	if (PN->getIncomingValue(i) == LF.OperandValToReplace) {
6034	bool needUpdateFixups = false;
6035	BasicBlock *BB = PN->getIncomingBlock(i);
6036
6037	// If this is a critical edge, split the edge so that we do not insert
6038	// the code on all predecessor/successor paths. We do this unless this
6039	// is the canonical backedge for this loop, which complicates post-inc
6040	// users.
6041	if (e != `1` && BB->getTerminator()->getNumSuccessors() > `1` &&
6042	!isa<IndirectBrInst>(Val: BB->getTerminator()) &&
6043	!isa<CatchSwitchInst>(Val: BB->getTerminator())) {
6044	BasicBlock *Parent = PN->getParent();
6045	Loop *PNLoop = LI.getLoopFor(BB: Parent);
6046	if (!PNLoop \|\| Parent != PNLoop->getHeader()) {
6047	// Split the critical edge.
6048	BasicBlock NewBB = nullptr*;
6049	if (!Parent->isLandingPad()) {
6050	CriticalEdgeSplittingOptions SplitOptions(&DT, &LI, MSSAU);
6051	SplitOptions =
6052	SplitOptions.setMergeIdenticalEdges().setKeepOneInputPHIs();
6053	if (ShouldPreserveLCSSA)
6054	SplitOptions = SplitOptions.setPreserveLCSSA();
6055	NewBB = SplitCriticalEdge(Src: BB, Dst: Parent, Options: SplitOptions);
6056	} else {
6057	SmallVector<BasicBlock *, `2`> NewBBs;
6058	DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
6059	SplitLandingPadPredecessors(OrigBB: Parent, Preds: BB, Suffix: "", Suffix2: "", NewBBs, DTU: &DTU, LI: &LI);
6060	NewBB = NewBBs [`0`];
6061	}
6062	// If NewBB==NULL, then SplitCriticalEdge refused to split because all
6063	// phi predecessors are identical. The simple thing to do is skip
6064	// splitting in this case rather than complicate the API.
6065	if (NewBB) {
6066	// If PN is outside of the loop and BB is in the loop, we want to
6067	// move the block to be immediately before the PHI block, not
6068	// immediately after BB.
6069	if (L->contains(BB) && !L->contains(Inst: PN))
6070	NewBB->moveBefore(MovePos: PN->getParent());
6071
6072	// Splitting the edge can reduce the number of PHI entries we have.
6073	e = PN->getNumIncomingValues();
6074	BB = NewBB;
6075	i = PN->getBasicBlockIndex(BB);
6076
6077	needUpdateFixups = true;
6078	}
6079	}
6080	}
6081
6082	std::pair<DenseMap<BasicBlock , Value >::iterator, bool> Pair =
6083	Inserted.try_emplace(Key: BB);
6084	if (!Pair.second)
6085	PN->setIncomingValue(i, V: Pair.first ->second);
6086	else {
6087	Value *FullV =
6088	Expand(LU, LF, F, IP: BB->getTerminator()->getIterator(), DeadInsts);
6089
6090	// If this is reuse-by-noop-cast, insert the noop cast.
6091	Type *OpTy = LF.OperandValToReplace->getType();
6092	if (FullV->getType() != OpTy)
6093	FullV = CastInst::Create(
6094	CastInst::getCastOpcode(Val: FullV, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false), S: FullV,
6095	Ty: LF.OperandValToReplace->getType(), Name: "tmp",
6096	InsertBefore: BB->getTerminator()->getIterator());
6097
6098	// If the incoming block for this value is not in the loop, it means the
6099	// current PHI is not in a loop exit, so we must create a LCSSA PHI for
6100	// the inserted value.
6101	if (auto *I = dyn_cast<Instruction>(Val: FullV))
6102	if (L->contains(Inst: I) && !L->contains(BB))
6103	InsertedNonLCSSAInsts.insert(X: I);
6104
6105	PN->setIncomingValue(i, V: FullV);
6106	Pair.first ->second = FullV;
6107	}
6108
6109	// If LSR splits critical edge and phi node has other pending
6110	// fixup operands, we need to update those pending fixups. Otherwise
6111	// formulae will not be implemented completely and some instructions
6112	// will not be eliminated.
6113	if (needUpdateFixups) {
6114	for (LSRUse &LU : Uses)
6115	for (LSRFixup &Fixup : LU.Fixups)
6116	// If fixup is supposed to rewrite some operand in the phi
6117	// that was just updated, it may be already moved to
6118	// another phi node. Such fixup requires update.
6119	if (Fixup.UserInst == PN) {
6120	// Check if the operand we try to replace still exists in the
6121	// original phi.
6122	bool foundInOriginalPHI = false;
6123	for (const auto &val : PN->incoming_values())
6124	if (val == Fixup.OperandValToReplace) {
6125	foundInOriginalPHI = true;
6126	break;
6127	}
6128
6129	// If fixup operand found in original PHI - nothing to do.
6130	if (foundInOriginalPHI)
6131	continue;
6132
6133	// Otherwise it might be moved to another PHI and requires update.
6134	// If fixup operand not found in any of the incoming blocks that
6135	// means we have already rewritten it - nothing to do.
6136	for (const auto &Block : PN->blocks())
6137	for (BasicBlock::iterator I = Block->begin(); isa<PHINode>(Val: I);
6138	++I) {
6139	PHINode *NewPN = cast<PHINode>(Val&: I);
6140	for (const auto &val : NewPN->incoming_values())
6141	if (val == Fixup.OperandValToReplace)
6142	Fixup.UserInst = NewPN;
6143	}
6144	}
6145	}
6146	}
6147	}
6148
6149	/// Emit instructions for the leading candidate expression for this LSRUse (this
6150	/// is called "expanding"), and update the UserInst to reference the newly
6151	/// expanded value.
6152	void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF,
6153	const Formula &F,
6154	SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
6155	// First, find an insertion point that dominates UserInst. For PHI nodes,
6156	// find the nearest block which dominates all the relevant uses.
6157	if (PHINode *PN = dyn_cast<PHINode>(Val: LF.UserInst)) {
6158	RewriteForPHI(PN, LU, LF, F, DeadInsts);
6159	} else {
6160	Value *FullV = Expand(LU, LF, F, IP: LF.UserInst->getIterator(), DeadInsts);
6161
6162	// If this is reuse-by-noop-cast, insert the noop cast.
6163	// For ICmpZero with pointer operands, Expand() already set both operands
6164	// in integer domain, so no cast is needed here.
6165	Type *OpTy = LF.OperandValToReplace->getType();
6166	if (FullV->getType() != OpTy &&
6167	!(LU.Kind == LSRUse::ICmpZero && OpTy->isPointerTy())) {
6168	Instruction *Cast =
6169	CastInst::Create(CastInst::getCastOpcode(Val: FullV, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false),
6170	S: FullV, Ty: OpTy, Name: "tmp", InsertBefore: LF.UserInst->getIterator());
6171	FullV = Cast;
6172	}
6173
6174	// Update the user. ICmpZero is handled specially here (for now) because
6175	// Expand may have updated one of the operands of the icmp already, and
6176	// its new value may happen to be equal to LF.OperandValToReplace, in
6177	// which case doing replaceUsesOfWith leads to replacing both operands
6178	// with the same value. TODO: Reorganize this.
6179	if (LU.Kind == LSRUse::ICmpZero)
6180	LF.UserInst->setOperand(i: `0`, Val: FullV);
6181	else
6182	LF.UserInst->replaceUsesOfWith(From: LF.OperandValToReplace, To: FullV);
6183	}
6184
6185	if (auto *OperandIsInstr = dyn_cast<Instruction>(Val: LF.OperandValToReplace))
6186	DeadInsts.emplace_back(Args&: OperandIsInstr);
6187	}
6188
6189	// Determine where to insert the transformed IV increment instruction for this
6190	// fixup. By default this is the default insert position, but if this is a
6191	// postincrement opportunity then we try to insert it in the same block as the
6192	// fixup user instruction, as this is needed for a postincrement instruction to
6193	// be generated.
6194	static Instruction getFixupInsertPos(const* TargetTransformInfo &TTI,
6195	const LSRFixup &Fixup, const LSRUse &LU,
6196	Instruction *IVIncInsertPos,
6197	DominatorTree &DT) {
6198	// Only address uses can be postincremented
6199	if (LU.Kind != LSRUse::Address)
6200	return IVIncInsertPos;
6201
6202	// Don't try to postincrement if it's not legal
6203	Instruction *I = Fixup.UserInst;
6204	Type *Ty = I->getType();
6205	if (!(isa<LoadInst>(Val: I) && TTI.isIndexedLoadLegal(Mode: TTI.MIM_PostInc, Ty)) &&
6206	!(isa<StoreInst>(Val: I) && TTI.isIndexedStoreLegal(Mode: TTI.MIM_PostInc, Ty)))
6207	return IVIncInsertPos;
6208
6209	// It's only legal to hoist to the user block if it dominates the default
6210	// insert position.
6211	BasicBlock *HoistBlock = I->getParent();
6212	BasicBlock *IVIncBlock = IVIncInsertPos->getParent();
6213	if (!DT.dominates(Def: I, BB: IVIncBlock))
6214	return IVIncInsertPos;
6215
6216	return HoistBlock->getTerminator();
6217	}
6218
6219	/// Rewrite all the fixup locations with new values, following the chosen
6220	/// solution.
6221	void LSRInstance::ImplementSolution(
6222	const SmallVectorImpl<const Formula *> &Solution) {
6223	// Keep track of instructions we may have made dead, so that
6224	// we can remove them after we are done working.
6225	SmallVector<WeakTrackingVH, `16`> DeadInsts;
6226
6227	// Mark phi nodes that terminate chains so the expander tries to reuse them.
6228	for (const IVChain &Chain : IVChainVec) {
6229	if (PHINode *PN = dyn_cast<PHINode>(Val: Chain.tailUserInst()))
6230	Rewriter.setChainedPhi(PN);
6231	}
6232
6233	// Expand the new value definitions and update the users.
6234	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx)
6235	for (const LSRFixup &Fixup : Uses [LUIdx].Fixups) {
6236	Instruction *InsertPos =
6237	getFixupInsertPos(TTI, Fixup, LU: Uses [LUIdx], IVIncInsertPos, DT);
6238	Rewriter.setIVIncInsertPos(L, Pos: InsertPos);
6239	Rewrite(LU: Uses [LUIdx], LF: Fixup, F: *Solution [LUIdx], DeadInsts);
6240	Changed = true;
6241	}
6242
6243	auto InsertedInsts = InsertedNonLCSSAInsts.takeVector();
6244	formLCSSAForInstructions(Worklist&: InsertedInsts, DT, LI, SE: &SE);
6245
6246	for (const IVChain &Chain : IVChainVec) {
6247	GenerateIVChain(Chain, DeadInsts);
6248	Changed = true;
6249	}
6250
6251	for (const WeakVH &IV : Rewriter.getInsertedIVs())
6252	if (IV && dyn_cast<Instruction>(Val: &*IV)->getParent())
6253	ScalarEvolutionIVs.push_back(Elt: IV);
6254
6255	// Clean up after ourselves. This must be done before deleting any
6256	// instructions.
6257	Rewriter.clear();
6258
6259	Changed \|= RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts,
6260	TLI: &TLI, MSSAU);
6261
6262	// In our cost analysis above, we assume that each addrec consumes exactly
6263	// one register, and arrange to have increments inserted just before the
6264	// latch to maximimize the chance this is true. However, if we reused
6265	// existing IVs, we now need to move the increments to match our
6266	// expectations. Otherwise, our cost modeling results in us having a
6267	// chosen a non-optimal result for the actual schedule. (And yes, this
6268	// scheduling decision does impact later codegen.)
6269	for (PHINode &PN : L->getHeader()->phis()) {
6270	BinaryOperator BO = nullptr*;
6271	Value Start = nullptr, Step = nullptr;
6272	if (!matchSimpleRecurrence(P: &PN, BO, Start, Step))
6273	continue;
6274
6275	switch (BO->getOpcode()) {
6276	case Instruction::Sub:
6277	if (BO->getOperand(i_nocapture: `0`) != &PN)
6278	// sub is non-commutative - match handling elsewhere in LSR
6279	continue;
6280	break;
6281	case Instruction::Add:
6282	break;
6283	default:
6284	continue;
6285	};
6286
6287	if (!isa<Constant>(Val: Step))
6288	// If not a constant step, might increase register pressure
6289	// (We assume constants have been canonicalized to RHS)
6290	continue;
6291
6292	if (BO->getParent() == IVIncInsertPos->getParent())
6293	// Only bother moving across blocks. Isel can handle block local case.
6294	continue;
6295
6296	// Can we legally schedule inc at the desired point?
6297	if (!llvm::all_of(Range: BO->uses(),
6298	P: [&](Use &U) {return DT.dominates(Def: IVIncInsertPos, U);}))
6299	continue;
6300	BO->moveBefore(InsertPos: IVIncInsertPos->getIterator());
6301	Changed = true;
6302	}
6303
6304
6305	}
6306
6307	LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
6308	DominatorTree &DT, LoopInfo &LI,
6309	const TargetTransformInfo &TTI, AssumptionCache &AC,
6310	TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU,
6311	bool PreserveLCSSA)
6312	: IU(IU), SE(SE), DT(DT), LI(LI), AC(AC), TLI(TLI), TTI(TTI), L(L),
6313	MSSAU(MSSAU), AMK(PreferredAddresingMode.getNumOccurrences() > `0`
6314	? PreferredAddresingMode
6315	: TTI.getPreferredAddressingMode(L, SE: &SE)),
6316	Rewriter (SE, "lsr", PreserveLCSSA), ShouldPreserveLCSSA(PreserveLCSSA),
6317	BaselineCost (L, SE, TTI, AMK) {
6318	// If LoopSimplify form is not available, stay out of trouble.
6319	if (!L->isLoopSimplifyForm())
6320	return;
6321
6322	// If there's no interesting work to be done, bail early.
6323	if (IU.empty()) return;
6324
6325	// If there's too much analysis to be done, bail early. We won't be able to
6326	// model the problem anyway.
6327	unsigned NumUsers = `0`;
6328	for (const IVStrideUse &U : IU) {
6329	if (++NumUsers > MaxIVUsers) {
6330	(void)U;
6331	LLVM_DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << U
6332	<< "\n");
6333	return;
6334	}
6335	// Bail out if we have a PHI on an EHPad that gets a value from a
6336	// CatchSwitchInst. Because the CatchSwitchInst cannot be split, there is
6337	// no good place to stick any instructions.
6338	if (auto *PN = dyn_cast<PHINode>(Val: U.getUser())) {
6339	auto FirstNonPHI = PN->getParent()->getFirstNonPHIIt();
6340	if (isa<FuncletPadInst>(Val: FirstNonPHI) \|\|
6341	isa<CatchSwitchInst>(Val: FirstNonPHI))
6342	for (BasicBlock *PredBB : PN->blocks())
6343	if (isa<CatchSwitchInst>(Val: PredBB->getFirstNonPHIIt()))
6344	return;
6345	}
6346	}
6347
6348	LLVM_DEBUG(dbgs() << "\nLSR on loop ";
6349	L->getHeader()->printAsOperand(dbgs(), /PrintType=/false);
6350	dbgs() << ":\n");
6351
6352	// Check if we expect this loop to use a hardware loop instruction, which will
6353	// be used when calculating the costs of formulas.
6354	HardwareLoopInfo HWLoopInfo(L);
6355	HardwareLoopProfitable =
6356	TTI.isHardwareLoopProfitable(L, SE, AC, LibInfo: &TLI, HWLoopInfo);
6357
6358	// Configure SCEVExpander already now, so the correct mode is used for
6359	// isSafeToExpand() checks.
6360	#if LLVM_ENABLE_ABI_BREAKING_CHECKS
6361	Rewriter.setDebugType(DEBUG_TYPE);
6362	#endif
6363	Rewriter.disableCanonicalMode();
6364	Rewriter.enableLSRMode();
6365
6366	// First, perform some low-level loop optimizations.
6367	OptimizeShadowIV();
6368	OptimizeLoopTermCond();
6369
6370	// If loop preparation eliminates all interesting IV users, bail.
6371	if (IU.empty()) return;
6372
6373	// Skip nested loops until we can model them better with formulae.
6374	if (!L->isInnermost()) {
6375	LLVM_DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n");
6376	return;
6377	}
6378
6379	// Start collecting data and preparing for the solver.
6380	// If number of registers is not the major cost, we cannot benefit from the
6381	// current profitable chain optimization which is based on number of
6382	// registers.
6383	// FIXME: add profitable chain optimization for other kinds major cost, for
6384	// example number of instructions.
6385	if (TTI.isNumRegsMajorCostOfLSR() \|\| StressIVChain)
6386	CollectChains();
6387	CollectInterestingTypesAndFactors();
6388	CollectFixupsAndInitialFormulae();
6389	CollectLoopInvariantFixupsAndFormulae();
6390
6391	if (Uses.empty())
6392	return;
6393
6394	LLVM_DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n";
6395	print_uses(dbgs()));
6396	LLVM_DEBUG(dbgs() << "The baseline solution requires ";
6397	BaselineCost.print(dbgs()); dbgs() << "\n");
6398
6399	// Now use the reuse data to generate a bunch of interesting ways
6400	// to formulate the values needed for the uses.
6401	GenerateAllReuseFormulae();
6402
6403	FilterOutUndesirableDedicatedRegisters();
6404	NarrowSearchSpaceUsingHeuristics();
6405
6406	SmallVector<const Formula *, `8`> Solution;
6407	Solve(Solution);
6408
6409	// Release memory that is no longer needed.
6410	Factors.clear();
6411	Types.clear();
6412	RegUses.clear();
6413
6414	if (Solution.empty())
6415	return;
6416
6417	#ifndef NDEBUG
6418	// Formulae should be legal.
6419	for (const LSRUse &LU : Uses) {
6420	for (const Formula &F : LU.Formulae)
6421	assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
6422	F) && "Illegal formula generated!");
6423	};
6424	#endif
6425
6426	// Now that we've decided what we want, make it so.
6427	ImplementSolution(Solution);
6428	}
6429
6430	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
6431	void LSRInstance::print_factors_and_types(raw_ostream &OS) const {
6432	if (Factors.empty() && Types.empty()) return;
6433
6434	OS << "LSR has identified the following interesting factors and types: ";
6435	ListSeparator LS;
6436
6437	for (int64_t Factor : Factors)
6438	OS << LS << `'*'` << Factor;
6439
6440	for (Type *Ty : Types)
6441	OS << LS << `'('` << *Ty << `')'`;
6442	OS << `'\n'`;
6443	}
6444
6445	void LSRInstance::print_fixups(raw_ostream &OS) const {
6446	OS << "LSR is examining the following fixup sites:\n";
6447	for (const LSRUse &LU : Uses)
6448	for (const LSRFixup &LF : LU.Fixups) {
6449	dbgs() << " ";
6450	LF.print(OS);
6451	OS << `'\n'`;
6452	}
6453	}
6454
6455	void LSRInstance::print_uses(raw_ostream &OS) const {
6456	OS << "LSR is examining the following uses:\n";
6457	for (const LSRUse &LU : Uses) {
6458	dbgs() << " ";
6459	LU.print(OS);
6460	OS << `'\n'`;
6461	for (const Formula &F : LU.Formulae) {
6462	OS << " ";
6463	F.print(OS);
6464	OS << `'\n'`;
6465	}
6466	}
6467	}
6468
6469	void LSRInstance::print(raw_ostream &OS) const {
6470	print_factors_and_types(OS);
6471	print_fixups(OS);
6472	print_uses(OS);
6473	}
6474
6475	LLVM_DUMP_METHOD void LSRInstance::dump() const {
6476	print(errs()); errs() << `'\n'`;
6477	}
6478	#endif
6479
6480	namespace {
6481
6482	class LoopStrengthReduce : public LoopPass {
6483	public:
6484	static char ID; // Pass ID, replacement for typeid
6485
6486	LoopStrengthReduce();
6487
6488	private:
6489	bool runOnLoop(Loop *L, LPPassManager &LPM) override;
6490	void getAnalysisUsage(AnalysisUsage &AU) const override;
6491	};
6492
6493	} // end anonymous namespace
6494
6495	LoopStrengthReduce::LoopStrengthReduce() : LoopPass (ID) {
6496	initializeLoopStrengthReducePass(*PassRegistry::getPassRegistry());
6497	}
6498
6499	void LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const {
6500	// We split critical edges, so we change the CFG. However, we do update
6501	// many analyses if they are around.
6502	AU.addPreservedID(ID&: LoopSimplifyID);
6503
6504	AU.addRequired<LoopInfoWrapperPass>();
6505	AU.addPreserved<LoopInfoWrapperPass>();
6506	AU.addRequiredID(ID&: LoopSimplifyID);
6507	AU.addRequired<DominatorTreeWrapperPass>();
6508	AU.addPreserved<DominatorTreeWrapperPass>();
6509	AU.addRequired<ScalarEvolutionWrapperPass>();
6510	AU.addPreserved<ScalarEvolutionWrapperPass>();
6511	AU.addRequired<AssumptionCacheTracker>();
6512	AU.addRequired<TargetLibraryInfoWrapperPass>();
6513	// Requiring LoopSimplify a second time here prevents IVUsers from running
6514	// twice, since LoopSimplify was invalidated by running ScalarEvolution.
6515	AU.addRequiredID(ID&: LoopSimplifyID);
6516	AU.addRequired<IVUsersWrapperPass>();
6517	AU.addPreserved<IVUsersWrapperPass>();
6518	AU.addRequired<TargetTransformInfoWrapperPass>();
6519	AU.addPreserved<MemorySSAWrapperPass>();
6520	}
6521
6522	namespace {
6523
6524	/// Enables more convenient iteration over a DWARF expression vector.
6525	static iterator_range<llvm::DIExpression::expr_op_iterator>
6526	ToDwarfOpIter(SmallVectorImpl<uint64_t> &Expr) {
6527	llvm::DIExpression::expr_op_iterator Begin =
6528	llvm::DIExpression::expr_op_iterator(Expr.begin());
6529	llvm::DIExpression::expr_op_iterator End =
6530	llvm::DIExpression::expr_op_iterator(Expr.end());
6531	return {Begin, End};
6532	}
6533
6534	struct SCEVDbgValueBuilder {
6535	SCEVDbgValueBuilder() = default;
6536	SCEVDbgValueBuilder(const SCEVDbgValueBuilder &Base) { clone(Base); }
6537
6538	void clone(const SCEVDbgValueBuilder &Base) {
6539	LocationOps = Base.LocationOps;
6540	Expr = Base.Expr;
6541	}
6542
6543	void clear() {
6544	LocationOps.clear();
6545	Expr.clear();
6546	}
6547
6548	/// The DIExpression as we translate the SCEV.
6549	SmallVector<uint64_t, `6`> Expr;
6550	/// The location ops of the DIExpression.
6551	SmallVector<Value *, `2`> LocationOps;
6552
6553	void pushOperator(uint64_t Op) { Expr.push_back(Elt: Op); }
6554	void pushUInt(uint64_t Operand) { Expr.push_back(Elt: Operand); }
6555
6556	/// Add a DW_OP_LLVM_arg to the expression, followed by the index of the value
6557	/// in the set of values referenced by the expression.
6558	void pushLocation(llvm::Value *V) {
6559	Expr.push_back(Elt: llvm::dwarf::DW_OP_LLVM_arg);
6560	auto *It = llvm::find(Range&: LocationOps, Val: V);
6561	unsigned ArgIndex = `0`;
6562	if (It != LocationOps.end()) {
6563	ArgIndex = std::distance(first: LocationOps.begin(), last: It);
6564	} else {
6565	ArgIndex = LocationOps.size();
6566	LocationOps.push_back(Elt: V);
6567	}
6568	Expr.push_back(Elt: ArgIndex);
6569	}
6570
6571	void pushValue(const SCEVUnknown *U) {
6572	llvm::Value *V = cast<SCEVUnknown>(Val: U)->getValue();
6573	pushLocation(V);
6574	}
6575
6576	bool pushConst(const SCEVConstant *C) {
6577	if (C->getAPInt().getSignificantBits() > `64`)
6578	return false;
6579	Expr.push_back(Elt: llvm::dwarf::DW_OP_consts);
6580	Expr.push_back(Elt: C->getAPInt().getSExtValue());
6581	return true;
6582	}
6583
6584	// Iterating the expression as DWARF ops is convenient when updating
6585	// DWARF_OP_LLVM_args.
6586	iterator_range<llvm::DIExpression::expr_op_iterator> expr_ops() {
6587	return ToDwarfOpIter(Expr);
6588	}
6589
6590	/// Several SCEV types are sequences of the same arithmetic operator applied
6591	/// to constants and values that may be extended or truncated.
6592	bool pushArithmeticExpr(const llvm::SCEVCommutativeExpr *CommExpr,
6593	uint64_t DwarfOp) {
6594	assert((isa<llvm::SCEVAddExpr>(CommExpr) \|\| isa<SCEVMulExpr>(CommExpr)) &&
6595	"Expected arithmetic SCEV type");
6596	bool Success = true;
6597	unsigned EmitOperator = `0`;
6598	for (const auto &Op : CommExpr->operands()) {
6599	Success &= pushSCEV(S: Op);
6600
6601	if (EmitOperator >= `1`)
6602	pushOperator(Op: DwarfOp);
6603	++EmitOperator;
6604	}
6605	return Success;
6606	}
6607
6608	// TODO: Identify and omit noop casts.
6609	bool pushCast(const llvm::SCEVCastExpr C, bool* IsSigned) {
6610	const llvm::SCEV *Inner = C->getOperand(i: `0`);
6611	const llvm::Type *Type = C->getType();
6612	uint64_t ToWidth = Type->getIntegerBitWidth();
6613	bool Success = pushSCEV(S: Inner);
6614	uint64_t CastOps[] = {dwarf::DW_OP_LLVM_convert, ToWidth,
6615	IsSigned ? llvm::dwarf::DW_ATE_signed
6616	: llvm::dwarf::DW_ATE_unsigned};
6617	for (const auto &Op : CastOps)
6618	pushOperator(Op);
6619	return Success;
6620	}
6621
6622	// TODO: MinMax - although these haven't been encountered in the test suite.
6623	bool pushSCEV(const llvm::SCEV *S) {
6624	bool Success = true;
6625	if (const SCEVConstant *StartInt = dyn_cast<SCEVConstant>(Val: S)) {
6626	Success &= pushConst(C: StartInt);
6627
6628	} else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Val: S)) {
6629	if (!U->getValue())
6630	return false;
6631	pushLocation(V: U->getValue());
6632
6633	} else if (const SCEVMulExpr *MulRec = dyn_cast<SCEVMulExpr>(Val: S)) {
6634	Success &= pushArithmeticExpr(CommExpr: MulRec, DwarfOp: llvm::dwarf::DW_OP_mul);
6635
6636	} else if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(Val: S)) {
6637	Success &= pushSCEV(S: UDiv->getLHS());
6638	Success &= pushSCEV(S: UDiv->getRHS());
6639	pushOperator(Op: llvm::dwarf::DW_OP_div);
6640
6641	} else if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(Val: S)) {
6642	// Assert if a new and unknown SCEVCastEXpr type is encountered.
6643	assert((isa<SCEVZeroExtendExpr>(Cast) \|\| isa<SCEVTruncateExpr>(Cast) \|\|
6644	isa<SCEVPtrToIntExpr>(Cast) \|\| isa<SCEVPtrToAddrExpr>(Cast) \|\|
6645	isa<SCEVSignExtendExpr>(Cast)) &&
6646	"Unexpected cast type in SCEV.");
6647	Success &= pushCast(C: Cast, IsSigned: (isa<SCEVSignExtendExpr>(Val: Cast)));
6648
6649	} else if (const SCEVAddExpr *AddExpr = dyn_cast<SCEVAddExpr>(Val: S)) {
6650	Success &= pushArithmeticExpr(CommExpr: AddExpr, DwarfOp: llvm::dwarf::DW_OP_plus);
6651
6652	} else if (isa<SCEVAddRecExpr>(Val: S)) {
6653	// Nested SCEVAddRecExpr are generated by nested loops and are currently
6654	// unsupported.
6655	return false;
6656
6657	} else {
6658	return false;
6659	}
6660	return Success;
6661	}
6662
6663	/// Return true if the combination of arithmetic operator and underlying
6664	/// SCEV constant value is an identity function.
6665	bool isIdentityFunction(uint64_t Op, const SCEV *S) {
6666	if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: S)) {
6667	if (C->getAPInt().getSignificantBits() > `64`)
6668	return false;
6669	int64_t I = C->getAPInt().getSExtValue();
6670	switch (Op) {
6671	case llvm::dwarf::DW_OP_plus:
6672	case llvm::dwarf::DW_OP_minus:
6673	return I == `0`;
6674	case llvm::dwarf::DW_OP_mul:
6675	case llvm::dwarf::DW_OP_div:
6676	return I == `1`;
6677	}
6678	}
6679	return false;
6680	}
6681
6682	/// Convert a SCEV of a value to a DIExpression that is pushed onto the
6683	/// builder's expression stack. The stack should already contain an
6684	/// expression for the iteration count, so that it can be multiplied by
6685	/// the stride and added to the start.
6686	/// Components of the expression are omitted if they are an identity function.
6687	/// Chain (non-affine) SCEVs are not supported.
6688	bool SCEVToValueExpr(const llvm::SCEVAddRecExpr &SAR, ScalarEvolution &SE) {
6689	assert(SAR.isAffine() && "Expected affine SCEV");
6690	const SCEV *Start = SAR.getStart();
6691	const SCEV *Stride = SAR.getStepRecurrence(SE);
6692
6693	// Skip pushing arithmetic noops.
6694	if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_mul, S: Stride)) {
6695	if (!pushSCEV(S: Stride))
6696	return false;
6697	pushOperator(Op: llvm::dwarf::DW_OP_mul);
6698	}
6699	if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_plus, S: Start)) {
6700	if (!pushSCEV(S: Start))
6701	return false;
6702	pushOperator(Op: llvm::dwarf::DW_OP_plus);
6703	}
6704	return true;
6705	}
6706
6707	/// Create an expression that is an offset from a value (usually the IV).
6708	void createOffsetExpr(int64_t Offset, Value *OffsetValue) {
6709	pushLocation(V: OffsetValue);
6710	DIExpression::appendOffset(Ops&: Expr, Offset);
6711	LLVM_DEBUG(
6712	dbgs() << "scev-salvage: Generated IV offset expression. Offset: "
6713	<< std::to_string(Offset) << "\n");
6714	}
6715
6716	/// Combine a translation of the SCEV and the IV to create an expression that
6717	/// recovers a location's value.
6718	/// returns true if an expression was created.
6719	bool createIterCountExpr(const SCEV *S,
6720	const SCEVDbgValueBuilder &IterationCount,
6721	ScalarEvolution &SE) {
6722	// SCEVs for SSA values are most frquently of the form
6723	// {start,+,stride}, but sometimes they are ({start,+,stride} + %a + ..).
6724	// This is because %a is a PHI node that is not the IV. However, these
6725	// SCEVs have not been observed to result in debuginfo-lossy optimisations,
6726	// so its not expected this point will be reached.
6727	if (!isa<SCEVAddRecExpr>(Val: S))
6728	return false;
6729
6730	LLVM_DEBUG(dbgs() << "scev-salvage: Location to salvage SCEV: " << *S
6731	<< `'\n'`);
6732
6733	const auto *Rec = cast<SCEVAddRecExpr>(Val: S);
6734	if (!Rec->isAffine())
6735	return false;
6736
6737	if (S->getExpressionSize() > MaxSCEVSalvageExpressionSize)
6738	return false;
6739
6740	// Initialise a new builder with the iteration count expression. In
6741	// combination with the value's SCEV this enables recovery.
6742	clone(Base: IterationCount);
6743	if (!SCEVToValueExpr(SAR: *Rec, SE))
6744	return false;
6745
6746	return true;
6747	}
6748
6749	/// Convert a SCEV of a value to a DIExpression that is pushed onto the
6750	/// builder's expression stack. The stack should already contain an
6751	/// expression for the iteration count, so that it can be multiplied by
6752	/// the stride and added to the start.
6753	/// Components of the expression are omitted if they are an identity function.
6754	bool SCEVToIterCountExpr(const llvm::SCEVAddRecExpr &SAR,
6755	ScalarEvolution &SE) {
6756	assert(SAR.isAffine() && "Expected affine SCEV");
6757	const SCEV *Start = SAR.getStart();
6758	const SCEV *Stride = SAR.getStepRecurrence(SE);
6759
6760	// Skip pushing arithmetic noops.
6761	if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_minus, S: Start)) {
6762	if (!pushSCEV(S: Start))
6763	return false;
6764	pushOperator(Op: llvm::dwarf::DW_OP_minus);
6765	}
6766	if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_div, S: Stride)) {
6767	if (!pushSCEV(S: Stride))
6768	return false;
6769	pushOperator(Op: llvm::dwarf::DW_OP_div);
6770	}
6771	return true;
6772	}
6773
6774	// Append the current expression and locations to a location list and an
6775	// expression list. Modify the DW_OP_LLVM_arg indexes to account for
6776	// the locations already present in the destination list.
6777	void appendToVectors(SmallVectorImpl<uint64_t> &DestExpr,
6778	SmallVectorImpl<Value *> &DestLocations) {
6779	assert(!DestLocations.empty() &&
6780	"Expected the locations vector to contain the IV");
6781	// The DWARF_OP_LLVM_arg arguments of the expression being appended must be
6782	// modified to account for the locations already in the destination vector.
6783	// All builders contain the IV as the first location op.
6784	assert(!LocationOps.empty() &&
6785	"Expected the location ops to contain the IV.");
6786	// DestIndexMap[n] contains the index in DestLocations for the nth
6787	// location in this SCEVDbgValueBuilder.
6788	SmallVector<uint64_t, `2`> DestIndexMap;
6789	for (const auto &Op : LocationOps) {
6790	auto It = find(Range&: DestLocations, Val: Op);
6791	if (It != DestLocations.end()) {
6792	// Location already exists in DestLocations, reuse existing ArgIndex.
6793	DestIndexMap.push_back(Elt: std::distance(first: DestLocations.begin(), last: It));
6794	continue;
6795	}
6796	// Location is not in DestLocations, add it.
6797	DestIndexMap.push_back(Elt: DestLocations.size());
6798	DestLocations.push_back(Elt: Op);
6799	}
6800
6801	for (const auto &Op : expr_ops()) {
6802	if (Op.getOp() != dwarf::DW_OP_LLVM_arg) {
6803	Op.appendToVector(V&: DestExpr);
6804	continue;
6805	}
6806
6807	DestExpr.push_back(Elt: dwarf::DW_OP_LLVM_arg);
6808	// `DW_OP_LLVM_arg n` represents the nth LocationOp in this SCEV,
6809	// DestIndexMap[n] contains its new index in DestLocations.
6810	uint64_t NewIndex = DestIndexMap [Op.getArg(I: `0`)];
6811	DestExpr.push_back(Elt: NewIndex);
6812	}
6813	}
6814	};
6815
6816	/// Holds all the required data to salvage a dbg.value using the pre-LSR SCEVs
6817	/// and DIExpression.
6818	struct DVIRecoveryRec {
6819	DVIRecoveryRec(DbgVariableRecord *DVR)
6820	: DbgRef(DVR), Expr(DVR->getExpression()), HadLocationArgList(false) {}
6821
6822	DbgVariableRecord *DbgRef;
6823	DIExpression *Expr;
6824	bool HadLocationArgList;
6825	SmallVector<WeakVH, `2`> LocationOps;
6826	SmallVector<const llvm::SCEV *, `2`> SCEVs;
6827	SmallVector<std::unique_ptr<SCEVDbgValueBuilder>, `2`> RecoveryExprs;
6828
6829	void clear() {
6830	for (auto &RE : RecoveryExprs)
6831	RE.reset();
6832	RecoveryExprs.clear();
6833	}
6834
6835	~DVIRecoveryRec() { clear(); }
6836	};
6837	} // namespace
6838
6839	/// Returns the total number of DW_OP_llvm_arg operands in the expression.
6840	/// This helps in determining if a DIArglist is necessary or can be omitted from
6841	/// the dbg.value.
6842	static unsigned numLLVMArgOps(SmallVectorImpl<uint64_t> &Expr) {
6843	auto expr_ops = ToDwarfOpIter(Expr);
6844	unsigned Count = `0`;
6845	for (auto Op : expr_ops)
6846	if (Op.getOp() == dwarf::DW_OP_LLVM_arg)
6847	Count++;
6848	return Count;
6849	}
6850
6851	/// Overwrites DVI with the location and Ops as the DIExpression. This will
6852	/// create an invalid expression if Ops has any dwarf::DW_OP_llvm_arg operands,
6853	/// because a DIArglist is not created for the first argument of the dbg.value.
6854	template <typename T>
6855	static void updateDVIWithLocation(T &DbgVal, Value *Location,
6856	SmallVectorImpl<uint64_t> &Ops) {
6857	assert(numLLVMArgOps(Ops) == `0` && "Expected expression that does not "
6858	"contain any DW_OP_llvm_arg operands.");
6859	DbgVal.setRawLocation(ValueAsMetadata::get(V: Location));
6860	DbgVal.setExpression(DIExpression::get(Context&: DbgVal.getContext(), Elements: Ops));
6861	}
6862
6863	/// Overwrite DVI with locations placed into a DIArglist.
6864	template <typename T>
6865	static void updateDVIWithLocations(T &DbgVal,
6866	SmallVectorImpl<Value *> &Locations,
6867	SmallVectorImpl<uint64_t> &Ops) {
6868	assert(numLLVMArgOps(Ops) != `0` &&
6869	"Expected expression that references DIArglist locations using "
6870	"DW_OP_llvm_arg operands.");
6871	SmallVector<ValueAsMetadata *, `3`> MetadataLocs;
6872	for (Value *V : Locations)
6873	MetadataLocs.push_back(Elt: ValueAsMetadata::get(V));
6874	auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
6875	DbgVal.setRawLocation(llvm::DIArgList::get(Context&: DbgVal.getContext(), Args: ValArrayRef));
6876	DbgVal.setExpression(DIExpression::get(Context&: DbgVal.getContext(), Elements: Ops));
6877	}
6878
6879	/// Write the new expression and new location ops for the dbg.value. If possible
6880	/// reduce the szie of the dbg.value by omitting DIArglist. This
6881	/// can be omitted if:
6882	/// 1. There is only a single location, refenced by a single DW_OP_llvm_arg.
6883	/// 2. The DW_OP_LLVM_arg is the first operand in the expression.
6884	static void UpdateDbgValue(DVIRecoveryRec &DVIRec,
6885	SmallVectorImpl<Value *> &NewLocationOps,
6886	SmallVectorImpl<uint64_t> &NewExpr) {
6887	DbgVariableRecord *DbgVal = DVIRec.DbgRef;
6888	unsigned NumLLVMArgs = numLLVMArgOps(Expr&: NewExpr);
6889	if (NumLLVMArgs == `0`) {
6890	// Location assumed to be on the stack.
6891	updateDVIWithLocation(DbgVal&: *DbgVal, Location: NewLocationOps [`0`], Ops&: NewExpr);
6892	} else if (NumLLVMArgs == `1` && NewExpr [`0`] == dwarf::DW_OP_LLVM_arg) {
6893	// There is only a single DW_OP_llvm_arg at the start of the expression,
6894	// so it can be omitted along with DIArglist.
6895	assert(NewExpr[`1`] == `0` &&
6896	"Lone LLVM_arg in a DIExpression should refer to location-op 0.");
6897	llvm::SmallVector<uint64_t, `6`> ShortenedOps(llvm::drop_begin(RangeOrContainer&: NewExpr, N: `2`));
6898	updateDVIWithLocation(DbgVal&: *DbgVal, Location: NewLocationOps [`0`], Ops&: ShortenedOps);
6899	} else {
6900	// Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
6901	updateDVIWithLocations(DbgVal&: *DbgVal, Locations&: NewLocationOps, Ops&: NewExpr);
6902	}
6903
6904	// If the DIExpression was previously empty then add the stack terminator.
6905	// Non-empty expressions have only had elements inserted into them and so
6906	// the terminator should already be present e.g. stack_value or fragment.
6907	DIExpression *SalvageExpr = DbgVal->getExpression();
6908	if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
6909	SalvageExpr = DIExpression::append(Expr: SalvageExpr, Ops: {dwarf::DW_OP_stack_value});
6910	DbgVal->setExpression(SalvageExpr);
6911	}
6912	}
6913
6914	/// Cached location ops may be erased during LSR, in which case a poison is
6915	/// required when restoring from the cache. The type of that location is no
6916	/// longer available, so just use int8. The poison will be replaced by one or
6917	/// more locations later when a SCEVDbgValueBuilder selects alternative
6918	/// locations to use for the salvage.
6919	static Value *getValueOrPoison(WeakVH &VH, LLVMContext &C) {
6920	return (VH) ? VH : PoisonValue::get(T: llvm::Type::getInt8Ty(C));
6921	}
6922
6923	/// Restore the DVI's pre-LSR arguments. Substitute undef for any erased values.
6924	static void restorePreTransformState(DVIRecoveryRec &DVIRec) {
6925	DbgVariableRecord *DbgVal = DVIRec.DbgRef;
6926	LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
6927	<< "scev-salvage: post-LSR: " << *DbgVal << `'\n'`);
6928	assert(DVIRec.Expr && "Expected an expression");
6929	DbgVal->setExpression(DVIRec.Expr);
6930
6931	// Even a single location-op may be inside a DIArgList and referenced with
6932	// DW_OP_LLVM_arg, which is valid only with a DIArgList.
6933	if (!DVIRec.HadLocationArgList) {
6934	assert(DVIRec.LocationOps.size() == `1` &&
6935	"Unexpected number of location ops.");
6936	// LSR's unsuccessful salvage attempt may have added DIArgList, which in
6937	// this case was not present before, so force the location back to a
6938	// single uncontained Value.
6939	Value *CachedValue =
6940	getValueOrPoison(VH&: DVIRec.LocationOps [`0`], C&: DbgVal->getContext());
6941	DbgVal->setRawLocation(ValueAsMetadata::get(V: CachedValue));
6942	} else {
6943	SmallVector<ValueAsMetadata *, `3`> MetadataLocs;
6944	for (WeakVH VH : DVIRec.LocationOps) {
6945	Value *CachedValue = getValueOrPoison(VH, C&: DbgVal->getContext());
6946	MetadataLocs.push_back(Elt: ValueAsMetadata::get(V: CachedValue));
6947	}
6948	auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
6949	DbgVal->setRawLocation(
6950	llvm::DIArgList::get(Context&: DbgVal->getContext(), Args: ValArrayRef));
6951	}
6952	LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << `'\n'`);
6953	}
6954
6955	static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
6956	llvm::PHINode *LSRInductionVar, DVIRecoveryRec &DVIRec,
6957	const SCEV *SCEVInductionVar,
6958	SCEVDbgValueBuilder IterCountExpr) {
6959
6960	if (!DVIRec.DbgRef->isKillLocation())
6961	return false;
6962
6963	// LSR may have caused several changes to the dbg.value in the failed salvage
6964	// attempt. So restore the DIExpression, the location ops and also the
6965	// location ops format, which is always DIArglist for multiple ops, but only
6966	// sometimes for a single op.
6967	restorePreTransformState(DVIRec);
6968
6969	// LocationOpIndexMap[i] will store the post-LSR location index of
6970	// the non-optimised out location at pre-LSR index i.
6971	SmallVector<int64_t, `2`> LocationOpIndexMap;
6972	LocationOpIndexMap.assign(NumElts: DVIRec.LocationOps.size(), Elt: -`1`);
6973	SmallVector<Value *, `2`> NewLocationOps;
6974	NewLocationOps.push_back(Elt: LSRInductionVar);
6975
6976	for (unsigned i = `0`; i < DVIRec.LocationOps.size(); i++) {
6977	WeakVH VH = DVIRec.LocationOps [i];
6978	// Place the locations not optimised out in the list first, avoiding
6979	// inserts later. The map is used to update the DIExpression's
6980	// DW_OP_LLVM_arg arguments as the expression is updated.
6981	if (VH && !isa<UndefValue>(Val: VH)) {
6982	NewLocationOps.push_back(Elt: VH);
6983	LocationOpIndexMap [i] = NewLocationOps.size() - `1`;
6984	LLVM_DEBUG(dbgs() << "scev-salvage: Location index " << i
6985	<< " now at index " << LocationOpIndexMap[i] << "\n");
6986	continue;
6987	}
6988
6989	// It's possible that a value referred to in the SCEV may have been
6990	// optimised out by LSR.
6991	if (SE.containsErasedValue(S: DVIRec.SCEVs [i]) \|\|
6992	SE.containsUndefs(S: DVIRec.SCEVs [i])) {
6993	LLVM_DEBUG(dbgs() << "scev-salvage: SCEV for location at index: " << i
6994	<< " refers to a location that is now undef or erased. "
6995	"Salvage abandoned.\n");
6996	return false;
6997	}
6998
6999	LLVM_DEBUG(dbgs() << "scev-salvage: salvaging location at index " << i
7000	<< " with SCEV: " << *DVIRec.SCEVs[i] << "\n");
7001
7002	DVIRec.RecoveryExprs [i] = std::make_unique<SCEVDbgValueBuilder>();
7003	SCEVDbgValueBuilder *SalvageExpr = DVIRec.RecoveryExprs [i].get();
7004
7005	// Create an offset-based salvage expression if possible, as it requires
7006	// less DWARF ops than an iteration count-based expression.
7007	if (std::optional<APInt> Offset =
7008	SE.computeConstantDifference(LHS: DVIRec.SCEVs [i], RHS: SCEVInductionVar)) {
7009	if (Offset ->getSignificantBits() <= `64`)
7010	SalvageExpr->createOffsetExpr(Offset: Offset ->getSExtValue(), OffsetValue: LSRInductionVar);
7011	else
7012	return false;
7013	} else if (!SalvageExpr->createIterCountExpr(S: DVIRec.SCEVs [i], IterationCount: IterCountExpr,
7014	SE))
7015	return false;
7016	}
7017
7018	// Merge the DbgValueBuilder generated expressions and the original
7019	// DIExpression, place the result into an new vector.
7020	SmallVector<uint64_t, `3`> NewExpr;
7021	if (DVIRec.Expr->getNumElements() == `0`) {
7022	assert(DVIRec.RecoveryExprs.size() == `1` &&
7023	"Expected only a single recovery expression for an empty "
7024	"DIExpression.");
7025	assert(DVIRec.RecoveryExprs[`0`] &&
7026	"Expected a SCEVDbgSalvageBuilder for location 0");
7027	SCEVDbgValueBuilder *B = DVIRec.RecoveryExprs [`0`].get();
7028	B->appendToVectors(DestExpr&: NewExpr, DestLocations&: NewLocationOps);
7029	}
7030	for (const auto &Op : DVIRec.Expr->expr_ops()) {
7031	// Most Ops needn't be updated.
7032	if (Op.getOp() != dwarf::DW_OP_LLVM_arg) {
7033	Op.appendToVector(V&: NewExpr);
7034	continue;
7035	}
7036
7037	uint64_t LocationArgIndex = Op.getArg(I: `0`);
7038	SCEVDbgValueBuilder *DbgBuilder =
7039	DVIRec.RecoveryExprs [LocationArgIndex].get();
7040	// The location doesn't have s SCEVDbgValueBuilder, so LSR did not
7041	// optimise it away. So just translate the argument to the updated
7042	// location index.
7043	if (!DbgBuilder) {
7044	NewExpr.push_back(Elt: dwarf::DW_OP_LLVM_arg);
7045	assert(LocationOpIndexMap[Op.getArg(`0`)] != -`1` &&
7046	"Expected a positive index for the location-op position.");
7047	NewExpr.push_back(Elt: LocationOpIndexMap [Op.getArg(I: `0`)]);
7048	continue;
7049	}
7050	// The location has a recovery expression.
7051	DbgBuilder->appendToVectors(DestExpr&: NewExpr, DestLocations&: NewLocationOps);
7052	}
7053
7054	UpdateDbgValue(DVIRec, NewLocationOps, NewExpr);
7055	LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: " << *DVIRec.DbgRef << "\n");
7056	return true;
7057	}
7058
7059	/// Obtain an expression for the iteration count, then attempt to salvage the
7060	/// dbg.value intrinsics.
7061	static void DbgRewriteSalvageableDVIs(
7062	llvm::Loop L, ScalarEvolution &SE, llvm::PHINode LSRInductionVar,
7063	SmallVector<std::unique_ptr<DVIRecoveryRec>, `2`> &DVIToUpdate) {
7064	if (DVIToUpdate.empty())
7065	return;
7066
7067	const llvm::SCEV *SCEVInductionVar = SE.getSCEV(V: LSRInductionVar);
7068	assert(SCEVInductionVar &&
7069	"Anticipated a SCEV for the post-LSR induction variable");
7070
7071	if (const SCEVAddRecExpr *IVAddRec =
7072	dyn_cast<SCEVAddRecExpr>(Val: SCEVInductionVar)) {
7073	if (!IVAddRec->isAffine())
7074	return;
7075
7076	// Prevent translation using excessive resources.
7077	if (IVAddRec->getExpressionSize() > MaxSCEVSalvageExpressionSize)
7078	return;
7079
7080	// The iteration count is required to recover location values.
7081	SCEVDbgValueBuilder IterCountExpr;
7082	IterCountExpr.pushLocation(V: LSRInductionVar);
7083	if (!IterCountExpr.SCEVToIterCountExpr(SAR: *IVAddRec, SE))
7084	return;
7085
7086	LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV: " << *SCEVInductionVar
7087	<< `'\n'`);
7088
7089	for (auto &DVIRec : DVIToUpdate) {
7090	SalvageDVI(L, SE, LSRInductionVar, DVIRec&: *DVIRec, SCEVInductionVar,
7091	IterCountExpr);
7092	}
7093	}
7094	}
7095
7096	/// Identify and cache salvageable DVI locations and expressions along with the
7097	/// corresponding SCEV(s). Also ensure that the DVI is not deleted between
7098	/// cacheing and salvaging.
7099	static void DbgGatherSalvagableDVI(
7100	Loop *L, ScalarEvolution &SE,
7101	SmallVector<std::unique_ptr<DVIRecoveryRec>, `2`> &SalvageableDVISCEVs) {
7102	for (const auto &B : L->getBlocks()) {
7103	for (auto &I : *B) {
7104	for (DbgVariableRecord &DbgVal : filterDbgVars(R: I.getDbgRecordRange())) {
7105	if (!DbgVal.isDbgValue() && !DbgVal.isDbgAssign())
7106	continue;
7107
7108	// Ensure that if any location op is undef that the dbg.vlue is not
7109	// cached.
7110	if (DbgVal.isKillLocation())
7111	continue;
7112
7113	// Check that the location op SCEVs are suitable for translation to
7114	// DIExpression.
7115	const auto &HasTranslatableLocationOps =
7116	[&](const DbgVariableRecord &DbgValToTranslate) -> bool {
7117	for (const auto LocOp : DbgValToTranslate.location_ops()) {
7118	if (!LocOp)
7119	return false;
7120
7121	if (!SE.isSCEVable(Ty: LocOp->getType()))
7122	return false;
7123
7124	const SCEV *S = SE.getSCEV(V: LocOp);
7125	if (SE.containsUndefs(S))
7126	return false;
7127	}
7128	return true;
7129	};
7130
7131	if (!HasTranslatableLocationOps (DbgVal))
7132	continue;
7133
7134	std::unique_ptr<DVIRecoveryRec> NewRec =
7135	std::make_unique<DVIRecoveryRec>(args: &DbgVal);
7136	// Each location Op may need a SCEVDbgValueBuilder in order to recover
7137	// it. Pre-allocating a vector will enable quick lookups of the builder
7138	// later during the salvage.
7139	NewRec ->RecoveryExprs.resize(N: DbgVal.getNumVariableLocationOps());
7140	for (const auto LocOp : DbgVal.location_ops()) {
7141	NewRec ->SCEVs.push_back(Elt: SE.getSCEV(V: LocOp));
7142	NewRec ->LocationOps.push_back(Elt: LocOp);
7143	NewRec ->HadLocationArgList = DbgVal.hasArgList();
7144	}
7145	SalvageableDVISCEVs.push_back(Elt: std::move(NewRec));
7146	}
7147	}
7148	}
7149	}
7150
7151	/// Ideally pick the PHI IV inserted by ScalarEvolutionExpander. As a fallback
7152	/// any PHi from the loop header is usable, but may have less chance of
7153	/// surviving subsequent transforms.
7154	static llvm::PHINode GetInductionVariable(const* Loop &L, ScalarEvolution &SE,
7155	const LSRInstance &LSR) {
7156
7157	auto IsSuitableIV = [&](PHINode *P) {
7158	if (!SE.isSCEVable(Ty: P->getType()))
7159	return false;
7160	if (const SCEVAddRecExpr *Rec = dyn_cast<SCEVAddRecExpr>(Val: SE.getSCEV(V: P)))
7161	return Rec->isAffine() && !SE.containsUndefs(S: SE.getSCEV(V: P));
7162	return false;
7163	};
7164
7165	// For now, just pick the first IV that was generated and inserted by
7166	// ScalarEvolution. Ideally pick an IV that is unlikely to be optimised away
7167	// by subsequent transforms.
7168	for (const WeakVH &IV : LSR.getScalarEvolutionIVs()) {
7169	if (!IV)
7170	continue;
7171
7172	// There should only be PHI node IVs.
7173	PHINode P = cast<PHINode>(Val: &IV);
7174
7175	if (IsSuitableIV (P))
7176	return P;
7177	}
7178
7179	for (PHINode &P : L.getHeader()->phis()) {
7180	if (IsSuitableIV (&P))
7181	return &P;
7182	}
7183	return nullptr;
7184	}
7185
7186	static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
7187	DominatorTree &DT, LoopInfo &LI,
7188	const TargetTransformInfo &TTI,
7189	AssumptionCache &AC, TargetLibraryInfo &TLI,
7190	MemorySSA MSSA, bool* PreserveLCSSA) {
7191
7192	// Debug preservation - before we start removing anything identify which DVI
7193	// meet the salvageable criteria and store their DIExpression and SCEVs.
7194	SmallVector<std::unique_ptr<DVIRecoveryRec>, `2`> SalvageableDVIRecords;
7195	DbgGatherSalvagableDVI(L, SE, SalvageableDVISCEVs&: SalvageableDVIRecords);
7196
7197	bool Changed = false;
7198	std::unique_ptr<MemorySSAUpdater> MSSAU;
7199	if (MSSA)
7200	MSSAU = std::make_unique<MemorySSAUpdater>(args&: MSSA);
7201
7202	// Run the main LSR transformation.
7203	const LSRInstance &Reducer =
7204	LSRInstance (L, IU, SE, DT, LI, TTI, AC, TLI, MSSAU.get(), PreserveLCSSA);
7205	Changed \|= Reducer.getChanged();
7206
7207	// Remove any extra phis created by processing inner loops.
7208	Changed \|= DeleteDeadPHIs(BB: L->getHeader(), TLI: &TLI, MSSAU: MSSAU.get());
7209	if (EnablePhiElim && L->isLoopSimplifyForm()) {
7210	SmallVector<WeakTrackingVH, `16`> DeadInsts;
7211	SCEVExpander Rewriter(SE, "lsr", false);
7212	#if LLVM_ENABLE_ABI_BREAKING_CHECKS
7213	Rewriter.setDebugType(DEBUG_TYPE);
7214	#endif
7215	unsigned numFolded = Rewriter.replaceCongruentIVs(L, DT: &DT, DeadInsts, TTI: &TTI);
7216	Rewriter.clear();
7217	if (numFolded) {
7218	Changed = true;
7219	RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts, TLI: &TLI,
7220	MSSAU: MSSAU.get());
7221	DeleteDeadPHIs(BB: L->getHeader(), TLI: &TLI, MSSAU: MSSAU.get());
7222	}
7223	}
7224	// LSR may at times remove all uses of an induction variable from a loop.
7225	// The only remaining use is the PHI in the exit block.
7226	// When this is the case, if the exit value of the IV can be calculated using
7227	// SCEV, we can replace the exit block PHI with the final value of the IV and
7228	// skip the updates in each loop iteration.
7229	if (L->isRecursivelyLCSSAForm(DT, LI) && L->getExitBlock()) {
7230	SmallVector<WeakTrackingVH, `16`> DeadInsts;
7231	SCEVExpander Rewriter(SE, "lsr", true);
7232	int Rewrites = rewriteLoopExitValues(L, LI: &LI, TLI: &TLI, SE: &SE, TTI: &TTI, Rewriter, DT: &DT,
7233	ReplaceExitValue: UnusedIndVarInLoop, DeadInsts);
7234	Rewriter.clear();
7235	if (Rewrites) {
7236	Changed = true;
7237	RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts, TLI: &TLI,
7238	MSSAU: MSSAU.get());
7239	DeleteDeadPHIs(BB: L->getHeader(), TLI: &TLI, MSSAU: MSSAU.get());
7240	}
7241	}
7242
7243	if (SalvageableDVIRecords.empty())
7244	return Changed;
7245
7246	// Obtain relevant IVs and attempt to rewrite the salvageable DVIs with
7247	// expressions composed using the derived iteration count.
7248	// TODO: Allow for multiple IV references for nested AddRecSCEVs
7249	for (const auto &L : LI) {
7250	if (llvm::PHINode IV = GetInductionVariable(L: L, SE, LSR: Reducer))
7251	DbgRewriteSalvageableDVIs(L, SE, LSRInductionVar: IV, DVIToUpdate&: SalvageableDVIRecords);
7252	else {
7253	LLVM_DEBUG(dbgs() << "scev-salvage: SCEV salvaging not possible. An IV "
7254	"could not be identified.\n");
7255	}
7256	}
7257
7258	for (auto &Rec : SalvageableDVIRecords)
7259	Rec ->clear();
7260	SalvageableDVIRecords.clear();
7261	return Changed;
7262	}
7263
7264	bool LoopStrengthReduce::runOnLoop(Loop L, LPPassManager & /LPM/*) {
7265	if (skipLoop(L))
7266	return false;
7267
7268	auto &IU = getAnalysis<IVUsersWrapperPass>().getIU();
7269	auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
7270	auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
7271	auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
7272	const auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
7273	F: *L->getHeader()->getParent());
7274	auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
7275	F&: *L->getHeader()->getParent());
7276	auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
7277	F: *L->getHeader()->getParent());
7278	auto *MSSAAnalysis = getAnalysisIfAvailable<MemorySSAWrapperPass>();
7279	MemorySSA MSSA = nullptr*;
7280	if (MSSAAnalysis)
7281	MSSA = &MSSAAnalysis->getMSSA();
7282	return ReduceLoopStrength(L, IU, SE, DT, LI, TTI, AC, TLI, MSSA,
7283	/PreserveLCSSA=/false);
7284	}
7285
7286	PreservedAnalyses LoopStrengthReducePass::run(Loop &L, LoopAnalysisManager &AM,
7287	LoopStandardAnalysisResults &AR,
7288	LPMUpdater &) {
7289	if (!ReduceLoopStrength(L: &L, IU&: AM.getResult<IVUsersAnalysis>(IR&: L, ExtraArgs&: AR), SE&: AR.SE,
7290	DT&: AR.DT, LI&: AR.LI, TTI: AR.TTI, AC&: AR.AC, TLI&: AR.TLI, MSSA: AR.MSSA,
7291	/PreserveLCSSA=/true))
7292	return PreservedAnalyses::all();
7293
7294	auto PA = getLoopPassPreservedAnalyses();
7295	if (AR.MSSA)
7296	PA.preserve<MemorySSAAnalysis>();
7297	return PA;
7298	}
7299
7300	char LoopStrengthReduce::ID = `0`;
7301
7302	INITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce",
7303	"Loop Strength Reduction", false, false)
7304	INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
7305	INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
7306	INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
7307	INITIALIZE_PASS_DEPENDENCY(IVUsersWrapperPass)
7308	INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
7309	INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
7310	INITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce",
7311	"Loop Strength Reduction", false, false)
7312
7313	Pass llvm::createLoopStrengthReducePass() { return* new LoopStrengthReduce (); }
7314

Browse the source code of llvm_projects/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp