LoopStrengthReduce.cpp source code [llvm_projects/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp]

1	//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===//
2	//
3	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4	// See https://llvm.org/LICENSE.txt for license information.
5	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6	//
7	//===----------------------------------------------------------------------===//
8	//
9	// This transformation analyzes and transforms the induction variables (and
10	// computations derived from them) into forms suitable for efficient execution
11	// on the target.
12	//
13	// This pass performs a strength reduction on array references inside loops that
14	// have as one or more of their components the loop induction variable, it
15	// rewrites expressions to take advantage of scaled-index addressing modes
16	// available on the target, and it performs a variety of other optimizations
17	// related to loop induction variables.
18	//
19	// Terminology note: this code has a lot of handling for "post-increment" or
20	// "post-inc" users. This is not talking about post-increment addressing modes;
21	// it is instead talking about code like this:
22	//
23	// %i = phi [ 0, %entry ], [ %i.next, %latch ]
24	// ...
25	// %i.next = add %i, 1
26	// %c = icmp eq %i.next, %n
27	//
28	// The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however
29	// it's useful to think about these as the same register, with some uses using
30	// the value of the register before the add and some using it after. In this
31	// example, the icmp is a post-increment user, since it uses %i.next, which is
32	// the value of the induction variable after the increment. The other common
33	// case of post-increment users is users outside the loop.
34	//
35	// TODO: More sophistication in the way Formulae are generated and filtered.
36	//
37	// TODO: Handle multiple loops at a time.
38	//
39	// TODO: Should the addressing mode BaseGV be changed to a ConstantExpr instead
40	// of a GlobalValue?
41	//
42	// TODO: When truncation is free, truncate ICmp users' operands to make it a
43	// smaller encoding (on x86 at least).
44	//
45	// TODO: When a negated register is used by an add (such as in a list of
46	// multiple base registers, or as the increment expression in an addrec),
47	// we may not actually need both reg and (-1 reg) in registers; the*
48	// negation can be implemented by using a sub instead of an add. The
49	// lack of support for taking this into consideration when making
50	// register pressure decisions is partly worked around by the "Special"
51	// use kind.
52	//
53	//===----------------------------------------------------------------------===//
54
55	#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
56	#include "llvm/ADT/APInt.h"
57	#include "llvm/ADT/DenseMap.h"
58	#include "llvm/ADT/DenseSet.h"
59	#include "llvm/ADT/PointerIntPair.h"
60	#include "llvm/ADT/STLExtras.h"
61	#include "llvm/ADT/SetVector.h"
62	#include "llvm/ADT/SmallBitVector.h"
63	#include "llvm/ADT/SmallPtrSet.h"
64	#include "llvm/ADT/SmallSet.h"
65	#include "llvm/ADT/SmallVector.h"
66	#include "llvm/ADT/Statistic.h"
67	#include "llvm/ADT/iterator_range.h"
68	#include "llvm/Analysis/AssumptionCache.h"
69	#include "llvm/Analysis/DomTreeUpdater.h"
70	#include "llvm/Analysis/IVUsers.h"
71	#include "llvm/Analysis/LoopAnalysisManager.h"
72	#include "llvm/Analysis/LoopInfo.h"
73	#include "llvm/Analysis/LoopPass.h"
74	#include "llvm/Analysis/MemorySSA.h"
75	#include "llvm/Analysis/MemorySSAUpdater.h"
76	#include "llvm/Analysis/ScalarEvolution.h"
77	#include "llvm/Analysis/ScalarEvolutionExpressions.h"
78	#include "llvm/Analysis/ScalarEvolutionNormalization.h"
79	#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
80	#include "llvm/Analysis/TargetLibraryInfo.h"
81	#include "llvm/Analysis/TargetTransformInfo.h"
82	#include "llvm/Analysis/ValueTracking.h"
83	#include "llvm/BinaryFormat/Dwarf.h"
84	#include "llvm/IR/BasicBlock.h"
85	#include "llvm/IR/Constant.h"
86	#include "llvm/IR/Constants.h"
87	#include "llvm/IR/DebugInfoMetadata.h"
88	#include "llvm/IR/DerivedTypes.h"
89	#include "llvm/IR/Dominators.h"
90	#include "llvm/IR/GlobalValue.h"
91	#include "llvm/IR/IRBuilder.h"
92	#include "llvm/IR/InstrTypes.h"
93	#include "llvm/IR/Instruction.h"
94	#include "llvm/IR/Instructions.h"
95	#include "llvm/IR/IntrinsicInst.h"
96	#include "llvm/IR/Module.h"
97	#include "llvm/IR/Operator.h"
98	#include "llvm/IR/Type.h"
99	#include "llvm/IR/Use.h"
100	#include "llvm/IR/User.h"
101	#include "llvm/IR/Value.h"
102	#include "llvm/IR/ValueHandle.h"
103	#include "llvm/InitializePasses.h"
104	#include "llvm/Pass.h"
105	#include "llvm/Support/Casting.h"
106	#include "llvm/Support/CommandLine.h"
107	#include "llvm/Support/Compiler.h"
108	#include "llvm/Support/Debug.h"
109	#include "llvm/Support/ErrorHandling.h"
110	#include "llvm/Support/MathExtras.h"
111	#include "llvm/Support/raw_ostream.h"
112	#include "llvm/Transforms/Scalar.h"
113	#include "llvm/Transforms/Utils.h"
114	#include "llvm/Transforms/Utils/BasicBlockUtils.h"
115	#include "llvm/Transforms/Utils/Local.h"
116	#include "llvm/Transforms/Utils/LoopUtils.h"
117	#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
118	#include <algorithm>
119	#include <cassert>
120	#include <cstddef>
121	#include <cstdint>
122	#include <iterator>
123	#include <limits>
124	#include <map>
125	#include <numeric>
126	#include <optional>
127	#include <utility>
128
129	using namespace llvm;
130	using namespace SCEVPatternMatch;
131
132	#define DEBUG_TYPE "loop-reduce"
133
134	/// MaxIVUsers is an arbitrary threshold that provides an early opportunity for
135	/// bail out. This threshold is far beyond the number of users that LSR can
136	/// conceivably solve, so it should not affect generated code, but catches the
137	/// worst cases before LSR burns too much compile time and stack space.
138	static const unsigned MaxIVUsers = `200`;
139
140	/// Limit the size of expression that SCEV-based salvaging will attempt to
141	/// translate into a DIExpression.
142	/// Choose a maximum size such that debuginfo is not excessively increased and
143	/// the salvaging is not too expensive for the compiler.
144	static const unsigned MaxSCEVSalvageExpressionSize = `64`;
145
146	// Cleanup congruent phis after LSR phi expansion.
147	static cl::opt<bool> EnablePhiElim(
148	"enable-lsr-phielim", cl::Hidden, cl::init(Val: true),
149	cl::desc("Enable LSR phi elimination"));
150
151	// The flag adds instruction count to solutions cost comparison.
152	static cl::opt<bool> InsnsCost(
153	"lsr-insns-cost", cl::Hidden, cl::init(Val: true),
154	cl::desc("Add instruction count to a LSR cost model"));
155
156	// Flag to choose how to narrow complex lsr solution
157	static cl::opt<bool> LSRExpNarrow(
158	"lsr-exp-narrow", cl::Hidden, cl::init(Val: false),
159	cl::desc("Narrow LSR complex solution using"
160	" expectation of registers number"));
161
162	// Flag to narrow search space by filtering non-optimal formulae with
163	// the same ScaledReg and Scale.
164	static cl::opt<bool> FilterSameScaledReg(
165	"lsr-filter-same-scaled-reg", cl::Hidden, cl::init(Val: true),
166	cl::desc("Narrow LSR search space by filtering non-optimal formulae"
167	" with the same ScaledReg and Scale"));
168
169	static cl::opt<TTI::AddressingModeKind> PreferredAddresingMode(
170	"lsr-preferred-addressing-mode", cl::Hidden, cl::init(Val: TTI::AMK_None),
171	cl::desc("A flag that overrides the target's preferred addressing mode."),
172	cl::values(
173	clEnumValN(TTI::AMK_None, "none", "Don't prefer any addressing mode"),
174	clEnumValN(TTI::AMK_PreIndexed, "preindexed",
175	"Prefer pre-indexed addressing mode"),
176	clEnumValN(TTI::AMK_PostIndexed, "postindexed",
177	"Prefer post-indexed addressing mode"),
178	clEnumValN(TTI::AMK_All, "all", "Consider all addressing modes")));
179
180	static cl::opt<unsigned> ComplexityLimit(
181	"lsr-complexity-limit", cl::Hidden,
182	cl::init(Val: std::numeric_limits<uint16_t>::max()),
183	cl::desc("LSR search space complexity limit"));
184
185	static cl::opt<unsigned> SetupCostDepthLimit(
186	"lsr-setupcost-depth-limit", cl::Hidden, cl::init(Val: `7`),
187	cl::desc("The limit on recursion depth for LSRs setup cost"));
188
189	static cl::opt<cl::boolOrDefault> AllowDropSolutionIfLessProfitable(
190	"lsr-drop-solution", cl::Hidden,
191	cl::desc("Attempt to drop solution if it is less profitable"));
192
193	static cl::opt<bool> EnableVScaleImmediates(
194	"lsr-enable-vscale-immediates", cl::Hidden, cl::init(Val: true),
195	cl::desc("Enable analysis of vscale-relative immediates in LSR"));
196
197	static cl::opt<bool> DropScaledForVScale(
198	"lsr-drop-scaled-reg-for-vscale", cl::Hidden, cl::init(Val: true),
199	cl::desc("Avoid using scaled registers with vscale-relative addressing"));
200
201	#ifndef NDEBUG
202	// Stress test IV chain generation.
203	static cl::opt<bool> StressIVChain(
204	"stress-ivchain", cl::Hidden, cl::init(false),
205	cl::desc("Stress test LSR IV chains"));
206	#else
207	static bool StressIVChain = false;
208	#endif
209
210	namespace {
211
212	struct MemAccessTy {
213	/// Used in situations where the accessed memory type is unknown.
214	static const unsigned UnknownAddressSpace =
215	std::numeric_limits<unsigned>::max();
216
217	Type MemTy = nullptr*;
218	unsigned AddrSpace = UnknownAddressSpace;
219
220	MemAccessTy() = default;
221	MemAccessTy(Type Ty, unsigned* AS) : MemTy(Ty), AddrSpace(AS) {}
222
223	bool operator==(MemAccessTy Other) const {
224	return MemTy == Other.MemTy && AddrSpace == Other.AddrSpace;
225	}
226
227	bool operator!=(MemAccessTy Other) const { return !(*this == Other); }
228
229	static MemAccessTy getUnknown(LLVMContext &Ctx,
230	unsigned AS = UnknownAddressSpace) {
231	return MemAccessTy (Type::getVoidTy(C&: Ctx), AS);
232	}
233
234	Type getType() { return* MemTy; }
235	};
236
237	/// This class holds data which is used to order reuse candidates.
238	class RegSortData {
239	public:
240	/// This represents the set of LSRUse indices which reference
241	/// a particular register.
242	SmallBitVector UsedByIndices;
243
244	void print(raw_ostream &OS) const;
245	void dump() const;
246	};
247
248	// An offset from an address that is either scalable or fixed. Used for
249	// per-target optimizations of addressing modes.
250	class Immediate : public details::FixedOrScalableQuantity<Immediate, int64_t> {
251	constexpr Immediate(ScalarTy MinVal, bool Scalable)
252	: FixedOrScalableQuantity (MinVal, Scalable) {}
253
254	constexpr Immediate(const FixedOrScalableQuantity<Immediate, int64_t> &V)
255	: FixedOrScalableQuantity (V) {}
256
257	public:
258	constexpr Immediate() = delete;
259
260	static constexpr Immediate getFixed(ScalarTy MinVal) {
261	return {MinVal, false};
262	}
263	static constexpr Immediate getScalable(ScalarTy MinVal) {
264	return {MinVal, true};
265	}
266	static constexpr Immediate get(ScalarTy MinVal, bool Scalable) {
267	return {MinVal, Scalable};
268	}
269	static constexpr Immediate getZero() { return {`0`, false}; }
270	static constexpr Immediate getFixedMin() {
271	return {std::numeric_limits<int64_t>::min(), false};
272	}
273	static constexpr Immediate getFixedMax() {
274	return {std::numeric_limits<int64_t>::max(), false};
275	}
276	static constexpr Immediate getScalableMin() {
277	return {std::numeric_limits<int64_t>::min(), true};
278	}
279	static constexpr Immediate getScalableMax() {
280	return {std::numeric_limits<int64_t>::max(), true};
281	}
282
283	constexpr bool isLessThanZero() const { return Quantity < `0`; }
284
285	constexpr bool isGreaterThanZero() const { return Quantity > `0`; }
286
287	constexpr bool isCompatibleImmediate(const Immediate &Imm) const {
288	return isZero() \|\| Imm.isZero() \|\| Imm.Scalable == Scalable;
289	}
290
291	constexpr bool isMin() const {
292	return Quantity == std::numeric_limits<ScalarTy>::min();
293	}
294
295	constexpr bool isMax() const {
296	return Quantity == std::numeric_limits<ScalarTy>::max();
297	}
298
299	// Arithmetic 'operators' that cast to unsigned types first.
300	constexpr Immediate addUnsigned(const Immediate &RHS) const {
301	assert(isCompatibleImmediate(RHS) && "Incompatible Immediates");
302	ScalarTy Value = (uint64_t)Quantity + RHS.getKnownMinValue();
303	return {Value, Scalable \|\| RHS.isScalable()};
304	}
305
306	constexpr Immediate subUnsigned(const Immediate &RHS) const {
307	assert(isCompatibleImmediate(RHS) && "Incompatible Immediates");
308	ScalarTy Value = (uint64_t)Quantity - RHS.getKnownMinValue();
309	return {Value, Scalable \|\| RHS.isScalable()};
310	}
311
312	// Scale the quantity by a constant without caring about runtime scalability.
313	constexpr Immediate mulUnsigned(const ScalarTy RHS) const {
314	ScalarTy Value = (uint64_t)Quantity * RHS;
315	return {Value, Scalable};
316	}
317
318	// Helpers for generating SCEVs with vscale terms where needed.
319	const SCEV getSCEV(ScalarEvolution &SE, Type Ty) const {
320	const SCEV *S = SE.getConstant(Ty, V: Quantity);
321	if (Scalable)
322	S = SE.getMulExpr(LHS: S, RHS: SE.getVScale(Ty: S->getType()));
323	return S;
324	}
325
326	const SCEV getNegativeSCEV(ScalarEvolution &SE, Type Ty) const {
327	const SCEV *NegS = SE.getConstant(Ty, V: -(uint64_t)Quantity);
328	if (Scalable)
329	NegS = SE.getMulExpr(LHS: NegS, RHS: SE.getVScale(Ty: NegS->getType()));
330	return NegS;
331	}
332
333	const SCEV getUnknownSCEV(ScalarEvolution &SE, Type Ty) const {
334	// TODO: Avoid implicit trunc?
335	// See https://github.com/llvm/llvm-project/issues/112510.
336	const SCEV *SU = SE.getUnknown(
337	V: ConstantInt::getSigned(Ty, V: Quantity, /ImplicitTrunc=/true));
338	if (Scalable)
339	SU = SE.getMulExpr(LHS: SU, RHS: SE.getVScale(Ty: SU->getType()));
340	return SU;
341	}
342	};
343
344	// This is needed for the Compare type of std::map when Immediate is used
345	// as a key. We don't need it to be fully correct against any value of vscale,
346	// just to make sure that vscale-related terms in the map are considered against
347	// each other rather than being mixed up and potentially missing opportunities.
348	struct KeyOrderTargetImmediate {
349	bool operator()(const Immediate &LHS, const Immediate &RHS) const {
350	if (LHS.isScalable() && !RHS.isScalable())
351	return false;
352	if (!LHS.isScalable() && RHS.isScalable())
353	return true;
354	return LHS.getKnownMinValue() < RHS.getKnownMinValue();
355	}
356	};
357
358	// This would be nicer if we could be generic instead of directly using size_t,
359	// but there doesn't seem to be a type trait for is_orderable or
360	// is_lessthan_comparable or similar.
361	struct KeyOrderSizeTAndImmediate {
362	bool operator()(const std::pair<size_t, Immediate> &LHS,
363	const std::pair<size_t, Immediate> &RHS) const {
364	size_t LSize = LHS.first;
365	size_t RSize = RHS.first;
366	if (LSize != RSize)
367	return LSize < RSize;
368	return KeyOrderTargetImmediate ()(LHS.second, RHS.second);
369	}
370	};
371	} // end anonymous namespace
372
373	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
374	void RegSortData::print(raw_ostream &OS) const {
375	OS << "[NumUses=" << UsedByIndices.count() << `']'`;
376	}
377
378	LLVM_DUMP_METHOD void RegSortData::dump() const {
379	print(errs()); errs() << `'\n'`;
380	}
381	#endif
382
383	namespace {
384
385	/// Map register candidates to information about how they are used.
386	class RegUseTracker {
387	using RegUsesTy = DenseMap<const SCEV *, RegSortData>;
388
389	RegUsesTy RegUsesMap;
390	SmallVector<const SCEV *, `16`> RegSequence;
391
392	public:
393	void countRegister(const SCEV *Reg, size_t LUIdx);
394	void dropRegister(const SCEV *Reg, size_t LUIdx);
395	void swapAndDropUse(size_t LUIdx, size_t LastLUIdx);
396
397	bool isRegUsedByUsesOtherThan(const SCEV Reg, size_t LUIdx) const*;
398
399	const SmallBitVector &getUsedByIndices(const SCEV Reg) const*;
400
401	void clear();
402
403	using iterator = SmallVectorImpl<const SCEV *>::iterator;
404	using const_iterator = SmallVectorImpl<const SCEV *>::const_iterator;
405
406	iterator begin() { return RegSequence.begin(); }
407	iterator end() { return RegSequence.end(); }
408	const_iterator begin() const { return RegSequence.begin(); }
409	const_iterator end() const { return RegSequence.end(); }
410	};
411
412	} // end anonymous namespace
413
414	void
415	RegUseTracker::countRegister(const SCEV *Reg, size_t LUIdx) {
416	std::pair<RegUsesTy::iterator, bool> Pair = RegUsesMap.try_emplace(Key: Reg);
417	RegSortData &RSD = Pair.first ->second;
418	if (Pair.second)
419	RegSequence.push_back(Elt: Reg);
420	RSD.UsedByIndices.resize(N: std::max(a: RSD.UsedByIndices.size(), b: LUIdx + `1`));
421	RSD.UsedByIndices.set(LUIdx);
422	}
423
424	void
425	RegUseTracker::dropRegister(const SCEV *Reg, size_t LUIdx) {
426	RegUsesTy::iterator It = RegUsesMap.find(Val: Reg);
427	assert(It != RegUsesMap.end());
428	RegSortData &RSD = It ->second;
429	assert(RSD.UsedByIndices.size() > LUIdx);
430	RSD.UsedByIndices.reset(Idx: LUIdx);
431	}
432
433	void
434	RegUseTracker::swapAndDropUse(size_t LUIdx, size_t LastLUIdx) {
435	assert(LUIdx <= LastLUIdx);
436
437	// Update RegUses. The data structure is not optimized for this purpose;
438	// we must iterate through it and update each of the bit vectors.
439	for (auto &Pair : RegUsesMap) {
440	SmallBitVector &UsedByIndices = Pair.second.UsedByIndices;
441	if (LUIdx < UsedByIndices.size())
442	UsedByIndices [LUIdx] =
443	LastLUIdx < UsedByIndices.size() ? UsedByIndices [LastLUIdx] : false;
444	UsedByIndices.resize(N: std::min(a: UsedByIndices.size(), b: LastLUIdx));
445	}
446	}
447
448	bool
449	RegUseTracker::isRegUsedByUsesOtherThan(const SCEV Reg, size_t LUIdx) const* {
450	RegUsesTy::const_iterator I = RegUsesMap.find(Val: Reg);
451	if (I == RegUsesMap.end())
452	return false;
453	const SmallBitVector &UsedByIndices = I ->second.UsedByIndices;
454	int i = UsedByIndices.find_first();
455	if (i == -`1`) return false;
456	if ((size_t)i != LUIdx) return true;
457	return UsedByIndices.find_next(Prev: i) != -`1`;
458	}
459
460	const SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV Reg) const* {
461	RegUsesTy::const_iterator I = RegUsesMap.find(Val: Reg);
462	assert(I != RegUsesMap.end() && "Unknown register!");
463	return I ->second.UsedByIndices;
464	}
465
466	void RegUseTracker::clear() {
467	RegUsesMap.clear();
468	RegSequence.clear();
469	}
470
471	namespace {
472
473	/// This class holds information that describes a formula for computing
474	/// satisfying a use. It may include broken-out immediates and scaled registers.
475	struct Formula {
476	/// Global base address used for complex addressing.
477	GlobalValue BaseGV = nullptr*;
478
479	/// Base offset for complex addressing.
480	Immediate BaseOffset = Immediate::getZero();
481
482	/// Whether any complex addressing has a base register.
483	bool HasBaseReg = false;
484
485	/// The scale of any complex addressing.
486	int64_t Scale = `0`;
487
488	/// The list of "base" registers for this use. When this is non-empty. The
489	/// canonical representation of a formula is
490	/// 1. BaseRegs.size > 1 implies ScaledReg != NULL and
491	/// 2. ScaledReg != NULL implies Scale != 1 \|\| !BaseRegs.empty().
492	/// 3. The reg containing recurrent expr related with currect loop in the
493	/// formula should be put in the ScaledReg.
494	/// #1 enforces that the scaled register is always used when at least two
495	/// registers are needed by the formula: e.g., reg1 + reg2 is reg1 + 1 reg2.*
496	/// #2 enforces that 1 reg is reg.*
497	/// #3 ensures invariant regs with respect to current loop can be combined
498	/// together in LSR codegen.
499	/// This invariant can be temporarily broken while building a formula.
500	/// However, every formula inserted into the LSRInstance must be in canonical
501	/// form.
502	SmallVector<const SCEV *, `4`> BaseRegs;
503
504	/// The 'scaled' register for this use. This should be non-null when Scale is
505	/// not zero.
506	const SCEV ScaledReg = nullptr*;
507
508	/// An additional constant offset which added near the use. This requires a
509	/// temporary register, but the offset itself can live in an add immediate
510	/// field rather than a register.
511	Immediate UnfoldedOffset = Immediate::getZero();
512
513	Formula() = default;
514
515	void initialMatch(const SCEV S, Loop L, ScalarEvolution &SE);
516
517	bool isCanonical(const Loop &L) const;
518
519	void canonicalize(const Loop &L);
520
521	bool unscale();
522
523	bool hasZeroEnd() const;
524
525	bool countsDownToZero() const;
526
527	size_t getNumRegs() const;
528	Type getType() const*;
529
530	void deleteBaseReg(const SCEV *&S);
531
532	bool referencesReg(const SCEV S) const*;
533	bool hasRegsUsedByUsesOtherThan(size_t LUIdx,
534	const RegUseTracker &RegUses) const;
535
536	void print(raw_ostream &OS) const;
537	void dump() const;
538	};
539
540	} // end anonymous namespace
541
542	/// Recursion helper for initialMatch.
543	static void DoInitialMatch(const SCEV S, Loop L,
544	SmallVectorImpl<SCEVUse> &Good,
545	SmallVectorImpl<SCEVUse> &Bad, ScalarEvolution &SE) {
546	// Collect expressions which properly dominate the loop header.
547	if (SE.properlyDominates(S, BB: L->getHeader())) {
548	Good.push_back(Elt: S);
549	return;
550	}
551
552	// Look at add operands.
553	if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
554	for (const SCEV *S : Add->operands())
555	DoInitialMatch(S, L, Good, Bad, SE);
556	return;
557	}
558
559	// Look at addrec operands.
560	const SCEV Start, Step;
561	const Loop *ARLoop;
562	if (match(S,
563	P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L&: ARLoop))) &&
564	!Start->isZero()) {
565	DoInitialMatch(S: Start, L, Good, Bad, SE);
566	DoInitialMatch(S: SE.getAddRecExpr(Start: SE.getConstant(Ty: S->getType(), V: `0`), Step,
567	// FIXME: AR->getNoWrapFlags()
568	L: ARLoop, Flags: SCEV::FlagAnyWrap),
569	L, Good, Bad, SE);
570	return;
571	}
572
573	// Handle a multiplication by -1 (negation) if it didn't fold.
574	if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: S))
575	if (Mul->getOperand(i: `0`)->isAllOnesValue()) {
576	SmallVector<SCEVUse, `4`> Ops(drop_begin(RangeOrContainer: Mul->operands()));
577	const SCEV *NewMul = SE.getMulExpr(Ops);
578
579	SmallVector<SCEVUse, `4`> MyGood;
580	SmallVector<SCEVUse, `4`> MyBad;
581	DoInitialMatch(S: NewMul, L, Good&: MyGood, Bad&: MyBad, SE);
582	const SCEV *NegOne = SE.getSCEV(V: ConstantInt::getAllOnesValue(
583	Ty: SE.getEffectiveSCEVType(Ty: NewMul->getType())));
584	for (const SCEV *S : MyGood)
585	Good.push_back(Elt: SE.getMulExpr(LHS: NegOne, RHS: S));
586	for (const SCEV *S : MyBad)
587	Bad.push_back(Elt: SE.getMulExpr(LHS: NegOne, RHS: S));
588	return;
589	}
590
591	// Ok, we can't do anything interesting. Just stuff the whole thing into a
592	// register and hope for the best.
593	Bad.push_back(Elt: S);
594	}
595
596	/// Incorporate loop-variant parts of S into this Formula, attempting to keep
597	/// all loop-invariant and loop-computable values in a single base register.
598	void Formula::initialMatch(const SCEV S, Loop L, ScalarEvolution &SE) {
599	SmallVector<SCEVUse, `4`> Good;
600	SmallVector<SCEVUse, `4`> Bad;
601	DoInitialMatch(S, L, Good, Bad, SE);
602	if (!Good.empty()) {
603	const SCEV *Sum = SE.getAddExpr(Ops&: Good);
604	if (!Sum->isZero())
605	BaseRegs.push_back(Elt: Sum);
606	HasBaseReg = true;
607	}
608	if (!Bad.empty()) {
609	const SCEV *Sum = SE.getAddExpr(Ops&: Bad);
610	if (!Sum->isZero())
611	BaseRegs.push_back(Elt: Sum);
612	HasBaseReg = true;
613	}
614	canonicalize(L: *L);
615	}
616
617	static bool containsAddRecDependentOnLoop(const SCEV S, const* Loop &L) {
618	return SCEVExprContains(Root: S, Pred: [&L](const SCEV *S) {
619	return isa<SCEVAddRecExpr>(Val: S) && (cast<SCEVAddRecExpr>(Val: S)->getLoop() == &L);
620	});
621	}
622
623	/// Check whether or not this formula satisfies the canonical
624	/// representation.
625	/// \see Formula::BaseRegs.
626	bool Formula::isCanonical(const Loop &L) const {
627	assert((Scale == `0` \|\| ScaledReg) &&
628	"ScaledReg must be non-null if Scale is non-zero");
629
630	if (!ScaledReg)
631	return BaseRegs.size() <= `1`;
632
633	if (Scale != `1`)
634	return true;
635
636	if (Scale == `1` && BaseRegs.empty())
637	return false;
638
639	if (containsAddRecDependentOnLoop(S: ScaledReg, L))
640	return true;
641
642	// If ScaledReg is not a recurrent expr, or it is but its loop is not current
643	// loop, meanwhile BaseRegs contains a recurrent expr reg related with current
644	// loop, we want to swap the reg in BaseRegs with ScaledReg.
645	return none_of(Range: BaseRegs, P: [&L](const SCEV *S) {
646	return containsAddRecDependentOnLoop(S, L);
647	});
648	}
649
650	/// Helper method to morph a formula into its canonical representation.
651	/// \see Formula::BaseRegs.
652	/// Every formula having more than one base register, must use the ScaledReg
653	/// field. Otherwise, we would have to do special cases everywhere in LSR
654	/// to treat reg1 + reg2 + ... the same way as reg1 + 1reg2 + ...*
655	/// On the other hand, 1reg should be canonicalized into reg.*
656	void Formula::canonicalize(const Loop &L) {
657	if (isCanonical(L))
658	return;
659
660	if (BaseRegs.empty()) {
661	// No base reg? Use scale reg with scale = 1 as such.
662	assert(ScaledReg && "Expected 1*reg => reg");
663	assert(Scale == `1` && "Expected 1*reg => reg");
664	BaseRegs.push_back(Elt: ScaledReg);
665	Scale = `0`;
666	ScaledReg = nullptr;
667	return;
668	}
669
670	// Keep the invariant sum in BaseRegs and one of the variant sum in ScaledReg.
671	if (!ScaledReg) {
672	ScaledReg = BaseRegs.pop_back_val();
673	Scale = `1`;
674	}
675
676	// If ScaledReg is an invariant with respect to L, find the reg from
677	// BaseRegs containing the recurrent expr related with Loop L. Swap the
678	// reg with ScaledReg.
679	if (!containsAddRecDependentOnLoop(S: ScaledReg, L)) {
680	auto I = find_if(Range&: BaseRegs, P: [&L](const SCEV *S) {
681	return containsAddRecDependentOnLoop(S, L);
682	});
683	if (I != BaseRegs.end())
684	std::swap(a&: ScaledReg, b&: *I);
685	}
686	assert(isCanonical(L) && "Failed to canonicalize?");
687	}
688
689	/// Get rid of the scale in the formula.
690	/// In other words, this method morphes reg1 + 1reg2 into reg1 + reg2.*
691	/// \return true if it was possible to get rid of the scale, false otherwise.
692	/// \note After this operation the formula may not be in the canonical form.
693	bool Formula::unscale() {
694	if (Scale != `1`)
695	return false;
696	Scale = `0`;
697	BaseRegs.push_back(Elt: ScaledReg);
698	ScaledReg = nullptr;
699	return true;
700	}
701
702	bool Formula::hasZeroEnd() const {
703	if (UnfoldedOffset \|\| BaseOffset)
704	return false;
705	if (BaseRegs.size() != `1` \|\| ScaledReg)
706	return false;
707	return true;
708	}
709
710	bool Formula::countsDownToZero() const {
711	if (!hasZeroEnd())
712	return false;
713	assert(BaseRegs.size() == `1` && "hasZeroEnd should mean one BaseReg");
714	const APInt *StepInt;
715	if (!match(S: BaseRegs [`0`], P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_scev_APInt(C&: StepInt))))
716	return false;
717	return StepInt->isNegative();
718	}
719
720	/// Return the total number of register operands used by this formula. This does
721	/// not include register uses implied by non-constant addrec strides.
722	size_t Formula::getNumRegs() const {
723	return !!ScaledReg + BaseRegs.size();
724	}
725
726	/// Return the type of this formula, if it has one, or null otherwise. This type
727	/// is meaningless except for the bit size.
728	Type Formula::getType() const* {
729	return !BaseRegs.empty() ? BaseRegs.front()->getType() :
730	ScaledReg ? ScaledReg->getType() :
731	BaseGV ? BaseGV->getType() :
732	nullptr;
733	}
734
735	/// Delete the given base reg from the BaseRegs list.
736	void Formula::deleteBaseReg(const SCEV *&S) {
737	if (&S != &BaseRegs.back())
738	std::swap(a&: S, b&: BaseRegs.back());
739	BaseRegs.pop_back();
740	}
741
742	/// Test if this formula references the given register.
743	bool Formula::referencesReg(const SCEV S) const* {
744	return S == ScaledReg \|\| is_contained(Range: BaseRegs, Element: S);
745	}
746
747	/// Test whether this formula uses registers which are used by uses other than
748	/// the use with the given index.
749	bool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx,
750	const RegUseTracker &RegUses) const {
751	if (ScaledReg)
752	if (RegUses.isRegUsedByUsesOtherThan(Reg: ScaledReg, LUIdx))
753	return true;
754	for (const SCEV *BaseReg : BaseRegs)
755	if (RegUses.isRegUsedByUsesOtherThan(Reg: BaseReg, LUIdx))
756	return true;
757	return false;
758	}
759
760	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
761	void Formula::print(raw_ostream &OS) const {
762	ListSeparator Plus(" + ");
763	if (BaseGV) {
764	OS << Plus;
765	BaseGV->printAsOperand(OS, /PrintType=/false);
766	}
767	if (BaseOffset.isNonZero())
768	OS << Plus << BaseOffset;
769
770	for (const SCEV *BaseReg : BaseRegs)
771	OS << Plus << "reg(" << *BaseReg << `')'`;
772
773	if (HasBaseReg && BaseRegs.empty())
774	OS << Plus << "error: HasBaseReg";
775	else if (!HasBaseReg && !BaseRegs.empty())
776	OS << Plus << "error: !HasBaseReg";
777
778	if (Scale != `0`) {
779	OS << Plus << Scale << "*reg(";
780	if (ScaledReg)
781	OS << *ScaledReg;
782	else
783	OS << "<unknown>";
784	OS << `')'`;
785	}
786	if (UnfoldedOffset.isNonZero())
787	OS << Plus << "imm(" << UnfoldedOffset << `')'`;
788	}
789
790	LLVM_DUMP_METHOD void Formula::dump() const {
791	print(errs()); errs() << `'\n'`;
792	}
793	#endif
794
795	/// Return true if the given addrec can be sign-extended without changing its
796	/// value.
797	static bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
798	Type *WideTy =
799	IntegerType::get(C&: SE.getContext(), NumBits: SE.getTypeSizeInBits(Ty: AR->getType()) + `1`);
800	return isa<SCEVAddRecExpr>(Val: SE.getSignExtendExpr(Op: AR, Ty: WideTy));
801	}
802
803	/// Return true if the given add can be sign-extended without changing its
804	/// value.
805	static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) {
806	Type *WideTy =
807	IntegerType::get(C&: SE.getContext(), NumBits: SE.getTypeSizeInBits(Ty: A->getType()) + `1`);
808	return isa<SCEVAddExpr>(Val: SE.getSignExtendExpr(Op: A, Ty: WideTy));
809	}
810
811	/// Return true if the given mul can be sign-extended without changing its
812	/// value.
813	static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) {
814	Type *WideTy =
815	IntegerType::get(C&: SE.getContext(),
816	NumBits: SE.getTypeSizeInBits(Ty: M->getType()) * M->getNumOperands());
817	return isa<SCEVMulExpr>(Val: SE.getSignExtendExpr(Op: M, Ty: WideTy));
818	}
819
820	/// Return an expression for LHS /s RHS, if it can be determined and if the
821	/// remainder is known to be zero, or null otherwise. If IgnoreSignificantBits
822	/// is true, expressions like (X Y) /s Y are simplified to X, ignoring that*
823	/// the multiplication may overflow, which is useful when the result will be
824	/// used in a context where the most significant bits are ignored.
825	static const SCEV getExactSDiv(const* SCEV LHS, const* SCEV *RHS,
826	ScalarEvolution &SE,
827	bool IgnoreSignificantBits = false) {
828	// Handle the trivial case, which works for any SCEV type.
829	if (LHS == RHS)
830	return SE.getConstant(Ty: LHS->getType(), V: `1`);
831
832	// Handle a few RHS special cases.
833	const SCEVConstant *RC = dyn_cast<SCEVConstant>(Val: RHS);
834	if (RC) {
835	const APInt &RA = RC->getAPInt();
836	// Handle x /s -1 as x -1, to give ScalarEvolution a chance to do*
837	// some folding.
838	if (RA.isAllOnes()) {
839	if (LHS->getType()->isPointerTy())
840	return nullptr;
841	return SE.getMulExpr(LHS, RHS: RC);
842	}
843	// Handle x /s 1 as x.
844	if (RA == `1`)
845	return LHS;
846	}
847
848	// Check for a division of a constant by a constant.
849	if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: LHS)) {
850	if (!RC)
851	return nullptr;
852	const APInt &LA = C->getAPInt();
853	const APInt &RA = RC->getAPInt();
854	if (LA.srem(RHS: RA) != `0`)
855	return nullptr;
856	return SE.getConstant(Val: LA.sdiv(RHS: RA));
857	}
858
859	// Distribute the sdiv over addrec operands, if the addrec doesn't overflow.
860	if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS)) {
861	if ((IgnoreSignificantBits \|\| isAddRecSExtable(AR, SE)) && AR->isAffine()) {
862	const SCEV *Step = getExactSDiv(LHS: AR->getStepRecurrence(SE), RHS, SE,
863	IgnoreSignificantBits);
864	if (!Step) return nullptr;
865	const SCEV *Start = getExactSDiv(LHS: AR->getStart(), RHS, SE,
866	IgnoreSignificantBits);
867	if (!Start) return nullptr;
868	// FlagNW is independent of the start value, step direction, and is
869	// preserved with smaller magnitude steps.
870	// FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
871	return SE.getAddRecExpr(Start, Step, L: AR->getLoop(), Flags: SCEV::FlagAnyWrap);
872	}
873	return nullptr;
874	}
875
876	// Distribute the sdiv over add operands, if the add doesn't overflow.
877	if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: LHS)) {
878	if (IgnoreSignificantBits \|\| isAddSExtable(A: Add, SE)) {
879	SmallVector<SCEVUse, `8`> Ops;
880	for (const SCEV *S : Add->operands()) {
881	const SCEV *Op = getExactSDiv(LHS: S, RHS, SE, IgnoreSignificantBits);
882	if (!Op) return nullptr;
883	Ops.push_back(Elt: Op);
884	}
885	return SE.getAddExpr(Ops);
886	}
887	return nullptr;
888	}
889
890	// Check for a multiply operand that we can pull RHS out of.
891	if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: LHS)) {
892	if (IgnoreSignificantBits \|\| isMulSExtable(M: Mul, SE)) {
893	// Handle special case C1XY /s C2XY.
894	if (const SCEVMulExpr *MulRHS = dyn_cast<SCEVMulExpr>(Val: RHS)) {
895	if (IgnoreSignificantBits \|\| isMulSExtable(M: MulRHS, SE)) {
896	const SCEVConstant *LC = dyn_cast<SCEVConstant>(Val: Mul->getOperand(i: `0`));
897	const SCEVConstant *RC =
898	dyn_cast<SCEVConstant>(Val: MulRHS->getOperand(i: `0`));
899	if (LC && RC) {
900	SmallVector<const SCEV *, `4`> LOps(drop_begin(RangeOrContainer: Mul->operands()));
901	SmallVector<const SCEV *, `4`> ROps(drop_begin(RangeOrContainer: MulRHS->operands()));
902	if (LOps == ROps)
903	return getExactSDiv(LHS: LC, RHS: RC, SE, IgnoreSignificantBits);
904	}
905	}
906	}
907
908	SmallVector<SCEVUse, `4`> Ops;
909	bool Found = false;
910	for (const SCEV *S : Mul->operands()) {
911	if (!Found)
912	if (const SCEV *Q = getExactSDiv(LHS: S, RHS, SE,
913	IgnoreSignificantBits)) {
914	S = Q;
915	Found = true;
916	}
917	Ops.push_back(Elt: S);
918	}
919	return Found ? SE.getMulExpr(Ops) : nullptr;
920	}
921	return nullptr;
922	}
923
924	// Otherwise we don't know.
925	return nullptr;
926	}
927
928	/// Extracts an immediate operand from \p Ops and replaces the operand with
929	/// zero. If \p PreferScalable is true and \p Ops contains both a scalable and
930	/// non-scalable offsets, the scalable offset will be extracted.
931	static Immediate ExtractImmediateOperand(MutableArrayRef<SCEVUse> Ops,
932	ScalarEvolution &SE,
933	bool PreferScalable) {
934	const APInt *C;
935	SCEVUse Op = nullptr*;
936	Immediate Result = Immediate::getZero();
937
938	// Ops are sorted by their SCEVType (the order of SCEVTypes enum). So, for an
939	// AddExpr the possible order of operands is:
940	// Constant < VScale < Truncate < ZeroExtend < SignExtend < MulExpr < ...
941
942	// This means fixed-size immediates will always appear on the LHS:
943	SCEVUse &S = Ops.front();
944	if (match(U: S, P: m_scev_APInt(C)) && !C->isZero() &&
945	C->getSignificantBits() <= `64`) {
946	Op = &S;
947	Result = Immediate::getFixed(MinVal: C->getSExtValue());
948	}
949
950	// But scalable immediates, which are MulExpr(Vscale, Constant), can appear
951	// later in the operand list:
952	if (EnableVScaleImmediates && (Result.isZero() \|\| PreferScalable)) {
953	for (SCEVUse &S : Ops) {
954	// We know anything past scMulExpr will not be a vscale immediate.
955	if (S ->getSCEVType() > scMulExpr)
956	break;
957	if (match(U: S, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEVVScale()))) {
958	Op = &S;
959	Result = Immediate::getScalable(MinVal: C->getSExtValue());
960	break;
961	}
962	}
963	}
964
965	if (Result.isNonZero()) {
966	SCEVUse &S = *Op;
967	S = SE.getConstant(Ty: S ->getType(), V: `0`);
968	}
969
970	return Result;
971	}
972
973	/// If S involves the addition of a constant integer value, return that integer
974	/// value, and mutate S to point to a new SCEV with that value excluded.
975	static Immediate ExtractImmediate(SCEVUse &S, ScalarEvolution &SE,
976	bool PreferScalable = false) {
977	if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: S)) {
978	SmallVector<SCEVUse, `8`> NewOps(Add->operands());
979	Immediate Result = ExtractImmediateOperand(Ops: NewOps, SE, PreferScalable);
980	if (Result.isZero())
981	Result = ExtractImmediate(S&: NewOps.front(), SE, PreferScalable);
982	if (Result.isNonZero())
983	S = SE.getAddExpr(Ops&: NewOps);
984	return Result;
985	} else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: S)) {
986	SmallVector<SCEVUse, `8`> NewOps(AR->operands());
987	Immediate Result = ExtractImmediate(S&: NewOps.front(), SE, PreferScalable);
988	if (Result.isNonZero())
989	S = SE.getAddRecExpr(Operands&: NewOps, L: AR->getLoop(),
990	// FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
991	Flags: SCEV::FlagAnyWrap);
992	return Result;
993	}
994	return ExtractImmediateOperand(Ops: {S}, SE, PreferScalable);
995	}
996
997	/// If S involves the addition of a GlobalValue address, return that symbol, and
998	/// mutate S to point to a new SCEV with that value excluded.
999	static GlobalValue *ExtractSymbol(SCEVUse &S, ScalarEvolution &SE) {
1000	if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Val&: S)) {
1001	if (GlobalValue *GV = dyn_cast<GlobalValue>(Val: U->getValue())) {
1002	S = SE.getConstant(Ty: GV->getType(), V: `0`);
1003	return GV;
1004	}
1005	} else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: S)) {
1006	SmallVector<SCEVUse, `8`> NewOps(Add->operands());
1007	GlobalValue *Result = ExtractSymbol(S&: NewOps.back(), SE);
1008	if (Result)
1009	S = SE.getAddExpr(Ops&: NewOps);
1010	return Result;
1011	} else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: S)) {
1012	SmallVector<SCEVUse, `8`> NewOps(AR->operands());
1013	GlobalValue *Result = ExtractSymbol(S&: NewOps.front(), SE);
1014	if (Result)
1015	S = SE.getAddRecExpr(Operands&: NewOps, L: AR->getLoop(),
1016	// FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
1017	Flags: SCEV::FlagAnyWrap);
1018	return Result;
1019	}
1020	return nullptr;
1021	}
1022
1023	/// Returns true if the specified instruction is using the specified value as an
1024	/// address.
1025	static bool isAddressUse(const TargetTransformInfo &TTI,
1026	Instruction Inst, Value OperandVal) {
1027	bool isAddress = isa<LoadInst>(Val: Inst);
1028	if (StoreInst *SI = dyn_cast<StoreInst>(Val: Inst)) {
1029	if (SI->getPointerOperand() == OperandVal)
1030	isAddress = true;
1031	} else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Inst)) {
1032	// Addressing modes can also be folded into prefetches and a variety
1033	// of intrinsics.
1034	switch (II->getIntrinsicID()) {
1035	case Intrinsic::memset:
1036	case Intrinsic::prefetch:
1037	case Intrinsic::masked_load:
1038	if (II->getArgOperand(i: `0`) == OperandVal)
1039	isAddress = true;
1040	break;
1041	case Intrinsic::masked_store:
1042	if (II->getArgOperand(i: `1`) == OperandVal)
1043	isAddress = true;
1044	break;
1045	case Intrinsic::memmove:
1046	case Intrinsic::memcpy:
1047	if (II->getArgOperand(i: `0`) == OperandVal \|\|
1048	II->getArgOperand(i: `1`) == OperandVal)
1049	isAddress = true;
1050	break;
1051	default: {
1052	MemIntrinsicInfo IntrInfo;
1053	if (TTI.getTgtMemIntrinsic(Inst: II, Info&: IntrInfo)) {
1054	if (IntrInfo.PtrVal == OperandVal)
1055	isAddress = true;
1056	}
1057	}
1058	}
1059	} else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
1060	if (RMW->getPointerOperand() == OperandVal)
1061	isAddress = true;
1062	} else if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
1063	if (CmpX->getPointerOperand() == OperandVal)
1064	isAddress = true;
1065	}
1066	return isAddress;
1067	}
1068
1069	/// Return the type of the memory being accessed.
1070	static MemAccessTy getAccessType(const TargetTransformInfo &TTI,
1071	Instruction Inst, Value OperandVal) {
1072	MemAccessTy AccessTy = MemAccessTy::getUnknown(Ctx&: Inst->getContext());
1073
1074	// First get the type of memory being accessed.
1075	if (Type *Ty = Inst->getAccessType())
1076	AccessTy.MemTy = Ty;
1077
1078	// Then get the pointer address space.
1079	if (const StoreInst *SI = dyn_cast<StoreInst>(Val: Inst)) {
1080	AccessTy.AddrSpace = SI->getPointerAddressSpace();
1081	} else if (const LoadInst *LI = dyn_cast<LoadInst>(Val: Inst)) {
1082	AccessTy.AddrSpace = LI->getPointerAddressSpace();
1083	} else if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
1084	AccessTy.AddrSpace = RMW->getPointerAddressSpace();
1085	} else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
1086	AccessTy.AddrSpace = CmpX->getPointerAddressSpace();
1087	} else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Inst)) {
1088	switch (II->getIntrinsicID()) {
1089	case Intrinsic::prefetch:
1090	case Intrinsic::memset:
1091	AccessTy.AddrSpace = II->getArgOperand(i: `0`)->getType()->getPointerAddressSpace();
1092	AccessTy.MemTy = OperandVal->getType();
1093	break;
1094	case Intrinsic::memmove:
1095	case Intrinsic::memcpy:
1096	AccessTy.AddrSpace = OperandVal->getType()->getPointerAddressSpace();
1097	AccessTy.MemTy = OperandVal->getType();
1098	break;
1099	case Intrinsic::masked_load:
1100	AccessTy.AddrSpace =
1101	II->getArgOperand(i: `0`)->getType()->getPointerAddressSpace();
1102	break;
1103	case Intrinsic::masked_store:
1104	AccessTy.AddrSpace =
1105	II->getArgOperand(i: `1`)->getType()->getPointerAddressSpace();
1106	break;
1107	default: {
1108	MemIntrinsicInfo IntrInfo;
1109	if (TTI.getTgtMemIntrinsic(Inst: II, Info&: IntrInfo) && IntrInfo.PtrVal) {
1110	AccessTy.AddrSpace
1111	= IntrInfo.PtrVal->getType()->getPointerAddressSpace();
1112	}
1113
1114	break;
1115	}
1116	}
1117	}
1118
1119	return AccessTy;
1120	}
1121
1122	/// Return true if this AddRec is already a phi in its loop.
1123	static bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
1124	for (PHINode &PN : AR->getLoop()->getHeader()->phis()) {
1125	if (SE.isSCEVable(Ty: PN.getType()) &&
1126	(SE.getEffectiveSCEVType(Ty: PN.getType()) ==
1127	SE.getEffectiveSCEVType(Ty: AR->getType())) &&
1128	SE.getSCEV(V: &PN) == AR)
1129	return true;
1130	}
1131	return false;
1132	}
1133
1134	/// Check if expanding this expression is likely to incur significant cost. This
1135	/// is tricky because SCEV doesn't track which expressions are actually computed
1136	/// by the current IR.
1137	///
1138	/// We currently allow expansion of IV increments that involve adds,
1139	/// multiplication by constants, and AddRecs from existing phis.
1140	///
1141	/// TODO: Allow UDivExpr if we can find an existing IV increment that is an
1142	/// obvious multiple of the UDivExpr.
1143	static bool isHighCostExpansion(const SCEV *S,
1144	SmallPtrSetImpl<const SCEV*> &Processed,
1145	ScalarEvolution &SE) {
1146	// Zero/One operand expressions
1147	switch (S->getSCEVType()) {
1148	case scUnknown:
1149	case scConstant:
1150	case scVScale:
1151	return false;
1152	case scTruncate:
1153	return isHighCostExpansion(S: cast<SCEVTruncateExpr>(Val: S)->getOperand(),
1154	Processed, SE);
1155	case scZeroExtend:
1156	return isHighCostExpansion(S: cast<SCEVZeroExtendExpr>(Val: S)->getOperand(),
1157	Processed, SE);
1158	case scSignExtend:
1159	return isHighCostExpansion(S: cast<SCEVSignExtendExpr>(Val: S)->getOperand(),
1160	Processed, SE);
1161	default:
1162	break;
1163	}
1164
1165	if (!Processed.insert(Ptr: S).second)
1166	return false;
1167
1168	if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
1169	for (const SCEV *S : Add->operands()) {
1170	if (isHighCostExpansion(S, Processed, SE))
1171	return true;
1172	}
1173	return false;
1174	}
1175
1176	const SCEV Op0, Op1;
1177	if (match(S, P: m_scev_Mul(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1)))) {
1178	// Multiplication by a constant is ok
1179	if (isa<SCEVConstant>(Val: Op0))
1180	return isHighCostExpansion(S: Op1, Processed, SE);
1181
1182	// If we have the value of one operand, check if an existing
1183	// multiplication already generates this expression.
1184	if (const auto *U = dyn_cast<SCEVUnknown>(Val: Op1)) {
1185	Value *UVal = U->getValue();
1186	for (User *UR : UVal->users()) {
1187	// If U is a constant, it may be used by a ConstantExpr.
1188	Instruction *UI = dyn_cast<Instruction>(Val: UR);
1189	if (UI && UI->getOpcode() == Instruction::Mul &&
1190	SE.isSCEVable(Ty: UI->getType())) {
1191	return SE.getSCEV(V: UI) == S;
1192	}
1193	}
1194	}
1195	}
1196
1197	if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
1198	if (isExistingPhi(AR, SE))
1199	return false;
1200	}
1201
1202	// Fow now, consider any other type of expression (div/mul/min/max) high cost.
1203	return true;
1204	}
1205
1206	namespace {
1207
1208	class LSRUse;
1209
1210	} // end anonymous namespace
1211
1212	/// Check if the addressing mode defined by \p F is completely
1213	/// folded in \p LU at isel time.
1214	/// This includes address-mode folding and special icmp tricks.
1215	/// This function returns true if \p LU can accommodate what \p F
1216	/// defines and up to 1 base + 1 scaled + offset.
1217	/// In other words, if \p F has several base registers, this function may
1218	/// still return true. Therefore, users still need to account for
1219	/// additional base registers and/or unfolded offsets to derive an
1220	/// accurate cost model.
1221	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1222	const LSRUse &LU, const Formula &F);
1223
1224	// Get the cost of the scaling factor used in F for LU.
1225	static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI,
1226	const LSRUse &LU, const Formula &F,
1227	const Loop &L);
1228
1229	namespace {
1230
1231	/// This class is used to measure and compare candidate formulae.
1232	class Cost {
1233	const Loop L = nullptr*;
1234	ScalarEvolution SE = nullptr*;
1235	const TargetTransformInfo TTI = nullptr*;
1236	TargetTransformInfo::LSRCost C;
1237	TTI::AddressingModeKind AMK = TTI::AMK_None;
1238
1239	public:
1240	Cost() = delete;
1241	Cost(const Loop L, ScalarEvolution &SE, const* TargetTransformInfo &TTI,
1242	TTI::AddressingModeKind AMK) :
1243	L(L), SE(&SE), TTI(&TTI), AMK(AMK) {
1244	C.Insns = `0`;
1245	C.NumRegs = `0`;
1246	C.AddRecCost = `0`;
1247	C.NumIVMuls = `0`;
1248	C.NumBaseAdds = `0`;
1249	C.ImmCost = `0`;
1250	C.SetupCost = `0`;
1251	C.ScaleCost = `0`;
1252	}
1253
1254	bool isLess(const Cost &Other) const;
1255
1256	void Lose();
1257
1258	#ifndef NDEBUG
1259	// Once any of the metrics loses, they must all remain losers.
1260	bool isValid() {
1261	return ((C.Insns \| C.NumRegs \| C.AddRecCost \| C.NumIVMuls \| C.NumBaseAdds
1262	\| C.ImmCost \| C.SetupCost \| C.ScaleCost) != ~`0u`)
1263	\|\| ((C.Insns & C.NumRegs & C.AddRecCost & C.NumIVMuls & C.NumBaseAdds
1264	& C.ImmCost & C.SetupCost & C.ScaleCost) == ~`0u`);
1265	}
1266	#endif
1267
1268	bool isLoser() {
1269	assert(isValid() && "invalid cost");
1270	return C.NumRegs == ~`0u`;
1271	}
1272
1273	void RateFormula(const Formula &F, SmallPtrSetImpl<const SCEV *> &Regs,
1274	const DenseSet<const SCEV > &VisitedRegs, const* LSRUse &LU,
1275	bool HardwareLoopProfitable,
1276	SmallPtrSetImpl<const SCEV > LoserRegs = nullptr);
1277
1278	void print(raw_ostream &OS) const;
1279	void dump() const;
1280
1281	private:
1282	void RateRegister(const Formula &F, const SCEV *Reg,
1283	SmallPtrSetImpl<const SCEV > &Regs, const* LSRUse &LU,
1284	bool HardwareLoopProfitable);
1285	void RatePrimaryRegister(const Formula &F, const SCEV *Reg,
1286	SmallPtrSetImpl<const SCEV *> &Regs,
1287	const LSRUse &LU, bool HardwareLoopProfitable,
1288	SmallPtrSetImpl<const SCEV > LoserRegs);
1289	};
1290
1291	/// An operand value in an instruction which is to be replaced with some
1292	/// equivalent, possibly strength-reduced, replacement.
1293	struct LSRFixup {
1294	/// The instruction which will be updated.
1295	Instruction UserInst = nullptr*;
1296
1297	/// The operand of the instruction which will be replaced. The operand may be
1298	/// used more than once; every instance will be replaced.
1299	Value OperandValToReplace = nullptr*;
1300
1301	/// If this user is to use the post-incremented value of an induction
1302	/// variable, this set is non-empty and holds the loops associated with the
1303	/// induction variable.
1304	PostIncLoopSet PostIncLoops;
1305
1306	/// A constant offset to be added to the LSRUse expression. This allows
1307	/// multiple fixups to share the same LSRUse with different offsets, for
1308	/// example in an unrolled loop.
1309	Immediate Offset = Immediate::getZero();
1310
1311	LSRFixup() = default;
1312
1313	bool isUseFullyOutsideLoop(const Loop L) const*;
1314
1315	void print(raw_ostream &OS) const;
1316	void dump() const;
1317	};
1318
1319	/// This class holds the state that LSR keeps for each use in IVUsers, as well
1320	/// as uses invented by LSR itself. It includes information about what kinds of
1321	/// things can be folded into the user, information about the user itself, and
1322	/// information about how the use may be satisfied. TODO: Represent multiple
1323	/// users of the same expression in common?
1324	class LSRUse {
1325	DenseSet<SmallVector<const SCEV *, `4`>> Uniquifier;
1326
1327	public:
1328	/// An enum for a kind of use, indicating what types of scaled and immediate
1329	/// operands it might support.
1330	enum KindType {
1331	Basic, ///< A normal use, with no folding.
1332	Special, ///< A special case of basic, allowing -1 scales.
1333	Address, ///< An address use; folding according to TargetLowering
1334	ICmpZero ///< An equality icmp with both operands folded into one.
1335	// TODO: Add a generic icmp too?
1336	};
1337
1338	using SCEVUseKindPair = PointerIntPair<const SCEV *, `2`, KindType>;
1339
1340	KindType Kind;
1341	MemAccessTy AccessTy;
1342
1343	/// The list of operands which are to be replaced.
1344	SmallVector<LSRFixup, `8`> Fixups;
1345
1346	/// Keep track of the min and max offsets of the fixups.
1347	Immediate MinOffset = Immediate::getFixedMax();
1348	Immediate MaxOffset = Immediate::getFixedMin();
1349
1350	/// This records whether all of the fixups using this LSRUse are outside of
1351	/// the loop, in which case some special-case heuristics may be used.
1352	bool AllFixupsOutsideLoop = true;
1353
1354	/// This records whether all of the fixups using this LSRUse are unconditional
1355	/// within the loop, meaning they will be executed on every path to the loop
1356	/// latch. This includes fixups before early exits.
1357	bool AllFixupsUnconditional = true;
1358
1359	/// RigidFormula is set to true to guarantee that this use will be associated
1360	/// with a single formula--the one that initially matched. Some SCEV
1361	/// expressions cannot be expanded. This allows LSR to consider the registers
1362	/// used by those expressions without the need to expand them later after
1363	/// changing the formula.
1364	bool RigidFormula = false;
1365
1366	/// A list of ways to build a value that can satisfy this user. After the
1367	/// list is populated, one of these is selected heuristically and used to
1368	/// formulate a replacement for OperandValToReplace in UserInst.
1369	SmallVector<Formula, `12`> Formulae;
1370
1371	/// The set of register candidates used by all formulae in this LSRUse.
1372	SmallPtrSet<const SCEV *, `4`> Regs;
1373
1374	LSRUse(KindType K, MemAccessTy AT) : Kind(K), AccessTy (AT) {}
1375
1376	LSRFixup &getNewFixup() {
1377	Fixups.push_back(Elt: LSRFixup ());
1378	return Fixups.back();
1379	}
1380
1381	void pushFixup(LSRFixup &f) {
1382	Fixups.push_back(Elt: f);
1383	if (Immediate::isKnownGT(LHS: f.Offset, RHS: MaxOffset))
1384	MaxOffset = f.Offset;
1385	if (Immediate::isKnownLT(LHS: f.Offset, RHS: MinOffset))
1386	MinOffset = f.Offset;
1387	}
1388
1389	bool HasFormulaWithSameRegs(const Formula &F) const;
1390	float getNotSelectedProbability(const SCEV Reg) const*;
1391	bool InsertFormula(const Formula &F, const Loop &L);
1392	void DeleteFormula(Formula &F);
1393	void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses);
1394
1395	void print(raw_ostream &OS) const;
1396	void dump() const;
1397	};
1398
1399	} // end anonymous namespace
1400
1401	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1402	LSRUse::KindType Kind, MemAccessTy AccessTy,
1403	GlobalValue *BaseGV, Immediate BaseOffset,
1404	bool HasBaseReg, int64_t Scale,
1405	Instruction Fixup = nullptr*);
1406
1407	static unsigned getSetupCost(const SCEV Reg, unsigned* Depth,
1408	const TargetTransformInfo &TTI) {
1409	if (isa<SCEVUnknown>(Val: Reg))
1410	return `1`;
1411	if (const auto *C = dyn_cast<SCEVConstant>(Val: Reg)) {
1412	if (TTI.getIntImmCost(Imm: C->getAPInt(), Ty: C->getType(),
1413	CostKind: TargetTransformInfo::TCK_RecipThroughput) ==
1414	TargetTransformInfo::TCC_Free)
1415	return `0`;
1416	return `1`;
1417	}
1418	if (Depth == `0`)
1419	return `0`;
1420	if (const auto *S = dyn_cast<SCEVAddRecExpr>(Val: Reg))
1421	return getSetupCost(Reg: S->getStart(), Depth: Depth - `1`, TTI);
1422	if (auto S = dyn_cast<SCEVIntegralCastExpr>(Val: Reg))
1423	return getSetupCost(Reg: S->getOperand(), Depth: Depth - `1`, TTI);
1424	if (auto S = dyn_cast<SCEVNAryExpr>(Val: Reg))
1425	return std::accumulate(first: S->operands().begin(), last: S->operands().end(), init: `0`,
1426	binary_op: [&](unsigned i, const SCEV *Reg) {
1427	return i + getSetupCost(Reg, Depth: Depth - `1`, TTI);
1428	});
1429	if (auto S = dyn_cast<SCEVUDivExpr>(Val: Reg))
1430	return getSetupCost(Reg: S->getLHS(), Depth: Depth - `1`, TTI) +
1431	getSetupCost(Reg: S->getRHS(), Depth: Depth - `1`, TTI);
1432	return `0`;
1433	}
1434
1435	/// Tally up interesting quantities from the given register.
1436	void Cost::RateRegister(const Formula &F, const SCEV *Reg,
1437	SmallPtrSetImpl<const SCEV > &Regs, const* LSRUse &LU,
1438	bool HardwareLoopProfitable) {
1439	if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Reg)) {
1440	// If this is an addrec for another loop, it should be an invariant
1441	// with respect to L since L is the innermost loop (at least
1442	// for now LSR only handles innermost loops).
1443	if (AR->getLoop() != L) {
1444	// If the AddRec exists, consider it's register free and leave it alone.
1445	if (isExistingPhi(AR, SE&: *SE) && !(AMK & TTI::AMK_PostIndexed))
1446	return;
1447
1448	// It is bad to allow LSR for current loop to add induction variables
1449	// for its sibling loops.
1450	if (!AR->getLoop()->contains(L)) {
1451	Lose();
1452	return;
1453	}
1454
1455	// Otherwise, it will be an invariant with respect to Loop L.
1456	++C.NumRegs;
1457	return;
1458	}
1459
1460	unsigned LoopCost = `1`;
1461	if (TTI->isIndexedLoadLegal(Mode: TTI->MIM_PostInc, Ty: AR->getType()) \|\|
1462	TTI->isIndexedStoreLegal(Mode: TTI->MIM_PostInc, Ty: AR->getType())) {
1463	const SCEV *Start;
1464	const APInt *Step;
1465	if (match(S: AR, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_scev_APInt(C&: Step)))) {
1466	// If the step size matches the base offset, we could use pre-indexed
1467	// addressing.
1468	bool CanPreIndex = (AMK & TTI::AMK_PreIndexed) &&
1469	F.BaseOffset.isFixed() &&
1470	*Step == F.BaseOffset.getFixedValue();
1471	bool CanPostIndex = (AMK & TTI::AMK_PostIndexed) &&
1472	!isa<SCEVConstant>(Val: Start) &&
1473	SE->isLoopInvariant(S: Start, L);
1474	// We can only pre or post index when the load/store is unconditional.
1475	if ((CanPreIndex \|\| CanPostIndex) && LU.AllFixupsUnconditional)
1476	LoopCost = `0`;
1477	}
1478	}
1479
1480	// If the loop counts down to zero and we'll be using a hardware loop then
1481	// the addrec will be combined into the hardware loop instruction.
1482	if (LU.Kind == LSRUse::ICmpZero && F.countsDownToZero() &&
1483	HardwareLoopProfitable)
1484	LoopCost = `0`;
1485	C.AddRecCost += LoopCost;
1486
1487	// Add the step value register, if it needs one.
1488	// TODO: The non-affine case isn't precisely modeled here.
1489	const SCEV *StepReg = AR->getOperand(i: `1`);
1490	if (!AR->isAffine() \|\| !isa<SCEVConstant>(Val: StepReg)) {
1491	// If the step amount is a constant multiplied by vscale then it can form
1492	// the immediate value of an add and doesn't use a register, so long as
1493	// the immediate value is legal.
1494	auto IsVScaleStep = [](const SCEV Reg, const* TargetTransformInfo *TTI) {
1495	const APInt *X;
1496	if (!match(S: Reg, P: m_scev_Mul(Op0: m_scev_APInt(C&: X), Op1: m_SCEVVScale())))
1497	return false;
1498	return TTI->isLegalAddScalableImmediate(Imm: X->getLimitedValue());
1499	};
1500	if (!Regs.count(Ptr: StepReg) && !IsVScaleStep (StepReg, TTI)) {
1501	RateRegister(F, Reg: StepReg, Regs, LU, HardwareLoopProfitable);
1502	if (isLoser())
1503	return;
1504	}
1505	}
1506	}
1507	++C.NumRegs;
1508
1509	// Rough heuristic; favor registers which don't require extra setup
1510	// instructions in the preheader.
1511	C.SetupCost += getSetupCost(Reg, Depth: SetupCostDepthLimit, TTI: *TTI);
1512	// Ensure we don't, even with the recusion limit, produce invalid costs.
1513	C.SetupCost = std::min<unsigned>(a: C.SetupCost, b: `1` << `16`);
1514
1515	C.NumIVMuls += isa<SCEVMulExpr>(Val: Reg) &&
1516	SE->hasComputableLoopEvolution(S: Reg, L);
1517	}
1518
1519	/// Record this register in the set. If we haven't seen it before, rate
1520	/// it. Optional LoserRegs provides a way to declare any formula that refers to
1521	/// one of those regs an instant loser.
1522	void Cost::RatePrimaryRegister(const Formula &F, const SCEV *Reg,
1523	SmallPtrSetImpl<const SCEV *> &Regs,
1524	const LSRUse &LU, bool HardwareLoopProfitable,
1525	SmallPtrSetImpl<const SCEV > LoserRegs) {
1526	if (LoserRegs && LoserRegs->count(Ptr: Reg)) {
1527	Lose();
1528	return;
1529	}
1530	if (Regs.insert(Ptr: Reg).second) {
1531	RateRegister(F, Reg, Regs, LU, HardwareLoopProfitable);
1532	if (LoserRegs && isLoser())
1533	LoserRegs->insert(Ptr: Reg);
1534	}
1535	}
1536
1537	void Cost::RateFormula(const Formula &F, SmallPtrSetImpl<const SCEV *> &Regs,
1538	const DenseSet<const SCEV *> &VisitedRegs,
1539	const LSRUse &LU, bool HardwareLoopProfitable,
1540	SmallPtrSetImpl<const SCEV > LoserRegs) {
1541	if (isLoser())
1542	return;
1543	assert(F.isCanonical(*L) && "Cost is accurate only for canonical formula");
1544	// Tally up the registers.
1545	unsigned PrevAddRecCost = C.AddRecCost;
1546	unsigned PrevNumRegs = C.NumRegs;
1547	unsigned PrevNumBaseAdds = C.NumBaseAdds;
1548	if (const SCEV *ScaledReg = F.ScaledReg) {
1549	if (VisitedRegs.count(V: ScaledReg)) {
1550	Lose();
1551	return;
1552	}
1553	RatePrimaryRegister(F, Reg: ScaledReg, Regs, LU, HardwareLoopProfitable,
1554	LoserRegs);
1555	if (isLoser())
1556	return;
1557	}
1558	for (const SCEV *BaseReg : F.BaseRegs) {
1559	if (VisitedRegs.count(V: BaseReg)) {
1560	Lose();
1561	return;
1562	}
1563	RatePrimaryRegister(F, Reg: BaseReg, Regs, LU, HardwareLoopProfitable,
1564	LoserRegs);
1565	if (isLoser())
1566	return;
1567	}
1568
1569	// Determine how many (unfolded) adds we'll need inside the loop.
1570	size_t NumBaseParts = F.getNumRegs();
1571	if (NumBaseParts > `1`)
1572	// Do not count the base and a possible second register if the target
1573	// allows to fold 2 registers.
1574	C.NumBaseAdds +=
1575	NumBaseParts - (`1` + (F.Scale && isAMCompletelyFolded(TTI: *TTI, LU, F)));
1576	C.NumBaseAdds += (F.UnfoldedOffset.isNonZero());
1577
1578	// Accumulate non-free scaling amounts.
1579	C.ScaleCost += getScalingFactorCost(TTI: TTI, LU, F, L: L).getValue();
1580
1581	// Tally up the non-zero immediates.
1582	for (const LSRFixup &Fixup : LU.Fixups) {
1583	if (Fixup.Offset.isCompatibleImmediate(Imm: F.BaseOffset)) {
1584	Immediate Offset = Fixup.Offset.addUnsigned(RHS: F.BaseOffset);
1585	if (F.BaseGV)
1586	C.ImmCost += `64`; // Handle symbolic values conservatively.
1587	// TODO: This should probably be the pointer size.
1588	else if (Offset.isNonZero())
1589	C.ImmCost +=
1590	APInt (`64`, Offset.getKnownMinValue(), true).getSignificantBits();
1591
1592	// Check with target if this offset with this instruction is
1593	// specifically not supported.
1594	if (LU.Kind == LSRUse::Address && Offset.isNonZero() &&
1595	!isAMCompletelyFolded(TTI: *TTI, Kind: LSRUse::Address, AccessTy: LU.AccessTy, BaseGV: F.BaseGV,
1596	BaseOffset: Offset, HasBaseReg: F.HasBaseReg, Scale: F.Scale, Fixup: Fixup.UserInst))
1597	C.NumBaseAdds++;
1598	} else {
1599	// Incompatible immediate type, increase cost to avoid using
1600	C.ImmCost += `2048`;
1601	}
1602	}
1603
1604	// If we don't count instruction cost exit here.
1605	if (!InsnsCost) {
1606	assert(isValid() && "invalid cost");
1607	return;
1608	}
1609
1610	// Treat every new register that exceeds TTI.getNumberOfRegisters() - 1 as
1611	// additional instruction (at least fill).
1612	// TODO: Need distinguish register class?
1613	unsigned TTIRegNum = TTI->getNumberOfRegisters(
1614	ClassID: TTI->getRegisterClassForType(Vector: false, Ty: F.getType())) - `1`;
1615	if (C.NumRegs > TTIRegNum) {
1616	// Cost already exceeded TTIRegNum, then only newly added register can add
1617	// new instructions.
1618	if (PrevNumRegs > TTIRegNum)
1619	C.Insns += (C.NumRegs - PrevNumRegs);
1620	else
1621	C.Insns += (C.NumRegs - TTIRegNum);
1622	}
1623
1624	// If ICmpZero formula ends with not 0, it could not be replaced by
1625	// just add or sub. We'll need to compare final result of AddRec.
1626	// That means we'll need an additional instruction. But if the target can
1627	// macro-fuse a compare with a branch, don't count this extra instruction.
1628	// For -10 + {0, +, 1}:
1629	// i = i + 1;
1630	// cmp i, 10
1631	//
1632	// For {-10, +, 1}:
1633	// i = i + 1;
1634	if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd() &&
1635	!TTI->canMacroFuseCmp())
1636	C.Insns++;
1637	// Each new AddRec adds 1 instruction to calculation.
1638	C.Insns += (C.AddRecCost - PrevAddRecCost);
1639
1640	// BaseAdds adds instructions for unfolded registers.
1641	if (LU.Kind != LSRUse::ICmpZero)
1642	C.Insns += C.NumBaseAdds - PrevNumBaseAdds;
1643	assert(isValid() && "invalid cost");
1644	}
1645
1646	/// Set this cost to a losing value.
1647	void Cost::Lose() {
1648	C.Insns = std::numeric_limits<unsigned>::max();
1649	C.NumRegs = std::numeric_limits<unsigned>::max();
1650	C.AddRecCost = std::numeric_limits<unsigned>::max();
1651	C.NumIVMuls = std::numeric_limits<unsigned>::max();
1652	C.NumBaseAdds = std::numeric_limits<unsigned>::max();
1653	C.ImmCost = std::numeric_limits<unsigned>::max();
1654	C.SetupCost = std::numeric_limits<unsigned>::max();
1655	C.ScaleCost = std::numeric_limits<unsigned>::max();
1656	}
1657
1658	/// Choose the lower cost.
1659	bool Cost::isLess(const Cost &Other) const {
1660	if (InsnsCost.getNumOccurrences() > `0` && InsnsCost &&
1661	C.Insns != Other.C.Insns)
1662	return C.Insns < Other.C.Insns;
1663	return TTI->isLSRCostLess(C1: C, C2: Other.C);
1664	}
1665
1666	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1667	void Cost::print(raw_ostream &OS) const {
1668	if (InsnsCost)
1669	OS << C.Insns << " instruction" << (C.Insns == `1` ? " " : "s ");
1670	OS << C.NumRegs << " reg" << (C.NumRegs == `1` ? "" : "s");
1671	if (C.AddRecCost != `0`)
1672	OS << ", with addrec cost " << C.AddRecCost;
1673	if (C.NumIVMuls != `0`)
1674	OS << ", plus " << C.NumIVMuls << " IV mul"
1675	<< (C.NumIVMuls == `1` ? "" : "s");
1676	if (C.NumBaseAdds != `0`)
1677	OS << ", plus " << C.NumBaseAdds << " base add"
1678	<< (C.NumBaseAdds == `1` ? "" : "s");
1679	if (C.ScaleCost != `0`)
1680	OS << ", plus " << C.ScaleCost << " scale cost";
1681	if (C.ImmCost != `0`)
1682	OS << ", plus " << C.ImmCost << " imm cost";
1683	if (C.SetupCost != `0`)
1684	OS << ", plus " << C.SetupCost << " setup cost";
1685	}
1686
1687	LLVM_DUMP_METHOD void Cost::dump() const {
1688	print(errs()); errs() << `'\n'`;
1689	}
1690	#endif
1691
1692	/// Test whether this fixup always uses its value outside of the given loop.
1693	bool LSRFixup::isUseFullyOutsideLoop(const Loop L) const* {
1694	// PHI nodes use their value in their incoming blocks.
1695	if (const PHINode *PN = dyn_cast<PHINode>(Val: UserInst)) {
1696	for (unsigned i = `0`, e = PN->getNumIncomingValues(); i != e; ++i)
1697	if (PN->getIncomingValue(i) == OperandValToReplace &&
1698	L->contains(BB: PN->getIncomingBlock(i)))
1699	return false;
1700	return true;
1701	}
1702
1703	return !L->contains(Inst: UserInst);
1704	}
1705
1706	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1707	void LSRFixup::print(raw_ostream &OS) const {
1708	OS << "UserInst=";
1709	// Store is common and interesting enough to be worth special-casing.
1710	if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) {
1711	OS << "store ";
1712	Store->getOperand(`0`)->printAsOperand(OS, /PrintType=/false);
1713	} else if (UserInst->getType()->isVoidTy())
1714	OS << UserInst->getOpcodeName();
1715	else
1716	UserInst->printAsOperand(OS, /PrintType=/false);
1717
1718	OS << ", OperandValToReplace=";
1719	OperandValToReplace->printAsOperand(OS, /PrintType=/false);
1720
1721	for (const Loop *PIL : PostIncLoops) {
1722	OS << ", PostIncLoop=";
1723	PIL->getHeader()->printAsOperand(OS, /PrintType=/false);
1724	}
1725
1726	if (Offset.isNonZero())
1727	OS << ", Offset=" << Offset;
1728	}
1729
1730	LLVM_DUMP_METHOD void LSRFixup::dump() const {
1731	print(errs()); errs() << `'\n'`;
1732	}
1733	#endif
1734
1735	/// Test whether this use as a formula which has the same registers as the given
1736	/// formula.
1737	bool LSRUse::HasFormulaWithSameRegs(const Formula &F) const {
1738	SmallVector<const SCEV *, `4`> Key = F.BaseRegs;
1739	if (F.ScaledReg) Key.push_back(Elt: F.ScaledReg);
1740	// Unstable sort by host order ok, because this is only used for uniquifying.
1741	llvm::sort(C&: Key);
1742	return Uniquifier.count(V: Key);
1743	}
1744
1745	/// The function returns a probability of selecting formula without Reg.
1746	float LSRUse::getNotSelectedProbability(const SCEV Reg) const* {
1747	unsigned FNum = `0`;
1748	for (const Formula &F : Formulae)
1749	if (F.referencesReg(S: Reg))
1750	FNum++;
1751	return ((float)(Formulae.size() - FNum)) / Formulae.size();
1752	}
1753
1754	/// If the given formula has not yet been inserted, add it to the list, and
1755	/// return true. Return false otherwise. The formula must be in canonical form.
1756	bool LSRUse::InsertFormula(const Formula &F, const Loop &L) {
1757	assert(F.isCanonical(L) && "Invalid canonical representation");
1758
1759	if (!Formulae.empty() && RigidFormula)
1760	return false;
1761
1762	SmallVector<const SCEV *, `4`> Key = F.BaseRegs;
1763	if (F.ScaledReg) Key.push_back(Elt: F.ScaledReg);
1764	// Unstable sort by host order ok, because this is only used for uniquifying.
1765	llvm::sort(C&: Key);
1766
1767	if (!Uniquifier.insert(V: Key).second)
1768	return false;
1769
1770	// Using a register to hold the value of 0 is not profitable.
1771	assert((!F.ScaledReg \|\| !F.ScaledReg->isZero()) &&
1772	"Zero allocated in a scaled register!");
1773	#ifndef NDEBUG
1774	for (const SCEV *BaseReg : F.BaseRegs)
1775	assert(!BaseReg->isZero() && "Zero allocated in a base register!");
1776	#endif
1777
1778	// Add the formula to the list.
1779	Formulae.push_back(Elt: F);
1780
1781	// Record registers now being used by this use.
1782	Regs.insert_range(R: F.BaseRegs);
1783	if (F.ScaledReg)
1784	Regs.insert(Ptr: F.ScaledReg);
1785
1786	return true;
1787	}
1788
1789	/// Remove the given formula from this use's list.
1790	void LSRUse::DeleteFormula(Formula &F) {
1791	if (&F != &Formulae.back())
1792	std::swap(a&: F, b&: Formulae.back());
1793	Formulae.pop_back();
1794	}
1795
1796	/// Recompute the Regs field, and update RegUses.
1797	void LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) {
1798	// Now that we've filtered out some formulae, recompute the Regs set.
1799	SmallPtrSet<const SCEV *, `4`> OldRegs = std::move(Regs);
1800	Regs.clear();
1801	for (const Formula &F : Formulae) {
1802	if (F.ScaledReg) Regs.insert(Ptr: F.ScaledReg);
1803	Regs.insert_range(R: F.BaseRegs);
1804	}
1805
1806	// Update the RegTracker.
1807	for (const SCEV *S : OldRegs)
1808	if (!Regs.count(Ptr: S))
1809	RegUses.dropRegister(Reg: S, LUIdx);
1810	}
1811
1812	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
1813	void LSRUse::print(raw_ostream &OS) const {
1814	OS << "LSR Use: Kind=";
1815	switch (Kind) {
1816	case Basic: OS << "Basic"; break;
1817	case Special: OS << "Special"; break;
1818	case ICmpZero: OS << "ICmpZero"; break;
1819	case Address:
1820	OS << "Address of ";
1821	if (AccessTy.MemTy->isPointerTy())
1822	OS << "pointer"; // the full pointer type could be really verbose
1823	else {
1824	OS << *AccessTy.MemTy;
1825	}
1826
1827	OS << " in addrspace(" << AccessTy.AddrSpace << `')'`;
1828	}
1829
1830	OS << ", Offsets={";
1831	bool NeedComma = false;
1832	for (const LSRFixup &Fixup : Fixups) {
1833	if (NeedComma) OS << `','`;
1834	OS << Fixup.Offset;
1835	NeedComma = true;
1836	}
1837	OS << `'}'`;
1838
1839	if (AllFixupsOutsideLoop)
1840	OS << ", all-fixups-outside-loop";
1841
1842	if (AllFixupsUnconditional)
1843	OS << ", all-fixups-unconditional";
1844	}
1845
1846	LLVM_DUMP_METHOD void LSRUse::dump() const {
1847	print(errs()); errs() << `'\n'`;
1848	}
1849	#endif
1850
1851	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1852	LSRUse::KindType Kind, MemAccessTy AccessTy,
1853	GlobalValue *BaseGV, Immediate BaseOffset,
1854	bool HasBaseReg, int64_t Scale,
1855	Instruction Fixup /* = nullptr /) {
1856	switch (Kind) {
1857	case LSRUse::Address: {
1858	int64_t FixedOffset =
1859	BaseOffset.isScalable() ? `0` : BaseOffset.getFixedValue();
1860	int64_t ScalableOffset =
1861	BaseOffset.isScalable() ? BaseOffset.getKnownMinValue() : `0`;
1862	return TTI.isLegalAddressingMode(Ty: AccessTy.MemTy, BaseGV, BaseOffset: FixedOffset,
1863	HasBaseReg, Scale, AddrSpace: AccessTy.AddrSpace,
1864	I: Fixup, ScalableOffset);
1865	}
1866	case LSRUse::ICmpZero:
1867	// There's not even a target hook for querying whether it would be legal to
1868	// fold a GV into an ICmp.
1869	if (BaseGV)
1870	return false;
1871
1872	// ICmp only has two operands; don't allow more than two non-trivial parts.
1873	if (Scale != `0` && HasBaseReg && BaseOffset.isNonZero())
1874	return false;
1875
1876	// ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by
1877	// putting the scaled register in the other operand of the icmp.
1878	if (Scale != `0` && Scale != -`1`)
1879	return false;
1880
1881	// If we have low-level target information, ask the target if it can fold an
1882	// integer immediate on an icmp.
1883	if (BaseOffset.isNonZero()) {
1884	// We don't have an interface to query whether the target supports
1885	// icmpzero against scalable quantities yet.
1886	if (BaseOffset.isScalable())
1887	return false;
1888
1889	// We have one of:
1890	// ICmpZero BaseReg + BaseOffset => ICmp BaseReg, -BaseOffset
1891	// ICmpZero -1ScaleReg + BaseOffset => ICmp ScaleReg, BaseOffset*
1892	// Offs is the ICmp immediate.
1893	if (Scale == `0`)
1894	// The cast does the right thing with
1895	// std::numeric_limits<int64_t>::min().
1896	BaseOffset = BaseOffset.getFixed(MinVal: -(uint64_t)BaseOffset.getFixedValue());
1897	return TTI.isLegalICmpImmediate(Imm: BaseOffset.getFixedValue());
1898	}
1899
1900	// ICmpZero BaseReg + -1ScaleReg => ICmp BaseReg, ScaleReg*
1901	return true;
1902
1903	case LSRUse::Basic:
1904	// Only handle single-register values.
1905	return !BaseGV && Scale == `0` && BaseOffset.isZero();
1906
1907	case LSRUse::Special:
1908	// Special case Basic to handle -1 scales.
1909	return !BaseGV && (Scale == `0` \|\| Scale == -`1`) && BaseOffset.isZero();
1910	}
1911
1912	llvm_unreachable("Invalid LSRUse Kind!");
1913	}
1914
1915	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1916	Immediate MinOffset, Immediate MaxOffset,
1917	LSRUse::KindType Kind, MemAccessTy AccessTy,
1918	GlobalValue *BaseGV, Immediate BaseOffset,
1919	bool HasBaseReg, int64_t Scale) {
1920	if (BaseOffset.isNonZero() &&
1921	(BaseOffset.isScalable() != MinOffset.isScalable() \|\|
1922	BaseOffset.isScalable() != MaxOffset.isScalable()))
1923	return false;
1924	// Check for overflow.
1925	int64_t Base = BaseOffset.getKnownMinValue();
1926	int64_t Min = MinOffset.getKnownMinValue();
1927	int64_t Max = MaxOffset.getKnownMinValue();
1928	if (((int64_t)((uint64_t)Base + Min) > Base) != (Min > `0`))
1929	return false;
1930	MinOffset = Immediate::get(MinVal: (uint64_t)Base + Min, Scalable: MinOffset.isScalable());
1931	if (((int64_t)((uint64_t)Base + Max) > Base) != (Max > `0`))
1932	return false;
1933	MaxOffset = Immediate::get(MinVal: (uint64_t)Base + Max, Scalable: MaxOffset.isScalable());
1934
1935	return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset: MinOffset,
1936	HasBaseReg, Scale) &&
1937	isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset: MaxOffset,
1938	HasBaseReg, Scale);
1939	}
1940
1941	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1942	Immediate MinOffset, Immediate MaxOffset,
1943	LSRUse::KindType Kind, MemAccessTy AccessTy,
1944	const Formula &F, const Loop &L) {
1945	// For the purpose of isAMCompletelyFolded either having a canonical formula
1946	// or a scale not equal to zero is correct.
1947	// Problems may arise from non canonical formulae having a scale == 0.
1948	// Strictly speaking it would best to just rely on canonical formulae.
1949	// However, when we generate the scaled formulae, we first check that the
1950	// scaling factor is profitable before computing the actual ScaledReg for
1951	// compile time sake.
1952	assert((F.isCanonical(L) \|\| F.Scale != `0`));
1953	return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1954	BaseGV: F.BaseGV, BaseOffset: F.BaseOffset, HasBaseReg: F.HasBaseReg, Scale: F.Scale);
1955	}
1956
1957	/// Test whether we know how to expand the current formula.
1958	static bool isLegalUse(const TargetTransformInfo &TTI, Immediate MinOffset,
1959	Immediate MaxOffset, LSRUse::KindType Kind,
1960	MemAccessTy AccessTy, GlobalValue *BaseGV,
1961	Immediate BaseOffset, bool HasBaseReg, int64_t Scale) {
1962	// We know how to expand completely foldable formulae.
1963	return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
1964	BaseOffset, HasBaseReg, Scale) \|\|
1965	// Or formulae that use a base register produced by a sum of base
1966	// registers.
1967	(Scale == `1` &&
1968	isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1969	BaseGV, BaseOffset, HasBaseReg: true, Scale: `0`));
1970	}
1971
1972	static bool isLegalUse(const TargetTransformInfo &TTI, Immediate MinOffset,
1973	Immediate MaxOffset, LSRUse::KindType Kind,
1974	MemAccessTy AccessTy, const Formula &F) {
1975	return isLegalUse(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV: F.BaseGV,
1976	BaseOffset: F.BaseOffset, HasBaseReg: F.HasBaseReg, Scale: F.Scale);
1977	}
1978
1979	static bool isLegalAddImmediate(const TargetTransformInfo &TTI,
1980	Immediate Offset) {
1981	if (Offset.isScalable())
1982	return TTI.isLegalAddScalableImmediate(Imm: Offset.getKnownMinValue());
1983
1984	return TTI.isLegalAddImmediate(Imm: Offset.getFixedValue());
1985	}
1986
1987	static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1988	const LSRUse &LU, const Formula &F) {
1989	// Target may want to look at the user instructions.
1990	if (LU.Kind == LSRUse::Address && TTI.LSRWithInstrQueries()) {
1991	for (const LSRFixup &Fixup : LU.Fixups)
1992	if (!isAMCompletelyFolded(TTI, Kind: LSRUse::Address, AccessTy: LU.AccessTy, BaseGV: F.BaseGV,
1993	BaseOffset: (F.BaseOffset + Fixup.Offset), HasBaseReg: F.HasBaseReg,
1994	Scale: F.Scale, Fixup: Fixup.UserInst))
1995	return false;
1996	return true;
1997	}
1998
1999	return isAMCompletelyFolded(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
2000	AccessTy: LU.AccessTy, BaseGV: F.BaseGV, BaseOffset: F.BaseOffset, HasBaseReg: F.HasBaseReg,
2001	Scale: F.Scale);
2002	}
2003
2004	static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI,
2005	const LSRUse &LU, const Formula &F,
2006	const Loop &L) {
2007	if (!F.Scale)
2008	return `0`;
2009
2010	// If the use is not completely folded in that instruction, we will have to
2011	// pay an extra cost only for scale != 1.
2012	if (!isAMCompletelyFolded(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
2013	AccessTy: LU.AccessTy, F, L))
2014	return F.Scale != `1`;
2015
2016	switch (LU.Kind) {
2017	case LSRUse::Address: {
2018	// Check the scaling factor cost with both the min and max offsets.
2019	int64_t ScalableMin = `0`, ScalableMax = `0`, FixedMin = `0`, FixedMax = `0`;
2020	if (F.BaseOffset.isScalable()) {
2021	ScalableMin = (F.BaseOffset + LU.MinOffset).getKnownMinValue();
2022	ScalableMax = (F.BaseOffset + LU.MaxOffset).getKnownMinValue();
2023	} else {
2024	FixedMin = (F.BaseOffset + LU.MinOffset).getFixedValue();
2025	FixedMax = (F.BaseOffset + LU.MaxOffset).getFixedValue();
2026	}
2027	InstructionCost ScaleCostMinOffset = TTI.getScalingFactorCost(
2028	Ty: LU.AccessTy.MemTy, BaseGV: F.BaseGV, BaseOffset: StackOffset::get(Fixed: FixedMin, Scalable: ScalableMin),
2029	HasBaseReg: F.HasBaseReg, Scale: F.Scale, AddrSpace: LU.AccessTy.AddrSpace);
2030	InstructionCost ScaleCostMaxOffset = TTI.getScalingFactorCost(
2031	Ty: LU.AccessTy.MemTy, BaseGV: F.BaseGV, BaseOffset: StackOffset::get(Fixed: FixedMax, Scalable: ScalableMax),
2032	HasBaseReg: F.HasBaseReg, Scale: F.Scale, AddrSpace: LU.AccessTy.AddrSpace);
2033
2034	assert(ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() &&
2035	"Legal addressing mode has an illegal cost!");
2036	return std::max(a: ScaleCostMinOffset, b: ScaleCostMaxOffset);
2037	}
2038	case LSRUse::ICmpZero:
2039	case LSRUse::Basic:
2040	case LSRUse::Special:
2041	// The use is completely folded, i.e., everything is folded into the
2042	// instruction.
2043	return `0`;
2044	}
2045
2046	llvm_unreachable("Invalid LSRUse Kind!");
2047	}
2048
2049	static bool isAlwaysFoldable(const TargetTransformInfo &TTI,
2050	LSRUse::KindType Kind, MemAccessTy AccessTy,
2051	GlobalValue *BaseGV, Immediate BaseOffset,
2052	bool HasBaseReg) {
2053	// Fast-path: zero is always foldable.
2054	if (BaseOffset.isZero() && !BaseGV)
2055	return true;
2056
2057	// Conservatively, create an address with an immediate and a
2058	// base and a scale.
2059	int64_t Scale = Kind == LSRUse::ICmpZero ? -`1` : `1`;
2060
2061	// Canonicalize a scale of 1 to a base register if the formula doesn't
2062	// already have a base register.
2063	if (!HasBaseReg && Scale == `1`) {
2064	Scale = `0`;
2065	HasBaseReg = true;
2066	}
2067
2068	// FIXME: Try with + without a scale? Maybe based on TTI?
2069	// I think basereg + scaledreg + immediateoffset isn't a good 'conservative'
2070	// default for many architectures, not just AArch64 SVE. More investigation
2071	// needed later to determine if this should be used more widely than just
2072	// on scalable types.
2073	if (HasBaseReg && BaseOffset.isNonZero() && Kind != LSRUse::ICmpZero &&
2074	AccessTy.MemTy && AccessTy.MemTy->isScalableTy() && DropScaledForVScale)
2075	Scale = `0`;
2076
2077	return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset,
2078	HasBaseReg, Scale);
2079	}
2080
2081	static bool isAlwaysFoldable(const TargetTransformInfo &TTI,
2082	ScalarEvolution &SE, Immediate MinOffset,
2083	Immediate MaxOffset, LSRUse::KindType Kind,
2084	MemAccessTy AccessTy, const SCEV *S,
2085	bool HasBaseReg) {
2086	// Fast-path: zero is always foldable.
2087	if (S->isZero()) return true;
2088
2089	// Conservatively, create an address with an immediate and a
2090	// base and a scale.
2091	SCEVUse SCopy = S;
2092	Immediate BaseOffset = ExtractImmediate(S&: SCopy, SE);
2093	GlobalValue *BaseGV = ExtractSymbol(S&: SCopy, SE);
2094
2095	// If there's anything else involved, it's not foldable.
2096	if (!SCopy ->isZero())
2097	return false;
2098
2099	// Fast-path: zero is always foldable.
2100	if (BaseOffset.isZero() && !BaseGV)
2101	return true;
2102
2103	if (BaseOffset.isScalable())
2104	return false;
2105
2106	// Conservatively, create an address with an immediate and a
2107	// base and a scale.
2108	int64_t Scale = Kind == LSRUse::ICmpZero ? -`1` : `1`;
2109
2110	return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
2111	BaseOffset, HasBaseReg, Scale);
2112	}
2113
2114	namespace {
2115
2116	/// An individual increment in a Chain of IV increments. Relate an IV user to
2117	/// an expression that computes the IV it uses from the IV used by the previous
2118	/// link in the Chain.
2119	///
2120	/// For the head of a chain, IncExpr holds the absolute SCEV expression for the
2121	/// original IVOperand. The head of the chain's IVOperand is only valid during
2122	/// chain collection, before LSR replaces IV users. During chain generation,
2123	/// IncExpr can be used to find the new IVOperand that computes the same
2124	/// expression.
2125	struct IVInc {
2126	Instruction *UserInst;
2127	Value* IVOperand;
2128	const SCEV *IncExpr;
2129
2130	IVInc(Instruction U, Value O, const SCEV *E)
2131	: UserInst(U), IVOperand(O), IncExpr(E) {}
2132	};
2133
2134	// The list of IV increments in program order. We typically add the head of a
2135	// chain without finding subsequent links.
2136	struct IVChain {
2137	SmallVector<IVInc, `1`> Incs;
2138	const SCEV ExprBase = nullptr*;
2139
2140	IVChain() = default;
2141	IVChain(const IVInc &Head, const SCEV *Base)
2142	: Incs (`1`, Head), ExprBase(Base) {}
2143
2144	using const_iterator = SmallVectorImpl<IVInc>::const_iterator;
2145
2146	// Return the first increment in the chain.
2147	const_iterator begin() const {
2148	assert(!Incs.empty());
2149	return std::next(x: Incs.begin());
2150	}
2151	const_iterator end() const {
2152	return Incs.end();
2153	}
2154
2155	// Returns true if this chain contains any increments.
2156	bool hasIncs() const { return Incs.size() >= `2`; }
2157
2158	// Add an IVInc to the end of this chain.
2159	void add(const IVInc &X) { Incs.push_back(Elt: X); }
2160
2161	// Returns the last UserInst in the chain.
2162	Instruction tailUserInst() const* { return Incs.back().UserInst; }
2163
2164	// Returns true if IncExpr can be profitably added to this chain.
2165	bool isProfitableIncrement(const SCEV *OperExpr,
2166	const SCEV *IncExpr,
2167	ScalarEvolution&);
2168	};
2169
2170	/// Helper for CollectChains to track multiple IV increment uses. Distinguish
2171	/// between FarUsers that definitely cross IV increments and NearUsers that may
2172	/// be used between IV increments.
2173	struct ChainUsers {
2174	SmallPtrSet<Instruction*, `4`> FarUsers;
2175	SmallPtrSet<Instruction*, `4`> NearUsers;
2176	};
2177
2178	/// This class holds state for the main loop strength reduction logic.
2179	class LSRInstance {
2180	IVUsers &IU;
2181	ScalarEvolution &SE;
2182	DominatorTree &DT;
2183	LoopInfo &LI;
2184	AssumptionCache &AC;
2185	TargetLibraryInfo &TLI;
2186	const TargetTransformInfo &TTI;
2187	Loop *const L;
2188	MemorySSAUpdater *MSSAU;
2189	TTI::AddressingModeKind AMK;
2190	mutable SCEVExpander Rewriter;
2191	bool Changed = false;
2192	bool HardwareLoopProfitable = false;
2193	bool ShouldPreserveLCSSA = false;
2194
2195	/// This is the insert position that the current loop's induction variable
2196	/// increment should be placed. In simple loops, this is the latch block's
2197	/// terminator. But in more complicated cases, this is a position which will
2198	/// dominate all the in-loop post-increment users.
2199	Instruction IVIncInsertPos = nullptr*;
2200
2201	/// Interesting factors between use strides.
2202	///
2203	/// We explicitly use a SetVector which contains a SmallSet, instead of the
2204	/// default, a SmallDenseSet, because we need to use the full range of
2205	/// int64_ts, and there's currently no good way of doing that with
2206	/// SmallDenseSet.
2207	SetVector<int64_t, SmallVector<int64_t, `8`>, SmallSet<int64_t, `8`>> Factors;
2208
2209	/// The cost of the current SCEV, the best solution by LSR will be dropped if
2210	/// the solution is not profitable.
2211	Cost BaselineCost;
2212
2213	/// Interesting use types, to facilitate truncation reuse.
2214	SmallSetVector<Type *, `4`> Types;
2215
2216	/// The list of interesting uses.
2217	mutable SmallVector<LSRUse, `16`> Uses;
2218
2219	/// Track which uses use which register candidates.
2220	RegUseTracker RegUses;
2221
2222	// Limit the number of chains to avoid quadratic behavior. We don't expect to
2223	// have more than a few IV increment chains in a loop. Missing a Chain falls
2224	// back to normal LSR behavior for those uses.
2225	static const unsigned MaxChains = `8`;
2226
2227	/// IV users can form a chain of IV increments.
2228	SmallVector<IVChain, MaxChains> IVChainVec;
2229
2230	/// IV users that belong to profitable IVChains.
2231	SmallPtrSet<Use*, MaxChains> IVIncSet;
2232
2233	/// Induction variables that were generated and inserted by the SCEV Expander.
2234	SmallVector<llvm::WeakVH, `2`> ScalarEvolutionIVs;
2235
2236	// Inserting instructions in the loop and using them as PHI's input could
2237	// break LCSSA in case if PHI's parent block is not a loop exit (i.e. the
2238	// corresponding incoming block is not loop exiting). So collect all such
2239	// instructions to form LCSSA for them later.
2240	SmallSetVector<Instruction *, `4`> InsertedNonLCSSAInsts;
2241
2242	void OptimizeShadowIV();
2243	bool FindIVUserForCond(Instruction Cond, IVStrideUse &CondUse);
2244	Instruction OptimizeMax(ICmpInst Cond, IVStrideUse *&CondUse);
2245	void OptimizeLoopTermCond();
2246
2247	void ChainInstruction(Instruction UserInst, Instruction IVOper,
2248	SmallVectorImpl<ChainUsers> &ChainUsersVec);
2249	void FinalizeChain(IVChain &Chain);
2250	void CollectChains();
2251	void GenerateIVChain(const IVChain &Chain,
2252	SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2253
2254	void CollectInterestingTypesAndFactors();
2255	void CollectFixupsAndInitialFormulae();
2256
2257	// Support for sharing of LSRUses between LSRFixups.
2258	using UseMapTy = DenseMap<LSRUse::SCEVUseKindPair, size_t>;
2259	UseMapTy UseMap;
2260
2261	bool reconcileNewOffset(LSRUse &LU, Immediate NewOffset, bool HasBaseReg,
2262	LSRUse::KindType Kind, MemAccessTy AccessTy);
2263
2264	std::pair<size_t, Immediate> getUse(const SCEV *&Expr, LSRUse::KindType Kind,
2265	MemAccessTy AccessTy);
2266
2267	void DeleteUse(LSRUse &LU, size_t LUIdx);
2268
2269	LSRUse FindUseWithSimilarFormula(const* Formula &F, const LSRUse &OrigLU);
2270
2271	void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2272	void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2273	void CountRegisters(const Formula &F, size_t LUIdx);
2274	bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
2275	bool IsFixupExecutedEachIncrement(const LSRFixup &LF) const;
2276
2277	void CollectLoopInvariantFixupsAndFormulae();
2278
2279	void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
2280	unsigned Depth = `0`);
2281
2282	void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
2283	const Formula &Base, unsigned Depth,
2284	size_t Idx, bool IsScaledReg = false);
2285	void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
2286	void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2287	const Formula &Base, size_t Idx,
2288	bool IsScaledReg = false);
2289	void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2290	void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2291	const Formula &Base,
2292	const SmallVectorImpl<Immediate> &Worklist,
2293	size_t Idx, bool IsScaledReg = false);
2294	void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2295	void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2296	void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2297	void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
2298	void GenerateCrossUseConstantOffsets();
2299	void GenerateAllReuseFormulae();
2300
2301	void FilterOutUndesirableDedicatedRegisters();
2302
2303	size_t EstimateSearchSpaceComplexity() const;
2304	void NarrowSearchSpaceByDetectingSupersets();
2305	void NarrowSearchSpaceByCollapsingUnrolledCode();
2306	void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
2307	void NarrowSearchSpaceByFilterFormulaWithSameScaledReg();
2308	void NarrowSearchSpaceByFilterPostInc();
2309	void NarrowSearchSpaceByMergingUsesOutsideLoop();
2310	void NarrowSearchSpaceByDeletingCostlyFormulas();
2311	void NarrowSearchSpaceByPickingWinnerRegs();
2312	void NarrowSearchSpaceUsingHeuristics();
2313
2314	void SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
2315	Cost &SolutionCost,
2316	SmallVectorImpl<const Formula *> &Workspace,
2317	const Cost &CurCost,
2318	const SmallPtrSet<const SCEV *, `16`> &CurRegs,
2319	DenseSet<const SCEV > &VisitedRegs) const*;
2320	void Solve(SmallVectorImpl<const Formula > &Solution) const*;
2321
2322	BasicBlock::iterator
2323	HoistInsertPosition(BasicBlock::iterator IP,
2324	const SmallVectorImpl<Instruction > &Inputs) const*;
2325	BasicBlock::iterator AdjustInsertPositionForExpand(BasicBlock::iterator IP,
2326	const LSRFixup &LF,
2327	const LSRUse &LU) const;
2328
2329	Value Expand(const* LSRUse &LU, const LSRFixup &LF, const Formula &F,
2330	BasicBlock::iterator IP,
2331	SmallVectorImpl<WeakTrackingVH> &DeadInsts) const;
2332	void RewriteForPHI(PHINode PN, const* LSRUse &LU, const LSRFixup &LF,
2333	const Formula &F,
2334	SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2335	void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2336	SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2337	void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution);
2338
2339	public:
2340	// TODO(boomanaiden154): The PreserveLCSSA flag is a hack to allow
2341	// experimentation with the NewPM which requires LCSSA preservation while
2342	// some of the details are worked out in LSR. Eventually it should be set
2343	// to true and removed.
2344	LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT,
2345	LoopInfo &LI, const TargetTransformInfo &TTI, AssumptionCache &AC,
2346	TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU,
2347	bool PreserveLCSSA);
2348
2349	bool getChanged() const { return Changed; }
2350	const SmallVectorImpl<WeakVH> &getScalarEvolutionIVs() const {
2351	return ScalarEvolutionIVs;
2352	}
2353
2354	void print_factors_and_types(raw_ostream &OS) const;
2355	void print_fixups(raw_ostream &OS) const;
2356	void print_uses(raw_ostream &OS) const;
2357	void print(raw_ostream &OS) const;
2358	void dump() const;
2359	};
2360
2361	} // end anonymous namespace
2362
2363	/// If IV is used in a int-to-float cast inside the loop then try to eliminate
2364	/// the cast operation.
2365	void LSRInstance::OptimizeShadowIV() {
2366	const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2367	if (isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount))
2368	return;
2369
2370	for (IVUsers::const_iterator UI = IU.begin(), E = IU.end();
2371	UI != E; / empty /) {
2372	IVUsers::const_iterator CandidateUI = UI;
2373	++UI;
2374	Instruction *ShadowUse = CandidateUI ->getUser();
2375	Type DestTy = nullptr*;
2376	bool IsSigned = false;
2377
2378	/ If shadow use is a int->float cast then insert a second IV*
2379	to eliminate this cast.
2380
2381	for (unsigned i = 0; i < n; ++i)
2382	foo((double)i);
2383
2384	is transformed into
2385
2386	double d = 0.0;
2387	for (unsigned i = 0; i < n; ++i, ++d)
2388	foo(d);
2389	*/
2390	if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(Val: CandidateUI ->getUser())) {
2391	IsSigned = false;
2392	DestTy = UCast->getDestTy();
2393	}
2394	else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(Val: CandidateUI ->getUser())) {
2395	IsSigned = true;
2396	DestTy = SCast->getDestTy();
2397	}
2398	if (!DestTy) continue;
2399
2400	// If target does not support DestTy natively then do not apply
2401	// this transformation.
2402	if (!TTI.isTypeLegal(Ty: DestTy)) continue;
2403
2404	PHINode *PH = dyn_cast<PHINode>(Val: ShadowUse->getOperand(i: `0`));
2405	if (!PH) continue;
2406	if (PH->getNumIncomingValues() != `2`) continue;
2407
2408	// If the calculation in integers overflows, the result in FP type will
2409	// differ. So we only can do this transformation if we are guaranteed to not
2410	// deal with overflowing values
2411	const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: SE.getSCEV(V: PH));
2412	if (!AR) continue;
2413	if (IsSigned && !AR->hasNoSignedWrap()) continue;
2414	if (!IsSigned && !AR->hasNoUnsignedWrap()) continue;
2415
2416	Type *SrcTy = PH->getType();
2417	int Mantissa = DestTy->getFPMantissaWidth();
2418	if (Mantissa == -`1`) continue;
2419	if ((int)SE.getTypeSizeInBits(Ty: SrcTy) > Mantissa)
2420	continue;
2421
2422	unsigned Entry, Latch;
2423	if (PH->getIncomingBlock(i: `0`) == L->getLoopPreheader()) {
2424	Entry = `0`;
2425	Latch = `1`;
2426	} else {
2427	Entry = `1`;
2428	Latch = `0`;
2429	}
2430
2431	ConstantInt *Init = dyn_cast<ConstantInt>(Val: PH->getIncomingValue(i: Entry));
2432	if (!Init) continue;
2433	Constant *NewInit = ConstantFP::get(Ty: DestTy, V: IsSigned ?
2434	(double)Init->getSExtValue() :
2435	(double)Init->getZExtValue());
2436
2437	BinaryOperator *Incr =
2438	dyn_cast<BinaryOperator>(Val: PH->getIncomingValue(i: Latch));
2439	if (!Incr) continue;
2440	if (Incr->getOpcode() != Instruction::Add
2441	&& Incr->getOpcode() != Instruction::Sub)
2442	continue;
2443
2444	/ Initialize new IV, double d = 0.0 in above example. /
2445	ConstantInt C = nullptr*;
2446	if (Incr->getOperand(i_nocapture: `0`) == PH)
2447	C = dyn_cast<ConstantInt>(Val: Incr->getOperand(i_nocapture: `1`));
2448	else if (Incr->getOperand(i_nocapture: `1`) == PH)
2449	C = dyn_cast<ConstantInt>(Val: Incr->getOperand(i_nocapture: `0`));
2450	else
2451	continue;
2452
2453	if (!C) continue;
2454
2455	// Ignore negative constants, as the code below doesn't handle them
2456	// correctly. TODO: Remove this restriction.
2457	if (!C->getValue().isStrictlyPositive())
2458	continue;
2459
2460	/ Add new PHINode. /
2461	PHINode *NewPH = PHINode::Create(Ty: DestTy, NumReservedValues: `2`, NameStr: "IV.S.", InsertBefore: PH->getIterator());
2462	NewPH->setDebugLoc(PH->getDebugLoc());
2463
2464	/ create new increment. '++d' in above example. /
2465	Constant *CFP = ConstantFP::get(Ty: DestTy, V: C->getZExtValue());
2466	BinaryOperator *NewIncr = BinaryOperator::Create(
2467	Op: Incr->getOpcode() == Instruction::Add ? Instruction::FAdd
2468	: Instruction::FSub,
2469	S1: NewPH, S2: CFP, Name: "IV.S.next.", InsertBefore: Incr->getIterator());
2470	NewIncr->setDebugLoc(Incr->getDebugLoc());
2471
2472	NewPH->addIncoming(V: NewInit, BB: PH->getIncomingBlock(i: Entry));
2473	NewPH->addIncoming(V: NewIncr, BB: PH->getIncomingBlock(i: Latch));
2474
2475	/ Remove cast operation /
2476	ShadowUse->replaceAllUsesWith(V: NewPH);
2477	ShadowUse->eraseFromParent();
2478	Changed = true;
2479	break;
2480	}
2481	}
2482
2483	/// If Cond has an operand that is an expression of an IV, set the IV user and
2484	/// stride information and return true, otherwise return false.
2485	bool LSRInstance::FindIVUserForCond(Instruction Cond, IVStrideUse &CondUse) {
2486	for (IVStrideUse &U : IU)
2487	if (U.getUser() == Cond) {
2488	// NOTE: we could handle setcc instructions with multiple uses here, but
2489	// InstCombine does it as well for simple uses, it's not clear that it
2490	// occurs enough in real life to handle.
2491	CondUse = &U;
2492	return true;
2493	}
2494	return false;
2495	}
2496
2497	/// Rewrite the loop's terminating condition if it uses a max computation.
2498	///
2499	/// This is a narrow solution to a specific, but acute, problem. For loops
2500	/// like this:
2501	///
2502	/// i = 0;
2503	/// do {
2504	/// p[i] = 0.0;
2505	/// } while (++i < n);
2506	///
2507	/// the trip count isn't just 'n', because 'n' might not be positive. And
2508	/// unfortunately this can come up even for loops where the user didn't use
2509	/// a C do-while loop. For example, seemingly well-behaved top-test loops
2510	/// will commonly be lowered like this:
2511	///
2512	/// if (n > 0) {
2513	/// i = 0;
2514	/// do {
2515	/// p[i] = 0.0;
2516	/// } while (++i < n);
2517	/// }
2518	///
2519	/// and then it's possible for subsequent optimization to obscure the if
2520	/// test in such a way that indvars can't find it.
2521	///
2522	/// When indvars can't find the if test in loops like this, it creates a
2523	/// max expression, which allows it to give the loop a canonical
2524	/// induction variable:
2525	///
2526	/// i = 0;
2527	/// max = n < 1 ? 1 : n;
2528	/// do {
2529	/// p[i] = 0.0;
2530	/// } while (++i != max);
2531	///
2532	/// Canonical induction variables are necessary because the loop passes
2533	/// are designed around them. The most obvious example of this is the
2534	/// LoopInfo analysis, which doesn't remember trip count values. It
2535	/// expects to be able to rediscover the trip count each time it is
2536	/// needed, and it does this using a simple analysis that only succeeds if
2537	/// the loop has a canonical induction variable.
2538	///
2539	/// However, when it comes time to generate code, the maximum operation
2540	/// can be quite costly, especially if it's inside of an outer loop.
2541	///
2542	/// This function solves this problem by detecting this type of loop and
2543	/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting
2544	/// the instructions for the maximum computation.
2545	Instruction LSRInstance::OptimizeMax(ICmpInst Cond, IVStrideUse *&CondUse) {
2546	// Check that the loop matches the pattern we're looking for.
2547	if (Cond->getPredicate() != CmpInst::ICMP_EQ &&
2548	Cond->getPredicate() != CmpInst::ICMP_NE)
2549	return Cond;
2550
2551	SelectInst *Sel = dyn_cast<SelectInst>(Val: Cond->getOperand(i_nocapture: `1`));
2552	if (!Sel \|\| !Sel->hasOneUse()) return Cond;
2553
2554	const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2555	if (isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount))
2556	return Cond;
2557	const SCEV *One = SE.getConstant(Ty: BackedgeTakenCount->getType(), V: `1`);
2558
2559	// Add one to the backedge-taken count to get the trip count.
2560	const SCEV *IterationCount = SE.getAddExpr(LHS: One, RHS: BackedgeTakenCount);
2561	if (IterationCount != SE.getSCEV(V: Sel)) return Cond;
2562
2563	// Check for a max calculation that matches the pattern. There's no check
2564	// for ICMP_ULE here because the comparison would be with zero, which
2565	// isn't interesting.
2566	CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
2567	const SCEVNAryExpr Max = nullptr*;
2568	if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(Val: BackedgeTakenCount)) {
2569	Pred = ICmpInst::ICMP_SLE;
2570	Max = S;
2571	} else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(Val: IterationCount)) {
2572	Pred = ICmpInst::ICMP_SLT;
2573	Max = S;
2574	} else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(Val: IterationCount)) {
2575	Pred = ICmpInst::ICMP_ULT;
2576	Max = U;
2577	} else {
2578	// No match; bail.
2579	return Cond;
2580	}
2581
2582	// To handle a max with more than two operands, this optimization would
2583	// require additional checking and setup.
2584	if (Max->getNumOperands() != `2`)
2585	return Cond;
2586
2587	const SCEV *MaxLHS = Max->getOperand(i: `0`);
2588	const SCEV *MaxRHS = Max->getOperand(i: `1`);
2589
2590	// ScalarEvolution canonicalizes constants to the left. For < and >, look
2591	// for a comparison with 1. For <= and >=, a comparison with zero.
2592	if (!MaxLHS \|\|
2593	(ICmpInst::isTrueWhenEqual(predicate: Pred) ? !MaxLHS->isZero() : (MaxLHS != One)))
2594	return Cond;
2595
2596	// Check the relevant induction variable for conformance to
2597	// the pattern.
2598	const SCEV *IV = SE.getSCEV(V: Cond->getOperand(i_nocapture: `0`));
2599	if (!match(S: IV,
2600	P: m_scev_AffineAddRec(Op0: m_scev_SpecificInt(V: `1`), Op1: m_scev_SpecificInt(V: `1`))))
2601	return Cond;
2602
2603	assert(cast<SCEVAddRecExpr>(IV)->getLoop() == L &&
2604	"Loop condition operand is an addrec in a different loop!");
2605
2606	// Check the right operand of the select, and remember it, as it will
2607	// be used in the new comparison instruction.
2608	Value NewRHS = nullptr*;
2609	if (ICmpInst::isTrueWhenEqual(predicate: Pred)) {
2610	// Look for n+1, and grab n.
2611	if (AddOperator *BO = dyn_cast<AddOperator>(Val: Sel->getOperand(i_nocapture: `1`)))
2612	if (ConstantInt *BO1 = dyn_cast<ConstantInt>(Val: BO->getOperand(i_nocapture: `1`)))
2613	if (BO1->isOne() && SE.getSCEV(V: BO->getOperand(i_nocapture: `0`)) == MaxRHS)
2614	NewRHS = BO->getOperand(i_nocapture: `0`);
2615	if (AddOperator *BO = dyn_cast<AddOperator>(Val: Sel->getOperand(i_nocapture: `2`)))
2616	if (ConstantInt *BO1 = dyn_cast<ConstantInt>(Val: BO->getOperand(i_nocapture: `1`)))
2617	if (BO1->isOne() && SE.getSCEV(V: BO->getOperand(i_nocapture: `0`)) == MaxRHS)
2618	NewRHS = BO->getOperand(i_nocapture: `0`);
2619	if (!NewRHS)
2620	return Cond;
2621	} else if (SE.getSCEV(V: Sel->getOperand(i_nocapture: `1`)) == MaxRHS)
2622	NewRHS = Sel->getOperand(i_nocapture: `1`);
2623	else if (SE.getSCEV(V: Sel->getOperand(i_nocapture: `2`)) == MaxRHS)
2624	NewRHS = Sel->getOperand(i_nocapture: `2`);
2625	else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(Val: MaxRHS))
2626	NewRHS = SU->getValue();
2627	else
2628	// Max doesn't match expected pattern.
2629	return Cond;
2630
2631	// Determine the new comparison opcode. It may be signed or unsigned,
2632	// and the original comparison may be either equality or inequality.
2633	if (Cond->getPredicate() == CmpInst::ICMP_EQ)
2634	Pred = CmpInst::getInversePredicate(pred: Pred);
2635
2636	// Ok, everything looks ok to change the condition into an SLT or SGE and
2637	// delete the max calculation.
2638	ICmpInst NewCond = new* ICmpInst (Cond->getIterator(), Pred,
2639	Cond->getOperand(i_nocapture: `0`), NewRHS, "scmp");
2640
2641	// Delete the max calculation instructions.
2642	NewCond->setDebugLoc(Cond->getDebugLoc());
2643	Cond->replaceAllUsesWith(V: NewCond);
2644	CondUse->setUser(NewCond);
2645	Instruction *Cmp = cast<Instruction>(Val: Sel->getOperand(i_nocapture: `0`));
2646	Cond->eraseFromParent();
2647	Sel->eraseFromParent();
2648	if (Cmp->use_empty()) {
2649	salvageDebugInfo(I&: *Cmp);
2650	Cmp->eraseFromParent();
2651	}
2652	return NewCond;
2653	}
2654
2655	/// Change loop terminating condition to use the postinc iv when possible.
2656	void
2657	LSRInstance::OptimizeLoopTermCond() {
2658	SmallPtrSet<Instruction *, `4`> PostIncs;
2659
2660	// We need a different set of heuristics for rotated and non-rotated loops.
2661	// If a loop is rotated then the latch is also the backedge, so inserting
2662	// post-inc expressions just before the latch is ideal. To reduce live ranges
2663	// it also makes sense to rewrite terminating conditions to use post-inc
2664	// expressions.
2665	//
2666	// If the loop is not rotated then the latch is not a backedge; the latch
2667	// check is done in the loop head. Adding post-inc expressions before the
2668	// latch will cause overlapping live-ranges of pre-inc and post-inc expressions
2669	// in the loop body. In this case we do not* want to use post-inc expressions*
2670	// in the latch check, and we want to insert post-inc expressions before
2671	// the backedge.
2672	BasicBlock *LatchBlock = L->getLoopLatch();
2673	SmallVector<BasicBlock*, `8`> ExitingBlocks;
2674	L->getExitingBlocks(ExitingBlocks);
2675	if (!llvm::is_contained(Range&: ExitingBlocks, Element: LatchBlock)) {
2676	// The backedge doesn't exit the loop; treat this as a head-tested loop.
2677	IVIncInsertPos = LatchBlock->getTerminator();
2678	return;
2679	}
2680
2681	// Otherwise treat this as a rotated loop.
2682	for (BasicBlock *ExitingBlock : ExitingBlocks) {
2683	// Get the terminating condition for the loop if possible. If we
2684	// can, we want to change it to use a post-incremented version of its
2685	// induction variable, to allow coalescing the live ranges for the IV into
2686	// one register value.
2687
2688	CondBrInst *TermBr = dyn_cast<CondBrInst>(Val: ExitingBlock->getTerminator());
2689	if (!TermBr)
2690	continue;
2691
2692	Instruction *Cond = dyn_cast<Instruction>(Val: TermBr->getCondition());
2693	// If the argument to TermBr is an extractelement, then the source of that
2694	// instruction is what's generated the condition.
2695	auto *Extract = dyn_cast_or_null<ExtractElementInst>(Val: Cond);
2696	if (Extract)
2697	Cond = dyn_cast<Instruction>(Val: Extract->getVectorOperand());
2698	// FIXME: We could do more here, like handling logical operations where one
2699	// side is a cmp that uses an induction variable.
2700	if (!Cond)
2701	continue;
2702
2703	// Search IVUsesByStride to find Cond's IVUse if there is one.
2704	IVStrideUse CondUse = nullptr*;
2705	if (!FindIVUserForCond(Cond, CondUse))
2706	continue;
2707
2708	// If the trip count is computed in terms of a max (due to ScalarEvolution
2709	// being unable to find a sufficient guard, for example), change the loop
2710	// comparison to use SLT or ULT instead of NE.
2711	// One consequence of doing this now is that it disrupts the count-down
2712	// optimization. That's not always a bad thing though, because in such
2713	// cases it may still be worthwhile to avoid a max.
2714	if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond))
2715	Cond = OptimizeMax(Cond: Cmp, CondUse);
2716
2717	// If this exiting block dominates the latch block, it may also use
2718	// the post-inc value if it won't be shared with other uses.
2719	// Check for dominance.
2720	if (!DT.dominates(A: ExitingBlock, B: LatchBlock))
2721	continue;
2722
2723	// Conservatively avoid trying to use the post-inc value in non-latch
2724	// exits if there may be pre-inc users in intervening blocks.
2725	if (LatchBlock != ExitingBlock)
2726	for (const IVStrideUse &UI : IU)
2727	// Test if the use is reachable from the exiting block. This dominator
2728	// query is a conservative approximation of reachability.
2729	if (&UI != CondUse &&
2730	!DT.properlyDominates(A: UI.getUser()->getParent(), B: ExitingBlock)) {
2731	// Conservatively assume there may be reuse if the quotient of their
2732	// strides could be a legal scale.
2733	const SCEV A = IU.getStride(IU: CondUse, L);
2734	const SCEV *B = IU.getStride(IU: UI, L);
2735	if (!A \|\| !B) continue;
2736	if (SE.getTypeSizeInBits(Ty: A->getType()) !=
2737	SE.getTypeSizeInBits(Ty: B->getType())) {
2738	if (SE.getTypeSizeInBits(Ty: A->getType()) >
2739	SE.getTypeSizeInBits(Ty: B->getType()))
2740	B = SE.getSignExtendExpr(Op: B, Ty: A->getType());
2741	else
2742	A = SE.getSignExtendExpr(Op: A, Ty: B->getType());
2743	}
2744	if (const SCEVConstant *D =
2745	dyn_cast_or_null<SCEVConstant>(Val: getExactSDiv(LHS: B, RHS: A, SE))) {
2746	const ConstantInt *C = D->getValue();
2747	// Stride of one or negative one can have reuse with non-addresses.
2748	if (C->isOne() \|\| C->isMinusOne())
2749	goto decline_post_inc;
2750	// Avoid weird situations.
2751	if (C->getValue().getSignificantBits() >= `64` \|\|
2752	C->getValue().isMinSignedValue())
2753	goto decline_post_inc;
2754	// Check for possible scaled-address reuse.
2755	if (isAddressUse(TTI, Inst: UI.getUser(), OperandVal: UI.getOperandValToReplace())) {
2756	MemAccessTy AccessTy =
2757	getAccessType(TTI, Inst: UI.getUser(), OperandVal: UI.getOperandValToReplace());
2758	int64_t Scale = C->getSExtValue();
2759	if (TTI.isLegalAddressingMode(Ty: AccessTy.MemTy, /BaseGV=/nullptr,
2760	/BaseOffset=/`0`,
2761	/HasBaseReg=/true, Scale,
2762	AddrSpace: AccessTy.AddrSpace))
2763	goto decline_post_inc;
2764	Scale = -Scale;
2765	if (TTI.isLegalAddressingMode(Ty: AccessTy.MemTy, /BaseGV=/nullptr,
2766	/BaseOffset=/`0`,
2767	/HasBaseReg=/true, Scale,
2768	AddrSpace: AccessTy.AddrSpace))
2769	goto decline_post_inc;
2770	}
2771	}
2772	}
2773
2774	LLVM_DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: "
2775	<< *Cond << `'\n'`);
2776
2777	// It's possible for the setcc instruction to be anywhere in the loop, and
2778	// possible for it to have multiple users. If it is not immediately before
2779	// the exiting block branch, move it.
2780	if (isa_and_nonnull<CmpInst>(Val: Cond) && Cond->getNextNode() != TermBr &&
2781	!Extract) {
2782	if (Cond->hasOneUse()) {
2783	Cond->moveBefore(InsertPos: TermBr->getIterator());
2784	} else {
2785	// Clone the terminating condition and insert into the loopend.
2786	Instruction *OldCond = Cond;
2787	Cond = Cond->clone();
2788	Cond->setName(L->getHeader()->getName() + ".termcond");
2789	Cond->insertInto(ParentBB: ExitingBlock, It: TermBr->getIterator());
2790
2791	// Clone the IVUse, as the old use still exists!
2792	CondUse = &IU.AddUser(User: Cond, Operand: CondUse->getOperandValToReplace());
2793	TermBr->replaceUsesOfWith(From: OldCond, To: Cond);
2794	}
2795	}
2796
2797	// If we get to here, we know that we can transform the setcc instruction to
2798	// use the post-incremented version of the IV, allowing us to coalesce the
2799	// live ranges for the IV correctly.
2800	CondUse->transformToPostInc(L);
2801	Changed = true;
2802
2803	PostIncs.insert(Ptr: Cond);
2804	decline_post_inc:;
2805	}
2806
2807	// Determine an insertion point for the loop induction variable increment. It
2808	// must dominate all the post-inc comparisons we just set up, and it must
2809	// dominate the loop latch edge.
2810	IVIncInsertPos = L->getLoopLatch()->getTerminator();
2811	for (Instruction *Inst : PostIncs)
2812	IVIncInsertPos = DT.findNearestCommonDominator(I1: IVIncInsertPos, I2: Inst);
2813	}
2814
2815	/// Determine if the given use can accommodate a fixup at the given offset and
2816	/// other details. If so, update the use and return true.
2817	bool LSRInstance::reconcileNewOffset(LSRUse &LU, Immediate NewOffset,
2818	bool HasBaseReg, LSRUse::KindType Kind,
2819	MemAccessTy AccessTy) {
2820	Immediate NewMinOffset = LU.MinOffset;
2821	Immediate NewMaxOffset = LU.MaxOffset;
2822	MemAccessTy NewAccessTy = AccessTy;
2823
2824	// Check for a mismatched kind. It's tempting to collapse mismatched kinds to
2825	// something conservative, however this can pessimize in the case that one of
2826	// the uses will have all its uses outside the loop, for example.
2827	if (LU.Kind != Kind)
2828	return false;
2829
2830	// Check for a mismatched access type, and fall back conservatively as needed.
2831	// TODO: Be less conservative when the type is similar and can use the same
2832	// addressing modes.
2833	if (Kind == LSRUse::Address) {
2834	if (AccessTy.MemTy != LU.AccessTy.MemTy) {
2835	NewAccessTy = MemAccessTy::getUnknown(Ctx&: AccessTy.MemTy->getContext(),
2836	AS: AccessTy.AddrSpace);
2837	}
2838	}
2839
2840	// Conservatively assume HasBaseReg is true for now.
2841	if (Immediate::isKnownLT(LHS: NewOffset, RHS: LU.MinOffset)) {
2842	if (!isAlwaysFoldable(TTI, Kind, AccessTy: NewAccessTy, /BaseGV=/nullptr,
2843	BaseOffset: LU.MaxOffset - NewOffset, HasBaseReg))
2844	return false;
2845	NewMinOffset = NewOffset;
2846	} else if (Immediate::isKnownGT(LHS: NewOffset, RHS: LU.MaxOffset)) {
2847	if (!isAlwaysFoldable(TTI, Kind, AccessTy: NewAccessTy, /BaseGV=/nullptr,
2848	BaseOffset: NewOffset - LU.MinOffset, HasBaseReg))
2849	return false;
2850	NewMaxOffset = NewOffset;
2851	}
2852
2853	// FIXME: We should be able to handle some level of scalable offset support
2854	// for 'void', but in order to get basic support up and running this is
2855	// being left out.
2856	if (NewAccessTy.MemTy && NewAccessTy.MemTy->isVoidTy() &&
2857	(NewMinOffset.isScalable() \|\| NewMaxOffset.isScalable()))
2858	return false;
2859
2860	// Update the use.
2861	LU.MinOffset = NewMinOffset;
2862	LU.MaxOffset = NewMaxOffset;
2863	LU.AccessTy = NewAccessTy;
2864	return true;
2865	}
2866
2867	/// Return an LSRUse index and an offset value for a fixup which needs the given
2868	/// expression, with the given kind and optional access type. Either reuse an
2869	/// existing use or create a new one, as needed.
2870	std::pair<size_t, Immediate> LSRInstance::getUse(const SCEV *&Expr,
2871	LSRUse::KindType Kind,
2872	MemAccessTy AccessTy) {
2873	const SCEV *Copy = Expr;
2874	SCEVUse ExprUse = Expr;
2875	Immediate Offset = ExtractImmediate(
2876	S&: ExprUse, SE, PreferScalable: AccessTy.MemTy && AccessTy.MemTy->isScalableTy());
2877	Expr = ExprUse;
2878
2879	// Basic uses can't accept any offset, for example.
2880	if (!isAlwaysFoldable(TTI, Kind, AccessTy, /BaseGV=/ nullptr,
2881	BaseOffset: Offset, /HasBaseReg=/ true)) {
2882	Expr = Copy;
2883	Offset = Immediate::getFixed(MinVal: `0`);
2884	}
2885
2886	std::pair<UseMapTy::iterator, bool> P =
2887	UseMap.try_emplace(Key: LSRUse::SCEVUseKindPair(Expr, Kind));
2888	if (!P.second) {
2889	// A use already existed with this base.
2890	size_t LUIdx = P.first ->second;
2891	LSRUse &LU = Uses [LUIdx];
2892	if (reconcileNewOffset(LU, NewOffset: Offset, /HasBaseReg=/true, Kind, AccessTy))
2893	// Reuse this use.
2894	return std::make_pair(x&: LUIdx, y&: Offset);
2895	}
2896
2897	// Create a new use.
2898	size_t LUIdx = Uses.size();
2899	P.first ->second = LUIdx;
2900	Uses.push_back(Elt: LSRUse (Kind, AccessTy));
2901	LSRUse &LU = Uses [LUIdx];
2902
2903	LU.MinOffset = Offset;
2904	LU.MaxOffset = Offset;
2905	return std::make_pair(x&: LUIdx, y&: Offset);
2906	}
2907
2908	/// Delete the given use from the Uses list.
2909	void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
2910	if (&LU != &Uses.back())
2911	std::swap(a&: LU, b&: Uses.back());
2912	Uses.pop_back();
2913
2914	// Update RegUses.
2915	RegUses.swapAndDropUse(LUIdx, LastLUIdx: Uses.size());
2916	}
2917
2918	/// Look for a use distinct from OrigLU which is has a formula that has the same
2919	/// registers as the given formula.
2920	LSRUse *
2921	LSRInstance::FindUseWithSimilarFormula(const Formula &OrigF,
2922	const LSRUse &OrigLU) {
2923	// Search all uses for the formula. This could be more clever.
2924	for (LSRUse &LU : Uses) {
2925	// Check whether this use is close enough to OrigLU, to see whether it's
2926	// worthwhile looking through its formulae.
2927	// Ignore ICmpZero uses because they may contain formulae generated by
2928	// GenerateICmpZeroScales, in which case adding fixup offsets may
2929	// be invalid.
2930	if (&LU != &OrigLU && LU.Kind != LSRUse::ICmpZero &&
2931	LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
2932	LU.HasFormulaWithSameRegs(F: OrigF)) {
2933	// Scan through this use's formulae.
2934	for (const Formula &F : LU.Formulae) {
2935	// Check to see if this formula has the same registers and symbols
2936	// as OrigF.
2937	if (F.BaseRegs == OrigF.BaseRegs &&
2938	F.ScaledReg == OrigF.ScaledReg &&
2939	F.BaseGV == OrigF.BaseGV &&
2940	F.Scale == OrigF.Scale &&
2941	F.UnfoldedOffset == OrigF.UnfoldedOffset) {
2942	if (F.BaseOffset.isZero())
2943	return &LU;
2944	// This is the formula where all the registers and symbols matched;
2945	// there aren't going to be any others. Since we declined it, we
2946	// can skip the rest of the formulae and proceed to the next LSRUse.
2947	break;
2948	}
2949	}
2950	}
2951	}
2952
2953	// Nothing looked good.
2954	return nullptr;
2955	}
2956
2957	void LSRInstance::CollectInterestingTypesAndFactors() {
2958	SmallSetVector<const SCEV *, `4`> Strides;
2959
2960	// Collect interesting types and strides.
2961	SmallVector<const SCEV *, `4`> Worklist;
2962	for (const IVStrideUse &U : IU) {
2963	const SCEV *Expr = IU.getExpr(IU: U);
2964	if (!Expr)
2965	continue;
2966
2967	// Collect interesting types.
2968	Types.insert(X: SE.getEffectiveSCEVType(Ty: Expr->getType()));
2969
2970	// Add strides for mentioned loops.
2971	Worklist.push_back(Elt: Expr);
2972	do {
2973	const SCEV *S = Worklist.pop_back_val();
2974	if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
2975	if (AR->getLoop() == L)
2976	Strides.insert(X: AR->getStepRecurrence(SE));
2977	Worklist.push_back(Elt: AR->getStart());
2978	} else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
2979	append_range(C&: Worklist, R: Add->operands());
2980	}
2981	} while (!Worklist.empty());
2982	}
2983
2984	// Compute interesting factors from the set of interesting strides.
2985	for (SmallSetVector<const SCEV *, `4`>::const_iterator
2986	I = Strides.begin(), E = Strides.end(); I != E; ++I)
2987	for (SmallSetVector<const SCEV *, `4`>::const_iterator NewStrideIter =
2988	std::next(x: I); NewStrideIter != E; ++NewStrideIter) {
2989	const SCEV OldStride = I;
2990	const SCEV NewStride = NewStrideIter;
2991
2992	if (SE.getTypeSizeInBits(Ty: OldStride->getType()) !=
2993	SE.getTypeSizeInBits(Ty: NewStride->getType())) {
2994	if (SE.getTypeSizeInBits(Ty: OldStride->getType()) >
2995	SE.getTypeSizeInBits(Ty: NewStride->getType()))
2996	NewStride = SE.getSignExtendExpr(Op: NewStride, Ty: OldStride->getType());
2997	else
2998	OldStride = SE.getSignExtendExpr(Op: OldStride, Ty: NewStride->getType());
2999	}
3000	if (const SCEVConstant *Factor =
3001	dyn_cast_or_null<SCEVConstant>(Val: getExactSDiv(LHS: NewStride, RHS: OldStride,
3002	SE, IgnoreSignificantBits: true))) {
3003	if (Factor->getAPInt().getSignificantBits() <= `64` && !Factor->isZero())
3004	Factors.insert(X: Factor->getAPInt().getSExtValue());
3005	} else if (const SCEVConstant *Factor =
3006	dyn_cast_or_null<SCEVConstant>(Val: getExactSDiv(LHS: OldStride,
3007	RHS: NewStride,
3008	SE, IgnoreSignificantBits: true))) {
3009	if (Factor->getAPInt().getSignificantBits() <= `64` && !Factor->isZero())
3010	Factors.insert(X: Factor->getAPInt().getSExtValue());
3011	}
3012	}
3013
3014	// If all uses use the same type, don't bother looking for truncation-based
3015	// reuse.
3016	if (Types.size() == `1`)
3017	Types.clear();
3018
3019	LLVM_DEBUG(print_factors_and_types(dbgs()));
3020	}
3021
3022	/// Helper for CollectChains that finds an IV operand (computed by an AddRec in
3023	/// this loop) within [OI,OE) or returns OE. If IVUsers mapped Instructions to
3024	/// IVStrideUses, we could partially skip this.
3025	static User::op_iterator
3026	findIVOperand(User::op_iterator OI, User::op_iterator OE,
3027	Loop *L, ScalarEvolution &SE) {
3028	for(; OI != OE; ++OI) {
3029	if (Instruction Oper = dyn_cast<Instruction>(Val&: OI)) {
3030	if (!SE.isSCEVable(Ty: Oper->getType()))
3031	continue;
3032
3033	if (const SCEVAddRecExpr *AR =
3034	dyn_cast<SCEVAddRecExpr>(Val: SE.getSCEV(V: Oper))) {
3035	if (AR->getLoop() == L)
3036	break;
3037	}
3038	}
3039	}
3040	return OI;
3041	}
3042
3043	/// IVChain logic must consistently peek base TruncInst operands, so wrap it in
3044	/// a convenient helper.
3045	static Value getWideOperand(Value Oper) {
3046	if (TruncInst *Trunc = dyn_cast<TruncInst>(Val: Oper))
3047	return Trunc->getOperand(i_nocapture: `0`);
3048	return Oper;
3049	}
3050
3051	/// Return an approximation of this SCEV expression's "base", or NULL for any
3052	/// constant. Returning the expression itself is conservative. Returning a
3053	/// deeper subexpression is more precise and valid as long as it isn't less
3054	/// complex than another subexpression. For expressions involving multiple
3055	/// unscaled values, we need to return the pointer-type SCEVUnknown. This avoids
3056	/// forming chains across objects, such as: PrevOper==a[i], IVOper==b[i],
3057	/// IVInc==b-a.
3058	///
3059	/// Since SCEVUnknown is the rightmost type, and pointers are the rightmost
3060	/// SCEVUnknown, we simply return the rightmost SCEV operand.
3061	static const SCEV getExprBase(const* SCEV *S) {
3062	switch (S->getSCEVType()) {
3063	default: // including scUnknown.
3064	return S;
3065	case scConstant:
3066	case scVScale:
3067	return nullptr;
3068	case scTruncate:
3069	return getExprBase(S: cast<SCEVTruncateExpr>(Val: S)->getOperand());
3070	case scZeroExtend:
3071	return getExprBase(S: cast<SCEVZeroExtendExpr>(Val: S)->getOperand());
3072	case scSignExtend:
3073	return getExprBase(S: cast<SCEVSignExtendExpr>(Val: S)->getOperand());
3074	case scAddExpr: {
3075	// Skip over scaled operands (scMulExpr) to follow add operands as long as
3076	// there's nothing more complex.
3077	// FIXME: not sure if we want to recognize negation.
3078	const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: S);
3079	for (const SCEV *SubExpr : reverse(C: Add->operands())) {
3080	if (SubExpr->getSCEVType() == scAddExpr)
3081	return getExprBase(S: SubExpr);
3082
3083	if (SubExpr->getSCEVType() != scMulExpr)
3084	return SubExpr;
3085	}
3086	return S; // all operands are scaled, be conservative.
3087	}
3088	case scAddRecExpr:
3089	return getExprBase(S: cast<SCEVAddRecExpr>(Val: S)->getStart());
3090	}
3091	llvm_unreachable("Unknown SCEV kind!");
3092	}
3093
3094	/// Return true if the chain increment is profitable to expand into a loop
3095	/// invariant value, which may require its own register. A profitable chain
3096	/// increment will be an offset relative to the same base. We allow such offsets
3097	/// to potentially be used as chain increment as long as it's not obviously
3098	/// expensive to expand using real instructions.
3099	bool IVChain::isProfitableIncrement(const SCEV *OperExpr,
3100	const SCEV *IncExpr,
3101	ScalarEvolution &SE) {
3102	// Aggressively form chains when -stress-ivchain.
3103	if (StressIVChain)
3104	return true;
3105
3106	// Do not replace a constant offset from IV head with a nonconstant IV
3107	// increment.
3108	if (!isa<SCEVConstant>(Val: IncExpr)) {
3109	const SCEV *HeadExpr = SE.getSCEV(V: getWideOperand(Oper: Incs [`0`].IVOperand));
3110	if (isa<SCEVConstant>(Val: SE.getMinusSCEV(LHS: OperExpr, RHS: HeadExpr)))
3111	return false;
3112	}
3113
3114	SmallPtrSet<const SCEV*, `8`> Processed;
3115	return !isHighCostExpansion(S: IncExpr, Processed, SE);
3116	}
3117
3118	/// Return true if the number of registers needed for the chain is estimated to
3119	/// be less than the number required for the individual IV users. First prohibit
3120	/// any IV users that keep the IV live across increments (the Users set should
3121	/// be empty). Next count the number and type of increments in the chain.
3122	///
3123	/// Chaining IVs can lead to considerable code bloat if ISEL doesn't
3124	/// effectively use postinc addressing modes. Only consider it profitable it the
3125	/// increments can be computed in fewer registers when chained.
3126	///
3127	/// TODO: Consider IVInc free if it's already used in another chains.
3128	static bool isProfitableChain(IVChain &Chain,
3129	SmallPtrSetImpl<Instruction *> &Users,
3130	ScalarEvolution &SE,
3131	const TargetTransformInfo &TTI) {
3132	if (StressIVChain)
3133	return true;
3134
3135	if (!Chain.hasIncs())
3136	return false;
3137
3138	if (!Users.empty()) {
3139	LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[`0`].UserInst << " users:\n";
3140	for (Instruction *Inst
3141	: Users) { dbgs() << " " << *Inst << "\n"; });
3142	return false;
3143	}
3144	assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
3145
3146	// The chain itself may require a register, so initialize cost to 1.
3147	int cost = `1`;
3148
3149	// A complete chain likely eliminates the need for keeping the original IV in
3150	// a register. LSR does not currently know how to form a complete chain unless
3151	// the header phi already exists.
3152	if (isa<PHINode>(Val: Chain.tailUserInst())
3153	&& SE.getSCEV(V: Chain.tailUserInst()) == Chain.Incs [`0`].IncExpr) {
3154	--cost;
3155	}
3156	const SCEV LastIncExpr = nullptr*;
3157	unsigned NumConstIncrements = `0`;
3158	unsigned NumVarIncrements = `0`;
3159	unsigned NumReusedIncrements = `0`;
3160
3161	if (TTI.isProfitableLSRChainElement(I: Chain.Incs [`0`].UserInst))
3162	return true;
3163
3164	for (const IVInc &Inc : Chain) {
3165	if (TTI.isProfitableLSRChainElement(I: Inc.UserInst))
3166	return true;
3167	if (Inc.IncExpr->isZero())
3168	continue;
3169
3170	// Incrementing by zero or some constant is neutral. We assume constants can
3171	// be folded into an addressing mode or an add's immediate operand.
3172	if (isa<SCEVConstant>(Val: Inc.IncExpr)) {
3173	++NumConstIncrements;
3174	continue;
3175	}
3176
3177	if (Inc.IncExpr == LastIncExpr)
3178	++NumReusedIncrements;
3179	else
3180	++NumVarIncrements;
3181
3182	LastIncExpr = Inc.IncExpr;
3183	}
3184	// An IV chain with a single increment is handled by LSR's postinc
3185	// uses. However, a chain with multiple increments requires keeping the IV's
3186	// value live longer than it needs to be if chained.
3187	if (NumConstIncrements > `1`)
3188	--cost;
3189
3190	// Materializing increment expressions in the preheader that didn't exist in
3191	// the original code may cost a register. For example, sign-extended array
3192	// indices can produce ridiculous increments like this:
3193	// IV + ((sext i32 (2 %s) to i64) + (-1 * (sext i32 %s to i64)))*
3194	cost += NumVarIncrements;
3195
3196	// Reusing variable increments likely saves a register to hold the multiple of
3197	// the stride.
3198	cost -= NumReusedIncrements;
3199
3200	LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[`0`].UserInst << " Cost: " << cost
3201	<< "\n");
3202
3203	return cost < `0`;
3204	}
3205
3206	/// Add this IV user to an existing chain or make it the head of a new chain.
3207	void LSRInstance::ChainInstruction(Instruction UserInst, Instruction IVOper,
3208	SmallVectorImpl<ChainUsers> &ChainUsersVec) {
3209	// When IVs are used as types of varying widths, they are generally converted
3210	// to a wider type with some uses remaining narrow under a (free) trunc.
3211	Value *const NextIV = getWideOperand(Oper: IVOper);
3212	const SCEV *const OperExpr = SE.getSCEV(V: NextIV);
3213	const SCEV *const OperExprBase = getExprBase(S: OperExpr);
3214
3215	// Visit all existing chains. Check if its IVOper can be computed as a
3216	// profitable loop invariant increment from the last link in the Chain.
3217	unsigned ChainIdx = `0`, NChains = IVChainVec.size();
3218	const SCEV LastIncExpr = nullptr*;
3219	for (; ChainIdx < NChains; ++ChainIdx) {
3220	IVChain &Chain = IVChainVec [ChainIdx];
3221
3222	// Prune the solution space aggressively by checking that both IV operands
3223	// are expressions that operate on the same unscaled SCEVUnknown. This
3224	// "base" will be canceled by the subsequent getMinusSCEV call. Checking
3225	// first avoids creating extra SCEV expressions.
3226	if (!StressIVChain && Chain.ExprBase != OperExprBase)
3227	continue;
3228
3229	Value *PrevIV = getWideOperand(Oper: Chain.Incs.back().IVOperand);
3230	if (PrevIV->getType() != NextIV->getType())
3231	continue;
3232
3233	// A phi node terminates a chain.
3234	if (isa<PHINode>(Val: UserInst) && isa<PHINode>(Val: Chain.tailUserInst()))
3235	continue;
3236
3237	// The increment must be loop-invariant so it can be kept in a register.
3238	const SCEV *PrevExpr = SE.getSCEV(V: PrevIV);
3239	const SCEV *IncExpr = SE.getMinusSCEV(LHS: OperExpr, RHS: PrevExpr);
3240	if (isa<SCEVCouldNotCompute>(Val: IncExpr) \|\| !SE.isLoopInvariant(S: IncExpr, L))
3241	continue;
3242
3243	if (Chain.isProfitableIncrement(OperExpr, IncExpr, SE)) {
3244	LastIncExpr = IncExpr;
3245	break;
3246	}
3247	}
3248	// If we haven't found a chain, create a new one, unless we hit the max. Don't
3249	// bother for phi nodes, because they must be last in the chain.
3250	if (ChainIdx == NChains) {
3251	if (isa<PHINode>(Val: UserInst))
3252	return;
3253	if (NChains >= MaxChains && !StressIVChain) {
3254	LLVM_DEBUG(dbgs() << "IV Chain Limit\n");
3255	return;
3256	}
3257	LastIncExpr = OperExpr;
3258	// IVUsers may have skipped over sign/zero extensions. We don't currently
3259	// attempt to form chains involving extensions unless they can be hoisted
3260	// into this loop's AddRec.
3261	if (!isa<SCEVAddRecExpr>(Val: LastIncExpr))
3262	return;
3263	++NChains;
3264	IVChainVec.push_back(Elt: IVChain (IVInc (UserInst, IVOper, LastIncExpr),
3265	OperExprBase));
3266	ChainUsersVec.resize(N: NChains);
3267	LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Head: (" << *UserInst
3268	<< ") IV=" << *LastIncExpr << "\n");
3269	} else {
3270	LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Inc: (" << *UserInst
3271	<< ") IV+" << *LastIncExpr << "\n");
3272	// Add this IV user to the end of the chain.
3273	IVChainVec [ChainIdx].add(X: IVInc (UserInst, IVOper, LastIncExpr));
3274	}
3275	IVChain &Chain = IVChainVec [ChainIdx];
3276
3277	SmallPtrSet<Instruction*,`4`> &NearUsers = ChainUsersVec [ChainIdx].NearUsers;
3278	// This chain's NearUsers become FarUsers.
3279	if (!LastIncExpr->isZero()) {
3280	ChainUsersVec [ChainIdx].FarUsers.insert_range(R&: NearUsers);
3281	NearUsers.clear();
3282	}
3283
3284	// All other uses of IVOperand become near uses of the chain.
3285	// We currently ignore intermediate values within SCEV expressions, assuming
3286	// they will eventually be used be the current chain, or can be computed
3287	// from one of the chain increments. To be more precise we could
3288	// transitively follow its user and only add leaf IV users to the set.
3289	for (User *U : IVOper->users()) {
3290	Instruction *OtherUse = dyn_cast<Instruction>(Val: U);
3291	if (!OtherUse)
3292	continue;
3293	// Uses in the chain will no longer be uses if the chain is formed.
3294	// Include the head of the chain in this iteration (not Chain.begin()).
3295	IVChain::const_iterator IncIter = Chain.Incs.begin();
3296	IVChain::const_iterator IncEnd = Chain.Incs.end();
3297	for( ; IncIter != IncEnd; ++IncIter) {
3298	if (IncIter->UserInst == OtherUse)
3299	break;
3300	}
3301	if (IncIter != IncEnd)
3302	continue;
3303
3304	if (SE.isSCEVable(Ty: OtherUse->getType())
3305	&& !isa<SCEVUnknown>(Val: SE.getSCEV(V: OtherUse))
3306	&& IU.isIVUserOrOperand(Inst: OtherUse)) {
3307	continue;
3308	}
3309	NearUsers.insert(Ptr: OtherUse);
3310	}
3311
3312	// Since this user is part of the chain, it's no longer considered a use
3313	// of the chain.
3314	ChainUsersVec [ChainIdx].FarUsers.erase(Ptr: UserInst);
3315	}
3316
3317	/// Populate the vector of Chains.
3318	///
3319	/// This decreases ILP at the architecture level. Targets with ample registers,
3320	/// multiple memory ports, and no register renaming probably don't want
3321	/// this. However, such targets should probably disable LSR altogether.
3322	///
3323	/// The job of LSR is to make a reasonable choice of induction variables across
3324	/// the loop. Subsequent passes can easily "unchain" computation exposing more
3325	/// ILP within the loop* if the target wants it.*
3326	///
3327	/// Finding the best IV chain is potentially a scheduling problem. Since LSR
3328	/// will not reorder memory operations, it will recognize this as a chain, but
3329	/// will generate redundant IV increments. Ideally this would be corrected later
3330	/// by a smart scheduler:
3331	/// = A[i]
3332	/// = A[i+x]
3333	/// A[i] =
3334	/// A[i+x] =
3335	///
3336	/// TODO: Walk the entire domtree within this loop, not just the path to the
3337	/// loop latch. This will discover chains on side paths, but requires
3338	/// maintaining multiple copies of the Chains state.
3339	void LSRInstance::CollectChains() {
3340	LLVM_DEBUG(dbgs() << "Collecting IV Chains.\n");
3341	SmallVector<ChainUsers, `8`> ChainUsersVec;
3342
3343	SmallVector<BasicBlock *,`8`> LatchPath;
3344	BasicBlock *LoopHeader = L->getHeader();
3345	for (DomTreeNode *Rung = DT.getNode(BB: L->getLoopLatch());
3346	Rung->getBlock() != LoopHeader; Rung = Rung->getIDom()) {
3347	LatchPath.push_back(Elt: Rung->getBlock());
3348	}
3349	LatchPath.push_back(Elt: LoopHeader);
3350
3351	// Walk the instruction stream from the loop header to the loop latch.
3352	for (BasicBlock *BB : reverse(C&: LatchPath)) {
3353	for (Instruction &I : *BB) {
3354	// Skip instructions that weren't seen by IVUsers analysis.
3355	if (isa<PHINode>(Val: I) \|\| !IU.isIVUserOrOperand(Inst: &I))
3356	continue;
3357
3358	// Skip ephemeral values, as they don't produce real code.
3359	if (IU.isEphemeral(V: &I))
3360	continue;
3361
3362	// Ignore users that are part of a SCEV expression. This way we only
3363	// consider leaf IV Users. This effectively rediscovers a portion of
3364	// IVUsers analysis but in program order this time.
3365	if (SE.isSCEVable(Ty: I.getType()) && !isa<SCEVUnknown>(Val: SE.getSCEV(V: &I)))
3366	continue;
3367
3368	// Remove this instruction from any NearUsers set it may be in.
3369	for (unsigned ChainIdx = `0`, NChains = IVChainVec.size();
3370	ChainIdx < NChains; ++ChainIdx) {
3371	ChainUsersVec [ChainIdx].NearUsers.erase(Ptr: &I);
3372	}
3373	// Search for operands that can be chained.
3374	SmallPtrSet<Instruction*, `4`> UniqueOperands;
3375	User::op_iterator IVOpEnd = I.op_end();
3376	User::op_iterator IVOpIter = findIVOperand(OI: I.op_begin(), OE: IVOpEnd, L, SE);
3377	while (IVOpIter != IVOpEnd) {
3378	Instruction IVOpInst = cast<Instruction>(Val&: IVOpIter);
3379	if (UniqueOperands.insert(Ptr: IVOpInst).second)
3380	ChainInstruction(UserInst: &I, IVOper: IVOpInst, ChainUsersVec);
3381	IVOpIter = findIVOperand(OI: std::next(x: IVOpIter), OE: IVOpEnd, L, SE);
3382	}
3383	} // Continue walking down the instructions.
3384	} // Continue walking down the domtree.
3385	// Visit phi backedges to determine if the chain can generate the IV postinc.
3386	for (PHINode &PN : L->getHeader()->phis()) {
3387	if (!SE.isSCEVable(Ty: PN.getType()))
3388	continue;
3389
3390	Instruction *IncV =
3391	dyn_cast<Instruction>(Val: PN.getIncomingValueForBlock(BB: L->getLoopLatch()));
3392	if (IncV)
3393	ChainInstruction(UserInst: &PN, IVOper: IncV, ChainUsersVec);
3394	}
3395	// Remove any unprofitable chains.
3396	unsigned ChainIdx = `0`;
3397	for (unsigned UsersIdx = `0`, NChains = IVChainVec.size();
3398	UsersIdx < NChains; ++UsersIdx) {
3399	if (!isProfitableChain(Chain&: IVChainVec [UsersIdx],
3400	Users&: ChainUsersVec [UsersIdx].FarUsers, SE, TTI))
3401	continue;
3402	// Preserve the chain at UsesIdx.
3403	if (ChainIdx != UsersIdx)
3404	IVChainVec [ChainIdx] = IVChainVec [UsersIdx];
3405	FinalizeChain(Chain&: IVChainVec [ChainIdx]);
3406	++ChainIdx;
3407	}
3408	IVChainVec.resize(N: ChainIdx);
3409	}
3410
3411	void LSRInstance::FinalizeChain(IVChain &Chain) {
3412	assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
3413	LLVM_DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[`0`].UserInst << "\n");
3414
3415	for (const IVInc &Inc : Chain) {
3416	LLVM_DEBUG(dbgs() << " Inc: " << *Inc.UserInst << "\n");
3417	auto UseI = find(Range: Inc.UserInst->operands(), Val: Inc.IVOperand);
3418	assert(UseI != Inc.UserInst->op_end() && "cannot find IV operand");
3419	IVIncSet.insert(Ptr: UseI);
3420	}
3421	}
3422
3423	/// Return true if the IVInc can be folded into an addressing mode.
3424	static bool canFoldIVIncExpr(const SCEV IncExpr, Instruction UserInst,
3425	Value Operand, const* TargetTransformInfo &TTI) {
3426	const SCEVConstant *IncConst = dyn_cast<SCEVConstant>(Val: IncExpr);
3427	Immediate IncOffset = Immediate::getZero();
3428	if (IncConst) {
3429	if (IncConst && IncConst->getAPInt().getSignificantBits() > `64`)
3430	return false;
3431	IncOffset = Immediate::getFixed(MinVal: IncConst->getValue()->getSExtValue());
3432	} else {
3433	// Look for mul(vscale, constant), to detect a scalable offset.
3434	const APInt *C;
3435	if (!match(S: IncExpr, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEVVScale())) \|\|
3436	C->getSignificantBits() > `64`)
3437	return false;
3438	IncOffset = Immediate::getScalable(MinVal: C->getSExtValue());
3439	}
3440
3441	if (!isAddressUse(TTI, Inst: UserInst, OperandVal: Operand))
3442	return false;
3443
3444	MemAccessTy AccessTy = getAccessType(TTI, Inst: UserInst, OperandVal: Operand);
3445	if (!isAlwaysFoldable(TTI, Kind: LSRUse::Address, AccessTy, /BaseGV=/nullptr,
3446	BaseOffset: IncOffset, /HasBaseReg=/false))
3447	return false;
3448
3449	return true;
3450	}
3451
3452	/// Generate an add or subtract for each IVInc in a chain to materialize the IV
3453	/// user's operand from the previous IV user's operand.
3454	void LSRInstance::GenerateIVChain(const IVChain &Chain,
3455	SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
3456	// Find the new IVOperand for the head of the chain. It may have been replaced
3457	// by LSR.
3458	const IVInc &Head = Chain.Incs [`0`];
3459	User::op_iterator IVOpEnd = Head.UserInst->op_end();
3460	// findIVOperand returns IVOpEnd if it can no longer find a valid IV user.
3461	User::op_iterator IVOpIter = findIVOperand(OI: Head.UserInst->op_begin(),
3462	OE: IVOpEnd, L, SE);
3463	Value IVSrc = nullptr*;
3464	while (IVOpIter != IVOpEnd) {
3465	IVSrc = getWideOperand(Oper: *IVOpIter);
3466
3467	// If this operand computes the expression that the chain needs, we may use
3468	// it. (Check this after setting IVSrc which is used below.)
3469	//
3470	// Note that if Head.IncExpr is wider than IVSrc, then this phi is too
3471	// narrow for the chain, so we can no longer use it. We do allow using a
3472	// wider phi, assuming the LSR checked for free truncation. In that case we
3473	// should already have a truncate on this operand such that
3474	// getSCEV(IVSrc) == IncExpr.
3475	if (SE.getSCEV(V: *IVOpIter) == Head.IncExpr
3476	\|\| SE.getSCEV(V: IVSrc) == Head.IncExpr) {
3477	break;
3478	}
3479	IVOpIter = findIVOperand(OI: std::next(x: IVOpIter), OE: IVOpEnd, L, SE);
3480	}
3481	if (IVOpIter == IVOpEnd) {
3482	// Gracefully give up on this chain.
3483	LLVM_DEBUG(dbgs() << "Concealed chain head: " << *Head.UserInst << "\n");
3484	return;
3485	}
3486	assert(IVSrc && "Failed to find IV chain source");
3487
3488	LLVM_DEBUG(dbgs() << "Generate chain at: " << *IVSrc << "\n");
3489	Type *IVTy = IVSrc->getType();
3490	Type *IntTy = SE.getEffectiveSCEVType(Ty: IVTy);
3491	const SCEV LeftOverExpr = nullptr*;
3492	const SCEV *Accum = SE.getZero(Ty: IntTy);
3493	SmallVector<std::pair<const SCEV , Value >> Bases;
3494	Bases.emplace_back(Args&: Accum, Args&: IVSrc);
3495
3496	for (const IVInc &Inc : Chain) {
3497	Instruction *InsertPt = Inc.UserInst;
3498	if (isa<PHINode>(Val: InsertPt))
3499	InsertPt = L->getLoopLatch()->getTerminator();
3500
3501	// IVOper will replace the current IV User's operand. IVSrc is the IV
3502	// value currently held in a register.
3503	Value *IVOper = IVSrc;
3504	if (!Inc.IncExpr->isZero()) {
3505	// IncExpr was the result of subtraction of two narrow values, so must
3506	// be signed.
3507	const SCEV *IncExpr = SE.getNoopOrSignExtend(V: Inc.IncExpr, Ty: IntTy);
3508	Accum = SE.getAddExpr(LHS: Accum, RHS: IncExpr);
3509	LeftOverExpr = LeftOverExpr
3510	? SE.getAddExpr(LHS: LeftOverExpr, RHS: IncExpr).getPointer()
3511	: IncExpr;
3512	}
3513
3514	// Look through each base to see if any can produce a nice addressing mode.
3515	bool FoundBase = false;
3516	for (auto [MapScev, MapIVOper] : reverse(C&: Bases)) {
3517	const SCEV *Remainder = SE.getMinusSCEV(LHS: Accum, RHS: MapScev);
3518	if (canFoldIVIncExpr(IncExpr: Remainder, UserInst: Inc.UserInst, Operand: Inc.IVOperand, TTI)) {
3519	if (!Remainder->isZero()) {
3520	Rewriter.clearPostInc();
3521	Value *IncV = Rewriter.expandCodeFor(SH: Remainder, Ty: IntTy, I: InsertPt);
3522	const SCEV *IVOperExpr =
3523	SE.getAddExpr(LHS: SE.getUnknown(V: MapIVOper), RHS: SE.getUnknown(V: IncV));
3524	IVOper = Rewriter.expandCodeFor(SH: IVOperExpr, Ty: IVTy, I: InsertPt);
3525	} else {
3526	IVOper = MapIVOper;
3527	}
3528
3529	FoundBase = true;
3530	break;
3531	}
3532	}
3533	if (!FoundBase && LeftOverExpr && !LeftOverExpr->isZero()) {
3534	// Expand the IV increment.
3535	Rewriter.clearPostInc();
3536	Value *IncV = Rewriter.expandCodeFor(SH: LeftOverExpr, Ty: IntTy, I: InsertPt);
3537	const SCEV *IVOperExpr = SE.getAddExpr(LHS: SE.getUnknown(V: IVSrc),
3538	RHS: SE.getUnknown(V: IncV));
3539	IVOper = Rewriter.expandCodeFor(SH: IVOperExpr, Ty: IVTy, I: InsertPt);
3540
3541	// If an IV increment can't be folded, use it as the next IV value.
3542	if (!canFoldIVIncExpr(IncExpr: LeftOverExpr, UserInst: Inc.UserInst, Operand: Inc.IVOperand, TTI)) {
3543	assert(IVTy == IVOper->getType() && "inconsistent IV increment type");
3544	Bases.emplace_back(Args&: Accum, Args&: IVOper);
3545	IVSrc = IVOper;
3546	LeftOverExpr = nullptr;
3547	}
3548	}
3549	Type *OperTy = Inc.IVOperand->getType();
3550	if (IVTy != OperTy) {
3551	assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) &&
3552	"cannot extend a chained IV");
3553	IRBuilder<> Builder(InsertPt);
3554	IVOper = Builder.CreateTruncOrBitCast(V: IVOper, DestTy: OperTy, Name: "lsr.chain");
3555	}
3556	Inc.UserInst->replaceUsesOfWith(From: Inc.IVOperand, To: IVOper);
3557	if (auto *OperandIsInstr = dyn_cast<Instruction>(Val: Inc.IVOperand))
3558	DeadInsts.emplace_back(Args&: OperandIsInstr);
3559	}
3560	// If LSR created a new, wider phi, we may also replace its postinc. We only
3561	// do this if we also found a wide value for the head of the chain.
3562	if (isa<PHINode>(Val: Chain.tailUserInst())) {
3563	for (PHINode &Phi : L->getHeader()->phis()) {
3564	if (Phi.getType() != IVSrc->getType())
3565	continue;
3566	Instruction *PostIncV = dyn_cast<Instruction>(
3567	Val: Phi.getIncomingValueForBlock(BB: L->getLoopLatch()));
3568	if (!PostIncV \|\| (SE.getSCEV(V: PostIncV) != SE.getSCEV(V: IVSrc)))
3569	continue;
3570	Value *IVOper = IVSrc;
3571	Type *PostIncTy = PostIncV->getType();
3572	if (IVTy != PostIncTy) {
3573	assert(PostIncTy->isPointerTy() && "mixing int/ptr IV types");
3574	IRBuilder<> Builder(L->getLoopLatch()->getTerminator());
3575	Builder.SetCurrentDebugLocation(PostIncV->getDebugLoc());
3576	IVOper = Builder.CreatePointerCast(V: IVSrc, DestTy: PostIncTy, Name: "lsr.chain");
3577	}
3578	Phi.replaceUsesOfWith(From: PostIncV, To: IVOper);
3579	DeadInsts.emplace_back(Args&: PostIncV);
3580	}
3581	}
3582	}
3583
3584	void LSRInstance::CollectFixupsAndInitialFormulae() {
3585	CondBrInst ExitBranch = nullptr*;
3586	bool SaveCmp = TTI.canSaveCmp(L, BI: &ExitBranch, SE: &SE, LI: &LI, DT: &DT, AC: &AC, LibInfo: &TLI);
3587
3588	// For calculating baseline cost
3589	SmallPtrSet<const SCEV *, `16`> Regs;
3590	DenseSet<const SCEV *> VisitedRegs;
3591	DenseSet<size_t> VisitedLSRUse;
3592
3593	for (const IVStrideUse &U : IU) {
3594	Instruction *UserInst = U.getUser();
3595	// Skip IV users that are part of profitable IV Chains.
3596	User::op_iterator UseI =
3597	find(Range: UserInst->operands(), Val: U.getOperandValToReplace());
3598	assert(UseI != UserInst->op_end() && "cannot find IV operand");
3599	if (IVIncSet.count(Ptr: UseI)) {
3600	LLVM_DEBUG(dbgs() << "Use is in profitable chain: " << **UseI << `'\n'`);
3601	continue;
3602	}
3603
3604	LSRUse::KindType Kind = LSRUse::Basic;
3605	MemAccessTy AccessTy;
3606	if (isAddressUse(TTI, Inst: UserInst, OperandVal: U.getOperandValToReplace())) {
3607	Kind = LSRUse::Address;
3608	AccessTy = getAccessType(TTI, Inst: UserInst, OperandVal: U.getOperandValToReplace());
3609	}
3610
3611	const SCEV *S = IU.getExpr(IU: U);
3612	if (!S)
3613	continue;
3614	PostIncLoopSet TmpPostIncLoops = U.getPostIncLoops();
3615
3616	// Equality (== and !=) ICmps are special. We can rewrite (i == N) as
3617	// (N - i == 0), and this allows (N - i) to be the expression that we work
3618	// with rather than just N or i, so we can consider the register
3619	// requirements for both N and i at the same time. Limiting this code to
3620	// equality icmps is not a problem because all interesting loops use
3621	// equality icmps, thanks to IndVarSimplify.
3622	if (ICmpInst *CI = dyn_cast<ICmpInst>(Val: UserInst)) {
3623	// If CI can be saved in some target, like replaced inside hardware loop
3624	// in PowerPC, no need to generate initial formulae for it.
3625	if (SaveCmp && CI == dyn_cast<ICmpInst>(Val: ExitBranch->getCondition()))
3626	continue;
3627	if (CI->isEquality()) {
3628	// Swap the operands if needed to put the OperandValToReplace on the
3629	// left, for consistency.
3630	Value *NV = CI->getOperand(i_nocapture: `1`);
3631	if (NV == U.getOperandValToReplace()) {
3632	CI->setOperand(i_nocapture: `1`, Val_nocapture: CI->getOperand(i_nocapture: `0`));
3633	CI->setOperand(i_nocapture: `0`, Val_nocapture: NV);
3634	NV = CI->getOperand(i_nocapture: `1`);
3635	Changed = true;
3636	}
3637
3638	// x == y --> x - y == 0
3639	const SCEV *N = SE.getSCEV(V: NV);
3640	if (SE.isLoopInvariant(S: N, L) && Rewriter.isSafeToExpand(S: N) &&
3641	(!NV->getType()->isPointerTy() \|\|
3642	SE.getPointerBase(V: N) == SE.getPointerBase(V: S))) {
3643	// S is normalized, so normalize N before folding it into S
3644	// to keep the result normalized.
3645	N = normalizeForPostIncUse(S: N, Loops: TmpPostIncLoops, SE);
3646	if (!N)
3647	continue;
3648	Kind = LSRUse::ICmpZero;
3649	S = SE.getMinusSCEV(LHS: N, RHS: S);
3650	} else if (L->isLoopInvariant(V: NV) &&
3651	(!isa<Instruction>(Val: NV) \|\|
3652	DT.dominates(Def: cast<Instruction>(Val: NV), BB: L->getHeader())) &&
3653	!NV->getType()->isPointerTy()) {
3654	// If we can't generally expand the expression (e.g. it contains
3655	// a divide), but it is already at a loop invariant point before the
3656	// loop, wrap it in an unknown (to prevent the expander from trying
3657	// to re-expand in a potentially unsafe way.) The restriction to
3658	// integer types is required because the unknown hides the base, and
3659	// SCEV can't compute the difference of two unknown pointers.
3660	N = SE.getUnknown(V: NV);
3661	N = normalizeForPostIncUse(S: N, Loops: TmpPostIncLoops, SE);
3662	if (!N)
3663	continue;
3664	Kind = LSRUse::ICmpZero;
3665	S = SE.getMinusSCEV(LHS: N, RHS: S);
3666	assert(!isa<SCEVCouldNotCompute>(S));
3667	}
3668
3669	// -1 and the negations of all interesting strides (except the negation
3670	// of -1) are now also interesting.
3671	for (size_t i = `0`, e = Factors.size(); i != e; ++i)
3672	if (Factors [i] != -`1`)
3673	Factors.insert(X: -(uint64_t)Factors [i]);
3674	Factors.insert(X: -`1`);
3675	}
3676	}
3677
3678	// Get or create an LSRUse.
3679	std::pair<size_t, Immediate> P = getUse(Expr&: S, Kind, AccessTy);
3680	size_t LUIdx = P.first;
3681	Immediate Offset = P.second;
3682	LSRUse &LU = Uses [LUIdx];
3683
3684	// Record the fixup.
3685	LSRFixup &LF = LU.getNewFixup();
3686	LF.UserInst = UserInst;
3687	LF.OperandValToReplace = U.getOperandValToReplace();
3688	LF.PostIncLoops = TmpPostIncLoops;
3689	LF.Offset = Offset;
3690	LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3691	LU.AllFixupsUnconditional &= IsFixupExecutedEachIncrement(LF);
3692
3693	// Create SCEV as Formula for calculating baseline cost
3694	if (!VisitedLSRUse.count(V: LUIdx) && !LF.isUseFullyOutsideLoop(L)) {
3695	Formula F;
3696	F.initialMatch(S, L, SE);
3697	BaselineCost.RateFormula(F, Regs, VisitedRegs, LU,
3698	HardwareLoopProfitable);
3699	VisitedLSRUse.insert(V: LUIdx);
3700	}
3701
3702	// If this is the first use of this LSRUse, give it a formula.
3703	if (LU.Formulae.empty()) {
3704	InsertInitialFormula(S, LU, LUIdx);
3705	CountRegisters(F: LU.Formulae.back(), LUIdx);
3706	}
3707	}
3708
3709	LLVM_DEBUG(print_fixups(dbgs()));
3710	}
3711
3712	/// Insert a formula for the given expression into the given use, separating out
3713	/// loop-variant portions from loop-invariant and loop-computable portions.
3714	void LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU,
3715	size_t LUIdx) {
3716	// Mark uses whose expressions cannot be expanded.
3717	if (!Rewriter.isSafeToExpand(S))
3718	LU.RigidFormula = true;
3719
3720	Formula F;
3721	F.initialMatch(S, L, SE);
3722	bool Inserted = InsertFormula(LU, LUIdx, F);
3723	assert(Inserted && "Initial formula already exists!"); (void)Inserted;
3724	}
3725
3726	/// Insert a simple single-register formula for the given expression into the
3727	/// given use.
3728	void
3729	LSRInstance::InsertSupplementalFormula(const SCEV *S,
3730	LSRUse &LU, size_t LUIdx) {
3731	Formula F;
3732	F.BaseRegs.push_back(Elt: S);
3733	F.HasBaseReg = true;
3734	bool Inserted = InsertFormula(LU, LUIdx, F);
3735	assert(Inserted && "Supplemental formula already exists!"); (void)Inserted;
3736	}
3737
3738	/// Note which registers are used by the given formula, updating RegUses.
3739	void LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) {
3740	if (F.ScaledReg)
3741	RegUses.countRegister(Reg: F.ScaledReg, LUIdx);
3742	for (const SCEV *BaseReg : F.BaseRegs)
3743	RegUses.countRegister(Reg: BaseReg, LUIdx);
3744	}
3745
3746	/// If the given formula has not yet been inserted, add it to the list, and
3747	/// return true. Return false otherwise.
3748	bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
3749	// Do not insert formula that we will not be able to expand.
3750	assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
3751	"Formula is illegal");
3752
3753	if (!LU.InsertFormula(F, L: *L))
3754	return false;
3755
3756	CountRegisters(F, LUIdx);
3757	return true;
3758	}
3759
3760	/// Test whether this fixup will be executed each time the corresponding IV
3761	/// increment instruction is executed.
3762	bool LSRInstance::IsFixupExecutedEachIncrement(const LSRFixup &LF) const {
3763	// If the fixup block dominates the IV increment block then there is no path
3764	// through the loop to the increment that doesn't pass through the fixup.
3765	return DT.dominates(A: LF.UserInst->getParent(), B: IVIncInsertPos->getParent());
3766	}
3767
3768	/// Check for other uses of loop-invariant values which we're tracking. These
3769	/// other uses will pin these values in registers, making them less profitable
3770	/// for elimination.
3771	/// TODO: This currently misses non-constant addrec step registers.
3772	/// TODO: Should this give more weight to users inside the loop?
3773	void
3774	LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
3775	SmallVector<const SCEV *, `8`> Worklist(RegUses.begin(), RegUses.end());
3776	SmallPtrSet<const SCEV *, `32`> Visited;
3777
3778	// Don't collect outside uses if we are favoring postinc - the instructions in
3779	// the loop are more important than the ones outside of it.
3780	if (AMK == TTI::AMK_PostIndexed)
3781	return;
3782
3783	while (!Worklist.empty()) {
3784	const SCEV *S = Worklist.pop_back_val();
3785
3786	// Don't process the same SCEV twice
3787	if (!Visited.insert(Ptr: S).second)
3788	continue;
3789
3790	if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(Val: S))
3791	append_range(C&: Worklist, R: N->operands());
3792	else if (const SCEVIntegralCastExpr *C = dyn_cast<SCEVIntegralCastExpr>(Val: S))
3793	Worklist.push_back(Elt: C->getOperand());
3794	else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(Val: S)) {
3795	Worklist.push_back(Elt: D->getLHS());
3796	Worklist.push_back(Elt: D->getRHS());
3797	} else if (const SCEVUnknown *US = dyn_cast<SCEVUnknown>(Val: S)) {
3798	const Value *V = US->getValue();
3799	if (const Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
3800	// Look for instructions defined outside the loop.
3801	if (L->contains(Inst)) continue;
3802	} else if (isa<Constant>(Val: V))
3803	// Constants can be re-materialized.
3804	continue;
3805	for (const Use &U : V->uses()) {
3806	const Instruction *UserInst = dyn_cast<Instruction>(Val: U.getUser());
3807	// Ignore non-instructions.
3808	if (!UserInst)
3809	continue;
3810	// Don't bother if the instruction is an EHPad.
3811	if (UserInst->isEHPad())
3812	continue;
3813	// Ignore instructions in other functions (as can happen with
3814	// Constants).
3815	if (UserInst->getParent()->getParent() != L->getHeader()->getParent())
3816	continue;
3817	// Ignore instructions not dominated by the loop.
3818	const BasicBlock *UseBB = !isa<PHINode>(Val: UserInst) ?
3819	UserInst->getParent() :
3820	cast<PHINode>(Val: UserInst)->getIncomingBlock(
3821	i: PHINode::getIncomingValueNumForOperand(i: U.getOperandNo()));
3822	if (!DT.dominates(A: L->getHeader(), B: UseBB))
3823	continue;
3824	// Don't bother if the instruction is in a BB which ends in an EHPad.
3825	if (UseBB->getTerminator()->isEHPad())
3826	continue;
3827
3828	// Ignore cases in which the currently-examined value could come from
3829	// a basic block terminated with an EHPad. This checks all incoming
3830	// blocks of the phi node since it is possible that the same incoming
3831	// value comes from multiple basic blocks, only some of which may end
3832	// in an EHPad. If any of them do, a subsequent rewrite attempt by this
3833	// pass would try to insert instructions into an EHPad, hitting an
3834	// assertion.
3835	if (isa<PHINode>(Val: UserInst)) {
3836	const auto *PhiNode = cast<PHINode>(Val: UserInst);
3837	bool HasIncompatibleEHPTerminatedBlock = false;
3838	llvm::Value *ExpectedValue = U;
3839	for (unsigned int I = `0`; I < PhiNode->getNumIncomingValues(); I++) {
3840	if (PhiNode->getIncomingValue(i: I) == ExpectedValue) {
3841	if (PhiNode->getIncomingBlock(i: I)->getTerminator()->isEHPad()) {
3842	HasIncompatibleEHPTerminatedBlock = true;
3843	break;
3844	}
3845	}
3846	}
3847	if (HasIncompatibleEHPTerminatedBlock) {
3848	continue;
3849	}
3850	}
3851
3852	// Don't bother rewriting PHIs in catchswitch blocks.
3853	if (isa<CatchSwitchInst>(Val: UserInst->getParent()->getTerminator()))
3854	continue;
3855	// Ignore uses which are part of other SCEV expressions, to avoid
3856	// analyzing them multiple times.
3857	if (SE.isSCEVable(Ty: UserInst->getType())) {
3858	const SCEV UserS = SE.getSCEV(V: const_cast<Instruction >(UserInst));
3859	// If the user is a no-op, look through to its uses.
3860	if (!isa<SCEVUnknown>(Val: UserS))
3861	continue;
3862	if (UserS == US) {
3863	Worklist.push_back(
3864	Elt: SE.getUnknown(V: const_cast<Instruction *>(UserInst)));
3865	continue;
3866	}
3867	}
3868	// Ignore icmp instructions which are already being analyzed.
3869	if (const ICmpInst *ICI = dyn_cast<ICmpInst>(Val: UserInst)) {
3870	unsigned OtherIdx = !U.getOperandNo();
3871	Value *OtherOp = ICI->getOperand(i_nocapture: OtherIdx);
3872	if (SE.hasComputableLoopEvolution(S: SE.getSCEV(V: OtherOp), L))
3873	continue;
3874	}
3875
3876	// Do not consider uses inside lifetime intrinsics. These are not
3877	// actually materialized.
3878	if (UserInst->isLifetimeStartOrEnd())
3879	continue;
3880
3881	std::pair<size_t, Immediate> P =
3882	getUse(Expr&: S, Kind: LSRUse::Basic, AccessTy: MemAccessTy ());
3883	size_t LUIdx = P.first;
3884	Immediate Offset = P.second;
3885	LSRUse &LU = Uses [LUIdx];
3886	LSRFixup &LF = LU.getNewFixup();
3887	LF.UserInst = const_cast<Instruction *>(UserInst);
3888	LF.OperandValToReplace = U;
3889	LF.Offset = Offset;
3890	LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3891	LU.AllFixupsUnconditional &= IsFixupExecutedEachIncrement(LF);
3892	InsertSupplementalFormula(S: US, LU, LUIdx);
3893	CountRegisters(F: LU.Formulae.back(), LUIdx: Uses.size() - `1`);
3894	break;
3895	}
3896	}
3897	}
3898	}
3899
3900	/// Split S into subexpressions which can be pulled out into separate
3901	/// registers. If C is non-null, multiply each subexpression by C.
3902	///
3903	/// Return remainder expression after factoring the subexpressions captured by
3904	/// Ops. If Ops is complete, return NULL.
3905	static const SCEV CollectSubexprs(const* SCEV S, const* SCEVConstant *C,
3906	SmallVectorImpl<const SCEV *> &Ops,
3907	const Loop *L,
3908	ScalarEvolution &SE,
3909	unsigned Depth = `0`) {
3910	// Arbitrarily cap recursion to protect compile time.
3911	if (Depth >= `3`)
3912	return S;
3913
3914	if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
3915	// Break out add operands.
3916	for (const SCEV *S : Add->operands()) {
3917	const SCEV *Remainder = CollectSubexprs(S, C, Ops, L, SE, Depth: Depth+`1`);
3918	if (Remainder)
3919	Ops.push_back(Elt: C ? SE.getMulExpr(LHS: C, RHS: Remainder).getPointer() : Remainder);
3920	}
3921	return nullptr;
3922	}
3923	const SCEV Start, Step;
3924	const SCEVConstant *Op0;
3925	const SCEV *Op1;
3926	if (match(S, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step)))) {
3927	// Split a non-zero base out of an addrec.
3928	if (Start->isZero())
3929	return S;
3930
3931	const SCEV *Remainder = CollectSubexprs(S: Start, C, Ops, L, SE, Depth: Depth + `1`);
3932	// Split the non-zero AddRec unless it is part of a nested recurrence that
3933	// does not pertain to this loop.
3934	if (Remainder && (cast<SCEVAddRecExpr>(Val: S)->getLoop() == L \|\|
3935	!isa<SCEVAddRecExpr>(Val: Remainder))) {
3936	Ops.push_back(Elt: C ? SE.getMulExpr(LHS: C, RHS: Remainder).getPointer() : Remainder);
3937	Remainder = nullptr;
3938	}
3939	if (Remainder != Start) {
3940	if (!Remainder)
3941	Remainder = SE.getConstant(Ty: S->getType(), V: `0`);
3942	return SE.getAddRecExpr(Start: Remainder, Step,
3943	L: cast<SCEVAddRecExpr>(Val: S)->getLoop(),
3944	// FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
3945	Flags: SCEV::FlagAnyWrap);
3946	}
3947	} else if (match(S, P: m_scev_Mul(Op0: m_SCEVConstant(V&: Op0), Op1: m_SCEV(V&: Op1)))) {
3948	// Break (C (a + b + c)) into Ca + Cb + Cc.
3949	C = C ? cast<SCEVConstant>(Val: SE.getMulExpr(LHS: C, RHS: Op0)) : Op0;
3950	const SCEV *Remainder = CollectSubexprs(S: Op1, C, Ops, L, SE, Depth: Depth + `1`);
3951	if (Remainder)
3952	Ops.push_back(Elt: SE.getMulExpr(LHS: C, RHS: Remainder));
3953	return nullptr;
3954	}
3955	return S;
3956	}
3957
3958	/// Return true if the SCEV represents a value that may end up as a
3959	/// post-increment operation.
3960	static bool mayUsePostIncMode(const TargetTransformInfo &TTI,
3961	LSRUse &LU, const SCEV S, const* Loop *L,
3962	ScalarEvolution &SE) {
3963	if (LU.Kind != LSRUse::Address \|\|
3964	!LU.AccessTy.getType()->isIntOrIntVectorTy())
3965	return false;
3966	const SCEV *Start;
3967	if (!match(S, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEVConstant())))
3968	return false;
3969	// Check if a post-indexed load/store can be used.
3970	if (TTI.isIndexedLoadLegal(Mode: TTI.MIM_PostInc, Ty: S->getType()) \|\|
3971	TTI.isIndexedStoreLegal(Mode: TTI.MIM_PostInc, Ty: S->getType())) {
3972	if (!isa<SCEVConstant>(Val: Start) && SE.isLoopInvariant(S: Start, L))
3973	return true;
3974	}
3975	return false;
3976	}
3977
3978	/// Helper function for LSRInstance::GenerateReassociations.
3979	void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
3980	const Formula &Base,
3981	unsigned Depth, size_t Idx,
3982	bool IsScaledReg) {
3983	const SCEV *BaseReg = IsScaledReg ? Base.ScaledReg : Base.BaseRegs [Idx];
3984	// Don't generate reassociations for the base register of a value that
3985	// may generate a post-increment operator. The reason is that the
3986	// reassociations cause extra base+register formula to be created,
3987	// and possibly chosen, but the post-increment is more efficient.
3988	if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, S: BaseReg, L, SE))
3989	return;
3990	SmallVector<const SCEV *, `8`> AddOps;
3991	const SCEV Remainder = CollectSubexprs(S: BaseReg, C: nullptr*, Ops&: AddOps, L, SE);
3992	if (Remainder)
3993	AddOps.push_back(Elt: Remainder);
3994
3995	if (AddOps.size() == `1`)
3996	return;
3997
3998	for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(),
3999	JE = AddOps.end();
4000	J != JE; ++J) {
4001	// Loop-variant "unknown" values are uninteresting; we won't be able to
4002	// do anything meaningful with them.
4003	if (isa<SCEVUnknown>(Val: J) && !SE.isLoopInvariant(S: J, L))
4004	continue;
4005
4006	// Don't pull a constant into a register if the constant could be folded
4007	// into an immediate field.
4008	if (isAlwaysFoldable(TTI, SE, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
4009	AccessTy: LU.AccessTy, S: *J, HasBaseReg: Base.getNumRegs() > `1`))
4010	continue;
4011
4012	// Collect all operands except J.*
4013	SmallVector<SCEVUse, `8`> InnerAddOps(std::as_const(t&: AddOps).begin(), J);
4014	InnerAddOps.append(in_start: std::next(x: J), in_end: std::as_const(t&: AddOps).end());
4015
4016	// Don't leave just a constant behind in a register if the constant could
4017	// be folded into an immediate field.
4018	if (InnerAddOps.size() == `1` &&
4019	isAlwaysFoldable(TTI, SE, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
4020	AccessTy: LU.AccessTy, S: InnerAddOps [`0`], HasBaseReg: Base.getNumRegs() > `1`))
4021	continue;
4022
4023	const SCEV *InnerSum = SE.getAddExpr(Ops&: InnerAddOps);
4024	if (InnerSum->isZero())
4025	continue;
4026	Formula F = Base;
4027
4028	if (F.UnfoldedOffset.isNonZero() && F.UnfoldedOffset.isScalable())
4029	continue;
4030
4031	// Add the remaining pieces of the add back into the new formula.
4032	const SCEVConstant *InnerSumSC = dyn_cast<SCEVConstant>(Val: InnerSum);
4033	if (InnerSumSC && SE.getTypeSizeInBits(Ty: InnerSumSC->getType()) <= `64` &&
4034	TTI.isLegalAddImmediate(Imm: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4035	InnerSumSC->getValue()->getZExtValue())) {
4036	F.UnfoldedOffset =
4037	Immediate::getFixed(MinVal: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4038	InnerSumSC->getValue()->getZExtValue());
4039	if (IsScaledReg) {
4040	F.ScaledReg = nullptr;
4041	F.Scale = `0`;
4042	} else
4043	F.BaseRegs.erase(CI: F.BaseRegs.begin() + Idx);
4044	} else if (IsScaledReg)
4045	F.ScaledReg = InnerSum;
4046	else
4047	F.BaseRegs [Idx] = InnerSum;
4048
4049	// Add J as its own register, or an unfolded immediate.
4050	const SCEVConstant SC = dyn_cast<SCEVConstant>(Val: J);
4051	if (SC && SE.getTypeSizeInBits(Ty: SC->getType()) <= `64` &&
4052	TTI.isLegalAddImmediate(Imm: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4053	SC->getValue()->getZExtValue()))
4054	F.UnfoldedOffset =
4055	Immediate::getFixed(MinVal: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4056	SC->getValue()->getZExtValue());
4057	else
4058	F.BaseRegs.push_back(Elt: *J);
4059	// We may have changed the number of register in base regs, adjust the
4060	// formula accordingly.
4061	F.canonicalize(L: *L);
4062
4063	if (InsertFormula(LU, LUIdx, F))
4064	// If that formula hadn't been seen before, recurse to find more like
4065	// it.
4066	// Add check on Log16(AddOps.size()) - same as Log2_32(AddOps.size()) >> 2)
4067	// Because just Depth is not enough to bound compile time.
4068	// This means that every time AddOps.size() is greater 16^x we will add
4069	// x to Depth.
4070	GenerateReassociations(LU, LUIdx, Base: LU.Formulae.back(),
4071	Depth: Depth + `1` + (Log2_32(Value: AddOps.size()) >> `2`));
4072	}
4073	}
4074
4075	/// Split out subexpressions from adds and the bases of addrecs.
4076	void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
4077	Formula Base, unsigned Depth) {
4078	assert(Base.isCanonical(*L) && "Input must be in the canonical form");
4079	// Arbitrarily cap recursion to protect compile time.
4080	if (Depth >= `3`)
4081	return;
4082
4083	for (size_t i = `0`, e = Base.BaseRegs.size(); i != e; ++i)
4084	GenerateReassociationsImpl(LU, LUIdx, Base, Depth, Idx: i);
4085
4086	if (Base.Scale == `1`)
4087	GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
4088	/ Idx / -`1`, / IsScaledReg / true);
4089	}
4090
4091	/// Generate a formula consisting of all of the loop-dominating registers added
4092	/// into a single register.
4093	void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
4094	Formula Base) {
4095	// This method is only interesting on a plurality of registers.
4096	if (Base.BaseRegs.size() + (Base.Scale == `1`) +
4097	(Base.UnfoldedOffset.isNonZero()) <=
4098	`1`)
4099	return;
4100
4101	// Flatten the representation, i.e., reg1 + 1reg2 => reg1 + reg2, before*
4102	// processing the formula.
4103	Base.unscale();
4104	SmallVector<SCEVUse, `4`> Ops;
4105	Formula NewBase = Base;
4106	NewBase.BaseRegs.clear();
4107	Type CombinedIntegerType = nullptr*;
4108	for (const SCEV *BaseReg : Base.BaseRegs) {
4109	if (SE.properlyDominates(S: BaseReg, BB: L->getHeader()) &&
4110	!SE.hasComputableLoopEvolution(S: BaseReg, L)) {
4111	if (!CombinedIntegerType)
4112	CombinedIntegerType = SE.getEffectiveSCEVType(Ty: BaseReg->getType());
4113	Ops.push_back(Elt: BaseReg);
4114	}
4115	else
4116	NewBase.BaseRegs.push_back(Elt: BaseReg);
4117	}
4118
4119	// If no register is relevant, we're done.
4120	if (Ops.size() == `0`)
4121	return;
4122
4123	// Utility function for generating the required variants of the combined
4124	// registers.
4125	auto GenerateFormula = [&](const SCEV *Sum) {
4126	Formula F = NewBase;
4127
4128	// TODO: If Sum is zero, it probably means ScalarEvolution missed an
4129	// opportunity to fold something. For now, just ignore such cases
4130	// rather than proceed with zero in a register.
4131	if (Sum->isZero())
4132	return;
4133
4134	F.BaseRegs.push_back(Elt: Sum);
4135	F.canonicalize(L: *L);
4136	(void)InsertFormula(LU, LUIdx, F);
4137	};
4138
4139	// If we collected at least two registers, generate a formula combining them.
4140	if (Ops.size() > `1`) {
4141	SmallVector<SCEVUse, `4`> OpsCopy(Ops); // Don't let SE modify Ops.
4142	GenerateFormula (SE.getAddExpr(Ops&: OpsCopy));
4143	}
4144
4145	// If we have an unfolded offset, generate a formula combining it with the
4146	// registers collected.
4147	if (NewBase.UnfoldedOffset.isNonZero() && NewBase.UnfoldedOffset.isFixed()) {
4148	assert(CombinedIntegerType && "Missing a type for the unfolded offset");
4149	Ops.push_back(Elt: SE.getConstant(Ty: CombinedIntegerType,
4150	V: NewBase.UnfoldedOffset.getFixedValue(), isSigned: true));
4151	NewBase.UnfoldedOffset = Immediate::getFixed(MinVal: `0`);
4152	GenerateFormula (SE.getAddExpr(Ops));
4153	}
4154	}
4155
4156	/// Helper function for LSRInstance::GenerateSymbolicOffsets.
4157	void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
4158	const Formula &Base, size_t Idx,
4159	bool IsScaledReg) {
4160	SCEVUse G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs [Idx];
4161	GlobalValue *GV = ExtractSymbol(S&: G, SE);
4162	if (G ->isZero() \|\| !GV)
4163	return;
4164	Formula F = Base;
4165	F.BaseGV = GV;
4166	if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy, F))
4167	return;
4168	if (IsScaledReg)
4169	F.ScaledReg = G;
4170	else
4171	F.BaseRegs [Idx] = G;
4172	(void)InsertFormula(LU, LUIdx, F);
4173	}
4174
4175	/// Generate reuse formulae using symbolic offsets.
4176	void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
4177	Formula Base) {
4178	// We can't add a symbolic offset if the address already contains one.
4179	if (Base.BaseGV) return;
4180
4181	for (size_t i = `0`, e = Base.BaseRegs.size(); i != e; ++i)
4182	GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, Idx: i);
4183	if (Base.Scale == `1`)
4184	GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, / Idx / -`1`,
4185	/ IsScaledReg / true);
4186	}
4187
4188	/// Helper function for LSRInstance::GenerateConstantOffsets.
4189	void LSRInstance::GenerateConstantOffsetsImpl(
4190	LSRUse &LU, unsigned LUIdx, const Formula &Base,
4191	const SmallVectorImpl<Immediate> &Worklist, size_t Idx, bool IsScaledReg) {
4192
4193	auto GenerateOffset = [&](const SCEV *G, Immediate Offset) {
4194	Formula F = Base;
4195	if (!Base.BaseOffset.isCompatibleImmediate(Imm: Offset))
4196	return;
4197	F.BaseOffset = Base.BaseOffset.subUnsigned(RHS: Offset);
4198
4199	if (isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy, F)) {
4200	// Add the offset to the base register.
4201	const SCEV *NewOffset = Offset.getSCEV(SE, Ty: G->getType());
4202	const SCEV *NewG = SE.getAddExpr(LHS: NewOffset, RHS: G);
4203	// If it cancelled out, drop the base register, otherwise update it.
4204	if (NewG->isZero()) {
4205	if (IsScaledReg) {
4206	F.Scale = `0`;
4207	F.ScaledReg = nullptr;
4208	} else
4209	F.deleteBaseReg(S&: F.BaseRegs [Idx]);
4210	F.canonicalize(L: *L);
4211	} else if (IsScaledReg)
4212	F.ScaledReg = NewG;
4213	else
4214	F.BaseRegs [Idx] = NewG;
4215
4216	(void)InsertFormula(LU, LUIdx, F);
4217	}
4218	};
4219
4220	SCEVUse G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs [Idx];
4221
4222	// With constant offsets and constant steps, we can generate pre-inc
4223	// accesses by having the offset equal the step. So, for access #0 with a
4224	// step of 8, we generate a G - 8 base which would require the first access
4225	// to be ((G - 8) + 8),+,8. The pre-indexed access then updates the pointer
4226	// for itself and hopefully becomes the base for other accesses. This means
4227	// means that a single pre-indexed access can be generated to become the new
4228	// base pointer for each iteration of the loop, resulting in no extra add/sub
4229	// instructions for pointer updating.
4230	if ((AMK & TTI::AMK_PreIndexed) && LU.Kind == LSRUse::Address) {
4231	const APInt *StepInt;
4232	if (match(U: G, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_scev_APInt(C&: StepInt)))) {
4233	int64_t Step = StepInt->isNegative() ? StepInt->getSExtValue()
4234	: StepInt->getZExtValue();
4235
4236	for (Immediate Offset : Worklist) {
4237	if (Offset.isFixed()) {
4238	Offset = Immediate::getFixed(MinVal: Offset.getFixedValue() - Step);
4239	GenerateOffset (G, Offset);
4240	}
4241	}
4242	}
4243	}
4244	for (Immediate Offset : Worklist)
4245	GenerateOffset (G, Offset);
4246
4247	// TODO: It likely makes sense to extract the immediate corresponding to the
4248	// access type (i.e., set PreferScalable to AccessTy.MemTy &&
4249	// AccessTy.MemTy->isScalableTy()).
4250	Immediate Imm = ExtractImmediate(S&: G, SE, /PreferScalable=/false);
4251	if (G ->isZero() \|\| Imm.isZero() \|\|
4252	!Base.BaseOffset.isCompatibleImmediate(Imm))
4253	return;
4254	Formula F = Base;
4255	F.BaseOffset = F.BaseOffset.addUnsigned(RHS: Imm);
4256	if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy, F))
4257	return;
4258	if (IsScaledReg) {
4259	F.ScaledReg = G;
4260	} else {
4261	F.BaseRegs [Idx] = G;
4262	// We may generate non canonical Formula if G is a recurrent expr reg
4263	// related with current loop while F.ScaledReg is not.
4264	F.canonicalize(L: *L);
4265	}
4266	(void)InsertFormula(LU, LUIdx, F);
4267	}
4268
4269	/// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets.
4270	void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
4271	Formula Base) {
4272	// TODO: For now, just add the min and max offset, because it usually isn't
4273	// worthwhile looking at everything inbetween.
4274	SmallVector<Immediate, `2`> Worklist;
4275	Worklist.push_back(Elt: LU.MinOffset);
4276	if (LU.MaxOffset != LU.MinOffset)
4277	Worklist.push_back(Elt: LU.MaxOffset);
4278
4279	for (size_t i = `0`, e = Base.BaseRegs.size(); i != e; ++i)
4280	GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, Idx: i);
4281	if (Base.Scale == `1`)
4282	GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, / Idx / -`1`,
4283	/ IsScaledReg / true);
4284	}
4285
4286	/// For ICmpZero, check to see if we can scale up the comparison. For example, x
4287	/// == y -> xc == yc.
4288	void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
4289	Formula Base) {
4290	if (LU.Kind != LSRUse::ICmpZero) return;
4291
4292	// Determine the integer type for the base formula.
4293	Type *IntTy = Base.getType();
4294	if (!IntTy) return;
4295	if (SE.getTypeSizeInBits(Ty: IntTy) > `64`) return;
4296
4297	// Don't do this if there is more than one offset.
4298	if (LU.MinOffset != LU.MaxOffset) return;
4299
4300	// Check if transformation is valid. It is illegal to multiply pointer.
4301	if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy())
4302	return;
4303	for (const SCEV *BaseReg : Base.BaseRegs)
4304	if (BaseReg->getType()->isPointerTy())
4305	return;
4306	assert(!Base.BaseGV && "ICmpZero use is not legal!");
4307
4308	// Check each interesting stride.
4309	for (int64_t Factor : Factors) {
4310	// Check that Factor can be represented by IntTy
4311	if (!ConstantInt::isValueValidForType(Ty: IntTy, V: Factor))
4312	continue;
4313	// Check that the multiplication doesn't overflow.
4314	if (Base.BaseOffset.isMin() && Factor == -`1`)
4315	continue;
4316	// Not supporting scalable immediates.
4317	if (Base.BaseOffset.isNonZero() && Base.BaseOffset.isScalable())
4318	continue;
4319	Immediate NewBaseOffset = Base.BaseOffset.mulUnsigned(RHS: Factor);
4320	assert(Factor != `0` && "Zero factor not expected!");
4321	if (NewBaseOffset.getFixedValue() / Factor !=
4322	Base.BaseOffset.getFixedValue())
4323	continue;
4324	// If the offset will be truncated at this use, check that it is in bounds.
4325	if (!IntTy->isPointerTy() &&
4326	!ConstantInt::isValueValidForType(Ty: IntTy, V: NewBaseOffset.getFixedValue()))
4327	continue;
4328
4329	// Check that multiplying with the use offset doesn't overflow.
4330	Immediate Offset = LU.MinOffset;
4331	if (Offset.isMin() && Factor == -`1`)
4332	continue;
4333	Offset = Offset.mulUnsigned(RHS: Factor);
4334	if (Offset.getFixedValue() / Factor != LU.MinOffset.getFixedValue())
4335	continue;
4336	// If the offset will be truncated at this use, check that it is in bounds.
4337	if (!IntTy->isPointerTy() &&
4338	!ConstantInt::isValueValidForType(Ty: IntTy, V: Offset.getFixedValue()))
4339	continue;
4340
4341	Formula F = Base;
4342	F.BaseOffset = NewBaseOffset;
4343
4344	// Check that this scale is legal.
4345	if (!isLegalUse(TTI, MinOffset: Offset, MaxOffset: Offset, Kind: LU.Kind, AccessTy: LU.AccessTy, F))
4346	continue;
4347
4348	// Compensate for the use having MinOffset built into it.
4349	F.BaseOffset = F.BaseOffset.addUnsigned(RHS: Offset).subUnsigned(RHS: LU.MinOffset);
4350
4351	const SCEV *FactorS = SE.getConstant(Ty: IntTy, V: Factor);
4352
4353	// Check that multiplying with each base register doesn't overflow.
4354	for (size_t i = `0`, e = F.BaseRegs.size(); i != e; ++i) {
4355	F.BaseRegs [i] = SE.getMulExpr(LHS: F.BaseRegs [i], RHS: FactorS);
4356	if (getExactSDiv(LHS: F.BaseRegs [i], RHS: FactorS, SE) != Base.BaseRegs [i])
4357	goto next;
4358	}
4359
4360	// Check that multiplying with the scaled register doesn't overflow.
4361	if (F.ScaledReg) {
4362	F.ScaledReg = SE.getMulExpr(LHS: F.ScaledReg, RHS: FactorS);
4363	if (getExactSDiv(LHS: F.ScaledReg, RHS: FactorS, SE) != Base.ScaledReg)
4364	continue;
4365	}
4366
4367	// Check that multiplying with the unfolded offset doesn't overflow.
4368	if (F.UnfoldedOffset.isNonZero()) {
4369	if (F.UnfoldedOffset.isMin() && Factor == -`1`)
4370	continue;
4371	F.UnfoldedOffset = F.UnfoldedOffset.mulUnsigned(RHS: Factor);
4372	if (F.UnfoldedOffset.getFixedValue() / Factor !=
4373	Base.UnfoldedOffset.getFixedValue())
4374	continue;
4375	// If the offset will be truncated, check that it is in bounds.
4376	if (!IntTy->isPointerTy() && !ConstantInt::isValueValidForType(
4377	Ty: IntTy, V: F.UnfoldedOffset.getFixedValue()))
4378	continue;
4379	}
4380
4381	// If we make it here and it's legal, add it.
4382	(void)InsertFormula(LU, LUIdx, F);
4383	next:;
4384	}
4385	}
4386
4387	/// Generate stride factor reuse formulae by making use of scaled-offset address
4388	/// modes, for example.
4389	void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
4390	// Determine the integer type for the base formula.
4391	Type *IntTy = Base.getType();
4392	if (!IntTy) return;
4393
4394	// If this Formula already has a scaled register, we can't add another one.
4395	// Try to unscale the formula to generate a better scale.
4396	if (Base.Scale != `0` && !Base.unscale())
4397	return;
4398
4399	assert(Base.Scale == `0` && "unscale did not did its job!");
4400
4401	// Check each interesting stride.
4402	for (int64_t Factor : Factors) {
4403	Base.Scale = Factor;
4404	Base.HasBaseReg = Base.BaseRegs.size() > `1`;
4405	// Check whether this scale is going to be legal.
4406	if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy,
4407	F: Base)) {
4408	// As a special-case, handle special out-of-loop Basic users specially.
4409	// TODO: Reconsider this special case.
4410	if (LU.Kind == LSRUse::Basic &&
4411	isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LSRUse::Special,
4412	AccessTy: LU.AccessTy, F: Base) &&
4413	LU.AllFixupsOutsideLoop)
4414	LU.Kind = LSRUse::Special;
4415	else
4416	continue;
4417	}
4418	// For an ICmpZero, negating a solitary base register won't lead to
4419	// new solutions.
4420	if (LU.Kind == LSRUse::ICmpZero && !Base.HasBaseReg &&
4421	Base.BaseOffset.isZero() && !Base.BaseGV)
4422	continue;
4423	// For each addrec base reg, if its loop is current loop, apply the scale.
4424	for (size_t i = `0`, e = Base.BaseRegs.size(); i != e; ++i) {
4425	const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Base.BaseRegs [i]);
4426	if (AR && (AR->getLoop() == L \|\| LU.AllFixupsOutsideLoop)) {
4427	const SCEV *FactorS = SE.getConstant(Ty: IntTy, V: Factor);
4428	if (FactorS->isZero())
4429	continue;
4430	// Divide out the factor, ignoring high bits, since we'll be
4431	// scaling the value back up in the end.
4432	if (const SCEV Quotient = getExactSDiv(LHS: AR, RHS: FactorS, SE, IgnoreSignificantBits: true*))
4433	if (!Quotient->isZero()) {
4434	// TODO: This could be optimized to avoid all the copying.
4435	Formula F = Base;
4436	F.ScaledReg = Quotient;
4437	F.deleteBaseReg(S&: F.BaseRegs [i]);
4438	// The canonical representation of 1reg is reg, which is already in*
4439	// Base. In that case, do not try to insert the formula, it will be
4440	// rejected anyway.
4441	if (F.Scale == `1` && (F.BaseRegs.empty() \|\|
4442	(AR->getLoop() != L && LU.AllFixupsOutsideLoop)))
4443	continue;
4444	// If AllFixupsOutsideLoop is true and F.Scale is 1, we may generate
4445	// non canonical Formula with ScaledReg's loop not being L.
4446	if (F.Scale == `1` && LU.AllFixupsOutsideLoop)
4447	F.canonicalize(L: *L);
4448	(void)InsertFormula(LU, LUIdx, F);
4449	}
4450	}
4451	}
4452	}
4453	}
4454
4455	/// Extend/Truncate \p Expr to \p ToTy considering post-inc uses in \p Loops.
4456	/// For all PostIncLoopSets in \p Loops, first de-normalize \p Expr, then
4457	/// perform the extension/truncate and normalize again, as the normalized form
4458	/// can result in folds that are not valid in the post-inc use contexts. The
4459	/// expressions for all PostIncLoopSets must match, otherwise return nullptr.
4460	static const SCEV *
4461	getAnyExtendConsideringPostIncUses(ArrayRef<PostIncLoopSet> Loops,
4462	const SCEV Expr, Type ToTy,
4463	ScalarEvolution &SE) {
4464	const SCEV Result = nullptr*;
4465	for (auto &L : Loops) {
4466	auto *DenormExpr = denormalizeForPostIncUse(S: Expr, Loops: L, SE);
4467	const SCEV *NewDenormExpr = SE.getAnyExtendExpr(Op: DenormExpr, Ty: ToTy);
4468	const SCEV *New = normalizeForPostIncUse(S: NewDenormExpr, Loops: L, SE);
4469	if (!New \|\| (Result && New != Result))
4470	return nullptr;
4471	Result = New;
4472	}
4473
4474	assert(Result && "failed to create expression");
4475	return Result;
4476	}
4477
4478	/// Generate reuse formulae from different IV types.
4479	void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
4480	// Don't bother truncating symbolic values.
4481	if (Base.BaseGV) return;
4482
4483	// Determine the integer type for the base formula.
4484	Type *DstTy = Base.getType();
4485	if (!DstTy) return;
4486	if (DstTy->isPointerTy())
4487	return;
4488
4489	// It is invalid to extend a pointer type so exit early if ScaledReg or
4490	// any of the BaseRegs are pointers.
4491	if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy())
4492	return;
4493	if (any_of(Range&: Base.BaseRegs,
4494	P: [](const SCEV S) { return* S->getType()->isPointerTy(); }))
4495	return;
4496
4497	SmallVector<PostIncLoopSet> Loops;
4498	for (auto &LF : LU.Fixups)
4499	Loops.push_back(Elt: LF.PostIncLoops);
4500
4501	for (Type *SrcTy : Types) {
4502	if (SrcTy != DstTy && TTI.isTruncateFree(Ty1: SrcTy, Ty2: DstTy)) {
4503	Formula F = Base;
4504
4505	// Sometimes SCEV is able to prove zero during ext transform. It may
4506	// happen if SCEV did not do all possible transforms while creating the
4507	// initial node (maybe due to depth limitations), but it can do them while
4508	// taking ext.
4509	if (F.ScaledReg) {
4510	const SCEV *NewScaledReg =
4511	getAnyExtendConsideringPostIncUses(Loops, Expr: F.ScaledReg, ToTy: SrcTy, SE);
4512	if (!NewScaledReg \|\| NewScaledReg->isZero())
4513	continue;
4514	F.ScaledReg = NewScaledReg;
4515	}
4516	bool HasZeroBaseReg = false;
4517	for (const SCEV *&BaseReg : F.BaseRegs) {
4518	const SCEV *NewBaseReg =
4519	getAnyExtendConsideringPostIncUses(Loops, Expr: BaseReg, ToTy: SrcTy, SE);
4520	if (!NewBaseReg \|\| NewBaseReg->isZero()) {
4521	HasZeroBaseReg = true;
4522	break;
4523	}
4524	BaseReg = NewBaseReg;
4525	}
4526	if (HasZeroBaseReg)
4527	continue;
4528
4529	// TODO: This assumes we've done basic processing on all uses and
4530	// have an idea what the register usage is.
4531	if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses))
4532	continue;
4533
4534	F.canonicalize(L: *L);
4535	(void)InsertFormula(LU, LUIdx, F);
4536	}
4537	}
4538	}
4539
4540	namespace {
4541
4542	/// Helper class for GenerateCrossUseConstantOffsets. It's used to defer
4543	/// modifications so that the search phase doesn't have to worry about the data
4544	/// structures moving underneath it.
4545	struct WorkItem {
4546	size_t LUIdx;
4547	Immediate Imm;
4548	const SCEV *OrigReg;
4549
4550	WorkItem(size_t LI, Immediate I, const SCEV *R)
4551	: LUIdx(LI), Imm (I), OrigReg(R) {}
4552
4553	void print(raw_ostream &OS) const;
4554	void dump() const;
4555	};
4556
4557	} // end anonymous namespace
4558
4559	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
4560	void WorkItem::print(raw_ostream &OS) const {
4561	OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx
4562	<< " , add offset " << Imm;
4563	}
4564
4565	LLVM_DUMP_METHOD void WorkItem::dump() const {
4566	print(errs()); errs() << `'\n'`;
4567	}
4568	#endif
4569
4570	/// Look for registers which are a constant distance apart and try to form reuse
4571	/// opportunities between them.
4572	void LSRInstance::GenerateCrossUseConstantOffsets() {
4573	// Group the registers by their value without any added constant offset.
4574	using ImmMapTy = std::map<Immediate, const SCEV *, KeyOrderTargetImmediate>;
4575
4576	DenseMap<const SCEV *, ImmMapTy> Map;
4577	DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap;
4578	SmallVector<const SCEV *, `8`> Sequence;
4579	for (const SCEV *Use : RegUses) {
4580	SCEVUse Reg = Use; // Make a copy for ExtractImmediate to modify.
4581	// TODO: Extract both scalable and fixed immediates (if present)?
4582	Immediate Imm = ExtractImmediate(S&: Reg, SE);
4583	auto Pair = Map.try_emplace(Key: Reg);
4584	if (Pair.second)
4585	Sequence.push_back(Elt: Reg);
4586	Pair.first ->second.insert(x: std::make_pair(x&: Imm, y&: Use));
4587	UsedByIndicesMap [Reg] \|= RegUses.getUsedByIndices(Reg: Use);
4588	}
4589
4590	// Now examine each set of registers with the same base value. Build up
4591	// a list of work to do and do the work in a separate step so that we're
4592	// not adding formulae and register counts while we're searching.
4593	SmallVector<WorkItem, `32`> WorkItems;
4594	SmallSet<std::pair<size_t, Immediate>, `32`, KeyOrderSizeTAndImmediate>
4595	UniqueItems;
4596	for (const SCEV *Reg : Sequence) {
4597	const ImmMapTy &Imms = Map.find(Val: Reg)->second;
4598
4599	// It's not worthwhile looking for reuse if there's only one offset.
4600	if (Imms.size() == `1`)
4601	continue;
4602
4603	LLVM_DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << `':'`;
4604	for (const auto &Entry
4605	: Imms) dbgs()
4606	<< `' '` << Entry.first;
4607	dbgs() << `'\n'`);
4608
4609	// Examine each offset.
4610	for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end();
4611	J != JE; ++J) {
4612	const SCEV *OrigReg = J ->second;
4613
4614	Immediate JImm = J ->first;
4615	const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg: OrigReg);
4616
4617	if (!isa<SCEVConstant>(Val: OrigReg) &&
4618	UsedByIndicesMap [Reg].count() == `1`) {
4619	LLVM_DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg
4620	<< `'\n'`);
4621	continue;
4622	}
4623
4624	// Conservatively examine offsets between this orig reg a few selected
4625	// other orig regs.
4626	Immediate First = Imms.begin()->first;
4627	Immediate Last = std::prev(x: Imms.end())->first;
4628	if (!First.isCompatibleImmediate(Imm: Last)) {
4629	LLVM_DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg
4630	<< "\n");
4631	continue;
4632	}
4633	// Only scalable if both terms are scalable, or if one is scalable and
4634	// the other is 0.
4635	bool Scalable = First.isScalable() \|\| Last.isScalable();
4636	int64_t FI = First.getKnownMinValue();
4637	int64_t LI = Last.getKnownMinValue();
4638	// Compute (First + Last) / 2 without overflow using the fact that
4639	// First + Last = 2 (First + Last) + (First ^ Last).*
4640	int64_t Avg = (FI & LI) + ((FI ^ LI) >> `1`);
4641	// If the result is negative and FI is odd and LI even (or vice versa),
4642	// we rounded towards -inf. Add 1 in that case, to round towards 0.
4643	Avg = Avg + ((FI ^ LI) & ((uint64_t)Avg >> `63`));
4644	ImmMapTy::const_iterator OtherImms[] = {
4645	Imms.begin(), std::prev(x: Imms.end()),
4646	Imms.lower_bound(x: Immediate::get(MinVal: Avg, Scalable))};
4647	for (const auto &M : OtherImms) {
4648	if (M == J \|\| M == JE) continue;
4649	if (!JImm.isCompatibleImmediate(Imm: M ->first))
4650	continue;
4651
4652	// Compute the difference between the two.
4653	Immediate Imm = JImm.subUnsigned(RHS: M ->first);
4654	for (unsigned LUIdx : UsedByIndices.set_bits())
4655	// Make a memo of this use, offset, and register tuple.
4656	if (UniqueItems.insert(V: std::make_pair(x&: LUIdx, y&: Imm)).second)
4657	WorkItems.push_back(Elt: WorkItem (LUIdx, Imm, OrigReg));
4658	}
4659	}
4660	}
4661
4662	Map.clear();
4663	Sequence.clear();
4664	UsedByIndicesMap.clear();
4665	UniqueItems.clear();
4666
4667	// Now iterate through the worklist and add new formulae.
4668	for (const WorkItem &WI : WorkItems) {
4669	size_t LUIdx = WI.LUIdx;
4670	LSRUse &LU = Uses [LUIdx];
4671	Immediate Imm = WI.Imm;
4672	const SCEV *OrigReg = WI.OrigReg;
4673
4674	Type *IntTy = SE.getEffectiveSCEVType(Ty: OrigReg->getType());
4675	const SCEV *NegImmS = Imm.getNegativeSCEV(SE, Ty: IntTy);
4676	unsigned BitWidth = SE.getTypeSizeInBits(Ty: IntTy);
4677
4678	// TODO: Use a more targeted data structure.
4679	for (size_t L = `0`, LE = LU.Formulae.size(); L != LE; ++L) {
4680	Formula F = LU.Formulae [L];
4681	// FIXME: The code for the scaled and unscaled registers looks
4682	// very similar but slightly different. Investigate if they
4683	// could be merged. That way, we would not have to unscale the
4684	// Formula.
4685	F.unscale();
4686	// Use the immediate in the scaled register.
4687	if (F.ScaledReg == OrigReg) {
4688	if (!F.BaseOffset.isCompatibleImmediate(Imm))
4689	continue;
4690	Immediate Offset = F.BaseOffset.addUnsigned(RHS: Imm.mulUnsigned(RHS: F.Scale));
4691	// Don't create 50 + reg(-50).
4692	const SCEV *S = Offset.getNegativeSCEV(SE, Ty: IntTy);
4693	if (F.referencesReg(S))
4694	continue;
4695	Formula NewF = F;
4696	NewF.BaseOffset = Offset;
4697	if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy,
4698	F: NewF))
4699	continue;
4700	NewF.ScaledReg = SE.getAddExpr(LHS: NegImmS, RHS: NewF.ScaledReg);
4701
4702	// If the new scale is a constant in a register, and adding the constant
4703	// value to the immediate would produce a value closer to zero than the
4704	// immediate itself, then the formula isn't worthwhile.
4705	if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: NewF.ScaledReg)) {
4706	// FIXME: Do we need to do something for scalable immediates here?
4707	// A scalable SCEV won't be constant, but we might still have
4708	// something in the offset? Bail out for now to be safe.
4709	if (NewF.BaseOffset.isNonZero() && NewF.BaseOffset.isScalable())
4710	continue;
4711	if (C->getValue()->isNegative() !=
4712	(NewF.BaseOffset.isLessThanZero()) &&
4713	(C->getAPInt().abs() * APInt (BitWidth, F.Scale))
4714	.ule(RHS: std::abs(i: NewF.BaseOffset.getFixedValue())))
4715	continue;
4716	}
4717
4718	// OK, looks good.
4719	NewF.canonicalize(L: *this->L);
4720	(void)InsertFormula(LU, LUIdx, F: NewF);
4721	} else {
4722	// Use the immediate in a base register.
4723	for (size_t N = `0`, NE = F.BaseRegs.size(); N != NE; ++N) {
4724	const SCEV *BaseReg = F.BaseRegs [N];
4725	if (BaseReg != OrigReg)
4726	continue;
4727	Formula NewF = F;
4728	if (!NewF.BaseOffset.isCompatibleImmediate(Imm) \|\|
4729	!NewF.UnfoldedOffset.isCompatibleImmediate(Imm) \|\|
4730	!NewF.BaseOffset.isCompatibleImmediate(Imm: NewF.UnfoldedOffset))
4731	continue;
4732	NewF.BaseOffset = NewF.BaseOffset.addUnsigned(RHS: Imm);
4733	if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset,
4734	Kind: LU.Kind, AccessTy: LU.AccessTy, F: NewF)) {
4735	if (AMK == TTI::AMK_PostIndexed &&
4736	mayUsePostIncMode(TTI, LU, S: OrigReg, L: this->L, SE))
4737	continue;
4738	Immediate NewUnfoldedOffset = NewF.UnfoldedOffset.addUnsigned(RHS: Imm);
4739	if (!isLegalAddImmediate(TTI, Offset: NewUnfoldedOffset))
4740	continue;
4741	NewF = F;
4742	NewF.UnfoldedOffset = NewUnfoldedOffset;
4743	}
4744	NewF.BaseRegs [N] = SE.getAddExpr(LHS: NegImmS, RHS: BaseReg);
4745
4746	// If the new formula has a constant in a register, and adding the
4747	// constant value to the immediate would produce a value closer to
4748	// zero than the immediate itself, then the formula isn't worthwhile.
4749	for (const SCEV *NewReg : NewF.BaseRegs)
4750	if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: NewReg)) {
4751	if (NewF.BaseOffset.isNonZero() && NewF.BaseOffset.isScalable())
4752	goto skip_formula;
4753	if ((C->getAPInt() + NewF.BaseOffset.getFixedValue())
4754	.abs()
4755	.slt(RHS: std::abs(i: NewF.BaseOffset.getFixedValue())) &&
4756	(C->getAPInt() + NewF.BaseOffset.getFixedValue())
4757	.countr_zero() >=
4758	(unsigned)llvm::countr_zero<uint64_t>(
4759	Val: NewF.BaseOffset.getFixedValue()))
4760	goto skip_formula;
4761	}
4762
4763	// Ok, looks good.
4764	NewF.canonicalize(L: *this->L);
4765	(void)InsertFormula(LU, LUIdx, F: NewF);
4766	break;
4767	skip_formula:;
4768	}
4769	}
4770	}
4771	}
4772	}
4773
4774	/// Generate formulae for each use.
4775	void
4776	LSRInstance::GenerateAllReuseFormulae() {
4777	// This is split into multiple loops so that hasRegsUsedByUsesOtherThan
4778	// queries are more precise.
4779	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4780	LSRUse &LU = Uses [LUIdx];
4781	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4782	GenerateReassociations(LU, LUIdx, Base: LU.Formulae [i]);
4783	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4784	GenerateCombinations(LU, LUIdx, Base: LU.Formulae [i]);
4785	}
4786	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4787	LSRUse &LU = Uses [LUIdx];
4788	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4789	GenerateSymbolicOffsets(LU, LUIdx, Base: LU.Formulae [i]);
4790	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4791	GenerateConstantOffsets(LU, LUIdx, Base: LU.Formulae [i]);
4792	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4793	GenerateICmpZeroScales(LU, LUIdx, Base: LU.Formulae [i]);
4794	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4795	GenerateScales(LU, LUIdx, Base: LU.Formulae [i]);
4796	}
4797	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4798	LSRUse &LU = Uses [LUIdx];
4799	for (size_t i = `0`, f = LU.Formulae.size(); i != f; ++i)
4800	GenerateTruncates(LU, LUIdx, Base: LU.Formulae [i]);
4801	}
4802
4803	GenerateCrossUseConstantOffsets();
4804
4805	LLVM_DEBUG(dbgs() << "\n"
4806	"After generating reuse formulae:\n";
4807	print_uses(dbgs()));
4808	}
4809
4810	/// If there are multiple formulae with the same set of registers used
4811	/// by other uses, pick the best one and delete the others.
4812	void LSRInstance::FilterOutUndesirableDedicatedRegisters() {
4813	DenseSet<const SCEV *> VisitedRegs;
4814	SmallPtrSet<const SCEV *, `16`> Regs;
4815	SmallPtrSet<const SCEV *, `16`> LoserRegs;
4816	#ifndef NDEBUG
4817	bool ChangedFormulae = false;
4818	#endif
4819
4820	// Collect the best formula for each unique set of shared registers. This
4821	// is reset for each use.
4822	using BestFormulaeTy = DenseMap<SmallVector<const SCEV *, `4`>, size_t>;
4823
4824	BestFormulaeTy BestFormulae;
4825
4826	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4827	LSRUse &LU = Uses [LUIdx];
4828	LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4829	dbgs() << `'\n'`);
4830
4831	bool Any = false;
4832	for (size_t FIdx = `0`, NumForms = LU.Formulae.size();
4833	FIdx != NumForms; ++FIdx) {
4834	Formula &F = LU.Formulae [FIdx];
4835
4836	// Some formulas are instant losers. For example, they may depend on
4837	// nonexistent AddRecs from other loops. These need to be filtered
4838	// immediately, otherwise heuristics could choose them over others leading
4839	// to an unsatisfactory solution. Passing LoserRegs into RateFormula here
4840	// avoids the need to recompute this information across formulae using the
4841	// same bad AddRec. Passing LoserRegs is also essential unless we remove
4842	// the corresponding bad register from the Regs set.
4843	Cost CostF(L, SE, TTI, AMK);
4844	Regs.clear();
4845	CostF.RateFormula(F, Regs, VisitedRegs, LU, HardwareLoopProfitable,
4846	LoserRegs: &LoserRegs);
4847	if (CostF.isLoser()) {
4848	// During initial formula generation, undesirable formulae are generated
4849	// by uses within other loops that have some non-trivial address mode or
4850	// use the postinc form of the IV. LSR needs to provide these formulae
4851	// as the basis of rediscovering the desired formula that uses an AddRec
4852	// corresponding to the existing phi. Once all formulae have been
4853	// generated, these initial losers may be pruned.
4854	LLVM_DEBUG(dbgs() << " Filtering loser "; F.print(dbgs());
4855	dbgs() << "\n");
4856	}
4857	else {
4858	SmallVector<const SCEV *, `4`> Key;
4859	for (const SCEV *Reg : F.BaseRegs) {
4860	if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx))
4861	Key.push_back(Elt: Reg);
4862	}
4863	if (F.ScaledReg &&
4864	RegUses.isRegUsedByUsesOtherThan(Reg: F.ScaledReg, LUIdx))
4865	Key.push_back(Elt: F.ScaledReg);
4866	// Unstable sort by host order ok, because this is only used for
4867	// uniquifying.
4868	llvm::sort(C&: Key);
4869
4870	std::pair<BestFormulaeTy::const_iterator, bool> P =
4871	BestFormulae.insert(KV: std::make_pair(x&: Key, y&: FIdx));
4872	if (P.second)
4873	continue;
4874
4875	Formula &Best = LU.Formulae [P.first ->second];
4876
4877	Cost CostBest(L, SE, TTI, AMK);
4878	Regs.clear();
4879	CostBest.RateFormula(F: Best, Regs, VisitedRegs, LU,
4880	HardwareLoopProfitable);
4881	if (CostF.isLess(Other: CostBest))
4882	std::swap(a&: F, b&: Best);
4883	LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
4884	dbgs() << "\n"
4885	" in favor of formula ";
4886	Best.print(dbgs()); dbgs() << `'\n'`);
4887	}
4888	#ifndef NDEBUG
4889	ChangedFormulae = true;
4890	#endif
4891	LU.DeleteFormula(F);
4892	--FIdx;
4893	--NumForms;
4894	Any = true;
4895	}
4896
4897	// Now that we've filtered out some formulae, recompute the Regs set.
4898	if (Any)
4899	LU.RecomputeRegs(LUIdx, RegUses);
4900
4901	// Reset this to prepare for the next use.
4902	BestFormulae.clear();
4903	}
4904
4905	LLVM_DEBUG(if (ChangedFormulae) {
4906	dbgs() << "\n"
4907	"After filtering out undesirable candidates:\n";
4908	print_uses(dbgs());
4909	});
4910	}
4911
4912	/// Estimate the worst-case number of solutions the solver might have to
4913	/// consider. It almost never considers this many solutions because it prune the
4914	/// search space, but the pruning isn't always sufficient.
4915	size_t LSRInstance::EstimateSearchSpaceComplexity() const {
4916	size_t Power = `1`;
4917	for (const LSRUse &LU : Uses) {
4918	size_t FSize = LU.Formulae.size();
4919	if (FSize >= ComplexityLimit) {
4920	Power = ComplexityLimit;
4921	break;
4922	}
4923	Power *= FSize;
4924	if (Power >= ComplexityLimit)
4925	break;
4926	}
4927	return Power;
4928	}
4929
4930	/// When one formula uses a superset of the registers of another formula, it
4931	/// won't help reduce register pressure (though it may not necessarily hurt
4932	/// register pressure); remove it to simplify the system.
4933	void LSRInstance::NarrowSearchSpaceByDetectingSupersets() {
4934	if (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
4935	LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
4936
4937	LLVM_DEBUG(dbgs() << "Narrowing the search space by eliminating formulae "
4938	"which use a superset of registers used by other "
4939	"formulae.\n");
4940
4941	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4942	LSRUse &LU = Uses [LUIdx];
4943	bool Any = false;
4944	for (size_t i = `0`, e = LU.Formulae.size(); i != e; ++i) {
4945	Formula &F = LU.Formulae [i];
4946	if (F.BaseOffset.isNonZero() && F.BaseOffset.isScalable())
4947	continue;
4948	// Look for a formula with a constant or GV in a register. If the use
4949	// also has a formula with that same value in an immediate field,
4950	// delete the one that uses a register.
4951	for (SmallVectorImpl<const SCEV *>::const_iterator
4952	I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) {
4953	if (const SCEVConstant C = dyn_cast<SCEVConstant>(Val: I)) {
4954	Formula NewF = F;
4955	//FIXME: Formulas should store bitwidth to do wrapping properly.
4956	// See PR41034.
4957	NewF.BaseOffset =
4958	Immediate::getFixed(MinVal: NewF.BaseOffset.getFixedValue() +
4959	(uint64_t)C->getValue()->getSExtValue());
4960	NewF.BaseRegs.erase(CI: NewF.BaseRegs.begin() +
4961	(I - F.BaseRegs.begin()));
4962	if (LU.HasFormulaWithSameRegs(F: NewF)) {
4963	LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());
4964	dbgs() << `'\n'`);
4965	LU.DeleteFormula(F);
4966	--i;
4967	--e;
4968	Any = true;
4969	break;
4970	}
4971	} else if (const SCEVUnknown U = dyn_cast<SCEVUnknown>(Val: I)) {
4972	if (GlobalValue *GV = dyn_cast<GlobalValue>(Val: U->getValue()))
4973	if (!F.BaseGV) {
4974	Formula NewF = F;
4975	NewF.BaseGV = GV;
4976	NewF.BaseRegs.erase(CI: NewF.BaseRegs.begin() +
4977	(I - F.BaseRegs.begin()));
4978	if (LU.HasFormulaWithSameRegs(F: NewF)) {
4979	LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());
4980	dbgs() << `'\n'`);
4981	LU.DeleteFormula(F);
4982	--i;
4983	--e;
4984	Any = true;
4985	break;
4986	}
4987	}
4988	}
4989	}
4990	}
4991	if (Any)
4992	LU.RecomputeRegs(LUIdx, RegUses);
4993	}
4994
4995	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
4996	}
4997	}
4998
4999	/// When there are many registers for expressions like A, A+1, A+2, etc.,
5000	/// allocate a single register for them.
5001	void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() {
5002	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5003	return;
5004
5005	LLVM_DEBUG(
5006	dbgs() << "The search space is too complex.\n"
5007	"Narrowing the search space by assuming that uses separated "
5008	"by a constant offset will use the same registers.\n");
5009
5010	// This is especially useful for unrolled loops.
5011
5012	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5013	LSRUse &LU = Uses [LUIdx];
5014	for (const Formula &F : LU.Formulae) {
5015	if (F.BaseOffset.isZero() \|\| (F.Scale != `0` && F.Scale != `1`))
5016	continue;
5017	assert((LU.Kind == LSRUse::Address \|\| LU.Kind == LSRUse::ICmpZero) &&
5018	"Only address and cmp uses expected to have nonzero BaseOffset");
5019
5020	LSRUse *LUThatHas = FindUseWithSimilarFormula(OrigF: F, OrigLU: LU);
5021	if (!LUThatHas)
5022	continue;
5023
5024	if (!reconcileNewOffset(LU&: LUThatHas, NewOffset: F.BaseOffset, /HasBaseReg=/* false,
5025	Kind: LU.Kind, AccessTy: LU.AccessTy))
5026	continue;
5027
5028	LLVM_DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << `'\n'`);
5029
5030	LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
5031	LUThatHas->AllFixupsUnconditional &= LU.AllFixupsUnconditional;
5032
5033	// Transfer the fixups of LU to LUThatHas.
5034	for (LSRFixup &Fixup : LU.Fixups) {
5035	Fixup.Offset += F.BaseOffset;
5036	LUThatHas->pushFixup(f&: Fixup);
5037	LLVM_DEBUG(dbgs() << "New fixup has offset " << Fixup.Offset << `'\n'`);
5038	}
5039
5040	#ifndef NDEBUG
5041	Type *FixupType = LUThatHas->Fixups[`0`].OperandValToReplace->getType();
5042	for (LSRFixup &Fixup : LUThatHas->Fixups)
5043	assert(Fixup.OperandValToReplace->getType() == FixupType &&
5044	"Expected all fixups to have the same type");
5045	#endif
5046
5047	// Delete formulae from the new use which are no longer legal.
5048	bool Any = false;
5049	for (size_t i = `0`, e = LUThatHas->Formulae.size(); i != e; ++i) {
5050	Formula &F = LUThatHas->Formulae [i];
5051	if (!isLegalUse(TTI, MinOffset: LUThatHas->MinOffset, MaxOffset: LUThatHas->MaxOffset,
5052	Kind: LUThatHas->Kind, AccessTy: LUThatHas->AccessTy, F)) {
5053	LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << `'\n'`);
5054	LUThatHas->DeleteFormula(F);
5055	--i;
5056	--e;
5057	Any = true;
5058	}
5059	}
5060
5061	if (Any)
5062	LUThatHas->RecomputeRegs(LUIdx: LUThatHas - &Uses.front(), RegUses);
5063
5064	// Delete the old use.
5065	DeleteUse(LU, LUIdx);
5066	--LUIdx;
5067	--NumUses;
5068	break;
5069	}
5070	}
5071
5072	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5073	}
5074
5075	/// Call FilterOutUndesirableDedicatedRegisters again, if necessary, now that
5076	/// we've done more filtering, as it may be able to find more formulae to
5077	/// eliminate.
5078	void LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){
5079	if (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
5080	LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5081
5082	LLVM_DEBUG(dbgs() << "Narrowing the search space by re-filtering out "
5083	"undesirable dedicated registers.\n");
5084
5085	FilterOutUndesirableDedicatedRegisters();
5086
5087	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5088	}
5089	}
5090
5091	/// If a LSRUse has multiple formulae with the same ScaledReg and Scale.
5092	/// Pick the best one and delete the others.
5093	/// This narrowing heuristic is to keep as many formulae with different
5094	/// Scale and ScaledReg pair as possible while narrowing the search space.
5095	/// The benefit is that it is more likely to find out a better solution
5096	/// from a formulae set with more Scale and ScaledReg variations than
5097	/// a formulae set with the same Scale and ScaledReg. The picking winner
5098	/// reg heuristic will often keep the formulae with the same Scale and
5099	/// ScaledReg and filter others, and we want to avoid that if possible.
5100	void LSRInstance::NarrowSearchSpaceByFilterFormulaWithSameScaledReg() {
5101	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5102	return;
5103
5104	LLVM_DEBUG(
5105	dbgs() << "The search space is too complex.\n"
5106	"Narrowing the search space by choosing the best Formula "
5107	"from the Formulae with the same Scale and ScaledReg.\n");
5108
5109	// Map the "Scale ScaledReg" pair to the best formula of current LSRUse.*
5110	using BestFormulaeTy = DenseMap<std::pair<const SCEV *, int64_t>, size_t>;
5111
5112	BestFormulaeTy BestFormulae;
5113	#ifndef NDEBUG
5114	bool ChangedFormulae = false;
5115	#endif
5116	DenseSet<const SCEV *> VisitedRegs;
5117	SmallPtrSet<const SCEV *, `16`> Regs;
5118
5119	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5120	LSRUse &LU = Uses [LUIdx];
5121	LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
5122	dbgs() << `'\n'`);
5123
5124	// Return true if Formula FA is better than Formula FB.
5125	auto IsBetterThan = [&](Formula &FA, Formula &FB) {
5126	// First we will try to choose the Formula with fewer new registers.
5127	// For a register used by current Formula, the more the register is
5128	// shared among LSRUses, the less we increase the register number
5129	// counter of the formula.
5130	size_t FARegNum = `0`;
5131	for (const SCEV *Reg : FA.BaseRegs) {
5132	const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg);
5133	FARegNum += (NumUses - UsedByIndices.count() + `1`);
5134	}
5135	size_t FBRegNum = `0`;
5136	for (const SCEV *Reg : FB.BaseRegs) {
5137	const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg);
5138	FBRegNum += (NumUses - UsedByIndices.count() + `1`);
5139	}
5140	if (FARegNum != FBRegNum)
5141	return FARegNum < FBRegNum;
5142
5143	// If the new register numbers are the same, choose the Formula with
5144	// less Cost.
5145	Cost CostFA(L, SE, TTI, AMK);
5146	Cost CostFB(L, SE, TTI, AMK);
5147	Regs.clear();
5148	CostFA.RateFormula(F: FA, Regs, VisitedRegs, LU, HardwareLoopProfitable);
5149	Regs.clear();
5150	CostFB.RateFormula(F: FB, Regs, VisitedRegs, LU, HardwareLoopProfitable);
5151	return CostFA.isLess(Other: CostFB);
5152	};
5153
5154	bool Any = false;
5155	for (size_t FIdx = `0`, NumForms = LU.Formulae.size(); FIdx != NumForms;
5156	++FIdx) {
5157	Formula &F = LU.Formulae [FIdx];
5158	if (!F.ScaledReg)
5159	continue;
5160	auto P = BestFormulae.insert(KV: {{F.ScaledReg, F.Scale}, FIdx});
5161	if (P.second)
5162	continue;
5163
5164	Formula &Best = LU.Formulae [P.first ->second];
5165	if (IsBetterThan (F, Best))
5166	std::swap(a&: F, b&: Best);
5167	LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
5168	dbgs() << "\n"
5169	" in favor of formula ";
5170	Best.print(dbgs()); dbgs() << `'\n'`);
5171	#ifndef NDEBUG
5172	ChangedFormulae = true;
5173	#endif
5174	LU.DeleteFormula(F);
5175	--FIdx;
5176	--NumForms;
5177	Any = true;
5178	}
5179	if (Any)
5180	LU.RecomputeRegs(LUIdx, RegUses);
5181
5182	// Reset this to prepare for the next use.
5183	BestFormulae.clear();
5184	}
5185
5186	LLVM_DEBUG(if (ChangedFormulae) {
5187	dbgs() << "\n"
5188	"After filtering out undesirable candidates:\n";
5189	print_uses(dbgs());
5190	});
5191	}
5192
5193	/// If we are over the complexity limit, filter out any post-inc prefering
5194	/// variables to only post-inc values.
5195	void LSRInstance::NarrowSearchSpaceByFilterPostInc() {
5196	if (AMK != TTI::AMK_PostIndexed)
5197	return;
5198	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5199	return;
5200
5201	LLVM_DEBUG(dbgs() << "The search space is too complex.\n"
5202	"Narrowing the search space by choosing the lowest "
5203	"register Formula for PostInc Uses.\n");
5204
5205	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5206	LSRUse &LU = Uses [LUIdx];
5207
5208	if (LU.Kind != LSRUse::Address)
5209	continue;
5210	if (!TTI.isIndexedLoadLegal(Mode: TTI.MIM_PostInc, Ty: LU.AccessTy.getType()) &&
5211	!TTI.isIndexedStoreLegal(Mode: TTI.MIM_PostInc, Ty: LU.AccessTy.getType()))
5212	continue;
5213
5214	size_t MinRegs = std::numeric_limits<size_t>::max();
5215	for (const Formula &F : LU.Formulae)
5216	MinRegs = std::min(a: F.getNumRegs(), b: MinRegs);
5217
5218	bool Any = false;
5219	for (size_t FIdx = `0`, NumForms = LU.Formulae.size(); FIdx != NumForms;
5220	++FIdx) {
5221	Formula &F = LU.Formulae [FIdx];
5222	if (F.getNumRegs() > MinRegs) {
5223	LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
5224	dbgs() << "\n");
5225	LU.DeleteFormula(F);
5226	--FIdx;
5227	--NumForms;
5228	Any = true;
5229	}
5230	}
5231	if (Any)
5232	LU.RecomputeRegs(LUIdx, RegUses);
5233
5234	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5235	break;
5236	}
5237
5238	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5239	}
5240
5241	void LSRInstance::NarrowSearchSpaceByMergingUsesOutsideLoop() {
5242	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5243	return;
5244
5245	LLVM_DEBUG(
5246	dbgs() << "The search space is too complex.\n"
5247	"Narrowing the search space by merging uses with fixups "
5248	"entirely outside the loop with uses inside the loop.\n");
5249
5250	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5251	LSRUse &LU = Uses [LUIdx];
5252	// Don't merge ICmpZero uses outside the loop, as ICmpZero needs to be
5253	// handled specially when expanding.
5254	if (!LU.AllFixupsOutsideLoop \|\| LU.Formulae.empty() \|\|
5255	LU.Kind == LSRUse::ICmpZero)
5256	continue;
5257
5258	LLVM_DEBUG(dbgs() << " Trying to eliminate use "; LU.print(dbgs());
5259	dbgs() << `'\n'`);
5260
5261	// Find a compatible LSRUse inside the loop that we could merge LU with
5262	LSRUse LUToMergeWith = nullptr*;
5263	const Formula &ThisF = LU.Formulae [`0`];
5264	for (LSRUse &OtherLU : Uses) {
5265	// Only merge with uses inside the loop
5266	if (OtherLU.AllFixupsOutsideLoop)
5267	continue;
5268	// Can't merge with ICmpZero uses as they're handled specially when
5269	// expanding
5270	if (OtherLU.Kind == LSRUse::ICmpZero)
5271	continue;
5272	// Can't merge with uses without any formulae
5273	if (OtherLU.Formulae.empty())
5274	continue;
5275	// Can't merge if LU's offsets aren't legal for all of OtherLU's formulae
5276	if (any_of(Range&: OtherLU.Formulae, P: [&](const Formula &F) {
5277	return !isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: OtherLU.Kind,
5278	AccessTy: OtherLU.AccessTy, F);
5279	}))
5280	continue;
5281	// We can merge with uses that have the same initial formula. We allow
5282	// merging of uses with different Kind and AccessTy which means that the
5283	// cost may end up being inaccurate, but it's also what we would have
5284	// gotten if we'd ignored uses outside the loop entirely.
5285	const Formula &OtherF = OtherLU.Formulae [`0`];
5286	if (ThisF.BaseRegs == OtherF.BaseRegs &&
5287	ThisF.ScaledReg == OtherF.ScaledReg &&
5288	ThisF.BaseGV == OtherF.BaseGV && ThisF.Scale == OtherF.Scale &&
5289	ThisF.UnfoldedOffset == OtherF.UnfoldedOffset &&
5290	ThisF.BaseOffset == OtherF.BaseOffset) {
5291	LUToMergeWith = &OtherLU;
5292	break;
5293	}
5294	}
5295	if (!LUToMergeWith)
5296	continue;
5297
5298	LLVM_DEBUG(dbgs() << " Merging with "; LUToMergeWith->print(dbgs());
5299	dbgs() << `'\n'`);
5300
5301	// Copy fixups
5302	for (LSRFixup &Fixup : LU.Fixups) {
5303	LUToMergeWith->pushFixup(f&: Fixup);
5304	}
5305
5306	// Delete the old use.
5307	DeleteUse(LU, LUIdx);
5308	--LUIdx;
5309	--NumUses;
5310	}
5311
5312	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5313	}
5314
5315	/// The function delete formulas with high registers number expectation.
5316	/// Assuming we don't know the value of each formula (already delete
5317	/// all inefficient), generate probability of not selecting for each
5318	/// register.
5319	/// For example,
5320	/// Use1:
5321	/// reg(a) + reg({0,+,1})
5322	/// reg(a) + reg({-1,+,1}) + 1
5323	/// reg({a,+,1})
5324	/// Use2:
5325	/// reg(b) + reg({0,+,1})
5326	/// reg(b) + reg({-1,+,1}) + 1
5327	/// reg({b,+,1})
5328	/// Use3:
5329	/// reg(c) + reg(b) + reg({0,+,1})
5330	/// reg(c) + reg({b,+,1})
5331	///
5332	/// Probability of not selecting
5333	/// Use1 Use2 Use3
5334	/// reg(a) (1/3) 1 * 1*
5335	/// reg(b) 1 (1/3) * (1/2)*
5336	/// reg({0,+,1}) (2/3) (2/3) * (1/2)*
5337	/// reg({-1,+,1}) (2/3) (2/3) * 1*
5338	/// reg({a,+,1}) (2/3) 1 * 1*
5339	/// reg({b,+,1}) 1 (2/3) * (2/3)*
5340	/// reg(c) 1 1 * 0*
5341	///
5342	/// Now count registers number mathematical expectation for each formula:
5343	/// Note that for each use we exclude probability if not selecting for the use.
5344	/// For example for Use1 probability for reg(a) would be just 1 1 (excluding*
5345	/// probabilty 1/3 of not selecting for Use1).
5346	/// Use1:
5347	/// reg(a) + reg({0,+,1}) 1 + 1/3 -- to be deleted
5348	/// reg(a) + reg({-1,+,1}) + 1 1 + 4/9 -- to be deleted
5349	/// reg({a,+,1}) 1
5350	/// Use2:
5351	/// reg(b) + reg({0,+,1}) 1/2 + 1/3 -- to be deleted
5352	/// reg(b) + reg({-1,+,1}) + 1 1/2 + 2/3 -- to be deleted
5353	/// reg({b,+,1}) 2/3
5354	/// Use3:
5355	/// reg(c) + reg(b) + reg({0,+,1}) 1 + 1/3 + 4/9 -- to be deleted
5356	/// reg(c) + reg({b,+,1}) 1 + 2/3
5357	void LSRInstance::NarrowSearchSpaceByDeletingCostlyFormulas() {
5358	if (EstimateSearchSpaceComplexity() < ComplexityLimit)
5359	return;
5360	// Ok, we have too many of formulae on our hands to conveniently handle.
5361	// Use a rough heuristic to thin out the list.
5362
5363	// Set of Regs wich will be 100% used in final solution.
5364	// Used in each formula of a solution (in example above this is reg(c)).
5365	// We can skip them in calculations.
5366	SmallPtrSet<const SCEV *, `4`> UniqRegs;
5367	LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5368
5369	// Map each register to probability of not selecting
5370	DenseMap <const SCEV , float*> RegNumMap;
5371	for (const SCEV *Reg : RegUses) {
5372	if (UniqRegs.count(Ptr: Reg))
5373	continue;
5374	float PNotSel = `1`;
5375	for (const LSRUse &LU : Uses) {
5376	if (!LU.Regs.count(Ptr: Reg))
5377	continue;
5378	float P = LU.getNotSelectedProbability(Reg);
5379	if (P != `0.0`)
5380	PNotSel *= P;
5381	else
5382	UniqRegs.insert(Ptr: Reg);
5383	}
5384	RegNumMap.insert(KV: std::make_pair(x&: Reg, y&: PNotSel));
5385	}
5386
5387	LLVM_DEBUG(
5388	dbgs() << "Narrowing the search space by deleting costly formulas\n");
5389
5390	// Delete formulas where registers number expectation is high.
5391	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5392	LSRUse &LU = Uses [LUIdx];
5393	// If nothing to delete - continue.
5394	if (LU.Formulae.size() < `2`)
5395	continue;
5396	// This is temporary solution to test performance. Float should be
5397	// replaced with round independent type (based on integers) to avoid
5398	// different results for different target builds.
5399	float FMinRegNum = LU.Formulae [`0`].getNumRegs();
5400	float FMinARegNum = LU.Formulae [`0`].getNumRegs();
5401	size_t MinIdx = `0`;
5402	for (size_t i = `0`, e = LU.Formulae.size(); i != e; ++i) {
5403	Formula &F = LU.Formulae [i];
5404	float FRegNum = `0`;
5405	float FARegNum = `0`;
5406	for (const SCEV *BaseReg : F.BaseRegs) {
5407	if (UniqRegs.count(Ptr: BaseReg))
5408	continue;
5409	FRegNum += RegNumMap [BaseReg] / LU.getNotSelectedProbability(Reg: BaseReg);
5410	if (isa<SCEVAddRecExpr>(Val: BaseReg))
5411	FARegNum +=
5412	RegNumMap [BaseReg] / LU.getNotSelectedProbability(Reg: BaseReg);
5413	}
5414	if (const SCEV *ScaledReg = F.ScaledReg) {
5415	if (!UniqRegs.count(Ptr: ScaledReg)) {
5416	FRegNum +=
5417	RegNumMap [ScaledReg] / LU.getNotSelectedProbability(Reg: ScaledReg);
5418	if (isa<SCEVAddRecExpr>(Val: ScaledReg))
5419	FARegNum +=
5420	RegNumMap [ScaledReg] / LU.getNotSelectedProbability(Reg: ScaledReg);
5421	}
5422	}
5423	if (FMinRegNum > FRegNum \|\|
5424	(FMinRegNum == FRegNum && FMinARegNum > FARegNum)) {
5425	FMinRegNum = FRegNum;
5426	FMinARegNum = FARegNum;
5427	MinIdx = i;
5428	}
5429	}
5430	LLVM_DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs());
5431	dbgs() << " with min reg num " << FMinRegNum << `'\n'`);
5432	if (MinIdx != `0`)
5433	std::swap(a&: LU.Formulae [MinIdx], b&: LU.Formulae [`0`]);
5434	while (LU.Formulae.size() != `1`) {
5435	LLVM_DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs());
5436	dbgs() << `'\n'`);
5437	LU.Formulae.pop_back();
5438	}
5439	LU.RecomputeRegs(LUIdx, RegUses);
5440	assert(LU.Formulae.size() == `1` && "Should be exactly 1 min regs formula");
5441	Formula &F = LU.Formulae [`0`];
5442	LLVM_DEBUG(dbgs() << " Leaving only "; F.print(dbgs()); dbgs() << `'\n'`);
5443	// When we choose the formula, the regs become unique.
5444	UniqRegs.insert_range(R&: F.BaseRegs);
5445	if (F.ScaledReg)
5446	UniqRegs.insert(Ptr: F.ScaledReg);
5447	}
5448	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5449	}
5450
5451	// Check if Best and Reg are SCEVs separated by a constant amount C, and if so
5452	// would the addressing offset +C would be legal where the negative offset -C is
5453	// not.
5454	static bool IsSimplerBaseSCEVForTarget(const TargetTransformInfo &TTI,
5455	ScalarEvolution &SE, const SCEV *Best,
5456	const SCEV *Reg,
5457	MemAccessTy AccessType) {
5458	if (Best->getType() != Reg->getType() \|\|
5459	(isa<SCEVAddRecExpr>(Val: Best) && isa<SCEVAddRecExpr>(Val: Reg) &&
5460	cast<SCEVAddRecExpr>(Val: Best)->getLoop() !=
5461	cast<SCEVAddRecExpr>(Val: Reg)->getLoop()))
5462	return false;
5463	std::optional<APInt> Diff = SE.computeConstantDifference(LHS: Best, RHS: Reg);
5464	if (!Diff)
5465	return false;
5466
5467	return TTI.isLegalAddressingMode(
5468	Ty: AccessType.MemTy, /BaseGV=/nullptr,
5469	/BaseOffset=/Diff ->getSExtValue(),
5470	/HasBaseReg=/true, /Scale=/`0`, AddrSpace: AccessType.AddrSpace) &&
5471	!TTI.isLegalAddressingMode(
5472	Ty: AccessType.MemTy, /BaseGV=/nullptr,
5473	/BaseOffset=/-Diff ->getSExtValue(),
5474	/HasBaseReg=/true, /Scale=/`0`, AddrSpace: AccessType.AddrSpace);
5475	}
5476
5477	/// Pick a register which seems likely to be profitable, and then in any use
5478	/// which has any reference to that register, delete all formulae which do not
5479	/// reference that register.
5480	void LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() {
5481	// With all other options exhausted, loop until the system is simple
5482	// enough to handle.
5483	SmallPtrSet<const SCEV *, `4`> Taken;
5484	while (EstimateSearchSpaceComplexity() >= ComplexityLimit) {
5485	// Ok, we have too many of formulae on our hands to conveniently handle.
5486	// Use a rough heuristic to thin out the list.
5487	LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5488
5489	// Pick the register which is used by the most LSRUses, which is likely
5490	// to be a good reuse register candidate.
5491	const SCEV Best = nullptr*;
5492	unsigned BestNum = `0`;
5493	for (const SCEV *Reg : RegUses) {
5494	if (Taken.count(Ptr: Reg))
5495	continue;
5496	if (!Best) {
5497	Best = Reg;
5498	BestNum = RegUses.getUsedByIndices(Reg).count();
5499	} else {
5500	unsigned Count = RegUses.getUsedByIndices(Reg).count();
5501	if (Count > BestNum) {
5502	Best = Reg;
5503	BestNum = Count;
5504	}
5505
5506	// If the scores are the same, but the Reg is simpler for the target
5507	// (for example {x,+,1} as opposed to {x+C,+,1}, where the target can
5508	// handle +C but not -C), opt for the simpler formula.
5509	if (Count == BestNum) {
5510	int LUIdx = RegUses.getUsedByIndices(Reg).find_first();
5511	if (LUIdx >= `0` && Uses [LUIdx].Kind == LSRUse::Address &&
5512	IsSimplerBaseSCEVForTarget(TTI, SE, Best, Reg,
5513	AccessType: Uses [LUIdx].AccessTy)) {
5514	Best = Reg;
5515	BestNum = Count;
5516	}
5517	}
5518	}
5519	}
5520	assert(Best && "Failed to find best LSRUse candidate");
5521
5522	LLVM_DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best
5523	<< " will yield profitable reuse.\n");
5524	Taken.insert(Ptr: Best);
5525
5526	// In any use with formulae which references this register, delete formulae
5527	// which don't reference it.
5528	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5529	LSRUse &LU = Uses [LUIdx];
5530	if (!LU.Regs.count(Ptr: Best)) continue;
5531
5532	bool Any = false;
5533	for (size_t i = `0`, e = LU.Formulae.size(); i != e; ++i) {
5534	Formula &F = LU.Formulae [i];
5535	if (!F.referencesReg(S: Best)) {
5536	LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << `'\n'`);
5537	LU.DeleteFormula(F);
5538	--e;
5539	--i;
5540	Any = true;
5541	assert(e != `0` && "Use has no formulae left! Is Regs inconsistent?");
5542	continue;
5543	}
5544	}
5545
5546	if (Any)
5547	LU.RecomputeRegs(LUIdx, RegUses);
5548	}
5549
5550	LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5551	}
5552	}
5553
5554	/// If there are an extraordinary number of formulae to choose from, use some
5555	/// rough heuristics to prune down the number of formulae. This keeps the main
5556	/// solver from taking an extraordinary amount of time in some worst-case
5557	/// scenarios.
5558	void LSRInstance::NarrowSearchSpaceUsingHeuristics() {
5559	NarrowSearchSpaceByDetectingSupersets();
5560	NarrowSearchSpaceByCollapsingUnrolledCode();
5561	NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
5562	if (FilterSameScaledReg)
5563	NarrowSearchSpaceByFilterFormulaWithSameScaledReg();
5564	NarrowSearchSpaceByFilterPostInc();
5565	NarrowSearchSpaceByMergingUsesOutsideLoop();
5566	if (LSRExpNarrow)
5567	NarrowSearchSpaceByDeletingCostlyFormulas();
5568	else
5569	NarrowSearchSpaceByPickingWinnerRegs();
5570	}
5571
5572	/// This is the recursive solver.
5573	void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
5574	Cost &SolutionCost,
5575	SmallVectorImpl<const Formula *> &Workspace,
5576	const Cost &CurCost,
5577	const SmallPtrSet<const SCEV *, `16`> &CurRegs,
5578	DenseSet<const SCEV > &VisitedRegs) const* {
5579	// Some ideas:
5580	// - prune more:
5581	// - use more aggressive filtering
5582	// - sort the formula so that the most profitable solutions are found first
5583	// - sort the uses too
5584	// - search faster:
5585	// - don't compute a cost, and then compare. compare while computing a cost
5586	// and bail early.
5587	// - track register sets with SmallBitVector
5588
5589	const LSRUse &LU = Uses [Workspace.size()];
5590
5591	// If this use references any register that's already a part of the
5592	// in-progress solution, consider it a requirement that a formula must
5593	// reference that register in order to be considered. This prunes out
5594	// unprofitable searching.
5595	SmallSetVector<const SCEV *, `4`> ReqRegs;
5596	for (const SCEV *S : CurRegs)
5597	if (LU.Regs.count(Ptr: S))
5598	ReqRegs.insert(X: S);
5599
5600	SmallPtrSet<const SCEV *, `16`> NewRegs;
5601	Cost NewCost(L, SE, TTI, AMK);
5602	for (const Formula &F : LU.Formulae) {
5603	// Ignore formulae which may not be ideal in terms of register reuse of
5604	// ReqRegs. The formula should use all required registers before
5605	// introducing new ones.
5606	// This can sometimes (notably when trying to favour postinc) lead to
5607	// sub-optimial decisions. There it is best left to the cost modelling to
5608	// get correct.
5609	if (!(AMK & TTI::AMK_PostIndexed) \|\| LU.Kind != LSRUse::Address) {
5610	int NumReqRegsToFind = std::min(a: F.getNumRegs(), b: ReqRegs.size());
5611	for (const SCEV *Reg : ReqRegs) {
5612	if ((F.ScaledReg && F.ScaledReg == Reg) \|\|
5613	is_contained(Range: F.BaseRegs, Element: Reg)) {
5614	--NumReqRegsToFind;
5615	if (NumReqRegsToFind == `0`)
5616	break;
5617	}
5618	}
5619	if (NumReqRegsToFind != `0`) {
5620	// If none of the formulae satisfied the required registers, then we could
5621	// clear ReqRegs and try again. Currently, we simply give up in this case.
5622	continue;
5623	}
5624	}
5625
5626	// Evaluate the cost of the current formula. If it's already worse than
5627	// the current best, prune the search at that point.
5628	NewCost = CurCost;
5629	NewRegs = CurRegs;
5630	NewCost.RateFormula(F, Regs&: NewRegs, VisitedRegs, LU, HardwareLoopProfitable);
5631	if (NewCost.isLess(Other: SolutionCost)) {
5632	Workspace.push_back(Elt: &F);
5633	if (Workspace.size() != Uses.size()) {
5634	SolveRecurse(Solution, SolutionCost, Workspace, CurCost: NewCost,
5635	CurRegs: NewRegs, VisitedRegs);
5636	if (F.getNumRegs() == `1` && Workspace.size() == `1`)
5637	VisitedRegs.insert(V: F.ScaledReg ? F.ScaledReg : F.BaseRegs [`0`]);
5638	} else {
5639	LLVM_DEBUG(dbgs() << "New best at "; NewCost.print(dbgs());
5640	dbgs() << ".\nRegs:\n";
5641	for (const SCEV *S : NewRegs) dbgs()
5642	<< "- " << *S << "\n";
5643	dbgs() << `'\n'`);
5644
5645	SolutionCost = NewCost;
5646	Solution = Workspace;
5647	}
5648	Workspace.pop_back();
5649	}
5650	}
5651	}
5652
5653	/// Choose one formula from each use. Return the results in the given Solution
5654	/// vector.
5655	void LSRInstance::Solve(SmallVectorImpl<const Formula > &Solution) const* {
5656	SmallVector<const Formula *, `8`> Workspace;
5657	Cost SolutionCost(L, SE, TTI, AMK);
5658	SolutionCost.Lose();
5659	Cost CurCost(L, SE, TTI, AMK);
5660	SmallPtrSet<const SCEV *, `16`> CurRegs;
5661	DenseSet<const SCEV *> VisitedRegs;
5662	Workspace.reserve(N: Uses.size());
5663
5664	// SolveRecurse does all the work.
5665	SolveRecurse(Solution, SolutionCost, Workspace, CurCost,
5666	CurRegs, VisitedRegs);
5667	if (Solution.empty()) {
5668	LLVM_DEBUG(dbgs() << "\nNo Satisfactory Solution\n");
5669	return;
5670	}
5671
5672	// Ok, we've now made all our decisions.
5673	LLVM_DEBUG(dbgs() << "\n"
5674	"The chosen solution requires ";
5675	SolutionCost.print(dbgs()); dbgs() << ":\n";
5676	for (size_t i = `0`, e = Uses.size(); i != e; ++i) {
5677	dbgs() << " ";
5678	Uses[i].print(dbgs());
5679	dbgs() << "\n"
5680	" ";
5681	Solution[i]->print(dbgs());
5682	dbgs() << `'\n'`;
5683	});
5684
5685	assert(Solution.size() == Uses.size() && "Malformed solution!");
5686
5687	const bool EnableDropUnprofitableSolution = [&] {
5688	switch (AllowDropSolutionIfLessProfitable) {
5689	case cl::boolOrDefault::BOU_TRUE:
5690	return true;
5691	case cl::boolOrDefault::BOU_FALSE:
5692	return false;
5693	case cl::boolOrDefault::BOU_UNSET:
5694	return TTI.shouldDropLSRSolutionIfLessProfitable();
5695	}
5696	llvm_unreachable("Unhandled cl::boolOrDefault enum");
5697	}();
5698
5699	if (BaselineCost.isLess(Other: SolutionCost)) {
5700	if (!EnableDropUnprofitableSolution)
5701	LLVM_DEBUG(
5702	dbgs() << "Baseline is more profitable than chosen solution, "
5703	"add option 'lsr-drop-solution' to drop LSR solution.\n");
5704	else {
5705	LLVM_DEBUG(dbgs() << "Baseline is more profitable than chosen "
5706	"solution, dropping LSR solution.\n";);
5707	Solution.clear();
5708	}
5709	}
5710	}
5711
5712	/// Helper for AdjustInsertPositionForExpand. Climb up the dominator tree far as
5713	/// we can go while still being dominated by the input positions. This helps
5714	/// canonicalize the insert position, which encourages sharing.
5715	BasicBlock::iterator
5716	LSRInstance::HoistInsertPosition(BasicBlock::iterator IP,
5717	const SmallVectorImpl<Instruction *> &Inputs)
5718	const {
5719	Instruction Tentative = &IP;
5720	while (true) {
5721	bool AllDominate = true;
5722	Instruction BetterPos = nullptr*;
5723	// Don't bother attempting to insert before a catchswitch, their basic block
5724	// cannot have other non-PHI instructions.
5725	if (isa<CatchSwitchInst>(Val: Tentative))
5726	return IP;
5727
5728	for (Instruction *Inst : Inputs) {
5729	if (Inst == Tentative \|\| !DT.dominates(Def: Inst, User: Tentative)) {
5730	AllDominate = false;
5731	break;
5732	}
5733	// Attempt to find an insert position in the middle of the block,
5734	// instead of at the end, so that it can be used for other expansions.
5735	if (Tentative->getParent() == Inst->getParent() &&
5736	(!BetterPos \|\| !DT.dominates(Def: Inst, User: BetterPos)))
5737	BetterPos = &*std::next(x: BasicBlock::iterator(Inst));
5738	}
5739	if (!AllDominate)
5740	break;
5741	if (BetterPos)
5742	IP = BetterPos->getIterator();
5743	else
5744	IP = Tentative->getIterator();
5745
5746	const Loop *IPLoop = LI.getLoopFor(BB: IP ->getParent());
5747	unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : `0`;
5748
5749	BasicBlock *IDom;
5750	for (DomTreeNode *Rung = DT.getNode(BB: IP ->getParent()); ; ) {
5751	if (!Rung) return IP;
5752	Rung = Rung->getIDom();
5753	if (!Rung) return IP;
5754	IDom = Rung->getBlock();
5755
5756	// Don't climb into a loop though.
5757	const Loop *IDomLoop = LI.getLoopFor(BB: IDom);
5758	unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : `0`;
5759	if (IDomDepth <= IPLoopDepth &&
5760	(IDomDepth != IPLoopDepth \|\| IDomLoop == IPLoop))
5761	break;
5762	}
5763
5764	Tentative = IDom->getTerminator();
5765	}
5766
5767	return IP;
5768	}
5769
5770	/// Determine an input position which will be dominated by the operands and
5771	/// which will dominate the result.
5772	BasicBlock::iterator LSRInstance::AdjustInsertPositionForExpand(
5773	BasicBlock::iterator LowestIP, const LSRFixup &LF, const LSRUse &LU) const {
5774	// Collect some instructions which must be dominated by the
5775	// expanding replacement. These must be dominated by any operands that
5776	// will be required in the expansion.
5777	SmallVector<Instruction *, `4`> Inputs;
5778	if (Instruction *I = dyn_cast<Instruction>(Val: LF.OperandValToReplace))
5779	Inputs.push_back(Elt: I);
5780	if (LU.Kind == LSRUse::ICmpZero)
5781	if (Instruction *I =
5782	dyn_cast<Instruction>(Val: cast<ICmpInst>(Val: LF.UserInst)->getOperand(i_nocapture: `1`)))
5783	Inputs.push_back(Elt: I);
5784	if (LF.PostIncLoops.count(Ptr: L)) {
5785	if (LF.isUseFullyOutsideLoop(L))
5786	Inputs.push_back(Elt: L->getLoopLatch()->getTerminator());
5787	else
5788	Inputs.push_back(Elt: IVIncInsertPos);
5789	}
5790	// The expansion must also be dominated by the increment positions of any
5791	// loops it for which it is using post-inc mode.
5792	for (const Loop *PIL : LF.PostIncLoops) {
5793	if (PIL == L) continue;
5794
5795	// Be dominated by the loop exit.
5796	SmallVector<BasicBlock *, `4`> ExitingBlocks;
5797	PIL->getExitingBlocks(ExitingBlocks);
5798	if (!ExitingBlocks.empty()) {
5799	BasicBlock *BB = ExitingBlocks [`0`];
5800	for (unsigned i = `1`, e = ExitingBlocks.size(); i != e; ++i)
5801	BB = DT.findNearestCommonDominator(A: BB, B: ExitingBlocks [i]);
5802	Inputs.push_back(Elt: BB->getTerminator());
5803	}
5804	}
5805
5806	assert(!isa<PHINode>(LowestIP) && !LowestIP->isEHPad() &&
5807	"Insertion point must be a normal instruction");
5808
5809	// Then, climb up the immediate dominator tree as far as we can go while
5810	// still being dominated by the input positions.
5811	BasicBlock::iterator IP = HoistInsertPosition(IP: LowestIP, Inputs);
5812
5813	// Don't insert instructions before PHI nodes.
5814	while (isa<PHINode>(Val: IP)) ++IP;
5815
5816	// Ignore landingpad instructions.
5817	while (IP ->isEHPad()) ++IP;
5818
5819	// Set IP below instructions recently inserted by SCEVExpander. This keeps the
5820	// IP consistent across expansions and allows the previously inserted
5821	// instructions to be reused by subsequent expansion.
5822	while (Rewriter.isInsertedInstruction(I: &*IP) && IP != LowestIP)
5823	++IP;
5824
5825	return IP;
5826	}
5827
5828	/// Emit instructions for the leading candidate expression for this LSRUse (this
5829	/// is called "expanding").
5830	Value LSRInstance::Expand(const* LSRUse &LU, const LSRFixup &LF,
5831	const Formula &F, BasicBlock::iterator IP,
5832	SmallVectorImpl<WeakTrackingVH> &DeadInsts) const {
5833	if (LU.RigidFormula)
5834	return LF.OperandValToReplace;
5835
5836	// Determine an input position which will be dominated by the operands and
5837	// which will dominate the result.
5838	IP = AdjustInsertPositionForExpand(LowestIP: IP, LF, LU);
5839	Rewriter.setInsertPoint(&*IP);
5840
5841	// Inform the Rewriter if we have a post-increment use, so that it can
5842	// perform an advantageous expansion.
5843	Rewriter.setPostInc(LF.PostIncLoops);
5844
5845	// This is the type that the user actually needs.
5846	Type *OpTy = LF.OperandValToReplace->getType();
5847	// This will be the type that we'll initially expand to.
5848	Type *Ty = F.getType();
5849	if (!Ty)
5850	// No type known; just expand directly to the ultimate type.
5851	Ty = OpTy;
5852	else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(Ty: OpTy))
5853	// Expand directly to the ultimate type if it's the right size.
5854	Ty = OpTy;
5855	// This is the type to do integer arithmetic in.
5856	Type *IntTy = SE.getEffectiveSCEVType(Ty);
5857	// For ICmpZero with pointer-typed operands, keep the comparison in the
5858	// integer domain to avoid generating inttoptr casts. Use IntTy (the
5859	// formula's arithmetic width) so that both icmp operands match even when
5860	// the IV is wider than the pointer.
5861	if (LU.Kind == LSRUse::ICmpZero && OpTy->isPointerTy()) {
5862	OpTy = IntTy;
5863	Ty = IntTy;
5864	}
5865
5866	// Build up a list of operands to add together to form the full base.
5867	SmallVector<SCEVUse, `8`> Ops;
5868
5869	// Expand the BaseRegs portion.
5870	for (const SCEV *Reg : F.BaseRegs) {
5871	assert(!Reg->isZero() && "Zero allocated in a base register!");
5872
5873	// If we're expanding for a post-inc user, make the post-inc adjustment.
5874	Reg = denormalizeForPostIncUse(S: Reg, Loops: LF.PostIncLoops, SE);
5875	Ops.push_back(Elt: SE.getUnknown(V: Rewriter.expandCodeFor(SH: Reg, Ty: nullptr)));
5876	}
5877
5878	// Expand the ScaledReg portion.
5879	Value ICmpScaledV = nullptr*;
5880	if (F.Scale != `0`) {
5881	const SCEV *ScaledS = F.ScaledReg;
5882
5883	// If we're expanding for a post-inc user, make the post-inc adjustment.
5884	PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
5885	ScaledS = denormalizeForPostIncUse(S: ScaledS, Loops, SE);
5886
5887	if (LU.Kind == LSRUse::ICmpZero) {
5888	// Expand ScaleReg as if it was part of the base regs.
5889	if (F.Scale == `1`)
5890	Ops.push_back(
5891	Elt: SE.getUnknown(V: Rewriter.expandCodeFor(SH: ScaledS, Ty: nullptr)));
5892	else {
5893	// An interesting way of "folding" with an icmp is to use a negated
5894	// scale, which we'll implement by inserting it into the other operand
5895	// of the icmp.
5896	assert(F.Scale == -`1` &&
5897	"The only scale supported by ICmpZero uses is -1!");
5898	ICmpScaledV = Rewriter.expandCodeFor(SH: ScaledS, Ty: nullptr);
5899	}
5900	} else {
5901	// Otherwise just expand the scaled register and an explicit scale,
5902	// which is expected to be matched as part of the address.
5903
5904	// Flush the operand list to suppress SCEVExpander hoisting address modes.
5905	// Unless the addressing mode will not be folded.
5906	if (!Ops.empty() && LU.Kind == LSRUse::Address &&
5907	isAMCompletelyFolded(TTI, LU, F)) {
5908	Value FullV = Rewriter.expandCodeFor(SH: SE.getAddExpr(Ops), Ty: nullptr*);
5909	Ops.clear();
5910	Ops.push_back(Elt: SE.getUnknown(V: FullV));
5911	}
5912	ScaledS = SE.getUnknown(V: Rewriter.expandCodeFor(SH: ScaledS, Ty: nullptr));
5913	if (F.Scale != `1`)
5914	ScaledS =
5915	SE.getMulExpr(LHS: ScaledS, RHS: SE.getConstant(Ty: ScaledS->getType(), V: F.Scale));
5916	Ops.push_back(Elt: ScaledS);
5917	}
5918	}
5919
5920	// Expand the GV portion.
5921	if (F.BaseGV) {
5922	// Flush the operand list to suppress SCEVExpander hoisting.
5923	if (!Ops.empty()) {
5924	Value *FullV = Rewriter.expandCodeFor(SH: SE.getAddExpr(Ops), Ty: IntTy);
5925	Ops.clear();
5926	Ops.push_back(Elt: SE.getUnknown(V: FullV));
5927	}
5928	Ops.push_back(Elt: SE.getUnknown(V: F.BaseGV));
5929	}
5930
5931	// Flush the operand list to suppress SCEVExpander hoisting of both folded and
5932	// unfolded offsets. LSR assumes they both live next to their uses.
5933	if (!Ops.empty()) {
5934	Value *FullV = Rewriter.expandCodeFor(SH: SE.getAddExpr(Ops), Ty);
5935	Ops.clear();
5936	Ops.push_back(Elt: SE.getUnknown(V: FullV));
5937	}
5938
5939	// FIXME: Are we sure we won't get a mismatch here? Is there a way to bail
5940	// out at this point, or should we generate a SCEV adding together mixed
5941	// offsets?
5942	assert(F.BaseOffset.isCompatibleImmediate(LF.Offset) &&
5943	"Expanding mismatched offsets\n");
5944	// Expand the immediate portion.
5945	Immediate Offset = F.BaseOffset.addUnsigned(RHS: LF.Offset);
5946	if (Offset.isNonZero()) {
5947	if (LU.Kind == LSRUse::ICmpZero) {
5948	// The other interesting way of "folding" with an ICmpZero is to use a
5949	// negated immediate.
5950	if (!ICmpScaledV) {
5951	// TODO: Avoid implicit trunc?
5952	// See https://github.com/llvm/llvm-project/issues/112510.
5953	ICmpScaledV = ConstantInt::getSigned(
5954	Ty: IntTy, V: -(uint64_t)Offset.getFixedValue(), /ImplicitTrunc=/true);
5955	} else {
5956	Ops.push_back(Elt: SE.getUnknown(V: ICmpScaledV));
5957	ICmpScaledV = ConstantInt::getSigned(Ty: IntTy, V: Offset.getFixedValue(),
5958	/ImplicitTrunc=/true);
5959	}
5960	} else {
5961	// Just add the immediate values. These again are expected to be matched
5962	// as part of the address.
5963	Ops.push_back(Elt: Offset.getUnknownSCEV(SE, Ty: IntTy));
5964	}
5965	}
5966
5967	// Expand the unfolded offset portion.
5968	Immediate UnfoldedOffset = F.UnfoldedOffset;
5969	if (UnfoldedOffset.isNonZero()) {
5970	// Just add the immediate values.
5971	Ops.push_back(Elt: UnfoldedOffset.getUnknownSCEV(SE, Ty: IntTy));
5972	}
5973
5974	// Emit instructions summing all the operands.
5975	const SCEV *FullS =
5976	Ops.empty() ? SE.getConstant(Ty: IntTy, V: `0`) : SE.getAddExpr(Ops).getPointer();
5977	Value *FullV = Rewriter.expandCodeFor(SH: FullS, Ty);
5978
5979	// We're done expanding now, so reset the rewriter.
5980	Rewriter.clearPostInc();
5981
5982	// An ICmpZero Formula represents an ICmp which we're handling as a
5983	// comparison against zero. Now that we've expanded an expression for that
5984	// form, update the ICmp's other operand.
5985	if (LU.Kind == LSRUse::ICmpZero) {
5986	ICmpInst *CI = cast<ICmpInst>(Val: LF.UserInst);
5987	if (auto *OperandIsInstr = dyn_cast<Instruction>(Val: CI->getOperand(i_nocapture: `1`)))
5988	DeadInsts.emplace_back(Args&: OperandIsInstr);
5989	assert(!F.BaseGV && "ICmp does not support folding a global value and "
5990	"a scale at the same time!");
5991	if (F.Scale == -`1`) {
5992	if (ICmpScaledV->getType() != OpTy) {
5993	Instruction *Cast = CastInst::Create(
5994	CastInst::getCastOpcode(Val: ICmpScaledV, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false),
5995	S: ICmpScaledV, Ty: OpTy, Name: "tmp", InsertBefore: CI->getIterator());
5996	ICmpScaledV = Cast;
5997	}
5998	CI->setOperand(i_nocapture: `1`, Val_nocapture: ICmpScaledV);
5999	} else {
6000	// A scale of 1 means that the scale has been expanded as part of the
6001	// base regs.
6002	assert((F.Scale == `0` \|\| F.Scale == `1`) &&
6003	"ICmp does not support folding a global value and "
6004	"a scale at the same time!");
6005	// TODO: Avoid implicit trunc?
6006	// See https://github.com/llvm/llvm-project/issues/112510.
6007	Constant *C = ConstantInt::getSigned(Ty: SE.getEffectiveSCEVType(Ty: OpTy),
6008	V: -(uint64_t)Offset.getFixedValue(),
6009	/ImplicitTrunc=/true);
6010	if (C->getType() != OpTy) {
6011	C = ConstantFoldCastOperand(
6012	Opcode: CastInst::getCastOpcode(Val: C, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false), C, DestTy: OpTy,
6013	DL: CI->getDataLayout());
6014	assert(C && "Cast of ConstantInt should have folded");
6015	}
6016
6017	CI->setOperand(i_nocapture: `1`, Val_nocapture: C);
6018	}
6019	}
6020
6021	return FullV;
6022	}
6023
6024	/// Helper for Rewrite. PHI nodes are special because the use of their operands
6025	/// effectively happens in their predecessor blocks, so the expression may need
6026	/// to be expanded in multiple places.
6027	void LSRInstance::RewriteForPHI(PHINode PN, const* LSRUse &LU,
6028	const LSRFixup &LF, const Formula &F,
6029	SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
6030	DenseMap<BasicBlock , Value > Inserted;
6031
6032	for (unsigned i = `0`, e = PN->getNumIncomingValues(); i != e; ++i)
6033	if (PN->getIncomingValue(i) == LF.OperandValToReplace) {
6034	bool needUpdateFixups = false;
6035	BasicBlock *BB = PN->getIncomingBlock(i);
6036
6037	// If this is a critical edge, split the edge so that we do not insert
6038	// the code on all predecessor/successor paths. We do this unless this
6039	// is the canonical backedge for this loop, which complicates post-inc
6040	// users.
6041	if (e != `1` && BB->getTerminator()->getNumSuccessors() > `1` &&
6042	!isa<IndirectBrInst>(Val: BB->getTerminator()) &&
6043	!isa<CatchSwitchInst>(Val: BB->getTerminator())) {
6044	BasicBlock *Parent = PN->getParent();
6045	Loop *PNLoop = LI.getLoopFor(BB: Parent);
6046	if (!PNLoop \|\| Parent != PNLoop->getHeader()) {
6047	// Split the critical edge.
6048	BasicBlock NewBB = nullptr*;
6049	if (!Parent->isLandingPad()) {
6050	CriticalEdgeSplittingOptions SplitOptions(&DT, &LI, MSSAU);
6051	SplitOptions =
6052	SplitOptions.setMergeIdenticalEdges().setKeepOneInputPHIs();
6053	if (ShouldPreserveLCSSA)
6054	SplitOptions = SplitOptions.setPreserveLCSSA();
6055	NewBB = SplitCriticalEdge(Src: BB, Dst: Parent, Options: SplitOptions);
6056	} else {
6057	SmallVector<BasicBlock *, `2`> NewBBs;
6058	DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
6059	SplitLandingPadPredecessors(OrigBB: Parent, Preds: BB, Suffix: "", Suffix2: "", NewBBs, DTU: &DTU, LI: &LI);
6060	NewBB = NewBBs [`0`];
6061	}
6062	// If NewBB==NULL, then SplitCriticalEdge refused to split because all
6063	// phi predecessors are identical. The simple thing to do is skip
6064	// splitting in this case rather than complicate the API.
6065	if (NewBB) {
6066	// If PN is outside of the loop and BB is in the loop, we want to
6067	// move the block to be immediately before the PHI block, not
6068	// immediately after BB.
6069	if (L->contains(BB) && !L->contains(Inst: PN))
6070	NewBB->moveBefore(MovePos: PN->getParent());
6071
6072	// Splitting the edge can reduce the number of PHI entries we have.
6073	e = PN->getNumIncomingValues();
6074	BB = NewBB;
6075	i = PN->getBasicBlockIndex(BB);
6076
6077	needUpdateFixups = true;
6078	}
6079	}
6080	}
6081
6082	std::pair<DenseMap<BasicBlock , Value >::iterator, bool> Pair =
6083	Inserted.try_emplace(Key: BB);
6084	if (!Pair.second)
6085	PN->setIncomingValue(i, V: Pair.first ->second);
6086	else {
6087	Value *FullV =
6088	Expand(LU, LF, F, IP: BB->getTerminator()->getIterator(), DeadInsts);
6089
6090	// If this is reuse-by-noop-cast, insert the noop cast.
6091	Type *OpTy = LF.OperandValToReplace->getType();
6092	if (FullV->getType() != OpTy)
6093	FullV = CastInst::Create(
6094	CastInst::getCastOpcode(Val: FullV, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false), S: FullV,
6095	Ty: LF.OperandValToReplace->getType(), Name: "tmp",
6096	InsertBefore: BB->getTerminator()->getIterator());
6097
6098	// If the incoming block for this value is not in the loop, it means the
6099	// current PHI is not in a loop exit, so we must create a LCSSA PHI for
6100	// the inserted value.
6101	if (auto *I = dyn_cast<Instruction>(Val: FullV))
6102	if (L->contains(Inst: I) && !L->contains(BB))
6103	InsertedNonLCSSAInsts.insert(X: I);
6104
6105	PN->setIncomingValue(i, V: FullV);
6106	Pair.first ->second = FullV;
6107	}
6108
6109	// If LSR splits critical edge and phi node has other pending
6110	// fixup operands, we need to update those pending fixups. Otherwise
6111	// formulae will not be implemented completely and some instructions
6112	// will not be eliminated.
6113	if (needUpdateFixups) {
6114	for (LSRUse &LU : Uses)
6115	for (LSRFixup &Fixup : LU.Fixups)
6116	// If fixup is supposed to rewrite some operand in the phi
6117	// that was just updated, it may be already moved to
6118	// another phi node. Such fixup requires update.
6119	if (Fixup.UserInst == PN) {
6120	// Check if the operand we try to replace still exists in the
6121	// original phi.
6122	bool foundInOriginalPHI = false;
6123	for (const auto &val : PN->incoming_values())
6124	if (val == Fixup.OperandValToReplace) {
6125	foundInOriginalPHI = true;
6126	break;
6127	}
6128
6129	// If fixup operand found in original PHI - nothing to do.
6130	if (foundInOriginalPHI)
6131	continue;
6132
6133	// Otherwise it might be moved to another PHI and requires update.
6134	// If fixup operand not found in any of the incoming blocks that
6135	// means we have already rewritten it - nothing to do.
6136	for (const auto &Block : PN->blocks())
6137	for (BasicBlock::iterator I = Block->begin(); isa<PHINode>(Val: I);
6138	++I) {
6139	PHINode *NewPN = cast<PHINode>(Val&: I);
6140	for (const auto &val : NewPN->incoming_values())
6141	if (val == Fixup.OperandValToReplace)
6142	Fixup.UserInst = NewPN;
6143	}
6144	}
6145	}
6146	}
6147	}
6148
6149	/// Emit instructions for the leading candidate expression for this LSRUse (this
6150	/// is called "expanding"), and update the UserInst to reference the newly
6151	/// expanded value.
6152	void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF,
6153	const Formula &F,
6154	SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
6155	// First, find an insertion point that dominates UserInst. For PHI nodes,
6156	// find the nearest block which dominates all the relevant uses.
6157	if (PHINode *PN = dyn_cast<PHINode>(Val: LF.UserInst)) {
6158	RewriteForPHI(PN, LU, LF, F, DeadInsts);
6159	} else {
6160	Value *FullV = Expand(LU, LF, F, IP: LF.UserInst->getIterator(), DeadInsts);
6161
6162	// If this is reuse-by-noop-cast, insert the noop cast.
6163	// For ICmpZero with pointer operands, Expand() already set both operands
6164	// in integer domain, so no cast is needed here.
6165	Type *OpTy = LF.OperandValToReplace->getType();
6166	if (FullV->getType() != OpTy &&
6167	!(LU.Kind == LSRUse::ICmpZero && OpTy->isPointerTy())) {
6168	Instruction *Cast =
6169	CastInst::Create(CastInst::getCastOpcode(Val: FullV, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false),
6170	S: FullV, Ty: OpTy, Name: "tmp", InsertBefore: LF.UserInst->getIterator());
6171	FullV = Cast;
6172	}
6173
6174	// Update the user. ICmpZero is handled specially here (for now) because
6175	// Expand may have updated one of the operands of the icmp already, and
6176	// its new value may happen to be equal to LF.OperandValToReplace, in
6177	// which case doing replaceUsesOfWith leads to replacing both operands
6178	// with the same value. TODO: Reorganize this.
6179	if (LU.Kind == LSRUse::ICmpZero)
6180	LF.UserInst->setOperand(i: `0`, Val: FullV);
6181	else
6182	LF.UserInst->replaceUsesOfWith(From: LF.OperandValToReplace, To: FullV);
6183	}
6184
6185	if (auto *OperandIsInstr = dyn_cast<Instruction>(Val: LF.OperandValToReplace))
6186	DeadInsts.emplace_back(Args&: OperandIsInstr);
6187	}
6188
6189	// Determine where to insert the transformed IV increment instruction for this
6190	// fixup. By default this is the default insert position, but if this is a
6191	// postincrement opportunity then we try to insert it in the same block as the
6192	// fixup user instruction, as this is needed for a postincrement instruction to
6193	// be generated.
6194	static Instruction getFixupInsertPos(const* TargetTransformInfo &TTI,
6195	const LSRFixup &Fixup, const LSRUse &LU,
6196	Instruction *IVIncInsertPos,
6197	DominatorTree &DT) {
6198	// Only address uses can be postincremented
6199	if (LU.Kind != LSRUse::Address)
6200	return IVIncInsertPos;
6201
6202	// Don't try to postincrement if it's not legal
6203	Instruction *I = Fixup.UserInst;
6204	Type *Ty = I->getType();
6205	if (!(isa<LoadInst>(Val: I) && TTI.isIndexedLoadLegal(Mode: TTI.MIM_PostInc, Ty)) &&
6206	!(isa<StoreInst>(Val: I) && TTI.isIndexedStoreLegal(Mode: TTI.MIM_PostInc, Ty)))
6207	return IVIncInsertPos;
6208
6209	// It's only legal to hoist to the user block if it dominates the default
6210	// insert position.
6211	BasicBlock *HoistBlock = I->getParent();
6212	BasicBlock *IVIncBlock = IVIncInsertPos->getParent();
6213	if (!DT.dominates(Def: I, BB: IVIncBlock))
6214	return IVIncInsertPos;
6215
6216	return HoistBlock->getTerminator();
6217	}
6218
6219	/// Rewrite all the fixup locations with new values, following the chosen
6220	/// solution.
6221	void LSRInstance::ImplementSolution(
6222	const SmallVectorImpl<const Formula *> &Solution) {
6223	// Keep track of instructions we may have made dead, so that
6224	// we can remove them after we are done working.
6225	SmallVector<WeakTrackingVH, `16`> DeadInsts;
6226
6227	// Mark phi nodes that terminate chains so the expander tries to reuse them.
6228	for (const IVChain &Chain : IVChainVec) {
6229	if (PHINode *PN = dyn_cast<PHINode>(Val: Chain.tailUserInst()))
6230	Rewriter.setChainedPhi(PN);
6231	}
6232
6233	// Expand the new value definitions and update the users.
6234	for (size_t LUIdx = `0`, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx)
6235	for (const LSRFixup &Fixup : Uses [LUIdx].Fixups) {
6236	Instruction *InsertPos =
6237	getFixupInsertPos(TTI, Fixup, LU: Uses [LUIdx], IVIncInsertPos, DT);
6238	Rewriter.setIVIncInsertPos(L, Pos: InsertPos);
6239	Rewrite(LU: Uses [LUIdx], LF: Fixup, F: *Solution [LUIdx], DeadInsts);
6240	Changed = true;
6241	}
6242
6243	auto InsertedInsts = InsertedNonLCSSAInsts.takeVector();
6244	formLCSSAForInstructions(Worklist&: InsertedInsts, DT, LI, SE: &SE);
6245
6246	for (const IVChain &Chain : IVChainVec) {
6247	GenerateIVChain(Chain, DeadInsts);
6248	Changed = true;
6249	}
6250
6251	for (const WeakVH &IV : Rewriter.getInsertedIVs())
6252	if (IV && dyn_cast<Instruction>(Val: &*IV)->getParent())
6253	ScalarEvolutionIVs.push_back(Elt: IV);
6254
6255	// Clean up after ourselves. This must be done before deleting any
6256	// instructions.
6257	Rewriter.clear();
6258
6259	Changed \|= RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts,
6260	TLI: &TLI, MSSAU);
6261
6262	// In our cost analysis above, we assume that each addrec consumes exactly
6263	// one register, and arrange to have increments inserted just before the
6264	// latch to maximimize the chance this is true. However, if we reused
6265	// existing IVs, we now need to move the increments to match our
6266	// expectations. Otherwise, our cost modeling results in us having a
6267	// chosen a non-optimal result for the actual schedule. (And yes, this
6268	// scheduling decision does impact later codegen.)
6269	for (PHINode &PN : L->getHeader()->phis()) {
6270	BinaryOperator BO = nullptr*;
6271	Value Start = nullptr, Step = nullptr;
6272	if (!matchSimpleRecurrence(P: &PN, BO, Start, Step))
6273	continue;
6274
6275	switch (BO->getOpcode()) {
6276	case Instruction::Sub:
6277	if (BO->getOperand(i_nocapture: `0`) != &PN)
6278	// sub is non-commutative - match handling elsewhere in LSR
6279	continue;
6280	break;
6281	case Instruction::Add:
6282	break;
6283	default:
6284	continue;
6285	};
6286
6287	if (!isa<Constant>(Val: Step))
6288	// If not a constant step, might increase register pressure
6289	// (We assume constants have been canonicalized to RHS)
6290	continue;
6291
6292	if (BO->getParent() == IVIncInsertPos->getParent())
6293	// Only bother moving across blocks. Isel can handle block local case.
6294	continue;
6295
6296	// Can we legally schedule inc at the desired point?
6297	if (!llvm::all_of(Range: BO->uses(),
6298	P: [&](Use &U) {return DT.dominates(Def: IVIncInsertPos, U);}))
6299	continue;
6300	BO->moveBefore(InsertPos: IVIncInsertPos->getIterator());
6301	Changed = true;
6302	}
6303
6304
6305	}
6306
6307	LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
6308	DominatorTree &DT, LoopInfo &LI,
6309	const TargetTransformInfo &TTI, AssumptionCache &AC,
6310	TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU,
6311	bool PreserveLCSSA)
6312	: IU(IU), SE(SE), DT(DT), LI(LI), AC(AC), TLI(TLI), TTI(TTI), L(L),
6313	MSSAU(MSSAU), AMK(PreferredAddresingMode.getNumOccurrences() > `0`
6314	? PreferredAddresingMode
6315	: TTI.getPreferredAddressingMode(L, SE: &SE)),
6316	Rewriter (SE, "lsr", PreserveLCSSA), ShouldPreserveLCSSA(PreserveLCSSA),
6317	BaselineCost (L, SE, TTI, AMK) {
6318	// If LoopSimplify form is not available, stay out of trouble.
6319	if (!L->isLoopSimplifyForm())
6320	return;
6321
6322	// If there's no interesting work to be done, bail early.
6323	if (IU.empty()) return;
6324
6325	// If there's too much analysis to be done, bail early. We won't be able to
6326	// model the problem anyway.
6327	unsigned NumUsers = `0`;
6328	for (const IVStrideUse &U : IU) {
6329	if (++NumUsers > MaxIVUsers) {
6330	(void)U;
6331	LLVM_DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << U
6332	<< "\n");
6333	return;
6334	}
6335	// Bail out if we have a PHI on an EHPad that gets a value from a
6336	// CatchSwitchInst. Because the CatchSwitchInst cannot be split, there is
6337	// no good place to stick any instructions.
6338	if (auto *PN = dyn_cast<PHINode>(Val: U.getUser())) {
6339	auto FirstNonPHI = PN->getParent()->getFirstNonPHIIt();
6340	if (isa<FuncletPadInst>(Val: FirstNonPHI) \|\|
6341	isa<CatchSwitchInst>(Val: FirstNonPHI))
6342	for (BasicBlock *PredBB : PN->blocks())
6343	if (isa<CatchSwitchInst>(Val: PredBB->getFirstNonPHIIt()))
6344	return;
6345	}
6346	}
6347
6348	LLVM_DEBUG(dbgs() << "\nLSR on loop ";
6349	L->getHeader()->printAsOperand(dbgs(), /PrintType=/false);
6350	dbgs() << ":\n");
6351
6352	// Check if we expect this loop to use a hardware loop instruction, which will
6353	// be used when calculating the costs of formulas.
6354	HardwareLoopInfo HWLoopInfo(L);
6355	HardwareLoopProfitable =
6356	TTI.isHardwareLoopProfitable(L, SE, AC, LibInfo: &TLI, HWLoopInfo);
6357
6358	// Configure SCEVExpander already now, so the correct mode is used for
6359	// isSafeToExpand() checks.
6360	#if LLVM_ENABLE_ABI_BREAKING_CHECKS
6361	Rewriter.setDebugType(DEBUG_TYPE);
6362	#endif
6363	Rewriter.disableCanonicalMode();
6364	Rewriter.enableLSRMode();
6365
6366	// First, perform some low-level loop optimizations.
6367	OptimizeShadowIV();
6368	OptimizeLoopTermCond();
6369
6370	// If loop preparation eliminates all interesting IV users, bail.
6371	if (IU.empty()) return;
6372
6373	// Skip nested loops until we can model them better with formulae.
6374	if (!L->isInnermost()) {
6375	LLVM_DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n");
6376	return;
6377	}
6378
6379	// Start collecting data and preparing for the solver.
6380	// If number of registers is not the major cost, we cannot benefit from the
6381	// current profitable chain optimization which is based on number of
6382	// registers.
6383	// FIXME: add profitable chain optimization for other kinds major cost, for
6384	// example number of instructions.
6385	if (TTI.isNumRegsMajorCostOfLSR() \|\| StressIVChain)
6386	CollectChains();
6387	CollectInterestingTypesAndFactors();
6388	CollectFixupsAndInitialFormulae();
6389	CollectLoopInvariantFixupsAndFormulae();
6390
6391	if (Uses.empty())
6392	return;
6393
6394	LLVM_DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n";
6395	print_uses(dbgs()));
6396	LLVM_DEBUG(dbgs() << "The baseline solution requires ";
6397	BaselineCost.print(dbgs()); dbgs() << "\n");
6398
6399	// Now use the reuse data to generate a bunch of interesting ways
6400	// to formulate the values needed for the uses.
6401	GenerateAllReuseFormulae();
6402
6403	FilterOutUndesirableDedicatedRegisters();
6404	NarrowSearchSpaceUsingHeuristics();
6405
6406	SmallVector<const Formula *, `8`> Solution;
6407	Solve(Solution);
6408
6409	// Release memory that is no longer needed.
6410	Factors.clear();
6411	Types.clear();
6412	RegUses.clear();
6413
6414	if (Solution.empty())
6415	return;
6416
6417	#ifndef NDEBUG
6418	// Formulae should be legal.
6419	for (const LSRUse &LU : Uses) {
6420	for (const Formula &F : LU.Formulae)
6421	assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
6422	F) && "Illegal formula generated!");
6423	};
6424	#endif
6425
6426	// Now that we've decided what we want, make it so.
6427	ImplementSolution(Solution);
6428	}
6429
6430	#if !defined(NDEBUG) \|\| defined(LLVM_ENABLE_DUMP)
6431	void LSRInstance::print_factors_and_types(raw_ostream &OS) const {
6432	if (Factors.empty() && Types.empty()) return;
6433
6434	OS << "LSR has identified the following interesting factors and types: ";
6435	ListSeparator LS;
6436
6437	for (int64_t Factor : Factors)
6438	OS << LS << `'*'` << Factor;
6439
6440	for (Type *Ty : Types)
6441	OS << LS << `'('` << *Ty << `')'`;
6442	OS << `'\n'`;
6443	}
6444
6445	void LSRInstance::print_fixups(raw_ostream &OS) const {
6446	OS << "LSR is examining the following fixup sites:\n";
6447	for (const LSRUse &LU : Uses)
6448	for (const LSRFixup &LF : LU.Fixups) {
6449	dbgs() << " ";
6450	LF.print(OS);
6451	OS << `'\n'`;
6452	}
6453	}
6454
6455	void LSRInstance::print_uses(raw_ostream &OS) const {
6456	OS << "LSR is examining the following uses:\n";
6457	for (const LSRUse &LU : Uses) {
6458	dbgs() << " ";
6459	LU.print(OS);
6460	OS << `'\n'`;
6461	for (const Formula &F : LU.Formulae) {
6462	OS << " ";
6463	F.print(OS);
6464	OS << `'\n'`;
6465	}
6466	}
6467	}
6468
6469	void LSRInstance::print(raw_ostream &OS) const {
6470	print_factors_and_types(OS);
6471	print_fixups(OS);
6472	print_uses(OS);
6473	}
6474
6475	LLVM_DUMP_METHOD void LSRInstance::dump() const {
6476	print(errs()); errs() << `'\n'`;
6477	}
6478	#endif
6479
6480	namespace {
6481
6482	class LoopStrengthReduce : public LoopPass {
6483	public:
6484	static char ID; // Pass ID, replacement for typeid
6485
6486	LoopStrengthReduce();
6487
6488	private:
6489	bool runOnLoop(Loop *L, LPPassManager &LPM) override;
6490	void getAnalysisUsage(AnalysisUsage &AU) const override;
6491	};
6492
6493	} // end anonymous namespace
6494
6495	LoopStrengthReduce::LoopStrengthReduce() : LoopPass (ID) {
6496	initializeLoopStrengthReducePass(*PassRegistry::getPassRegistry());
6497	}
6498
6499	void LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const {
6500	// We split critical edges, so we change the CFG. However, we do update
6501	// many analyses if they are around.
6502	AU.addPreservedID(ID&: LoopSimplifyID);
6503
6504	AU.addRequired<LoopInfoWrapperPass>();
6505	AU.addPreserved<LoopInfoWrapperPass>();
6506	AU.addRequiredID(ID&: LoopSimplifyID);
6507	AU.addRequired<DominatorTreeWrapperPass>();
6508	AU.addPreserved<DominatorTreeWrapperPass>();
6509	AU.addRequired<ScalarEvolutionWrapperPass>();
6510	AU.addPreserved<ScalarEvolutionWrapperPass>();
6511	AU.addRequired<AssumptionCacheTracker>();
6512	AU.addRequired<TargetLibraryInfoWrapperPass>();
6513	// Requiring LoopSimplify a second time here prevents IVUsers from running
6514	// twice, since LoopSimplify was invalidated by running ScalarEvolution.
6515	AU.addRequiredID(ID&: LoopSimplifyID);
6516	AU.addRequired<IVUsersWrapperPass>();
6517	AU.addPreserved<IVUsersWrapperPass>();
6518	AU.addRequired<TargetTransformInfoWrapperPass>();
6519	AU.addPreserved<MemorySSAWrapperPass>();
6520	}
6521
6522	namespace {
6523
6524	/// Enables more convenient iteration over a DWARF expression vector.
6525	static iterator_range<llvm::DIExpression::expr_op_iterator>
6526	ToDwarfOpIter(SmallVectorImpl<uint64_t> &Expr) {
6527	llvm::DIExpression::expr_op_iterator Begin =
6528	llvm::DIExpression::expr_op_iterator(Expr.begin());
6529	llvm::DIExpression::expr_op_iterator End =
6530	llvm::DIExpression::expr_op_iterator(Expr.end());
6531	return {Begin, End};
6532	}
6533
6534	struct SCEVDbgValueBuilder {
6535	SCEVDbgValueBuilder() = default;
6536	SCEVDbgValueBuilder(const SCEVDbgValueBuilder &Base) { clone(Base); }
6537
6538	void clone(const SCEVDbgValueBuilder &Base) {
6539	LocationOps = Base.LocationOps;
6540	Expr = Base.Expr;
6541	}
6542
6543	void clear() {
6544	LocationOps.clear();
6545	Expr.clear();
6546	}
6547
6548	/// The DIExpression as we translate the SCEV.
6549	SmallVector<uint64_t, `6`> Expr;
6550	/// The location ops of the DIExpression.
6551	SmallVector<Value *, `2`> LocationOps;
6552
6553	void pushOperator(uint64_t Op) { Expr.push_back(Elt: Op); }
6554	void pushUInt(uint64_t Operand) { Expr.push_back(Elt: Operand); }
6555
6556	/// Add a DW_OP_LLVM_arg to the expression, followed by the index of the value
6557	/// in the set of values referenced by the expression.
6558	void pushLocation(llvm::Value *V) {
6559	Expr.push_back(Elt: llvm::dwarf::DW_OP_LLVM_arg);
6560	auto *It = llvm::find(Range&: LocationOps, Val: V);
6561	unsigned ArgIndex = `0`;
6562	if (It != LocationOps.end()) {
6563	ArgIndex = std::distance(first: LocationOps.begin(), last: It);
6564	} else {
6565	ArgIndex = LocationOps.size();
6566	LocationOps.push_back(Elt: V);
6567	}
6568	Expr.push_back(Elt: ArgIndex);
6569	}
6570
6571	void pushValue(const SCEVUnknown *U) {
6572	llvm::Value *V = cast<SCEVUnknown>(Val: U)->getValue();
6573	pushLocation(V);
6574	}
6575
6576	bool pushConst(const SCEVConstant *C) {
6577	if (C->getAPInt().getSignificantBits() > `64`)
6578	return false;
6579	Expr.push_back(Elt: llvm::dwarf::DW_OP_consts);
6580	Expr.push_back(Elt: C->getAPInt().getSExtValue());
6581	return true;
6582	}
6583
6584	// Iterating the expression as DWARF ops is convenient when updating
6585	// DWARF_OP_LLVM_args.
6586	iterator_range<llvm::DIExpression::expr_op_iterator> expr_ops() {
6587	return ToDwarfOpIter(Expr);
6588	}
6589
6590	/// Several SCEV types are sequences of the same arithmetic operator applied
6591	/// to constants and values that may be extended or truncated.
6592	bool pushArithmeticExpr(const llvm::SCEVCommutativeExpr *CommExpr,
6593	uint64_t DwarfOp) {
6594	assert((isa<llvm::SCEVAddExpr>(CommExpr) \|\| isa<SCEVMulExpr>(CommExpr)) &&
6595	"Expected arithmetic SCEV type");
6596	bool Success = true;
6597	unsigned EmitOperator = `0`;
6598	for (const auto &Op : CommExpr->operands()) {
6599	Success &= pushSCEV(S: Op);
6600
6601	if (EmitOperator >= `1`)
6602	pushOperator(Op: DwarfOp);
6603	++EmitOperator;
6604	}
6605	return Success;
6606	}
6607
6608	// TODO: Identify and omit noop casts.
6609	bool pushCast(const llvm::SCEVCastExpr C, bool* IsSigned) {
6610	const llvm::SCEV *Inner = C->getOperand(i: `0`);
6611	const llvm::Type *Type = C->getType();
6612	uint64_t ToWidth = Type->getIntegerBitWidth();
6613	bool Success = pushSCEV(S: Inner);
6614	uint64_t CastOps[] = {dwarf::DW_OP_LLVM_convert, ToWidth,
6615	IsSigned ? llvm::dwarf::DW_ATE_signed
6616	: llvm::dwarf::DW_ATE_unsigned};
6617	for (const auto &Op : CastOps)
6618	pushOperator(Op);
6619	return Success;
6620	}
6621
6622	// TODO: MinMax - although these haven't been encountered in the test suite.
6623	bool pushSCEV(const llvm::SCEV *S) {
6624	bool Success = true;
6625	if (const SCEVConstant *StartInt = dyn_cast<SCEVConstant>(Val: S)) {
6626	Success &= pushConst(C: StartInt);
6627
6628	} else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Val: S)) {
6629	if (!U->getValue())
6630	return false;
6631	pushLocation(V: U->getValue());
6632
6633	} else if (const SCEVMulExpr *MulRec = dyn_cast<SCEVMulExpr>(Val: S)) {
6634	Success &= pushArithmeticExpr(CommExpr: MulRec, DwarfOp: llvm::dwarf::DW_OP_mul);
6635
6636	} else if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(Val: S)) {
6637	Success &= pushSCEV(S: UDiv->getLHS());
6638	Success &= pushSCEV(S: UDiv->getRHS());
6639	pushOperator(Op: llvm::dwarf::DW_OP_div);
6640
6641	} else if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(Val: S)) {
6642	// Assert if a new and unknown SCEVCastEXpr type is encountered.
6643	assert((isa<SCEVZeroExtendExpr>(Cast) \|\| isa<SCEVTruncateExpr>(Cast) \|\|
6644	isa<SCEVPtrToAddrExpr>(Cast) \|\| isa<SCEVSignExtendExpr>(Cast)) &&
6645	"Unexpected cast type in SCEV.");
6646	Success &= pushCast(C: Cast, IsSigned: (isa<SCEVSignExtendExpr>(Val: Cast)));
6647
6648	} else if (const SCEVAddExpr *AddExpr = dyn_cast<SCEVAddExpr>(Val: S)) {
6649	Success &= pushArithmeticExpr(CommExpr: AddExpr, DwarfOp: llvm::dwarf::DW_OP_plus);
6650
6651	} else if (isa<SCEVAddRecExpr>(Val: S)) {
6652	// Nested SCEVAddRecExpr are generated by nested loops and are currently
6653	// unsupported.
6654	return false;
6655
6656	} else {
6657	return false;
6658	}
6659	return Success;
6660	}
6661
6662	/// Return true if the combination of arithmetic operator and underlying
6663	/// SCEV constant value is an identity function.
6664	bool isIdentityFunction(uint64_t Op, const SCEV *S) {
6665	if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: S)) {
6666	if (C->getAPInt().getSignificantBits() > `64`)
6667	return false;
6668	int64_t I = C->getAPInt().getSExtValue();
6669	switch (Op) {
6670	case llvm::dwarf::DW_OP_plus:
6671	case llvm::dwarf::DW_OP_minus:
6672	return I == `0`;
6673	case llvm::dwarf::DW_OP_mul:
6674	case llvm::dwarf::DW_OP_div:
6675	return I == `1`;
6676	}
6677	}
6678	return false;
6679	}
6680
6681	/// Convert a SCEV of a value to a DIExpression that is pushed onto the
6682	/// builder's expression stack. The stack should already contain an
6683	/// expression for the iteration count, so that it can be multiplied by
6684	/// the stride and added to the start.
6685	/// Components of the expression are omitted if they are an identity function.
6686	/// Chain (non-affine) SCEVs are not supported.
6687	bool SCEVToValueExpr(const llvm::SCEVAddRecExpr &SAR, ScalarEvolution &SE) {
6688	assert(SAR.isAffine() && "Expected affine SCEV");
6689	const SCEV *Start = SAR.getStart();
6690	const SCEV *Stride = SAR.getStepRecurrence(SE);
6691
6692	// Skip pushing arithmetic noops.
6693	if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_mul, S: Stride)) {
6694	if (!pushSCEV(S: Stride))
6695	return false;
6696	pushOperator(Op: llvm::dwarf::DW_OP_mul);
6697	}
6698	if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_plus, S: Start)) {
6699	if (!pushSCEV(S: Start))
6700	return false;
6701	pushOperator(Op: llvm::dwarf::DW_OP_plus);
6702	}
6703	return true;
6704	}
6705
6706	/// Create an expression that is an offset from a value (usually the IV).
6707	void createOffsetExpr(int64_t Offset, Value *OffsetValue) {
6708	pushLocation(V: OffsetValue);
6709	DIExpression::appendOffset(Ops&: Expr, Offset);
6710	LLVM_DEBUG(
6711	dbgs() << "scev-salvage: Generated IV offset expression. Offset: "
6712	<< std::to_string(Offset) << "\n");
6713	}
6714
6715	/// Combine a translation of the SCEV and the IV to create an expression that
6716	/// recovers a location's value.
6717	/// returns true if an expression was created.
6718	bool createIterCountExpr(const SCEV *S,
6719	const SCEVDbgValueBuilder &IterationCount,
6720	ScalarEvolution &SE) {
6721	// SCEVs for SSA values are most frquently of the form
6722	// {start,+,stride}, but sometimes they are ({start,+,stride} + %a + ..).
6723	// This is because %a is a PHI node that is not the IV. However, these
6724	// SCEVs have not been observed to result in debuginfo-lossy optimisations,
6725	// so its not expected this point will be reached.
6726	if (!isa<SCEVAddRecExpr>(Val: S))
6727	return false;
6728
6729	LLVM_DEBUG(dbgs() << "scev-salvage: Location to salvage SCEV: " << *S
6730	<< `'\n'`);
6731
6732	const auto *Rec = cast<SCEVAddRecExpr>(Val: S);
6733	if (!Rec->isAffine())
6734	return false;
6735
6736	if (S->getExpressionSize() > MaxSCEVSalvageExpressionSize)
6737	return false;
6738
6739	// Initialise a new builder with the iteration count expression. In
6740	// combination with the value's SCEV this enables recovery.
6741	clone(Base: IterationCount);
6742	if (!SCEVToValueExpr(SAR: *Rec, SE))
6743	return false;
6744
6745	return true;
6746	}
6747
6748	/// Convert a SCEV of a value to a DIExpression that is pushed onto the
6749	/// builder's expression stack. The stack should already contain an
6750	/// expression for the iteration count, so that it can be multiplied by
6751	/// the stride and added to the start.
6752	/// Components of the expression are omitted if they are an identity function.
6753	bool SCEVToIterCountExpr(const llvm::SCEVAddRecExpr &SAR,
6754	ScalarEvolution &SE) {
6755	assert(SAR.isAffine() && "Expected affine SCEV");
6756	const SCEV *Start = SAR.getStart();
6757	const SCEV *Stride = SAR.getStepRecurrence(SE);
6758
6759	// Skip pushing arithmetic noops.
6760	if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_minus, S: Start)) {
6761	if (!pushSCEV(S: Start))
6762	return false;
6763	pushOperator(Op: llvm::dwarf::DW_OP_minus);
6764	}
6765	if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_div, S: Stride)) {
6766	if (!pushSCEV(S: Stride))
6767	return false;
6768	pushOperator(Op: llvm::dwarf::DW_OP_div);
6769	}
6770	return true;
6771	}
6772
6773	// Append the current expression and locations to a location list and an
6774	// expression list. Modify the DW_OP_LLVM_arg indexes to account for
6775	// the locations already present in the destination list.
6776	void appendToVectors(SmallVectorImpl<uint64_t> &DestExpr,
6777	SmallVectorImpl<Value *> &DestLocations) {
6778	assert(!DestLocations.empty() &&
6779	"Expected the locations vector to contain the IV");
6780	// The DWARF_OP_LLVM_arg arguments of the expression being appended must be
6781	// modified to account for the locations already in the destination vector.
6782	// All builders contain the IV as the first location op.
6783	assert(!LocationOps.empty() &&
6784	"Expected the location ops to contain the IV.");
6785	// DestIndexMap[n] contains the index in DestLocations for the nth
6786	// location in this SCEVDbgValueBuilder.
6787	SmallVector<uint64_t, `2`> DestIndexMap;
6788	for (const auto &Op : LocationOps) {
6789	auto It = find(Range&: DestLocations, Val: Op);
6790	if (It != DestLocations.end()) {
6791	// Location already exists in DestLocations, reuse existing ArgIndex.
6792	DestIndexMap.push_back(Elt: std::distance(first: DestLocations.begin(), last: It));
6793	continue;
6794	}
6795	// Location is not in DestLocations, add it.
6796	DestIndexMap.push_back(Elt: DestLocations.size());
6797	DestLocations.push_back(Elt: Op);
6798	}
6799
6800	for (const auto &Op : expr_ops()) {
6801	auto Arg = dyn_cast<DIExpression::ArgOp>(Val: Op);
6802	if (!Arg) {
6803	Op.appendToVector(V&: DestExpr);
6804	continue;
6805	}
6806
6807	DestExpr.push_back(Elt: dwarf::DW_OP_LLVM_arg);
6808	// `DW_OP_LLVM_arg n` represents the nth LocationOp in this SCEV,
6809	// DestIndexMap[n] contains its new index in DestLocations.
6810	uint64_t NewIndex = DestIndexMap [Arg.getIndex()];
6811	DestExpr.push_back(Elt: NewIndex);
6812	}
6813	}
6814	};
6815
6816	/// Holds all the required data to salvage a dbg.value using the pre-LSR SCEVs
6817	/// and DIExpression.
6818	struct DVIRecoveryRec {
6819	DVIRecoveryRec(DbgVariableRecord *DVR)
6820	: DbgRef(DVR), Expr(DVR->getExpression()), HadLocationArgList(false) {}
6821
6822	DbgVariableRecord *DbgRef;
6823	DIExpression *Expr;
6824	bool HadLocationArgList;
6825	SmallVector<WeakVH, `2`> LocationOps;
6826	SmallVector<const llvm::SCEV *, `2`> SCEVs;
6827	SmallVector<std::unique_ptr<SCEVDbgValueBuilder>, `2`> RecoveryExprs;
6828
6829	void clear() {
6830	for (auto &RE : RecoveryExprs)
6831	RE.reset();
6832	RecoveryExprs.clear();
6833	}
6834
6835	~DVIRecoveryRec() { clear(); }
6836	};
6837	} // namespace
6838
6839	/// Returns the total number of DW_OP_llvm_arg operands in the expression.
6840	/// This helps in determining if a DIArglist is necessary or can be omitted from
6841	/// the dbg.value.
6842	static unsigned numLLVMArgOps(SmallVectorImpl<uint64_t> &Expr) {
6843	auto expr_ops = ToDwarfOpIter(Expr);
6844	unsigned Count = `0`;
6845	for (auto Op : expr_ops)
6846	if (Op.getOp() == dwarf::DW_OP_LLVM_arg)
6847	Count++;
6848	return Count;
6849	}
6850
6851	/// Overwrites DVI with the location and Ops as the DIExpression. This will
6852	/// create an invalid expression if Ops has any dwarf::DW_OP_llvm_arg operands,
6853	/// because a DIArglist is not created for the first argument of the dbg.value.
6854	template <typename T>
6855	static void updateDVIWithLocation(T &DbgVal, Value *Location,
6856	SmallVectorImpl<uint64_t> &Ops) {
6857	assert(numLLVMArgOps(Ops) == `0` && "Expected expression that does not "
6858	"contain any DW_OP_llvm_arg operands.");
6859	DbgVal.setRawLocation(ValueAsMetadata::get(V: Location));
6860	DbgVal.setExpression(DIExpression::get(Context&: DbgVal.getContext(), Elements: Ops));
6861	}
6862
6863	/// Overwrite DVI with locations placed into a DIArglist.
6864	template <typename T>
6865	static void updateDVIWithLocations(T &DbgVal,
6866	SmallVectorImpl<Value *> &Locations,
6867	SmallVectorImpl<uint64_t> &Ops) {
6868	assert(numLLVMArgOps(Ops) != `0` &&
6869	"Expected expression that references DIArglist locations using "
6870	"DW_OP_llvm_arg operands.");
6871	SmallVector<ValueAsMetadata *, `3`> MetadataLocs;
6872	for (Value *V : Locations)
6873	MetadataLocs.push_back(Elt: ValueAsMetadata::get(V));
6874	auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
6875	DbgVal.setRawLocation(llvm::DIArgList::get(Context&: DbgVal.getContext(), Args: ValArrayRef));
6876	DbgVal.setExpression(DIExpression::get(Context&: DbgVal.getContext(), Elements: Ops));
6877	}
6878
6879	/// Write the new expression and new location ops for the dbg.value. If possible
6880	/// reduce the szie of the dbg.value by omitting DIArglist. This
6881	/// can be omitted if:
6882	/// 1. There is only a single location, refenced by a single DW_OP_llvm_arg.
6883	/// 2. The DW_OP_LLVM_arg is the first operand in the expression.
6884	static void UpdateDbgValue(DVIRecoveryRec &DVIRec,
6885	SmallVectorImpl<Value *> &NewLocationOps,
6886	SmallVectorImpl<uint64_t> &NewExpr) {
6887	DbgVariableRecord *DbgVal = DVIRec.DbgRef;
6888	unsigned NumLLVMArgs = numLLVMArgOps(Expr&: NewExpr);
6889	if (NumLLVMArgs == `0`) {
6890	// Location assumed to be on the stack.
6891	updateDVIWithLocation(DbgVal&: *DbgVal, Location: NewLocationOps [`0`], Ops&: NewExpr);
6892	} else if (NumLLVMArgs == `1` && NewExpr [`0`] == dwarf::DW_OP_LLVM_arg) {
6893	// There is only a single DW_OP_llvm_arg at the start of the expression,
6894	// so it can be omitted along with DIArglist.
6895	assert(NewExpr[`1`] == `0` &&
6896	"Lone LLVM_arg in a DIExpression should refer to location-op 0.");
6897	llvm::SmallVector<uint64_t, `6`> ShortenedOps(llvm::drop_begin(RangeOrContainer&: NewExpr, N: `2`));
6898	updateDVIWithLocation(DbgVal&: *DbgVal, Location: NewLocationOps [`0`], Ops&: ShortenedOps);
6899	} else {
6900	// Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
6901	updateDVIWithLocations(DbgVal&: *DbgVal, Locations&: NewLocationOps, Ops&: NewExpr);
6902	}
6903
6904	// If the DIExpression was previously empty then add the stack terminator.
6905	// Non-empty expressions have only had elements inserted into them and so
6906	// the terminator should already be present e.g. stack_value or fragment.
6907	DIExpression *SalvageExpr = DbgVal->getExpression();
6908	if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
6909	SalvageExpr = DIExpression::append(Expr: SalvageExpr, Ops: {dwarf::DW_OP_stack_value});
6910	DbgVal->setExpression(SalvageExpr);
6911	}
6912	}
6913
6914	/// Cached location ops may be erased during LSR, in which case a poison is
6915	/// required when restoring from the cache. The type of that location is no
6916	/// longer available, so just use int8. The poison will be replaced by one or
6917	/// more locations later when a SCEVDbgValueBuilder selects alternative
6918	/// locations to use for the salvage.
6919	static Value *getValueOrPoison(WeakVH &VH, LLVMContext &C) {
6920	return (VH) ? VH : PoisonValue::get(T: llvm::Type::getInt8Ty(C));
6921	}
6922
6923	/// Restore the DVI's pre-LSR arguments. Substitute undef for any erased values.
6924	static void restorePreTransformState(DVIRecoveryRec &DVIRec) {
6925	DbgVariableRecord *DbgVal = DVIRec.DbgRef;
6926	LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
6927	<< "scev-salvage: post-LSR: " << *DbgVal << `'\n'`);
6928	assert(DVIRec.Expr && "Expected an expression");
6929	DbgVal->setExpression(DVIRec.Expr);
6930
6931	// Even a single location-op may be inside a DIArgList and referenced with
6932	// DW_OP_LLVM_arg, which is valid only with a DIArgList.
6933	if (!DVIRec.HadLocationArgList) {
6934	assert(DVIRec.LocationOps.size() == `1` &&
6935	"Unexpected number of location ops.");
6936	// LSR's unsuccessful salvage attempt may have added DIArgList, which in
6937	// this case was not present before, so force the location back to a
6938	// single uncontained Value.
6939	Value *CachedValue =
6940	getValueOrPoison(VH&: DVIRec.LocationOps [`0`], C&: DbgVal->getContext());
6941	DbgVal->setRawLocation(ValueAsMetadata::get(V: CachedValue));
6942	} else {
6943	SmallVector<ValueAsMetadata *, `3`> MetadataLocs;
6944	for (WeakVH VH : DVIRec.LocationOps) {
6945	Value *CachedValue = getValueOrPoison(VH, C&: DbgVal->getContext());
6946	MetadataLocs.push_back(Elt: ValueAsMetadata::get(V: CachedValue));
6947	}
6948	auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
6949	DbgVal->setRawLocation(
6950	llvm::DIArgList::get(Context&: DbgVal->getContext(), Args: ValArrayRef));
6951	}
6952	LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << `'\n'`);
6953	}
6954
6955	static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
6956	llvm::PHINode *LSRInductionVar, DVIRecoveryRec &DVIRec,
6957	const SCEV *SCEVInductionVar,
6958	SCEVDbgValueBuilder IterCountExpr) {
6959
6960	if (!DVIRec.DbgRef->isKillLocation())
6961	return false;
6962
6963	// LSR may have caused several changes to the dbg.value in the failed salvage
6964	// attempt. So restore the DIExpression, the location ops and also the
6965	// location ops format, which is always DIArglist for multiple ops, but only
6966	// sometimes for a single op.
6967	restorePreTransformState(DVIRec);
6968
6969	// LocationOpIndexMap[i] will store the post-LSR location index of
6970	// the non-optimised out location at pre-LSR index i.
6971	SmallVector<int64_t, `2`> LocationOpIndexMap;
6972	LocationOpIndexMap.assign(NumElts: DVIRec.LocationOps.size(), Elt: -`1`);
6973	SmallVector<Value *, `2`> NewLocationOps;
6974	NewLocationOps.push_back(Elt: LSRInductionVar);
6975
6976	for (unsigned i = `0`; i < DVIRec.LocationOps.size(); i++) {
6977	WeakVH VH = DVIRec.LocationOps [i];
6978	// Place the locations not optimised out in the list first, avoiding
6979	// inserts later. The map is used to update the DIExpression's
6980	// DW_OP_LLVM_arg arguments as the expression is updated.
6981	if (VH && !isa<UndefValue>(Val: VH)) {
6982	NewLocationOps.push_back(Elt: VH);
6983	LocationOpIndexMap [i] = NewLocationOps.size() - `1`;
6984	LLVM_DEBUG(dbgs() << "scev-salvage: Location index " << i
6985	<< " now at index " << LocationOpIndexMap[i] << "\n");
6986	continue;
6987	}
6988
6989	// It's possible that a value referred to in the SCEV may have been
6990	// optimised out by LSR.
6991	if (SE.containsErasedValue(S: DVIRec.SCEVs [i]) \|\|
6992	SE.containsUndefs(S: DVIRec.SCEVs [i])) {
6993	LLVM_DEBUG(dbgs() << "scev-salvage: SCEV for location at index: " << i
6994	<< " refers to a location that is now undef or erased. "
6995	"Salvage abandoned.\n");
6996	return false;
6997	}
6998
6999	LLVM_DEBUG(dbgs() << "scev-salvage: salvaging location at index " << i
7000	<< " with SCEV: " << *DVIRec.SCEVs[i] << "\n");
7001
7002	DVIRec.RecoveryExprs [i] = std::make_unique<SCEVDbgValueBuilder>();
7003	SCEVDbgValueBuilder *SalvageExpr = DVIRec.RecoveryExprs [i].get();
7004
7005	// Create an offset-based salvage expression if possible, as it requires
7006	// less DWARF ops than an iteration count-based expression.
7007	if (std::optional<APInt> Offset =
7008	SE.computeConstantDifference(LHS: DVIRec.SCEVs [i], RHS: SCEVInductionVar)) {
7009	if (Offset ->getSignificantBits() <= `64`)
7010	SalvageExpr->createOffsetExpr(Offset: Offset ->getSExtValue(), OffsetValue: LSRInductionVar);
7011	else
7012	return false;
7013	} else if (!SalvageExpr->createIterCountExpr(S: DVIRec.SCEVs [i], IterationCount: IterCountExpr,
7014	SE))
7015	return false;
7016	}
7017
7018	// Merge the DbgValueBuilder generated expressions and the original
7019	// DIExpression, place the result into an new vector.
7020	SmallVector<uint64_t, `3`> NewExpr;
7021	if (DVIRec.Expr->getNumElements() == `0`) {
7022	assert(DVIRec.RecoveryExprs.size() == `1` &&
7023	"Expected only a single recovery expression for an empty "
7024	"DIExpression.");
7025	assert(DVIRec.RecoveryExprs[`0`] &&
7026	"Expected a SCEVDbgSalvageBuilder for location 0");
7027	SCEVDbgValueBuilder *B = DVIRec.RecoveryExprs [`0`].get();
7028	B->appendToVectors(DestExpr&: NewExpr, DestLocations&: NewLocationOps);
7029	}
7030	for (const auto &Op : DVIRec.Expr->expr_ops()) {
7031	// Most Ops needn't be updated.
7032	auto Arg = dyn_cast<DIExpression::ArgOp>(Val: Op);
7033	if (!Arg) {
7034	Op.appendToVector(V&: NewExpr);
7035	continue;
7036	}
7037
7038	uint64_t LocationArgIndex = Arg.getIndex();
7039	SCEVDbgValueBuilder *DbgBuilder =
7040	DVIRec.RecoveryExprs [LocationArgIndex].get();
7041	// The location doesn't have s SCEVDbgValueBuilder, so LSR did not
7042	// optimise it away. So just translate the argument to the updated
7043	// location index.
7044	if (!DbgBuilder) {
7045	NewExpr.push_back(Elt: dwarf::DW_OP_LLVM_arg);
7046	assert(LocationOpIndexMap[LocationArgIndex] != -`1` &&
7047	"Expected a positive index for the location-op position.");
7048	NewExpr.push_back(Elt: LocationOpIndexMap [LocationArgIndex]);
7049	continue;
7050	}
7051	// The location has a recovery expression.
7052	DbgBuilder->appendToVectors(DestExpr&: NewExpr, DestLocations&: NewLocationOps);
7053	}
7054
7055	UpdateDbgValue(DVIRec, NewLocationOps, NewExpr);
7056	LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: " << *DVIRec.DbgRef << "\n");
7057	return true;
7058	}
7059
7060	/// Obtain an expression for the iteration count, then attempt to salvage the
7061	/// dbg.value intrinsics.
7062	static void DbgRewriteSalvageableDVIs(
7063	llvm::Loop L, ScalarEvolution &SE, llvm::PHINode LSRInductionVar,
7064	SmallVector<std::unique_ptr<DVIRecoveryRec>, `2`> &DVIToUpdate) {
7065	if (DVIToUpdate.empty())
7066	return;
7067
7068	const llvm::SCEV *SCEVInductionVar = SE.getSCEV(V: LSRInductionVar);
7069	assert(SCEVInductionVar &&
7070	"Anticipated a SCEV for the post-LSR induction variable");
7071
7072	if (const SCEVAddRecExpr *IVAddRec =
7073	dyn_cast<SCEVAddRecExpr>(Val: SCEVInductionVar)) {
7074	if (!IVAddRec->isAffine())
7075	return;
7076
7077	// Prevent translation using excessive resources.
7078	if (IVAddRec->getExpressionSize() > MaxSCEVSalvageExpressionSize)
7079	return;
7080
7081	// The iteration count is required to recover location values.
7082	SCEVDbgValueBuilder IterCountExpr;
7083	IterCountExpr.pushLocation(V: LSRInductionVar);
7084	if (!IterCountExpr.SCEVToIterCountExpr(SAR: *IVAddRec, SE))
7085	return;
7086
7087	LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV: " << *SCEVInductionVar
7088	<< `'\n'`);
7089
7090	for (auto &DVIRec : DVIToUpdate) {
7091	SalvageDVI(L, SE, LSRInductionVar, DVIRec&: *DVIRec, SCEVInductionVar,
7092	IterCountExpr);
7093	}
7094	}
7095	}
7096
7097	/// Identify and cache salvageable DVI locations and expressions along with the
7098	/// corresponding SCEV(s). Also ensure that the DVI is not deleted between
7099	/// cacheing and salvaging.
7100	static void DbgGatherSalvagableDVI(
7101	Loop *L, ScalarEvolution &SE,
7102	SmallVector<std::unique_ptr<DVIRecoveryRec>, `2`> &SalvageableDVISCEVs) {
7103	for (const auto &B : L->getBlocks()) {
7104	for (auto &I : *B) {
7105	for (DbgVariableRecord &DbgVal : filterDbgVars(R: I.getDbgRecordRange())) {
7106	if (!DbgVal.isDbgValue() && !DbgVal.isDbgAssign())
7107	continue;
7108
7109	// Ensure that if any location op is undef that the dbg.vlue is not
7110	// cached.
7111	if (DbgVal.isKillLocation())
7112	continue;
7113
7114	// Check that the location op SCEVs are suitable for translation to
7115	// DIExpression.
7116	const auto &HasTranslatableLocationOps =
7117	[&](const DbgVariableRecord &DbgValToTranslate) -> bool {
7118	for (const auto LocOp : DbgValToTranslate.location_ops()) {
7119	if (!LocOp)
7120	return false;
7121
7122	if (!SE.isSCEVable(Ty: LocOp->getType()))
7123	return false;
7124
7125	const SCEV *S = SE.getSCEV(V: LocOp);
7126	if (SE.containsUndefs(S))
7127	return false;
7128	}
7129	return true;
7130	};
7131
7132	if (!HasTranslatableLocationOps (DbgVal))
7133	continue;
7134
7135	std::unique_ptr<DVIRecoveryRec> NewRec =
7136	std::make_unique<DVIRecoveryRec>(args: &DbgVal);
7137	// Each location Op may need a SCEVDbgValueBuilder in order to recover
7138	// it. Pre-allocating a vector will enable quick lookups of the builder
7139	// later during the salvage.
7140	NewRec ->RecoveryExprs.resize(N: DbgVal.getNumVariableLocationOps());
7141	for (const auto LocOp : DbgVal.location_ops()) {
7142	NewRec ->SCEVs.push_back(Elt: SE.getSCEV(V: LocOp));
7143	NewRec ->LocationOps.push_back(Elt: LocOp);
7144	NewRec ->HadLocationArgList = DbgVal.hasArgList();
7145	}
7146	SalvageableDVISCEVs.push_back(Elt: std::move(NewRec));
7147	}
7148	}
7149	}
7150	}
7151
7152	/// Ideally pick the PHI IV inserted by ScalarEvolutionExpander. As a fallback
7153	/// any PHi from the loop header is usable, but may have less chance of
7154	/// surviving subsequent transforms.
7155	static llvm::PHINode GetInductionVariable(const* Loop &L, ScalarEvolution &SE,
7156	const LSRInstance &LSR) {
7157
7158	auto IsSuitableIV = [&](PHINode *P) {
7159	if (!SE.isSCEVable(Ty: P->getType()))
7160	return false;
7161	if (const SCEVAddRecExpr *Rec = dyn_cast<SCEVAddRecExpr>(Val: SE.getSCEV(V: P)))
7162	return Rec->isAffine() && !SE.containsUndefs(S: SE.getSCEV(V: P));
7163	return false;
7164	};
7165
7166	// For now, just pick the first IV that was generated and inserted by
7167	// ScalarEvolution. Ideally pick an IV that is unlikely to be optimised away
7168	// by subsequent transforms.
7169	for (const WeakVH &IV : LSR.getScalarEvolutionIVs()) {
7170	if (!IV)
7171	continue;
7172
7173	// There should only be PHI node IVs.
7174	PHINode P = cast<PHINode>(Val: &IV);
7175
7176	if (IsSuitableIV (P))
7177	return P;
7178	}
7179
7180	for (PHINode &P : L.getHeader()->phis()) {
7181	if (IsSuitableIV (&P))
7182	return &P;
7183	}
7184	return nullptr;
7185	}
7186
7187	static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
7188	DominatorTree &DT, LoopInfo &LI,
7189	const TargetTransformInfo &TTI,
7190	AssumptionCache &AC, TargetLibraryInfo &TLI,
7191	MemorySSA MSSA, bool* PreserveLCSSA) {
7192
7193	// Debug preservation - before we start removing anything identify which DVI
7194	// meet the salvageable criteria and store their DIExpression and SCEVs.
7195	SmallVector<std::unique_ptr<DVIRecoveryRec>, `2`> SalvageableDVIRecords;
7196	DbgGatherSalvagableDVI(L, SE, SalvageableDVISCEVs&: SalvageableDVIRecords);
7197
7198	bool Changed = false;
7199	std::unique_ptr<MemorySSAUpdater> MSSAU;
7200	if (MSSA)
7201	MSSAU = std::make_unique<MemorySSAUpdater>(args&: MSSA);
7202
7203	// Run the main LSR transformation.
7204	const LSRInstance &Reducer =
7205	LSRInstance (L, IU, SE, DT, LI, TTI, AC, TLI, MSSAU.get(), PreserveLCSSA);
7206	Changed \|= Reducer.getChanged();
7207
7208	// Remove any extra phis created by processing inner loops.
7209	Changed \|= DeleteDeadPHIs(BB: L->getHeader(), TLI: &TLI, MSSAU: MSSAU.get());
7210	if (EnablePhiElim && L->isLoopSimplifyForm()) {
7211	SmallVector<WeakTrackingVH, `16`> DeadInsts;
7212	SCEVExpander Rewriter(SE, "lsr", false);
7213	#if LLVM_ENABLE_ABI_BREAKING_CHECKS
7214	Rewriter.setDebugType(DEBUG_TYPE);
7215	#endif
7216	unsigned numFolded = Rewriter.replaceCongruentIVs(L, DT: &DT, DeadInsts, TTI: &TTI);
7217	Rewriter.clear();
7218	if (numFolded) {
7219	Changed = true;
7220	RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts, TLI: &TLI,
7221	MSSAU: MSSAU.get());
7222	DeleteDeadPHIs(BB: L->getHeader(), TLI: &TLI, MSSAU: MSSAU.get());
7223	}
7224	}
7225	// LSR may at times remove all uses of an induction variable from a loop.
7226	// The only remaining use is the PHI in the exit block.
7227	// When this is the case, if the exit value of the IV can be calculated using
7228	// SCEV, we can replace the exit block PHI with the final value of the IV and
7229	// skip the updates in each loop iteration.
7230	if (L->isRecursivelyLCSSAForm(DT, LI) && L->getExitBlock()) {
7231	SmallVector<WeakTrackingVH, `16`> DeadInsts;
7232	SCEVExpander Rewriter(SE, "lsr", true);
7233	int Rewrites = rewriteLoopExitValues(L, LI: &LI, TLI: &TLI, SE: &SE, TTI: &TTI, Rewriter, DT: &DT,
7234	ReplaceExitValue: UnusedIndVarInLoop, DeadInsts);
7235	Rewriter.clear();
7236	if (Rewrites) {
7237	Changed = true;
7238	RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts, TLI: &TLI,
7239	MSSAU: MSSAU.get());
7240	DeleteDeadPHIs(BB: L->getHeader(), TLI: &TLI, MSSAU: MSSAU.get());
7241	}
7242	}
7243
7244	if (SalvageableDVIRecords.empty())
7245	return Changed;
7246
7247	// Obtain relevant IVs and attempt to rewrite the salvageable DVIs with
7248	// expressions composed using the derived iteration count.
7249	// TODO: Allow for multiple IV references for nested AddRecSCEVs
7250	for (const auto &L : LI) {
7251	if (llvm::PHINode IV = GetInductionVariable(L: L, SE, LSR: Reducer))
7252	DbgRewriteSalvageableDVIs(L, SE, LSRInductionVar: IV, DVIToUpdate&: SalvageableDVIRecords);
7253	else {
7254	LLVM_DEBUG(dbgs() << "scev-salvage: SCEV salvaging not possible. An IV "
7255	"could not be identified.\n");
7256	}
7257	}
7258
7259	for (auto &Rec : SalvageableDVIRecords)
7260	Rec ->clear();
7261	SalvageableDVIRecords.clear();
7262	return Changed;
7263	}
7264
7265	bool LoopStrengthReduce::runOnLoop(Loop L, LPPassManager & /LPM/*) {
7266	if (skipLoop(L))
7267	return false;
7268
7269	auto &IU = getAnalysis<IVUsersWrapperPass>().getIU();
7270	auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
7271	auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
7272	auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
7273	const auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
7274	F: *L->getHeader()->getParent());
7275	auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
7276	F&: *L->getHeader()->getParent());
7277	auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
7278	F: *L->getHeader()->getParent());
7279	auto *MSSAAnalysis = getAnalysisIfAvailable<MemorySSAWrapperPass>();
7280	MemorySSA MSSA = nullptr*;
7281	if (MSSAAnalysis)
7282	MSSA = &MSSAAnalysis->getMSSA();
7283	return ReduceLoopStrength(L, IU, SE, DT, LI, TTI, AC, TLI, MSSA,
7284	/PreserveLCSSA=/false);
7285	}
7286
7287	PreservedAnalyses LoopStrengthReducePass::run(Loop &L, LoopAnalysisManager &AM,
7288	LoopStandardAnalysisResults &AR,
7289	LPMUpdater &) {
7290	if (!ReduceLoopStrength(L: &L, IU&: AM.getResult<IVUsersAnalysis>(IR&: L, ExtraArgs&: AR), SE&: AR.SE,
7291	DT&: AR.DT, LI&: AR.LI, TTI: AR.TTI, AC&: AR.AC, TLI&: AR.TLI, MSSA: AR.MSSA,
7292	/PreserveLCSSA=/true))
7293	return PreservedAnalyses::all();
7294
7295	auto PA = getLoopPassPreservedAnalyses();
7296	if (AR.MSSA)
7297	PA.preserve<MemorySSAAnalysis>();
7298	return PA;
7299	}
7300
7301	char LoopStrengthReduce::ID = `0`;
7302
7303	INITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce",
7304	"Loop Strength Reduction", false, false)
7305	INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
7306	INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
7307	INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
7308	INITIALIZE_PASS_DEPENDENCY(IVUsersWrapperPass)
7309	INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
7310	INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
7311	INITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce",
7312	"Loop Strength Reduction", false, false)
7313
7314	Pass llvm::createLoopStrengthReducePass() { return* new LoopStrengthReduce (); }
7315

Browse the source code of llvm_projects/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp