1//===- LoopStrengthReduce.cpp - Strength Reduce IVs in Loops --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This transformation analyzes and transforms the induction variables (and
10// computations derived from them) into forms suitable for efficient execution
11// on the target.
12//
13// This pass performs a strength reduction on array references inside loops that
14// have as one or more of their components the loop induction variable, it
15// rewrites expressions to take advantage of scaled-index addressing modes
16// available on the target, and it performs a variety of other optimizations
17// related to loop induction variables.
18//
19// Terminology note: this code has a lot of handling for "post-increment" or
20// "post-inc" users. This is not talking about post-increment addressing modes;
21// it is instead talking about code like this:
22//
23// %i = phi [ 0, %entry ], [ %i.next, %latch ]
24// ...
25// %i.next = add %i, 1
26// %c = icmp eq %i.next, %n
27//
28// The SCEV for %i is {0,+,1}<%L>. The SCEV for %i.next is {1,+,1}<%L>, however
29// it's useful to think about these as the same register, with some uses using
30// the value of the register before the add and some using it after. In this
31// example, the icmp is a post-increment user, since it uses %i.next, which is
32// the value of the induction variable after the increment. The other common
33// case of post-increment users is users outside the loop.
34//
35// TODO: More sophistication in the way Formulae are generated and filtered.
36//
37// TODO: Handle multiple loops at a time.
38//
39// TODO: Should the addressing mode BaseGV be changed to a ConstantExpr instead
40// of a GlobalValue?
41//
42// TODO: When truncation is free, truncate ICmp users' operands to make it a
43// smaller encoding (on x86 at least).
44//
45// TODO: When a negated register is used by an add (such as in a list of
46// multiple base registers, or as the increment expression in an addrec),
47// we may not actually need both reg and (-1 * reg) in registers; the
48// negation can be implemented by using a sub instead of an add. The
49// lack of support for taking this into consideration when making
50// register pressure decisions is partly worked around by the "Special"
51// use kind.
52//
53//===----------------------------------------------------------------------===//
54
55#include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
56#include "ScalarOptions.h"
57#include "llvm/ADT/APInt.h"
58#include "llvm/ADT/DenseMap.h"
59#include "llvm/ADT/DenseSet.h"
60#include "llvm/ADT/PointerIntPair.h"
61#include "llvm/ADT/STLExtras.h"
62#include "llvm/ADT/SetVector.h"
63#include "llvm/ADT/SmallBitVector.h"
64#include "llvm/ADT/SmallPtrSet.h"
65#include "llvm/ADT/SmallSet.h"
66#include "llvm/ADT/SmallVector.h"
67#include "llvm/ADT/Statistic.h"
68#include "llvm/ADT/iterator_range.h"
69#include "llvm/Analysis/AssumptionCache.h"
70#include "llvm/Analysis/DomTreeUpdater.h"
71#include "llvm/Analysis/IVUsers.h"
72#include "llvm/Analysis/LoopAnalysisManager.h"
73#include "llvm/Analysis/LoopInfo.h"
74#include "llvm/Analysis/LoopPass.h"
75#include "llvm/Analysis/MemorySSA.h"
76#include "llvm/Analysis/MemorySSAUpdater.h"
77#include "llvm/Analysis/ScalarEvolution.h"
78#include "llvm/Analysis/ScalarEvolutionExpressions.h"
79#include "llvm/Analysis/ScalarEvolutionNormalization.h"
80#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
81#include "llvm/Analysis/TargetLibraryInfo.h"
82#include "llvm/Analysis/TargetTransformInfo.h"
83#include "llvm/Analysis/ValueTracking.h"
84#include "llvm/BinaryFormat/Dwarf.h"
85#include "llvm/IR/BasicBlock.h"
86#include "llvm/IR/Constant.h"
87#include "llvm/IR/Constants.h"
88#include "llvm/IR/DebugInfoMetadata.h"
89#include "llvm/IR/DerivedTypes.h"
90#include "llvm/IR/Dominators.h"
91#include "llvm/IR/GlobalValue.h"
92#include "llvm/IR/IRBuilder.h"
93#include "llvm/IR/InstrTypes.h"
94#include "llvm/IR/Instruction.h"
95#include "llvm/IR/Instructions.h"
96#include "llvm/IR/IntrinsicInst.h"
97#include "llvm/IR/Module.h"
98#include "llvm/IR/Operator.h"
99#include "llvm/IR/Type.h"
100#include "llvm/IR/Use.h"
101#include "llvm/IR/User.h"
102#include "llvm/IR/Value.h"
103#include "llvm/IR/ValueHandle.h"
104#include "llvm/InitializePasses.h"
105#include "llvm/Pass.h"
106#include "llvm/Support/Casting.h"
107#include "llvm/Support/Compiler.h"
108#include "llvm/Support/Debug.h"
109#include "llvm/Support/ErrorHandling.h"
110#include "llvm/Support/MathExtras.h"
111#include "llvm/Support/raw_ostream.h"
112#include "llvm/Transforms/Scalar.h"
113#include "llvm/Transforms/Utils.h"
114#include "llvm/Transforms/Utils/BasicBlockUtils.h"
115#include "llvm/Transforms/Utils/Local.h"
116#include "llvm/Transforms/Utils/LoopUtils.h"
117#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
118#include <algorithm>
119#include <cassert>
120#include <cstddef>
121#include <cstdint>
122#include <iterator>
123#include <limits>
124#include <map>
125#include <numeric>
126#include <optional>
127#include <utility>
128
129using namespace llvm;
130using namespace SCEVPatternMatch;
131
132#define DEBUG_TYPE "loop-reduce"
133
134/// MaxIVUsers is an arbitrary threshold that provides an early opportunity for
135/// bail out. This threshold is far beyond the number of users that LSR can
136/// conceivably solve, so it should not affect generated code, but catches the
137/// worst cases before LSR burns too much compile time and stack space.
138static const unsigned MaxIVUsers = 200;
139
140/// Limit the size of expression that SCEV-based salvaging will attempt to
141/// translate into a DIExpression.
142/// Choose a maximum size such that debuginfo is not excessively increased and
143/// the salvaging is not too expensive for the compiler.
144static const unsigned MaxSCEVSalvageExpressionSize = 64;
145
146static bool stressIVChain(const ScalarOptions &Opts) {
147#ifndef NDEBUG
148 return Opts.stress_ivchain;
149#else
150 return false;
151#endif
152}
153
154namespace {
155
156struct MemAccessTy {
157 /// Used in situations where the accessed memory type is unknown.
158 static const unsigned UnknownAddressSpace =
159 std::numeric_limits<unsigned>::max();
160
161 Type *MemTy = nullptr;
162 unsigned AddrSpace = UnknownAddressSpace;
163
164 MemAccessTy() = default;
165 MemAccessTy(Type *Ty, unsigned AS) : MemTy(Ty), AddrSpace(AS) {}
166
167 bool operator==(MemAccessTy Other) const {
168 return MemTy == Other.MemTy && AddrSpace == Other.AddrSpace;
169 }
170
171 bool operator!=(MemAccessTy Other) const { return !(*this == Other); }
172
173 static MemAccessTy getUnknown(LLVMContext &Ctx,
174 unsigned AS = UnknownAddressSpace) {
175 return MemAccessTy(Type::getVoidTy(C&: Ctx), AS);
176 }
177
178 Type *getType() { return MemTy; }
179};
180
181/// This class holds data which is used to order reuse candidates.
182class RegSortData {
183public:
184 /// This represents the set of LSRUse indices which reference
185 /// a particular register.
186 SmallBitVector UsedByIndices;
187
188 void print(raw_ostream &OS) const;
189 void dump() const;
190};
191
192// An offset from an address that is either scalable or fixed. Used for
193// per-target optimizations of addressing modes.
194class Immediate : public details::FixedOrScalableQuantity<Immediate, int64_t> {
195 constexpr Immediate(ScalarTy MinVal, bool Scalable)
196 : FixedOrScalableQuantity(MinVal, Scalable) {}
197
198 constexpr Immediate(const FixedOrScalableQuantity<Immediate, int64_t> &V)
199 : FixedOrScalableQuantity(V) {}
200
201public:
202 constexpr Immediate() = delete;
203
204 static constexpr Immediate getFixed(ScalarTy MinVal) {
205 return {MinVal, false};
206 }
207 static constexpr Immediate getScalable(ScalarTy MinVal) {
208 return {MinVal, true};
209 }
210 static constexpr Immediate get(ScalarTy MinVal, bool Scalable) {
211 return {MinVal, Scalable};
212 }
213 static constexpr Immediate getZero() { return {0, false}; }
214 static constexpr Immediate getFixedMin() {
215 return {std::numeric_limits<int64_t>::min(), false};
216 }
217 static constexpr Immediate getFixedMax() {
218 return {std::numeric_limits<int64_t>::max(), false};
219 }
220 static constexpr Immediate getScalableMin() {
221 return {std::numeric_limits<int64_t>::min(), true};
222 }
223 static constexpr Immediate getScalableMax() {
224 return {std::numeric_limits<int64_t>::max(), true};
225 }
226
227 constexpr bool isLessThanZero() const { return Quantity < 0; }
228
229 constexpr bool isGreaterThanZero() const { return Quantity > 0; }
230
231 constexpr bool isCompatibleImmediate(const Immediate &Imm) const {
232 return isZero() || Imm.isZero() || Imm.Scalable == Scalable;
233 }
234
235 constexpr bool isMin() const {
236 return Quantity == std::numeric_limits<ScalarTy>::min();
237 }
238
239 constexpr bool isMax() const {
240 return Quantity == std::numeric_limits<ScalarTy>::max();
241 }
242
243 // Arithmetic 'operators' that cast to unsigned types first.
244 constexpr Immediate addUnsigned(const Immediate &RHS) const {
245 assert(isCompatibleImmediate(RHS) && "Incompatible Immediates");
246 ScalarTy Value = (uint64_t)Quantity + RHS.getKnownMinValue();
247 return {Value, Scalable || RHS.isScalable()};
248 }
249
250 constexpr Immediate subUnsigned(const Immediate &RHS) const {
251 assert(isCompatibleImmediate(RHS) && "Incompatible Immediates");
252 ScalarTy Value = (uint64_t)Quantity - RHS.getKnownMinValue();
253 return {Value, Scalable || RHS.isScalable()};
254 }
255
256 // Scale the quantity by a constant without caring about runtime scalability.
257 constexpr Immediate mulUnsigned(const ScalarTy RHS) const {
258 ScalarTy Value = (uint64_t)Quantity * RHS;
259 return {Value, Scalable};
260 }
261
262 // Helpers for generating SCEVs with vscale terms where needed.
263 const SCEV *getSCEV(ScalarEvolution &SE, Type *Ty) const {
264 const SCEV *S = SE.getConstant(Ty, V: Quantity);
265 if (Scalable)
266 S = SE.getMulExpr(LHS: S, RHS: SE.getVScale(Ty: S->getType()));
267 return S;
268 }
269
270 const SCEV *getNegativeSCEV(ScalarEvolution &SE, Type *Ty) const {
271 const SCEV *NegS = SE.getConstant(Ty, V: -(uint64_t)Quantity);
272 if (Scalable)
273 NegS = SE.getMulExpr(LHS: NegS, RHS: SE.getVScale(Ty: NegS->getType()));
274 return NegS;
275 }
276
277 const SCEV *getUnknownSCEV(ScalarEvolution &SE, Type *Ty) const {
278 // TODO: Avoid implicit trunc?
279 // See https://github.com/llvm/llvm-project/issues/112510.
280 const SCEV *SU = SE.getUnknown(
281 V: ConstantInt::getSigned(Ty, V: Quantity, /*ImplicitTrunc=*/true));
282 if (Scalable)
283 SU = SE.getMulExpr(LHS: SU, RHS: SE.getVScale(Ty: SU->getType()));
284 return SU;
285 }
286};
287
288// This is needed for the Compare type of std::map when Immediate is used
289// as a key. We don't need it to be fully correct against any value of vscale,
290// just to make sure that vscale-related terms in the map are considered against
291// each other rather than being mixed up and potentially missing opportunities.
292struct KeyOrderTargetImmediate {
293 bool operator()(const Immediate &LHS, const Immediate &RHS) const {
294 if (LHS.isScalable() && !RHS.isScalable())
295 return false;
296 if (!LHS.isScalable() && RHS.isScalable())
297 return true;
298 return LHS.getKnownMinValue() < RHS.getKnownMinValue();
299 }
300};
301
302// This would be nicer if we could be generic instead of directly using size_t,
303// but there doesn't seem to be a type trait for is_orderable or
304// is_lessthan_comparable or similar.
305struct KeyOrderSizeTAndImmediate {
306 bool operator()(const std::pair<size_t, Immediate> &LHS,
307 const std::pair<size_t, Immediate> &RHS) const {
308 size_t LSize = LHS.first;
309 size_t RSize = RHS.first;
310 if (LSize != RSize)
311 return LSize < RSize;
312 return KeyOrderTargetImmediate()(LHS.second, RHS.second);
313 }
314};
315} // end anonymous namespace
316
317#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
318void RegSortData::print(raw_ostream &OS) const {
319 OS << "[NumUses=" << UsedByIndices.count() << ']';
320}
321
322LLVM_DUMP_METHOD void RegSortData::dump() const {
323 print(errs()); errs() << '\n';
324}
325#endif
326
327namespace {
328
329/// Map register candidates to information about how they are used.
330class RegUseTracker {
331 using RegUsesTy = DenseMap<const SCEV *, RegSortData>;
332
333 RegUsesTy RegUsesMap;
334 SmallVector<const SCEV *, 16> RegSequence;
335
336public:
337 void countRegister(const SCEV *Reg, size_t LUIdx);
338 void dropRegister(const SCEV *Reg, size_t LUIdx);
339 void swapAndDropUse(size_t LUIdx, size_t LastLUIdx);
340
341 bool isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const;
342
343 const SmallBitVector &getUsedByIndices(const SCEV *Reg) const;
344
345 void clear();
346
347 using iterator = SmallVectorImpl<const SCEV *>::iterator;
348 using const_iterator = SmallVectorImpl<const SCEV *>::const_iterator;
349
350 iterator begin() { return RegSequence.begin(); }
351 iterator end() { return RegSequence.end(); }
352 const_iterator begin() const { return RegSequence.begin(); }
353 const_iterator end() const { return RegSequence.end(); }
354};
355
356} // end anonymous namespace
357
358void
359RegUseTracker::countRegister(const SCEV *Reg, size_t LUIdx) {
360 std::pair<RegUsesTy::iterator, bool> Pair = RegUsesMap.try_emplace(Key: Reg);
361 RegSortData &RSD = Pair.first->second;
362 if (Pair.second)
363 RegSequence.push_back(Elt: Reg);
364 RSD.UsedByIndices.resize(N: std::max(a: RSD.UsedByIndices.size(), b: LUIdx + 1));
365 RSD.UsedByIndices.set(LUIdx);
366}
367
368void
369RegUseTracker::dropRegister(const SCEV *Reg, size_t LUIdx) {
370 RegUsesTy::iterator It = RegUsesMap.find(Val: Reg);
371 assert(It != RegUsesMap.end());
372 RegSortData &RSD = It->second;
373 assert(RSD.UsedByIndices.size() > LUIdx);
374 RSD.UsedByIndices.reset(Idx: LUIdx);
375}
376
377void
378RegUseTracker::swapAndDropUse(size_t LUIdx, size_t LastLUIdx) {
379 assert(LUIdx <= LastLUIdx);
380
381 // Update RegUses. The data structure is not optimized for this purpose;
382 // we must iterate through it and update each of the bit vectors.
383 for (auto &Pair : RegUsesMap) {
384 SmallBitVector &UsedByIndices = Pair.second.UsedByIndices;
385 if (LUIdx < UsedByIndices.size())
386 UsedByIndices[LUIdx] =
387 LastLUIdx < UsedByIndices.size() ? UsedByIndices[LastLUIdx] : false;
388 UsedByIndices.resize(N: std::min(a: UsedByIndices.size(), b: LastLUIdx));
389 }
390}
391
392bool
393RegUseTracker::isRegUsedByUsesOtherThan(const SCEV *Reg, size_t LUIdx) const {
394 RegUsesTy::const_iterator I = RegUsesMap.find(Val: Reg);
395 if (I == RegUsesMap.end())
396 return false;
397 const SmallBitVector &UsedByIndices = I->second.UsedByIndices;
398 int i = UsedByIndices.find_first();
399 if (i == -1) return false;
400 if ((size_t)i != LUIdx) return true;
401 return UsedByIndices.find_next(Prev: i) != -1;
402}
403
404const SmallBitVector &RegUseTracker::getUsedByIndices(const SCEV *Reg) const {
405 RegUsesTy::const_iterator I = RegUsesMap.find(Val: Reg);
406 assert(I != RegUsesMap.end() && "Unknown register!");
407 return I->second.UsedByIndices;
408}
409
410void RegUseTracker::clear() {
411 RegUsesMap.clear();
412 RegSequence.clear();
413}
414
415namespace {
416
417/// This class holds information that describes a formula for computing
418/// satisfying a use. It may include broken-out immediates and scaled registers.
419struct Formula {
420 /// Global base address used for complex addressing.
421 GlobalValue *BaseGV = nullptr;
422
423 /// Base offset for complex addressing.
424 Immediate BaseOffset = Immediate::getZero();
425
426 /// Whether any complex addressing has a base register.
427 bool HasBaseReg = false;
428
429 /// The scale of any complex addressing.
430 int64_t Scale = 0;
431
432 /// The list of "base" registers for this use. When this is non-empty. The
433 /// canonical representation of a formula is
434 /// 1. BaseRegs.size > 1 implies ScaledReg != NULL and
435 /// 2. ScaledReg != NULL implies Scale != 1 || !BaseRegs.empty().
436 /// 3. The reg containing recurrent expr related with currect loop in the
437 /// formula should be put in the ScaledReg.
438 /// #1 enforces that the scaled register is always used when at least two
439 /// registers are needed by the formula: e.g., reg1 + reg2 is reg1 + 1 * reg2.
440 /// #2 enforces that 1 * reg is reg.
441 /// #3 ensures invariant regs with respect to current loop can be combined
442 /// together in LSR codegen.
443 /// This invariant can be temporarily broken while building a formula.
444 /// However, every formula inserted into the LSRInstance must be in canonical
445 /// form.
446 SmallVector<const SCEV *, 4> BaseRegs;
447
448 /// The 'scaled' register for this use. This should be non-null when Scale is
449 /// not zero.
450 const SCEV *ScaledReg = nullptr;
451
452 /// An additional constant offset which added near the use. This requires a
453 /// temporary register, but the offset itself can live in an add immediate
454 /// field rather than a register.
455 Immediate UnfoldedOffset = Immediate::getZero();
456
457 Formula() = default;
458
459 void initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE);
460
461 bool isCanonical(const Loop &L) const;
462
463 void canonicalize(const Loop &L);
464
465 bool unscale();
466
467 bool hasZeroEnd() const;
468
469 bool countsDownToZero() const;
470
471 size_t getNumRegs() const;
472 Type *getType() const;
473
474 void deleteBaseReg(const SCEV *&S);
475
476 bool referencesReg(const SCEV *S) const;
477 bool hasRegsUsedByUsesOtherThan(size_t LUIdx,
478 const RegUseTracker &RegUses) const;
479
480 void print(raw_ostream &OS) const;
481 void dump() const;
482};
483
484} // end anonymous namespace
485
486/// Recursion helper for initialMatch.
487static void DoInitialMatch(const SCEV *S, Loop *L,
488 SmallVectorImpl<SCEVUse> &Good,
489 SmallVectorImpl<SCEVUse> &Bad, ScalarEvolution &SE) {
490 // Collect expressions which properly dominate the loop header.
491 if (SE.properlyDominates(S, BB: L->getHeader())) {
492 Good.push_back(Elt: S);
493 return;
494 }
495
496 // Look at add operands.
497 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
498 for (const SCEV *S : Add->operands())
499 DoInitialMatch(S, L, Good, Bad, SE);
500 return;
501 }
502
503 // Look at addrec operands.
504 const SCEV *Start, *Step;
505 const Loop *ARLoop;
506 if (match(S,
507 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step), L: m_Loop(L&: ARLoop))) &&
508 !Start->isZero()) {
509 DoInitialMatch(S: Start, L, Good, Bad, SE);
510 DoInitialMatch(S: SE.getAddRecExpr(Start: SE.getConstant(Ty: S->getType(), V: 0), Step,
511 // FIXME: AR->getNoWrapFlags()
512 L: ARLoop, Flags: SCEV::FlagNone),
513 L, Good, Bad, SE);
514 return;
515 }
516
517 // Handle a multiplication by -1 (negation) if it didn't fold.
518 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: S))
519 if (Mul->getOperand(i: 0)->isAllOnesValue()) {
520 SmallVector<SCEVUse, 4> Ops(drop_begin(RangeOrContainer: Mul->operands()));
521 const SCEV *NewMul = SE.getMulExpr(Ops);
522
523 SmallVector<SCEVUse, 4> MyGood;
524 SmallVector<SCEVUse, 4> MyBad;
525 DoInitialMatch(S: NewMul, L, Good&: MyGood, Bad&: MyBad, SE);
526 const SCEV *NegOne = SE.getSCEV(V: ConstantInt::getAllOnesValue(
527 Ty: SE.getEffectiveSCEVType(Ty: NewMul->getType())));
528 for (const SCEV *S : MyGood)
529 Good.push_back(Elt: SE.getMulExpr(LHS: NegOne, RHS: S));
530 for (const SCEV *S : MyBad)
531 Bad.push_back(Elt: SE.getMulExpr(LHS: NegOne, RHS: S));
532 return;
533 }
534
535 // Ok, we can't do anything interesting. Just stuff the whole thing into a
536 // register and hope for the best.
537 Bad.push_back(Elt: S);
538}
539
540/// Incorporate loop-variant parts of S into this Formula, attempting to keep
541/// all loop-invariant and loop-computable values in a single base register.
542void Formula::initialMatch(const SCEV *S, Loop *L, ScalarEvolution &SE) {
543 SmallVector<SCEVUse, 4> Good;
544 SmallVector<SCEVUse, 4> Bad;
545 DoInitialMatch(S, L, Good, Bad, SE);
546 if (!Good.empty()) {
547 const SCEV *Sum = SE.getAddExpr(Ops&: Good);
548 if (!Sum->isZero())
549 BaseRegs.push_back(Elt: Sum);
550 HasBaseReg = true;
551 }
552 if (!Bad.empty()) {
553 const SCEV *Sum = SE.getAddExpr(Ops&: Bad);
554 if (!Sum->isZero())
555 BaseRegs.push_back(Elt: Sum);
556 HasBaseReg = true;
557 }
558 canonicalize(L: *L);
559}
560
561static bool containsAddRecDependentOnLoop(const SCEV *S, const Loop &L) {
562 return SCEVExprContains(Root: S, Pred: [&L](const SCEV *S) {
563 return isa<SCEVAddRecExpr>(Val: S) && (cast<SCEVAddRecExpr>(Val: S)->getLoop() == &L);
564 });
565}
566
567/// Check whether or not this formula satisfies the canonical
568/// representation.
569/// \see Formula::BaseRegs.
570bool Formula::isCanonical(const Loop &L) const {
571 assert((Scale == 0 || ScaledReg) &&
572 "ScaledReg must be non-null if Scale is non-zero");
573
574 if (!ScaledReg)
575 return BaseRegs.size() <= 1;
576
577 if (Scale != 1)
578 return true;
579
580 if (Scale == 1 && BaseRegs.empty())
581 return false;
582
583 if (containsAddRecDependentOnLoop(S: ScaledReg, L))
584 return true;
585
586 // If ScaledReg is not a recurrent expr, or it is but its loop is not current
587 // loop, meanwhile BaseRegs contains a recurrent expr reg related with current
588 // loop, we want to swap the reg in BaseRegs with ScaledReg.
589 return none_of(Range: BaseRegs, P: [&L](const SCEV *S) {
590 return containsAddRecDependentOnLoop(S, L);
591 });
592}
593
594/// Helper method to morph a formula into its canonical representation.
595/// \see Formula::BaseRegs.
596/// Every formula having more than one base register, must use the ScaledReg
597/// field. Otherwise, we would have to do special cases everywhere in LSR
598/// to treat reg1 + reg2 + ... the same way as reg1 + 1*reg2 + ...
599/// On the other hand, 1*reg should be canonicalized into reg.
600void Formula::canonicalize(const Loop &L) {
601 if (isCanonical(L))
602 return;
603
604 if (BaseRegs.empty()) {
605 // No base reg? Use scale reg with scale = 1 as such.
606 assert(ScaledReg && "Expected 1*reg => reg");
607 assert(Scale == 1 && "Expected 1*reg => reg");
608 BaseRegs.push_back(Elt: ScaledReg);
609 Scale = 0;
610 ScaledReg = nullptr;
611 return;
612 }
613
614 // Keep the invariant sum in BaseRegs and one of the variant sum in ScaledReg.
615 if (!ScaledReg) {
616 ScaledReg = BaseRegs.pop_back_val();
617 Scale = 1;
618 }
619
620 // If ScaledReg is an invariant with respect to L, find the reg from
621 // BaseRegs containing the recurrent expr related with Loop L. Swap the
622 // reg with ScaledReg.
623 if (!containsAddRecDependentOnLoop(S: ScaledReg, L)) {
624 auto I = find_if(Range&: BaseRegs, P: [&L](const SCEV *S) {
625 return containsAddRecDependentOnLoop(S, L);
626 });
627 if (I != BaseRegs.end())
628 std::swap(a&: ScaledReg, b&: *I);
629 }
630 assert(isCanonical(L) && "Failed to canonicalize?");
631}
632
633/// Get rid of the scale in the formula.
634/// In other words, this method morphes reg1 + 1*reg2 into reg1 + reg2.
635/// \return true if it was possible to get rid of the scale, false otherwise.
636/// \note After this operation the formula may not be in the canonical form.
637bool Formula::unscale() {
638 if (Scale != 1)
639 return false;
640 Scale = 0;
641 BaseRegs.push_back(Elt: ScaledReg);
642 ScaledReg = nullptr;
643 return true;
644}
645
646bool Formula::hasZeroEnd() const {
647 if (UnfoldedOffset || BaseOffset)
648 return false;
649 if (BaseRegs.size() != 1 || ScaledReg)
650 return false;
651 return true;
652}
653
654bool Formula::countsDownToZero() const {
655 if (!hasZeroEnd())
656 return false;
657 assert(BaseRegs.size() == 1 && "hasZeroEnd should mean one BaseReg");
658 const APInt *StepInt;
659 if (!match(S: BaseRegs[0], P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_scev_APInt(C&: StepInt))))
660 return false;
661 return StepInt->isNegative();
662}
663
664/// Return the total number of register operands used by this formula. This does
665/// not include register uses implied by non-constant addrec strides.
666size_t Formula::getNumRegs() const {
667 return !!ScaledReg + BaseRegs.size();
668}
669
670/// Return the type of this formula, if it has one, or null otherwise. This type
671/// is meaningless except for the bit size.
672Type *Formula::getType() const {
673 return !BaseRegs.empty() ? BaseRegs.front()->getType() :
674 ScaledReg ? ScaledReg->getType() :
675 BaseGV ? BaseGV->getType() :
676 nullptr;
677}
678
679/// Delete the given base reg from the BaseRegs list.
680void Formula::deleteBaseReg(const SCEV *&S) {
681 if (&S != &BaseRegs.back())
682 std::swap(a&: S, b&: BaseRegs.back());
683 BaseRegs.pop_back();
684}
685
686/// Test if this formula references the given register.
687bool Formula::referencesReg(const SCEV *S) const {
688 return S == ScaledReg || is_contained(Range: BaseRegs, Element: S);
689}
690
691/// Test whether this formula uses registers which are used by uses other than
692/// the use with the given index.
693bool Formula::hasRegsUsedByUsesOtherThan(size_t LUIdx,
694 const RegUseTracker &RegUses) const {
695 if (ScaledReg)
696 if (RegUses.isRegUsedByUsesOtherThan(Reg: ScaledReg, LUIdx))
697 return true;
698 for (const SCEV *BaseReg : BaseRegs)
699 if (RegUses.isRegUsedByUsesOtherThan(Reg: BaseReg, LUIdx))
700 return true;
701 return false;
702}
703
704#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
705void Formula::print(raw_ostream &OS) const {
706 ListSeparator Plus(" + ");
707 if (BaseGV) {
708 OS << Plus;
709 BaseGV->printAsOperand(OS, /*PrintType=*/false);
710 }
711 if (BaseOffset.isNonZero())
712 OS << Plus << BaseOffset;
713
714 for (const SCEV *BaseReg : BaseRegs)
715 OS << Plus << "reg(" << *BaseReg << ')';
716
717 if (HasBaseReg && BaseRegs.empty())
718 OS << Plus << "**error: HasBaseReg**";
719 else if (!HasBaseReg && !BaseRegs.empty())
720 OS << Plus << "**error: !HasBaseReg**";
721
722 if (Scale != 0) {
723 OS << Plus << Scale << "*reg(";
724 if (ScaledReg)
725 OS << *ScaledReg;
726 else
727 OS << "<unknown>";
728 OS << ')';
729 }
730 if (UnfoldedOffset.isNonZero())
731 OS << Plus << "imm(" << UnfoldedOffset << ')';
732}
733
734LLVM_DUMP_METHOD void Formula::dump() const {
735 print(errs()); errs() << '\n';
736}
737#endif
738
739/// Return true if the given addrec can be sign-extended without changing its
740/// value.
741static bool isAddRecSExtable(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
742 Type *WideTy =
743 IntegerType::get(C&: SE.getContext(), NumBits: SE.getTypeSizeInBits(Ty: AR->getType()) + 1);
744 return isa<SCEVAddRecExpr>(Val: SE.getSignExtendExpr(Op: AR, Ty: WideTy));
745}
746
747/// Return true if the given add can be sign-extended without changing its
748/// value.
749static bool isAddSExtable(const SCEVAddExpr *A, ScalarEvolution &SE) {
750 Type *WideTy =
751 IntegerType::get(C&: SE.getContext(), NumBits: SE.getTypeSizeInBits(Ty: A->getType()) + 1);
752 return isa<SCEVAddExpr>(Val: SE.getSignExtendExpr(Op: A, Ty: WideTy));
753}
754
755/// Return true if the given mul can be sign-extended without changing its
756/// value.
757static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) {
758 Type *WideTy =
759 IntegerType::get(C&: SE.getContext(),
760 NumBits: SE.getTypeSizeInBits(Ty: M->getType()) * M->getNumOperands());
761 return isa<SCEVMulExpr>(Val: SE.getSignExtendExpr(Op: M, Ty: WideTy));
762}
763
764/// Return an expression for LHS /s RHS, if it can be determined and if the
765/// remainder is known to be zero, or null otherwise. If IgnoreSignificantBits
766/// is true, expressions like (X * Y) /s Y are simplified to X, ignoring that
767/// the multiplication may overflow, which is useful when the result will be
768/// used in a context where the most significant bits are ignored.
769static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
770 ScalarEvolution &SE,
771 bool IgnoreSignificantBits = false) {
772 // A stride can fold to zero once the callers sign-extend it.
773 if (RHS->isZero())
774 return nullptr;
775
776 // Handle the trivial case, which works for any SCEV type.
777 if (LHS == RHS)
778 return SE.getConstant(Ty: LHS->getType(), V: 1);
779
780 // Handle a few RHS special cases.
781 const SCEVConstant *RC = dyn_cast<SCEVConstant>(Val: RHS);
782 if (RC) {
783 const APInt &RA = RC->getAPInt();
784 // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do
785 // some folding.
786 if (RA.isAllOnes()) {
787 if (LHS->getType()->isPointerTy())
788 return nullptr;
789 return SE.getMulExpr(LHS, RHS: RC);
790 }
791 // Handle x /s 1 as x.
792 if (RA == 1)
793 return LHS;
794 }
795
796 // Check for a division of a constant by a constant.
797 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: LHS)) {
798 if (!RC)
799 return nullptr;
800 const APInt &LA = C->getAPInt();
801 const APInt &RA = RC->getAPInt();
802 if (LA.srem(RHS: RA) != 0)
803 return nullptr;
804 return SE.getConstant(Val: LA.sdiv(RHS: RA));
805 }
806
807 // Distribute the sdiv over addrec operands, if the addrec doesn't overflow.
808 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: LHS)) {
809 if ((IgnoreSignificantBits || isAddRecSExtable(AR, SE)) && AR->isAffine()) {
810 const SCEV *Step = getExactSDiv(LHS: AR->getStepRecurrence(SE), RHS, SE,
811 IgnoreSignificantBits);
812 if (!Step) return nullptr;
813 const SCEV *Start = getExactSDiv(LHS: AR->getStart(), RHS, SE,
814 IgnoreSignificantBits);
815 if (!Start) return nullptr;
816 // FlagNW is independent of the start value, step direction, and is
817 // preserved with smaller magnitude steps.
818 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
819 return SE.getAddRecExpr(Start, Step, L: AR->getLoop(), Flags: SCEV::FlagNone);
820 }
821 return nullptr;
822 }
823
824 // Distribute the sdiv over add operands, if the add doesn't overflow.
825 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: LHS)) {
826 if (IgnoreSignificantBits || isAddSExtable(A: Add, SE)) {
827 SmallVector<SCEVUse, 8> Ops;
828 for (const SCEV *S : Add->operands()) {
829 const SCEV *Op = getExactSDiv(LHS: S, RHS, SE, IgnoreSignificantBits);
830 if (!Op) return nullptr;
831 Ops.push_back(Elt: Op);
832 }
833 return SE.getAddExpr(Ops);
834 }
835 return nullptr;
836 }
837
838 // Check for a multiply operand that we can pull RHS out of.
839 if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(Val: LHS)) {
840 if (IgnoreSignificantBits || isMulSExtable(M: Mul, SE)) {
841 // Handle special case C1*X*Y /s C2*X*Y.
842 if (const SCEVMulExpr *MulRHS = dyn_cast<SCEVMulExpr>(Val: RHS)) {
843 if (IgnoreSignificantBits || isMulSExtable(M: MulRHS, SE)) {
844 const SCEVConstant *LC = dyn_cast<SCEVConstant>(Val: Mul->getOperand(i: 0));
845 const SCEVConstant *RC =
846 dyn_cast<SCEVConstant>(Val: MulRHS->getOperand(i: 0));
847 if (LC && RC) {
848 SmallVector<const SCEV *, 4> LOps(drop_begin(RangeOrContainer: Mul->operands()));
849 SmallVector<const SCEV *, 4> ROps(drop_begin(RangeOrContainer: MulRHS->operands()));
850 if (LOps == ROps)
851 return getExactSDiv(LHS: LC, RHS: RC, SE, IgnoreSignificantBits);
852 }
853 }
854 }
855
856 SmallVector<SCEVUse, 4> Ops;
857 bool Found = false;
858 for (const SCEV *S : Mul->operands()) {
859 if (!Found)
860 if (const SCEV *Q = getExactSDiv(LHS: S, RHS, SE,
861 IgnoreSignificantBits)) {
862 S = Q;
863 Found = true;
864 }
865 Ops.push_back(Elt: S);
866 }
867 return Found ? SE.getMulExpr(Ops) : nullptr;
868 }
869 return nullptr;
870 }
871
872 // Otherwise we don't know.
873 return nullptr;
874}
875
876/// Extracts an immediate operand from \p Ops and replaces the operand with
877/// zero. If \p PreferScalable is true and \p Ops contains both a scalable and
878/// non-scalable offsets, the scalable offset will be extracted.
879static Immediate extractImmediateOperand(const ScalarOptions &Opts,
880 MutableArrayRef<SCEVUse> Ops,
881 ScalarEvolution &SE,
882 bool PreferScalable) {
883 const APInt *C;
884 SCEVUse *Op = nullptr;
885 Immediate Result = Immediate::getZero();
886
887 // Ops are sorted by their SCEVType (the order of SCEVTypes enum). So, for an
888 // AddExpr the possible order of operands is:
889 // Constant < VScale < Truncate < ZeroExtend < SignExtend < MulExpr < ...
890
891 // This means fixed-size immediates will always appear on the LHS:
892 SCEVUse &S = Ops.front();
893 if (match(U: S, P: m_scev_APInt(C)) && !C->isZero() &&
894 C->getSignificantBits() <= 64) {
895 Op = &S;
896 Result = Immediate::getFixed(MinVal: C->getSExtValue());
897 }
898
899 // But scalable immediates, which are MulExpr(Vscale, Constant), can appear
900 // later in the operand list:
901 if (Opts.lsr_enable_vscale_immediates &&
902 (Result.isZero() || PreferScalable)) {
903 for (SCEVUse &S : Ops) {
904 // We know anything past scMulExpr will not be a vscale immediate.
905 if (S->getSCEVType() > scMulExpr)
906 break;
907 if (match(U: S, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEVVScale()))) {
908 Op = &S;
909 Result = Immediate::getScalable(MinVal: C->getSExtValue());
910 break;
911 }
912 }
913 }
914
915 if (Result.isNonZero()) {
916 SCEVUse &S = *Op;
917 S = SE.getConstant(Ty: S->getType(), V: 0);
918 }
919
920 return Result;
921}
922
923/// If S involves the addition of a constant integer value, return that integer
924/// value, and mutate S to point to a new SCEV with that value excluded.
925static Immediate extractImmediate(const ScalarOptions &Opts, SCEVUse &S,
926 ScalarEvolution &SE,
927 bool PreferScalable = false) {
928 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: S)) {
929 SmallVector<SCEVUse, 8> NewOps(Add->operands());
930 Immediate Result =
931 extractImmediateOperand(Opts, Ops: NewOps, SE, PreferScalable);
932 if (Result.isZero())
933 Result = extractImmediate(Opts, S&: NewOps.front(), SE, PreferScalable);
934 if (Result.isNonZero())
935 S = SE.getAddExpr(Ops&: NewOps);
936 return Result;
937 } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: S)) {
938 SmallVector<SCEVUse, 8> NewOps(AR->operands());
939 Immediate Result =
940 extractImmediate(Opts, S&: NewOps.front(), SE, PreferScalable);
941 if (Result.isNonZero())
942 S = SE.getAddRecExpr(Operands&: NewOps, L: AR->getLoop(),
943 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
944 Flags: SCEV::FlagNone);
945 return Result;
946 }
947 return extractImmediateOperand(Opts, Ops: {S}, SE, PreferScalable);
948}
949
950/// If S involves the addition of a GlobalValue address, return that symbol, and
951/// mutate S to point to a new SCEV with that value excluded.
952static GlobalValue *ExtractSymbol(SCEVUse &S, ScalarEvolution &SE) {
953 if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Val&: S)) {
954 if (GlobalValue *GV = dyn_cast<GlobalValue>(Val: U->getValue())) {
955 S = SE.getConstant(Ty: GV->getType(), V: 0);
956 return GV;
957 }
958 } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val&: S)) {
959 SmallVector<SCEVUse, 8> NewOps(Add->operands());
960 GlobalValue *Result = ExtractSymbol(S&: NewOps.back(), SE);
961 if (Result)
962 S = SE.getAddExpr(Ops&: NewOps);
963 return Result;
964 } else if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val&: S)) {
965 SmallVector<SCEVUse, 8> NewOps(AR->operands());
966 GlobalValue *Result = ExtractSymbol(S&: NewOps.front(), SE);
967 if (Result)
968 S = SE.getAddRecExpr(Operands&: NewOps, L: AR->getLoop(),
969 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
970 Flags: SCEV::FlagNone);
971 return Result;
972 }
973 return nullptr;
974}
975
976/// Returns true if the specified instruction is using the specified value as an
977/// address.
978static bool isAddressUse(const TargetTransformInfo &TTI,
979 Instruction *Inst, Value *OperandVal) {
980 bool isAddress = isa<LoadInst>(Val: Inst);
981 if (StoreInst *SI = dyn_cast<StoreInst>(Val: Inst)) {
982 if (SI->getPointerOperand() == OperandVal)
983 isAddress = true;
984 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Inst)) {
985 // Addressing modes can also be folded into prefetches and a variety
986 // of intrinsics.
987 switch (II->getIntrinsicID()) {
988 case Intrinsic::memset:
989 case Intrinsic::prefetch:
990 case Intrinsic::masked_load:
991 if (II->getArgOperand(i: 0) == OperandVal)
992 isAddress = true;
993 break;
994 case Intrinsic::masked_store:
995 if (II->getArgOperand(i: 1) == OperandVal)
996 isAddress = true;
997 break;
998 case Intrinsic::memmove:
999 case Intrinsic::memcpy:
1000 if (II->getArgOperand(i: 0) == OperandVal ||
1001 II->getArgOperand(i: 1) == OperandVal)
1002 isAddress = true;
1003 break;
1004 default: {
1005 MemIntrinsicInfo IntrInfo;
1006 if (TTI.getTgtMemIntrinsic(Inst: II, Info&: IntrInfo)) {
1007 if (IntrInfo.PtrVal == OperandVal)
1008 isAddress = true;
1009 }
1010 }
1011 }
1012 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
1013 if (RMW->getPointerOperand() == OperandVal)
1014 isAddress = true;
1015 } else if (AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
1016 if (CmpX->getPointerOperand() == OperandVal)
1017 isAddress = true;
1018 }
1019 return isAddress;
1020}
1021
1022/// Return the type of the memory being accessed.
1023static MemAccessTy getAccessType(const TargetTransformInfo &TTI,
1024 Instruction *Inst, Value *OperandVal) {
1025 MemAccessTy AccessTy = MemAccessTy::getUnknown(Ctx&: Inst->getContext());
1026
1027 // First get the type of memory being accessed.
1028 if (Type *Ty = Inst->getAccessType())
1029 AccessTy.MemTy = Ty;
1030
1031 // Then get the pointer address space.
1032 if (const StoreInst *SI = dyn_cast<StoreInst>(Val: Inst)) {
1033 AccessTy.AddrSpace = SI->getPointerAddressSpace();
1034 } else if (const LoadInst *LI = dyn_cast<LoadInst>(Val: Inst)) {
1035 AccessTy.AddrSpace = LI->getPointerAddressSpace();
1036 } else if (const AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
1037 AccessTy.AddrSpace = RMW->getPointerAddressSpace();
1038 } else if (const AtomicCmpXchgInst *CmpX = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
1039 AccessTy.AddrSpace = CmpX->getPointerAddressSpace();
1040 } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: Inst)) {
1041 switch (II->getIntrinsicID()) {
1042 case Intrinsic::prefetch:
1043 case Intrinsic::memset:
1044 AccessTy.AddrSpace = II->getArgOperand(i: 0)->getType()->getPointerAddressSpace();
1045 AccessTy.MemTy = OperandVal->getType();
1046 break;
1047 case Intrinsic::memmove:
1048 case Intrinsic::memcpy:
1049 AccessTy.AddrSpace = OperandVal->getType()->getPointerAddressSpace();
1050 AccessTy.MemTy = OperandVal->getType();
1051 break;
1052 case Intrinsic::masked_load:
1053 AccessTy.AddrSpace =
1054 II->getArgOperand(i: 0)->getType()->getPointerAddressSpace();
1055 break;
1056 case Intrinsic::masked_store:
1057 AccessTy.AddrSpace =
1058 II->getArgOperand(i: 1)->getType()->getPointerAddressSpace();
1059 break;
1060 default: {
1061 MemIntrinsicInfo IntrInfo;
1062 if (TTI.getTgtMemIntrinsic(Inst: II, Info&: IntrInfo) && IntrInfo.PtrVal) {
1063 AccessTy.AddrSpace
1064 = IntrInfo.PtrVal->getType()->getPointerAddressSpace();
1065 }
1066
1067 break;
1068 }
1069 }
1070 }
1071
1072 return AccessTy;
1073}
1074
1075/// Return true if this AddRec is already a phi in its loop.
1076static bool isExistingPhi(const SCEVAddRecExpr *AR, ScalarEvolution &SE) {
1077 for (PHINode &PN : AR->getLoop()->getHeader()->phis()) {
1078 if (SE.isSCEVable(Ty: PN.getType()) &&
1079 (SE.getEffectiveSCEVType(Ty: PN.getType()) ==
1080 SE.getEffectiveSCEVType(Ty: AR->getType())) &&
1081 SE.getSCEV(V: &PN) == AR)
1082 return true;
1083 }
1084 return false;
1085}
1086
1087/// Check if expanding this expression is likely to incur significant cost. This
1088/// is tricky because SCEV doesn't track which expressions are actually computed
1089/// by the current IR.
1090///
1091/// We currently allow expansion of IV increments that involve adds,
1092/// multiplication by constants, and AddRecs from existing phis.
1093///
1094/// TODO: Allow UDivExpr if we can find an existing IV increment that is an
1095/// obvious multiple of the UDivExpr.
1096static bool isHighCostExpansion(const SCEV *S,
1097 SmallPtrSetImpl<const SCEV*> &Processed,
1098 ScalarEvolution &SE) {
1099 // Zero/One operand expressions
1100 switch (S->getSCEVType()) {
1101 case scUnknown:
1102 case scConstant:
1103 case scVScale:
1104 return false;
1105 case scTruncate:
1106 return isHighCostExpansion(S: cast<SCEVTruncateExpr>(Val: S)->getOperand(),
1107 Processed, SE);
1108 case scZeroExtend:
1109 return isHighCostExpansion(S: cast<SCEVZeroExtendExpr>(Val: S)->getOperand(),
1110 Processed, SE);
1111 case scSignExtend:
1112 return isHighCostExpansion(S: cast<SCEVSignExtendExpr>(Val: S)->getOperand(),
1113 Processed, SE);
1114 default:
1115 break;
1116 }
1117
1118 if (!Processed.insert(Ptr: S).second)
1119 return false;
1120
1121 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
1122 for (const SCEV *S : Add->operands()) {
1123 if (isHighCostExpansion(S, Processed, SE))
1124 return true;
1125 }
1126 return false;
1127 }
1128
1129 const SCEV *Op0, *Op1;
1130 if (match(S, P: m_scev_Mul(Op0: m_SCEV(V&: Op0), Op1: m_SCEV(V&: Op1)))) {
1131 // Multiplication by a constant is ok
1132 if (isa<SCEVConstant>(Val: Op0))
1133 return isHighCostExpansion(S: Op1, Processed, SE);
1134
1135 // If we have the value of one operand, check if an existing
1136 // multiplication already generates this expression.
1137 if (const auto *U = dyn_cast<SCEVUnknown>(Val: Op1)) {
1138 Value *UVal = U->getValue();
1139 for (User *UR : UVal->users()) {
1140 // If U is a constant, it may be used by a ConstantExpr.
1141 Instruction *UI = dyn_cast<Instruction>(Val: UR);
1142 if (UI && UI->getOpcode() == Instruction::Mul &&
1143 SE.isSCEVable(Ty: UI->getType())) {
1144 return SE.getSCEV(V: UI) == S;
1145 }
1146 }
1147 }
1148 }
1149
1150 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
1151 if (isExistingPhi(AR, SE))
1152 return false;
1153 }
1154
1155 // Fow now, consider any other type of expression (div/mul/min/max) high cost.
1156 return true;
1157}
1158
1159namespace {
1160
1161class LSRUse;
1162
1163} // end anonymous namespace
1164
1165/// Check if the addressing mode defined by \p F is completely
1166/// folded in \p LU at isel time.
1167/// This includes address-mode folding and special icmp tricks.
1168/// This function returns true if \p LU can accommodate what \p F
1169/// defines and up to 1 base + 1 scaled + offset.
1170/// In other words, if \p F has several base registers, this function may
1171/// still return true. Therefore, users still need to account for
1172/// additional base registers and/or unfolded offsets to derive an
1173/// accurate cost model.
1174static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1175 const LSRUse &LU, const Formula &F);
1176
1177// Get the cost of the scaling factor used in F for LU.
1178static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI,
1179 const LSRUse &LU, const Formula &F,
1180 const Loop &L);
1181
1182namespace {
1183
1184/// This class is used to measure and compare candidate formulae.
1185class Cost {
1186 const ScalarOptions *Opts = nullptr;
1187 const Loop *L = nullptr;
1188 ScalarEvolution *SE = nullptr;
1189 const TargetTransformInfo *TTI = nullptr;
1190 TargetTransformInfo::LSRCost C;
1191 TTI::AddressingModeKind AMK = TTI::AMK_None;
1192
1193public:
1194 Cost() = delete;
1195 Cost(const ScalarOptions &Opts, const Loop *L, ScalarEvolution &SE,
1196 const TargetTransformInfo &TTI, TTI::AddressingModeKind AMK)
1197 : Opts(&Opts), L(L), SE(&SE), TTI(&TTI), AMK(AMK) {
1198 C.Insns = 0;
1199 C.NumRegs = 0;
1200 C.AddRecCost = 0;
1201 C.NumIVMuls = 0;
1202 C.NumBaseAdds = 0;
1203 C.ImmCost = 0;
1204 C.SetupCost = 0;
1205 C.ScaleCost = 0;
1206 C.NumPrePostIndex = 0;
1207 }
1208
1209 bool isLess(const Cost &Other) const;
1210
1211 void Lose();
1212
1213#ifndef NDEBUG
1214 // Once any of the metrics loses, they must all remain losers.
1215 bool isValid() {
1216 return ((C.Insns | C.NumRegs | C.AddRecCost | C.NumIVMuls | C.NumBaseAdds |
1217 C.ImmCost | C.SetupCost | C.ScaleCost | C.NumPrePostIndex) !=
1218 ~0u) ||
1219 ((C.Insns & C.NumRegs & C.AddRecCost & C.NumIVMuls & C.NumBaseAdds &
1220 C.ImmCost & C.SetupCost & C.ScaleCost & C.NumPrePostIndex) == ~0u);
1221 }
1222#endif
1223
1224 bool isLoser() {
1225 assert(isValid() && "invalid cost");
1226 return C.NumRegs == ~0u;
1227 }
1228
1229 void RateFormula(const Formula &F, SmallPtrSetImpl<const SCEV *> &Regs,
1230 const DenseSet<const SCEV *> &VisitedRegs, const LSRUse &LU,
1231 bool HardwareLoopProfitable,
1232 SmallPtrSetImpl<const SCEV *> *LoserRegs = nullptr);
1233
1234 void print(raw_ostream &OS) const;
1235 void dump() const;
1236
1237private:
1238 void RateRegister(const Formula &F, const SCEV *Reg,
1239 SmallPtrSetImpl<const SCEV *> &Regs, const LSRUse &LU,
1240 bool HardwareLoopProfitable);
1241 void RatePrimaryRegister(const Formula &F, const SCEV *Reg,
1242 SmallPtrSetImpl<const SCEV *> &Regs,
1243 const LSRUse &LU, bool HardwareLoopProfitable,
1244 SmallPtrSetImpl<const SCEV *> *LoserRegs);
1245};
1246
1247/// An operand value in an instruction which is to be replaced with some
1248/// equivalent, possibly strength-reduced, replacement.
1249struct LSRFixup {
1250 /// The instruction which will be updated.
1251 Instruction *UserInst = nullptr;
1252
1253 /// The operand of the instruction which will be replaced. The operand may be
1254 /// used more than once; every instance will be replaced.
1255 Value *OperandValToReplace = nullptr;
1256
1257 /// If this user is to use the post-incremented value of an induction
1258 /// variable, this set is non-empty and holds the loops associated with the
1259 /// induction variable.
1260 PostIncLoopSet PostIncLoops;
1261
1262 /// A constant offset to be added to the LSRUse expression. This allows
1263 /// multiple fixups to share the same LSRUse with different offsets, for
1264 /// example in an unrolled loop.
1265 Immediate Offset = Immediate::getZero();
1266
1267 LSRFixup() = default;
1268
1269 bool isUseFullyOutsideLoop(const Loop *L) const;
1270
1271 void print(raw_ostream &OS) const;
1272 void dump() const;
1273};
1274
1275/// This class holds the state that LSR keeps for each use in IVUsers, as well
1276/// as uses invented by LSR itself. It includes information about what kinds of
1277/// things can be folded into the user, information about the user itself, and
1278/// information about how the use may be satisfied. TODO: Represent multiple
1279/// users of the same expression in common?
1280class LSRUse {
1281 DenseSet<SmallVector<const SCEV *, 4>> Uniquifier;
1282
1283public:
1284 /// An enum for a kind of use, indicating what types of scaled and immediate
1285 /// operands it might support.
1286 enum KindType {
1287 Basic, ///< A normal use, with no folding.
1288 Special, ///< A special case of basic, allowing -1 scales.
1289 Address, ///< An address use; folding according to TargetLowering
1290 ICmpZero ///< An equality icmp with both operands folded into one.
1291 // TODO: Add a generic icmp too?
1292 };
1293
1294 using SCEVUseKindPair = PointerIntPair<const SCEV *, 2, KindType>;
1295
1296 KindType Kind;
1297 MemAccessTy AccessTy;
1298
1299 /// The list of operands which are to be replaced.
1300 SmallVector<LSRFixup, 8> Fixups;
1301
1302 /// Keep track of the min and max offsets of the fixups.
1303 Immediate MinOffset = Immediate::getFixedMax();
1304 Immediate MaxOffset = Immediate::getFixedMin();
1305
1306 /// This records whether all of the fixups using this LSRUse are outside of
1307 /// the loop, in which case some special-case heuristics may be used.
1308 bool AllFixupsOutsideLoop = true;
1309
1310 /// This records whether all of the fixups using this LSRUse are unconditional
1311 /// within the loop, meaning they will be executed on every path to the loop
1312 /// latch. This includes fixups before early exits.
1313 bool AllFixupsUnconditional = true;
1314
1315 /// RigidFormula is set to true to guarantee that this use will be associated
1316 /// with a single formula--the one that initially matched. Some SCEV
1317 /// expressions cannot be expanded. This allows LSR to consider the registers
1318 /// used by those expressions without the need to expand them later after
1319 /// changing the formula.
1320 bool RigidFormula = false;
1321
1322 /// A list of ways to build a value that can satisfy this user. After the
1323 /// list is populated, one of these is selected heuristically and used to
1324 /// formulate a replacement for OperandValToReplace in UserInst.
1325 SmallVector<Formula, 12> Formulae;
1326
1327 /// The set of register candidates used by all formulae in this LSRUse.
1328 SmallPtrSet<const SCEV *, 4> Regs;
1329
1330 LSRUse(KindType K, MemAccessTy AT) : Kind(K), AccessTy(AT) {}
1331
1332 LSRFixup &getNewFixup() {
1333 Fixups.push_back(Elt: LSRFixup());
1334 return Fixups.back();
1335 }
1336
1337 void pushFixup(LSRFixup &f) {
1338 Fixups.push_back(Elt: f);
1339 if (Immediate::isKnownGT(LHS: f.Offset, RHS: MaxOffset))
1340 MaxOffset = f.Offset;
1341 if (Immediate::isKnownLT(LHS: f.Offset, RHS: MinOffset))
1342 MinOffset = f.Offset;
1343 }
1344
1345 bool HasFormulaWithSameRegs(const Formula &F) const;
1346 float getNotSelectedProbability(const SCEV *Reg) const;
1347 bool InsertFormula(const Formula &F, const Loop &L);
1348 void DeleteFormula(Formula &F);
1349 void RecomputeRegs(size_t LUIdx, RegUseTracker &Reguses);
1350
1351 void print(raw_ostream &OS) const;
1352 void dump() const;
1353};
1354
1355} // end anonymous namespace
1356
1357static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1358 LSRUse::KindType Kind, MemAccessTy AccessTy,
1359 GlobalValue *BaseGV, Immediate BaseOffset,
1360 bool HasBaseReg, int64_t Scale,
1361 Instruction *Fixup = nullptr);
1362
1363static unsigned getSetupCost(const SCEV *Reg, unsigned Depth,
1364 const TargetTransformInfo &TTI) {
1365 if (isa<SCEVUnknown>(Val: Reg))
1366 return 1;
1367 if (const auto *C = dyn_cast<SCEVConstant>(Val: Reg)) {
1368 if (TTI.getIntImmCost(Imm: C->getAPInt(), Ty: C->getType(),
1369 CostKind: TargetTransformInfo::TCK_RecipThroughput) ==
1370 TargetTransformInfo::TCC_Free)
1371 return 0;
1372 return 1;
1373 }
1374 if (Depth == 0)
1375 return 0;
1376 if (const auto *S = dyn_cast<SCEVAddRecExpr>(Val: Reg))
1377 return getSetupCost(Reg: S->getStart(), Depth: Depth - 1, TTI);
1378 if (auto S = dyn_cast<SCEVIntegralCastExpr>(Val: Reg))
1379 return getSetupCost(Reg: S->getOperand(), Depth: Depth - 1, TTI);
1380 if (auto S = dyn_cast<SCEVNAryExpr>(Val: Reg))
1381 return std::accumulate(first: S->operands().begin(), last: S->operands().end(), init: 0,
1382 binary_op: [&](unsigned i, const SCEV *Reg) {
1383 return i + getSetupCost(Reg, Depth: Depth - 1, TTI);
1384 });
1385 if (auto S = dyn_cast<SCEVUDivExpr>(Val: Reg))
1386 return getSetupCost(Reg: S->getLHS(), Depth: Depth - 1, TTI) +
1387 getSetupCost(Reg: S->getRHS(), Depth: Depth - 1, TTI);
1388 return 0;
1389}
1390
1391/// Tally up interesting quantities from the given register.
1392void Cost::RateRegister(const Formula &F, const SCEV *Reg,
1393 SmallPtrSetImpl<const SCEV *> &Regs, const LSRUse &LU,
1394 bool HardwareLoopProfitable) {
1395 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Reg)) {
1396 // If this is an addrec for another loop, it should be an invariant
1397 // with respect to L since L is the innermost loop (at least
1398 // for now LSR only handles innermost loops).
1399 if (AR->getLoop() != L) {
1400 // If the AddRec exists, consider it's register free and leave it alone.
1401 if (isExistingPhi(AR, SE&: *SE) && !(AMK & TTI::AMK_PostIndexed))
1402 return;
1403
1404 // It is bad to allow LSR for current loop to add induction variables
1405 // for its sibling loops.
1406 if (!AR->getLoop()->contains(L)) {
1407 Lose();
1408 return;
1409 }
1410
1411 // Otherwise, it will be an invariant with respect to Loop L.
1412 ++C.NumRegs;
1413 return;
1414 }
1415
1416 unsigned LoopCost = 1;
1417 if (LU.Kind == LSRUse::Address &&
1418 (TTI->isIndexedLoadLegal(Mode: TTI->MIM_PostInc, Ty: AR->getType()) ||
1419 TTI->isIndexedStoreLegal(Mode: TTI->MIM_PostInc, Ty: AR->getType()))) {
1420 const SCEV *Start;
1421 const APInt *Step;
1422 if (match(S: AR, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_scev_APInt(C&: Step)))) {
1423 // If the step size matches the base offset, we could use pre-indexed
1424 // addressing.
1425 bool CanPreIndex = (AMK & TTI::AMK_PreIndexed) &&
1426 F.BaseOffset.isFixed() &&
1427 *Step == F.BaseOffset.getFixedValue();
1428 bool CanPostIndex = (AMK & TTI::AMK_PostIndexed) &&
1429 !isa<SCEVConstant>(Val: Start) &&
1430 SE->isLoopInvariant(S: Start, L);
1431 // We can only pre or post index when the load/store is unconditional.
1432 if ((CanPreIndex || CanPostIndex) && LU.AllFixupsUnconditional) {
1433 ++C.NumPrePostIndex;
1434 LoopCost = 0;
1435 }
1436 }
1437 }
1438
1439 // If the loop counts down to zero and we'll be using a hardware loop then
1440 // the addrec will be combined into the hardware loop instruction.
1441 if (LU.Kind == LSRUse::ICmpZero && F.countsDownToZero() &&
1442 HardwareLoopProfitable)
1443 LoopCost = 0;
1444 C.AddRecCost += LoopCost;
1445
1446 // Add the step value register, if it needs one.
1447 // TODO: The non-affine case isn't precisely modeled here.
1448 const SCEV *StepReg = AR->getOperand(i: 1);
1449 if (!AR->isAffine() || !isa<SCEVConstant>(Val: StepReg)) {
1450 // If the step amount is a constant multiplied by vscale then it can form
1451 // the immediate value of an add and doesn't use a register, so long as
1452 // the immediate value is legal.
1453 auto IsVScaleStep = [](const SCEV *Reg, const TargetTransformInfo *TTI) {
1454 const APInt *X;
1455 if (!match(S: Reg, P: m_scev_Mul(Op0: m_scev_APInt(C&: X), Op1: m_SCEVVScale())))
1456 return false;
1457 return TTI->isLegalAddScalableImmediate(Imm: X->getLimitedValue());
1458 };
1459 if (!Regs.count(Ptr: StepReg) && !IsVScaleStep(StepReg, TTI)) {
1460 RateRegister(F, Reg: StepReg, Regs, LU, HardwareLoopProfitable);
1461 if (isLoser())
1462 return;
1463 }
1464 }
1465 }
1466 ++C.NumRegs;
1467
1468 // Rough heuristic; favor registers which don't require extra setup
1469 // instructions in the preheader.
1470 C.SetupCost += getSetupCost(Reg, Depth: Opts->lsr_setupcost_depth_limit, TTI: *TTI);
1471 // Ensure we don't, even with the recusion limit, produce invalid costs.
1472 C.SetupCost = std::min<unsigned>(a: C.SetupCost, b: 1 << 16);
1473
1474 C.NumIVMuls += isa<SCEVMulExpr>(Val: Reg) &&
1475 SE->hasComputableLoopEvolution(S: Reg, L);
1476}
1477
1478/// Record this register in the set. If we haven't seen it before, rate
1479/// it. Optional LoserRegs provides a way to declare any formula that refers to
1480/// one of those regs an instant loser.
1481void Cost::RatePrimaryRegister(const Formula &F, const SCEV *Reg,
1482 SmallPtrSetImpl<const SCEV *> &Regs,
1483 const LSRUse &LU, bool HardwareLoopProfitable,
1484 SmallPtrSetImpl<const SCEV *> *LoserRegs) {
1485 if (LoserRegs && LoserRegs->count(Ptr: Reg)) {
1486 Lose();
1487 return;
1488 }
1489 if (Regs.insert(Ptr: Reg).second) {
1490 RateRegister(F, Reg, Regs, LU, HardwareLoopProfitable);
1491 if (LoserRegs && isLoser())
1492 LoserRegs->insert(Ptr: Reg);
1493 }
1494}
1495
1496void Cost::RateFormula(const Formula &F, SmallPtrSetImpl<const SCEV *> &Regs,
1497 const DenseSet<const SCEV *> &VisitedRegs,
1498 const LSRUse &LU, bool HardwareLoopProfitable,
1499 SmallPtrSetImpl<const SCEV *> *LoserRegs) {
1500 if (isLoser())
1501 return;
1502 assert(F.isCanonical(*L) && "Cost is accurate only for canonical formula");
1503 // Tally up the registers.
1504 unsigned PrevAddRecCost = C.AddRecCost;
1505 unsigned PrevNumRegs = C.NumRegs;
1506 unsigned PrevNumBaseAdds = C.NumBaseAdds;
1507 if (const SCEV *ScaledReg = F.ScaledReg) {
1508 if (VisitedRegs.count(V: ScaledReg)) {
1509 Lose();
1510 return;
1511 }
1512 RatePrimaryRegister(F, Reg: ScaledReg, Regs, LU, HardwareLoopProfitable,
1513 LoserRegs);
1514 if (isLoser())
1515 return;
1516 }
1517 for (const SCEV *BaseReg : F.BaseRegs) {
1518 if (VisitedRegs.count(V: BaseReg)) {
1519 Lose();
1520 return;
1521 }
1522 RatePrimaryRegister(F, Reg: BaseReg, Regs, LU, HardwareLoopProfitable,
1523 LoserRegs);
1524 if (isLoser())
1525 return;
1526 }
1527
1528 // Determine how many (unfolded) adds we'll need inside the loop.
1529 size_t NumBaseParts = F.getNumRegs();
1530 if (NumBaseParts > 1)
1531 // Do not count the base and a possible second register if the target
1532 // allows to fold 2 registers.
1533 C.NumBaseAdds +=
1534 NumBaseParts - (1 + (F.Scale && isAMCompletelyFolded(TTI: *TTI, LU, F)));
1535 C.NumBaseAdds += (F.UnfoldedOffset.isNonZero());
1536
1537 // Accumulate non-free scaling amounts.
1538 C.ScaleCost += getScalingFactorCost(TTI: *TTI, LU, F, L: *L).getValue();
1539
1540 // Tally up the non-zero immediates.
1541 for (const LSRFixup &Fixup : LU.Fixups) {
1542 if (Fixup.Offset.isCompatibleImmediate(Imm: F.BaseOffset)) {
1543 Immediate Offset = Fixup.Offset.addUnsigned(RHS: F.BaseOffset);
1544 if (F.BaseGV)
1545 C.ImmCost += 64; // Handle symbolic values conservatively.
1546 // TODO: This should probably be the pointer size.
1547 else if (Offset.isNonZero())
1548 C.ImmCost +=
1549 APInt(64, Offset.getKnownMinValue(), true).getSignificantBits();
1550
1551 // Check with target if this offset with this instruction is
1552 // specifically not supported.
1553 if (LU.Kind == LSRUse::Address && Offset.isNonZero() &&
1554 !isAMCompletelyFolded(TTI: *TTI, Kind: LSRUse::Address, AccessTy: LU.AccessTy, BaseGV: F.BaseGV,
1555 BaseOffset: Offset, HasBaseReg: F.HasBaseReg, Scale: F.Scale, Fixup: Fixup.UserInst))
1556 C.NumBaseAdds++;
1557 } else {
1558 // Incompatible immediate type, increase cost to avoid using
1559 C.ImmCost += 2048;
1560 }
1561 }
1562
1563 // If we don't count instruction cost exit here.
1564 if (!valueOr(X: Opts->lsr_insns_cost, Default: true)) {
1565 assert(isValid() && "invalid cost");
1566 return;
1567 }
1568
1569 // Treat every new register that exceeds TTI.getNumberOfRegisters() - 1 as
1570 // additional instruction (at least fill).
1571 // TODO: Need distinguish register class?
1572 unsigned TTIRegNum = TTI->getNumberOfRegisters(
1573 ClassID: TTI->getRegisterClassForType(Vector: false, Ty: F.getType())) - 1;
1574 if (C.NumRegs > TTIRegNum) {
1575 // Cost already exceeded TTIRegNum, then only newly added register can add
1576 // new instructions.
1577 if (PrevNumRegs > TTIRegNum)
1578 C.Insns += (C.NumRegs - PrevNumRegs);
1579 else
1580 C.Insns += (C.NumRegs - TTIRegNum);
1581 }
1582
1583 // If ICmpZero formula ends with not 0, it could not be replaced by
1584 // just add or sub. We'll need to compare final result of AddRec.
1585 // That means we'll need an additional instruction. But if the target can
1586 // macro-fuse a compare with a branch, don't count this extra instruction.
1587 // For -10 + {0, +, 1}:
1588 // i = i + 1;
1589 // cmp i, 10
1590 //
1591 // For {-10, +, 1}:
1592 // i = i + 1;
1593 if (LU.Kind == LSRUse::ICmpZero && !F.hasZeroEnd() &&
1594 !TTI->canMacroFuseCmp())
1595 C.Insns++;
1596 // Each new AddRec adds 1 instruction to calculation.
1597 C.Insns += (C.AddRecCost - PrevAddRecCost);
1598
1599 // BaseAdds adds instructions for unfolded registers.
1600 if (LU.Kind != LSRUse::ICmpZero)
1601 C.Insns += C.NumBaseAdds - PrevNumBaseAdds;
1602 assert(isValid() && "invalid cost");
1603}
1604
1605/// Set this cost to a losing value.
1606void Cost::Lose() {
1607 C.Insns = std::numeric_limits<unsigned>::max();
1608 C.NumRegs = std::numeric_limits<unsigned>::max();
1609 C.AddRecCost = std::numeric_limits<unsigned>::max();
1610 C.NumIVMuls = std::numeric_limits<unsigned>::max();
1611 C.NumBaseAdds = std::numeric_limits<unsigned>::max();
1612 C.ImmCost = std::numeric_limits<unsigned>::max();
1613 C.SetupCost = std::numeric_limits<unsigned>::max();
1614 C.ScaleCost = std::numeric_limits<unsigned>::max();
1615 C.NumPrePostIndex = std::numeric_limits<unsigned>::max();
1616}
1617
1618/// Choose the lower cost.
1619bool Cost::isLess(const Cost &Other) const {
1620 if (Opts->lsr_insns_cost == BoolOrDefault::True && C.Insns != Other.C.Insns)
1621 return C.Insns < Other.C.Insns;
1622 return TTI->isLSRCostLess(C1: C, C2: Other.C);
1623}
1624
1625#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1626void Cost::print(raw_ostream &OS) const {
1627 if (valueOr(Opts->lsr_insns_cost, true))
1628 OS << C.Insns << " instruction" << (C.Insns == 1 ? " " : "s ");
1629 OS << C.NumRegs << " reg" << (C.NumRegs == 1 ? "" : "s");
1630 if (C.AddRecCost != 0)
1631 OS << ", with addrec cost " << C.AddRecCost;
1632 if (C.NumIVMuls != 0)
1633 OS << ", plus " << C.NumIVMuls << " IV mul"
1634 << (C.NumIVMuls == 1 ? "" : "s");
1635 if (C.NumBaseAdds != 0)
1636 OS << ", plus " << C.NumBaseAdds << " base add"
1637 << (C.NumBaseAdds == 1 ? "" : "s");
1638 if (C.ScaleCost != 0)
1639 OS << ", plus " << C.ScaleCost << " scale cost";
1640 if (C.ImmCost != 0)
1641 OS << ", plus " << C.ImmCost << " imm cost";
1642 if (C.SetupCost != 0)
1643 OS << ", plus " << C.SetupCost << " setup cost";
1644 if (C.NumPrePostIndex != 0)
1645 OS << ", plus " << C.NumPrePostIndex << " pre/post-index";
1646}
1647
1648LLVM_DUMP_METHOD void Cost::dump() const {
1649 print(errs()); errs() << '\n';
1650}
1651#endif
1652
1653/// Test whether this fixup always uses its value outside of the given loop.
1654bool LSRFixup::isUseFullyOutsideLoop(const Loop *L) const {
1655 // PHI nodes use their value in their incoming blocks.
1656 if (const PHINode *PN = dyn_cast<PHINode>(Val: UserInst)) {
1657 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
1658 if (PN->getIncomingValue(i) == OperandValToReplace &&
1659 L->contains(BB: PN->getIncomingBlock(i)))
1660 return false;
1661 return true;
1662 }
1663
1664 return !L->contains(Inst: UserInst);
1665}
1666
1667#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1668void LSRFixup::print(raw_ostream &OS) const {
1669 OS << "UserInst=";
1670 // Store is common and interesting enough to be worth special-casing.
1671 if (StoreInst *Store = dyn_cast<StoreInst>(UserInst)) {
1672 OS << "store ";
1673 Store->getOperand(0)->printAsOperand(OS, /*PrintType=*/false);
1674 } else if (UserInst->getType()->isVoidTy())
1675 OS << UserInst->getOpcodeName();
1676 else
1677 UserInst->printAsOperand(OS, /*PrintType=*/false);
1678
1679 OS << ", OperandValToReplace=";
1680 OperandValToReplace->printAsOperand(OS, /*PrintType=*/false);
1681
1682 for (const Loop *PIL : PostIncLoops) {
1683 OS << ", PostIncLoop=";
1684 PIL->getHeader()->printAsOperand(OS, /*PrintType=*/false);
1685 }
1686
1687 if (Offset.isNonZero())
1688 OS << ", Offset=" << Offset;
1689}
1690
1691LLVM_DUMP_METHOD void LSRFixup::dump() const {
1692 print(errs()); errs() << '\n';
1693}
1694#endif
1695
1696/// Test whether this use as a formula which has the same registers as the given
1697/// formula.
1698bool LSRUse::HasFormulaWithSameRegs(const Formula &F) const {
1699 SmallVector<const SCEV *, 4> Key = F.BaseRegs;
1700 if (F.ScaledReg) Key.push_back(Elt: F.ScaledReg);
1701 // Unstable sort by host order ok, because this is only used for uniquifying.
1702 llvm::sort(C&: Key);
1703 return Uniquifier.count(V: Key);
1704}
1705
1706/// The function returns a probability of selecting formula without Reg.
1707float LSRUse::getNotSelectedProbability(const SCEV *Reg) const {
1708 unsigned FNum = 0;
1709 for (const Formula &F : Formulae)
1710 if (F.referencesReg(S: Reg))
1711 FNum++;
1712 return ((float)(Formulae.size() - FNum)) / Formulae.size();
1713}
1714
1715/// If the given formula has not yet been inserted, add it to the list, and
1716/// return true. Return false otherwise. The formula must be in canonical form.
1717bool LSRUse::InsertFormula(const Formula &F, const Loop &L) {
1718 assert(F.isCanonical(L) && "Invalid canonical representation");
1719
1720 if (!Formulae.empty() && RigidFormula)
1721 return false;
1722
1723 SmallVector<const SCEV *, 4> Key = F.BaseRegs;
1724 if (F.ScaledReg) Key.push_back(Elt: F.ScaledReg);
1725 // Unstable sort by host order ok, because this is only used for uniquifying.
1726 llvm::sort(C&: Key, Comp: std::less<>());
1727
1728 if (!Uniquifier.insert(V: Key).second)
1729 return false;
1730
1731 // Using a register to hold the value of 0 is not profitable.
1732 assert((!F.ScaledReg || !F.ScaledReg->isZero()) &&
1733 "Zero allocated in a scaled register!");
1734#ifndef NDEBUG
1735 for (const SCEV *BaseReg : F.BaseRegs)
1736 assert(!BaseReg->isZero() && "Zero allocated in a base register!");
1737#endif
1738
1739 // Add the formula to the list.
1740 Formulae.push_back(Elt: F);
1741
1742 // Record registers now being used by this use.
1743 Regs.insert_range(R: F.BaseRegs);
1744 if (F.ScaledReg)
1745 Regs.insert(Ptr: F.ScaledReg);
1746
1747 return true;
1748}
1749
1750/// Remove the given formula from this use's list.
1751void LSRUse::DeleteFormula(Formula &F) {
1752 if (&F != &Formulae.back())
1753 std::swap(a&: F, b&: Formulae.back());
1754 Formulae.pop_back();
1755}
1756
1757/// Recompute the Regs field, and update RegUses.
1758void LSRUse::RecomputeRegs(size_t LUIdx, RegUseTracker &RegUses) {
1759 // Now that we've filtered out some formulae, recompute the Regs set.
1760 SmallPtrSet<const SCEV *, 4> OldRegs = std::move(Regs);
1761 Regs.clear();
1762 for (const Formula &F : Formulae) {
1763 if (F.ScaledReg) Regs.insert(Ptr: F.ScaledReg);
1764 Regs.insert_range(R: F.BaseRegs);
1765 }
1766
1767 // Update the RegTracker.
1768 for (const SCEV *S : OldRegs)
1769 if (!Regs.count(Ptr: S))
1770 RegUses.dropRegister(Reg: S, LUIdx);
1771}
1772
1773#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1774void LSRUse::print(raw_ostream &OS) const {
1775 OS << "LSR Use: Kind=";
1776 switch (Kind) {
1777 case Basic: OS << "Basic"; break;
1778 case Special: OS << "Special"; break;
1779 case ICmpZero: OS << "ICmpZero"; break;
1780 case Address:
1781 OS << "Address of ";
1782 if (AccessTy.MemTy->isPointerTy())
1783 OS << "pointer"; // the full pointer type could be really verbose
1784 else {
1785 OS << *AccessTy.MemTy;
1786 }
1787
1788 OS << " in addrspace(" << AccessTy.AddrSpace << ')';
1789 }
1790
1791 OS << ", Offsets={";
1792 bool NeedComma = false;
1793 for (const LSRFixup &Fixup : Fixups) {
1794 if (NeedComma) OS << ',';
1795 OS << Fixup.Offset;
1796 NeedComma = true;
1797 }
1798 OS << '}';
1799
1800 if (AllFixupsOutsideLoop)
1801 OS << ", all-fixups-outside-loop";
1802
1803 if (AllFixupsUnconditional)
1804 OS << ", all-fixups-unconditional";
1805}
1806
1807LLVM_DUMP_METHOD void LSRUse::dump() const {
1808 print(errs()); errs() << '\n';
1809}
1810#endif
1811
1812static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1813 LSRUse::KindType Kind, MemAccessTy AccessTy,
1814 GlobalValue *BaseGV, Immediate BaseOffset,
1815 bool HasBaseReg, int64_t Scale,
1816 Instruction *Fixup /* = nullptr */) {
1817 switch (Kind) {
1818 case LSRUse::Address: {
1819 int64_t FixedOffset =
1820 BaseOffset.isScalable() ? 0 : BaseOffset.getFixedValue();
1821 int64_t ScalableOffset =
1822 BaseOffset.isScalable() ? BaseOffset.getKnownMinValue() : 0;
1823 return TTI.isLegalAddressingMode(Ty: AccessTy.MemTy, BaseGV, BaseOffset: FixedOffset,
1824 HasBaseReg, Scale, AddrSpace: AccessTy.AddrSpace,
1825 I: Fixup, ScalableOffset);
1826 }
1827 case LSRUse::ICmpZero:
1828 // There's not even a target hook for querying whether it would be legal to
1829 // fold a GV into an ICmp.
1830 if (BaseGV)
1831 return false;
1832
1833 // ICmp only has two operands; don't allow more than two non-trivial parts.
1834 if (Scale != 0 && HasBaseReg && BaseOffset.isNonZero())
1835 return false;
1836
1837 // ICmp only supports no scale or a -1 scale, as we can "fold" a -1 scale by
1838 // putting the scaled register in the other operand of the icmp.
1839 if (Scale != 0 && Scale != -1)
1840 return false;
1841
1842 // If we have low-level target information, ask the target if it can fold an
1843 // integer immediate on an icmp.
1844 if (BaseOffset.isNonZero()) {
1845 // We don't have an interface to query whether the target supports
1846 // icmpzero against scalable quantities yet.
1847 if (BaseOffset.isScalable())
1848 return false;
1849
1850 // We have one of:
1851 // ICmpZero BaseReg + BaseOffset => ICmp BaseReg, -BaseOffset
1852 // ICmpZero -1*ScaleReg + BaseOffset => ICmp ScaleReg, BaseOffset
1853 // Offs is the ICmp immediate.
1854 if (Scale == 0)
1855 // The cast does the right thing with
1856 // std::numeric_limits<int64_t>::min().
1857 BaseOffset = BaseOffset.getFixed(MinVal: -(uint64_t)BaseOffset.getFixedValue());
1858 return TTI.isLegalICmpImmediate(Imm: BaseOffset.getFixedValue());
1859 }
1860
1861 // ICmpZero BaseReg + -1*ScaleReg => ICmp BaseReg, ScaleReg
1862 return true;
1863
1864 case LSRUse::Basic:
1865 // Only handle single-register values.
1866 return !BaseGV && Scale == 0 && BaseOffset.isZero();
1867
1868 case LSRUse::Special:
1869 // Special case Basic to handle -1 scales.
1870 return !BaseGV && (Scale == 0 || Scale == -1) && BaseOffset.isZero();
1871 }
1872
1873 llvm_unreachable("Invalid LSRUse Kind!");
1874}
1875
1876static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1877 Immediate MinOffset, Immediate MaxOffset,
1878 LSRUse::KindType Kind, MemAccessTy AccessTy,
1879 GlobalValue *BaseGV, Immediate BaseOffset,
1880 bool HasBaseReg, int64_t Scale) {
1881 if (BaseOffset.isNonZero() &&
1882 (BaseOffset.isScalable() != MinOffset.isScalable() ||
1883 BaseOffset.isScalable() != MaxOffset.isScalable()))
1884 return false;
1885 // Check for overflow.
1886 int64_t Base = BaseOffset.getKnownMinValue();
1887 int64_t Min = MinOffset.getKnownMinValue();
1888 int64_t Max = MaxOffset.getKnownMinValue();
1889 if (((int64_t)((uint64_t)Base + Min) > Base) != (Min > 0))
1890 return false;
1891 MinOffset = Immediate::get(MinVal: (uint64_t)Base + Min, Scalable: MinOffset.isScalable());
1892 if (((int64_t)((uint64_t)Base + Max) > Base) != (Max > 0))
1893 return false;
1894 MaxOffset = Immediate::get(MinVal: (uint64_t)Base + Max, Scalable: MaxOffset.isScalable());
1895
1896 return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset: MinOffset,
1897 HasBaseReg, Scale) &&
1898 isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset: MaxOffset,
1899 HasBaseReg, Scale);
1900}
1901
1902static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1903 Immediate MinOffset, Immediate MaxOffset,
1904 LSRUse::KindType Kind, MemAccessTy AccessTy,
1905 const Formula &F, const Loop &L) {
1906 // For the purpose of isAMCompletelyFolded either having a canonical formula
1907 // or a scale not equal to zero is correct.
1908 // Problems may arise from non canonical formulae having a scale == 0.
1909 // Strictly speaking it would best to just rely on canonical formulae.
1910 // However, when we generate the scaled formulae, we first check that the
1911 // scaling factor is profitable before computing the actual ScaledReg for
1912 // compile time sake.
1913 assert((F.isCanonical(L) || F.Scale != 0));
1914 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1915 BaseGV: F.BaseGV, BaseOffset: F.BaseOffset, HasBaseReg: F.HasBaseReg, Scale: F.Scale);
1916}
1917
1918/// Test whether we know how to expand the current formula.
1919static bool isLegalUse(const TargetTransformInfo &TTI, Immediate MinOffset,
1920 Immediate MaxOffset, LSRUse::KindType Kind,
1921 MemAccessTy AccessTy, GlobalValue *BaseGV,
1922 Immediate BaseOffset, bool HasBaseReg, int64_t Scale) {
1923 // We know how to expand completely foldable formulae.
1924 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
1925 BaseOffset, HasBaseReg, Scale) ||
1926 // Or formulae that use a base register produced by a sum of base
1927 // registers.
1928 (Scale == 1 &&
1929 isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy,
1930 BaseGV, BaseOffset, HasBaseReg: true, Scale: 0));
1931}
1932
1933static bool isLegalUse(const TargetTransformInfo &TTI, Immediate MinOffset,
1934 Immediate MaxOffset, LSRUse::KindType Kind,
1935 MemAccessTy AccessTy, const Formula &F) {
1936 return isLegalUse(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV: F.BaseGV,
1937 BaseOffset: F.BaseOffset, HasBaseReg: F.HasBaseReg, Scale: F.Scale);
1938}
1939
1940static bool isLegalAddImmediate(const TargetTransformInfo &TTI,
1941 Immediate Offset) {
1942 if (Offset.isScalable())
1943 return TTI.isLegalAddScalableImmediate(Imm: Offset.getKnownMinValue());
1944
1945 return TTI.isLegalAddImmediate(Imm: Offset.getFixedValue());
1946}
1947
1948static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
1949 const LSRUse &LU, const Formula &F) {
1950 // Target may want to look at the user instructions.
1951 if (LU.Kind == LSRUse::Address && TTI.LSRWithInstrQueries()) {
1952 for (const LSRFixup &Fixup : LU.Fixups)
1953 if (!isAMCompletelyFolded(TTI, Kind: LSRUse::Address, AccessTy: LU.AccessTy, BaseGV: F.BaseGV,
1954 BaseOffset: (F.BaseOffset + Fixup.Offset), HasBaseReg: F.HasBaseReg,
1955 Scale: F.Scale, Fixup: Fixup.UserInst))
1956 return false;
1957 return true;
1958 }
1959
1960 return isAMCompletelyFolded(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
1961 AccessTy: LU.AccessTy, BaseGV: F.BaseGV, BaseOffset: F.BaseOffset, HasBaseReg: F.HasBaseReg,
1962 Scale: F.Scale);
1963}
1964
1965static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI,
1966 const LSRUse &LU, const Formula &F,
1967 const Loop &L) {
1968 if (!F.Scale)
1969 return 0;
1970
1971 // If the use is not completely folded in that instruction, we will have to
1972 // pay an extra cost only for scale != 1.
1973 if (!isAMCompletelyFolded(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
1974 AccessTy: LU.AccessTy, F, L))
1975 return F.Scale != 1;
1976
1977 switch (LU.Kind) {
1978 case LSRUse::Address: {
1979 // Check the scaling factor cost with both the min and max offsets.
1980 int64_t ScalableMin = 0, ScalableMax = 0, FixedMin = 0, FixedMax = 0;
1981 if (F.BaseOffset.isScalable()) {
1982 ScalableMin = (F.BaseOffset + LU.MinOffset).getKnownMinValue();
1983 ScalableMax = (F.BaseOffset + LU.MaxOffset).getKnownMinValue();
1984 } else {
1985 FixedMin = (F.BaseOffset + LU.MinOffset).getFixedValue();
1986 FixedMax = (F.BaseOffset + LU.MaxOffset).getFixedValue();
1987 }
1988 InstructionCost ScaleCostMinOffset = TTI.getScalingFactorCost(
1989 Ty: LU.AccessTy.MemTy, BaseGV: F.BaseGV, BaseOffset: StackOffset::get(Fixed: FixedMin, Scalable: ScalableMin),
1990 HasBaseReg: F.HasBaseReg, Scale: F.Scale, AddrSpace: LU.AccessTy.AddrSpace);
1991 InstructionCost ScaleCostMaxOffset = TTI.getScalingFactorCost(
1992 Ty: LU.AccessTy.MemTy, BaseGV: F.BaseGV, BaseOffset: StackOffset::get(Fixed: FixedMax, Scalable: ScalableMax),
1993 HasBaseReg: F.HasBaseReg, Scale: F.Scale, AddrSpace: LU.AccessTy.AddrSpace);
1994
1995 assert(ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() &&
1996 "Legal addressing mode has an illegal cost!");
1997 return std::max(a: ScaleCostMinOffset, b: ScaleCostMaxOffset);
1998 }
1999 case LSRUse::ICmpZero:
2000 case LSRUse::Basic:
2001 case LSRUse::Special:
2002 // The use is completely folded, i.e., everything is folded into the
2003 // instruction.
2004 return 0;
2005 }
2006
2007 llvm_unreachable("Invalid LSRUse Kind!");
2008}
2009
2010static bool isAlwaysFoldable(const ScalarOptions &Opts,
2011 const TargetTransformInfo &TTI,
2012 LSRUse::KindType Kind, MemAccessTy AccessTy,
2013 GlobalValue *BaseGV, Immediate BaseOffset,
2014 bool HasBaseReg) {
2015 // Fast-path: zero is always foldable.
2016 if (BaseOffset.isZero() && !BaseGV)
2017 return true;
2018
2019 // Conservatively, create an address with an immediate and a
2020 // base and a scale.
2021 int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1;
2022
2023 // Canonicalize a scale of 1 to a base register if the formula doesn't
2024 // already have a base register.
2025 if (!HasBaseReg && Scale == 1) {
2026 Scale = 0;
2027 HasBaseReg = true;
2028 }
2029
2030 // FIXME: Try with + without a scale? Maybe based on TTI?
2031 // I think basereg + scaledreg + immediateoffset isn't a good 'conservative'
2032 // default for many architectures, not just AArch64 SVE. More investigation
2033 // needed later to determine if this should be used more widely than just
2034 // on scalable types.
2035 if (HasBaseReg && BaseOffset.isNonZero() && Kind != LSRUse::ICmpZero &&
2036 AccessTy.MemTy && AccessTy.MemTy->isScalableTy() &&
2037 Opts.lsr_drop_scaled_reg_for_vscale)
2038 Scale = 0;
2039
2040 return isAMCompletelyFolded(TTI, Kind, AccessTy, BaseGV, BaseOffset,
2041 HasBaseReg, Scale);
2042}
2043
2044static bool isAlwaysFoldable(const ScalarOptions &Opts,
2045 const TargetTransformInfo &TTI,
2046 ScalarEvolution &SE, Immediate MinOffset,
2047 Immediate MaxOffset, LSRUse::KindType Kind,
2048 MemAccessTy AccessTy, const SCEV *S,
2049 bool HasBaseReg) {
2050 // Fast-path: zero is always foldable.
2051 if (S->isZero()) return true;
2052
2053 // Conservatively, create an address with an immediate and a
2054 // base and a scale.
2055 SCEVUse SCopy = S;
2056 Immediate BaseOffset = extractImmediate(Opts, S&: SCopy, SE);
2057 GlobalValue *BaseGV = ExtractSymbol(S&: SCopy, SE);
2058
2059 // If there's anything else involved, it's not foldable.
2060 if (!SCopy->isZero())
2061 return false;
2062
2063 // Fast-path: zero is always foldable.
2064 if (BaseOffset.isZero() && !BaseGV)
2065 return true;
2066
2067 if (BaseOffset.isScalable())
2068 return false;
2069
2070 // Conservatively, create an address with an immediate and a
2071 // base and a scale.
2072 int64_t Scale = Kind == LSRUse::ICmpZero ? -1 : 1;
2073
2074 return isAMCompletelyFolded(TTI, MinOffset, MaxOffset, Kind, AccessTy, BaseGV,
2075 BaseOffset, HasBaseReg, Scale);
2076}
2077
2078namespace {
2079
2080/// An individual increment in a Chain of IV increments. Relate an IV user to
2081/// an expression that computes the IV it uses from the IV used by the previous
2082/// link in the Chain.
2083///
2084/// For the head of a chain, IncExpr holds the absolute SCEV expression for the
2085/// original IVOperand. The head of the chain's IVOperand is only valid during
2086/// chain collection, before LSR replaces IV users. During chain generation,
2087/// IncExpr can be used to find the new IVOperand that computes the same
2088/// expression.
2089struct IVInc {
2090 Instruction *UserInst;
2091 Value* IVOperand;
2092 const SCEV *IncExpr;
2093
2094 IVInc(Instruction *U, Value *O, const SCEV *E)
2095 : UserInst(U), IVOperand(O), IncExpr(E) {}
2096};
2097
2098// The list of IV increments in program order. We typically add the head of a
2099// chain without finding subsequent links.
2100struct IVChain {
2101 SmallVector<IVInc, 1> Incs;
2102 const SCEV *ExprBase = nullptr;
2103
2104 IVChain() = default;
2105 IVChain(const IVInc &Head, const SCEV *Base)
2106 : Incs(1, Head), ExprBase(Base) {}
2107
2108 using const_iterator = SmallVectorImpl<IVInc>::const_iterator;
2109
2110 // Return the first increment in the chain.
2111 const_iterator begin() const {
2112 assert(!Incs.empty());
2113 return std::next(x: Incs.begin());
2114 }
2115 const_iterator end() const {
2116 return Incs.end();
2117 }
2118
2119 // Returns true if this chain contains any increments.
2120 bool hasIncs() const { return Incs.size() >= 2; }
2121
2122 // Add an IVInc to the end of this chain.
2123 void add(const IVInc &X) { Incs.push_back(Elt: X); }
2124
2125 // Returns the last UserInst in the chain.
2126 Instruction *tailUserInst() const { return Incs.back().UserInst; }
2127
2128 // Returns true if IncExpr can be profitably added to this chain.
2129 bool isProfitableIncrement(const SCEV *OperExpr,
2130 const SCEV *IncExpr,
2131 ScalarEvolution&);
2132};
2133
2134/// Helper for CollectChains to track multiple IV increment uses. Distinguish
2135/// between FarUsers that definitely cross IV increments and NearUsers that may
2136/// be used between IV increments.
2137struct ChainUsers {
2138 SmallPtrSet<Instruction*, 4> FarUsers;
2139 SmallPtrSet<Instruction*, 4> NearUsers;
2140};
2141
2142/// This class holds state for the main loop strength reduction logic.
2143class LSRInstance {
2144 const ScalarOptions &Opts;
2145 IVUsers &IU;
2146 ScalarEvolution &SE;
2147 DominatorTree &DT;
2148 LoopInfo &LI;
2149 AssumptionCache &AC;
2150 TargetLibraryInfo &TLI;
2151 const TargetTransformInfo &TTI;
2152 Loop *const L;
2153 MemorySSAUpdater *MSSAU;
2154 TTI::AddressingModeKind AMK;
2155 mutable SCEVExpander Rewriter;
2156 bool Changed = false;
2157 bool HardwareLoopProfitable = false;
2158 bool ShouldPreserveLCSSA = false;
2159
2160 /// This is the insert position that the current loop's induction variable
2161 /// increment should be placed. In simple loops, this is the latch block's
2162 /// terminator. But in more complicated cases, this is a position which will
2163 /// dominate all the in-loop post-increment users.
2164 Instruction *IVIncInsertPos = nullptr;
2165
2166 /// Interesting factors between use strides.
2167 ///
2168 /// We explicitly use a SetVector which contains a SmallSet, instead of the
2169 /// default, a SmallDenseSet, because we need to use the full range of
2170 /// int64_ts, and there's currently no good way of doing that with
2171 /// SmallDenseSet.
2172 SetVector<int64_t, SmallVector<int64_t, 8>, SmallSet<int64_t, 8>> Factors;
2173
2174 /// The cost of the current SCEV, the best solution by LSR will be dropped if
2175 /// the solution is not profitable.
2176 Cost BaselineCost;
2177
2178 /// Interesting use types, to facilitate truncation reuse.
2179 SmallSetVector<Type *, 4> Types;
2180
2181 /// The list of interesting uses.
2182 mutable SmallVector<LSRUse, 16> Uses;
2183
2184 /// Track which uses use which register candidates.
2185 RegUseTracker RegUses;
2186
2187 // Limit the number of chains to avoid quadratic behavior. We don't expect to
2188 // have more than a few IV increment chains in a loop. Missing a Chain falls
2189 // back to normal LSR behavior for those uses.
2190 static const unsigned MaxChains = 8;
2191
2192 /// IV users can form a chain of IV increments.
2193 SmallVector<IVChain, MaxChains> IVChainVec;
2194
2195 /// IV users that belong to profitable IVChains.
2196 SmallPtrSet<Use*, MaxChains> IVIncSet;
2197
2198 /// Induction variables that were generated and inserted by the SCEV Expander.
2199 SmallVector<llvm::WeakVH, 2> ScalarEvolutionIVs;
2200
2201 // Inserting instructions in the loop and using them as PHI's input could
2202 // break LCSSA in case if PHI's parent block is not a loop exit (i.e. the
2203 // corresponding incoming block is not loop exiting). So collect all such
2204 // instructions to form LCSSA for them later.
2205 SmallSetVector<Instruction *, 4> InsertedNonLCSSAInsts;
2206
2207 void OptimizeShadowIV();
2208 bool FindIVUserForCond(Instruction *Cond, IVStrideUse *&CondUse);
2209 Instruction *OptimizeMax(ICmpInst *Cond, IVStrideUse *&CondUse);
2210 ICmpInst *OptimizeLoopTermCondToEquality(ICmpInst *Cond,
2211 IVStrideUse *&CondUse,
2212 BasicBlock *ExitingBlock);
2213 void OptimizeLoopTermCond();
2214
2215 void ChainInstruction(Instruction *UserInst, Instruction *IVOper,
2216 SmallVectorImpl<ChainUsers> &ChainUsersVec);
2217 void FinalizeChain(IVChain &Chain);
2218 void CollectChains();
2219 void GenerateIVChain(const IVChain &Chain,
2220 SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2221
2222 void CollectInterestingTypesAndFactors();
2223 void CollectFixupsAndInitialFormulae();
2224
2225 // Support for sharing of LSRUses between LSRFixups.
2226 using UseMapTy = DenseMap<LSRUse::SCEVUseKindPair, size_t>;
2227 UseMapTy UseMap;
2228
2229 bool reconcileNewOffset(LSRUse &LU, Immediate NewOffset, bool HasBaseReg,
2230 LSRUse::KindType Kind, MemAccessTy AccessTy);
2231
2232 std::pair<size_t, Immediate> getUse(const SCEV *&Expr, LSRUse::KindType Kind,
2233 MemAccessTy AccessTy);
2234
2235 void DeleteUse(LSRUse &LU, size_t LUIdx);
2236
2237 LSRUse *FindUseWithSimilarFormula(const Formula &F, const LSRUse &OrigLU);
2238
2239 void InsertInitialFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2240 void InsertSupplementalFormula(const SCEV *S, LSRUse &LU, size_t LUIdx);
2241 void CountRegisters(const Formula &F, size_t LUIdx);
2242 bool InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F);
2243 bool IsFixupExecutedEachIncrement(const LSRFixup &LF) const;
2244
2245 void CollectLoopInvariantFixupsAndFormulae();
2246
2247 void GenerateReassociations(LSRUse &LU, unsigned LUIdx, Formula Base,
2248 unsigned Depth = 0);
2249
2250 void GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
2251 const Formula &Base, unsigned Depth,
2252 size_t Idx, bool IsScaledReg = false);
2253 void GenerateCombinations(LSRUse &LU, unsigned LUIdx, Formula Base);
2254 void GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2255 const Formula &Base, size_t Idx,
2256 bool IsScaledReg = false);
2257 void GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2258 void GenerateConstantOffsetsImpl(LSRUse &LU, unsigned LUIdx,
2259 const Formula &Base,
2260 const SmallVectorImpl<Immediate> &Worklist,
2261 size_t Idx, bool IsScaledReg = false);
2262 void GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx, Formula Base);
2263 void GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2264 void GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base);
2265 void GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base);
2266 void GenerateCrossUseConstantOffsets();
2267 void GenerateAllReuseFormulae();
2268
2269 void FilterOutUndesirableDedicatedRegisters();
2270
2271 size_t EstimateSearchSpaceComplexity() const;
2272 void NarrowSearchSpaceByDetectingSupersets();
2273 void NarrowSearchSpaceByCollapsingUnrolledCode();
2274 void NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
2275 void NarrowSearchSpaceByFilterFormulaWithSameScaledReg();
2276 void NarrowSearchSpaceByFilterPostInc();
2277 void NarrowSearchSpaceByMergingUsesOutsideLoop();
2278 void NarrowSearchSpaceByDeletingCostlyFormulas();
2279 void NarrowSearchSpaceByPickingWinnerRegs();
2280 void NarrowSearchSpaceUsingHeuristics();
2281
2282 void SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
2283 Cost &SolutionCost,
2284 SmallVectorImpl<const Formula *> &Workspace,
2285 const Cost &CurCost,
2286 const SmallPtrSet<const SCEV *, 16> &CurRegs,
2287 DenseSet<const SCEV *> &VisitedRegs) const;
2288 void Solve(SmallVectorImpl<const Formula *> &Solution) const;
2289
2290 BasicBlock::iterator
2291 HoistInsertPosition(BasicBlock::iterator IP,
2292 const SmallVectorImpl<Instruction *> &Inputs) const;
2293 BasicBlock::iterator AdjustInsertPositionForExpand(BasicBlock::iterator IP,
2294 const LSRFixup &LF,
2295 const LSRUse &LU) const;
2296
2297 Value *Expand(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2298 BasicBlock::iterator IP,
2299 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const;
2300 void RewriteForPHI(PHINode *PN, const LSRUse &LU, const LSRFixup &LF,
2301 const Formula &F,
2302 SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2303 void Rewrite(const LSRUse &LU, const LSRFixup &LF, const Formula &F,
2304 SmallVectorImpl<WeakTrackingVH> &DeadInsts);
2305 void ImplementSolution(const SmallVectorImpl<const Formula *> &Solution);
2306
2307public:
2308 // TODO(boomanaiden154): The PreserveLCSSA flag is a hack to allow
2309 // experimentation with the NewPM which requires LCSSA preservation while
2310 // some of the details are worked out in LSR. Eventually it should be set
2311 // to true and removed.
2312 LSRInstance(const ScalarOptions &Opts, Loop *L, IVUsers &IU,
2313 ScalarEvolution &SE, DominatorTree &DT, LoopInfo &LI,
2314 const TargetTransformInfo &TTI, AssumptionCache &AC,
2315 TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU,
2316 bool PreserveLCSSA);
2317
2318 bool getChanged() const { return Changed; }
2319 const SmallVectorImpl<WeakVH> &getScalarEvolutionIVs() const {
2320 return ScalarEvolutionIVs;
2321 }
2322
2323 void print_factors_and_types(raw_ostream &OS) const;
2324 void print_fixups(raw_ostream &OS) const;
2325 void print_uses(raw_ostream &OS) const;
2326 void print(raw_ostream &OS) const;
2327 void dump() const;
2328};
2329
2330} // end anonymous namespace
2331
2332/// If IV is used in a int-to-float cast inside the loop then try to eliminate
2333/// the cast operation.
2334void LSRInstance::OptimizeShadowIV() {
2335 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2336 if (isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount))
2337 return;
2338
2339 for (IVUsers::const_iterator UI = IU.begin(), E = IU.end();
2340 UI != E; /* empty */) {
2341 IVUsers::const_iterator CandidateUI = UI;
2342 ++UI;
2343 Instruction *ShadowUse = CandidateUI->getUser();
2344 Type *DestTy = nullptr;
2345 bool IsSigned = false;
2346
2347 /* If shadow use is a int->float cast then insert a second IV
2348 to eliminate this cast.
2349
2350 for (unsigned i = 0; i < n; ++i)
2351 foo((double)i);
2352
2353 is transformed into
2354
2355 double d = 0.0;
2356 for (unsigned i = 0; i < n; ++i, ++d)
2357 foo(d);
2358 */
2359 if (UIToFPInst *UCast = dyn_cast<UIToFPInst>(Val: CandidateUI->getUser())) {
2360 IsSigned = false;
2361 DestTy = UCast->getDestTy();
2362 }
2363 else if (SIToFPInst *SCast = dyn_cast<SIToFPInst>(Val: CandidateUI->getUser())) {
2364 IsSigned = true;
2365 DestTy = SCast->getDestTy();
2366 }
2367 if (!DestTy) continue;
2368
2369 // If target does not support DestTy natively then do not apply
2370 // this transformation.
2371 if (!TTI.isTypeLegal(Ty: DestTy)) continue;
2372
2373 PHINode *PH = dyn_cast<PHINode>(Val: ShadowUse->getOperand(i: 0));
2374 if (!PH) continue;
2375 if (PH->getNumIncomingValues() != 2) continue;
2376
2377 // If the calculation in integers overflows, the result in FP type will
2378 // differ. So we only can do this transformation if we are guaranteed to not
2379 // deal with overflowing values
2380 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: SE.getSCEV(V: PH));
2381 if (!AR) continue;
2382 if (IsSigned && !AR->hasNoSignedWrap()) continue;
2383 if (!IsSigned && !AR->hasNoUnsignedWrap()) continue;
2384
2385 Type *SrcTy = PH->getType();
2386 int Mantissa = DestTy->getFPMantissaWidth();
2387 if (Mantissa == -1) continue;
2388 if ((int)SE.getTypeSizeInBits(Ty: SrcTy) > Mantissa)
2389 continue;
2390
2391 unsigned Entry, Latch;
2392 if (PH->getIncomingBlock(i: 0) == L->getLoopPreheader()) {
2393 Entry = 0;
2394 Latch = 1;
2395 } else {
2396 Entry = 1;
2397 Latch = 0;
2398 }
2399
2400 ConstantInt *Init = dyn_cast<ConstantInt>(Val: PH->getIncomingValue(i: Entry));
2401 if (!Init) continue;
2402 Constant *NewInit = ConstantFP::get(Ty: DestTy, V: IsSigned ?
2403 (double)Init->getSExtValue() :
2404 (double)Init->getZExtValue());
2405
2406 BinaryOperator *Incr =
2407 dyn_cast<BinaryOperator>(Val: PH->getIncomingValue(i: Latch));
2408 if (!Incr) continue;
2409 if (Incr->getOpcode() != Instruction::Add
2410 && Incr->getOpcode() != Instruction::Sub)
2411 continue;
2412
2413 /* Initialize new IV, double d = 0.0 in above example. */
2414 ConstantInt *C = nullptr;
2415 if (Incr->getOperand(i_nocapture: 0) == PH)
2416 C = dyn_cast<ConstantInt>(Val: Incr->getOperand(i_nocapture: 1));
2417 else if (Incr->getOperand(i_nocapture: 1) == PH)
2418 C = dyn_cast<ConstantInt>(Val: Incr->getOperand(i_nocapture: 0));
2419 else
2420 continue;
2421
2422 if (!C) continue;
2423
2424 // Ignore negative constants, as the code below doesn't handle them
2425 // correctly. TODO: Remove this restriction.
2426 if (!C->getValue().isStrictlyPositive())
2427 continue;
2428
2429 /* Add new PHINode. */
2430 PHINode *NewPH = PHINode::Create(Ty: DestTy, NumReservedValues: 2, NameStr: "IV.S.", InsertBefore: PH->getIterator());
2431 NewPH->setDebugLoc(PH->getDebugLoc());
2432
2433 /* create new increment. '++d' in above example. */
2434 Constant *CFP = ConstantFP::get(Ty: DestTy, V: C->getZExtValue());
2435 BinaryOperator *NewIncr = BinaryOperator::Create(
2436 Op: Incr->getOpcode() == Instruction::Add ? Instruction::FAdd
2437 : Instruction::FSub,
2438 S1: NewPH, S2: CFP, Name: "IV.S.next.", InsertBefore: Incr->getIterator());
2439 NewIncr->setDebugLoc(Incr->getDebugLoc());
2440
2441 NewPH->addIncoming(V: NewInit, BB: PH->getIncomingBlock(i: Entry));
2442 NewPH->addIncoming(V: NewIncr, BB: PH->getIncomingBlock(i: Latch));
2443
2444 /* Remove cast operation */
2445 ShadowUse->replaceAllUsesWith(V: NewPH);
2446 ShadowUse->eraseFromParent();
2447 Changed = true;
2448 break;
2449 }
2450}
2451
2452/// If Cond has an operand that is an expression of an IV, set the IV user and
2453/// stride information and return true, otherwise return false.
2454bool LSRInstance::FindIVUserForCond(Instruction *Cond, IVStrideUse *&CondUse) {
2455 for (IVStrideUse &U : IU)
2456 if (U.getUser() == Cond) {
2457 // NOTE: we could handle setcc instructions with multiple uses here, but
2458 // InstCombine does it as well for simple uses, it's not clear that it
2459 // occurs enough in real life to handle.
2460 CondUse = &U;
2461 return true;
2462 }
2463 return false;
2464}
2465
2466/// Replace a relational loop exit with an equality comparison when the target
2467/// prefers the exact exit value as an equality immediate.
2468ICmpInst *LSRInstance::OptimizeLoopTermCondToEquality(
2469 ICmpInst *Cond, IVStrideUse *&CondUse, BasicBlock *ExitingBlock) {
2470 if (Cond->isEquality() || ExitingBlock != L->getLoopLatch())
2471 return Cond;
2472
2473 auto *IV = dyn_cast<PHINode>(Val: CondUse->getOperandValToReplace());
2474 if (!IV || !IV->getType()->isIntegerTy() || IV->getParent() != L->getHeader())
2475 return Cond;
2476
2477 const auto *AR = dyn_cast<SCEVAddRecExpr>(Val: SE.getSCEV(V: IV));
2478 if (!AR || AR->getLoop() != L || !AR->isAffine())
2479 return Cond;
2480
2481 const auto *Step = dyn_cast<SCEVConstant>(Val: AR->getStepRecurrence(SE));
2482 if (!Step || Step->isZero())
2483 return Cond;
2484
2485 const SCEV *ExitCount = SE.getExitCount(L, ExitingBlock);
2486 if (isa<SCEVCouldNotCompute>(Val: ExitCount))
2487 return Cond;
2488
2489 // A W-bit add recurrence with step S has period
2490 // 2^W / gcd(2^W, S) = 2^(W - countr_zero(S)). Require the exit count to be
2491 // smaller without relying on poison-derived no-wrap flags, so equality
2492 // cannot match on an earlier iteration.
2493 const APInt &StepValue = Step->getAPInt();
2494 unsigned RecurrencePeriodLog2 =
2495 StepValue.getBitWidth() - StepValue.countr_zero();
2496 if (!SE.getUnsignedRangeMax(S: ExitCount).isIntN(N: RecurrencePeriodLog2))
2497 return Cond;
2498
2499 const auto *ExitValue =
2500 dyn_cast<SCEVConstant>(Val: AR->evaluateAtIteration(It: ExitCount, SE));
2501 if (!ExitValue || !TTI.preferLoopExitWithEqualityComparison(
2502 Cond: *Cond, PostIncExitValue: ExitValue->getAPInt() + StepValue))
2503 return Cond;
2504
2505 LLVM_DEBUG(dbgs() << " Rewriting loop exit to compare " << *IV << " against "
2506 << *ExitValue << '\n');
2507
2508 auto *TermBr = cast<CondBrInst>(Val: ExitingBlock->getTerminator());
2509 bool TrueContinues = L->contains(BB: TermBr->getSuccessor(i: 0));
2510 assert(TrueContinues != L->contains(TermBr->getSuccessor(1)) &&
2511 "Expected exactly one in-loop successor");
2512
2513 auto Pred = TrueContinues ? ICmpInst::ICMP_NE : ICmpInst::ICMP_EQ;
2514 auto *NewCond = new ICmpInst(TermBr->getIterator(), Pred, IV,
2515 ExitValue->getValue(), "lsr.exitcond");
2516 NewCond->setDebugLoc(Cond->getDebugLoc());
2517 TermBr->setCondition(NewCond);
2518
2519 if (Cond->use_empty()) {
2520 CondUse->setUser(NewCond);
2521 Cond->eraseFromParent();
2522 } else {
2523 CondUse = &IU.AddUser(User: NewCond, Operand: IV);
2524 }
2525
2526 Changed = true;
2527 return NewCond;
2528}
2529
2530/// Rewrite the loop's terminating condition if it uses a max computation.
2531///
2532/// This is a narrow solution to a specific, but acute, problem. For loops
2533/// like this:
2534///
2535/// i = 0;
2536/// do {
2537/// p[i] = 0.0;
2538/// } while (++i < n);
2539///
2540/// the trip count isn't just 'n', because 'n' might not be positive. And
2541/// unfortunately this can come up even for loops where the user didn't use
2542/// a C do-while loop. For example, seemingly well-behaved top-test loops
2543/// will commonly be lowered like this:
2544///
2545/// if (n > 0) {
2546/// i = 0;
2547/// do {
2548/// p[i] = 0.0;
2549/// } while (++i < n);
2550/// }
2551///
2552/// and then it's possible for subsequent optimization to obscure the if
2553/// test in such a way that indvars can't find it.
2554///
2555/// When indvars can't find the if test in loops like this, it creates a
2556/// max expression, which allows it to give the loop a canonical
2557/// induction variable:
2558///
2559/// i = 0;
2560/// max = n < 1 ? 1 : n;
2561/// do {
2562/// p[i] = 0.0;
2563/// } while (++i != max);
2564///
2565/// Canonical induction variables are necessary because the loop passes
2566/// are designed around them. The most obvious example of this is the
2567/// LoopInfo analysis, which doesn't remember trip count values. It
2568/// expects to be able to rediscover the trip count each time it is
2569/// needed, and it does this using a simple analysis that only succeeds if
2570/// the loop has a canonical induction variable.
2571///
2572/// However, when it comes time to generate code, the maximum operation
2573/// can be quite costly, especially if it's inside of an outer loop.
2574///
2575/// This function solves this problem by detecting this type of loop and
2576/// rewriting their conditions from ICMP_NE back to ICMP_SLT, and deleting
2577/// the instructions for the maximum computation.
2578Instruction *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse *&CondUse) {
2579 // Check that the loop matches the pattern we're looking for.
2580 if (Cond->getPredicate() != CmpInst::ICMP_EQ &&
2581 Cond->getPredicate() != CmpInst::ICMP_NE)
2582 return Cond;
2583
2584 SelectInst *Sel = dyn_cast<SelectInst>(Val: Cond->getOperand(i_nocapture: 1));
2585 if (!Sel || !Sel->hasOneUse()) return Cond;
2586
2587 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2588 if (isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount))
2589 return Cond;
2590 const SCEV *One = SE.getConstant(Ty: BackedgeTakenCount->getType(), V: 1);
2591
2592 // Add one to the backedge-taken count to get the trip count.
2593 const SCEV *IterationCount = SE.getAddExpr(LHS: One, RHS: BackedgeTakenCount);
2594 if (IterationCount != SE.getSCEV(V: Sel)) return Cond;
2595
2596 // Check for a max calculation that matches the pattern. There's no check
2597 // for ICMP_ULE here because the comparison would be with zero, which
2598 // isn't interesting.
2599 CmpInst::Predicate Pred = ICmpInst::BAD_ICMP_PREDICATE;
2600 const SCEVNAryExpr *Max = nullptr;
2601 if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(Val: BackedgeTakenCount)) {
2602 Pred = ICmpInst::ICMP_SLE;
2603 Max = S;
2604 } else if (const SCEVSMaxExpr *S = dyn_cast<SCEVSMaxExpr>(Val: IterationCount)) {
2605 Pred = ICmpInst::ICMP_SLT;
2606 Max = S;
2607 } else if (const SCEVUMaxExpr *U = dyn_cast<SCEVUMaxExpr>(Val: IterationCount)) {
2608 Pred = ICmpInst::ICMP_ULT;
2609 Max = U;
2610 } else {
2611 // No match; bail.
2612 return Cond;
2613 }
2614
2615 // To handle a max with more than two operands, this optimization would
2616 // require additional checking and setup.
2617 if (Max->getNumOperands() != 2)
2618 return Cond;
2619
2620 const SCEV *MaxLHS = Max->getOperand(i: 0);
2621 const SCEV *MaxRHS = Max->getOperand(i: 1);
2622
2623 // ScalarEvolution canonicalizes constants to the left. For < and >, look
2624 // for a comparison with 1. For <= and >=, a comparison with zero.
2625 if (!MaxLHS ||
2626 (ICmpInst::isTrueWhenEqual(predicate: Pred) ? !MaxLHS->isZero() : (MaxLHS != One)))
2627 return Cond;
2628
2629 // Check the relevant induction variable for conformance to
2630 // the pattern.
2631 const SCEV *IV = SE.getSCEV(V: Cond->getOperand(i_nocapture: 0));
2632 if (!match(S: IV,
2633 P: m_scev_AffineAddRec(Op0: m_scev_SpecificInt(V: 1), Op1: m_scev_SpecificInt(V: 1))))
2634 return Cond;
2635
2636 assert(cast<SCEVAddRecExpr>(IV)->getLoop() == L &&
2637 "Loop condition operand is an addrec in a different loop!");
2638
2639 // Check the right operand of the select, and remember it, as it will
2640 // be used in the new comparison instruction.
2641 Value *NewRHS = nullptr;
2642 if (ICmpInst::isTrueWhenEqual(predicate: Pred)) {
2643 // Look for n+1, and grab n.
2644 if (AddOperator *BO = dyn_cast<AddOperator>(Val: Sel->getOperand(i_nocapture: 1)))
2645 if (ConstantInt *BO1 = dyn_cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1)))
2646 if (BO1->isOne() && SE.getSCEV(V: BO->getOperand(i_nocapture: 0)) == MaxRHS)
2647 NewRHS = BO->getOperand(i_nocapture: 0);
2648 if (AddOperator *BO = dyn_cast<AddOperator>(Val: Sel->getOperand(i_nocapture: 2)))
2649 if (ConstantInt *BO1 = dyn_cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1)))
2650 if (BO1->isOne() && SE.getSCEV(V: BO->getOperand(i_nocapture: 0)) == MaxRHS)
2651 NewRHS = BO->getOperand(i_nocapture: 0);
2652 if (!NewRHS)
2653 return Cond;
2654 } else if (SE.getSCEV(V: Sel->getOperand(i_nocapture: 1)) == MaxRHS)
2655 NewRHS = Sel->getOperand(i_nocapture: 1);
2656 else if (SE.getSCEV(V: Sel->getOperand(i_nocapture: 2)) == MaxRHS)
2657 NewRHS = Sel->getOperand(i_nocapture: 2);
2658 else if (const SCEVUnknown *SU = dyn_cast<SCEVUnknown>(Val: MaxRHS))
2659 NewRHS = SU->getValue();
2660 else
2661 // Max doesn't match expected pattern.
2662 return Cond;
2663
2664 // Determine the new comparison opcode. It may be signed or unsigned,
2665 // and the original comparison may be either equality or inequality.
2666 if (Cond->getPredicate() == CmpInst::ICMP_EQ)
2667 Pred = CmpInst::getInversePredicate(pred: Pred);
2668
2669 // Ok, everything looks ok to change the condition into an SLT or SGE and
2670 // delete the max calculation.
2671 ICmpInst *NewCond = new ICmpInst(Cond->getIterator(), Pred,
2672 Cond->getOperand(i_nocapture: 0), NewRHS, "scmp");
2673
2674 // Delete the max calculation instructions.
2675 NewCond->setDebugLoc(Cond->getDebugLoc());
2676 Cond->replaceAllUsesWith(V: NewCond);
2677 CondUse->setUser(NewCond);
2678 Instruction *Cmp = cast<Instruction>(Val: Sel->getOperand(i_nocapture: 0));
2679 Cond->eraseFromParent();
2680 Sel->eraseFromParent();
2681 if (Cmp->use_empty()) {
2682 salvageDebugInfo(I&: *Cmp);
2683 Cmp->eraseFromParent();
2684 }
2685 return NewCond;
2686}
2687
2688/// Change loop terminating condition to use the postinc iv when possible.
2689void
2690LSRInstance::OptimizeLoopTermCond() {
2691 SmallPtrSet<Instruction *, 4> PostIncs;
2692
2693 // We need a different set of heuristics for rotated and non-rotated loops.
2694 // If a loop is rotated then the latch is also the backedge, so inserting
2695 // post-inc expressions just before the latch is ideal. To reduce live ranges
2696 // it also makes sense to rewrite terminating conditions to use post-inc
2697 // expressions.
2698 //
2699 // If the loop is not rotated then the latch is not a backedge; the latch
2700 // check is done in the loop head. Adding post-inc expressions before the
2701 // latch will cause overlapping live-ranges of pre-inc and post-inc expressions
2702 // in the loop body. In this case we do *not* want to use post-inc expressions
2703 // in the latch check, and we want to insert post-inc expressions before
2704 // the backedge.
2705 BasicBlock *LatchBlock = L->getLoopLatch();
2706 SmallVector<BasicBlock*, 8> ExitingBlocks;
2707 L->getExitingBlocks(ExitingBlocks);
2708 if (!llvm::is_contained(Range&: ExitingBlocks, Element: LatchBlock)) {
2709 // The backedge doesn't exit the loop; treat this as a head-tested loop.
2710 IVIncInsertPos = LatchBlock->getTerminator();
2711 return;
2712 }
2713
2714 // Otherwise treat this as a rotated loop.
2715 for (BasicBlock *ExitingBlock : ExitingBlocks) {
2716 // Get the terminating condition for the loop if possible. If we
2717 // can, we want to change it to use a post-incremented version of its
2718 // induction variable, to allow coalescing the live ranges for the IV into
2719 // one register value.
2720
2721 CondBrInst *TermBr = dyn_cast<CondBrInst>(Val: ExitingBlock->getTerminator());
2722 if (!TermBr)
2723 continue;
2724
2725 Instruction *Cond = dyn_cast<Instruction>(Val: TermBr->getCondition());
2726 // If the argument to TermBr is an extractelement, then the source of that
2727 // instruction is what's generated the condition.
2728 auto *Extract = dyn_cast_or_null<ExtractElementInst>(Val: Cond);
2729 if (Extract)
2730 Cond = dyn_cast<Instruction>(Val: Extract->getVectorOperand());
2731 // FIXME: We could do more here, like handling logical operations where one
2732 // side is a cmp that uses an induction variable.
2733 if (!Cond)
2734 continue;
2735
2736 // Search IVUsesByStride to find Cond's IVUse if there is one.
2737 IVStrideUse *CondUse = nullptr;
2738 if (!FindIVUserForCond(Cond, CondUse))
2739 continue;
2740
2741 // If the trip count is computed in terms of a max (due to ScalarEvolution
2742 // being unable to find a sufficient guard, for example), change the loop
2743 // comparison to use SLT or ULT instead of NE.
2744 // One consequence of doing this now is that it disrupts the count-down
2745 // optimization. That's not always a bad thing though, because in such
2746 // cases it may still be worthwhile to avoid a max.
2747 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond))
2748 Cond = OptimizeMax(Cond: Cmp, CondUse);
2749
2750 // If this exiting block dominates the latch block, it may also use
2751 // the post-inc value if it won't be shared with other uses.
2752 // Check for dominance.
2753 if (!DT.dominates(A: ExitingBlock, B: LatchBlock))
2754 continue;
2755
2756 if (auto *Cmp = dyn_cast<ICmpInst>(Val: Cond))
2757 Cond = OptimizeLoopTermCondToEquality(Cond: Cmp, CondUse, ExitingBlock);
2758
2759 // Conservatively avoid trying to use the post-inc value in non-latch
2760 // exits if there may be pre-inc users in intervening blocks.
2761 if (LatchBlock != ExitingBlock)
2762 for (const IVStrideUse &UI : IU)
2763 // Test if the use is reachable from the exiting block. This dominator
2764 // query is a conservative approximation of reachability.
2765 if (&UI != CondUse &&
2766 !DT.properlyDominates(A: UI.getUser()->getParent(), B: ExitingBlock)) {
2767 // Conservatively assume there may be reuse if the quotient of their
2768 // strides could be a legal scale.
2769 const SCEV *A = IU.getStride(IU: *CondUse, L);
2770 const SCEV *B = IU.getStride(IU: UI, L);
2771 if (!A || !B) continue;
2772 if (SE.getTypeSizeInBits(Ty: A->getType()) !=
2773 SE.getTypeSizeInBits(Ty: B->getType())) {
2774 if (SE.getTypeSizeInBits(Ty: A->getType()) >
2775 SE.getTypeSizeInBits(Ty: B->getType()))
2776 B = SE.getSignExtendExpr(Op: B, Ty: A->getType());
2777 else
2778 A = SE.getSignExtendExpr(Op: A, Ty: B->getType());
2779 }
2780 if (const SCEVConstant *D =
2781 dyn_cast_or_null<SCEVConstant>(Val: getExactSDiv(LHS: B, RHS: A, SE))) {
2782 const ConstantInt *C = D->getValue();
2783 // Stride of one or negative one can have reuse with non-addresses.
2784 if (C->isOne() || C->isMinusOne())
2785 goto decline_post_inc;
2786 // Avoid weird situations.
2787 if (C->getValue().getSignificantBits() >= 64 ||
2788 C->getValue().isMinSignedValue())
2789 goto decline_post_inc;
2790 // Check for possible scaled-address reuse.
2791 if (isAddressUse(TTI, Inst: UI.getUser(), OperandVal: UI.getOperandValToReplace())) {
2792 MemAccessTy AccessTy =
2793 getAccessType(TTI, Inst: UI.getUser(), OperandVal: UI.getOperandValToReplace());
2794 int64_t Scale = C->getSExtValue();
2795 if (TTI.isLegalAddressingMode(Ty: AccessTy.MemTy, /*BaseGV=*/nullptr,
2796 /*BaseOffset=*/0,
2797 /*HasBaseReg=*/true, Scale,
2798 AddrSpace: AccessTy.AddrSpace))
2799 goto decline_post_inc;
2800 Scale = -Scale;
2801 if (TTI.isLegalAddressingMode(Ty: AccessTy.MemTy, /*BaseGV=*/nullptr,
2802 /*BaseOffset=*/0,
2803 /*HasBaseReg=*/true, Scale,
2804 AddrSpace: AccessTy.AddrSpace))
2805 goto decline_post_inc;
2806 }
2807 }
2808 }
2809
2810 LLVM_DEBUG(dbgs() << " Change loop exiting icmp to use postinc iv: "
2811 << *Cond << '\n');
2812
2813 // It's possible for the setcc instruction to be anywhere in the loop, and
2814 // possible for it to have multiple users. If it is not immediately before
2815 // the exiting block branch, move it.
2816 if (isa_and_nonnull<CmpInst>(Val: Cond) && Cond->getNextNode() != TermBr &&
2817 !Extract) {
2818 if (Cond->hasOneUse()) {
2819 Cond->moveBefore(InsertPos: TermBr->getIterator());
2820 } else {
2821 // Clone the terminating condition and insert into the loopend.
2822 Instruction *OldCond = Cond;
2823 Cond = Cond->clone();
2824 Cond->setName(L->getHeader()->getName() + ".termcond");
2825 Cond->insertInto(ParentBB: ExitingBlock, It: TermBr->getIterator());
2826
2827 // Clone the IVUse, as the old use still exists!
2828 CondUse = &IU.AddUser(User: Cond, Operand: CondUse->getOperandValToReplace());
2829 TermBr->replaceUsesOfWith(From: OldCond, To: Cond);
2830 }
2831 }
2832
2833 // If we get to here, we know that we can transform the setcc instruction to
2834 // use the post-incremented version of the IV, allowing us to coalesce the
2835 // live ranges for the IV correctly.
2836 CondUse->transformToPostInc(L);
2837 Changed = true;
2838
2839 PostIncs.insert(Ptr: Cond);
2840 decline_post_inc:;
2841 }
2842
2843 // Determine an insertion point for the loop induction variable increment. It
2844 // must dominate all the post-inc comparisons we just set up, and it must
2845 // dominate the loop latch edge.
2846 IVIncInsertPos = L->getLoopLatch()->getTerminator();
2847 for (Instruction *Inst : PostIncs)
2848 IVIncInsertPos = DT.findNearestCommonDominator(I1: IVIncInsertPos, I2: Inst);
2849}
2850
2851/// Determine if the given use can accommodate a fixup at the given offset and
2852/// other details. If so, update the use and return true.
2853bool LSRInstance::reconcileNewOffset(LSRUse &LU, Immediate NewOffset,
2854 bool HasBaseReg, LSRUse::KindType Kind,
2855 MemAccessTy AccessTy) {
2856 Immediate NewMinOffset = LU.MinOffset;
2857 Immediate NewMaxOffset = LU.MaxOffset;
2858 MemAccessTy NewAccessTy = AccessTy;
2859
2860 // Check for a mismatched kind. It's tempting to collapse mismatched kinds to
2861 // something conservative, however this can pessimize in the case that one of
2862 // the uses will have all its uses outside the loop, for example.
2863 if (LU.Kind != Kind)
2864 return false;
2865
2866 // Check for a mismatched access type, and fall back conservatively as needed.
2867 // TODO: Be less conservative when the type is similar and can use the same
2868 // addressing modes.
2869 if (Kind == LSRUse::Address) {
2870 if (AccessTy.MemTy != LU.AccessTy.MemTy) {
2871 NewAccessTy = MemAccessTy::getUnknown(Ctx&: AccessTy.MemTy->getContext(),
2872 AS: AccessTy.AddrSpace);
2873 }
2874 }
2875
2876 // Conservatively assume HasBaseReg is true for now.
2877 if (Immediate::isKnownLT(LHS: NewOffset, RHS: LU.MinOffset)) {
2878 if (!isAlwaysFoldable(Opts, TTI, Kind, AccessTy: NewAccessTy, /*BaseGV=*/nullptr,
2879 BaseOffset: LU.MaxOffset - NewOffset, HasBaseReg))
2880 return false;
2881 NewMinOffset = NewOffset;
2882 } else if (Immediate::isKnownGT(LHS: NewOffset, RHS: LU.MaxOffset)) {
2883 if (!isAlwaysFoldable(Opts, TTI, Kind, AccessTy: NewAccessTy, /*BaseGV=*/nullptr,
2884 BaseOffset: NewOffset - LU.MinOffset, HasBaseReg))
2885 return false;
2886 NewMaxOffset = NewOffset;
2887 }
2888
2889 // FIXME: We should be able to handle some level of scalable offset support
2890 // for 'void', but in order to get basic support up and running this is
2891 // being left out.
2892 if (NewAccessTy.MemTy && NewAccessTy.MemTy->isVoidTy() &&
2893 (NewMinOffset.isScalable() || NewMaxOffset.isScalable()))
2894 return false;
2895
2896 // Update the use.
2897 LU.MinOffset = NewMinOffset;
2898 LU.MaxOffset = NewMaxOffset;
2899 LU.AccessTy = NewAccessTy;
2900 return true;
2901}
2902
2903/// Return an LSRUse index and an offset value for a fixup which needs the given
2904/// expression, with the given kind and optional access type. Either reuse an
2905/// existing use or create a new one, as needed.
2906std::pair<size_t, Immediate> LSRInstance::getUse(const SCEV *&Expr,
2907 LSRUse::KindType Kind,
2908 MemAccessTy AccessTy) {
2909 const SCEV *Copy = Expr;
2910 SCEVUse ExprUse = Expr;
2911 Immediate Offset = extractImmediate(
2912 Opts, S&: ExprUse, SE, PreferScalable: AccessTy.MemTy && AccessTy.MemTy->isScalableTy());
2913 Expr = ExprUse;
2914
2915 // Basic uses can't accept any offset, for example.
2916 if (!isAlwaysFoldable(Opts, TTI, Kind, AccessTy, /*BaseGV=*/nullptr, BaseOffset: Offset,
2917 /*HasBaseReg=*/true)) {
2918 Expr = Copy;
2919 Offset = Immediate::getFixed(MinVal: 0);
2920 }
2921
2922 std::pair<UseMapTy::iterator, bool> P =
2923 UseMap.try_emplace(Key: LSRUse::SCEVUseKindPair(Expr, Kind));
2924 if (!P.second) {
2925 // A use already existed with this base.
2926 size_t LUIdx = P.first->second;
2927 LSRUse &LU = Uses[LUIdx];
2928 if (reconcileNewOffset(LU, NewOffset: Offset, /*HasBaseReg=*/true, Kind, AccessTy))
2929 // Reuse this use.
2930 return std::make_pair(x&: LUIdx, y&: Offset);
2931 }
2932
2933 // Create a new use.
2934 size_t LUIdx = Uses.size();
2935 P.first->second = LUIdx;
2936 Uses.push_back(Elt: LSRUse(Kind, AccessTy));
2937 LSRUse &LU = Uses[LUIdx];
2938
2939 LU.MinOffset = Offset;
2940 LU.MaxOffset = Offset;
2941 return std::make_pair(x&: LUIdx, y&: Offset);
2942}
2943
2944/// Delete the given use from the Uses list.
2945void LSRInstance::DeleteUse(LSRUse &LU, size_t LUIdx) {
2946 if (&LU != &Uses.back())
2947 std::swap(a&: LU, b&: Uses.back());
2948 Uses.pop_back();
2949
2950 // Update RegUses.
2951 RegUses.swapAndDropUse(LUIdx, LastLUIdx: Uses.size());
2952}
2953
2954/// Look for a use distinct from OrigLU which is has a formula that has the same
2955/// registers as the given formula.
2956LSRUse *
2957LSRInstance::FindUseWithSimilarFormula(const Formula &OrigF,
2958 const LSRUse &OrigLU) {
2959 // Search all uses for the formula. This could be more clever.
2960 for (LSRUse &LU : Uses) {
2961 // Check whether this use is close enough to OrigLU, to see whether it's
2962 // worthwhile looking through its formulae.
2963 // Ignore ICmpZero uses because they may contain formulae generated by
2964 // GenerateICmpZeroScales, in which case adding fixup offsets may
2965 // be invalid.
2966 if (&LU != &OrigLU && LU.Kind != LSRUse::ICmpZero &&
2967 LU.Kind == OrigLU.Kind && OrigLU.AccessTy == LU.AccessTy &&
2968 LU.HasFormulaWithSameRegs(F: OrigF)) {
2969 // Scan through this use's formulae.
2970 for (const Formula &F : LU.Formulae) {
2971 // Check to see if this formula has the same registers and symbols
2972 // as OrigF.
2973 if (F.BaseRegs == OrigF.BaseRegs &&
2974 F.ScaledReg == OrigF.ScaledReg &&
2975 F.BaseGV == OrigF.BaseGV &&
2976 F.Scale == OrigF.Scale &&
2977 F.UnfoldedOffset == OrigF.UnfoldedOffset) {
2978 if (F.BaseOffset.isZero())
2979 return &LU;
2980 // This is the formula where all the registers and symbols matched;
2981 // there aren't going to be any others. Since we declined it, we
2982 // can skip the rest of the formulae and proceed to the next LSRUse.
2983 break;
2984 }
2985 }
2986 }
2987 }
2988
2989 // Nothing looked good.
2990 return nullptr;
2991}
2992
2993void LSRInstance::CollectInterestingTypesAndFactors() {
2994 SmallSetVector<const SCEV *, 4> Strides;
2995
2996 // Collect interesting types and strides.
2997 SmallVector<const SCEV *, 4> Worklist;
2998 for (const IVStrideUse &U : IU) {
2999 const SCEV *Expr = IU.getExpr(IU: U);
3000 if (!Expr)
3001 continue;
3002
3003 // Collect interesting types.
3004 Types.insert(X: SE.getEffectiveSCEVType(Ty: Expr->getType()));
3005
3006 // Add strides for mentioned loops.
3007 Worklist.push_back(Elt: Expr);
3008 do {
3009 const SCEV *S = Worklist.pop_back_val();
3010 if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: S)) {
3011 if (AR->getLoop() == L)
3012 Strides.insert(X: AR->getStepRecurrence(SE));
3013 Worklist.push_back(Elt: AR->getStart());
3014 } else if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
3015 append_range(C&: Worklist, R: Add->operands());
3016 }
3017 } while (!Worklist.empty());
3018 }
3019
3020 // Compute interesting factors from the set of interesting strides.
3021 for (SmallSetVector<const SCEV *, 4>::const_iterator
3022 I = Strides.begin(), E = Strides.end(); I != E; ++I)
3023 for (SmallSetVector<const SCEV *, 4>::const_iterator NewStrideIter =
3024 std::next(x: I); NewStrideIter != E; ++NewStrideIter) {
3025 const SCEV *OldStride = *I;
3026 const SCEV *NewStride = *NewStrideIter;
3027
3028 if (SE.getTypeSizeInBits(Ty: OldStride->getType()) !=
3029 SE.getTypeSizeInBits(Ty: NewStride->getType())) {
3030 if (SE.getTypeSizeInBits(Ty: OldStride->getType()) >
3031 SE.getTypeSizeInBits(Ty: NewStride->getType()))
3032 NewStride = SE.getSignExtendExpr(Op: NewStride, Ty: OldStride->getType());
3033 else
3034 OldStride = SE.getSignExtendExpr(Op: OldStride, Ty: NewStride->getType());
3035 }
3036 if (const SCEVConstant *Factor =
3037 dyn_cast_or_null<SCEVConstant>(Val: getExactSDiv(LHS: NewStride, RHS: OldStride,
3038 SE, IgnoreSignificantBits: true))) {
3039 if (Factor->getAPInt().getSignificantBits() <= 64 && !Factor->isZero())
3040 Factors.insert(X: Factor->getAPInt().getSExtValue());
3041 } else if (const SCEVConstant *Factor =
3042 dyn_cast_or_null<SCEVConstant>(Val: getExactSDiv(LHS: OldStride,
3043 RHS: NewStride,
3044 SE, IgnoreSignificantBits: true))) {
3045 if (Factor->getAPInt().getSignificantBits() <= 64 && !Factor->isZero())
3046 Factors.insert(X: Factor->getAPInt().getSExtValue());
3047 }
3048 }
3049
3050 // If all uses use the same type, don't bother looking for truncation-based
3051 // reuse.
3052 if (Types.size() == 1)
3053 Types.clear();
3054
3055 LLVM_DEBUG(print_factors_and_types(dbgs()));
3056}
3057
3058/// Helper for CollectChains that finds an IV operand (computed by an AddRec in
3059/// this loop) within [OI,OE) or returns OE. If IVUsers mapped Instructions to
3060/// IVStrideUses, we could partially skip this.
3061static User::op_iterator
3062findIVOperand(User::op_iterator OI, User::op_iterator OE,
3063 Loop *L, ScalarEvolution &SE) {
3064 for(; OI != OE; ++OI) {
3065 if (Instruction *Oper = dyn_cast<Instruction>(Val&: *OI)) {
3066 if (!SE.isSCEVable(Ty: Oper->getType()))
3067 continue;
3068
3069 if (const SCEVAddRecExpr *AR =
3070 dyn_cast<SCEVAddRecExpr>(Val: SE.getSCEV(V: Oper))) {
3071 if (AR->getLoop() == L)
3072 break;
3073 }
3074 }
3075 }
3076 return OI;
3077}
3078
3079/// IVChain logic must consistently peek base TruncInst operands, so wrap it in
3080/// a convenient helper.
3081static Value *getWideOperand(Value *Oper) {
3082 if (TruncInst *Trunc = dyn_cast<TruncInst>(Val: Oper))
3083 return Trunc->getOperand(i_nocapture: 0);
3084 return Oper;
3085}
3086
3087/// Return an approximation of this SCEV expression's "base", or NULL for any
3088/// constant. Returning the expression itself is conservative. Returning a
3089/// deeper subexpression is more precise and valid as long as it isn't less
3090/// complex than another subexpression. For expressions involving multiple
3091/// unscaled values, we need to return the pointer-type SCEVUnknown. This avoids
3092/// forming chains across objects, such as: PrevOper==a[i], IVOper==b[i],
3093/// IVInc==b-a.
3094///
3095/// Since SCEVUnknown is the rightmost type, and pointers are the rightmost
3096/// SCEVUnknown, we simply return the rightmost SCEV operand.
3097static const SCEV *getExprBase(const SCEV *S) {
3098 switch (S->getSCEVType()) {
3099 default: // including scUnknown.
3100 return S;
3101 case scConstant:
3102 case scVScale:
3103 return nullptr;
3104 case scTruncate:
3105 return getExprBase(S: cast<SCEVTruncateExpr>(Val: S)->getOperand());
3106 case scZeroExtend:
3107 return getExprBase(S: cast<SCEVZeroExtendExpr>(Val: S)->getOperand());
3108 case scSignExtend:
3109 return getExprBase(S: cast<SCEVSignExtendExpr>(Val: S)->getOperand());
3110 case scAddExpr: {
3111 // Skip over scaled operands (scMulExpr) to follow add operands as long as
3112 // there's nothing more complex.
3113 // FIXME: not sure if we want to recognize negation.
3114 const SCEVAddExpr *Add = cast<SCEVAddExpr>(Val: S);
3115 for (const SCEV *SubExpr : reverse(C: Add->operands())) {
3116 if (SubExpr->getSCEVType() == scAddExpr)
3117 return getExprBase(S: SubExpr);
3118
3119 if (SubExpr->getSCEVType() != scMulExpr)
3120 return SubExpr;
3121 }
3122 return S; // all operands are scaled, be conservative.
3123 }
3124 case scAddRecExpr:
3125 return getExprBase(S: cast<SCEVAddRecExpr>(Val: S)->getStart());
3126 }
3127 llvm_unreachable("Unknown SCEV kind!");
3128}
3129
3130/// Return true if the chain increment is profitable to expand into a loop
3131/// invariant value, which may require its own register. A profitable chain
3132/// increment will be an offset relative to the same base. We allow such offsets
3133/// to potentially be used as chain increment as long as it's not obviously
3134/// expensive to expand using real instructions.
3135bool IVChain::isProfitableIncrement(const SCEV *OperExpr,
3136 const SCEV *IncExpr,
3137 ScalarEvolution &SE) {
3138 // Do not replace a constant offset from IV head with a nonconstant IV
3139 // increment.
3140 if (!isa<SCEVConstant>(Val: IncExpr)) {
3141 const SCEV *HeadExpr = SE.getSCEV(V: getWideOperand(Oper: Incs[0].IVOperand));
3142 if (isa<SCEVConstant>(Val: SE.getMinusSCEV(LHS: OperExpr, RHS: HeadExpr)))
3143 return false;
3144 }
3145
3146 SmallPtrSet<const SCEV*, 8> Processed;
3147 return !isHighCostExpansion(S: IncExpr, Processed, SE);
3148}
3149
3150/// Return true if the number of registers needed for the chain is estimated to
3151/// be less than the number required for the individual IV users. First prohibit
3152/// any IV users that keep the IV live across increments (the Users set should
3153/// be empty). Next count the number and type of increments in the chain.
3154///
3155/// Chaining IVs can lead to considerable code bloat if ISEL doesn't
3156/// effectively use postinc addressing modes. Only consider it profitable it the
3157/// increments can be computed in fewer registers when chained.
3158///
3159/// TODO: Consider IVInc free if it's already used in another chains.
3160static bool isProfitableChain(IVChain &Chain,
3161 SmallPtrSetImpl<Instruction *> &Users,
3162 ScalarEvolution &SE,
3163 const TargetTransformInfo &TTI) {
3164 if (!Chain.hasIncs())
3165 return false;
3166
3167 if (!Users.empty()) {
3168 LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " users:\n";
3169 for (Instruction *Inst
3170 : Users) { dbgs() << " " << *Inst << "\n"; });
3171 return false;
3172 }
3173 assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
3174
3175 // The chain itself may require a register, so initialize cost to 1.
3176 int cost = 1;
3177
3178 // A complete chain likely eliminates the need for keeping the original IV in
3179 // a register. LSR does not currently know how to form a complete chain unless
3180 // the header phi already exists.
3181 if (isa<PHINode>(Val: Chain.tailUserInst())
3182 && SE.getSCEV(V: Chain.tailUserInst()) == Chain.Incs[0].IncExpr) {
3183 --cost;
3184 }
3185 const SCEV *LastIncExpr = nullptr;
3186 unsigned NumConstIncrements = 0;
3187 unsigned NumVarIncrements = 0;
3188 unsigned NumReusedIncrements = 0;
3189
3190 if (TTI.isProfitableLSRChainElement(I: Chain.Incs[0].UserInst))
3191 return true;
3192
3193 for (const IVInc &Inc : Chain) {
3194 if (TTI.isProfitableLSRChainElement(I: Inc.UserInst))
3195 return true;
3196 if (Inc.IncExpr->isZero())
3197 continue;
3198
3199 // Incrementing by zero or some constant is neutral. We assume constants can
3200 // be folded into an addressing mode or an add's immediate operand.
3201 if (isa<SCEVConstant>(Val: Inc.IncExpr)) {
3202 ++NumConstIncrements;
3203 continue;
3204 }
3205
3206 if (Inc.IncExpr == LastIncExpr)
3207 ++NumReusedIncrements;
3208 else
3209 ++NumVarIncrements;
3210
3211 LastIncExpr = Inc.IncExpr;
3212 }
3213 // An IV chain with a single increment is handled by LSR's postinc
3214 // uses. However, a chain with multiple increments requires keeping the IV's
3215 // value live longer than it needs to be if chained.
3216 if (NumConstIncrements > 1)
3217 --cost;
3218
3219 // Materializing increment expressions in the preheader that didn't exist in
3220 // the original code may cost a register. For example, sign-extended array
3221 // indices can produce ridiculous increments like this:
3222 // IV + ((sext i32 (2 * %s) to i64) + (-1 * (sext i32 %s to i64)))
3223 cost += NumVarIncrements;
3224
3225 // Reusing variable increments likely saves a register to hold the multiple of
3226 // the stride.
3227 cost -= NumReusedIncrements;
3228
3229 LLVM_DEBUG(dbgs() << "Chain: " << *Chain.Incs[0].UserInst << " Cost: " << cost
3230 << "\n");
3231
3232 return cost < 0;
3233}
3234
3235/// Add this IV user to an existing chain or make it the head of a new chain.
3236void LSRInstance::ChainInstruction(Instruction *UserInst, Instruction *IVOper,
3237 SmallVectorImpl<ChainUsers> &ChainUsersVec) {
3238 // When IVs are used as types of varying widths, they are generally converted
3239 // to a wider type with some uses remaining narrow under a (free) trunc.
3240 Value *const NextIV = getWideOperand(Oper: IVOper);
3241 const SCEV *const OperExpr = SE.getSCEV(V: NextIV);
3242 const SCEV *const OperExprBase = getExprBase(S: OperExpr);
3243
3244 // Visit all existing chains. Check if its IVOper can be computed as a
3245 // profitable loop invariant increment from the last link in the Chain.
3246 unsigned ChainIdx = 0, NChains = IVChainVec.size();
3247 const SCEV *LastIncExpr = nullptr;
3248 for (; ChainIdx < NChains; ++ChainIdx) {
3249 IVChain &Chain = IVChainVec[ChainIdx];
3250
3251 // Prune the solution space aggressively by checking that both IV operands
3252 // are expressions that operate on the same unscaled SCEVUnknown. This
3253 // "base" will be canceled by the subsequent getMinusSCEV call. Checking
3254 // first avoids creating extra SCEV expressions.
3255 if (!stressIVChain(Opts) && Chain.ExprBase != OperExprBase)
3256 continue;
3257
3258 Value *PrevIV = getWideOperand(Oper: Chain.Incs.back().IVOperand);
3259 if (PrevIV->getType() != NextIV->getType())
3260 continue;
3261
3262 // A phi node terminates a chain.
3263 if (isa<PHINode>(Val: UserInst) && isa<PHINode>(Val: Chain.tailUserInst()))
3264 continue;
3265
3266 // The increment must be loop-invariant so it can be kept in a register.
3267 const SCEV *PrevExpr = SE.getSCEV(V: PrevIV);
3268 const SCEV *IncExpr = SE.getMinusSCEV(LHS: OperExpr, RHS: PrevExpr);
3269 if (isa<SCEVCouldNotCompute>(Val: IncExpr) || !SE.isLoopInvariant(S: IncExpr, L))
3270 continue;
3271
3272 // Aggressively form chains when -stress-ivchain.
3273 if (stressIVChain(Opts) ||
3274 Chain.isProfitableIncrement(OperExpr, IncExpr, SE)) {
3275 LastIncExpr = IncExpr;
3276 break;
3277 }
3278 }
3279 // If we haven't found a chain, create a new one, unless we hit the max. Don't
3280 // bother for phi nodes, because they must be last in the chain.
3281 if (ChainIdx == NChains) {
3282 if (isa<PHINode>(Val: UserInst))
3283 return;
3284 if (NChains >= MaxChains && !stressIVChain(Opts)) {
3285 LLVM_DEBUG(dbgs() << "IV Chain Limit\n");
3286 return;
3287 }
3288 LastIncExpr = OperExpr;
3289 // IVUsers may have skipped over sign/zero extensions. We don't currently
3290 // attempt to form chains involving extensions unless they can be hoisted
3291 // into this loop's AddRec.
3292 if (!isa<SCEVAddRecExpr>(Val: LastIncExpr))
3293 return;
3294 ++NChains;
3295 IVChainVec.push_back(Elt: IVChain(IVInc(UserInst, IVOper, LastIncExpr),
3296 OperExprBase));
3297 ChainUsersVec.resize(N: NChains);
3298 LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Head: (" << *UserInst
3299 << ") IV=" << *LastIncExpr << "\n");
3300 } else {
3301 LLVM_DEBUG(dbgs() << "IV Chain#" << ChainIdx << " Inc: (" << *UserInst
3302 << ") IV+" << *LastIncExpr << "\n");
3303 // Add this IV user to the end of the chain.
3304 IVChainVec[ChainIdx].add(X: IVInc(UserInst, IVOper, LastIncExpr));
3305 }
3306 IVChain &Chain = IVChainVec[ChainIdx];
3307
3308 SmallPtrSet<Instruction*,4> &NearUsers = ChainUsersVec[ChainIdx].NearUsers;
3309 // This chain's NearUsers become FarUsers.
3310 if (!LastIncExpr->isZero()) {
3311 ChainUsersVec[ChainIdx].FarUsers.insert_range(R&: NearUsers);
3312 NearUsers.clear();
3313 }
3314
3315 // All other uses of IVOperand become near uses of the chain.
3316 // We currently ignore intermediate values within SCEV expressions, assuming
3317 // they will eventually be used be the current chain, or can be computed
3318 // from one of the chain increments. To be more precise we could
3319 // transitively follow its user and only add leaf IV users to the set.
3320 for (User *U : IVOper->users()) {
3321 Instruction *OtherUse = dyn_cast<Instruction>(Val: U);
3322 if (!OtherUse)
3323 continue;
3324 // Uses in the chain will no longer be uses if the chain is formed.
3325 // Include the head of the chain in this iteration (not Chain.begin()).
3326 IVChain::const_iterator IncIter = Chain.Incs.begin();
3327 IVChain::const_iterator IncEnd = Chain.Incs.end();
3328 for( ; IncIter != IncEnd; ++IncIter) {
3329 if (IncIter->UserInst == OtherUse)
3330 break;
3331 }
3332 if (IncIter != IncEnd)
3333 continue;
3334
3335 if (SE.isSCEVable(Ty: OtherUse->getType())
3336 && !isa<SCEVUnknown>(Val: SE.getSCEV(V: OtherUse))
3337 && IU.isIVUserOrOperand(Inst: OtherUse)) {
3338 continue;
3339 }
3340 NearUsers.insert(Ptr: OtherUse);
3341 }
3342
3343 // Since this user is part of the chain, it's no longer considered a use
3344 // of the chain.
3345 ChainUsersVec[ChainIdx].FarUsers.erase(Ptr: UserInst);
3346}
3347
3348/// Populate the vector of Chains.
3349///
3350/// This decreases ILP at the architecture level. Targets with ample registers,
3351/// multiple memory ports, and no register renaming probably don't want
3352/// this. However, such targets should probably disable LSR altogether.
3353///
3354/// The job of LSR is to make a reasonable choice of induction variables across
3355/// the loop. Subsequent passes can easily "unchain" computation exposing more
3356/// ILP *within the loop* if the target wants it.
3357///
3358/// Finding the best IV chain is potentially a scheduling problem. Since LSR
3359/// will not reorder memory operations, it will recognize this as a chain, but
3360/// will generate redundant IV increments. Ideally this would be corrected later
3361/// by a smart scheduler:
3362/// = A[i]
3363/// = A[i+x]
3364/// A[i] =
3365/// A[i+x] =
3366///
3367/// TODO: Walk the entire domtree within this loop, not just the path to the
3368/// loop latch. This will discover chains on side paths, but requires
3369/// maintaining multiple copies of the Chains state.
3370void LSRInstance::CollectChains() {
3371 LLVM_DEBUG(dbgs() << "Collecting IV Chains.\n");
3372 SmallVector<ChainUsers, 8> ChainUsersVec;
3373
3374 SmallVector<BasicBlock *,8> LatchPath;
3375 BasicBlock *LoopHeader = L->getHeader();
3376 for (DomTreeNode *Rung = DT.getNode(BB: L->getLoopLatch());
3377 Rung->getBlock() != LoopHeader; Rung = Rung->getIDom()) {
3378 LatchPath.push_back(Elt: Rung->getBlock());
3379 }
3380 LatchPath.push_back(Elt: LoopHeader);
3381
3382 // Walk the instruction stream from the loop header to the loop latch.
3383 for (BasicBlock *BB : reverse(C&: LatchPath)) {
3384 for (Instruction &I : *BB) {
3385 // Skip instructions that weren't seen by IVUsers analysis.
3386 if (isa<PHINode>(Val: I) || !IU.isIVUserOrOperand(Inst: &I))
3387 continue;
3388
3389 // Skip ephemeral values, as they don't produce real code.
3390 if (IU.isEphemeral(V: &I))
3391 continue;
3392
3393 // Ignore users that are part of a SCEV expression. This way we only
3394 // consider leaf IV Users. This effectively rediscovers a portion of
3395 // IVUsers analysis but in program order this time.
3396 if (SE.isSCEVable(Ty: I.getType()) && !isa<SCEVUnknown>(Val: SE.getSCEV(V: &I)))
3397 continue;
3398
3399 // Remove this instruction from any NearUsers set it may be in.
3400 for (unsigned ChainIdx = 0, NChains = IVChainVec.size();
3401 ChainIdx < NChains; ++ChainIdx) {
3402 ChainUsersVec[ChainIdx].NearUsers.erase(Ptr: &I);
3403 }
3404 // Search for operands that can be chained.
3405 SmallPtrSet<Instruction*, 4> UniqueOperands;
3406 User::op_iterator IVOpEnd = I.op_end();
3407 User::op_iterator IVOpIter = findIVOperand(OI: I.op_begin(), OE: IVOpEnd, L, SE);
3408 while (IVOpIter != IVOpEnd) {
3409 Instruction *IVOpInst = cast<Instruction>(Val&: *IVOpIter);
3410 if (UniqueOperands.insert(Ptr: IVOpInst).second)
3411 ChainInstruction(UserInst: &I, IVOper: IVOpInst, ChainUsersVec);
3412 IVOpIter = findIVOperand(OI: std::next(x: IVOpIter), OE: IVOpEnd, L, SE);
3413 }
3414 } // Continue walking down the instructions.
3415 } // Continue walking down the domtree.
3416 // Visit phi backedges to determine if the chain can generate the IV postinc.
3417 for (PHINode &PN : L->getHeader()->phis()) {
3418 if (!SE.isSCEVable(Ty: PN.getType()))
3419 continue;
3420
3421 Instruction *IncV =
3422 dyn_cast<Instruction>(Val: PN.getIncomingValueForBlock(BB: L->getLoopLatch()));
3423 if (IncV)
3424 ChainInstruction(UserInst: &PN, IVOper: IncV, ChainUsersVec);
3425 }
3426 // Remove any unprofitable chains.
3427 unsigned ChainIdx = 0;
3428 for (unsigned UsersIdx = 0, NChains = IVChainVec.size();
3429 UsersIdx < NChains; ++UsersIdx) {
3430 if (!stressIVChain(Opts) &&
3431 !isProfitableChain(Chain&: IVChainVec[UsersIdx],
3432 Users&: ChainUsersVec[UsersIdx].FarUsers, SE, TTI))
3433 continue;
3434 // Preserve the chain at UsesIdx.
3435 if (ChainIdx != UsersIdx)
3436 IVChainVec[ChainIdx] = IVChainVec[UsersIdx];
3437 FinalizeChain(Chain&: IVChainVec[ChainIdx]);
3438 ++ChainIdx;
3439 }
3440 IVChainVec.resize(N: ChainIdx);
3441}
3442
3443void LSRInstance::FinalizeChain(IVChain &Chain) {
3444 assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
3445 LLVM_DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n");
3446
3447 for (const IVInc &Inc : Chain) {
3448 LLVM_DEBUG(dbgs() << " Inc: " << *Inc.UserInst << "\n");
3449 auto UseI = find(Range: Inc.UserInst->operands(), Val: Inc.IVOperand);
3450 assert(UseI != Inc.UserInst->op_end() && "cannot find IV operand");
3451 IVIncSet.insert(Ptr: UseI);
3452 }
3453}
3454
3455/// Return true if the IVInc can be folded into an addressing mode.
3456static bool canFoldIVIncExpr(const ScalarOptions &Opts, const SCEV *IncExpr,
3457 Instruction *UserInst, Value *Operand,
3458 const TargetTransformInfo &TTI) {
3459 const SCEVConstant *IncConst = dyn_cast<SCEVConstant>(Val: IncExpr);
3460 Immediate IncOffset = Immediate::getZero();
3461 if (IncConst) {
3462 if (IncConst && IncConst->getAPInt().getSignificantBits() > 64)
3463 return false;
3464 IncOffset = Immediate::getFixed(MinVal: IncConst->getValue()->getSExtValue());
3465 } else {
3466 // Look for mul(vscale, constant), to detect a scalable offset.
3467 const APInt *C;
3468 if (!match(S: IncExpr, P: m_scev_Mul(Op0: m_scev_APInt(C), Op1: m_SCEVVScale())) ||
3469 C->getSignificantBits() > 64)
3470 return false;
3471 IncOffset = Immediate::getScalable(MinVal: C->getSExtValue());
3472 }
3473
3474 if (!isAddressUse(TTI, Inst: UserInst, OperandVal: Operand))
3475 return false;
3476
3477 MemAccessTy AccessTy = getAccessType(TTI, Inst: UserInst, OperandVal: Operand);
3478 if (!isAlwaysFoldable(Opts, TTI, Kind: LSRUse::Address, AccessTy,
3479 /*BaseGV=*/nullptr, BaseOffset: IncOffset, /*HasBaseReg=*/false))
3480 return false;
3481
3482 return true;
3483}
3484
3485/// Generate an add or subtract for each IVInc in a chain to materialize the IV
3486/// user's operand from the previous IV user's operand.
3487void LSRInstance::GenerateIVChain(const IVChain &Chain,
3488 SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
3489 // Find the new IVOperand for the head of the chain. It may have been replaced
3490 // by LSR.
3491 const IVInc &Head = Chain.Incs[0];
3492 User::op_iterator IVOpEnd = Head.UserInst->op_end();
3493 // findIVOperand returns IVOpEnd if it can no longer find a valid IV user.
3494 User::op_iterator IVOpIter = findIVOperand(OI: Head.UserInst->op_begin(),
3495 OE: IVOpEnd, L, SE);
3496 Value *IVSrc = nullptr;
3497 while (IVOpIter != IVOpEnd) {
3498 IVSrc = getWideOperand(Oper: *IVOpIter);
3499
3500 // If this operand computes the expression that the chain needs, we may use
3501 // it. (Check this after setting IVSrc which is used below.)
3502 //
3503 // Note that if Head.IncExpr is wider than IVSrc, then this phi is too
3504 // narrow for the chain, so we can no longer use it. We do allow using a
3505 // wider phi, assuming the LSR checked for free truncation. In that case we
3506 // should already have a truncate on this operand such that
3507 // getSCEV(IVSrc) == IncExpr.
3508 if (SE.getSCEV(V: *IVOpIter) == Head.IncExpr
3509 || SE.getSCEV(V: IVSrc) == Head.IncExpr) {
3510 break;
3511 }
3512 IVOpIter = findIVOperand(OI: std::next(x: IVOpIter), OE: IVOpEnd, L, SE);
3513 }
3514 if (IVOpIter == IVOpEnd) {
3515 // Gracefully give up on this chain.
3516 LLVM_DEBUG(dbgs() << "Concealed chain head: " << *Head.UserInst << "\n");
3517 return;
3518 }
3519 assert(IVSrc && "Failed to find IV chain source");
3520
3521 LLVM_DEBUG(dbgs() << "Generate chain at: " << *IVSrc << "\n");
3522 Type *IVTy = IVSrc->getType();
3523 Type *IntTy = SE.getEffectiveSCEVType(Ty: IVTy);
3524 const SCEV *LeftOverExpr = nullptr;
3525 const SCEV *Accum = SE.getZero(Ty: IntTy);
3526 SmallVector<std::pair<const SCEV *, Value *>> Bases;
3527 Bases.emplace_back(Args&: Accum, Args&: IVSrc);
3528
3529 for (const IVInc &Inc : Chain) {
3530 Instruction *InsertPt = Inc.UserInst;
3531 if (isa<PHINode>(Val: InsertPt))
3532 InsertPt = L->getLoopLatch()->getTerminator();
3533
3534 // IVOper will replace the current IV User's operand. IVSrc is the IV
3535 // value currently held in a register.
3536 Value *IVOper = IVSrc;
3537 if (!Inc.IncExpr->isZero()) {
3538 // IncExpr was the result of subtraction of two narrow values, so must
3539 // be signed.
3540 const SCEV *IncExpr = SE.getNoopOrSignExtend(V: Inc.IncExpr, Ty: IntTy);
3541 Accum = SE.getAddExpr(LHS: Accum, RHS: IncExpr);
3542 LeftOverExpr = LeftOverExpr
3543 ? SE.getAddExpr(LHS: LeftOverExpr, RHS: IncExpr).getPointer()
3544 : IncExpr;
3545 }
3546
3547 // Look through each base to see if any can produce a nice addressing mode.
3548 bool FoundBase = false;
3549 for (auto [MapScev, MapIVOper] : reverse(C&: Bases)) {
3550 const SCEV *Remainder = SE.getMinusSCEV(LHS: Accum, RHS: MapScev);
3551 if (canFoldIVIncExpr(Opts, IncExpr: Remainder, UserInst: Inc.UserInst, Operand: Inc.IVOperand, TTI)) {
3552 if (!Remainder->isZero()) {
3553 Rewriter.clearPostInc();
3554 Value *IncV = Rewriter.expandCodeFor(SH: Remainder, Ty: IntTy, I: InsertPt);
3555 const SCEV *IVOperExpr =
3556 SE.getAddExpr(LHS: SE.getUnknown(V: MapIVOper), RHS: SE.getUnknown(V: IncV));
3557 IVOper = Rewriter.expandCodeFor(SH: IVOperExpr, Ty: IVTy, I: InsertPt);
3558 } else {
3559 IVOper = MapIVOper;
3560 }
3561
3562 FoundBase = true;
3563 break;
3564 }
3565 }
3566 if (!FoundBase && LeftOverExpr && !LeftOverExpr->isZero()) {
3567 // Expand the IV increment.
3568 Rewriter.clearPostInc();
3569 Value *IncV = Rewriter.expandCodeFor(SH: LeftOverExpr, Ty: IntTy, I: InsertPt);
3570 const SCEV *IVOperExpr = SE.getAddExpr(LHS: SE.getUnknown(V: IVSrc),
3571 RHS: SE.getUnknown(V: IncV));
3572 IVOper = Rewriter.expandCodeFor(SH: IVOperExpr, Ty: IVTy, I: InsertPt);
3573
3574 // If an IV increment can't be folded, use it as the next IV value.
3575 if (!canFoldIVIncExpr(Opts, IncExpr: LeftOverExpr, UserInst: Inc.UserInst, Operand: Inc.IVOperand,
3576 TTI)) {
3577 assert(IVTy == IVOper->getType() && "inconsistent IV increment type");
3578 Bases.emplace_back(Args&: Accum, Args&: IVOper);
3579 IVSrc = IVOper;
3580 LeftOverExpr = nullptr;
3581 }
3582 }
3583 Type *OperTy = Inc.IVOperand->getType();
3584 if (IVTy != OperTy) {
3585 assert(SE.getTypeSizeInBits(IVTy) >= SE.getTypeSizeInBits(OperTy) &&
3586 "cannot extend a chained IV");
3587 IRBuilder<> Builder(InsertPt);
3588 IVOper = Builder.CreateTruncOrBitCast(V: IVOper, DestTy: OperTy, Name: "lsr.chain");
3589 }
3590 Inc.UserInst->replaceUsesOfWith(From: Inc.IVOperand, To: IVOper);
3591 if (auto *OperandIsInstr = dyn_cast<Instruction>(Val: Inc.IVOperand))
3592 DeadInsts.emplace_back(Args&: OperandIsInstr);
3593 }
3594 // If LSR created a new, wider phi, we may also replace its postinc. We only
3595 // do this if we also found a wide value for the head of the chain.
3596 if (isa<PHINode>(Val: Chain.tailUserInst())) {
3597 for (PHINode &Phi : L->getHeader()->phis()) {
3598 if (Phi.getType() != IVSrc->getType())
3599 continue;
3600 Instruction *PostIncV = dyn_cast<Instruction>(
3601 Val: Phi.getIncomingValueForBlock(BB: L->getLoopLatch()));
3602 if (!PostIncV || (SE.getSCEV(V: PostIncV) != SE.getSCEV(V: IVSrc)))
3603 continue;
3604 Value *IVOper = IVSrc;
3605 Type *PostIncTy = PostIncV->getType();
3606 if (IVTy != PostIncTy) {
3607 assert(PostIncTy->isPointerTy() && "mixing int/ptr IV types");
3608 IRBuilder<> Builder(L->getLoopLatch()->getTerminator());
3609 Builder.SetCurrentDebugLocation(PostIncV->getDebugLoc());
3610 IVOper = Builder.CreatePointerCast(V: IVSrc, DestTy: PostIncTy, Name: "lsr.chain");
3611 }
3612 Phi.replaceUsesOfWith(From: PostIncV, To: IVOper);
3613 DeadInsts.emplace_back(Args&: PostIncV);
3614 }
3615 }
3616}
3617
3618void LSRInstance::CollectFixupsAndInitialFormulae() {
3619 CondBrInst *ExitBranch = nullptr;
3620 bool SaveCmp = TTI.canSaveCmp(L, BI: &ExitBranch, SE: &SE, LI: &LI, DT: &DT, AC: &AC, LibInfo: &TLI);
3621
3622 // For calculating baseline cost
3623 SmallPtrSet<const SCEV *, 16> Regs;
3624 DenseSet<const SCEV *> VisitedRegs;
3625 DenseSet<size_t> VisitedLSRUse;
3626
3627 for (const IVStrideUse &U : IU) {
3628 Instruction *UserInst = U.getUser();
3629 // Skip IV users that are part of profitable IV Chains.
3630 User::op_iterator UseI =
3631 find(Range: UserInst->operands(), Val: U.getOperandValToReplace());
3632 assert(UseI != UserInst->op_end() && "cannot find IV operand");
3633 if (IVIncSet.count(Ptr: UseI)) {
3634 LLVM_DEBUG(dbgs() << "Use is in profitable chain: " << **UseI << '\n');
3635 continue;
3636 }
3637
3638 LSRUse::KindType Kind = LSRUse::Basic;
3639 MemAccessTy AccessTy;
3640 if (isAddressUse(TTI, Inst: UserInst, OperandVal: U.getOperandValToReplace())) {
3641 Kind = LSRUse::Address;
3642 AccessTy = getAccessType(TTI, Inst: UserInst, OperandVal: U.getOperandValToReplace());
3643 }
3644
3645 const SCEV *S = IU.getExpr(IU: U);
3646 if (!S)
3647 continue;
3648 PostIncLoopSet TmpPostIncLoops = U.getPostIncLoops();
3649
3650 // Equality (== and !=) ICmps are special. We can rewrite (i == N) as
3651 // (N - i == 0), and this allows (N - i) to be the expression that we work
3652 // with rather than just N or i, so we can consider the register
3653 // requirements for both N and i at the same time. Limiting this code to
3654 // equality icmps is not a problem because all interesting loops use
3655 // equality icmps, thanks to IndVarSimplify.
3656 if (ICmpInst *CI = dyn_cast<ICmpInst>(Val: UserInst)) {
3657 // If CI can be saved in some target, like replaced inside hardware loop
3658 // in PowerPC, no need to generate initial formulae for it.
3659 if (SaveCmp && CI == dyn_cast<ICmpInst>(Val: ExitBranch->getCondition()))
3660 continue;
3661 if (CI->isEquality()) {
3662 // Swap the operands if needed to put the OperandValToReplace on the
3663 // left, for consistency.
3664 Value *NV = CI->getOperand(i_nocapture: 1);
3665 if (NV == U.getOperandValToReplace()) {
3666 CI->setOperand(i_nocapture: 1, Val_nocapture: CI->getOperand(i_nocapture: 0));
3667 CI->setOperand(i_nocapture: 0, Val_nocapture: NV);
3668 NV = CI->getOperand(i_nocapture: 1);
3669 Changed = true;
3670 }
3671
3672 // x == y --> x - y == 0
3673 const SCEV *N = SE.getSCEV(V: NV);
3674 if (SE.isLoopInvariant(S: N, L) && Rewriter.isSafeToExpand(S: N) &&
3675 (!NV->getType()->isPointerTy() ||
3676 SE.getPointerBase(V: N) == SE.getPointerBase(V: S))) {
3677 // S is normalized, so normalize N before folding it into S
3678 // to keep the result normalized.
3679 N = normalizeForPostIncUse(S: N, Loops: TmpPostIncLoops, SE);
3680 if (!N)
3681 continue;
3682 Kind = LSRUse::ICmpZero;
3683 S = SE.getMinusSCEV(LHS: N, RHS: S);
3684 } else if (L->isLoopInvariant(V: NV) &&
3685 (!isa<Instruction>(Val: NV) ||
3686 DT.dominates(Def: cast<Instruction>(Val: NV), BB: L->getHeader())) &&
3687 !NV->getType()->isPointerTy()) {
3688 // If we can't generally expand the expression (e.g. it contains
3689 // a divide), but it is already at a loop invariant point before the
3690 // loop, wrap it in an unknown (to prevent the expander from trying
3691 // to re-expand in a potentially unsafe way.) The restriction to
3692 // integer types is required because the unknown hides the base, and
3693 // SCEV can't compute the difference of two unknown pointers.
3694 N = SE.getUnknown(V: NV);
3695 N = normalizeForPostIncUse(S: N, Loops: TmpPostIncLoops, SE);
3696 if (!N)
3697 continue;
3698 Kind = LSRUse::ICmpZero;
3699 S = SE.getMinusSCEV(LHS: N, RHS: S);
3700 assert(!isa<SCEVCouldNotCompute>(S));
3701 }
3702
3703 // -1 and the negations of all interesting strides (except the negation
3704 // of -1) are now also interesting.
3705 for (size_t i = 0, e = Factors.size(); i != e; ++i)
3706 if (Factors[i] != -1)
3707 Factors.insert(X: -(uint64_t)Factors[i]);
3708 Factors.insert(X: -1);
3709 }
3710 }
3711
3712 // Get or create an LSRUse.
3713 std::pair<size_t, Immediate> P = getUse(Expr&: S, Kind, AccessTy);
3714 size_t LUIdx = P.first;
3715 Immediate Offset = P.second;
3716 LSRUse &LU = Uses[LUIdx];
3717
3718 // Record the fixup.
3719 LSRFixup &LF = LU.getNewFixup();
3720 LF.UserInst = UserInst;
3721 LF.OperandValToReplace = U.getOperandValToReplace();
3722 LF.PostIncLoops = TmpPostIncLoops;
3723 LF.Offset = Offset;
3724 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3725 LU.AllFixupsUnconditional &= IsFixupExecutedEachIncrement(LF);
3726
3727 // Create SCEV as Formula for calculating baseline cost
3728 if (!VisitedLSRUse.count(V: LUIdx) && !LF.isUseFullyOutsideLoop(L)) {
3729 Formula F;
3730 F.initialMatch(S, L, SE);
3731 BaselineCost.RateFormula(F, Regs, VisitedRegs, LU,
3732 HardwareLoopProfitable);
3733 VisitedLSRUse.insert(V: LUIdx);
3734 }
3735
3736 // If this is the first use of this LSRUse, give it a formula.
3737 if (LU.Formulae.empty()) {
3738 InsertInitialFormula(S, LU, LUIdx);
3739 CountRegisters(F: LU.Formulae.back(), LUIdx);
3740 }
3741 }
3742
3743 LLVM_DEBUG(print_fixups(dbgs()));
3744}
3745
3746/// Insert a formula for the given expression into the given use, separating out
3747/// loop-variant portions from loop-invariant and loop-computable portions.
3748void LSRInstance::InsertInitialFormula(const SCEV *S, LSRUse &LU,
3749 size_t LUIdx) {
3750 // Mark uses whose expressions cannot be expanded.
3751 if (!Rewriter.isSafeToExpand(S))
3752 LU.RigidFormula = true;
3753
3754 Formula F;
3755 F.initialMatch(S, L, SE);
3756 bool Inserted = InsertFormula(LU, LUIdx, F);
3757 assert(Inserted && "Initial formula already exists!"); (void)Inserted;
3758}
3759
3760/// Insert a simple single-register formula for the given expression into the
3761/// given use.
3762void
3763LSRInstance::InsertSupplementalFormula(const SCEV *S,
3764 LSRUse &LU, size_t LUIdx) {
3765 Formula F;
3766 F.BaseRegs.push_back(Elt: S);
3767 F.HasBaseReg = true;
3768 bool Inserted = InsertFormula(LU, LUIdx, F);
3769 assert(Inserted && "Supplemental formula already exists!"); (void)Inserted;
3770}
3771
3772/// Note which registers are used by the given formula, updating RegUses.
3773void LSRInstance::CountRegisters(const Formula &F, size_t LUIdx) {
3774 if (F.ScaledReg)
3775 RegUses.countRegister(Reg: F.ScaledReg, LUIdx);
3776 for (const SCEV *BaseReg : F.BaseRegs)
3777 RegUses.countRegister(Reg: BaseReg, LUIdx);
3778}
3779
3780/// If the given formula has not yet been inserted, add it to the list, and
3781/// return true. Return false otherwise.
3782bool LSRInstance::InsertFormula(LSRUse &LU, unsigned LUIdx, const Formula &F) {
3783 // Do not insert formula that we will not be able to expand.
3784 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F) &&
3785 "Formula is illegal");
3786
3787 if (!LU.InsertFormula(F, L: *L))
3788 return false;
3789
3790 CountRegisters(F, LUIdx);
3791 return true;
3792}
3793
3794/// Test whether this fixup will be executed each time the corresponding IV
3795/// increment instruction is executed.
3796bool LSRInstance::IsFixupExecutedEachIncrement(const LSRFixup &LF) const {
3797 // If the fixup block dominates the IV increment block then there is no path
3798 // through the loop to the increment that doesn't pass through the fixup.
3799 return DT.dominates(A: LF.UserInst->getParent(), B: IVIncInsertPos->getParent());
3800}
3801
3802/// Check for other uses of loop-invariant values which we're tracking. These
3803/// other uses will pin these values in registers, making them less profitable
3804/// for elimination.
3805/// TODO: This currently misses non-constant addrec step registers.
3806/// TODO: Should this give more weight to users inside the loop?
3807void
3808LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
3809 SmallVector<const SCEV *, 8> Worklist(RegUses.begin(), RegUses.end());
3810 SmallPtrSet<const SCEV *, 32> Visited;
3811
3812 // Don't collect outside uses if we are favoring postinc - the instructions in
3813 // the loop are more important than the ones outside of it.
3814 if (AMK == TTI::AMK_PostIndexed)
3815 return;
3816
3817 while (!Worklist.empty()) {
3818 const SCEV *S = Worklist.pop_back_val();
3819
3820 // Don't process the same SCEV twice
3821 if (!Visited.insert(Ptr: S).second)
3822 continue;
3823
3824 if (const SCEVNAryExpr *N = dyn_cast<SCEVNAryExpr>(Val: S))
3825 append_range(C&: Worklist, R: N->operands());
3826 else if (const SCEVIntegralCastExpr *C = dyn_cast<SCEVIntegralCastExpr>(Val: S))
3827 Worklist.push_back(Elt: C->getOperand());
3828 else if (const SCEVUDivExpr *D = dyn_cast<SCEVUDivExpr>(Val: S)) {
3829 Worklist.push_back(Elt: D->getLHS());
3830 Worklist.push_back(Elt: D->getRHS());
3831 } else if (const SCEVUnknown *US = dyn_cast<SCEVUnknown>(Val: S)) {
3832 const Value *V = US->getValue();
3833 if (const Instruction *Inst = dyn_cast<Instruction>(Val: V)) {
3834 // Look for instructions defined outside the loop.
3835 if (L->contains(Inst)) continue;
3836 } else if (isa<Constant>(Val: V))
3837 // Constants can be re-materialized.
3838 continue;
3839 for (const Use &U : V->uses()) {
3840 const Instruction *UserInst = dyn_cast<Instruction>(Val: U.getUser());
3841 // Ignore non-instructions.
3842 if (!UserInst)
3843 continue;
3844 // Don't bother if the instruction is an EHPad.
3845 if (UserInst->isEHPad())
3846 continue;
3847 // Ignore instructions in other functions (as can happen with
3848 // Constants).
3849 if (UserInst->getParent()->getParent() != L->getHeader()->getParent())
3850 continue;
3851 // Ignore instructions not dominated by the loop.
3852 const BasicBlock *UseBB = !isa<PHINode>(Val: UserInst) ?
3853 UserInst->getParent() :
3854 cast<PHINode>(Val: UserInst)->getIncomingBlock(
3855 i: PHINode::getIncomingValueNumForOperand(i: U.getOperandNo()));
3856 if (!DT.dominates(A: L->getHeader(), B: UseBB))
3857 continue;
3858 // Don't bother if the instruction is in a BB which ends in an EHPad.
3859 if (UseBB->getTerminator()->isEHPad())
3860 continue;
3861
3862 // Ignore cases in which the currently-examined value could come from
3863 // a basic block terminated with an EHPad. This checks all incoming
3864 // blocks of the phi node since it is possible that the same incoming
3865 // value comes from multiple basic blocks, only some of which may end
3866 // in an EHPad. If any of them do, a subsequent rewrite attempt by this
3867 // pass would try to insert instructions into an EHPad, hitting an
3868 // assertion.
3869 if (isa<PHINode>(Val: UserInst)) {
3870 const auto *PhiNode = cast<PHINode>(Val: UserInst);
3871 bool HasIncompatibleEHPTerminatedBlock = false;
3872 llvm::Value *ExpectedValue = U;
3873 for (unsigned int I = 0; I < PhiNode->getNumIncomingValues(); I++) {
3874 if (PhiNode->getIncomingValue(i: I) == ExpectedValue) {
3875 if (PhiNode->getIncomingBlock(i: I)->getTerminator()->isEHPad()) {
3876 HasIncompatibleEHPTerminatedBlock = true;
3877 break;
3878 }
3879 }
3880 }
3881 if (HasIncompatibleEHPTerminatedBlock) {
3882 continue;
3883 }
3884 }
3885
3886 // Don't bother rewriting PHIs in catchswitch blocks.
3887 if (isa<CatchSwitchInst>(Val: UserInst->getParent()->getTerminator()))
3888 continue;
3889 // Ignore uses which are part of other SCEV expressions, to avoid
3890 // analyzing them multiple times.
3891 if (SE.isSCEVable(Ty: UserInst->getType())) {
3892 const SCEV *UserS = SE.getSCEV(V: const_cast<Instruction *>(UserInst));
3893 // If the user is a no-op, look through to its uses.
3894 if (!isa<SCEVUnknown>(Val: UserS))
3895 continue;
3896 if (UserS == US) {
3897 Worklist.push_back(
3898 Elt: SE.getUnknown(V: const_cast<Instruction *>(UserInst)));
3899 continue;
3900 }
3901 }
3902 // Ignore icmp instructions which are already being analyzed.
3903 if (const ICmpInst *ICI = dyn_cast<ICmpInst>(Val: UserInst)) {
3904 unsigned OtherIdx = !U.getOperandNo();
3905 Value *OtherOp = ICI->getOperand(i_nocapture: OtherIdx);
3906 if (SE.hasComputableLoopEvolution(S: SE.getSCEV(V: OtherOp), L))
3907 continue;
3908 }
3909
3910 // Do not consider uses inside lifetime intrinsics. These are not
3911 // actually materialized.
3912 if (UserInst->isLifetimeStartOrEnd())
3913 continue;
3914
3915 std::pair<size_t, Immediate> P =
3916 getUse(Expr&: S, Kind: LSRUse::Basic, AccessTy: MemAccessTy());
3917 size_t LUIdx = P.first;
3918 Immediate Offset = P.second;
3919 LSRUse &LU = Uses[LUIdx];
3920 LSRFixup &LF = LU.getNewFixup();
3921 LF.UserInst = const_cast<Instruction *>(UserInst);
3922 LF.OperandValToReplace = U;
3923 LF.Offset = Offset;
3924 LU.AllFixupsOutsideLoop &= LF.isUseFullyOutsideLoop(L);
3925 LU.AllFixupsUnconditional &= IsFixupExecutedEachIncrement(LF);
3926 InsertSupplementalFormula(S: US, LU, LUIdx);
3927 CountRegisters(F: LU.Formulae.back(), LUIdx: Uses.size() - 1);
3928 break;
3929 }
3930 }
3931 }
3932}
3933
3934/// Split S into subexpressions which can be pulled out into separate
3935/// registers. If C is non-null, multiply each subexpression by C.
3936///
3937/// Return remainder expression after factoring the subexpressions captured by
3938/// Ops. If Ops is complete, return NULL.
3939static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C,
3940 SmallVectorImpl<const SCEV *> &Ops,
3941 const Loop *L,
3942 ScalarEvolution &SE,
3943 unsigned Depth = 0) {
3944 // Arbitrarily cap recursion to protect compile time.
3945 if (Depth >= 3)
3946 return S;
3947
3948 if (const SCEVAddExpr *Add = dyn_cast<SCEVAddExpr>(Val: S)) {
3949 // Break out add operands.
3950 for (const SCEV *S : Add->operands()) {
3951 const SCEV *Remainder = CollectSubexprs(S, C, Ops, L, SE, Depth: Depth+1);
3952 if (Remainder)
3953 Ops.push_back(Elt: C ? SE.getMulExpr(LHS: C, RHS: Remainder).getPointer() : Remainder);
3954 }
3955 return nullptr;
3956 }
3957 const SCEV *Start, *Step;
3958 const SCEVConstant *Op0;
3959 const SCEV *Op1;
3960 if (match(S, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step)))) {
3961 // Split a non-zero base out of an addrec.
3962 if (Start->isZero())
3963 return S;
3964
3965 const SCEV *Remainder = CollectSubexprs(S: Start, C, Ops, L, SE, Depth: Depth + 1);
3966 // Split the non-zero AddRec unless it is part of a nested recurrence that
3967 // does not pertain to this loop.
3968 if (Remainder && (cast<SCEVAddRecExpr>(Val: S)->getLoop() == L ||
3969 !isa<SCEVAddRecExpr>(Val: Remainder))) {
3970 Ops.push_back(Elt: C ? SE.getMulExpr(LHS: C, RHS: Remainder).getPointer() : Remainder);
3971 Remainder = nullptr;
3972 }
3973 if (Remainder != Start) {
3974 if (!Remainder)
3975 Remainder = SE.getConstant(Ty: S->getType(), V: 0);
3976 return SE.getAddRecExpr(Start: Remainder, Step,
3977 L: cast<SCEVAddRecExpr>(Val: S)->getLoop(),
3978 // FIXME: AR->getNoWrapFlags(SCEV::FlagNW)
3979 Flags: SCEV::FlagNone);
3980 }
3981 } else if (match(S, P: m_scev_Mul(Op0: m_SCEVConstant(V&: Op0), Op1: m_SCEV(V&: Op1)))) {
3982 // Break (C * (a + b + c)) into C*a + C*b + C*c.
3983 C = C ? cast<SCEVConstant>(Val: SE.getMulExpr(LHS: C, RHS: Op0)) : Op0;
3984 const SCEV *Remainder = CollectSubexprs(S: Op1, C, Ops, L, SE, Depth: Depth + 1);
3985 if (Remainder)
3986 Ops.push_back(Elt: SE.getMulExpr(LHS: C, RHS: Remainder));
3987 return nullptr;
3988 }
3989 return S;
3990}
3991
3992/// Return true if the SCEV represents a value that may end up as a
3993/// post-increment operation.
3994static bool mayUsePostIncMode(const TargetTransformInfo &TTI,
3995 LSRUse &LU, const SCEV *S, const Loop *L,
3996 ScalarEvolution &SE) {
3997 if (LU.Kind != LSRUse::Address ||
3998 !LU.AccessTy.getType()->isIntOrIntVectorTy())
3999 return false;
4000 const SCEV *Start;
4001 if (!match(S, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEVConstant())))
4002 return false;
4003 // Check if a post-indexed load/store can be used.
4004 if (TTI.isIndexedLoadLegal(Mode: TTI.MIM_PostInc, Ty: S->getType()) ||
4005 TTI.isIndexedStoreLegal(Mode: TTI.MIM_PostInc, Ty: S->getType())) {
4006 if (!isa<SCEVConstant>(Val: Start) && SE.isLoopInvariant(S: Start, L))
4007 return true;
4008 }
4009 return false;
4010}
4011
4012/// Helper function for LSRInstance::GenerateReassociations.
4013void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
4014 const Formula &Base,
4015 unsigned Depth, size_t Idx,
4016 bool IsScaledReg) {
4017 const SCEV *BaseReg = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
4018 // Don't generate reassociations for the base register of a value that
4019 // may generate a post-increment operator. The reason is that the
4020 // reassociations cause extra base+register formula to be created,
4021 // and possibly chosen, but the post-increment is more efficient.
4022 if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, S: BaseReg, L, SE))
4023 return;
4024 SmallVector<const SCEV *, 8> AddOps;
4025 const SCEV *Remainder = CollectSubexprs(S: BaseReg, C: nullptr, Ops&: AddOps, L, SE);
4026 if (Remainder)
4027 AddOps.push_back(Elt: Remainder);
4028
4029 if (AddOps.size() == 1)
4030 return;
4031
4032 for (SmallVectorImpl<const SCEV *>::const_iterator J = AddOps.begin(),
4033 JE = AddOps.end();
4034 J != JE; ++J) {
4035 // Loop-variant "unknown" values are uninteresting; we won't be able to
4036 // do anything meaningful with them.
4037 if (isa<SCEVUnknown>(Val: *J) && !SE.isLoopInvariant(S: *J, L))
4038 continue;
4039
4040 // Don't pull a constant into a register if the constant could be folded
4041 // into an immediate field.
4042 if (isAlwaysFoldable(Opts, TTI, SE, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
4043 AccessTy: LU.AccessTy, S: *J, HasBaseReg: Base.getNumRegs() > 1))
4044 continue;
4045
4046 // Collect all operands except *J.
4047 SmallVector<SCEVUse, 8> InnerAddOps(std::as_const(t&: AddOps).begin(), J);
4048 InnerAddOps.append(in_start: std::next(x: J), in_end: std::as_const(t&: AddOps).end());
4049
4050 // Don't leave just a constant behind in a register if the constant could
4051 // be folded into an immediate field.
4052 if (InnerAddOps.size() == 1 &&
4053 isAlwaysFoldable(Opts, TTI, SE, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind,
4054 AccessTy: LU.AccessTy, S: InnerAddOps[0], HasBaseReg: Base.getNumRegs() > 1))
4055 continue;
4056
4057 const SCEV *InnerSum = SE.getAddExpr(Ops&: InnerAddOps);
4058 if (InnerSum->isZero())
4059 continue;
4060 Formula F = Base;
4061
4062 if (F.UnfoldedOffset.isNonZero() && F.UnfoldedOffset.isScalable())
4063 continue;
4064
4065 // Add the remaining pieces of the add back into the new formula.
4066 const SCEVConstant *InnerSumSC = dyn_cast<SCEVConstant>(Val: InnerSum);
4067 if (InnerSumSC && SE.getTypeSizeInBits(Ty: InnerSumSC->getType()) <= 64 &&
4068 TTI.isLegalAddImmediate(Imm: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4069 InnerSumSC->getValue()->getZExtValue())) {
4070 F.UnfoldedOffset =
4071 Immediate::getFixed(MinVal: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4072 InnerSumSC->getValue()->getZExtValue());
4073 if (IsScaledReg) {
4074 F.ScaledReg = nullptr;
4075 F.Scale = 0;
4076 } else
4077 F.BaseRegs.erase(CI: F.BaseRegs.begin() + Idx);
4078 } else if (IsScaledReg)
4079 F.ScaledReg = InnerSum;
4080 else
4081 F.BaseRegs[Idx] = InnerSum;
4082
4083 // Add J as its own register, or an unfolded immediate.
4084 const SCEVConstant *SC = dyn_cast<SCEVConstant>(Val: *J);
4085 if (SC && SE.getTypeSizeInBits(Ty: SC->getType()) <= 64 &&
4086 TTI.isLegalAddImmediate(Imm: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4087 SC->getValue()->getZExtValue()))
4088 F.UnfoldedOffset =
4089 Immediate::getFixed(MinVal: (uint64_t)F.UnfoldedOffset.getFixedValue() +
4090 SC->getValue()->getZExtValue());
4091 else
4092 F.BaseRegs.push_back(Elt: *J);
4093 // We may have changed the number of register in base regs, adjust the
4094 // formula accordingly.
4095 F.canonicalize(L: *L);
4096
4097 if (InsertFormula(LU, LUIdx, F))
4098 // If that formula hadn't been seen before, recurse to find more like
4099 // it.
4100 // Add check on Log16(AddOps.size()) - same as Log2_32(AddOps.size()) >> 2)
4101 // Because just Depth is not enough to bound compile time.
4102 // This means that every time AddOps.size() is greater 16^x we will add
4103 // x to Depth.
4104 GenerateReassociations(LU, LUIdx, Base: LU.Formulae.back(),
4105 Depth: Depth + 1 + (Log2_32(Value: AddOps.size()) >> 2));
4106 }
4107}
4108
4109/// Split out subexpressions from adds and the bases of addrecs.
4110void LSRInstance::GenerateReassociations(LSRUse &LU, unsigned LUIdx,
4111 Formula Base, unsigned Depth) {
4112 assert(Base.isCanonical(*L) && "Input must be in the canonical form");
4113 // Arbitrarily cap recursion to protect compile time.
4114 if (Depth >= 3)
4115 return;
4116
4117 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
4118 GenerateReassociationsImpl(LU, LUIdx, Base, Depth, Idx: i);
4119
4120 if (Base.Scale == 1)
4121 GenerateReassociationsImpl(LU, LUIdx, Base, Depth,
4122 /* Idx */ -1, /* IsScaledReg */ true);
4123}
4124
4125/// Generate a formula consisting of all of the loop-dominating registers added
4126/// into a single register.
4127void LSRInstance::GenerateCombinations(LSRUse &LU, unsigned LUIdx,
4128 Formula Base) {
4129 // This method is only interesting on a plurality of registers.
4130 if (Base.BaseRegs.size() + (Base.Scale == 1) +
4131 (Base.UnfoldedOffset.isNonZero()) <=
4132 1)
4133 return;
4134
4135 // Flatten the representation, i.e., reg1 + 1*reg2 => reg1 + reg2, before
4136 // processing the formula.
4137 Base.unscale();
4138 SmallVector<SCEVUse, 4> Ops;
4139 Formula NewBase = Base;
4140 NewBase.BaseRegs.clear();
4141 Type *CombinedIntegerType = nullptr;
4142 for (const SCEV *BaseReg : Base.BaseRegs) {
4143 if (SE.properlyDominates(S: BaseReg, BB: L->getHeader()) &&
4144 !SE.hasComputableLoopEvolution(S: BaseReg, L)) {
4145 if (!CombinedIntegerType)
4146 CombinedIntegerType = SE.getEffectiveSCEVType(Ty: BaseReg->getType());
4147 Ops.push_back(Elt: BaseReg);
4148 }
4149 else
4150 NewBase.BaseRegs.push_back(Elt: BaseReg);
4151 }
4152
4153 // If no register is relevant, we're done.
4154 if (Ops.size() == 0)
4155 return;
4156
4157 // Utility function for generating the required variants of the combined
4158 // registers.
4159 auto GenerateFormula = [&](const SCEV *Sum) {
4160 Formula F = NewBase;
4161
4162 // TODO: If Sum is zero, it probably means ScalarEvolution missed an
4163 // opportunity to fold something. For now, just ignore such cases
4164 // rather than proceed with zero in a register.
4165 if (Sum->isZero())
4166 return;
4167
4168 F.BaseRegs.push_back(Elt: Sum);
4169 F.canonicalize(L: *L);
4170 (void)InsertFormula(LU, LUIdx, F);
4171 };
4172
4173 // If we collected at least two registers, generate a formula combining them.
4174 if (Ops.size() > 1) {
4175 SmallVector<SCEVUse, 4> OpsCopy(Ops); // Don't let SE modify Ops.
4176 GenerateFormula(SE.getAddExpr(Ops&: OpsCopy));
4177 }
4178
4179 // If we have an unfolded offset, generate a formula combining it with the
4180 // registers collected.
4181 if (NewBase.UnfoldedOffset.isNonZero() && NewBase.UnfoldedOffset.isFixed()) {
4182 assert(CombinedIntegerType && "Missing a type for the unfolded offset");
4183 Ops.push_back(Elt: SE.getConstant(Ty: CombinedIntegerType,
4184 V: NewBase.UnfoldedOffset.getFixedValue(), isSigned: true));
4185 NewBase.UnfoldedOffset = Immediate::getFixed(MinVal: 0);
4186 GenerateFormula(SE.getAddExpr(Ops));
4187 }
4188}
4189
4190/// Helper function for LSRInstance::GenerateSymbolicOffsets.
4191void LSRInstance::GenerateSymbolicOffsetsImpl(LSRUse &LU, unsigned LUIdx,
4192 const Formula &Base, size_t Idx,
4193 bool IsScaledReg) {
4194 SCEVUse G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
4195 GlobalValue *GV = ExtractSymbol(S&: G, SE);
4196 if (G->isZero() || !GV)
4197 return;
4198 Formula F = Base;
4199 F.BaseGV = GV;
4200 if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy, F))
4201 return;
4202 if (IsScaledReg)
4203 F.ScaledReg = G;
4204 else
4205 F.BaseRegs[Idx] = G;
4206 (void)InsertFormula(LU, LUIdx, F);
4207}
4208
4209/// Generate reuse formulae using symbolic offsets.
4210void LSRInstance::GenerateSymbolicOffsets(LSRUse &LU, unsigned LUIdx,
4211 Formula Base) {
4212 // We can't add a symbolic offset if the address already contains one.
4213 if (Base.BaseGV) return;
4214
4215 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
4216 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, Idx: i);
4217 if (Base.Scale == 1)
4218 GenerateSymbolicOffsetsImpl(LU, LUIdx, Base, /* Idx */ -1,
4219 /* IsScaledReg */ true);
4220}
4221
4222/// Helper function for LSRInstance::GenerateConstantOffsets.
4223void LSRInstance::GenerateConstantOffsetsImpl(
4224 LSRUse &LU, unsigned LUIdx, const Formula &Base,
4225 const SmallVectorImpl<Immediate> &Worklist, size_t Idx, bool IsScaledReg) {
4226
4227 auto GenerateOffset = [&](const SCEV *G, Immediate Offset) {
4228 Formula F = Base;
4229 if (!Base.BaseOffset.isCompatibleImmediate(Imm: Offset))
4230 return;
4231 F.BaseOffset = Base.BaseOffset.subUnsigned(RHS: Offset);
4232
4233 if (isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy, F)) {
4234 // Add the offset to the base register.
4235 const SCEV *NewOffset = Offset.getSCEV(SE, Ty: G->getType());
4236 const SCEV *NewG = SE.getAddExpr(LHS: NewOffset, RHS: G);
4237 // If it cancelled out, drop the base register, otherwise update it.
4238 if (NewG->isZero()) {
4239 if (IsScaledReg) {
4240 F.Scale = 0;
4241 F.ScaledReg = nullptr;
4242 } else
4243 F.deleteBaseReg(S&: F.BaseRegs[Idx]);
4244 F.canonicalize(L: *L);
4245 } else if (IsScaledReg)
4246 F.ScaledReg = NewG;
4247 else
4248 F.BaseRegs[Idx] = NewG;
4249
4250 (void)InsertFormula(LU, LUIdx, F);
4251 }
4252 };
4253
4254 SCEVUse G = IsScaledReg ? Base.ScaledReg : Base.BaseRegs[Idx];
4255
4256 // With constant offsets and constant steps, we can generate pre-inc
4257 // accesses by having the offset equal the step. So, for access #0 with a
4258 // step of 8, we generate a G - 8 base which would require the first access
4259 // to be ((G - 8) + 8),+,8. The pre-indexed access then updates the pointer
4260 // for itself and hopefully becomes the base for other accesses. This means
4261 // means that a single pre-indexed access can be generated to become the new
4262 // base pointer for each iteration of the loop, resulting in no extra add/sub
4263 // instructions for pointer updating.
4264 if ((AMK & TTI::AMK_PreIndexed) && LU.Kind == LSRUse::Address) {
4265 const APInt *StepInt;
4266 if (match(U: G, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_scev_APInt(C&: StepInt)))) {
4267 int64_t Step = StepInt->isNegative() ? StepInt->getSExtValue()
4268 : StepInt->getZExtValue();
4269
4270 for (Immediate Offset : Worklist) {
4271 if (Offset.isFixed()) {
4272 Offset = Immediate::getFixed(MinVal: Offset.getFixedValue() - Step);
4273 GenerateOffset(G, Offset);
4274 }
4275 }
4276 }
4277 }
4278 for (Immediate Offset : Worklist)
4279 GenerateOffset(G, Offset);
4280
4281 // TODO: It likely makes sense to extract the immediate corresponding to the
4282 // access type (i.e., set PreferScalable to AccessTy.MemTy &&
4283 // AccessTy.MemTy->isScalableTy()).
4284 Immediate Imm = extractImmediate(Opts, S&: G, SE, /*PreferScalable=*/false);
4285 if (G->isZero() || Imm.isZero() ||
4286 !Base.BaseOffset.isCompatibleImmediate(Imm))
4287 return;
4288 Formula F = Base;
4289 F.BaseOffset = F.BaseOffset.addUnsigned(RHS: Imm);
4290 if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy, F))
4291 return;
4292 if (IsScaledReg) {
4293 F.ScaledReg = G;
4294 } else {
4295 F.BaseRegs[Idx] = G;
4296 // We may generate non canonical Formula if G is a recurrent expr reg
4297 // related with current loop while F.ScaledReg is not.
4298 F.canonicalize(L: *L);
4299 }
4300 (void)InsertFormula(LU, LUIdx, F);
4301}
4302
4303/// GenerateConstantOffsets - Generate reuse formulae using symbolic offsets.
4304void LSRInstance::GenerateConstantOffsets(LSRUse &LU, unsigned LUIdx,
4305 Formula Base) {
4306 // TODO: For now, just add the min and max offset, because it usually isn't
4307 // worthwhile looking at everything inbetween.
4308 SmallVector<Immediate, 2> Worklist;
4309 Worklist.push_back(Elt: LU.MinOffset);
4310 if (LU.MaxOffset != LU.MinOffset)
4311 Worklist.push_back(Elt: LU.MaxOffset);
4312
4313 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i)
4314 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, Idx: i);
4315 if (Base.Scale == 1)
4316 GenerateConstantOffsetsImpl(LU, LUIdx, Base, Worklist, /* Idx */ -1,
4317 /* IsScaledReg */ true);
4318}
4319
4320/// For ICmpZero, check to see if we can scale up the comparison. For example, x
4321/// == y -> x*c == y*c.
4322void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
4323 Formula Base) {
4324 if (LU.Kind != LSRUse::ICmpZero) return;
4325
4326 // Determine the integer type for the base formula.
4327 Type *IntTy = Base.getType();
4328 if (!IntTy) return;
4329 if (SE.getTypeSizeInBits(Ty: IntTy) > 64) return;
4330
4331 // Don't do this if there is more than one offset.
4332 if (LU.MinOffset != LU.MaxOffset) return;
4333
4334 // Check if transformation is valid. It is illegal to multiply pointer.
4335 if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy())
4336 return;
4337 for (const SCEV *BaseReg : Base.BaseRegs)
4338 if (BaseReg->getType()->isPointerTy())
4339 return;
4340 assert(!Base.BaseGV && "ICmpZero use is not legal!");
4341
4342 // Check each interesting stride.
4343 for (int64_t Factor : Factors) {
4344 // Check that Factor can be represented by IntTy
4345 if (!ConstantInt::isValueValidForType(Ty: IntTy, V: Factor))
4346 continue;
4347 // Check that the multiplication doesn't overflow.
4348 if (Base.BaseOffset.isMin() && Factor == -1)
4349 continue;
4350 // Not supporting scalable immediates.
4351 if (Base.BaseOffset.isNonZero() && Base.BaseOffset.isScalable())
4352 continue;
4353 Immediate NewBaseOffset = Base.BaseOffset.mulUnsigned(RHS: Factor);
4354 assert(Factor != 0 && "Zero factor not expected!");
4355 if (NewBaseOffset.getFixedValue() / Factor !=
4356 Base.BaseOffset.getFixedValue())
4357 continue;
4358 // If the offset will be truncated at this use, check that it is in bounds.
4359 if (!IntTy->isPointerTy() &&
4360 !ConstantInt::isValueValidForType(Ty: IntTy, V: NewBaseOffset.getFixedValue()))
4361 continue;
4362
4363 // Check that multiplying with the use offset doesn't overflow.
4364 Immediate Offset = LU.MinOffset;
4365 if (Offset.isMin() && Factor == -1)
4366 continue;
4367 Offset = Offset.mulUnsigned(RHS: Factor);
4368 if (Offset.getFixedValue() / Factor != LU.MinOffset.getFixedValue())
4369 continue;
4370 // If the offset will be truncated at this use, check that it is in bounds.
4371 if (!IntTy->isPointerTy() &&
4372 !ConstantInt::isValueValidForType(Ty: IntTy, V: Offset.getFixedValue()))
4373 continue;
4374
4375 Formula F = Base;
4376 F.BaseOffset = NewBaseOffset;
4377
4378 // Check that this scale is legal.
4379 if (!isLegalUse(TTI, MinOffset: Offset, MaxOffset: Offset, Kind: LU.Kind, AccessTy: LU.AccessTy, F))
4380 continue;
4381
4382 // Compensate for the use having MinOffset built into it.
4383 F.BaseOffset = F.BaseOffset.addUnsigned(RHS: Offset).subUnsigned(RHS: LU.MinOffset);
4384
4385 const SCEV *FactorS = SE.getConstant(Ty: IntTy, V: Factor);
4386
4387 // Check that multiplying with each base register doesn't overflow.
4388 for (size_t i = 0, e = F.BaseRegs.size(); i != e; ++i) {
4389 F.BaseRegs[i] = SE.getMulExpr(LHS: F.BaseRegs[i], RHS: FactorS);
4390 if (getExactSDiv(LHS: F.BaseRegs[i], RHS: FactorS, SE) != Base.BaseRegs[i])
4391 goto next;
4392 }
4393
4394 // Check that multiplying with the scaled register doesn't overflow.
4395 if (F.ScaledReg) {
4396 F.ScaledReg = SE.getMulExpr(LHS: F.ScaledReg, RHS: FactorS);
4397 if (getExactSDiv(LHS: F.ScaledReg, RHS: FactorS, SE) != Base.ScaledReg)
4398 continue;
4399 }
4400
4401 // Check that multiplying with the unfolded offset doesn't overflow.
4402 if (F.UnfoldedOffset.isNonZero()) {
4403 if (F.UnfoldedOffset.isMin() && Factor == -1)
4404 continue;
4405 F.UnfoldedOffset = F.UnfoldedOffset.mulUnsigned(RHS: Factor);
4406 if (F.UnfoldedOffset.getFixedValue() / Factor !=
4407 Base.UnfoldedOffset.getFixedValue())
4408 continue;
4409 // If the offset will be truncated, check that it is in bounds.
4410 if (!IntTy->isPointerTy() && !ConstantInt::isValueValidForType(
4411 Ty: IntTy, V: F.UnfoldedOffset.getFixedValue()))
4412 continue;
4413 }
4414
4415 // If we make it here and it's legal, add it.
4416 (void)InsertFormula(LU, LUIdx, F);
4417 next:;
4418 }
4419}
4420
4421/// Generate stride factor reuse formulae by making use of scaled-offset address
4422/// modes, for example.
4423void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
4424 // Determine the integer type for the base formula.
4425 Type *IntTy = Base.getType();
4426 if (!IntTy) return;
4427
4428 // If this Formula already has a scaled register, we can't add another one.
4429 // Try to unscale the formula to generate a better scale.
4430 if (Base.Scale != 0 && !Base.unscale())
4431 return;
4432
4433 assert(Base.Scale == 0 && "unscale did not did its job!");
4434
4435 // Check each interesting stride.
4436 for (int64_t Factor : Factors) {
4437 Base.Scale = Factor;
4438 Base.HasBaseReg = Base.BaseRegs.size() > 1;
4439 // Check whether this scale is going to be legal.
4440 if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy,
4441 F: Base)) {
4442 // As a special-case, handle special out-of-loop Basic users specially.
4443 // TODO: Reconsider this special case.
4444 if (LU.Kind == LSRUse::Basic &&
4445 isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LSRUse::Special,
4446 AccessTy: LU.AccessTy, F: Base) &&
4447 LU.AllFixupsOutsideLoop)
4448 LU.Kind = LSRUse::Special;
4449 else
4450 continue;
4451 }
4452 // For an ICmpZero, negating a solitary base register won't lead to
4453 // new solutions.
4454 if (LU.Kind == LSRUse::ICmpZero && !Base.HasBaseReg &&
4455 Base.BaseOffset.isZero() && !Base.BaseGV)
4456 continue;
4457 // For each addrec base reg, if its loop is current loop, apply the scale.
4458 for (size_t i = 0, e = Base.BaseRegs.size(); i != e; ++i) {
4459 const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Val: Base.BaseRegs[i]);
4460 if (AR && (AR->getLoop() == L || LU.AllFixupsOutsideLoop)) {
4461 const SCEV *FactorS = SE.getConstant(Ty: IntTy, V: Factor);
4462 if (FactorS->isZero())
4463 continue;
4464 // Divide out the factor, ignoring high bits, since we'll be
4465 // scaling the value back up in the end.
4466 if (const SCEV *Quotient = getExactSDiv(LHS: AR, RHS: FactorS, SE, IgnoreSignificantBits: true))
4467 if (!Quotient->isZero()) {
4468 // TODO: This could be optimized to avoid all the copying.
4469 Formula F = Base;
4470 F.ScaledReg = Quotient;
4471 F.deleteBaseReg(S&: F.BaseRegs[i]);
4472 // The canonical representation of 1*reg is reg, which is already in
4473 // Base. In that case, do not try to insert the formula, it will be
4474 // rejected anyway.
4475 if (F.Scale == 1 && (F.BaseRegs.empty() ||
4476 (AR->getLoop() != L && LU.AllFixupsOutsideLoop)))
4477 continue;
4478 // If AllFixupsOutsideLoop is true and F.Scale is 1, we may generate
4479 // non canonical Formula with ScaledReg's loop not being L.
4480 if (F.Scale == 1 && LU.AllFixupsOutsideLoop)
4481 F.canonicalize(L: *L);
4482 (void)InsertFormula(LU, LUIdx, F);
4483 }
4484 }
4485 }
4486 }
4487}
4488
4489/// Extend/Truncate \p Expr to \p ToTy considering post-inc uses in \p Loops.
4490/// For all PostIncLoopSets in \p Loops, first de-normalize \p Expr, then
4491/// perform the extension/truncate and normalize again, as the normalized form
4492/// can result in folds that are not valid in the post-inc use contexts. The
4493/// expressions for all PostIncLoopSets must match, otherwise return nullptr.
4494static const SCEV *
4495getAnyExtendConsideringPostIncUses(ArrayRef<PostIncLoopSet> Loops,
4496 const SCEV *Expr, Type *ToTy,
4497 ScalarEvolution &SE) {
4498 const SCEV *Result = nullptr;
4499 for (auto &L : Loops) {
4500 auto *DenormExpr = denormalizeForPostIncUse(S: Expr, Loops: L, SE);
4501 const SCEV *NewDenormExpr = SE.getAnyExtendExpr(Op: DenormExpr, Ty: ToTy);
4502 const SCEV *New = normalizeForPostIncUse(S: NewDenormExpr, Loops: L, SE);
4503 if (!New || (Result && New != Result))
4504 return nullptr;
4505 Result = New;
4506 }
4507
4508 assert(Result && "failed to create expression");
4509 return Result;
4510}
4511
4512/// Generate reuse formulae from different IV types.
4513void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
4514 // Don't bother truncating symbolic values.
4515 if (Base.BaseGV) return;
4516
4517 // Determine the integer type for the base formula.
4518 Type *DstTy = Base.getType();
4519 if (!DstTy) return;
4520 if (DstTy->isPointerTy())
4521 return;
4522
4523 // It is invalid to extend a pointer type so exit early if ScaledReg or
4524 // any of the BaseRegs are pointers.
4525 if (Base.ScaledReg && Base.ScaledReg->getType()->isPointerTy())
4526 return;
4527 if (any_of(Range&: Base.BaseRegs,
4528 P: [](const SCEV *S) { return S->getType()->isPointerTy(); }))
4529 return;
4530
4531 SmallVector<PostIncLoopSet> Loops;
4532 for (auto &LF : LU.Fixups)
4533 Loops.push_back(Elt: LF.PostIncLoops);
4534
4535 for (Type *SrcTy : Types) {
4536 if (SrcTy != DstTy && TTI.isTruncateFree(Ty1: SrcTy, Ty2: DstTy)) {
4537 Formula F = Base;
4538
4539 // Sometimes SCEV is able to prove zero during ext transform. It may
4540 // happen if SCEV did not do all possible transforms while creating the
4541 // initial node (maybe due to depth limitations), but it can do them while
4542 // taking ext.
4543 if (F.ScaledReg) {
4544 const SCEV *NewScaledReg =
4545 getAnyExtendConsideringPostIncUses(Loops, Expr: F.ScaledReg, ToTy: SrcTy, SE);
4546 if (!NewScaledReg || NewScaledReg->isZero())
4547 continue;
4548 F.ScaledReg = NewScaledReg;
4549 }
4550 bool HasZeroBaseReg = false;
4551 for (const SCEV *&BaseReg : F.BaseRegs) {
4552 const SCEV *NewBaseReg =
4553 getAnyExtendConsideringPostIncUses(Loops, Expr: BaseReg, ToTy: SrcTy, SE);
4554 if (!NewBaseReg || NewBaseReg->isZero()) {
4555 HasZeroBaseReg = true;
4556 break;
4557 }
4558 BaseReg = NewBaseReg;
4559 }
4560 if (HasZeroBaseReg)
4561 continue;
4562
4563 // TODO: This assumes we've done basic processing on all uses and
4564 // have an idea what the register usage is.
4565 if (!F.hasRegsUsedByUsesOtherThan(LUIdx, RegUses))
4566 continue;
4567
4568 F.canonicalize(L: *L);
4569 (void)InsertFormula(LU, LUIdx, F);
4570 }
4571 }
4572}
4573
4574namespace {
4575
4576/// Helper class for GenerateCrossUseConstantOffsets. It's used to defer
4577/// modifications so that the search phase doesn't have to worry about the data
4578/// structures moving underneath it.
4579struct WorkItem {
4580 size_t LUIdx;
4581 Immediate Imm;
4582 const SCEV *OrigReg;
4583
4584 WorkItem(size_t LI, Immediate I, const SCEV *R)
4585 : LUIdx(LI), Imm(I), OrigReg(R) {}
4586
4587 void print(raw_ostream &OS) const;
4588 void dump() const;
4589};
4590
4591} // end anonymous namespace
4592
4593#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
4594void WorkItem::print(raw_ostream &OS) const {
4595 OS << "in formulae referencing " << *OrigReg << " in use " << LUIdx
4596 << " , add offset " << Imm;
4597}
4598
4599LLVM_DUMP_METHOD void WorkItem::dump() const {
4600 print(errs()); errs() << '\n';
4601}
4602#endif
4603
4604/// Look for registers which are a constant distance apart and try to form reuse
4605/// opportunities between them.
4606void LSRInstance::GenerateCrossUseConstantOffsets() {
4607 // Group the registers by their value without any added constant offset.
4608 using ImmMapTy = std::map<Immediate, const SCEV *, KeyOrderTargetImmediate>;
4609
4610 DenseMap<const SCEV *, ImmMapTy> Map;
4611 DenseMap<const SCEV *, SmallBitVector> UsedByIndicesMap;
4612 SmallVector<const SCEV *, 8> Sequence;
4613 for (const SCEV *Use : RegUses) {
4614 SCEVUse Reg = Use; // Make a copy for extractImmediate to modify.
4615 // TODO: Extract both scalable and fixed immediates (if present)?
4616 Immediate Imm = extractImmediate(Opts, S&: Reg, SE);
4617 auto Pair = Map.try_emplace(Key: Reg);
4618 if (Pair.second)
4619 Sequence.push_back(Elt: Reg);
4620 Pair.first->second.insert(x: std::make_pair(x&: Imm, y&: Use));
4621 UsedByIndicesMap[Reg] |= RegUses.getUsedByIndices(Reg: Use);
4622 }
4623
4624 // Now examine each set of registers with the same base value. Build up
4625 // a list of work to do and do the work in a separate step so that we're
4626 // not adding formulae and register counts while we're searching.
4627 SmallVector<WorkItem, 32> WorkItems;
4628 SmallSet<std::pair<size_t, Immediate>, 32, KeyOrderSizeTAndImmediate>
4629 UniqueItems;
4630 for (const SCEV *Reg : Sequence) {
4631 const ImmMapTy &Imms = Map.find(Val: Reg)->second;
4632
4633 // It's not worthwhile looking for reuse if there's only one offset.
4634 if (Imms.size() == 1)
4635 continue;
4636
4637 LLVM_DEBUG(dbgs() << "Generating cross-use offsets for " << *Reg << ':';
4638 for (const auto &Entry
4639 : Imms) dbgs()
4640 << ' ' << Entry.first;
4641 dbgs() << '\n');
4642
4643 // Examine each offset.
4644 for (ImmMapTy::const_iterator J = Imms.begin(), JE = Imms.end();
4645 J != JE; ++J) {
4646 const SCEV *OrigReg = J->second;
4647
4648 Immediate JImm = J->first;
4649 const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg: OrigReg);
4650
4651 if (!isa<SCEVConstant>(Val: OrigReg) &&
4652 UsedByIndicesMap[Reg].count() == 1) {
4653 LLVM_DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg
4654 << '\n');
4655 continue;
4656 }
4657
4658 // Conservatively examine offsets between this orig reg a few selected
4659 // other orig regs.
4660 Immediate First = Imms.begin()->first;
4661 Immediate Last = std::prev(x: Imms.end())->first;
4662 if (!First.isCompatibleImmediate(Imm: Last)) {
4663 LLVM_DEBUG(dbgs() << "Skipping cross-use reuse for " << *OrigReg
4664 << "\n");
4665 continue;
4666 }
4667 // Only scalable if both terms are scalable, or if one is scalable and
4668 // the other is 0.
4669 bool Scalable = First.isScalable() || Last.isScalable();
4670 int64_t FI = First.getKnownMinValue();
4671 int64_t LI = Last.getKnownMinValue();
4672 // Compute (First + Last) / 2 without overflow using the fact that
4673 // First + Last = 2 * (First + Last) + (First ^ Last).
4674 int64_t Avg = (FI & LI) + ((FI ^ LI) >> 1);
4675 // If the result is negative and FI is odd and LI even (or vice versa),
4676 // we rounded towards -inf. Add 1 in that case, to round towards 0.
4677 Avg = Avg + ((FI ^ LI) & ((uint64_t)Avg >> 63));
4678 ImmMapTy::const_iterator OtherImms[] = {
4679 Imms.begin(), std::prev(x: Imms.end()),
4680 Imms.lower_bound(x: Immediate::get(MinVal: Avg, Scalable))};
4681 for (const auto &M : OtherImms) {
4682 if (M == J || M == JE) continue;
4683 if (!JImm.isCompatibleImmediate(Imm: M->first))
4684 continue;
4685
4686 // Compute the difference between the two.
4687 Immediate Imm = JImm.subUnsigned(RHS: M->first);
4688 for (unsigned LUIdx : UsedByIndices.set_bits())
4689 // Make a memo of this use, offset, and register tuple.
4690 if (UniqueItems.insert(V: std::make_pair(x&: LUIdx, y&: Imm)).second)
4691 WorkItems.push_back(Elt: WorkItem(LUIdx, Imm, OrigReg));
4692 }
4693 }
4694 }
4695
4696 Map.clear();
4697 Sequence.clear();
4698 UsedByIndicesMap.clear();
4699 UniqueItems.clear();
4700
4701 // Now iterate through the worklist and add new formulae.
4702 for (const WorkItem &WI : WorkItems) {
4703 size_t LUIdx = WI.LUIdx;
4704 LSRUse &LU = Uses[LUIdx];
4705 Immediate Imm = WI.Imm;
4706 const SCEV *OrigReg = WI.OrigReg;
4707
4708 Type *IntTy = SE.getEffectiveSCEVType(Ty: OrigReg->getType());
4709 const SCEV *NegImmS = Imm.getNegativeSCEV(SE, Ty: IntTy);
4710 unsigned BitWidth = SE.getTypeSizeInBits(Ty: IntTy);
4711
4712 // TODO: Use a more targeted data structure.
4713 for (size_t L = 0, LE = LU.Formulae.size(); L != LE; ++L) {
4714 Formula F = LU.Formulae[L];
4715 // FIXME: The code for the scaled and unscaled registers looks
4716 // very similar but slightly different. Investigate if they
4717 // could be merged. That way, we would not have to unscale the
4718 // Formula.
4719 F.unscale();
4720 // Use the immediate in the scaled register.
4721 if (F.ScaledReg == OrigReg) {
4722 if (!F.BaseOffset.isCompatibleImmediate(Imm))
4723 continue;
4724 Immediate Offset = F.BaseOffset.addUnsigned(RHS: Imm.mulUnsigned(RHS: F.Scale));
4725 // Don't create 50 + reg(-50).
4726 const SCEV *S = Offset.getNegativeSCEV(SE, Ty: IntTy);
4727 if (F.referencesReg(S))
4728 continue;
4729 Formula NewF = F;
4730 NewF.BaseOffset = Offset;
4731 if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: LU.Kind, AccessTy: LU.AccessTy,
4732 F: NewF))
4733 continue;
4734 NewF.ScaledReg = SE.getAddExpr(LHS: NegImmS, RHS: NewF.ScaledReg);
4735
4736 // If the new scale is a constant in a register, and adding the constant
4737 // value to the immediate would produce a value closer to zero than the
4738 // immediate itself, then the formula isn't worthwhile.
4739 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: NewF.ScaledReg)) {
4740 // FIXME: Do we need to do something for scalable immediates here?
4741 // A scalable SCEV won't be constant, but we might still have
4742 // something in the offset? Bail out for now to be safe.
4743 if (NewF.BaseOffset.isNonZero() && NewF.BaseOffset.isScalable())
4744 continue;
4745 if (C->getValue()->isNegative() !=
4746 (NewF.BaseOffset.isLessThanZero()) &&
4747 (C->getAPInt().abs() * APInt(BitWidth, F.Scale))
4748 .ule(RHS: std::abs(i: NewF.BaseOffset.getFixedValue())))
4749 continue;
4750 }
4751
4752 // OK, looks good.
4753 NewF.canonicalize(L: *this->L);
4754 (void)InsertFormula(LU, LUIdx, F: NewF);
4755 } else {
4756 // Use the immediate in a base register.
4757 for (size_t N = 0, NE = F.BaseRegs.size(); N != NE; ++N) {
4758 const SCEV *BaseReg = F.BaseRegs[N];
4759 if (BaseReg != OrigReg)
4760 continue;
4761 Formula NewF = F;
4762 if (!NewF.BaseOffset.isCompatibleImmediate(Imm) ||
4763 !NewF.UnfoldedOffset.isCompatibleImmediate(Imm) ||
4764 !NewF.BaseOffset.isCompatibleImmediate(Imm: NewF.UnfoldedOffset))
4765 continue;
4766 NewF.BaseOffset = NewF.BaseOffset.addUnsigned(RHS: Imm);
4767 if (!isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset,
4768 Kind: LU.Kind, AccessTy: LU.AccessTy, F: NewF)) {
4769 if (AMK == TTI::AMK_PostIndexed &&
4770 mayUsePostIncMode(TTI, LU, S: OrigReg, L: this->L, SE))
4771 continue;
4772 Immediate NewUnfoldedOffset = NewF.UnfoldedOffset.addUnsigned(RHS: Imm);
4773 if (!isLegalAddImmediate(TTI, Offset: NewUnfoldedOffset))
4774 continue;
4775 NewF = F;
4776 NewF.UnfoldedOffset = NewUnfoldedOffset;
4777 }
4778 NewF.BaseRegs[N] = SE.getAddExpr(LHS: NegImmS, RHS: BaseReg);
4779
4780 // If the new formula has a constant in a register, and adding the
4781 // constant value to the immediate would produce a value closer to
4782 // zero than the immediate itself, then the formula isn't worthwhile.
4783 for (const SCEV *NewReg : NewF.BaseRegs)
4784 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: NewReg)) {
4785 if (NewF.BaseOffset.isNonZero() && NewF.BaseOffset.isScalable())
4786 goto skip_formula;
4787 if ((C->getAPInt() + NewF.BaseOffset.getFixedValue())
4788 .abs()
4789 .slt(RHS: std::abs(i: NewF.BaseOffset.getFixedValue())) &&
4790 (C->getAPInt() + NewF.BaseOffset.getFixedValue())
4791 .countr_zero() >=
4792 (unsigned)llvm::countr_zero<uint64_t>(
4793 Val: NewF.BaseOffset.getFixedValue()))
4794 goto skip_formula;
4795 }
4796
4797 // Ok, looks good.
4798 NewF.canonicalize(L: *this->L);
4799 (void)InsertFormula(LU, LUIdx, F: NewF);
4800 break;
4801 skip_formula:;
4802 }
4803 }
4804 }
4805 }
4806}
4807
4808/// Generate formulae for each use.
4809void
4810LSRInstance::GenerateAllReuseFormulae() {
4811 // This is split into multiple loops so that hasRegsUsedByUsesOtherThan
4812 // queries are more precise.
4813 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4814 LSRUse &LU = Uses[LUIdx];
4815 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4816 GenerateReassociations(LU, LUIdx, Base: LU.Formulae[i]);
4817 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4818 GenerateCombinations(LU, LUIdx, Base: LU.Formulae[i]);
4819 }
4820 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4821 LSRUse &LU = Uses[LUIdx];
4822 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4823 GenerateSymbolicOffsets(LU, LUIdx, Base: LU.Formulae[i]);
4824 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4825 GenerateConstantOffsets(LU, LUIdx, Base: LU.Formulae[i]);
4826 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4827 GenerateICmpZeroScales(LU, LUIdx, Base: LU.Formulae[i]);
4828 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4829 GenerateScales(LU, LUIdx, Base: LU.Formulae[i]);
4830 }
4831 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4832 LSRUse &LU = Uses[LUIdx];
4833 for (size_t i = 0, f = LU.Formulae.size(); i != f; ++i)
4834 GenerateTruncates(LU, LUIdx, Base: LU.Formulae[i]);
4835 }
4836
4837 GenerateCrossUseConstantOffsets();
4838
4839 LLVM_DEBUG(dbgs() << "\n"
4840 "After generating reuse formulae:\n";
4841 print_uses(dbgs()));
4842}
4843
4844/// If there are multiple formulae with the same set of registers used
4845/// by other uses, pick the best one and delete the others.
4846void LSRInstance::FilterOutUndesirableDedicatedRegisters() {
4847 DenseSet<const SCEV *> VisitedRegs;
4848 SmallPtrSet<const SCEV *, 16> Regs;
4849 SmallPtrSet<const SCEV *, 16> LoserRegs;
4850#ifndef NDEBUG
4851 bool ChangedFormulae = false;
4852#endif
4853
4854 // Collect the best formula for each unique set of shared registers. This
4855 // is reset for each use.
4856 using BestFormulaeTy = DenseMap<SmallVector<const SCEV *, 4>, size_t>;
4857
4858 BestFormulaeTy BestFormulae;
4859
4860 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4861 LSRUse &LU = Uses[LUIdx];
4862 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
4863 dbgs() << '\n');
4864
4865 bool Any = false;
4866 for (size_t FIdx = 0, NumForms = LU.Formulae.size();
4867 FIdx != NumForms; ++FIdx) {
4868 Formula &F = LU.Formulae[FIdx];
4869
4870 // Some formulas are instant losers. For example, they may depend on
4871 // nonexistent AddRecs from other loops. These need to be filtered
4872 // immediately, otherwise heuristics could choose them over others leading
4873 // to an unsatisfactory solution. Passing LoserRegs into RateFormula here
4874 // avoids the need to recompute this information across formulae using the
4875 // same bad AddRec. Passing LoserRegs is also essential unless we remove
4876 // the corresponding bad register from the Regs set.
4877 Cost CostF(Opts, L, SE, TTI, AMK);
4878 Regs.clear();
4879 CostF.RateFormula(F, Regs, VisitedRegs, LU, HardwareLoopProfitable,
4880 LoserRegs: &LoserRegs);
4881 if (CostF.isLoser()) {
4882 // During initial formula generation, undesirable formulae are generated
4883 // by uses within other loops that have some non-trivial address mode or
4884 // use the postinc form of the IV. LSR needs to provide these formulae
4885 // as the basis of rediscovering the desired formula that uses an AddRec
4886 // corresponding to the existing phi. Once all formulae have been
4887 // generated, these initial losers may be pruned.
4888 LLVM_DEBUG(dbgs() << " Filtering loser "; F.print(dbgs());
4889 dbgs() << "\n");
4890 }
4891 else {
4892 SmallVector<const SCEV *, 4> Key;
4893 for (const SCEV *Reg : F.BaseRegs) {
4894 if (RegUses.isRegUsedByUsesOtherThan(Reg, LUIdx))
4895 Key.push_back(Elt: Reg);
4896 }
4897 if (F.ScaledReg &&
4898 RegUses.isRegUsedByUsesOtherThan(Reg: F.ScaledReg, LUIdx))
4899 Key.push_back(Elt: F.ScaledReg);
4900 // Unstable sort by host order ok, because this is only used for
4901 // uniquifying.
4902 llvm::sort(C&: Key);
4903
4904 std::pair<BestFormulaeTy::const_iterator, bool> P =
4905 BestFormulae.insert(KV: std::make_pair(x&: Key, y&: FIdx));
4906 if (P.second)
4907 continue;
4908
4909 Formula &Best = LU.Formulae[P.first->second];
4910
4911 Cost CostBest(Opts, L, SE, TTI, AMK);
4912 Regs.clear();
4913 CostBest.RateFormula(F: Best, Regs, VisitedRegs, LU,
4914 HardwareLoopProfitable);
4915 if (CostF.isLess(Other: CostBest))
4916 std::swap(a&: F, b&: Best);
4917 LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
4918 dbgs() << "\n"
4919 " in favor of formula ";
4920 Best.print(dbgs()); dbgs() << '\n');
4921 }
4922#ifndef NDEBUG
4923 ChangedFormulae = true;
4924#endif
4925 LU.DeleteFormula(F);
4926 --FIdx;
4927 --NumForms;
4928 Any = true;
4929 }
4930
4931 // Now that we've filtered out some formulae, recompute the Regs set.
4932 if (Any)
4933 LU.RecomputeRegs(LUIdx, RegUses);
4934
4935 // Reset this to prepare for the next use.
4936 BestFormulae.clear();
4937 }
4938
4939 LLVM_DEBUG(if (ChangedFormulae) {
4940 dbgs() << "\n"
4941 "After filtering out undesirable candidates:\n";
4942 print_uses(dbgs());
4943 });
4944}
4945
4946/// Estimate the worst-case number of solutions the solver might have to
4947/// consider. It almost never considers this many solutions because it prune the
4948/// search space, but the pruning isn't always sufficient.
4949size_t LSRInstance::EstimateSearchSpaceComplexity() const {
4950 size_t Power = 1;
4951 for (const LSRUse &LU : Uses) {
4952 size_t FSize = LU.Formulae.size();
4953 if (FSize >= Opts.lsr_complexity_limit) {
4954 Power = Opts.lsr_complexity_limit;
4955 break;
4956 }
4957 Power *= FSize;
4958 if (Power >= Opts.lsr_complexity_limit)
4959 break;
4960 }
4961 return Power;
4962}
4963
4964/// When one formula uses a superset of the registers of another formula, it
4965/// won't help reduce register pressure (though it may not necessarily hurt
4966/// register pressure); remove it to simplify the system.
4967void LSRInstance::NarrowSearchSpaceByDetectingSupersets() {
4968 if (EstimateSearchSpaceComplexity() >= Opts.lsr_complexity_limit) {
4969 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
4970
4971 LLVM_DEBUG(dbgs() << "Narrowing the search space by eliminating formulae "
4972 "which use a superset of registers used by other "
4973 "formulae.\n");
4974
4975 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
4976 LSRUse &LU = Uses[LUIdx];
4977 bool Any = false;
4978 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
4979 Formula &F = LU.Formulae[i];
4980 if (F.BaseOffset.isNonZero() && F.BaseOffset.isScalable())
4981 continue;
4982 // Look for a formula with a constant or GV in a register. If the use
4983 // also has a formula with that same value in an immediate field,
4984 // delete the one that uses a register.
4985 for (SmallVectorImpl<const SCEV *>::const_iterator
4986 I = F.BaseRegs.begin(), E = F.BaseRegs.end(); I != E; ++I) {
4987 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: *I)) {
4988 Formula NewF = F;
4989 //FIXME: Formulas should store bitwidth to do wrapping properly.
4990 // See PR41034.
4991 NewF.BaseOffset =
4992 Immediate::getFixed(MinVal: NewF.BaseOffset.getFixedValue() +
4993 (uint64_t)C->getValue()->getSExtValue());
4994 NewF.BaseRegs.erase(CI: NewF.BaseRegs.begin() +
4995 (I - F.BaseRegs.begin()));
4996 if (LU.HasFormulaWithSameRegs(F: NewF)) {
4997 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());
4998 dbgs() << '\n');
4999 LU.DeleteFormula(F);
5000 --i;
5001 --e;
5002 Any = true;
5003 break;
5004 }
5005 } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Val: *I)) {
5006 if (GlobalValue *GV = dyn_cast<GlobalValue>(Val: U->getValue()))
5007 if (!F.BaseGV) {
5008 Formula NewF = F;
5009 NewF.BaseGV = GV;
5010 NewF.BaseRegs.erase(CI: NewF.BaseRegs.begin() +
5011 (I - F.BaseRegs.begin()));
5012 if (LU.HasFormulaWithSameRegs(F: NewF)) {
5013 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs());
5014 dbgs() << '\n');
5015 LU.DeleteFormula(F);
5016 --i;
5017 --e;
5018 Any = true;
5019 break;
5020 }
5021 }
5022 }
5023 }
5024 }
5025 if (Any)
5026 LU.RecomputeRegs(LUIdx, RegUses);
5027 }
5028
5029 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5030 }
5031}
5032
5033/// When there are many registers for expressions like A, A+1, A+2, etc.,
5034/// allocate a single register for them.
5035void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() {
5036 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5037 return;
5038
5039 LLVM_DEBUG(
5040 dbgs() << "The search space is too complex.\n"
5041 "Narrowing the search space by assuming that uses separated "
5042 "by a constant offset will use the same registers.\n");
5043
5044 // This is especially useful for unrolled loops.
5045
5046 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5047 LSRUse &LU = Uses[LUIdx];
5048 for (const Formula &F : LU.Formulae) {
5049 if (F.BaseOffset.isZero() || (F.Scale != 0 && F.Scale != 1))
5050 continue;
5051 assert((LU.Kind == LSRUse::Address || LU.Kind == LSRUse::ICmpZero) &&
5052 "Only address and cmp uses expected to have nonzero BaseOffset");
5053
5054 LSRUse *LUThatHas = FindUseWithSimilarFormula(OrigF: F, OrigLU: LU);
5055 if (!LUThatHas)
5056 continue;
5057
5058 if (!reconcileNewOffset(LU&: *LUThatHas, NewOffset: F.BaseOffset, /*HasBaseReg=*/ false,
5059 Kind: LU.Kind, AccessTy: LU.AccessTy))
5060 continue;
5061
5062 LLVM_DEBUG(dbgs() << " Deleting use "; LU.print(dbgs()); dbgs() << '\n');
5063
5064 LUThatHas->AllFixupsOutsideLoop &= LU.AllFixupsOutsideLoop;
5065 LUThatHas->AllFixupsUnconditional &= LU.AllFixupsUnconditional;
5066
5067 // Transfer the fixups of LU to LUThatHas.
5068 for (LSRFixup &Fixup : LU.Fixups) {
5069 Fixup.Offset += F.BaseOffset;
5070 LUThatHas->pushFixup(f&: Fixup);
5071 LLVM_DEBUG(dbgs() << "New fixup has offset " << Fixup.Offset << '\n');
5072 }
5073
5074#ifndef NDEBUG
5075 Type *FixupType = LUThatHas->Fixups[0].OperandValToReplace->getType();
5076 for (LSRFixup &Fixup : LUThatHas->Fixups)
5077 assert(Fixup.OperandValToReplace->getType() == FixupType &&
5078 "Expected all fixups to have the same type");
5079#endif
5080
5081 // Delete formulae from the new use which are no longer legal.
5082 bool Any = false;
5083 for (size_t i = 0, e = LUThatHas->Formulae.size(); i != e; ++i) {
5084 Formula &F = LUThatHas->Formulae[i];
5085 if (!isLegalUse(TTI, MinOffset: LUThatHas->MinOffset, MaxOffset: LUThatHas->MaxOffset,
5086 Kind: LUThatHas->Kind, AccessTy: LUThatHas->AccessTy, F)) {
5087 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n');
5088 LUThatHas->DeleteFormula(F);
5089 --i;
5090 --e;
5091 Any = true;
5092 }
5093 }
5094
5095 if (Any)
5096 LUThatHas->RecomputeRegs(LUIdx: LUThatHas - &Uses.front(), RegUses);
5097
5098 // Delete the old use.
5099 DeleteUse(LU, LUIdx);
5100 --LUIdx;
5101 --NumUses;
5102 break;
5103 }
5104 }
5105
5106 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5107}
5108
5109/// Call FilterOutUndesirableDedicatedRegisters again, if necessary, now that
5110/// we've done more filtering, as it may be able to find more formulae to
5111/// eliminate.
5112void LSRInstance::NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters(){
5113 if (EstimateSearchSpaceComplexity() >= Opts.lsr_complexity_limit) {
5114 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5115
5116 LLVM_DEBUG(dbgs() << "Narrowing the search space by re-filtering out "
5117 "undesirable dedicated registers.\n");
5118
5119 FilterOutUndesirableDedicatedRegisters();
5120
5121 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5122 }
5123}
5124
5125/// If a LSRUse has multiple formulae with the same ScaledReg and Scale.
5126/// Pick the best one and delete the others.
5127/// This narrowing heuristic is to keep as many formulae with different
5128/// Scale and ScaledReg pair as possible while narrowing the search space.
5129/// The benefit is that it is more likely to find out a better solution
5130/// from a formulae set with more Scale and ScaledReg variations than
5131/// a formulae set with the same Scale and ScaledReg. The picking winner
5132/// reg heuristic will often keep the formulae with the same Scale and
5133/// ScaledReg and filter others, and we want to avoid that if possible.
5134void LSRInstance::NarrowSearchSpaceByFilterFormulaWithSameScaledReg() {
5135 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5136 return;
5137
5138 LLVM_DEBUG(
5139 dbgs() << "The search space is too complex.\n"
5140 "Narrowing the search space by choosing the best Formula "
5141 "from the Formulae with the same Scale and ScaledReg.\n");
5142
5143 // Map the "Scale * ScaledReg" pair to the best formula of current LSRUse.
5144 using BestFormulaeTy = DenseMap<std::pair<const SCEV *, int64_t>, size_t>;
5145
5146 BestFormulaeTy BestFormulae;
5147#ifndef NDEBUG
5148 bool ChangedFormulae = false;
5149#endif
5150 DenseSet<const SCEV *> VisitedRegs;
5151 SmallPtrSet<const SCEV *, 16> Regs;
5152
5153 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5154 LSRUse &LU = Uses[LUIdx];
5155 LLVM_DEBUG(dbgs() << "Filtering for use "; LU.print(dbgs());
5156 dbgs() << '\n');
5157
5158 // Return true if Formula FA is better than Formula FB.
5159 auto IsBetterThan = [&](Formula &FA, Formula &FB) {
5160 // First we will try to choose the Formula with fewer new registers.
5161 // For a register used by current Formula, the more the register is
5162 // shared among LSRUses, the less we increase the register number
5163 // counter of the formula.
5164 size_t FARegNum = 0;
5165 for (const SCEV *Reg : FA.BaseRegs) {
5166 const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg);
5167 FARegNum += (NumUses - UsedByIndices.count() + 1);
5168 }
5169 size_t FBRegNum = 0;
5170 for (const SCEV *Reg : FB.BaseRegs) {
5171 const SmallBitVector &UsedByIndices = RegUses.getUsedByIndices(Reg);
5172 FBRegNum += (NumUses - UsedByIndices.count() + 1);
5173 }
5174 if (FARegNum != FBRegNum)
5175 return FARegNum < FBRegNum;
5176
5177 // If the new register numbers are the same, choose the Formula with
5178 // less Cost.
5179 Cost CostFA(Opts, L, SE, TTI, AMK);
5180 Cost CostFB(Opts, L, SE, TTI, AMK);
5181 Regs.clear();
5182 CostFA.RateFormula(F: FA, Regs, VisitedRegs, LU, HardwareLoopProfitable);
5183 Regs.clear();
5184 CostFB.RateFormula(F: FB, Regs, VisitedRegs, LU, HardwareLoopProfitable);
5185 return CostFA.isLess(Other: CostFB);
5186 };
5187
5188 bool Any = false;
5189 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
5190 ++FIdx) {
5191 Formula &F = LU.Formulae[FIdx];
5192 if (!F.ScaledReg)
5193 continue;
5194 auto P = BestFormulae.insert(KV: {{F.ScaledReg, F.Scale}, FIdx});
5195 if (P.second)
5196 continue;
5197
5198 Formula &Best = LU.Formulae[P.first->second];
5199 if (IsBetterThan(F, Best))
5200 std::swap(a&: F, b&: Best);
5201 LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
5202 dbgs() << "\n"
5203 " in favor of formula ";
5204 Best.print(dbgs()); dbgs() << '\n');
5205#ifndef NDEBUG
5206 ChangedFormulae = true;
5207#endif
5208 LU.DeleteFormula(F);
5209 --FIdx;
5210 --NumForms;
5211 Any = true;
5212 }
5213 if (Any)
5214 LU.RecomputeRegs(LUIdx, RegUses);
5215
5216 // Reset this to prepare for the next use.
5217 BestFormulae.clear();
5218 }
5219
5220 LLVM_DEBUG(if (ChangedFormulae) {
5221 dbgs() << "\n"
5222 "After filtering out undesirable candidates:\n";
5223 print_uses(dbgs());
5224 });
5225}
5226
5227/// If we are over the complexity limit, filter out any post-inc prefering
5228/// variables to only post-inc values.
5229void LSRInstance::NarrowSearchSpaceByFilterPostInc() {
5230 if (AMK != TTI::AMK_PostIndexed)
5231 return;
5232 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5233 return;
5234
5235 LLVM_DEBUG(dbgs() << "The search space is too complex.\n"
5236 "Narrowing the search space by choosing the lowest "
5237 "register Formula for PostInc Uses.\n");
5238
5239 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5240 LSRUse &LU = Uses[LUIdx];
5241
5242 if (LU.Kind != LSRUse::Address)
5243 continue;
5244 if (!TTI.isIndexedLoadLegal(Mode: TTI.MIM_PostInc, Ty: LU.AccessTy.getType()) &&
5245 !TTI.isIndexedStoreLegal(Mode: TTI.MIM_PostInc, Ty: LU.AccessTy.getType()))
5246 continue;
5247
5248 size_t MinRegs = std::numeric_limits<size_t>::max();
5249 for (const Formula &F : LU.Formulae)
5250 MinRegs = std::min(a: F.getNumRegs(), b: MinRegs);
5251
5252 bool Any = false;
5253 for (size_t FIdx = 0, NumForms = LU.Formulae.size(); FIdx != NumForms;
5254 ++FIdx) {
5255 Formula &F = LU.Formulae[FIdx];
5256 if (F.getNumRegs() > MinRegs) {
5257 LLVM_DEBUG(dbgs() << " Filtering out formula "; F.print(dbgs());
5258 dbgs() << "\n");
5259 LU.DeleteFormula(F);
5260 --FIdx;
5261 --NumForms;
5262 Any = true;
5263 }
5264 }
5265 if (Any)
5266 LU.RecomputeRegs(LUIdx, RegUses);
5267
5268 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5269 break;
5270 }
5271
5272 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5273}
5274
5275void LSRInstance::NarrowSearchSpaceByMergingUsesOutsideLoop() {
5276 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5277 return;
5278
5279 LLVM_DEBUG(
5280 dbgs() << "The search space is too complex.\n"
5281 "Narrowing the search space by merging uses with fixups "
5282 "entirely outside the loop with uses inside the loop.\n");
5283
5284 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5285 LSRUse &LU = Uses[LUIdx];
5286 // Don't merge ICmpZero uses outside the loop, as ICmpZero needs to be
5287 // handled specially when expanding.
5288 if (!LU.AllFixupsOutsideLoop || LU.Formulae.empty() ||
5289 LU.Kind == LSRUse::ICmpZero)
5290 continue;
5291
5292 LLVM_DEBUG(dbgs() << " Trying to eliminate use "; LU.print(dbgs());
5293 dbgs() << '\n');
5294
5295 // Find a compatible LSRUse inside the loop that we could merge LU with
5296 LSRUse *LUToMergeWith = nullptr;
5297 const Formula &ThisF = LU.Formulae[0];
5298 for (LSRUse &OtherLU : Uses) {
5299 // Only merge with uses inside the loop
5300 if (OtherLU.AllFixupsOutsideLoop)
5301 continue;
5302 // Can't merge with ICmpZero uses as they're handled specially when
5303 // expanding
5304 if (OtherLU.Kind == LSRUse::ICmpZero)
5305 continue;
5306 // Can't merge with uses without any formulae
5307 if (OtherLU.Formulae.empty())
5308 continue;
5309 // Can't merge if LU's offsets aren't legal for all of OtherLU's formulae
5310 if (any_of(Range&: OtherLU.Formulae, P: [&](const Formula &F) {
5311 return !isLegalUse(TTI, MinOffset: LU.MinOffset, MaxOffset: LU.MaxOffset, Kind: OtherLU.Kind,
5312 AccessTy: OtherLU.AccessTy, F);
5313 }))
5314 continue;
5315 // We can merge with uses that have the same initial formula. We allow
5316 // merging of uses with different Kind and AccessTy which means that the
5317 // cost may end up being inaccurate, but it's also what we would have
5318 // gotten if we'd ignored uses outside the loop entirely.
5319 const Formula &OtherF = OtherLU.Formulae[0];
5320 if (ThisF.BaseRegs == OtherF.BaseRegs &&
5321 ThisF.ScaledReg == OtherF.ScaledReg &&
5322 ThisF.BaseGV == OtherF.BaseGV && ThisF.Scale == OtherF.Scale &&
5323 ThisF.UnfoldedOffset == OtherF.UnfoldedOffset &&
5324 ThisF.BaseOffset == OtherF.BaseOffset) {
5325 LUToMergeWith = &OtherLU;
5326 break;
5327 }
5328 }
5329 if (!LUToMergeWith)
5330 continue;
5331
5332 LLVM_DEBUG(dbgs() << " Merging with "; LUToMergeWith->print(dbgs());
5333 dbgs() << '\n');
5334
5335 // Copy fixups
5336 for (LSRFixup &Fixup : LU.Fixups) {
5337 LUToMergeWith->pushFixup(f&: Fixup);
5338 }
5339
5340 // Delete the old use.
5341 DeleteUse(LU, LUIdx);
5342 --LUIdx;
5343 --NumUses;
5344 }
5345
5346 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5347}
5348
5349/// The function delete formulas with high registers number expectation.
5350/// Assuming we don't know the value of each formula (already delete
5351/// all inefficient), generate probability of not selecting for each
5352/// register.
5353/// For example,
5354/// Use1:
5355/// reg(a) + reg({0,+,1})
5356/// reg(a) + reg({-1,+,1}) + 1
5357/// reg({a,+,1})
5358/// Use2:
5359/// reg(b) + reg({0,+,1})
5360/// reg(b) + reg({-1,+,1}) + 1
5361/// reg({b,+,1})
5362/// Use3:
5363/// reg(c) + reg(b) + reg({0,+,1})
5364/// reg(c) + reg({b,+,1})
5365///
5366/// Probability of not selecting
5367/// Use1 Use2 Use3
5368/// reg(a) (1/3) * 1 * 1
5369/// reg(b) 1 * (1/3) * (1/2)
5370/// reg({0,+,1}) (2/3) * (2/3) * (1/2)
5371/// reg({-1,+,1}) (2/3) * (2/3) * 1
5372/// reg({a,+,1}) (2/3) * 1 * 1
5373/// reg({b,+,1}) 1 * (2/3) * (2/3)
5374/// reg(c) 1 * 1 * 0
5375///
5376/// Now count registers number mathematical expectation for each formula:
5377/// Note that for each use we exclude probability if not selecting for the use.
5378/// For example for Use1 probability for reg(a) would be just 1 * 1 (excluding
5379/// probabilty 1/3 of not selecting for Use1).
5380/// Use1:
5381/// reg(a) + reg({0,+,1}) 1 + 1/3 -- to be deleted
5382/// reg(a) + reg({-1,+,1}) + 1 1 + 4/9 -- to be deleted
5383/// reg({a,+,1}) 1
5384/// Use2:
5385/// reg(b) + reg({0,+,1}) 1/2 + 1/3 -- to be deleted
5386/// reg(b) + reg({-1,+,1}) + 1 1/2 + 2/3 -- to be deleted
5387/// reg({b,+,1}) 2/3
5388/// Use3:
5389/// reg(c) + reg(b) + reg({0,+,1}) 1 + 1/3 + 4/9 -- to be deleted
5390/// reg(c) + reg({b,+,1}) 1 + 2/3
5391void LSRInstance::NarrowSearchSpaceByDeletingCostlyFormulas() {
5392 if (EstimateSearchSpaceComplexity() < Opts.lsr_complexity_limit)
5393 return;
5394 // Ok, we have too many of formulae on our hands to conveniently handle.
5395 // Use a rough heuristic to thin out the list.
5396
5397 // Set of Regs wich will be 100% used in final solution.
5398 // Used in each formula of a solution (in example above this is reg(c)).
5399 // We can skip them in calculations.
5400 SmallPtrSet<const SCEV *, 4> UniqRegs;
5401 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5402
5403 // Map each register to probability of not selecting
5404 DenseMap <const SCEV *, float> RegNumMap;
5405 for (const SCEV *Reg : RegUses) {
5406 if (UniqRegs.count(Ptr: Reg))
5407 continue;
5408 float PNotSel = 1;
5409 for (const LSRUse &LU : Uses) {
5410 if (!LU.Regs.count(Ptr: Reg))
5411 continue;
5412 float P = LU.getNotSelectedProbability(Reg);
5413 if (P != 0.0)
5414 PNotSel *= P;
5415 else
5416 UniqRegs.insert(Ptr: Reg);
5417 }
5418 RegNumMap.insert(KV: std::make_pair(x&: Reg, y&: PNotSel));
5419 }
5420
5421 LLVM_DEBUG(
5422 dbgs() << "Narrowing the search space by deleting costly formulas\n");
5423
5424 // Delete formulas where registers number expectation is high.
5425 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5426 LSRUse &LU = Uses[LUIdx];
5427 // If nothing to delete - continue.
5428 if (LU.Formulae.size() < 2)
5429 continue;
5430 // This is temporary solution to test performance. Float should be
5431 // replaced with round independent type (based on integers) to avoid
5432 // different results for different target builds.
5433 float FMinRegNum = LU.Formulae[0].getNumRegs();
5434 float FMinARegNum = LU.Formulae[0].getNumRegs();
5435 size_t MinIdx = 0;
5436 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
5437 Formula &F = LU.Formulae[i];
5438 float FRegNum = 0;
5439 float FARegNum = 0;
5440 for (const SCEV *BaseReg : F.BaseRegs) {
5441 if (UniqRegs.count(Ptr: BaseReg))
5442 continue;
5443 FRegNum += RegNumMap[BaseReg] / LU.getNotSelectedProbability(Reg: BaseReg);
5444 if (isa<SCEVAddRecExpr>(Val: BaseReg))
5445 FARegNum +=
5446 RegNumMap[BaseReg] / LU.getNotSelectedProbability(Reg: BaseReg);
5447 }
5448 if (const SCEV *ScaledReg = F.ScaledReg) {
5449 if (!UniqRegs.count(Ptr: ScaledReg)) {
5450 FRegNum +=
5451 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(Reg: ScaledReg);
5452 if (isa<SCEVAddRecExpr>(Val: ScaledReg))
5453 FARegNum +=
5454 RegNumMap[ScaledReg] / LU.getNotSelectedProbability(Reg: ScaledReg);
5455 }
5456 }
5457 if (FMinRegNum > FRegNum ||
5458 (FMinRegNum == FRegNum && FMinARegNum > FARegNum)) {
5459 FMinRegNum = FRegNum;
5460 FMinARegNum = FARegNum;
5461 MinIdx = i;
5462 }
5463 }
5464 LLVM_DEBUG(dbgs() << " The formula "; LU.Formulae[MinIdx].print(dbgs());
5465 dbgs() << " with min reg num " << FMinRegNum << '\n');
5466 if (MinIdx != 0)
5467 std::swap(a&: LU.Formulae[MinIdx], b&: LU.Formulae[0]);
5468 while (LU.Formulae.size() != 1) {
5469 LLVM_DEBUG(dbgs() << " Deleting "; LU.Formulae.back().print(dbgs());
5470 dbgs() << '\n');
5471 LU.Formulae.pop_back();
5472 }
5473 LU.RecomputeRegs(LUIdx, RegUses);
5474 assert(LU.Formulae.size() == 1 && "Should be exactly 1 min regs formula");
5475 Formula &F = LU.Formulae[0];
5476 LLVM_DEBUG(dbgs() << " Leaving only "; F.print(dbgs()); dbgs() << '\n');
5477 // When we choose the formula, the regs become unique.
5478 UniqRegs.insert_range(R&: F.BaseRegs);
5479 if (F.ScaledReg)
5480 UniqRegs.insert(Ptr: F.ScaledReg);
5481 }
5482 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5483}
5484
5485// Check if Best and Reg are SCEVs separated by a constant amount C, and if so
5486// would the addressing offset +C would be legal where the negative offset -C is
5487// not.
5488static bool IsSimplerBaseSCEVForTarget(const TargetTransformInfo &TTI,
5489 ScalarEvolution &SE, const SCEV *Best,
5490 const SCEV *Reg,
5491 MemAccessTy AccessType) {
5492 if (Best->getType() != Reg->getType() ||
5493 (isa<SCEVAddRecExpr>(Val: Best) && isa<SCEVAddRecExpr>(Val: Reg) &&
5494 cast<SCEVAddRecExpr>(Val: Best)->getLoop() !=
5495 cast<SCEVAddRecExpr>(Val: Reg)->getLoop()))
5496 return false;
5497 std::optional<APInt> Diff = SE.computeConstantDifference(LHS: Best, RHS: Reg);
5498 if (!Diff)
5499 return false;
5500
5501 return TTI.isLegalAddressingMode(
5502 Ty: AccessType.MemTy, /*BaseGV=*/nullptr,
5503 /*BaseOffset=*/Diff->getSExtValue(),
5504 /*HasBaseReg=*/true, /*Scale=*/0, AddrSpace: AccessType.AddrSpace) &&
5505 !TTI.isLegalAddressingMode(
5506 Ty: AccessType.MemTy, /*BaseGV=*/nullptr,
5507 /*BaseOffset=*/-Diff->getSExtValue(),
5508 /*HasBaseReg=*/true, /*Scale=*/0, AddrSpace: AccessType.AddrSpace);
5509}
5510
5511/// Pick a register which seems likely to be profitable, and then in any use
5512/// which has any reference to that register, delete all formulae which do not
5513/// reference that register.
5514void LSRInstance::NarrowSearchSpaceByPickingWinnerRegs() {
5515 // With all other options exhausted, loop until the system is simple
5516 // enough to handle.
5517 SmallPtrSet<const SCEV *, 4> Taken;
5518 while (EstimateSearchSpaceComplexity() >= Opts.lsr_complexity_limit) {
5519 // Ok, we have too many of formulae on our hands to conveniently handle.
5520 // Use a rough heuristic to thin out the list.
5521 LLVM_DEBUG(dbgs() << "The search space is too complex.\n");
5522
5523 // Pick the register which is used by the most LSRUses, which is likely
5524 // to be a good reuse register candidate.
5525 const SCEV *Best = nullptr;
5526 unsigned BestNum = 0;
5527 for (const SCEV *Reg : RegUses) {
5528 if (Taken.count(Ptr: Reg))
5529 continue;
5530 if (!Best) {
5531 Best = Reg;
5532 BestNum = RegUses.getUsedByIndices(Reg).count();
5533 } else {
5534 unsigned Count = RegUses.getUsedByIndices(Reg).count();
5535 if (Count > BestNum) {
5536 Best = Reg;
5537 BestNum = Count;
5538 }
5539
5540 // If the scores are the same, but the Reg is simpler for the target
5541 // (for example {x,+,1} as opposed to {x+C,+,1}, where the target can
5542 // handle +C but not -C), opt for the simpler formula.
5543 if (Count == BestNum) {
5544 int LUIdx = RegUses.getUsedByIndices(Reg).find_first();
5545 if (LUIdx >= 0 && Uses[LUIdx].Kind == LSRUse::Address &&
5546 IsSimplerBaseSCEVForTarget(TTI, SE, Best, Reg,
5547 AccessType: Uses[LUIdx].AccessTy)) {
5548 Best = Reg;
5549 BestNum = Count;
5550 }
5551 }
5552 }
5553 }
5554 assert(Best && "Failed to find best LSRUse candidate");
5555
5556 LLVM_DEBUG(dbgs() << "Narrowing the search space by assuming " << *Best
5557 << " will yield profitable reuse.\n");
5558 Taken.insert(Ptr: Best);
5559
5560 // In any use with formulae which references this register, delete formulae
5561 // which don't reference it.
5562 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx) {
5563 LSRUse &LU = Uses[LUIdx];
5564 if (!LU.Regs.count(Ptr: Best)) continue;
5565
5566 bool Any = false;
5567 for (size_t i = 0, e = LU.Formulae.size(); i != e; ++i) {
5568 Formula &F = LU.Formulae[i];
5569 if (!F.referencesReg(S: Best)) {
5570 LLVM_DEBUG(dbgs() << " Deleting "; F.print(dbgs()); dbgs() << '\n');
5571 LU.DeleteFormula(F);
5572 --e;
5573 --i;
5574 Any = true;
5575 assert(e != 0 && "Use has no formulae left! Is Regs inconsistent?");
5576 continue;
5577 }
5578 }
5579
5580 if (Any)
5581 LU.RecomputeRegs(LUIdx, RegUses);
5582 }
5583
5584 LLVM_DEBUG(dbgs() << "After pre-selection:\n"; print_uses(dbgs()));
5585 }
5586}
5587
5588/// If there are an extraordinary number of formulae to choose from, use some
5589/// rough heuristics to prune down the number of formulae. This keeps the main
5590/// solver from taking an extraordinary amount of time in some worst-case
5591/// scenarios.
5592void LSRInstance::NarrowSearchSpaceUsingHeuristics() {
5593 NarrowSearchSpaceByDetectingSupersets();
5594 NarrowSearchSpaceByCollapsingUnrolledCode();
5595 NarrowSearchSpaceByRefilteringUndesirableDedicatedRegisters();
5596 if (Opts.lsr_filter_same_scaled_reg)
5597 NarrowSearchSpaceByFilterFormulaWithSameScaledReg();
5598 NarrowSearchSpaceByFilterPostInc();
5599 NarrowSearchSpaceByMergingUsesOutsideLoop();
5600 if (Opts.lsr_exp_narrow)
5601 NarrowSearchSpaceByDeletingCostlyFormulas();
5602 else
5603 NarrowSearchSpaceByPickingWinnerRegs();
5604}
5605
5606/// This is the recursive solver.
5607void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
5608 Cost &SolutionCost,
5609 SmallVectorImpl<const Formula *> &Workspace,
5610 const Cost &CurCost,
5611 const SmallPtrSet<const SCEV *, 16> &CurRegs,
5612 DenseSet<const SCEV *> &VisitedRegs) const {
5613 // Some ideas:
5614 // - prune more:
5615 // - use more aggressive filtering
5616 // - sort the formula so that the most profitable solutions are found first
5617 // - sort the uses too
5618 // - search faster:
5619 // - don't compute a cost, and then compare. compare while computing a cost
5620 // and bail early.
5621 // - track register sets with SmallBitVector
5622
5623 const LSRUse &LU = Uses[Workspace.size()];
5624
5625 // If this use references any register that's already a part of the
5626 // in-progress solution, consider it a requirement that a formula must
5627 // reference that register in order to be considered. This prunes out
5628 // unprofitable searching.
5629 SmallSetVector<const SCEV *, 4> ReqRegs;
5630 for (const SCEV *S : CurRegs)
5631 if (LU.Regs.count(Ptr: S))
5632 ReqRegs.insert(X: S);
5633
5634 SmallPtrSet<const SCEV *, 16> NewRegs;
5635 Cost NewCost(Opts, L, SE, TTI, AMK);
5636 for (const Formula &F : LU.Formulae) {
5637 // Ignore formulae which may not be ideal in terms of register reuse of
5638 // ReqRegs. The formula should use all required registers before
5639 // introducing new ones.
5640 // This can sometimes (notably when trying to favour postinc) lead to
5641 // sub-optimial decisions. There it is best left to the cost modelling to
5642 // get correct.
5643 if (!(AMK & TTI::AMK_PostIndexed) || LU.Kind != LSRUse::Address) {
5644 int NumReqRegsToFind = std::min(a: F.getNumRegs(), b: ReqRegs.size());
5645 for (const SCEV *Reg : ReqRegs) {
5646 if ((F.ScaledReg && F.ScaledReg == Reg) ||
5647 is_contained(Range: F.BaseRegs, Element: Reg)) {
5648 --NumReqRegsToFind;
5649 if (NumReqRegsToFind == 0)
5650 break;
5651 }
5652 }
5653 if (NumReqRegsToFind != 0) {
5654 // If none of the formulae satisfied the required registers, then we could
5655 // clear ReqRegs and try again. Currently, we simply give up in this case.
5656 continue;
5657 }
5658 }
5659
5660 // Evaluate the cost of the current formula. If it's already worse than
5661 // the current best, prune the search at that point.
5662 NewCost = CurCost;
5663 NewRegs = CurRegs;
5664 NewCost.RateFormula(F, Regs&: NewRegs, VisitedRegs, LU, HardwareLoopProfitable);
5665 if (NewCost.isLess(Other: SolutionCost)) {
5666 Workspace.push_back(Elt: &F);
5667 if (Workspace.size() != Uses.size()) {
5668 SolveRecurse(Solution, SolutionCost, Workspace, CurCost: NewCost,
5669 CurRegs: NewRegs, VisitedRegs);
5670 if (F.getNumRegs() == 1 && Workspace.size() == 1)
5671 VisitedRegs.insert(V: F.ScaledReg ? F.ScaledReg : F.BaseRegs[0]);
5672 } else {
5673 LLVM_DEBUG(dbgs() << "New best at "; NewCost.print(dbgs());
5674 dbgs() << ".\nRegs:\n";
5675 for (const SCEV *S : NewRegs) dbgs()
5676 << "- " << *S << "\n";
5677 dbgs() << '\n');
5678
5679 SolutionCost = NewCost;
5680 Solution = Workspace;
5681 }
5682 Workspace.pop_back();
5683 }
5684 }
5685}
5686
5687/// Choose one formula from each use. Return the results in the given Solution
5688/// vector.
5689void LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const {
5690 SmallVector<const Formula *, 8> Workspace;
5691 Cost SolutionCost(Opts, L, SE, TTI, AMK);
5692 SolutionCost.Lose();
5693 Cost CurCost(Opts, L, SE, TTI, AMK);
5694 SmallPtrSet<const SCEV *, 16> CurRegs;
5695 DenseSet<const SCEV *> VisitedRegs;
5696 Workspace.reserve(N: Uses.size());
5697
5698 // SolveRecurse does all the work.
5699 SolveRecurse(Solution, SolutionCost, Workspace, CurCost,
5700 CurRegs, VisitedRegs);
5701 if (Solution.empty()) {
5702 LLVM_DEBUG(dbgs() << "\nNo Satisfactory Solution\n");
5703 return;
5704 }
5705
5706 // Ok, we've now made all our decisions.
5707 LLVM_DEBUG(dbgs() << "\n"
5708 "The chosen solution requires ";
5709 SolutionCost.print(dbgs()); dbgs() << ":\n";
5710 for (size_t i = 0, e = Uses.size(); i != e; ++i) {
5711 dbgs() << " ";
5712 Uses[i].print(dbgs());
5713 dbgs() << "\n"
5714 " ";
5715 Solution[i]->print(dbgs());
5716 dbgs() << '\n';
5717 });
5718
5719 assert(Solution.size() == Uses.size() && "Malformed solution!");
5720
5721 const bool EnableDropUnprofitableSolution = valueOr(
5722 X: Opts.lsr_drop_solution, Default: TTI.shouldDropLSRSolutionIfLessProfitable());
5723
5724 if (BaselineCost.isLess(Other: SolutionCost)) {
5725 if (!EnableDropUnprofitableSolution)
5726 LLVM_DEBUG(
5727 dbgs() << "Baseline is more profitable than chosen solution, "
5728 "add option 'lsr-drop-solution' to drop LSR solution.\n");
5729 else {
5730 LLVM_DEBUG(dbgs() << "Baseline is more profitable than chosen "
5731 "solution, dropping LSR solution.\n";);
5732 Solution.clear();
5733 }
5734 }
5735}
5736
5737/// Helper for AdjustInsertPositionForExpand. Climb up the dominator tree far as
5738/// we can go while still being dominated by the input positions. This helps
5739/// canonicalize the insert position, which encourages sharing.
5740BasicBlock::iterator
5741LSRInstance::HoistInsertPosition(BasicBlock::iterator IP,
5742 const SmallVectorImpl<Instruction *> &Inputs)
5743 const {
5744 Instruction *Tentative = &*IP;
5745 while (true) {
5746 bool AllDominate = true;
5747 Instruction *BetterPos = nullptr;
5748 // Don't bother attempting to insert before a catchswitch, their basic block
5749 // cannot have other non-PHI instructions.
5750 if (isa<CatchSwitchInst>(Val: Tentative))
5751 return IP;
5752
5753 for (Instruction *Inst : Inputs) {
5754 if (Inst == Tentative || !DT.dominates(Def: Inst, User: Tentative)) {
5755 AllDominate = false;
5756 break;
5757 }
5758 // Attempt to find an insert position in the middle of the block,
5759 // instead of at the end, so that it can be used for other expansions.
5760 if (Tentative->getParent() == Inst->getParent() &&
5761 (!BetterPos || !DT.dominates(Def: Inst, User: BetterPos)))
5762 BetterPos = &*std::next(x: BasicBlock::iterator(Inst));
5763 }
5764 if (!AllDominate)
5765 break;
5766 if (BetterPos)
5767 IP = BetterPos->getIterator();
5768 else
5769 IP = Tentative->getIterator();
5770
5771 const Loop *IPLoop = LI.getLoopFor(BB: IP->getParent());
5772 unsigned IPLoopDepth = IPLoop ? IPLoop->getLoopDepth() : 0;
5773
5774 BasicBlock *IDom;
5775 for (DomTreeNode *Rung = DT.getNode(BB: IP->getParent()); ; ) {
5776 if (!Rung) return IP;
5777 Rung = Rung->getIDom();
5778 if (!Rung) return IP;
5779 IDom = Rung->getBlock();
5780
5781 // Don't climb into a loop though.
5782 const Loop *IDomLoop = LI.getLoopFor(BB: IDom);
5783 unsigned IDomDepth = IDomLoop ? IDomLoop->getLoopDepth() : 0;
5784 if (IDomDepth <= IPLoopDepth &&
5785 (IDomDepth != IPLoopDepth || IDomLoop == IPLoop))
5786 break;
5787 }
5788
5789 Tentative = IDom->getTerminator();
5790 }
5791
5792 return IP;
5793}
5794
5795/// Determine an input position which will be dominated by the operands and
5796/// which will dominate the result.
5797BasicBlock::iterator LSRInstance::AdjustInsertPositionForExpand(
5798 BasicBlock::iterator LowestIP, const LSRFixup &LF, const LSRUse &LU) const {
5799 // Collect some instructions which must be dominated by the
5800 // expanding replacement. These must be dominated by any operands that
5801 // will be required in the expansion.
5802 SmallVector<Instruction *, 4> Inputs;
5803 if (Instruction *I = dyn_cast<Instruction>(Val: LF.OperandValToReplace))
5804 Inputs.push_back(Elt: I);
5805 if (LU.Kind == LSRUse::ICmpZero)
5806 if (Instruction *I =
5807 dyn_cast<Instruction>(Val: cast<ICmpInst>(Val: LF.UserInst)->getOperand(i_nocapture: 1)))
5808 Inputs.push_back(Elt: I);
5809 if (LF.PostIncLoops.count(Ptr: L)) {
5810 if (LF.isUseFullyOutsideLoop(L))
5811 Inputs.push_back(Elt: L->getLoopLatch()->getTerminator());
5812 else
5813 Inputs.push_back(Elt: IVIncInsertPos);
5814 }
5815 // The expansion must also be dominated by the increment positions of any
5816 // loops it for which it is using post-inc mode.
5817 for (const Loop *PIL : LF.PostIncLoops) {
5818 if (PIL == L) continue;
5819
5820 // Be dominated by the loop exit.
5821 SmallVector<BasicBlock *, 4> ExitingBlocks;
5822 PIL->getExitingBlocks(ExitingBlocks);
5823 if (!ExitingBlocks.empty()) {
5824 BasicBlock *BB = ExitingBlocks[0];
5825 for (unsigned i = 1, e = ExitingBlocks.size(); i != e; ++i)
5826 BB = DT.findNearestCommonDominator(A: BB, B: ExitingBlocks[i]);
5827 Inputs.push_back(Elt: BB->getTerminator());
5828 }
5829 }
5830
5831 assert(!isa<PHINode>(LowestIP) && !LowestIP->isEHPad() &&
5832 "Insertion point must be a normal instruction");
5833
5834 // Then, climb up the immediate dominator tree as far as we can go while
5835 // still being dominated by the input positions.
5836 BasicBlock::iterator IP = HoistInsertPosition(IP: LowestIP, Inputs);
5837
5838 // Don't insert instructions before PHI nodes.
5839 while (isa<PHINode>(Val: IP)) ++IP;
5840
5841 // Ignore landingpad instructions.
5842 while (IP->isEHPad()) ++IP;
5843
5844 // Set IP below instructions recently inserted by SCEVExpander. This keeps the
5845 // IP consistent across expansions and allows the previously inserted
5846 // instructions to be reused by subsequent expansion.
5847 while (Rewriter.isInsertedInstruction(I: &*IP) && IP != LowestIP)
5848 ++IP;
5849
5850 return IP;
5851}
5852
5853/// Emit instructions for the leading candidate expression for this LSRUse (this
5854/// is called "expanding").
5855Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF,
5856 const Formula &F, BasicBlock::iterator IP,
5857 SmallVectorImpl<WeakTrackingVH> &DeadInsts) const {
5858 if (LU.RigidFormula)
5859 return LF.OperandValToReplace;
5860
5861 // Determine an input position which will be dominated by the operands and
5862 // which will dominate the result.
5863 IP = AdjustInsertPositionForExpand(LowestIP: IP, LF, LU);
5864 Rewriter.setInsertPoint(&*IP);
5865
5866 // Inform the Rewriter if we have a post-increment use, so that it can
5867 // perform an advantageous expansion.
5868 Rewriter.setPostInc(LF.PostIncLoops);
5869
5870 // This is the type that the user actually needs.
5871 Type *OpTy = LF.OperandValToReplace->getType();
5872 // This will be the type that we'll initially expand to.
5873 Type *Ty = F.getType();
5874 if (!Ty)
5875 // No type known; just expand directly to the ultimate type.
5876 Ty = OpTy;
5877 else if (SE.getEffectiveSCEVType(Ty) == SE.getEffectiveSCEVType(Ty: OpTy))
5878 // Expand directly to the ultimate type if it's the right size.
5879 Ty = OpTy;
5880 // This is the type to do integer arithmetic in.
5881 Type *IntTy = SE.getEffectiveSCEVType(Ty);
5882 // For ICmpZero with pointer-typed operands, keep the comparison in the
5883 // integer domain to avoid generating inttoptr casts. Use IntTy (the
5884 // formula's arithmetic width) so that both icmp operands match even when
5885 // the IV is wider than the pointer.
5886 if (LU.Kind == LSRUse::ICmpZero && OpTy->isPointerTy()) {
5887 OpTy = IntTy;
5888 Ty = IntTy;
5889 }
5890
5891 // Build up a list of operands to add together to form the full base.
5892 SmallVector<SCEVUse, 8> Ops;
5893
5894 // Expand the BaseRegs portion.
5895 for (const SCEV *Reg : F.BaseRegs) {
5896 assert(!Reg->isZero() && "Zero allocated in a base register!");
5897
5898 // If we're expanding for a post-inc user, make the post-inc adjustment.
5899 Reg = denormalizeForPostIncUse(S: Reg, Loops: LF.PostIncLoops, SE);
5900 Ops.push_back(Elt: SE.getUnknown(V: Rewriter.expandCodeFor(SH: Reg, Ty: nullptr)));
5901 }
5902
5903 // Expand the ScaledReg portion.
5904 Value *ICmpScaledV = nullptr;
5905 if (F.Scale != 0) {
5906 const SCEV *ScaledS = F.ScaledReg;
5907
5908 // If we're expanding for a post-inc user, make the post-inc adjustment.
5909 PostIncLoopSet &Loops = const_cast<PostIncLoopSet &>(LF.PostIncLoops);
5910 ScaledS = denormalizeForPostIncUse(S: ScaledS, Loops, SE);
5911
5912 if (LU.Kind == LSRUse::ICmpZero) {
5913 // Expand ScaleReg as if it was part of the base regs.
5914 if (F.Scale == 1)
5915 Ops.push_back(
5916 Elt: SE.getUnknown(V: Rewriter.expandCodeFor(SH: ScaledS, Ty: nullptr)));
5917 else {
5918 // An interesting way of "folding" with an icmp is to use a negated
5919 // scale, which we'll implement by inserting it into the other operand
5920 // of the icmp.
5921 assert(F.Scale == -1 &&
5922 "The only scale supported by ICmpZero uses is -1!");
5923 ICmpScaledV = Rewriter.expandCodeFor(SH: ScaledS, Ty: nullptr);
5924 }
5925 } else {
5926 // Otherwise just expand the scaled register and an explicit scale,
5927 // which is expected to be matched as part of the address.
5928
5929 // Flush the operand list to suppress SCEVExpander hoisting address modes.
5930 // Unless the addressing mode will not be folded.
5931 if (!Ops.empty() && LU.Kind == LSRUse::Address &&
5932 isAMCompletelyFolded(TTI, LU, F)) {
5933 Value *FullV = Rewriter.expandCodeFor(SH: SE.getAddExpr(Ops), Ty: nullptr);
5934 Ops.clear();
5935 Ops.push_back(Elt: SE.getUnknown(V: FullV));
5936 }
5937 ScaledS = SE.getUnknown(V: Rewriter.expandCodeFor(SH: ScaledS, Ty: nullptr));
5938 if (F.Scale != 1)
5939 ScaledS =
5940 SE.getMulExpr(LHS: ScaledS, RHS: SE.getConstant(Ty: ScaledS->getType(), V: F.Scale));
5941 Ops.push_back(Elt: ScaledS);
5942 }
5943 }
5944
5945 // Expand the GV portion.
5946 if (F.BaseGV) {
5947 // Flush the operand list to suppress SCEVExpander hoisting.
5948 if (!Ops.empty()) {
5949 Value *FullV = Rewriter.expandCodeFor(SH: SE.getAddExpr(Ops), Ty: IntTy);
5950 Ops.clear();
5951 Ops.push_back(Elt: SE.getUnknown(V: FullV));
5952 }
5953 Ops.push_back(Elt: SE.getUnknown(V: F.BaseGV));
5954 }
5955
5956 // Flush the operand list to suppress SCEVExpander hoisting of both folded and
5957 // unfolded offsets. LSR assumes they both live next to their uses.
5958 if (!Ops.empty()) {
5959 Value *FullV = Rewriter.expandCodeFor(SH: SE.getAddExpr(Ops), Ty);
5960 Ops.clear();
5961 Ops.push_back(Elt: SE.getUnknown(V: FullV));
5962 }
5963
5964 // FIXME: Are we sure we won't get a mismatch here? Is there a way to bail
5965 // out at this point, or should we generate a SCEV adding together mixed
5966 // offsets?
5967 assert(F.BaseOffset.isCompatibleImmediate(LF.Offset) &&
5968 "Expanding mismatched offsets\n");
5969 // Expand the immediate portion.
5970 Immediate Offset = F.BaseOffset.addUnsigned(RHS: LF.Offset);
5971 if (Offset.isNonZero()) {
5972 if (LU.Kind == LSRUse::ICmpZero) {
5973 // The other interesting way of "folding" with an ICmpZero is to use a
5974 // negated immediate.
5975 if (!ICmpScaledV) {
5976 // TODO: Avoid implicit trunc?
5977 // See https://github.com/llvm/llvm-project/issues/112510.
5978 ICmpScaledV = ConstantInt::getSigned(
5979 Ty: IntTy, V: -(uint64_t)Offset.getFixedValue(), /*ImplicitTrunc=*/true);
5980 } else {
5981 Ops.push_back(Elt: SE.getUnknown(V: ICmpScaledV));
5982 ICmpScaledV = ConstantInt::getSigned(Ty: IntTy, V: Offset.getFixedValue(),
5983 /*ImplicitTrunc=*/true);
5984 }
5985 } else {
5986 // Just add the immediate values. These again are expected to be matched
5987 // as part of the address.
5988 Ops.push_back(Elt: Offset.getUnknownSCEV(SE, Ty: IntTy));
5989 }
5990 }
5991
5992 // Expand the unfolded offset portion.
5993 Immediate UnfoldedOffset = F.UnfoldedOffset;
5994 if (UnfoldedOffset.isNonZero()) {
5995 // Just add the immediate values.
5996 Ops.push_back(Elt: UnfoldedOffset.getUnknownSCEV(SE, Ty: IntTy));
5997 }
5998
5999 // Emit instructions summing all the operands.
6000 const SCEV *FullS =
6001 Ops.empty() ? SE.getConstant(Ty: IntTy, V: 0) : SE.getAddExpr(Ops).getPointer();
6002 Value *FullV = Rewriter.expandCodeFor(SH: FullS, Ty);
6003
6004 // We're done expanding now, so reset the rewriter.
6005 Rewriter.clearPostInc();
6006
6007 // An ICmpZero Formula represents an ICmp which we're handling as a
6008 // comparison against zero. Now that we've expanded an expression for that
6009 // form, update the ICmp's other operand.
6010 if (LU.Kind == LSRUse::ICmpZero) {
6011 ICmpInst *CI = cast<ICmpInst>(Val: LF.UserInst);
6012 if (auto *OperandIsInstr = dyn_cast<Instruction>(Val: CI->getOperand(i_nocapture: 1)))
6013 DeadInsts.emplace_back(Args&: OperandIsInstr);
6014 assert(!F.BaseGV && "ICmp does not support folding a global value and "
6015 "a scale at the same time!");
6016 if (F.Scale == -1) {
6017 if (ICmpScaledV->getType() != OpTy) {
6018 Instruction *Cast = CastInst::Create(
6019 CastInst::getCastOpcode(Val: ICmpScaledV, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false),
6020 S: ICmpScaledV, Ty: OpTy, Name: "tmp", InsertBefore: CI->getIterator());
6021 ICmpScaledV = Cast;
6022 }
6023 CI->setOperand(i_nocapture: 1, Val_nocapture: ICmpScaledV);
6024 } else {
6025 // A scale of 1 means that the scale has been expanded as part of the
6026 // base regs.
6027 assert((F.Scale == 0 || F.Scale == 1) &&
6028 "ICmp does not support folding a global value and "
6029 "a scale at the same time!");
6030 // TODO: Avoid implicit trunc?
6031 // See https://github.com/llvm/llvm-project/issues/112510.
6032 Constant *C = ConstantInt::getSigned(Ty: SE.getEffectiveSCEVType(Ty: OpTy),
6033 V: -(uint64_t)Offset.getFixedValue(),
6034 /*ImplicitTrunc=*/true);
6035 if (C->getType() != OpTy) {
6036 C = ConstantFoldCastOperand(
6037 Opcode: CastInst::getCastOpcode(Val: C, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false), C, DestTy: OpTy,
6038 DL: CI->getDataLayout());
6039 assert(C && "Cast of ConstantInt should have folded");
6040 }
6041
6042 CI->setOperand(i_nocapture: 1, Val_nocapture: C);
6043 }
6044 }
6045
6046 return FullV;
6047}
6048
6049/// Helper for Rewrite. PHI nodes are special because the use of their operands
6050/// effectively happens in their predecessor blocks, so the expression may need
6051/// to be expanded in multiple places.
6052void LSRInstance::RewriteForPHI(PHINode *PN, const LSRUse &LU,
6053 const LSRFixup &LF, const Formula &F,
6054 SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
6055 DenseMap<BasicBlock *, Value *> Inserted;
6056
6057 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
6058 if (PN->getIncomingValue(i) == LF.OperandValToReplace) {
6059 bool needUpdateFixups = false;
6060 BasicBlock *BB = PN->getIncomingBlock(i);
6061
6062 // If this is a critical edge, split the edge so that we do not insert
6063 // the code on all predecessor/successor paths. We do this unless this
6064 // is the canonical backedge for this loop, which complicates post-inc
6065 // users.
6066 if (e != 1 && BB->getTerminator()->getNumSuccessors() > 1 &&
6067 !isa<IndirectBrInst>(Val: BB->getTerminator()) &&
6068 !isa<CatchSwitchInst>(Val: BB->getTerminator())) {
6069 BasicBlock *Parent = PN->getParent();
6070 Loop *PNLoop = LI.getLoopFor(BB: Parent);
6071 if (!PNLoop || Parent != PNLoop->getHeader()) {
6072 // Split the critical edge.
6073 BasicBlock *NewBB = nullptr;
6074 if (!Parent->isLandingPad()) {
6075 CriticalEdgeSplittingOptions SplitOptions(&DT, &LI, MSSAU);
6076 SplitOptions =
6077 SplitOptions.setMergeIdenticalEdges().setKeepOneInputPHIs();
6078 if (ShouldPreserveLCSSA)
6079 SplitOptions = SplitOptions.setPreserveLCSSA();
6080 NewBB = SplitCriticalEdge(Src: BB, Dst: Parent, Options: SplitOptions);
6081 } else {
6082 SmallVector<BasicBlock *, 2> NewBBs;
6083 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
6084 SplitLandingPadPredecessors(OrigBB: Parent, Preds: BB, Suffix: "", Suffix2: "", NewBBs, DTU: &DTU, LI: &LI,
6085 MSSAU, PreserveLCSSA: ShouldPreserveLCSSA);
6086 NewBB = NewBBs[0];
6087 }
6088 // If NewBB==NULL, then SplitCriticalEdge refused to split because all
6089 // phi predecessors are identical. The simple thing to do is skip
6090 // splitting in this case rather than complicate the API.
6091 if (NewBB) {
6092 // If PN is outside of the loop and BB is in the loop, we want to
6093 // move the block to be immediately before the PHI block, not
6094 // immediately after BB.
6095 if (L->contains(BB) && !L->contains(Inst: PN))
6096 NewBB->moveBefore(MovePos: PN->getParent());
6097
6098 // Splitting the edge can reduce the number of PHI entries we have.
6099 e = PN->getNumIncomingValues();
6100 BB = NewBB;
6101 i = PN->getBasicBlockIndex(BB);
6102
6103 needUpdateFixups = true;
6104
6105 if (auto *OldIncomingInst =
6106 dyn_cast<Instruction>(Val: PN->getIncomingValue(i)))
6107 DeadInsts.emplace_back(Args&: OldIncomingInst);
6108 }
6109 }
6110 }
6111
6112 std::pair<DenseMap<BasicBlock *, Value *>::iterator, bool> Pair =
6113 Inserted.try_emplace(Key: BB);
6114 if (!Pair.second)
6115 PN->setIncomingValue(i, V: Pair.first->second);
6116 else {
6117 Value *FullV =
6118 Expand(LU, LF, F, IP: BB->getTerminator()->getIterator(), DeadInsts);
6119
6120 // If this is reuse-by-noop-cast, insert the noop cast.
6121 Type *OpTy = LF.OperandValToReplace->getType();
6122 if (FullV->getType() != OpTy)
6123 FullV = CastInst::Create(
6124 CastInst::getCastOpcode(Val: FullV, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false), S: FullV,
6125 Ty: LF.OperandValToReplace->getType(), Name: "tmp",
6126 InsertBefore: BB->getTerminator()->getIterator());
6127
6128 // If the incoming block for this value is not in the loop, it means the
6129 // current PHI is not in a loop exit, so we must create a LCSSA PHI for
6130 // the inserted value.
6131 if (auto *I = dyn_cast<Instruction>(Val: FullV))
6132 if (L->contains(Inst: I) && !L->contains(BB))
6133 InsertedNonLCSSAInsts.insert(X: I);
6134
6135 PN->setIncomingValue(i, V: FullV);
6136 Pair.first->second = FullV;
6137 }
6138
6139 // If LSR splits critical edge and phi node has other pending
6140 // fixup operands, we need to update those pending fixups. Otherwise
6141 // formulae will not be implemented completely and some instructions
6142 // will not be eliminated.
6143 if (needUpdateFixups) {
6144 for (LSRUse &LU : Uses)
6145 for (LSRFixup &Fixup : LU.Fixups)
6146 // If fixup is supposed to rewrite some operand in the phi
6147 // that was just updated, it may be already moved to
6148 // another phi node. Such fixup requires update.
6149 if (Fixup.UserInst == PN) {
6150 // Check if the operand we try to replace still exists in the
6151 // original phi.
6152 bool foundInOriginalPHI = false;
6153 for (const auto &val : PN->incoming_values())
6154 if (val == Fixup.OperandValToReplace) {
6155 foundInOriginalPHI = true;
6156 break;
6157 }
6158
6159 // If fixup operand found in original PHI - nothing to do.
6160 if (foundInOriginalPHI)
6161 continue;
6162
6163 // Otherwise it might be moved to another PHI and requires update.
6164 // If fixup operand not found in any of the incoming blocks that
6165 // means we have already rewritten it - nothing to do.
6166 for (const auto &Block : PN->blocks())
6167 for (BasicBlock::iterator I = Block->begin(); isa<PHINode>(Val: I);
6168 ++I) {
6169 PHINode *NewPN = cast<PHINode>(Val&: I);
6170 for (const auto &val : NewPN->incoming_values())
6171 if (val == Fixup.OperandValToReplace)
6172 Fixup.UserInst = NewPN;
6173 }
6174 }
6175 }
6176 }
6177}
6178
6179/// Emit instructions for the leading candidate expression for this LSRUse (this
6180/// is called "expanding"), and update the UserInst to reference the newly
6181/// expanded value.
6182void LSRInstance::Rewrite(const LSRUse &LU, const LSRFixup &LF,
6183 const Formula &F,
6184 SmallVectorImpl<WeakTrackingVH> &DeadInsts) {
6185 // First, find an insertion point that dominates UserInst. For PHI nodes,
6186 // find the nearest block which dominates all the relevant uses.
6187 if (PHINode *PN = dyn_cast<PHINode>(Val: LF.UserInst)) {
6188 RewriteForPHI(PN, LU, LF, F, DeadInsts);
6189 } else {
6190 Value *FullV = Expand(LU, LF, F, IP: LF.UserInst->getIterator(), DeadInsts);
6191
6192 // If this is reuse-by-noop-cast, insert the noop cast.
6193 // For ICmpZero with pointer operands, Expand() already set both operands
6194 // in integer domain, so no cast is needed here.
6195 Type *OpTy = LF.OperandValToReplace->getType();
6196 if (FullV->getType() != OpTy &&
6197 !(LU.Kind == LSRUse::ICmpZero && OpTy->isPointerTy())) {
6198 Instruction *Cast =
6199 CastInst::Create(CastInst::getCastOpcode(Val: FullV, SrcIsSigned: false, Ty: OpTy, DstIsSigned: false),
6200 S: FullV, Ty: OpTy, Name: "tmp", InsertBefore: LF.UserInst->getIterator());
6201 FullV = Cast;
6202 }
6203
6204 // Update the user. ICmpZero is handled specially here (for now) because
6205 // Expand may have updated one of the operands of the icmp already, and
6206 // its new value may happen to be equal to LF.OperandValToReplace, in
6207 // which case doing replaceUsesOfWith leads to replacing both operands
6208 // with the same value. TODO: Reorganize this.
6209 if (LU.Kind == LSRUse::ICmpZero)
6210 LF.UserInst->setOperand(i: 0, Val: FullV);
6211 else
6212 LF.UserInst->replaceUsesOfWith(From: LF.OperandValToReplace, To: FullV);
6213 }
6214
6215 if (auto *OperandIsInstr = dyn_cast<Instruction>(Val: LF.OperandValToReplace))
6216 DeadInsts.emplace_back(Args&: OperandIsInstr);
6217}
6218
6219// Determine where to insert the transformed IV increment instruction for this
6220// fixup. By default this is the default insert position, but if this is a
6221// postincrement opportunity then we try to insert it in the same block as the
6222// fixup user instruction, as this is needed for a postincrement instruction to
6223// be generated.
6224static Instruction *getFixupInsertPos(const TargetTransformInfo &TTI,
6225 const LSRFixup &Fixup, const LSRUse &LU,
6226 Instruction *IVIncInsertPos,
6227 DominatorTree &DT) {
6228 // Only address uses can be postincremented
6229 if (LU.Kind != LSRUse::Address)
6230 return IVIncInsertPos;
6231
6232 // Don't try to postincrement if it's not legal
6233 Instruction *I = Fixup.UserInst;
6234 Type *Ty = I->getType();
6235 if (!(isa<LoadInst>(Val: I) && TTI.isIndexedLoadLegal(Mode: TTI.MIM_PostInc, Ty)) &&
6236 !(isa<StoreInst>(Val: I) && TTI.isIndexedStoreLegal(Mode: TTI.MIM_PostInc, Ty)))
6237 return IVIncInsertPos;
6238
6239 // It's only legal to hoist to the user block if it dominates the default
6240 // insert position.
6241 BasicBlock *HoistBlock = I->getParent();
6242 BasicBlock *IVIncBlock = IVIncInsertPos->getParent();
6243 if (!DT.dominates(Def: I, BB: IVIncBlock))
6244 return IVIncInsertPos;
6245
6246 return HoistBlock->getTerminator();
6247}
6248
6249/// Rewrite all the fixup locations with new values, following the chosen
6250/// solution.
6251void LSRInstance::ImplementSolution(
6252 const SmallVectorImpl<const Formula *> &Solution) {
6253 // Keep track of instructions we may have made dead, so that
6254 // we can remove them after we are done working.
6255 SmallVector<WeakTrackingVH, 16> DeadInsts;
6256
6257 // Mark phi nodes that terminate chains so the expander tries to reuse them.
6258 for (const IVChain &Chain : IVChainVec) {
6259 if (PHINode *PN = dyn_cast<PHINode>(Val: Chain.tailUserInst()))
6260 Rewriter.setChainedPhi(PN);
6261 }
6262
6263 // Expand the new value definitions and update the users.
6264 for (size_t LUIdx = 0, NumUses = Uses.size(); LUIdx != NumUses; ++LUIdx)
6265 for (const LSRFixup &Fixup : Uses[LUIdx].Fixups) {
6266 Instruction *InsertPos =
6267 getFixupInsertPos(TTI, Fixup, LU: Uses[LUIdx], IVIncInsertPos, DT);
6268 Rewriter.setIVIncInsertPos(L, Pos: InsertPos);
6269 Rewrite(LU: Uses[LUIdx], LF: Fixup, F: *Solution[LUIdx], DeadInsts);
6270 Changed = true;
6271 }
6272
6273 auto InsertedInsts = InsertedNonLCSSAInsts.takeVector();
6274 formLCSSAForInstructions(Worklist&: InsertedInsts, DT, LI, SE: &SE);
6275
6276 for (const IVChain &Chain : IVChainVec) {
6277 GenerateIVChain(Chain, DeadInsts);
6278 Changed = true;
6279 }
6280
6281 for (const WeakVH &IV : Rewriter.getInsertedIVs())
6282 if (IV && dyn_cast<Instruction>(Val: &*IV)->getParent())
6283 ScalarEvolutionIVs.push_back(Elt: IV);
6284
6285 // Clean up after ourselves. This must be done before deleting any
6286 // instructions.
6287 Rewriter.clear();
6288
6289 Changed |= RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts,
6290 TLI: &TLI, MSSAU);
6291
6292 // In our cost analysis above, we assume that each addrec consumes exactly
6293 // one register, and arrange to have increments inserted just before the
6294 // latch to maximimize the chance this is true. However, if we reused
6295 // existing IVs, we now need to move the increments to match our
6296 // expectations. Otherwise, our cost modeling results in us having a
6297 // chosen a non-optimal result for the actual schedule. (And yes, this
6298 // scheduling decision does impact later codegen.)
6299 for (PHINode &PN : L->getHeader()->phis()) {
6300 BinaryOperator *BO = nullptr;
6301 Value *Start = nullptr, *Step = nullptr;
6302 if (!matchSimpleRecurrence(P: &PN, BO, Start, Step))
6303 continue;
6304
6305 switch (BO->getOpcode()) {
6306 case Instruction::Sub:
6307 if (BO->getOperand(i_nocapture: 0) != &PN)
6308 // sub is non-commutative - match handling elsewhere in LSR
6309 continue;
6310 break;
6311 case Instruction::Add:
6312 break;
6313 default:
6314 continue;
6315 };
6316
6317 if (!isa<Constant>(Val: Step))
6318 // If not a constant step, might increase register pressure
6319 // (We assume constants have been canonicalized to RHS)
6320 continue;
6321
6322 if (BO->getParent() == IVIncInsertPos->getParent())
6323 // Only bother moving across blocks. Isel can handle block local case.
6324 continue;
6325
6326 // Can we legally schedule inc at the desired point?
6327 if (!llvm::all_of(Range: BO->uses(),
6328 P: [&](Use &U) {return DT.dominates(Def: IVIncInsertPos, U);}))
6329 continue;
6330 BO->moveBefore(InsertPos: IVIncInsertPos->getIterator());
6331 Changed = true;
6332 }
6333
6334
6335}
6336
6337LSRInstance::LSRInstance(const ScalarOptions &Opts, Loop *L, IVUsers &IU,
6338 ScalarEvolution &SE, DominatorTree &DT, LoopInfo &LI,
6339 const TargetTransformInfo &TTI, AssumptionCache &AC,
6340 TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU,
6341 bool PreserveLCSSA)
6342 : Opts(Opts), IU(IU), SE(SE), DT(DT), LI(LI), AC(AC), TLI(TLI), TTI(TTI),
6343 L(L), MSSAU(MSSAU), AMK(Opts.lsr_preferred_addressing_mode.value_or(
6344 u: TTI.getPreferredAddressingMode(L, SE: &SE))),
6345 Rewriter(SE, "lsr", PreserveLCSSA), ShouldPreserveLCSSA(PreserveLCSSA),
6346 BaselineCost(Opts, L, SE, TTI, AMK) {
6347 // If LoopSimplify form is not available, stay out of trouble.
6348 if (!L->isLoopSimplifyForm())
6349 return;
6350
6351 // If there's no interesting work to be done, bail early.
6352 if (IU.empty()) return;
6353
6354 // If there's too much analysis to be done, bail early. We won't be able to
6355 // model the problem anyway.
6356 unsigned NumUsers = 0;
6357 for (const IVStrideUse &U : IU) {
6358 if (++NumUsers > MaxIVUsers) {
6359 (void)U;
6360 LLVM_DEBUG(dbgs() << "LSR skipping loop, too many IV Users in " << U
6361 << "\n");
6362 return;
6363 }
6364 // Bail out if we have a PHI on an EHPad that gets a value from a
6365 // CatchSwitchInst. Because the CatchSwitchInst cannot be split, there is
6366 // no good place to stick any instructions.
6367 if (auto *PN = dyn_cast<PHINode>(Val: U.getUser())) {
6368 auto FirstNonPHI = PN->getParent()->getFirstNonPHIIt();
6369 if (isa<FuncletPadInst>(Val: FirstNonPHI) ||
6370 isa<CatchSwitchInst>(Val: FirstNonPHI))
6371 for (BasicBlock *PredBB : PN->blocks())
6372 if (isa<CatchSwitchInst>(Val: PredBB->getFirstNonPHIIt()))
6373 return;
6374 }
6375 }
6376
6377 LLVM_DEBUG(dbgs() << "\nLSR on loop ";
6378 L->getHeader()->printAsOperand(dbgs(), /*PrintType=*/false);
6379 dbgs() << ":\n");
6380
6381 // Check if we expect this loop to use a hardware loop instruction, which will
6382 // be used when calculating the costs of formulas.
6383 HardwareLoopInfo HWLoopInfo(L);
6384 HardwareLoopProfitable =
6385 TTI.isHardwareLoopProfitable(L, SE, AC, LibInfo: &TLI, HWLoopInfo);
6386
6387 // Configure SCEVExpander already now, so the correct mode is used for
6388 // isSafeToExpand() checks.
6389#if LLVM_ENABLE_ABI_BREAKING_CHECKS
6390 Rewriter.setDebugType(DEBUG_TYPE);
6391#endif
6392 Rewriter.disableCanonicalMode();
6393 Rewriter.enableLSRMode();
6394
6395 // First, perform some low-level loop optimizations.
6396 OptimizeShadowIV();
6397 OptimizeLoopTermCond();
6398
6399 // If loop preparation eliminates all interesting IV users, bail.
6400 if (IU.empty()) return;
6401
6402 // Skip nested loops until we can model them better with formulae.
6403 if (!L->isInnermost()) {
6404 LLVM_DEBUG(dbgs() << "LSR skipping outer loop " << *L << "\n");
6405 return;
6406 }
6407
6408 // Start collecting data and preparing for the solver.
6409 // If number of registers is not the major cost, we cannot benefit from the
6410 // current profitable chain optimization which is based on number of
6411 // registers.
6412 // FIXME: add profitable chain optimization for other kinds major cost, for
6413 // example number of instructions.
6414 if (TTI.isNumRegsMajorCostOfLSR() || stressIVChain(Opts))
6415 CollectChains();
6416 CollectInterestingTypesAndFactors();
6417 CollectFixupsAndInitialFormulae();
6418 CollectLoopInvariantFixupsAndFormulae();
6419
6420 if (Uses.empty())
6421 return;
6422
6423 LLVM_DEBUG(dbgs() << "LSR found " << Uses.size() << " uses:\n";
6424 print_uses(dbgs()));
6425 LLVM_DEBUG(dbgs() << "The baseline solution requires ";
6426 BaselineCost.print(dbgs()); dbgs() << "\n");
6427
6428 // Now use the reuse data to generate a bunch of interesting ways
6429 // to formulate the values needed for the uses.
6430 GenerateAllReuseFormulae();
6431
6432 FilterOutUndesirableDedicatedRegisters();
6433 NarrowSearchSpaceUsingHeuristics();
6434
6435 SmallVector<const Formula *, 8> Solution;
6436 Solve(Solution);
6437
6438 // Release memory that is no longer needed.
6439 Factors.clear();
6440 Types.clear();
6441 RegUses.clear();
6442
6443 if (Solution.empty())
6444 return;
6445
6446#ifndef NDEBUG
6447 // Formulae should be legal.
6448 for (const LSRUse &LU : Uses) {
6449 for (const Formula &F : LU.Formulae)
6450 assert(isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy,
6451 F) && "Illegal formula generated!");
6452 };
6453#endif
6454
6455 // Now that we've decided what we want, make it so.
6456 ImplementSolution(Solution);
6457}
6458
6459#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
6460void LSRInstance::print_factors_and_types(raw_ostream &OS) const {
6461 if (Factors.empty() && Types.empty()) return;
6462
6463 OS << "LSR has identified the following interesting factors and types: ";
6464 ListSeparator LS;
6465
6466 for (int64_t Factor : Factors)
6467 OS << LS << '*' << Factor;
6468
6469 for (Type *Ty : Types)
6470 OS << LS << '(' << *Ty << ')';
6471 OS << '\n';
6472}
6473
6474void LSRInstance::print_fixups(raw_ostream &OS) const {
6475 OS << "LSR is examining the following fixup sites:\n";
6476 for (const LSRUse &LU : Uses)
6477 for (const LSRFixup &LF : LU.Fixups) {
6478 dbgs() << " ";
6479 LF.print(OS);
6480 OS << '\n';
6481 }
6482}
6483
6484void LSRInstance::print_uses(raw_ostream &OS) const {
6485 OS << "LSR is examining the following uses:\n";
6486 for (const LSRUse &LU : Uses) {
6487 dbgs() << " ";
6488 LU.print(OS);
6489 OS << '\n';
6490 for (const Formula &F : LU.Formulae) {
6491 OS << " ";
6492 F.print(OS);
6493 OS << '\n';
6494 }
6495 }
6496}
6497
6498void LSRInstance::print(raw_ostream &OS) const {
6499 print_factors_and_types(OS);
6500 print_fixups(OS);
6501 print_uses(OS);
6502}
6503
6504LLVM_DUMP_METHOD void LSRInstance::dump() const {
6505 print(errs()); errs() << '\n';
6506}
6507#endif
6508
6509namespace {
6510
6511class LoopStrengthReduce : public LoopPass {
6512public:
6513 static char ID; // Pass ID, replacement for typeid
6514
6515 LoopStrengthReduce();
6516
6517private:
6518 bool runOnLoop(Loop *L, LPPassManager &LPM) override;
6519 void getAnalysisUsage(AnalysisUsage &AU) const override;
6520};
6521
6522} // end anonymous namespace
6523
6524LoopStrengthReduce::LoopStrengthReduce() : LoopPass(ID) {
6525 initializeLoopStrengthReducePass(*PassRegistry::getPassRegistry());
6526}
6527
6528void LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const {
6529 // We split critical edges, so we change the CFG. However, we do update
6530 // many analyses if they are around.
6531 AU.addPreservedID(ID&: LoopSimplifyID);
6532
6533 AU.addRequired<LoopInfoWrapperPass>();
6534 AU.addPreserved<LoopInfoWrapperPass>();
6535 AU.addRequiredID(ID&: LoopSimplifyID);
6536 AU.addRequired<DominatorTreeWrapperPass>();
6537 AU.addPreserved<DominatorTreeWrapperPass>();
6538 AU.addRequired<ScalarEvolutionWrapperPass>();
6539 AU.addPreserved<ScalarEvolutionWrapperPass>();
6540 AU.addRequired<AssumptionCacheTracker>();
6541 AU.addRequired<TargetLibraryInfoWrapperPass>();
6542 // Requiring LoopSimplify a second time here prevents IVUsers from running
6543 // twice, since LoopSimplify was invalidated by running ScalarEvolution.
6544 AU.addRequiredID(ID&: LoopSimplifyID);
6545 AU.addRequired<IVUsersWrapperPass>();
6546 AU.addPreserved<IVUsersWrapperPass>();
6547 AU.addRequired<TargetTransformInfoWrapperPass>();
6548 AU.addPreserved<MemorySSAWrapperPass>();
6549}
6550
6551namespace {
6552
6553/// Enables more convenient iteration over a DWARF expression vector.
6554static iterator_range<llvm::DIExpression::expr_op_iterator>
6555ToDwarfOpIter(SmallVectorImpl<uint64_t> &Expr) {
6556 llvm::DIExpression::expr_op_iterator Begin =
6557 llvm::DIExpression::expr_op_iterator(Expr.begin());
6558 llvm::DIExpression::expr_op_iterator End =
6559 llvm::DIExpression::expr_op_iterator(Expr.end());
6560 return {Begin, End};
6561}
6562
6563struct SCEVDbgValueBuilder {
6564 SCEVDbgValueBuilder() = default;
6565 SCEVDbgValueBuilder(const SCEVDbgValueBuilder &Base) { clone(Base); }
6566
6567 void clone(const SCEVDbgValueBuilder &Base) {
6568 LocationOps = Base.LocationOps;
6569 Expr = Base.Expr;
6570 }
6571
6572 void clear() {
6573 LocationOps.clear();
6574 Expr.clear();
6575 }
6576
6577 /// The DIExpression as we translate the SCEV.
6578 SmallVector<uint64_t, 6> Expr;
6579 /// The location ops of the DIExpression.
6580 SmallVector<Value *, 2> LocationOps;
6581
6582 void pushOperator(uint64_t Op) { Expr.push_back(Elt: Op); }
6583 void pushUInt(uint64_t Operand) { Expr.push_back(Elt: Operand); }
6584
6585 /// Add a DW_OP_LLVM_arg to the expression, followed by the index of the value
6586 /// in the set of values referenced by the expression.
6587 void pushLocation(llvm::Value *V) {
6588 Expr.push_back(Elt: llvm::dwarf::DW_OP_LLVM_arg);
6589 auto *It = llvm::find(Range&: LocationOps, Val: V);
6590 unsigned ArgIndex = 0;
6591 if (It != LocationOps.end()) {
6592 ArgIndex = std::distance(first: LocationOps.begin(), last: It);
6593 } else {
6594 ArgIndex = LocationOps.size();
6595 LocationOps.push_back(Elt: V);
6596 }
6597 Expr.push_back(Elt: ArgIndex);
6598 }
6599
6600 void pushValue(const SCEVUnknown *U) {
6601 llvm::Value *V = cast<SCEVUnknown>(Val: U)->getValue();
6602 pushLocation(V);
6603 }
6604
6605 bool pushConst(const SCEVConstant *C) {
6606 if (C->getAPInt().getSignificantBits() > 64)
6607 return false;
6608 Expr.push_back(Elt: llvm::dwarf::DW_OP_consts);
6609 Expr.push_back(Elt: C->getAPInt().getSExtValue());
6610 return true;
6611 }
6612
6613 // Iterating the expression as DWARF ops is convenient when updating
6614 // DWARF_OP_LLVM_args.
6615 iterator_range<llvm::DIExpression::expr_op_iterator> expr_ops() {
6616 return ToDwarfOpIter(Expr);
6617 }
6618
6619 /// Several SCEV types are sequences of the same arithmetic operator applied
6620 /// to constants and values that may be extended or truncated.
6621 bool pushArithmeticExpr(const llvm::SCEVCommutativeExpr *CommExpr,
6622 uint64_t DwarfOp) {
6623 assert((isa<llvm::SCEVAddExpr>(CommExpr) || isa<SCEVMulExpr>(CommExpr)) &&
6624 "Expected arithmetic SCEV type");
6625 bool Success = true;
6626 unsigned EmitOperator = 0;
6627 for (const auto &Op : CommExpr->operands()) {
6628 Success &= pushSCEV(S: Op);
6629
6630 if (EmitOperator >= 1)
6631 pushOperator(Op: DwarfOp);
6632 ++EmitOperator;
6633 }
6634 return Success;
6635 }
6636
6637 // TODO: Identify and omit noop casts.
6638 bool pushCast(const llvm::SCEVCastExpr *C, bool IsSigned) {
6639 const llvm::SCEV *Inner = C->getOperand(i: 0);
6640 const llvm::Type *Type = C->getType();
6641 uint64_t ToWidth = Type->getIntegerBitWidth();
6642 bool Success = pushSCEV(S: Inner);
6643 uint64_t CastOps[] = {dwarf::DW_OP_LLVM_convert, ToWidth,
6644 IsSigned ? llvm::dwarf::DW_ATE_signed
6645 : llvm::dwarf::DW_ATE_unsigned};
6646 for (const auto &Op : CastOps)
6647 pushOperator(Op);
6648 return Success;
6649 }
6650
6651 // TODO: MinMax - although these haven't been encountered in the test suite.
6652 bool pushSCEV(const llvm::SCEV *S) {
6653 bool Success = true;
6654 if (const SCEVConstant *StartInt = dyn_cast<SCEVConstant>(Val: S)) {
6655 Success &= pushConst(C: StartInt);
6656
6657 } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Val: S)) {
6658 if (!U->getValue())
6659 return false;
6660 pushLocation(V: U->getValue());
6661
6662 } else if (const SCEVMulExpr *MulRec = dyn_cast<SCEVMulExpr>(Val: S)) {
6663 Success &= pushArithmeticExpr(CommExpr: MulRec, DwarfOp: llvm::dwarf::DW_OP_mul);
6664
6665 } else if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(Val: S)) {
6666 Success &= pushSCEV(S: UDiv->getLHS());
6667 Success &= pushSCEV(S: UDiv->getRHS());
6668 pushOperator(Op: llvm::dwarf::DW_OP_div);
6669
6670 } else if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(Val: S)) {
6671 // Assert if a new and unknown SCEVCastEXpr type is encountered.
6672 assert((isa<SCEVZeroExtendExpr>(Cast) || isa<SCEVTruncateExpr>(Cast) ||
6673 isa<SCEVPtrToAddrExpr>(Cast) || isa<SCEVSignExtendExpr>(Cast)) &&
6674 "Unexpected cast type in SCEV.");
6675 Success &= pushCast(C: Cast, IsSigned: (isa<SCEVSignExtendExpr>(Val: Cast)));
6676
6677 } else if (const SCEVAddExpr *AddExpr = dyn_cast<SCEVAddExpr>(Val: S)) {
6678 Success &= pushArithmeticExpr(CommExpr: AddExpr, DwarfOp: llvm::dwarf::DW_OP_plus);
6679
6680 } else if (isa<SCEVAddRecExpr>(Val: S)) {
6681 // Nested SCEVAddRecExpr are generated by nested loops and are currently
6682 // unsupported.
6683 return false;
6684
6685 } else {
6686 return false;
6687 }
6688 return Success;
6689 }
6690
6691 /// Return true if the combination of arithmetic operator and underlying
6692 /// SCEV constant value is an identity function.
6693 bool isIdentityFunction(uint64_t Op, const SCEV *S) {
6694 if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Val: S)) {
6695 if (C->getAPInt().getSignificantBits() > 64)
6696 return false;
6697 int64_t I = C->getAPInt().getSExtValue();
6698 switch (Op) {
6699 case llvm::dwarf::DW_OP_plus:
6700 case llvm::dwarf::DW_OP_minus:
6701 return I == 0;
6702 case llvm::dwarf::DW_OP_mul:
6703 case llvm::dwarf::DW_OP_div:
6704 return I == 1;
6705 }
6706 }
6707 return false;
6708 }
6709
6710 /// Convert a SCEV of a value to a DIExpression that is pushed onto the
6711 /// builder's expression stack. The stack should already contain an
6712 /// expression for the iteration count, so that it can be multiplied by
6713 /// the stride and added to the start.
6714 /// Components of the expression are omitted if they are an identity function.
6715 /// Chain (non-affine) SCEVs are not supported.
6716 bool SCEVToValueExpr(const llvm::SCEVAddRecExpr &SAR, ScalarEvolution &SE) {
6717 assert(SAR.isAffine() && "Expected affine SCEV");
6718 const SCEV *Start = SAR.getStart();
6719 const SCEV *Stride = SAR.getStepRecurrence(SE);
6720
6721 // Skip pushing arithmetic noops.
6722 if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_mul, S: Stride)) {
6723 if (!pushSCEV(S: Stride))
6724 return false;
6725 pushOperator(Op: llvm::dwarf::DW_OP_mul);
6726 }
6727 if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_plus, S: Start)) {
6728 if (!pushSCEV(S: Start))
6729 return false;
6730 pushOperator(Op: llvm::dwarf::DW_OP_plus);
6731 }
6732 return true;
6733 }
6734
6735 /// Create an expression that is an offset from a value (usually the IV).
6736 void createOffsetExpr(int64_t Offset, Value *OffsetValue) {
6737 pushLocation(V: OffsetValue);
6738 DIExpression::appendOffset(Ops&: Expr, Offset);
6739 LLVM_DEBUG(
6740 dbgs() << "scev-salvage: Generated IV offset expression. Offset: "
6741 << std::to_string(Offset) << "\n");
6742 }
6743
6744 /// Combine a translation of the SCEV and the IV to create an expression that
6745 /// recovers a location's value.
6746 /// returns true if an expression was created.
6747 bool createIterCountExpr(const SCEV *S,
6748 const SCEVDbgValueBuilder &IterationCount,
6749 ScalarEvolution &SE) {
6750 // SCEVs for SSA values are most frquently of the form
6751 // {start,+,stride}, but sometimes they are ({start,+,stride} + %a + ..).
6752 // This is because %a is a PHI node that is not the IV. However, these
6753 // SCEVs have not been observed to result in debuginfo-lossy optimisations,
6754 // so its not expected this point will be reached.
6755 if (!isa<SCEVAddRecExpr>(Val: S))
6756 return false;
6757
6758 LLVM_DEBUG(dbgs() << "scev-salvage: Location to salvage SCEV: " << *S
6759 << '\n');
6760
6761 const auto *Rec = cast<SCEVAddRecExpr>(Val: S);
6762 if (!Rec->isAffine())
6763 return false;
6764
6765 if (S->getExpressionSize() > MaxSCEVSalvageExpressionSize)
6766 return false;
6767
6768 // Initialise a new builder with the iteration count expression. In
6769 // combination with the value's SCEV this enables recovery.
6770 clone(Base: IterationCount);
6771 if (!SCEVToValueExpr(SAR: *Rec, SE))
6772 return false;
6773
6774 return true;
6775 }
6776
6777 /// Convert a SCEV of a value to a DIExpression that is pushed onto the
6778 /// builder's expression stack. The stack should already contain an
6779 /// expression for the iteration count, so that it can be multiplied by
6780 /// the stride and added to the start.
6781 /// Components of the expression are omitted if they are an identity function.
6782 bool SCEVToIterCountExpr(const llvm::SCEVAddRecExpr &SAR,
6783 ScalarEvolution &SE) {
6784 assert(SAR.isAffine() && "Expected affine SCEV");
6785 const SCEV *Start = SAR.getStart();
6786 const SCEV *Stride = SAR.getStepRecurrence(SE);
6787
6788 // Skip pushing arithmetic noops.
6789 if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_minus, S: Start)) {
6790 if (!pushSCEV(S: Start))
6791 return false;
6792 pushOperator(Op: llvm::dwarf::DW_OP_minus);
6793 }
6794 if (!isIdentityFunction(Op: llvm::dwarf::DW_OP_div, S: Stride)) {
6795 if (!pushSCEV(S: Stride))
6796 return false;
6797 pushOperator(Op: llvm::dwarf::DW_OP_div);
6798 }
6799 return true;
6800 }
6801
6802 // Append the current expression and locations to a location list and an
6803 // expression list. Modify the DW_OP_LLVM_arg indexes to account for
6804 // the locations already present in the destination list.
6805 void appendToVectors(SmallVectorImpl<uint64_t> &DestExpr,
6806 SmallVectorImpl<Value *> &DestLocations) {
6807 assert(!DestLocations.empty() &&
6808 "Expected the locations vector to contain the IV");
6809 // The DWARF_OP_LLVM_arg arguments of the expression being appended must be
6810 // modified to account for the locations already in the destination vector.
6811 // All builders contain the IV as the first location op.
6812 assert(!LocationOps.empty() &&
6813 "Expected the location ops to contain the IV.");
6814 // DestIndexMap[n] contains the index in DestLocations for the nth
6815 // location in this SCEVDbgValueBuilder.
6816 SmallVector<uint64_t, 2> DestIndexMap;
6817 for (const auto &Op : LocationOps) {
6818 auto It = find(Range&: DestLocations, Val: Op);
6819 if (It != DestLocations.end()) {
6820 // Location already exists in DestLocations, reuse existing ArgIndex.
6821 DestIndexMap.push_back(Elt: std::distance(first: DestLocations.begin(), last: It));
6822 continue;
6823 }
6824 // Location is not in DestLocations, add it.
6825 DestIndexMap.push_back(Elt: DestLocations.size());
6826 DestLocations.push_back(Elt: Op);
6827 }
6828
6829 for (const auto &Op : expr_ops()) {
6830 auto Arg = dyn_cast<DIExpression::ArgOp>(Val: Op);
6831 if (!Arg) {
6832 Op.appendToVector(V&: DestExpr);
6833 continue;
6834 }
6835
6836 DestExpr.push_back(Elt: dwarf::DW_OP_LLVM_arg);
6837 // `DW_OP_LLVM_arg n` represents the nth LocationOp in this SCEV,
6838 // DestIndexMap[n] contains its new index in DestLocations.
6839 uint64_t NewIndex = DestIndexMap[Arg.getIndex()];
6840 DestExpr.push_back(Elt: NewIndex);
6841 }
6842 }
6843};
6844
6845/// Holds all the required data to salvage a dbg.value using the pre-LSR SCEVs
6846/// and DIExpression.
6847struct DVIRecoveryRec {
6848 DVIRecoveryRec(DbgVariableRecord *DVR)
6849 : DbgRef(DVR), Expr(DVR->getExpression()), HadLocationArgList(false) {}
6850
6851 DbgVariableRecord *DbgRef;
6852 DIExpression *Expr;
6853 bool HadLocationArgList;
6854 SmallVector<WeakVH, 2> LocationOps;
6855 SmallVector<const llvm::SCEV *, 2> SCEVs;
6856 SmallVector<std::unique_ptr<SCEVDbgValueBuilder>, 2> RecoveryExprs;
6857
6858 void clear() {
6859 for (auto &RE : RecoveryExprs)
6860 RE.reset();
6861 RecoveryExprs.clear();
6862 }
6863
6864 ~DVIRecoveryRec() { clear(); }
6865};
6866} // namespace
6867
6868/// Returns the total number of DW_OP_llvm_arg operands in the expression.
6869/// This helps in determining if a DIArglist is necessary or can be omitted from
6870/// the dbg.value.
6871static unsigned numLLVMArgOps(SmallVectorImpl<uint64_t> &Expr) {
6872 auto expr_ops = ToDwarfOpIter(Expr);
6873 unsigned Count = 0;
6874 for (auto Op : expr_ops)
6875 if (Op.getOp() == dwarf::DW_OP_LLVM_arg)
6876 Count++;
6877 return Count;
6878}
6879
6880/// Overwrites DVI with the location and Ops as the DIExpression. This will
6881/// create an invalid expression if Ops has any dwarf::DW_OP_llvm_arg operands,
6882/// because a DIArglist is not created for the first argument of the dbg.value.
6883template <typename T>
6884static void updateDVIWithLocation(T &DbgVal, Value *Location,
6885 SmallVectorImpl<uint64_t> &Ops) {
6886 assert(numLLVMArgOps(Ops) == 0 && "Expected expression that does not "
6887 "contain any DW_OP_llvm_arg operands.");
6888 DbgVal.setRawLocation(ValueAsMetadata::get(V: Location));
6889 DbgVal.setExpression(DIExpression::get(Context&: DbgVal.getContext(), Elements: Ops));
6890}
6891
6892/// Overwrite DVI with locations placed into a DIArglist.
6893template <typename T>
6894static void updateDVIWithLocations(T &DbgVal,
6895 SmallVectorImpl<Value *> &Locations,
6896 SmallVectorImpl<uint64_t> &Ops) {
6897 assert(numLLVMArgOps(Ops) != 0 &&
6898 "Expected expression that references DIArglist locations using "
6899 "DW_OP_llvm_arg operands.");
6900 SmallVector<ValueAsMetadata *, 3> MetadataLocs;
6901 for (Value *V : Locations)
6902 MetadataLocs.push_back(Elt: ValueAsMetadata::get(V));
6903 auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
6904 DbgVal.setRawLocation(llvm::DIArgList::get(Context&: DbgVal.getContext(), Args: ValArrayRef));
6905 DbgVal.setExpression(DIExpression::get(Context&: DbgVal.getContext(), Elements: Ops));
6906}
6907
6908/// Write the new expression and new location ops for the dbg.value. If possible
6909/// reduce the szie of the dbg.value by omitting DIArglist. This
6910/// can be omitted if:
6911/// 1. There is only a single location, refenced by a single DW_OP_llvm_arg.
6912/// 2. The DW_OP_LLVM_arg is the first operand in the expression.
6913static void UpdateDbgValue(DVIRecoveryRec &DVIRec,
6914 SmallVectorImpl<Value *> &NewLocationOps,
6915 SmallVectorImpl<uint64_t> &NewExpr) {
6916 DbgVariableRecord *DbgVal = DVIRec.DbgRef;
6917 unsigned NumLLVMArgs = numLLVMArgOps(Expr&: NewExpr);
6918 if (NumLLVMArgs == 0) {
6919 // Location assumed to be on the stack.
6920 updateDVIWithLocation(DbgVal&: *DbgVal, Location: NewLocationOps[0], Ops&: NewExpr);
6921 } else if (NumLLVMArgs == 1 && NewExpr[0] == dwarf::DW_OP_LLVM_arg) {
6922 // There is only a single DW_OP_llvm_arg at the start of the expression,
6923 // so it can be omitted along with DIArglist.
6924 assert(NewExpr[1] == 0 &&
6925 "Lone LLVM_arg in a DIExpression should refer to location-op 0.");
6926 llvm::SmallVector<uint64_t, 6> ShortenedOps(llvm::drop_begin(RangeOrContainer&: NewExpr, N: 2));
6927 updateDVIWithLocation(DbgVal&: *DbgVal, Location: NewLocationOps[0], Ops&: ShortenedOps);
6928 } else {
6929 // Multiple DW_OP_llvm_arg, so DIArgList is strictly necessary.
6930 updateDVIWithLocations(DbgVal&: *DbgVal, Locations&: NewLocationOps, Ops&: NewExpr);
6931 }
6932
6933 // If the DIExpression was previously empty then add the stack terminator.
6934 // Non-empty expressions have only had elements inserted into them and so
6935 // the terminator should already be present e.g. stack_value or fragment.
6936 DIExpression *SalvageExpr = DbgVal->getExpression();
6937 if (!DVIRec.Expr->isComplex() && SalvageExpr->isComplex()) {
6938 SalvageExpr = DIExpression::append(Expr: SalvageExpr, Ops: {dwarf::DW_OP_stack_value});
6939 DbgVal->setExpression(SalvageExpr);
6940 }
6941}
6942
6943/// Cached location ops may be erased during LSR, in which case a poison is
6944/// required when restoring from the cache. The type of that location is no
6945/// longer available, so just use int8. The poison will be replaced by one or
6946/// more locations later when a SCEVDbgValueBuilder selects alternative
6947/// locations to use for the salvage.
6948static Value *getValueOrPoison(WeakVH &VH, LLVMContext &C) {
6949 return (VH) ? VH : PoisonValue::get(T: llvm::Type::getInt8Ty(C));
6950}
6951
6952/// Restore the DVI's pre-LSR arguments. Substitute undef for any erased values.
6953static void restorePreTransformState(DVIRecoveryRec &DVIRec) {
6954 DbgVariableRecord *DbgVal = DVIRec.DbgRef;
6955 LLVM_DEBUG(dbgs() << "scev-salvage: restore dbg.value to pre-LSR state\n"
6956 << "scev-salvage: post-LSR: " << *DbgVal << '\n');
6957 assert(DVIRec.Expr && "Expected an expression");
6958 DbgVal->setExpression(DVIRec.Expr);
6959
6960 // Even a single location-op may be inside a DIArgList and referenced with
6961 // DW_OP_LLVM_arg, which is valid only with a DIArgList.
6962 if (!DVIRec.HadLocationArgList) {
6963 assert(DVIRec.LocationOps.size() == 1 &&
6964 "Unexpected number of location ops.");
6965 // LSR's unsuccessful salvage attempt may have added DIArgList, which in
6966 // this case was not present before, so force the location back to a
6967 // single uncontained Value.
6968 Value *CachedValue =
6969 getValueOrPoison(VH&: DVIRec.LocationOps[0], C&: DbgVal->getContext());
6970 DbgVal->setRawLocation(ValueAsMetadata::get(V: CachedValue));
6971 } else {
6972 SmallVector<ValueAsMetadata *, 3> MetadataLocs;
6973 for (WeakVH VH : DVIRec.LocationOps) {
6974 Value *CachedValue = getValueOrPoison(VH, C&: DbgVal->getContext());
6975 MetadataLocs.push_back(Elt: ValueAsMetadata::get(V: CachedValue));
6976 }
6977 auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(MetadataLocs);
6978 DbgVal->setRawLocation(
6979 llvm::DIArgList::get(Context&: DbgVal->getContext(), Args: ValArrayRef));
6980 }
6981 LLVM_DEBUG(dbgs() << "scev-salvage: pre-LSR: " << *DbgVal << '\n');
6982}
6983
6984static bool SalvageDVI(llvm::Loop *L, ScalarEvolution &SE,
6985 llvm::PHINode *LSRInductionVar, DVIRecoveryRec &DVIRec,
6986 const SCEV *SCEVInductionVar,
6987 SCEVDbgValueBuilder IterCountExpr) {
6988
6989 if (!DVIRec.DbgRef->isKillLocation())
6990 return false;
6991
6992 // LSR may have caused several changes to the dbg.value in the failed salvage
6993 // attempt. So restore the DIExpression, the location ops and also the
6994 // location ops format, which is always DIArglist for multiple ops, but only
6995 // sometimes for a single op.
6996 restorePreTransformState(DVIRec);
6997
6998 // LocationOpIndexMap[i] will store the post-LSR location index of
6999 // the non-optimised out location at pre-LSR index i.
7000 SmallVector<int64_t, 2> LocationOpIndexMap;
7001 LocationOpIndexMap.assign(NumElts: DVIRec.LocationOps.size(), Elt: -1);
7002 SmallVector<Value *, 2> NewLocationOps;
7003 NewLocationOps.push_back(Elt: LSRInductionVar);
7004
7005 for (unsigned i = 0; i < DVIRec.LocationOps.size(); i++) {
7006 WeakVH VH = DVIRec.LocationOps[i];
7007 // Place the locations not optimised out in the list first, avoiding
7008 // inserts later. The map is used to update the DIExpression's
7009 // DW_OP_LLVM_arg arguments as the expression is updated.
7010 if (VH && !isa<UndefValue>(Val: VH)) {
7011 NewLocationOps.push_back(Elt: VH);
7012 LocationOpIndexMap[i] = NewLocationOps.size() - 1;
7013 LLVM_DEBUG(dbgs() << "scev-salvage: Location index " << i
7014 << " now at index " << LocationOpIndexMap[i] << "\n");
7015 continue;
7016 }
7017
7018 // It's possible that a value referred to in the SCEV may have been
7019 // optimised out by LSR.
7020 if (SE.containsErasedValue(S: DVIRec.SCEVs[i]) ||
7021 SE.containsUndefs(S: DVIRec.SCEVs[i])) {
7022 LLVM_DEBUG(dbgs() << "scev-salvage: SCEV for location at index: " << i
7023 << " refers to a location that is now undef or erased. "
7024 "Salvage abandoned.\n");
7025 return false;
7026 }
7027
7028 LLVM_DEBUG(dbgs() << "scev-salvage: salvaging location at index " << i
7029 << " with SCEV: " << *DVIRec.SCEVs[i] << "\n");
7030
7031 DVIRec.RecoveryExprs[i] = std::make_unique<SCEVDbgValueBuilder>();
7032 SCEVDbgValueBuilder *SalvageExpr = DVIRec.RecoveryExprs[i].get();
7033
7034 // Create an offset-based salvage expression if possible, as it requires
7035 // less DWARF ops than an iteration count-based expression.
7036 if (std::optional<APInt> Offset =
7037 SE.computeConstantDifference(LHS: DVIRec.SCEVs[i], RHS: SCEVInductionVar)) {
7038 if (Offset->getSignificantBits() <= 64)
7039 SalvageExpr->createOffsetExpr(Offset: Offset->getSExtValue(), OffsetValue: LSRInductionVar);
7040 else
7041 return false;
7042 } else if (!SalvageExpr->createIterCountExpr(S: DVIRec.SCEVs[i], IterationCount: IterCountExpr,
7043 SE))
7044 return false;
7045 }
7046
7047 // Merge the DbgValueBuilder generated expressions and the original
7048 // DIExpression, place the result into an new vector.
7049 SmallVector<uint64_t, 3> NewExpr;
7050 if (DVIRec.Expr->getNumElements() == 0) {
7051 assert(DVIRec.RecoveryExprs.size() == 1 &&
7052 "Expected only a single recovery expression for an empty "
7053 "DIExpression.");
7054 assert(DVIRec.RecoveryExprs[0] &&
7055 "Expected a SCEVDbgSalvageBuilder for location 0");
7056 SCEVDbgValueBuilder *B = DVIRec.RecoveryExprs[0].get();
7057 B->appendToVectors(DestExpr&: NewExpr, DestLocations&: NewLocationOps);
7058 }
7059 for (const auto &Op : DVIRec.Expr->expr_ops()) {
7060 // Most Ops needn't be updated.
7061 auto Arg = dyn_cast<DIExpression::ArgOp>(Val: Op);
7062 if (!Arg) {
7063 Op.appendToVector(V&: NewExpr);
7064 continue;
7065 }
7066
7067 uint64_t LocationArgIndex = Arg.getIndex();
7068 SCEVDbgValueBuilder *DbgBuilder =
7069 DVIRec.RecoveryExprs[LocationArgIndex].get();
7070 // The location doesn't have s SCEVDbgValueBuilder, so LSR did not
7071 // optimise it away. So just translate the argument to the updated
7072 // location index.
7073 if (!DbgBuilder) {
7074 NewExpr.push_back(Elt: dwarf::DW_OP_LLVM_arg);
7075 assert(LocationOpIndexMap[LocationArgIndex] != -1 &&
7076 "Expected a positive index for the location-op position.");
7077 NewExpr.push_back(Elt: LocationOpIndexMap[LocationArgIndex]);
7078 continue;
7079 }
7080 // The location has a recovery expression.
7081 DbgBuilder->appendToVectors(DestExpr&: NewExpr, DestLocations&: NewLocationOps);
7082 }
7083
7084 UpdateDbgValue(DVIRec, NewLocationOps, NewExpr);
7085 LLVM_DEBUG(dbgs() << "scev-salvage: Updated DVI: " << *DVIRec.DbgRef << "\n");
7086 return true;
7087}
7088
7089/// Obtain an expression for the iteration count, then attempt to salvage the
7090/// dbg.value intrinsics.
7091static void DbgRewriteSalvageableDVIs(
7092 llvm::Loop *L, ScalarEvolution &SE, llvm::PHINode *LSRInductionVar,
7093 SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &DVIToUpdate) {
7094 if (DVIToUpdate.empty())
7095 return;
7096
7097 const llvm::SCEV *SCEVInductionVar = SE.getSCEV(V: LSRInductionVar);
7098 assert(SCEVInductionVar &&
7099 "Anticipated a SCEV for the post-LSR induction variable");
7100
7101 if (const SCEVAddRecExpr *IVAddRec =
7102 dyn_cast<SCEVAddRecExpr>(Val: SCEVInductionVar)) {
7103 if (!IVAddRec->isAffine())
7104 return;
7105
7106 // Prevent translation using excessive resources.
7107 if (IVAddRec->getExpressionSize() > MaxSCEVSalvageExpressionSize)
7108 return;
7109
7110 // The iteration count is required to recover location values.
7111 SCEVDbgValueBuilder IterCountExpr;
7112 IterCountExpr.pushLocation(V: LSRInductionVar);
7113 if (!IterCountExpr.SCEVToIterCountExpr(SAR: *IVAddRec, SE))
7114 return;
7115
7116 LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV: " << *SCEVInductionVar
7117 << '\n');
7118
7119 for (auto &DVIRec : DVIToUpdate) {
7120 SalvageDVI(L, SE, LSRInductionVar, DVIRec&: *DVIRec, SCEVInductionVar,
7121 IterCountExpr);
7122 }
7123 }
7124}
7125
7126/// Identify and cache salvageable DVI locations and expressions along with the
7127/// corresponding SCEV(s). Also ensure that the DVI is not deleted between
7128/// cacheing and salvaging.
7129static void DbgGatherSalvagableDVI(
7130 Loop *L, ScalarEvolution &SE,
7131 SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> &SalvageableDVISCEVs) {
7132 for (const auto &B : L->getBlocks()) {
7133 for (auto &I : *B) {
7134 for (DbgVariableRecord &DbgVal : filterDbgVars(R: I.getDbgRecordRange())) {
7135 if (!DbgVal.isDbgValue() && !DbgVal.isDbgAssign())
7136 continue;
7137
7138 // Ensure that if any location op is undef that the dbg.vlue is not
7139 // cached.
7140 if (DbgVal.isKillLocation())
7141 continue;
7142
7143 // Check that the location op SCEVs are suitable for translation to
7144 // DIExpression.
7145 const auto &HasTranslatableLocationOps =
7146 [&](const DbgVariableRecord &DbgValToTranslate) -> bool {
7147 for (const auto LocOp : DbgValToTranslate.location_ops()) {
7148 if (!LocOp)
7149 return false;
7150
7151 if (!SE.isSCEVable(Ty: LocOp->getType()))
7152 return false;
7153
7154 const SCEV *S = SE.getSCEV(V: LocOp);
7155 if (SE.containsUndefs(S))
7156 return false;
7157 }
7158 return true;
7159 };
7160
7161 if (!HasTranslatableLocationOps(DbgVal))
7162 continue;
7163
7164 std::unique_ptr<DVIRecoveryRec> NewRec =
7165 std::make_unique<DVIRecoveryRec>(args: &DbgVal);
7166 // Each location Op may need a SCEVDbgValueBuilder in order to recover
7167 // it. Pre-allocating a vector will enable quick lookups of the builder
7168 // later during the salvage.
7169 NewRec->RecoveryExprs.resize(N: DbgVal.getNumVariableLocationOps());
7170 for (const auto LocOp : DbgVal.location_ops()) {
7171 NewRec->SCEVs.push_back(Elt: SE.getSCEV(V: LocOp));
7172 NewRec->LocationOps.push_back(Elt: LocOp);
7173 NewRec->HadLocationArgList = DbgVal.hasArgList();
7174 }
7175 SalvageableDVISCEVs.push_back(Elt: std::move(NewRec));
7176 }
7177 }
7178 }
7179}
7180
7181/// Ideally pick the PHI IV inserted by ScalarEvolutionExpander. As a fallback
7182/// any PHi from the loop header is usable, but may have less chance of
7183/// surviving subsequent transforms.
7184static llvm::PHINode *GetInductionVariable(const Loop &L, ScalarEvolution &SE,
7185 const LSRInstance &LSR) {
7186
7187 auto IsSuitableIV = [&](PHINode *P) {
7188 if (!SE.isSCEVable(Ty: P->getType()))
7189 return false;
7190 if (const SCEVAddRecExpr *Rec = dyn_cast<SCEVAddRecExpr>(Val: SE.getSCEV(V: P)))
7191 return Rec->isAffine() && !SE.containsUndefs(S: SE.getSCEV(V: P));
7192 return false;
7193 };
7194
7195 // For now, just pick the first IV that was generated and inserted by
7196 // ScalarEvolution. Ideally pick an IV that is unlikely to be optimised away
7197 // by subsequent transforms.
7198 for (const WeakVH &IV : LSR.getScalarEvolutionIVs()) {
7199 if (!IV)
7200 continue;
7201
7202 // There should only be PHI node IVs.
7203 PHINode *P = cast<PHINode>(Val: &*IV);
7204
7205 if (IsSuitableIV(P))
7206 return P;
7207 }
7208
7209 for (PHINode &P : L.getHeader()->phis()) {
7210 if (IsSuitableIV(&P))
7211 return &P;
7212 }
7213 return nullptr;
7214}
7215
7216static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
7217 DominatorTree &DT, LoopInfo &LI,
7218 const TargetTransformInfo &TTI,
7219 AssumptionCache &AC, TargetLibraryInfo &TLI,
7220 MemorySSA *MSSA, bool PreserveLCSSA) {
7221 const ScalarOptions &Opts = getScalarOptions(L: *L);
7222
7223 // Debug preservation - before we start removing anything identify which DVI
7224 // meet the salvageable criteria and store their DIExpression and SCEVs.
7225 SmallVector<std::unique_ptr<DVIRecoveryRec>, 2> SalvageableDVIRecords;
7226 DbgGatherSalvagableDVI(L, SE, SalvageableDVISCEVs&: SalvageableDVIRecords);
7227
7228 bool Changed = false;
7229 std::unique_ptr<MemorySSAUpdater> MSSAU;
7230 if (MSSA)
7231 MSSAU = std::make_unique<MemorySSAUpdater>(args&: MSSA);
7232
7233 // Run the main LSR transformation.
7234 const LSRInstance &Reducer = LSRInstance(Opts, L, IU, SE, DT, LI, TTI, AC,
7235 TLI, MSSAU.get(), PreserveLCSSA);
7236 Changed |= Reducer.getChanged();
7237
7238 // Remove any extra phis created by processing inner loops.
7239 Changed |= DeleteDeadPHIs(BB: L->getHeader(), TLI: &TLI, MSSAU: MSSAU.get());
7240 if (Opts.enable_lsr_phielim && L->isLoopSimplifyForm()) {
7241 SmallVector<WeakTrackingVH, 16> DeadInsts;
7242 SCEVExpander Rewriter(SE, "lsr", false);
7243#if LLVM_ENABLE_ABI_BREAKING_CHECKS
7244 Rewriter.setDebugType(DEBUG_TYPE);
7245#endif
7246 unsigned numFolded = Rewriter.replaceCongruentIVs(L, DT: &DT, DeadInsts, TTI: &TTI);
7247 Rewriter.clear();
7248 if (numFolded) {
7249 Changed = true;
7250 RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts, TLI: &TLI,
7251 MSSAU: MSSAU.get());
7252 DeleteDeadPHIs(BB: L->getHeader(), TLI: &TLI, MSSAU: MSSAU.get());
7253 }
7254 }
7255
7256 // We might end up with duplicate phi nodes when preserving LCSSA. Clean them
7257 // up here, as otherwise rewriteLoopExitValues will bail due to the extra
7258 // uses.
7259 SmallVector<BasicBlock *, 4> ExitBlocks;
7260 L->getUniqueExitBlocks(ExitBlocks);
7261 for (BasicBlock *ExitBB : ExitBlocks)
7262 Changed |= EliminateDuplicatePHINodes(BB: ExitBB);
7263
7264 // LSR may at times remove all uses of an induction variable from a loop.
7265 // The only remaining use is the PHI in the exit block.
7266 // When this is the case, if the exit value of the IV can be calculated using
7267 // SCEV, we can replace the exit block PHI with the final value of the IV and
7268 // skip the updates in each loop iteration.
7269 if (L->isRecursivelyLCSSAForm(DT, LI) && L->getExitBlock()) {
7270 SmallVector<WeakTrackingVH, 16> DeadInsts;
7271 SCEVExpander Rewriter(SE, "lsr", true);
7272 int Rewrites = rewriteLoopExitValues(L, LI: &LI, TLI: &TLI, SE: &SE, TTI: &TTI, Rewriter, DT: &DT,
7273 ReplaceExitValue: UnusedIndVarInLoop, DeadInsts);
7274 Rewriter.clear();
7275 if (Rewrites) {
7276 Changed = true;
7277 RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts, TLI: &TLI,
7278 MSSAU: MSSAU.get());
7279 DeleteDeadPHIs(BB: L->getHeader(), TLI: &TLI, MSSAU: MSSAU.get());
7280 }
7281 }
7282
7283 if (SalvageableDVIRecords.empty())
7284 return Changed;
7285
7286 // Obtain relevant IVs and attempt to rewrite the salvageable DVIs with
7287 // expressions composed using the derived iteration count.
7288 // TODO: Allow for multiple IV references for nested AddRecSCEVs
7289 for (const auto &L : LI) {
7290 if (llvm::PHINode *IV = GetInductionVariable(L: *L, SE, LSR: Reducer))
7291 DbgRewriteSalvageableDVIs(L, SE, LSRInductionVar: IV, DVIToUpdate&: SalvageableDVIRecords);
7292 else {
7293 LLVM_DEBUG(dbgs() << "scev-salvage: SCEV salvaging not possible. An IV "
7294 "could not be identified.\n");
7295 }
7296 }
7297
7298 for (auto &Rec : SalvageableDVIRecords)
7299 Rec->clear();
7300 SalvageableDVIRecords.clear();
7301 return Changed;
7302}
7303
7304bool LoopStrengthReduce::runOnLoop(Loop *L, LPPassManager & /*LPM*/) {
7305 if (skipLoop(L))
7306 return false;
7307
7308 auto &IU = getAnalysis<IVUsersWrapperPass>().getIU();
7309 auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
7310 auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
7311 auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
7312 const auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
7313 F: *L->getHeader()->getParent());
7314 auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(
7315 F&: *L->getHeader()->getParent());
7316 auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
7317 F: *L->getHeader()->getParent());
7318 auto *MSSAAnalysis = getAnalysisIfAvailable<MemorySSAWrapperPass>();
7319 MemorySSA *MSSA = nullptr;
7320 if (MSSAAnalysis)
7321 MSSA = &MSSAAnalysis->getMSSA();
7322 return ReduceLoopStrength(L, IU, SE, DT, LI, TTI, AC, TLI, MSSA,
7323 /*PreserveLCSSA=*/false);
7324}
7325
7326PreservedAnalyses LoopStrengthReducePass::run(Loop &L, LoopAnalysisManager &AM,
7327 LoopStandardAnalysisResults &AR,
7328 LPMUpdater &) {
7329 if (!ReduceLoopStrength(L: &L, IU&: AM.getResult<IVUsersAnalysis>(IR&: L, ExtraArgs&: AR), SE&: AR.SE,
7330 DT&: AR.DT, LI&: AR.LI, TTI: AR.TTI, AC&: AR.AC, TLI&: AR.TLI, MSSA: AR.MSSA,
7331 /*PreserveLCSSA=*/true))
7332 return PreservedAnalyses::all();
7333
7334 auto PA = getLoopPassPreservedAnalyses();
7335 if (AR.MSSA)
7336 PA.preserve<MemorySSAAnalysis>();
7337 return PA;
7338}
7339
7340char LoopStrengthReduce::ID = 0;
7341
7342INITIALIZE_PASS_BEGIN(LoopStrengthReduce, "loop-reduce",
7343 "Loop Strength Reduction", false, false)
7344INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
7345INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
7346INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
7347INITIALIZE_PASS_DEPENDENCY(IVUsersWrapperPass)
7348INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
7349INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
7350INITIALIZE_PASS_END(LoopStrengthReduce, "loop-reduce",
7351 "Loop Strength Reduction", false, false)
7352
7353Pass *llvm::createLoopStrengthReducePass() { return new LoopStrengthReduce(); }
7354