1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
56#include "llvm/Transforms/Vectorize/LoopVectorize.h"
57#include "LoopVectorizationPlanner.h"
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanAnalysis.h"
61#include "VPlanCFG.h"
62#include "VPlanHelpers.h"
63#include "VPlanPatternMatch.h"
64#include "VPlanTransforms.h"
65#include "VPlanUtils.h"
66#include "VPlanVerifier.h"
67#include "VectorizeOptions.h"
68#include "llvm/ADT/APInt.h"
69#include "llvm/ADT/ArrayRef.h"
70#include "llvm/ADT/DenseMap.h"
71#include "llvm/ADT/Hashing.h"
72#include "llvm/ADT/MapVector.h"
73#include "llvm/ADT/STLExtras.h"
74#include "llvm/ADT/SmallPtrSet.h"
75#include "llvm/ADT/SmallVector.h"
76#include "llvm/ADT/Statistic.h"
77#include "llvm/ADT/StringRef.h"
78#include "llvm/ADT/Twine.h"
79#include "llvm/ADT/TypeSwitch.h"
80#include "llvm/ADT/iterator_range.h"
81#include "llvm/Analysis/AssumptionCache.h"
82#include "llvm/Analysis/BasicAliasAnalysis.h"
83#include "llvm/Analysis/BlockFrequencyInfo.h"
84#include "llvm/Analysis/BranchProbabilityInfo.h"
85#include "llvm/Analysis/CFG.h"
86#include "llvm/Analysis/CodeMetrics.h"
87#include "llvm/Analysis/CycleAnalysis.h"
88#include "llvm/Analysis/DemandedBits.h"
89#include "llvm/Analysis/GlobalsModRef.h"
90#include "llvm/Analysis/LoopAccessAnalysis.h"
91#include "llvm/Analysis/LoopAnalysisManager.h"
92#include "llvm/Analysis/LoopInfo.h"
93#include "llvm/Analysis/LoopIterator.h"
94#include "llvm/Analysis/OptimizationRemarkEmitter.h"
95#include "llvm/Analysis/ProfileSummaryInfo.h"
96#include "llvm/Analysis/ScalarEvolution.h"
97#include "llvm/Analysis/ScalarEvolutionExpressions.h"
98#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
99#include "llvm/Analysis/TargetLibraryInfo.h"
100#include "llvm/Analysis/TargetTransformInfo.h"
101#include "llvm/Analysis/ValueTracking.h"
102#include "llvm/Analysis/VectorUtils.h"
103#include "llvm/IR/Attributes.h"
104#include "llvm/IR/BasicBlock.h"
105#include "llvm/IR/CFG.h"
106#include "llvm/IR/Constant.h"
107#include "llvm/IR/Constants.h"
108#include "llvm/IR/DataLayout.h"
109#include "llvm/IR/DebugInfo.h"
110#include "llvm/IR/DebugLoc.h"
111#include "llvm/IR/DerivedTypes.h"
112#include "llvm/IR/DiagnosticInfo.h"
113#include "llvm/IR/Dominators.h"
114#include "llvm/IR/Function.h"
115#include "llvm/IR/IRBuilder.h"
116#include "llvm/IR/InstrTypes.h"
117#include "llvm/IR/Instruction.h"
118#include "llvm/IR/Instructions.h"
119#include "llvm/IR/IntrinsicInst.h"
120#include "llvm/IR/Intrinsics.h"
121#include "llvm/IR/MDBuilder.h"
122#include "llvm/IR/Metadata.h"
123#include "llvm/IR/Module.h"
124#include "llvm/IR/Operator.h"
125#include "llvm/IR/PatternMatch.h"
126#include "llvm/IR/ProfDataUtils.h"
127#include "llvm/IR/Type.h"
128#include "llvm/IR/Use.h"
129#include "llvm/IR/User.h"
130#include "llvm/IR/Value.h"
131#include "llvm/IR/Verifier.h"
132#include "llvm/Support/Casting.h"
133#include "llvm/Support/CommandLine.h"
134#include "llvm/Support/Debug.h"
135#include "llvm/Support/ErrorHandling.h"
136#include "llvm/Support/InstructionCost.h"
137#include "llvm/Support/MathExtras.h"
138#include "llvm/Support/NativeFormatting.h"
139#include "llvm/Support/raw_ostream.h"
140#include "llvm/Transforms/Utils/BasicBlockUtils.h"
141#include "llvm/Transforms/Utils/InjectTLIMappings.h"
142#include "llvm/Transforms/Utils/Local.h"
143#include "llvm/Transforms/Utils/LoopSimplify.h"
144#include "llvm/Transforms/Utils/LoopUtils.h"
145#include "llvm/Transforms/Utils/LoopVersioning.h"
146#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
147#include "llvm/Transforms/Utils/SizeOpts.h"
148#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
149#include <algorithm>
150#include <cassert>
151#include <cmath>
152#include <cstdint>
153#include <functional>
154#include <iterator>
155#include <memory>
156#include <string>
157#include <tuple>
158#include <utility>
159
160using namespace llvm;
161using namespace SCEVPatternMatch;
162using namespace LoopVectorizationUtils;
163
164#define LV_NAME "loop-vectorize"
165#define DEBUG_TYPE LV_NAME
166
167#ifndef NDEBUG
168const char VerboseDebug[] = DEBUG_TYPE "-verbose";
169#endif
170
171STATISTIC(LoopsVectorized, "Number of loops vectorized");
172STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
173STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
174STATISTIC(LoopsEarlyExitVectorized, "Number of early exit loops vectorized");
175STATISTIC(LoopsPartialAliasVectorized,
176 "Number of partial aliasing loops vectorized");
177
178static cl::opt<ElementCount> EpilogueVectorizationForceVF(
179 "epilogue-vectorization-force-VF", cl::init(Val: ElementCount::getFixed(MinVal: 1)),
180 cl::Hidden,
181 cl::desc("When epilogue vectorization is enabled, and a value greater than "
182 "1 is specified, forces the given VF for all applicable epilogue "
183 "loops. Note: This allows all scalable VFs >= vscale x 1."));
184
185// Returns true if the epilogue VF has been set to a non-zero value other than
186// VF=1 (scalar).
187static bool hasForcedEpilogueVF() {
188 return EpilogueVectorizationForceVF.isNonZero() &&
189 EpilogueVectorizationForceVF != ElementCount::getFixed(MinVal: 1);
190}
191
192// Likelyhood of bypassing the vectorized loop because there are zero trips left
193// after prolog. See `emitIterationCountCheck`.
194static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
195
196/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
197/// ElementCount to include loops whose trip count is a function of vscale.
198static ElementCount getSmallConstantTripCount(ScalarEvolution *SE,
199 const Loop *L) {
200 if (unsigned ExpectedTC = SE->getSmallConstantTripCount(L))
201 return ElementCount::getFixed(MinVal: ExpectedTC);
202
203 const SCEV *BTC = SE->getBackedgeTakenCount(L);
204 if (isa<SCEVCouldNotCompute>(Val: BTC))
205 return ElementCount::getFixed(MinVal: 0);
206
207 const SCEV *ExitCount = SE->getTripCountFromExitCount(ExitCount: BTC, EvalTy: BTC->getType(), L);
208 if (isa<SCEVVScale>(Val: ExitCount))
209 return ElementCount::getScalable(MinVal: 1);
210
211 const APInt *Scale;
212 if (match(S: ExitCount, P: m_scev_Mul(Op0: m_scev_APInt(C&: Scale), Op1: m_SCEVVScale())))
213 if (cast<SCEVMulExpr>(Val: ExitCount)->hasNoUnsignedWrap())
214 if (Scale->getActiveBits() <= 32)
215 return ElementCount::getScalable(MinVal: Scale->getZExtValue());
216
217 return ElementCount::getFixed(MinVal: 0);
218}
219
220/// Get the maximum trip count for \p L from the SCEV unsigned range, excluding
221/// zero from the range. Only valid when not folding the tail, as the minimum
222/// iteration count check guards against a zero trip count. Returns 0 if
223/// unknown.
224static unsigned getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE,
225 Loop *L) {
226 const SCEV *BTC = PSE.getBackedgeTakenCount();
227 if (isa<SCEVCouldNotCompute>(Val: BTC))
228 return 0;
229 ScalarEvolution *SE = PSE.getSE();
230 const SCEV *TripCount = SE->getTripCountFromExitCount(ExitCount: BTC, EvalTy: BTC->getType(), L);
231 ConstantRange TCRange = SE->getUnsignedRange(S: TripCount);
232 APInt MaxTCFromRange = TCRange.getUnsignedMax();
233 if (!MaxTCFromRange.isZero() && MaxTCFromRange.getActiveBits() <= 32)
234 return MaxTCFromRange.getZExtValue();
235 return 0;
236}
237
238/// Returns "best known" trip count, which is either a valid positive trip count
239/// or std::nullopt when an estimate cannot be made (including when the trip
240/// count would overflow), for the specified loop \p L as defined by the
241/// following procedure:
242/// 1) Returns exact trip count if it is known.
243/// 2) Returns expected trip count according to profile data if any.
244/// 3) Returns upper bound estimate if known, if \p CanUseConstantMax, and
245/// if \p ComputeUpperBoundOnly is false.
246/// 4) Returns the maximum trip count from the SCEV range excluding zero,
247/// if \p CanUseConstantMax and \p CanExcludeZeroTrips.
248/// 5) Returns std::nullopt if all of the above failed.
249static std::optional<ElementCount> getSmallBestKnownTC(
250 const VectorizeOptions &Opts, PredicatedScalarEvolution &PSE, Loop *L,
251 bool CanUseConstantMax = true, bool CanExcludeZeroTrips = false,
252 bool ComputeUpperBoundOnly = false) {
253 // Check if exact trip count is known.
254 if (auto ExpectedTC = getSmallConstantTripCount(SE: PSE.getSE(), L))
255 return ExpectedTC;
256
257 // Check if there is an expected trip count available from profile data.
258 // An estimate of zero means the loop is estimated not to be entered; it is
259 // not a usable trip count for the profitability decisions below (and would
260 // e.g. divide by zero when scaling runtime check cost), so treat it as
261 // unknown.
262 if (Opts.loop_vectorize_with_block_frequency && !ComputeUpperBoundOnly)
263 if (unsigned EstimatedTC = getLoopEstimatedTripCount(L).value_or(u: 0))
264 return ElementCount::getFixed(MinVal: EstimatedTC);
265
266 if (!CanUseConstantMax)
267 return std::nullopt;
268
269 // Check if upper bound estimate is known.
270 if (unsigned ExpectedTC = PSE.getSmallConstantMaxTripCount())
271 return ElementCount::getFixed(MinVal: ExpectedTC);
272
273 // Get the maximum trip count from the SCEV range excluding zero. This is
274 // only safe when not folding the tail, as the minimum iteration count check
275 // prevents entering the vector loop with a zero trip count.
276 if (CanUseConstantMax && CanExcludeZeroTrips)
277 if (unsigned RefinedTC = getMaxTCFromNonZeroRange(PSE, L))
278 return ElementCount::getFixed(MinVal: RefinedTC);
279
280 return std::nullopt;
281}
282
283namespace {
284// Forward declare GeneratedRTChecks.
285class GeneratedRTChecks;
286
287using SCEV2ValueTy = DenseMap<const SCEV *, Value *>;
288} // namespace
289
290namespace llvm {
291
292AnalysisKey ShouldRunExtraVectorPasses::Key;
293
294/// InnerLoopVectorizer vectorizes loops which contain only one basic
295/// block to a specified vectorization factor (VF).
296/// This class performs the widening of scalars into vectors, or multiple
297/// scalars. This class also implements the following features:
298/// * It inserts an epilogue loop for handling loops that don't have iteration
299/// counts that are known to be a multiple of the vectorization factor.
300/// * It handles the code generation for reduction variables.
301/// * Scalarization (implementation using scalars) of un-vectorizable
302/// instructions.
303/// InnerLoopVectorizer does not perform any vectorization-legality
304/// checks, and relies on the caller to check for the different legality
305/// aspects. The InnerLoopVectorizer relies on the
306/// LoopVectorizationLegality class to provide information about the induction
307/// and reduction variables that were found to a given vectorization factor.
308class InnerLoopVectorizer {
309public:
310 InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
311 LoopInfo *LI, DominatorTree *DT,
312 const TargetTransformInfo *TTI, AssumptionCache *AC,
313 ElementCount VecWidth, unsigned UnrollFactor,
314 GeneratedRTChecks &RTChecks, VPlan &Plan)
315 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TTI(TTI), AC(AC),
316 VF(VecWidth), UF(UnrollFactor), Builder(PSE.getSE()->getModule()),
317 RTChecks(RTChecks), Plan(Plan),
318 VectorPHVPBB(cast<VPBasicBlock>(
319 Val: Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
320
321 virtual ~InnerLoopVectorizer() = default;
322
323 /// Creates a basic block for the scalar preheader.
324 /// EpilogueVectorizerEpilogueLoop overrides the method to create additional
325 /// blocks and checks needed for epilogue vectorization.
326 virtual BasicBlock *createVectorizedLoopSkeleton();
327
328 /// Fix the vectorized code, taking care of header phi's, and more.
329 void fixVectorizedLoop(VPTransformState &State);
330
331protected:
332 friend class LoopVectorizationPlanner;
333
334 /// Create and return a new IR basic block for the scalar preheader whose name
335 /// is prefixed with \p Prefix.
336 BasicBlock *createScalarPreheader(StringRef Prefix);
337
338 /// The original loop.
339 Loop *OrigLoop;
340
341 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
342 /// dynamic knowledge to simplify SCEV expressions and converts them to a
343 /// more usable form.
344 PredicatedScalarEvolution &PSE;
345
346 /// Loop Info.
347 LoopInfo *LI;
348
349 /// Dominator Tree.
350 DominatorTree *DT;
351
352 /// Target Transform Info.
353 const TargetTransformInfo *TTI;
354
355 /// Assumption Cache.
356 AssumptionCache *AC;
357
358 /// The vectorization SIMD factor to use. Each vector will have this many
359 /// vector elements.
360 ElementCount VF;
361
362 /// The vectorization unroll factor to use. Each scalar is vectorized to this
363 /// many different vector instructions.
364 unsigned UF;
365
366 /// The builder that we use
367 IRBuilder<> Builder;
368
369 // --- Vectorization state ---
370
371 /// Structure to hold information about generated runtime checks, responsible
372 /// for cleaning the checks, if vectorization turns out unprofitable.
373 GeneratedRTChecks &RTChecks;
374
375 VPlan &Plan;
376
377 /// The vector preheader block of \p Plan, used as target for check blocks
378 /// introduced during skeleton creation.
379 VPBasicBlock *VectorPHVPBB;
380};
381
382/// A specialized derived class of inner loop vectorizer that performs
383/// vectorization of *epilogue* loops in the process of vectorizing loops and
384/// their epilogues. The idea is to run the vplan on a given loop twice, firstly
385/// to vectorize the main loop, and secondly to complete the skeleton from the
386/// first step and vectorize the epilogue. This helps us avoid regenerating and
387/// recomputing runtime safety checks, and shortens the iteration-count-check
388/// path length for loops whose iteration count is so small that the main vector
389/// loop is completely skipped.
390class EpilogueVectorizerEpilogueLoop : public InnerLoopVectorizer {
391 VPlan &MainPlan;
392
393public:
394 VPIRBasicBlock *VecEpilogueIterationCountCheck = nullptr;
395
396 EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
397 LoopInfo *LI, DominatorTree *DT,
398 const TargetTransformInfo *TTI,
399 AssumptionCache *AC, ElementCount VecWidth,
400 unsigned UnrollFactor,
401 GeneratedRTChecks &Checks, VPlan &Plan,
402 VPlan &MainPlan)
403 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, VecWidth,
404 UnrollFactor, Checks, Plan),
405 MainPlan(MainPlan) {}
406 /// Implements the interface for creating a vectorized skeleton using the
407 /// *epilogue loop* strategy (i.e., the second pass of VPlan execution).
408 BasicBlock *createVectorizedLoopSkeleton() final;
409};
410} // end namespace llvm
411
412/// Look for a meaningful debug location on the instruction or its operands.
413static DebugLoc getDebugLocFromInstOrOperands(Instruction *I) {
414 if (!I)
415 return DebugLoc::getUnknown();
416
417 DebugLoc Empty;
418 if (I->getDebugLoc() != Empty)
419 return I->getDebugLoc();
420
421 for (Use &Op : I->operands()) {
422 if (Instruction *OpInst = dyn_cast<Instruction>(Val&: Op))
423 if (OpInst->getDebugLoc() != Empty)
424 return OpInst->getDebugLoc();
425 }
426
427 return I->getDebugLoc();
428}
429
430namespace llvm {
431
432/// Return the runtime value for VF.
433Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF) {
434 return B.CreateElementCount(Ty, EC: VF);
435}
436
437} // end namespace llvm
438
439namespace llvm {
440
441// Loop vectorization cost-model hints how the epilogue/tail loop should be
442// lowered.
443enum EpilogueLowering {
444
445 // The default: allowing epilogues.
446 CM_EpilogueAllowed,
447
448 // Vectorization with OptForSize: don't allow epilogues.
449 CM_EpilogueNotAllowedOptSize,
450
451 // A special case of vectorisation with OptForSize: loops with a very small
452 // trip count are considered for vectorization under OptForSize, thereby
453 // making sure the cost of their loop body is dominant, free of runtime
454 // guards and scalar iteration overheads.
455 CM_EpilogueNotAllowedLowTripLoop,
456
457 // Loop hint indicating an epilogue is undesired, apply tail folding.
458 CM_EpilogueNotNeededFoldTail,
459
460 // Directive indicating we must either fold the epilogue/tail or not vectorize
461 CM_EpilogueNotAllowedFoldTail
462};
463
464enum class AliasMaskingStatus { NotDecided, Disabled, Enabled };
465
466/// LoopVectorizationCostModel - estimates the expected speedups due to
467/// vectorization.
468/// In many cases vectorization is not profitable. This can happen because of
469/// a number of reasons. In this class we mainly attempt to predict the
470/// expected speedup/slowdowns due to the supported instruction set. We use the
471/// TargetTransformInfo to query the different backends for the cost of
472/// different operations.
473class LoopVectorizationCostModel {
474 friend class LoopVectorizationPlanner;
475
476public:
477 LoopVectorizationCostModel(EpilogueLowering SEL, Loop *L,
478 PredicatedScalarEvolution &PSE, LoopInfo *LI,
479 LoopVectorizationLegality *Legal,
480 const TargetTransformInfo &TTI,
481 const TargetLibraryInfo *TLI, AssumptionCache *AC,
482 OptimizationRemarkEmitter *ORE,
483 std::function<BlockFrequencyInfo &()> GetBFI,
484 const Function *F, InterleavedAccessInfo &IAI,
485 VFSelectionContext &Config)
486 : Opts(Config.getOpts()), Config(Config), EpilogueLoweringStatus(SEL),
487 TheLoop(L), PSE(PSE), LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), AC(AC),
488 ORE(ORE), GetBFI(GetBFI), TheFunction(F), InterleaveInfo(IAI) {}
489
490 /// \return An upper bound for the vectorization factors (both fixed and
491 /// scalable). If the factors are 0, vectorization and interleaving should be
492 /// avoided up front.
493 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
494
495 /// Memory access instruction may be vectorized in more than one way.
496 /// Form of instruction after vectorization depends on cost.
497 /// This function takes cost-based decisions for Load/Store instructions
498 /// and collects them in a map. This decisions map is used for building
499 /// the lists of loop-uniform and loop-scalar instructions.
500 /// The calculated cost is saved with widening decision in order to
501 /// avoid redundant calculations.
502 void setCostBasedWideningDecision(ElementCount VF);
503
504 /// Collect values we want to ignore in the cost model.
505 void collectValuesToIgnore();
506
507 /// \returns True if it is more profitable to scalarize instruction \p I for
508 /// vectorization factor \p VF.
509 bool isProfitableToScalarize(Instruction *I, ElementCount VF) const {
510 assert(VF.isVector() &&
511 "Profitable to scalarize relevant only for VF > 1.");
512 assert(
513 TheLoop->isInnermost() &&
514 "cost-model should not be used for outer loops (in VPlan-native path)");
515
516 auto Scalars = InstsToScalarize.find(Key: VF);
517 assert(Scalars != InstsToScalarize.end() &&
518 "VF not yet analyzed for scalarization profitability");
519 return Scalars->second.contains(Key: I);
520 }
521
522 /// Returns true if \p I is known to be uniform after vectorization.
523 bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const {
524 assert(
525 TheLoop->isInnermost() &&
526 "cost-model should not be used for outer loops (in VPlan-native path)");
527
528 // If VF is scalar, then all instructions are trivially uniform.
529 if (VF.isScalar())
530 return true;
531
532 // Pseudo probes must be duplicated per vector lane so that the
533 // profiled loop trip count is not undercounted.
534 if (isa<PseudoProbeInst>(Val: I))
535 return false;
536
537 auto UniformsPerVF = Uniforms.find(Val: VF);
538 assert(UniformsPerVF != Uniforms.end() &&
539 "VF not yet analyzed for uniformity");
540 return UniformsPerVF->second.count(Ptr: I);
541 }
542
543 /// Returns true if \p I is known to be scalar after vectorization.
544 bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const {
545 assert(
546 TheLoop->isInnermost() &&
547 "cost-model should not be used for outer loops (in VPlan-native path)");
548 if (VF.isScalar())
549 return true;
550
551 auto ScalarsPerVF = Scalars.find(Val: VF);
552 assert(ScalarsPerVF != Scalars.end() &&
553 "Scalar values are not calculated for VF");
554 return ScalarsPerVF->second.count(Ptr: I);
555 }
556
557 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
558 /// for vectorization factor \p VF.
559 bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const {
560 const auto &MinBWs = Config.getMinimalBitwidths();
561 // Truncs must truncate at most to their destination type.
562 if (isa_and_nonnull<TruncInst>(Val: I) && MinBWs.contains(Key: I) &&
563 I->getType()->getScalarSizeInBits() < MinBWs.lookup(Key: I))
564 return false;
565 return VF.isVector() && MinBWs.contains(Key: I) &&
566 !isProfitableToScalarize(I, VF) &&
567 !isScalarAfterVectorization(I, VF);
568 }
569
570 /// Decision that was taken during cost calculation for memory instruction.
571 enum InstWidening {
572 CM_Unknown,
573 CM_Widen, // For consecutive accesses with stride +1.
574 CM_Widen_Reverse, // For consecutive accesses with stride -1.
575 CM_Interleave,
576 CM_GatherScatter,
577 CM_Scalarize,
578 /// A widening decision that has been invalidated after replacing the
579 /// corresponding recipe during VPlan transforms.
580 /// TODO: Remove once the legacy exit cost computation is retired.
581 CM_InvalidatedDecision
582 };
583
584#ifndef NDEBUG
585 static constexpr StringLiteral getInstWideningStr(InstWidening W) {
586 constexpr StringLiteral WideningStr[] = {
587 "Unknown", "Widen", "Widen_Reverse", "Interleave",
588 "GatherScatter", "Scalarize", "InvalidatedDecision"};
589 return WideningStr[W];
590 }
591#endif
592
593 /// Save vectorization decision \p W and \p Cost taken by the cost model for
594 /// instruction \p I and vector width \p VF.
595 void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W,
596 InstructionCost Cost) {
597 assert(VF.isVector() && "Expected VF >=2");
598 LLVM_DEBUG(dbgs() << "LV: Setting widening decision to "
599 << getInstWideningStr(W) << " for VF " << VF
600 << " and instruction: " << *I << '\n');
601 WideningDecisions[{I, VF}] = {W, Cost};
602 }
603
604 /// Save vectorization decision \p W and \p Cost taken by the cost model for
605 /// interleaving group \p Grp and vector width \p VF.
606 void setWideningDecision(const InterleaveGroup<Instruction> *Grp,
607 ElementCount VF, InstWidening W,
608 InstructionCost Cost) {
609 assert(VF.isVector() && "Expected VF >=2");
610 /// Broadcast this decicion to all instructions inside the group.
611 /// When interleaving, the cost will only be assigned one instruction, the
612 /// insert position. For other cases, add the appropriate fraction of the
613 /// total cost to each instruction. This ensures accurate costs are used,
614 /// even if the insert position instruction is not used.
615 InstructionCost InsertPosCost = Cost;
616 InstructionCost OtherMemberCost = 0;
617 if (W != CM_Interleave)
618 OtherMemberCost = InsertPosCost = Cost / Grp->getNumMembers();
619 ;
620 for (auto *I : Grp->members()) {
621 LLVM_DEBUG(dbgs() << "LV: Setting widening decision to "
622 << getInstWideningStr(W) << " for VF " << VF
623 << " and instruction: " << *I << '\n');
624 if (Grp->getInsertPos() == I)
625 WideningDecisions[{I, VF}] = {W, InsertPosCost};
626 else
627 WideningDecisions[{I, VF}] = {W, OtherMemberCost};
628 }
629 }
630
631 /// Return the cost model decision for the given instruction \p I and vector
632 /// width \p VF. Return CM_Unknown if this instruction did not pass
633 /// through the cost modeling.
634 InstWidening getWideningDecision(Instruction *I, ElementCount VF) const {
635 assert(VF.isVector() && "Expected VF to be a vector VF");
636 assert(
637 TheLoop->isInnermost() &&
638 "cost-model should not be used for outer loops (in VPlan-native path)");
639
640 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
641 auto Itr = WideningDecisions.find(Val: InstOnVF);
642 if (Itr == WideningDecisions.end())
643 return CM_Unknown;
644 return Itr->second.first;
645 }
646
647 /// Return the vectorization cost for the given instruction \p I and vector
648 /// width \p VF.
649 InstructionCost getWideningCost(Instruction *I, ElementCount VF) {
650 assert(VF.isVector() && "Expected VF >=2");
651 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
652 assert(WideningDecisions.contains(InstOnVF) &&
653 "The cost is not calculated");
654 return WideningDecisions[InstOnVF].second;
655 }
656
657 /// Return True if instruction \p I is an optimizable truncate whose operand
658 /// is an induction variable. Such a truncate will be removed by adding a new
659 /// induction variable with the destination type.
660 bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) {
661 // If the instruction is not a truncate, return false.
662 auto *Trunc = dyn_cast<TruncInst>(Val: I);
663 if (!Trunc)
664 return false;
665
666 // Get the source and destination types of the truncate.
667 Type *SrcTy = toVectorTy(Scalar: Trunc->getSrcTy(), EC: VF);
668 Type *DestTy = toVectorTy(Scalar: Trunc->getDestTy(), EC: VF);
669
670 // If the truncate is free for the given types, return false. Replacing a
671 // free truncate with an induction variable would add an induction variable
672 // update instruction to each iteration of the loop. We exclude from this
673 // check the primary induction variable since it will need an update
674 // instruction regardless.
675 Value *Op = Trunc->getOperand(i_nocapture: 0);
676 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(Ty1: SrcTy, Ty2: DestTy))
677 return false;
678
679 // If the truncated value is not an induction variable, return false.
680 return Legal->isInductionPhi(V: Op);
681 }
682
683 /// Collects the instructions to scalarize for each predicated instruction in
684 /// the loop.
685 void collectInstsToScalarize(ElementCount VF);
686
687 /// Collect values that will not be widened, including Uniforms, Scalars, and
688 /// Instructions to Scalarize for the given \p VF.
689 /// The sets depend on CM decision for Load/Store instructions
690 /// that may be vectorized as interleave, gather-scatter or scalarized.
691 /// Also make a decision on what to do about call instructions in the loop
692 /// at that VF -- scalarize, call a known vector routine, or call a
693 /// vector intrinsic.
694 void collectNonVectorizedAndSetWideningDecisions(ElementCount VF) {
695 // Do the analysis once.
696 if (VF.isScalar() || Uniforms.contains(Val: VF))
697 return;
698 setCostBasedWideningDecision(VF);
699 collectLoopUniforms(VF);
700 collectLoopScalars(VF);
701 collectInstsToScalarize(VF);
702 }
703
704 /// Given costs for both strategies, return true if the scalar predication
705 /// lowering should be used for div/rem. This incorporates an override
706 /// option so it is not simply a cost comparison.
707 bool isDivRemScalarWithPredication(InstructionCost ScalarCost,
708 InstructionCost MaskedCost) const {
709 return !valueOr(X: Opts.force_widen_divrem_via_masked_intrinsic,
710 Default: !(ScalarCost < MaskedCost));
711 }
712
713 /// Returns true if \p I is an instruction which requires predication and
714 /// for which our chosen predication strategy is scalarization (i.e. we
715 /// don't have an alternate strategy such as masking available).
716 /// \p VF is the vectorization factor that will be used to vectorize \p I.
717 bool isScalarWithPredication(Instruction *I, ElementCount VF);
718
719 /// Wrapper function for LoopVectorizationLegality::isMaskRequired,
720 /// that passes the Instruction \p I and if we fold tail.
721 bool isMaskRequired(Instruction *I) const;
722
723 /// Returns true if \p I is an instruction that needs to be predicated
724 /// at runtime. The result is independent of the predication mechanism.
725 /// Superset of instructions that return true for isScalarWithPredication.
726 bool isPredicatedInst(Instruction *I) const;
727
728 /// A helper function that returns how much we should divide the cost of a
729 /// predicated block by. Typically this is the reciprocal of the block
730 /// probability, i.e. if we return X we are assuming the predicated block will
731 /// execute once for every X iterations of the loop header so the block should
732 /// only contribute 1/X of its cost to the total cost calculation, but when
733 /// optimizing for code size it will just be 1 as code size costs don't depend
734 /// on execution probabilities.
735 ///
736 /// Note that if a block wasn't originally predicated but was predicated due
737 /// to tail folding, the divisor will still be 1 because it will execute for
738 /// every iteration of the loop header.
739 inline uint64_t
740 getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind,
741 const BasicBlock *BB);
742
743 /// Returns true if an artificially high cost for emulated masked memrefs
744 /// should be used.
745 bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF) const;
746
747 /// Return the costs for our two available strategies for lowering a
748 /// div/rem operation which requires speculating at least one lane.
749 /// First result is for scalarization (will be invalid for scalable
750 /// vectors); second is for the masked intrinsic strategy.
751 std::pair<InstructionCost, InstructionCost>
752 getDivRemSpeculationCost(Instruction *I, ElementCount VF);
753
754 /// If \p I is a memory instruction with a consecutive pointer that can be
755 /// widened, returns the widening kind (CM_Widen or CM_Widen_Reverse) and
756 /// std::nullopt otherwise.
757 std::optional<InstWidening> memoryInstructionCanBeWidened(Instruction *I,
758 ElementCount VF);
759
760 /// Returns true if \p I is a memory instruction in an interleaved-group
761 /// of memory accesses that can be vectorized with wide vector loads/stores
762 /// and shuffles.
763 bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const;
764
765 /// Returns true if the target machine supports masked loads or stores
766 /// for \p I's data type and alignment. The caller must ensure the access is
767 /// consecutive or part of an interleave group.
768 bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const;
769
770 /// Returns true if the target machine supports gather or scatter for \p I's
771 /// data type and alignment.
772 bool isLegalGatherOrScatter(Instruction *I, ElementCount VF) const;
773
774 /// Check if \p Instr belongs to any interleaved access group.
775 bool isAccessInterleaved(Instruction *Instr) const {
776 return InterleaveInfo.isInterleaved(Instr);
777 }
778
779 /// Get the interleaved access group that \p Instr belongs to.
780 const InterleaveGroup<Instruction> *
781 getInterleavedAccessGroup(Instruction *Instr) const {
782 return InterleaveInfo.getInterleaveGroup(Instr);
783 }
784
785 /// Returns true if we're required to use a scalar epilogue for at least
786 /// the final iteration of the original loop.
787 bool requiresScalarEpilogue(bool IsVectorizing) const {
788 if (!isEpilogueAllowed()) {
789 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
790 return false;
791 }
792 // If we might exit from anywhere but the latch and early exit vectorization
793 // is disabled, we must run the exiting iteration in scalar form.
794 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
795 !(Opts.enable_early_exit_vectorization &&
796 Legal->hasUncountableEarlyExit())) {
797 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
798 "from latch block\n");
799 return true;
800 }
801 if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
802 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
803 "interleaved group requires scalar epilogue\n");
804 return true;
805 }
806 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
807 return false;
808 }
809
810 /// Returns true if an epilogue is allowed (e.g., not prevented by
811 /// optsize or a loop hint annotation).
812 bool isEpilogueAllowed() const {
813 return EpilogueLoweringStatus == CM_EpilogueAllowed;
814 }
815
816 /// Returns the TailFoldingStyle that is best for the current loop.
817 TailFoldingStyle getTailFoldingStyle() const {
818 return ChosenTailFoldingStyle;
819 }
820
821 /// Selects and saves TailFoldingStyle.
822 /// \param IsScalableVF true if scalable vector factors enabled.
823 /// \param UserIC User specific interleave count.
824 void setTailFoldingStyle(bool IsScalableVF, unsigned UserIC) {
825 assert(ChosenTailFoldingStyle == TailFoldingStyle::None &&
826 "Tail folding must not be selected yet.");
827 if (!Legal->canFoldTailByMasking()) {
828 ChosenTailFoldingStyle = TailFoldingStyle::None;
829 return;
830 }
831
832 // Default to TTI preference, but allow command line override.
833 ChosenTailFoldingStyle = Opts.force_tail_folding_style.value_or(
834 u: TTI.getPreferredTailFoldingStyle());
835
836 if (ChosenTailFoldingStyle != TailFoldingStyle::DataWithEVL)
837 return;
838 // Override EVL styles if needed.
839 // FIXME: Investigate opportunity for fixed vector factor.
840 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
841 TTI.hasActiveVectorLength() &&
842 !Opts.enable_vplan_native_path;
843 if (EVLIsLegal)
844 return;
845 // If for some reason EVL mode is unsupported, fallback to an epilogue
846 // if it's allowed, or DataWithoutLaneMask otherwise.
847 if (EpilogueLoweringStatus == CM_EpilogueAllowed ||
848 EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail)
849 ChosenTailFoldingStyle = TailFoldingStyle::None;
850 else
851 ChosenTailFoldingStyle = TailFoldingStyle::DataWithoutLaneMask;
852
853 LLVM_DEBUG(
854 dbgs() << "LV: Preference for VP intrinsics indicated. Will "
855 "not try to generate VP Intrinsics "
856 << (UserIC > 1
857 ? "since interleave count specified is greater than 1.\n"
858 : "due to non-interleaving reasons.\n"));
859 }
860
861 /// Returns true if all loop blocks should be masked to fold tail loop.
862 bool foldTailByMasking() const {
863 return getTailFoldingStyle() != TailFoldingStyle::None;
864 }
865
866 void tryToEnablePartialAliasMasking() {
867 assert(foldTailByMasking() && "Expected tail folding to be enabled!");
868 assert(!foldTailWithEVL() &&
869 "Did not expect to enable alias masking with EVL!");
870 assert(PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided);
871
872 // Assume we fail to enable alias masking (in case we early exit).
873 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
874
875 // Note: FixedOrderRecurrences are not supported yet as we cannot handle
876 // the required `splice.right` with the alias-mask.
877 if (!Opts.force_partial_aliasing_vectorization ||
878 !Legal->getFixedOrderRecurrences().empty())
879 return;
880
881 const RuntimePointerChecking *Checks = Legal->getRuntimePointerChecking();
882 if (!Checks)
883 return;
884
885 auto DiffChecks = Checks->getDiffChecks();
886 if (!DiffChecks || DiffChecks->empty())
887 return;
888
889 [[maybe_unused]] auto HasPointerArgs = [](CallBase *CB) {
890 return any_of(Range: CB->args(), P: [](Value const *Arg) {
891 return Arg->getType()->isPointerTy();
892 });
893 };
894
895 for (BasicBlock *BB : TheLoop->blocks()) {
896 for (Instruction &I : *BB) {
897 if (!isa<LoadInst, StoreInst>(Val: I)) {
898 [[maybe_unused]] auto *Call = dyn_cast<CallInst>(Val: &I);
899 assert(
900 (!I.mayReadOrWriteMemory() || (Call && !HasPointerArgs(Call))) &&
901 "Skipped unexpected memory access");
902 continue;
903 }
904
905 Type *ScalarTy = getLoadStoreType(I: &I);
906 Value *Ptr = getLoadStorePointerOperand(V: &I);
907
908 // Currently, we can't handle alias masking in reverse. Reversing the
909 // alias mask is not correct (or necessary). When combined with
910 // tail-folding the active lane mask should only be reversed where the
911 // alias-mask is true.
912 if (Legal->isConsecutivePtr(AccessTy: ScalarTy, Ptr) == -1)
913 return;
914 }
915 }
916
917 PartialAliasMaskingStatus = AliasMaskingStatus::Enabled;
918 }
919
920 /// Returns true if all loop blocks should have partial aliases masked.
921 bool maskPartialAliasing() const {
922 return PartialAliasMaskingStatus == AliasMaskingStatus::Enabled;
923 }
924
925 /// Returns true if the instructions in this block requires predication
926 /// for any reason, e.g. because tail folding now requires a predicate
927 /// or because the block in the original loop was predicated.
928 bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const {
929 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
930 }
931
932 /// Returns true if VP intrinsics with explicit vector length support should
933 /// be generated in the tail folded loop.
934 bool foldTailWithEVL() const {
935 return getTailFoldingStyle() == TailFoldingStyle::DataWithEVL;
936 }
937
938 /// Returns true if the predicated reduction select should be used to set the
939 /// incoming value for the reduction phi.
940 bool usePredicatedReductionSelect(RecurKind RecurrenceKind,
941 bool HasUsesOutsideReductionChain) const {
942 // Force to use predicated reduction select since the EVL of the
943 // second-to-last iteration might not be VF*UF.
944 if (foldTailWithEVL())
945 return true;
946
947 // Force a predicated select with alias-masking to avoid propagating poison
948 // values to the header phi for lanes outside the alias-mask.
949 if (maskPartialAliasing())
950 return true;
951
952 // Note: For FindLast recurrences and multi-use reductions we prefer a
953 // predicated select to simplify matching in handleFindLastReductions() and
954 // handleMultiUseReductions() respectively, rather than handle multiple
955 // cases.
956 if (RecurrenceDescriptor::isFindLastRecurrenceKind(Kind: RecurrenceKind) ||
957 HasUsesOutsideReductionChain)
958 return true;
959
960 return Opts.prefer_predicated_reduction_select ||
961 TTI.preferPredicatedReductionSelect();
962 }
963
964 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
965 /// with factor VF. Return the cost of the instruction, including
966 /// scalarization overhead if it's needed.
967 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;
968
969 /// Estimate cost of a call instruction CI if it were vectorized with factor
970 /// VF. Return the cost of the instruction, including scalarization overhead
971 /// if it's needed.
972 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const;
973
974 /// Invalidates decisions already taken by the cost model.
975 void invalidateCostModelingDecisions() {
976 WideningDecisions.clear();
977 Uniforms.clear();
978 Scalars.clear();
979 }
980
981 /// Returns the expected execution cost. The unit of the cost does
982 /// not matter because we use the 'cost' units to compare different
983 /// vector widths. The cost that is returned is *not* normalized by
984 /// the factor width.
985 InstructionCost expectedCost(ElementCount VF);
986
987 /// Returns the execution time cost of an instruction for a given vector
988 /// width. Vector width of one means scalar.
989 InstructionCost getInstructionCost(Instruction *I, ElementCount VF);
990
991 /// Returns true if \p Op should be considered invariant and if it is
992 /// trivially hoistable.
993 bool shouldConsiderInvariant(Value *Op);
994
995 /// Returns true if \p I has been forced to be scalarized at \p VF.
996 bool isForcedScalar(Instruction *I, ElementCount VF) const {
997 auto FS = ForcedScalars.find(Val: VF);
998 return FS != ForcedScalars.end() && FS->second.contains(key: I);
999 }
1000
1001private:
1002 const VectorizeOptions &Opts;
1003
1004 unsigned NumPredStores = 0;
1005
1006 /// VF selection state independent of cost-modeling decisions.
1007 VFSelectionContext &Config;
1008
1009 /// Wrapper around LoopVectorizationLegality::isUniform() that takes into
1010 /// account if alias-masking is enabled. We consider the VF to be unknown when
1011 /// alias masking.
1012 bool isUniform(Value *V, ElementCount VF) const {
1013 // With alias-masking our runtime VF is [2, VF] (and not necessarily a
1014 // power-of-two). Something that is uniform for VF may not be for the full
1015 // range.
1016 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
1017 "alias-mask status must be decided already");
1018 return Legal->isUniform(V, VF: PartialAliasMaskingStatus ==
1019 AliasMaskingStatus::Disabled
1020 ? std::optional(VF)
1021 : std::nullopt);
1022 }
1023
1024 /// Wrapper around LoopVectorizationLegality::isUniformMemOp() that takes into
1025 /// account if alias-masking is enabled. We consider the VF to be unknown when
1026 /// alias masking.
1027 bool isUniformMemOp(Instruction &I, ElementCount VF) const {
1028 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
1029 "alias-mask status must be decided already");
1030 return Legal->isUniformMemOp(I, VF: PartialAliasMaskingStatus ==
1031 AliasMaskingStatus::Disabled
1032 ? std::optional(VF)
1033 : std::nullopt);
1034 }
1035
1036 /// Calculate vectorization cost of memory instruction \p I.
1037 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
1038
1039 /// The cost computation for scalarized memory instruction.
1040 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1041
1042 /// The cost computation for interleaving group of memory instructions.
1043 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF) const;
1044
1045 /// The cost computation for Gather/Scatter instruction.
1046 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF) const;
1047
1048 /// The cost computation for widening instruction \p I with consecutive
1049 /// memory access.
1050 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF,
1051 InstWidening Kind);
1052
1053 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1054 /// Load: scalar load + broadcast.
1055 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1056 /// element)
1057 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF) const;
1058
1059 /// Estimate the overhead of scalarizing an instruction. This is a
1060 /// convenience wrapper for the type-based getScalarizationOverhead API.
1061 InstructionCost getScalarizationOverhead(Instruction *I,
1062 ElementCount VF) const;
1063
1064 /// A type representing the costs for instructions if they were to be
1065 /// scalarized rather than vectorized. The entries are Instruction-Cost
1066 /// pairs.
1067 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
1068
1069 /// A set containing all BasicBlocks that are known to present after
1070 /// vectorization as a predicated block.
1071 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
1072 PredicatedBBsAfterVectorization;
1073
1074 /// Records whether it is allowed to have the original scalar loop execute at
1075 /// least once. This may be needed as a fallback loop in case runtime
1076 /// aliasing/dependence checks fail, or to handle the tail/remainder
1077 /// iterations when the trip count is unknown or doesn't divide by the VF,
1078 /// or as a peel-loop to handle gaps in interleave-groups.
1079 /// Under optsize and when the trip count is very small we don't allow any
1080 /// iterations to execute in the scalar loop.
1081 EpilogueLowering EpilogueLoweringStatus = CM_EpilogueAllowed;
1082
1083 /// Control finally chosen tail folding style.
1084 TailFoldingStyle ChosenTailFoldingStyle = TailFoldingStyle::None;
1085
1086 /// If partial alias masking is enabled/disabled or not decided.
1087 AliasMaskingStatus PartialAliasMaskingStatus = AliasMaskingStatus::NotDecided;
1088
1089 /// A map holding scalar costs for different vectorization factors. The
1090 /// presence of a cost for an instruction in the mapping indicates that the
1091 /// instruction will be scalarized when vectorizing with the associated
1092 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1093 MapVector<ElementCount, ScalarCostsTy> InstsToScalarize;
1094
1095 /// Holds the instructions known to be uniform after vectorization.
1096 /// The data is collected per VF.
1097 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1098
1099 /// Holds the instructions known to be scalar after vectorization.
1100 /// The data is collected per VF.
1101 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1102
1103 /// Holds the instructions (address computations) that are forced to be
1104 /// scalarized.
1105 DenseMap<ElementCount, SmallSetVector<Instruction *, 4>> ForcedScalars;
1106
1107 /// Returns the expected difference in cost from scalarizing the expression
1108 /// feeding a predicated instruction \p PredInst. The instructions to
1109 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1110 /// non-negative return value implies the expression will be scalarized.
1111 /// Currently, only single-use chains are considered for scalarization.
1112 InstructionCost computePredInstDiscount(Instruction *PredInst,
1113 ScalarCostsTy &ScalarCosts,
1114 ElementCount VF);
1115
1116 /// Collect the instructions that are uniform after vectorization. An
1117 /// instruction is uniform if we represent it with a single scalar value in
1118 /// the vectorized loop corresponding to each vector iteration. Examples of
1119 /// uniform instructions include pointer operands of consecutive or
1120 /// interleaved memory accesses. Note that although uniformity implies an
1121 /// instruction will be scalar, the reverse is not true. In general, a
1122 /// scalarized instruction will be represented by VF scalar values in the
1123 /// vectorized loop, each corresponding to an iteration of the original
1124 /// scalar loop.
1125 void collectLoopUniforms(ElementCount VF);
1126
1127 /// Collect the instructions that are scalar after vectorization. An
1128 /// instruction is scalar if it is known to be uniform or will be scalarized
1129 /// during vectorization. collectLoopScalars should only add non-uniform nodes
1130 /// to the list if they are used by a load/store instruction that is marked as
1131 /// CM_Scalarize. Non-uniform scalarized instructions will be represented by
1132 /// VF values in the vectorized loop, each corresponding to an iteration of
1133 /// the original scalar loop.
1134 void collectLoopScalars(ElementCount VF);
1135
1136 /// Keeps cost model vectorization decision and cost for instructions.
1137 /// Right now it is used for memory instructions only.
1138 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1139 std::pair<InstWidening, InstructionCost>>;
1140
1141 DecisionList WideningDecisions;
1142
1143 /// Returns true if \p V is expected to be vectorized and it needs to be
1144 /// extracted.
1145 bool needsExtract(Value *V, ElementCount VF) const {
1146 Instruction *I = dyn_cast<Instruction>(Val: V);
1147 if (VF.isScalar() || !I || !TheLoop->contains(Inst: I) ||
1148 TheLoop->isLoopInvariant(V: I) ||
1149 getWideningDecision(I, VF) == CM_Scalarize)
1150 return false;
1151
1152 // Assume we can vectorize V (and hence we need extraction) if the
1153 // scalars are not computed yet. This can happen, because it is called
1154 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1155 // the scalars are collected. That should be a safe assumption in most
1156 // cases, because we check if the operands have vectorizable types
1157 // beforehand in LoopVectorizationLegality.
1158 return !Scalars.contains(Val: VF) || !isScalarAfterVectorization(I, VF);
1159 };
1160
1161 /// Returns a range containing only operands needing to be extracted.
1162 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
1163 ElementCount VF) const {
1164
1165 SmallPtrSet<const Value *, 4> UniqueOperands;
1166 SmallVector<Value *, 4> Res;
1167 for (Value *Op : Ops) {
1168 if (isa<Constant>(Val: Op) || !UniqueOperands.insert(Ptr: Op).second ||
1169 !needsExtract(V: Op, VF))
1170 continue;
1171 Res.push_back(Elt: Op);
1172 }
1173 return Res;
1174 }
1175
1176public:
1177 /// The loop that we evaluate.
1178 Loop *TheLoop;
1179
1180 /// Predicated scalar evolution analysis.
1181 PredicatedScalarEvolution &PSE;
1182
1183 /// Loop Info analysis.
1184 LoopInfo *LI;
1185
1186 /// Vectorization legality.
1187 LoopVectorizationLegality *Legal;
1188
1189 /// Vector target information.
1190 const TargetTransformInfo &TTI;
1191
1192 /// Target Library Info.
1193 const TargetLibraryInfo *TLI;
1194
1195 /// Assumption cache.
1196 AssumptionCache *AC;
1197
1198 /// Interface to emit optimization remarks.
1199 OptimizationRemarkEmitter *ORE;
1200
1201 /// A function to lazily fetch BlockFrequencyInfo. This avoids computing it
1202 /// unless necessary, e.g. when the loop isn't legal to vectorize or when
1203 /// there is no predication.
1204 std::function<BlockFrequencyInfo &()> GetBFI;
1205 /// The BlockFrequencyInfo returned from GetBFI.
1206 BlockFrequencyInfo *BFI = nullptr;
1207 /// Returns the BlockFrequencyInfo for the function if cached, otherwise
1208 /// fetches it via GetBFI. Avoids an indirect call to the std::function.
1209 BlockFrequencyInfo &getBFI() {
1210 if (!BFI)
1211 BFI = &GetBFI();
1212 return *BFI;
1213 }
1214
1215 const Function *TheFunction;
1216
1217 /// The interleave access information contains groups of interleaved accesses
1218 /// with the same stride and close to each other.
1219 InterleavedAccessInfo &InterleaveInfo;
1220
1221 /// Values to ignore in the cost model.
1222 SmallPtrSet<const Value *, 16> ValuesToIgnore;
1223
1224 /// Values to ignore in the cost model when VF > 1.
1225 SmallPtrSet<const Value *, 16> VecValuesToIgnore;
1226};
1227} // end namespace llvm
1228
1229namespace {
1230/// Helper struct to manage generating runtime checks for vectorization.
1231///
1232/// The runtime checks are created up-front in temporary blocks to allow better
1233/// estimating the cost and un-linked from the existing IR. After deciding to
1234/// vectorize, the checks are attached to VPlan as IR or recipes. If deciding
1235/// not to vectorize, the temporary blocks are completely removed.
1236class GeneratedRTChecks {
1237 const VectorizeOptions &Opts;
1238
1239 /// Basic block which contains the generated SCEV checks, if any.
1240 BasicBlock *SCEVCheckBlock = nullptr;
1241
1242 /// The value representing the result of the generated SCEV checks. If it is
1243 /// nullptr no SCEV checks have been generated.
1244 Value *SCEVCheckCond = nullptr;
1245
1246 /// Basic block which contains the generated memory runtime checks, if any.
1247 BasicBlock *MemCheckBlock = nullptr;
1248
1249 /// The value representing the result of the generated memory runtime checks.
1250 /// If it is nullptr no memory runtime checks have been generated.
1251 Value *MemRuntimeCheckCond = nullptr;
1252
1253 /// Whether checks were generated, retained after their IR is replaced or
1254 /// removed during VPlan execution.
1255 bool HasChecks = false;
1256
1257 DominatorTree *DT;
1258 LoopInfo *LI;
1259 TargetTransformInfo *TTI;
1260
1261 SCEVExpander SCEVExp;
1262 SCEVExpander MemCheckExp;
1263
1264 bool CostTooHigh = false;
1265
1266 Loop *OuterLoop = nullptr;
1267
1268 PredicatedScalarEvolution &PSE;
1269
1270 /// The kind of cost that we are calculating
1271 TTI::TargetCostKind CostKind;
1272
1273 /// True if the loop is alias-masked (which allows us to omit diff checks).
1274 bool LoopUsesPartialAliasMasking = false;
1275
1276public:
1277 GeneratedRTChecks(const VectorizeOptions &Opts,
1278 PredicatedScalarEvolution &PSE, DominatorTree *DT,
1279 LoopInfo *LI, TargetTransformInfo *TTI,
1280 TTI::TargetCostKind CostKind,
1281 bool LoopUsesPartialAliasMasking)
1282 : Opts(Opts), DT(DT), LI(LI), TTI(TTI),
1283 SCEVExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1284 MemCheckExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1285 PSE(PSE), CostKind(CostKind),
1286 LoopUsesPartialAliasMasking(LoopUsesPartialAliasMasking) {}
1287
1288 /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
1289 /// accurately estimate the cost of the runtime checks. The blocks are
1290 /// un-linked from the IR and attached to VPlan as IR or recipes if
1291 /// profitable. Otherwise, the check blocks are removed completely.
1292 void create(Loop *L, const LoopAccessInfo &LAI,
1293 const SCEVPredicate &UnionPred, ElementCount VF, unsigned IC,
1294 OptimizationRemarkEmitter &ORE) {
1295
1296 // Hard cutoff to limit compile-time increase in case a very large number of
1297 // runtime checks needs to be generated.
1298 // TODO: Skip cutoff if the loop is guaranteed to execute, e.g. due to
1299 // profile info.
1300 CostTooHigh = LAI.getNumRuntimePointerChecks() >
1301 VectorizerParams::VectorizeMemoryCheckThreshold;
1302 if (CostTooHigh) {
1303 // Mark runtime checks as never succeeding when they exceed the threshold.
1304 MemRuntimeCheckCond = ConstantInt::getTrue(Context&: L->getHeader()->getContext());
1305 SCEVCheckCond = ConstantInt::getTrue(Context&: L->getHeader()->getContext());
1306 ORE.emit(RemarkBuilder: [&]() {
1307 return OptimizationRemarkAnalysisAliasing(
1308 DEBUG_TYPE, "TooManyMemoryRuntimeChecks", L->getStartLoc(),
1309 L->getHeader())
1310 << "loop not vectorized: too many memory checks needed";
1311 });
1312 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
1313 return;
1314 }
1315
1316 BasicBlock *LoopHeader = L->getHeader();
1317 BasicBlock *Preheader = L->getLoopPreheader();
1318
1319 // Use SplitBlock to create blocks for SCEV & memory runtime checks to
1320 // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
1321 // may be used by SCEVExpander. The blocks will be un-linked from their
1322 // predecessors and removed from LI & DT at the end of the function.
1323 if (!UnionPred.isAlwaysTrue()) {
1324 SCEVCheckBlock = SplitBlock(Old: Preheader, SplitPt: Preheader->getTerminator(), DT, LI,
1325 MSSAU: nullptr, BBName: "vector.scevcheck");
1326
1327 SCEVCheckCond = SCEVExp.expandCodeForPredicate(
1328 Pred: &UnionPred, Loc: SCEVCheckBlock->getTerminator());
1329 if (isa<Constant>(Val: SCEVCheckCond)) {
1330 // Clean up directly after expanding the predicate to a constant, to
1331 // avoid further expansions re-using anything left over from SCEVExp.
1332 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1333 SCEVCleaner.cleanup();
1334 }
1335 }
1336
1337 const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
1338 // TODO: We need to estimate the cost of alias-masking in
1339 // GeneratedRTChecks::getCost(). We can't check the MemCheckBlock as the
1340 // alias-mask is generated later in VPlan.
1341 if (RtPtrChecking.Need && !LoopUsesPartialAliasMasking) {
1342 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1343 MemCheckBlock = SplitBlock(Old: Pred, SplitPt: Pred->getTerminator(), DT, LI, MSSAU: nullptr,
1344 BBName: "vector.memcheck");
1345
1346 auto DiffChecks = RtPtrChecking.getDiffChecks();
1347 if (DiffChecks) {
1348 MemRuntimeCheckCond = addDiffRuntimeChecks(
1349 Loc: MemCheckBlock->getTerminator(), Checks: *DiffChecks, Expander&: MemCheckExp, VF, IC);
1350 } else {
1351 MemRuntimeCheckCond = addRuntimeChecks(
1352 Loc: MemCheckBlock->getTerminator(), TheLoop: L, PointerChecks: RtPtrChecking.getChecks(),
1353 Expander&: MemCheckExp, HoistRuntimeChecks: VectorizerParams::HoistRuntimeChecks);
1354 }
1355 assert(MemRuntimeCheckCond &&
1356 "no RT checks generated although RtPtrChecking "
1357 "claimed checks are required");
1358 }
1359
1360 SCEVExp.eraseDeadInstructions(Root: SCEVCheckCond);
1361 HasChecks = getSCEVChecks().first || getMemRuntimeChecks().first;
1362
1363 if (!MemCheckBlock && !SCEVCheckBlock)
1364 return;
1365
1366 // Unhook the temporary block with the checks, update various places
1367 // accordingly.
1368 if (SCEVCheckBlock)
1369 SCEVCheckBlock->replaceAllUsesWith(V: Preheader);
1370 if (MemCheckBlock)
1371 MemCheckBlock->replaceAllUsesWith(V: Preheader);
1372
1373 if (SCEVCheckBlock) {
1374 SCEVCheckBlock->getTerminator()->moveBefore(
1375 InsertPos: Preheader->getTerminator()->getIterator());
1376 auto *UI = new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
1377 UI->setDebugLoc(DebugLoc::getTemporary());
1378 Preheader->getTerminator()->eraseFromParent();
1379 }
1380 if (MemCheckBlock) {
1381 MemCheckBlock->getTerminator()->moveBefore(
1382 InsertPos: Preheader->getTerminator()->getIterator());
1383 auto *UI = new UnreachableInst(Preheader->getContext(), MemCheckBlock);
1384 UI->setDebugLoc(DebugLoc::getTemporary());
1385 Preheader->getTerminator()->eraseFromParent();
1386 }
1387
1388 DT->changeImmediateDominator(BB: LoopHeader, NewBB: Preheader);
1389 if (MemCheckBlock) {
1390 DT->eraseNode(BB: MemCheckBlock);
1391 LI->removeBlock(BB: MemCheckBlock);
1392 }
1393 if (SCEVCheckBlock) {
1394 DT->eraseNode(BB: SCEVCheckBlock);
1395 LI->removeBlock(BB: SCEVCheckBlock);
1396 }
1397
1398 // Outer loop is used as part of the later cost calculations.
1399 OuterLoop = L->getParentLoop();
1400 }
1401
1402 InstructionCost getCost() {
1403 if (SCEVCheckBlock || MemCheckBlock)
1404 LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
1405
1406 if (CostTooHigh) {
1407 InstructionCost Cost;
1408 Cost.setInvalid();
1409 LLVM_DEBUG(dbgs() << " number of checks exceeded threshold\n");
1410 return Cost;
1411 }
1412
1413 InstructionCost RTCheckCost = 0;
1414 if (SCEVCheckBlock)
1415 for (Instruction &I : *SCEVCheckBlock) {
1416 if (SCEVCheckBlock->getTerminator() == &I)
1417 continue;
1418 InstructionCost C = TTI->getInstructionCost(U: &I, CostKind);
1419 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1420 RTCheckCost += C;
1421 }
1422 if (MemCheckBlock) {
1423 InstructionCost MemCheckCost = 0;
1424 for (Instruction &I : *MemCheckBlock) {
1425 if (MemCheckBlock->getTerminator() == &I)
1426 continue;
1427 InstructionCost C = TTI->getInstructionCost(U: &I, CostKind);
1428 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1429 MemCheckCost += C;
1430 }
1431
1432 // If the runtime memory checks are being created inside an outer loop
1433 // we should find out if these checks are outer loop invariant. If so,
1434 // the checks will likely be hoisted out and so the effective cost will
1435 // reduce according to the outer loop trip count.
1436 if (OuterLoop) {
1437 ScalarEvolution *SE = MemCheckExp.getSE();
1438 // TODO: If profitable, we could refine this further by analysing every
1439 // individual memory check, since there could be a mixture of loop
1440 // variant and invariant checks that mean the final condition is
1441 // variant.
1442 const SCEV *Cond = SE->getSCEV(V: MemRuntimeCheckCond);
1443 if (SE->isLoopInvariant(S: Cond, L: OuterLoop)) {
1444 // It seems reasonable to assume that we can reduce the effective
1445 // cost of the checks even when we know nothing about the trip
1446 // count. Assume that the outer loop executes at least twice.
1447 unsigned BestTripCount = 2;
1448
1449 // Get the best known TC estimate.
1450 if (auto EstimatedTC = getSmallBestKnownTC(
1451 Opts, PSE, L: OuterLoop, /* CanUseConstantMax = */ false))
1452 if (EstimatedTC->isFixed())
1453 BestTripCount = EstimatedTC->getFixedValue();
1454
1455 InstructionCost NewMemCheckCost = MemCheckCost / BestTripCount;
1456
1457 // Let's ensure the cost is always at least 1.
1458 NewMemCheckCost = std::max(a: NewMemCheckCost.getValue(),
1459 b: (InstructionCost::CostType)1);
1460
1461 if (BestTripCount > 1)
1462 LLVM_DEBUG(dbgs()
1463 << "We expect runtime memory checks to be hoisted "
1464 << "out of the outer loop. Cost reduced from "
1465 << MemCheckCost << " to " << NewMemCheckCost << '\n');
1466
1467 MemCheckCost = NewMemCheckCost;
1468 }
1469 }
1470
1471 RTCheckCost += MemCheckCost;
1472 }
1473
1474 if (SCEVCheckBlock || MemCheckBlock)
1475 LLVM_DEBUG(dbgs() << "Total cost of runtime checks: " << RTCheckCost
1476 << "\n");
1477
1478 return RTCheckCost;
1479 }
1480
1481 /// Remove the created SCEV & memory runtime check blocks & instructions, if
1482 /// unused.
1483 ~GeneratedRTChecks() {
1484 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1485 bool SCEVChecksUsed = !SCEVCheckBlock || !pred_empty(BB: SCEVCheckBlock);
1486 if (SCEVChecksUsed)
1487 SCEVCleaner.markResultUsed();
1488
1489 if (MemCheckBlock && pred_empty(BB: MemCheckBlock))
1490 eraseMemCheckBlock();
1491
1492 SCEVCleaner.cleanup();
1493
1494 if (!SCEVChecksUsed)
1495 SCEVCheckBlock->eraseFromParent();
1496 }
1497
1498 /// Retrieves the SCEVCheckCond and SCEVCheckBlock that were generated as IR
1499 /// outside VPlan.
1500 std::pair<Value *, BasicBlock *> getSCEVChecks() const {
1501 using namespace llvm::PatternMatch;
1502 if (!SCEVCheckCond || match(V: SCEVCheckCond, P: m_ZeroInt()))
1503 return {nullptr, nullptr};
1504
1505 return {SCEVCheckCond, SCEVCheckBlock};
1506 }
1507
1508 /// Retrieves the MemCheckCond and MemCheckBlock that were generated as IR
1509 /// outside VPlan.
1510 std::pair<Value *, BasicBlock *> getMemRuntimeChecks() const {
1511 using namespace llvm::PatternMatch;
1512 if (MemRuntimeCheckCond && match(V: MemRuntimeCheckCond, P: m_ZeroInt()))
1513 return {nullptr, nullptr};
1514 return {MemRuntimeCheckCond, MemCheckBlock};
1515 }
1516
1517 /// Return true if any runtime checks have been added
1518 bool hasChecks() const { return HasChecks; }
1519
1520 /// Erase the memory check block, its instructions and their SCEV expansions.
1521 void eraseMemCheckBlock() {
1522 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1523 auto &SE = *MemCheckExp.getSE();
1524 // Memory runtime check generation creates compares that use expanded
1525 // values. Remove them before running the SCEVExpanderCleaner.
1526 for (auto &I : make_early_inc_range(Range: reverse(C&: *MemCheckBlock))) {
1527 if (MemCheckExp.isInsertedInstruction(I: &I))
1528 continue;
1529 SE.forgetValue(V: &I);
1530 I.eraseFromParent();
1531 }
1532 MemCheckCleaner.cleanup();
1533 MemCheckBlock->eraseFromParent();
1534 MemCheckBlock = nullptr;
1535 MemRuntimeCheckCond = nullptr;
1536 }
1537};
1538} // namespace
1539
1540static bool useActiveLaneMask(TailFoldingStyle Style) {
1541 return Style == TailFoldingStyle::Data ||
1542 Style == TailFoldingStyle::DataAndControlFlow;
1543}
1544
1545static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style) {
1546 return Style == TailFoldingStyle::DataAndControlFlow;
1547}
1548
1549// Return true if \p OuterLp is an outer loop annotated with hints for explicit
1550// vectorization. The loop needs to be annotated with #pragma omp simd
1551// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
1552// vector length information is not provided, vectorization is not considered
1553// explicit. Interleave hints are not allowed either. These limitations will be
1554// relaxed in the future.
1555// Please, note that we are currently forced to abuse the pragma 'clang
1556// vectorize' semantics. This pragma provides *auto-vectorization hints*
1557// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
1558// provides *explicit vectorization hints* (LV can bypass legal checks and
1559// assume that vectorization is legal). However, both hints are implemented
1560// using the same metadata (llvm.loop.vectorize, processed by
1561// LoopVectorizeHints). This will be fixed in the future when the native IR
1562// representation for pragma 'omp simd' is introduced.
1563static bool isExplicitVecOuterLoop(const VectorizeOptions &Opts, Loop *OuterLp,
1564 OptimizationRemarkEmitter *ORE) {
1565 assert(!OuterLp->isInnermost() && "This is not an outer loop");
1566 LoopVectorizeHints Hints(Opts, OuterLp, true /*DisableInterleaving*/, *ORE);
1567
1568 // Only outer loops with an explicit vectorization hint are supported.
1569 // Unannotated outer loops are ignored.
1570 if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
1571 return false;
1572
1573 Function *Fn = OuterLp->getHeader()->getParent();
1574 if (!Hints.allowVectorization(F: Fn, L: OuterLp,
1575 VectorizeOnlyWhenForced: true /*VectorizeOnlyWhenForced*/)) {
1576 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
1577 return false;
1578 }
1579
1580 if (Hints.getInterleave() > 1) {
1581 // TODO: Interleave support is future work.
1582 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
1583 "outer loops.\n");
1584 Hints.emitRemarkWithHints();
1585 return false;
1586 }
1587
1588 return true;
1589}
1590
1591static void collectSupportedLoops(const VectorizeOptions &Opts, Loop &L,
1592 LoopInfo *LI, OptimizationRemarkEmitter *ORE,
1593 SmallVectorImpl<Loop *> &V) {
1594 // Collect inner loops and outer loops without irreducible control flow. For
1595 // now, only collect outer loops that have explicit vectorization hints. If we
1596 // are stress testing the VPlan H-CFG construction, we collect the outermost
1597 // loop of every loop nest.
1598 if (L.isInnermost() || Opts.vplan_build_outerloop_stress_test ||
1599 (Opts.enable_vplan_native_path &&
1600 isExplicitVecOuterLoop(Opts, OuterLp: &L, ORE))) {
1601 LoopBlocksRPO RPOT(&L);
1602 RPOT.perform(LI);
1603 if (!containsIrreducibleCFG<const BasicBlock *>(RPOTraversal&: RPOT, LI: *LI)) {
1604 V.push_back(Elt: &L);
1605 // TODO: Collect inner loops inside marked outer loops in case
1606 // vectorization fails for the outer loop. Do not invoke
1607 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
1608 // already known to be reducible. We can use an inherited attribute for
1609 // that.
1610 return;
1611 }
1612 }
1613 for (Loop *InnerL : L)
1614 collectSupportedLoops(Opts, L&: *InnerL, LI, ORE, V);
1615}
1616
1617//===----------------------------------------------------------------------===//
1618// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
1619// LoopVectorizationCostModel and LoopVectorizationPlanner.
1620//===----------------------------------------------------------------------===//
1621
1622/// For the given VF and UF and maximum trip count computed for the loop, return
1623/// whether the induction variable might overflow in the vectorized loop. If not,
1624/// then we know a runtime overflow check always evaluates to false and can be
1625/// removed.
1626static bool isIndvarOverflowCheckKnownFalse(
1627 const VectorizeOptions &Opts, const LoopVectorizationCostModel *Cost,
1628 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
1629 // Always be conservative if we don't know the exact unroll factor.
1630 uint64_t MaxUF = UF ? *UF
1631 : std::max(a: Cost->TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions: false),
1632 b: Cost->TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions: true));
1633
1634 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
1635 APInt MaxUIntTripCount = IdxTy->getMask();
1636
1637 // We know the runtime overflow check is known false iff the (max) trip-count
1638 // is known and (max) trip-count + (VF * UF) does not overflow in the type of
1639 // the vector loop induction variable.
1640 if (std::optional<ElementCount> TC = getSmallBestKnownTC(
1641 Opts, PSE&: Cost->PSE, L: Cost->TheLoop,
1642 /*CanUseConstantMax=*/true, /*CanExcludeZeroTrips=*/false,
1643 /*ComputeUpperBoundOnly=*/true)) {
1644 // Compute the maximum runtime values of VF and the trip count.
1645 std::optional<uint64_t> MaxStep =
1646 getMaxRuntimeElementCount(EC: VF * MaxUF, F: *Cost->TheFunction);
1647 std::optional<uint64_t> MaxTC =
1648 getMaxRuntimeElementCount(EC: *TC, F: *Cost->TheFunction);
1649 if (!MaxStep || !MaxTC)
1650 return false;
1651
1652 // Bail out if the maximum trip count is not representable in the induction
1653 // variable's type.
1654 if (MaxUIntTripCount.ult(RHS: *MaxTC))
1655 return false;
1656
1657 return (MaxUIntTripCount - *MaxTC).ugt(RHS: *MaxStep);
1658 }
1659
1660 return false;
1661}
1662
1663// Return whether we allow using masked interleave-groups (for dealing with
1664// strided loads/stores that reside in predicated blocks, or for dealing
1665// with gaps).
1666static bool useMaskedInterleavedAccesses(const VectorizeOptions &Opts,
1667 const TargetTransformInfo &TTI) {
1668 // If an override option has been passed in for interleaved accesses, use it.
1669 return valueOr(X: Opts.enable_masked_interleaved_mem_accesses,
1670 Default: TTI.enableMaskedInterleavedAccessVectorization());
1671}
1672
1673/// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
1674/// VPBB are moved to the end of the newly created VPIRBasicBlock. All
1675/// predecessors and successors of VPBB, if any, are rewired to the new
1676/// VPIRBasicBlock. If \p VPBB may be unreachable, \p Plan must be passed.
1677static VPIRBasicBlock *replaceVPBBWithIRVPBB(VPBasicBlock *VPBB,
1678 BasicBlock *IRBB,
1679 VPlan *Plan = nullptr) {
1680 if (!Plan)
1681 Plan = VPBB->getPlan();
1682 VPIRBasicBlock *IRVPBB = Plan->createEmptyVPIRBasicBlock(IRBB);
1683 auto IP = IRVPBB->begin();
1684 for (auto &R : make_early_inc_range(Range: VPBB->phis()))
1685 R.moveBefore(BB&: *IRVPBB, I: IP);
1686
1687 for (auto &R :
1688 make_early_inc_range(Range: make_range(x: VPBB->getFirstNonPhi(), y: VPBB->end())))
1689 R.moveBefore(BB&: *IRVPBB, I: IRVPBB->end());
1690
1691 VPBlockUtils::reassociateBlocks(Old: VPBB, New: IRVPBB);
1692 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
1693 return IRVPBB;
1694}
1695
1696BasicBlock *InnerLoopVectorizer::createScalarPreheader(StringRef Prefix) {
1697 BasicBlock *VectorPH = OrigLoop->getLoopPreheader();
1698 assert(VectorPH && "Invalid loop structure");
1699
1700 // NOTE: The Plan's scalar preheader VPBB isn't replaced with a VPIRBasicBlock
1701 // wrapping the newly created scalar preheader here at the moment, because the
1702 // Plan's scalar preheader may be unreachable at this point. Instead it is
1703 // replaced in executePlan.
1704 return SplitBlock(Old: VectorPH, SplitPt: VectorPH->getTerminator(), DT, LI, MSSAU: nullptr,
1705 BBName: Twine(Prefix) + "scalar.ph");
1706}
1707
1708/// Knowing that loop \p L executes a single vector iteration, add instructions
1709/// that will get simplified and thus should not have any cost to \p
1710/// InstsToIgnore.
1711static void addFullyUnrolledInstructionsToIgnore(
1712 Loop *L, const LoopVectorizationLegality::InductionList &IL,
1713 SmallPtrSetImpl<Instruction *> &InstsToIgnore) {
1714 auto *Cmp = L->getLatchCmpInst();
1715 if (Cmp)
1716 InstsToIgnore.insert(Ptr: Cmp);
1717 for (PHINode *IV : IL.keys()) {
1718 // The induction is free: a widened induction generates a vector phi with
1719 // its start value and an increment that is dead without a backedge.
1720 InstsToIgnore.insert(Ptr: IV);
1721
1722 // Get next iteration value of the induction variable.
1723 Instruction *IVInst =
1724 cast<Instruction>(Val: IV->getIncomingValueForBlock(BB: L->getLoopLatch()));
1725 if (all_of(Range: IVInst->users(),
1726 P: [&](const User *U) { return U == IV || U == Cmp; }))
1727 InstsToIgnore.insert(Ptr: IVInst);
1728 }
1729}
1730
1731BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
1732 // Create a new IR basic block for the scalar preheader.
1733 BasicBlock *ScalarPH = createScalarPreheader(Prefix: "");
1734 return ScalarPH->getSinglePredecessor();
1735}
1736
1737namespace {
1738
1739struct CSEDenseMapInfo {
1740 static bool canHandle(const Instruction *I) {
1741 return isa<InsertElementInst>(Val: I) || isa<ExtractElementInst>(Val: I) ||
1742 isa<ShuffleVectorInst>(Val: I) || isa<GetElementPtrInst>(Val: I);
1743 }
1744
1745 static unsigned getHashValue(const Instruction *I) {
1746 assert(canHandle(I) && "Unknown instruction!");
1747 return hash_combine(args: I->getOpcode(),
1748 args: hash_combine_range(R: I->operand_values()));
1749 }
1750
1751 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
1752 return LHS->isIdenticalTo(I: RHS);
1753 }
1754};
1755
1756} // end anonymous namespace
1757
1758/// FIXME: This legacy common-subexpression-elimination routine is scheduled for
1759/// removal, in favor of the VPlan-based one.
1760static void legacyCSE(BasicBlock *BB) {
1761 // Perform simple cse.
1762 SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
1763 for (Instruction &In : llvm::make_early_inc_range(Range&: *BB)) {
1764 if (!CSEDenseMapInfo::canHandle(I: &In))
1765 continue;
1766
1767 // Check if we can replace this instruction with any of the
1768 // visited instructions.
1769 if (Instruction *V = CSEMap.lookup(Val: &In)) {
1770 In.replaceAllUsesWith(V);
1771 In.eraseFromParent();
1772 continue;
1773 }
1774
1775 CSEMap[&In] = &In;
1776 }
1777}
1778
1779/// This function attempts to return a value that represents the ElementCount
1780/// at runtime. For fixed-width VFs we know this precisely at compile
1781/// time, but for scalable VFs we calculate it based on an estimate of the
1782/// vscale value.
1783static unsigned estimateElementCount(ElementCount VF,
1784 std::optional<unsigned> VScale) {
1785 unsigned EstimatedVF = VF.getKnownMinValue();
1786 if (VF.isScalable())
1787 if (VScale)
1788 EstimatedVF *= *VScale;
1789 assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
1790 return EstimatedVF;
1791}
1792
1793/// Returns the vector library variant function of \p CI usable at \p VF,
1794/// respecting \p MaskRequired, or nullptr if none is found: a mapping with
1795/// matching VF, masked if required, whose vector function is declared in the
1796/// module.
1797static Function *getVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
1798 bool MaskRequired,
1799 const TargetLibraryInfo *TLI) {
1800 if (!TLI || CI.isNoBuiltin())
1801 return nullptr;
1802 for (const VFInfo &Info : VFDatabase::getMappings(CI))
1803 if (Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()))
1804 if (Function *F = CI.getModule()->getFunction(Name: Info.VectorName))
1805 return F;
1806 return nullptr;
1807}
1808
1809/// Returns true iff \p CI has a library vector variant usable at \p VF.
1810static bool hasVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
1811 bool MaskRequired,
1812 const TargetLibraryInfo *TLI) {
1813 return getVectorLibraryVariantFor(CI, VF, MaskRequired, TLI) != nullptr;
1814}
1815
1816InstructionCost
1817LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
1818 ElementCount VF) const {
1819 Type *RetTy = CI->getType();
1820 SmallVector<Type *, 4> Tys;
1821 for (auto &ArgOp : CI->args())
1822 Tys.push_back(Elt: ArgOp->getType());
1823
1824 InstructionCost ScalarCallCost = TTI.getCallInstrCost(
1825 F: CI->getCalledFunction(), RetTy, Tys, CostKind: Config.CostKind);
1826
1827 // Cost of the scalar call (scalar VF) or its scalarization (vector VF). The
1828 // scalarization cost is only meaningful for fixed VFs.
1829 InstructionCost Cost = VF.isScalable()
1830 ? InstructionCost::getInvalid()
1831 : ScalarCallCost * VF.getKnownMinValue() +
1832 getScalarizationOverhead(I: CI, VF);
1833
1834 // The call may be vectorized at this VF, via a vector intrinsic or a vector
1835 // library variant.
1836 if (getVectorIntrinsicIDForCall(CI, TLI))
1837 Cost = std::min(a: Cost, b: getVectorIntrinsicCost(CI, VF));
1838
1839 if (Function *Variant =
1840 getVectorLibraryVariantFor(CI: *CI, VF, MaskRequired: isMaskRequired(I: CI), TLI))
1841 Cost = std::min(a: Cost,
1842 b: TTI.getCallInstrCost(
1843 /*F=*/nullptr, RetTy: Variant->getReturnType(),
1844 Tys: Variant->getFunctionType()->params(), CostKind: Config.CostKind));
1845
1846 return Cost;
1847}
1848
1849static Type *maybeVectorizeType(Type *Ty, ElementCount VF) {
1850 if (VF.isScalar() || !canVectorizeTy(Ty))
1851 return Ty;
1852 return toVectorizedTy(Ty, EC: VF);
1853}
1854
1855InstructionCost
1856LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
1857 ElementCount VF) const {
1858 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1859 assert(ID && "Expected intrinsic call!");
1860 Type *RetTy = maybeVectorizeType(Ty: CI->getType(), VF);
1861 FastMathFlags FMF;
1862 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: CI))
1863 FMF = FPMO->getFastMathFlags();
1864
1865 SmallVector<const Value *> Arguments(CI->args());
1866 FunctionType *FTy = CI->getCalledFunction()->getFunctionType();
1867 SmallVector<Type *> ParamTys;
1868 std::transform(first: FTy->param_begin(), last: FTy->param_end(),
1869 result: std::back_inserter(x&: ParamTys),
1870 unary_op: [&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
1871
1872 IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
1873 dyn_cast<IntrinsicInst>(Val: CI),
1874 InstructionCost::getInvalid());
1875 return TTI.getIntrinsicInstrCost(ICA: CostAttrs, CostKind: Config.CostKind);
1876}
1877
1878void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
1879 // Don't apply optimizations below when no (vector) loop remains, as they all
1880 // require one at the moment.
1881 VPBasicBlock *HeaderVPBB =
1882 vputils::getFirstLoopHeader(Plan&: *State.Plan, VPDT&: State.VPDT);
1883 if (!HeaderVPBB)
1884 return;
1885
1886 BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
1887
1888 // Remove redundant induction instructions.
1889 legacyCSE(BB: HeaderBB);
1890}
1891
1892void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
1893 // We should not collect Scalars more than once per VF. Right now, this
1894 // function is called from collectUniformsAndScalars(), which already does
1895 // this check. Collecting Scalars for VF=1 does not make any sense.
1896 assert(VF.isVector() && !Scalars.contains(VF) &&
1897 "This function should not be visited twice for the same VF");
1898
1899 // This avoids any chances of creating a REPLICATE recipe during planning
1900 // since that would result in generation of scalarized code during execution,
1901 // which is not supported for scalable vectors.
1902 if (VF.isScalable()) {
1903 Scalars[VF].insert_range(R&: Uniforms[VF]);
1904 return;
1905 }
1906
1907 SmallSetVector<Instruction *, 8> Worklist;
1908
1909 // These sets are used to seed the analysis with pointers used by memory
1910 // accesses that will remain scalar.
1911 SmallSetVector<Instruction *, 8> ScalarPtrs;
1912 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
1913 auto *Latch = TheLoop->getLoopLatch();
1914
1915 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
1916 // The pointer operands of loads and stores will be scalar as long as the
1917 // memory access is not a gather/scatter or histogram operation. The value
1918 // operand of a store will remain scalar if the store is scalarized.
1919 auto IsScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
1920 InstWidening WideningDecision = getWideningDecision(I: MemAccess, VF);
1921 assert(WideningDecision != CM_Unknown &&
1922 "Widening decision should be ready at this moment");
1923 auto *Store = dyn_cast<StoreInst>(Val: MemAccess);
1924 if (Store && Ptr == Store->getValueOperand())
1925 return WideningDecision == CM_Scalarize;
1926 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&
1927 "Ptr is neither a value or pointer operand");
1928 return WideningDecision != CM_GatherScatter &&
1929 !(Store && Legal->getHistogramInfo(I: Store));
1930 };
1931
1932 // A helper that returns true if the given value is a getelementptr
1933 // instruction contained in the loop.
1934 auto IsLoopVaryingGEP = [&](Value *V) {
1935 return isa<GetElementPtrInst>(Val: V) && !TheLoop->isLoopInvariant(V);
1936 };
1937
1938 // A helper that evaluates a memory access's use of a pointer. If the use will
1939 // be a scalar use and the pointer is only used by memory accesses, we place
1940 // the pointer in ScalarPtrs. Otherwise, the pointer is placed in
1941 // PossibleNonScalarPtrs.
1942 auto EvaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
1943 // We only care about bitcast and getelementptr instructions contained in
1944 // the loop.
1945 if (!IsLoopVaryingGEP(Ptr))
1946 return;
1947
1948 // If the pointer has already been identified as scalar (e.g., if it was
1949 // also identified as uniform), there's nothing to do.
1950 auto *I = cast<Instruction>(Val: Ptr);
1951 if (Worklist.count(key: I))
1952 return;
1953
1954 // If the use of the pointer will be a scalar use, and all users of the
1955 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
1956 // place the pointer in PossibleNonScalarPtrs.
1957 if (IsScalarUse(MemAccess, Ptr) &&
1958 all_of(Range: I->users(), P: IsaPred<LoadInst, StoreInst>))
1959 ScalarPtrs.insert(X: I);
1960 else
1961 PossibleNonScalarPtrs.insert(Ptr: I);
1962 };
1963
1964 // We seed the scalars analysis with three classes of instructions: (1)
1965 // instructions marked uniform-after-vectorization and (2) bitcast,
1966 // getelementptr and (pointer) phi instructions used by memory accesses
1967 // requiring a scalar use.
1968 //
1969 // (1) Add to the worklist all instructions that have been identified as
1970 // uniform-after-vectorization.
1971 Worklist.insert_range(R&: Uniforms[VF]);
1972
1973 // (2) Add to the worklist all bitcast and getelementptr instructions used by
1974 // memory accesses requiring a scalar use. The pointer operands of loads and
1975 // stores will be scalar unless the operation is a gather or scatter.
1976 // The value operand of a store will remain scalar if the store is scalarized.
1977 for (auto *BB : TheLoop->blocks())
1978 for (auto &I : *BB) {
1979 if (auto *Load = dyn_cast<LoadInst>(Val: &I)) {
1980 EvaluatePtrUse(Load, Load->getPointerOperand());
1981 } else if (auto *Store = dyn_cast<StoreInst>(Val: &I)) {
1982 EvaluatePtrUse(Store, Store->getPointerOperand());
1983 EvaluatePtrUse(Store, Store->getValueOperand());
1984 }
1985 }
1986 for (auto *I : ScalarPtrs)
1987 if (!PossibleNonScalarPtrs.count(Ptr: I)) {
1988 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n");
1989 Worklist.insert(X: I);
1990 }
1991
1992 // Insert the forced scalars.
1993 // FIXME: Currently VPWidenPHIRecipe() often creates a dead vector
1994 // induction variable when the PHI user is scalarized.
1995 auto ForcedScalar = ForcedScalars.find(Val: VF);
1996 if (ForcedScalar != ForcedScalars.end())
1997 for (auto *I : ForcedScalar->second) {
1998 LLVM_DEBUG(dbgs() << "LV: Found (forced) scalar instruction: " << *I << "\n");
1999 Worklist.insert(X: I);
2000 }
2001
2002 // Expand the worklist by looking through any bitcasts and getelementptr
2003 // instructions we've already identified as scalar. This is similar to the
2004 // expansion step in collectLoopUniforms(); however, here we're only
2005 // expanding to include additional bitcasts and getelementptr instructions.
2006 unsigned Idx = 0;
2007 while (Idx != Worklist.size()) {
2008 Instruction *Dst = Worklist[Idx++];
2009 if (!IsLoopVaryingGEP(Dst->getOperand(i: 0)))
2010 continue;
2011 auto *Src = cast<Instruction>(Val: Dst->getOperand(i: 0));
2012 if (llvm::all_of(Range: Src->users(), P: [&](User *U) -> bool {
2013 auto *J = cast<Instruction>(Val: U);
2014 return !TheLoop->contains(Inst: J) || Worklist.count(key: J) ||
2015 ((isa<LoadInst>(Val: J) || isa<StoreInst>(Val: J)) &&
2016 IsScalarUse(J, Src));
2017 })) {
2018 Worklist.insert(X: Src);
2019 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n");
2020 }
2021 }
2022
2023 // An induction variable will remain scalar if all users of the induction
2024 // variable and induction variable update remain scalar.
2025 for (const auto &Induction : Legal->getInductionVars()) {
2026 auto *Ind = Induction.first;
2027 auto *IndUpdate = cast<Instruction>(Val: Ind->getIncomingValueForBlock(BB: Latch));
2028
2029 // If tail-folding is applied, the primary induction variable will be used
2030 // to feed a vector compare.
2031 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
2032 continue;
2033
2034 // Returns true if \p Indvar is a pointer induction that is used directly by
2035 // load/store instruction \p I.
2036 auto IsDirectLoadStoreFromPtrIndvar = [&](Instruction *Indvar,
2037 Instruction *I) {
2038 return Induction.second.getKind() ==
2039 InductionDescriptor::IK_PtrInduction &&
2040 (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I)) &&
2041 Indvar == getLoadStorePointerOperand(V: I) && IsScalarUse(I, Indvar);
2042 };
2043
2044 // Determine if all users of the induction variable are scalar after
2045 // vectorization.
2046 bool ScalarInd = all_of(Range: Ind->users(), P: [&](User *U) -> bool {
2047 auto *I = cast<Instruction>(Val: U);
2048 return I == IndUpdate || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2049 IsDirectLoadStoreFromPtrIndvar(Ind, I);
2050 });
2051 if (!ScalarInd)
2052 continue;
2053
2054 // If the induction variable update is a fixed-order recurrence, neither the
2055 // induction variable or its update should be marked scalar after
2056 // vectorization.
2057 auto *IndUpdatePhi = dyn_cast<PHINode>(Val: IndUpdate);
2058 if (IndUpdatePhi && Legal->isFixedOrderRecurrence(Phi: IndUpdatePhi))
2059 continue;
2060
2061 // Determine if all users of the induction variable update instruction are
2062 // scalar after vectorization.
2063 bool ScalarIndUpdate = all_of(Range: IndUpdate->users(), P: [&](User *U) -> bool {
2064 auto *I = cast<Instruction>(Val: U);
2065 return I == Ind || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2066 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
2067 });
2068 if (!ScalarIndUpdate)
2069 continue;
2070
2071 // The induction variable and its update instruction will remain scalar.
2072 Worklist.insert(X: Ind);
2073 Worklist.insert(X: IndUpdate);
2074 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n");
2075 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate
2076 << "\n");
2077 }
2078
2079 Scalars[VF].insert_range(R&: Worklist);
2080}
2081
2082bool LoopVectorizationCostModel::isLegalMaskedLoadOrStore(
2083 Instruction *I, ElementCount VF) const {
2084 assert(isa<LoadInst>(I) || isa<StoreInst>(I));
2085 return Config.isLegalMaskedLoadOrStore(IsLoad: isa<LoadInst>(Val: I), ScalarTy: getLoadStoreType(I),
2086 Alignment: getLoadStoreAlignment(I),
2087 AddressSpace: getLoadStoreAddressSpace(I));
2088}
2089
2090bool LoopVectorizationCostModel::isLegalGatherOrScatter(Instruction *I,
2091 ElementCount VF) const {
2092 assert((isa<LoadInst, StoreInst>(I)));
2093 return Config.isLegalGatherOrScatter(IsLoad: isa<LoadInst>(Val: I), ScalarTy: getLoadStoreType(I),
2094 Alignment: getLoadStoreAlignment(I), VF);
2095}
2096
2097bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
2098 ElementCount VF) {
2099 if (!isPredicatedInst(I))
2100 return false;
2101
2102 // Do we have a non-scalar lowering for this predicated
2103 // instruction? No - it is scalar with predication.
2104 switch(I->getOpcode()) {
2105 default:
2106 return true;
2107 case Instruction::Call: {
2108 if (VF.isScalar())
2109 return true;
2110 auto *CI = cast<CallInst>(Val: I);
2111 // A vector intrinsic or library variant lowering avoids scalarization.
2112 return !getVectorIntrinsicIDForCall(CI, TLI) &&
2113 !hasVectorLibraryVariantFor(CI: *CI, VF, MaskRequired: isMaskRequired(I: CI), TLI);
2114 }
2115 case Instruction::Load:
2116 case Instruction::Store: {
2117 bool IsConsecutive = Legal->isConsecutivePtr(AccessTy: getLoadStoreType(I),
2118 Ptr: getLoadStorePointerOperand(V: I));
2119 return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) &&
2120 !isLegalGatherOrScatter(I, VF);
2121 }
2122 case Instruction::UDiv:
2123 case Instruction::SDiv:
2124 case Instruction::SRem:
2125 case Instruction::URem: {
2126 // We have the option to use the llvm.masked.udiv intrinsics to avoid
2127 // predication. The cost based decision here will always select the masked
2128 // intrinsics for scalable vectors as scalarization isn't legal.
2129 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
2130 return isDivRemScalarWithPredication(ScalarCost, MaskedCost);
2131 }
2132 }
2133}
2134
2135bool LoopVectorizationCostModel::isMaskRequired(Instruction *I) const {
2136 return Legal->isMaskRequired(I, TailFolded: foldTailByMasking());
2137}
2138
2139// TODO: Fold into LoopVectorizationLegality::isMaskRequired.
2140bool LoopVectorizationCostModel::isPredicatedInst(Instruction *I) const {
2141 // TODO: We can use the loop-preheader as context point here and get
2142 // context sensitive reasoning for isSafeToSpeculativelyExecute.
2143 if (isSafeToSpeculativelyExecute(I) ||
2144 (isa<LoadInst, StoreInst, CallInst>(Val: I) && !isMaskRequired(I)) ||
2145 isa<UncondBrInst, CondBrInst, SwitchInst, PHINode, AllocaInst>(Val: I))
2146 return false;
2147
2148 // If the instruction was executed conditionally in the original scalar loop,
2149 // predication is needed with a mask whose lanes are all possibly inactive.
2150 if (Legal->blockNeedsPredication(BB: I->getParent()))
2151 return true;
2152
2153 // If we're not folding the tail by masking and not vectorizing a loop with
2154 // uncountable exits and side effects, predication is unnecessary.
2155 if (!foldTailByMasking() && !Legal->hasUncountableExitWithSideEffects())
2156 return false;
2157
2158 // All that remain are instructions with side-effects originally executed in
2159 // the loop unconditionally, but now execute under a tail-fold mask (only)
2160 // having at least one active lane (the first). If the side-effects of the
2161 // instruction are invariant, executing it w/o (the tail-folding) mask is safe
2162 // - it will cause the same side-effects as when masked.
2163 switch(I->getOpcode()) {
2164 default:
2165 llvm_unreachable(
2166 "instruction should have been considered by earlier checks");
2167 case Instruction::Call:
2168 // Side-effects of a Call are assumed to be non-invariant, needing a
2169 // (fold-tail) mask.
2170 assert(isMaskRequired(I) &&
2171 "should have returned earlier for calls not needing a mask");
2172 return true;
2173 case Instruction::Load:
2174 // If the address is loop invariant no predication is needed.
2175 return !Legal->isInvariant(V: getLoadStorePointerOperand(V: I));
2176 case Instruction::Store: {
2177 // For stores, we need to prove both speculation safety (which follows from
2178 // the same argument as loads), but also must prove the value being stored
2179 // is correct. The easiest form of the later is to require that all values
2180 // stored are the same.
2181 return !(Legal->isInvariant(V: getLoadStorePointerOperand(V: I)) &&
2182 TheLoop->isLoopInvariant(V: cast<StoreInst>(Val: I)->getValueOperand()));
2183 }
2184 case Instruction::UDiv:
2185 case Instruction::URem:
2186 // If the divisor is loop-invariant no predication is needed.
2187 return !Legal->isInvariant(V: I->getOperand(i: 1));
2188 case Instruction::SDiv:
2189 case Instruction::SRem:
2190 // Conservative for now, since masked-off lanes may be poison and could
2191 // trigger signed overflow.
2192 return true;
2193 }
2194}
2195
2196uint64_t LoopVectorizationCostModel::getPredBlockCostDivisor(
2197 TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) {
2198 if (CostKind == TTI::TCK_CodeSize)
2199 return 1;
2200 // If the block wasn't originally predicated then return early to avoid
2201 // computing BlockFrequencyInfo unnecessarily.
2202 if (!Legal->blockNeedsPredication(BB))
2203 return 1;
2204
2205 uint64_t HeaderFreq =
2206 getBFI().getBlockFreq(BB: TheLoop->getHeader()).getFrequency();
2207 uint64_t BBFreq = getBFI().getBlockFreq(BB).getFrequency();
2208 assert(HeaderFreq >= BBFreq &&
2209 "Header has smaller block freq than dominated BB?");
2210 return std::round(x: (double)HeaderFreq / BBFreq);
2211}
2212
2213static Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode) {
2214 switch (Opcode) {
2215 case Instruction::UDiv:
2216 return Intrinsic::masked_udiv;
2217 case Instruction::SDiv:
2218 return Intrinsic::masked_sdiv;
2219 case Instruction::URem:
2220 return Intrinsic::masked_urem;
2221 case Instruction::SRem:
2222 return Intrinsic::masked_srem;
2223 default:
2224 llvm_unreachable("Unexpected opcode");
2225 }
2226}
2227
2228std::pair<InstructionCost, InstructionCost>
2229LoopVectorizationCostModel::getDivRemSpeculationCost(Instruction *I,
2230 ElementCount VF) {
2231 assert(I->getOpcode() == Instruction::UDiv ||
2232 I->getOpcode() == Instruction::SDiv ||
2233 I->getOpcode() == Instruction::SRem ||
2234 I->getOpcode() == Instruction::URem);
2235 assert(!isSafeToSpeculativelyExecute(I));
2236
2237 // Scalarization isn't legal for scalable vector types
2238 InstructionCost ScalarizationCost = InstructionCost::getInvalid();
2239 if (!VF.isScalable()) {
2240 // Get the scalarization cost and scale this amount by the probability of
2241 // executing the predicated block. If the instruction is not predicated,
2242 // we fall through to the next case.
2243 ScalarizationCost = 0;
2244
2245 // These instructions have a non-void type, so account for the phi nodes
2246 // that we will create. This cost is likely to be zero. The phi node
2247 // cost, if any, should be scaled by the block probability because it
2248 // models a copy at the end of each predicated block.
2249 ScalarizationCost += VF.getFixedValue() *
2250 TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
2251
2252 // The cost of the non-predicated instruction.
2253 ScalarizationCost +=
2254 VF.getFixedValue() * TTI.getArithmeticInstrCost(
2255 Opcode: I->getOpcode(), Ty: I->getType(), CostKind: Config.CostKind);
2256
2257 // The cost of insertelement and extractelement instructions needed for
2258 // scalarization.
2259 ScalarizationCost += getScalarizationOverhead(I, VF);
2260
2261 // Scale the cost by the probability of executing the predicated blocks.
2262 // This assumes the predicated block for each vector lane is equally
2263 // likely.
2264 ScalarizationCost =
2265 ScalarizationCost /
2266 getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
2267 }
2268
2269 auto *VecTy = toVectorTy(Scalar: I->getType(), EC: VF);
2270 auto *MaskTy = toVectorTy(Scalar: Type::getInt1Ty(C&: I->getContext()), EC: VF);
2271 IntrinsicCostAttributes ICA(getMaskedDivRemIntrinsic(Opcode: I->getOpcode()), VecTy,
2272 {VecTy, VecTy, MaskTy});
2273 InstructionCost MaskedCost = TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind);
2274 return {ScalarizationCost, MaskedCost};
2275}
2276
2277bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
2278 Instruction *I, ElementCount VF) const {
2279 assert(isAccessInterleaved(I) && "Expecting interleaved access.");
2280 assert(getWideningDecision(I, VF) == CM_Unknown &&
2281 "Decision should not be set yet.");
2282 auto *Group = getInterleavedAccessGroup(Instr: I);
2283 assert(Group && "Must have a group.");
2284 unsigned InterleaveFactor = Group->getFactor();
2285
2286 // If the instruction's allocated size doesn't equal its type size, it
2287 // requires padding and will be scalarized.
2288 auto &DL = I->getDataLayout();
2289 auto *ScalarTy = getLoadStoreType(I);
2290 if (hasIrregularType(Ty: ScalarTy, DL))
2291 return false;
2292
2293 // For scalable vectors, the interleave factors must be <= 8 since we require
2294 // the (de)interleaveN intrinsics instead of shufflevectors.
2295 if (VF.isScalable() && InterleaveFactor > 8)
2296 return false;
2297
2298 // If the group involves a non-integral pointer, we may not be able to
2299 // losslessly cast all values to a common type.
2300 bool ScalarNI = DL.isNonIntegralPointerType(Ty: ScalarTy);
2301 for (Instruction *Member : Group->members()) {
2302 auto *MemberTy = getLoadStoreType(I: Member);
2303 bool MemberNI = DL.isNonIntegralPointerType(Ty: MemberTy);
2304 // Don't coerce non-integral pointers to integers or vice versa.
2305 if (MemberNI != ScalarNI)
2306 // TODO: Consider adding special nullptr value case here
2307 return false;
2308 if (MemberNI && ScalarNI &&
2309 ScalarTy->getPointerAddressSpace() !=
2310 MemberTy->getPointerAddressSpace())
2311 return false;
2312 }
2313
2314 // Check if masking is required.
2315 // A Group may need masking for one of two reasons: it resides in a block that
2316 // needs predication, or it was decided to use masking to deal with gaps
2317 // (either a gap at the end of a load-access that may result in a speculative
2318 // load, or any gaps in a store-access).
2319 bool PredicatedAccessRequiresMasking =
2320 blockNeedsPredicationForAnyReason(BB: I->getParent()) && isMaskRequired(I);
2321 bool LoadAccessWithGapsRequiresEpilogMasking =
2322 isa<LoadInst>(Val: I) && Group->requiresScalarEpilogue() &&
2323 !isEpilogueAllowed();
2324 bool StoreAccessWithGapsRequiresMasking =
2325 isa<StoreInst>(Val: I) && !Group->isFull();
2326 if (!PredicatedAccessRequiresMasking &&
2327 !LoadAccessWithGapsRequiresEpilogMasking &&
2328 !StoreAccessWithGapsRequiresMasking)
2329 return true;
2330
2331 // If masked interleaving is required, we expect that the user/target had
2332 // enabled it, because otherwise it either wouldn't have been created or
2333 // it should have been invalidated by the CostModel.
2334 assert(useMaskedInterleavedAccesses(Opts, TTI) &&
2335 "Masked interleave-groups for predicated accesses are not enabled.");
2336
2337 if (Group->isReverse())
2338 return false;
2339
2340 // TODO: Support interleaved access that requires a gap mask for scalable VFs.
2341 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2342 StoreAccessWithGapsRequiresMasking;
2343 if (VF.isScalable() && NeedsMaskForGaps)
2344 return false;
2345
2346 return isLegalMaskedLoadOrStore(I, VF);
2347}
2348
2349std::optional<LoopVectorizationCostModel::InstWidening>
2350LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
2351 ElementCount VF) {
2352 // Get and ensure we have a valid memory instruction.
2353 assert((isa<LoadInst, StoreInst>(I)) && "Invalid memory instruction");
2354
2355 auto *Ptr = getLoadStorePointerOperand(V: I);
2356 auto *ScalarTy = getLoadStoreType(I);
2357
2358 // In order to be widened, the pointer should be consecutive, first of all.
2359 int Stride = Legal->isConsecutivePtr(AccessTy: ScalarTy, Ptr);
2360 if (!Stride)
2361 return std::nullopt;
2362
2363 // If the instruction is a store located in a predicated block, it will be
2364 // scalarized.
2365 if (isScalarWithPredication(I, VF))
2366 return std::nullopt;
2367
2368 // If the instruction's allocated size doesn't equal it's type size, it
2369 // requires padding and will be scalarized.
2370 auto &DL = I->getDataLayout();
2371 if (hasIrregularType(Ty: ScalarTy, DL))
2372 return std::nullopt;
2373
2374 return Stride == 1 ? CM_Widen : CM_Widen_Reverse;
2375}
2376
2377void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
2378 // We should not collect Uniforms more than once per VF. Right now,
2379 // this function is called from collectUniformsAndScalars(), which
2380 // already does this check. Collecting Uniforms for VF=1 does not make any
2381 // sense.
2382
2383 assert(VF.isVector() && !Uniforms.contains(VF) &&
2384 "This function should not be visited twice for the same VF");
2385
2386 // Visit the list of Uniforms. If we find no uniform value, we won't
2387 // analyze again. Uniforms.count(VF) will return 1.
2388 Uniforms[VF].clear();
2389
2390 // Now we know that the loop is vectorizable!
2391 // Collect instructions inside the loop that will remain uniform after
2392 // vectorization.
2393
2394 // Global values, params and instructions outside of current loop are out of
2395 // scope.
2396 auto IsOutOfScope = [&](Value *V) -> bool {
2397 Instruction *I = dyn_cast<Instruction>(Val: V);
2398 return (!I || !TheLoop->contains(Inst: I));
2399 };
2400
2401 // Worklist containing uniform instructions demanding lane 0.
2402 SetVector<Instruction *> Worklist;
2403
2404 // Add uniform instructions demanding lane 0 to the worklist. Instructions
2405 // that require predication must not be considered uniform after
2406 // vectorization, because that would create an erroneous replicating region
2407 // where only a single instance out of VF should be formed.
2408 auto AddToWorklistIfAllowed = [&](Instruction *I) -> void {
2409 if (IsOutOfScope(I)) {
2410 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "
2411 << *I << "\n");
2412 return;
2413 }
2414 if (isPredicatedInst(I)) {
2415 LLVM_DEBUG(
2416 dbgs() << "LV: Found not uniform due to requiring predication: " << *I
2417 << "\n");
2418 return;
2419 }
2420 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n");
2421 Worklist.insert(X: I);
2422 };
2423
2424 // Start with the conditional branches exiting the loop. If the branch
2425 // condition is an instruction contained in the loop that is only used by the
2426 // branch, it is uniform. Note conditions from uncountable early exits are not
2427 // uniform.
2428 SmallVector<BasicBlock *> Exiting;
2429 TheLoop->getExitingBlocks(ExitingBlocks&: Exiting);
2430 for (BasicBlock *E : Exiting) {
2431 if (Legal->hasUncountableEarlyExit() && TheLoop->getLoopLatch() != E)
2432 continue;
2433 auto *Cmp = dyn_cast<Instruction>(Val: E->getTerminator()->getOperand(i: 0));
2434 if (!Cmp || !TheLoop->contains(Inst: Cmp) || !Cmp->hasOneUse())
2435 continue;
2436
2437 // If we have an exit condition that is actually two conditions (one
2438 // countable and the other uncountable) combined via an or, only add the
2439 // countable comparison as a uniform value.
2440 if (Legal->hasUncountableExitWithSideEffects() &&
2441 TheLoop->getLoopLatch() == E) {
2442 if (Instruction *Countable =
2443 Legal->findCountableComparisonInCombinedCondition(Cond: Cmp)) {
2444 if (Countable->hasOneUse())
2445 AddToWorklistIfAllowed(Countable);
2446 continue;
2447 }
2448 }
2449
2450 // Normal exit comparisons are uniform.
2451 AddToWorklistIfAllowed(Cmp);
2452 }
2453
2454 auto PrevVF = VF.divideCoefficientBy(RHS: 2);
2455 // Return true if all lanes perform the same memory operation, and we can
2456 // thus choose to execute only one.
2457 auto IsUniformMemOpUse = [&](Instruction *I) {
2458 // If the value was already known to not be uniform for the previous
2459 // (smaller VF), it cannot be uniform for the larger VF.
2460 if (PrevVF.isVector()) {
2461 auto Iter = Uniforms.find(Val: PrevVF);
2462 if (Iter != Uniforms.end() && !Iter->second.contains(Ptr: I))
2463 return false;
2464 }
2465 if (!isUniformMemOp(I&: *I, VF))
2466 return false;
2467 if (isa<LoadInst>(Val: I))
2468 // Loading the same address always produces the same result - at least
2469 // assuming aliasing and ordering which have already been checked.
2470 return true;
2471 // Storing the same value on every iteration.
2472 return TheLoop->isLoopInvariant(V: cast<StoreInst>(Val: I)->getValueOperand());
2473 };
2474
2475 auto IsUniformDecision = [&](Instruction *I, ElementCount VF) {
2476 InstWidening WideningDecision = getWideningDecision(I, VF);
2477 assert(WideningDecision != CM_Unknown &&
2478 "Widening decision should be ready at this moment");
2479
2480 if (IsUniformMemOpUse(I))
2481 return true;
2482
2483 return (WideningDecision == CM_Widen ||
2484 WideningDecision == CM_Widen_Reverse ||
2485 WideningDecision == CM_Interleave);
2486 };
2487
2488 // Returns true if Ptr is the pointer operand of a memory access instruction
2489 // I, I is known to not require scalarization, and the pointer is not also
2490 // stored.
2491 auto IsVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
2492 if (isa<StoreInst>(Val: I) && I->getOperand(i: 0) == Ptr)
2493 return false;
2494 return getLoadStorePointerOperand(V: I) == Ptr &&
2495 (IsUniformDecision(I, VF) || Legal->isInvariant(V: Ptr));
2496 };
2497
2498 // Holds a list of values which are known to have at least one uniform use.
2499 // Note that there may be other uses which aren't uniform. A "uniform use"
2500 // here is something which only demands lane 0 of the unrolled iterations;
2501 // it does not imply that all lanes produce the same value (e.g. this is not
2502 // the usual meaning of uniform)
2503 SetVector<Value *> HasUniformUse;
2504
2505 // Scan the loop for instructions which are either a) known to have only
2506 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
2507 for (auto *BB : TheLoop->blocks())
2508 for (auto &I : *BB) {
2509 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &I)) {
2510 switch (II->getIntrinsicID()) {
2511 case Intrinsic::sideeffect:
2512 case Intrinsic::experimental_noalias_scope_decl:
2513 case Intrinsic::assume:
2514 case Intrinsic::lifetime_start:
2515 case Intrinsic::lifetime_end:
2516 if (TheLoop->hasLoopInvariantOperands(I: &I))
2517 AddToWorklistIfAllowed(&I);
2518 break;
2519 default:
2520 break;
2521 }
2522 }
2523
2524 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: &I)) {
2525 if (IsOutOfScope(EVI->getAggregateOperand())) {
2526 AddToWorklistIfAllowed(EVI);
2527 continue;
2528 }
2529 // Only ExtractValue instructions where the aggregate value comes from a
2530 // call are allowed to be non-uniform.
2531 assert(isa<CallInst>(EVI->getAggregateOperand()) &&
2532 "Expected aggregate value to be call return value");
2533 }
2534
2535 // If there's no pointer operand, there's nothing to do.
2536 auto *Ptr = getLoadStorePointerOperand(V: &I);
2537 if (!Ptr)
2538 continue;
2539
2540 // If the pointer can be proven to be uniform, always add it to the
2541 // worklist.
2542 if (isa<Instruction>(Val: Ptr) && isUniform(V: Ptr, VF))
2543 AddToWorklistIfAllowed(cast<Instruction>(Val: Ptr));
2544
2545 if (IsUniformMemOpUse(&I))
2546 AddToWorklistIfAllowed(&I);
2547
2548 if (IsVectorizedMemAccessUse(&I, Ptr))
2549 HasUniformUse.insert(X: Ptr);
2550 }
2551
2552 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
2553 // demanding) users. Since loops are assumed to be in LCSSA form, this
2554 // disallows uses outside the loop as well.
2555 for (auto *V : HasUniformUse) {
2556 if (IsOutOfScope(V))
2557 continue;
2558 auto *I = cast<Instruction>(Val: V);
2559 bool UsersAreMemAccesses = all_of(Range: I->users(), P: [&](User *U) -> bool {
2560 auto *UI = cast<Instruction>(Val: U);
2561 return TheLoop->contains(Inst: UI) && IsVectorizedMemAccessUse(UI, V);
2562 });
2563 if (UsersAreMemAccesses)
2564 AddToWorklistIfAllowed(I);
2565 }
2566
2567 // Expand Worklist in topological order: whenever a new instruction
2568 // is added , its users should be already inside Worklist. It ensures
2569 // a uniform instruction will only be used by uniform instructions.
2570 unsigned Idx = 0;
2571 while (Idx != Worklist.size()) {
2572 Instruction *I = Worklist[Idx++];
2573
2574 for (auto *OV : I->operand_values()) {
2575 // isOutOfScope operands cannot be uniform instructions.
2576 if (IsOutOfScope(OV))
2577 continue;
2578 // First order recurrence Phi's should typically be considered
2579 // non-uniform.
2580 auto *OP = dyn_cast<PHINode>(Val: OV);
2581 if (OP && Legal->isFixedOrderRecurrence(Phi: OP))
2582 continue;
2583 // If all the users of the operand are uniform, then add the
2584 // operand into the uniform worklist.
2585 auto *OI = cast<Instruction>(Val: OV);
2586 if (llvm::all_of(Range: OI->users(), P: [&](User *U) -> bool {
2587 auto *J = cast<Instruction>(Val: U);
2588 return Worklist.count(key: J) || IsVectorizedMemAccessUse(J, OI);
2589 }))
2590 AddToWorklistIfAllowed(OI);
2591 }
2592 }
2593
2594 // For an instruction to be added into Worklist above, all its users inside
2595 // the loop should also be in Worklist. However, this condition cannot be
2596 // true for phi nodes that form a cyclic dependence. We must process phi
2597 // nodes separately. An induction variable will remain uniform if all users
2598 // of the induction variable and induction variable update remain uniform.
2599 // The code below handles both pointer and non-pointer induction variables.
2600 BasicBlock *Latch = TheLoop->getLoopLatch();
2601 for (PHINode *Ind : Legal->getInductionVars().keys()) {
2602 auto *IndUpdate = cast<Instruction>(Val: Ind->getIncomingValueForBlock(BB: Latch));
2603
2604 // Determine if all users of the induction variable are uniform after
2605 // vectorization.
2606 bool UniformInd = all_of(Range: Ind->users(), P: [&](User *U) -> bool {
2607 auto *I = cast<Instruction>(Val: U);
2608 return I == IndUpdate || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2609 IsVectorizedMemAccessUse(I, Ind);
2610 });
2611 if (!UniformInd)
2612 continue;
2613
2614 // Determine if all users of the induction variable update instruction are
2615 // uniform after vectorization.
2616 bool UniformIndUpdate = all_of(Range: IndUpdate->users(), P: [&](User *U) -> bool {
2617 auto *I = cast<Instruction>(Val: U);
2618 return I == Ind || Worklist.count(key: I) ||
2619 IsVectorizedMemAccessUse(I, IndUpdate);
2620 });
2621 if (!UniformIndUpdate)
2622 continue;
2623
2624 // The induction variable and its update instruction will remain uniform.
2625 AddToWorklistIfAllowed(Ind);
2626 AddToWorklistIfAllowed(IndUpdate);
2627 }
2628
2629 Uniforms[VF].insert_range(R&: Worklist);
2630}
2631
2632FixedScalableVFPair
2633LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
2634 // Make sure once we return PartialAliasMaskingStatus is not "NotDecided".
2635 scope_exit EnsureAliasMaskingStatusIsDecidedOnReturn([this] {
2636 if (PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided)
2637 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
2638 });
2639
2640 // For outer loops, use simple type-based heuristic VF. No cost model or
2641 // memory dependence analysis is available.
2642 if (!TheLoop->isInnermost()) {
2643 return Config.computeVPlanOuterloopVF(UserVF);
2644 }
2645
2646 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
2647 // TODO: It may be useful to do since it's still likely to be dynamically
2648 // uniform if the target can skip.
2649 reportVectorizationFailure(
2650 DebugMsg: "Not inserting runtime ptr check for divergent target",
2651 OREMsg: "runtime pointer checks needed. Not enabled for divergent target",
2652 ORETag: "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
2653 return FixedScalableVFPair::getNone();
2654 }
2655
2656 ScalarEvolution *SE = PSE.getSE();
2657 ElementCount TC = getSmallConstantTripCount(SE, L: TheLoop);
2658 unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
2659 if (!MaxTC && EpilogueLoweringStatus == CM_EpilogueAllowed)
2660 MaxTC = getMaxTCFromNonZeroRange(PSE, L: TheLoop);
2661 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
2662 if (TC != ElementCount::getFixed(MinVal: MaxTC))
2663 LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
2664 if (TC.isScalar()) {
2665 reportVectorizationFailure(
2666 DebugMsg: "Single iteration (non) loop",
2667 OREMsg: "loop trip count is one, irrelevant for vectorization",
2668 ORETag: "SingleIterationLoop", ORE, TheLoop);
2669 return FixedScalableVFPair::getNone();
2670 }
2671
2672 // If BTC matches the widest induction type and is -1 then the trip count
2673 // computation will wrap to 0 and the vector trip count will be 0. Do not try
2674 // to vectorize.
2675 const SCEV *BTC = SE->getBackedgeTakenCount(L: TheLoop);
2676 if (!isa<SCEVCouldNotCompute>(Val: BTC) &&
2677 BTC->getType()->getScalarSizeInBits() >=
2678 Legal->getWidestInductionType()->getScalarSizeInBits() &&
2679 SE->isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: BTC,
2680 RHS: SE->getMinusOne(Ty: BTC->getType()))) {
2681 reportVectorizationFailure(
2682 DebugMsg: "Trip count computation wrapped",
2683 OREMsg: "backedge-taken count is -1, loop trip count wrapped to 0",
2684 ORETag: "TripCountWrapped", ORE, TheLoop);
2685 return FixedScalableVFPair::getNone();
2686 }
2687
2688 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
2689 "No cost-modeling decisions should have been taken at this point");
2690
2691 switch (EpilogueLoweringStatus) {
2692 case CM_EpilogueAllowed:
2693 return Config.computeFeasibleMaxVF(MaxTripCount: MaxTC, UserVF, UserIC, FoldTailByMasking: false,
2694 RequiresScalarEpilogue: requiresScalarEpilogue(IsVectorizing: true));
2695 case CM_EpilogueNotAllowedFoldTail:
2696 [[fallthrough]];
2697 case CM_EpilogueNotNeededFoldTail:
2698 LLVM_DEBUG(dbgs() << "LV: tail-folding hint/switch found.\n"
2699 << "LV: Not allowing epilogue, creating tail-folded "
2700 << "vector loop.\n");
2701 break;
2702 case CM_EpilogueNotAllowedLowTripLoop:
2703 // fallthrough as a special case of OptForSize
2704 case CM_EpilogueNotAllowedOptSize:
2705 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize)
2706 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
2707 else
2708 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
2709 << "count.\n");
2710
2711 // Bail if runtime checks are required, which are not good when optimising
2712 // for size.
2713 if (Config.runtimeChecksRequired())
2714 return FixedScalableVFPair::getNone();
2715
2716 break;
2717 }
2718
2719 // Now try the tail folding
2720
2721 // Invalidate interleave groups that require an epilogue if we can't mask
2722 // the interleave-group.
2723 if (!useMaskedInterleavedAccesses(Opts, TTI)) {
2724 // Note: There is no need to invalidate any cost modeling decisions here, as
2725 // none were taken so far (see assertion above).
2726 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
2727 }
2728
2729 FixedScalableVFPair MaxFactors = Config.computeFeasibleMaxVF(
2730 MaxTripCount: MaxTC, UserVF, UserIC, FoldTailByMasking: true, RequiresScalarEpilogue: requiresScalarEpilogue(IsVectorizing: true));
2731
2732 // Avoid tail folding if the trip count is known to be a multiple of any VF
2733 // we choose.
2734 std::optional<uint64_t> MaxPowerOf2RuntimeVF =
2735 MaxFactors.FixedVF.getFixedValue();
2736 if (MaxFactors.ScalableVF) {
2737 if (std::optional<uint64_t> MaxRuntimeScalableVF =
2738 getMaxRuntimeElementCount(EC: MaxFactors.ScalableVF, F: *TheFunction))
2739 MaxPowerOf2RuntimeVF =
2740 std::max(a: *MaxPowerOf2RuntimeVF, b: *MaxRuntimeScalableVF);
2741 else
2742 MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
2743 }
2744
2745 auto NoScalarEpilogueNeeded = [this, &UserIC](uint64_t MaxRuntimeVF) {
2746 // Return false if the loop is neither a single-latch-exit loop nor an
2747 // early-exit loop as tail-folding is not supported in that case.
2748 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
2749 !Legal->hasUncountableEarlyExit())
2750 return false;
2751 uint64_t MaxVFtimesIC = MaxRuntimeVF * std::max<uint64_t>(a: UserIC, b: 1);
2752 ScalarEvolution *SE = PSE.getSE();
2753 // Calling getSymbolicMaxBackedgeTakenCount enables support for loops
2754 // with uncountable exits. For countable loops, the symbolic maximum must
2755 // remain identical to the known back-edge taken count.
2756 const SCEV *BackedgeTakenCount = PSE.getSymbolicMaxBackedgeTakenCount();
2757 assert((Legal->hasUncountableEarlyExit() ||
2758 BackedgeTakenCount == PSE.getBackedgeTakenCount()) &&
2759 "Invalid loop count");
2760 const SCEV *ExitCount = SE->getAddExpr(
2761 LHS: BackedgeTakenCount, RHS: SE->getOne(Ty: BackedgeTakenCount->getType()));
2762 const SCEV *Rem = SE->getURemExpr(
2763 LHS: SE->applyLoopGuards(Expr: ExitCount, L: TheLoop),
2764 RHS: SE->getConstant(Ty: BackedgeTakenCount->getType(), V: MaxVFtimesIC));
2765 return Rem->isZero();
2766 };
2767
2768 if (MaxPowerOf2RuntimeVF > 0u) {
2769 assert((UserVF.isNonZero() || isPowerOf2_64(*MaxPowerOf2RuntimeVF)) &&
2770 "MaxFixedVF must be a power of 2");
2771 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
2772 // Accept MaxFixedVF if we do not have a tail.
2773 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
2774 return MaxFactors;
2775 }
2776 }
2777
2778 auto ExpectedTC = getSmallBestKnownTC(Opts, PSE, L: TheLoop);
2779 if (ExpectedTC && ExpectedTC->isFixed() &&
2780 ExpectedTC->getFixedValue() <=
2781 TTI.getMinTripCountTailFoldingThreshold()) {
2782 // If we have a low-trip-count, and the fixed-width VF is known to divide
2783 // the trip count the fixed-width factor in preference to allow the
2784 // generation of a non-predicated loop.
2785 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
2786 NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
2787 LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
2788 "remain for any chosen VF.\n");
2789 MaxFactors.ScalableVF = ElementCount::getScalable(MinVal: 0);
2790 return MaxFactors;
2791 }
2792
2793 // Allow cases where the ExactTC == (VF * IC) + 1.
2794 //
2795 // This produces 1 vector iteration, and 1 scalar iteration with no
2796 // remainder. Later passes will eliminate the loop and leave straight-line
2797 // code as the both iteration counts are statically known.
2798 //
2799 // If a function is marked as minsize/optsize or OptForSize is set, do not
2800 // allow this form of transformation as this will increase CodeSize.
2801 //
2802 // For loops with small bodies, the cost model is not currently reliable
2803 // enough to accurately determine if vectorization is beneficial.
2804 unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
2805 unsigned MaxVFForTC = llvm::bit_floor(Value: TC.getFixedValue());
2806 if (TC.getFixedValue() - MaxVFForTC == 1 && MaxVFForTC / EffectiveIC > 1 &&
2807 MaxVFForTC <= (MaxFactors.FixedVF.getFixedValue() * EffectiveIC) &&
2808 !Config.OptForSize) {
2809 unsigned NumOfInstructions = llvm::sum_of(
2810 Range: llvm::map_range(C: TheLoop->blocks(),
2811 F: [](BasicBlock *BB) { return BB->size(); }),
2812 Init: unsigned(0));
2813 if (NumOfInstructions > Opts.low_trip_count_loop_body_size_limit) {
2814 unsigned VF = MaxVFForTC / EffectiveIC;
2815 LLVM_DEBUG(dbgs() << "LV: Picking MaxVF=" << VF
2816 << " with 1 scalar iteration remaining.\n");
2817 MaxFactors.FixedVF = ElementCount::getFixed(MinVal: VF);
2818 MaxFactors.ScalableVF = ElementCount::getScalable(MinVal: 0);
2819 return MaxFactors;
2820 }
2821 }
2822
2823 reportVectorizationFailure(
2824 DebugMsg: "The trip count is below the minial threshold value.",
2825 OREMsg: "loop trip count is too low, avoiding vectorization", ORETag: "LowTripCount",
2826 ORE, TheLoop);
2827 return FixedScalableVFPair::getNone();
2828 }
2829
2830 // If we don't know the precise trip count, or if the trip count that we
2831 // found modulo the vectorization factor is not zero, try to fold the tail
2832 // by masking.
2833 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
2834 bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
2835 setTailFoldingStyle(IsScalableVF: ContainsScalableVF, UserIC);
2836 if (foldTailByMasking()) {
2837 if (foldTailWithEVL()) {
2838 LLVM_DEBUG(
2839 dbgs()
2840 << "LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
2841 "try to generate VP Intrinsics with scalable vector "
2842 "factors only.\n");
2843 // Tail folded loop using VP intrinsics restricts the VF to be scalable
2844 // for now.
2845 // TODO: extend it for fixed vectors, if required.
2846 assert(ContainsScalableVF && "Expected scalable vector factor.");
2847
2848 MaxFactors.FixedVF = ElementCount::getFixed(MinVal: 1);
2849 } else {
2850 tryToEnablePartialAliasMasking();
2851 }
2852 return MaxFactors;
2853 }
2854
2855 // If there was a tail-folding hint/switch, but we can't fold the tail by
2856 // masking, fallback to a vectorization with an epilogue.
2857 if (EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail) {
2858 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with an "
2859 "epilogue instead.\n");
2860 EpilogueLoweringStatus = CM_EpilogueAllowed;
2861 return MaxFactors;
2862 }
2863
2864 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail) {
2865 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
2866 return FixedScalableVFPair::getNone();
2867 }
2868
2869 if (TC.isZero()) {
2870 reportVectorizationFailure(
2871 DebugMsg: "unable to calculate the loop count due to complex control flow",
2872 ORETag: "UnknownLoopCountComplexCFG", ORE, TheLoop);
2873 return FixedScalableVFPair::getNone();
2874 }
2875
2876 reportVectorizationFailure(
2877 DebugMsg: "Cannot optimize for size and vectorize at the same time.",
2878 OREMsg: "cannot optimize for size and vectorize at the same time. "
2879 "Enable vectorization of this loop with '#pragma clang loop "
2880 "vectorize(enable)' when compiling with -Os/-Oz",
2881 ORETag: "NoTailLoopWithOptForSize", ORE, TheLoop);
2882 return FixedScalableVFPair::getNone();
2883}
2884
2885void LoopVectorizationPlanner::emitInvalidCostRemarks(
2886 OptimizationRemarkEmitter *ORE) {
2887 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
2888 SmallVector<RecipeVFPair> InvalidCosts;
2889 for (const auto &Plan : VPlans) {
2890 for (ElementCount VF : Plan->vectorFactors()) {
2891 // The VPlan-based cost model is designed for computing vector cost.
2892 // Querying VPlan-based cost model with a scarlar VF will cause some
2893 // errors because we expect the VF is vector for most of the widen
2894 // recipes.
2895 if (VF.isScalar())
2896 continue;
2897
2898 VPCostContext CostCtx(*TLI, *Plan, *CM, Config,
2899 /*ReusePrintingSlotTracker=*/true);
2900 precomputeCosts(Plan&: *Plan, VF, CostCtx);
2901 auto Iter = vp_depth_first_deep(G: Plan->getVectorLoopRegion()->getEntry());
2902 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: Iter)) {
2903 for (auto &R : *VPBB) {
2904 if (!R.cost(VF, Ctx&: CostCtx).isValid())
2905 InvalidCosts.emplace_back(Args: &R, Args&: VF);
2906 }
2907 }
2908 }
2909 }
2910 if (InvalidCosts.empty())
2911 return;
2912
2913 // Emit a report of VFs with invalid costs in the loop.
2914
2915 // Group the remarks per recipe, keeping the recipe order from InvalidCosts.
2916 DenseMap<VPRecipeBase *, unsigned> Numbering;
2917 unsigned I = 0;
2918 for (auto &Pair : InvalidCosts)
2919 if (Numbering.try_emplace(Key: Pair.first, Args&: I).second)
2920 ++I;
2921
2922 // Sort the list, first on recipe(number) then on VF.
2923 sort(C&: InvalidCosts, Comp: [&Numbering](RecipeVFPair &A, RecipeVFPair &B) {
2924 unsigned NA = Numbering[A.first];
2925 unsigned NB = Numbering[B.first];
2926 if (NA != NB)
2927 return NA < NB;
2928 return ElementCount::isKnownLT(LHS: A.second, RHS: B.second);
2929 });
2930
2931 // For a list of ordered recipe-VF pairs:
2932 // [(load, VF1), (load, VF2), (store, VF1)]
2933 // group the recipes together to emit separate remarks for:
2934 // load (VF1, VF2)
2935 // store (VF1)
2936 auto Tail = ArrayRef<RecipeVFPair>(InvalidCosts);
2937 auto Subset = ArrayRef<RecipeVFPair>();
2938 do {
2939 if (Subset.empty())
2940 Subset = Tail.take_front(N: 1);
2941
2942 VPRecipeBase *R = Subset.front().first;
2943
2944 unsigned Opcode =
2945 TypeSwitch<const VPRecipeBase *, unsigned>(R)
2946 .Case(caseFn: [](const VPHeaderPHIRecipe *R) { return Instruction::PHI; })
2947 .Case(
2948 caseFn: [](const VPWidenStoreRecipe *R) { return Instruction::Store; })
2949 .Case(caseFn: [](const VPWidenLoadRecipe *R) { return Instruction::Load; })
2950 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
2951 caseFn: [](const auto *R) { return Instruction::Call; })
2952 .Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
2953 VPWidenCastRecipe>(
2954 caseFn: [](const auto *R) { return R->getOpcode(); })
2955 .Case(caseFn: [](const VPInterleaveRecipe *R) {
2956 return R->getStoredValues().empty() ? Instruction::Load
2957 : Instruction::Store;
2958 })
2959 .Case(caseFn: [](const VPReductionRecipe *R) {
2960 return RecurrenceDescriptor::getOpcode(Kind: R->getRecurrenceKind());
2961 });
2962
2963 // If the next recipe is different, or if there are no other pairs,
2964 // emit a remark for the collated subset. e.g.
2965 // [(load, VF1), (load, VF2))]
2966 // to emit:
2967 // remark: invalid costs for 'load' at VF=(VF1, VF2)
2968 if (Subset == Tail || Tail[Subset.size()].first != R) {
2969 std::string OutString;
2970 raw_string_ostream OS(OutString);
2971 assert(!Subset.empty() && "Unexpected empty range");
2972 OS << "Recipe with invalid costs prevented vectorization at VF=(";
2973 for (const auto &Pair : Subset)
2974 OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second;
2975 OS << "):";
2976 if (Opcode == Instruction::Call) {
2977 StringRef Name = "";
2978 if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(Val: R)) {
2979 Name = Int->getIntrinsicName();
2980 } else {
2981 auto *WidenCall = dyn_cast<VPWidenCallRecipe>(Val: R);
2982 Function *CalledFn =
2983 WidenCall
2984 ? WidenCall->getCalledScalarFunction()
2985 : cast<Function>(Val: R->getLastOperand()->getLiveInIRValue());
2986 Name = CalledFn->getName();
2987 }
2988 OS << " call to " << Name;
2989 } else
2990 OS << " " << Instruction::getOpcodeName(Opcode);
2991 reportVectorizationInfo(Msg: OutString, ORETag: "InvalidCost", ORE, TheLoop: OrigLoop, I: nullptr,
2992 DL: R->getDebugLoc());
2993 Tail = Tail.drop_front(N: Subset.size());
2994 Subset = {};
2995 } else
2996 // Grow the subset by one element
2997 Subset = Tail.take_front(N: Subset.size() + 1);
2998 } while (!Tail.empty());
2999}
3000
3001/// Check if any recipe of \p Plan will generate a vector value, which will be
3002/// assigned a vector register.
3003static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
3004 const TargetTransformInfo &TTI) {
3005 assert(VF.isVector() && "Checking a scalar VF?");
3006 DenseSet<VPRecipeBase *> EphemeralRecipes;
3007 collectEphemeralRecipesForVPlan(Plan, EphRecipes&: EphemeralRecipes);
3008 // Set of already visited types.
3009 DenseSet<Type *> Visited;
3010 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3011 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
3012 for (VPRecipeBase &R : *VPBB) {
3013 if (EphemeralRecipes.contains(V: &R))
3014 continue;
3015 // Continue early if the recipe is considered to not produce a vector
3016 // result. Note that this includes VPInstruction where some opcodes may
3017 // produce a vector, to preserve existing behavior as VPInstructions model
3018 // aspects not directly mapped to existing IR instructions.
3019 switch (R.getVPRecipeID()) {
3020 case VPRecipeBase::VPDerivedIVSC:
3021 case VPRecipeBase::VPScalarIVStepsSC:
3022 case VPRecipeBase::VPReplicateSC:
3023 case VPRecipeBase::VPInstructionSC:
3024 case VPRecipeBase::VPCurrentIterationPHISC:
3025 case VPRecipeBase::VPVectorPointerSC:
3026 case VPRecipeBase::VPVectorEndPointerSC:
3027 case VPRecipeBase::VPExpandSCEVSC:
3028 case VPRecipeBase::VPPredInstPHISC:
3029 case VPRecipeBase::VPBranchOnMaskSC:
3030 continue;
3031 case VPRecipeBase::VPReductionSC:
3032 case VPRecipeBase::VPActiveLaneMaskPHISC:
3033 case VPRecipeBase::VPWidenCallSC:
3034 case VPRecipeBase::VPWidenCanonicalIVSC:
3035 case VPRecipeBase::VPWidenCastSC:
3036 case VPRecipeBase::VPWidenGEPSC:
3037 case VPRecipeBase::VPWidenIntrinsicSC:
3038 case VPRecipeBase::VPWidenMemIntrinsicSC:
3039 case VPRecipeBase::VPWidenSC:
3040 case VPRecipeBase::VPBlendSC:
3041 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
3042 case VPRecipeBase::VPHistogramSC:
3043 case VPRecipeBase::VPWidenPHISC:
3044 case VPRecipeBase::VPWidenIntOrFpInductionSC:
3045 case VPRecipeBase::VPWidenPointerInductionSC:
3046 case VPRecipeBase::VPReductionPHISC:
3047 case VPRecipeBase::VPInterleaveEVLSC:
3048 case VPRecipeBase::VPInterleaveSC:
3049 case VPRecipeBase::VPWidenLoadEVLSC:
3050 case VPRecipeBase::VPWidenLoadSC:
3051 case VPRecipeBase::VPWidenStoreEVLSC:
3052 case VPRecipeBase::VPWidenStoreSC:
3053 break;
3054 default:
3055 llvm_unreachable("unhandled recipe");
3056 }
3057
3058 auto WillGenerateTargetVectors = [&TTI, VF](Type *VectorTy) {
3059 unsigned NumLegalParts = TTI.getNumberOfParts(Tp: VectorTy);
3060 if (!NumLegalParts)
3061 return false;
3062 if (VF.isScalable()) {
3063 // <vscale x 1 x iN> is assumed to be profitable over iN because
3064 // scalable registers are a distinct register class from scalar
3065 // ones. If we ever find a target which wants to lower scalable
3066 // vectors back to scalars, we'll need to update this code to
3067 // explicitly ask TTI about the register class uses for each part.
3068 return NumLegalParts <= VF.getKnownMinValue();
3069 }
3070 // Two or more elements that share a register - are vectorized.
3071 return NumLegalParts < VF.getFixedValue();
3072 };
3073
3074 // If no def nor is a store, e.g., branches, continue - no value to check.
3075 if (R.getNumDefinedValues() == 0 &&
3076 !isa<VPWidenStoreRecipe, VPWidenStoreEVLRecipe, VPInterleaveBase>(Val: &R))
3077 continue;
3078 // For multi-def recipes, currently only interleaved loads, suffice to
3079 // check first def only.
3080 // For stores check their stored value; for interleaved stores suffice
3081 // the check first stored value only. In all cases this is the second
3082 // operand.
3083 VPValue *ToCheck =
3084 R.getNumDefinedValues() >= 1 ? R.getVPValue(I: 0) : R.getOperand(N: 1);
3085 Type *ScalarTy = ToCheck->getScalarType();
3086 if (!Visited.insert(V: {ScalarTy}).second)
3087 continue;
3088 Type *WideTy = toVectorizedTy(Ty: ScalarTy, EC: VF);
3089 if (any_of(Range: getContainedTypes(Ty: WideTy), P: WillGenerateTargetVectors))
3090 return true;
3091 }
3092 }
3093
3094 return false;
3095}
3096
3097static bool hasReplicatorRegion(VPlan &Plan) {
3098 return any_of(Range: VPBlockUtils::blocksOnly<VPRegionBlock>(Range: vp_depth_first_shallow(
3099 G: Plan.getVectorLoopRegion()->getEntry())),
3100 P: [](auto *VPRB) { return VPRB->isReplicator(); });
3101}
3102
3103/// Returns true if the VPlan contains a VPReductionPHIRecipe with
3104/// FindLast recurrence kind.
3105static bool hasFindLastReductionPhi(VPlan &Plan) {
3106 return any_of(Range: make_isa_range<VPReductionPHIRecipe>(
3107 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
3108 P: [](VPReductionPHIRecipe &RedPhi) {
3109 return RecurrenceDescriptor::isFindLastRecurrenceKind(
3110 Kind: RedPhi.getRecurrenceKind());
3111 });
3112}
3113
3114/// Determine how to lower the epilogue for the vector epilogue loop.
3115/// Check if there are any conflicts that prevent tail-folding the epilogue.
3116/// \return CM_EpilogueNotNeededFoldTail if epilogue tail-folding is possible,
3117/// otherwise CM_EpilogueAllowed.
3118static EpilogueLowering getEpilogueTailLowering(
3119 const VectorizeOptions &Opts, const LoopVectorizationCostModel &MainCM,
3120 const Loop *L, OptimizationRemarkEmitter *ORE,
3121 LoopVectorizationLegality &LVL, const LoopVectorizeHints &Hints,
3122 TargetTransformInfo *TTI) {
3123 // Epilogue TF is only enabled when explicitly requested via command line.
3124 if (Opts.epilogue_tail_folding_policy != TailFoldingPolicyTy::PreferFoldTail)
3125 return CM_EpilogueAllowed;
3126
3127 if (!Opts.enable_epilogue_vectorization) {
3128 reportVectorizationInfo(
3129 Msg: "Options conflict, epilogue vectorization is disallowed while "
3130 "epilogue tail-folding allowed!",
3131 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
3132 return CM_EpilogueAllowed;
3133 }
3134
3135 if (!Hints.getWidth() || !hasForcedEpilogueVF()) {
3136 reportVectorizationInfo(Msg: "For now, epilogue tail-folding can't be "
3137 "applied without forced main/epilogue loop VF",
3138 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
3139 return CM_EpilogueAllowed;
3140 }
3141
3142 if (ElementCount::isKnownLE(LHS: Hints.getWidth(), RHS: EpilogueVectorizationForceVF)) {
3143 reportVectorizationInfo(Msg: "For now, epilogue tail-folding can't be applied "
3144 "when VF of the main loop <= VF of the epilogue",
3145 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
3146 return CM_EpilogueAllowed;
3147 }
3148
3149 if (!L->isInnermost()) {
3150 reportVectorizationInfo(
3151 Msg: "Epilogue tail-folding is not supported for outer loop",
3152 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3153 return CM_EpilogueAllowed;
3154 }
3155
3156 // If scalar epilogue is explicitly required, we can't apply TF.
3157 if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
3158 reportVectorizationInfo(
3159 Msg: "Epilogue tail-folding can't be applied because scalar epilogue is "
3160 "required. Fall back to a normal epilogue",
3161 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3162 return CM_EpilogueAllowed;
3163 }
3164
3165 // If having epilogue is NOT allowed, then no epilogue to apply TF for.
3166 if (!MainCM.isEpilogueAllowed()) {
3167 reportVectorizationInfo(Msg: "Not applying tail-folding to the epilogue, since "
3168 "no epilogue is allowed.",
3169 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3170 return CM_EpilogueAllowed;
3171 }
3172
3173 if (L->getExitingBlock() != L->getLoopLatch() ||
3174 LVL.hasUncountableEarlyExit()) {
3175 reportVectorizationInfo(
3176 Msg: "Epilogue tail-folding is not supported yet for early-exit loops",
3177 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3178 return CM_EpilogueAllowed;
3179 }
3180
3181 // The epilogue reuses the main loop's interleave groups, so it can't be
3182 // tail-folded if the target can't mask interleaved accesses.
3183 // TODO: Add support once the epilogue has its own IAI, separate from the main
3184 // loop's.
3185 if (MainCM.InterleaveInfo.hasGroups() &&
3186 !useMaskedInterleavedAccesses(Opts, TTI: *TTI)) {
3187 reportVectorizationInfo(
3188 Msg: "Epilogue tail-folding is not supported with interleaved accesses "
3189 "when masking them isn't supported",
3190 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3191 return CM_EpilogueAllowed;
3192 }
3193
3194 if (Opts.force_partial_aliasing_vectorization) {
3195 reportVectorizationInfo(
3196 Msg: "Epilogue tail-folding is not supported with alias masking",
3197 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3198 return CM_EpilogueAllowed;
3199 }
3200
3201 if (!LVL.getReductionVars().empty()) {
3202 reportVectorizationInfo(
3203 Msg: "Epilogue tail-folding is not supported with reductions",
3204 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3205 return CM_EpilogueAllowed;
3206 }
3207
3208 if (!LVL.getFixedOrderRecurrences().empty()) {
3209 reportVectorizationInfo(
3210 Msg: "Epilogue tail-folding is not supported with fixed-order recurrence",
3211 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3212 return CM_EpilogueAllowed;
3213 }
3214
3215 // TODO: This is conservative: it rejects any target that prefers EVL, even
3216 // for fixed-width epilogue VFs where EVL won't be chosen. Move this check to
3217 // where the epilogue's TF style is known once epilogue TF is supported.
3218 TailFoldingStyle TFStyle = Opts.force_tail_folding_style.value_or(
3219 u: TTI->getPreferredTailFoldingStyle());
3220 // TODO: Remove once EVL recipes support cloning.
3221 if (TFStyle == TailFoldingStyle::DataWithEVL) {
3222 reportVectorizationInfo(Msg: "Epilogue tail-folding is not supported yet with "
3223 "EVL-based tail-folding",
3224 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
3225 return CM_EpilogueAllowed;
3226 }
3227
3228 // We can apply tail-folding on the vectorized epilogue loop.
3229 return CM_EpilogueNotNeededFoldTail;
3230}
3231
3232bool VFSelectionContext::isEpilogueVectorizationProfitable(
3233 const ElementCount VF, const unsigned IC) const {
3234 // FIXME: We need a much better cost-model to take different parameters such
3235 // as register pressure, code size increase and cost of extra branches into
3236 // account. For now we apply a very crude heuristic and only consider loops
3237 // with vectorization factors larger than a certain value.
3238
3239 // Allow the target to opt out.
3240 if (!TTI.preferEpilogueVectorization(Iters: VF * IC))
3241 return false;
3242
3243 unsigned MinVFThreshold = Opts.epilogue_vectorization_minimum_VF.value_or(
3244 u: TTI.getEpilogueVectorizationMinVF());
3245 return estimateElementCount(VF: VF * IC, VScale: getVScaleForTuning()) >= MinVFThreshold;
3246}
3247
3248std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
3249 VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC,
3250 bool ScalarEpilogueAllowed) {
3251 if (!Opts.enable_epilogue_vectorization) {
3252 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
3253 return nullptr;
3254 }
3255
3256 if (!ScalarEpilogueAllowed) {
3257 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
3258 "epilogue is allowed.\n");
3259 return nullptr;
3260 }
3261
3262 if (vputils::findIncomingAliasMask(Plan: MainPlan)) {
3263 LLVM_DEBUG(
3264 dbgs()
3265 << "LEV: Epilogue vectorization not supported with alias masking.\n");
3266 return nullptr;
3267 }
3268
3269 // Not really a cost consideration, but check for unsupported cases here to
3270 // simplify the logic.
3271 if (!isCandidateForEpilogueVectorization(MainPlan)) {
3272 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
3273 "is not a supported candidate.\n");
3274 return nullptr;
3275 }
3276
3277 if (hasForcedEpilogueVF()) {
3278 if (estimateElementCount(VF: EpilogueVectorizationForceVF,
3279 VScale: Config.getVScaleForTuning()) >=
3280 IC * estimateElementCount(VF: MainLoopVF, VScale: Config.getVScaleForTuning())) {
3281 // Note that the main loop leaves IC * MainLoopVF iterations iff a scalar
3282 // epilogue is required, but then the epilogue loop also requires a scalar
3283 // epilogue.
3284 LLVM_DEBUG(dbgs() << "LEV: Forced epilogue VF results in dead epilogue "
3285 "vector loop, skipping vectorizing epilogue.\n");
3286 return nullptr;
3287 }
3288
3289 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
3290 if (hasPlanWithVF(VF: EpilogueVectorizationForceVF)) {
3291 std::unique_ptr<VPlan> Clone(
3292 getPlanFor(VF: EpilogueVectorizationForceVF).duplicate());
3293 Clone->setVF(EpilogueVectorizationForceVF);
3294 return Clone;
3295 }
3296
3297 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization forced factor is not "
3298 "viable.\n");
3299 return nullptr;
3300 }
3301
3302 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
3303 LLVM_DEBUG(
3304 dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n");
3305 return nullptr;
3306 }
3307
3308 if (!Config.isEpilogueVectorizationProfitable(VF: MainLoopVF, IC)) {
3309 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
3310 "this loop\n");
3311 return nullptr;
3312 }
3313
3314 // Check if a plan's vector loop processes fewer iterations than VF (e.g. when
3315 // interleave groups have been narrowed) narrowInterleaveGroups) and return
3316 // the adjusted, effective VF.
3317 using namespace VPlanPatternMatch;
3318 auto GetEffectiveVF = [](VPlan &Plan, ElementCount VF) -> ElementCount {
3319 auto *Exiting = Plan.getVectorLoopRegion()->getExitingBasicBlock();
3320 if (match(V: &Exiting->back(),
3321 P: m_BranchOnCount(Op0: m_Add(Op0: m_CanonicalIV(), Op1: m_Specific(VPV: &Plan.getUF())),
3322 Op1: m_VPValue())))
3323 return ElementCount::get(MinVal: 1, Scalable: VF.isScalable());
3324 return VF;
3325 };
3326
3327 // Check if the main loop processes fewer than MainLoopVF elements per
3328 // iteration (e.g. due to narrowing interleave groups). Adjust MainLoopVF
3329 // as needed.
3330 MainLoopVF = GetEffectiveVF(MainPlan, MainLoopVF);
3331
3332 // If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know
3333 // the main loop handles 8 lanes per iteration. We could still benefit from
3334 // vectorizing the epilogue loop with VF=4.
3335 ElementCount EstimatedRuntimeVF = ElementCount::getFixed(
3336 MinVal: estimateElementCount(VF: MainLoopVF, VScale: Config.getVScaleForTuning()));
3337
3338 Type *TCType = Legal->getWidestInductionType();
3339 const SCEV *RemainingIterations = nullptr;
3340 unsigned MaxTripCount = 0;
3341 const SCEV *TC = vputils::getSCEVExprForVPValue(V: MainPlan.getTripCount(), PSE);
3342 assert(!isa<SCEVCouldNotCompute>(TC) && "Trip count SCEV must be computable");
3343 const SCEV *KnownMinTC;
3344 bool ScalableTC = match(S: TC, P: m_scev_c_Mul(Op0: m_SCEV(V&: KnownMinTC), Op1: m_SCEVVScale()));
3345 bool ScalableRemIter = false;
3346 ScalarEvolution &SE = *PSE.getSE();
3347 // Use versions of TC and VF in which both are either scalable or fixed.
3348 if (ScalableTC == MainLoopVF.isScalable()) {
3349 ScalableRemIter = ScalableTC;
3350 RemainingIterations =
3351 SE.getURemExpr(LHS: TC, RHS: SE.getElementCount(Ty: TCType, EC: MainLoopVF * IC));
3352 } else if (ScalableTC) {
3353 const SCEV *EstimatedTC = SE.getMulExpr(
3354 LHS: KnownMinTC,
3355 RHS: SE.getConstant(Ty: TCType, V: Config.getVScaleForTuning().value_or(u: 1)));
3356 RemainingIterations = SE.getURemExpr(
3357 LHS: EstimatedTC, RHS: SE.getElementCount(Ty: TCType, EC: MainLoopVF * IC));
3358 } else
3359 RemainingIterations =
3360 SE.getURemExpr(LHS: TC, RHS: SE.getElementCount(Ty: TCType, EC: EstimatedRuntimeVF * IC));
3361
3362 // No iterations left to process in the epilogue.
3363 if (RemainingIterations->isZero())
3364 return nullptr;
3365
3366 if (MainLoopVF.isFixed()) {
3367 MaxTripCount = MainLoopVF.getFixedValue() * IC - 1;
3368 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_ULT, LHS: RemainingIterations,
3369 RHS: SE.getConstant(Ty: TCType, V: MaxTripCount))) {
3370 MaxTripCount = SE.getUnsignedRangeMax(S: RemainingIterations).getZExtValue();
3371 }
3372 LLVM_DEBUG(dbgs() << "LEV: Maximum Trip Count for Epilogue: "
3373 << MaxTripCount << "\n");
3374 }
3375
3376 auto SkipVF = [&](const SCEV *VF, const SCEV *RemIter) -> bool {
3377 return SE.isKnownPredicate(Pred: CmpInst::ICMP_UGT, LHS: VF, RHS: RemIter);
3378 };
3379 VectorizationFactor Result = VectorizationFactor::Disabled();
3380 VPlan *BestPlan = nullptr;
3381 for (auto &NextVF : ProfitableVFs) {
3382 // Skip candidate VFs without a corresponding VPlan.
3383 if (!hasPlanWithVF(VF: NextVF.Width))
3384 continue;
3385
3386 VPlan &CurrentPlan = getPlanFor(VF: NextVF.Width);
3387 ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
3388 // Skip fixed vector VFs > than the estimated runtime VF, or any VF > than
3389 // the VF of the main loop.
3390 if ((!EffectiveVF.isScalable() && MainLoopVF.isScalable() &&
3391 ElementCount::isKnownGT(LHS: EffectiveVF, RHS: EstimatedRuntimeVF)) ||
3392 ElementCount::isKnownGT(LHS: EffectiveVF, RHS: MainLoopVF))
3393 continue;
3394
3395 // If EffectiveVF is greater than the number of remaining iterations, the
3396 // epilogue loop would be dead. Skip such factors. If the epilogue plan
3397 // also has narrowed interleave groups, use the effective VF since
3398 // the epilogue step will be reduced to its IC.
3399 // TODO: We should also consider comparing against a scalable
3400 // RemainingIterations when SCEV be able to evaluate non-canonical
3401 // vscale-based expressions.
3402 if (!ScalableRemIter) {
3403 // Handle the case where EffectiveVF and RemainingIterations are in
3404 // different numerical spaces.
3405 if (EffectiveVF.isScalable())
3406 EffectiveVF = ElementCount::getFixed(
3407 MinVal: estimateElementCount(VF: EffectiveVF, VScale: Config.getVScaleForTuning()));
3408 if (SkipVF(SE.getElementCount(Ty: TCType, EC: EffectiveVF), RemainingIterations))
3409 continue;
3410 }
3411
3412 if (Result.Width.isScalar() ||
3413 isMoreProfitable(A: NextVF, B: Result, MaxTripCount,
3414 HasTail: !MainPlan.hasTailFolded(),
3415 /*IsEpilogue*/ true)) {
3416 Result = NextVF;
3417 BestPlan = &CurrentPlan;
3418 }
3419 }
3420
3421 if (!BestPlan)
3422 return nullptr;
3423
3424 LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
3425 << Result.Width << "\n");
3426 std::unique_ptr<VPlan> Clone(BestPlan->duplicate());
3427 Clone->setVF(Result.Width);
3428 return Clone;
3429}
3430
3431unsigned
3432LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
3433 InstructionCost LoopCost) {
3434 // -- The interleave heuristics --
3435 // We interleave the loop in order to expose ILP and reduce the loop overhead.
3436 // There are many micro-architectural considerations that we can't predict
3437 // at this level. For example, frontend pressure (on decode or fetch) due to
3438 // code size, or the number and capabilities of the execution ports.
3439 //
3440 // We use the following heuristics to select the interleave count:
3441 // 1. If the code has reductions, then we interleave to break the cross
3442 // iteration dependency.
3443 // 2. If the loop is really small, then we interleave to reduce the loop
3444 // overhead.
3445 // 3. We don't interleave if we think that we will spill registers to memory
3446 // due to the increased register pressure.
3447
3448 // Do not interleave tail-folded loops, as the overhead of multiple
3449 // instructions to calculate the predicate is likely not beneficial.
3450 // If an epilogue is not allowed for any other reason, do not interleave.
3451 if (!CM->isEpilogueAllowed())
3452 return 1;
3453
3454 if (any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3455 P: IsaPred<VPCurrentIterationPHIRecipe>)) {
3456 LLVM_DEBUG(dbgs() << "LV: Loop requires variable-length step. "
3457 "Unroll factor forced to be 1.\n");
3458 return 1;
3459 }
3460
3461 // We used the distance for the interleave count.
3462 if (!Legal->isSafeForAnyVectorWidth())
3463 return 1;
3464
3465 // We don't attempt to perform interleaving for loops with uncountable early
3466 // exits because the VPInstruction::AnyOf code cannot currently handle
3467 // multiple parts.
3468 if (Plan.hasEarlyExit())
3469 return 1;
3470
3471 const bool HasReductions =
3472 any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3473 P: IsaPred<VPReductionPHIRecipe>);
3474
3475 // FIXME: implement interleaving for FindLast transform correctly.
3476 if (hasFindLastReductionPhi(Plan))
3477 return 1;
3478
3479 VPRegisterUsage R = calculateRegisterUsageForPlan(Plan, VFs: {VF}, TTI)[0];
3480
3481 // If we did not calculate the cost for VF (because the user selected the VF)
3482 // then we calculate the cost of VF here.
3483 if (LoopCost == 0) {
3484 if (VF.isScalar())
3485 LoopCost = CM->expectedCost(VF);
3486 else
3487 LoopCost = cost(Plan, VF, RU: &R, PlanCM&: *CM);
3488 assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
3489
3490 // Loop body is free and there is no need for interleaving.
3491 if (LoopCost == 0)
3492 return 1;
3493 }
3494
3495 // We divide by these constants so assume that we have at least one
3496 // instruction that uses at least one register.
3497 for (auto &Pair : R.MaxLocalUsers) {
3498 Pair.second = std::max(a: Pair.second, b: 1U);
3499 }
3500
3501 // We calculate the interleave count using the following formula.
3502 // Subtract the number of loop invariants from the number of available
3503 // registers. These registers are used by all of the interleaved instances.
3504 // Next, divide the remaining registers by the number of registers that is
3505 // required by the loop, in order to estimate how many parallel instances
3506 // fit without causing spills. All of this is rounded down if necessary to be
3507 // a power of two. We want power of two interleave count to simplify any
3508 // addressing operations or alignment considerations.
3509 // We also want power of two interleave counts to ensure that the induction
3510 // variable of the vector loop wraps to zero, when tail is folded by masking;
3511 // this currently happens when OptForSize, in which case IC is set to 1 above.
3512 unsigned IC = UINT_MAX;
3513
3514 for (const auto &Pair : R.MaxLocalUsers) {
3515 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(ClassID: Pair.first);
3516 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters
3517 << " registers of "
3518 << TTI.getRegisterClassName(Pair.first)
3519 << " register class\n");
3520 TargetNumRegisters = (VF.isScalar() ? Opts.force_target_num_scalar_regs
3521 : Opts.force_target_num_vector_regs)
3522 .value_or(u&: TargetNumRegisters);
3523 unsigned MaxLocalUsers = Pair.second;
3524 unsigned LoopInvariantRegs = 0;
3525 if (R.LoopInvariantRegs.contains(Key: Pair.first))
3526 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
3527
3528 unsigned TmpIC = llvm::bit_floor(Value: (TargetNumRegisters - LoopInvariantRegs) /
3529 MaxLocalUsers);
3530 // Don't count the induction variable as interleaved.
3531 if (Opts.enable_ind_var_reg_heur) {
3532 TmpIC = llvm::bit_floor(Value: (TargetNumRegisters - LoopInvariantRegs - 1) /
3533 std::max(a: 1U, b: (MaxLocalUsers - 1)));
3534 }
3535
3536 IC = std::min(a: IC, b: TmpIC);
3537 }
3538
3539 // Clamp the interleave ranges to reasonable counts.
3540 bool HasUnorderedReductions =
3541 HasReductions &&
3542 !any_of(Range: make_isa_range<VPReductionPHIRecipe>(
3543 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
3544 P: [](VPReductionPHIRecipe &RedR) { return RedR.isOrdered(); });
3545 unsigned MaxInterleaveCount =
3546 TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions);
3547 LLVM_DEBUG(dbgs() << "LV: MaxInterleaveFactor for the target is "
3548 << MaxInterleaveCount << "\n");
3549
3550 // Check if the user has overridden the max.
3551 MaxInterleaveCount = (VF.isScalar() ? Opts.force_target_max_scalar_interleave
3552 : Opts.force_target_max_vector_interleave)
3553 .value_or(u&: MaxInterleaveCount);
3554
3555 // Try to get the exact trip count, or an estimate based on profiling data or
3556 // ConstantMax from PSE, failing that.
3557 auto BestKnownTC =
3558 getSmallBestKnownTC(Opts, PSE, L: OrigLoop,
3559 /*CanUseConstantMax=*/true,
3560 /*CanExcludeZeroTrips=*/CM->isEpilogueAllowed());
3561
3562 // For fixed length VFs treat a scalable trip count as unknown.
3563 if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
3564 // Re-evaluate trip counts and VFs to be in the same numerical space.
3565 unsigned AvailableTC =
3566 estimateElementCount(VF: *BestKnownTC, VScale: Config.getVScaleForTuning());
3567 unsigned EstimatedVF =
3568 estimateElementCount(VF, VScale: Config.getVScaleForTuning());
3569
3570 // At least one iteration must be scalar when this constraint holds. So the
3571 // maximum available iterations for interleaving is one less.
3572 if (Plan.requiresScalarEpilogue())
3573 --AvailableTC;
3574
3575 unsigned InterleaveCountLB = bit_floor(Value: std::max(
3576 a: 1u, b: std::min(a: AvailableTC / (EstimatedVF * 2), b: MaxInterleaveCount)));
3577
3578 if (getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop).isNonZero()) {
3579 // If the best known trip count is exact, we select between two
3580 // prospective ICs, where
3581 //
3582 // 1) the aggressive IC is capped by the trip count divided by VF
3583 // 2) the conservative IC is capped by the trip count divided by (VF * 2)
3584 //
3585 // The final IC is selected in a way that the epilogue loop trip count is
3586 // minimized while maximizing the IC itself, so that we either run the
3587 // vector loop at least once if it generates a small epilogue loop, or
3588 // else we run the vector loop at least twice.
3589
3590 unsigned InterleaveCountUB = bit_floor(Value: std::max(
3591 a: 1u, b: std::min(a: AvailableTC / EstimatedVF, b: MaxInterleaveCount)));
3592 MaxInterleaveCount = InterleaveCountLB;
3593
3594 if (InterleaveCountUB != InterleaveCountLB) {
3595 unsigned TailTripCountUB =
3596 (AvailableTC % (EstimatedVF * InterleaveCountUB));
3597 unsigned TailTripCountLB =
3598 (AvailableTC % (EstimatedVF * InterleaveCountLB));
3599 // If both produce same scalar tail, maximize the IC to do the same work
3600 // in fewer vector loop iterations
3601 if (TailTripCountUB == TailTripCountLB)
3602 MaxInterleaveCount = InterleaveCountUB;
3603 }
3604 } else {
3605 // If trip count is an estimated compile time constant, limit the
3606 // IC to be capped by the trip count divided by VF * 2, such that the
3607 // vector loop runs at least twice to make interleaving seem profitable
3608 // when there is an epilogue loop present. Since exact Trip count is not
3609 // known we choose to be conservative in our IC estimate.
3610 MaxInterleaveCount = InterleaveCountLB;
3611 }
3612 }
3613
3614 assert(MaxInterleaveCount > 0 &&
3615 "Maximum interleave count must be greater than 0");
3616
3617 // Clamp the calculated IC to be between the 1 and the max interleave count
3618 // that the target and trip count allows.
3619 if (IC > MaxInterleaveCount)
3620 IC = MaxInterleaveCount;
3621 else
3622 // Make sure IC is greater than 0.
3623 IC = std::max(a: 1u, b: IC);
3624
3625 assert(IC > 0 && "Interleave count must be greater than 0.");
3626
3627 // Interleave if we vectorized this loop and there is a reduction that could
3628 // benefit from interleaving.
3629 if (VF.isVector() && HasReductions) {
3630 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
3631 return IC;
3632 }
3633
3634 // For any scalar loop that either requires runtime checks or tail-folding we
3635 // are better off leaving this to the unroller. Note that if we've already
3636 // vectorized the loop we will have done the runtime check and so interleaving
3637 // won't require further checks.
3638 bool ScalarInterleavingRequiresPredication =
3639 (VF.isScalar() && any_of(Range: OrigLoop->blocks(), P: [this](BasicBlock *BB) {
3640 return Legal->blockNeedsPredication(BB);
3641 }));
3642 bool ScalarInterleavingRequiresRuntimePointerCheck =
3643 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need);
3644
3645 // We want to interleave small loops in order to reduce the loop overhead and
3646 // potentially expose ILP opportunities.
3647 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
3648 << "LV: IC is " << IC << '\n'
3649 << "LV: VF is " << VF << '\n');
3650 const bool AggressivelyInterleave =
3651 TTI.enableAggressiveInterleaving(LoopHasReductions: HasReductions);
3652 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
3653 !ScalarInterleavingRequiresPredication &&
3654 LoopCost < Opts.small_loop_cost) {
3655 // We assume that the cost overhead is 1 and we use the cost model
3656 // to estimate the cost of the loop and interleave until the cost of the
3657 // loop overhead is about 5% of the cost of the loop.
3658 unsigned SmallIC =
3659 std::min(a: IC, b: (unsigned)llvm::bit_floor<uint64_t>(Value: Opts.small_loop_cost /
3660 LoopCost.getValue()));
3661
3662 // Interleave until store/load ports (estimated by max interleave count) are
3663 // saturated.
3664 unsigned NumStores = 0;
3665 unsigned NumLoads = 0;
3666 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3667 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
3668 for (VPRecipeBase &R : *VPBB) {
3669 if (isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Val: &R)) {
3670 NumLoads++;
3671 continue;
3672 }
3673 if (isa<VPWidenStoreRecipe, VPWidenStoreEVLRecipe>(Val: &R)) {
3674 NumStores++;
3675 continue;
3676 }
3677
3678 if (auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(Val: &R)) {
3679 if (unsigned StoreOps = InterleaveR->getNumStoreOperands())
3680 NumStores += StoreOps;
3681 else
3682 NumLoads += InterleaveR->getNumDefinedValues();
3683 continue;
3684 }
3685 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
3686 NumLoads += isa<LoadInst>(Val: RepR->getUnderlyingInstr());
3687 NumStores += isa<StoreInst>(Val: RepR->getUnderlyingInstr());
3688 continue;
3689 }
3690 if (isa<VPHistogramRecipe>(Val: &R)) {
3691 NumLoads++;
3692 NumStores++;
3693 continue;
3694 }
3695 }
3696 }
3697 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
3698 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
3699
3700 // There is little point in interleaving for reductions containing selects
3701 // and compares when VF=1 since it may just create more overhead than it's
3702 // worth for loops with small trip counts. This is because we still have to
3703 // do the final reduction after the loop.
3704 bool HasSelectCmpReductions =
3705 HasReductions &&
3706 any_of(Range: make_isa_range<VPReductionPHIRecipe>(
3707 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
3708 P: [](VPReductionPHIRecipe &RedR) {
3709 return RecurrenceDescriptor::isAnyOfRecurrenceKind(
3710 Kind: RedR.getRecurrenceKind()) ||
3711 RecurrenceDescriptor::isFindIVRecurrenceKind(
3712 Kind: RedR.getRecurrenceKind());
3713 });
3714 if (HasSelectCmpReductions) {
3715 LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
3716 return 1;
3717 }
3718
3719 // If we have a scalar reduction (vector reductions are already dealt with
3720 // by this point), we can increase the critical path length if the loop
3721 // we're interleaving is inside another loop. For tree-wise reductions
3722 // set the limit to 2, and for ordered reductions it's best to disable
3723 // interleaving entirely.
3724 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
3725 bool HasOrderedReductions =
3726 any_of(Range: make_isa_range<VPReductionPHIRecipe>(
3727 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
3728 P: [](VPReductionPHIRecipe &RedR) { return RedR.isOrdered(); });
3729 if (HasOrderedReductions) {
3730 LLVM_DEBUG(
3731 dbgs() << "LV: Not interleaving scalar ordered reductions.\n");
3732 return 1;
3733 }
3734
3735 unsigned F = Opts.max_nested_scalar_reduction_interleave;
3736 SmallIC = std::min(a: SmallIC, b: F);
3737 StoresIC = std::min(a: StoresIC, b: F);
3738 LoadsIC = std::min(a: LoadsIC, b: F);
3739 }
3740
3741 if (Opts.enable_loadstore_runtime_interleave &&
3742 std::max(a: StoresIC, b: LoadsIC) > SmallIC) {
3743 LLVM_DEBUG(
3744 dbgs() << "LV: Interleaving to saturate store or load ports.\n");
3745 return std::max(a: StoresIC, b: LoadsIC);
3746 }
3747
3748 // If there are scalar reductions and TTI has enabled aggressive
3749 // interleaving for reductions, we will interleave to expose ILP.
3750 if (VF.isScalar() && AggressivelyInterleave) {
3751 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3752 // Interleave no less than SmallIC but not as aggressive as the normal IC
3753 // to satisfy the rare situation when resources are too limited.
3754 return std::max(a: IC / 2, b: SmallIC);
3755 }
3756
3757 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
3758 return SmallIC;
3759 }
3760
3761 // Interleave if this is a large loop (small loops are already dealt with by
3762 // this point) that could benefit from interleaving.
3763 if (AggressivelyInterleave) {
3764 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3765 return IC;
3766 }
3767
3768 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n");
3769 return 1;
3770}
3771
3772bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(
3773 Instruction *I, ElementCount VF) const {
3774 // TODO: Cost model for emulated masked load/store is completely
3775 // broken. This hack guides the cost model to use an artificially
3776 // high enough value to practically disable vectorization with such
3777 // operations, except where previously deployed legality hack allowed
3778 // using very low cost values. This is to avoid regressions coming simply
3779 // from moving "masked load/store" check from legality to cost model.
3780 // Masked Load/Gather emulation was previously never allowed.
3781 // Limited number of Masked Store/Scatter emulation was allowed.
3782 assert((isPredicatedInst(I)) &&
3783 "Expecting a scalar emulated instruction");
3784 return isa<LoadInst>(Val: I) ||
3785 (isa<StoreInst>(Val: I) && NumPredStores > Opts.vectorize_num_stores_pred);
3786}
3787
3788void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
3789 assert(VF.isVector() && "Expected VF >= 2");
3790
3791 // If we've already collected the instructions to scalarize or the predicated
3792 // BBs after vectorization, there's nothing to do. Collection may already have
3793 // occurred if we have a user-selected VF and are now computing the expected
3794 // cost for interleaving.
3795 if (InstsToScalarize.contains(Key: VF) ||
3796 PredicatedBBsAfterVectorization.contains(Val: VF))
3797 return;
3798
3799 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's
3800 // not profitable to scalarize any instructions, the presence of VF in the
3801 // map will indicate that we've analyzed it already.
3802 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
3803
3804 // Find all the instructions that are scalar with predication in the loop and
3805 // determine if it would be better to not if-convert the blocks they are in.
3806 // If so, we also record the instructions to scalarize.
3807 for (BasicBlock *BB : TheLoop->blocks()) {
3808 if (!blockNeedsPredicationForAnyReason(BB))
3809 continue;
3810 for (Instruction &I : *BB)
3811 if (isScalarWithPredication(I: &I, VF)) {
3812 ScalarCostsTy ScalarCosts;
3813 // Do not apply discount logic for:
3814 // 1. Scalars after vectorization, as there will only be a single copy
3815 // of the instruction.
3816 // 2. Scalable VF, as that would lead to invalid scalarization costs.
3817 // 3. Emulated masked memrefs, if a hacked cost is needed.
3818 if (!isScalarAfterVectorization(I: &I, VF) && !VF.isScalable() &&
3819 !useEmulatedMaskMemRefHack(I: &I, VF) &&
3820 computePredInstDiscount(PredInst: &I, ScalarCosts, VF) >= 0) {
3821 for (const auto &[I, IC] : ScalarCosts)
3822 ScalarCostsVF.insert(KV: {I, IC});
3823 }
3824 // Remember that BB will remain after vectorization.
3825 PredicatedBBsAfterVectorization[VF].insert(Ptr: BB);
3826 for (auto *Pred : predecessors(BB)) {
3827 if (Pred->getSingleSuccessor() == BB)
3828 PredicatedBBsAfterVectorization[VF].insert(Ptr: Pred);
3829 }
3830 }
3831 }
3832}
3833
3834InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
3835 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) {
3836 assert(!isUniformAfterVectorization(PredInst, VF) &&
3837 "Instruction marked uniform-after-vectorization will be predicated");
3838
3839 // Initialize the discount to zero, meaning that the scalar version and the
3840 // vector version cost the same.
3841 InstructionCost Discount = 0;
3842
3843 // Holds instructions to analyze. The instructions we visit are mapped in
3844 // ScalarCosts. Those instructions are the ones that would be scalarized if
3845 // we find that the scalar version costs less.
3846 SmallVector<Instruction *, 8> Worklist;
3847
3848 // Returns true if the given instruction can be scalarized.
3849 auto CanBeScalarized = [&](Instruction *I) -> bool {
3850 // We only attempt to scalarize instructions forming a single-use chain
3851 // from the original predicated block that would otherwise be vectorized.
3852 // Although not strictly necessary, we give up on instructions we know will
3853 // already be scalar to avoid traversing chains that are unlikely to be
3854 // beneficial.
3855 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
3856 isScalarAfterVectorization(I, VF))
3857 return false;
3858
3859 // If the instruction is scalar with predication, it will be analyzed
3860 // separately. We ignore it within the context of PredInst.
3861 if (isScalarWithPredication(I, VF))
3862 return false;
3863
3864 // If any of the instruction's operands are uniform after vectorization,
3865 // the instruction cannot be scalarized. This prevents, for example, a
3866 // masked load from being scalarized.
3867 //
3868 // We assume we will only emit a value for lane zero of an instruction
3869 // marked uniform after vectorization, rather than VF identical values.
3870 // Thus, if we scalarize an instruction that uses a uniform, we would
3871 // create uses of values corresponding to the lanes we aren't emitting code
3872 // for. This behavior can be changed by allowing getScalarValue to clone
3873 // the lane zero values for uniforms rather than asserting.
3874 for (Use &U : I->operands())
3875 if (auto *J = dyn_cast<Instruction>(Val: U.get()))
3876 if (isUniformAfterVectorization(I: J, VF))
3877 return false;
3878
3879 // Otherwise, we can scalarize the instruction.
3880 return true;
3881 };
3882
3883 // Compute the expected cost discount from scalarizing the entire expression
3884 // feeding the predicated instruction. We currently only consider expressions
3885 // that are single-use instruction chains.
3886 Worklist.push_back(Elt: PredInst);
3887 while (!Worklist.empty()) {
3888 Instruction *I = Worklist.pop_back_val();
3889
3890 // If we've already analyzed the instruction, there's nothing to do.
3891 if (ScalarCosts.contains(Key: I))
3892 continue;
3893
3894 // Cannot scalarize fixed-order recurrence phis at the moment.
3895 if (isa<PHINode>(Val: I) && Legal->isFixedOrderRecurrence(Phi: cast<PHINode>(Val: I)))
3896 continue;
3897
3898 // Compute the cost of the vector instruction. Note that this cost already
3899 // includes the scalarization overhead of the predicated instruction.
3900 InstructionCost VectorCost = getInstructionCost(I, VF);
3901
3902 // Compute the cost of the scalarized instruction. This cost is the cost of
3903 // the instruction as if it wasn't if-converted and instead remained in the
3904 // predicated block. We will scale this cost by block probability after
3905 // computing the scalarization overhead.
3906 InstructionCost ScalarCost =
3907 VF.getFixedValue() * getInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1));
3908
3909 // Compute the scalarization overhead of needed insertelement instructions
3910 // and phi nodes.
3911 if (isScalarWithPredication(I, VF) && !I->getType()->isVoidTy()) {
3912 Type *WideTy = toVectorizedTy(Ty: I->getType(), EC: VF);
3913 for (Type *VectorTy : getContainedTypes(Ty: WideTy)) {
3914 ScalarCost += TTI.getScalarizationOverhead(
3915 Ty: cast<VectorType>(Val: VectorTy), DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
3916 /*Insert=*/true,
3917 /*Extract=*/false, CostKind: Config.CostKind);
3918 }
3919 ScalarCost += VF.getFixedValue() *
3920 TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
3921 }
3922
3923 // Compute the scalarization overhead of needed extractelement
3924 // instructions. For each of the instruction's operands, if the operand can
3925 // be scalarized, add it to the worklist; otherwise, account for the
3926 // overhead.
3927 for (Use &U : I->operands())
3928 if (auto *J = dyn_cast<Instruction>(Val: U.get())) {
3929 assert(canVectorizeTy(J->getType()) &&
3930 "Instruction has non-scalar type");
3931 if (CanBeScalarized(J))
3932 Worklist.push_back(Elt: J);
3933 else if (needsExtract(V: J, VF)) {
3934 Type *WideTy = toVectorizedTy(Ty: J->getType(), EC: VF);
3935 for (Type *VectorTy : getContainedTypes(Ty: WideTy)) {
3936 ScalarCost += TTI.getScalarizationOverhead(
3937 Ty: cast<VectorType>(Val: VectorTy),
3938 DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()), /*Insert*/ false,
3939 /*Extract*/ true, CostKind: Config.CostKind);
3940 }
3941 }
3942 }
3943
3944 // Scale the total scalar cost by block probability.
3945 ScalarCost /= getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
3946
3947 // Compute the discount. A non-negative discount means the vector version
3948 // of the instruction costs more, and scalarizing would be beneficial.
3949 Discount += VectorCost - ScalarCost;
3950 ScalarCosts[I] = ScalarCost;
3951 }
3952
3953 return Discount;
3954}
3955
3956InstructionCost LoopVectorizationCostModel::expectedCost(ElementCount VF) {
3957 InstructionCost Cost;
3958 assert(VF.isScalar() && "must only be called for scalar VFs");
3959
3960 // For each block.
3961 for (BasicBlock *BB : TheLoop->blocks()) {
3962 InstructionCost BlockCost;
3963
3964 // For each instruction in the old loop.
3965 for (Instruction &I : *BB) {
3966 // Skip ignored values.
3967 if (ValuesToIgnore.count(Ptr: &I) ||
3968 (VF.isVector() && VecValuesToIgnore.count(Ptr: &I)))
3969 continue;
3970
3971 InstructionCost C = getInstructionCost(I: &I, VF);
3972
3973 // Check if we should override the cost.
3974 if (C.isValid() && Opts.force_target_instruction_cost)
3975 C = InstructionCost(*Opts.force_target_instruction_cost);
3976
3977 BlockCost += C;
3978 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C << " for VF "
3979 << VF << " For instruction: " << I << '\n');
3980 }
3981
3982 // In the scalar loop, we may not always execute the predicated block, if it
3983 // is an if-else block. Thus, scale the block's cost by the probability of
3984 // executing it. getPredBlockCostDivisor will return 1 for blocks that are
3985 // only predicated by the header mask when folding the tail.
3986 Cost += BlockCost / getPredBlockCostDivisor(CostKind: Config.CostKind, BB);
3987 }
3988
3989 return Cost;
3990}
3991
3992/// Gets the address access SCEV for Ptr, if it should be used for cost modeling
3993/// according to isAddressSCEVForCost.
3994///
3995/// This SCEV can be sent to the Target in order to estimate the address
3996/// calculation cost.
3997static const SCEV *getAddressAccessSCEV(
3998 Value *Ptr,
3999 PredicatedScalarEvolution &PSE,
4000 const Loop *TheLoop) {
4001 const SCEV *Addr = PSE.getSCEV(V: Ptr);
4002 return vputils::isAddressSCEVForCost(Addr, SE&: *PSE.getSE(), L: TheLoop) ? Addr
4003 : nullptr;
4004}
4005
4006InstructionCost
4007LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
4008 ElementCount VF) {
4009 assert(VF.isVector() &&
4010 "Scalarization cost of instruction implies vectorization.");
4011 if (VF.isScalable())
4012 return InstructionCost::getInvalid();
4013
4014 Type *ValTy = getLoadStoreType(I);
4015 auto *SE = PSE.getSE();
4016
4017 unsigned AS = getLoadStoreAddressSpace(I);
4018 Value *Ptr = getLoadStorePointerOperand(V: I);
4019 Type *PtrTy = toVectorTy(Scalar: Ptr->getType(), EC: VF);
4020 // NOTE: PtrTy is a vector to signal `TTI::getAddressComputationCost`
4021 // that it is being called from this specific place.
4022
4023 // Figure out whether the access is strided and get the stride value
4024 // if it's known in compile time
4025 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, PSE, TheLoop);
4026
4027 // Get the cost of the scalar memory instruction and address computation.
4028 InstructionCost Cost =
4029 VF.getFixedValue() *
4030 TTI.getAddressComputationCost(PtrTy, SE, Ptr: PtrSCEV, CostKind: Config.CostKind);
4031
4032 // Don't pass *I here, since it is scalar but will actually be part of a
4033 // vectorized loop where the user of it is a vectorized instruction.
4034 const Align Alignment = getLoadStoreAlignment(I);
4035 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4036 Cost += VF.getFixedValue() *
4037 TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: ValTy->getScalarType(), Alignment,
4038 AddressSpace: AS, CostKind: Config.CostKind, OpdInfo: OpInfo);
4039
4040 // Get the overhead of the extractelement and insertelement instructions
4041 // we might create due to scalarization.
4042 Cost += getScalarizationOverhead(I, VF);
4043
4044 // If we have a predicated load/store, it will need extra i1 extracts and
4045 // conditional branches, but may not be executed for each vector lane. Scale
4046 // the cost by the probability of executing the predicated block.
4047 if (isPredicatedInst(I)) {
4048 Cost /= getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
4049
4050 // Add the cost of an i1 extract and a branch
4051 auto *VecI1Ty =
4052 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: ValTy->getContext()), EC: VF);
4053 Cost += TTI.getScalarizationOverhead(
4054 Ty: VecI1Ty, DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4055 /*Insert=*/false, /*Extract=*/true, CostKind: Config.CostKind);
4056 Cost += TTI.getCFInstrCost(Opcode: Instruction::CondBr, CostKind: Config.CostKind);
4057
4058 if (useEmulatedMaskMemRefHack(I, VF))
4059 // Artificially setting to a high enough value to practically disable
4060 // vectorization with such operations.
4061 Cost = 3000000;
4062 }
4063
4064 return Cost;
4065}
4066
4067InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
4068 Instruction *I, ElementCount VF, InstWidening Kind) {
4069 assert((Kind == CM_Widen || Kind == CM_Widen_Reverse) &&
4070 "Expected a consecutive widening decision");
4071 Type *ValTy = getLoadStoreType(I);
4072 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4073 unsigned AS = getLoadStoreAddressSpace(I);
4074
4075 const Align Alignment = getLoadStoreAlignment(I);
4076 InstructionCost Cost = 0;
4077 if (isMaskRequired(I)) {
4078 unsigned IID = I->getOpcode() == Instruction::Load
4079 ? Intrinsic::masked_load
4080 : Intrinsic::masked_store;
4081 Cost += TTI.getMemIntrinsicInstrCost(
4082 MICA: MemIntrinsicCostAttributes(IID, VectorTy, Alignment, AS),
4083 CostKind: Config.CostKind);
4084 } else {
4085 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4086 Cost += TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: VectorTy, Alignment, AddressSpace: AS,
4087 CostKind: Config.CostKind, OpdInfo: OpInfo, I);
4088 }
4089
4090 if (Kind == CM_Widen_Reverse)
4091 Cost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Reverse, DstTy: VectorTy,
4092 SrcTy: VectorTy, CostKind: Config.CostKind, Mask: {}, Index: 0);
4093 return Cost;
4094}
4095
4096InstructionCost
4097LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
4098 ElementCount VF) const {
4099 assert(isUniformMemOp(*I, VF));
4100
4101 Type *ValTy = getLoadStoreType(I);
4102 Type *PtrTy = getLoadStorePointerOperand(V: I)->getType();
4103 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4104 const Align Alignment = getLoadStoreAlignment(I);
4105 unsigned AS = getLoadStoreAddressSpace(I);
4106 if (isa<LoadInst>(Val: I)) {
4107 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4108 CostKind: Config.CostKind) +
4109 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: ValTy, Alignment, AddressSpace: AS,
4110 CostKind: Config.CostKind) +
4111 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Broadcast, DstTy: VectorTy,
4112 SrcTy: VectorTy, CostKind: Config.CostKind);
4113 }
4114 StoreInst *SI = cast<StoreInst>(Val: I);
4115
4116 bool IsLoopInvariantStoreValue = Legal->isInvariant(V: SI->getValueOperand());
4117 // TODO: We have existing tests that request the cost of extracting element
4118 // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent
4119 // the actual generated code, which involves extracting the last element of
4120 // a scalable vector where the lane to extract is unknown at compile time.
4121 InstructionCost Cost =
4122 TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr, CostKind: Config.CostKind) +
4123 TTI.getMemoryOpCost(Opcode: Instruction::Store, Src: ValTy, Alignment, AddressSpace: AS,
4124 CostKind: Config.CostKind);
4125 if (!IsLoopInvariantStoreValue)
4126 Cost += TTI.getIndexedVectorInstrCostFromEnd(Opcode: Instruction::ExtractElement,
4127 Val: VectorTy, CostKind: Config.CostKind, Index: 0);
4128 return Cost;
4129}
4130
4131InstructionCost
4132LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
4133 ElementCount VF) const {
4134 Type *ValTy = getLoadStoreType(I);
4135 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4136 const Align Alignment = getLoadStoreAlignment(I);
4137 Value *Ptr = getLoadStorePointerOperand(V: I);
4138 Type *PtrTy = Ptr->getType();
4139
4140 if (!isUniform(V: Ptr, VF))
4141 PtrTy = toVectorTy(Scalar: PtrTy, EC: VF);
4142
4143 unsigned IID = I->getOpcode() == Instruction::Load
4144 ? Intrinsic::masked_gather
4145 : Intrinsic::masked_scatter;
4146 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4147 CostKind: Config.CostKind) +
4148 TTI.getMemIntrinsicInstrCost(
4149 MICA: MemIntrinsicCostAttributes(IID, VectorTy, Ptr, isMaskRequired(I),
4150 Alignment, I),
4151 CostKind: Config.CostKind);
4152}
4153
4154InstructionCost
4155LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
4156 ElementCount VF) const {
4157 const auto *Group = getInterleavedAccessGroup(Instr: I);
4158 assert(Group && "Fail to get an interleaved access group.");
4159
4160 Instruction *InsertPos = Group->getInsertPos();
4161 Type *ValTy = getLoadStoreType(I: InsertPos);
4162 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4163 unsigned AS = getLoadStoreAddressSpace(I: InsertPos);
4164
4165 unsigned InterleaveFactor = Group->getFactor();
4166 auto *WideVecTy = VectorType::get(ElementType: ValTy, EC: VF * InterleaveFactor);
4167
4168 // Holds the indices of existing members in the interleaved group.
4169 SmallVector<unsigned, 4> Indices;
4170 for (unsigned IF = 0; IF < InterleaveFactor; IF++)
4171 if (Group->getMember(Index: IF))
4172 Indices.push_back(Elt: IF);
4173
4174 // Calculate the cost of the whole interleaved group.
4175 bool UseMaskForGaps =
4176 (Group->requiresScalarEpilogue() && !isEpilogueAllowed()) ||
4177 (isa<StoreInst>(Val: I) && !Group->isFull());
4178 InstructionCost Cost = TTI.getInterleavedMemoryOpCost(
4179 Opcode: InsertPos->getOpcode(), VecTy: WideVecTy, Factor: Group->getFactor(), Indices,
4180 Alignment: Group->getAlign(), AddressSpace: AS, CostKind: Config.CostKind, UseMaskForCond: isMaskRequired(I),
4181 UseMaskForGaps);
4182
4183 if (Group->isReverse()) {
4184 // TODO: Add support for reversed masked interleaved access.
4185 assert(!isMaskRequired(I) &&
4186 "Reverse masked interleaved access not supported.");
4187 Cost += Group->getNumMembers() *
4188 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Reverse, DstTy: VectorTy,
4189 SrcTy: VectorTy, CostKind: Config.CostKind, Mask: {}, Index: 0);
4190 }
4191 return Cost;
4192}
4193
4194InstructionCost
4195LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
4196 ElementCount VF) {
4197 // Calculate scalar cost only. Vectorization cost should be ready at this
4198 // moment.
4199 if (VF.isScalar()) {
4200 Type *ValTy = getLoadStoreType(I);
4201 Type *PtrTy = getLoadStorePointerOperand(V: I)->getType();
4202 const Align Alignment = getLoadStoreAlignment(I);
4203 unsigned AS = getLoadStoreAddressSpace(I);
4204
4205 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4206 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4207 CostKind: Config.CostKind) +
4208 TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: ValTy, Alignment, AddressSpace: AS,
4209 CostKind: Config.CostKind, OpdInfo: OpInfo, I);
4210 }
4211 return getWideningCost(I, VF);
4212}
4213
4214InstructionCost
4215LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
4216 ElementCount VF) const {
4217
4218 // There is no mechanism yet to create a scalable scalarization loop,
4219 // so this is currently Invalid.
4220 if (VF.isScalable())
4221 return InstructionCost::getInvalid();
4222
4223 if (VF.isScalar())
4224 return 0;
4225
4226 InstructionCost Cost = 0;
4227 Type *RetTy = toVectorizedTy(Ty: I->getType(), EC: VF);
4228 if (!RetTy->isVoidTy() &&
4229 (!isa<LoadInst>(Val: I) || !TTI.supportsEfficientVectorElementLoadStore())) {
4230
4231 TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None;
4232 if (isa<LoadInst>(Val: I))
4233 VIC = TTI::VectorInstrContext::Load;
4234 else if (isa<StoreInst>(Val: I))
4235 VIC = TTI::VectorInstrContext::Store;
4236
4237 for (Type *VectorTy : getContainedTypes(Ty: RetTy)) {
4238 Cost += TTI.getScalarizationOverhead(
4239 Ty: cast<VectorType>(Val: VectorTy), DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4240 /*Insert=*/true, /*Extract=*/false, CostKind: Config.CostKind,
4241 /*ForPoisonSrc=*/true, VL: {}, VIC);
4242 }
4243 }
4244
4245 // Some targets keep addresses scalar.
4246 if (isa<LoadInst>(Val: I) && !TTI.prefersVectorizedAddressing())
4247 return Cost;
4248
4249 // Some targets support efficient element stores.
4250 if (isa<StoreInst>(Val: I) && TTI.supportsEfficientVectorElementLoadStore())
4251 return Cost;
4252
4253 // Collect operands to consider.
4254 CallInst *CI = dyn_cast<CallInst>(Val: I);
4255 Instruction::op_range Ops = CI ? CI->args() : I->operands();
4256
4257 // Skip operands that do not require extraction/scalarization and do not incur
4258 // any overhead.
4259 SmallVector<Type *> Tys;
4260 for (auto *V : filterExtractingOperands(Ops, VF))
4261 Tys.push_back(Elt: maybeVectorizeType(Ty: V->getType(), VF));
4262
4263 TTI::VectorInstrContext OperandVIC = isa<StoreInst>(Val: I)
4264 ? TTI::VectorInstrContext::Store
4265 : TTI::VectorInstrContext::None;
4266 return Cost +
4267 TTI.getOperandsScalarizationOverhead(Tys, CostKind: Config.CostKind, VIC: OperandVIC);
4268}
4269
4270void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
4271 if (VF.isScalar())
4272 return;
4273
4274 // TODO: We should generate better code and update the cost model for
4275 // predicated uniform stores. Today they are treated as any other
4276 // predicated store (see added test cases in
4277 // invariant-store-vectorization.ll).
4278 NumPredStores = 0;
4279 for (BasicBlock *BB : TheLoop->blocks())
4280 for (Instruction &I : *BB)
4281 if (isa<StoreInst>(Val: &I) && isScalarWithPredication(I: &I, VF))
4282 ++NumPredStores;
4283
4284 for (BasicBlock *BB : TheLoop->blocks()) {
4285 // For each instruction in the old loop.
4286 for (Instruction &I : *BB) {
4287 Value *Ptr = getLoadStorePointerOperand(V: &I);
4288
4289 // Ignore if no pointer found or a widening decision has already been
4290 // made.
4291 if (!Ptr || (getWideningDecision(I: &I, VF) != CM_Unknown))
4292 continue;
4293
4294 LLVM_DEBUG(dbgs() << "LV: Memory widening: calculating best strategy for "
4295 << I << '\n');
4296 if (isUniformMemOp(I, VF)) {
4297 auto IsLegalToScalarize = [&]() {
4298 if (!VF.isScalable())
4299 // Scalarization of fixed length vectors "just works".
4300 return true;
4301
4302 // We have dedicated lowering for unpredicated uniform loads and
4303 // stores. Note that even with tail folding we know that at least
4304 // one lane is active (i.e. generalized predication is not possible
4305 // here), and the logic below depends on this fact.
4306 if (!foldTailByMasking())
4307 return true;
4308
4309 // For scalable vectors, a uniform memop load is always
4310 // uniform-by-parts and we know how to scalarize that.
4311 if (isa<LoadInst>(Val: I))
4312 return true;
4313
4314 // A uniform store isn't neccessarily uniform-by-part
4315 // and we can't assume scalarization.
4316 auto &SI = cast<StoreInst>(Val&: I);
4317 return TheLoop->isLoopInvariant(V: SI.getValueOperand());
4318 };
4319
4320 const InstructionCost GatherScatterCost =
4321 isLegalGatherOrScatter(I: &I, VF) ? getGatherScatterCost(I: &I, VF)
4322 : InstructionCost::getInvalid();
4323
4324 // Load: Scalar load + broadcast
4325 // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
4326 // FIXME: This cost is a significant under-estimate for tail folded
4327 // memory ops.
4328 const InstructionCost ScalarizationCost =
4329 IsLegalToScalarize() ? getUniformMemOpCost(I: &I, VF)
4330 : InstructionCost::getInvalid();
4331
4332 // Choose better solution for the current VF, Note that Invalid
4333 // costs compare as maximumal large. If both are invalid, we get
4334 // scalable invalid which signals a failure and a vectorization abort.
4335 LLVM_DEBUG(dbgs() << "LV: Memory widening: uniform memory op has "
4336 "GatherScatterCost = "
4337 << GatherScatterCost << ", ScalarizationCost = "
4338 << ScalarizationCost << '\n');
4339 if (GatherScatterCost < ScalarizationCost)
4340 setWideningDecision(I: &I, VF, W: CM_GatherScatter, Cost: GatherScatterCost);
4341 else
4342 setWideningDecision(I: &I, VF, W: CM_Scalarize, Cost: ScalarizationCost);
4343 continue;
4344 }
4345
4346 // We assume that widening is the best solution when possible.
4347 if (std::optional<InstWidening> Decision =
4348 memoryInstructionCanBeWidened(I: &I, VF)) {
4349 InstructionCost WidenCost = getConsecutiveMemOpCost(I: &I, VF, Kind: *Decision);
4350 LLVM_DEBUG(
4351 dbgs() << "LV: Memory widening: can be widened normally with cost "
4352 << WidenCost << '\n');
4353 setWideningDecision(I: &I, VF, W: *Decision, Cost: WidenCost);
4354 continue;
4355 }
4356
4357 // Choose between Interleaving, Gather/Scatter or Scalarization.
4358 InstructionCost InterleaveCost = InstructionCost::getInvalid();
4359 unsigned NumAccesses = 1;
4360 if (isAccessInterleaved(Instr: &I)) {
4361 const auto *Group = getInterleavedAccessGroup(Instr: &I);
4362 assert(Group && "Fail to get an interleaved access group.");
4363
4364 NumAccesses = Group->getNumMembers();
4365 if (interleavedAccessCanBeWidened(I: &I, VF))
4366 InterleaveCost = getInterleaveGroupCost(I: &I, VF);
4367 }
4368
4369 InstructionCost GatherScatterCost =
4370 isLegalGatherOrScatter(I: &I, VF)
4371 ? getGatherScatterCost(I: &I, VF) * NumAccesses
4372 : InstructionCost::getInvalid();
4373
4374 InstructionCost ScalarizationCost =
4375 getMemInstScalarizationCost(I: &I, VF) * NumAccesses;
4376
4377 // Choose better solution for the current VF,
4378 // write down this decision and use it during vectorization.
4379 InstructionCost Cost;
4380 InstWidening Decision;
4381 if (InterleaveCost <= GatherScatterCost &&
4382 InterleaveCost < ScalarizationCost) {
4383 Decision = CM_Interleave;
4384 Cost = InterleaveCost;
4385 } else if (GatherScatterCost < ScalarizationCost) {
4386 Decision = CM_GatherScatter;
4387 Cost = GatherScatterCost;
4388 } else {
4389 Decision = CM_Scalarize;
4390 Cost = ScalarizationCost;
4391 }
4392 LLVM_DEBUG(
4393 dbgs() << "LV: Memory widening: InterleaveCost = " << InterleaveCost
4394 << ", GatherScatterCost = " << GatherScatterCost
4395 << ", ScalarizationCost = " << ScalarizationCost << '\n');
4396
4397 // If the instructions belongs to an interleave group, the whole group
4398 // receives the same decision. The whole group receives the cost, but
4399 // the cost will actually be assigned to one instruction.
4400 if (const auto *Group = getInterleavedAccessGroup(Instr: &I)) {
4401 if (Decision == CM_Scalarize) {
4402 for (Instruction *I : Group->members())
4403 setWideningDecision(I, VF, W: Decision,
4404 Cost: getMemInstScalarizationCost(I, VF));
4405 } else {
4406 setWideningDecision(Grp: Group, VF, W: Decision, Cost);
4407 }
4408 } else
4409 setWideningDecision(I: &I, VF, W: Decision, Cost);
4410 }
4411 }
4412
4413 // Make sure that any load of address and any other address computation
4414 // remains scalar unless there is gather/scatter support. This avoids
4415 // inevitable extracts into address registers, and also has the benefit of
4416 // activating LSR more, since that pass can't optimize vectorized
4417 // addresses.
4418 if (TTI.prefersVectorizedAddressing())
4419 return;
4420
4421 // Start with all scalar pointer uses.
4422 SmallSetVector<Instruction *, 8> AddrDefs;
4423 for (BasicBlock *BB : TheLoop->blocks())
4424 for (Instruction &I : *BB) {
4425 Instruction *PtrDef =
4426 dyn_cast_or_null<Instruction>(Val: getLoadStorePointerOperand(V: &I));
4427 if (PtrDef && TheLoop->contains(Inst: PtrDef) &&
4428 getWideningDecision(I: &I, VF) != CM_GatherScatter)
4429 AddrDefs.insert(X: PtrDef);
4430 }
4431
4432 // Add all instructions used to generate the addresses.
4433 SmallVector<Instruction *, 4> Worklist;
4434 append_range(C&: Worklist, R&: AddrDefs);
4435 while (!Worklist.empty()) {
4436 Instruction *I = Worklist.pop_back_val();
4437 for (auto &Op : I->operands())
4438 if (auto *InstOp = dyn_cast<Instruction>(Val&: Op))
4439 if (TheLoop->contains(Inst: InstOp) && !isa<PHINode>(Val: InstOp) &&
4440 AddrDefs.insert(X: InstOp))
4441 Worklist.push_back(Elt: InstOp);
4442 }
4443
4444 auto UpdateMemOpUserCost = [this, VF](LoadInst *LI) {
4445 // If there are direct memory op users of the newly scalarized load,
4446 // their cost may have changed because there's no scalarization
4447 // overhead for the operand. Update it.
4448 for (User *U : LI->users()) {
4449 if (!isa<LoadInst, StoreInst>(Val: U))
4450 continue;
4451 if (getWideningDecision(I: cast<Instruction>(Val: U), VF) != CM_Scalarize)
4452 continue;
4453 auto UI = cast<Instruction>(Val: U);
4454 LLVM_DEBUG(
4455 dbgs() << "LV: Memory widening: updating decision for load user "
4456 << *UI << '\n');
4457 setWideningDecision(
4458 I: UI, VF, W: CM_Scalarize,
4459 Cost: getMemInstScalarizationCost(I: cast<Instruction>(Val: U), VF));
4460 }
4461 };
4462 for (auto *I : AddrDefs) {
4463 if (isa<LoadInst>(Val: I)) {
4464 // Setting the desired widening decision should ideally be handled in
4465 // by cost functions, but since this involves the task of finding out
4466 // if the loaded register is involved in an address computation, it is
4467 // instead changed here when we know this is the case.
4468 InstWidening Decision = getWideningDecision(I, VF);
4469 if (!isPredicatedInst(I) &&
4470 (Decision == CM_Widen || Decision == CM_Widen_Reverse ||
4471 (!isUniformMemOp(I&: *I, VF) && Decision == CM_Scalarize))) {
4472 // Scalarize a widened load of address or update the cost of a scalar
4473 // load of an address.
4474 LLVM_DEBUG(dbgs() << "LV: Memory widening: updating decision for load "
4475 << *I << '\n');
4476 setWideningDecision(
4477 I, VF, W: CM_Scalarize,
4478 Cost: (VF.getKnownMinValue() *
4479 getMemoryInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1))));
4480 UpdateMemOpUserCost(cast<LoadInst>(Val: I));
4481 } else if (const auto *Group = getInterleavedAccessGroup(Instr: I)) {
4482 // Scalarize all members of this interleaved group when any member
4483 // is used as an address. The address-used load skips scalarization
4484 // overhead, other members include it.
4485 for (Instruction *Member : Group->members()) {
4486 InstructionCost Cost = AddrDefs.contains(key: Member)
4487 ? (VF.getKnownMinValue() *
4488 getMemoryInstructionCost(
4489 I: Member, VF: ElementCount::getFixed(MinVal: 1)))
4490 : getMemInstScalarizationCost(I: Member, VF);
4491 LLVM_DEBUG(
4492 dbgs()
4493 << "LV: Memory widening: updating decision for interleave member "
4494 << *Member << '\n');
4495 setWideningDecision(I: Member, VF, W: CM_Scalarize, Cost);
4496 UpdateMemOpUserCost(cast<LoadInst>(Val: Member));
4497 }
4498 }
4499 } else {
4500 // Cannot scalarize fixed-order recurrence phis at the moment.
4501 if (isa<PHINode>(Val: I) && Legal->isFixedOrderRecurrence(Phi: cast<PHINode>(Val: I)))
4502 continue;
4503
4504 // Make sure I gets scalarized and a cost estimate without
4505 // scalarization overhead.
4506 ForcedScalars[VF].insert(X: I);
4507 }
4508 }
4509}
4510
4511bool LoopVectorizationCostModel::shouldConsiderInvariant(Value *Op) {
4512 if (!Legal->isInvariant(V: Op))
4513 return false;
4514 // Consider Op invariant, if it or its operands aren't predicated
4515 // instruction in the loop. In that case, it is not trivially hoistable.
4516 auto *OpI = dyn_cast<Instruction>(Val: Op);
4517 return !OpI || !TheLoop->contains(Inst: OpI) ||
4518 (!isPredicatedInst(I: OpI) &&
4519 (!isa<PHINode>(Val: OpI) || OpI->getParent() != TheLoop->getHeader()) &&
4520 all_of(Range: OpI->operands(),
4521 P: [this](Value *Op) { return shouldConsiderInvariant(Op); }));
4522}
4523
4524InstructionCost
4525LoopVectorizationCostModel::getInstructionCost(Instruction *I,
4526 ElementCount VF) {
4527 // If we know that this instruction will remain uniform, check the cost of
4528 // the scalar version.
4529 if (isUniformAfterVectorization(I, VF))
4530 VF = ElementCount::getFixed(MinVal: 1);
4531
4532 if (VF.isVector() && isProfitableToScalarize(I, VF))
4533 return InstsToScalarize[VF][I];
4534
4535 // Forced scalars do not have any scalarization overhead.
4536 auto ForcedScalar = ForcedScalars.find(Val: VF);
4537 if (VF.isVector() && ForcedScalar != ForcedScalars.end()) {
4538 auto InstSet = ForcedScalar->second;
4539 if (InstSet.count(key: I))
4540 return getInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1)) *
4541 VF.getKnownMinValue();
4542 }
4543
4544 const auto &MinBWs = Config.getMinimalBitwidths();
4545 uint64_t InstrMinBWs = MinBWs.lookup(Key: I);
4546 Type *RetTy = I->getType();
4547 if (canTruncateToMinimalBitwidth(I, VF))
4548 RetTy = IntegerType::get(C&: RetTy->getContext(), NumBits: InstrMinBWs);
4549 auto *SE = PSE.getSE();
4550
4551 Type *VectorTy;
4552 if (isScalarAfterVectorization(I, VF)) {
4553 [[maybe_unused]] auto HasSingleCopyAfterVectorization =
4554 [this](Instruction *I, ElementCount VF) -> bool {
4555 if (VF.isScalar())
4556 return true;
4557
4558 auto Scalarized = InstsToScalarize.find(Key: VF);
4559 assert(Scalarized != InstsToScalarize.end() &&
4560 "VF not yet analyzed for scalarization profitability");
4561 return !Scalarized->second.count(Key: I) &&
4562 llvm::all_of(Range: I->users(), P: [&](User *U) {
4563 auto *UI = cast<Instruction>(Val: U);
4564 return !Scalarized->second.count(Key: UI);
4565 });
4566 };
4567
4568 // With the exception of GEPs and PHIs, after scalarization there should
4569 // only be one copy of the instruction generated in the loop. This is
4570 // because the VF is either 1, or any instructions that need scalarizing
4571 // have already been dealt with by the time we get here. As a result,
4572 // it means we don't have to multiply the instruction cost by VF.
4573 assert(I->getOpcode() == Instruction::GetElementPtr ||
4574 I->getOpcode() == Instruction::PHI ||
4575 (I->getOpcode() == Instruction::BitCast &&
4576 I->getType()->isPointerTy()) ||
4577 HasSingleCopyAfterVectorization(I, VF));
4578 VectorTy = RetTy;
4579 } else
4580 VectorTy = toVectorizedTy(Ty: RetTy, EC: VF);
4581
4582 if (VF.isVector() && VectorTy->isVectorTy() &&
4583 !TTI.getNumberOfParts(Tp: VectorTy))
4584 return InstructionCost::getInvalid();
4585
4586 // TODO: We need to estimate the cost of intrinsic calls.
4587 switch (I->getOpcode()) {
4588 case Instruction::GetElementPtr:
4589 // We mark this instruction as zero-cost because the cost of GEPs in
4590 // vectorized code depends on whether the corresponding memory instruction
4591 // is scalarized or not. Therefore, we handle GEPs with the memory
4592 // instruction cost.
4593 return 0;
4594 case Instruction::UncondBr:
4595 case Instruction::CondBr: {
4596 // In cases of scalarized and predicated instructions, there will be VF
4597 // predicated blocks in the vectorized loop. Each branch around these
4598 // blocks requires also an extract of its vector compare i1 element.
4599 // Note that the conditional branch from the loop latch will be replaced by
4600 // a single branch controlling the loop, so there is no extra overhead from
4601 // scalarization.
4602 bool ScalarPredicatedBB = false;
4603 CondBrInst *BI = dyn_cast<CondBrInst>(Val: I);
4604 if (VF.isVector() && BI &&
4605 (PredicatedBBsAfterVectorization[VF].count(Ptr: BI->getSuccessor(i: 0)) ||
4606 PredicatedBBsAfterVectorization[VF].count(Ptr: BI->getSuccessor(i: 1))) &&
4607 BI->getParent() != TheLoop->getLoopLatch())
4608 ScalarPredicatedBB = true;
4609
4610 if (ScalarPredicatedBB) {
4611 // Not possible to scalarize scalable vector with predicated instructions.
4612 if (VF.isScalable())
4613 return InstructionCost::getInvalid();
4614 // Return cost for branches around scalarized and predicated blocks.
4615 auto *VecI1Ty =
4616 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: RetTy->getContext()), EC: VF);
4617 return (TTI.getScalarizationOverhead(
4618 Ty: VecI1Ty, DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4619 /*Insert*/ false, /*Extract*/ true, CostKind: Config.CostKind) +
4620 (TTI.getCFInstrCost(Opcode: Instruction::CondBr, CostKind: Config.CostKind) *
4621 VF.getFixedValue()));
4622 }
4623
4624 if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar())
4625 // The back-edge branch will remain, as will all scalar branches.
4626 return TTI.getCFInstrCost(Opcode: Instruction::UncondBr, CostKind: Config.CostKind);
4627
4628 // This branch will be eliminated by if-conversion.
4629 return 0;
4630 // Note: We currently assume zero cost for an unconditional branch inside
4631 // a predicated block since it will become a fall-through, although we
4632 // may decide in the future to call TTI for all branches.
4633 }
4634 case Instruction::Switch: {
4635 if (VF.isScalar())
4636 return TTI.getCFInstrCost(Opcode: Instruction::Switch, CostKind: Config.CostKind);
4637 auto *Switch = cast<SwitchInst>(Val: I);
4638 return Switch->getNumCases() *
4639 TTI.getCmpSelInstrCost(
4640 Opcode: Instruction::ICmp,
4641 ValTy: toVectorTy(Scalar: Switch->getCondition()->getType(), EC: VF),
4642 CondTy: toVectorTy(Scalar: Type::getInt1Ty(C&: I->getContext()), EC: VF),
4643 VecPred: CmpInst::ICMP_EQ, CostKind: Config.CostKind);
4644 }
4645 case Instruction::PHI: {
4646 auto *Phi = cast<PHINode>(Val: I);
4647
4648 // First-order recurrences are replaced by vector shuffles inside the loop.
4649 if (VF.isVector() && Legal->isFixedOrderRecurrence(Phi)) {
4650 return TTI.getShuffleCost(
4651 Kind: TargetTransformInfo::SK_Splice, DstTy: cast<VectorType>(Val: VectorTy),
4652 SrcTy: cast<VectorType>(Val: VectorTy), CostKind: Config.CostKind, Mask: {}, Index: -1);
4653 }
4654
4655 // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
4656 // converted into select instructions. We require N - 1 selects per phi
4657 // node, where N is the number of incoming values.
4658 if (VF.isVector() && Phi->getParent() != TheLoop->getHeader()) {
4659 Type *ResultTy = Phi->getType();
4660
4661 // All instructions in an Any-of reduction chain are narrowed to bool.
4662 // Check if that is the case for this phi node.
4663 auto *HeaderUser = cast_if_present<PHINode>(
4664 Val: find_singleton<User>(Range: Phi->users(), P: [this](User *U, bool) -> User * {
4665 auto *Phi = dyn_cast<PHINode>(Val: U);
4666 if (Phi && Phi->getParent() == TheLoop->getHeader())
4667 return Phi;
4668 return nullptr;
4669 }));
4670 if (HeaderUser) {
4671 auto &ReductionVars = Legal->getReductionVars();
4672 auto Iter = ReductionVars.find(Key: HeaderUser);
4673 if (Iter != ReductionVars.end() &&
4674 RecurrenceDescriptor::isAnyOfRecurrenceKind(
4675 Kind: Iter->second.getRecurrenceKind()))
4676 ResultTy = Type::getInt1Ty(C&: Phi->getContext());
4677 }
4678 return (Phi->getNumIncomingValues() - 1) *
4679 TTI.getCmpSelInstrCost(
4680 Opcode: Instruction::Select, ValTy: toVectorTy(Scalar: ResultTy, EC: VF),
4681 CondTy: toVectorTy(Scalar: Type::getInt1Ty(C&: Phi->getContext()), EC: VF),
4682 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind: Config.CostKind);
4683 }
4684
4685 // When tail folding with EVL, if the phi is part of an out of loop
4686 // reduction then it will be transformed into a wide vp_merge.
4687 if (VF.isVector() && foldTailWithEVL() &&
4688 Legal->getReductionVars().contains(Key: Phi) &&
4689 !Config.isInLoopReduction(Phi)) {
4690 IntrinsicCostAttributes ICA(
4691 Intrinsic::vp_merge, toVectorTy(Scalar: Phi->getType(), EC: VF),
4692 {toVectorTy(Scalar: Type::getInt1Ty(C&: Phi->getContext()), EC: VF)});
4693 return TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind);
4694 }
4695
4696 return TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
4697 }
4698 case Instruction::UDiv:
4699 case Instruction::SDiv:
4700 case Instruction::URem:
4701 case Instruction::SRem:
4702 if (VF.isVector() && isPredicatedInst(I)) {
4703 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
4704 return isDivRemScalarWithPredication(ScalarCost, MaskedCost) ? ScalarCost
4705 : MaskedCost;
4706 }
4707 // We've proven all lanes safe to speculate, fall through.
4708 [[fallthrough]];
4709 case Instruction::Add:
4710 case Instruction::Sub: {
4711 auto Info = Legal->getHistogramInfo(I);
4712 if (Info && VF.isVector()) {
4713 const HistogramInfo *HGram = Info.value();
4714 // Assume that a non-constant update value (or a constant != 1) requires
4715 // a multiply, and add that into the cost.
4716 InstructionCost MulCost = TTI::TCC_Free;
4717 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: I->getOperand(i: 1));
4718 if (!RHS || RHS->getZExtValue() != 1)
4719 MulCost = TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: VectorTy,
4720 CostKind: Config.CostKind);
4721
4722 // Find the cost of the histogram operation itself.
4723 Type *PtrTy = VectorType::get(ElementType: HGram->Load->getPointerOperandType(), EC: VF);
4724 Type *ScalarTy = I->getType();
4725 Type *MaskTy = VectorType::get(ElementType: Type::getInt1Ty(C&: I->getContext()), EC: VF);
4726 IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
4727 Type::getVoidTy(C&: I->getContext()),
4728 {PtrTy, ScalarTy, MaskTy});
4729
4730 // Add the costs together with the add/sub operation.
4731 return TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind) + MulCost +
4732 TTI.getArithmeticInstrCost(Opcode: I->getOpcode(), Ty: VectorTy,
4733 CostKind: Config.CostKind);
4734 }
4735 [[fallthrough]];
4736 }
4737 case Instruction::FAdd:
4738 case Instruction::FSub:
4739 case Instruction::Mul:
4740 case Instruction::FMul:
4741 case Instruction::FDiv:
4742 case Instruction::FRem:
4743 case Instruction::Shl:
4744 case Instruction::LShr:
4745 case Instruction::AShr:
4746 case Instruction::And:
4747 case Instruction::Or:
4748 case Instruction::Xor: {
4749 // If we're speculating on the stride being 1, the multiplication may
4750 // fold away. We can generalize this for all operations using the notion
4751 // of neutral elements. (TODO)
4752 if (I->getOpcode() == Instruction::Mul &&
4753 ((TheLoop->isLoopInvariant(V: I->getOperand(i: 0)) &&
4754 PSE.getSCEV(V: I->getOperand(i: 0))->isOne()) ||
4755 (TheLoop->isLoopInvariant(V: I->getOperand(i: 1)) &&
4756 PSE.getSCEV(V: I->getOperand(i: 1))->isOne())))
4757 return 0;
4758
4759 // Certain instructions can be cheaper to vectorize if they have a constant
4760 // second vector operand. One example of this are shifts on x86.
4761 Value *Op2 = I->getOperand(i: 1);
4762 if (!isa<Constant>(Val: Op2) && TheLoop->isLoopInvariant(V: Op2) &&
4763 PSE.getSE()->isSCEVable(Ty: Op2->getType()) &&
4764 isa<SCEVConstant>(Val: PSE.getSCEV(V: Op2))) {
4765 Op2 = cast<SCEVConstant>(Val: PSE.getSCEV(V: Op2))->getValue();
4766 }
4767 auto Op2Info = TTI.getOperandInfo(V: Op2);
4768 if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue &&
4769 shouldConsiderInvariant(Op: Op2))
4770 Op2Info.Kind = TargetTransformInfo::OK_UniformValue;
4771
4772 SmallVector<const Value *, 4> Operands(I->operand_values());
4773 return TTI.getArithmeticInstrCost(
4774 Opcode: I->getOpcode(), Ty: VectorTy, CostKind: Config.CostKind,
4775 Opd1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
4776 Opd2Info: Op2Info, Args: Operands, CtxI: I, TLibInfo: TLI);
4777 }
4778 case Instruction::FNeg: {
4779 return TTI.getArithmeticInstrCost(
4780 Opcode: I->getOpcode(), Ty: VectorTy, CostKind: Config.CostKind,
4781 Opd1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
4782 Opd2Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
4783 Args: I->getOperand(i: 0), CtxI: I);
4784 }
4785 case Instruction::Select: {
4786 SelectInst *SI = cast<SelectInst>(Val: I);
4787 const SCEV *CondSCEV = SE->getSCEV(V: SI->getCondition());
4788 bool ScalarCond = (SE->isLoopInvariant(S: CondSCEV, L: TheLoop));
4789
4790 const Value *Op0, *Op1;
4791 using namespace llvm::PatternMatch;
4792 if (!ScalarCond && (match(V: I, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))) ||
4793 match(V: I, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))) {
4794 // select x, y, false --> x & y
4795 // select x, true, y --> x | y
4796 const auto [Op1VK, Op1VP] = TTI::getOperandInfo(V: Op0);
4797 const auto [Op2VK, Op2VP] = TTI::getOperandInfo(V: Op1);
4798 assert(Op0->getType()->getScalarSizeInBits() == 1 &&
4799 Op1->getType()->getScalarSizeInBits() == 1);
4800
4801 return TTI.getArithmeticInstrCost(
4802 Opcode: match(V: I, P: m_LogicalOr()) ? Instruction::Or : Instruction::And,
4803 Ty: VectorTy, CostKind: Config.CostKind, Opd1Info: {.Kind: Op1VK, .Properties: Op1VP}, Opd2Info: {.Kind: Op2VK, .Properties: Op2VP}, Args: {Op0, Op1},
4804 CtxI: I);
4805 }
4806
4807 Type *CondTy = SI->getCondition()->getType();
4808 if (!ScalarCond)
4809 CondTy = VectorType::get(ElementType: CondTy, EC: VF);
4810
4811 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
4812 if (auto *Cmp = dyn_cast<CmpInst>(Val: SI->getCondition()))
4813 Pred = Cmp->getPredicate();
4814 return TTI.getCmpSelInstrCost(
4815 Opcode: I->getOpcode(), ValTy: VectorTy, CondTy, VecPred: Pred, CostKind: Config.CostKind,
4816 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
4817 }
4818 case Instruction::ICmp:
4819 case Instruction::FCmp: {
4820 Type *ValTy = I->getOperand(i: 0)->getType();
4821
4822 if (canTruncateToMinimalBitwidth(I, VF)) {
4823 [[maybe_unused]] Instruction *Op0AsInstruction =
4824 dyn_cast<Instruction>(Val: I->getOperand(i: 0));
4825 assert((!canTruncateToMinimalBitwidth(Op0AsInstruction, VF) ||
4826 InstrMinBWs == MinBWs.lookup(Op0AsInstruction)) &&
4827 "if both the operand and the compare are marked for "
4828 "truncation, they must have the same bitwidth");
4829 ValTy = IntegerType::get(C&: ValTy->getContext(), NumBits: InstrMinBWs);
4830 }
4831
4832 VectorTy = toVectorTy(Scalar: ValTy, EC: VF);
4833 return TTI.getCmpSelInstrCost(
4834 Opcode: I->getOpcode(), ValTy: VectorTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VectorTy),
4835 VecPred: cast<CmpInst>(Val: I)->getPredicate(), CostKind: Config.CostKind,
4836 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
4837 }
4838 case Instruction::Store:
4839 case Instruction::Load: {
4840 ElementCount Width = VF;
4841 if (Width.isVector()) {
4842 InstWidening Decision = getWideningDecision(I, VF: Width);
4843 assert(Decision != CM_Unknown &&
4844 "CM decision should be taken at this point");
4845 if (getWideningCost(I, VF) == InstructionCost::getInvalid())
4846 return InstructionCost::getInvalid();
4847 if (Decision == CM_Scalarize)
4848 Width = ElementCount::getFixed(MinVal: 1);
4849 }
4850 VectorTy = toVectorTy(Scalar: getLoadStoreType(I), EC: Width);
4851 return getMemoryInstructionCost(I, VF);
4852 }
4853 case Instruction::BitCast:
4854 if (I->getType()->isPointerTy())
4855 return 0;
4856 [[fallthrough]];
4857 case Instruction::ZExt:
4858 case Instruction::SExt:
4859 case Instruction::FPToUI:
4860 case Instruction::FPToSI:
4861 case Instruction::FPExt:
4862 case Instruction::PtrToInt:
4863 case Instruction::IntToPtr:
4864 case Instruction::SIToFP:
4865 case Instruction::UIToFP:
4866 case Instruction::Trunc:
4867 case Instruction::FPTrunc: {
4868 // Computes the CastContextHint from a Load/Store instruction.
4869 auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
4870 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
4871 "Expected a load or a store!");
4872
4873 if (VF.isScalar() || !TheLoop->contains(Inst: I))
4874 return TTI::CastContextHint::Normal;
4875
4876 switch (getWideningDecision(I, VF)) {
4877 case LoopVectorizationCostModel::CM_GatherScatter:
4878 return TTI::CastContextHint::GatherScatter;
4879 case LoopVectorizationCostModel::CM_Interleave:
4880 return TTI::CastContextHint::Interleave;
4881 case LoopVectorizationCostModel::CM_Scalarize:
4882 case LoopVectorizationCostModel::CM_Widen:
4883 return isPredicatedInst(I) ? TTI::CastContextHint::Masked
4884 : TTI::CastContextHint::Normal;
4885 case LoopVectorizationCostModel::CM_Widen_Reverse:
4886 return TTI::CastContextHint::Reversed;
4887 case LoopVectorizationCostModel::CM_Unknown:
4888 llvm_unreachable("Instr did not go through cost modelling?");
4889 case LoopVectorizationCostModel::CM_InvalidatedDecision:
4890 return TTI::CastContextHint::None;
4891 }
4892
4893 llvm_unreachable("Unhandled case!");
4894 };
4895
4896 unsigned Opcode = I->getOpcode();
4897 TTI::CastContextHint CCH = TTI::CastContextHint::None;
4898 // For Trunc, the context is the only user, which must be a StoreInst.
4899 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
4900 if (I->hasOneUse())
4901 if (StoreInst *Store = dyn_cast<StoreInst>(Val: *I->user_begin()))
4902 CCH = ComputeCCH(Store);
4903 }
4904 // For Z/Sext, the context is the operand, which must be a LoadInst.
4905 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
4906 Opcode == Instruction::FPExt) {
4907 if (LoadInst *Load = dyn_cast<LoadInst>(Val: I->getOperand(i: 0)))
4908 CCH = ComputeCCH(Load);
4909 }
4910
4911 // We optimize the truncation of induction variables having constant
4912 // integer steps. The cost of these truncations is the same as the scalar
4913 // operation.
4914 if (isOptimizableIVTruncate(I, VF)) {
4915 auto *Trunc = cast<TruncInst>(Val: I);
4916 return TTI.getCastInstrCost(Opcode: Instruction::Trunc, Dst: Trunc->getDestTy(),
4917 Src: Trunc->getSrcTy(), CCH, CostKind: Config.CostKind,
4918 I: Trunc);
4919 }
4920
4921 Type *SrcScalarTy = I->getOperand(i: 0)->getType();
4922 Instruction *Op0AsInstruction = dyn_cast<Instruction>(Val: I->getOperand(i: 0));
4923 if (canTruncateToMinimalBitwidth(I: Op0AsInstruction, VF))
4924 SrcScalarTy = IntegerType::get(C&: SrcScalarTy->getContext(),
4925 NumBits: MinBWs.lookup(Key: Op0AsInstruction));
4926 Type *SrcVecTy =
4927 VectorTy->isVectorTy() ? toVectorTy(Scalar: SrcScalarTy, EC: VF) : SrcScalarTy;
4928
4929 if (canTruncateToMinimalBitwidth(I, VF)) {
4930 // If the result type is <= the source type, there will be no extend
4931 // after truncating the users to the minimal required bitwidth.
4932 if (VectorTy->getScalarSizeInBits() <= SrcVecTy->getScalarSizeInBits() &&
4933 (I->getOpcode() == Instruction::ZExt ||
4934 I->getOpcode() == Instruction::SExt))
4935 return 0;
4936 }
4937
4938 return TTI.getCastInstrCost(Opcode, Dst: VectorTy, Src: SrcVecTy, CCH,
4939 CostKind: Config.CostKind, I);
4940 }
4941 case Instruction::Call:
4942 return getVectorCallCost(CI: cast<CallInst>(Val: I), VF);
4943 case Instruction::ExtractValue:
4944 return TTI.getInstructionCost(U: I, CostKind: Config.CostKind);
4945 case Instruction::Alloca:
4946 // We cannot easily widen alloca to a scalable alloca, as
4947 // the result would need to be a vector of pointers.
4948 if (VF.isScalable())
4949 return InstructionCost::getInvalid();
4950 return TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: RetTy, CostKind: Config.CostKind);
4951 case Instruction::Freeze:
4952 return TTI::TCC_Free;
4953 default:
4954 // This opcode is unknown. Assume that it is the same as 'mul'.
4955 return TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: VectorTy,
4956 CostKind: Config.CostKind);
4957 } // end of switch.
4958}
4959
4960void LoopVectorizationCostModel::collectValuesToIgnore() {
4961 // Ignore ephemeral values.
4962 CodeMetrics::collectEphemeralValues(L: TheLoop, AC, EphValues&: ValuesToIgnore);
4963
4964 SmallVector<Value *, 4> DeadInterleavePointerOps;
4965 SmallVector<Value *, 4> DeadOps;
4966
4967 // If a scalar epilogue is required, users outside the loop won't use
4968 // live-outs from the vector loop but from the scalar epilogue. Ignore them if
4969 // that is the case.
4970 bool RequiresScalarEpilogue = requiresScalarEpilogue(IsVectorizing: true);
4971 auto IsLiveOutDead = [this, RequiresScalarEpilogue](User *U) {
4972 return RequiresScalarEpilogue &&
4973 !TheLoop->contains(BB: cast<Instruction>(Val: U)->getParent());
4974 };
4975
4976 LoopBlocksDFS DFS(TheLoop);
4977 DFS.perform(LI);
4978 for (BasicBlock *BB : reverse(C: make_range(x: DFS.beginRPO(), y: DFS.endRPO())))
4979 for (Instruction &I : reverse(C&: *BB)) {
4980 if (VecValuesToIgnore.contains(Ptr: &I) || ValuesToIgnore.contains(Ptr: &I))
4981 continue;
4982
4983 // Add instructions that would be trivially dead and are only used by
4984 // values already ignored to DeadOps to seed worklist.
4985 if (wouldInstructionBeTriviallyDead(I: &I, TLI) &&
4986 all_of(Range: I.users(), P: [this, IsLiveOutDead](User *U) {
4987 return VecValuesToIgnore.contains(Ptr: U) ||
4988 ValuesToIgnore.contains(Ptr: U) || IsLiveOutDead(U);
4989 }))
4990 DeadOps.push_back(Elt: &I);
4991
4992 // For interleave groups, we only create a pointer for the start of the
4993 // interleave group. Queue up addresses of group members except the insert
4994 // position for further processing.
4995 if (isAccessInterleaved(Instr: &I)) {
4996 auto *Group = getInterleavedAccessGroup(Instr: &I);
4997 if (Group->getInsertPos() == &I)
4998 continue;
4999 Value *PointerOp = getLoadStorePointerOperand(V: &I);
5000 DeadInterleavePointerOps.push_back(Elt: PointerOp);
5001 }
5002
5003 // Queue branches for analysis. They are dead, if their successors only
5004 // contain dead instructions.
5005 if (isa<CondBrInst>(Val: &I))
5006 DeadOps.push_back(Elt: &I);
5007 }
5008
5009 // Mark ops feeding interleave group members as free, if they are only used
5010 // by other dead computations.
5011 for (unsigned I = 0; I != DeadInterleavePointerOps.size(); ++I) {
5012 auto *Op = dyn_cast<Instruction>(Val: DeadInterleavePointerOps[I]);
5013 if (!Op || !TheLoop->contains(Inst: Op) || any_of(Range: Op->users(), P: [this](User *U) {
5014 Instruction *UI = cast<Instruction>(Val: U);
5015 return !VecValuesToIgnore.contains(Ptr: U) &&
5016 (!isAccessInterleaved(Instr: UI) ||
5017 getInterleavedAccessGroup(Instr: UI)->getInsertPos() == UI);
5018 }))
5019 continue;
5020 VecValuesToIgnore.insert(Ptr: Op);
5021 append_range(C&: DeadInterleavePointerOps, R: Op->operands());
5022 }
5023
5024 // Mark ops that would be trivially dead and are only used by ignored
5025 // instructions as free.
5026 BasicBlock *Header = TheLoop->getHeader();
5027
5028 // Returns true if the block contains only dead instructions. Such blocks will
5029 // be removed by VPlan-to-VPlan transforms and won't be considered by the
5030 // VPlan-based cost model, so skip them in the legacy cost-model as well.
5031 auto IsEmptyBlock = [this](BasicBlock *BB) {
5032 return all_of(Range&: *BB, P: [this](Instruction &I) {
5033 return ValuesToIgnore.contains(Ptr: &I) || VecValuesToIgnore.contains(Ptr: &I) ||
5034 isa<UncondBrInst>(Val: &I);
5035 });
5036 };
5037 SmallPtrSet<Instruction *, 16> ProcessedDeadOps;
5038 for (unsigned I = 0; I != DeadOps.size(); ++I) {
5039 auto *Op = dyn_cast<Instruction>(Val: DeadOps[I]);
5040
5041 // Check if the branch should be considered dead.
5042 if (auto *Br = dyn_cast_or_null<CondBrInst>(Val: Op)) {
5043 BasicBlock *ThenBB = Br->getSuccessor(i: 0);
5044 BasicBlock *ElseBB = Br->getSuccessor(i: 1);
5045 // Don't considers branches leaving the loop for simplification.
5046 if (!TheLoop->contains(BB: ThenBB) || !TheLoop->contains(BB: ElseBB))
5047 continue;
5048 bool ThenEmpty = IsEmptyBlock(ThenBB);
5049 bool ElseEmpty = IsEmptyBlock(ElseBB);
5050 if ((ThenEmpty && ElseEmpty) ||
5051 (ThenEmpty && ThenBB->getSingleSuccessor() == ElseBB &&
5052 ElseBB->phis().empty()) ||
5053 (ElseEmpty && ElseBB->getSingleSuccessor() == ThenBB &&
5054 ThenBB->phis().empty())) {
5055 VecValuesToIgnore.insert(Ptr: Br);
5056 DeadOps.push_back(Elt: Br->getCondition());
5057 }
5058 continue;
5059 }
5060
5061 // Skip any op that shouldn't be considered dead.
5062 if (!Op || !TheLoop->contains(Inst: Op) ||
5063 (isa<PHINode>(Val: Op) && Op->getParent() == Header) ||
5064 !wouldInstructionBeTriviallyDead(I: Op, TLI) ||
5065 any_of(Range: Op->users(), P: [this, IsLiveOutDead](User *U) {
5066 return !VecValuesToIgnore.contains(Ptr: U) &&
5067 !ValuesToIgnore.contains(Ptr: U) && !IsLiveOutDead(U);
5068 }))
5069 continue;
5070
5071 // If all of Op's users are in ValuesToIgnore, add it to ValuesToIgnore
5072 // which applies for both scalar and vector versions. Otherwise it is only
5073 // dead in vector versions, so only add it to VecValuesToIgnore.
5074 bool BecameScalarDead = false;
5075 if (all_of(Range: Op->users(),
5076 P: [this](User *U) { return ValuesToIgnore.contains(Ptr: U); }))
5077 BecameScalarDead = ValuesToIgnore.insert(Ptr: Op).second;
5078
5079 VecValuesToIgnore.insert(Ptr: Op);
5080 // Shared operands may be queued more than once. Propagate deadness once,
5081 // and again if an instruction previously dead only in the vector loop
5082 // becomes dead in the scalar loop too.
5083 if (ProcessedDeadOps.insert(Ptr: Op).second || BecameScalarDead)
5084 append_range(C&: DeadOps, R: Op->operands());
5085 }
5086
5087 // Ignore type-promoting instructions we identified during reduction
5088 // detection.
5089 for (const RecurrenceDescriptor &RedDes :
5090 Legal->getReductionVars().values()) {
5091 VecValuesToIgnore.insert_range(R: RedDes.getCastInsts());
5092 }
5093 // Ignore type-casting instructions we identified during induction
5094 // detection.
5095 for (const InductionDescriptor &IndDes : Legal->getInductionVars().values())
5096 VecValuesToIgnore.insert_range(R: IndDes.getCastInsts());
5097}
5098
5099void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
5100 CM->collectValuesToIgnore();
5101 Config.collectElementTypesForWidening(ValuesToIgnore: &CM->ValuesToIgnore);
5102
5103 FixedScalableVFPair MaxFactors = CM->computeMaxVF(UserVF, UserIC);
5104 if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
5105 return;
5106
5107 Config.collectInLoopReductions();
5108 // Cases that may be vectorized may be optimized by unit stride predicates.
5109 // TODO: Currently unit stride predicates are added unconditionally, even if
5110 // they are not used for the selected VF (e.g. when only interleaving).
5111 if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
5112 Legal->collectUnitStridePredicates();
5113
5114 auto VPlan1 = tryToBuildVPlan1(PlanCM&: *CM);
5115 if (!VPlan1)
5116 return;
5117
5118 LLVM_DEBUG(dbgs() << "LV: VPlan created successfully. Loop can be "
5119 "vectorized.\n");
5120
5121 if (!OrigLoop->isInnermost()) {
5122 // For outer loops, computeMaxVF returns a single non-scalar VF; build a
5123 // plan for that VF only.
5124 ElementCount VF =
5125 MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
5126 buildVPlans(VPlan1&: *VPlan1, MinVF: VF, MaxVF: VF, PlanCM&: *CM);
5127 LLVM_DEBUG(printPlans(dbgs()));
5128 return;
5129 }
5130
5131 // Compute the minimal bitwidths required for integer operations in the loop
5132 // for later use by the cost model.
5133 Config.computeMinimalBitwidths();
5134
5135 // Invalidate interleave groups if all blocks of loop will be predicated.
5136 if (CM->blockNeedsPredicationForAnyReason(BB: OrigLoop->getHeader()) &&
5137 !useMaskedInterleavedAccesses(Opts, TTI)) {
5138 LLVM_DEBUG(
5139 dbgs()
5140 << "LV: Invalidate all interleaved groups due to fold-tail by masking "
5141 "which requires masked-interleaved support.\n");
5142 if (CM->InterleaveInfo.invalidateGroups())
5143 // Invalidating interleave groups also requires invalidating all decisions
5144 // based on them, which includes widening decisions and uniform and scalar
5145 // values.
5146 CM->invalidateCostModelingDecisions();
5147 }
5148
5149 if (CM->foldTailByMasking())
5150 Legal->prepareToFoldTailByMasking();
5151
5152 ElementCount MaxUserVF =
5153 UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
5154 if (UserVF) {
5155 if (!ElementCount::isKnownLE(LHS: UserVF, RHS: MaxUserVF)) {
5156 reportVectorizationInfo(
5157 Msg: "UserVF ignored because it may be larger than the maximal safe VF",
5158 ORETag: "InvalidUserVF", ORE, TheLoop: OrigLoop);
5159 } else {
5160 assert(isPowerOf2_32(UserVF.getKnownMinValue()) &&
5161 "VF needs to be a power of two");
5162 // Collect the instructions (and their associated costs) that will be more
5163 // profitable to scalarize.
5164 CM->collectNonVectorizedAndSetWideningDecisions(VF: UserVF);
5165 buildVPlans(VPlan1&: *VPlan1, MinVF: UserVF, MaxVF: UserVF, PlanCM&: *CM);
5166 ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
5167 if (EpilogueUserVF.isVector() &&
5168 ElementCount::isKnownLT(LHS: EpilogueUserVF, RHS: UserVF)) {
5169 CM->collectNonVectorizedAndSetWideningDecisions(VF: EpilogueUserVF);
5170 buildVPlans(VPlan1&: *VPlan1, MinVF: EpilogueUserVF, MaxVF: EpilogueUserVF, PlanCM&: *CM);
5171 }
5172 if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
5173 // For scalar VF, skip VPlan cost check as VPlan cost is designed for
5174 // vector VFs only.
5175 if (UserVF.isScalar() ||
5176 cost(Plan&: *VPlans.front(), VF: UserVF, /*RU=*/nullptr, PlanCM&: *CM).isValid()) {
5177 LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
5178 LLVM_DEBUG(printPlans(dbgs()));
5179 return;
5180 }
5181 }
5182 VPlans.clear();
5183 reportVectorizationInfo(Msg: "UserVF ignored because of invalid costs.",
5184 ORETag: "InvalidCost", ORE, TheLoop: OrigLoop);
5185 }
5186 }
5187
5188 // Collect the Vectorization Factor Candidates.
5189 SmallVector<ElementCount> VFCandidates;
5190 for (auto VF = ElementCount::getFixed(MinVal: 1);
5191 ElementCount::isKnownLE(LHS: VF, RHS: MaxFactors.FixedVF); VF *= 2)
5192 VFCandidates.push_back(Elt: VF);
5193 for (auto VF = ElementCount::getScalable(MinVal: 1);
5194 ElementCount::isKnownLE(LHS: VF, RHS: MaxFactors.ScalableVF); VF *= 2)
5195 VFCandidates.push_back(Elt: VF);
5196
5197 for (const auto &VF : VFCandidates) {
5198 // Collect Uniform and Scalar instructions after vectorization with VF.
5199 CM->collectNonVectorizedAndSetWideningDecisions(VF);
5200 }
5201
5202 buildVPlans(VPlan1&: *VPlan1, MinVF: ElementCount::getFixed(MinVal: 1), MaxVF: MaxFactors.FixedVF, PlanCM&: *CM);
5203 buildVPlans(VPlan1&: *VPlan1, MinVF: ElementCount::getScalable(MinVal: 1), MaxVF: MaxFactors.ScalableVF,
5204 PlanCM&: *CM);
5205
5206 LLVM_DEBUG(printPlans(dbgs()));
5207}
5208
5209VPCostContext::VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan,
5210 LoopVectorizationCostModel &CM,
5211 VFSelectionContext &Config,
5212 bool ReusePrintingSlotTracker)
5213 : Opts(Config.getOpts()), TTI(Config.getTTI()), TLI(TLI),
5214 LLVMCtx(Plan.getContext()), CM(CM), Config(Config),
5215 CostKind(Config.CostKind), PSE(Config.getPSE()), L(Config.getLoop()) {
5216#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5217 if (ReusePrintingSlotTracker)
5218 PlanForSlotTracker = &Plan;
5219#endif
5220}
5221
5222InstructionCost VPCostContext::getLegacyCost(Instruction *UI,
5223 ElementCount VF) const {
5224 InstructionCost Cost = CM.getInstructionCost(I: UI, VF);
5225 if (Cost.isValid() && Opts.force_target_instruction_cost)
5226 return InstructionCost(*Opts.force_target_instruction_cost);
5227 return Cost;
5228}
5229
5230bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const {
5231 return CM.ValuesToIgnore.contains(Ptr: UI) ||
5232 (IsVector && CM.VecValuesToIgnore.contains(Ptr: UI)) ||
5233 SkipCostComputation.contains(Ptr: UI);
5234}
5235
5236void VPCostContext::invalidateWideningDecision(Instruction *I,
5237 ElementCount VF) {
5238 CM.setWideningDecision(I, VF,
5239 W: LoopVectorizationCostModel::CM_InvalidatedDecision, Cost: 0);
5240}
5241
5242bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const {
5243 return CM.isScalarWithPredication(I, VF) ||
5244 CM.isUniformAfterVectorization(I, VF) || CM.isForcedScalar(I, VF) ||
5245 (VF.isVector() && CM.isProfitableToScalarize(I, VF));
5246}
5247
5248bool VPCostContext::isMaskRequired(Instruction *I) const {
5249 return CM.isMaskRequired(I);
5250}
5251
5252bool VPCostContext::executesAtMostOnce(const VPlan &Plan, ElementCount VF) {
5253 auto *TC = dyn_cast_if_present<ConstantInt>(
5254 Val: Plan.getTripCount()->getUnderlyingValue());
5255 return TC && TC->getValue().ule(RHS: VF.getKnownMinValue());
5256}
5257
5258InstructionCost
5259LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
5260 VPCostContext &CostCtx) const {
5261 InstructionCost Cost;
5262
5263 // If the vector loop gets executed exactly once with the given VF, ignore the
5264 // costs of comparison and induction instructions, as they'll get simplified
5265 // away.
5266 // TODO: Remove this code after stepping away from the legacy cost model and
5267 // adding code to simplify VPlans before calculating their costs.
5268 auto TC = getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop);
5269 if (TC == VF && !Plan.hasTailFolded())
5270 addFullyUnrolledInstructionsToIgnore(L: OrigLoop, IL: Legal->getInductionVars(),
5271 InstsToIgnore&: CostCtx.SkipCostComputation);
5272
5273 // Pre-compute the costs for branches except for the backedge, as the number
5274 // of replicate regions in a VPlan may not directly match the number of
5275 // branches, which would lead to different decisions.
5276 // TODO: Compute cost of branches for each replicate region in the VPlan,
5277 // which is more accurate than the legacy cost model.
5278 for (BasicBlock *BB : OrigLoop->blocks()) {
5279 if (CostCtx.skipCostComputation(UI: BB->getTerminator(), IsVector: VF.isVector()))
5280 continue;
5281 CostCtx.SkipCostComputation.insert(Ptr: BB->getTerminator());
5282 if (BB == OrigLoop->getLoopLatch())
5283 continue;
5284 auto BranchCost = CostCtx.getLegacyCost(UI: BB->getTerminator(), VF);
5285 Cost += BranchCost;
5286 }
5287
5288 // Don't apply special costs when instruction cost is forced to make sure the
5289 // forced cost is used for each recipe.
5290 if (Opts.force_target_instruction_cost)
5291 return Cost;
5292
5293 // Pre-compute costs for instructions that are forced-scalar or profitable to
5294 // scalarize. For most such instructions, their scalarization costs are
5295 // accounted for here using the legacy cost model. However, some opcodes
5296 // are excluded from these precomputed scalarization costs and are instead
5297 // modeled later by the VPlan cost model (see UseVPlanCostModel below).
5298 for (Instruction *ForcedScalar : CostCtx.CM.ForcedScalars[VF]) {
5299 if (CostCtx.skipCostComputation(UI: ForcedScalar, IsVector: VF.isVector()))
5300 continue;
5301 CostCtx.SkipCostComputation.insert(Ptr: ForcedScalar);
5302 InstructionCost ForcedCost = CostCtx.getLegacyCost(UI: ForcedScalar, VF);
5303 LLVM_DEBUG({
5304 dbgs() << "Cost of " << ForcedCost << " for VF " << VF
5305 << ": forced scalar " << *ForcedScalar << "\n";
5306 });
5307 Cost += ForcedCost;
5308 }
5309
5310 // Don't apply legacy scalarization costs if nothing remains scalar &
5311 // predicated.
5312 if (!hasReplicatorRegion(Plan))
5313 return Cost;
5314
5315 auto UseVPlanCostModel = [](Instruction *I) -> bool {
5316 switch (I->getOpcode()) {
5317 case Instruction::SDiv:
5318 case Instruction::UDiv:
5319 case Instruction::SRem:
5320 case Instruction::URem:
5321 return true;
5322 default:
5323 return false;
5324 }
5325 };
5326 // Only charge legacy scalarization costs for instructions VPlan replicates.
5327 SmallPtrSet<Instruction *, 8> Replicated;
5328 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
5329 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry())))
5330 for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(Range&: *VPBB))
5331 Replicated.insert(Ptr: RepR.getUnderlyingInstr());
5332 for (const auto &[Scalarized, ScalarCost] : CostCtx.CM.InstsToScalarize[VF]) {
5333 if (!Replicated.contains(Ptr: Scalarized) || UseVPlanCostModel(Scalarized) ||
5334 CostCtx.skipCostComputation(UI: Scalarized, IsVector: VF.isVector()))
5335 continue;
5336 CostCtx.SkipCostComputation.insert(Ptr: Scalarized);
5337 LLVM_DEBUG({
5338 dbgs() << "Cost of " << ScalarCost << " for VF " << VF
5339 << ": profitable to scalarize " << *Scalarized << "\n";
5340 });
5341 Cost += ScalarCost;
5342 }
5343
5344 return Cost;
5345}
5346
5347#ifndef NDEBUG
5348/// Returns the frequency with which \p VPBB executes, as recorded on its
5349/// recipes. All recipes of a block share the same frequency.
5350static std::optional<VPExecutionFrequency>
5351getRecordedExecutionFrequency(const VPBasicBlock *VPBB) {
5352 if (VPBB->empty())
5353 return std::nullopt;
5354 return cast<VPInstruction>(&VPBB->front())->getExecutionFrequency();
5355}
5356#endif
5357
5358InstructionCost
5359LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
5360 VPRegisterUsage *RU,
5361 LoopVectorizationCostModel &PlanCM) const {
5362 VPCostContext CostCtx(*TLI, Plan, PlanCM, Config,
5363 /*ReusePrintingSlotTracker=*/true);
5364 InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
5365 LLVM_DEBUG(dbgs() << "Precomputed costs for VF " << VF << ": " << Cost
5366 << '\n');
5367
5368 // Now compute and add the VPlan-based cost.
5369 Cost += Plan.cost(VF, Ctx&: CostCtx);
5370
5371 // Add the cost of spills due to excess register usage
5372 if (RU && Config.shouldConsiderRegPressureForVF(VF)) {
5373 InstructionCost SpillCost = RU->spillCost(
5374 TTI, CostKind: Config.CostKind, OverrideMaxNumRegs: Opts.force_target_num_vector_regs.value_or(u: 0));
5375 LLVM_DEBUG(dbgs() << "Spill costs for VF " << VF << ": " << SpillCost
5376 << '\n');
5377 Cost += SpillCost;
5378 }
5379
5380#ifndef NDEBUG
5381 unsigned EstimatedWidth =
5382 estimateElementCount(VF, Config.getVScaleForTuning());
5383 LLVM_DEBUG(dbgs() << "Cost for VF " << VF << ": " << Cost
5384 << " (Estimated cost per lane: ");
5385 if (Cost.isValid()) {
5386 APFloat CostPerLane(APFloat::IEEEdouble());
5387 APFloat EstimatedWidthAsAPFloat(APFloat::IEEEdouble());
5388 (void)CostPerLane.convertFromAPInt(APInt(64, (uint64_t)Cost.getValue()),
5389 false, APFloat::rmTowardZero);
5390 (void)EstimatedWidthAsAPFloat.convertFromAPInt(
5391 APInt(64, (uint64_t)EstimatedWidth), false, APFloat::rmTowardZero);
5392 (void)CostPerLane.divide(EstimatedWidthAsAPFloat, APFloat::rmTowardZero);
5393
5394 SmallString<16> Str;
5395 CostPerLane.toString(Str, 3);
5396 LLVM_DEBUG(dbgs() << Str);
5397 } else /* No point dividing an invalid cost - it will still be invalid */
5398 LLVM_DEBUG(dbgs() << "Invalid");
5399 LLVM_DEBUG(dbgs() << ")\n");
5400#endif
5401 return Cost;
5402}
5403
5404std::pair<VectorizationFactor, VPlan *>
5405LoopVectorizationPlanner::computeBestVF() {
5406 if (VPlans.empty())
5407 return {VectorizationFactor::Disabled(), nullptr};
5408 // If there is a single VPlan with a single VF, return it directly.
5409 VPlan &FirstPlan = *VPlans[0];
5410
5411 ElementCount UserVF = Config.getHints().getWidth();
5412 if (VPlans.size() == 1) {
5413 // For outer loops, the plan has a single vector VF determined by the
5414 // heuristic.
5415 assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
5416 FirstPlan.isOuterLoop()) &&
5417 "must have a single scalar VF, UserVF or an outer loop");
5418 return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
5419 }
5420
5421 if (hasPlanWithVF(VF: UserVF) && hasForcedEpilogueVF() && VPlans.size() == 2) {
5422 assert(VPlans[0]->getSingleVF() == UserVF &&
5423 "expected second plan to be for the forced UserVF");
5424 assert(VPlans[1]->getSingleVF() == EpilogueVectorizationForceVF &&
5425 "expected first plan to be for the forced epilogue VF");
5426 return {VectorizationFactor(UserVF, 0, 0), VPlans[0].get()};
5427 }
5428
5429 LLVM_DEBUG(dbgs() << "LV: Computing best VF using cost kind: "
5430 << (Config.CostKind == TTI::TCK_RecipThroughput
5431 ? "Reciprocal Throughput\n"
5432 : Config.CostKind == TTI::TCK_Latency
5433 ? "Instruction Latency\n"
5434 : Config.CostKind == TTI::TCK_CodeSize ? "Code Size\n"
5435 : Config.CostKind == TTI::TCK_SizeAndLatency
5436 ? "Code Size and Latency\n"
5437 : "Unknown\n"));
5438
5439 ElementCount ScalarVF = ElementCount::getFixed(MinVal: 1);
5440 assert(FirstPlan.hasVF(ScalarVF) &&
5441 "More than a single plan/VF w/o any plan having scalar VF");
5442
5443 // TODO: Compute scalar cost using VPlan-based cost model.
5444 InstructionCost ScalarCost = CM->expectedCost(VF: ScalarVF);
5445 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
5446 VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
5447 VectorizationFactor BestFactor = ScalarFactor;
5448
5449 bool ForceVectorization =
5450 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
5451 if (ForceVectorization) {
5452 // Ignore scalar width, because the user explicitly wants vectorization.
5453 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
5454 // evaluation.
5455 BestFactor.Cost = InstructionCost::getMax();
5456 }
5457
5458 VPlan *PlanForBestVF = &FirstPlan;
5459 ElementCount ExactTC = getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop);
5460
5461 for (auto &P : VPlans) {
5462 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
5463 P->vectorFactors().end());
5464
5465 // For loops where the Trip Count is below the TailFoldingThreshold, only
5466 // consider the largest VF to result in at most one vector iteration, and at
5467 // most one scalar iteration.
5468 // FIXME: Encode this decision directly in LVPlanner.
5469 if (!ForceVectorization && P->hasScalarTail() && ExactTC.isFixed() &&
5470 ExactTC.getFixedValue() > 0 &&
5471 ExactTC.getFixedValue() <= TTI.getMinTripCountTailFoldingThreshold()) {
5472 VFs = VFs.take_back(N: 1);
5473 }
5474
5475 SmallVector<VPRegisterUsage, 8> RUs;
5476 bool ConsiderRegPressure = any_of(Range&: VFs, P: [this](ElementCount VF) {
5477 return Config.shouldConsiderRegPressureForVF(VF);
5478 });
5479 if (ConsiderRegPressure)
5480 RUs = calculateRegisterUsageForPlan(Plan&: *P, VFs, TTI);
5481
5482 for (unsigned I = 0; I < VFs.size(); I++) {
5483 ElementCount VF = VFs[I];
5484 if (VF.isScalar())
5485 continue;
5486 if (!ForceVectorization && !willGenerateVectors(Plan&: *P, VF, TTI)) {
5487 LLVM_DEBUG(
5488 dbgs()
5489 << "LV: Not considering vector loop of width " << VF
5490 << " because it will not generate any vector instructions.\n");
5491 continue;
5492 }
5493 if (Config.OptForSize && !ForceVectorization && hasReplicatorRegion(Plan&: *P)) {
5494 LLVM_DEBUG(
5495 dbgs()
5496 << "LV: Not considering vector loop of width " << VF
5497 << " because it would cause replicated blocks to be generated,"
5498 << " which isn't allowed when optimizing for size.\n");
5499 continue;
5500 }
5501
5502 InstructionCost Cost =
5503 cost(Plan&: *P, VF, RU: ConsiderRegPressure ? &RUs[I] : nullptr, PlanCM&: *CM);
5504 VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
5505
5506 if (isMoreProfitable(A: CurrentFactor, B: BestFactor, HasTail: P->hasScalarTail())) {
5507 BestFactor = CurrentFactor;
5508 PlanForBestVF = P.get();
5509 }
5510
5511 // If profitable add it to ProfitableVF list.
5512 if (isMoreProfitable(A: CurrentFactor, B: ScalarFactor, HasTail: P->hasScalarTail()))
5513 ProfitableVFs.push_back(Elt: CurrentFactor);
5514 }
5515 }
5516
5517 VPlan &BestPlan = *PlanForBestVF;
5518
5519 assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
5520 "when vectorizing, the scalar cost must be computed.");
5521
5522 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << BestFactor.Width << ".\n");
5523 return {BestFactor, &BestPlan};
5524}
5525
5526LoopVectorizationPlanner::LoopVectorizationPlanner(
5527 Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
5528 const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
5529 std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
5530 PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE,
5531 std::function<const BranchProbabilityInfo &()> GetBPI)
5532 : Opts(Config.getOpts()), OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
5533 Legal(Legal), CM(std::move(CM)), Config(Config), PSE(PSE), ORE(ORE),
5534 GetBPI(GetBPI) {}
5535
5536LoopVectorizationPlanner::~LoopVectorizationPlanner() = default;
5537
5538void LoopVectorizationPlanner::clearCostModel() { CM.reset(); }
5539
5540DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
5541 ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
5542 InnerLoopVectorizer &ILV, DominatorTree *DT,
5543 EpilogueVectorizationKind EpilogueVecKind) {
5544 assert(BestVPlan.hasVF(BestVF) &&
5545 "Trying to execute plan with unsupported VF");
5546 assert(BestVPlan.hasUF(BestUF) &&
5547 "Trying to execute plan with unsupported UF");
5548 if (BestVPlan.hasEarlyExit())
5549 ++LoopsEarlyExitVectorized;
5550
5551 RUN_VPLAN_PASS(VPlanTransforms::replaceWideCanonicalIVWithWideIV, BestVPlan,
5552 *PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF);
5553 if (!BestVPlan.hasTailFolded() &&
5554 TTI.hasMultiVectorLoadStore(NumVectors: BestUF,
5555 Mask: TargetTransformInfo::MaskSource::None))
5556 RUN_VPLAN_PASS(VPlanTransforms::widenMemoryAccessesByUF, BestVPlan, BestVF,
5557 BestUF, TTI);
5558 // TODO: Move to VPlan transform stage once the transition to the VPlan-based
5559 // cost model is complete for better cost estimates.
5560 RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
5561 RUN_VPLAN_PASS(VPlanTransforms::materializeUnpacks, BestVPlan);
5562 RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
5563 RUN_VPLAN_PASS(VPlanTransforms::replicateByVF, BestVPlan, BestVF);
5564 bool HasBranchWeights =
5565 hasBranchWeightMD(I: *OrigLoop->getLoopLatch()->getTerminator());
5566 if (HasBranchWeights) {
5567 std::optional<unsigned> VScale = Config.getVScaleForTuning();
5568 RUN_VPLAN_PASS(VPlanTransforms::addBranchWeightToMiddleTerminator,
5569 BestVPlan, BestVF, VScale);
5570 }
5571
5572 if (vputils::findIncomingAliasMask(Plan: BestVPlan)) {
5573 assert(BestVPlan.hasTailFolded() && "Expected tail folding to be enabled");
5574 RUN_VPLAN_PASS(VPlanTransforms::materializeAliasMaskCheckBlock, BestVPlan,
5575 *Legal->getRuntimePointerChecking()->getDiffChecks(),
5576 HasBranchWeights);
5577 ++LoopsPartialAliasVectorized;
5578 }
5579
5580 // Retrieving VectorPH now when it's easier while VPlan still has Regions.
5581 VPBasicBlock *VectorPH = cast<VPBasicBlock>(Val: BestVPlan.getVectorPreheader());
5582
5583 RUN_VPLAN_PASS(VPlanTransforms::materializeConstantVectorTripCount, BestVPlan,
5584 BestVF, BestUF, PSE);
5585 RUN_VPLAN_PASS(VPlanTransforms::optimizeForVFAndUF, BestVPlan, BestVF, BestUF,
5586 PSE);
5587 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
5588 // Check if scalar epilogue is required, before simplifying constant branches.
5589 const bool RequiresScalarEpilogue = BestVPlan.requiresScalarEpilogue();
5590 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5591 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5592 /*OnlyLatches=*/false);
5593 if (BestVPlan.getEntry()->getSingleSuccessor() ==
5594 BestVPlan.getScalarPreheader()) {
5595 // TODO: The vector loop would be dead, should not even try to vectorize.
5596 ORE->emit(RemarkBuilder: [&]() {
5597 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationDead",
5598 OrigLoop->getStartLoc(),
5599 OrigLoop->getHeader())
5600 << "Created vector loop never executes due to insufficient trip "
5601 "count.";
5602 });
5603 return DenseMap<const SCEV *, Value *>();
5604 }
5605
5606 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, BestVPlan);
5607
5608 RUN_VPLAN_PASS(VPlanTransforms::convertToConcreteRecipes, BestVPlan);
5609 // Convert the exit condition to AVLNext == 0 for EVL tail folded loops.
5610 RUN_VPLAN_PASS(VPlanTransforms::convertEVLExitCond, BestVPlan);
5611 // Regions are dissolved after optimizing for VF and UF, which completely
5612 // removes unneeded loop regions first.
5613 const bool HasTailFolded = BestVPlan.hasTailFolded();
5614 RUN_VPLAN_PASS(VPlanTransforms::dissolveLoopRegions, BestVPlan);
5615 // Expand BranchOnTwoConds after dissolution, when latch has direct access to
5616 // its successors.
5617 RUN_VPLAN_PASS(VPlanTransforms::expandBranchOnTwoConds, BestVPlan);
5618 // Convert loops with variable-length stepping after regions are dissolved.
5619 RUN_VPLAN_PASS(VPlanTransforms::convertToVariableLengthStep, BestVPlan);
5620 // Remove dead back-edges for single-iteration loops with BranchOnCond(true).
5621 // Only process loop latches to avoid removing edges from the middle block,
5622 // which may be needed for epilogue vectorization.
5623 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5624 /*OnlyLatches=*/true);
5625 RUN_VPLAN_PASS(VPlanTransforms::materializeBackedgeTakenCount, BestVPlan,
5626 VectorPH);
5627 std::optional<uint64_t> MaxRuntimeStep = getMaxRuntimeElementCount(
5628 EC: BestVF * BestUF, F: *OrigLoop->getHeader()->getParent());
5629
5630 assert((LI->getUniqueLatchExitBlock(*OrigLoop) || RequiresScalarEpilogue) &&
5631 "loops not exiting via the latch without required epilogue?");
5632 RUN_VPLAN_PASS(VPlanTransforms::materializeVectorTripCount, BestVPlan,
5633 VectorPH, HasTailFolded, RequiresScalarEpilogue,
5634 &BestVPlan.getVFxUF(), MaxRuntimeStep);
5635 RUN_VPLAN_PASS(VPlanTransforms::materializeFactors, BestVPlan, VectorPH,
5636 BestVF);
5637 // Limit expansions to VPInstruction to when not vectorizing the epilogue.
5638 // Currently this code path still relies on code re-using SCEVs expanded
5639 // directly to IR instructions.
5640 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5641 RUN_VPLAN_PASS(VPlanTransforms::expandSCEVsToVPInstructions, BestVPlan,
5642 *PSE.getSE());
5643 RUN_VPLAN_PASS(VPlanTransforms::removeRedundantExpandSCEVRecipes, BestVPlan);
5644 RUN_VPLAN_PASS(VPlanTransforms::cse, BestVPlan, VPDominatorTree(BestVPlan));
5645 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
5646 // Removing branches and incoming values may expose additional simplification
5647 // opportunities.
5648 if (RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5649 /*OnlyLatches=*/EpilogueVecKind !=
5650 EpilogueVectorizationKind::None))
5651 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
5652 RUN_VPLAN_PASS(VPlanTransforms::simplifyKnownEVL, BestVPlan, BestVF, PSE);
5653
5654 // 0. Generate SCEV-dependent code in the entry, including TripCount, before
5655 // making any changes to the CFG.
5656 DenseMap<const SCEV *, Value *> ExpandedSCEVs =
5657 RUN_VPLAN_PASS(VPlanTransforms::expandSCEVs, BestVPlan, *PSE.getSE());
5658
5659 // Perform the actual loop transformation.
5660 VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
5661 OrigLoop->getParentLoop());
5662
5663#ifdef EXPENSIVE_CHECKS
5664 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
5665#endif
5666
5667 // 1. Set up the skeleton for vectorization, including vector pre-header and
5668 // middle block. The vector loop is created during VPlan execution.
5669 State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
5670 if (VPBasicBlock *ScalarPH = BestVPlan.getScalarPreheader())
5671 replaceVPBBWithIRVPBB(VPBB: ScalarPH, IRBB: State.CFG.PrevBB->getSingleSuccessor(),
5672 Plan: &BestVPlan);
5673 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, BestVPlan);
5674
5675 assert(verifyVPlanIsValid(BestVPlan) && "final VPlan is invalid");
5676
5677 // After vectorization, the exit blocks of the original loop will have
5678 // additional predecessors. Invalidate SCEVs for the exit phis in case SE
5679 // looked through single-entry phis.
5680 ScalarEvolution &SE = *PSE.getSE();
5681 for (VPIRBasicBlock *Exit : BestVPlan.getExitBlocks()) {
5682 if (!Exit->hasPredecessors())
5683 continue;
5684 for (VPRecipeBase &PhiR : Exit->phis())
5685 SE.forgetLcssaPhiWithNewPredecessor(L: OrigLoop,
5686 V: &cast<VPIRPhi>(Val&: PhiR).getIRPhi());
5687 }
5688
5689 // Query whether the target wants loops it vectorizes to remain eligible for
5690 // runtime unrolling. Do this here, on the original loop and before its SCEV
5691 // is forgotten below.
5692 TargetTransformInfo::UnrollingPreferences UP;
5693 TTI.getUnrollingPreferences(L: OrigLoop, SE, UP, ORE);
5694 bool UnrollVectorizedLoop = UP.UnrollVectorizedLoop;
5695
5696 // Forget the original loop and block dispositions.
5697 SE.forgetLoop(L: OrigLoop);
5698 SE.forgetBlockAndLoopDispositions();
5699
5700 //===------------------------------------------------===//
5701 //
5702 // Notice: any optimization or new instruction that go
5703 // into the code below should also be implemented in
5704 // the cost-model.
5705 //
5706 //===------------------------------------------------===//
5707
5708 // Retrieve loop information before executing the plan, which may remove the
5709 // original loop, if it becomes unreachable.
5710 MDNode *LID = OrigLoop->getLoopID();
5711 unsigned OrigLoopInvocationWeight = 0;
5712 std::optional<unsigned> OrigAverageTripCount =
5713 getLoopEstimatedTripCount(L: OrigLoop, EstimatedLoopInvocationWeight: &OrigLoopInvocationWeight);
5714
5715 BestVPlan.execute(State: &State);
5716
5717 // 2.6. Maintain Loop Hints
5718 // Keep all loop hints from the original loop on the vector loop (we'll
5719 // replace the vectorizer-specific hints below).
5720 VPBasicBlock *HeaderVPBB = vputils::getFirstLoopHeader(Plan&: BestVPlan, VPDT&: State.VPDT);
5721 // Add metadata to disable runtime unrolling a scalar loop when there
5722 // are no runtime checks about strides and memory. A scalar loop that is
5723 // rarely used is not worth unrolling.
5724 bool DisableRuntimeUnroll = !ILV.RTChecks.hasChecks() && !BestVF.isScalar();
5725 updateLoopMetadataAndProfileInfo(
5726 VectorLoop: HeaderVPBB ? LI->getLoopFor(BB: State.CFG.VPBB2IRBB.lookup(Val: HeaderVPBB))
5727 : nullptr,
5728 HeaderVPBB, Plan: BestVPlan,
5729 VectorizingEpilogue: EpilogueVecKind == EpilogueVectorizationKind::Epilogue, OrigLoopID: LID,
5730 OrigAverageTripCount, OrigLoopInvocationWeight,
5731 EstimatedVFxUF: estimateElementCount(VF: BestVF * BestUF, VScale: Config.getVScaleForTuning()),
5732 DisableRuntimeUnroll, UnrollVectorizedLoop);
5733
5734 // 3. Fix the vectorized code: take care of header phi's, live-outs,
5735 // predication, updating analyses.
5736 ILV.fixVectorizedLoop(State);
5737
5738 // Wrap the generated blocks in VPIRBasicBlocks, so they can be used in the
5739 // epilogue plan.
5740 if (EpilogueVecKind == EpilogueVectorizationKind::MainLoop)
5741 for (VPBasicBlock *VPBB : to_vector(Range: VPBlockUtils::blocksAs<VPBasicBlock>(
5742 Range: vp_depth_first_shallow(G: BestVPlan.getEntry()))))
5743 if (!isa<VPIRBasicBlock>(Val: VPBB))
5744 replaceVPBBWithIRVPBB(VPBB, IRBB: State.CFG.VPBB2IRBB.at(Val: VPBB), Plan: &BestVPlan);
5745
5746 return ExpandedSCEVs;
5747}
5748
5749//===--------------------------------------------------------------------===//
5750// EpilogueVectorizerEpilogueLoop
5751//===--------------------------------------------------------------------===//
5752
5753/// This function creates a new scalar preheader, using the previous one as
5754/// entry block to the epilogue VPlan. The minimum iteration check is being
5755/// represented in VPlan.
5756BasicBlock *EpilogueVectorizerEpilogueLoop::createVectorizedLoopSkeleton() {
5757 BasicBlock *NewScalarPH = createScalarPreheader(Prefix: "vec.epilog.");
5758 BasicBlock *OriginalScalarPH = NewScalarPH->getSinglePredecessor();
5759 OriginalScalarPH->setName("vec.epilog.iter.check");
5760 VPIRBasicBlock *NewEntry = Plan.createVPIRBasicBlock(IRBB: OriginalScalarPH);
5761 VPBasicBlock *OldEntry = Plan.getEntry();
5762 for (auto &R : make_early_inc_range(Range&: *OldEntry)) {
5763 // Skip moving VPIRInstructions (including VPIRPhis), which are unmovable by
5764 // defining.
5765 if (isa<VPIRInstruction>(Val: &R))
5766 continue;
5767 R.moveBefore(BB&: *NewEntry, I: NewEntry->end());
5768 }
5769
5770 VPBlockUtils::reassociateBlocks(Old: OldEntry, New: NewEntry);
5771
5772 VecEpilogueIterationCountCheck = NewEntry;
5773
5774 // Model the skeleton from the main vector loop in the epilogue plan.
5775 RUN_VPLAN_PASS(VPlanTransforms::modelGeneratedMainLoopBlocks, Plan, MainPlan,
5776 NewEntry);
5777
5778 return OriginalScalarPH;
5779}
5780
5781bool VPRecipeBuilder::isPredicatedInst(Instruction *I) const {
5782 return CM.isPredicatedInst(I);
5783}
5784
5785bool VPRecipeBuilder::prefersVectorizedAddressing() const {
5786 return CM.TTI.prefersVectorizedAddressing();
5787}
5788
5789VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
5790 VFRange &Range) {
5791 assert((VPI->getOpcode() == Instruction::Load ||
5792 VPI->getOpcode() == Instruction::Store) &&
5793 "Must be called with either a load or store");
5794 Instruction *I = VPI->getUnderlyingInstr();
5795
5796 auto WillWiden = [&](ElementCount VF) -> bool {
5797 LoopVectorizationCostModel::InstWidening Decision =
5798 CM.getWideningDecision(I, VF);
5799 assert(Decision != LoopVectorizationCostModel::CM_Unknown &&
5800 "CM decision should be taken at this point.");
5801 if (Decision == LoopVectorizationCostModel::CM_Interleave)
5802 return true;
5803 if (CM.isScalarAfterVectorization(I, VF) ||
5804 CM.isProfitableToScalarize(I, VF))
5805 return false;
5806 return Decision != LoopVectorizationCostModel::CM_Scalarize;
5807 };
5808
5809 if (!LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: WillWiden, Range))
5810 return nullptr;
5811
5812 // If a mask is not required, drop it - use unmasked version for safe loads.
5813 // TODO: Determine if mask is needed in VPlan.
5814 VPValue *Mask = CM.isMaskRequired(I) ? VPI->getMask() : nullptr;
5815
5816 // Determine if the pointer operand of the access is either consecutive or
5817 // reverse consecutive.
5818 LoopVectorizationCostModel::InstWidening Decision =
5819 CM.getWideningDecision(I, VF: Range.Start);
5820 bool Reverse = Decision == LoopVectorizationCostModel::CM_Widen_Reverse;
5821 bool Consecutive =
5822 Reverse || Decision == LoopVectorizationCostModel::CM_Widen;
5823
5824 VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(N: 0)
5825 : VPI->getOperand(N: 1);
5826 Builder.setInsertPoint(VPI);
5827 if (Consecutive) {
5828 Ptr = Builder.createConsecutiveVectorPointer(Ptr, SourceElementTy: getLoadStoreType(I),
5829 Reverse, DL: VPI->getDebugLoc());
5830 }
5831
5832 if (Reverse && Mask)
5833 Mask = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Mask, DL: I->getDebugLoc());
5834
5835 if (VPI->getOpcode() == Instruction::Load) {
5836 auto *Load = cast<LoadInst>(Val: I);
5837 auto *LoadR = Builder.createWidenLoad(Load&: *Load, Addr: Ptr, Mask, Consecutive, Metadata: *VPI,
5838 DL: Load->getDebugLoc());
5839 if (Reverse)
5840 return Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: LoadR,
5841 DL: LoadR->getDebugLoc());
5842 return LoadR;
5843 }
5844
5845 StoreInst *Store = cast<StoreInst>(Val: I);
5846 VPValue *StoredVal = VPI->getOperand(N: 0);
5847 if (Reverse)
5848 StoredVal = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: StoredVal,
5849 DL: Store->getDebugLoc());
5850 return Builder.createWidenStore(Store&: *Store, Addr: Ptr, StoredVal, Mask, Consecutive,
5851 Metadata: *VPI, DL: Store->getDebugLoc());
5852}
5853
5854bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
5855 assert((!isa<UncondBrInst, CondBrInst, PHINode, LoadInst, StoreInst>(I)) &&
5856 "Instruction should have been handled earlier");
5857 // Instruction should be widened, unless it is scalar after vectorization,
5858 // scalarization is profitable or it is predicated.
5859 auto WillScalarize = [this, I](ElementCount VF) -> bool {
5860 return CM.isScalarAfterVectorization(I, VF) ||
5861 CM.isProfitableToScalarize(I, VF) ||
5862 CM.isScalarWithPredication(I, VF);
5863 };
5864 return !LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: WillScalarize,
5865 Range);
5866}
5867
5868VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
5869 auto *I = VPI->getUnderlyingInstr();
5870 switch (VPI->getOpcode()) {
5871 default:
5872 return nullptr;
5873 case Instruction::SDiv:
5874 case Instruction::UDiv:
5875 case Instruction::SRem:
5876 case Instruction::URem:
5877 // If not provably safe, use a masked intrinsic.
5878 if (CM.isPredicatedInst(I))
5879 return new VPWidenIntrinsicRecipe(
5880 getMaskedDivRemIntrinsic(Opcode: VPI->getOpcode()), VPI->operands(),
5881 I->getType(), {}, {}, VPI->getDebugLoc());
5882 [[fallthrough]];
5883 case Instruction::Add:
5884 case Instruction::And:
5885 case Instruction::AShr:
5886 case Instruction::FAdd:
5887 case Instruction::FCmp:
5888 case Instruction::FDiv:
5889 case Instruction::FMul:
5890 case Instruction::FNeg:
5891 case Instruction::FRem:
5892 case Instruction::FSub:
5893 case Instruction::ICmp:
5894 case Instruction::LShr:
5895 case Instruction::Mul:
5896 case Instruction::Or:
5897 case Instruction::Select:
5898 case Instruction::Shl:
5899 case Instruction::Sub:
5900 case Instruction::Xor:
5901 case Instruction::Freeze:
5902 case Instruction::ExtractValue:
5903 return new VPWidenRecipe(*I, VPI->operandsWithoutMask(), *VPI, *VPI,
5904 VPI->getDebugLoc());
5905 };
5906}
5907
5908VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
5909 if (VPI->getOpcode() != Instruction::Store)
5910 return nullptr;
5911
5912 auto HistInfo =
5913 Legal->getHistogramInfo(I: cast<StoreInst>(Val: VPI->getUnderlyingInstr()));
5914 if (!HistInfo)
5915 return nullptr;
5916
5917 const HistogramInfo *HI = *HistInfo;
5918
5919 SmallVector<VPValue *, 3> HGramOps;
5920 // Bucket address.
5921 HGramOps.push_back(Elt: VPI->getOperand(N: 1));
5922 // Increment value.
5923 HGramOps.push_back(Elt: Plan.getOrAddLiveIn(V: HI->Update->getOperand(i: 1)));
5924
5925 // In case of predicated execution (due to tail-folding, or conditional
5926 // execution, or both), pass the relevant mask.
5927 if (CM.isMaskRequired(I: HI->Store))
5928 HGramOps.push_back(Elt: VPI->getMask());
5929
5930 return new VPHistogramRecipe(HI->UpdateKind, HGramOps,
5931 cast<VPIRMetadata>(Val&: *VPI), VPI->getDebugLoc());
5932}
5933
5934bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
5935 VPInstruction *VPI, VPBuilder &FinalRedStoresBuilder) {
5936 StoreInst *SI;
5937 if ((SI = dyn_cast<StoreInst>(Val: VPI->getUnderlyingInstr())) &&
5938 Legal->isInvariantAddressOfReduction(V: SI->getPointerOperand())) {
5939 // Only create recipe for the final invariant store of the reduction.
5940 if (Legal->isInvariantStoreOfReduction(SI)) {
5941 VPValue *Val = VPI->getOperand(N: 0);
5942 VPValue *Addr = VPI->getOperand(N: 1);
5943 // We need to store the exiting value of the reduction, so use the blend
5944 // if tail folded.
5945 if (auto *Blend = VPlanPatternMatch::findUserOf<VPBlendRecipe>(V: Val))
5946 Val = Blend;
5947 [[maybe_unused]] auto *Rdx =
5948 VPlanPatternMatch::findUserOf<VPReductionPHIRecipe>(V: Val);
5949 assert((isa<VPIRValue>(Val) || !Rdx || Rdx->getBackedgeValue() == Val) &&
5950 "Store of reduction thats not the backedge value?");
5951 auto *Recipe = new VPReplicateRecipe(
5952 SI, {Val, Addr}, true /* IsUniform */, nullptr /*Mask*/, *VPI, *VPI,
5953 VPI->getDebugLoc());
5954 FinalRedStoresBuilder.insert(R: Recipe);
5955 }
5956 VPI->eraseFromParent();
5957 return true;
5958 }
5959
5960 return false;
5961}
5962
5963VPSingleDefRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
5964 VFRange &Range) {
5965 auto *I = VPI->getUnderlyingInstr();
5966 bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
5967 Predicate: [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
5968 Range);
5969
5970 bool IsPredicated = CM.isPredicatedInst(I);
5971
5972 // Even if the instruction is not marked as uniform, there are certain
5973 // intrinsic calls that can be effectively treated as such, so we check for
5974 // them here. Conservatively, we only do this for scalable vectors, since
5975 // for fixed-width VFs we can always fall back on full scalarization.
5976 if (!IsUniform && Range.Start.isScalable() && isa<IntrinsicInst>(Val: I)) {
5977 switch (cast<IntrinsicInst>(Val: I)->getIntrinsicID()) {
5978 case Intrinsic::assume:
5979 case Intrinsic::lifetime_start:
5980 case Intrinsic::lifetime_end:
5981 // For scalable vectors if one of the operands is variant then we still
5982 // want to mark as uniform, which will generate one instruction for just
5983 // the first lane of the vector. We can't scalarize the call in the same
5984 // way as for fixed-width vectors because we don't know how many lanes
5985 // there are.
5986 //
5987 // The reasons for doing it this way for scalable vectors are:
5988 // 1. For the assume intrinsic generating the instruction for the first
5989 // lane is still be better than not generating any at all. For
5990 // example, the input may be a splat across all lanes.
5991 // 2. For the lifetime start/end intrinsics the pointer operand only
5992 // does anything useful when the input comes from a stack object,
5993 // which suggests it should always be uniform. For non-stack objects
5994 // the effect is to poison the object, which still allows us to
5995 // remove the call.
5996 IsUniform = true;
5997 break;
5998 default:
5999 break;
6000 }
6001 }
6002 VPValue *BlockInMask = nullptr;
6003 if (!IsPredicated) {
6004 // Finalize the recipe for Instr, first if it is not predicated.
6005 LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n");
6006 } else {
6007 LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n");
6008 // Instructions marked for predication are replicated and a mask operand is
6009 // added initially. Masked replicate recipes will later be placed under an
6010 // if-then construct to prevent side-effects. Generate recipes to compute
6011 // the block mask for this region.
6012 BlockInMask = VPI->getMask();
6013 }
6014
6015 // Note that there is some custom logic to mark some intrinsics as uniform
6016 // manually above for scalable vectors, which this assert needs to account for
6017 // as well.
6018 assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
6019 (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
6020 "Should not predicate a uniform recipe");
6021 if (IsUniform) {
6022 return VPBuilder::createSingleScalarOp(
6023 Opcode: VPI->getOpcode(), Operands: VPI->operandsWithoutMask(), Mask: BlockInMask, Flags: *VPI, Metadata: *VPI,
6024 DL: VPI->getDebugLoc(), ResultTy: VPI->getScalarType(), UV: I);
6025 }
6026 auto *Recipe = new VPReplicateRecipe(I, VPI->operandsWithoutMask(),
6027 /*IsSingleScalar=*/false, BlockInMask,
6028 *VPI, *VPI, VPI->getDebugLoc());
6029 return Recipe;
6030}
6031
6032VPRecipeBase *
6033VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
6034 VFRange &Range) {
6035 assert(!R->isPhi() && "phis must be handled earlier");
6036 auto *VPI = cast<VPInstruction>(Val: R);
6037 assert(VPI->getOpcode() != Instruction::Call &&
6038 "Call should have been handled by makeCallWideningDecisions");
6039
6040 // All widen recipes below deal only with VF > 1.
6041 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6042 Predicate: [&](ElementCount VF) { return VF.isScalar(); }, Range))
6043 return nullptr;
6044
6045 Instruction *Instr = R->getUnderlyingInstr();
6046 assert(!is_contained({Instruction::Load, Instruction::Store},
6047 VPI->getOpcode()) &&
6048 "Should have been handled prior to this!");
6049
6050 // An extractvalue takes the shape of its operand: we cannot extract a lane
6051 // from a widened struct, nor build a struct from replicated lanes.
6052 if (VPI->getOpcode() == Instruction::ExtractValue &&
6053 !vputils::isSingleScalar(VPV: VPI->getOperand(N: 0)))
6054 if (VPRecipeBase *OpR = VPI->getOperand(N: 0)->getDefiningRecipe())
6055 if (!vputils::doesGeneratePerAllLanes(R: OpR))
6056 return tryToWiden(VPI);
6057 if (VPI->getOpcode() == Instruction::ExtractValue)
6058 return new VPReplicateRecipe(Instr, VPI->operandsWithoutMask(),
6059 vputils::isSingleScalar(VPV: VPI->getOperand(N: 0)),
6060 /*Mask=*/nullptr, *VPI, *VPI,
6061 VPI->getDebugLoc());
6062
6063 if (!shouldWiden(I: Instr, Range))
6064 return nullptr;
6065
6066 if (VPI->getOpcode() == Instruction::GetElementPtr) {
6067 auto *GEP = cast<GetElementPtrInst>(Val: Instr);
6068 return new VPWidenGEPRecipe(GEP->getSourceElementType(),
6069 VPI->operandsWithoutMask(), *VPI,
6070 VPI->getDebugLoc(), GEP);
6071 }
6072
6073 if (Instruction::isCast(Opcode: VPI->getOpcode())) {
6074 auto *CI = cast<CastInst>(Val: Instr);
6075 return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(N: 0),
6076 VPI->getScalarType(), CI, *VPI, *VPI,
6077 VPI->getDebugLoc());
6078 }
6079
6080 return tryToWiden(VPI);
6081}
6082
6083// To allow RUN_VPLAN_PASS to print the VPlan after VF/UF independent
6084// optimizations.
6085static void printOptimizedVPlan(VPlan &) {}
6086
6087#ifndef NDEBUG
6088/// Cross-check the execution frequencies recorded in \p Plan against
6089/// BlockFrequencyInfo for the blocks of \p OrigLoop.
6090/// FIXME: Temporary verification aid, to be removed.
6091static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
6092 LoopInfo *LI,
6093 LoopVectorizationCostModel &CM) {
6094 // Limited to loops with the latch as only exiting block
6095 if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
6096 return true;
6097
6098 // Visit the loop body in the same order as recordExecutionFrequencies. Both
6099 // are reverse post-orders of the same CFG, so indices correspond.
6100 VPBasicBlock *Header = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan).first;
6101 SmallVector<VPBasicBlock *> Blocks = vp_rpo_plain_cfg_loop_body(Header);
6102 assert(Blocks.size() == OrigLoop->getNumBlocks() &&
6103 "loop body and original loop must have the same blocks");
6104
6105 LoopBlocksRPO OrigRPO(OrigLoop);
6106 OrigRPO.perform(LI);
6107
6108 // Only request the expensive BFI once the cheap bail-outs are past.
6109 BlockFrequencyInfo &BFI = CM.getBFI();
6110 uint64_t HeaderFreq = BFI.getBlockFreq(OrigLoop->getHeader()).getFrequency();
6111 if (HeaderFreq == 0)
6112 return true;
6113
6114 // BFI's fixed-point mass propagation loses up to 1 ULP per edge, so bound the
6115 // error by the number of edges in the region.
6116 uint64_t Edges = 0;
6117 for (const VPBasicBlock *VPBB : Blocks)
6118 Edges += VPBB->getNumSuccessors();
6119 uint64_t Tolerance = Edges + BranchProbability::getDenominator() / HeaderFreq;
6120
6121 for (const auto &[VPBB, BB] :
6122 zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
6123 // Nothing to check for blocks without a recorded frequency.
6124 std::optional<VPExecutionFrequency> Freq =
6125 getRecordedExecutionFrequency(VPBB);
6126 if (!Freq)
6127 continue;
6128 BranchProbability Computed = vputils::getExecutionProbability(Freq->Freq);
6129
6130 // Clamp to the header's frequency, which BFI's rounding may exceed.
6131 uint64_t BBFreq = BFI.getBlockFreq(BB).getFrequency();
6132 BranchProbability Expected = BranchProbability::getBranchProbability(
6133 std::min(BBFreq, HeaderFreq), HeaderFreq);
6134 if (AbsoluteDifference(Computed.getNumerator(), Expected.getNumerator()) <=
6135 Tolerance)
6136 continue;
6137
6138 errs() << "Block frequency mismatch for " << VPBB->getName() << ": VPlan "
6139 << Computed << ", BlockFrequencyInfo " << Expected << "\n";
6140 return false;
6141 }
6142 return true;
6143}
6144#endif
6145
6146VPlanPtr
6147LoopVectorizationPlanner::tryToBuildVPlan1(LoopVectorizationCostModel &PlanCM) {
6148 bool IsInnerLoop = OrigLoop->isInnermost();
6149
6150 // Set up loop versioning for inner loops with memory runtime checks.
6151 // Outer loops don't have LoopAccessInfo since canVectorizeMemory() is not
6152 // called for them.
6153 std::optional<LoopVersioning> LVer;
6154 if (IsInnerLoop) {
6155 const LoopAccessInfo *LAI = Legal->getLAI();
6156 LVer.emplace(args: *LAI, args: LAI->getRuntimePointerChecking()->getChecks(), args&: OrigLoop,
6157 args&: LI, args&: DT, args: PSE.getSE());
6158 if (!LAI->getRuntimePointerChecking()->getChecks().empty() &&
6159 !LAI->getRuntimePointerChecking()->getDiffChecks()) {
6160 // Only use noalias metadata when using memory checks guaranteeing no
6161 // overlap across all iterations.
6162 LVer->prepareNoAliasMetadata();
6163 }
6164 }
6165
6166 // Create initial base VPlan0, to serve as common starting point for all
6167 // candidates built later for specific VF ranges.
6168 auto VPlan0 = VPlanTransforms::buildVPlan0(
6169 TheLoop: OrigLoop, LI&: *LI, InductionTy: Legal->getWidestInductionType(), PSE,
6170 LVer: LVer ? &*LVer : nullptr, GetBPI);
6171
6172 VPDominatorTree VPDT(*VPlan0);
6173 if (const LoopAccessInfo *LAI = Legal->getLAI())
6174 RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
6175 LAI->getSymbolicStrides(), VPDT);
6176 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, *VPlan0);
6177 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
6178 if (IsInnerLoop) {
6179 RUN_VPLAN_PASS(VPlanTransforms::recordExecutionFrequencies, *VPlan0);
6180 assert(verifyExecutionFrequenciesMatchBFI(*VPlan0, OrigLoop, LI, PlanCM) &&
6181 "execution frequencies do not match the loop's block frequencies");
6182 }
6183
6184 // Create recipes for header phis. For outer loops, reductions, recurrences
6185 // and in-loop reductions are empty since legality doesn't detect them.
6186 if (!RUN_VPLAN_PASS(
6187 VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE, *OrigLoop, ORE,
6188 VPDT, Legal->getInductionVars(), Legal->getReductionVars(),
6189 Legal->getFixedOrderRecurrences(), Config.getInLoopReductions(),
6190 Config.getHints().allowReordering())) {
6191 return nullptr;
6192 }
6193
6194 if (const LoopAccessInfo *LAI = Legal->getLAI())
6195 RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
6196 LAI->getSymbolicStrides(), VPDT);
6197
6198 // Add surviving induction predicates to PSE and check constraints.
6199 bool ForceVectorization =
6200 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
6201 bool OptForSize =
6202 !ForceVectorization &&
6203 (PlanCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
6204 PlanCM.EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
6205 unsigned SCEVCheckThreshold = ForceVectorization
6206 ? Opts.pragma_vectorize_scev_check_threshold
6207 : Opts.vectorize_scev_check_threshold;
6208 if (!RUN_VPLAN_PASS(VPlanTransforms::finalizeSCEVPredicates, *VPlan0, PSE,
6209 OptForSize, SCEVCheckThreshold, ORE, OrigLoop))
6210 return nullptr;
6211
6212 RUN_VPLAN_PASS(VPlanTransforms::addMiddleCheck, *VPlan0);
6213
6214 // If we're vectorizing a loop with an uncountable exit, make sure that the
6215 // recipes are safe to handle.
6216 // TODO: Remove this once we can properly check the VPlan itself for the
6217 // presence of an uncountable exit.
6218 if (Legal->hasUncountableEarlyExit()) {
6219 if (!RUN_VPLAN_PASS(VPlanTransforms::splitCombinedExits, *VPlan0, PSE,
6220 OrigLoop))
6221 return nullptr;
6222
6223 // TODO: Check target preference for style.
6224 UncountableExitStyle EEStyle = UncountableExitStyle::ReadOnly;
6225 if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0,
6226 ORE, OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
6227 EEStyle)) {
6228 return nullptr;
6229 }
6230 } else {
6231 RUN_VPLAN_PASS(VPlanTransforms::handleCountableEarlyExits, *VPlan0);
6232 }
6233
6234 RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
6235 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
6236 if (PlanCM.foldTailByMasking())
6237 RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
6238
6239 RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
6240
6241 return VPlan0;
6242}
6243
6244void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
6245 ElementCount MaxVF,
6246 LoopVectorizationCostModel &PlanCM) {
6247 if (ElementCount::isKnownGT(LHS: MinVF, RHS: MaxVF))
6248 return;
6249
6250 auto MaxVFTimes2 = MaxVF * 2;
6251 for (ElementCount VF = MinVF; ElementCount::isKnownLT(LHS: VF, RHS: MaxVFTimes2);) {
6252 VFRange SubRange = {VF, MaxVFTimes2};
6253 auto Plan = tryToBuildVPlan(InitialPlan: std::unique_ptr<VPlan>(VPlan1.duplicate()),
6254 Range&: SubRange, PlanCM);
6255 VF = SubRange.End;
6256
6257 if (!Plan)
6258 continue;
6259
6260 // Now optimize the initial VPlan.
6261 RUN_VPLAN_PASS(VPlanTransforms::hoistPredicatedLoads, *Plan, PSE, OrigLoop);
6262 RUN_VPLAN_PASS(VPlanTransforms::sinkPredicatedStores, *Plan, PSE, OrigLoop);
6263 RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
6264 Config.getMinimalBitwidths());
6265 RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
6266 // TODO: try to put addExplicitVectorLength close to addActiveLaneMask
6267 if (PlanCM.foldTailWithEVL()) {
6268 RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
6269 Config.getMaxSafeElements());
6270 RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
6271 }
6272
6273 if (auto P =
6274 RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
6275 VPlans.push_back(Elt: std::move(P));
6276
6277 TailFoldingStyle Style = PlanCM.getTailFoldingStyle();
6278 RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
6279 useActiveLaneMask(Style),
6280 useActiveLaneMaskForControlFlow(Style));
6281
6282 RUN_VPLAN_PASS_NO_VERIFY(printOptimizedVPlan, *Plan);
6283 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6284 VPlans.push_back(Elt: std::move(Plan));
6285 }
6286}
6287
6288VPlanPtr
6289LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan, VFRange &Range,
6290 LoopVectorizationCostModel &PlanCM) {
6291
6292 // For outer loops, the plan only needs basic recipe conversion and induction
6293 // live-out optimization; the full inner-loop recipe building below does not
6294 // apply (no widening decisions, interleave groups, reductions, etc.).
6295 if (Plan->isOuterLoop()) {
6296 for (ElementCount VF : Range)
6297 Plan->addVF(VF);
6298 if (!RUN_VPLAN_PASS(VPlanTransforms::tryToConvertVPInstructionsToVPRecipes,
6299 *Plan, *TLI, PSE, OrigLoop))
6300 return nullptr;
6301 RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
6302 OrigLoop);
6303 return Plan;
6304 }
6305
6306 using namespace llvm::VPlanPatternMatch;
6307 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
6308
6309 // ---------------------------------------------------------------------------
6310 // Build initial VPlan: Scan the body of the loop in a topological order to
6311 // visit each basic block after having visited its predecessor basic blocks.
6312 // ---------------------------------------------------------------------------
6313
6314 bool RequiresScalarEpilogueCheck =
6315 LoopVectorizationPlanner::getDecisionAndClampRange(
6316 Predicate: [&PlanCM](ElementCount VF) {
6317 return !PlanCM.requiresScalarEpilogue(IsVectorizing: VF.isVector());
6318 },
6319 Range);
6320 // Update the branch in the middle block if a scalar epilogue is required.
6321 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6322 if (!RequiresScalarEpilogueCheck && MiddleVPBB->getNumSuccessors() == 2) {
6323 auto *BranchOnCond = cast<VPInstruction>(Val: MiddleVPBB->getTerminator());
6324 assert(MiddleVPBB->getSuccessors()[1] == Plan->getScalarPreheader() &&
6325 "second successor must be scalar preheader");
6326 BranchOnCond->setOperand(I: 0, New: Plan->getFalse());
6327 }
6328
6329 // Don't use getDecisionAndClampRange here, because we don't know the UF
6330 // so this function is better to be conservative, rather than to split
6331 // it up into different VPlans.
6332 // TODO: Consider using getDecisionAndClampRange here to split up VPlans.
6333 bool IVUpdateMayOverflow = false;
6334 for (ElementCount VF : Range)
6335 IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(Opts, Cost: &PlanCM, VF);
6336
6337 TailFoldingStyle Style = PlanCM.getTailFoldingStyle();
6338 // Use NUW for the induction increment if we proved that it won't overflow in
6339 // the vector loop or when not folding the tail. In the later case, we know
6340 // that the canonical induction increment will not overflow as the vector trip
6341 // count is >= increment and a multiple of the increment.
6342 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
6343 bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
6344 if (!HasNUW) {
6345 auto *IVInc =
6346 LoopRegion->getExitingBasicBlock()->getTerminator()->getOperand(N: 0);
6347 assert(match(IVInc,
6348 m_VPInstruction<Instruction::Add>(
6349 m_Specific(LoopRegion->getCanonicalIV()), m_VPValue())) &&
6350 "Did not find the canonical IV increment");
6351 LoopRegion->clearCanonicalIVNUW(Increment: cast<VPInstruction>(Val: IVInc));
6352 }
6353
6354 // ---------------------------------------------------------------------------
6355 // Pre-construction: record ingredients whose recipes we'll need to further
6356 // process after constructing the initial VPlan.
6357 // ---------------------------------------------------------------------------
6358
6359 // For each interleave group which is relevant for this (possibly trimmed)
6360 // Range, add it to the set of groups to be later applied to the VPlan and add
6361 // placeholders for its members' Recipes which we'll be replacing with a
6362 // single VPInterleaveRecipe.
6363 for (InterleaveGroup<Instruction> *IG :
6364 PlanCM.InterleaveInfo.getInterleaveGroups()) {
6365 auto ApplyIG = [IG, &PlanCM](ElementCount VF) -> bool {
6366 bool Result = (VF.isVector() && // Query is illegal for VF == 1
6367 PlanCM.getWideningDecision(I: IG->getInsertPos(), VF) ==
6368 LoopVectorizationCostModel::CM_Interleave);
6369 // For scalable vectors, the interleave factors must be <= 8 since we
6370 // require the (de)interleaveN intrinsics instead of shufflevectors.
6371 assert((!Result || !VF.isScalable() || IG->getFactor() <= 8) &&
6372 "Unsupported interleave factor for scalable vectors");
6373 return Result;
6374 };
6375 if (!getDecisionAndClampRange(Predicate: ApplyIG, Range))
6376 continue;
6377 InterleaveGroups.insert(Ptr: IG);
6378 }
6379
6380 // ---------------------------------------------------------------------------
6381 // Construct wide recipes and apply predication for original scalar
6382 // VPInstructions in the loop.
6383 // ---------------------------------------------------------------------------
6384 VPRecipeBuilder RecipeBuilder(*Plan, Legal, PlanCM, Builder);
6385
6386 RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
6387 Range.Start);
6388
6389 VPCostContext CostCtx(*TLI, *Plan, PlanCM, Config);
6390
6391 RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
6392 RecipeBuilder, CostCtx);
6393
6394 RUN_VPLAN_PASS(VPlanTransforms::makeScalarizationDecisions, *Plan, Range);
6395
6396 // Widened calls may only use the first lane of some operands.
6397 if (RUN_VPLAN_PASS(VPlanTransforms::makeCallWideningDecisions, *Plan, Range,
6398 RecipeBuilder, CostCtx))
6399 RUN_VPLAN_PASS(VPlanTransforms::makeScalarizationDecisions, *Plan, Range);
6400
6401 RUN_VPLAN_PASS(VPlanTransforms::narrowInductionTruncates, *Plan, Range, TTI,
6402 PSE);
6403
6404 // Convert remaining VPInstructions to widen or replicate recipes.
6405 // TODO: This legacy code should eventually be migrated to VPlan.
6406 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
6407 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6408 Range: vp_depth_first_shallow(G: HeaderVPBB))) {
6409 // All types but VPInstructions are already widened and don't need extra
6410 // processing. We process VPInstructions below.
6411 assert(
6412 all_of(
6413 make_range(VPBB->getFirstNonPhi(), VPBB->end()),
6414 IsaPred<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe,
6415 VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
6416 VPWidenCallRecipe, VPWidenIntrinsicRecipe,
6417 VPVectorPointerRecipe, VPVectorEndPointerRecipe,
6418 VPHistogramRecipe, VPInstruction>) &&
6419 "Unexpected recipe");
6420 for (VPInstruction &VPI :
6421 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range&: *VPBB))) {
6422 // We represent single-scalar casts directly as VPInstructions.
6423 if (Instruction::isCast(Opcode: VPI.getOpcode()) &&
6424 vputils::onlyFirstLaneUsed(Def: &VPI))
6425 continue;
6426
6427 // Only VPInstrutions with an underlying value need to be processed.
6428 if (!VPI.getUnderlyingValue())
6429 continue;
6430
6431 Builder.setInsertPoint(&VPI);
6432
6433 VPRecipeBase *Recipe =
6434 RecipeBuilder.tryToCreateWidenNonPhiRecipe(R: &VPI, Range);
6435 if (!Recipe)
6436 Recipe = RecipeBuilder.handleReplication(VPI: &VPI, Range);
6437 Builder.insert(R: Recipe);
6438
6439 if (Recipe->getNumDefinedValues() == 1) {
6440 VPI.replaceAllUsesWith(New: Recipe->getVPSingleValue());
6441 } else {
6442 assert(Recipe->getNumDefinedValues() == 0 &&
6443 "Unexpected multidef recipe");
6444 }
6445 VPI.eraseFromParent();
6446 }
6447 }
6448
6449 assert(isa<VPRegionBlock>(LoopRegion) &&
6450 !LoopRegion->getEntryBasicBlock()->empty() &&
6451 "entry block must be set to a VPRegionBlock having a non-empty entry "
6452 "VPBasicBlock");
6453
6454 RUN_VPLAN_PASS(VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers, *Plan,
6455 Range);
6456
6457 // ---------------------------------------------------------------------------
6458 // Transform initial VPlan: Apply previously taken decisions, in order, to
6459 // bring the VPlan to its final state.
6460 // ---------------------------------------------------------------------------
6461
6462 addReductionResultComputation(Plan, MinVF: Range.Start, PlanCM);
6463
6464 // Optimize FindIV reductions to use sentinel-based approach when possible.
6465 RUN_VPLAN_PASS(VPlanTransforms::optimizeFindIVReductions, *Plan, PSE,
6466 *OrigLoop);
6467 RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
6468 OrigLoop);
6469
6470 // Apply mandatory transformation to handle reductions with multiple in-loop
6471 // uses if possible, bail out otherwise.
6472 if (!RUN_VPLAN_PASS(VPlanTransforms::handleMultiUseReductions, *Plan, ORE,
6473 OrigLoop))
6474 return nullptr;
6475 // Apply mandatory transformation to handle FP maxnum/minnum reduction with
6476 // NaNs if possible, bail out otherwise.
6477 if (!RUN_VPLAN_PASS(VPlanTransforms::handleMaxMinNumReductions, *Plan))
6478 return nullptr;
6479
6480 // Create whole-vector selects for find-last recurrences.
6481 if (!RUN_VPLAN_PASS(VPlanTransforms::handleFindLastReductions, *Plan))
6482 return nullptr;
6483
6484 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, *Plan, false);
6485
6486 // Create partial reduction recipes for scaled reductions and transform
6487 // recipes to abstract recipes if it is legal and beneficial and clamp the
6488 // range for better cost estimation.
6489 RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
6490 Range);
6491 RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
6492 Range);
6493
6494 // Interleave memory: for each Interleave Group we marked earlier as relevant
6495 // for this VPlan, replace the Recipes widening its memory instructions with a
6496 // single VPInterleaveRecipe at its insertion point.
6497 RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
6498 InterleaveGroups, PlanCM.isEpilogueAllowed());
6499
6500 // Convert memory recipes to strided access recipes if the strided access is
6501 // legal and profitable.
6502 RUN_VPLAN_PASS(VPlanTransforms::convertToStridedAccesses, *Plan, PSE,
6503 *OrigLoop, CostCtx, Range);
6504
6505 // Ensure scalar VF plans only contain VF=1, as required by hasScalarVFOnly.
6506 if (Range.Start.isScalar())
6507 Range.End = Range.Start * 2;
6508
6509 for (ElementCount VF : Range)
6510 Plan->addVF(VF);
6511 Plan->setName("Initial VPlan");
6512
6513 RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
6514
6515 if (PlanCM.maskPartialAliasing())
6516 RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
6517
6518 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6519 return Plan;
6520}
6521
6522void LoopVectorizationPlanner::addReductionResultComputation(
6523 VPlanPtr &Plan, ElementCount MinVF, LoopVectorizationCostModel &PlanCM) {
6524 using namespace VPlanPatternMatch;
6525 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
6526 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6527 VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
6528 Builder.setInsertPoint(&*std::prev(x: std::prev(x: LatchVPBB->end())));
6529 VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
6530 VPValue *HeaderMask = Plan->getVectorLoopRegion()->getHeaderMask();
6531 for (VPRecipeBase &R : make_early_inc_range(
6532 Range: Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
6533 VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
6534 if (!PhiR)
6535 continue;
6536
6537 // Clean up reductions that have become invariant.
6538 if (PhiR->getBackedgeValue() == PhiR) {
6539 PhiR->replaceAllUsesWith(New: PhiR->getStartValue());
6540 PhiR->eraseFromParent();
6541 continue;
6542 }
6543
6544 RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
6545 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
6546 PN: cast<PHINode>(Val: PhiR->getUnderlyingInstr()));
6547 Type *PhiTy = PhiR->getScalarType();
6548
6549 // Convert a VPBlendRecipe backedge to a select.
6550 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: PhiR->getBackedgeValue())) {
6551 if (Blend->getNumIncomingValues() == 2 &&
6552 Blend->getMask(Idx: 0) == HeaderMask) {
6553 auto *Sel = VPBuilder(Blend).createSelect(
6554 Cond: Blend->getMask(Idx: 0), TrueVal: Blend->getIncomingValue(Idx: 0),
6555 FalseVal: Blend->getIncomingValue(Idx: 1), DL: {}, Name: "", Flags: *Blend);
6556 Blend->replaceAllUsesWith(New: Sel);
6557 Blend->eraseFromParent();
6558 }
6559 }
6560
6561 auto *OrigExitingVPV = PhiR->getBackedgeValue();
6562 auto *NewExitingVPV = OrigExitingVPV;
6563
6564 // Remove the predicated select if the target doesn't want it.
6565 VPValue *V;
6566 if (!PlanCM.usePredicatedReductionSelect(
6567 RecurrenceKind, HasUsesOutsideReductionChain: PhiR->hasUsesOutsideReductionChain()) &&
6568 match(V: PhiR->getBackedgeValue(),
6569 P: m_Select(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue(V), Op2: m_Specific(VPV: PhiR))))
6570 PhiR->setBackedgeValue(V);
6571
6572 // We want code in the middle block to appear to execute on the location of
6573 // the scalar loop's latch terminator because: (a) it is all compiler
6574 // generated, (b) these instructions are always executed after evaluating
6575 // the latch conditional branch, and (c) other passes may add new
6576 // predecessors which terminate on this line. This is the easiest way to
6577 // ensure we don't accidentally cause an extra step back into the loop while
6578 // debugging.
6579 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
6580
6581 // TODO: At the moment ComputeReductionResult also drives creation of the
6582 // bc.merge.rdx phi nodes, hence it needs to be created unconditionally here
6583 // even for in-loop reductions, until the reduction resume value handling is
6584 // also modeled in VPlan.
6585 VPInstruction *FinalReductionResult;
6586 VPBuilder::InsertPointGuard Guard(Builder);
6587 Builder.setInsertPoint(TheBB: MiddleVPBB, IP);
6588 // For AnyOf reductions, find the select among PhiR's users and convert
6589 // the reduction phi to operate on bools before creating the final
6590 // reduction result.
6591 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RecurrenceKind)) {
6592 auto *AnyOfSelect = cast<VPSingleDefRecipe>(
6593 Val: findUserOf(V: PhiR, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())));
6594 VPValue *Start = PhiR->getStartValue();
6595 bool TrueValIsPhi = AnyOfSelect->getOperand(N: 1) == PhiR;
6596 // NewVal is the non-phi operand of the select.
6597 VPValue *NewVal = TrueValIsPhi ? AnyOfSelect->getOperand(N: 2)
6598 : AnyOfSelect->getOperand(N: 1);
6599
6600 // Adjust AnyOf reductions; replace the reduction phi for the selected
6601 // value with a boolean reduction phi node to check if the condition is
6602 // true in any iteration. The final value is selected by the final
6603 // ComputeReductionResult.
6604 VPValue *Cmp = AnyOfSelect->getOperand(N: 0);
6605 // If the compare is checking the reduction PHI node, adjust it to check
6606 // the start value.
6607 if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
6608 CmpR->replaceUsesOfWith(From: PhiR, To: PhiR->getStartValue());
6609 Builder.setInsertPoint(AnyOfSelect);
6610
6611 // If the true value of the select is the reduction phi, the new value
6612 // is selected if the negated condition is true in any iteration.
6613 if (TrueValIsPhi)
6614 Cmp = Builder.createNot(Operand: Cmp);
6615
6616 // Build a fresh i1 chain (phi, or, and i1 versions of any blend/select
6617 // the exiting value flows through).
6618 auto *NewPhiR =
6619 PhiR->cloneWithOperands(Start: Plan->getFalse(), BackedgeValue: Plan->getFalse());
6620 NewPhiR->insertBefore(InsertPos: PhiR);
6621 VPValue *NewExiting = Builder.createOr(LHS: NewPhiR, RHS: Cmp);
6622
6623 // The exiting value may flow through a chain of VPBlendRecipes and
6624 // select recipes (VPInstruction, VPWidenRecipe or VPReplicateRecipe with
6625 // Select opcode) before reaching OrigExitingVPV. Clone each chain link
6626 // in topological order so each clone refers to the already-rewritten i1
6627 // operands via Substitutions.
6628 DenseMap<VPValue *, VPValue *> Substitutions = {{AnyOfSelect, NewExiting},
6629 {PhiR, NewPhiR}};
6630 std::function<void(VPSingleDefRecipe *)> CloneChain =
6631 [&](VPSingleDefRecipe *Old) {
6632 if (Substitutions.contains(Val: Old))
6633 return;
6634 SmallVector<VPValue *> NewOps;
6635 for (VPValue *Op : Old->operands()) {
6636 if (isa<VPBlendRecipe>(Val: Op) ||
6637 match(V: Op, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())))
6638 CloneChain(cast<VPSingleDefRecipe>(Val: Op));
6639 NewOps.push_back(Elt: Substitutions.lookup_or(Val: Op, Default&: Op));
6640 }
6641 VPSingleDefRecipe *New;
6642 if (auto *B = dyn_cast<VPBlendRecipe>(Val: Old))
6643 New = B->cloneWithOperands(NewOperands: NewOps);
6644 else if (auto *W = dyn_cast<VPWidenRecipe>(Val: Old))
6645 New = W->cloneWithOperands(NewOperands: NewOps);
6646 else if (auto *Rep = dyn_cast<VPReplicateRecipe>(Val: Old))
6647 New = Rep->cloneWithOperands(NewOperands: NewOps);
6648 else
6649 New = cast<VPInstruction>(Val: Old)->cloneWithOperands(NewOperands: NewOps);
6650 New->insertBefore(InsertPos: Old);
6651 Substitutions[Old] = New;
6652 };
6653
6654 if (OrigExitingVPV != AnyOfSelect) {
6655 CloneChain(cast<VPSingleDefRecipe>(Val: OrigExitingVPV));
6656 NewExiting = Substitutions.lookup(Val: OrigExitingVPV);
6657 }
6658 NewPhiR->setOperand(I: 1, New: NewExiting);
6659 PhiR->replaceAllUsesWith(New: Plan->getPoison(Ty: PhiR->getScalarType()));
6660
6661 Builder.setInsertPoint(TheBB: MiddleVPBB, IP);
6662 FinalReductionResult =
6663 Builder.createAnyOfReduction(ChainOp: NewExiting, TrueVal: NewVal, FalseVal: Start, DL: ExitDL);
6664 } else {
6665 // If the vector reduction can be performed in a smaller type, we
6666 // truncate then extend the loop exit value to enable InstCombine to
6667 // evaluate the entire expression in the smaller type.
6668 VPValue *ReductionOp = NewExitingVPV;
6669 Instruction::CastOps ExtendOpc = Instruction::CastOpsEnd;
6670 if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
6671 assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
6672 assert(!RecurrenceDescriptor::isMinMaxRecurrenceKind(RecurrenceKind) &&
6673 "Unexpected truncated min-max recurrence!");
6674 Type *RdxTy = RdxDesc.getRecurrenceType();
6675 ExtendOpc = RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
6676 {
6677 VPBuilder::InsertPointGuard Guard(Builder);
6678 Builder.setInsertPoint(
6679 TheBB: NewExitingVPV->getDefiningRecipe()->getParent(),
6680 IP: std::next(x: NewExitingVPV->getDefiningRecipe()->getIterator()));
6681 ReductionOp =
6682 Builder.createWidenCast(Opcode: Instruction::Trunc, Op: NewExitingVPV, ResultTy: RdxTy);
6683 VPWidenCastRecipe *Extnd =
6684 Builder.createWidenCast(Opcode: ExtendOpc, Op: ReductionOp, ResultTy: PhiTy);
6685 if (PhiR->getOperand(N: 1) == NewExitingVPV)
6686 PhiR->setOperand(I: 1, New: Extnd);
6687 }
6688 }
6689
6690 VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
6691 PhiR->getFastMathFlagsOrNone());
6692 FinalReductionResult = Builder.createNaryOp(
6693 Opcode: VPInstruction::ComputeReductionResult, Operands: {ReductionOp}, Flags, DL: ExitDL);
6694 if (ExtendOpc != Instruction::CastOpsEnd)
6695 FinalReductionResult = Builder.createScalarCast(
6696 Opcode: ExtendOpc, Op: FinalReductionResult, ResultTy: PhiTy, DL: {});
6697 }
6698
6699 // Update all users outside the vector region. Also replace redundant
6700 // extracts.
6701 for (auto *U : to_vector(Range: OrigExitingVPV->users())) {
6702 auto *Parent = cast<VPRecipeBase>(Val: U)->getParent();
6703 if (FinalReductionResult == U || Parent->getParent())
6704 continue;
6705 // Skip ComputeReductionResult and FindIV reductions when they are not the
6706 // final result.
6707 if (match(U, P: m_VPInstruction<VPInstruction::ComputeReductionResult>()) ||
6708 (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind: RecurrenceKind) &&
6709 match(U, P: m_VPInstruction<Instruction::ICmp>())))
6710 continue;
6711 U->replaceUsesOfWith(From: OrigExitingVPV, To: FinalReductionResult);
6712
6713 // Look through ExtractLastPart.
6714 if (match(U, P: m_ExtractLastPart(Op0: m_VPValue())))
6715 U = cast<VPInstruction>(Val: U)->getSingleUser();
6716
6717 if (match(U, P: m_CombineOr(Ps: m_ExtractLane(Op0: m_VPValue(), Op1: m_VPValue()),
6718 Ps: m_ExtractLastLane(Op0: m_VPValue()))))
6719 cast<VPInstruction>(Val: U)->replaceAllUsesWith(New: FinalReductionResult);
6720 }
6721
6722 RecurKind RK = PhiR->getRecurrenceKind();
6723 if ((!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK) &&
6724 !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind: RK) &&
6725 !RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind: RK) &&
6726 !RecurrenceDescriptor::isFindLastRecurrenceKind(Kind: RK))) {
6727 VPBuilder PHBuilder(Plan->getVectorPreheader());
6728 VPValue *Iden = Plan->getOrAddLiveIn(
6729 V: getRecurrenceIdentity(K: RK, Tp: PhiTy, FMF: PhiR->getFastMathFlagsOrNone()));
6730 auto *ScaleFactorVPV = Plan->getConstantInt(BitWidth: 32, Val: 1);
6731 VPValue *StartV = PHBuilder.createNaryOp(
6732 Opcode: VPInstruction::ReductionStartVector,
6733 Operands: {PhiR->getStartValue(), Iden, ScaleFactorVPV}, Flags: *PhiR);
6734 PhiR->setOperand(I: 0, New: StartV);
6735 }
6736 }
6737
6738 RUN_VPLAN_PASS(VPlanTransforms::clearReductionWrapFlags, *Plan);
6739}
6740
6741void LoopVectorizationPlanner::attachRuntimeChecks(
6742 VPlan &Plan, GeneratedRTChecks &RTChecks, ElementCount VF, unsigned UF,
6743 bool HasBranchWeights) const {
6744 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
6745 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(N: 0)) {
6746 assert((!Config.OptForSize ||
6747 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled) &&
6748 "Cannot SCEV check stride or overflow when optimizing for size");
6749 RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan, SCEVCheckCond,
6750 SCEVCheckBlock, HasBranchWeights);
6751 }
6752 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
6753 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(N: 0)) {
6754 // VPlan-native path does not do any analysis for runtime checks
6755 // currently.
6756 assert((!Opts.enable_vplan_native_path || !Plan.isOuterLoop()) &&
6757 "Runtime checks are not supported for outer loops yet");
6758
6759 if (Config.OptForSize) {
6760 assert(
6761 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled &&
6762 "Cannot emit memory checks when optimizing for size, unless forced "
6763 "to vectorize.");
6764 ORE->emit(RemarkBuilder: [&]() {
6765 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
6766 OrigLoop->getStartLoc(),
6767 OrigLoop->getHeader())
6768 << "Code-size may be reduced by not forcing "
6769 "vectorization, or by source-code modifications "
6770 "eliminating the need for runtime checks "
6771 "(e.g., adding 'restrict').";
6772 });
6773 }
6774 const auto &RtPtrChecking = *Legal->getRuntimePointerChecking();
6775 auto DiffChecks = RtPtrChecking.getDiffChecks();
6776 auto HasUnsupportedBounds = [&]() {
6777 // VPSCEVExpander expands AddRecs in the plan's entry, not the check
6778 // block.
6779 auto IsUnsupported = [](const SCEV *S) {
6780 return SCEVExprContains(Root: S, Pred: IsaPred<SCEVAddRecExpr>);
6781 };
6782 if (DiffChecks)
6783 return any_of(Range&: *DiffChecks, P: [&](const PointerDiffInfo &C) {
6784 return IsUnsupported(C.SrcStart) || IsUnsupported(C.SinkStart);
6785 });
6786 return any_of(Range: RtPtrChecking.CheckingGroups,
6787 P: [&](const RuntimeCheckingPtrGroup &CG) {
6788 return IsUnsupported(CG.Low) || IsUnsupported(CG.High);
6789 });
6790 };
6791 // The VPlan expander cannot hoist bounds out of an enclosing loop.
6792 if (OrigLoop->getParentLoop() || HasUnsupportedBounds())
6793 return RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan,
6794 MemCheckCond, MemCheckBlock, HasBranchWeights);
6795
6796 // Erase the temporary IR before recipe expansion can reuse its values.
6797 RTChecks.eraseMemCheckBlock();
6798 RUN_VPLAN_PASS(VPlanTransforms::attachMemoryChecks, Plan, RtPtrChecking,
6799 *PSE.getSE(), VF, UF, OrigLoop->getStartLoc(),
6800 HasBranchWeights);
6801 }
6802}
6803
6804void LoopVectorizationPlanner::addMinimumIterationCheck(
6805 VPlan &Plan, ElementCount VF, unsigned UF,
6806 ElementCount MinProfitableTripCount) const {
6807 const uint32_t *BranchWeights =
6808 hasBranchWeightMD(I: *OrigLoop->getLoopLatch()->getTerminator())
6809 ? &MinItersBypassWeights[0]
6810 : nullptr;
6811 RUN_VPLAN_PASS(VPlanTransforms::addMinimumIterationCheck, Plan, VF, UF,
6812 MinProfitableTripCount, Plan.requiresScalarEpilogue(),
6813 Plan.hasTailFolded(), OrigLoop, BranchWeights,
6814 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
6815 PSE, Plan.getEntry());
6816}
6817
6818// Determine how to lower the epilogue, which depends on 1) optimising
6819// for minimum code-size, 2) tail-folding compiler options, 3) loop
6820// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
6821// is suitable for tail-folding.
6822// This function determines epilogue lowering for the main vector loop while
6823// epilogue lowering for the tail-folded epilogue path will be handled
6824// separately in getEpilogueTailLowering.
6825static EpilogueLowering
6826getEpilogueLowering(const VectorizeOptions &Opts, Function *F, Loop *L,
6827 LoopVectorizeHints &Hints, bool OptForSize,
6828 TargetTransformInfo *TTI, TargetLibraryInfo *TLI,
6829 LoopVectorizationLegality &LVL,
6830 InterleavedAccessInfo *IAI) {
6831 // 1) OptSize takes precedence over all other options, i.e. if this is set,
6832 // don't look at hints or options, and don't request an epilogue.
6833 if (F->hasOptSize() ||
6834 (OptForSize && Hints.getForce() != LoopVectorizeHints::FK_Enabled))
6835 return CM_EpilogueNotAllowedOptSize;
6836
6837 // 2) If set, obey the directives
6838 if (Opts.tail_folding_policy) {
6839 switch (*Opts.tail_folding_policy) {
6840 case TailFoldingPolicyTy::None:
6841 return CM_EpilogueAllowed;
6842 case TailFoldingPolicyTy::PreferFoldTail:
6843 return CM_EpilogueNotNeededFoldTail;
6844 case TailFoldingPolicyTy::MustFoldTail:
6845 return CM_EpilogueNotAllowedFoldTail;
6846 };
6847 }
6848
6849 // 3) If set, obey the hints
6850 switch (Hints.getPredicate()) {
6851 case LoopVectorizeHints::FK_Enabled:
6852 return CM_EpilogueNotNeededFoldTail;
6853 case LoopVectorizeHints::FK_Disabled:
6854 return CM_EpilogueAllowed;
6855 };
6856
6857 // 4) if the TTI hook indicates this is profitable, request tail-folding.
6858 TailFoldingInfo TFI(TLI, &LVL, IAI);
6859 if (TTI->preferTailFoldingOverEpilogue(TFI: &TFI))
6860 return CM_EpilogueNotNeededFoldTail;
6861
6862 return CM_EpilogueAllowed;
6863}
6864
6865// Emit a remark if there are stores to floats that required a floating point
6866// extension. If the vectorized loop was generated with floating point there
6867// will be a performance penalty from the conversion overhead and the change in
6868// the vector width.
6869static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
6870 SmallVector<Instruction *, 4> Worklist;
6871 for (BasicBlock *BB : L->getBlocks()) {
6872 for (Instruction &Inst : *BB) {
6873 if (auto *S = dyn_cast<StoreInst>(Val: &Inst)) {
6874 if (S->getValueOperand()->getType()->isFloatTy())
6875 Worklist.push_back(Elt: S);
6876 }
6877 }
6878 }
6879
6880 // Traverse the floating point stores upwards searching, for floating point
6881 // conversions.
6882 SmallPtrSet<const Instruction *, 4> Visited;
6883 SmallPtrSet<const Instruction *, 4> EmittedRemark;
6884 while (!Worklist.empty()) {
6885 auto *I = Worklist.pop_back_val();
6886 if (!L->contains(Inst: I))
6887 continue;
6888 if (!Visited.insert(Ptr: I).second)
6889 continue;
6890
6891 // Emit a remark if the floating point store required a floating
6892 // point conversion.
6893 // TODO: More work could be done to identify the root cause such as a
6894 // constant or a function return type and point the user to it.
6895 if (isa<FPExtInst>(Val: I) && EmittedRemark.insert(Ptr: I).second)
6896 ORE->emit(RemarkBuilder: [&]() {
6897 return OptimizationRemarkAnalysis(LV_NAME, "VectorMixedPrecision",
6898 I->getDebugLoc(), L->getHeader())
6899 << "floating point conversion changes vector width. "
6900 << "Mixed floating point precision requires an up/down "
6901 << "cast that will negatively impact performance.";
6902 });
6903
6904 for (Use &Op : I->operands())
6905 if (auto *OpI = dyn_cast<Instruction>(Val&: Op))
6906 Worklist.push_back(Elt: OpI);
6907 }
6908}
6909
6910/// For loops with uncountable early exits, find the cost of doing work when
6911/// exiting the loop early, such as calculating the final exit values of
6912/// variables used outside the loop.
6913/// TODO: This is currently overly pessimistic because the loop may not take
6914/// the early exit, but better to keep this conservative for now. In future,
6915/// it might be possible to relax this by using branch probabilities.
6916static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx,
6917 VPlan &Plan, ElementCount VF) {
6918 InstructionCost Cost = 0;
6919 for (auto *ExitVPBB : Plan.getExitBlocks()) {
6920 for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
6921 // If the predecessor is not the middle.block, then it must be the
6922 // vector.early.exit block, which may contain work to calculate the exit
6923 // values of variables used outside the loop.
6924 if (PredVPBB != Plan.getMiddleBlock()) {
6925 LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
6926 << PredVPBB->getName() << ":\n");
6927 Cost += PredVPBB->cost(VF, Ctx&: CostCtx);
6928 }
6929 }
6930 }
6931 return Cost;
6932}
6933
6934/// This function determines whether or not it's still profitable to vectorize
6935/// the loop given the extra work we have to do outside of the loop:
6936/// 1. Perform the runtime checks before entering the loop to ensure it's safe
6937/// to vectorize.
6938/// 2. In the case of loops with uncountable early exits, we may have to do
6939/// extra work when exiting the loop early, such as calculating the final
6940/// exit values of variables used outside the loop.
6941/// 3. The middle block.
6942static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
6943 VectorizationFactor &VF, Loop *L,
6944 PredicatedScalarEvolution &PSE,
6945 VPCostContext &CostCtx, VPlan &Plan,
6946 EpilogueLowering SEL,
6947 std::optional<unsigned> VScale) {
6948 InstructionCost RtC = Checks.getCost();
6949 if (!RtC.isValid())
6950 return false;
6951
6952 // When interleaving only scalar and vector cost will be equal, which in turn
6953 // would lead to a divide by 0. Fall back to hard threshold.
6954 if (VF.Width.isScalar()) {
6955 // TODO: Should we rename VectorizeMemoryCheckThreshold?
6956 if (RtC > VectorizerParams::VectorizeMemoryCheckThreshold) {
6957 LLVM_DEBUG(
6958 dbgs()
6959 << "LV: Interleaving only is not profitable due to runtime checks\n");
6960 return false;
6961 }
6962 return true;
6963 }
6964
6965 // The scalar cost should only be 0 when vectorizing with a user specified
6966 // VF/IC. In those cases, runtime checks should always be generated.
6967 uint64_t ScalarC = VF.ScalarCost.getValue();
6968 if (ScalarC == 0)
6969 return true;
6970
6971 InstructionCost TotalCost = RtC;
6972 // Add on the cost of any work required in the vector early exit block, if
6973 // one exists.
6974 TotalCost += calculateEarlyExitCost(CostCtx, Plan, VF: VF.Width);
6975 TotalCost += Plan.getMiddleBlock()->cost(VF: VF.Width, Ctx&: CostCtx);
6976
6977 // First, compute the minimum iteration count required so that the vector
6978 // loop outperforms the scalar loop.
6979 // The total cost of the scalar loop is
6980 // ScalarC * TC
6981 // where
6982 // * TC is the actual trip count of the loop.
6983 // * ScalarC is the cost of a single scalar iteration.
6984 //
6985 // The total cost of the vector loop is
6986 // TotalCost + VecC * (TC / VF) + EpiC
6987 // where
6988 // * TotalCost is the sum of the costs cost of
6989 // - the generated runtime checks, i.e. RtC
6990 // - performing any additional work in the vector.early.exit block for
6991 // loops with uncountable early exits.
6992 // - the middle block, if ExpectedTC <= VF.Width.
6993 // * VecC is the cost of a single vector iteration.
6994 // * TC is the actual trip count of the loop
6995 // * VF is the vectorization factor
6996 // * EpiCost is the cost of the generated epilogue, including the cost
6997 // of the remaining scalar operations.
6998 //
6999 // Vectorization is profitable once the total vector cost is less than the
7000 // total scalar cost:
7001 // TotalCost + VecC * (TC / VF) + EpiC < ScalarC * TC
7002 //
7003 // Now we can compute the minimum required trip count TC as
7004 // VF * (TotalCost + EpiC) / (ScalarC * VF - VecC) < TC
7005 //
7006 // For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
7007 // the computations are performed on doubles, not integers and the result
7008 // is rounded up, hence we get an upper estimate of the TC.
7009 unsigned IntVF = estimateElementCount(VF: VF.Width, VScale);
7010 uint64_t Div = ScalarC * IntVF - VF.Cost.getValue();
7011 uint64_t MinTC1 =
7012 Div == 0 ? 0 : divideCeil(Numerator: TotalCost.getValue() * IntVF, Denominator: Div);
7013
7014 // Second, compute a minimum iteration count so that the cost of the
7015 // runtime checks is only a fraction of the total scalar loop cost. This
7016 // adds a loop-dependent bound on the overhead incurred if the runtime
7017 // checks fail. In case the runtime checks fail, the cost is RtC + ScalarC
7018 // * TC. To bound the runtime check to be a fraction 1/X of the scalar
7019 // cost, compute
7020 // RtC < ScalarC * TC * (1 / X) ==> RtC * X / ScalarC < TC
7021 uint64_t MinTC2 = divideCeil(Numerator: RtC.getValue() * 10, Denominator: ScalarC);
7022
7023 // Now pick the larger minimum. If it is not a multiple of VF and an epilogue
7024 // is allowed, choose the next closest multiple of VF. This should partly
7025 // compensate for ignoring the epilogue cost.
7026 uint64_t MinTC = std::max(a: MinTC1, b: MinTC2);
7027 if (SEL == CM_EpilogueAllowed)
7028 MinTC = alignTo(Value: MinTC, Align: IntVF);
7029 VF.MinProfitableTripCount = ElementCount::getFixed(MinVal: MinTC);
7030
7031 LLVM_DEBUG(
7032 dbgs() << "LV: Minimum required TC for runtime checks to be profitable:"
7033 << VF.MinProfitableTripCount << "\n");
7034
7035 // Skip vectorization if the expected trip count is less than the minimum
7036 // required trip count.
7037 if (auto ExpectedTC = getSmallBestKnownTC(Opts: CostCtx.Opts, PSE, L)) {
7038 if (ElementCount::isKnownLT(LHS: *ExpectedTC, RHS: VF.MinProfitableTripCount)) {
7039 LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
7040 "trip count < minimum profitable VF ("
7041 << *ExpectedTC << " < " << VF.MinProfitableTripCount
7042 << ")\n");
7043
7044 return false;
7045 }
7046 }
7047 return true;
7048}
7049
7050LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
7051 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced),
7052 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced) {}
7053
7054/// ResumeForEpilogue markers in the main plan, used by the epilogue plan.
7055struct MainPlanResumeMarkers {
7056 VPInstruction *CanIVResume;
7057 VPInstruction *VectorTC;
7058 SmallVector<VPInstruction *> ResumeValues;
7059};
7060
7061/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
7062/// vectorization.
7063static MainPlanResumeMarkers preparePlanForMainVectorLoop(VPlan &MainPlan) {
7064 using namespace VPlanPatternMatch;
7065 // When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
7066 // introduce multiple uses of undef/poison. If the reduction start value may
7067 // be undef or poison it needs to be frozen and the frozen start has to be
7068 // used when computing the reduction result. We also need to use the frozen
7069 // value in the resume phi generated by the main vector loop, as this is also
7070 // used to compute the reduction result after the epilogue vector loop. The
7071 // epilogue plan re-uses the frozen values.
7072 VPBuilder Builder(MainPlan.getEntry());
7073 for (VPInstruction &VPI :
7074 make_isa_range<VPInstruction>(Range&: *MainPlan.getMiddleBlock())) {
7075 VPValue *OrigStart;
7076 if (!matchFindIVResult(VPI: &VPI, ReducedIV: m_VPValue(), Start: m_VPValue(V&: OrigStart)))
7077 continue;
7078 if (isGuaranteedNotToBeUndefOrPoison(V: OrigStart->getLiveInIRValue()))
7079 continue;
7080 VPInstruction *Freeze = Builder.createFreeze(Op: OrigStart, DL: {}, Name: "fr");
7081 VPI.setOperand(I: 2, New: Freeze);
7082 OrigStart->replaceUsesWithIf(New: Freeze, ShouldReplace: IsaPred<VPPhi>);
7083 }
7084
7085 VPValue *VectorTC = nullptr;
7086 auto *Term =
7087 MainPlan.getVectorLoopRegion()->getExitingBasicBlock()->getTerminator();
7088 [[maybe_unused]] bool MatchedTC =
7089 match(V: Term, P: m_BranchOnCount(Op0: m_VPValue(), Op1: m_VPValue(V&: VectorTC)));
7090 assert(MatchedTC && "must match vector trip count");
7091
7092 VPBasicBlock *MiddleVPBB = MainPlan.getMiddleBlock();
7093 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
7094 VPInstruction *VectorTCMarker = MiddleBuilder.createNaryOp(
7095 Opcode: VPInstruction::ResumeForEpilogue,
7096 Operands: {VectorTC, MainPlan.getZero(Ty: VectorTC->getScalarType())});
7097
7098 // If there is a suitable resume value for the canonical induction in the
7099 // scalar (which will become vector) epilogue loop, use it and move it to the
7100 // beginning of the scalar preheader. Otherwise create it below.
7101 VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader();
7102 auto ResumePhiIter =
7103 find_if(Range: MainScalarPH->phis(), P: [VectorTC](VPRecipeBase &R) {
7104 return match(V: &R, P: m_VPInstruction<Instruction::PHI>(Ops: m_Specific(VPV: VectorTC),
7105 Ops: m_ZeroInt()));
7106 });
7107 VPPhi *ResumePhi = nullptr;
7108 if (ResumePhiIter == MainScalarPH->phis().end()) {
7109 assert(MainPlan.getVectorLoopRegion()->getCanonicalIV() &&
7110 "canonical IV must exist");
7111 Type *Ty = VectorTC->getScalarType();
7112 VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
7113 ResumePhi = ScalarPHBuilder.createScalarPhi(
7114 IncomingValues: {VectorTC, MainPlan.getZero(Ty)}, DL: {}, Name: "vec.epilog.resume.val");
7115 } else {
7116 ResumePhi = cast<VPPhi>(Val: &*ResumePhiIter);
7117 ResumePhi->setName("vec.epilog.resume.val");
7118 if (&MainScalarPH->front() != ResumePhi)
7119 ResumePhi->moveBefore(BB&: *MainScalarPH, I: MainScalarPH->begin());
7120 }
7121
7122 // Create a ResumeForEpilogue for the canonical IV resume and its bypass value
7123 // as the first non-phi, to keep them alive for the epilogue.
7124 VPBuilder ResumeBuilder(MainScalarPH);
7125 VPInstruction *CanIVResume = ResumeBuilder.createNaryOp(
7126 Opcode: VPInstruction::ResumeForEpilogue, Operands: {ResumePhi, ResumePhi->getOperand(N: 1)});
7127
7128 // Create ResumeForEpilogue instructions for the resume phis of the
7129 // VPIRPhis and their bypass values in the scalar header of the main plan and
7130 // return them so they can be used as resume values when vectorizing the
7131 // epilogue.
7132 auto ResumeValues = to_vector(
7133 Range: map_range(C: MainPlan.getScalarHeader()->phis(), F: [&](VPRecipeBase &R) {
7134 assert(isa<VPIRPhi>(R) &&
7135 "only VPIRPhis expected in the scalar header");
7136 VPValue *MainResumePhi = R.getOperand(N: 0);
7137 VPValue *Bypass = MainResumePhi->getDefiningRecipe()->getOperand(N: 1);
7138 return ResumeBuilder.createNaryOp(Opcode: VPInstruction::ResumeForEpilogue,
7139 Operands: {MainResumePhi, Bypass});
7140 }));
7141 return {.CanIVResume: CanIVResume, .VectorTC: VectorTCMarker, .ResumeValues: std::move(ResumeValues)};
7142}
7143
7144/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
7145/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes.
7146static void preparePlanForEpilogueVectorLoop(
7147 VPlan &MainPlan, VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs,
7148 ElementCount MainLoopVF, unsigned MainLoopUF, ElementCount EpilogueVF,
7149 LoopVectorizationPlanner &LVP, VFSelectionContext &Config,
7150 ScalarEvolution &SE, const MainPlanResumeMarkers &Markers) {
7151 // Build a map from the scalar-header PHI to the ResumeForEpilogue markers
7152 // from the main plan.
7153 // TODO: Replace the IR PHI key.
7154 DenseMap<PHINode *, VPInstruction *> IRPhiToResumeForEpi;
7155 for (auto [HeaderPhi, ResumeForEpi] :
7156 zip_equal(t: MainPlan.getScalarHeader()->phis(), u: Markers.ResumeValues))
7157 IRPhiToResumeForEpi[&cast<VPIRPhi>(Val&: HeaderPhi).getIRPhi()] = ResumeForEpi;
7158 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
7159 VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
7160 Header->setName("vec.epilog.vector.body");
7161
7162 VPValue *IV = VectorLoop->getCanonicalIV();
7163 // When vectorizing the epilogue loop, the canonical induction needs to start
7164 // at the resume value from the main vector loop. Find the resume value
7165 // created during execution of the main VPlan. Add this resume value as an
7166 // offset to the canonical IV of the epilogue loop.
7167 VPValue *VPV = Plan.getOrAddLiveIn(V: Markers.CanIVResume->getUnderlyingValue());
7168 assert(all_of(IV->users(),
7169 [](const VPUser *U) {
7170 if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe>(U))
7171 return true;
7172 auto *VPI = dyn_cast<VPInstruction>(U);
7173 if (!VPI)
7174 return false;
7175 // Only users linear in the canonical IV (e.g. scaling it by
7176 // a stride) stay correct when offsetting it below; others,
7177 // like header masks, may need an epilogue-relative IV.
7178 return Instruction::isCast(VPI->getOpcode()) ||
7179 is_contained({Instruction::Add, Instruction::Mul,
7180 Instruction::Shl},
7181 VPI->getOpcode());
7182 }) &&
7183 "the canonical IV should only be used by its increment, casts, "
7184 "stride scaling, ScalarIVSteps or DerivedIV when resetting the start "
7185 "value");
7186 VPBuilder Builder(Header, Header->getFirstNonPhi());
7187 VPInstruction *Add = Builder.createAdd(LHS: IV, RHS: VPV);
7188 // Replace all users of the canonical IV and its increment with the offset
7189 // version, except for the Add itself and the canonical IV increment.
7190 auto *Increment = vputils::findCanonicalIVIncrement(Plan);
7191 assert(Increment && "Must have a canonical IV increment at this point");
7192 IV->replaceUsesWithIf(
7193 New: Add, ShouldReplace: [Add, Increment](VPUser *U) { return U != Add && U != Increment; });
7194 VPInstruction *OffsetIVInc =
7195 VPBuilder::getToInsertAfter(R: Increment).createAdd(LHS: Increment, RHS: VPV);
7196 Increment->replaceAllUsesWith(New: OffsetIVInc);
7197 OffsetIVInc->setOperand(I: 0, New: Increment);
7198
7199 // Resume values must be created in the vector preheader.
7200 VPBasicBlock *VectorPH = Plan.getVectorPreheader();
7201 VPBuilder PHBuilder(VectorPH, VectorPH->getFirstNonPhi());
7202
7203 // Ensure that the start values for all header phi recipes are updated before
7204 // vectorizing the epilogue loop.
7205 for (VPRecipeBase &R : Header->phis()) {
7206 VPValue *ResumeVPV = nullptr;
7207 // TODO: Move setting of resume values to prepareToExecute.
7208 if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(Val: &R)) {
7209 // Find the reduction result by searching users of the phi or its backedge
7210 // value.
7211 auto IsReductionResult = [](VPRecipeBase *R) {
7212 auto *VPI = dyn_cast<VPInstruction>(Val: R);
7213 return VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult;
7214 };
7215 auto *RdxResult = cast<VPInstruction>(
7216 Val: vputils::findRecipe(Start: ReductionPhi->getBackedgeValue(), Pred: IsReductionResult));
7217 assert(RdxResult && "expected to find reduction result");
7218
7219 VPInstruction *ResumeForEpi = IRPhiToResumeForEpi.at(
7220 Val: cast<PHINode>(Val: ReductionPhi->getUnderlyingInstr()));
7221
7222 // Check for FindIV pattern by looking for icmp user of RdxResult.
7223 // The pattern is: select(icmp ne RdxResult, Sentinel), RdxResult, Start
7224 using namespace VPlanPatternMatch;
7225 VPValue *SentinelVPV = nullptr;
7226 bool IsFindIV = any_of(Range: RdxResult->users(), P: [&](VPUser *U) {
7227 return match(U, P: VPlanPatternMatch::m_SpecificICmp(
7228 MatchPred: ICmpInst::ICMP_NE, Op0: m_Specific(VPV: RdxResult),
7229 Op1: m_VPValue(V&: SentinelVPV)));
7230 });
7231
7232 RecurKind RK = ReductionPhi->getRecurrenceKind();
7233 ResumeVPV = Plan.getOrAddLiveIn(V: ResumeForEpi->getUnderlyingValue());
7234 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK) || IsFindIV) {
7235 VPValue *BypassOp = ResumeForEpi->getOperand(N: 1);
7236 assert((isa<VPIRValue>(BypassOp) ||
7237 VPlanPatternMatch::match(
7238 BypassOp,
7239 m_VPInstruction<Instruction::Freeze>(m_VPValue()))) &&
7240 "expected live-in or Freeze");
7241 VPValue *StartV = Plan.getOrAddLiveIn(V: BypassOp->getUnderlyingValue());
7242 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK)) {
7243 // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
7244 // start value; compare the final value from the main vector loop
7245 // to the start value.
7246 ResumeVPV = PHBuilder.createICmp(Pred: CmpInst::ICMP_NE, A: ResumeVPV, B: StartV);
7247 } else {
7248 assert(isa<VPIRValue>(SentinelVPV) &&
7249 "sentinel must be a live-in to be used in the preheader");
7250 // Use the start value frozen for the main plan, if any, when
7251 // computing the reduction result.
7252 if (match(V: BypassOp, P: m_Freeze(Op0: m_VPValue())))
7253 for (VPUser *U : RdxResult->users()) {
7254 auto *VPI = dyn_cast<VPInstruction>(Val: U);
7255 if (VPI && matchFindIVResult(VPI, ReducedIV: m_VPValue(), Start: m_VPValue()))
7256 VPI->setOperand(I: 2, New: StartV);
7257 }
7258
7259 // Adjust resume: select(icmp eq ResumeVPV, StartV), Sentinel,
7260 // ResumeVPV
7261 VPValue *Cmp =
7262 PHBuilder.createICmp(Pred: CmpInst::ICMP_EQ, A: ResumeVPV, B: StartV);
7263 ResumeVPV = PHBuilder.createSelect(Cond: Cmp, TrueVal: SentinelVPV, FalseVal: ResumeVPV);
7264 }
7265 // TODO: materializeBroadcasts does not cover values in the vector
7266 // preheader.
7267 ReductionPhi->setStartValue(
7268 PHBuilder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: ResumeVPV));
7269 continue;
7270 }
7271 if (auto *VPI = dyn_cast<VPInstruction>(Val: ReductionPhi->getStartValue())) {
7272 assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
7273 "unexpected start value");
7274 // Partial sub-reductions always start at 0 and account for the
7275 // reduction start value in a final subtraction. Update it to use the
7276 // resume value from the main vector loop.
7277 if (ReductionPhi->getVFScaleFactor() > 1 &&
7278 RecurrenceDescriptor::isSubRecurrenceKind(Kind: RK)) {
7279 auto *Sub = cast<VPInstruction>(Val: RdxResult->getSingleUser());
7280 assert((Sub->getOpcode() == Instruction::Sub ||
7281 Sub->getOpcode() == Instruction::FSub) &&
7282 "Unexpected opcode");
7283 assert(isa<VPIRValue>(Sub->getOperand(0)) &&
7284 "Expected operand to match the original start value of the "
7285 "reduction");
7286 // For integer sub-reductions, verify start value is zero.
7287 // For FP sub-reductions, verify start value is negative zero.
7288 [[maybe_unused]] auto StartValueIsIdentity = [&] {
7289 Value *IdentityValue =
7290 getRecurrenceIdentity(K: RK, Tp: ResumeVPV->getScalarType(),
7291 FMF: ReductionPhi->getFastMathFlagsOrNone());
7292 auto *StartValue = dyn_cast<VPIRValue>(Val: VPI->getOperand(N: 0));
7293 return StartValue && StartValue->getValue() == IdentityValue;
7294 };
7295 assert(StartValueIsIdentity() &&
7296 "Expected start value for partial sub-reduction to be zero "
7297 "(or negative zero)");
7298
7299 Sub->setOperand(I: 0, New: ResumeVPV);
7300 } else {
7301 // The start vector may be shared by multiple reductions, so create
7302 // a new one using the resume.
7303 auto *NewStart = cast<VPInstruction>(Val: VPI->clone());
7304 NewStart->setOperand(I: 0, New: ResumeVPV);
7305 NewStart->insertBefore(InsertPos: VPI);
7306 ReductionPhi->setStartValue(NewStart);
7307 }
7308 continue;
7309 }
7310 } else {
7311 // Retrieve the induction resume value via ResumeForEpilogue.
7312 PHINode *IndPhi = cast<VPWidenInductionRecipe>(Val: &R)->getPHINode();
7313 ResumeVPV = Plan.getOrAddLiveIn(
7314 V: IRPhiToResumeForEpi.at(Val: IndPhi)->getUnderlyingValue());
7315 }
7316 assert(ResumeVPV && "Must have a resume value");
7317 cast<VPHeaderPHIRecipe>(Val: &R)->setStartValue(ResumeVPV);
7318 }
7319
7320 // Re-use the trip count and steps expanded for the main loop, as skeleton
7321 // creation needs it as a value that dominates both the scalar and vector
7322 // epilogue loops.
7323 // TODO: This is a workaround needed for epilogue vectorization and it
7324 // should be removed once induction resume value creation is done
7325 // directly in VPlan.
7326 for (VPExpandSCEVRecipe &ExpandR : make_early_inc_range(
7327 Range: make_isa_range<VPExpandSCEVRecipe>(Range&: *Plan.getEntry()))) {
7328 assert(ExpandedSCEVs.contains(ExpandR.getSCEV()) &&
7329 "Epilogue plan needs a SCEV not expanded for the main loop");
7330 VPValue *ExpandedVal =
7331 Plan.getOrAddLiveIn(V: ExpandedSCEVs.lookup(Val: ExpandR.getSCEV()));
7332 ExpandR.replaceAllUsesWith(New: ExpandedVal);
7333 if (Plan.getTripCount() == &ExpandR)
7334 Plan.resetTripCount(NewTripCount: ExpandedVal);
7335 ExpandR.eraseFromParent();
7336 }
7337
7338 auto VScale = Config.getVScaleForTuning();
7339 unsigned MainLoopStep = estimateElementCount(VF: MainLoopVF * MainLoopUF, VScale);
7340 unsigned EpilogueLoopStep = estimateElementCount(VF: EpilogueVF, VScale);
7341 RUN_VPLAN_PASS(VPlanTransforms::addMinimumVectorEpilogueIterationCheck, Plan,
7342 Plan.getOrAddLiveIn(Markers.VectorTC->getUnderlyingValue()),
7343 Plan.requiresScalarEpilogue(), EpilogueVF, MainLoopStep,
7344 EpilogueLoopStep, SE);
7345}
7346
7347static void
7348fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, VPlan &BestEpiPlan,
7349 ArrayRef<VPInstruction *> ResumeValues) {
7350 auto *ScalarPH = cast<VPIRBasicBlock>(Val: BestEpiPlan.getScalarPreheader());
7351 BasicBlock *PH = ScalarPH->getIRBasicBlock();
7352 if (ScalarPH->hasPredecessors()) {
7353 // Fix resume values for inductions and reductions from the additional
7354 // bypass block using the incoming values from the main loop's resume phis.
7355 // ResumeValues correspond 1:1 with the scalar loop header phis.
7356 for (auto [ResumeV, HeaderPhi] :
7357 zip(t&: ResumeValues, u: BestEpiPlan.getScalarHeader()->phis())) {
7358 auto *HeaderPhiR = cast<VPIRPhi>(Val: &HeaderPhi);
7359 auto *EpiResumePhi =
7360 cast<PHINode>(Val: HeaderPhiR->getIRPhi().getIncomingValueForBlock(BB: PH));
7361 if (EpiResumePhi->getBasicBlockIndex(BB: BypassBlock) == -1)
7362 continue;
7363 auto *MainResumePhi = cast<PHINode>(Val: ResumeV->getUnderlyingValue());
7364 EpiResumePhi->setIncomingValueForBlock(
7365 BB: BypassBlock, V: MainResumePhi->getIncomingValueForBlock(BB: BypassBlock));
7366 }
7367 }
7368}
7369
7370/// Connect the epilogue vector loop generated for \p EpiPlan to the main vector
7371/// loop, after both plans have executed, updating the branch from the iteration
7372/// count check of the main loop, as well as updating various phis.
7373static void connectEpilogueVectorLoop(VPlan &EpiPlan, DominatorTree *DT,
7374 VPIRBasicBlock *VecEpilogueIterCheckVPBB,
7375 ArrayRef<VPInstruction *> ResumeValues) {
7376 ArrayRef<VPBlockBase *> Preds = VecEpilogueIterCheckVPBB->getPredecessors();
7377 BasicBlock *MainLoopIterationCountCheck =
7378 cast<VPIRBasicBlock>(Val: Preds.front())->getIRBasicBlock();
7379 BasicBlock *VecEpilogueIterationCountCheck =
7380 VecEpilogueIterCheckVPBB->getIRBasicBlock();
7381 BasicBlock *VecEpiloguePreHeader =
7382 cast<CondBrInst>(Val: VecEpilogueIterationCountCheck->getTerminator())
7383 ->getSuccessor(i: 1);
7384 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
7385
7386 MainLoopIterationCountCheck->getTerminator()->replaceSuccessorWith(
7387 OldBB: VecEpilogueIterationCountCheck, NewBB: VecEpiloguePreHeader);
7388 DTU.applyUpdates(Updates: {{DominatorTree::Delete, MainLoopIterationCountCheck,
7389 VecEpilogueIterationCountCheck},
7390 {DominatorTree::Insert, MainLoopIterationCountCheck,
7391 VecEpiloguePreHeader}});
7392
7393 // The vec.epilog.iter.check block may contain Phi nodes from inductions
7394 // or reductions which merge control-flow from the latch block and the
7395 // middle block. Update the incoming values here and move the Phi into the
7396 // preheader.
7397 SmallVector<PHINode *, 4> PhisInBlock(
7398 llvm::make_pointer_range(Range: VecEpilogueIterationCountCheck->phis()));
7399
7400 for (PHINode *Phi : PhisInBlock) {
7401 Phi->moveBefore(InsertPos: VecEpiloguePreHeader->getFirstNonPHIIt());
7402 Phi->replaceIncomingBlockWith(
7403 Old: VecEpilogueIterationCountCheck->getSinglePredecessor(),
7404 New: VecEpilogueIterationCountCheck);
7405 }
7406
7407 // VecEpilogueIterationCountCheck conditionally skips over the epilogue loop
7408 // after executing the main loop. We need to update the resume values of
7409 // inductions and reductions during epilogue vectorization.
7410 fixScalarResumeValuesFromBypass(BypassBlock: VecEpilogueIterationCountCheck, BestEpiPlan&: EpiPlan,
7411 ResumeValues);
7412
7413 // Remove dead phis that were moved to the epilogue preheader but are unused
7414 // (e.g., resume phis for inductions not widened in the epilogue vector loop).
7415 for (PHINode &Phi : make_early_inc_range(Range: VecEpiloguePreHeader->phis()))
7416 if (Phi.use_empty())
7417 Phi.eraseFromParent();
7418}
7419
7420bool LoopVectorizePass::processLoop(Loop *L) {
7421 const VectorizeOptions &Opts =
7422 L->getHeader()->getContext().getLibraryOptions().get<VectorizeOptions>();
7423 assert((Opts.enable_vplan_native_path || L->isInnermost()) &&
7424 "VPlan-native path is not enabled. Only process inner loops.");
7425
7426 LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in '"
7427 << L->getHeader()->getParent()->getName() << "' from "
7428 << L->getLocStr() << "\n");
7429
7430 LoopVectorizeHints Hints(
7431 Opts, L, InterleaveOnlyWhenForced || !Opts.interleave_loops, *ORE, TTI);
7432
7433 LLVM_DEBUG(
7434 dbgs() << "LV: Loop hints:"
7435 << " force="
7436 << (Hints.getForce() == LoopVectorizeHints::FK_Disabled
7437 ? "disabled"
7438 : (Hints.getForce() == LoopVectorizeHints::FK_Enabled
7439 ? "enabled"
7440 : "?"))
7441 << " width=" << Hints.getWidth()
7442 << " interleave=" << Hints.getInterleave() << "\n");
7443
7444 // Function containing loop
7445 Function *F = L->getHeader()->getParent();
7446
7447 // Looking at the diagnostic output is the only way to determine if a loop
7448 // was vectorized (other than looking at the IR or machine code), so it
7449 // is important to generate an optimization remark for each loop. Most of
7450 // these messages are generated as OptimizationRemarkAnalysis. Remarks
7451 // generated as OptimizationRemark and OptimizationRemarkMissed are
7452 // less verbose reporting vectorized loops and unvectorized loops that may
7453 // benefit from vectorization, respectively.
7454
7455 if (!Hints.allowVectorization(
7456 F, L, VectorizeOnlyWhenForced: VectorizeOnlyWhenForced || !Opts.vectorize_loops)) {
7457 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
7458 return false;
7459 }
7460
7461 PredicatedScalarEvolution PSE(*SE, *L);
7462
7463 // Query this against the original loop and save it here because the profile
7464 // of the original loop header may change as the transformation happens.
7465 bool OptForSize = llvm::shouldOptimizeForSize(
7466 BB: L->getHeader(), PSI,
7467 BFI: PSI && PSI->hasProfileSummary() ? &GetBFI() : nullptr,
7468 QueryType: PGSOQueryType::IRPass);
7469
7470 // Check if it is legal to vectorize the loop.
7471 LoopVectorizationRequirements Requirements;
7472 LoopVectorizationLegality LVL(Opts, L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE,
7473 &Requirements, &Hints, DB, AC,
7474 /*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
7475 if (!LVL.canVectorize(UseVPlanNativePath: Opts.enable_vplan_native_path)) {
7476 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
7477 Hints.emitRemarkWithHints();
7478 return false;
7479 }
7480
7481 bool IsInnerLoop = L->isInnermost();
7482
7483 // Outer loops require a computable trip count.
7484 if (!IsInnerLoop && isa<SCEVCouldNotCompute>(Val: PSE.getBackedgeTakenCount())) {
7485 LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n");
7486 return false;
7487 }
7488
7489 if (LVL.hasUncountableEarlyExit()) {
7490 if (!Opts.enable_early_exit_vectorization) {
7491 reportVectorizationFailure(DebugMsg: "Auto-vectorization of loops with uncountable "
7492 "early exit is not enabled",
7493 ORETag: "UncountableEarlyExitLoopsDisabled", ORE, TheLoop: L);
7494 return false;
7495 }
7496 if (LVL.hasUncountableExitWithSideEffects() &&
7497 !Opts.enable_early_exit_vectorization_with_side_effects) {
7498 reportVectorizationFailure(DebugMsg: "Auto-vectorization of loops with uncountable "
7499 "early exit and side effects is not enabled",
7500 ORETag: "UncountableEarlyExitSideEffectLoopsDisabled",
7501 ORE, TheLoop: L);
7502 return false;
7503 }
7504 }
7505
7506 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
7507 // If an override option has been passed in for interleaved accesses, use it.
7508 bool UseInterleaved =
7509 IsInnerLoop && valueOr(X: Opts.enable_interleaved_mem_accesses,
7510 Default: TTI->enableInterleavedAccessVectorization());
7511
7512 // Analyze interleaved memory accesses.
7513 if (UseInterleaved)
7514 IAI.analyzeInterleaving(EnableMaskedInterleavedGroup: useMaskedInterleavedAccesses(Opts, TTI: *TTI));
7515
7516 if (LVL.hasUncountableEarlyExit()) {
7517 BasicBlock *LoopLatch = L->getLoopLatch();
7518 if (IAI.requiresScalarEpilogue() ||
7519 any_of(Range: LVL.getCountableExitingBlocks(), P: not_equal_to(Arg&: LoopLatch))) {
7520 reportVectorizationFailure(DebugMsg: "Auto-vectorization of early exit loops "
7521 "requiring a scalar epilogue is unsupported",
7522 ORETag: "UncountableEarlyExitUnsupported", ORE, TheLoop: L);
7523 return false;
7524 }
7525 }
7526
7527 // Check the function attributes and profiles to find out if this function
7528 // should be optimized for size.
7529 EpilogueLowering SEL =
7530 getEpilogueLowering(Opts, F, L, Hints, OptForSize, TTI, TLI, LVL, IAI: &IAI);
7531
7532 // Check the loop for a trip count threshold: vectorize loops with a tiny trip
7533 // count by optimizing for size, to minimize overheads.
7534 auto ExpectedTC = getSmallBestKnownTC(Opts, PSE, L);
7535 if (ExpectedTC && ExpectedTC->isFixed() &&
7536 ExpectedTC->getFixedValue() < Opts.vectorizer_min_trip_count) {
7537 LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
7538 << "This loop is worth vectorizing only if no scalar "
7539 << "iteration overheads are incurred.");
7540 if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
7541 LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
7542 else {
7543 LLVM_DEBUG(dbgs() << "\n");
7544 // Tail-folded loops are efficient even when the loop
7545 // iteration count is low. However, setting the epilogue policy to
7546 // `CM_EpilogueNotAllowedLowTripLoop` prevents vectorizing loops
7547 // with runtime checks. It's more effective to let
7548 // `isOutsideLoopWorkProfitable` determine if vectorization is
7549 // beneficial for the loop. If the trip count is below the target's
7550 // minimum for tail-folding, the tail cannot be folded, so treat it like
7551 // any other low trip count loop.
7552 if (SEL != CM_EpilogueNotNeededFoldTail ||
7553 ExpectedTC->getFixedValue() <=
7554 TTI->getMinTripCountTailFoldingThreshold())
7555 SEL = CM_EpilogueNotAllowedLowTripLoop;
7556 }
7557 }
7558
7559 // Check the function attributes to see if implicit floats or vectors are
7560 // allowed.
7561 if (F->hasFnAttribute(Kind: Attribute::NoImplicitFloat)) {
7562 reportVectorizationFailure(
7563 DebugMsg: "Can't vectorize when the NoImplicitFloat attribute is used",
7564 OREMsg: "loop not vectorized due to NoImplicitFloat attribute",
7565 ORETag: "NoImplicitFloat", ORE, TheLoop: L);
7566 Hints.emitRemarkWithHints();
7567 return false;
7568 }
7569
7570 // Check if the target supports potentially unsafe FP vectorization.
7571 // FIXME: Add a check for the type of safety issue (denormal, signaling)
7572 // for the target we're vectorizing for, to make sure none of the
7573 // additional fp-math flags can help.
7574 if (Hints.isPotentiallyUnsafe() &&
7575 TTI->isFPVectorizationPotentiallyUnsafe()) {
7576 reportVectorizationFailure(
7577 DebugMsg: "Potentially unsafe FP op prevents vectorization",
7578 OREMsg: "loop not vectorized due to unsafe FP support.", ORETag: "UnsafeFP", ORE, TheLoop: L);
7579 Hints.emitRemarkWithHints();
7580 return false;
7581 }
7582
7583 // If the flag is set, use that instead and override the TTI behaviour.
7584 bool AllowOrderedReductions =
7585 valueOr(X: Opts.force_ordered_reductions, Default: TTI->enableOrderedReductions());
7586 if (!LVL.canVectorizeFPMath(EnableStrictReductions: AllowOrderedReductions)) {
7587 ORE->emit(RemarkBuilder: [&]() {
7588 auto *ExactFPMathInst = Requirements.getExactFPInst();
7589 return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE, "CantReorderFPOps",
7590 ExactFPMathInst->getDebugLoc(),
7591 ExactFPMathInst->getParent())
7592 << "loop not vectorized: cannot prove it is safe to reorder "
7593 "floating-point operations";
7594 });
7595 LLVM_DEBUG(dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
7596 "reorder floating-point operations\n");
7597 Hints.emitRemarkWithHints();
7598 return false;
7599 }
7600
7601 // Use the cost model.
7602 VFSelectionContext Config(Opts, *TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
7603 OptForSize);
7604 // Use the planner for vectorization.
7605 LoopVectorizationPlanner LVP(
7606 L, LI, DT, TLI, *TTI, &LVL,
7607 std::make_unique<LoopVectorizationCostModel>(
7608 args&: SEL, args&: L, args&: PSE, args&: LI, args: &LVL, args&: *TTI, args&: TLI, args&: AC, args&: ORE, args&: GetBFI, args&: F, args&: IAI, args&: Config),
7609 Config, PSE, ORE, GetBPI);
7610
7611 EpilogueLowering EpilogueTailLoweringStatus = getEpilogueTailLowering(
7612 Opts, MainCM: LVP.getCostModel(), L, ORE, LVL, Hints, TTI);
7613 if (EpilogueTailLoweringStatus ==
7614 EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
7615 // TODO: Apply tail-folding on the vectorized epilogue loop.
7616 LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
7617 reportVectorizationInfo(
7618 Msg: "The epilogue-tail-folding policy prefer-fold-tail is not supported "
7619 "yet, fall back to a normal epilogue",
7620 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
7621 }
7622
7623 // Get user vectorization factor and interleave count.
7624 ElementCount UserVF = Hints.getWidth();
7625 unsigned UserIC = Hints.getInterleave();
7626 // Outer loops don't have LoopAccessInfo, so skip the safety check and reset
7627 // UserIC (interleaving is not supported for outer loops).
7628 if (!IsInnerLoop)
7629 UserIC = 0;
7630 else if (UserIC > 1 && !LVL.isSafeForAnyVectorWidth())
7631 UserIC = 1;
7632
7633 // Plan how to best vectorize.
7634 LVP.plan(UserVF, UserIC);
7635 auto [VF, BestPlanPtr] = LVP.computeBestVF();
7636 unsigned IC = 1;
7637
7638 // For VPlan build stress testing of outer loops, bail after plan
7639 // construction.
7640 if (!IsInnerLoop && Opts.vplan_build_outerloop_stress_test)
7641 return false;
7642
7643 if (IsInnerLoop && ORE->allowExtraAnalysis(LV_NAME))
7644 LVP.emitInvalidCostRemarks(ORE);
7645
7646 assert((IsInnerLoop || !LVP.getCostModel().maskPartialAliasing()) &&
7647 "Did not expect to alias-mask outer loop");
7648
7649 GeneratedRTChecks Checks(Opts, PSE, DT, LI, TTI, Config.CostKind,
7650 LVP.getCostModel().maskPartialAliasing());
7651 if (IsInnerLoop && LVP.hasPlanWithVF(VF: VF.Width)) {
7652 // Select the interleave count.
7653 IC = LVP.selectInterleaveCount(Plan&: *BestPlanPtr, VF: VF.Width, LoopCost: VF.Cost);
7654
7655 unsigned SelectedIC = UserIC > 0 ? UserIC : IC;
7656 // Optimistically generate runtime checks if they are needed. Drop them if
7657 // they turn out to not be profitable.
7658 if (VF.Width.isVector() || SelectedIC > 1) {
7659 Checks.create(L, LAI: *LVL.getLAI(), UnionPred: PSE.getPredicate(), VF: VF.Width, IC: SelectedIC,
7660 ORE&: *ORE);
7661
7662 // Bail out early if either the SCEV or memory runtime checks are known to
7663 // fail. In that case, the vector loop would never execute.
7664 using namespace llvm::PatternMatch;
7665 if ((Checks.getSCEVChecks().first &&
7666 match(V: Checks.getSCEVChecks().first, P: m_One())) ||
7667 (Checks.getMemRuntimeChecks().first &&
7668 match(V: Checks.getMemRuntimeChecks().first, P: m_One()))) {
7669 reportVectorizationFailure(
7670 DebugMsg: "runtime checks are known to fail, so we will never enter the "
7671 "vector loop",
7672 ORETag: "RuntimeChecksNeverEnterVectorLoop", ORE, TheLoop: L);
7673 return false;
7674 }
7675 }
7676
7677 // Check if it is profitable to vectorize with runtime checks.
7678 bool ForceVectorization =
7679 Hints.getForce() == LoopVectorizeHints::FK_Enabled;
7680 VPCostContext CostCtx(*TLI, *BestPlanPtr, LVP.getCostModel(), Config,
7681 /*ReusePrintingSlotTracker=*/true);
7682 if (!ForceVectorization &&
7683 !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx, Plan&: *BestPlanPtr,
7684 SEL, VScale: Config.getVScaleForTuning())) {
7685 ORE->emit(RemarkBuilder: [&]() {
7686 return OptimizationRemarkAnalysisAliasing(
7687 DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
7688 L->getHeader())
7689 << "loop not vectorized: cannot prove it is safe to reorder "
7690 "memory operations";
7691 });
7692 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
7693 Hints.emitRemarkWithHints();
7694 return false;
7695 }
7696 }
7697
7698 // Identify the diagnostic messages that should be produced.
7699 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
7700 bool VectorizeLoop = true, InterleaveLoop = true;
7701 if (VF.Width.isScalar()) {
7702 if (LVP.hasVectorPlan()) {
7703 LLVM_DEBUG(
7704 dbgs() << "LV: Vectorization is possible but not beneficial.\n");
7705 VecDiagMsg = {
7706 "VectorizationNotBeneficial",
7707 "the cost-model indicates that vectorization is not beneficial"};
7708 } else {
7709 LLVM_DEBUG(dbgs() << "LV: Vectorization is not possible. Failed to "
7710 "create any vector VPlans.\n");
7711 VecDiagMsg = {"VectorizationNotPossible",
7712 "vectorization is not possible"};
7713 }
7714 VectorizeLoop = false;
7715 }
7716
7717 if (UserIC == 1 && Hints.getInterleave() > 1) {
7718 assert(!LVL.isSafeForAnyVectorWidth() &&
7719 "UserIC should only be ignored due to unsafe dependencies");
7720 LLVM_DEBUG(dbgs() << "LV: Ignoring user-specified interleave count.\n");
7721 IntDiagMsg = {"InterleavingUnsafe",
7722 "Ignoring user-specified interleave count due to possibly "
7723 "unsafe dependencies in the loop."};
7724 InterleaveLoop = false;
7725 } else if (!LVP.hasPlanWithVF(VF: VF.Width) && UserIC > 1) {
7726 // Tell the user interleaving was avoided up-front, despite being explicitly
7727 // requested.
7728 LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
7729 "interleaving should be avoided up front\n");
7730 IntDiagMsg = {"InterleavingAvoided",
7731 "Ignoring UserIC, because interleaving was avoided up front"};
7732 InterleaveLoop = false;
7733 } else if (IC == 1 && UserIC <= 1) {
7734 // Tell the user interleaving is not beneficial.
7735 if (LVP.hasAPlan()) {
7736 LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
7737 IntDiagMsg = {
7738 "InterleavingNotBeneficial",
7739 "the cost-model indicates that interleaving is not beneficial"};
7740 if (UserIC == 1) {
7741 IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
7742 IntDiagMsg.second +=
7743 " and is explicitly disabled or interleave count is set to 1";
7744 }
7745 } else {
7746 LLVM_DEBUG(dbgs() << "LV: Interleaving is not possible. Failed to create"
7747 << " any vplans\n");
7748 IntDiagMsg = {"InterleavingNotPossible", "interleaving is not possible"};
7749 }
7750 InterleaveLoop = false;
7751 } else if (IC > 1 && UserIC == 1) {
7752 // Tell the user interleaving is beneficial, but it explicitly disabled.
7753 LLVM_DEBUG(dbgs() << "LV: Interleaving is beneficial but is explicitly "
7754 "disabled.\n");
7755 IntDiagMsg = {"InterleavingBeneficialButDisabled",
7756 "the cost-model indicates that interleaving is beneficial "
7757 "but is explicitly disabled or interleave count is set to 1"};
7758 InterleaveLoop = false;
7759 }
7760
7761 // If there is a histogram in the loop, do not just interleave without
7762 // vectorizing. The order of operations will be incorrect without the
7763 // histogram intrinsics, which are only used for recipes with VF > 1.
7764 if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
7765 LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
7766 << "to histogram operations.\n");
7767 IntDiagMsg = {
7768 "HistogramPreventsScalarInterleaving",
7769 "Unable to interleave without vectorization due to constraints on "
7770 "the order of histogram operations"};
7771 InterleaveLoop = false;
7772 }
7773
7774 // Override IC if user provided an interleave count.
7775 IC = UserIC > 0 ? UserIC : IC;
7776
7777 if (LVP.getCostModel().maskPartialAliasing()) {
7778 LLVM_DEBUG(
7779 dbgs()
7780 << "LV: Not interleaving due to partial aliasing vectorization.\n");
7781 IntDiagMsg = {
7782 "PartialAliasingVectorization",
7783 "Unable to interleave due to partial aliasing vectorization."};
7784 InterleaveLoop = false;
7785 IC = 1;
7786 }
7787
7788 // FIXME: Enable interleaving for EE-with-side-effects.
7789 if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) {
7790 LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n");
7791 IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving",
7792 "Unable to interleave due to early exit with side effects."};
7793 InterleaveLoop = false;
7794 IC = 1;
7795 }
7796
7797 // Emit diagnostic messages, if any.
7798 if (!VectorizeLoop && !InterleaveLoop) {
7799 // Do not vectorize or interleaving the loop.
7800 ORE->emit(RemarkBuilder: [&]() {
7801 return OptimizationRemarkMissed(LV_NAME, VecDiagMsg.first,
7802 L->getStartLoc(), L->getHeader())
7803 << VecDiagMsg.second;
7804 });
7805 ORE->emit(RemarkBuilder: [&]() {
7806 return OptimizationRemarkMissed(LV_NAME, IntDiagMsg.first,
7807 L->getStartLoc(), L->getHeader())
7808 << IntDiagMsg.second;
7809 });
7810 return false;
7811 }
7812
7813 if (!VectorizeLoop && InterleaveLoop) {
7814 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
7815 ORE->emit(RemarkBuilder: [&]() {
7816 return OptimizationRemarkAnalysis(LV_NAME, VecDiagMsg.first,
7817 L->getStartLoc(), L->getHeader())
7818 << VecDiagMsg.second;
7819 });
7820 } else if (VectorizeLoop && !InterleaveLoop) {
7821 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
7822 << ") in " << L->getLocStr() << '\n');
7823 ORE->emit(RemarkBuilder: [&]() {
7824 return OptimizationRemarkAnalysis(LV_NAME, IntDiagMsg.first,
7825 L->getStartLoc(), L->getHeader())
7826 << IntDiagMsg.second;
7827 });
7828 } else if (VectorizeLoop && InterleaveLoop) {
7829 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
7830 << ") in " << L->getLocStr() << '\n');
7831 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
7832 }
7833
7834 // Report the vectorization decision.
7835 if (VF.Width.isScalar()) {
7836 using namespace ore;
7837 assert(IC > 1);
7838 ORE->emit(RemarkBuilder: [&]() {
7839 return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(),
7840 L->getHeader())
7841 << "interleaved loop (interleaved count: "
7842 << NV("InterleaveCount", IC) << ")";
7843 });
7844 } else {
7845 // Report the vectorization decision.
7846 reportVectorization(ORE, TheLoop: L, VFWidth: VF.Width, IC);
7847 }
7848 if (ORE->allowExtraAnalysis(LV_NAME))
7849 checkMixedPrecision(L, ORE);
7850
7851 // If we decided that it is *legal* to interleave or vectorize the loop, then
7852 // do it.
7853
7854 // Whether a scalar epilogue may be created is decided by the epilogue
7855 // lowering policy.
7856 // TODO: Also move check to be based on VPlan.
7857 bool ScalarEpilogueAllowed = LVP.getCostModel().isEpilogueAllowed();
7858
7859 // Destroy the cost model before executing any plan, so that code generation
7860 // cannot rely on cost-modeling decisions.
7861 LVP.clearCostModel();
7862
7863 VPlan &BestPlan = *BestPlanPtr;
7864 // Consider vectorizing the epilogue too if it's profitable.
7865 std::unique_ptr<VPlan> EpiPlan =
7866 LVP.selectBestEpiloguePlan(MainPlan&: BestPlan, MainLoopVF: VF.Width, IC, ScalarEpilogueAllowed);
7867 bool HasBranchWeights =
7868 hasBranchWeightMD(I: *L->getLoopLatch()->getTerminator());
7869 if (EpiPlan) {
7870 VPlan &BestEpiPlan = *EpiPlan;
7871 VPlan &BestMainPlan = BestPlan;
7872 ElementCount EpilogueVF = BestEpiPlan.getSingleVF();
7873
7874 // The first pass vectorizes the main loop and creates a scalar epilogue
7875 // to be vectorized by executing the plan (potentially with a different
7876 // factor) again shortly afterwards.
7877 BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
7878 BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
7879 MainPlanResumeMarkers Markers = preparePlanForMainVectorLoop(MainPlan&: BestMainPlan);
7880
7881 // Add minimum iteration check for the epilogue plan, followed by runtime
7882 // checks for the main plan.
7883 LVP.addMinimumIterationCheck(Plan&: BestMainPlan, VF: EpilogueVF, /*UF=*/1,
7884 MinProfitableTripCount: ElementCount::getFixed(MinVal: 0));
7885 LVP.attachRuntimeChecks(Plan&: BestMainPlan, RTChecks&: Checks, VF: VF.Width, UF: IC,
7886 HasBranchWeights);
7887 RUN_VPLAN_PASS(VPlanTransforms::addIterationCountCheckBlock, BestMainPlan,
7888 VF.Width, IC, BestMainPlan.requiresScalarEpilogue(), L,
7889 HasBranchWeights ? MinItersBypassWeights : nullptr,
7890 L->getLoopPredecessor()->getTerminator()->getDebugLoc(),
7891 PSE);
7892
7893 LLVM_DEBUG({
7894 dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
7895 << "Main Loop VF:" << VF.Width << ", Main Loop UF:" << IC
7896 << ", Epilogue Loop VF:" << EpilogueVF << ", Epilogue Loop UF:1\n";
7897 });
7898 InnerLoopVectorizer MainILV(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
7899 BestMainPlan);
7900 auto ExpandedSCEVs = LVP.executePlan(
7901 BestVF: VF.Width, BestUF: IC, BestVPlan&: BestMainPlan, ILV&: MainILV, DT,
7902 EpilogueVecKind: LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
7903 ++LoopsVectorized;
7904 DEBUG_WITH_TYPE(VerboseDebug, {
7905 dbgs() << "intermediate fn:\n" << *L->getHeader()->getParent() << "\n";
7906 });
7907
7908 BasicBlock *EntryBB =
7909 cast<VPIRBasicBlock>(Val: BestMainPlan.getEntry())->getIRBasicBlock();
7910 EntryBB->setName("iter.check");
7911
7912 // Second pass vectorizes the epilogue and adjusts the control flow
7913 // edges from the first pass.
7914 EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TTI, AC,
7915 EpilogueVF, /*UnrollFactor=*/1,
7916 Checks, BestEpiPlan, BestMainPlan);
7917 preparePlanForEpilogueVectorLoop(MainPlan&: BestMainPlan, Plan&: BestEpiPlan, L,
7918 ExpandedSCEVs, MainLoopVF: VF.Width, MainLoopUF: IC, EpilogueVF,
7919 LVP, Config, SE&: *PSE.getSE(), Markers);
7920 RUN_VPLAN_PASS(VPlanTransforms::simplifyLiveInsWithSCEV, BestEpiPlan, PSE);
7921 LLVM_DEBUG({
7922 dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
7923 << "Epilogue Loop VF:" << EpilogueVF << ", Epilogue Loop UF:1\n";
7924 });
7925 LVP.executePlan(
7926 BestVF: EpilogueVF, /*BestUF=*/1, BestVPlan&: BestEpiPlan, ILV&: EpilogILV, DT,
7927 EpilogueVecKind: LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
7928 DEBUG_WITH_TYPE(VerboseDebug, {
7929 dbgs() << "final fn:\n" << *L->getHeader()->getParent() << "\n";
7930 });
7931 connectEpilogueVectorLoop(EpiPlan&: BestEpiPlan, DT,
7932 VecEpilogueIterCheckVPBB: EpilogILV.VecEpilogueIterationCountCheck,
7933 ResumeValues: Markers.ResumeValues);
7934 ++LoopsEpilogueVectorized;
7935 } else {
7936 InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
7937 BestPlan);
7938 LVP.addMinimumIterationCheck(Plan&: BestPlan, VF: VF.Width, UF: IC,
7939 MinProfitableTripCount: VF.MinProfitableTripCount);
7940 LVP.attachRuntimeChecks(Plan&: BestPlan, RTChecks&: Checks, VF: VF.Width, UF: IC, HasBranchWeights);
7941
7942 if (!IsInnerLoop)
7943 LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" << F->getName()
7944 << "\"\n");
7945 LVP.executePlan(BestVF: VF.Width, BestUF: IC, BestVPlan&: BestPlan, ILV&: LB, DT);
7946 ++LoopsVectorized;
7947 }
7948
7949 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
7950 "DT not preserved correctly");
7951
7952 return true;
7953}
7954
7955LoopVectorizeResult LoopVectorizePass::runImpl(Function &F) {
7956 CFGChanged = false;
7957
7958 // Don't attempt if
7959 // 1. the target claims to have no vector registers, and
7960 // 2. interleaving won't help ILP.
7961 //
7962 // The second condition is necessary because, even if the target has no
7963 // vector registers, loop vectorization may still enable scalar
7964 // interleaving.
7965 if (!TTI->getNumberOfRegisters(ClassID: TTI->getRegisterClassForType(Vector: true)) &&
7966 (TTI->getMaxInterleaveFactor(VF: ElementCount::getFixed(MinVal: 1), HasUnorderedReductions: false) < 2 ||
7967 TTI->getMaxInterleaveFactor(VF: ElementCount::getFixed(MinVal: 1), HasUnorderedReductions: true) < 2))
7968 return LoopVectorizeResult(false, false);
7969
7970 bool Changed = false;
7971
7972 // The vectorizer requires loops to be in simplified form.
7973 // Since simplification may add new inner loops, it has to run before the
7974 // legality and profitability checks. This means running the loop vectorizer
7975 // will simplify all loops, regardless of whether anything end up being
7976 // vectorized.
7977 for (const auto &L : *LI)
7978 Changed |= CFGChanged |=
7979 simplifyLoop(L, DT, LI, SE, AC, MSSAU: nullptr, PreserveLCSSA: false /* PreserveLCSSA */);
7980
7981 // Build up a worklist of inner-loops to vectorize. This is necessary as
7982 // the act of vectorizing or partially unrolling a loop creates new loops
7983 // and can invalidate iterators across the loops.
7984 SmallVector<Loop *, 8> Worklist;
7985
7986 const VectorizeOptions &Opts =
7987 F.getContext().getLibraryOptions().get<VectorizeOptions>();
7988 for (Loop *L : *LI)
7989 collectSupportedLoops(Opts, L&: *L, LI, ORE, V&: Worklist);
7990
7991 LoopsAnalyzed += Worklist.size();
7992
7993 // Now walk the identified inner loops.
7994 while (!Worklist.empty()) {
7995 Loop *L = Worklist.pop_back_val();
7996
7997 // For the inner loops we actually process, form LCSSA to simplify the
7998 // transform.
7999 Changed |= formLCSSARecursively(L&: *L, DT: *DT, LI, SE);
8000
8001 Changed |= CFGChanged |= processLoop(L);
8002
8003 if (Changed) {
8004 LAIs->clear();
8005
8006#ifndef NDEBUG
8007 if (VerifySCEV)
8008 SE->verify();
8009#endif
8010 }
8011 }
8012
8013 // Verify once per function rather than once per processed loop, which would
8014 // make the pass quadratic in the number of loops.
8015 assert((!Changed || !verifyFunction(F, &dbgs())) &&
8016 "Invalid IR produced by LoopVectorize");
8017
8018 // Process each loop nest in the function.
8019 return LoopVectorizeResult(Changed, CFGChanged);
8020}
8021
8022PreservedAnalyses LoopVectorizePass::run(Function &F,
8023 FunctionAnalysisManager &AM) {
8024 LI = &AM.getResult<LoopAnalysis>(IR&: F);
8025 // There are no loops in the function. Return before computing other
8026 // expensive analyses.
8027 if (LI->empty())
8028 return PreservedAnalyses::all();
8029 SE = &AM.getResult<ScalarEvolutionAnalysis>(IR&: F);
8030 TTI = &AM.getResult<TargetIRAnalysis>(IR&: F);
8031 DT = &AM.getResult<DominatorTreeAnalysis>(IR&: F);
8032 TLI = &AM.getResult<TargetLibraryAnalysis>(IR&: F);
8033 AC = &AM.getResult<AssumptionAnalysis>(IR&: F);
8034 DB = &AM.getResult<DemandedBitsAnalysis>(IR&: F);
8035 ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F);
8036 LAIs = &AM.getResult<LoopAccessAnalysis>(IR&: F);
8037 AA = &AM.getResult<AAManager>(IR&: F);
8038
8039 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
8040 PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(IR&: *F.getParent());
8041 // CycleInfo cached by an earlier pass is invalidated when the CFG changes.
8042 // Both BlockFrequencyAnalysis and BranchProbabilityAnalysis depend on it, so
8043 // drop the stale result before either is (re-)computed.
8044 auto ClearStaleCycleInfo = [this, &AM, &F] {
8045 if (CFGChanged && AM.getCachedResult<CycleAnalysis>(IR&: F))
8046 AM.clearAnalysis<CycleAnalysis>(IR&: F);
8047 };
8048 GetBFI = [&AM, &F, ClearStaleCycleInfo]() -> BlockFrequencyInfo & {
8049 ClearStaleCycleInfo();
8050 return AM.getResult<BlockFrequencyAnalysis>(IR&: F);
8051 };
8052 GetBPI = [&AM, &F, ClearStaleCycleInfo]() -> const BranchProbabilityInfo & {
8053 ClearStaleCycleInfo();
8054 return AM.getResult<BranchProbabilityAnalysis>(IR&: F);
8055 };
8056 LoopVectorizeResult Result = runImpl(F);
8057 if (!Result.MadeAnyChange)
8058 return PreservedAnalyses::all();
8059 PreservedAnalyses PA;
8060
8061 if (isAssignmentTrackingEnabled(M: *F.getParent())) {
8062 for (auto &BB : F)
8063 RemoveRedundantDbgInstrs(BB: &BB);
8064 }
8065
8066 PA.preserve<LoopAnalysis>();
8067 PA.preserve<DominatorTreeAnalysis>();
8068 PA.preserve<ScalarEvolutionAnalysis>();
8069 PA.preserve<LoopAccessAnalysis>();
8070
8071 if (Result.MadeCFGChange) {
8072 // Making CFG changes likely means a loop got vectorized. Indicate that
8073 // extra simplification passes should be run.
8074 // TODO: MadeCFGChanges is not a prefect proxy. Extra passes should only
8075 // be run if runtime checks have been added.
8076 AM.getResult<ShouldRunExtraVectorPasses>(IR&: F);
8077 PA.preserve<ShouldRunExtraVectorPasses>();
8078 } else {
8079 PA.preserveSet<CFGAnalyses>();
8080 }
8081 return PA;
8082}
8083
8084void LoopVectorizePass::printPipeline(
8085 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
8086 static_cast<PassInfoMixin<LoopVectorizePass> *>(this)->printPipeline(
8087 OS, MapClassName2PassName);
8088
8089 OS << '<';
8090 OS << (InterleaveOnlyWhenForced ? "" : "no-") << "interleave-forced-only;";
8091 OS << (VectorizeOnlyWhenForced ? "" : "no-") << "vectorize-forced-only;";
8092 OS << '>';
8093}
8094