1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
56#include "llvm/Transforms/Vectorize/LoopVectorize.h"
57#include "LoopVectorizationPlanner.h"
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanAnalysis.h"
61#include "VPlanCFG.h"
62#include "VPlanHelpers.h"
63#include "VPlanPatternMatch.h"
64#include "VPlanTransforms.h"
65#include "VPlanUtils.h"
66#include "VPlanVerifier.h"
67#include "llvm/ADT/APInt.h"
68#include "llvm/ADT/ArrayRef.h"
69#include "llvm/ADT/DenseMap.h"
70#include "llvm/ADT/Hashing.h"
71#include "llvm/ADT/MapVector.h"
72#include "llvm/ADT/STLExtras.h"
73#include "llvm/ADT/SmallPtrSet.h"
74#include "llvm/ADT/SmallVector.h"
75#include "llvm/ADT/Statistic.h"
76#include "llvm/ADT/StringRef.h"
77#include "llvm/ADT/Twine.h"
78#include "llvm/ADT/TypeSwitch.h"
79#include "llvm/ADT/iterator_range.h"
80#include "llvm/Analysis/AssumptionCache.h"
81#include "llvm/Analysis/BasicAliasAnalysis.h"
82#include "llvm/Analysis/BlockFrequencyInfo.h"
83#include "llvm/Analysis/CFG.h"
84#include "llvm/Analysis/CodeMetrics.h"
85#include "llvm/Analysis/CycleAnalysis.h"
86#include "llvm/Analysis/DemandedBits.h"
87#include "llvm/Analysis/GlobalsModRef.h"
88#include "llvm/Analysis/LoopAccessAnalysis.h"
89#include "llvm/Analysis/LoopAnalysisManager.h"
90#include "llvm/Analysis/LoopInfo.h"
91#include "llvm/Analysis/LoopIterator.h"
92#include "llvm/Analysis/OptimizationRemarkEmitter.h"
93#include "llvm/Analysis/ProfileSummaryInfo.h"
94#include "llvm/Analysis/ScalarEvolution.h"
95#include "llvm/Analysis/ScalarEvolutionExpressions.h"
96#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
97#include "llvm/Analysis/TargetLibraryInfo.h"
98#include "llvm/Analysis/TargetTransformInfo.h"
99#include "llvm/Analysis/ValueTracking.h"
100#include "llvm/Analysis/VectorUtils.h"
101#include "llvm/IR/Attributes.h"
102#include "llvm/IR/BasicBlock.h"
103#include "llvm/IR/CFG.h"
104#include "llvm/IR/Constant.h"
105#include "llvm/IR/Constants.h"
106#include "llvm/IR/DataLayout.h"
107#include "llvm/IR/DebugInfo.h"
108#include "llvm/IR/DebugLoc.h"
109#include "llvm/IR/DerivedTypes.h"
110#include "llvm/IR/DiagnosticInfo.h"
111#include "llvm/IR/Dominators.h"
112#include "llvm/IR/Function.h"
113#include "llvm/IR/IRBuilder.h"
114#include "llvm/IR/InstrTypes.h"
115#include "llvm/IR/Instruction.h"
116#include "llvm/IR/Instructions.h"
117#include "llvm/IR/IntrinsicInst.h"
118#include "llvm/IR/Intrinsics.h"
119#include "llvm/IR/MDBuilder.h"
120#include "llvm/IR/Metadata.h"
121#include "llvm/IR/Module.h"
122#include "llvm/IR/Operator.h"
123#include "llvm/IR/PatternMatch.h"
124#include "llvm/IR/ProfDataUtils.h"
125#include "llvm/IR/Type.h"
126#include "llvm/IR/Use.h"
127#include "llvm/IR/User.h"
128#include "llvm/IR/Value.h"
129#include "llvm/IR/Verifier.h"
130#include "llvm/Support/Casting.h"
131#include "llvm/Support/CommandLine.h"
132#include "llvm/Support/Debug.h"
133#include "llvm/Support/ErrorHandling.h"
134#include "llvm/Support/InstructionCost.h"
135#include "llvm/Support/MathExtras.h"
136#include "llvm/Support/NativeFormatting.h"
137#include "llvm/Support/raw_ostream.h"
138#include "llvm/Transforms/Utils/BasicBlockUtils.h"
139#include "llvm/Transforms/Utils/InjectTLIMappings.h"
140#include "llvm/Transforms/Utils/Local.h"
141#include "llvm/Transforms/Utils/LoopSimplify.h"
142#include "llvm/Transforms/Utils/LoopUtils.h"
143#include "llvm/Transforms/Utils/LoopVersioning.h"
144#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
145#include "llvm/Transforms/Utils/SizeOpts.h"
146#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
147#include <algorithm>
148#include <cassert>
149#include <cmath>
150#include <cstdint>
151#include <functional>
152#include <iterator>
153#include <memory>
154#include <string>
155#include <tuple>
156#include <utility>
157
158using namespace llvm;
159using namespace SCEVPatternMatch;
160using namespace LoopVectorizationUtils;
161
162#define LV_NAME "loop-vectorize"
163#define DEBUG_TYPE LV_NAME
164
165#ifndef NDEBUG
166const char VerboseDebug[] = DEBUG_TYPE "-verbose";
167#endif
168
169STATISTIC(LoopsVectorized, "Number of loops vectorized");
170STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
171STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
172STATISTIC(LoopsEarlyExitVectorized, "Number of early exit loops vectorized");
173STATISTIC(LoopsPartialAliasVectorized,
174 "Number of partial aliasing loops vectorized");
175
176static cl::opt<bool> EnableEpilogueVectorization(
177 "enable-epilogue-vectorization", cl::init(Val: true), cl::Hidden,
178 cl::desc("Enable vectorization of epilogue loops."));
179
180static cl::opt<ElementCount> EpilogueVectorizationForceVF(
181 "epilogue-vectorization-force-VF", cl::init(Val: ElementCount::getFixed(MinVal: 1)),
182 cl::Hidden,
183 cl::desc("When epilogue vectorization is enabled, and a value greater than "
184 "1 is specified, forces the given VF for all applicable epilogue "
185 "loops. Note: This allows all scalable VFs >= vscale x 1."));
186
187static cl::opt<unsigned> EpilogueVectorizationMinVF(
188 "epilogue-vectorization-minimum-VF", cl::Hidden,
189 cl::desc("Only loops with vectorization factor equal to or larger than "
190 "the specified value are considered for epilogue vectorization."));
191
192/// Loops with a known constant trip count below this number are vectorized only
193/// if no scalar iteration overheads are incurred.
194static cl::opt<unsigned> TinyTripCountVectorThreshold(
195 "vectorizer-min-trip-count", cl::init(Val: 16), cl::Hidden,
196 cl::desc("Loops with a constant trip count that is smaller than this "
197 "value are vectorized only if no scalar iteration overheads "
198 "are incurred."));
199
200static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
201 "vectorize-memory-check-threshold", cl::init(Val: 128), cl::Hidden,
202 cl::desc("The maximum allowed number of runtime memory checks"));
203
204static cl::opt<bool> ForcePartialAliasingVectorization(
205 "force-partial-aliasing-vectorization", cl::init(Val: false), cl::Hidden,
206 cl::desc("Replace pointer diff checks with alias masks."));
207
208/// Option tail-folding-policy controls the tail-folding strategy and lists all
209/// available options. The vectorizer will attempt to fold the tail-loop into
210/// the vector loop (main/epilogue loops) and predicate the instructions
211/// accordingly. If tail-folding fails, there are different fallback strategies
212/// depending on these values:
213enum class TailFoldingPolicyTy { None = 0, PreferFoldTail, MustFoldTail };
214
215static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
216 "tail-folding-policy", cl::init(Val: TailFoldingPolicyTy::None), cl::Hidden,
217 cl::desc("Tail-folding preferences over creating an epilogue loop."),
218 cl::values(
219 clEnumValN(TailFoldingPolicyTy::None, "dont-fold-tail",
220 "Don't tail-fold loops."),
221 clEnumValN(TailFoldingPolicyTy::PreferFoldTail, "prefer-fold-tail",
222 "prefer tail-folding, otherwise create an epilogue when "
223 "appropriate."),
224 clEnumValN(TailFoldingPolicyTy::MustFoldTail, "must-fold-tail",
225 "always tail-fold, don't attempt vectorization if "
226 "tail-folding fails.")));
227
228static cl::opt<TailFoldingPolicyTy> EpilogueTailFoldingPolicy(
229 "epilogue-tail-folding-policy", cl::Hidden,
230 cl::desc(
231 "Epilogue-tail-folding preferences over creating an epilogue loop."),
232 cl::values(
233 clEnumValN(TailFoldingPolicyTy::None, "dont-fold-tail",
234 "Don't tail-fold loops."),
235 clEnumValN(TailFoldingPolicyTy::PreferFoldTail, "prefer-fold-tail",
236 "prefer tail-folding, otherwise create an epilogue when "
237 "appropriate.")));
238
239static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
240 "force-tail-folding-style", cl::desc("Force the tail folding style"),
241 cl::init(Val: TailFoldingStyle::None),
242 cl::values(
243 clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"),
244 clEnumValN(
245 TailFoldingStyle::Data, "data",
246 "Create lane mask for data only, using active.lane.mask intrinsic"),
247 clEnumValN(TailFoldingStyle::DataWithoutLaneMask,
248 "data-without-lane-mask",
249 "Create lane mask with compare/stepvector"),
250 clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control",
251 "Create lane mask using active.lane.mask intrinsic, and use "
252 "it for both data and control flow"),
253 clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl",
254 "Use predicated EVL instructions for tail folding. If EVL "
255 "is unsupported, fallback to data-without-lane-mask.")));
256
257static cl::opt<bool> EnableInterleavedMemAccesses(
258 "enable-interleaved-mem-accesses", cl::init(Val: false), cl::Hidden,
259 cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
260
261/// An interleave-group may need masking if it resides in a block that needs
262/// predication, or in order to mask away gaps.
263static cl::opt<bool> EnableMaskedInterleavedMemAccesses(
264 "enable-masked-interleaved-mem-accesses", cl::init(Val: false), cl::Hidden,
265 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));
266
267static cl::opt<unsigned> ForceTargetNumScalarRegs(
268 "force-target-num-scalar-regs", cl::init(Val: 0), cl::Hidden,
269 cl::desc("A flag that overrides the target's number of scalar registers."));
270
271static cl::opt<unsigned> ForceTargetNumVectorRegs(
272 "force-target-num-vector-regs", cl::init(Val: 0), cl::Hidden,
273 cl::desc("A flag that overrides the target's number of vector registers."));
274
275static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor(
276 "force-target-max-scalar-interleave", cl::init(Val: 0), cl::Hidden,
277 cl::desc("A flag that overrides the target's max interleave factor for "
278 "scalar loops."));
279
280static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(
281 "force-target-max-vector-interleave", cl::init(Val: 0), cl::Hidden,
282 cl::desc("A flag that overrides the target's max interleave factor for "
283 "vectorized loops."));
284
285cl::opt<unsigned> llvm::ForceTargetInstructionCost(
286 "force-target-instruction-cost", cl::init(Val: 0), cl::Hidden,
287 cl::desc("A flag that overrides the target's expected cost for "
288 "an instruction to a single constant value. Mostly "
289 "useful for getting consistent testing."));
290
291static cl::opt<unsigned> SmallLoopCost(
292 "small-loop-cost", cl::init(Val: 20), cl::Hidden,
293 cl::desc(
294 "The cost of a loop that is considered 'small' by the interleaver."));
295
296static cl::opt<bool> LoopVectorizeWithBlockFrequency(
297 "loop-vectorize-with-block-frequency", cl::init(Val: true), cl::Hidden,
298 cl::desc("Enable the use of the block frequency analysis to access PGO "
299 "heuristics minimizing code growth in cold regions and being more "
300 "aggressive in hot regions."));
301
302// Runtime interleave loops for load/store throughput.
303static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
304 "enable-loadstore-runtime-interleave", cl::init(Val: true), cl::Hidden,
305 cl::desc(
306 "Enable runtime interleaving until load/store ports are saturated"));
307
308/// The number of stores in a loop that are allowed to need predication.
309cl::opt<unsigned> NumberOfStoresToPredicate(
310 "vectorize-num-stores-pred", cl::init(Val: 1), cl::Hidden,
311 cl::desc("Max number of stores to be predicated behind an if."));
312
313// TODO: Move size-based thresholds out of legality checking, make cost based
314// decisions instead of hard thresholds.
315static cl::opt<unsigned> VectorizeSCEVCheckThreshold(
316 "vectorize-scev-check-threshold", cl::init(Val: 16), cl::Hidden,
317 cl::desc("The maximum number of SCEV checks allowed."));
318
319static cl::opt<unsigned> PragmaVectorizeSCEVCheckThreshold(
320 "pragma-vectorize-scev-check-threshold", cl::init(Val: 128), cl::Hidden,
321 cl::desc("The maximum number of SCEV checks allowed with a "
322 "vectorize(enable) pragma"));
323
324static cl::opt<bool> EnableIndVarRegisterHeur(
325 "enable-ind-var-reg-heur", cl::init(Val: true), cl::Hidden,
326 cl::desc("Count the induction variable only once when interleaving"));
327
328static cl::opt<unsigned> MaxNestedScalarReductionIC(
329 "max-nested-scalar-reduction-interleave", cl::init(Val: 2), cl::Hidden,
330 cl::desc("The maximum interleave count to use when interleaving a scalar "
331 "reduction in a nested loop."));
332
333static cl::opt<bool> ForceOrderedReductions(
334 "force-ordered-reductions", cl::init(Val: false), cl::Hidden,
335 cl::desc("Enable the vectorisation of loops with in-order (strict) "
336 "FP reductions"));
337
338static cl::opt<bool> PreferPredicatedReductionSelect(
339 "prefer-predicated-reduction-select", cl::init(Val: false), cl::Hidden,
340 cl::desc(
341 "Prefer predicating a reduction operation over an after loop select."));
342
343cl::opt<bool> llvm::EnableVPlanNativePath(
344 "enable-vplan-native-path", cl::Hidden,
345 cl::desc("Enable VPlan-native vectorization path with "
346 "support for outer loop vectorization."));
347
348cl::opt<bool>
349 llvm::VerifyEachVPlan("vplan-verify-each",
350#ifdef EXPENSIVE_CHECKS
351 cl::init(true),
352#else
353 cl::init(Val: false),
354#endif
355 cl::Hidden,
356 cl::desc("Verify VPlans after VPlan transforms."));
357
358#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
359cl::opt<bool> llvm::VPlanPrintBeforeAll(
360 "vplan-print-before-all", cl::init(false), cl::Hidden,
361 cl::desc("Print VPlans before all VPlan transformations."));
362
363cl::opt<bool> llvm::VPlanPrintAfterAll(
364 "vplan-print-after-all", cl::init(false), cl::Hidden,
365 cl::desc("Print VPlans after all VPlan transformations."));
366
367cl::list<std::string> llvm::VPlanPrintBeforePasses(
368 "vplan-print-before", cl::Hidden,
369 cl::desc("Print VPlans before specified VPlan transformations (regexp)."));
370
371cl::list<std::string> llvm::VPlanPrintAfterPasses(
372 "vplan-print-after", cl::Hidden,
373 cl::desc("Print VPlans after specified VPlan transformations (regexp)."));
374
375cl::opt<bool> llvm::VPlanPrintVectorRegionScope(
376 "vplan-print-vector-region-scope", cl::init(false), cl::Hidden,
377 cl::desc("Limit VPlan printing to vector loop region in "
378 "`-vplan-print-after*` if the plan has one."));
379#endif
380
381// This flag enables the stress testing of the VPlan H-CFG construction in the
382// VPlan-native vectorization path. It must be used in conjuction with
383// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
384// verification of the H-CFGs built.
385cl::opt<bool> VPlanBuildOuterloopStressTest(
386 "vplan-build-outerloop-stress-test", cl::init(Val: false), cl::Hidden,
387 cl::desc(
388 "Build VPlan for every supported loop nest in the function and bail "
389 "out right after the build (stress test the VPlan H-CFG construction "
390 "in the VPlan-native vectorization path)."));
391
392cl::opt<bool> llvm::EnableLoopInterleaving(
393 "interleave-loops", cl::init(Val: true), cl::Hidden,
394 cl::desc("Enable loop interleaving in Loop vectorization passes"));
395cl::opt<bool> llvm::EnableLoopVectorization(
396 "vectorize-loops", cl::init(Val: true), cl::Hidden,
397 cl::desc("Run the Loop vectorization passes"));
398
399static cl::opt<cl::boolOrDefault>
400 ForceMaskedDivRem("force-widen-divrem-via-masked-intrinsic", cl::Hidden,
401 cl::desc("Override cost based masked intrinsic widening "
402 "for div/rem instructions"));
403
404static cl::opt<bool> EnableEarlyExitVectorization(
405 "enable-early-exit-vectorization", cl::init(Val: true), cl::Hidden,
406 cl::desc(
407 "Enable vectorization of early exit loops with uncountable exits."));
408
409static cl::opt<bool> EnableEarlyExitVectorizationWithSideEffects(
410 "enable-early-exit-vectorization-with-side-effects", cl::init(Val: false),
411 cl::Hidden,
412 cl::desc("Enable vectorization of early exit loops with uncountable exits "
413 "and side effects"));
414
415// Returns true if the epilogue VF has been set to a non-zero value other than
416// VF=1 (scalar).
417static bool hasForcedEpilogueVF() {
418 return EpilogueVectorizationForceVF.isNonZero() &&
419 EpilogueVectorizationForceVF != ElementCount::getFixed(MinVal: 1);
420}
421
422// Likelyhood of bypassing the vectorized loop because there are zero trips left
423// after prolog. See `emitIterationCountCheck`.
424static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
425
426/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
427/// ElementCount to include loops whose trip count is a function of vscale.
428static ElementCount getSmallConstantTripCount(ScalarEvolution *SE,
429 const Loop *L) {
430 if (unsigned ExpectedTC = SE->getSmallConstantTripCount(L))
431 return ElementCount::getFixed(MinVal: ExpectedTC);
432
433 const SCEV *BTC = SE->getBackedgeTakenCount(L);
434 if (isa<SCEVCouldNotCompute>(Val: BTC))
435 return ElementCount::getFixed(MinVal: 0);
436
437 const SCEV *ExitCount = SE->getTripCountFromExitCount(ExitCount: BTC, EvalTy: BTC->getType(), L);
438 if (isa<SCEVVScale>(Val: ExitCount))
439 return ElementCount::getScalable(MinVal: 1);
440
441 const APInt *Scale;
442 if (match(S: ExitCount, P: m_scev_Mul(Op0: m_scev_APInt(C&: Scale), Op1: m_SCEVVScale())))
443 if (cast<SCEVMulExpr>(Val: ExitCount)->hasNoUnsignedWrap())
444 if (Scale->getActiveBits() <= 32)
445 return ElementCount::getScalable(MinVal: Scale->getZExtValue());
446
447 return ElementCount::getFixed(MinVal: 0);
448}
449
450/// Get the maximum trip count for \p L from the SCEV unsigned range, excluding
451/// zero from the range. Only valid when not folding the tail, as the minimum
452/// iteration count check guards against a zero trip count. Returns 0 if
453/// unknown.
454static unsigned getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE,
455 Loop *L) {
456 const SCEV *BTC = PSE.getBackedgeTakenCount();
457 if (isa<SCEVCouldNotCompute>(Val: BTC))
458 return 0;
459 ScalarEvolution *SE = PSE.getSE();
460 const SCEV *TripCount = SE->getTripCountFromExitCount(ExitCount: BTC, EvalTy: BTC->getType(), L);
461 ConstantRange TCRange = SE->getUnsignedRange(S: TripCount);
462 APInt MaxTCFromRange = TCRange.getUnsignedMax();
463 if (!MaxTCFromRange.isZero() && MaxTCFromRange.getActiveBits() <= 32)
464 return MaxTCFromRange.getZExtValue();
465 return 0;
466}
467
468/// Returns "best known" trip count, which is either a valid positive trip count
469/// or std::nullopt when an estimate cannot be made (including when the trip
470/// count would overflow), for the specified loop \p L as defined by the
471/// following procedure:
472/// 1) Returns exact trip count if it is known.
473/// 2) Returns expected trip count according to profile data if any.
474/// 3) Returns upper bound estimate if known, if \p CanUseConstantMax, and
475/// if \p ComputeUpperBoundOnly is false.
476/// 4) Returns the maximum trip count from the SCEV range excluding zero,
477/// if \p CanUseConstantMax and \p CanExcludeZeroTrips.
478/// 5) Returns std::nullopt if all of the above failed.
479static std::optional<ElementCount> getSmallBestKnownTC(
480 PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax = true,
481 bool CanExcludeZeroTrips = false, bool ComputeUpperBoundOnly = false) {
482 // Check if exact trip count is known.
483 if (auto ExpectedTC = getSmallConstantTripCount(SE: PSE.getSE(), L))
484 return ExpectedTC;
485
486 // Check if there is an expected trip count available from profile data.
487 // An estimate of zero means the loop is estimated not to be entered; it is
488 // not a usable trip count for the profitability decisions below (and would
489 // e.g. divide by zero when scaling runtime check cost), so treat it as
490 // unknown.
491 if (LoopVectorizeWithBlockFrequency && !ComputeUpperBoundOnly)
492 if (unsigned EstimatedTC = getLoopEstimatedTripCount(L).value_or(u: 0))
493 return ElementCount::getFixed(MinVal: EstimatedTC);
494
495 if (!CanUseConstantMax)
496 return std::nullopt;
497
498 // Check if upper bound estimate is known.
499 if (unsigned ExpectedTC = PSE.getSmallConstantMaxTripCount())
500 return ElementCount::getFixed(MinVal: ExpectedTC);
501
502 // Get the maximum trip count from the SCEV range excluding zero. This is
503 // only safe when not folding the tail, as the minimum iteration count check
504 // prevents entering the vector loop with a zero trip count.
505 if (CanUseConstantMax && CanExcludeZeroTrips)
506 if (unsigned RefinedTC = getMaxTCFromNonZeroRange(PSE, L))
507 return ElementCount::getFixed(MinVal: RefinedTC);
508
509 return std::nullopt;
510}
511
512namespace {
513// Forward declare GeneratedRTChecks.
514class GeneratedRTChecks;
515
516using SCEV2ValueTy = DenseMap<const SCEV *, Value *>;
517} // namespace
518
519namespace llvm {
520
521AnalysisKey ShouldRunExtraVectorPasses::Key;
522
523/// InnerLoopVectorizer vectorizes loops which contain only one basic
524/// block to a specified vectorization factor (VF).
525/// This class performs the widening of scalars into vectors, or multiple
526/// scalars. This class also implements the following features:
527/// * It inserts an epilogue loop for handling loops that don't have iteration
528/// counts that are known to be a multiple of the vectorization factor.
529/// * It handles the code generation for reduction variables.
530/// * Scalarization (implementation using scalars) of un-vectorizable
531/// instructions.
532/// InnerLoopVectorizer does not perform any vectorization-legality
533/// checks, and relies on the caller to check for the different legality
534/// aspects. The InnerLoopVectorizer relies on the
535/// LoopVectorizationLegality class to provide information about the induction
536/// and reduction variables that were found to a given vectorization factor.
537class InnerLoopVectorizer {
538public:
539 InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
540 LoopInfo *LI, DominatorTree *DT,
541 const TargetTransformInfo *TTI, AssumptionCache *AC,
542 ElementCount VecWidth, unsigned UnrollFactor,
543 GeneratedRTChecks &RTChecks, VPlan &Plan)
544 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TTI(TTI), AC(AC),
545 VF(VecWidth), UF(UnrollFactor), Builder(PSE.getSE()->getContext()),
546 RTChecks(RTChecks), Plan(Plan),
547 VectorPHVPBB(cast<VPBasicBlock>(
548 Val: Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
549
550 virtual ~InnerLoopVectorizer() = default;
551
552 /// Creates a basic block for the scalar preheader. Both
553 /// EpilogueVectorizerMainLoop and EpilogueVectorizerEpilogueLoop overwrite
554 /// the method to create additional blocks and checks needed for epilogue
555 /// vectorization.
556 virtual BasicBlock *createVectorizedLoopSkeleton();
557
558 /// Fix the vectorized code, taking care of header phi's, and more.
559 void fixVectorizedLoop(VPTransformState &State);
560
561protected:
562 friend class LoopVectorizationPlanner;
563
564 /// Create and return a new IR basic block for the scalar preheader whose name
565 /// is prefixed with \p Prefix.
566 BasicBlock *createScalarPreheader(StringRef Prefix);
567
568 /// Allow subclasses to override and print debug traces before/after vplan
569 /// execution, when trace information is requested.
570 virtual void printDebugTracesAtStart() {}
571 virtual void printDebugTracesAtEnd() {}
572
573 /// The original loop.
574 Loop *OrigLoop;
575
576 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
577 /// dynamic knowledge to simplify SCEV expressions and converts them to a
578 /// more usable form.
579 PredicatedScalarEvolution &PSE;
580
581 /// Loop Info.
582 LoopInfo *LI;
583
584 /// Dominator Tree.
585 DominatorTree *DT;
586
587 /// Target Transform Info.
588 const TargetTransformInfo *TTI;
589
590 /// Assumption Cache.
591 AssumptionCache *AC;
592
593 /// The vectorization SIMD factor to use. Each vector will have this many
594 /// vector elements.
595 ElementCount VF;
596
597 /// The vectorization unroll factor to use. Each scalar is vectorized to this
598 /// many different vector instructions.
599 unsigned UF;
600
601 /// The builder that we use
602 IRBuilder<> Builder;
603
604 // --- Vectorization state ---
605
606 /// Structure to hold information about generated runtime checks, responsible
607 /// for cleaning the checks, if vectorization turns out unprofitable.
608 GeneratedRTChecks &RTChecks;
609
610 VPlan &Plan;
611
612 /// The vector preheader block of \p Plan, used as target for check blocks
613 /// introduced during skeleton creation.
614 VPBasicBlock *VectorPHVPBB;
615};
616
617/// Encapsulate information regarding vectorization of a loop and its epilogue.
618/// This information is meant to be updated and used across two stages of
619/// epilogue vectorization.
620struct EpilogueLoopVectorizationInfo {
621 ElementCount MainLoopVF = ElementCount::getFixed(MinVal: 0);
622 unsigned MainLoopUF = 0;
623 ElementCount EpilogueVF = ElementCount::getFixed(MinVal: 0);
624 unsigned EpilogueUF = 0;
625 BasicBlock *MainLoopIterationCountCheck = nullptr;
626 BasicBlock *EpilogueIterationCountCheck = nullptr;
627 Value *VectorTripCount = nullptr;
628 VPlan &EpiloguePlan;
629
630 EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF,
631 ElementCount EVF, unsigned EUF,
632 VPlan &EpiloguePlan)
633 : MainLoopVF(MVF), MainLoopUF(MUF), EpilogueVF(EVF), EpilogueUF(EUF),
634 EpiloguePlan(EpiloguePlan) {
635 assert(EUF == 1 &&
636 "A high UF for the epilogue loop is likely not beneficial.");
637 }
638};
639
640/// An extension of the inner loop vectorizer that creates a skeleton for a
641/// vectorized loop that has its epilogue (residual) also vectorized.
642/// The idea is to run the vplan on a given loop twice, firstly to setup the
643/// skeleton and vectorize the main loop, and secondly to complete the skeleton
644/// from the first step and vectorize the epilogue. This is achieved by
645/// deriving two concrete strategy classes from this base class and invoking
646/// them in succession from the loop vectorizer planner.
647class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer {
648public:
649 InnerLoopAndEpilogueVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
650 LoopInfo *LI, DominatorTree *DT,
651 const TargetTransformInfo *TTI,
652 AssumptionCache *AC,
653 EpilogueLoopVectorizationInfo &EPI,
654 GeneratedRTChecks &Checks, VPlan &Plan,
655 ElementCount VecWidth, unsigned UnrollFactor)
656 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, VecWidth,
657 UnrollFactor, Checks, Plan),
658 EPI(EPI) {}
659
660 /// Holds and updates state information required to vectorize the main loop
661 /// and its epilogue in two separate passes. This setup helps us avoid
662 /// regenerating and recomputing runtime safety checks. It also helps us to
663 /// shorten the iteration-count-check path length for the cases where the
664 /// iteration count of the loop is so small that the main vector loop is
665 /// completely skipped.
666 EpilogueLoopVectorizationInfo &EPI;
667};
668
669/// A specialized derived class of inner loop vectorizer that performs
670/// vectorization of *main* loops in the process of vectorizing loops and their
671/// epilogues.
672class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
673public:
674 EpilogueVectorizerMainLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
675 LoopInfo *LI, DominatorTree *DT,
676 const TargetTransformInfo *TTI,
677 AssumptionCache *AC,
678 EpilogueLoopVectorizationInfo &EPI,
679 GeneratedRTChecks &Check, VPlan &Plan)
680 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, EPI,
681 Check, Plan, EPI.MainLoopVF,
682 EPI.MainLoopUF) {}
683
684protected:
685 void printDebugTracesAtStart() override;
686 void printDebugTracesAtEnd() override;
687};
688
689// A specialized derived class of inner loop vectorizer that performs
690// vectorization of *epilogue* loops in the process of vectorizing loops and
691// their epilogues.
692class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
693public:
694 EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
695 LoopInfo *LI, DominatorTree *DT,
696 const TargetTransformInfo *TTI,
697 AssumptionCache *AC,
698 EpilogueLoopVectorizationInfo &EPI,
699 GeneratedRTChecks &Checks, VPlan &Plan)
700 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, EPI,
701 Checks, Plan, EPI.EpilogueVF,
702 EPI.EpilogueUF) {}
703 /// Implements the interface for creating a vectorized skeleton using the
704 /// *epilogue loop* strategy (i.e., the second pass of VPlan execution).
705 BasicBlock *createVectorizedLoopSkeleton() final;
706
707protected:
708 void printDebugTracesAtStart() override;
709 void printDebugTracesAtEnd() override;
710};
711} // end namespace llvm
712
713/// Look for a meaningful debug location on the instruction or its operands.
714static DebugLoc getDebugLocFromInstOrOperands(Instruction *I) {
715 if (!I)
716 return DebugLoc::getUnknown();
717
718 DebugLoc Empty;
719 if (I->getDebugLoc() != Empty)
720 return I->getDebugLoc();
721
722 for (Use &Op : I->operands()) {
723 if (Instruction *OpInst = dyn_cast<Instruction>(Val&: Op))
724 if (OpInst->getDebugLoc() != Empty)
725 return OpInst->getDebugLoc();
726 }
727
728 return I->getDebugLoc();
729}
730
731namespace llvm {
732
733/// Return the runtime value for VF.
734Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF) {
735 return B.CreateElementCount(Ty, EC: VF);
736}
737
738} // end namespace llvm
739
740namespace llvm {
741
742// Loop vectorization cost-model hints how the epilogue/tail loop should be
743// lowered.
744enum EpilogueLowering {
745
746 // The default: allowing epilogues.
747 CM_EpilogueAllowed,
748
749 // Vectorization with OptForSize: don't allow epilogues.
750 CM_EpilogueNotAllowedOptSize,
751
752 // A special case of vectorisation with OptForSize: loops with a very small
753 // trip count are considered for vectorization under OptForSize, thereby
754 // making sure the cost of their loop body is dominant, free of runtime
755 // guards and scalar iteration overheads.
756 CM_EpilogueNotAllowedLowTripLoop,
757
758 // Loop hint indicating an epilogue is undesired, apply tail folding.
759 CM_EpilogueNotNeededFoldTail,
760
761 // Directive indicating we must either fold the epilogue/tail or not vectorize
762 CM_EpilogueNotAllowedFoldTail
763};
764
765enum class AliasMaskingStatus { NotDecided, Disabled, Enabled };
766
767/// LoopVectorizationCostModel - estimates the expected speedups due to
768/// vectorization.
769/// In many cases vectorization is not profitable. This can happen because of
770/// a number of reasons. In this class we mainly attempt to predict the
771/// expected speedup/slowdowns due to the supported instruction set. We use the
772/// TargetTransformInfo to query the different backends for the cost of
773/// different operations.
774class LoopVectorizationCostModel {
775 friend class LoopVectorizationPlanner;
776
777public:
778 LoopVectorizationCostModel(EpilogueLowering SEL, Loop *L,
779 PredicatedScalarEvolution &PSE, LoopInfo *LI,
780 LoopVectorizationLegality *Legal,
781 const TargetTransformInfo &TTI,
782 const TargetLibraryInfo *TLI, AssumptionCache *AC,
783 OptimizationRemarkEmitter *ORE,
784 std::function<BlockFrequencyInfo &()> GetBFI,
785 const Function *F, InterleavedAccessInfo &IAI,
786 VFSelectionContext &Config)
787 : Config(Config), EpilogueLoweringStatus(SEL), TheLoop(L), PSE(PSE),
788 LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), AC(AC), ORE(ORE),
789 GetBFI(GetBFI), TheFunction(F), InterleaveInfo(IAI) {}
790
791 /// \return An upper bound for the vectorization factors (both fixed and
792 /// scalable). If the factors are 0, vectorization and interleaving should be
793 /// avoided up front.
794 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
795
796 /// Memory access instruction may be vectorized in more than one way.
797 /// Form of instruction after vectorization depends on cost.
798 /// This function takes cost-based decisions for Load/Store instructions
799 /// and collects them in a map. This decisions map is used for building
800 /// the lists of loop-uniform and loop-scalar instructions.
801 /// The calculated cost is saved with widening decision in order to
802 /// avoid redundant calculations.
803 void setCostBasedWideningDecision(ElementCount VF);
804
805 /// Collect values we want to ignore in the cost model.
806 void collectValuesToIgnore();
807
808 /// \returns True if it is more profitable to scalarize instruction \p I for
809 /// vectorization factor \p VF.
810 bool isProfitableToScalarize(Instruction *I, ElementCount VF) const {
811 assert(VF.isVector() &&
812 "Profitable to scalarize relevant only for VF > 1.");
813 assert(
814 TheLoop->isInnermost() &&
815 "cost-model should not be used for outer loops (in VPlan-native path)");
816
817 auto Scalars = InstsToScalarize.find(Key: VF);
818 assert(Scalars != InstsToScalarize.end() &&
819 "VF not yet analyzed for scalarization profitability");
820 return Scalars->second.contains(Key: I);
821 }
822
823 /// Returns true if \p I is known to be uniform after vectorization.
824 bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const {
825 assert(
826 TheLoop->isInnermost() &&
827 "cost-model should not be used for outer loops (in VPlan-native path)");
828
829 // If VF is scalar, then all instructions are trivially uniform.
830 if (VF.isScalar())
831 return true;
832
833 // Pseudo probes must be duplicated per vector lane so that the
834 // profiled loop trip count is not undercounted.
835 if (isa<PseudoProbeInst>(Val: I))
836 return false;
837
838 auto UniformsPerVF = Uniforms.find(Val: VF);
839 assert(UniformsPerVF != Uniforms.end() &&
840 "VF not yet analyzed for uniformity");
841 return UniformsPerVF->second.count(Ptr: I);
842 }
843
844 /// Returns true if \p I is known to be scalar after vectorization.
845 bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const {
846 assert(
847 TheLoop->isInnermost() &&
848 "cost-model should not be used for outer loops (in VPlan-native path)");
849 if (VF.isScalar())
850 return true;
851
852 auto ScalarsPerVF = Scalars.find(Val: VF);
853 assert(ScalarsPerVF != Scalars.end() &&
854 "Scalar values are not calculated for VF");
855 return ScalarsPerVF->second.count(Ptr: I);
856 }
857
858 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
859 /// for vectorization factor \p VF.
860 bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const {
861 const auto &MinBWs = Config.getMinimalBitwidths();
862 // Truncs must truncate at most to their destination type.
863 if (isa_and_nonnull<TruncInst>(Val: I) && MinBWs.contains(Key: I) &&
864 I->getType()->getScalarSizeInBits() < MinBWs.lookup(Key: I))
865 return false;
866 return VF.isVector() && MinBWs.contains(Key: I) &&
867 !isProfitableToScalarize(I, VF) &&
868 !isScalarAfterVectorization(I, VF);
869 }
870
871 /// Decision that was taken during cost calculation for memory instruction.
872 enum InstWidening {
873 CM_Unknown,
874 CM_Widen, // For consecutive accesses with stride +1.
875 CM_Widen_Reverse, // For consecutive accesses with stride -1.
876 CM_Interleave,
877 CM_GatherScatter,
878 CM_Scalarize,
879 /// A widening decision that has been invalidated after replacing the
880 /// corresponding recipe during VPlan transforms.
881 /// TODO: Remove once the legacy exit cost computation is retired.
882 CM_InvalidatedDecision
883 };
884
885 /// Save vectorization decision \p W and \p Cost taken by the cost model for
886 /// instruction \p I and vector width \p VF.
887 void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W,
888 InstructionCost Cost) {
889 assert(VF.isVector() && "Expected VF >=2");
890 WideningDecisions[{I, VF}] = {W, Cost};
891 }
892
893 /// Save vectorization decision \p W and \p Cost taken by the cost model for
894 /// interleaving group \p Grp and vector width \p VF.
895 void setWideningDecision(const InterleaveGroup<Instruction> *Grp,
896 ElementCount VF, InstWidening W,
897 InstructionCost Cost) {
898 assert(VF.isVector() && "Expected VF >=2");
899 /// Broadcast this decicion to all instructions inside the group.
900 /// When interleaving, the cost will only be assigned one instruction, the
901 /// insert position. For other cases, add the appropriate fraction of the
902 /// total cost to each instruction. This ensures accurate costs are used,
903 /// even if the insert position instruction is not used.
904 InstructionCost InsertPosCost = Cost;
905 InstructionCost OtherMemberCost = 0;
906 if (W != CM_Interleave)
907 OtherMemberCost = InsertPosCost = Cost / Grp->getNumMembers();
908 ;
909 for (auto *I : Grp->members()) {
910 if (Grp->getInsertPos() == I)
911 WideningDecisions[{I, VF}] = {W, InsertPosCost};
912 else
913 WideningDecisions[{I, VF}] = {W, OtherMemberCost};
914 }
915 }
916
917 /// Return the cost model decision for the given instruction \p I and vector
918 /// width \p VF. Return CM_Unknown if this instruction did not pass
919 /// through the cost modeling.
920 InstWidening getWideningDecision(Instruction *I, ElementCount VF) const {
921 assert(VF.isVector() && "Expected VF to be a vector VF");
922 assert(
923 TheLoop->isInnermost() &&
924 "cost-model should not be used for outer loops (in VPlan-native path)");
925
926 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
927 auto Itr = WideningDecisions.find(Val: InstOnVF);
928 if (Itr == WideningDecisions.end())
929 return CM_Unknown;
930 return Itr->second.first;
931 }
932
933 /// Return the vectorization cost for the given instruction \p I and vector
934 /// width \p VF.
935 InstructionCost getWideningCost(Instruction *I, ElementCount VF) {
936 assert(VF.isVector() && "Expected VF >=2");
937 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
938 assert(WideningDecisions.contains(InstOnVF) &&
939 "The cost is not calculated");
940 return WideningDecisions[InstOnVF].second;
941 }
942
943 /// Return True if instruction \p I is an optimizable truncate whose operand
944 /// is an induction variable. Such a truncate will be removed by adding a new
945 /// induction variable with the destination type.
946 bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) {
947 // If the instruction is not a truncate, return false.
948 auto *Trunc = dyn_cast<TruncInst>(Val: I);
949 if (!Trunc)
950 return false;
951
952 // Get the source and destination types of the truncate.
953 Type *SrcTy = toVectorTy(Scalar: Trunc->getSrcTy(), EC: VF);
954 Type *DestTy = toVectorTy(Scalar: Trunc->getDestTy(), EC: VF);
955
956 // If the truncate is free for the given types, return false. Replacing a
957 // free truncate with an induction variable would add an induction variable
958 // update instruction to each iteration of the loop. We exclude from this
959 // check the primary induction variable since it will need an update
960 // instruction regardless.
961 Value *Op = Trunc->getOperand(i_nocapture: 0);
962 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(Ty1: SrcTy, Ty2: DestTy))
963 return false;
964
965 // If the truncated value is not an induction variable, return false.
966 return Legal->isInductionPhi(V: Op);
967 }
968
969 /// Collects the instructions to scalarize for each predicated instruction in
970 /// the loop.
971 void collectInstsToScalarize(ElementCount VF);
972
973 /// Collect values that will not be widened, including Uniforms, Scalars, and
974 /// Instructions to Scalarize for the given \p VF.
975 /// The sets depend on CM decision for Load/Store instructions
976 /// that may be vectorized as interleave, gather-scatter or scalarized.
977 /// Also make a decision on what to do about call instructions in the loop
978 /// at that VF -- scalarize, call a known vector routine, or call a
979 /// vector intrinsic.
980 void collectNonVectorizedAndSetWideningDecisions(ElementCount VF) {
981 // Do the analysis once.
982 if (VF.isScalar() || Uniforms.contains(Val: VF))
983 return;
984 setCostBasedWideningDecision(VF);
985 collectLoopUniforms(VF);
986 collectLoopScalars(VF);
987 collectInstsToScalarize(VF);
988 }
989
990 /// Given costs for both strategies, return true if the scalar predication
991 /// lowering should be used for div/rem. This incorporates an override
992 /// option so it is not simply a cost comparison.
993 bool isDivRemScalarWithPredication(InstructionCost ScalarCost,
994 InstructionCost MaskedCost) const {
995 switch (ForceMaskedDivRem) {
996 case cl::boolOrDefault::BOU_UNSET:
997 return ScalarCost < MaskedCost;
998 case cl::boolOrDefault::BOU_TRUE:
999 return false;
1000 case cl::boolOrDefault::BOU_FALSE:
1001 return true;
1002 }
1003 llvm_unreachable("impossible case value");
1004 }
1005
1006 /// Returns true if \p I is an instruction which requires predication and
1007 /// for which our chosen predication strategy is scalarization (i.e. we
1008 /// don't have an alternate strategy such as masking available).
1009 /// \p VF is the vectorization factor that will be used to vectorize \p I.
1010 bool isScalarWithPredication(Instruction *I, ElementCount VF);
1011
1012 /// Wrapper function for LoopVectorizationLegality::isMaskRequired,
1013 /// that passes the Instruction \p I and if we fold tail.
1014 bool isMaskRequired(Instruction *I) const;
1015
1016 /// Returns true if \p I is an instruction that needs to be predicated
1017 /// at runtime. The result is independent of the predication mechanism.
1018 /// Superset of instructions that return true for isScalarWithPredication.
1019 bool isPredicatedInst(Instruction *I) const;
1020
1021 /// A helper function that returns how much we should divide the cost of a
1022 /// predicated block by. Typically this is the reciprocal of the block
1023 /// probability, i.e. if we return X we are assuming the predicated block will
1024 /// execute once for every X iterations of the loop header so the block should
1025 /// only contribute 1/X of its cost to the total cost calculation, but when
1026 /// optimizing for code size it will just be 1 as code size costs don't depend
1027 /// on execution probabilities.
1028 ///
1029 /// Note that if a block wasn't originally predicated but was predicated due
1030 /// to tail folding, the divisor will still be 1 because it will execute for
1031 /// every iteration of the loop header.
1032 inline uint64_t
1033 getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind,
1034 const BasicBlock *BB);
1035
1036 /// Returns true if an artificially high cost for emulated masked memrefs
1037 /// should be used.
1038 bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF) const;
1039
1040 /// Return the costs for our two available strategies for lowering a
1041 /// div/rem operation which requires speculating at least one lane.
1042 /// First result is for scalarization (will be invalid for scalable
1043 /// vectors); second is for the masked intrinsic strategy.
1044 std::pair<InstructionCost, InstructionCost>
1045 getDivRemSpeculationCost(Instruction *I, ElementCount VF);
1046
1047 /// If \p I is a memory instruction with a consecutive pointer that can be
1048 /// widened, returns the widening kind (CM_Widen or CM_Widen_Reverse) and
1049 /// std::nullopt otherwise.
1050 std::optional<InstWidening> memoryInstructionCanBeWidened(Instruction *I,
1051 ElementCount VF);
1052
1053 /// Returns true if \p I is a memory instruction in an interleaved-group
1054 /// of memory accesses that can be vectorized with wide vector loads/stores
1055 /// and shuffles.
1056 bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const;
1057
1058 /// Returns true if the target machine supports masked loads or stores
1059 /// for \p I's data type and alignment. The caller must ensure the access is
1060 /// consecutive or part of an interleave group.
1061 bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const;
1062
1063 /// Check if \p Instr belongs to any interleaved access group.
1064 bool isAccessInterleaved(Instruction *Instr) const {
1065 return InterleaveInfo.isInterleaved(Instr);
1066 }
1067
1068 /// Get the interleaved access group that \p Instr belongs to.
1069 const InterleaveGroup<Instruction> *
1070 getInterleavedAccessGroup(Instruction *Instr) const {
1071 return InterleaveInfo.getInterleaveGroup(Instr);
1072 }
1073
1074 /// Returns true if we're required to use a scalar epilogue for at least
1075 /// the final iteration of the original loop.
1076 bool requiresScalarEpilogue(bool IsVectorizing) const {
1077 if (!isEpilogueAllowed()) {
1078 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1079 return false;
1080 }
1081 // If we might exit from anywhere but the latch and early exit vectorization
1082 // is disabled, we must run the exiting iteration in scalar form.
1083 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
1084 !(EnableEarlyExitVectorization && Legal->hasUncountableEarlyExit())) {
1085 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
1086 "from latch block\n");
1087 return true;
1088 }
1089 if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
1090 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
1091 "interleaved group requires scalar epilogue\n");
1092 return true;
1093 }
1094 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1095 return false;
1096 }
1097
1098 /// Returns true if an epilogue is allowed (e.g., not prevented by
1099 /// optsize or a loop hint annotation).
1100 bool isEpilogueAllowed() const {
1101 return EpilogueLoweringStatus == CM_EpilogueAllowed;
1102 }
1103
1104 /// Returns true if tail-folding is preferred over an epilogue.
1105 bool preferTailFoldedLoop() const {
1106 return EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail ||
1107 EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail;
1108 }
1109
1110 /// Returns the TailFoldingStyle that is best for the current loop.
1111 TailFoldingStyle getTailFoldingStyle() const {
1112 return ChosenTailFoldingStyle;
1113 }
1114
1115 /// Selects and saves TailFoldingStyle.
1116 /// \param IsScalableVF true if scalable vector factors enabled.
1117 /// \param UserIC User specific interleave count.
1118 void setTailFoldingStyle(bool IsScalableVF, unsigned UserIC) {
1119 assert(ChosenTailFoldingStyle == TailFoldingStyle::None &&
1120 "Tail folding must not be selected yet.");
1121 if (!Legal->canFoldTailByMasking()) {
1122 ChosenTailFoldingStyle = TailFoldingStyle::None;
1123 return;
1124 }
1125
1126 // Default to TTI preference, but allow command line override.
1127 ChosenTailFoldingStyle = TTI.getPreferredTailFoldingStyle();
1128 if (ForceTailFoldingStyle.getNumOccurrences())
1129 ChosenTailFoldingStyle = ForceTailFoldingStyle.getValue();
1130
1131 if (ChosenTailFoldingStyle != TailFoldingStyle::DataWithEVL)
1132 return;
1133 // Override EVL styles if needed.
1134 // FIXME: Investigate opportunity for fixed vector factor.
1135 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
1136 TTI.hasActiveVectorLength() && !EnableVPlanNativePath;
1137 if (EVLIsLegal)
1138 return;
1139 // If for some reason EVL mode is unsupported, fallback to an epilogue
1140 // if it's allowed, or DataWithoutLaneMask otherwise.
1141 if (EpilogueLoweringStatus == CM_EpilogueAllowed ||
1142 EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail)
1143 ChosenTailFoldingStyle = TailFoldingStyle::None;
1144 else
1145 ChosenTailFoldingStyle = TailFoldingStyle::DataWithoutLaneMask;
1146
1147 LLVM_DEBUG(
1148 dbgs() << "LV: Preference for VP intrinsics indicated. Will "
1149 "not try to generate VP Intrinsics "
1150 << (UserIC > 1
1151 ? "since interleave count specified is greater than 1.\n"
1152 : "due to non-interleaving reasons.\n"));
1153 }
1154
1155 /// Returns true if all loop blocks should be masked to fold tail loop.
1156 bool foldTailByMasking() const {
1157 return getTailFoldingStyle() != TailFoldingStyle::None;
1158 }
1159
1160 void tryToEnablePartialAliasMasking() {
1161 assert(foldTailByMasking() && "Expected tail folding to be enabled!");
1162 assert(!foldTailWithEVL() &&
1163 "Did not expect to enable alias masking with EVL!");
1164 assert(PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided);
1165
1166 // Assume we fail to enable alias masking (in case we early exit).
1167 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
1168
1169 // Note: FixedOrderRecurrences are not supported yet as we cannot handle
1170 // the required `splice.right` with the alias-mask.
1171 if (!ForcePartialAliasingVectorization ||
1172 !Legal->getFixedOrderRecurrences().empty())
1173 return;
1174
1175 const RuntimePointerChecking *Checks = Legal->getRuntimePointerChecking();
1176 if (!Checks)
1177 return;
1178
1179 auto DiffChecks = Checks->getDiffChecks();
1180 if (!DiffChecks || DiffChecks->empty())
1181 return;
1182
1183 [[maybe_unused]] auto HasPointerArgs = [](CallBase *CB) {
1184 return any_of(Range: CB->args(), P: [](Value const *Arg) {
1185 return Arg->getType()->isPointerTy();
1186 });
1187 };
1188
1189 for (BasicBlock *BB : TheLoop->blocks()) {
1190 for (Instruction &I : *BB) {
1191 if (!isa<LoadInst, StoreInst>(Val: I)) {
1192 [[maybe_unused]] auto *Call = dyn_cast<CallInst>(Val: &I);
1193 assert(
1194 (!I.mayReadOrWriteMemory() || (Call && !HasPointerArgs(Call))) &&
1195 "Skipped unexpected memory access");
1196 continue;
1197 }
1198
1199 Type *ScalarTy = getLoadStoreType(I: &I);
1200 Value *Ptr = getLoadStorePointerOperand(V: &I);
1201
1202 // Currently, we can't handle alias masking in reverse. Reversing the
1203 // alias mask is not correct (or necessary). When combined with
1204 // tail-folding the active lane mask should only be reversed where the
1205 // alias-mask is true.
1206 if (Legal->isConsecutivePtr(AccessTy: ScalarTy, Ptr) == -1)
1207 return;
1208 }
1209 }
1210
1211 PartialAliasMaskingStatus = AliasMaskingStatus::Enabled;
1212 }
1213
1214 /// Returns true if all loop blocks should have partial aliases masked.
1215 bool maskPartialAliasing() const {
1216 return PartialAliasMaskingStatus == AliasMaskingStatus::Enabled;
1217 }
1218
1219 /// Returns true if the instructions in this block requires predication
1220 /// for any reason, e.g. because tail folding now requires a predicate
1221 /// or because the block in the original loop was predicated.
1222 bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const {
1223 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
1224 }
1225
1226 /// Returns true if VP intrinsics with explicit vector length support should
1227 /// be generated in the tail folded loop.
1228 bool foldTailWithEVL() const {
1229 return getTailFoldingStyle() == TailFoldingStyle::DataWithEVL;
1230 }
1231
1232 /// Returns true if the predicated reduction select should be used to set the
1233 /// incoming value for the reduction phi.
1234 bool usePredicatedReductionSelect(RecurKind RecurrenceKind) const {
1235 // Force to use predicated reduction select since the EVL of the
1236 // second-to-last iteration might not be VF*UF.
1237 if (foldTailWithEVL())
1238 return true;
1239
1240 // Force a predicated select with alias-masking to avoid propagating poison
1241 // values to the header phi for lanes outside the alias-mask.
1242 if (maskPartialAliasing())
1243 return true;
1244
1245 // Note: For FindLast recurrences we prefer a predicated select to simplify
1246 // matching in handleFindLastReductions(), rather than handle multiple
1247 // cases.
1248 if (RecurrenceDescriptor::isFindLastRecurrenceKind(Kind: RecurrenceKind))
1249 return true;
1250
1251 return PreferPredicatedReductionSelect ||
1252 TTI.preferPredicatedReductionSelect();
1253 }
1254
1255 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
1256 /// with factor VF. Return the cost of the instruction, including
1257 /// scalarization overhead if it's needed.
1258 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;
1259
1260 /// Estimate cost of a call instruction CI if it were vectorized with factor
1261 /// VF. Return the cost of the instruction, including scalarization overhead
1262 /// if it's needed.
1263 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const;
1264
1265 /// Invalidates decisions already taken by the cost model.
1266 void invalidateCostModelingDecisions() {
1267 WideningDecisions.clear();
1268 Uniforms.clear();
1269 Scalars.clear();
1270 }
1271
1272 /// Returns the expected execution cost. The unit of the cost does
1273 /// not matter because we use the 'cost' units to compare different
1274 /// vector widths. The cost that is returned is *not* normalized by
1275 /// the factor width.
1276 InstructionCost expectedCost(ElementCount VF);
1277
1278 /// Returns the execution time cost of an instruction for a given vector
1279 /// width. Vector width of one means scalar.
1280 InstructionCost getInstructionCost(Instruction *I, ElementCount VF);
1281
1282 /// Return the cost of instructions in an inloop reduction pattern, if I is
1283 /// part of that pattern.
1284 std::optional<InstructionCost> getReductionPatternCost(Instruction *I,
1285 ElementCount VF,
1286 Type *VectorTy) const;
1287
1288 /// Returns true if \p Op should be considered invariant and if it is
1289 /// trivially hoistable.
1290 bool shouldConsiderInvariant(Value *Op);
1291
1292 /// Returns true if \p I has been forced to be scalarized at \p VF.
1293 bool isForcedScalar(Instruction *I, ElementCount VF) const {
1294 auto FS = ForcedScalars.find(Val: VF);
1295 return FS != ForcedScalars.end() && FS->second.contains(key: I);
1296 }
1297
1298private:
1299 unsigned NumPredStores = 0;
1300
1301 /// VF selection state independent of cost-modeling decisions.
1302 VFSelectionContext &Config;
1303
1304 /// Wrapper around LoopVectorizationLegality::isUniform() that takes into
1305 /// account if alias-masking is enabled. We consider the VF to be unknown when
1306 /// alias masking.
1307 bool isUniform(Value *V, ElementCount VF) const {
1308 // With alias-masking our runtime VF is [2, VF] (and not necessarily a
1309 // power-of-two). Something that is uniform for VF may not be for the full
1310 // range.
1311 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
1312 "alias-mask status must be decided already");
1313 return Legal->isUniform(V, VF: PartialAliasMaskingStatus ==
1314 AliasMaskingStatus::Disabled
1315 ? std::optional(VF)
1316 : std::nullopt);
1317 }
1318
1319 /// Wrapper around LoopVectorizationLegality::isUniformMemOp() that takes into
1320 /// account if alias-masking is enabled. We consider the VF to be unknown when
1321 /// alias masking.
1322 bool isUniformMemOp(Instruction &I, ElementCount VF) const {
1323 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
1324 "alias-mask status must be decided already");
1325 return Legal->isUniformMemOp(I, VF: PartialAliasMaskingStatus ==
1326 AliasMaskingStatus::Disabled
1327 ? std::optional(VF)
1328 : std::nullopt);
1329 }
1330
1331 /// Calculate vectorization cost of memory instruction \p I.
1332 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
1333
1334 /// The cost computation for scalarized memory instruction.
1335 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1336
1337 /// The cost computation for interleaving group of memory instructions.
1338 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF) const;
1339
1340 /// The cost computation for Gather/Scatter instruction.
1341 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF) const;
1342
1343 /// The cost computation for widening instruction \p I with consecutive
1344 /// memory access.
1345 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF,
1346 InstWidening Kind);
1347
1348 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1349 /// Load: scalar load + broadcast.
1350 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1351 /// element)
1352 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF) const;
1353
1354 /// Estimate the overhead of scalarizing an instruction. This is a
1355 /// convenience wrapper for the type-based getScalarizationOverhead API.
1356 InstructionCost getScalarizationOverhead(Instruction *I,
1357 ElementCount VF) const;
1358
1359 /// A type representing the costs for instructions if they were to be
1360 /// scalarized rather than vectorized. The entries are Instruction-Cost
1361 /// pairs.
1362 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
1363
1364 /// A set containing all BasicBlocks that are known to present after
1365 /// vectorization as a predicated block.
1366 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
1367 PredicatedBBsAfterVectorization;
1368
1369 /// Records whether it is allowed to have the original scalar loop execute at
1370 /// least once. This may be needed as a fallback loop in case runtime
1371 /// aliasing/dependence checks fail, or to handle the tail/remainder
1372 /// iterations when the trip count is unknown or doesn't divide by the VF,
1373 /// or as a peel-loop to handle gaps in interleave-groups.
1374 /// Under optsize and when the trip count is very small we don't allow any
1375 /// iterations to execute in the scalar loop.
1376 EpilogueLowering EpilogueLoweringStatus = CM_EpilogueAllowed;
1377
1378 /// Control finally chosen tail folding style.
1379 TailFoldingStyle ChosenTailFoldingStyle = TailFoldingStyle::None;
1380
1381 /// If partial alias masking is enabled/disabled or not decided.
1382 AliasMaskingStatus PartialAliasMaskingStatus = AliasMaskingStatus::NotDecided;
1383
1384 /// A map holding scalar costs for different vectorization factors. The
1385 /// presence of a cost for an instruction in the mapping indicates that the
1386 /// instruction will be scalarized when vectorizing with the associated
1387 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1388 MapVector<ElementCount, ScalarCostsTy> InstsToScalarize;
1389
1390 /// Holds the instructions known to be uniform after vectorization.
1391 /// The data is collected per VF.
1392 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1393
1394 /// Holds the instructions known to be scalar after vectorization.
1395 /// The data is collected per VF.
1396 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1397
1398 /// Holds the instructions (address computations) that are forced to be
1399 /// scalarized.
1400 DenseMap<ElementCount, SmallSetVector<Instruction *, 4>> ForcedScalars;
1401
1402 /// Returns the expected difference in cost from scalarizing the expression
1403 /// feeding a predicated instruction \p PredInst. The instructions to
1404 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1405 /// non-negative return value implies the expression will be scalarized.
1406 /// Currently, only single-use chains are considered for scalarization.
1407 InstructionCost computePredInstDiscount(Instruction *PredInst,
1408 ScalarCostsTy &ScalarCosts,
1409 ElementCount VF);
1410
1411 /// Collect the instructions that are uniform after vectorization. An
1412 /// instruction is uniform if we represent it with a single scalar value in
1413 /// the vectorized loop corresponding to each vector iteration. Examples of
1414 /// uniform instructions include pointer operands of consecutive or
1415 /// interleaved memory accesses. Note that although uniformity implies an
1416 /// instruction will be scalar, the reverse is not true. In general, a
1417 /// scalarized instruction will be represented by VF scalar values in the
1418 /// vectorized loop, each corresponding to an iteration of the original
1419 /// scalar loop.
1420 void collectLoopUniforms(ElementCount VF);
1421
1422 /// Collect the instructions that are scalar after vectorization. An
1423 /// instruction is scalar if it is known to be uniform or will be scalarized
1424 /// during vectorization. collectLoopScalars should only add non-uniform nodes
1425 /// to the list if they are used by a load/store instruction that is marked as
1426 /// CM_Scalarize. Non-uniform scalarized instructions will be represented by
1427 /// VF values in the vectorized loop, each corresponding to an iteration of
1428 /// the original scalar loop.
1429 void collectLoopScalars(ElementCount VF);
1430
1431 /// Keeps cost model vectorization decision and cost for instructions.
1432 /// Right now it is used for memory instructions only.
1433 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1434 std::pair<InstWidening, InstructionCost>>;
1435
1436 DecisionList WideningDecisions;
1437
1438 /// Returns true if \p V is expected to be vectorized and it needs to be
1439 /// extracted.
1440 bool needsExtract(Value *V, ElementCount VF) const {
1441 Instruction *I = dyn_cast<Instruction>(Val: V);
1442 if (VF.isScalar() || !I || !TheLoop->contains(Inst: I) ||
1443 TheLoop->isLoopInvariant(V: I) ||
1444 getWideningDecision(I, VF) == CM_Scalarize)
1445 return false;
1446
1447 // Assume we can vectorize V (and hence we need extraction) if the
1448 // scalars are not computed yet. This can happen, because it is called
1449 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1450 // the scalars are collected. That should be a safe assumption in most
1451 // cases, because we check if the operands have vectorizable types
1452 // beforehand in LoopVectorizationLegality.
1453 return !Scalars.contains(Val: VF) || !isScalarAfterVectorization(I, VF);
1454 };
1455
1456 /// Returns a range containing only operands needing to be extracted.
1457 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
1458 ElementCount VF) const {
1459
1460 SmallPtrSet<const Value *, 4> UniqueOperands;
1461 SmallVector<Value *, 4> Res;
1462 for (Value *Op : Ops) {
1463 if (isa<Constant>(Val: Op) || !UniqueOperands.insert(Ptr: Op).second ||
1464 !needsExtract(V: Op, VF))
1465 continue;
1466 Res.push_back(Elt: Op);
1467 }
1468 return Res;
1469 }
1470
1471public:
1472 /// The loop that we evaluate.
1473 Loop *TheLoop;
1474
1475 /// Predicated scalar evolution analysis.
1476 PredicatedScalarEvolution &PSE;
1477
1478 /// Loop Info analysis.
1479 LoopInfo *LI;
1480
1481 /// Vectorization legality.
1482 LoopVectorizationLegality *Legal;
1483
1484 /// Vector target information.
1485 const TargetTransformInfo &TTI;
1486
1487 /// Target Library Info.
1488 const TargetLibraryInfo *TLI;
1489
1490 /// Assumption cache.
1491 AssumptionCache *AC;
1492
1493 /// Interface to emit optimization remarks.
1494 OptimizationRemarkEmitter *ORE;
1495
1496 /// A function to lazily fetch BlockFrequencyInfo. This avoids computing it
1497 /// unless necessary, e.g. when the loop isn't legal to vectorize or when
1498 /// there is no predication.
1499 std::function<BlockFrequencyInfo &()> GetBFI;
1500 /// The BlockFrequencyInfo returned from GetBFI.
1501 BlockFrequencyInfo *BFI = nullptr;
1502 /// Returns the BlockFrequencyInfo for the function if cached, otherwise
1503 /// fetches it via GetBFI. Avoids an indirect call to the std::function.
1504 BlockFrequencyInfo &getBFI() {
1505 if (!BFI)
1506 BFI = &GetBFI();
1507 return *BFI;
1508 }
1509
1510 const Function *TheFunction;
1511
1512 /// The interleave access information contains groups of interleaved accesses
1513 /// with the same stride and close to each other.
1514 InterleavedAccessInfo &InterleaveInfo;
1515
1516 /// Values to ignore in the cost model.
1517 SmallPtrSet<const Value *, 16> ValuesToIgnore;
1518
1519 /// Values to ignore in the cost model when VF > 1.
1520 SmallPtrSet<const Value *, 16> VecValuesToIgnore;
1521};
1522} // end namespace llvm
1523
1524namespace {
1525/// Helper struct to manage generating runtime checks for vectorization.
1526///
1527/// The runtime checks are created up-front in temporary blocks to allow better
1528/// estimating the cost and un-linked from the existing IR. After deciding to
1529/// vectorize, the checks are moved back. If deciding not to vectorize, the
1530/// temporary blocks are completely removed.
1531class GeneratedRTChecks {
1532 /// Basic block which contains the generated SCEV checks, if any.
1533 BasicBlock *SCEVCheckBlock = nullptr;
1534
1535 /// The value representing the result of the generated SCEV checks. If it is
1536 /// nullptr no SCEV checks have been generated.
1537 Value *SCEVCheckCond = nullptr;
1538
1539 /// Basic block which contains the generated memory runtime checks, if any.
1540 BasicBlock *MemCheckBlock = nullptr;
1541
1542 /// The value representing the result of the generated memory runtime checks.
1543 /// If it is nullptr no memory runtime checks have been generated.
1544 Value *MemRuntimeCheckCond = nullptr;
1545
1546 DominatorTree *DT;
1547 LoopInfo *LI;
1548 TargetTransformInfo *TTI;
1549
1550 SCEVExpander SCEVExp;
1551 SCEVExpander MemCheckExp;
1552
1553 bool CostTooHigh = false;
1554
1555 Loop *OuterLoop = nullptr;
1556
1557 PredicatedScalarEvolution &PSE;
1558
1559 /// The kind of cost that we are calculating
1560 TTI::TargetCostKind CostKind;
1561
1562 /// True if the loop is alias-masked (which allows us to omit diff checks).
1563 bool LoopUsesPartialAliasMasking = false;
1564
1565public:
1566 GeneratedRTChecks(PredicatedScalarEvolution &PSE, DominatorTree *DT,
1567 LoopInfo *LI, TargetTransformInfo *TTI,
1568 TTI::TargetCostKind CostKind,
1569 bool LoopUsesPartialAliasMasking)
1570 : DT(DT), LI(LI), TTI(TTI),
1571 SCEVExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1572 MemCheckExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1573 PSE(PSE), CostKind(CostKind),
1574 LoopUsesPartialAliasMasking(LoopUsesPartialAliasMasking) {}
1575
1576 /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
1577 /// accurately estimate the cost of the runtime checks. The blocks are
1578 /// un-linked from the IR and are added back during vector code generation. If
1579 /// there is no vector code generation, the check blocks are removed
1580 /// completely.
1581 void create(Loop *L, const LoopAccessInfo &LAI,
1582 const SCEVPredicate &UnionPred, ElementCount VF, unsigned IC,
1583 OptimizationRemarkEmitter &ORE) {
1584
1585 // Hard cutoff to limit compile-time increase in case a very large number of
1586 // runtime checks needs to be generated.
1587 // TODO: Skip cutoff if the loop is guaranteed to execute, e.g. due to
1588 // profile info.
1589 CostTooHigh =
1590 LAI.getNumRuntimePointerChecks() > VectorizeMemoryCheckThreshold;
1591 if (CostTooHigh) {
1592 // Mark runtime checks as never succeeding when they exceed the threshold.
1593 MemRuntimeCheckCond = ConstantInt::getTrue(Context&: L->getHeader()->getContext());
1594 SCEVCheckCond = ConstantInt::getTrue(Context&: L->getHeader()->getContext());
1595 ORE.emit(RemarkBuilder: [&]() {
1596 return OptimizationRemarkAnalysisAliasing(
1597 DEBUG_TYPE, "TooManyMemoryRuntimeChecks", L->getStartLoc(),
1598 L->getHeader())
1599 << "loop not vectorized: too many memory checks needed";
1600 });
1601 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
1602 return;
1603 }
1604
1605 BasicBlock *LoopHeader = L->getHeader();
1606 BasicBlock *Preheader = L->getLoopPreheader();
1607
1608 // Use SplitBlock to create blocks for SCEV & memory runtime checks to
1609 // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
1610 // may be used by SCEVExpander. The blocks will be un-linked from their
1611 // predecessors and removed from LI & DT at the end of the function.
1612 if (!UnionPred.isAlwaysTrue()) {
1613 SCEVCheckBlock = SplitBlock(Old: Preheader, SplitPt: Preheader->getTerminator(), DT, LI,
1614 MSSAU: nullptr, BBName: "vector.scevcheck");
1615
1616 SCEVCheckCond = SCEVExp.expandCodeForPredicate(
1617 Pred: &UnionPred, Loc: SCEVCheckBlock->getTerminator());
1618 if (isa<Constant>(Val: SCEVCheckCond)) {
1619 // Clean up directly after expanding the predicate to a constant, to
1620 // avoid further expansions re-using anything left over from SCEVExp.
1621 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1622 SCEVCleaner.cleanup();
1623 }
1624 }
1625
1626 const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
1627 // TODO: We need to estimate the cost of alias-masking in
1628 // GeneratedRTChecks::getCost(). We can't check the MemCheckBlock as the
1629 // alias-mask is generated later in VPlan.
1630 if (RtPtrChecking.Need && !LoopUsesPartialAliasMasking) {
1631 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1632 MemCheckBlock = SplitBlock(Old: Pred, SplitPt: Pred->getTerminator(), DT, LI, MSSAU: nullptr,
1633 BBName: "vector.memcheck");
1634
1635 auto DiffChecks = RtPtrChecking.getDiffChecks();
1636 if (DiffChecks) {
1637 MemRuntimeCheckCond = addDiffRuntimeChecks(
1638 Loc: MemCheckBlock->getTerminator(), Checks: *DiffChecks, Expander&: MemCheckExp, VF, IC);
1639 } else {
1640 MemRuntimeCheckCond = addRuntimeChecks(
1641 Loc: MemCheckBlock->getTerminator(), TheLoop: L, PointerChecks: RtPtrChecking.getChecks(),
1642 Expander&: MemCheckExp, HoistRuntimeChecks: VectorizerParams::HoistRuntimeChecks);
1643 }
1644 assert(MemRuntimeCheckCond &&
1645 "no RT checks generated although RtPtrChecking "
1646 "claimed checks are required");
1647 }
1648
1649 SCEVExp.eraseDeadInstructions(Root: SCEVCheckCond);
1650
1651 if (!MemCheckBlock && !SCEVCheckBlock)
1652 return;
1653
1654 // Unhook the temporary block with the checks, update various places
1655 // accordingly.
1656 if (SCEVCheckBlock)
1657 SCEVCheckBlock->replaceAllUsesWith(V: Preheader);
1658 if (MemCheckBlock)
1659 MemCheckBlock->replaceAllUsesWith(V: Preheader);
1660
1661 if (SCEVCheckBlock) {
1662 SCEVCheckBlock->getTerminator()->moveBefore(
1663 InsertPos: Preheader->getTerminator()->getIterator());
1664 auto *UI = new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
1665 UI->setDebugLoc(DebugLoc::getTemporary());
1666 Preheader->getTerminator()->eraseFromParent();
1667 }
1668 if (MemCheckBlock) {
1669 MemCheckBlock->getTerminator()->moveBefore(
1670 InsertPos: Preheader->getTerminator()->getIterator());
1671 auto *UI = new UnreachableInst(Preheader->getContext(), MemCheckBlock);
1672 UI->setDebugLoc(DebugLoc::getTemporary());
1673 Preheader->getTerminator()->eraseFromParent();
1674 }
1675
1676 DT->changeImmediateDominator(BB: LoopHeader, NewBB: Preheader);
1677 if (MemCheckBlock) {
1678 DT->eraseNode(BB: MemCheckBlock);
1679 LI->removeBlock(BB: MemCheckBlock);
1680 }
1681 if (SCEVCheckBlock) {
1682 DT->eraseNode(BB: SCEVCheckBlock);
1683 LI->removeBlock(BB: SCEVCheckBlock);
1684 }
1685
1686 // Outer loop is used as part of the later cost calculations.
1687 OuterLoop = L->getParentLoop();
1688 }
1689
1690 InstructionCost getCost() {
1691 if (SCEVCheckBlock || MemCheckBlock)
1692 LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
1693
1694 if (CostTooHigh) {
1695 InstructionCost Cost;
1696 Cost.setInvalid();
1697 LLVM_DEBUG(dbgs() << " number of checks exceeded threshold\n");
1698 return Cost;
1699 }
1700
1701 InstructionCost RTCheckCost = 0;
1702 if (SCEVCheckBlock)
1703 for (Instruction &I : *SCEVCheckBlock) {
1704 if (SCEVCheckBlock->getTerminator() == &I)
1705 continue;
1706 InstructionCost C = TTI->getInstructionCost(U: &I, CostKind);
1707 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1708 RTCheckCost += C;
1709 }
1710 if (MemCheckBlock) {
1711 InstructionCost MemCheckCost = 0;
1712 for (Instruction &I : *MemCheckBlock) {
1713 if (MemCheckBlock->getTerminator() == &I)
1714 continue;
1715 InstructionCost C = TTI->getInstructionCost(U: &I, CostKind);
1716 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1717 MemCheckCost += C;
1718 }
1719
1720 // If the runtime memory checks are being created inside an outer loop
1721 // we should find out if these checks are outer loop invariant. If so,
1722 // the checks will likely be hoisted out and so the effective cost will
1723 // reduce according to the outer loop trip count.
1724 if (OuterLoop) {
1725 ScalarEvolution *SE = MemCheckExp.getSE();
1726 // TODO: If profitable, we could refine this further by analysing every
1727 // individual memory check, since there could be a mixture of loop
1728 // variant and invariant checks that mean the final condition is
1729 // variant.
1730 const SCEV *Cond = SE->getSCEV(V: MemRuntimeCheckCond);
1731 if (SE->isLoopInvariant(S: Cond, L: OuterLoop)) {
1732 // It seems reasonable to assume that we can reduce the effective
1733 // cost of the checks even when we know nothing about the trip
1734 // count. Assume that the outer loop executes at least twice.
1735 unsigned BestTripCount = 2;
1736
1737 // Get the best known TC estimate.
1738 if (auto EstimatedTC = getSmallBestKnownTC(
1739 PSE, L: OuterLoop, /* CanUseConstantMax = */ false))
1740 if (EstimatedTC->isFixed())
1741 BestTripCount = EstimatedTC->getFixedValue();
1742
1743 InstructionCost NewMemCheckCost = MemCheckCost / BestTripCount;
1744
1745 // Let's ensure the cost is always at least 1.
1746 NewMemCheckCost = std::max(a: NewMemCheckCost.getValue(),
1747 b: (InstructionCost::CostType)1);
1748
1749 if (BestTripCount > 1)
1750 LLVM_DEBUG(dbgs()
1751 << "We expect runtime memory checks to be hoisted "
1752 << "out of the outer loop. Cost reduced from "
1753 << MemCheckCost << " to " << NewMemCheckCost << '\n');
1754
1755 MemCheckCost = NewMemCheckCost;
1756 }
1757 }
1758
1759 RTCheckCost += MemCheckCost;
1760 }
1761
1762 if (SCEVCheckBlock || MemCheckBlock)
1763 LLVM_DEBUG(dbgs() << "Total cost of runtime checks: " << RTCheckCost
1764 << "\n");
1765
1766 return RTCheckCost;
1767 }
1768
1769 /// Remove the created SCEV & memory runtime check blocks & instructions, if
1770 /// unused.
1771 ~GeneratedRTChecks() {
1772 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1773 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1774 bool SCEVChecksUsed = !SCEVCheckBlock || !pred_empty(BB: SCEVCheckBlock);
1775 bool MemChecksUsed = !MemCheckBlock || !pred_empty(BB: MemCheckBlock);
1776 if (SCEVChecksUsed)
1777 SCEVCleaner.markResultUsed();
1778
1779 if (MemChecksUsed) {
1780 MemCheckCleaner.markResultUsed();
1781 } else {
1782 auto &SE = *MemCheckExp.getSE();
1783 // Memory runtime check generation creates compares that use expanded
1784 // values. Remove them before running the SCEVExpanderCleaners.
1785 for (auto &I : make_early_inc_range(Range: reverse(C&: *MemCheckBlock))) {
1786 if (MemCheckExp.isInsertedInstruction(I: &I))
1787 continue;
1788 SE.forgetValue(V: &I);
1789 I.eraseFromParent();
1790 }
1791 }
1792 MemCheckCleaner.cleanup();
1793 SCEVCleaner.cleanup();
1794
1795 if (!SCEVChecksUsed)
1796 SCEVCheckBlock->eraseFromParent();
1797 if (!MemChecksUsed)
1798 MemCheckBlock->eraseFromParent();
1799 }
1800
1801 /// Retrieves the SCEVCheckCond and SCEVCheckBlock that were generated as IR
1802 /// outside VPlan.
1803 std::pair<Value *, BasicBlock *> getSCEVChecks() const {
1804 using namespace llvm::PatternMatch;
1805 if (!SCEVCheckCond || match(V: SCEVCheckCond, P: m_ZeroInt()))
1806 return {nullptr, nullptr};
1807
1808 return {SCEVCheckCond, SCEVCheckBlock};
1809 }
1810
1811 /// Retrieves the MemCheckCond and MemCheckBlock that were generated as IR
1812 /// outside VPlan.
1813 std::pair<Value *, BasicBlock *> getMemRuntimeChecks() const {
1814 using namespace llvm::PatternMatch;
1815 if (MemRuntimeCheckCond && match(V: MemRuntimeCheckCond, P: m_ZeroInt()))
1816 return {nullptr, nullptr};
1817 return {MemRuntimeCheckCond, MemCheckBlock};
1818 }
1819
1820 /// Return true if any runtime checks have been added
1821 bool hasChecks() const {
1822 return getSCEVChecks().first || getMemRuntimeChecks().first;
1823 }
1824};
1825} // namespace
1826
1827static bool useActiveLaneMask(TailFoldingStyle Style) {
1828 return Style == TailFoldingStyle::Data ||
1829 Style == TailFoldingStyle::DataAndControlFlow;
1830}
1831
1832static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style) {
1833 return Style == TailFoldingStyle::DataAndControlFlow;
1834}
1835
1836// Return true if \p OuterLp is an outer loop annotated with hints for explicit
1837// vectorization. The loop needs to be annotated with #pragma omp simd
1838// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
1839// vector length information is not provided, vectorization is not considered
1840// explicit. Interleave hints are not allowed either. These limitations will be
1841// relaxed in the future.
1842// Please, note that we are currently forced to abuse the pragma 'clang
1843// vectorize' semantics. This pragma provides *auto-vectorization hints*
1844// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
1845// provides *explicit vectorization hints* (LV can bypass legal checks and
1846// assume that vectorization is legal). However, both hints are implemented
1847// using the same metadata (llvm.loop.vectorize, processed by
1848// LoopVectorizeHints). This will be fixed in the future when the native IR
1849// representation for pragma 'omp simd' is introduced.
1850static bool isExplicitVecOuterLoop(Loop *OuterLp,
1851 OptimizationRemarkEmitter *ORE) {
1852 assert(!OuterLp->isInnermost() && "This is not an outer loop");
1853 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
1854
1855 // Only outer loops with an explicit vectorization hint are supported.
1856 // Unannotated outer loops are ignored.
1857 if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
1858 return false;
1859
1860 Function *Fn = OuterLp->getHeader()->getParent();
1861 if (!Hints.allowVectorization(F: Fn, L: OuterLp,
1862 VectorizeOnlyWhenForced: true /*VectorizeOnlyWhenForced*/)) {
1863 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
1864 return false;
1865 }
1866
1867 if (Hints.getInterleave() > 1) {
1868 // TODO: Interleave support is future work.
1869 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
1870 "outer loops.\n");
1871 Hints.emitRemarkWithHints();
1872 return false;
1873 }
1874
1875 return true;
1876}
1877
1878static void collectSupportedLoops(Loop &L, LoopInfo *LI,
1879 OptimizationRemarkEmitter *ORE,
1880 SmallVectorImpl<Loop *> &V) {
1881 // Collect inner loops and outer loops without irreducible control flow. For
1882 // now, only collect outer loops that have explicit vectorization hints. If we
1883 // are stress testing the VPlan H-CFG construction, we collect the outermost
1884 // loop of every loop nest.
1885 if (L.isInnermost() || VPlanBuildOuterloopStressTest ||
1886 (EnableVPlanNativePath && isExplicitVecOuterLoop(OuterLp: &L, ORE))) {
1887 LoopBlocksRPO RPOT(&L);
1888 RPOT.perform(LI);
1889 if (!containsIrreducibleCFG<const BasicBlock *>(RPOTraversal&: RPOT, LI: *LI)) {
1890 V.push_back(Elt: &L);
1891 // TODO: Collect inner loops inside marked outer loops in case
1892 // vectorization fails for the outer loop. Do not invoke
1893 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
1894 // already known to be reducible. We can use an inherited attribute for
1895 // that.
1896 return;
1897 }
1898 }
1899 for (Loop *InnerL : L)
1900 collectSupportedLoops(L&: *InnerL, LI, ORE, V);
1901}
1902
1903//===----------------------------------------------------------------------===//
1904// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
1905// LoopVectorizationCostModel and LoopVectorizationPlanner.
1906//===----------------------------------------------------------------------===//
1907
1908/// For the given VF and UF and maximum trip count computed for the loop, return
1909/// whether the induction variable might overflow in the vectorized loop. If not,
1910/// then we know a runtime overflow check always evaluates to false and can be
1911/// removed.
1912static bool isIndvarOverflowCheckKnownFalse(
1913 const LoopVectorizationCostModel *Cost,
1914 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
1915 // Always be conservative if we don't know the exact unroll factor.
1916 uint64_t MaxUF = UF ? *UF
1917 : std::max(a: Cost->TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions: false),
1918 b: Cost->TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions: true));
1919
1920 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
1921 APInt MaxUIntTripCount = IdxTy->getMask();
1922
1923 // We know the runtime overflow check is known false iff the (max) trip-count
1924 // is known and (max) trip-count + (VF * UF) does not overflow in the type of
1925 // the vector loop induction variable.
1926 if (std::optional<ElementCount> TC = getSmallBestKnownTC(
1927 PSE&: Cost->PSE, L: Cost->TheLoop,
1928 /*CanUseConstantMax=*/true, /*CanExcludeZeroTrips=*/false,
1929 /*ComputeUpperBoundOnly=*/true)) {
1930 uint64_t MaxVF = VF.getKnownMinValue();
1931 uint64_t MaxTC = TC->getKnownMinValue();
1932 if (VF.isScalable() || TC->isScalable()) {
1933 std::optional<unsigned> MaxVScale =
1934 getMaxVScale(F: *Cost->TheFunction, TTI: Cost->TTI);
1935 if (!MaxVScale)
1936 return false;
1937 if (VF.isScalable())
1938 MaxVF *= *MaxVScale;
1939 if (TC->isScalable())
1940 MaxTC *= *MaxVScale;
1941 }
1942
1943 // Bail out if the maximum trip count is not representable in the induction
1944 // variable's type.
1945 if (MaxUIntTripCount.ult(RHS: MaxTC))
1946 return false;
1947
1948 uint64_t MaxStep = MaxVF * MaxUF;
1949 return (MaxUIntTripCount - MaxTC).ugt(RHS: MaxStep);
1950 }
1951
1952 return false;
1953}
1954
1955// Return whether we allow using masked interleave-groups (for dealing with
1956// strided loads/stores that reside in predicated blocks, or for dealing
1957// with gaps).
1958static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
1959 // If an override option has been passed in for interleaved accesses, use it.
1960 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
1961 return EnableMaskedInterleavedMemAccesses;
1962
1963 return TTI.enableMaskedInterleavedAccessVectorization();
1964}
1965
1966/// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
1967/// VPBB are moved to the end of the newly created VPIRBasicBlock. All
1968/// predecessors and successors of VPBB, if any, are rewired to the new
1969/// VPIRBasicBlock. If \p VPBB may be unreachable, \p Plan must be passed.
1970static VPIRBasicBlock *replaceVPBBWithIRVPBB(VPBasicBlock *VPBB,
1971 BasicBlock *IRBB,
1972 VPlan *Plan = nullptr) {
1973 if (!Plan)
1974 Plan = VPBB->getPlan();
1975 VPIRBasicBlock *IRVPBB = Plan->createVPIRBasicBlock(IRBB);
1976 auto IP = IRVPBB->begin();
1977 for (auto &R : make_early_inc_range(Range: VPBB->phis()))
1978 R.moveBefore(BB&: *IRVPBB, I: IP);
1979
1980 for (auto &R :
1981 make_early_inc_range(Range: make_range(x: VPBB->getFirstNonPhi(), y: VPBB->end())))
1982 R.moveBefore(BB&: *IRVPBB, I: IRVPBB->end());
1983
1984 VPBlockUtils::reassociateBlocks(Old: VPBB, New: IRVPBB);
1985 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
1986 return IRVPBB;
1987}
1988
1989BasicBlock *InnerLoopVectorizer::createScalarPreheader(StringRef Prefix) {
1990 BasicBlock *VectorPH = OrigLoop->getLoopPreheader();
1991 assert(VectorPH && "Invalid loop structure");
1992
1993 // NOTE: The Plan's scalar preheader VPBB isn't replaced with a VPIRBasicBlock
1994 // wrapping the newly created scalar preheader here at the moment, because the
1995 // Plan's scalar preheader may be unreachable at this point. Instead it is
1996 // replaced in executePlan.
1997 return SplitBlock(Old: VectorPH, SplitPt: VectorPH->getTerminator(), DT, LI, MSSAU: nullptr,
1998 BBName: Twine(Prefix) + "scalar.ph");
1999}
2000
2001/// Knowing that loop \p L executes a single vector iteration, add instructions
2002/// that will get simplified and thus should not have any cost to \p
2003/// InstsToIgnore.
2004static void addFullyUnrolledInstructionsToIgnore(
2005 Loop *L, const LoopVectorizationLegality::InductionList &IL,
2006 SmallPtrSetImpl<Instruction *> &InstsToIgnore) {
2007 auto *Cmp = L->getLatchCmpInst();
2008 if (Cmp)
2009 InstsToIgnore.insert(Ptr: Cmp);
2010 for (const auto &KV : IL) {
2011 // Extract the key by hand so that it can be used in the lambda below. Note
2012 // that captured structured bindings are a C++20 extension.
2013 const PHINode *IV = KV.first;
2014
2015 // Get next iteration value of the induction variable.
2016 Instruction *IVInst =
2017 cast<Instruction>(Val: IV->getIncomingValueForBlock(BB: L->getLoopLatch()));
2018 if (all_of(Range: IVInst->users(),
2019 P: [&](const User *U) { return U == IV || U == Cmp; }))
2020 InstsToIgnore.insert(Ptr: IVInst);
2021 }
2022}
2023
2024BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
2025 // Create a new IR basic block for the scalar preheader.
2026 BasicBlock *ScalarPH = createScalarPreheader(Prefix: "");
2027 return ScalarPH->getSinglePredecessor();
2028}
2029
2030namespace {
2031
2032struct CSEDenseMapInfo {
2033 static bool canHandle(const Instruction *I) {
2034 return isa<InsertElementInst>(Val: I) || isa<ExtractElementInst>(Val: I) ||
2035 isa<ShuffleVectorInst>(Val: I) || isa<GetElementPtrInst>(Val: I);
2036 }
2037
2038 static unsigned getHashValue(const Instruction *I) {
2039 assert(canHandle(I) && "Unknown instruction!");
2040 return hash_combine(args: I->getOpcode(),
2041 args: hash_combine_range(R: I->operand_values()));
2042 }
2043
2044 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
2045 return LHS->isIdenticalTo(I: RHS);
2046 }
2047};
2048
2049} // end anonymous namespace
2050
2051/// FIXME: This legacy common-subexpression-elimination routine is scheduled for
2052/// removal, in favor of the VPlan-based one.
2053static void legacyCSE(BasicBlock *BB) {
2054 // Perform simple cse.
2055 SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
2056 for (Instruction &In : llvm::make_early_inc_range(Range&: *BB)) {
2057 if (!CSEDenseMapInfo::canHandle(I: &In))
2058 continue;
2059
2060 // Check if we can replace this instruction with any of the
2061 // visited instructions.
2062 if (Instruction *V = CSEMap.lookup(Val: &In)) {
2063 In.replaceAllUsesWith(V);
2064 In.eraseFromParent();
2065 continue;
2066 }
2067
2068 CSEMap[&In] = &In;
2069 }
2070}
2071
2072/// This function attempts to return a value that represents the ElementCount
2073/// at runtime. For fixed-width VFs we know this precisely at compile
2074/// time, but for scalable VFs we calculate it based on an estimate of the
2075/// vscale value.
2076static unsigned estimateElementCount(ElementCount VF,
2077 std::optional<unsigned> VScale) {
2078 unsigned EstimatedVF = VF.getKnownMinValue();
2079 if (VF.isScalable())
2080 if (VScale)
2081 EstimatedVF *= *VScale;
2082 assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
2083 return EstimatedVF;
2084}
2085
2086/// Returns the vector library variant function of \p CI usable at \p VF,
2087/// respecting \p MaskRequired, or nullptr if none is found: a mapping with
2088/// matching VF, masked if required, whose vector function is declared in the
2089/// module.
2090static Function *getVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
2091 bool MaskRequired,
2092 const TargetLibraryInfo *TLI) {
2093 if (!TLI || CI.isNoBuiltin())
2094 return nullptr;
2095 for (const VFInfo &Info : VFDatabase::getMappings(CI))
2096 if (Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()))
2097 if (Function *F = CI.getModule()->getFunction(Name: Info.VectorName))
2098 return F;
2099 return nullptr;
2100}
2101
2102/// Returns true iff \p CI has a library vector variant usable at \p VF.
2103static bool hasVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
2104 bool MaskRequired,
2105 const TargetLibraryInfo *TLI) {
2106 return getVectorLibraryVariantFor(CI, VF, MaskRequired, TLI) != nullptr;
2107}
2108
2109InstructionCost
2110LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
2111 ElementCount VF) const {
2112 Type *RetTy = CI->getType();
2113 SmallVector<Type *, 4> Tys;
2114 for (auto &ArgOp : CI->args())
2115 Tys.push_back(Elt: ArgOp->getType());
2116
2117 InstructionCost ScalarCallCost = TTI.getCallInstrCost(
2118 F: CI->getCalledFunction(), RetTy, Tys, CostKind: Config.CostKind);
2119
2120 // Cost of the scalar call (scalar VF) or its scalarization (vector VF). The
2121 // scalarization cost is only meaningful for fixed VFs.
2122 InstructionCost Cost = VF.isScalable()
2123 ? InstructionCost::getInvalid()
2124 : ScalarCallCost * VF.getKnownMinValue() +
2125 getScalarizationOverhead(I: CI, VF);
2126
2127 // The call may be vectorized at this VF, via a vector intrinsic or a vector
2128 // library variant.
2129 if (getVectorIntrinsicIDForCall(CI, TLI))
2130 Cost = std::min(a: Cost, b: getVectorIntrinsicCost(CI, VF));
2131
2132 if (Function *Variant =
2133 getVectorLibraryVariantFor(CI: *CI, VF, MaskRequired: isMaskRequired(I: CI), TLI))
2134 Cost = std::min(a: Cost,
2135 b: TTI.getCallInstrCost(
2136 /*F=*/nullptr, RetTy: Variant->getReturnType(),
2137 Tys: Variant->getFunctionType()->params(), CostKind: Config.CostKind));
2138
2139 return Cost;
2140}
2141
2142static Type *maybeVectorizeType(Type *Ty, ElementCount VF) {
2143 if (VF.isScalar() || !canVectorizeTy(Ty))
2144 return Ty;
2145 return toVectorizedTy(Ty, EC: VF);
2146}
2147
2148InstructionCost
2149LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
2150 ElementCount VF) const {
2151 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2152 assert(ID && "Expected intrinsic call!");
2153 Type *RetTy = maybeVectorizeType(Ty: CI->getType(), VF);
2154 FastMathFlags FMF;
2155 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: CI))
2156 FMF = FPMO->getFastMathFlags();
2157
2158 SmallVector<const Value *> Arguments(CI->args());
2159 FunctionType *FTy = CI->getCalledFunction()->getFunctionType();
2160 SmallVector<Type *> ParamTys;
2161 std::transform(first: FTy->param_begin(), last: FTy->param_end(),
2162 result: std::back_inserter(x&: ParamTys),
2163 unary_op: [&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
2164
2165 IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
2166 dyn_cast<IntrinsicInst>(Val: CI),
2167 InstructionCost::getInvalid());
2168 return TTI.getIntrinsicInstrCost(ICA: CostAttrs, CostKind: Config.CostKind);
2169}
2170
2171void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
2172 // Don't apply optimizations below when no (vector) loop remains, as they all
2173 // require one at the moment.
2174 VPBasicBlock *HeaderVPBB =
2175 vputils::getFirstLoopHeader(Plan&: *State.Plan, VPDT&: State.VPDT);
2176 if (!HeaderVPBB)
2177 return;
2178
2179 BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
2180
2181 // Remove redundant induction instructions.
2182 legacyCSE(BB: HeaderBB);
2183}
2184
2185void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
2186 // We should not collect Scalars more than once per VF. Right now, this
2187 // function is called from collectUniformsAndScalars(), which already does
2188 // this check. Collecting Scalars for VF=1 does not make any sense.
2189 assert(VF.isVector() && !Scalars.contains(VF) &&
2190 "This function should not be visited twice for the same VF");
2191
2192 // This avoids any chances of creating a REPLICATE recipe during planning
2193 // since that would result in generation of scalarized code during execution,
2194 // which is not supported for scalable vectors.
2195 if (VF.isScalable()) {
2196 Scalars[VF].insert_range(R&: Uniforms[VF]);
2197 return;
2198 }
2199
2200 SmallSetVector<Instruction *, 8> Worklist;
2201
2202 // These sets are used to seed the analysis with pointers used by memory
2203 // accesses that will remain scalar.
2204 SmallSetVector<Instruction *, 8> ScalarPtrs;
2205 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
2206 auto *Latch = TheLoop->getLoopLatch();
2207
2208 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
2209 // The pointer operands of loads and stores will be scalar as long as the
2210 // memory access is not a gather/scatter or histogram operation. The value
2211 // operand of a store will remain scalar if the store is scalarized.
2212 auto IsScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
2213 InstWidening WideningDecision = getWideningDecision(I: MemAccess, VF);
2214 assert(WideningDecision != CM_Unknown &&
2215 "Widening decision should be ready at this moment");
2216 auto *Store = dyn_cast<StoreInst>(Val: MemAccess);
2217 if (Store && Ptr == Store->getValueOperand())
2218 return WideningDecision == CM_Scalarize;
2219 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&
2220 "Ptr is neither a value or pointer operand");
2221 return WideningDecision != CM_GatherScatter &&
2222 !(Store && Legal->getHistogramInfo(I: Store));
2223 };
2224
2225 // A helper that returns true if the given value is a getelementptr
2226 // instruction contained in the loop.
2227 auto IsLoopVaryingGEP = [&](Value *V) {
2228 return isa<GetElementPtrInst>(Val: V) && !TheLoop->isLoopInvariant(V);
2229 };
2230
2231 // A helper that evaluates a memory access's use of a pointer. If the use will
2232 // be a scalar use and the pointer is only used by memory accesses, we place
2233 // the pointer in ScalarPtrs. Otherwise, the pointer is placed in
2234 // PossibleNonScalarPtrs.
2235 auto EvaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
2236 // We only care about bitcast and getelementptr instructions contained in
2237 // the loop.
2238 if (!IsLoopVaryingGEP(Ptr))
2239 return;
2240
2241 // If the pointer has already been identified as scalar (e.g., if it was
2242 // also identified as uniform), there's nothing to do.
2243 auto *I = cast<Instruction>(Val: Ptr);
2244 if (Worklist.count(key: I))
2245 return;
2246
2247 // If the use of the pointer will be a scalar use, and all users of the
2248 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
2249 // place the pointer in PossibleNonScalarPtrs.
2250 if (IsScalarUse(MemAccess, Ptr) &&
2251 all_of(Range: I->users(), P: IsaPred<LoadInst, StoreInst>))
2252 ScalarPtrs.insert(X: I);
2253 else
2254 PossibleNonScalarPtrs.insert(Ptr: I);
2255 };
2256
2257 // We seed the scalars analysis with three classes of instructions: (1)
2258 // instructions marked uniform-after-vectorization and (2) bitcast,
2259 // getelementptr and (pointer) phi instructions used by memory accesses
2260 // requiring a scalar use.
2261 //
2262 // (1) Add to the worklist all instructions that have been identified as
2263 // uniform-after-vectorization.
2264 Worklist.insert_range(R&: Uniforms[VF]);
2265
2266 // (2) Add to the worklist all bitcast and getelementptr instructions used by
2267 // memory accesses requiring a scalar use. The pointer operands of loads and
2268 // stores will be scalar unless the operation is a gather or scatter.
2269 // The value operand of a store will remain scalar if the store is scalarized.
2270 for (auto *BB : TheLoop->blocks())
2271 for (auto &I : *BB) {
2272 if (auto *Load = dyn_cast<LoadInst>(Val: &I)) {
2273 EvaluatePtrUse(Load, Load->getPointerOperand());
2274 } else if (auto *Store = dyn_cast<StoreInst>(Val: &I)) {
2275 EvaluatePtrUse(Store, Store->getPointerOperand());
2276 EvaluatePtrUse(Store, Store->getValueOperand());
2277 }
2278 }
2279 for (auto *I : ScalarPtrs)
2280 if (!PossibleNonScalarPtrs.count(Ptr: I)) {
2281 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n");
2282 Worklist.insert(X: I);
2283 }
2284
2285 // Insert the forced scalars.
2286 // FIXME: Currently VPWidenPHIRecipe() often creates a dead vector
2287 // induction variable when the PHI user is scalarized.
2288 auto ForcedScalar = ForcedScalars.find(Val: VF);
2289 if (ForcedScalar != ForcedScalars.end())
2290 for (auto *I : ForcedScalar->second) {
2291 LLVM_DEBUG(dbgs() << "LV: Found (forced) scalar instruction: " << *I << "\n");
2292 Worklist.insert(X: I);
2293 }
2294
2295 // Expand the worklist by looking through any bitcasts and getelementptr
2296 // instructions we've already identified as scalar. This is similar to the
2297 // expansion step in collectLoopUniforms(); however, here we're only
2298 // expanding to include additional bitcasts and getelementptr instructions.
2299 unsigned Idx = 0;
2300 while (Idx != Worklist.size()) {
2301 Instruction *Dst = Worklist[Idx++];
2302 if (!IsLoopVaryingGEP(Dst->getOperand(i: 0)))
2303 continue;
2304 auto *Src = cast<Instruction>(Val: Dst->getOperand(i: 0));
2305 if (llvm::all_of(Range: Src->users(), P: [&](User *U) -> bool {
2306 auto *J = cast<Instruction>(Val: U);
2307 return !TheLoop->contains(Inst: J) || Worklist.count(key: J) ||
2308 ((isa<LoadInst>(Val: J) || isa<StoreInst>(Val: J)) &&
2309 IsScalarUse(J, Src));
2310 })) {
2311 Worklist.insert(X: Src);
2312 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n");
2313 }
2314 }
2315
2316 // An induction variable will remain scalar if all users of the induction
2317 // variable and induction variable update remain scalar.
2318 for (const auto &Induction : Legal->getInductionVars()) {
2319 auto *Ind = Induction.first;
2320 auto *IndUpdate = cast<Instruction>(Val: Ind->getIncomingValueForBlock(BB: Latch));
2321
2322 // If tail-folding is applied, the primary induction variable will be used
2323 // to feed a vector compare.
2324 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
2325 continue;
2326
2327 // Returns true if \p Indvar is a pointer induction that is used directly by
2328 // load/store instruction \p I.
2329 auto IsDirectLoadStoreFromPtrIndvar = [&](Instruction *Indvar,
2330 Instruction *I) {
2331 return Induction.second.getKind() ==
2332 InductionDescriptor::IK_PtrInduction &&
2333 (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I)) &&
2334 Indvar == getLoadStorePointerOperand(V: I) && IsScalarUse(I, Indvar);
2335 };
2336
2337 // Determine if all users of the induction variable are scalar after
2338 // vectorization.
2339 bool ScalarInd = all_of(Range: Ind->users(), P: [&](User *U) -> bool {
2340 auto *I = cast<Instruction>(Val: U);
2341 return I == IndUpdate || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2342 IsDirectLoadStoreFromPtrIndvar(Ind, I);
2343 });
2344 if (!ScalarInd)
2345 continue;
2346
2347 // If the induction variable update is a fixed-order recurrence, neither the
2348 // induction variable or its update should be marked scalar after
2349 // vectorization.
2350 auto *IndUpdatePhi = dyn_cast<PHINode>(Val: IndUpdate);
2351 if (IndUpdatePhi && Legal->isFixedOrderRecurrence(Phi: IndUpdatePhi))
2352 continue;
2353
2354 // Determine if all users of the induction variable update instruction are
2355 // scalar after vectorization.
2356 bool ScalarIndUpdate = all_of(Range: IndUpdate->users(), P: [&](User *U) -> bool {
2357 auto *I = cast<Instruction>(Val: U);
2358 return I == Ind || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2359 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
2360 });
2361 if (!ScalarIndUpdate)
2362 continue;
2363
2364 // The induction variable and its update instruction will remain scalar.
2365 Worklist.insert(X: Ind);
2366 Worklist.insert(X: IndUpdate);
2367 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n");
2368 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate
2369 << "\n");
2370 }
2371
2372 Scalars[VF].insert_range(R&: Worklist);
2373}
2374
2375bool LoopVectorizationCostModel::isLegalMaskedLoadOrStore(
2376 Instruction *I, ElementCount VF) const {
2377 assert(isa<LoadInst>(I) || isa<StoreInst>(I));
2378 return Config.isLegalMaskedLoadOrStore(IsLoad: isa<LoadInst>(Val: I), ScalarTy: getLoadStoreType(I),
2379 Alignment: getLoadStoreAlignment(I),
2380 AddressSpace: getLoadStoreAddressSpace(I));
2381}
2382
2383bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
2384 ElementCount VF) {
2385 if (!isPredicatedInst(I))
2386 return false;
2387
2388 // Do we have a non-scalar lowering for this predicated
2389 // instruction? No - it is scalar with predication.
2390 switch(I->getOpcode()) {
2391 default:
2392 return true;
2393 case Instruction::Call: {
2394 if (VF.isScalar())
2395 return true;
2396 auto *CI = cast<CallInst>(Val: I);
2397 // A vector intrinsic or library variant lowering avoids scalarization.
2398 return !getVectorIntrinsicIDForCall(CI, TLI) &&
2399 !hasVectorLibraryVariantFor(CI: *CI, VF, MaskRequired: isMaskRequired(I: CI), TLI);
2400 }
2401 case Instruction::Load:
2402 case Instruction::Store: {
2403 bool IsConsecutive = Legal->isConsecutivePtr(AccessTy: getLoadStoreType(I),
2404 Ptr: getLoadStorePointerOperand(V: I));
2405 return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) &&
2406 !Config.isLegalGatherOrScatter(V: I, VF);
2407 }
2408 case Instruction::UDiv:
2409 case Instruction::SDiv:
2410 case Instruction::SRem:
2411 case Instruction::URem: {
2412 // We have the option to use the llvm.masked.udiv intrinsics to avoid
2413 // predication. The cost based decision here will always select the masked
2414 // intrinsics for scalable vectors as scalarization isn't legal.
2415 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
2416 return isDivRemScalarWithPredication(ScalarCost, MaskedCost);
2417 }
2418 }
2419}
2420
2421bool LoopVectorizationCostModel::isMaskRequired(Instruction *I) const {
2422 return Legal->isMaskRequired(I, TailFolded: foldTailByMasking());
2423}
2424
2425// TODO: Fold into LoopVectorizationLegality::isMaskRequired.
2426bool LoopVectorizationCostModel::isPredicatedInst(Instruction *I) const {
2427 // TODO: We can use the loop-preheader as context point here and get
2428 // context sensitive reasoning for isSafeToSpeculativelyExecute.
2429 if (isSafeToSpeculativelyExecute(I) ||
2430 (isa<LoadInst, StoreInst, CallInst>(Val: I) && !isMaskRequired(I)) ||
2431 isa<UncondBrInst, CondBrInst, SwitchInst, PHINode, AllocaInst>(Val: I))
2432 return false;
2433
2434 // If the instruction was executed conditionally in the original scalar loop,
2435 // predication is needed with a mask whose lanes are all possibly inactive.
2436 if (Legal->blockNeedsPredication(BB: I->getParent()))
2437 return true;
2438
2439 // If we're not folding the tail by masking and not vectorizing a loop with
2440 // uncountable exits and side effects, predication is unnecessary.
2441 if (!foldTailByMasking() && !Legal->hasUncountableExitWithSideEffects())
2442 return false;
2443
2444 // All that remain are instructions with side-effects originally executed in
2445 // the loop unconditionally, but now execute under a tail-fold mask (only)
2446 // having at least one active lane (the first). If the side-effects of the
2447 // instruction are invariant, executing it w/o (the tail-folding) mask is safe
2448 // - it will cause the same side-effects as when masked.
2449 switch(I->getOpcode()) {
2450 default:
2451 llvm_unreachable(
2452 "instruction should have been considered by earlier checks");
2453 case Instruction::Call:
2454 // Side-effects of a Call are assumed to be non-invariant, needing a
2455 // (fold-tail) mask.
2456 assert(isMaskRequired(I) &&
2457 "should have returned earlier for calls not needing a mask");
2458 return true;
2459 case Instruction::Load:
2460 // If the address is loop invariant no predication is needed.
2461 return !Legal->isInvariant(V: getLoadStorePointerOperand(V: I));
2462 case Instruction::Store: {
2463 // For stores, we need to prove both speculation safety (which follows from
2464 // the same argument as loads), but also must prove the value being stored
2465 // is correct. The easiest form of the later is to require that all values
2466 // stored are the same.
2467 return !(Legal->isInvariant(V: getLoadStorePointerOperand(V: I)) &&
2468 TheLoop->isLoopInvariant(V: cast<StoreInst>(Val: I)->getValueOperand()));
2469 }
2470 case Instruction::UDiv:
2471 case Instruction::URem:
2472 // If the divisor is loop-invariant no predication is needed.
2473 return !Legal->isInvariant(V: I->getOperand(i: 1));
2474 case Instruction::SDiv:
2475 case Instruction::SRem:
2476 // Conservative for now, since masked-off lanes may be poison and could
2477 // trigger signed overflow.
2478 return true;
2479 }
2480}
2481
2482uint64_t LoopVectorizationCostModel::getPredBlockCostDivisor(
2483 TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) {
2484 if (CostKind == TTI::TCK_CodeSize)
2485 return 1;
2486 // If the block wasn't originally predicated then return early to avoid
2487 // computing BlockFrequencyInfo unnecessarily.
2488 if (!Legal->blockNeedsPredication(BB))
2489 return 1;
2490
2491 uint64_t HeaderFreq =
2492 getBFI().getBlockFreq(BB: TheLoop->getHeader()).getFrequency();
2493 uint64_t BBFreq = getBFI().getBlockFreq(BB).getFrequency();
2494 assert(HeaderFreq >= BBFreq &&
2495 "Header has smaller block freq than dominated BB?");
2496 return std::round(x: (double)HeaderFreq / BBFreq);
2497}
2498
2499static Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode) {
2500 switch (Opcode) {
2501 case Instruction::UDiv:
2502 return Intrinsic::masked_udiv;
2503 case Instruction::SDiv:
2504 return Intrinsic::masked_sdiv;
2505 case Instruction::URem:
2506 return Intrinsic::masked_urem;
2507 case Instruction::SRem:
2508 return Intrinsic::masked_srem;
2509 default:
2510 llvm_unreachable("Unexpected opcode");
2511 }
2512}
2513
2514std::pair<InstructionCost, InstructionCost>
2515LoopVectorizationCostModel::getDivRemSpeculationCost(Instruction *I,
2516 ElementCount VF) {
2517 assert(I->getOpcode() == Instruction::UDiv ||
2518 I->getOpcode() == Instruction::SDiv ||
2519 I->getOpcode() == Instruction::SRem ||
2520 I->getOpcode() == Instruction::URem);
2521 assert(!isSafeToSpeculativelyExecute(I));
2522
2523 // Scalarization isn't legal for scalable vector types
2524 InstructionCost ScalarizationCost = InstructionCost::getInvalid();
2525 if (!VF.isScalable()) {
2526 // Get the scalarization cost and scale this amount by the probability of
2527 // executing the predicated block. If the instruction is not predicated,
2528 // we fall through to the next case.
2529 ScalarizationCost = 0;
2530
2531 // These instructions have a non-void type, so account for the phi nodes
2532 // that we will create. This cost is likely to be zero. The phi node
2533 // cost, if any, should be scaled by the block probability because it
2534 // models a copy at the end of each predicated block.
2535 ScalarizationCost += VF.getFixedValue() *
2536 TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
2537
2538 // The cost of the non-predicated instruction.
2539 ScalarizationCost +=
2540 VF.getFixedValue() * TTI.getArithmeticInstrCost(
2541 Opcode: I->getOpcode(), Ty: I->getType(), CostKind: Config.CostKind);
2542
2543 // The cost of insertelement and extractelement instructions needed for
2544 // scalarization.
2545 ScalarizationCost += getScalarizationOverhead(I, VF);
2546
2547 // Scale the cost by the probability of executing the predicated blocks.
2548 // This assumes the predicated block for each vector lane is equally
2549 // likely.
2550 ScalarizationCost =
2551 ScalarizationCost /
2552 getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
2553 }
2554
2555 auto *VecTy = toVectorTy(Scalar: I->getType(), EC: VF);
2556 auto *MaskTy = toVectorTy(Scalar: Type::getInt1Ty(C&: I->getContext()), EC: VF);
2557 IntrinsicCostAttributes ICA(getMaskedDivRemIntrinsic(Opcode: I->getOpcode()), VecTy,
2558 {VecTy, VecTy, MaskTy});
2559 InstructionCost MaskedCost = TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind);
2560 return {ScalarizationCost, MaskedCost};
2561}
2562
2563bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
2564 Instruction *I, ElementCount VF) const {
2565 assert(isAccessInterleaved(I) && "Expecting interleaved access.");
2566 assert(getWideningDecision(I, VF) == CM_Unknown &&
2567 "Decision should not be set yet.");
2568 auto *Group = getInterleavedAccessGroup(Instr: I);
2569 assert(Group && "Must have a group.");
2570 unsigned InterleaveFactor = Group->getFactor();
2571
2572 // If the instruction's allocated size doesn't equal its type size, it
2573 // requires padding and will be scalarized.
2574 auto &DL = I->getDataLayout();
2575 auto *ScalarTy = getLoadStoreType(I);
2576 if (hasIrregularType(Ty: ScalarTy, DL))
2577 return false;
2578
2579 // For scalable vectors, the interleave factors must be <= 8 since we require
2580 // the (de)interleaveN intrinsics instead of shufflevectors.
2581 if (VF.isScalable() && InterleaveFactor > 8)
2582 return false;
2583
2584 // If the group involves a non-integral pointer, we may not be able to
2585 // losslessly cast all values to a common type.
2586 bool ScalarNI = DL.isNonIntegralPointerType(Ty: ScalarTy);
2587 for (Instruction *Member : Group->members()) {
2588 auto *MemberTy = getLoadStoreType(I: Member);
2589 bool MemberNI = DL.isNonIntegralPointerType(Ty: MemberTy);
2590 // Don't coerce non-integral pointers to integers or vice versa.
2591 if (MemberNI != ScalarNI)
2592 // TODO: Consider adding special nullptr value case here
2593 return false;
2594 if (MemberNI && ScalarNI &&
2595 ScalarTy->getPointerAddressSpace() !=
2596 MemberTy->getPointerAddressSpace())
2597 return false;
2598 }
2599
2600 // Check if masking is required.
2601 // A Group may need masking for one of two reasons: it resides in a block that
2602 // needs predication, or it was decided to use masking to deal with gaps
2603 // (either a gap at the end of a load-access that may result in a speculative
2604 // load, or any gaps in a store-access).
2605 bool PredicatedAccessRequiresMasking =
2606 blockNeedsPredicationForAnyReason(BB: I->getParent()) && isMaskRequired(I);
2607 bool LoadAccessWithGapsRequiresEpilogMasking =
2608 isa<LoadInst>(Val: I) && Group->requiresScalarEpilogue() &&
2609 !isEpilogueAllowed();
2610 bool StoreAccessWithGapsRequiresMasking =
2611 isa<StoreInst>(Val: I) && !Group->isFull();
2612 if (!PredicatedAccessRequiresMasking &&
2613 !LoadAccessWithGapsRequiresEpilogMasking &&
2614 !StoreAccessWithGapsRequiresMasking)
2615 return true;
2616
2617 // If masked interleaving is required, we expect that the user/target had
2618 // enabled it, because otherwise it either wouldn't have been created or
2619 // it should have been invalidated by the CostModel.
2620 assert(useMaskedInterleavedAccesses(TTI) &&
2621 "Masked interleave-groups for predicated accesses are not enabled.");
2622
2623 if (Group->isReverse())
2624 return false;
2625
2626 // TODO: Support interleaved access that requires a gap mask for scalable VFs.
2627 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2628 StoreAccessWithGapsRequiresMasking;
2629 if (VF.isScalable() && NeedsMaskForGaps)
2630 return false;
2631
2632 return isLegalMaskedLoadOrStore(I, VF);
2633}
2634
2635std::optional<LoopVectorizationCostModel::InstWidening>
2636LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
2637 ElementCount VF) {
2638 // Get and ensure we have a valid memory instruction.
2639 assert((isa<LoadInst, StoreInst>(I)) && "Invalid memory instruction");
2640
2641 auto *Ptr = getLoadStorePointerOperand(V: I);
2642 auto *ScalarTy = getLoadStoreType(I);
2643
2644 // In order to be widened, the pointer should be consecutive, first of all.
2645 int Stride = Legal->isConsecutivePtr(AccessTy: ScalarTy, Ptr);
2646 if (!Stride)
2647 return std::nullopt;
2648
2649 // If the instruction is a store located in a predicated block, it will be
2650 // scalarized.
2651 if (isScalarWithPredication(I, VF))
2652 return std::nullopt;
2653
2654 // If the instruction's allocated size doesn't equal it's type size, it
2655 // requires padding and will be scalarized.
2656 auto &DL = I->getDataLayout();
2657 if (hasIrregularType(Ty: ScalarTy, DL))
2658 return std::nullopt;
2659
2660 return Stride == 1 ? CM_Widen : CM_Widen_Reverse;
2661}
2662
2663void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
2664 // We should not collect Uniforms more than once per VF. Right now,
2665 // this function is called from collectUniformsAndScalars(), which
2666 // already does this check. Collecting Uniforms for VF=1 does not make any
2667 // sense.
2668
2669 assert(VF.isVector() && !Uniforms.contains(VF) &&
2670 "This function should not be visited twice for the same VF");
2671
2672 // Visit the list of Uniforms. If we find no uniform value, we won't
2673 // analyze again. Uniforms.count(VF) will return 1.
2674 Uniforms[VF].clear();
2675
2676 // Now we know that the loop is vectorizable!
2677 // Collect instructions inside the loop that will remain uniform after
2678 // vectorization.
2679
2680 // Global values, params and instructions outside of current loop are out of
2681 // scope.
2682 auto IsOutOfScope = [&](Value *V) -> bool {
2683 Instruction *I = dyn_cast<Instruction>(Val: V);
2684 return (!I || !TheLoop->contains(Inst: I));
2685 };
2686
2687 // Worklist containing uniform instructions demanding lane 0.
2688 SetVector<Instruction *> Worklist;
2689
2690 // Add uniform instructions demanding lane 0 to the worklist. Instructions
2691 // that require predication must not be considered uniform after
2692 // vectorization, because that would create an erroneous replicating region
2693 // where only a single instance out of VF should be formed.
2694 auto AddToWorklistIfAllowed = [&](Instruction *I) -> void {
2695 if (IsOutOfScope(I)) {
2696 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "
2697 << *I << "\n");
2698 return;
2699 }
2700 if (isPredicatedInst(I)) {
2701 LLVM_DEBUG(
2702 dbgs() << "LV: Found not uniform due to requiring predication: " << *I
2703 << "\n");
2704 return;
2705 }
2706 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n");
2707 Worklist.insert(X: I);
2708 };
2709
2710 // Start with the conditional branches exiting the loop. If the branch
2711 // condition is an instruction contained in the loop that is only used by the
2712 // branch, it is uniform. Note conditions from uncountable early exits are not
2713 // uniform.
2714 SmallVector<BasicBlock *> Exiting;
2715 TheLoop->getExitingBlocks(ExitingBlocks&: Exiting);
2716 for (BasicBlock *E : Exiting) {
2717 if (Legal->hasUncountableEarlyExit() && TheLoop->getLoopLatch() != E)
2718 continue;
2719 auto *Cmp = dyn_cast<Instruction>(Val: E->getTerminator()->getOperand(i: 0));
2720 if (Cmp && TheLoop->contains(Inst: Cmp) && Cmp->hasOneUse())
2721 AddToWorklistIfAllowed(Cmp);
2722 }
2723
2724 auto PrevVF = VF.divideCoefficientBy(RHS: 2);
2725 // Return true if all lanes perform the same memory operation, and we can
2726 // thus choose to execute only one.
2727 auto IsUniformMemOpUse = [&](Instruction *I) {
2728 // If the value was already known to not be uniform for the previous
2729 // (smaller VF), it cannot be uniform for the larger VF.
2730 if (PrevVF.isVector()) {
2731 auto Iter = Uniforms.find(Val: PrevVF);
2732 if (Iter != Uniforms.end() && !Iter->second.contains(Ptr: I))
2733 return false;
2734 }
2735 if (!isUniformMemOp(I&: *I, VF))
2736 return false;
2737 if (isa<LoadInst>(Val: I))
2738 // Loading the same address always produces the same result - at least
2739 // assuming aliasing and ordering which have already been checked.
2740 return true;
2741 // Storing the same value on every iteration.
2742 return TheLoop->isLoopInvariant(V: cast<StoreInst>(Val: I)->getValueOperand());
2743 };
2744
2745 auto IsUniformDecision = [&](Instruction *I, ElementCount VF) {
2746 InstWidening WideningDecision = getWideningDecision(I, VF);
2747 assert(WideningDecision != CM_Unknown &&
2748 "Widening decision should be ready at this moment");
2749
2750 if (IsUniformMemOpUse(I))
2751 return true;
2752
2753 return (WideningDecision == CM_Widen ||
2754 WideningDecision == CM_Widen_Reverse ||
2755 WideningDecision == CM_Interleave);
2756 };
2757
2758 // Returns true if Ptr is the pointer operand of a memory access instruction
2759 // I, I is known to not require scalarization, and the pointer is not also
2760 // stored.
2761 auto IsVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
2762 if (isa<StoreInst>(Val: I) && I->getOperand(i: 0) == Ptr)
2763 return false;
2764 return getLoadStorePointerOperand(V: I) == Ptr &&
2765 (IsUniformDecision(I, VF) || Legal->isInvariant(V: Ptr));
2766 };
2767
2768 // Holds a list of values which are known to have at least one uniform use.
2769 // Note that there may be other uses which aren't uniform. A "uniform use"
2770 // here is something which only demands lane 0 of the unrolled iterations;
2771 // it does not imply that all lanes produce the same value (e.g. this is not
2772 // the usual meaning of uniform)
2773 SetVector<Value *> HasUniformUse;
2774
2775 // Scan the loop for instructions which are either a) known to have only
2776 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
2777 for (auto *BB : TheLoop->blocks())
2778 for (auto &I : *BB) {
2779 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &I)) {
2780 switch (II->getIntrinsicID()) {
2781 case Intrinsic::sideeffect:
2782 case Intrinsic::experimental_noalias_scope_decl:
2783 case Intrinsic::assume:
2784 case Intrinsic::lifetime_start:
2785 case Intrinsic::lifetime_end:
2786 if (TheLoop->hasLoopInvariantOperands(I: &I))
2787 AddToWorklistIfAllowed(&I);
2788 break;
2789 default:
2790 break;
2791 }
2792 }
2793
2794 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: &I)) {
2795 if (IsOutOfScope(EVI->getAggregateOperand())) {
2796 AddToWorklistIfAllowed(EVI);
2797 continue;
2798 }
2799 // Only ExtractValue instructions where the aggregate value comes from a
2800 // call are allowed to be non-uniform.
2801 assert(isa<CallInst>(EVI->getAggregateOperand()) &&
2802 "Expected aggregate value to be call return value");
2803 }
2804
2805 // If there's no pointer operand, there's nothing to do.
2806 auto *Ptr = getLoadStorePointerOperand(V: &I);
2807 if (!Ptr)
2808 continue;
2809
2810 // If the pointer can be proven to be uniform, always add it to the
2811 // worklist.
2812 if (isa<Instruction>(Val: Ptr) && isUniform(V: Ptr, VF))
2813 AddToWorklistIfAllowed(cast<Instruction>(Val: Ptr));
2814
2815 if (IsUniformMemOpUse(&I))
2816 AddToWorklistIfAllowed(&I);
2817
2818 if (IsVectorizedMemAccessUse(&I, Ptr))
2819 HasUniformUse.insert(X: Ptr);
2820 }
2821
2822 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
2823 // demanding) users. Since loops are assumed to be in LCSSA form, this
2824 // disallows uses outside the loop as well.
2825 for (auto *V : HasUniformUse) {
2826 if (IsOutOfScope(V))
2827 continue;
2828 auto *I = cast<Instruction>(Val: V);
2829 bool UsersAreMemAccesses = all_of(Range: I->users(), P: [&](User *U) -> bool {
2830 auto *UI = cast<Instruction>(Val: U);
2831 return TheLoop->contains(Inst: UI) && IsVectorizedMemAccessUse(UI, V);
2832 });
2833 if (UsersAreMemAccesses)
2834 AddToWorklistIfAllowed(I);
2835 }
2836
2837 // Expand Worklist in topological order: whenever a new instruction
2838 // is added , its users should be already inside Worklist. It ensures
2839 // a uniform instruction will only be used by uniform instructions.
2840 unsigned Idx = 0;
2841 while (Idx != Worklist.size()) {
2842 Instruction *I = Worklist[Idx++];
2843
2844 for (auto *OV : I->operand_values()) {
2845 // isOutOfScope operands cannot be uniform instructions.
2846 if (IsOutOfScope(OV))
2847 continue;
2848 // First order recurrence Phi's should typically be considered
2849 // non-uniform.
2850 auto *OP = dyn_cast<PHINode>(Val: OV);
2851 if (OP && Legal->isFixedOrderRecurrence(Phi: OP))
2852 continue;
2853 // If all the users of the operand are uniform, then add the
2854 // operand into the uniform worklist.
2855 auto *OI = cast<Instruction>(Val: OV);
2856 if (llvm::all_of(Range: OI->users(), P: [&](User *U) -> bool {
2857 auto *J = cast<Instruction>(Val: U);
2858 return Worklist.count(key: J) || IsVectorizedMemAccessUse(J, OI);
2859 }))
2860 AddToWorklistIfAllowed(OI);
2861 }
2862 }
2863
2864 // For an instruction to be added into Worklist above, all its users inside
2865 // the loop should also be in Worklist. However, this condition cannot be
2866 // true for phi nodes that form a cyclic dependence. We must process phi
2867 // nodes separately. An induction variable will remain uniform if all users
2868 // of the induction variable and induction variable update remain uniform.
2869 // The code below handles both pointer and non-pointer induction variables.
2870 BasicBlock *Latch = TheLoop->getLoopLatch();
2871 for (const auto &Induction : Legal->getInductionVars()) {
2872 auto *Ind = Induction.first;
2873 auto *IndUpdate = cast<Instruction>(Val: Ind->getIncomingValueForBlock(BB: Latch));
2874
2875 // Determine if all users of the induction variable are uniform after
2876 // vectorization.
2877 bool UniformInd = all_of(Range: Ind->users(), P: [&](User *U) -> bool {
2878 auto *I = cast<Instruction>(Val: U);
2879 return I == IndUpdate || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2880 IsVectorizedMemAccessUse(I, Ind);
2881 });
2882 if (!UniformInd)
2883 continue;
2884
2885 // Determine if all users of the induction variable update instruction are
2886 // uniform after vectorization.
2887 bool UniformIndUpdate = all_of(Range: IndUpdate->users(), P: [&](User *U) -> bool {
2888 auto *I = cast<Instruction>(Val: U);
2889 return I == Ind || Worklist.count(key: I) ||
2890 IsVectorizedMemAccessUse(I, IndUpdate);
2891 });
2892 if (!UniformIndUpdate)
2893 continue;
2894
2895 // The induction variable and its update instruction will remain uniform.
2896 AddToWorklistIfAllowed(Ind);
2897 AddToWorklistIfAllowed(IndUpdate);
2898 }
2899
2900 Uniforms[VF].insert_range(R&: Worklist);
2901}
2902
2903FixedScalableVFPair
2904LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
2905 // Make sure once we return PartialAliasMaskingStatus is not "NotDecided".
2906 scope_exit EnsureAliasMaskingStatusIsDecidedOnReturn([this] {
2907 if (PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided)
2908 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
2909 });
2910
2911 // For outer loops, use simple type-based heuristic VF. No cost model or
2912 // memory dependence analysis is available.
2913 if (!TheLoop->isInnermost()) {
2914 return Config.computeVPlanOuterloopVF(UserVF);
2915 }
2916
2917 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
2918 // TODO: It may be useful to do since it's still likely to be dynamically
2919 // uniform if the target can skip.
2920 reportVectorizationFailure(
2921 DebugMsg: "Not inserting runtime ptr check for divergent target",
2922 OREMsg: "runtime pointer checks needed. Not enabled for divergent target",
2923 ORETag: "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
2924 return FixedScalableVFPair::getNone();
2925 }
2926
2927 ScalarEvolution *SE = PSE.getSE();
2928 ElementCount TC = getSmallConstantTripCount(SE, L: TheLoop);
2929 unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
2930 if (!MaxTC && EpilogueLoweringStatus == CM_EpilogueAllowed)
2931 MaxTC = getMaxTCFromNonZeroRange(PSE, L: TheLoop);
2932 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
2933 if (TC != ElementCount::getFixed(MinVal: MaxTC))
2934 LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
2935 if (TC.isScalar()) {
2936 reportVectorizationFailure(
2937 DebugMsg: "Single iteration (non) loop",
2938 OREMsg: "loop trip count is one, irrelevant for vectorization",
2939 ORETag: "SingleIterationLoop", ORE, TheLoop);
2940 return FixedScalableVFPair::getNone();
2941 }
2942
2943 // If BTC matches the widest induction type and is -1 then the trip count
2944 // computation will wrap to 0 and the vector trip count will be 0. Do not try
2945 // to vectorize.
2946 const SCEV *BTC = SE->getBackedgeTakenCount(L: TheLoop);
2947 if (!isa<SCEVCouldNotCompute>(Val: BTC) &&
2948 BTC->getType()->getScalarSizeInBits() >=
2949 Legal->getWidestInductionType()->getScalarSizeInBits() &&
2950 SE->isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: BTC,
2951 RHS: SE->getMinusOne(Ty: BTC->getType()))) {
2952 reportVectorizationFailure(
2953 DebugMsg: "Trip count computation wrapped",
2954 OREMsg: "backedge-taken count is -1, loop trip count wrapped to 0",
2955 ORETag: "TripCountWrapped", ORE, TheLoop);
2956 return FixedScalableVFPair::getNone();
2957 }
2958
2959 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
2960 "No cost-modeling decisions should have been taken at this point");
2961
2962 switch (EpilogueLoweringStatus) {
2963 case CM_EpilogueAllowed:
2964 return Config.computeFeasibleMaxVF(MaxTripCount: MaxTC, UserVF, UserIC, FoldTailByMasking: false,
2965 RequiresScalarEpilogue: requiresScalarEpilogue(IsVectorizing: true));
2966 case CM_EpilogueNotAllowedFoldTail:
2967 [[fallthrough]];
2968 case CM_EpilogueNotNeededFoldTail:
2969 LLVM_DEBUG(dbgs() << "LV: tail-folding hint/switch found.\n"
2970 << "LV: Not allowing epilogue, creating tail-folded "
2971 << "vector loop.\n");
2972 break;
2973 case CM_EpilogueNotAllowedLowTripLoop:
2974 // fallthrough as a special case of OptForSize
2975 case CM_EpilogueNotAllowedOptSize:
2976 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize)
2977 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
2978 else
2979 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
2980 << "count.\n");
2981
2982 // Bail if runtime checks are required, which are not good when optimising
2983 // for size.
2984 if (Config.runtimeChecksRequired())
2985 return FixedScalableVFPair::getNone();
2986
2987 break;
2988 }
2989
2990 // Now try the tail folding
2991
2992 // Invalidate interleave groups that require an epilogue if we can't mask
2993 // the interleave-group.
2994 if (!useMaskedInterleavedAccesses(TTI)) {
2995 // Note: There is no need to invalidate any cost modeling decisions here, as
2996 // none were taken so far (see assertion above).
2997 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
2998 }
2999
3000 FixedScalableVFPair MaxFactors = Config.computeFeasibleMaxVF(
3001 MaxTripCount: MaxTC, UserVF, UserIC, FoldTailByMasking: true, RequiresScalarEpilogue: requiresScalarEpilogue(IsVectorizing: true));
3002
3003 // Avoid tail folding if the trip count is known to be a multiple of any VF
3004 // we choose.
3005 std::optional<unsigned> MaxPowerOf2RuntimeVF =
3006 MaxFactors.FixedVF.getFixedValue();
3007 if (MaxFactors.ScalableVF) {
3008 std::optional<unsigned> MaxVScale = getMaxVScale(F: *TheFunction, TTI);
3009 if (MaxVScale) {
3010 MaxPowerOf2RuntimeVF = std::max<unsigned>(
3011 a: *MaxPowerOf2RuntimeVF,
3012 b: *MaxVScale * MaxFactors.ScalableVF.getKnownMinValue());
3013 } else
3014 MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
3015 }
3016
3017 auto NoScalarEpilogueNeeded = [this, &UserIC](unsigned MaxVF) {
3018 // Return false if the loop is neither a single-latch-exit loop nor an
3019 // early-exit loop as tail-folding is not supported in that case.
3020 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
3021 !Legal->hasUncountableEarlyExit())
3022 return false;
3023 unsigned MaxVFtimesIC = UserIC ? MaxVF * UserIC : MaxVF;
3024 ScalarEvolution *SE = PSE.getSE();
3025 // Calling getSymbolicMaxBackedgeTakenCount enables support for loops
3026 // with uncountable exits. For countable loops, the symbolic maximum must
3027 // remain identical to the known back-edge taken count.
3028 const SCEV *BackedgeTakenCount = PSE.getSymbolicMaxBackedgeTakenCount();
3029 assert((Legal->hasUncountableEarlyExit() ||
3030 BackedgeTakenCount == PSE.getBackedgeTakenCount()) &&
3031 "Invalid loop count");
3032 const SCEV *ExitCount = SE->getAddExpr(
3033 LHS: BackedgeTakenCount, RHS: SE->getOne(Ty: BackedgeTakenCount->getType()));
3034 const SCEV *Rem = SE->getURemExpr(
3035 LHS: SE->applyLoopGuards(Expr: ExitCount, L: TheLoop),
3036 RHS: SE->getConstant(Ty: BackedgeTakenCount->getType(), V: MaxVFtimesIC));
3037 return Rem->isZero();
3038 };
3039
3040 if (MaxPowerOf2RuntimeVF > 0u) {
3041 assert((UserVF.isNonZero() || isPowerOf2_32(*MaxPowerOf2RuntimeVF)) &&
3042 "MaxFixedVF must be a power of 2");
3043 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
3044 // Accept MaxFixedVF if we do not have a tail.
3045 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
3046 return MaxFactors;
3047 }
3048 }
3049
3050 auto ExpectedTC = getSmallBestKnownTC(PSE, L: TheLoop);
3051 if (ExpectedTC && ExpectedTC->isFixed() &&
3052 ExpectedTC->getFixedValue() <=
3053 TTI.getMinTripCountTailFoldingThreshold()) {
3054 if (MaxPowerOf2RuntimeVF > 0u) {
3055 // If we have a low-trip-count, and the fixed-width VF is known to divide
3056 // the trip count but the scalable factor does not, use the fixed-width
3057 // factor in preference to allow the generation of a non-predicated loop.
3058 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
3059 NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
3060 LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
3061 "remain for any chosen VF.\n");
3062 MaxFactors.ScalableVF = ElementCount::getScalable(MinVal: 0);
3063 return MaxFactors;
3064 }
3065 }
3066
3067 reportVectorizationFailure(
3068 DebugMsg: "The trip count is below the minial threshold value.",
3069 OREMsg: "loop trip count is too low, avoiding vectorization", ORETag: "LowTripCount",
3070 ORE, TheLoop);
3071 return FixedScalableVFPair::getNone();
3072 }
3073
3074 // If we don't know the precise trip count, or if the trip count that we
3075 // found modulo the vectorization factor is not zero, try to fold the tail
3076 // by masking.
3077 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
3078 bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
3079 setTailFoldingStyle(IsScalableVF: ContainsScalableVF, UserIC);
3080 if (foldTailByMasking()) {
3081 if (foldTailWithEVL()) {
3082 LLVM_DEBUG(
3083 dbgs()
3084 << "LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
3085 "try to generate VP Intrinsics with scalable vector "
3086 "factors only.\n");
3087 // Tail folded loop using VP intrinsics restricts the VF to be scalable
3088 // for now.
3089 // TODO: extend it for fixed vectors, if required.
3090 assert(ContainsScalableVF && "Expected scalable vector factor.");
3091
3092 MaxFactors.FixedVF = ElementCount::getFixed(MinVal: 1);
3093 } else {
3094 tryToEnablePartialAliasMasking();
3095 }
3096 return MaxFactors;
3097 }
3098
3099 // If there was a tail-folding hint/switch, but we can't fold the tail by
3100 // masking, fallback to a vectorization with an epilogue.
3101 if (EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail) {
3102 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with an "
3103 "epilogue instead.\n");
3104 EpilogueLoweringStatus = CM_EpilogueAllowed;
3105 return MaxFactors;
3106 }
3107
3108 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail) {
3109 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
3110 return FixedScalableVFPair::getNone();
3111 }
3112
3113 if (TC.isZero()) {
3114 reportVectorizationFailure(
3115 DebugMsg: "unable to calculate the loop count due to complex control flow",
3116 ORETag: "UnknownLoopCountComplexCFG", ORE, TheLoop);
3117 return FixedScalableVFPair::getNone();
3118 }
3119
3120 reportVectorizationFailure(
3121 DebugMsg: "Cannot optimize for size and vectorize at the same time.",
3122 OREMsg: "cannot optimize for size and vectorize at the same time. "
3123 "Enable vectorization of this loop with '#pragma clang loop "
3124 "vectorize(enable)' when compiling with -Os/-Oz",
3125 ORETag: "NoTailLoopWithOptForSize", ORE, TheLoop);
3126 return FixedScalableVFPair::getNone();
3127}
3128
3129void LoopVectorizationPlanner::emitInvalidCostRemarks(
3130 OptimizationRemarkEmitter *ORE) {
3131 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
3132 SmallVector<RecipeVFPair> InvalidCosts;
3133 for (const auto &Plan : VPlans) {
3134 for (ElementCount VF : Plan->vectorFactors()) {
3135 // The VPlan-based cost model is designed for computing vector cost.
3136 // Querying VPlan-based cost model with a scarlar VF will cause some
3137 // errors because we expect the VF is vector for most of the widen
3138 // recipes.
3139 if (VF.isScalar())
3140 continue;
3141
3142 VPCostContext CostCtx(*TLI, *Plan, *CM, Config,
3143 /*ReusePrintingSlotTracker=*/true);
3144 precomputeCosts(Plan&: *Plan, VF, CostCtx);
3145 auto Iter = vp_depth_first_deep(G: Plan->getVectorLoopRegion()->getEntry());
3146 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: Iter)) {
3147 for (auto &R : *VPBB) {
3148 if (!R.cost(VF, Ctx&: CostCtx).isValid())
3149 InvalidCosts.emplace_back(Args: &R, Args&: VF);
3150 }
3151 }
3152 }
3153 }
3154 if (InvalidCosts.empty())
3155 return;
3156
3157 // Emit a report of VFs with invalid costs in the loop.
3158
3159 // Group the remarks per recipe, keeping the recipe order from InvalidCosts.
3160 DenseMap<VPRecipeBase *, unsigned> Numbering;
3161 unsigned I = 0;
3162 for (auto &Pair : InvalidCosts)
3163 if (Numbering.try_emplace(Key: Pair.first, Args&: I).second)
3164 ++I;
3165
3166 // Sort the list, first on recipe(number) then on VF.
3167 sort(C&: InvalidCosts, Comp: [&Numbering](RecipeVFPair &A, RecipeVFPair &B) {
3168 unsigned NA = Numbering[A.first];
3169 unsigned NB = Numbering[B.first];
3170 if (NA != NB)
3171 return NA < NB;
3172 return ElementCount::isKnownLT(LHS: A.second, RHS: B.second);
3173 });
3174
3175 // For a list of ordered recipe-VF pairs:
3176 // [(load, VF1), (load, VF2), (store, VF1)]
3177 // group the recipes together to emit separate remarks for:
3178 // load (VF1, VF2)
3179 // store (VF1)
3180 auto Tail = ArrayRef<RecipeVFPair>(InvalidCosts);
3181 auto Subset = ArrayRef<RecipeVFPair>();
3182 do {
3183 if (Subset.empty())
3184 Subset = Tail.take_front(N: 1);
3185
3186 VPRecipeBase *R = Subset.front().first;
3187
3188 unsigned Opcode =
3189 TypeSwitch<const VPRecipeBase *, unsigned>(R)
3190 .Case(caseFn: [](const VPHeaderPHIRecipe *R) { return Instruction::PHI; })
3191 .Case(
3192 caseFn: [](const VPWidenStoreRecipe *R) { return Instruction::Store; })
3193 .Case(caseFn: [](const VPWidenLoadRecipe *R) { return Instruction::Load; })
3194 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
3195 caseFn: [](const auto *R) { return Instruction::Call; })
3196 .Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
3197 VPWidenCastRecipe>(
3198 caseFn: [](const auto *R) { return R->getOpcode(); })
3199 .Case(caseFn: [](const VPInterleaveRecipe *R) {
3200 return R->getStoredValues().empty() ? Instruction::Load
3201 : Instruction::Store;
3202 })
3203 .Case(caseFn: [](const VPReductionRecipe *R) {
3204 return RecurrenceDescriptor::getOpcode(Kind: R->getRecurrenceKind());
3205 });
3206
3207 // If the next recipe is different, or if there are no other pairs,
3208 // emit a remark for the collated subset. e.g.
3209 // [(load, VF1), (load, VF2))]
3210 // to emit:
3211 // remark: invalid costs for 'load' at VF=(VF1, VF2)
3212 if (Subset == Tail || Tail[Subset.size()].first != R) {
3213 std::string OutString;
3214 raw_string_ostream OS(OutString);
3215 assert(!Subset.empty() && "Unexpected empty range");
3216 OS << "Recipe with invalid costs prevented vectorization at VF=(";
3217 for (const auto &Pair : Subset)
3218 OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second;
3219 OS << "):";
3220 if (Opcode == Instruction::Call) {
3221 StringRef Name = "";
3222 if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(Val: R)) {
3223 Name = Int->getIntrinsicName();
3224 } else {
3225 auto *WidenCall = dyn_cast<VPWidenCallRecipe>(Val: R);
3226 Function *CalledFn =
3227 WidenCall ? WidenCall->getCalledScalarFunction()
3228 : cast<Function>(Val: R->getOperand(N: R->getNumOperands() - 1)
3229 ->getLiveInIRValue());
3230 Name = CalledFn->getName();
3231 }
3232 OS << " call to " << Name;
3233 } else
3234 OS << " " << Instruction::getOpcodeName(Opcode);
3235 reportVectorizationInfo(Msg: OutString, ORETag: "InvalidCost", ORE, TheLoop: OrigLoop, I: nullptr,
3236 DL: R->getDebugLoc());
3237 Tail = Tail.drop_front(N: Subset.size());
3238 Subset = {};
3239 } else
3240 // Grow the subset by one element
3241 Subset = Tail.take_front(N: Subset.size() + 1);
3242 } while (!Tail.empty());
3243}
3244
3245/// Check if any recipe of \p Plan will generate a vector value, which will be
3246/// assigned a vector register.
3247static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
3248 const TargetTransformInfo &TTI) {
3249 assert(VF.isVector() && "Checking a scalar VF?");
3250 DenseSet<VPRecipeBase *> EphemeralRecipes;
3251 collectEphemeralRecipesForVPlan(Plan, EphRecipes&: EphemeralRecipes);
3252 // Set of already visited types.
3253 DenseSet<Type *> Visited;
3254 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3255 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
3256 for (VPRecipeBase &R : *VPBB) {
3257 if (EphemeralRecipes.contains(V: &R))
3258 continue;
3259 // Continue early if the recipe is considered to not produce a vector
3260 // result. Note that this includes VPInstruction where some opcodes may
3261 // produce a vector, to preserve existing behavior as VPInstructions model
3262 // aspects not directly mapped to existing IR instructions.
3263 switch (R.getVPRecipeID()) {
3264 case VPRecipeBase::VPDerivedIVSC:
3265 case VPRecipeBase::VPScalarIVStepsSC:
3266 case VPRecipeBase::VPReplicateSC:
3267 case VPRecipeBase::VPInstructionSC:
3268 case VPRecipeBase::VPCurrentIterationPHISC:
3269 case VPRecipeBase::VPVectorPointerSC:
3270 case VPRecipeBase::VPVectorEndPointerSC:
3271 case VPRecipeBase::VPExpandSCEVSC:
3272 case VPRecipeBase::VPPredInstPHISC:
3273 case VPRecipeBase::VPBranchOnMaskSC:
3274 continue;
3275 case VPRecipeBase::VPReductionSC:
3276 case VPRecipeBase::VPActiveLaneMaskPHISC:
3277 case VPRecipeBase::VPWidenCallSC:
3278 case VPRecipeBase::VPWidenCanonicalIVSC:
3279 case VPRecipeBase::VPWidenCastSC:
3280 case VPRecipeBase::VPWidenGEPSC:
3281 case VPRecipeBase::VPWidenIntrinsicSC:
3282 case VPRecipeBase::VPWidenMemIntrinsicSC:
3283 case VPRecipeBase::VPWidenSC:
3284 case VPRecipeBase::VPBlendSC:
3285 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
3286 case VPRecipeBase::VPHistogramSC:
3287 case VPRecipeBase::VPWidenPHISC:
3288 case VPRecipeBase::VPWidenIntOrFpInductionSC:
3289 case VPRecipeBase::VPWidenPointerInductionSC:
3290 case VPRecipeBase::VPReductionPHISC:
3291 case VPRecipeBase::VPInterleaveEVLSC:
3292 case VPRecipeBase::VPInterleaveSC:
3293 case VPRecipeBase::VPWidenLoadEVLSC:
3294 case VPRecipeBase::VPWidenLoadSC:
3295 case VPRecipeBase::VPWidenStoreEVLSC:
3296 case VPRecipeBase::VPWidenStoreSC:
3297 break;
3298 default:
3299 llvm_unreachable("unhandled recipe");
3300 }
3301
3302 auto WillGenerateTargetVectors = [&TTI, VF](Type *VectorTy) {
3303 unsigned NumLegalParts = TTI.getNumberOfParts(Tp: VectorTy);
3304 if (!NumLegalParts)
3305 return false;
3306 if (VF.isScalable()) {
3307 // <vscale x 1 x iN> is assumed to be profitable over iN because
3308 // scalable registers are a distinct register class from scalar
3309 // ones. If we ever find a target which wants to lower scalable
3310 // vectors back to scalars, we'll need to update this code to
3311 // explicitly ask TTI about the register class uses for each part.
3312 return NumLegalParts <= VF.getKnownMinValue();
3313 }
3314 // Two or more elements that share a register - are vectorized.
3315 return NumLegalParts < VF.getFixedValue();
3316 };
3317
3318 // If no def nor is a store, e.g., branches, continue - no value to check.
3319 if (R.getNumDefinedValues() == 0 &&
3320 !isa<VPWidenStoreRecipe, VPWidenStoreEVLRecipe, VPInterleaveBase>(Val: &R))
3321 continue;
3322 // For multi-def recipes, currently only interleaved loads, suffice to
3323 // check first def only.
3324 // For stores check their stored value; for interleaved stores suffice
3325 // the check first stored value only. In all cases this is the second
3326 // operand.
3327 VPValue *ToCheck =
3328 R.getNumDefinedValues() >= 1 ? R.getVPValue(I: 0) : R.getOperand(N: 1);
3329 Type *ScalarTy = ToCheck->getScalarType();
3330 if (!Visited.insert(V: {ScalarTy}).second)
3331 continue;
3332 Type *WideTy = toVectorizedTy(Ty: ScalarTy, EC: VF);
3333 if (any_of(Range: getContainedTypes(Ty: WideTy), P: WillGenerateTargetVectors))
3334 return true;
3335 }
3336 }
3337
3338 return false;
3339}
3340
3341static bool hasReplicatorRegion(VPlan &Plan) {
3342 return any_of(Range: VPBlockUtils::blocksOnly<VPRegionBlock>(Range: vp_depth_first_shallow(
3343 G: Plan.getVectorLoopRegion()->getEntry())),
3344 P: [](auto *VPRB) { return VPRB->isReplicator(); });
3345}
3346
3347/// Returns true if the VPlan contains a VPReductionPHIRecipe with
3348/// FindLast recurrence kind.
3349static bool hasFindLastReductionPhi(VPlan &Plan) {
3350 return any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3351 P: [](VPRecipeBase &R) {
3352 auto *RedPhi = dyn_cast<VPReductionPHIRecipe>(Val: &R);
3353 return RedPhi &&
3354 RecurrenceDescriptor::isFindLastRecurrenceKind(
3355 Kind: RedPhi->getRecurrenceKind());
3356 });
3357}
3358bool VFSelectionContext::isEpilogueVectorizationProfitable(
3359 const ElementCount VF, const unsigned IC) const {
3360 // FIXME: We need a much better cost-model to take different parameters such
3361 // as register pressure, code size increase and cost of extra branches into
3362 // account. For now we apply a very crude heuristic and only consider loops
3363 // with vectorization factors larger than a certain value.
3364
3365 // Allow the target to opt out.
3366 if (!TTI.preferEpilogueVectorization(Iters: VF * IC))
3367 return false;
3368
3369 unsigned MinVFThreshold = EpilogueVectorizationMinVF.getNumOccurrences() > 0
3370 ? EpilogueVectorizationMinVF
3371 : TTI.getEpilogueVectorizationMinVF();
3372 return estimateElementCount(VF: VF * IC, VScale: getVScaleForTuning()) >= MinVFThreshold;
3373}
3374
3375std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
3376 VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC,
3377 bool ScalarEpilogueAllowed) {
3378 if (!EnableEpilogueVectorization) {
3379 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
3380 return nullptr;
3381 }
3382
3383 if (!ScalarEpilogueAllowed) {
3384 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
3385 "epilogue is allowed.\n");
3386 return nullptr;
3387 }
3388
3389 if (vputils::findIncomingAliasMask(Plan: MainPlan)) {
3390 LLVM_DEBUG(
3391 dbgs()
3392 << "LEV: Epilogue vectorization not supported with alias masking.\n");
3393 return nullptr;
3394 }
3395
3396 // Not really a cost consideration, but check for unsupported cases here to
3397 // simplify the logic.
3398 if (!isCandidateForEpilogueVectorization(MainPlan)) {
3399 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
3400 "is not a supported candidate.\n");
3401 return nullptr;
3402 }
3403
3404 if (hasForcedEpilogueVF()) {
3405 if (estimateElementCount(VF: EpilogueVectorizationForceVF,
3406 VScale: Config.getVScaleForTuning()) >=
3407 IC * estimateElementCount(VF: MainLoopVF, VScale: Config.getVScaleForTuning())) {
3408 // Note that the main loop leaves IC * MainLoopVF iterations iff a scalar
3409 // epilogue is required, but then the epilogue loop also requires a scalar
3410 // epilogue.
3411 LLVM_DEBUG(dbgs() << "LEV: Forced epilogue VF results in dead epilogue "
3412 "vector loop, skipping vectorizing epilogue.\n");
3413 return nullptr;
3414 }
3415
3416 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
3417 if (hasPlanWithVF(VF: EpilogueVectorizationForceVF)) {
3418 std::unique_ptr<VPlan> Clone(
3419 getPlanFor(VF: EpilogueVectorizationForceVF).duplicate());
3420 Clone->setVF(EpilogueVectorizationForceVF);
3421 return Clone;
3422 }
3423
3424 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization forced factor is not "
3425 "viable.\n");
3426 return nullptr;
3427 }
3428
3429 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
3430 LLVM_DEBUG(
3431 dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n");
3432 return nullptr;
3433 }
3434
3435 if (!Config.isEpilogueVectorizationProfitable(VF: MainLoopVF, IC)) {
3436 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
3437 "this loop\n");
3438 return nullptr;
3439 }
3440
3441 // Check if a plan's vector loop processes fewer iterations than VF (e.g. when
3442 // interleave groups have been narrowed) narrowInterleaveGroups) and return
3443 // the adjusted, effective VF.
3444 using namespace VPlanPatternMatch;
3445 auto GetEffectiveVF = [](VPlan &Plan, ElementCount VF) -> ElementCount {
3446 auto *Exiting = Plan.getVectorLoopRegion()->getExitingBasicBlock();
3447 if (match(V: &Exiting->back(),
3448 P: m_BranchOnCount(Op0: m_Add(Op0: m_CanonicalIV(), Op1: m_Specific(VPV: &Plan.getUF())),
3449 Op1: m_VPValue())))
3450 return ElementCount::get(MinVal: 1, Scalable: VF.isScalable());
3451 return VF;
3452 };
3453
3454 // Check if the main loop processes fewer than MainLoopVF elements per
3455 // iteration (e.g. due to narrowing interleave groups). Adjust MainLoopVF
3456 // as needed.
3457 MainLoopVF = GetEffectiveVF(MainPlan, MainLoopVF);
3458
3459 // If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know
3460 // the main loop handles 8 lanes per iteration. We could still benefit from
3461 // vectorizing the epilogue loop with VF=4.
3462 ElementCount EstimatedRuntimeVF = ElementCount::getFixed(
3463 MinVal: estimateElementCount(VF: MainLoopVF, VScale: Config.getVScaleForTuning()));
3464
3465 Type *TCType = Legal->getWidestInductionType();
3466 const SCEV *RemainingIterations = nullptr;
3467 unsigned MaxTripCount = 0;
3468 const SCEV *TC = vputils::getSCEVExprForVPValue(V: MainPlan.getTripCount(), PSE);
3469 assert(!isa<SCEVCouldNotCompute>(TC) && "Trip count SCEV must be computable");
3470 const SCEV *KnownMinTC;
3471 bool ScalableTC = match(S: TC, P: m_scev_c_Mul(Op0: m_SCEV(V&: KnownMinTC), Op1: m_SCEVVScale()));
3472 bool ScalableRemIter = false;
3473 ScalarEvolution &SE = *PSE.getSE();
3474 // Use versions of TC and VF in which both are either scalable or fixed.
3475 if (ScalableTC == MainLoopVF.isScalable()) {
3476 ScalableRemIter = ScalableTC;
3477 RemainingIterations =
3478 SE.getURemExpr(LHS: TC, RHS: SE.getElementCount(Ty: TCType, EC: MainLoopVF * IC));
3479 } else if (ScalableTC) {
3480 const SCEV *EstimatedTC = SE.getMulExpr(
3481 LHS: KnownMinTC,
3482 RHS: SE.getConstant(Ty: TCType, V: Config.getVScaleForTuning().value_or(u: 1)));
3483 RemainingIterations = SE.getURemExpr(
3484 LHS: EstimatedTC, RHS: SE.getElementCount(Ty: TCType, EC: MainLoopVF * IC));
3485 } else
3486 RemainingIterations =
3487 SE.getURemExpr(LHS: TC, RHS: SE.getElementCount(Ty: TCType, EC: EstimatedRuntimeVF * IC));
3488
3489 // No iterations left to process in the epilogue.
3490 if (RemainingIterations->isZero())
3491 return nullptr;
3492
3493 if (MainLoopVF.isFixed()) {
3494 MaxTripCount = MainLoopVF.getFixedValue() * IC - 1;
3495 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_ULT, LHS: RemainingIterations,
3496 RHS: SE.getConstant(Ty: TCType, V: MaxTripCount))) {
3497 MaxTripCount = SE.getUnsignedRangeMax(S: RemainingIterations).getZExtValue();
3498 }
3499 LLVM_DEBUG(dbgs() << "LEV: Maximum Trip Count for Epilogue: "
3500 << MaxTripCount << "\n");
3501 }
3502
3503 auto SkipVF = [&](const SCEV *VF, const SCEV *RemIter) -> bool {
3504 return SE.isKnownPredicate(Pred: CmpInst::ICMP_UGT, LHS: VF, RHS: RemIter);
3505 };
3506 VectorizationFactor Result = VectorizationFactor::Disabled();
3507 VPlan *BestPlan = nullptr;
3508 for (auto &NextVF : ProfitableVFs) {
3509 // Skip candidate VFs without a corresponding VPlan.
3510 if (!hasPlanWithVF(VF: NextVF.Width))
3511 continue;
3512
3513 VPlan &CurrentPlan = getPlanFor(VF: NextVF.Width);
3514 ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
3515 // Skip fixed vector VFs > than the estimated runtime VF, or any VF > than
3516 // the VF of the main loop.
3517 if ((!EffectiveVF.isScalable() && MainLoopVF.isScalable() &&
3518 ElementCount::isKnownGT(LHS: EffectiveVF, RHS: EstimatedRuntimeVF)) ||
3519 ElementCount::isKnownGT(LHS: EffectiveVF, RHS: MainLoopVF))
3520 continue;
3521
3522 // If EffectiveVF is greater than the number of remaining iterations, the
3523 // epilogue loop would be dead. Skip such factors. If the epilogue plan
3524 // also has narrowed interleave groups, use the effective VF since
3525 // the epilogue step will be reduced to its IC.
3526 // TODO: We should also consider comparing against a scalable
3527 // RemainingIterations when SCEV be able to evaluate non-canonical
3528 // vscale-based expressions.
3529 if (!ScalableRemIter) {
3530 // Handle the case where EffectiveVF and RemainingIterations are in
3531 // different numerical spaces.
3532 if (EffectiveVF.isScalable())
3533 EffectiveVF = ElementCount::getFixed(
3534 MinVal: estimateElementCount(VF: EffectiveVF, VScale: Config.getVScaleForTuning()));
3535 if (SkipVF(SE.getElementCount(Ty: TCType, EC: EffectiveVF), RemainingIterations))
3536 continue;
3537 }
3538
3539 if (Result.Width.isScalar() ||
3540 isMoreProfitable(A: NextVF, B: Result, MaxTripCount,
3541 HasTail: !MainPlan.hasTailFolded(),
3542 /*IsEpilogue*/ true)) {
3543 Result = NextVF;
3544 BestPlan = &CurrentPlan;
3545 }
3546 }
3547
3548 if (!BestPlan)
3549 return nullptr;
3550
3551 LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
3552 << Result.Width << "\n");
3553 std::unique_ptr<VPlan> Clone(BestPlan->duplicate());
3554 Clone->setVF(Result.Width);
3555 return Clone;
3556}
3557
3558unsigned
3559LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
3560 InstructionCost LoopCost) {
3561 // -- The interleave heuristics --
3562 // We interleave the loop in order to expose ILP and reduce the loop overhead.
3563 // There are many micro-architectural considerations that we can't predict
3564 // at this level. For example, frontend pressure (on decode or fetch) due to
3565 // code size, or the number and capabilities of the execution ports.
3566 //
3567 // We use the following heuristics to select the interleave count:
3568 // 1. If the code has reductions, then we interleave to break the cross
3569 // iteration dependency.
3570 // 2. If the loop is really small, then we interleave to reduce the loop
3571 // overhead.
3572 // 3. We don't interleave if we think that we will spill registers to memory
3573 // due to the increased register pressure.
3574
3575 // Do not interleave tail-folded loops, as the overhead of multiple
3576 // instructions to calculate the predicate is likely not beneficial.
3577 // If an epilogue is not allowed for any other reason, do not interleave.
3578 if (!CM->isEpilogueAllowed())
3579 return 1;
3580
3581 if (any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3582 P: IsaPred<VPCurrentIterationPHIRecipe>)) {
3583 LLVM_DEBUG(dbgs() << "LV: Loop requires variable-length step. "
3584 "Unroll factor forced to be 1.\n");
3585 return 1;
3586 }
3587
3588 // We used the distance for the interleave count.
3589 if (!Legal->isSafeForAnyVectorWidth())
3590 return 1;
3591
3592 // We don't attempt to perform interleaving for loops with uncountable early
3593 // exits because the VPInstruction::AnyOf code cannot currently handle
3594 // multiple parts.
3595 if (Plan.hasEarlyExit())
3596 return 1;
3597
3598 const bool HasReductions =
3599 any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3600 P: IsaPred<VPReductionPHIRecipe>);
3601
3602 // FIXME: implement interleaving for FindLast transform correctly.
3603 if (hasFindLastReductionPhi(Plan))
3604 return 1;
3605
3606 VPRegisterUsage R = calculateRegisterUsageForPlan(Plan, VFs: {VF}, TTI)[0];
3607
3608 // If we did not calculate the cost for VF (because the user selected the VF)
3609 // then we calculate the cost of VF here.
3610 if (LoopCost == 0) {
3611 if (VF.isScalar())
3612 LoopCost = CM->expectedCost(VF);
3613 else
3614 LoopCost = cost(Plan, VF, RU: &R);
3615 assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
3616
3617 // Loop body is free and there is no need for interleaving.
3618 if (LoopCost == 0)
3619 return 1;
3620 }
3621
3622 // We divide by these constants so assume that we have at least one
3623 // instruction that uses at least one register.
3624 for (auto &Pair : R.MaxLocalUsers) {
3625 Pair.second = std::max(a: Pair.second, b: 1U);
3626 }
3627
3628 // We calculate the interleave count using the following formula.
3629 // Subtract the number of loop invariants from the number of available
3630 // registers. These registers are used by all of the interleaved instances.
3631 // Next, divide the remaining registers by the number of registers that is
3632 // required by the loop, in order to estimate how many parallel instances
3633 // fit without causing spills. All of this is rounded down if necessary to be
3634 // a power of two. We want power of two interleave count to simplify any
3635 // addressing operations or alignment considerations.
3636 // We also want power of two interleave counts to ensure that the induction
3637 // variable of the vector loop wraps to zero, when tail is folded by masking;
3638 // this currently happens when OptForSize, in which case IC is set to 1 above.
3639 unsigned IC = UINT_MAX;
3640
3641 for (const auto &Pair : R.MaxLocalUsers) {
3642 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(ClassID: Pair.first);
3643 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters
3644 << " registers of "
3645 << TTI.getRegisterClassName(Pair.first)
3646 << " register class\n");
3647 if (VF.isScalar()) {
3648 if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)
3649 TargetNumRegisters = ForceTargetNumScalarRegs;
3650 } else {
3651 if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)
3652 TargetNumRegisters = ForceTargetNumVectorRegs;
3653 }
3654 unsigned MaxLocalUsers = Pair.second;
3655 unsigned LoopInvariantRegs = 0;
3656 if (R.LoopInvariantRegs.contains(Key: Pair.first))
3657 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
3658
3659 unsigned TmpIC = llvm::bit_floor(Value: (TargetNumRegisters - LoopInvariantRegs) /
3660 MaxLocalUsers);
3661 // Don't count the induction variable as interleaved.
3662 if (EnableIndVarRegisterHeur) {
3663 TmpIC = llvm::bit_floor(Value: (TargetNumRegisters - LoopInvariantRegs - 1) /
3664 std::max(a: 1U, b: (MaxLocalUsers - 1)));
3665 }
3666
3667 IC = std::min(a: IC, b: TmpIC);
3668 }
3669
3670 // Clamp the interleave ranges to reasonable counts.
3671 bool HasUnorderedReductions =
3672 HasReductions &&
3673 !any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3674 P: [](VPRecipeBase &R) {
3675 auto *RedR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
3676 return RedR && RedR->isOrdered();
3677 });
3678 unsigned MaxInterleaveCount =
3679 TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions);
3680 LLVM_DEBUG(dbgs() << "LV: MaxInterleaveFactor for the target is "
3681 << MaxInterleaveCount << "\n");
3682
3683 // Check if the user has overridden the max.
3684 if (VF.isScalar()) {
3685 if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
3686 MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
3687 } else {
3688 if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)
3689 MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
3690 }
3691
3692 // Try to get the exact trip count, or an estimate based on profiling data or
3693 // ConstantMax from PSE, failing that.
3694 auto BestKnownTC =
3695 getSmallBestKnownTC(PSE, L: OrigLoop,
3696 /*CanUseConstantMax=*/true,
3697 /*CanExcludeZeroTrips=*/CM->isEpilogueAllowed());
3698
3699 // For fixed length VFs treat a scalable trip count as unknown.
3700 if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
3701 // Re-evaluate trip counts and VFs to be in the same numerical space.
3702 unsigned AvailableTC =
3703 estimateElementCount(VF: *BestKnownTC, VScale: Config.getVScaleForTuning());
3704 unsigned EstimatedVF =
3705 estimateElementCount(VF, VScale: Config.getVScaleForTuning());
3706
3707 // At least one iteration must be scalar when this constraint holds. So the
3708 // maximum available iterations for interleaving is one less.
3709 if (Plan.requiresScalarEpilogue())
3710 --AvailableTC;
3711
3712 unsigned InterleaveCountLB = bit_floor(Value: std::max(
3713 a: 1u, b: std::min(a: AvailableTC / (EstimatedVF * 2), b: MaxInterleaveCount)));
3714
3715 if (getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop).isNonZero()) {
3716 // If the best known trip count is exact, we select between two
3717 // prospective ICs, where
3718 //
3719 // 1) the aggressive IC is capped by the trip count divided by VF
3720 // 2) the conservative IC is capped by the trip count divided by (VF * 2)
3721 //
3722 // The final IC is selected in a way that the epilogue loop trip count is
3723 // minimized while maximizing the IC itself, so that we either run the
3724 // vector loop at least once if it generates a small epilogue loop, or
3725 // else we run the vector loop at least twice.
3726
3727 unsigned InterleaveCountUB = bit_floor(Value: std::max(
3728 a: 1u, b: std::min(a: AvailableTC / EstimatedVF, b: MaxInterleaveCount)));
3729 MaxInterleaveCount = InterleaveCountLB;
3730
3731 if (InterleaveCountUB != InterleaveCountLB) {
3732 unsigned TailTripCountUB =
3733 (AvailableTC % (EstimatedVF * InterleaveCountUB));
3734 unsigned TailTripCountLB =
3735 (AvailableTC % (EstimatedVF * InterleaveCountLB));
3736 // If both produce same scalar tail, maximize the IC to do the same work
3737 // in fewer vector loop iterations
3738 if (TailTripCountUB == TailTripCountLB)
3739 MaxInterleaveCount = InterleaveCountUB;
3740 }
3741 } else {
3742 // If trip count is an estimated compile time constant, limit the
3743 // IC to be capped by the trip count divided by VF * 2, such that the
3744 // vector loop runs at least twice to make interleaving seem profitable
3745 // when there is an epilogue loop present. Since exact Trip count is not
3746 // known we choose to be conservative in our IC estimate.
3747 MaxInterleaveCount = InterleaveCountLB;
3748 }
3749 }
3750
3751 assert(MaxInterleaveCount > 0 &&
3752 "Maximum interleave count must be greater than 0");
3753
3754 // Clamp the calculated IC to be between the 1 and the max interleave count
3755 // that the target and trip count allows.
3756 if (IC > MaxInterleaveCount)
3757 IC = MaxInterleaveCount;
3758 else
3759 // Make sure IC is greater than 0.
3760 IC = std::max(a: 1u, b: IC);
3761
3762 assert(IC > 0 && "Interleave count must be greater than 0.");
3763
3764 // Interleave if we vectorized this loop and there is a reduction that could
3765 // benefit from interleaving.
3766 if (VF.isVector() && HasReductions) {
3767 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
3768 return IC;
3769 }
3770
3771 // For any scalar loop that either requires runtime checks or tail-folding we
3772 // are better off leaving this to the unroller. Note that if we've already
3773 // vectorized the loop we will have done the runtime check and so interleaving
3774 // won't require further checks.
3775 bool ScalarInterleavingRequiresPredication =
3776 (VF.isScalar() && any_of(Range: OrigLoop->blocks(), P: [this](BasicBlock *BB) {
3777 return Legal->blockNeedsPredication(BB);
3778 }));
3779 bool ScalarInterleavingRequiresRuntimePointerCheck =
3780 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need);
3781
3782 // We want to interleave small loops in order to reduce the loop overhead and
3783 // potentially expose ILP opportunities.
3784 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
3785 << "LV: IC is " << IC << '\n'
3786 << "LV: VF is " << VF << '\n');
3787 const bool AggressivelyInterleave =
3788 TTI.enableAggressiveInterleaving(LoopHasReductions: HasReductions);
3789 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
3790 !ScalarInterleavingRequiresPredication && LoopCost < SmallLoopCost) {
3791 // We assume that the cost overhead is 1 and we use the cost model
3792 // to estimate the cost of the loop and interleave until the cost of the
3793 // loop overhead is about 5% of the cost of the loop.
3794 unsigned SmallIC = std::min(a: IC, b: (unsigned)llvm::bit_floor<uint64_t>(
3795 Value: SmallLoopCost / LoopCost.getValue()));
3796
3797 // Interleave until store/load ports (estimated by max interleave count) are
3798 // saturated.
3799 unsigned NumStores = 0;
3800 unsigned NumLoads = 0;
3801 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3802 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
3803 for (VPRecipeBase &R : *VPBB) {
3804 if (isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Val: &R)) {
3805 NumLoads++;
3806 continue;
3807 }
3808 if (isa<VPWidenStoreRecipe, VPWidenStoreEVLRecipe>(Val: &R)) {
3809 NumStores++;
3810 continue;
3811 }
3812
3813 if (auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(Val: &R)) {
3814 if (unsigned StoreOps = InterleaveR->getNumStoreOperands())
3815 NumStores += StoreOps;
3816 else
3817 NumLoads += InterleaveR->getNumDefinedValues();
3818 continue;
3819 }
3820 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
3821 NumLoads += isa<LoadInst>(Val: RepR->getUnderlyingInstr());
3822 NumStores += isa<StoreInst>(Val: RepR->getUnderlyingInstr());
3823 continue;
3824 }
3825 if (isa<VPHistogramRecipe>(Val: &R)) {
3826 NumLoads++;
3827 NumStores++;
3828 continue;
3829 }
3830 }
3831 }
3832 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
3833 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
3834
3835 // There is little point in interleaving for reductions containing selects
3836 // and compares when VF=1 since it may just create more overhead than it's
3837 // worth for loops with small trip counts. This is because we still have to
3838 // do the final reduction after the loop.
3839 bool HasSelectCmpReductions =
3840 HasReductions &&
3841 any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3842 P: [](VPRecipeBase &R) {
3843 auto *RedR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
3844 return RedR && (RecurrenceDescriptor::isAnyOfRecurrenceKind(
3845 Kind: RedR->getRecurrenceKind()) ||
3846 RecurrenceDescriptor::isFindIVRecurrenceKind(
3847 Kind: RedR->getRecurrenceKind()));
3848 });
3849 if (HasSelectCmpReductions) {
3850 LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
3851 return 1;
3852 }
3853
3854 // If we have a scalar reduction (vector reductions are already dealt with
3855 // by this point), we can increase the critical path length if the loop
3856 // we're interleaving is inside another loop. For tree-wise reductions
3857 // set the limit to 2, and for ordered reductions it's best to disable
3858 // interleaving entirely.
3859 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
3860 bool HasOrderedReductions =
3861 any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3862 P: [](VPRecipeBase &R) {
3863 auto *RedR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
3864
3865 return RedR && RedR->isOrdered();
3866 });
3867 if (HasOrderedReductions) {
3868 LLVM_DEBUG(
3869 dbgs() << "LV: Not interleaving scalar ordered reductions.\n");
3870 return 1;
3871 }
3872
3873 unsigned F = MaxNestedScalarReductionIC;
3874 SmallIC = std::min(a: SmallIC, b: F);
3875 StoresIC = std::min(a: StoresIC, b: F);
3876 LoadsIC = std::min(a: LoadsIC, b: F);
3877 }
3878
3879 if (EnableLoadStoreRuntimeInterleave &&
3880 std::max(a: StoresIC, b: LoadsIC) > SmallIC) {
3881 LLVM_DEBUG(
3882 dbgs() << "LV: Interleaving to saturate store or load ports.\n");
3883 return std::max(a: StoresIC, b: LoadsIC);
3884 }
3885
3886 // If there are scalar reductions and TTI has enabled aggressive
3887 // interleaving for reductions, we will interleave to expose ILP.
3888 if (VF.isScalar() && AggressivelyInterleave) {
3889 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3890 // Interleave no less than SmallIC but not as aggressive as the normal IC
3891 // to satisfy the rare situation when resources are too limited.
3892 return std::max(a: IC / 2, b: SmallIC);
3893 }
3894
3895 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
3896 return SmallIC;
3897 }
3898
3899 // Interleave if this is a large loop (small loops are already dealt with by
3900 // this point) that could benefit from interleaving.
3901 if (AggressivelyInterleave) {
3902 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3903 return IC;
3904 }
3905
3906 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n");
3907 return 1;
3908}
3909
3910bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(
3911 Instruction *I, ElementCount VF) const {
3912 // TODO: Cost model for emulated masked load/store is completely
3913 // broken. This hack guides the cost model to use an artificially
3914 // high enough value to practically disable vectorization with such
3915 // operations, except where previously deployed legality hack allowed
3916 // using very low cost values. This is to avoid regressions coming simply
3917 // from moving "masked load/store" check from legality to cost model.
3918 // Masked Load/Gather emulation was previously never allowed.
3919 // Limited number of Masked Store/Scatter emulation was allowed.
3920 assert((isPredicatedInst(I)) &&
3921 "Expecting a scalar emulated instruction");
3922 return isa<LoadInst>(Val: I) ||
3923 (isa<StoreInst>(Val: I) &&
3924 NumPredStores > NumberOfStoresToPredicate);
3925}
3926
3927void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
3928 assert(VF.isVector() && "Expected VF >= 2");
3929
3930 // If we've already collected the instructions to scalarize or the predicated
3931 // BBs after vectorization, there's nothing to do. Collection may already have
3932 // occurred if we have a user-selected VF and are now computing the expected
3933 // cost for interleaving.
3934 if (InstsToScalarize.contains(Key: VF) ||
3935 PredicatedBBsAfterVectorization.contains(Val: VF))
3936 return;
3937
3938 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's
3939 // not profitable to scalarize any instructions, the presence of VF in the
3940 // map will indicate that we've analyzed it already.
3941 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
3942
3943 // Find all the instructions that are scalar with predication in the loop and
3944 // determine if it would be better to not if-convert the blocks they are in.
3945 // If so, we also record the instructions to scalarize.
3946 for (BasicBlock *BB : TheLoop->blocks()) {
3947 if (!blockNeedsPredicationForAnyReason(BB))
3948 continue;
3949 for (Instruction &I : *BB)
3950 if (isScalarWithPredication(I: &I, VF)) {
3951 ScalarCostsTy ScalarCosts;
3952 // Do not apply discount logic for:
3953 // 1. Scalars after vectorization, as there will only be a single copy
3954 // of the instruction.
3955 // 2. Scalable VF, as that would lead to invalid scalarization costs.
3956 // 3. Emulated masked memrefs, if a hacked cost is needed.
3957 if (!isScalarAfterVectorization(I: &I, VF) && !VF.isScalable() &&
3958 !useEmulatedMaskMemRefHack(I: &I, VF) &&
3959 computePredInstDiscount(PredInst: &I, ScalarCosts, VF) >= 0) {
3960 for (const auto &[I, IC] : ScalarCosts)
3961 ScalarCostsVF.insert(KV: {I, IC});
3962 }
3963 // Remember that BB will remain after vectorization.
3964 PredicatedBBsAfterVectorization[VF].insert(Ptr: BB);
3965 for (auto *Pred : predecessors(BB)) {
3966 if (Pred->getSingleSuccessor() == BB)
3967 PredicatedBBsAfterVectorization[VF].insert(Ptr: Pred);
3968 }
3969 }
3970 }
3971}
3972
3973InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
3974 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) {
3975 assert(!isUniformAfterVectorization(PredInst, VF) &&
3976 "Instruction marked uniform-after-vectorization will be predicated");
3977
3978 // Initialize the discount to zero, meaning that the scalar version and the
3979 // vector version cost the same.
3980 InstructionCost Discount = 0;
3981
3982 // Holds instructions to analyze. The instructions we visit are mapped in
3983 // ScalarCosts. Those instructions are the ones that would be scalarized if
3984 // we find that the scalar version costs less.
3985 SmallVector<Instruction *, 8> Worklist;
3986
3987 // Returns true if the given instruction can be scalarized.
3988 auto CanBeScalarized = [&](Instruction *I) -> bool {
3989 // We only attempt to scalarize instructions forming a single-use chain
3990 // from the original predicated block that would otherwise be vectorized.
3991 // Although not strictly necessary, we give up on instructions we know will
3992 // already be scalar to avoid traversing chains that are unlikely to be
3993 // beneficial.
3994 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
3995 isScalarAfterVectorization(I, VF))
3996 return false;
3997
3998 // If the instruction is scalar with predication, it will be analyzed
3999 // separately. We ignore it within the context of PredInst.
4000 if (isScalarWithPredication(I, VF))
4001 return false;
4002
4003 // If any of the instruction's operands are uniform after vectorization,
4004 // the instruction cannot be scalarized. This prevents, for example, a
4005 // masked load from being scalarized.
4006 //
4007 // We assume we will only emit a value for lane zero of an instruction
4008 // marked uniform after vectorization, rather than VF identical values.
4009 // Thus, if we scalarize an instruction that uses a uniform, we would
4010 // create uses of values corresponding to the lanes we aren't emitting code
4011 // for. This behavior can be changed by allowing getScalarValue to clone
4012 // the lane zero values for uniforms rather than asserting.
4013 for (Use &U : I->operands())
4014 if (auto *J = dyn_cast<Instruction>(Val: U.get()))
4015 if (isUniformAfterVectorization(I: J, VF))
4016 return false;
4017
4018 // Otherwise, we can scalarize the instruction.
4019 return true;
4020 };
4021
4022 // Compute the expected cost discount from scalarizing the entire expression
4023 // feeding the predicated instruction. We currently only consider expressions
4024 // that are single-use instruction chains.
4025 Worklist.push_back(Elt: PredInst);
4026 while (!Worklist.empty()) {
4027 Instruction *I = Worklist.pop_back_val();
4028
4029 // If we've already analyzed the instruction, there's nothing to do.
4030 if (ScalarCosts.contains(Key: I))
4031 continue;
4032
4033 // Cannot scalarize fixed-order recurrence phis at the moment.
4034 if (isa<PHINode>(Val: I) && Legal->isFixedOrderRecurrence(Phi: cast<PHINode>(Val: I)))
4035 continue;
4036
4037 // Compute the cost of the vector instruction. Note that this cost already
4038 // includes the scalarization overhead of the predicated instruction.
4039 InstructionCost VectorCost = getInstructionCost(I, VF);
4040
4041 // Compute the cost of the scalarized instruction. This cost is the cost of
4042 // the instruction as if it wasn't if-converted and instead remained in the
4043 // predicated block. We will scale this cost by block probability after
4044 // computing the scalarization overhead.
4045 InstructionCost ScalarCost =
4046 VF.getFixedValue() * getInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1));
4047
4048 // Compute the scalarization overhead of needed insertelement instructions
4049 // and phi nodes.
4050 if (isScalarWithPredication(I, VF) && !I->getType()->isVoidTy()) {
4051 Type *WideTy = toVectorizedTy(Ty: I->getType(), EC: VF);
4052 for (Type *VectorTy : getContainedTypes(Ty: WideTy)) {
4053 ScalarCost += TTI.getScalarizationOverhead(
4054 Ty: cast<VectorType>(Val: VectorTy), DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4055 /*Insert=*/true,
4056 /*Extract=*/false, CostKind: Config.CostKind);
4057 }
4058 ScalarCost += VF.getFixedValue() *
4059 TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
4060 }
4061
4062 // Compute the scalarization overhead of needed extractelement
4063 // instructions. For each of the instruction's operands, if the operand can
4064 // be scalarized, add it to the worklist; otherwise, account for the
4065 // overhead.
4066 for (Use &U : I->operands())
4067 if (auto *J = dyn_cast<Instruction>(Val: U.get())) {
4068 assert(canVectorizeTy(J->getType()) &&
4069 "Instruction has non-scalar type");
4070 if (CanBeScalarized(J))
4071 Worklist.push_back(Elt: J);
4072 else if (needsExtract(V: J, VF)) {
4073 Type *WideTy = toVectorizedTy(Ty: J->getType(), EC: VF);
4074 for (Type *VectorTy : getContainedTypes(Ty: WideTy)) {
4075 ScalarCost += TTI.getScalarizationOverhead(
4076 Ty: cast<VectorType>(Val: VectorTy),
4077 DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()), /*Insert*/ false,
4078 /*Extract*/ true, CostKind: Config.CostKind);
4079 }
4080 }
4081 }
4082
4083 // Scale the total scalar cost by block probability.
4084 ScalarCost /= getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
4085
4086 // Compute the discount. A non-negative discount means the vector version
4087 // of the instruction costs more, and scalarizing would be beneficial.
4088 Discount += VectorCost - ScalarCost;
4089 ScalarCosts[I] = ScalarCost;
4090 }
4091
4092 return Discount;
4093}
4094
4095InstructionCost LoopVectorizationCostModel::expectedCost(ElementCount VF) {
4096 InstructionCost Cost;
4097 assert(VF.isScalar() && "must only be called for scalar VFs");
4098
4099 // For each block.
4100 for (BasicBlock *BB : TheLoop->blocks()) {
4101 InstructionCost BlockCost;
4102
4103 // For each instruction in the old loop.
4104 for (Instruction &I : *BB) {
4105 // Skip ignored values.
4106 if (ValuesToIgnore.count(Ptr: &I) ||
4107 (VF.isVector() && VecValuesToIgnore.count(Ptr: &I)))
4108 continue;
4109
4110 InstructionCost C = getInstructionCost(I: &I, VF);
4111
4112 // Check if we should override the cost.
4113 if (C.isValid() && ForceTargetInstructionCost.getNumOccurrences() > 0)
4114 C = InstructionCost(ForceTargetInstructionCost);
4115
4116 BlockCost += C;
4117 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C << " for VF "
4118 << VF << " For instruction: " << I << '\n');
4119 }
4120
4121 // In the scalar loop, we may not always execute the predicated block, if it
4122 // is an if-else block. Thus, scale the block's cost by the probability of
4123 // executing it. getPredBlockCostDivisor will return 1 for blocks that are
4124 // only predicated by the header mask when folding the tail.
4125 Cost += BlockCost / getPredBlockCostDivisor(CostKind: Config.CostKind, BB);
4126 }
4127
4128 return Cost;
4129}
4130
4131/// Gets the address access SCEV for Ptr, if it should be used for cost modeling
4132/// according to isAddressSCEVForCost.
4133///
4134/// This SCEV can be sent to the Target in order to estimate the address
4135/// calculation cost.
4136static const SCEV *getAddressAccessSCEV(
4137 Value *Ptr,
4138 PredicatedScalarEvolution &PSE,
4139 const Loop *TheLoop) {
4140 const SCEV *Addr = PSE.getSCEV(V: Ptr);
4141 return vputils::isAddressSCEVForCost(Addr, SE&: *PSE.getSE(), L: TheLoop) ? Addr
4142 : nullptr;
4143}
4144
4145InstructionCost
4146LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
4147 ElementCount VF) {
4148 assert(VF.isVector() &&
4149 "Scalarization cost of instruction implies vectorization.");
4150 if (VF.isScalable())
4151 return InstructionCost::getInvalid();
4152
4153 Type *ValTy = getLoadStoreType(I);
4154 auto *SE = PSE.getSE();
4155
4156 unsigned AS = getLoadStoreAddressSpace(I);
4157 Value *Ptr = getLoadStorePointerOperand(V: I);
4158 Type *PtrTy = toVectorTy(Scalar: Ptr->getType(), EC: VF);
4159 // NOTE: PtrTy is a vector to signal `TTI::getAddressComputationCost`
4160 // that it is being called from this specific place.
4161
4162 // Figure out whether the access is strided and get the stride value
4163 // if it's known in compile time
4164 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, PSE, TheLoop);
4165
4166 // Get the cost of the scalar memory instruction and address computation.
4167 InstructionCost Cost =
4168 VF.getFixedValue() *
4169 TTI.getAddressComputationCost(PtrTy, SE, Ptr: PtrSCEV, CostKind: Config.CostKind);
4170
4171 // Don't pass *I here, since it is scalar but will actually be part of a
4172 // vectorized loop where the user of it is a vectorized instruction.
4173 const Align Alignment = getLoadStoreAlignment(I);
4174 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4175 Cost += VF.getFixedValue() *
4176 TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: ValTy->getScalarType(), Alignment,
4177 AddressSpace: AS, CostKind: Config.CostKind, OpdInfo: OpInfo);
4178
4179 // Get the overhead of the extractelement and insertelement instructions
4180 // we might create due to scalarization.
4181 Cost += getScalarizationOverhead(I, VF);
4182
4183 // If we have a predicated load/store, it will need extra i1 extracts and
4184 // conditional branches, but may not be executed for each vector lane. Scale
4185 // the cost by the probability of executing the predicated block.
4186 if (isPredicatedInst(I)) {
4187 Cost /= getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
4188
4189 // Add the cost of an i1 extract and a branch
4190 auto *VecI1Ty =
4191 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: ValTy->getContext()), EC: VF);
4192 Cost += TTI.getScalarizationOverhead(
4193 Ty: VecI1Ty, DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4194 /*Insert=*/false, /*Extract=*/true, CostKind: Config.CostKind);
4195 Cost += TTI.getCFInstrCost(Opcode: Instruction::CondBr, CostKind: Config.CostKind);
4196
4197 if (useEmulatedMaskMemRefHack(I, VF))
4198 // Artificially setting to a high enough value to practically disable
4199 // vectorization with such operations.
4200 Cost = 3000000;
4201 }
4202
4203 return Cost;
4204}
4205
4206InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
4207 Instruction *I, ElementCount VF, InstWidening Kind) {
4208 assert((Kind == CM_Widen || Kind == CM_Widen_Reverse) &&
4209 "Expected a consecutive widening decision");
4210 Type *ValTy = getLoadStoreType(I);
4211 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4212 unsigned AS = getLoadStoreAddressSpace(I);
4213
4214 const Align Alignment = getLoadStoreAlignment(I);
4215 InstructionCost Cost = 0;
4216 if (isMaskRequired(I)) {
4217 unsigned IID = I->getOpcode() == Instruction::Load
4218 ? Intrinsic::masked_load
4219 : Intrinsic::masked_store;
4220 Cost += TTI.getMemIntrinsicInstrCost(
4221 MICA: MemIntrinsicCostAttributes(IID, VectorTy, Alignment, AS),
4222 CostKind: Config.CostKind);
4223 } else {
4224 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4225 Cost += TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: VectorTy, Alignment, AddressSpace: AS,
4226 CostKind: Config.CostKind, OpdInfo: OpInfo, I);
4227 }
4228
4229 if (Kind == CM_Widen_Reverse)
4230 Cost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Reverse, DstTy: VectorTy,
4231 SrcTy: VectorTy, CostKind: Config.CostKind, Mask: {}, Index: 0);
4232 return Cost;
4233}
4234
4235InstructionCost
4236LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
4237 ElementCount VF) const {
4238 assert(isUniformMemOp(*I, VF));
4239
4240 Type *ValTy = getLoadStoreType(I);
4241 Type *PtrTy = getLoadStorePointerOperand(V: I)->getType();
4242 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4243 const Align Alignment = getLoadStoreAlignment(I);
4244 unsigned AS = getLoadStoreAddressSpace(I);
4245 if (isa<LoadInst>(Val: I)) {
4246 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4247 CostKind: Config.CostKind) +
4248 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: ValTy, Alignment, AddressSpace: AS,
4249 CostKind: Config.CostKind) +
4250 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Broadcast, DstTy: VectorTy,
4251 SrcTy: VectorTy, CostKind: Config.CostKind);
4252 }
4253 StoreInst *SI = cast<StoreInst>(Val: I);
4254
4255 bool IsLoopInvariantStoreValue = Legal->isInvariant(V: SI->getValueOperand());
4256 // TODO: We have existing tests that request the cost of extracting element
4257 // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent
4258 // the actual generated code, which involves extracting the last element of
4259 // a scalable vector where the lane to extract is unknown at compile time.
4260 InstructionCost Cost =
4261 TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr, CostKind: Config.CostKind) +
4262 TTI.getMemoryOpCost(Opcode: Instruction::Store, Src: ValTy, Alignment, AddressSpace: AS,
4263 CostKind: Config.CostKind);
4264 if (!IsLoopInvariantStoreValue)
4265 Cost += TTI.getIndexedVectorInstrCostFromEnd(Opcode: Instruction::ExtractElement,
4266 Val: VectorTy, CostKind: Config.CostKind, Index: 0);
4267 return Cost;
4268}
4269
4270InstructionCost
4271LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
4272 ElementCount VF) const {
4273 Type *ValTy = getLoadStoreType(I);
4274 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4275 const Align Alignment = getLoadStoreAlignment(I);
4276 Value *Ptr = getLoadStorePointerOperand(V: I);
4277 Type *PtrTy = Ptr->getType();
4278
4279 if (!isUniform(V: Ptr, VF))
4280 PtrTy = toVectorTy(Scalar: PtrTy, EC: VF);
4281
4282 unsigned IID = I->getOpcode() == Instruction::Load
4283 ? Intrinsic::masked_gather
4284 : Intrinsic::masked_scatter;
4285 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4286 CostKind: Config.CostKind) +
4287 TTI.getMemIntrinsicInstrCost(
4288 MICA: MemIntrinsicCostAttributes(IID, VectorTy, Ptr, isMaskRequired(I),
4289 Alignment, I),
4290 CostKind: Config.CostKind);
4291}
4292
4293InstructionCost
4294LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
4295 ElementCount VF) const {
4296 const auto *Group = getInterleavedAccessGroup(Instr: I);
4297 assert(Group && "Fail to get an interleaved access group.");
4298
4299 Instruction *InsertPos = Group->getInsertPos();
4300 Type *ValTy = getLoadStoreType(I: InsertPos);
4301 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4302 unsigned AS = getLoadStoreAddressSpace(I: InsertPos);
4303
4304 unsigned InterleaveFactor = Group->getFactor();
4305 auto *WideVecTy = VectorType::get(ElementType: ValTy, EC: VF * InterleaveFactor);
4306
4307 // Holds the indices of existing members in the interleaved group.
4308 SmallVector<unsigned, 4> Indices;
4309 for (unsigned IF = 0; IF < InterleaveFactor; IF++)
4310 if (Group->getMember(Index: IF))
4311 Indices.push_back(Elt: IF);
4312
4313 // Calculate the cost of the whole interleaved group.
4314 bool UseMaskForGaps =
4315 (Group->requiresScalarEpilogue() && !isEpilogueAllowed()) ||
4316 (isa<StoreInst>(Val: I) && !Group->isFull());
4317 InstructionCost Cost = TTI.getInterleavedMemoryOpCost(
4318 Opcode: InsertPos->getOpcode(), VecTy: WideVecTy, Factor: Group->getFactor(), Indices,
4319 Alignment: Group->getAlign(), AddressSpace: AS, CostKind: Config.CostKind, UseMaskForCond: isMaskRequired(I),
4320 UseMaskForGaps);
4321
4322 if (Group->isReverse()) {
4323 // TODO: Add support for reversed masked interleaved access.
4324 assert(!isMaskRequired(I) &&
4325 "Reverse masked interleaved access not supported.");
4326 Cost += Group->getNumMembers() *
4327 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Reverse, DstTy: VectorTy,
4328 SrcTy: VectorTy, CostKind: Config.CostKind, Mask: {}, Index: 0);
4329 }
4330 return Cost;
4331}
4332
4333std::optional<InstructionCost>
4334LoopVectorizationCostModel::getReductionPatternCost(Instruction *I,
4335 ElementCount VF,
4336 Type *Ty) const {
4337 using namespace llvm::PatternMatch;
4338 // Early exit for no inloop reductions
4339 if (Config.getInLoopReductions().empty() || VF.isScalar() ||
4340 !isa<VectorType>(Val: Ty))
4341 return std::nullopt;
4342 auto *VectorTy = cast<VectorType>(Val: Ty);
4343
4344 // We are looking for a pattern of, and finding the minimal acceptable cost:
4345 // reduce(mul(ext(A), ext(B))) or
4346 // reduce(mul(A, B)) or
4347 // reduce(ext(A)) or
4348 // reduce(A).
4349 // The basic idea is that we walk down the tree to do that, finding the root
4350 // reduction instruction in InLoopReductionImmediateChains. From there we find
4351 // the pattern of mul/ext and test the cost of the entire pattern vs the cost
4352 // of the components. If the reduction cost is lower then we return it for the
4353 // reduction instruction and 0 for the other instructions in the pattern. If
4354 // it is not we return an invalid cost specifying the orignal cost method
4355 // should be used.
4356 Instruction *RetI = I;
4357 if (match(V: RetI, P: m_ZExtOrSExt(Op: m_Value()))) {
4358 if (!RetI->hasOneUser())
4359 return std::nullopt;
4360 RetI = RetI->user_back();
4361 }
4362
4363 if (match(V: RetI, P: m_OneUse(SubPattern: m_Mul(L: m_Value(), R: m_Value()))) &&
4364 RetI->user_back()->getOpcode() == Instruction::Add) {
4365 RetI = RetI->user_back();
4366 }
4367
4368 // Test if the found instruction is a reduction, and if not return an invalid
4369 // cost specifying the parent to use the original cost modelling.
4370 Instruction *LastChain = Config.getInLoopReductionImmediateChain(I: RetI);
4371 if (!LastChain)
4372 return std::nullopt;
4373
4374 // Find the reduction this chain is a part of and calculate the basic cost of
4375 // the reduction on its own.
4376 Instruction *ReductionPhi = LastChain;
4377 while (!isa<PHINode>(Val: ReductionPhi))
4378 ReductionPhi = Config.getInLoopReductionImmediateChain(I: ReductionPhi);
4379
4380 const RecurrenceDescriptor &RdxDesc =
4381 Legal->getRecurrenceDescriptor(PN: cast<PHINode>(Val: ReductionPhi));
4382
4383 InstructionCost BaseCost;
4384 RecurKind RK = RdxDesc.getRecurrenceKind();
4385 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind: RK)) {
4386 Intrinsic::ID MinMaxID = getMinMaxReductionIntrinsicOp(RK);
4387 BaseCost = TTI.getMinMaxReductionCost(
4388 IID: MinMaxID, Ty: VectorTy, FMF: RdxDesc.getFastMathFlags(), CostKind: Config.CostKind);
4389 } else {
4390 BaseCost = TTI.getArithmeticReductionCost(Opcode: RdxDesc.getOpcode(), Ty: VectorTy,
4391 FMF: RdxDesc.getFastMathFlags(),
4392 CostKind: Config.CostKind);
4393 }
4394
4395 // For a call to the llvm.fmuladd intrinsic we need to add the cost of a
4396 // normal fmul instruction to the cost of the fadd reduction.
4397 if (RK == RecurKind::FMulAdd)
4398 BaseCost += TTI.getArithmeticInstrCost(Opcode: Instruction::FMul, Ty: VectorTy,
4399 CostKind: Config.CostKind);
4400
4401 // If we're using ordered reductions then we can just return the base cost
4402 // here, since getArithmeticReductionCost calculates the full ordered
4403 // reduction cost when FP reassociation is not allowed.
4404 if (Config.useOrderedReductions(RdxDesc))
4405 return BaseCost;
4406
4407 // Get the operand that was not the reduction chain and match it to one of the
4408 // patterns, returning the better cost if it is found.
4409 Instruction *RedOp = RetI->getOperand(i: 1) == LastChain
4410 ? dyn_cast<Instruction>(Val: RetI->getOperand(i: 0))
4411 : dyn_cast<Instruction>(Val: RetI->getOperand(i: 1));
4412
4413 VectorTy = VectorType::get(ElementType: I->getOperand(i: 0)->getType(), Other: VectorTy);
4414
4415 Instruction *Op0, *Op1;
4416 if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
4417 match(V: RedOp,
4418 P: m_ZExtOrSExt(Op: m_Mul(L: m_Instruction(I&: Op0), R: m_Instruction(I&: Op1)))) &&
4419 match(V: Op0, P: m_ZExtOrSExt(Op: m_Value())) &&
4420 Op0->getOpcode() == Op1->getOpcode() &&
4421 Op0->getOperand(i: 0)->getType() == Op1->getOperand(i: 0)->getType() &&
4422 !TheLoop->isLoopInvariant(V: Op0) && !TheLoop->isLoopInvariant(V: Op1) &&
4423 (Op0->getOpcode() == RedOp->getOpcode() || Op0 == Op1)) {
4424
4425 // Matched reduce.add(ext(mul(ext(A), ext(B)))
4426 // Note that the extend opcodes need to all match, or if A==B they will have
4427 // been converted to zext(mul(sext(A), sext(A))) as it is known positive,
4428 // which is equally fine.
4429 bool IsUnsigned = isa<ZExtInst>(Val: Op0);
4430 auto *ExtType = VectorType::get(ElementType: Op0->getOperand(i: 0)->getType(), Other: VectorTy);
4431 auto *MulType = VectorType::get(ElementType: Op0->getType(), Other: VectorTy);
4432
4433 InstructionCost ExtCost =
4434 TTI.getCastInstrCost(Opcode: Op0->getOpcode(), Dst: MulType, Src: ExtType,
4435 CCH: TTI::CastContextHint::None, CostKind: Config.CostKind, I: Op0);
4436 InstructionCost MulCost =
4437 TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: MulType, CostKind: Config.CostKind);
4438 InstructionCost Ext2Cost = TTI.getCastInstrCost(
4439 Opcode: RedOp->getOpcode(), Dst: VectorTy, Src: MulType, CCH: TTI::CastContextHint::None,
4440 CostKind: Config.CostKind, I: RedOp);
4441
4442 InstructionCost RedCost = TTI.getMulAccReductionCost(
4443 IsUnsigned, RedOpcode: RdxDesc.getOpcode(), ResTy: RdxDesc.getRecurrenceType(), Ty: ExtType,
4444 CostKind: Config.CostKind);
4445
4446 if (RedCost.isValid() &&
4447 RedCost < ExtCost * 2 + MulCost + Ext2Cost + BaseCost)
4448 return I == RetI ? RedCost : 0;
4449 } else if (RedOp && match(V: RedOp, P: m_ZExtOrSExt(Op: m_Value())) &&
4450 !TheLoop->isLoopInvariant(V: RedOp)) {
4451 // Matched reduce(ext(A))
4452 bool IsUnsigned = isa<ZExtInst>(Val: RedOp);
4453 auto *ExtType = VectorType::get(ElementType: RedOp->getOperand(i: 0)->getType(), Other: VectorTy);
4454 InstructionCost RedCost = TTI.getExtendedReductionCost(
4455 Opcode: RdxDesc.getOpcode(), IsUnsigned, ResTy: RdxDesc.getRecurrenceType(), Ty: ExtType,
4456 FMF: RdxDesc.getFastMathFlags(), CostKind: Config.CostKind);
4457
4458 InstructionCost ExtCost = TTI.getCastInstrCost(
4459 Opcode: RedOp->getOpcode(), Dst: VectorTy, Src: ExtType, CCH: TTI::CastContextHint::None,
4460 CostKind: Config.CostKind, I: RedOp);
4461 if (RedCost.isValid() && RedCost < BaseCost + ExtCost)
4462 return I == RetI ? RedCost : 0;
4463 } else if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
4464 match(V: RedOp, P: m_Mul(L: m_Instruction(I&: Op0), R: m_Instruction(I&: Op1)))) {
4465 if (match(V: Op0, P: m_ZExtOrSExt(Op: m_Value())) &&
4466 Op0->getOpcode() == Op1->getOpcode() &&
4467 !TheLoop->isLoopInvariant(V: Op0) && !TheLoop->isLoopInvariant(V: Op1)) {
4468 bool IsUnsigned = isa<ZExtInst>(Val: Op0);
4469 Type *Op0Ty = Op0->getOperand(i: 0)->getType();
4470 Type *Op1Ty = Op1->getOperand(i: 0)->getType();
4471 Type *LargestOpTy =
4472 Op0Ty->getIntegerBitWidth() < Op1Ty->getIntegerBitWidth() ? Op1Ty
4473 : Op0Ty;
4474 auto *ExtType = VectorType::get(ElementType: LargestOpTy, Other: VectorTy);
4475
4476 // Matched reduce.add(mul(ext(A), ext(B))), where the two ext may be of
4477 // different sizes. We take the largest type as the ext to reduce, and add
4478 // the remaining cost as, for example reduce(mul(ext(ext(A)), ext(B))).
4479 InstructionCost ExtCost0 = TTI.getCastInstrCost(
4480 Opcode: Op0->getOpcode(), Dst: VectorTy, Src: VectorType::get(ElementType: Op0Ty, Other: VectorTy),
4481 CCH: TTI::CastContextHint::None, CostKind: Config.CostKind, I: Op0);
4482 InstructionCost ExtCost1 = TTI.getCastInstrCost(
4483 Opcode: Op1->getOpcode(), Dst: VectorTy, Src: VectorType::get(ElementType: Op1Ty, Other: VectorTy),
4484 CCH: TTI::CastContextHint::None, CostKind: Config.CostKind, I: Op1);
4485 InstructionCost MulCost = TTI.getArithmeticInstrCost(
4486 Opcode: Instruction::Mul, Ty: VectorTy, CostKind: Config.CostKind);
4487
4488 InstructionCost RedCost = TTI.getMulAccReductionCost(
4489 IsUnsigned, RedOpcode: RdxDesc.getOpcode(), ResTy: RdxDesc.getRecurrenceType(), Ty: ExtType,
4490 CostKind: Config.CostKind);
4491 InstructionCost ExtraExtCost = 0;
4492 if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
4493 Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
4494 ExtraExtCost = TTI.getCastInstrCost(
4495 Opcode: ExtraExtOp->getOpcode(), Dst: ExtType,
4496 Src: VectorType::get(ElementType: ExtraExtOp->getOperand(i: 0)->getType(), Other: VectorTy),
4497 CCH: TTI::CastContextHint::None, CostKind: Config.CostKind, I: ExtraExtOp);
4498 }
4499
4500 if (RedCost.isValid() &&
4501 (RedCost + ExtraExtCost) < (ExtCost0 + ExtCost1 + MulCost + BaseCost))
4502 return I == RetI ? RedCost : 0;
4503 } else if (!match(V: I, P: m_ZExtOrSExt(Op: m_Value()))) {
4504 // Matched reduce.add(mul())
4505 InstructionCost MulCost = TTI.getArithmeticInstrCost(
4506 Opcode: Instruction::Mul, Ty: VectorTy, CostKind: Config.CostKind);
4507
4508 InstructionCost RedCost = TTI.getMulAccReductionCost(
4509 IsUnsigned: true, RedOpcode: RdxDesc.getOpcode(), ResTy: RdxDesc.getRecurrenceType(), Ty: VectorTy,
4510 CostKind: Config.CostKind);
4511
4512 if (RedCost.isValid() && RedCost < MulCost + BaseCost)
4513 return I == RetI ? RedCost : 0;
4514 }
4515 }
4516
4517 return I == RetI ? std::optional<InstructionCost>(BaseCost) : std::nullopt;
4518}
4519
4520InstructionCost
4521LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
4522 ElementCount VF) {
4523 // Calculate scalar cost only. Vectorization cost should be ready at this
4524 // moment.
4525 if (VF.isScalar()) {
4526 Type *ValTy = getLoadStoreType(I);
4527 Type *PtrTy = getLoadStorePointerOperand(V: I)->getType();
4528 const Align Alignment = getLoadStoreAlignment(I);
4529 unsigned AS = getLoadStoreAddressSpace(I);
4530
4531 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4532 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4533 CostKind: Config.CostKind) +
4534 TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: ValTy, Alignment, AddressSpace: AS,
4535 CostKind: Config.CostKind, OpdInfo: OpInfo, I);
4536 }
4537 return getWideningCost(I, VF);
4538}
4539
4540InstructionCost
4541LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
4542 ElementCount VF) const {
4543
4544 // There is no mechanism yet to create a scalable scalarization loop,
4545 // so this is currently Invalid.
4546 if (VF.isScalable())
4547 return InstructionCost::getInvalid();
4548
4549 if (VF.isScalar())
4550 return 0;
4551
4552 InstructionCost Cost = 0;
4553 Type *RetTy = toVectorizedTy(Ty: I->getType(), EC: VF);
4554 if (!RetTy->isVoidTy() &&
4555 (!isa<LoadInst>(Val: I) || !TTI.supportsEfficientVectorElementLoadStore())) {
4556
4557 TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None;
4558 if (isa<LoadInst>(Val: I))
4559 VIC = TTI::VectorInstrContext::Load;
4560 else if (isa<StoreInst>(Val: I))
4561 VIC = TTI::VectorInstrContext::Store;
4562
4563 for (Type *VectorTy : getContainedTypes(Ty: RetTy)) {
4564 Cost += TTI.getScalarizationOverhead(
4565 Ty: cast<VectorType>(Val: VectorTy), DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4566 /*Insert=*/true, /*Extract=*/false, CostKind: Config.CostKind,
4567 /*ForPoisonSrc=*/true, VL: {}, VIC);
4568 }
4569 }
4570
4571 // Some targets keep addresses scalar.
4572 if (isa<LoadInst>(Val: I) && !TTI.prefersVectorizedAddressing())
4573 return Cost;
4574
4575 // Some targets support efficient element stores.
4576 if (isa<StoreInst>(Val: I) && TTI.supportsEfficientVectorElementLoadStore())
4577 return Cost;
4578
4579 // Collect operands to consider.
4580 CallInst *CI = dyn_cast<CallInst>(Val: I);
4581 Instruction::op_range Ops = CI ? CI->args() : I->operands();
4582
4583 // Skip operands that do not require extraction/scalarization and do not incur
4584 // any overhead.
4585 SmallVector<Type *> Tys;
4586 for (auto *V : filterExtractingOperands(Ops, VF))
4587 Tys.push_back(Elt: maybeVectorizeType(Ty: V->getType(), VF));
4588
4589 TTI::VectorInstrContext OperandVIC = isa<StoreInst>(Val: I)
4590 ? TTI::VectorInstrContext::Store
4591 : TTI::VectorInstrContext::None;
4592 return Cost +
4593 TTI.getOperandsScalarizationOverhead(Tys, CostKind: Config.CostKind, VIC: OperandVIC);
4594}
4595
4596void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
4597 if (VF.isScalar())
4598 return;
4599
4600 // TODO: We should generate better code and update the cost model for
4601 // predicated uniform stores. Today they are treated as any other
4602 // predicated store (see added test cases in
4603 // invariant-store-vectorization.ll).
4604 NumPredStores = 0;
4605 for (BasicBlock *BB : TheLoop->blocks())
4606 for (Instruction &I : *BB)
4607 if (isa<StoreInst>(Val: &I) && isScalarWithPredication(I: &I, VF))
4608 ++NumPredStores;
4609
4610 for (BasicBlock *BB : TheLoop->blocks()) {
4611 // For each instruction in the old loop.
4612 for (Instruction &I : *BB) {
4613 Value *Ptr = getLoadStorePointerOperand(V: &I);
4614 if (!Ptr)
4615 continue;
4616
4617 if (isUniformMemOp(I, VF)) {
4618 auto IsLegalToScalarize = [&]() {
4619 if (!VF.isScalable())
4620 // Scalarization of fixed length vectors "just works".
4621 return true;
4622
4623 // We have dedicated lowering for unpredicated uniform loads and
4624 // stores. Note that even with tail folding we know that at least
4625 // one lane is active (i.e. generalized predication is not possible
4626 // here), and the logic below depends on this fact.
4627 if (!foldTailByMasking())
4628 return true;
4629
4630 // For scalable vectors, a uniform memop load is always
4631 // uniform-by-parts and we know how to scalarize that.
4632 if (isa<LoadInst>(Val: I))
4633 return true;
4634
4635 // A uniform store isn't neccessarily uniform-by-part
4636 // and we can't assume scalarization.
4637 auto &SI = cast<StoreInst>(Val&: I);
4638 return TheLoop->isLoopInvariant(V: SI.getValueOperand());
4639 };
4640
4641 const InstructionCost GatherScatterCost =
4642 Config.isLegalGatherOrScatter(V: &I, VF)
4643 ? getGatherScatterCost(I: &I, VF)
4644 : InstructionCost::getInvalid();
4645
4646 // Load: Scalar load + broadcast
4647 // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
4648 // FIXME: This cost is a significant under-estimate for tail folded
4649 // memory ops.
4650 const InstructionCost ScalarizationCost =
4651 IsLegalToScalarize() ? getUniformMemOpCost(I: &I, VF)
4652 : InstructionCost::getInvalid();
4653
4654 // Choose better solution for the current VF, Note that Invalid
4655 // costs compare as maximumal large. If both are invalid, we get
4656 // scalable invalid which signals a failure and a vectorization abort.
4657 if (GatherScatterCost < ScalarizationCost)
4658 setWideningDecision(I: &I, VF, W: CM_GatherScatter, Cost: GatherScatterCost);
4659 else
4660 setWideningDecision(I: &I, VF, W: CM_Scalarize, Cost: ScalarizationCost);
4661 continue;
4662 }
4663
4664 // We assume that widening is the best solution when possible.
4665 if (std::optional<InstWidening> Decision =
4666 memoryInstructionCanBeWidened(I: &I, VF)) {
4667 setWideningDecision(I: &I, VF, W: *Decision,
4668 Cost: getConsecutiveMemOpCost(I: &I, VF, Kind: *Decision));
4669 continue;
4670 }
4671
4672 // Choose between Interleaving, Gather/Scatter or Scalarization.
4673 InstructionCost InterleaveCost = InstructionCost::getInvalid();
4674 unsigned NumAccesses = 1;
4675 if (isAccessInterleaved(Instr: &I)) {
4676 const auto *Group = getInterleavedAccessGroup(Instr: &I);
4677 assert(Group && "Fail to get an interleaved access group.");
4678
4679 // Make one decision for the whole group.
4680 if (getWideningDecision(I: &I, VF) != CM_Unknown)
4681 continue;
4682
4683 NumAccesses = Group->getNumMembers();
4684 if (interleavedAccessCanBeWidened(I: &I, VF))
4685 InterleaveCost = getInterleaveGroupCost(I: &I, VF);
4686 }
4687
4688 InstructionCost GatherScatterCost =
4689 Config.isLegalGatherOrScatter(V: &I, VF)
4690 ? getGatherScatterCost(I: &I, VF) * NumAccesses
4691 : InstructionCost::getInvalid();
4692
4693 InstructionCost ScalarizationCost =
4694 getMemInstScalarizationCost(I: &I, VF) * NumAccesses;
4695
4696 // Choose better solution for the current VF,
4697 // write down this decision and use it during vectorization.
4698 InstructionCost Cost;
4699 InstWidening Decision;
4700 if (InterleaveCost <= GatherScatterCost &&
4701 InterleaveCost < ScalarizationCost) {
4702 Decision = CM_Interleave;
4703 Cost = InterleaveCost;
4704 } else if (GatherScatterCost < ScalarizationCost) {
4705 Decision = CM_GatherScatter;
4706 Cost = GatherScatterCost;
4707 } else {
4708 Decision = CM_Scalarize;
4709 Cost = ScalarizationCost;
4710 }
4711 // If the instructions belongs to an interleave group, the whole group
4712 // receives the same decision. The whole group receives the cost, but
4713 // the cost will actually be assigned to one instruction.
4714 if (const auto *Group = getInterleavedAccessGroup(Instr: &I)) {
4715 if (Decision == CM_Scalarize) {
4716 for (Instruction *I : Group->members())
4717 setWideningDecision(I, VF, W: Decision,
4718 Cost: getMemInstScalarizationCost(I, VF));
4719 } else {
4720 setWideningDecision(Grp: Group, VF, W: Decision, Cost);
4721 }
4722 } else
4723 setWideningDecision(I: &I, VF, W: Decision, Cost);
4724 }
4725 }
4726
4727 // Make sure that any load of address and any other address computation
4728 // remains scalar unless there is gather/scatter support. This avoids
4729 // inevitable extracts into address registers, and also has the benefit of
4730 // activating LSR more, since that pass can't optimize vectorized
4731 // addresses.
4732 if (TTI.prefersVectorizedAddressing())
4733 return;
4734
4735 // Start with all scalar pointer uses.
4736 SmallSetVector<Instruction *, 8> AddrDefs;
4737 for (BasicBlock *BB : TheLoop->blocks())
4738 for (Instruction &I : *BB) {
4739 Instruction *PtrDef =
4740 dyn_cast_or_null<Instruction>(Val: getLoadStorePointerOperand(V: &I));
4741 if (PtrDef && TheLoop->contains(Inst: PtrDef) &&
4742 getWideningDecision(I: &I, VF) != CM_GatherScatter)
4743 AddrDefs.insert(X: PtrDef);
4744 }
4745
4746 // Add all instructions used to generate the addresses.
4747 SmallVector<Instruction *, 4> Worklist;
4748 append_range(C&: Worklist, R&: AddrDefs);
4749 while (!Worklist.empty()) {
4750 Instruction *I = Worklist.pop_back_val();
4751 for (auto &Op : I->operands())
4752 if (auto *InstOp = dyn_cast<Instruction>(Val&: Op))
4753 if (TheLoop->contains(Inst: InstOp) && !isa<PHINode>(Val: InstOp) &&
4754 AddrDefs.insert(X: InstOp))
4755 Worklist.push_back(Elt: InstOp);
4756 }
4757
4758 auto UpdateMemOpUserCost = [this, VF](LoadInst *LI) {
4759 // If there are direct memory op users of the newly scalarized load,
4760 // their cost may have changed because there's no scalarization
4761 // overhead for the operand. Update it.
4762 for (User *U : LI->users()) {
4763 if (!isa<LoadInst, StoreInst>(Val: U))
4764 continue;
4765 if (getWideningDecision(I: cast<Instruction>(Val: U), VF) != CM_Scalarize)
4766 continue;
4767 setWideningDecision(
4768 I: cast<Instruction>(Val: U), VF, W: CM_Scalarize,
4769 Cost: getMemInstScalarizationCost(I: cast<Instruction>(Val: U), VF));
4770 }
4771 };
4772 for (auto *I : AddrDefs) {
4773 if (isa<LoadInst>(Val: I)) {
4774 // Setting the desired widening decision should ideally be handled in
4775 // by cost functions, but since this involves the task of finding out
4776 // if the loaded register is involved in an address computation, it is
4777 // instead changed here when we know this is the case.
4778 InstWidening Decision = getWideningDecision(I, VF);
4779 if (!isPredicatedInst(I) &&
4780 (Decision == CM_Widen || Decision == CM_Widen_Reverse ||
4781 (!isUniformMemOp(I&: *I, VF) && Decision == CM_Scalarize))) {
4782 // Scalarize a widened load of address or update the cost of a scalar
4783 // load of an address.
4784 setWideningDecision(
4785 I, VF, W: CM_Scalarize,
4786 Cost: (VF.getKnownMinValue() *
4787 getMemoryInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1))));
4788 UpdateMemOpUserCost(cast<LoadInst>(Val: I));
4789 } else if (const auto *Group = getInterleavedAccessGroup(Instr: I)) {
4790 // Scalarize all members of this interleaved group when any member
4791 // is used as an address. The address-used load skips scalarization
4792 // overhead, other members include it.
4793 for (Instruction *Member : Group->members()) {
4794 InstructionCost Cost = AddrDefs.contains(key: Member)
4795 ? (VF.getKnownMinValue() *
4796 getMemoryInstructionCost(
4797 I: Member, VF: ElementCount::getFixed(MinVal: 1)))
4798 : getMemInstScalarizationCost(I: Member, VF);
4799 setWideningDecision(I: Member, VF, W: CM_Scalarize, Cost);
4800 UpdateMemOpUserCost(cast<LoadInst>(Val: Member));
4801 }
4802 }
4803 } else {
4804 // Cannot scalarize fixed-order recurrence phis at the moment.
4805 if (isa<PHINode>(Val: I) && Legal->isFixedOrderRecurrence(Phi: cast<PHINode>(Val: I)))
4806 continue;
4807
4808 // Make sure I gets scalarized and a cost estimate without
4809 // scalarization overhead.
4810 ForcedScalars[VF].insert(X: I);
4811 }
4812 }
4813}
4814
4815bool LoopVectorizationCostModel::shouldConsiderInvariant(Value *Op) {
4816 if (!Legal->isInvariant(V: Op))
4817 return false;
4818 // Consider Op invariant, if it or its operands aren't predicated
4819 // instruction in the loop. In that case, it is not trivially hoistable.
4820 auto *OpI = dyn_cast<Instruction>(Val: Op);
4821 return !OpI || !TheLoop->contains(Inst: OpI) ||
4822 (!isPredicatedInst(I: OpI) &&
4823 (!isa<PHINode>(Val: OpI) || OpI->getParent() != TheLoop->getHeader()) &&
4824 all_of(Range: OpI->operands(),
4825 P: [this](Value *Op) { return shouldConsiderInvariant(Op); }));
4826}
4827
4828InstructionCost
4829LoopVectorizationCostModel::getInstructionCost(Instruction *I,
4830 ElementCount VF) {
4831 // If we know that this instruction will remain uniform, check the cost of
4832 // the scalar version.
4833 if (isUniformAfterVectorization(I, VF))
4834 VF = ElementCount::getFixed(MinVal: 1);
4835
4836 if (VF.isVector() && isProfitableToScalarize(I, VF))
4837 return InstsToScalarize[VF][I];
4838
4839 // Forced scalars do not have any scalarization overhead.
4840 auto ForcedScalar = ForcedScalars.find(Val: VF);
4841 if (VF.isVector() && ForcedScalar != ForcedScalars.end()) {
4842 auto InstSet = ForcedScalar->second;
4843 if (InstSet.count(key: I))
4844 return getInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1)) *
4845 VF.getKnownMinValue();
4846 }
4847
4848 const auto &MinBWs = Config.getMinimalBitwidths();
4849 uint64_t InstrMinBWs = MinBWs.lookup(Key: I);
4850 Type *RetTy = I->getType();
4851 if (canTruncateToMinimalBitwidth(I, VF))
4852 RetTy = IntegerType::get(C&: RetTy->getContext(), NumBits: InstrMinBWs);
4853 auto *SE = PSE.getSE();
4854
4855 Type *VectorTy;
4856 if (isScalarAfterVectorization(I, VF)) {
4857 [[maybe_unused]] auto HasSingleCopyAfterVectorization =
4858 [this](Instruction *I, ElementCount VF) -> bool {
4859 if (VF.isScalar())
4860 return true;
4861
4862 auto Scalarized = InstsToScalarize.find(Key: VF);
4863 assert(Scalarized != InstsToScalarize.end() &&
4864 "VF not yet analyzed for scalarization profitability");
4865 return !Scalarized->second.count(Key: I) &&
4866 llvm::all_of(Range: I->users(), P: [&](User *U) {
4867 auto *UI = cast<Instruction>(Val: U);
4868 return !Scalarized->second.count(Key: UI);
4869 });
4870 };
4871
4872 // With the exception of GEPs and PHIs, after scalarization there should
4873 // only be one copy of the instruction generated in the loop. This is
4874 // because the VF is either 1, or any instructions that need scalarizing
4875 // have already been dealt with by the time we get here. As a result,
4876 // it means we don't have to multiply the instruction cost by VF.
4877 assert(I->getOpcode() == Instruction::GetElementPtr ||
4878 I->getOpcode() == Instruction::PHI ||
4879 (I->getOpcode() == Instruction::BitCast &&
4880 I->getType()->isPointerTy()) ||
4881 HasSingleCopyAfterVectorization(I, VF));
4882 VectorTy = RetTy;
4883 } else
4884 VectorTy = toVectorizedTy(Ty: RetTy, EC: VF);
4885
4886 if (VF.isVector() && VectorTy->isVectorTy() &&
4887 !TTI.getNumberOfParts(Tp: VectorTy))
4888 return InstructionCost::getInvalid();
4889
4890 // TODO: We need to estimate the cost of intrinsic calls.
4891 switch (I->getOpcode()) {
4892 case Instruction::GetElementPtr:
4893 // We mark this instruction as zero-cost because the cost of GEPs in
4894 // vectorized code depends on whether the corresponding memory instruction
4895 // is scalarized or not. Therefore, we handle GEPs with the memory
4896 // instruction cost.
4897 return 0;
4898 case Instruction::UncondBr:
4899 case Instruction::CondBr: {
4900 // In cases of scalarized and predicated instructions, there will be VF
4901 // predicated blocks in the vectorized loop. Each branch around these
4902 // blocks requires also an extract of its vector compare i1 element.
4903 // Note that the conditional branch from the loop latch will be replaced by
4904 // a single branch controlling the loop, so there is no extra overhead from
4905 // scalarization.
4906 bool ScalarPredicatedBB = false;
4907 CondBrInst *BI = dyn_cast<CondBrInst>(Val: I);
4908 if (VF.isVector() && BI &&
4909 (PredicatedBBsAfterVectorization[VF].count(Ptr: BI->getSuccessor(i: 0)) ||
4910 PredicatedBBsAfterVectorization[VF].count(Ptr: BI->getSuccessor(i: 1))) &&
4911 BI->getParent() != TheLoop->getLoopLatch())
4912 ScalarPredicatedBB = true;
4913
4914 if (ScalarPredicatedBB) {
4915 // Not possible to scalarize scalable vector with predicated instructions.
4916 if (VF.isScalable())
4917 return InstructionCost::getInvalid();
4918 // Return cost for branches around scalarized and predicated blocks.
4919 auto *VecI1Ty =
4920 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: RetTy->getContext()), EC: VF);
4921 return (TTI.getScalarizationOverhead(
4922 Ty: VecI1Ty, DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4923 /*Insert*/ false, /*Extract*/ true, CostKind: Config.CostKind) +
4924 (TTI.getCFInstrCost(Opcode: Instruction::CondBr, CostKind: Config.CostKind) *
4925 VF.getFixedValue()));
4926 }
4927
4928 if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar())
4929 // The back-edge branch will remain, as will all scalar branches.
4930 return TTI.getCFInstrCost(Opcode: Instruction::UncondBr, CostKind: Config.CostKind);
4931
4932 // This branch will be eliminated by if-conversion.
4933 return 0;
4934 // Note: We currently assume zero cost for an unconditional branch inside
4935 // a predicated block since it will become a fall-through, although we
4936 // may decide in the future to call TTI for all branches.
4937 }
4938 case Instruction::Switch: {
4939 if (VF.isScalar())
4940 return TTI.getCFInstrCost(Opcode: Instruction::Switch, CostKind: Config.CostKind);
4941 auto *Switch = cast<SwitchInst>(Val: I);
4942 return Switch->getNumCases() *
4943 TTI.getCmpSelInstrCost(
4944 Opcode: Instruction::ICmp,
4945 ValTy: toVectorTy(Scalar: Switch->getCondition()->getType(), EC: VF),
4946 CondTy: toVectorTy(Scalar: Type::getInt1Ty(C&: I->getContext()), EC: VF),
4947 VecPred: CmpInst::ICMP_EQ, CostKind: Config.CostKind);
4948 }
4949 case Instruction::PHI: {
4950 auto *Phi = cast<PHINode>(Val: I);
4951
4952 // First-order recurrences are replaced by vector shuffles inside the loop.
4953 if (VF.isVector() && Legal->isFixedOrderRecurrence(Phi)) {
4954 return TTI.getShuffleCost(
4955 Kind: TargetTransformInfo::SK_Splice, DstTy: cast<VectorType>(Val: VectorTy),
4956 SrcTy: cast<VectorType>(Val: VectorTy), CostKind: Config.CostKind, Mask: {}, Index: -1);
4957 }
4958
4959 // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
4960 // converted into select instructions. We require N - 1 selects per phi
4961 // node, where N is the number of incoming values.
4962 if (VF.isVector() && Phi->getParent() != TheLoop->getHeader()) {
4963 Type *ResultTy = Phi->getType();
4964
4965 // All instructions in an Any-of reduction chain are narrowed to bool.
4966 // Check if that is the case for this phi node.
4967 auto *HeaderUser = cast_if_present<PHINode>(
4968 Val: find_singleton<User>(Range: Phi->users(), P: [this](User *U, bool) -> User * {
4969 auto *Phi = dyn_cast<PHINode>(Val: U);
4970 if (Phi && Phi->getParent() == TheLoop->getHeader())
4971 return Phi;
4972 return nullptr;
4973 }));
4974 if (HeaderUser) {
4975 auto &ReductionVars = Legal->getReductionVars();
4976 auto Iter = ReductionVars.find(Key: HeaderUser);
4977 if (Iter != ReductionVars.end() &&
4978 RecurrenceDescriptor::isAnyOfRecurrenceKind(
4979 Kind: Iter->second.getRecurrenceKind()))
4980 ResultTy = Type::getInt1Ty(C&: Phi->getContext());
4981 }
4982 return (Phi->getNumIncomingValues() - 1) *
4983 TTI.getCmpSelInstrCost(
4984 Opcode: Instruction::Select, ValTy: toVectorTy(Scalar: ResultTy, EC: VF),
4985 CondTy: toVectorTy(Scalar: Type::getInt1Ty(C&: Phi->getContext()), EC: VF),
4986 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind: Config.CostKind);
4987 }
4988
4989 // When tail folding with EVL, if the phi is part of an out of loop
4990 // reduction then it will be transformed into a wide vp_merge.
4991 if (VF.isVector() && foldTailWithEVL() &&
4992 Legal->getReductionVars().contains(Key: Phi) &&
4993 !Config.isInLoopReduction(Phi)) {
4994 IntrinsicCostAttributes ICA(
4995 Intrinsic::vp_merge, toVectorTy(Scalar: Phi->getType(), EC: VF),
4996 {toVectorTy(Scalar: Type::getInt1Ty(C&: Phi->getContext()), EC: VF)});
4997 return TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind);
4998 }
4999
5000 return TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
5001 }
5002 case Instruction::UDiv:
5003 case Instruction::SDiv:
5004 case Instruction::URem:
5005 case Instruction::SRem:
5006 if (VF.isVector() && isPredicatedInst(I)) {
5007 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
5008 return isDivRemScalarWithPredication(ScalarCost, MaskedCost) ? ScalarCost
5009 : MaskedCost;
5010 }
5011 // We've proven all lanes safe to speculate, fall through.
5012 [[fallthrough]];
5013 case Instruction::Add:
5014 case Instruction::Sub: {
5015 auto Info = Legal->getHistogramInfo(I);
5016 if (Info && VF.isVector()) {
5017 const HistogramInfo *HGram = Info.value();
5018 // Assume that a non-constant update value (or a constant != 1) requires
5019 // a multiply, and add that into the cost.
5020 InstructionCost MulCost = TTI::TCC_Free;
5021 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: I->getOperand(i: 1));
5022 if (!RHS || RHS->getZExtValue() != 1)
5023 MulCost = TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: VectorTy,
5024 CostKind: Config.CostKind);
5025
5026 // Find the cost of the histogram operation itself.
5027 Type *PtrTy = VectorType::get(ElementType: HGram->Load->getPointerOperandType(), EC: VF);
5028 Type *ScalarTy = I->getType();
5029 Type *MaskTy = VectorType::get(ElementType: Type::getInt1Ty(C&: I->getContext()), EC: VF);
5030 IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
5031 Type::getVoidTy(C&: I->getContext()),
5032 {PtrTy, ScalarTy, MaskTy});
5033
5034 // Add the costs together with the add/sub operation.
5035 return TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind) + MulCost +
5036 TTI.getArithmeticInstrCost(Opcode: I->getOpcode(), Ty: VectorTy,
5037 CostKind: Config.CostKind);
5038 }
5039 [[fallthrough]];
5040 }
5041 case Instruction::FAdd:
5042 case Instruction::FSub:
5043 case Instruction::Mul:
5044 case Instruction::FMul:
5045 case Instruction::FDiv:
5046 case Instruction::FRem:
5047 case Instruction::Shl:
5048 case Instruction::LShr:
5049 case Instruction::AShr:
5050 case Instruction::And:
5051 case Instruction::Or:
5052 case Instruction::Xor: {
5053 // If we're speculating on the stride being 1, the multiplication may
5054 // fold away. We can generalize this for all operations using the notion
5055 // of neutral elements. (TODO)
5056 if (I->getOpcode() == Instruction::Mul &&
5057 ((TheLoop->isLoopInvariant(V: I->getOperand(i: 0)) &&
5058 PSE.getSCEV(V: I->getOperand(i: 0))->isOne()) ||
5059 (TheLoop->isLoopInvariant(V: I->getOperand(i: 1)) &&
5060 PSE.getSCEV(V: I->getOperand(i: 1))->isOne())))
5061 return 0;
5062
5063 // Detect reduction patterns
5064 if (auto RedCost = getReductionPatternCost(I, VF, Ty: VectorTy))
5065 return *RedCost;
5066
5067 // Certain instructions can be cheaper to vectorize if they have a constant
5068 // second vector operand. One example of this are shifts on x86.
5069 Value *Op2 = I->getOperand(i: 1);
5070 if (!isa<Constant>(Val: Op2) && TheLoop->isLoopInvariant(V: Op2) &&
5071 PSE.getSE()->isSCEVable(Ty: Op2->getType()) &&
5072 isa<SCEVConstant>(Val: PSE.getSCEV(V: Op2))) {
5073 Op2 = cast<SCEVConstant>(Val: PSE.getSCEV(V: Op2))->getValue();
5074 }
5075 auto Op2Info = TTI.getOperandInfo(V: Op2);
5076 if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue &&
5077 shouldConsiderInvariant(Op: Op2))
5078 Op2Info.Kind = TargetTransformInfo::OK_UniformValue;
5079
5080 SmallVector<const Value *, 4> Operands(I->operand_values());
5081 return TTI.getArithmeticInstrCost(
5082 Opcode: I->getOpcode(), Ty: VectorTy, CostKind: Config.CostKind,
5083 Opd1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
5084 Opd2Info: Op2Info, Args: Operands, CxtI: I, TLibInfo: TLI);
5085 }
5086 case Instruction::FNeg: {
5087 return TTI.getArithmeticInstrCost(
5088 Opcode: I->getOpcode(), Ty: VectorTy, CostKind: Config.CostKind,
5089 Opd1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
5090 Opd2Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
5091 Args: I->getOperand(i: 0), CxtI: I);
5092 }
5093 case Instruction::Select: {
5094 SelectInst *SI = cast<SelectInst>(Val: I);
5095 const SCEV *CondSCEV = SE->getSCEV(V: SI->getCondition());
5096 bool ScalarCond = (SE->isLoopInvariant(S: CondSCEV, L: TheLoop));
5097
5098 const Value *Op0, *Op1;
5099 using namespace llvm::PatternMatch;
5100 if (!ScalarCond && (match(V: I, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))) ||
5101 match(V: I, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))) {
5102 // select x, y, false --> x & y
5103 // select x, true, y --> x | y
5104 const auto [Op1VK, Op1VP] = TTI::getOperandInfo(V: Op0);
5105 const auto [Op2VK, Op2VP] = TTI::getOperandInfo(V: Op1);
5106 assert(Op0->getType()->getScalarSizeInBits() == 1 &&
5107 Op1->getType()->getScalarSizeInBits() == 1);
5108
5109 return TTI.getArithmeticInstrCost(
5110 Opcode: match(V: I, P: m_LogicalOr()) ? Instruction::Or : Instruction::And,
5111 Ty: VectorTy, CostKind: Config.CostKind, Opd1Info: {.Kind: Op1VK, .Properties: Op1VP}, Opd2Info: {.Kind: Op2VK, .Properties: Op2VP}, Args: {Op0, Op1},
5112 CxtI: I);
5113 }
5114
5115 Type *CondTy = SI->getCondition()->getType();
5116 if (!ScalarCond)
5117 CondTy = VectorType::get(ElementType: CondTy, EC: VF);
5118
5119 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
5120 if (auto *Cmp = dyn_cast<CmpInst>(Val: SI->getCondition()))
5121 Pred = Cmp->getPredicate();
5122 return TTI.getCmpSelInstrCost(
5123 Opcode: I->getOpcode(), ValTy: VectorTy, CondTy, VecPred: Pred, CostKind: Config.CostKind,
5124 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
5125 }
5126 case Instruction::ICmp:
5127 case Instruction::FCmp: {
5128 Type *ValTy = I->getOperand(i: 0)->getType();
5129
5130 if (canTruncateToMinimalBitwidth(I, VF)) {
5131 [[maybe_unused]] Instruction *Op0AsInstruction =
5132 dyn_cast<Instruction>(Val: I->getOperand(i: 0));
5133 assert((!canTruncateToMinimalBitwidth(Op0AsInstruction, VF) ||
5134 InstrMinBWs == MinBWs.lookup(Op0AsInstruction)) &&
5135 "if both the operand and the compare are marked for "
5136 "truncation, they must have the same bitwidth");
5137 ValTy = IntegerType::get(C&: ValTy->getContext(), NumBits: InstrMinBWs);
5138 }
5139
5140 VectorTy = toVectorTy(Scalar: ValTy, EC: VF);
5141 return TTI.getCmpSelInstrCost(
5142 Opcode: I->getOpcode(), ValTy: VectorTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VectorTy),
5143 VecPred: cast<CmpInst>(Val: I)->getPredicate(), CostKind: Config.CostKind,
5144 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
5145 }
5146 case Instruction::Store:
5147 case Instruction::Load: {
5148 ElementCount Width = VF;
5149 if (Width.isVector()) {
5150 InstWidening Decision = getWideningDecision(I, VF: Width);
5151 assert(Decision != CM_Unknown &&
5152 "CM decision should be taken at this point");
5153 if (getWideningCost(I, VF) == InstructionCost::getInvalid())
5154 return InstructionCost::getInvalid();
5155 if (Decision == CM_Scalarize)
5156 Width = ElementCount::getFixed(MinVal: 1);
5157 }
5158 VectorTy = toVectorTy(Scalar: getLoadStoreType(I), EC: Width);
5159 return getMemoryInstructionCost(I, VF);
5160 }
5161 case Instruction::BitCast:
5162 if (I->getType()->isPointerTy())
5163 return 0;
5164 [[fallthrough]];
5165 case Instruction::ZExt:
5166 case Instruction::SExt:
5167 case Instruction::FPToUI:
5168 case Instruction::FPToSI:
5169 case Instruction::FPExt:
5170 case Instruction::PtrToInt:
5171 case Instruction::IntToPtr:
5172 case Instruction::SIToFP:
5173 case Instruction::UIToFP:
5174 case Instruction::Trunc:
5175 case Instruction::FPTrunc: {
5176 // Computes the CastContextHint from a Load/Store instruction.
5177 auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
5178 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
5179 "Expected a load or a store!");
5180
5181 if (VF.isScalar() || !TheLoop->contains(Inst: I))
5182 return TTI::CastContextHint::Normal;
5183
5184 switch (getWideningDecision(I, VF)) {
5185 case LoopVectorizationCostModel::CM_GatherScatter:
5186 return TTI::CastContextHint::GatherScatter;
5187 case LoopVectorizationCostModel::CM_Interleave:
5188 return TTI::CastContextHint::Interleave;
5189 case LoopVectorizationCostModel::CM_Scalarize:
5190 case LoopVectorizationCostModel::CM_Widen:
5191 return isPredicatedInst(I) ? TTI::CastContextHint::Masked
5192 : TTI::CastContextHint::Normal;
5193 case LoopVectorizationCostModel::CM_Widen_Reverse:
5194 return TTI::CastContextHint::Reversed;
5195 case LoopVectorizationCostModel::CM_Unknown:
5196 llvm_unreachable("Instr did not go through cost modelling?");
5197 case LoopVectorizationCostModel::CM_InvalidatedDecision:
5198 return TTI::CastContextHint::None;
5199 }
5200
5201 llvm_unreachable("Unhandled case!");
5202 };
5203
5204 unsigned Opcode = I->getOpcode();
5205 TTI::CastContextHint CCH = TTI::CastContextHint::None;
5206 // For Trunc, the context is the only user, which must be a StoreInst.
5207 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
5208 if (I->hasOneUse())
5209 if (StoreInst *Store = dyn_cast<StoreInst>(Val: *I->user_begin()))
5210 CCH = ComputeCCH(Store);
5211 }
5212 // For Z/Sext, the context is the operand, which must be a LoadInst.
5213 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
5214 Opcode == Instruction::FPExt) {
5215 if (LoadInst *Load = dyn_cast<LoadInst>(Val: I->getOperand(i: 0)))
5216 CCH = ComputeCCH(Load);
5217 }
5218
5219 // We optimize the truncation of induction variables having constant
5220 // integer steps. The cost of these truncations is the same as the scalar
5221 // operation.
5222 if (isOptimizableIVTruncate(I, VF)) {
5223 auto *Trunc = cast<TruncInst>(Val: I);
5224 return TTI.getCastInstrCost(Opcode: Instruction::Trunc, Dst: Trunc->getDestTy(),
5225 Src: Trunc->getSrcTy(), CCH, CostKind: Config.CostKind,
5226 I: Trunc);
5227 }
5228
5229 // Detect reduction patterns
5230 if (auto RedCost = getReductionPatternCost(I, VF, Ty: VectorTy))
5231 return *RedCost;
5232
5233 Type *SrcScalarTy = I->getOperand(i: 0)->getType();
5234 Instruction *Op0AsInstruction = dyn_cast<Instruction>(Val: I->getOperand(i: 0));
5235 if (canTruncateToMinimalBitwidth(I: Op0AsInstruction, VF))
5236 SrcScalarTy = IntegerType::get(C&: SrcScalarTy->getContext(),
5237 NumBits: MinBWs.lookup(Key: Op0AsInstruction));
5238 Type *SrcVecTy =
5239 VectorTy->isVectorTy() ? toVectorTy(Scalar: SrcScalarTy, EC: VF) : SrcScalarTy;
5240
5241 if (canTruncateToMinimalBitwidth(I, VF)) {
5242 // If the result type is <= the source type, there will be no extend
5243 // after truncating the users to the minimal required bitwidth.
5244 if (VectorTy->getScalarSizeInBits() <= SrcVecTy->getScalarSizeInBits() &&
5245 (I->getOpcode() == Instruction::ZExt ||
5246 I->getOpcode() == Instruction::SExt))
5247 return 0;
5248 }
5249
5250 return TTI.getCastInstrCost(Opcode, Dst: VectorTy, Src: SrcVecTy, CCH,
5251 CostKind: Config.CostKind, I);
5252 }
5253 case Instruction::Call:
5254 return getVectorCallCost(CI: cast<CallInst>(Val: I), VF);
5255 case Instruction::ExtractValue:
5256 return TTI.getInstructionCost(U: I, CostKind: Config.CostKind);
5257 case Instruction::Alloca:
5258 // We cannot easily widen alloca to a scalable alloca, as
5259 // the result would need to be a vector of pointers.
5260 if (VF.isScalable())
5261 return InstructionCost::getInvalid();
5262 return TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: RetTy, CostKind: Config.CostKind);
5263 case Instruction::Freeze:
5264 return TTI::TCC_Free;
5265 default:
5266 // This opcode is unknown. Assume that it is the same as 'mul'.
5267 return TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: VectorTy,
5268 CostKind: Config.CostKind);
5269 } // end of switch.
5270}
5271
5272void LoopVectorizationCostModel::collectValuesToIgnore() {
5273 // Ignore ephemeral values.
5274 CodeMetrics::collectEphemeralValues(L: TheLoop, AC, EphValues&: ValuesToIgnore);
5275
5276 SmallVector<Value *, 4> DeadInterleavePointerOps;
5277 SmallVector<Value *, 4> DeadOps;
5278
5279 // If a scalar epilogue is required, users outside the loop won't use
5280 // live-outs from the vector loop but from the scalar epilogue. Ignore them if
5281 // that is the case.
5282 bool RequiresScalarEpilogue = requiresScalarEpilogue(IsVectorizing: true);
5283 auto IsLiveOutDead = [this, RequiresScalarEpilogue](User *U) {
5284 return RequiresScalarEpilogue &&
5285 !TheLoop->contains(BB: cast<Instruction>(Val: U)->getParent());
5286 };
5287
5288 LoopBlocksDFS DFS(TheLoop);
5289 DFS.perform(LI);
5290 for (BasicBlock *BB : reverse(C: make_range(x: DFS.beginRPO(), y: DFS.endRPO())))
5291 for (Instruction &I : reverse(C&: *BB)) {
5292 if (VecValuesToIgnore.contains(Ptr: &I) || ValuesToIgnore.contains(Ptr: &I))
5293 continue;
5294
5295 // Add instructions that would be trivially dead and are only used by
5296 // values already ignored to DeadOps to seed worklist.
5297 if (wouldInstructionBeTriviallyDead(I: &I, TLI) &&
5298 all_of(Range: I.users(), P: [this, IsLiveOutDead](User *U) {
5299 return VecValuesToIgnore.contains(Ptr: U) ||
5300 ValuesToIgnore.contains(Ptr: U) || IsLiveOutDead(U);
5301 }))
5302 DeadOps.push_back(Elt: &I);
5303
5304 // For interleave groups, we only create a pointer for the start of the
5305 // interleave group. Queue up addresses of group members except the insert
5306 // position for further processing.
5307 if (isAccessInterleaved(Instr: &I)) {
5308 auto *Group = getInterleavedAccessGroup(Instr: &I);
5309 if (Group->getInsertPos() == &I)
5310 continue;
5311 Value *PointerOp = getLoadStorePointerOperand(V: &I);
5312 DeadInterleavePointerOps.push_back(Elt: PointerOp);
5313 }
5314
5315 // Queue branches for analysis. They are dead, if their successors only
5316 // contain dead instructions.
5317 if (isa<CondBrInst>(Val: &I))
5318 DeadOps.push_back(Elt: &I);
5319 }
5320
5321 // Mark ops feeding interleave group members as free, if they are only used
5322 // by other dead computations.
5323 for (unsigned I = 0; I != DeadInterleavePointerOps.size(); ++I) {
5324 auto *Op = dyn_cast<Instruction>(Val: DeadInterleavePointerOps[I]);
5325 if (!Op || !TheLoop->contains(Inst: Op) || any_of(Range: Op->users(), P: [this](User *U) {
5326 Instruction *UI = cast<Instruction>(Val: U);
5327 return !VecValuesToIgnore.contains(Ptr: U) &&
5328 (!isAccessInterleaved(Instr: UI) ||
5329 getInterleavedAccessGroup(Instr: UI)->getInsertPos() == UI);
5330 }))
5331 continue;
5332 VecValuesToIgnore.insert(Ptr: Op);
5333 append_range(C&: DeadInterleavePointerOps, R: Op->operands());
5334 }
5335
5336 // Mark ops that would be trivially dead and are only used by ignored
5337 // instructions as free.
5338 BasicBlock *Header = TheLoop->getHeader();
5339
5340 // Returns true if the block contains only dead instructions. Such blocks will
5341 // be removed by VPlan-to-VPlan transforms and won't be considered by the
5342 // VPlan-based cost model, so skip them in the legacy cost-model as well.
5343 auto IsEmptyBlock = [this](BasicBlock *BB) {
5344 return all_of(Range&: *BB, P: [this](Instruction &I) {
5345 return ValuesToIgnore.contains(Ptr: &I) || VecValuesToIgnore.contains(Ptr: &I) ||
5346 isa<UncondBrInst>(Val: &I);
5347 });
5348 };
5349 for (unsigned I = 0; I != DeadOps.size(); ++I) {
5350 auto *Op = dyn_cast<Instruction>(Val: DeadOps[I]);
5351
5352 // Check if the branch should be considered dead.
5353 if (auto *Br = dyn_cast_or_null<CondBrInst>(Val: Op)) {
5354 BasicBlock *ThenBB = Br->getSuccessor(i: 0);
5355 BasicBlock *ElseBB = Br->getSuccessor(i: 1);
5356 // Don't considers branches leaving the loop for simplification.
5357 if (!TheLoop->contains(BB: ThenBB) || !TheLoop->contains(BB: ElseBB))
5358 continue;
5359 bool ThenEmpty = IsEmptyBlock(ThenBB);
5360 bool ElseEmpty = IsEmptyBlock(ElseBB);
5361 if ((ThenEmpty && ElseEmpty) ||
5362 (ThenEmpty && ThenBB->getSingleSuccessor() == ElseBB &&
5363 ElseBB->phis().empty()) ||
5364 (ElseEmpty && ElseBB->getSingleSuccessor() == ThenBB &&
5365 ThenBB->phis().empty())) {
5366 VecValuesToIgnore.insert(Ptr: Br);
5367 DeadOps.push_back(Elt: Br->getCondition());
5368 }
5369 continue;
5370 }
5371
5372 // Skip any op that shouldn't be considered dead.
5373 if (!Op || !TheLoop->contains(Inst: Op) ||
5374 (isa<PHINode>(Val: Op) && Op->getParent() == Header) ||
5375 !wouldInstructionBeTriviallyDead(I: Op, TLI) ||
5376 any_of(Range: Op->users(), P: [this, IsLiveOutDead](User *U) {
5377 return !VecValuesToIgnore.contains(Ptr: U) &&
5378 !ValuesToIgnore.contains(Ptr: U) && !IsLiveOutDead(U);
5379 }))
5380 continue;
5381
5382 // If all of Op's users are in ValuesToIgnore, add it to ValuesToIgnore
5383 // which applies for both scalar and vector versions. Otherwise it is only
5384 // dead in vector versions, so only add it to VecValuesToIgnore.
5385 if (all_of(Range: Op->users(),
5386 P: [this](User *U) { return ValuesToIgnore.contains(Ptr: U); }))
5387 ValuesToIgnore.insert(Ptr: Op);
5388
5389 VecValuesToIgnore.insert(Ptr: Op);
5390 append_range(C&: DeadOps, R: Op->operands());
5391 }
5392
5393 // Ignore type-promoting instructions we identified during reduction
5394 // detection.
5395 for (const auto &Reduction : Legal->getReductionVars()) {
5396 const RecurrenceDescriptor &RedDes = Reduction.second;
5397 const SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
5398 VecValuesToIgnore.insert_range(R: Casts);
5399 }
5400 // Ignore type-casting instructions we identified during induction
5401 // detection.
5402 for (const auto &Induction : Legal->getInductionVars()) {
5403 const InductionDescriptor &IndDes = Induction.second;
5404 VecValuesToIgnore.insert_range(R: IndDes.getCastInsts());
5405 }
5406}
5407
5408void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
5409 CM->collectValuesToIgnore();
5410 Config.collectElementTypesForWidening(ValuesToIgnore: &CM->ValuesToIgnore);
5411
5412 FixedScalableVFPair MaxFactors = CM->computeMaxVF(UserVF, UserIC);
5413 if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
5414 return;
5415
5416 Config.collectInLoopReductions();
5417 // Cases that may be vectorized may be optimized by unit stride predicates.
5418 // TODO: Currently unit stride predicates are added unconditionally, even if
5419 // they are not used for the selected VF (e.g. when only interleaving).
5420 if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
5421 Legal->collectUnitStridePredicates();
5422
5423 auto VPlan1 = tryToBuildVPlan1();
5424 if (!VPlan1)
5425 return;
5426
5427 if (!OrigLoop->isInnermost()) {
5428 // For outer loops, computeMaxVF returns a single non-scalar VF; build a
5429 // plan for that VF only.
5430 ElementCount VF =
5431 MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
5432 buildVPlans(VPlan1&: *VPlan1, MinVF: VF, MaxVF: VF);
5433 LLVM_DEBUG(printPlans(dbgs()));
5434 return;
5435 }
5436
5437 // Compute the minimal bitwidths required for integer operations in the loop
5438 // for later use by the cost model.
5439 Config.computeMinimalBitwidths();
5440
5441 // Invalidate interleave groups if all blocks of loop will be predicated.
5442 if (CM->blockNeedsPredicationForAnyReason(BB: OrigLoop->getHeader()) &&
5443 !useMaskedInterleavedAccesses(TTI)) {
5444 LLVM_DEBUG(
5445 dbgs()
5446 << "LV: Invalidate all interleaved groups due to fold-tail by masking "
5447 "which requires masked-interleaved support.\n");
5448 if (CM->InterleaveInfo.invalidateGroups())
5449 // Invalidating interleave groups also requires invalidating all decisions
5450 // based on them, which includes widening decisions and uniform and scalar
5451 // values.
5452 CM->invalidateCostModelingDecisions();
5453 }
5454
5455 if (CM->foldTailByMasking())
5456 Legal->prepareToFoldTailByMasking();
5457
5458 ElementCount MaxUserVF =
5459 UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
5460 if (UserVF) {
5461 if (!ElementCount::isKnownLE(LHS: UserVF, RHS: MaxUserVF)) {
5462 reportVectorizationInfo(
5463 Msg: "UserVF ignored because it may be larger than the maximal safe VF",
5464 ORETag: "InvalidUserVF", ORE, TheLoop: OrigLoop);
5465 } else {
5466 assert(isPowerOf2_32(UserVF.getKnownMinValue()) &&
5467 "VF needs to be a power of two");
5468 // Collect the instructions (and their associated costs) that will be more
5469 // profitable to scalarize.
5470 CM->collectNonVectorizedAndSetWideningDecisions(VF: UserVF);
5471 buildVPlans(VPlan1&: *VPlan1, MinVF: UserVF, MaxVF: UserVF);
5472 ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
5473 if (EpilogueUserVF.isVector() &&
5474 ElementCount::isKnownLT(LHS: EpilogueUserVF, RHS: UserVF)) {
5475 CM->collectNonVectorizedAndSetWideningDecisions(VF: EpilogueUserVF);
5476 buildVPlans(VPlan1&: *VPlan1, MinVF: EpilogueUserVF, MaxVF: EpilogueUserVF);
5477 }
5478 if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
5479 // For scalar VF, skip VPlan cost check as VPlan cost is designed for
5480 // vector VFs only.
5481 if (UserVF.isScalar() ||
5482 cost(Plan&: *VPlans.front(), VF: UserVF, /*RU=*/nullptr).isValid()) {
5483 LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
5484 LLVM_DEBUG(printPlans(dbgs()));
5485 return;
5486 }
5487 }
5488 VPlans.clear();
5489 reportVectorizationInfo(Msg: "UserVF ignored because of invalid costs.",
5490 ORETag: "InvalidCost", ORE, TheLoop: OrigLoop);
5491 }
5492 }
5493
5494 // Collect the Vectorization Factor Candidates.
5495 SmallVector<ElementCount> VFCandidates;
5496 for (auto VF = ElementCount::getFixed(MinVal: 1);
5497 ElementCount::isKnownLE(LHS: VF, RHS: MaxFactors.FixedVF); VF *= 2)
5498 VFCandidates.push_back(Elt: VF);
5499 for (auto VF = ElementCount::getScalable(MinVal: 1);
5500 ElementCount::isKnownLE(LHS: VF, RHS: MaxFactors.ScalableVF); VF *= 2)
5501 VFCandidates.push_back(Elt: VF);
5502
5503 for (const auto &VF : VFCandidates) {
5504 // Collect Uniform and Scalar instructions after vectorization with VF.
5505 CM->collectNonVectorizedAndSetWideningDecisions(VF);
5506 }
5507
5508 buildVPlans(VPlan1&: *VPlan1, MinVF: ElementCount::getFixed(MinVal: 1), MaxVF: MaxFactors.FixedVF);
5509 buildVPlans(VPlan1&: *VPlan1, MinVF: ElementCount::getScalable(MinVal: 1), MaxVF: MaxFactors.ScalableVF);
5510
5511 LLVM_DEBUG(printPlans(dbgs()));
5512}
5513
5514VPCostContext::VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan,
5515 LoopVectorizationCostModel &CM,
5516 VFSelectionContext &Config,
5517 bool ReusePrintingSlotTracker)
5518 : TTI(Config.getTTI()), TLI(TLI), LLVMCtx(Plan.getContext()), CM(CM),
5519 Config(Config), CostKind(Config.CostKind), PSE(Config.getPSE()),
5520 L(Config.getLoop()) {
5521#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5522 if (ReusePrintingSlotTracker)
5523 PlanForSlotTracker = &Plan;
5524#endif
5525}
5526
5527InstructionCost VPCostContext::getLegacyCost(Instruction *UI,
5528 ElementCount VF) const {
5529 InstructionCost Cost = CM.getInstructionCost(I: UI, VF);
5530 if (Cost.isValid() && ForceTargetInstructionCost.getNumOccurrences())
5531 return InstructionCost(ForceTargetInstructionCost);
5532 return Cost;
5533}
5534
5535bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const {
5536 return CM.ValuesToIgnore.contains(Ptr: UI) ||
5537 (IsVector && CM.VecValuesToIgnore.contains(Ptr: UI)) ||
5538 SkipCostComputation.contains(Ptr: UI);
5539}
5540
5541void VPCostContext::invalidateWideningDecision(Instruction *I,
5542 ElementCount VF) {
5543 CM.setWideningDecision(I, VF,
5544 W: LoopVectorizationCostModel::CM_InvalidatedDecision, Cost: 0);
5545}
5546
5547uint64_t VPCostContext::getPredBlockCostDivisor(BasicBlock *BB) const {
5548 return CM.getPredBlockCostDivisor(CostKind, BB);
5549}
5550
5551bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const {
5552 return CM.isScalarWithPredication(I, VF) ||
5553 CM.isUniformAfterVectorization(I, VF) || CM.isForcedScalar(I, VF) ||
5554 (VF.isVector() && CM.isProfitableToScalarize(I, VF));
5555}
5556
5557bool VPCostContext::isMaskRequired(Instruction *I) const {
5558 return CM.isMaskRequired(I);
5559}
5560
5561InstructionCost
5562LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
5563 VPCostContext &CostCtx) const {
5564 InstructionCost Cost;
5565 // Cost modeling for inductions is inaccurate in the legacy cost model
5566 // compared to the recipes that are generated. To match here initially during
5567 // VPlan cost model bring up directly use the induction costs from the legacy
5568 // cost model. Note that we do this as pre-processing; the VPlan may not have
5569 // any recipes associated with the original induction increment instruction
5570 // and may replace truncates with VPWidenIntOrFpInductionRecipe. We precompute
5571 // the cost of induction phis and increments (both that are represented by
5572 // recipes and those that are not), to avoid distinguishing between them here,
5573 // and skip all recipes that represent induction phis and increments (the
5574 // former case) later on, if they exist, to avoid counting them twice.
5575 // Similarly we pre-compute the cost of any optimized truncates.
5576 // Inductions that are represented by a VPWidenIntOrFpInductionRecipe are an
5577 // exception: their cost is computed by the recipe's computeCost (see below),
5578 // so they are not precomputed here.
5579 // TODO: Switch to more accurate costing based on VPlan.
5580
5581 // If the vector loop gets executed exactly once with the given VF, ignore the
5582 // costs of comparison and induction instructions, as they'll get simplified
5583 // away.
5584 // TODO: Remove this code after stepping away from the legacy cost model and
5585 // adding code to simplify VPlans before calculating their costs.
5586 auto TC = getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop);
5587 SmallPtrSet<const Value *, 4> WidenedIVs;
5588 if (TC == VF && !Plan.hasTailFolded()) {
5589 addFullyUnrolledInstructionsToIgnore(L: OrigLoop, IL: Legal->getInductionVars(),
5590 InstsToIgnore&: CostCtx.SkipCostComputation);
5591 } else {
5592 // Inductions represented by a VPWidenIntOrFpInductionRecipe have their cost
5593 // computed by the recipe, so collect their phis to skip the legacy
5594 // increment cost below.
5595 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
5596 for (VPRecipeBase &R : *LoopRegion->getEntryBasicBlock())
5597 if (auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: &R)) {
5598 if (PHINode *IVPhi = WideIV->getPHINode())
5599 WidenedIVs.insert(Ptr: IVPhi);
5600 }
5601 }
5602
5603 for (const auto &[IV, IndDesc] : Legal->getInductionVars()) {
5604 if (WidenedIVs.contains(Ptr: IV))
5605 continue;
5606 Instruction *IVInc = cast<Instruction>(
5607 Val: IV->getIncomingValueForBlock(BB: OrigLoop->getLoopLatch()));
5608 SmallVector<Instruction *> IVInsts = {IVInc};
5609 for (unsigned I = 0; I != IVInsts.size(); I++) {
5610 for (Value *Op : IVInsts[I]->operands()) {
5611 auto *OpI = dyn_cast<Instruction>(Val: Op);
5612 if (Op == IV || !OpI || !OrigLoop->contains(Inst: OpI) || !Op->hasOneUse())
5613 continue;
5614 IVInsts.push_back(Elt: OpI);
5615 }
5616 }
5617 IVInsts.push_back(Elt: IV);
5618 for (User *U : IV->users()) {
5619 auto *CI = cast<Instruction>(Val: U);
5620 if (!CostCtx.CM.isOptimizableIVTruncate(I: CI, VF))
5621 continue;
5622 IVInsts.push_back(Elt: CI);
5623 }
5624
5625 for (Instruction *IVInst : IVInsts) {
5626 if (CostCtx.skipCostComputation(UI: IVInst, IsVector: VF.isVector()))
5627 continue;
5628 InstructionCost InductionCost = CostCtx.getLegacyCost(UI: IVInst, VF);
5629 LLVM_DEBUG({
5630 dbgs() << "Cost of " << InductionCost << " for VF " << VF
5631 << ": induction instruction " << *IVInst << "\n";
5632 });
5633 Cost += InductionCost;
5634 CostCtx.SkipCostComputation.insert(Ptr: IVInst);
5635 }
5636 }
5637
5638 // Pre-compute the costs for branches except for the backedge, as the number
5639 // of replicate regions in a VPlan may not directly match the number of
5640 // branches, which would lead to different decisions.
5641 // TODO: Compute cost of branches for each replicate region in the VPlan,
5642 // which is more accurate than the legacy cost model.
5643 for (BasicBlock *BB : OrigLoop->blocks()) {
5644 if (CostCtx.skipCostComputation(UI: BB->getTerminator(), IsVector: VF.isVector()))
5645 continue;
5646 CostCtx.SkipCostComputation.insert(Ptr: BB->getTerminator());
5647 if (BB == OrigLoop->getLoopLatch())
5648 continue;
5649 auto BranchCost = CostCtx.getLegacyCost(UI: BB->getTerminator(), VF);
5650 Cost += BranchCost;
5651 }
5652
5653 // Don't apply special costs when instruction cost is forced to make sure the
5654 // forced cost is used for each recipe.
5655 if (ForceTargetInstructionCost.getNumOccurrences())
5656 return Cost;
5657
5658 // Pre-compute costs for instructions that are forced-scalar or profitable to
5659 // scalarize. For most such instructions, their scalarization costs are
5660 // accounted for here using the legacy cost model. However, some opcodes
5661 // are excluded from these precomputed scalarization costs and are instead
5662 // modeled later by the VPlan cost model (see UseVPlanCostModel below).
5663 for (Instruction *ForcedScalar : CostCtx.CM.ForcedScalars[VF]) {
5664 if (CostCtx.skipCostComputation(UI: ForcedScalar, IsVector: VF.isVector()))
5665 continue;
5666 CostCtx.SkipCostComputation.insert(Ptr: ForcedScalar);
5667 InstructionCost ForcedCost = CostCtx.getLegacyCost(UI: ForcedScalar, VF);
5668 LLVM_DEBUG({
5669 dbgs() << "Cost of " << ForcedCost << " for VF " << VF
5670 << ": forced scalar " << *ForcedScalar << "\n";
5671 });
5672 Cost += ForcedCost;
5673 }
5674
5675 // Don't apply legacy scalarization costs if nothing remains scalar &
5676 // predicated.
5677 if (!hasReplicatorRegion(Plan))
5678 return Cost;
5679
5680 auto UseVPlanCostModel = [](Instruction *I) -> bool {
5681 switch (I->getOpcode()) {
5682 case Instruction::SDiv:
5683 case Instruction::UDiv:
5684 case Instruction::SRem:
5685 case Instruction::URem:
5686 return true;
5687 default:
5688 return false;
5689 }
5690 };
5691 for (const auto &[Scalarized, ScalarCost] : CostCtx.CM.InstsToScalarize[VF]) {
5692 if (UseVPlanCostModel(Scalarized) ||
5693 CostCtx.skipCostComputation(UI: Scalarized, IsVector: VF.isVector()))
5694 continue;
5695 CostCtx.SkipCostComputation.insert(Ptr: Scalarized);
5696 LLVM_DEBUG({
5697 dbgs() << "Cost of " << ScalarCost << " for VF " << VF
5698 << ": profitable to scalarize " << *Scalarized << "\n";
5699 });
5700 Cost += ScalarCost;
5701 }
5702
5703 return Cost;
5704}
5705
5706InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
5707 VPRegisterUsage *RU) const {
5708 VPCostContext CostCtx(*TLI, Plan, *CM, Config,
5709 /*ReusePrintingSlotTracker=*/true);
5710 InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
5711
5712 // Now compute and add the VPlan-based cost.
5713 Cost += Plan.cost(VF, Ctx&: CostCtx);
5714
5715 // Add the cost of spills due to excess register usage
5716 if (RU && Config.shouldConsiderRegPressureForVF(VF))
5717 Cost += RU->spillCost(TTI, CostKind: Config.CostKind, OverrideMaxNumRegs: ForceTargetNumVectorRegs);
5718
5719#ifndef NDEBUG
5720 unsigned EstimatedWidth =
5721 estimateElementCount(VF, Config.getVScaleForTuning());
5722 LLVM_DEBUG(dbgs() << "Cost for VF " << VF << ": " << Cost
5723 << " (Estimated cost per lane: ");
5724 if (Cost.isValid()) {
5725 APFloat CostPerLane(APFloat::IEEEdouble());
5726 APFloat EstimatedWidthAsAPFloat(APFloat::IEEEdouble());
5727 (void)CostPerLane.convertFromAPInt(APInt(64, (uint64_t)Cost.getValue()),
5728 false, APFloat::rmTowardZero);
5729 (void)EstimatedWidthAsAPFloat.convertFromAPInt(
5730 APInt(64, (uint64_t)EstimatedWidth), false, APFloat::rmTowardZero);
5731 (void)CostPerLane.divide(EstimatedWidthAsAPFloat, APFloat::rmTowardZero);
5732
5733 SmallString<16> Str;
5734 CostPerLane.toString(Str, 3);
5735 LLVM_DEBUG(dbgs() << Str);
5736 } else /* No point dividing an invalid cost - it will still be invalid */
5737 LLVM_DEBUG(dbgs() << "Invalid");
5738 LLVM_DEBUG(dbgs() << ")\n");
5739#endif
5740 return Cost;
5741}
5742
5743std::pair<VectorizationFactor, VPlan *>
5744LoopVectorizationPlanner::computeBestVF() {
5745 if (VPlans.empty())
5746 return {VectorizationFactor::Disabled(), nullptr};
5747 // If there is a single VPlan with a single VF, return it directly.
5748 VPlan &FirstPlan = *VPlans[0];
5749
5750 ElementCount UserVF = Config.getHints().getWidth();
5751 if (VPlans.size() == 1) {
5752 // For outer loops, the plan has a single vector VF determined by the
5753 // heuristic.
5754 assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
5755 FirstPlan.isOuterLoop()) &&
5756 "must have a single scalar VF, UserVF or an outer loop");
5757 return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
5758 }
5759
5760 if (hasPlanWithVF(VF: UserVF) && hasForcedEpilogueVF() && VPlans.size() == 2) {
5761 assert(VPlans[0]->getSingleVF() == UserVF &&
5762 "expected second plan to be for the forced UserVF");
5763 assert(VPlans[1]->getSingleVF() == EpilogueVectorizationForceVF &&
5764 "expected first plan to be for the forced epilogue VF");
5765 return {VectorizationFactor(UserVF, 0, 0), VPlans[0].get()};
5766 }
5767
5768 LLVM_DEBUG(dbgs() << "LV: Computing best VF using cost kind: "
5769 << (Config.CostKind == TTI::TCK_RecipThroughput
5770 ? "Reciprocal Throughput\n"
5771 : Config.CostKind == TTI::TCK_Latency
5772 ? "Instruction Latency\n"
5773 : Config.CostKind == TTI::TCK_CodeSize ? "Code Size\n"
5774 : Config.CostKind == TTI::TCK_SizeAndLatency
5775 ? "Code Size and Latency\n"
5776 : "Unknown\n"));
5777
5778 ElementCount ScalarVF = ElementCount::getFixed(MinVal: 1);
5779 assert(FirstPlan.hasVF(ScalarVF) &&
5780 "More than a single plan/VF w/o any plan having scalar VF");
5781
5782 // TODO: Compute scalar cost using VPlan-based cost model.
5783 InstructionCost ScalarCost = CM->expectedCost(VF: ScalarVF);
5784 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
5785 VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
5786 VectorizationFactor BestFactor = ScalarFactor;
5787
5788 bool ForceVectorization =
5789 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
5790 if (ForceVectorization) {
5791 // Ignore scalar width, because the user explicitly wants vectorization.
5792 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
5793 // evaluation.
5794 BestFactor.Cost = InstructionCost::getMax();
5795 }
5796
5797 VPlan *PlanForBestVF = &FirstPlan;
5798
5799 for (auto &P : VPlans) {
5800 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
5801 P->vectorFactors().end());
5802
5803 SmallVector<VPRegisterUsage, 8> RUs;
5804 bool ConsiderRegPressure = any_of(Range&: VFs, P: [this](ElementCount VF) {
5805 return Config.shouldConsiderRegPressureForVF(VF);
5806 });
5807 if (ConsiderRegPressure)
5808 RUs = calculateRegisterUsageForPlan(Plan&: *P, VFs, TTI);
5809
5810 for (unsigned I = 0; I < VFs.size(); I++) {
5811 ElementCount VF = VFs[I];
5812 if (VF.isScalar())
5813 continue;
5814 if (!ForceVectorization && !willGenerateVectors(Plan&: *P, VF, TTI)) {
5815 LLVM_DEBUG(
5816 dbgs()
5817 << "LV: Not considering vector loop of width " << VF
5818 << " because it will not generate any vector instructions.\n");
5819 continue;
5820 }
5821 if (Config.OptForSize && !ForceVectorization && hasReplicatorRegion(Plan&: *P)) {
5822 LLVM_DEBUG(
5823 dbgs()
5824 << "LV: Not considering vector loop of width " << VF
5825 << " because it would cause replicated blocks to be generated,"
5826 << " which isn't allowed when optimizing for size.\n");
5827 continue;
5828 }
5829
5830 InstructionCost Cost =
5831 cost(Plan&: *P, VF, RU: ConsiderRegPressure ? &RUs[I] : nullptr);
5832 VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
5833
5834 if (isMoreProfitable(A: CurrentFactor, B: BestFactor, HasTail: P->hasScalarTail())) {
5835 BestFactor = CurrentFactor;
5836 PlanForBestVF = P.get();
5837 }
5838
5839 // If profitable add it to ProfitableVF list.
5840 if (isMoreProfitable(A: CurrentFactor, B: ScalarFactor, HasTail: P->hasScalarTail()))
5841 ProfitableVFs.push_back(Elt: CurrentFactor);
5842 }
5843 }
5844
5845 VPlan &BestPlan = *PlanForBestVF;
5846
5847 assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
5848 "when vectorizing, the scalar cost must be computed.");
5849
5850 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << BestFactor.Width << ".\n");
5851 return {BestFactor, &BestPlan};
5852}
5853
5854LoopVectorizationPlanner::LoopVectorizationPlanner(
5855 Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
5856 const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
5857 std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
5858 InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
5859 OptimizationRemarkEmitter *ORE)
5860 : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
5861 CM(std::move(CM)), Config(Config), IAI(IAI), PSE(PSE), ORE(ORE) {}
5862
5863LoopVectorizationPlanner::~LoopVectorizationPlanner() = default;
5864
5865void LoopVectorizationPlanner::clearCostModel() { CM.reset(); }
5866
5867DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
5868 ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
5869 InnerLoopVectorizer &ILV, DominatorTree *DT,
5870 EpilogueVectorizationKind EpilogueVecKind) {
5871 assert(BestVPlan.hasVF(BestVF) &&
5872 "Trying to execute plan with unsupported VF");
5873 assert(BestVPlan.hasUF(BestUF) &&
5874 "Trying to execute plan with unsupported UF");
5875 if (BestVPlan.hasEarlyExit())
5876 ++LoopsEarlyExitVectorized;
5877
5878 RUN_VPLAN_PASS(VPlanTransforms::replaceWideCanonicalIVWithWideIV, BestVPlan,
5879 *PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF);
5880 // TODO: Move to VPlan transform stage once the transition to the VPlan-based
5881 // cost model is complete for better cost estimates.
5882 RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
5883 RUN_VPLAN_PASS(VPlanTransforms::materializePacksAndUnpacks, BestVPlan);
5884 RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
5885 RUN_VPLAN_PASS(VPlanTransforms::replicateByVF, BestVPlan, BestVF);
5886 bool HasBranchWeights =
5887 hasBranchWeightMD(I: *OrigLoop->getLoopLatch()->getTerminator());
5888 if (HasBranchWeights) {
5889 std::optional<unsigned> VScale = Config.getVScaleForTuning();
5890 RUN_VPLAN_PASS(VPlanTransforms::addBranchWeightToMiddleTerminator,
5891 BestVPlan, BestVF, VScale);
5892 }
5893
5894 if (vputils::findIncomingAliasMask(Plan: BestVPlan)) {
5895 assert(BestVPlan.hasTailFolded() && "Expected tail folding to be enabled");
5896 RUN_VPLAN_PASS(VPlanTransforms::materializeAliasMaskCheckBlock, BestVPlan,
5897 *Legal->getRuntimePointerChecking()->getDiffChecks(),
5898 HasBranchWeights);
5899 ++LoopsPartialAliasVectorized;
5900 }
5901
5902 // Retrieving VectorPH now when it's easier while VPlan still has Regions.
5903 VPBasicBlock *VectorPH = cast<VPBasicBlock>(Val: BestVPlan.getVectorPreheader());
5904
5905 RUN_VPLAN_PASS(VPlanTransforms::materializeConstantVectorTripCount, BestVPlan,
5906 BestVF, BestUF, PSE);
5907 RUN_VPLAN_PASS(VPlanTransforms::optimizeForVFAndUF, BestVPlan, BestVF, BestUF,
5908 PSE);
5909 RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, BestVPlan);
5910 // Check if scalar epilogue is required, before simplifying constant branches.
5911 const bool RequiresScalarEpilogue = BestVPlan.requiresScalarEpilogue();
5912 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5913 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5914 /*OnlyLatches=*/false);
5915 if (BestVPlan.getEntry()->getSingleSuccessor() ==
5916 BestVPlan.getScalarPreheader()) {
5917 // TODO: The vector loop would be dead, should not even try to vectorize.
5918 ORE->emit(RemarkBuilder: [&]() {
5919 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationDead",
5920 OrigLoop->getStartLoc(),
5921 OrigLoop->getHeader())
5922 << "Created vector loop never executes due to insufficient trip "
5923 "count.";
5924 });
5925 return DenseMap<const SCEV *, Value *>();
5926 }
5927
5928 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, BestVPlan);
5929
5930 RUN_VPLAN_PASS(VPlanTransforms::convertToConcreteRecipes, BestVPlan);
5931 // Convert the exit condition to AVLNext == 0 for EVL tail folded loops.
5932 RUN_VPLAN_PASS(VPlanTransforms::convertEVLExitCond, BestVPlan);
5933 // Regions are dissolved after optimizing for VF and UF, which completely
5934 // removes unneeded loop regions first.
5935 const bool HasTailFolded = BestVPlan.hasTailFolded();
5936 RUN_VPLAN_PASS(VPlanTransforms::dissolveLoopRegions, BestVPlan);
5937 // Expand BranchOnTwoConds after dissolution, when latch has direct access to
5938 // its successors.
5939 RUN_VPLAN_PASS(VPlanTransforms::expandBranchOnTwoConds, BestVPlan);
5940 // Convert loops with variable-length stepping after regions are dissolved.
5941 RUN_VPLAN_PASS(VPlanTransforms::convertToVariableLengthStep, BestVPlan);
5942 // Remove dead back-edges for single-iteration loops with BranchOnCond(true).
5943 // Only process loop latches to avoid removing edges from the middle block,
5944 // which may be needed for epilogue vectorization.
5945 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5946 /*OnlyLatches=*/true);
5947 RUN_VPLAN_PASS(VPlanTransforms::materializeBackedgeTakenCount, BestVPlan,
5948 VectorPH);
5949 std::optional<uint64_t> MaxRuntimeStep;
5950 if (auto MaxVScale = getMaxVScale(F: *OrigLoop->getHeader()->getParent(), TTI))
5951 MaxRuntimeStep = uint64_t(*MaxVScale) * BestVF.getKnownMinValue() * BestUF;
5952 assert((LI->getUniqueLatchExitBlock(*OrigLoop) || RequiresScalarEpilogue) &&
5953 "loops not exiting via the latch without required epilogue?");
5954 RUN_VPLAN_PASS(VPlanTransforms::materializeVectorTripCount, BestVPlan,
5955 VectorPH, HasTailFolded, RequiresScalarEpilogue,
5956 &BestVPlan.getVFxUF(), MaxRuntimeStep);
5957 RUN_VPLAN_PASS(VPlanTransforms::materializeFactors, BestVPlan, VectorPH,
5958 BestVF);
5959 // Limit expansions to VPInstruction to when not vectorizing the epilogue.
5960 // Currently this code path still relies on code re-using SCEVs expanded
5961 // directly to IR instructions.
5962 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5963 RUN_VPLAN_PASS(VPlanTransforms::expandSCEVsToVPInstructions, BestVPlan,
5964 *PSE.getSE());
5965 RUN_VPLAN_PASS(VPlanTransforms::cse, BestVPlan);
5966 RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, BestVPlan);
5967 // Removing branches and incoming values may expose additional simplification
5968 // opportunities.
5969 if (RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5970 /*OnlyLatches=*/EpilogueVecKind !=
5971 EpilogueVectorizationKind::None))
5972 RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, BestVPlan);
5973 RUN_VPLAN_PASS(VPlanTransforms::simplifyKnownEVL, BestVPlan, BestVF, PSE);
5974
5975 // 0. Generate SCEV-dependent code in the entry, including TripCount, before
5976 // making any changes to the CFG.
5977 DenseMap<const SCEV *, Value *> ExpandedSCEVs =
5978 RUN_VPLAN_PASS(VPlanTransforms::expandSCEVs, BestVPlan, *PSE.getSE());
5979
5980 // Perform the actual loop transformation.
5981 VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
5982 OrigLoop->getParentLoop());
5983
5984#ifdef EXPENSIVE_CHECKS
5985 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
5986#endif
5987
5988 // 1. Set up the skeleton for vectorization, including vector pre-header and
5989 // middle block. The vector loop is created during VPlan execution.
5990 State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
5991 if (VPBasicBlock *ScalarPH = BestVPlan.getScalarPreheader())
5992 replaceVPBBWithIRVPBB(VPBB: ScalarPH, IRBB: State.CFG.PrevBB->getSingleSuccessor(),
5993 Plan: &BestVPlan);
5994 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, BestVPlan);
5995
5996 assert(verifyVPlanIsValid(BestVPlan) && "final VPlan is invalid");
5997
5998 // After vectorization, the exit blocks of the original loop will have
5999 // additional predecessors. Invalidate SCEVs for the exit phis in case SE
6000 // looked through single-entry phis.
6001 ScalarEvolution &SE = *PSE.getSE();
6002 for (VPIRBasicBlock *Exit : BestVPlan.getExitBlocks()) {
6003 if (!Exit->hasPredecessors())
6004 continue;
6005 for (VPRecipeBase &PhiR : Exit->phis())
6006 SE.forgetLcssaPhiWithNewPredecessor(L: OrigLoop,
6007 V: &cast<VPIRPhi>(Val&: PhiR).getIRPhi());
6008 }
6009
6010 // Query whether the target wants loops it vectorizes to remain eligible for
6011 // runtime unrolling. Do this here, on the original loop and before its SCEV
6012 // is forgotten below.
6013 TargetTransformInfo::UnrollingPreferences UP;
6014 TTI.getUnrollingPreferences(L: OrigLoop, SE, UP, ORE);
6015 bool UnrollVectorizedLoop = UP.UnrollVectorizedLoop;
6016
6017 // Forget the original loop and block dispositions.
6018 SE.forgetLoop(L: OrigLoop);
6019 SE.forgetBlockAndLoopDispositions();
6020
6021 ILV.printDebugTracesAtStart();
6022
6023 //===------------------------------------------------===//
6024 //
6025 // Notice: any optimization or new instruction that go
6026 // into the code below should also be implemented in
6027 // the cost-model.
6028 //
6029 //===------------------------------------------------===//
6030
6031 // Retrieve loop information before executing the plan, which may remove the
6032 // original loop, if it becomes unreachable.
6033 MDNode *LID = OrigLoop->getLoopID();
6034 unsigned OrigLoopInvocationWeight = 0;
6035 std::optional<unsigned> OrigAverageTripCount =
6036 getLoopEstimatedTripCount(L: OrigLoop, EstimatedLoopInvocationWeight: &OrigLoopInvocationWeight);
6037
6038 BestVPlan.execute(State: &State);
6039
6040 // 2.6. Maintain Loop Hints
6041 // Keep all loop hints from the original loop on the vector loop (we'll
6042 // replace the vectorizer-specific hints below).
6043 VPBasicBlock *HeaderVPBB = vputils::getFirstLoopHeader(Plan&: BestVPlan, VPDT&: State.VPDT);
6044 // Add metadata to disable runtime unrolling a scalar loop when there
6045 // are no runtime checks about strides and memory. A scalar loop that is
6046 // rarely used is not worth unrolling.
6047 bool DisableRuntimeUnroll = !ILV.RTChecks.hasChecks() && !BestVF.isScalar();
6048 updateLoopMetadataAndProfileInfo(
6049 VectorLoop: HeaderVPBB ? LI->getLoopFor(BB: State.CFG.VPBB2IRBB.lookup(Val: HeaderVPBB))
6050 : nullptr,
6051 HeaderVPBB, Plan: BestVPlan,
6052 VectorizingEpilogue: EpilogueVecKind == EpilogueVectorizationKind::Epilogue, OrigLoopID: LID,
6053 OrigAverageTripCount, OrigLoopInvocationWeight,
6054 EstimatedVFxUF: estimateElementCount(VF: BestVF * BestUF, VScale: Config.getVScaleForTuning()),
6055 DisableRuntimeUnroll, UnrollVectorizedLoop);
6056
6057 // 3. Fix the vectorized code: take care of header phi's, live-outs,
6058 // predication, updating analyses.
6059 ILV.fixVectorizedLoop(State);
6060
6061 ILV.printDebugTracesAtEnd();
6062
6063 return ExpandedSCEVs;
6064}
6065
6066//===--------------------------------------------------------------------===//
6067// EpilogueVectorizerMainLoop
6068//===--------------------------------------------------------------------===//
6069
6070void EpilogueVectorizerMainLoop::printDebugTracesAtStart() {
6071 LLVM_DEBUG({
6072 dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
6073 << "Main Loop VF:" << EPI.MainLoopVF
6074 << ", Main Loop UF:" << EPI.MainLoopUF
6075 << ", Epilogue Loop VF:" << EPI.EpilogueVF
6076 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
6077 });
6078}
6079
6080void EpilogueVectorizerMainLoop::printDebugTracesAtEnd() {
6081 DEBUG_WITH_TYPE(VerboseDebug, {
6082 dbgs() << "intermediate fn:\n"
6083 << *OrigLoop->getHeader()->getParent() << "\n";
6084 });
6085}
6086
6087//===--------------------------------------------------------------------===//
6088// EpilogueVectorizerEpilogueLoop
6089//===--------------------------------------------------------------------===//
6090
6091/// This function creates a new scalar preheader, using the previous one as
6092/// entry block to the epilogue VPlan. The minimum iteration check is being
6093/// represented in VPlan.
6094BasicBlock *EpilogueVectorizerEpilogueLoop::createVectorizedLoopSkeleton() {
6095 BasicBlock *NewScalarPH = createScalarPreheader(Prefix: "vec.epilog.");
6096 BasicBlock *OriginalScalarPH = NewScalarPH->getSinglePredecessor();
6097 OriginalScalarPH->setName("vec.epilog.iter.check");
6098 VPIRBasicBlock *NewEntry = Plan.createVPIRBasicBlock(IRBB: OriginalScalarPH);
6099 VPBasicBlock *OldEntry = Plan.getEntry();
6100 for (auto &R : make_early_inc_range(Range&: *OldEntry)) {
6101 // Skip moving VPIRInstructions (including VPIRPhis), which are unmovable by
6102 // defining.
6103 if (isa<VPIRInstruction>(Val: &R))
6104 continue;
6105 R.moveBefore(BB&: *NewEntry, I: NewEntry->end());
6106 }
6107
6108 VPBlockUtils::reassociateBlocks(Old: OldEntry, New: NewEntry);
6109 Plan.setEntry(NewEntry);
6110 // OldEntry is now dead and will be cleaned up when the plan gets destroyed.
6111
6112 return OriginalScalarPH;
6113}
6114
6115void EpilogueVectorizerEpilogueLoop::printDebugTracesAtStart() {
6116 LLVM_DEBUG({
6117 dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
6118 << "Epilogue Loop VF:" << EPI.EpilogueVF
6119 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
6120 });
6121}
6122
6123void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
6124 DEBUG_WITH_TYPE(VerboseDebug, {
6125 dbgs() << "final fn:\n" << *OrigLoop->getHeader()->getParent() << "\n";
6126 });
6127}
6128
6129bool VPRecipeBuilder::isPredicatedInst(Instruction *I) const {
6130 return CM.isPredicatedInst(I);
6131}
6132
6133bool VPRecipeBuilder::prefersVectorizedAddressing() const {
6134 return CM.TTI.prefersVectorizedAddressing();
6135}
6136
6137VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
6138 VFRange &Range) {
6139 assert((VPI->getOpcode() == Instruction::Load ||
6140 VPI->getOpcode() == Instruction::Store) &&
6141 "Must be called with either a load or store");
6142 Instruction *I = VPI->getUnderlyingInstr();
6143
6144 auto WillWiden = [&](ElementCount VF) -> bool {
6145 LoopVectorizationCostModel::InstWidening Decision =
6146 CM.getWideningDecision(I, VF);
6147 assert(Decision != LoopVectorizationCostModel::CM_Unknown &&
6148 "CM decision should be taken at this point.");
6149 if (Decision == LoopVectorizationCostModel::CM_Interleave)
6150 return true;
6151 if (CM.isScalarAfterVectorization(I, VF) ||
6152 CM.isProfitableToScalarize(I, VF))
6153 return false;
6154 return Decision != LoopVectorizationCostModel::CM_Scalarize;
6155 };
6156
6157 if (!LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: WillWiden, Range))
6158 return nullptr;
6159
6160 // If a mask is not required, drop it - use unmasked version for safe loads.
6161 // TODO: Determine if mask is needed in VPlan.
6162 VPValue *Mask = CM.isMaskRequired(I) ? VPI->getMask() : nullptr;
6163
6164 // Determine if the pointer operand of the access is either consecutive or
6165 // reverse consecutive.
6166 LoopVectorizationCostModel::InstWidening Decision =
6167 CM.getWideningDecision(I, VF: Range.Start);
6168 bool Reverse = Decision == LoopVectorizationCostModel::CM_Widen_Reverse;
6169 bool Consecutive =
6170 Reverse || Decision == LoopVectorizationCostModel::CM_Widen;
6171
6172 VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(N: 0)
6173 : VPI->getOperand(N: 1);
6174 Builder.setInsertPoint(VPI);
6175 if (Consecutive) {
6176 Ptr = Builder.createConsecutiveVectorPointer(Ptr, SourceElementTy: getLoadStoreType(I),
6177 Reverse, DL: VPI->getDebugLoc());
6178 }
6179
6180 if (Reverse && Mask)
6181 Mask = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Mask, DL: I->getDebugLoc());
6182
6183 if (VPI->getOpcode() == Instruction::Load) {
6184 auto *Load = cast<LoadInst>(Val: I);
6185 auto *LoadR = Builder.createWidenLoad(Load&: *Load, Addr: Ptr, Mask, Consecutive, Metadata: *VPI,
6186 DL: Load->getDebugLoc());
6187 if (Reverse)
6188 return Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: LoadR,
6189 DL: LoadR->getDebugLoc());
6190 return LoadR;
6191 }
6192
6193 StoreInst *Store = cast<StoreInst>(Val: I);
6194 VPValue *StoredVal = VPI->getOperand(N: 0);
6195 if (Reverse)
6196 StoredVal = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: StoredVal,
6197 DL: Store->getDebugLoc());
6198 return Builder.createWidenStore(Store&: *Store, Addr: Ptr, StoredVal, Mask, Consecutive,
6199 Metadata: *VPI, DL: Store->getDebugLoc());
6200}
6201
6202VPWidenIntOrFpInductionRecipe *
6203VPRecipeBuilder::tryToOptimizeInductionTruncate(VPInstruction *VPI,
6204 VFRange &Range) {
6205 auto *I = cast<TruncInst>(Val: VPI->getUnderlyingInstr());
6206 // Optimize the special case where the source is a constant integer
6207 // induction variable. Notice that we can only optimize the 'trunc' case
6208 // because (a) FP conversions lose precision, (b) sext/zext may wrap, and
6209 // (c) other casts depend on pointer size.
6210
6211 // Determine whether \p K is a truncation based on an induction variable that
6212 // can be optimized.
6213 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
6214 Predicate: bind_front(Fn: &LoopVectorizationCostModel::isOptimizableIVTruncate, BindArgs&: CM,
6215 BindArgs&: I),
6216 Range))
6217 return nullptr;
6218
6219 auto *WidenIV = cast<VPWidenIntOrFpInductionRecipe>(
6220 Val: VPI->getOperand(N: 0)->getDefiningRecipe());
6221 PHINode *Phi = WidenIV->getPHINode();
6222 VPIRValue *Start = WidenIV->getStartValue();
6223 const InductionDescriptor &IndDesc = WidenIV->getInductionDescriptor();
6224
6225 // Wrap flags from the original induction do not apply to the truncated type,
6226 // so do not propagate them.
6227 VPIRFlags Flags = VPIRFlags::WrapFlagsTy(false, false);
6228 VPValue *Step =
6229 vputils::getOrCreateVPValueForSCEVExpr(Plan, Expr: IndDesc.getStep());
6230 return new VPWidenIntOrFpInductionRecipe(
6231 Phi, Start, Step, &Plan.getVF(), IndDesc, I, Flags, VPI->getDebugLoc());
6232}
6233
6234bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
6235 assert((!isa<UncondBrInst, CondBrInst, PHINode, LoadInst, StoreInst>(I)) &&
6236 "Instruction should have been handled earlier");
6237 // Instruction should be widened, unless it is scalar after vectorization,
6238 // scalarization is profitable or it is predicated.
6239 auto WillScalarize = [this, I](ElementCount VF) -> bool {
6240 return CM.isScalarAfterVectorization(I, VF) ||
6241 CM.isProfitableToScalarize(I, VF) ||
6242 CM.isScalarWithPredication(I, VF);
6243 };
6244 return !LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: WillScalarize,
6245 Range);
6246}
6247
6248VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
6249 auto *I = VPI->getUnderlyingInstr();
6250 switch (VPI->getOpcode()) {
6251 default:
6252 return nullptr;
6253 case Instruction::SDiv:
6254 case Instruction::UDiv:
6255 case Instruction::SRem:
6256 case Instruction::URem:
6257 // If not provably safe, use a masked intrinsic.
6258 if (CM.isPredicatedInst(I))
6259 return new VPWidenIntrinsicRecipe(
6260 getMaskedDivRemIntrinsic(Opcode: VPI->getOpcode()), VPI->operands(),
6261 I->getType(), {}, {}, VPI->getDebugLoc());
6262 [[fallthrough]];
6263 case Instruction::Add:
6264 case Instruction::And:
6265 case Instruction::AShr:
6266 case Instruction::FAdd:
6267 case Instruction::FCmp:
6268 case Instruction::FDiv:
6269 case Instruction::FMul:
6270 case Instruction::FNeg:
6271 case Instruction::FRem:
6272 case Instruction::FSub:
6273 case Instruction::ICmp:
6274 case Instruction::LShr:
6275 case Instruction::Mul:
6276 case Instruction::Or:
6277 case Instruction::Select:
6278 case Instruction::Shl:
6279 case Instruction::Sub:
6280 case Instruction::Xor:
6281 case Instruction::Freeze:
6282 return new VPWidenRecipe(*I, VPI->operandsWithoutMask(), *VPI, *VPI,
6283 VPI->getDebugLoc());
6284 case Instruction::ExtractValue: {
6285 SmallVector<VPValue *> NewOps(VPI->operandsWithoutMask());
6286 auto *EVI = cast<ExtractValueInst>(Val: I);
6287 assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
6288 unsigned Idx = EVI->getIndices()[0];
6289 NewOps.push_back(Elt: Plan.getConstantInt(BitWidth: 32, Val: Idx));
6290 return new VPWidenRecipe(*I, NewOps, *VPI, *VPI, VPI->getDebugLoc());
6291 }
6292 };
6293}
6294
6295VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
6296 if (VPI->getOpcode() != Instruction::Store)
6297 return nullptr;
6298
6299 auto HistInfo =
6300 Legal->getHistogramInfo(I: cast<StoreInst>(Val: VPI->getUnderlyingInstr()));
6301 if (!HistInfo)
6302 return nullptr;
6303
6304 const HistogramInfo *HI = *HistInfo;
6305 // FIXME: Support other operations.
6306 unsigned Opcode = HI->Update->getOpcode();
6307 assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
6308 "Histogram update operation must be an Add or Sub");
6309
6310 SmallVector<VPValue *, 3> HGramOps;
6311 // Bucket address.
6312 HGramOps.push_back(Elt: VPI->getOperand(N: 1));
6313 // Increment value.
6314 HGramOps.push_back(Elt: Plan.getOrAddLiveIn(V: HI->Update->getOperand(i: 1)));
6315
6316 // In case of predicated execution (due to tail-folding, or conditional
6317 // execution, or both), pass the relevant mask.
6318 if (CM.isMaskRequired(I: HI->Store))
6319 HGramOps.push_back(Elt: VPI->getMask());
6320
6321 return new VPHistogramRecipe(Opcode, HGramOps, cast<VPIRMetadata>(Val&: *VPI),
6322 VPI->getDebugLoc());
6323}
6324
6325bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
6326 VPInstruction *VPI, VPBuilder &FinalRedStoresBuilder) {
6327 StoreInst *SI;
6328 if ((SI = dyn_cast<StoreInst>(Val: VPI->getUnderlyingInstr())) &&
6329 Legal->isInvariantAddressOfReduction(V: SI->getPointerOperand())) {
6330 // Only create recipe for the final invariant store of the reduction.
6331 if (Legal->isInvariantStoreOfReduction(SI)) {
6332 VPValue *Val = VPI->getOperand(N: 0);
6333 VPValue *Addr = VPI->getOperand(N: 1);
6334 // We need to store the exiting value of the reduction, so use the blend
6335 // if tail folded.
6336 if (auto *Blend = VPlanPatternMatch::findUserOf<VPBlendRecipe>(V: Val))
6337 Val = Blend;
6338 [[maybe_unused]] auto *Rdx =
6339 VPlanPatternMatch::findUserOf<VPReductionPHIRecipe>(V: Val);
6340 assert((isa<VPIRValue>(Val) || !Rdx || Rdx->getBackedgeValue() == Val) &&
6341 "Store of reduction thats not the backedge value?");
6342 auto *Recipe = new VPReplicateRecipe(
6343 SI, {Val, Addr}, true /* IsUniform */, nullptr /*Mask*/, *VPI, *VPI,
6344 VPI->getDebugLoc());
6345 FinalRedStoresBuilder.insert(R: Recipe);
6346 }
6347 VPI->eraseFromParent();
6348 return true;
6349 }
6350
6351 return false;
6352}
6353
6354VPSingleDefRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
6355 VFRange &Range) {
6356 auto *I = VPI->getUnderlyingInstr();
6357 bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
6358 Predicate: [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
6359 Range);
6360
6361 bool IsPredicated = CM.isPredicatedInst(I);
6362
6363 // Even if the instruction is not marked as uniform, there are certain
6364 // intrinsic calls that can be effectively treated as such, so we check for
6365 // them here. Conservatively, we only do this for scalable vectors, since
6366 // for fixed-width VFs we can always fall back on full scalarization.
6367 if (!IsUniform && Range.Start.isScalable() && isa<IntrinsicInst>(Val: I)) {
6368 switch (cast<IntrinsicInst>(Val: I)->getIntrinsicID()) {
6369 case Intrinsic::assume:
6370 case Intrinsic::lifetime_start:
6371 case Intrinsic::lifetime_end:
6372 // For scalable vectors if one of the operands is variant then we still
6373 // want to mark as uniform, which will generate one instruction for just
6374 // the first lane of the vector. We can't scalarize the call in the same
6375 // way as for fixed-width vectors because we don't know how many lanes
6376 // there are.
6377 //
6378 // The reasons for doing it this way for scalable vectors are:
6379 // 1. For the assume intrinsic generating the instruction for the first
6380 // lane is still be better than not generating any at all. For
6381 // example, the input may be a splat across all lanes.
6382 // 2. For the lifetime start/end intrinsics the pointer operand only
6383 // does anything useful when the input comes from a stack object,
6384 // which suggests it should always be uniform. For non-stack objects
6385 // the effect is to poison the object, which still allows us to
6386 // remove the call.
6387 IsUniform = true;
6388 break;
6389 default:
6390 break;
6391 }
6392 }
6393 VPValue *BlockInMask = nullptr;
6394 if (!IsPredicated) {
6395 // Finalize the recipe for Instr, first if it is not predicated.
6396 LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n");
6397 } else {
6398 LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n");
6399 // Instructions marked for predication are replicated and a mask operand is
6400 // added initially. Masked replicate recipes will later be placed under an
6401 // if-then construct to prevent side-effects. Generate recipes to compute
6402 // the block mask for this region.
6403 BlockInMask = VPI->getMask();
6404 }
6405
6406 // Note that there is some custom logic to mark some intrinsics as uniform
6407 // manually above for scalable vectors, which this assert needs to account for
6408 // as well.
6409 assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
6410 (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
6411 "Should not predicate a uniform recipe");
6412 if (IsUniform) {
6413 return VPBuilder::createSingleScalarOp(
6414 Opcode: VPI->getOpcode(), Operands: VPI->operandsWithoutMask(), Mask: BlockInMask, Flags: *VPI, Metadata: *VPI,
6415 DL: VPI->getDebugLoc(), UV: I);
6416 }
6417 auto *Recipe = new VPReplicateRecipe(I, VPI->operandsWithoutMask(),
6418 /*IsSingleScalar=*/false, BlockInMask,
6419 *VPI, *VPI, VPI->getDebugLoc());
6420 return Recipe;
6421}
6422
6423VPRecipeBase *
6424VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
6425 VFRange &Range) {
6426 assert(!R->isPhi() && "phis must be handled earlier");
6427 // First, check for specific widening recipes that deal with optimizing
6428 // truncates and memory operations.
6429 auto *VPI = cast<VPInstruction>(Val: R);
6430 assert(VPI->getOpcode() != Instruction::Call &&
6431 "Call should have been handled by makeCallWideningDecisions");
6432
6433 VPRecipeBase *Recipe;
6434 if (VPI->getOpcode() == Instruction::Trunc &&
6435 (Recipe = tryToOptimizeInductionTruncate(VPI, Range)))
6436 return Recipe;
6437
6438 // All widen recipes below deal only with VF > 1.
6439 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6440 Predicate: [&](ElementCount VF) { return VF.isScalar(); }, Range))
6441 return nullptr;
6442
6443 Instruction *Instr = R->getUnderlyingInstr();
6444 assert(!is_contained({Instruction::Load, Instruction::Store},
6445 VPI->getOpcode()) &&
6446 "Should have been handled prior to this!");
6447
6448 if (!shouldWiden(I: Instr, Range))
6449 return nullptr;
6450
6451 if (VPI->getOpcode() == Instruction::GetElementPtr) {
6452 auto *GEP = cast<GetElementPtrInst>(Val: Instr);
6453 return new VPWidenGEPRecipe(GEP->getSourceElementType(),
6454 VPI->operandsWithoutMask(), *VPI,
6455 VPI->getDebugLoc(), GEP);
6456 }
6457
6458 if (Instruction::isCast(Opcode: VPI->getOpcode())) {
6459 auto *CI = cast<CastInst>(Val: Instr);
6460 auto *CastR = cast<VPInstructionWithType>(Val: VPI);
6461 return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(N: 0),
6462 CastR->getResultType(), CI, *VPI, *VPI,
6463 VPI->getDebugLoc());
6464 }
6465
6466 return tryToWiden(VPI);
6467}
6468
6469// To allow RUN_VPLAN_PASS to print the VPlan after VF/UF independent
6470// optimizations.
6471static void printOptimizedVPlan(VPlan &) {}
6472
6473VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
6474 bool IsInnerLoop = OrigLoop->isInnermost();
6475
6476 // Set up loop versioning for inner loops with memory runtime checks.
6477 // Outer loops don't have LoopAccessInfo since canVectorizeMemory() is not
6478 // called for them.
6479 std::optional<LoopVersioning> LVer;
6480 if (IsInnerLoop) {
6481 const LoopAccessInfo *LAI = Legal->getLAI();
6482 LVer.emplace(args: *LAI, args: LAI->getRuntimePointerChecking()->getChecks(), args&: OrigLoop,
6483 args&: LI, args&: DT, args: PSE.getSE());
6484 if (!LAI->getRuntimePointerChecking()->getChecks().empty() &&
6485 !LAI->getRuntimePointerChecking()->getDiffChecks()) {
6486 // Only use noalias metadata when using memory checks guaranteeing no
6487 // overlap across all iterations.
6488 LVer->prepareNoAliasMetadata();
6489 }
6490 }
6491
6492 // Create initial base VPlan0, to serve as common starting point for all
6493 // candidates built later for specific VF ranges.
6494 auto VPlan0 = VPlanTransforms::buildVPlan0(TheLoop: OrigLoop, LI&: *LI,
6495 InductionTy: Legal->getWidestInductionType(),
6496 PSE, LVer: LVer ? &*LVer : nullptr);
6497
6498 VPDominatorTree VPDT(*VPlan0);
6499 if (const LoopAccessInfo *LAI = Legal->getLAI())
6500 RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
6501 LAI->getSymbolicStrides(), VPDT);
6502 RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);
6503 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
6504
6505 // Create recipes for header phis. For outer loops, reductions, recurrences
6506 // and in-loop reductions are empty since legality doesn't detect them.
6507 if (!RUN_VPLAN_PASS(
6508 VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE, *OrigLoop,
6509 VPDT, Legal->getInductionVars(), Legal->getReductionVars(),
6510 Legal->getFixedOrderRecurrences(), Config.getInLoopReductions(),
6511 Config.getHints().allowReordering())) {
6512 return nullptr;
6513 }
6514
6515 if (const LoopAccessInfo *LAI = Legal->getLAI())
6516 RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
6517 LAI->getSymbolicStrides(), VPDT);
6518
6519 // Add surviving induction predicates to PSE and check constraints.
6520 bool ForceVectorization =
6521 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
6522 bool OptForSize =
6523 !ForceVectorization &&
6524 (CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
6525 CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
6526 unsigned SCEVCheckThreshold = ForceVectorization
6527 ? PragmaVectorizeSCEVCheckThreshold
6528 : VectorizeSCEVCheckThreshold;
6529 if (!RUN_VPLAN_PASS(VPlanTransforms::finalizeSCEVPredicates, *VPlan0, PSE,
6530 OptForSize, SCEVCheckThreshold, ORE, OrigLoop))
6531 return nullptr;
6532
6533 RUN_VPLAN_PASS(VPlanTransforms::addMiddleCheck, *VPlan0);
6534
6535 // If we're vectorizing a loop with an uncountable exit, make sure that the
6536 // recipes are safe to handle.
6537 // TODO: Remove this once we can properly check the VPlan itself for both
6538 // the presence of an uncountable exit and the presence of stores in
6539 // the loop inside handleUncountableEarlyExits itself.
6540 if (Legal->hasUncountableEarlyExit()) {
6541 // TODO: Check target preference for style.
6542 UncountableExitStyle EEStyle =
6543 Legal->hasUncountableExitWithSideEffects()
6544 ? UncountableExitStyle::MaskedHandleExitInScalarLoop
6545 : UncountableExitStyle::ReadOnly;
6546 if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0,
6547 OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
6548 EEStyle))
6549 return nullptr;
6550 } else {
6551 RUN_VPLAN_PASS(VPlanTransforms::handleCountableEarlyExits, *VPlan0);
6552 }
6553
6554 RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
6555 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
6556 if (CM->foldTailByMasking())
6557 RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
6558 RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
6559
6560 return VPlan0;
6561}
6562
6563void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
6564 ElementCount MaxVF) {
6565 if (ElementCount::isKnownGT(LHS: MinVF, RHS: MaxVF))
6566 return;
6567
6568 auto MaxVFTimes2 = MaxVF * 2;
6569 for (ElementCount VF = MinVF; ElementCount::isKnownLT(LHS: VF, RHS: MaxVFTimes2);) {
6570 VFRange SubRange = {VF, MaxVFTimes2};
6571 auto Plan =
6572 tryToBuildVPlan(InitialPlan: std::unique_ptr<VPlan>(VPlan1.duplicate()), Range&: SubRange);
6573 VF = SubRange.End;
6574
6575 if (!Plan)
6576 continue;
6577
6578 // Now optimize the initial VPlan.
6579 RUN_VPLAN_PASS(VPlanTransforms::hoistPredicatedLoads, *Plan, PSE, OrigLoop);
6580 RUN_VPLAN_PASS(VPlanTransforms::sinkPredicatedStores, *Plan, PSE, OrigLoop);
6581 RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
6582 Config.getMinimalBitwidths());
6583 RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
6584 // TODO: try to put addExplicitVectorLength close to addActiveLaneMask
6585 if (CM->foldTailWithEVL()) {
6586 RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
6587 Config.getMaxSafeElements());
6588 RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
6589 }
6590
6591 if (auto P =
6592 RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
6593 VPlans.push_back(Elt: std::move(P));
6594
6595 TailFoldingStyle Style = CM->getTailFoldingStyle();
6596 RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
6597 useActiveLaneMask(Style),
6598 useActiveLaneMaskForControlFlow(Style));
6599
6600 RUN_VPLAN_PASS_NO_VERIFY(printOptimizedVPlan, *Plan);
6601 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6602 VPlans.push_back(Elt: std::move(Plan));
6603 }
6604}
6605
6606VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
6607 VFRange &Range) {
6608
6609 // For outer loops, the plan only needs basic recipe conversion and induction
6610 // live-out optimization; the full inner-loop recipe building below does not
6611 // apply (no widening decisions, interleave groups, reductions, etc.).
6612 if (Plan->isOuterLoop()) {
6613 for (ElementCount VF : Range)
6614 Plan->addVF(VF);
6615 if (!RUN_VPLAN_PASS(VPlanTransforms::tryToConvertVPInstructionsToVPRecipes,
6616 *Plan, *TLI, PSE, OrigLoop))
6617 return nullptr;
6618 RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
6619 OrigLoop);
6620 return Plan;
6621 }
6622
6623 using namespace llvm::VPlanPatternMatch;
6624 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
6625
6626 // ---------------------------------------------------------------------------
6627 // Build initial VPlan: Scan the body of the loop in a topological order to
6628 // visit each basic block after having visited its predecessor basic blocks.
6629 // ---------------------------------------------------------------------------
6630
6631 bool RequiresScalarEpilogueCheck =
6632 LoopVectorizationPlanner::getDecisionAndClampRange(
6633 Predicate: [this](ElementCount VF) {
6634 return !CM->requiresScalarEpilogue(IsVectorizing: VF.isVector());
6635 },
6636 Range);
6637 // Update the branch in the middle block if a scalar epilogue is required.
6638 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6639 if (!RequiresScalarEpilogueCheck && MiddleVPBB->getNumSuccessors() == 2) {
6640 auto *BranchOnCond = cast<VPInstruction>(Val: MiddleVPBB->getTerminator());
6641 assert(MiddleVPBB->getSuccessors()[1] == Plan->getScalarPreheader() &&
6642 "second successor must be scalar preheader");
6643 BranchOnCond->setOperand(I: 0, New: Plan->getFalse());
6644 }
6645
6646 // Don't use getDecisionAndClampRange here, because we don't know the UF
6647 // so this function is better to be conservative, rather than to split
6648 // it up into different VPlans.
6649 // TODO: Consider using getDecisionAndClampRange here to split up VPlans.
6650 bool IVUpdateMayOverflow = false;
6651 for (ElementCount VF : Range)
6652 IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(Cost: CM.get(), VF);
6653
6654 TailFoldingStyle Style = CM->getTailFoldingStyle();
6655 // Use NUW for the induction increment if we proved that it won't overflow in
6656 // the vector loop or when not folding the tail. In the later case, we know
6657 // that the canonical induction increment will not overflow as the vector trip
6658 // count is >= increment and a multiple of the increment.
6659 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
6660 bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
6661 if (!HasNUW) {
6662 auto *IVInc =
6663 LoopRegion->getExitingBasicBlock()->getTerminator()->getOperand(N: 0);
6664 assert(match(IVInc,
6665 m_VPInstruction<Instruction::Add>(
6666 m_Specific(LoopRegion->getCanonicalIV()), m_VPValue())) &&
6667 "Did not find the canonical IV increment");
6668 LoopRegion->clearCanonicalIVNUW(Increment: cast<VPInstruction>(Val: IVInc));
6669 }
6670
6671 // ---------------------------------------------------------------------------
6672 // Pre-construction: record ingredients whose recipes we'll need to further
6673 // process after constructing the initial VPlan.
6674 // ---------------------------------------------------------------------------
6675
6676 // For each interleave group which is relevant for this (possibly trimmed)
6677 // Range, add it to the set of groups to be later applied to the VPlan and add
6678 // placeholders for its members' Recipes which we'll be replacing with a
6679 // single VPInterleaveRecipe.
6680 for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
6681 auto ApplyIG = [IG, this](ElementCount VF) -> bool {
6682 bool Result = (VF.isVector() && // Query is illegal for VF == 1
6683 CM->getWideningDecision(I: IG->getInsertPos(), VF) ==
6684 LoopVectorizationCostModel::CM_Interleave);
6685 // For scalable vectors, the interleave factors must be <= 8 since we
6686 // require the (de)interleaveN intrinsics instead of shufflevectors.
6687 assert((!Result || !VF.isScalable() || IG->getFactor() <= 8) &&
6688 "Unsupported interleave factor for scalable vectors");
6689 return Result;
6690 };
6691 if (!getDecisionAndClampRange(Predicate: ApplyIG, Range))
6692 continue;
6693 InterleaveGroups.insert(Ptr: IG);
6694 }
6695
6696 // ---------------------------------------------------------------------------
6697 // Construct wide recipes and apply predication for original scalar
6698 // VPInstructions in the loop.
6699 // ---------------------------------------------------------------------------
6700 VPRecipeBuilder RecipeBuilder(*Plan, Legal, *CM, Builder);
6701
6702 // Scan the body of the loop in a topological order to visit each basic block
6703 // after having visited its predecessor basic blocks.
6704 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
6705 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
6706 HeaderVPBB);
6707
6708 RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
6709 Range.Start);
6710
6711 VPCostContext CostCtx(*TLI, *Plan, *CM, Config);
6712
6713 RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
6714 RecipeBuilder, CostCtx);
6715
6716 RUN_VPLAN_PASS(VPlanTransforms::makeScalarizationDecisions, *Plan, Range);
6717
6718 RUN_VPLAN_PASS(VPlanTransforms::makeCallWideningDecisions, *Plan, Range,
6719 RecipeBuilder, CostCtx);
6720
6721 // Now process all other blocks and instructions.
6722 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
6723 // Convert input VPInstructions to widened recipes.
6724 for (VPRecipeBase &R : make_early_inc_range(
6725 Range: make_range(x: VPBB->getFirstNonPhi(), y: VPBB->end()))) {
6726 // Skip recipes that do not need transforming or have already been
6727 // transformed.
6728 if (isa<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe,
6729 VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
6730 VPWidenCallRecipe, VPWidenIntrinsicRecipe, VPVectorPointerRecipe,
6731 VPVectorEndPointerRecipe, VPHistogramRecipe>(Val: &R) ||
6732 (isa<VPInstructionWithType>(Val: R) &&
6733 Instruction::isCast(Opcode: cast<VPInstructionWithType>(Val&: R).getOpcode()) &&
6734 vputils::onlyFirstLaneUsed(Def: R.getVPSingleValue())))
6735 continue;
6736 auto *VPI = cast<VPInstruction>(Val: &R);
6737 if (!VPI->getUnderlyingValue())
6738 continue;
6739
6740 // TODO: Gradually replace uses of underlying instruction by analyses on
6741 // VPlan. Migrate code relying on the underlying instruction from VPlan0
6742 // to construct recipes below to not use the underlying instruction.
6743 Instruction *Instr = cast<Instruction>(Val: VPI->getUnderlyingValue());
6744 Builder.setInsertPoint(VPI);
6745
6746 VPRecipeBase *Recipe =
6747 RecipeBuilder.tryToCreateWidenNonPhiRecipe(R: VPI, Range);
6748 if (!Recipe)
6749 Recipe =
6750 RecipeBuilder.handleReplication(VPI: cast<VPInstruction>(Val: VPI), Range);
6751
6752 if (isa<VPWidenIntOrFpInductionRecipe>(Val: Recipe) && isa<TruncInst>(Val: Instr)) {
6753 // Optimized a truncate to VPWidenIntOrFpInductionRecipe. It needs to be
6754 // moved to the phi section in the header.
6755 Recipe->insertBefore(BB&: *HeaderVPBB, IP: HeaderVPBB->getFirstNonPhi());
6756 } else {
6757 Builder.insert(R: Recipe);
6758 }
6759 if (Recipe->getNumDefinedValues() == 1) {
6760 VPI->replaceAllUsesWith(New: Recipe->getVPSingleValue());
6761 } else {
6762 assert(Recipe->getNumDefinedValues() == 0 &&
6763 "Unexpected multidef recipe");
6764 }
6765 R.eraseFromParent();
6766 }
6767 }
6768
6769 assert(isa<VPRegionBlock>(LoopRegion) &&
6770 !LoopRegion->getEntryBasicBlock()->empty() &&
6771 "entry block must be set to a VPRegionBlock having a non-empty entry "
6772 "VPBasicBlock");
6773
6774 RUN_VPLAN_PASS(VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers, *Plan,
6775 Range);
6776
6777 // ---------------------------------------------------------------------------
6778 // Transform initial VPlan: Apply previously taken decisions, in order, to
6779 // bring the VPlan to its final state.
6780 // ---------------------------------------------------------------------------
6781
6782 addReductionResultComputation(Plan, RecipeBuilder, MinVF: Range.Start);
6783
6784 // Optimize FindIV reductions to use sentinel-based approach when possible.
6785 RUN_VPLAN_PASS(VPlanTransforms::optimizeFindIVReductions, *Plan, PSE,
6786 *OrigLoop);
6787 RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
6788 OrigLoop);
6789
6790 // Apply mandatory transformation to handle reductions with multiple in-loop
6791 // uses if possible, bail out otherwise.
6792 if (!RUN_VPLAN_PASS(VPlanTransforms::handleMultiUseReductions, *Plan, ORE,
6793 OrigLoop))
6794 return nullptr;
6795 // Apply mandatory transformation to handle FP maxnum/minnum reduction with
6796 // NaNs if possible, bail out otherwise.
6797 if (!RUN_VPLAN_PASS(VPlanTransforms::handleMaxMinNumReductions, *Plan))
6798 return nullptr;
6799
6800 // Create whole-vector selects for find-last recurrences.
6801 if (!RUN_VPLAN_PASS(VPlanTransforms::handleFindLastReductions, *Plan))
6802 return nullptr;
6803
6804 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, *Plan, false);
6805
6806 // Create partial reduction recipes for scaled reductions and transform
6807 // recipes to abstract recipes if it is legal and beneficial and clamp the
6808 // range for better cost estimation.
6809 RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
6810 Range);
6811 RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
6812 Range);
6813
6814 // Interleave memory: for each Interleave Group we marked earlier as relevant
6815 // for this VPlan, replace the Recipes widening its memory instructions with a
6816 // single VPInterleaveRecipe at its insertion point.
6817 RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
6818 InterleaveGroups, CM->isEpilogueAllowed());
6819
6820 // Convert memory recipes to strided access recipes if the strided access is
6821 // legal and profitable.
6822 RUN_VPLAN_PASS(VPlanTransforms::convertToStridedAccesses, *Plan, PSE,
6823 *OrigLoop, CostCtx, Range);
6824
6825 // Ensure scalar VF plans only contain VF=1, as required by hasScalarVFOnly.
6826 if (Range.Start.isScalar())
6827 Range.End = Range.Start * 2;
6828
6829 for (ElementCount VF : Range)
6830 Plan->addVF(VF);
6831 Plan->setName("Initial VPlan");
6832
6833 RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
6834
6835 if (CM->maskPartialAliasing())
6836 RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
6837
6838 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6839 return Plan;
6840}
6841
6842void LoopVectorizationPlanner::addReductionResultComputation(
6843 VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
6844 using namespace VPlanPatternMatch;
6845 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
6846 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6847 VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
6848 Builder.setInsertPoint(&*std::prev(x: std::prev(x: LatchVPBB->end())));
6849 VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
6850 VPValue *HeaderMask = Plan->getVectorLoopRegion()->getHeaderMask();
6851 for (VPRecipeBase &R :
6852 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
6853 VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
6854 if (!PhiR)
6855 continue;
6856
6857 RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
6858 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
6859 PN: cast<PHINode>(Val: PhiR->getUnderlyingInstr()));
6860 Type *PhiTy = PhiR->getScalarType();
6861
6862 // Convert a VPBlendRecipe backedge to a select.
6863 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: PhiR->getBackedgeValue())) {
6864 if (Blend->getNumIncomingValues() == 2 &&
6865 Blend->getMask(Idx: 0) == HeaderMask) {
6866 auto *Sel = VPBuilder(Blend).createSelect(
6867 Cond: Blend->getMask(Idx: 0), TrueVal: Blend->getIncomingValue(Idx: 0),
6868 FalseVal: Blend->getIncomingValue(Idx: 1), DL: {}, Name: "", Flags: *Blend);
6869 Blend->replaceAllUsesWith(New: Sel);
6870 Blend->eraseFromParent();
6871 }
6872 }
6873
6874 auto *OrigExitingVPV = PhiR->getBackedgeValue();
6875 auto *NewExitingVPV = OrigExitingVPV;
6876
6877 // Remove the predicated select if the target doesn't want it.
6878 VPValue *V;
6879 if (!CM->usePredicatedReductionSelect(RecurrenceKind) &&
6880 match(V: PhiR->getBackedgeValue(),
6881 P: m_Select(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue(V), Op2: m_Specific(VPV: PhiR))))
6882 PhiR->setBackedgeValue(V);
6883
6884 // We want code in the middle block to appear to execute on the location of
6885 // the scalar loop's latch terminator because: (a) it is all compiler
6886 // generated, (b) these instructions are always executed after evaluating
6887 // the latch conditional branch, and (c) other passes may add new
6888 // predecessors which terminate on this line. This is the easiest way to
6889 // ensure we don't accidentally cause an extra step back into the loop while
6890 // debugging.
6891 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
6892
6893 // TODO: At the moment ComputeReductionResult also drives creation of the
6894 // bc.merge.rdx phi nodes, hence it needs to be created unconditionally here
6895 // even for in-loop reductions, until the reduction resume value handling is
6896 // also modeled in VPlan.
6897 VPInstruction *FinalReductionResult;
6898 VPBuilder::InsertPointGuard Guard(Builder);
6899 Builder.setInsertPoint(TheBB: MiddleVPBB, IP);
6900 // For AnyOf reductions, find the select among PhiR's users and convert
6901 // the reduction phi to operate on bools before creating the final
6902 // reduction result.
6903 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RecurrenceKind)) {
6904 auto *AnyOfSelect = cast<VPSingleDefRecipe>(
6905 Val: findUserOf(V: PhiR, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())));
6906 VPValue *Start = PhiR->getStartValue();
6907 bool TrueValIsPhi = AnyOfSelect->getOperand(N: 1) == PhiR;
6908 // NewVal is the non-phi operand of the select.
6909 VPValue *NewVal = TrueValIsPhi ? AnyOfSelect->getOperand(N: 2)
6910 : AnyOfSelect->getOperand(N: 1);
6911
6912 // Adjust AnyOf reductions; replace the reduction phi for the selected
6913 // value with a boolean reduction phi node to check if the condition is
6914 // true in any iteration. The final value is selected by the final
6915 // ComputeReductionResult.
6916 VPValue *Cmp = AnyOfSelect->getOperand(N: 0);
6917 // If the compare is checking the reduction PHI node, adjust it to check
6918 // the start value.
6919 if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
6920 CmpR->replaceUsesOfWith(From: PhiR, To: PhiR->getStartValue());
6921 Builder.setInsertPoint(AnyOfSelect);
6922
6923 // If the true value of the select is the reduction phi, the new value
6924 // is selected if the negated condition is true in any iteration.
6925 if (TrueValIsPhi)
6926 Cmp = Builder.createNot(Operand: Cmp);
6927
6928 // Build a fresh i1 chain (phi, or, and i1 versions of any blend/select
6929 // the exiting value flows through).
6930 auto *NewPhiR =
6931 PhiR->cloneWithOperands(Start: Plan->getFalse(), BackedgeValue: Plan->getFalse());
6932 NewPhiR->insertBefore(InsertPos: PhiR);
6933 VPValue *NewExiting = Builder.createOr(LHS: NewPhiR, RHS: Cmp);
6934
6935 // The exiting value may flow through a chain of VPBlendRecipes and
6936 // select recipes (VPInstruction, VPWidenRecipe or VPReplicateRecipe with
6937 // Select opcode) before reaching OrigExitingVPV. Clone each chain link
6938 // in topological order so each clone refers to the already-rewritten i1
6939 // operands via Substitutions.
6940 DenseMap<VPValue *, VPValue *> Substitutions = {{AnyOfSelect, NewExiting},
6941 {PhiR, NewPhiR}};
6942 std::function<void(VPSingleDefRecipe *)> CloneChain =
6943 [&](VPSingleDefRecipe *Old) {
6944 if (Substitutions.contains(Val: Old))
6945 return;
6946 SmallVector<VPValue *> NewOps;
6947 for (VPValue *Op : Old->operands()) {
6948 if (isa<VPBlendRecipe>(Val: Op) ||
6949 match(V: Op, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())))
6950 CloneChain(cast<VPSingleDefRecipe>(Val: Op));
6951 NewOps.push_back(Elt: Substitutions.lookup_or(Val: Op, Default&: Op));
6952 }
6953 VPSingleDefRecipe *New;
6954 if (auto *B = dyn_cast<VPBlendRecipe>(Val: Old))
6955 New = B->cloneWithOperands(NewOperands: NewOps);
6956 else if (auto *W = dyn_cast<VPWidenRecipe>(Val: Old))
6957 New = W->cloneWithOperands(NewOperands: NewOps);
6958 else if (auto *Rep = dyn_cast<VPReplicateRecipe>(Val: Old))
6959 New = Rep->cloneWithOperands(NewOperands: NewOps);
6960 else
6961 New = cast<VPInstruction>(Val: Old)->cloneWithOperands(NewOperands: NewOps);
6962 New->insertBefore(InsertPos: Old);
6963 Substitutions[Old] = New;
6964 };
6965
6966 if (OrigExitingVPV != AnyOfSelect) {
6967 CloneChain(cast<VPSingleDefRecipe>(Val: OrigExitingVPV));
6968 NewExiting = Substitutions.lookup(Val: OrigExitingVPV);
6969 }
6970 NewPhiR->setOperand(I: 1, New: NewExiting);
6971 PhiR->replaceAllUsesWith(New: Plan->getPoison(Ty: PhiR->getScalarType()));
6972
6973 Builder.setInsertPoint(TheBB: MiddleVPBB, IP);
6974 FinalReductionResult =
6975 Builder.createAnyOfReduction(ChainOp: NewExiting, TrueVal: NewVal, FalseVal: Start, DL: ExitDL);
6976 } else {
6977 // If the vector reduction can be performed in a smaller type, we
6978 // truncate then extend the loop exit value to enable InstCombine to
6979 // evaluate the entire expression in the smaller type.
6980 VPValue *ReductionOp = NewExitingVPV;
6981 Instruction::CastOps ExtendOpc = Instruction::CastOpsEnd;
6982 if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
6983 assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
6984 assert(!RecurrenceDescriptor::isMinMaxRecurrenceKind(RecurrenceKind) &&
6985 "Unexpected truncated min-max recurrence!");
6986 Type *RdxTy = RdxDesc.getRecurrenceType();
6987 ExtendOpc = RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
6988 {
6989 VPBuilder::InsertPointGuard Guard(Builder);
6990 Builder.setInsertPoint(
6991 TheBB: NewExitingVPV->getDefiningRecipe()->getParent(),
6992 IP: std::next(x: NewExitingVPV->getDefiningRecipe()->getIterator()));
6993 ReductionOp =
6994 Builder.createWidenCast(Opcode: Instruction::Trunc, Op: NewExitingVPV, ResultTy: RdxTy);
6995 VPWidenCastRecipe *Extnd =
6996 Builder.createWidenCast(Opcode: ExtendOpc, Op: ReductionOp, ResultTy: PhiTy);
6997 if (PhiR->getOperand(N: 1) == NewExitingVPV)
6998 PhiR->setOperand(I: 1, New: Extnd);
6999 }
7000 }
7001
7002 VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
7003 PhiR->getFastMathFlagsOrNone());
7004 FinalReductionResult = Builder.createNaryOp(
7005 Opcode: VPInstruction::ComputeReductionResult, Operands: {ReductionOp}, Flags, DL: ExitDL);
7006 if (ExtendOpc != Instruction::CastOpsEnd)
7007 FinalReductionResult = Builder.createScalarCast(
7008 Opcode: ExtendOpc, Op: FinalReductionResult, ResultTy: PhiTy, DL: {});
7009 }
7010
7011 // Update all users outside the vector region. Also replace redundant
7012 // extracts.
7013 for (auto *U : to_vector(Range: OrigExitingVPV->users())) {
7014 auto *Parent = cast<VPRecipeBase>(Val: U)->getParent();
7015 if (FinalReductionResult == U || Parent->getParent())
7016 continue;
7017 // Skip ComputeReductionResult and FindIV reductions when they are not the
7018 // final result.
7019 if (match(U, P: m_VPInstruction<VPInstruction::ComputeReductionResult>()) ||
7020 (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind: RecurrenceKind) &&
7021 match(U, P: m_VPInstruction<Instruction::ICmp>())))
7022 continue;
7023 U->replaceUsesOfWith(From: OrigExitingVPV, To: FinalReductionResult);
7024
7025 // Look through ExtractLastPart.
7026 if (match(U, P: m_ExtractLastPart(Op0: m_VPValue())))
7027 U = cast<VPInstruction>(Val: U)->getSingleUser();
7028
7029 if (match(U, P: m_CombineOr(Ps: m_ExtractLane(Op0: m_VPValue(), Op1: m_VPValue()),
7030 Ps: m_ExtractLastLane(Op0: m_VPValue()))))
7031 cast<VPInstruction>(Val: U)->replaceAllUsesWith(New: FinalReductionResult);
7032 }
7033
7034 RecurKind RK = PhiR->getRecurrenceKind();
7035 if ((!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK) &&
7036 !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind: RK) &&
7037 !RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind: RK) &&
7038 !RecurrenceDescriptor::isFindLastRecurrenceKind(Kind: RK))) {
7039 VPBuilder PHBuilder(Plan->getVectorPreheader());
7040 VPValue *Iden = Plan->getOrAddLiveIn(
7041 V: getRecurrenceIdentity(K: RK, Tp: PhiTy, FMF: PhiR->getFastMathFlagsOrNone()));
7042 auto *ScaleFactorVPV = Plan->getConstantInt(BitWidth: 32, Val: 1);
7043 VPValue *StartV = PHBuilder.createNaryOp(
7044 Opcode: VPInstruction::ReductionStartVector,
7045 Operands: {PhiR->getStartValue(), Iden, ScaleFactorVPV}, Flags: *PhiR);
7046 PhiR->setOperand(I: 0, New: StartV);
7047 }
7048 }
7049
7050 RUN_VPLAN_PASS(VPlanTransforms::clearReductionWrapFlags, *Plan);
7051}
7052
7053void LoopVectorizationPlanner::attachRuntimeChecks(
7054 VPlan &Plan, GeneratedRTChecks &RTChecks, bool HasBranchWeights) const {
7055 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
7056 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(N: 0)) {
7057 assert((!Config.OptForSize ||
7058 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled) &&
7059 "Cannot SCEV check stride or overflow when optimizing for size");
7060 RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan, SCEVCheckCond,
7061 SCEVCheckBlock, HasBranchWeights);
7062 }
7063 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
7064 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(N: 0)) {
7065 // VPlan-native path does not do any analysis for runtime checks
7066 // currently.
7067 assert((!EnableVPlanNativePath || !Plan.isOuterLoop()) &&
7068 "Runtime checks are not supported for outer loops yet");
7069
7070 if (Config.OptForSize) {
7071 assert(
7072 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled &&
7073 "Cannot emit memory checks when optimizing for size, unless forced "
7074 "to vectorize.");
7075 ORE->emit(RemarkBuilder: [&]() {
7076 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
7077 OrigLoop->getStartLoc(),
7078 OrigLoop->getHeader())
7079 << "Code-size may be reduced by not forcing "
7080 "vectorization, or by source-code modifications "
7081 "eliminating the need for runtime checks "
7082 "(e.g., adding 'restrict').";
7083 });
7084 }
7085 RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan, MemCheckCond,
7086 MemCheckBlock, HasBranchWeights);
7087 }
7088}
7089
7090void LoopVectorizationPlanner::addMinimumIterationCheck(
7091 VPlan &Plan, ElementCount VF, unsigned UF,
7092 ElementCount MinProfitableTripCount) const {
7093 const uint32_t *BranchWeights =
7094 hasBranchWeightMD(I: *OrigLoop->getLoopLatch()->getTerminator())
7095 ? &MinItersBypassWeights[0]
7096 : nullptr;
7097 RUN_VPLAN_PASS(VPlanTransforms::addMinimumIterationCheck, Plan, VF, UF,
7098 MinProfitableTripCount, Plan.requiresScalarEpilogue(),
7099 Plan.hasTailFolded(), OrigLoop, BranchWeights,
7100 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
7101 PSE, Plan.getEntry());
7102}
7103
7104// Determine how to lower the epilogue, which depends on 1) optimising
7105// for minimum code-size, 2) tail-folding compiler options, 3) loop
7106// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
7107// is suitable for tail-folding.
7108// This function determines epilogue lowering for the main vector loop while
7109// epilogue lowering for the tail-folded epilogue path will be handled
7110// separately in getEpilogueTailLowering.
7111static EpilogueLowering
7112getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
7113 bool OptForSize, TargetTransformInfo *TTI,
7114 TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL,
7115 InterleavedAccessInfo *IAI) {
7116 // 1) OptSize takes precedence over all other options, i.e. if this is set,
7117 // don't look at hints or options, and don't request an epilogue.
7118 if (F->hasOptSize() ||
7119 (OptForSize && Hints.getForce() != LoopVectorizeHints::FK_Enabled))
7120 return CM_EpilogueNotAllowedOptSize;
7121
7122 // 2) If set, obey the directives
7123 if (TailFoldingPolicy.getNumOccurrences()) {
7124 switch (TailFoldingPolicy) {
7125 case TailFoldingPolicyTy::None:
7126 return CM_EpilogueAllowed;
7127 case TailFoldingPolicyTy::PreferFoldTail:
7128 return CM_EpilogueNotNeededFoldTail;
7129 case TailFoldingPolicyTy::MustFoldTail:
7130 return CM_EpilogueNotAllowedFoldTail;
7131 };
7132 }
7133
7134 // 3) If set, obey the hints
7135 switch (Hints.getPredicate()) {
7136 case LoopVectorizeHints::FK_Enabled:
7137 return CM_EpilogueNotNeededFoldTail;
7138 case LoopVectorizeHints::FK_Disabled:
7139 return CM_EpilogueAllowed;
7140 };
7141
7142 // 4) if the TTI hook indicates this is profitable, request tail-folding.
7143 TailFoldingInfo TFI(TLI, &LVL, IAI);
7144 if (TTI->preferTailFoldingOverEpilogue(TFI: &TFI))
7145 return CM_EpilogueNotNeededFoldTail;
7146
7147 return CM_EpilogueAllowed;
7148}
7149
7150/// Determine how to lower the epilogue for the vector epilogue loop.
7151/// Check if there are any conflicts that prevent tail-folding the epilogue.
7152/// \return CM_EpilogueNotNeededFoldTail if epilogue tail-folding is possible,
7153/// otherwise CM_EpilogueAllowed.
7154static EpilogueLowering
7155getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
7156 OptimizationRemarkEmitter *ORE,
7157 LoopVectorizationLegality &LVL,
7158 LoopVectorizeHints &Hints) {
7159 // Epilogue TF is only enabled when explicitly requested via command line.
7160 if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
7161 EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
7162 return CM_EpilogueAllowed;
7163
7164 if (!EnableEpilogueVectorization) {
7165 reportVectorizationInfo(
7166 Msg: "Options conflict, epilogue vectorization is disallowed while "
7167 "epilogue tail-folding allowed!",
7168 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
7169 return CM_EpilogueAllowed;
7170 }
7171
7172 if (!Hints.getWidth() || !hasForcedEpilogueVF()) {
7173 reportVectorizationInfo(Msg: "For now, epilogue tail-folding can't be "
7174 "applied without forced main/epilogue loop VF",
7175 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
7176 return CM_EpilogueAllowed;
7177 }
7178
7179 if (ElementCount::isKnownLE(LHS: Hints.getWidth(), RHS: EpilogueVectorizationForceVF)) {
7180 reportVectorizationInfo(Msg: "For now, epilogue tail-folding can't be applied "
7181 "when VF of the main loop <= VF of the epilogue",
7182 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
7183 return CM_EpilogueAllowed;
7184 }
7185
7186 if (!L->isInnermost()) {
7187 reportVectorizationInfo(
7188 Msg: "Epilogue tail-folding is not supported for outer loop",
7189 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
7190 return CM_EpilogueAllowed;
7191 }
7192
7193 // If scalar epilogue is explicitly required, we can't apply TF.
7194 if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
7195 reportVectorizationInfo(
7196 Msg: "Epilogue tail-folding can't be applied because scalar epilogue is "
7197 "required. Fall back to a normal epilogue",
7198 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
7199 return CM_EpilogueAllowed;
7200 }
7201
7202 // If having epilogue is NOT allowed, then no epilogue to apply TF for.
7203 if (!MainCM.isEpilogueAllowed()) {
7204 reportVectorizationInfo(Msg: "Not applying tail-folding to the epilogue, since "
7205 "no epilogue is allowed.",
7206 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
7207 return CM_EpilogueAllowed;
7208 }
7209
7210 if (L->getExitingBlock() != L->getLoopLatch() ||
7211 LVL.hasUncountableEarlyExit()) {
7212 reportVectorizationInfo(
7213 Msg: "Epilogue tail-folding is not supported yet for early-exit loops",
7214 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
7215 return CM_EpilogueAllowed;
7216 }
7217
7218 // We can apply tail-folding on the vectorized epilogue loop.
7219 return CM_EpilogueNotNeededFoldTail;
7220}
7221
7222// Emit a remark if there are stores to floats that required a floating point
7223// extension. If the vectorized loop was generated with floating point there
7224// will be a performance penalty from the conversion overhead and the change in
7225// the vector width.
7226static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
7227 SmallVector<Instruction *, 4> Worklist;
7228 for (BasicBlock *BB : L->getBlocks()) {
7229 for (Instruction &Inst : *BB) {
7230 if (auto *S = dyn_cast<StoreInst>(Val: &Inst)) {
7231 if (S->getValueOperand()->getType()->isFloatTy())
7232 Worklist.push_back(Elt: S);
7233 }
7234 }
7235 }
7236
7237 // Traverse the floating point stores upwards searching, for floating point
7238 // conversions.
7239 SmallPtrSet<const Instruction *, 4> Visited;
7240 SmallPtrSet<const Instruction *, 4> EmittedRemark;
7241 while (!Worklist.empty()) {
7242 auto *I = Worklist.pop_back_val();
7243 if (!L->contains(Inst: I))
7244 continue;
7245 if (!Visited.insert(Ptr: I).second)
7246 continue;
7247
7248 // Emit a remark if the floating point store required a floating
7249 // point conversion.
7250 // TODO: More work could be done to identify the root cause such as a
7251 // constant or a function return type and point the user to it.
7252 if (isa<FPExtInst>(Val: I) && EmittedRemark.insert(Ptr: I).second)
7253 ORE->emit(RemarkBuilder: [&]() {
7254 return OptimizationRemarkAnalysis(LV_NAME, "VectorMixedPrecision",
7255 I->getDebugLoc(), L->getHeader())
7256 << "floating point conversion changes vector width. "
7257 << "Mixed floating point precision requires an up/down "
7258 << "cast that will negatively impact performance.";
7259 });
7260
7261 for (Use &Op : I->operands())
7262 if (auto *OpI = dyn_cast<Instruction>(Val&: Op))
7263 Worklist.push_back(Elt: OpI);
7264 }
7265}
7266
7267/// For loops with uncountable early exits, find the cost of doing work when
7268/// exiting the loop early, such as calculating the final exit values of
7269/// variables used outside the loop.
7270/// TODO: This is currently overly pessimistic because the loop may not take
7271/// the early exit, but better to keep this conservative for now. In future,
7272/// it might be possible to relax this by using branch probabilities.
7273static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx,
7274 VPlan &Plan, ElementCount VF) {
7275 InstructionCost Cost = 0;
7276 for (auto *ExitVPBB : Plan.getExitBlocks()) {
7277 for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
7278 // If the predecessor is not the middle.block, then it must be the
7279 // vector.early.exit block, which may contain work to calculate the exit
7280 // values of variables used outside the loop.
7281 if (PredVPBB != Plan.getMiddleBlock()) {
7282 LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
7283 << PredVPBB->getName() << ":\n");
7284 Cost += PredVPBB->cost(VF, Ctx&: CostCtx);
7285 }
7286 }
7287 }
7288 return Cost;
7289}
7290
7291/// This function determines whether or not it's still profitable to vectorize
7292/// the loop given the extra work we have to do outside of the loop:
7293/// 1. Perform the runtime checks before entering the loop to ensure it's safe
7294/// to vectorize.
7295/// 2. In the case of loops with uncountable early exits, we may have to do
7296/// extra work when exiting the loop early, such as calculating the final
7297/// exit values of variables used outside the loop.
7298/// 3. The middle block.
7299static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
7300 VectorizationFactor &VF, Loop *L,
7301 PredicatedScalarEvolution &PSE,
7302 VPCostContext &CostCtx, VPlan &Plan,
7303 EpilogueLowering SEL,
7304 std::optional<unsigned> VScale) {
7305 InstructionCost RtC = Checks.getCost();
7306 if (!RtC.isValid())
7307 return false;
7308
7309 // When interleaving only scalar and vector cost will be equal, which in turn
7310 // would lead to a divide by 0. Fall back to hard threshold.
7311 if (VF.Width.isScalar()) {
7312 // TODO: Should we rename VectorizeMemoryCheckThreshold?
7313 if (RtC > VectorizeMemoryCheckThreshold) {
7314 LLVM_DEBUG(
7315 dbgs()
7316 << "LV: Interleaving only is not profitable due to runtime checks\n");
7317 return false;
7318 }
7319 return true;
7320 }
7321
7322 // The scalar cost should only be 0 when vectorizing with a user specified
7323 // VF/IC. In those cases, runtime checks should always be generated.
7324 uint64_t ScalarC = VF.ScalarCost.getValue();
7325 if (ScalarC == 0)
7326 return true;
7327
7328 InstructionCost TotalCost = RtC;
7329 // Add on the cost of any work required in the vector early exit block, if
7330 // one exists.
7331 TotalCost += calculateEarlyExitCost(CostCtx, Plan, VF: VF.Width);
7332 TotalCost += Plan.getMiddleBlock()->cost(VF: VF.Width, Ctx&: CostCtx);
7333
7334 // First, compute the minimum iteration count required so that the vector
7335 // loop outperforms the scalar loop.
7336 // The total cost of the scalar loop is
7337 // ScalarC * TC
7338 // where
7339 // * TC is the actual trip count of the loop.
7340 // * ScalarC is the cost of a single scalar iteration.
7341 //
7342 // The total cost of the vector loop is
7343 // TotalCost + VecC * (TC / VF) + EpiC
7344 // where
7345 // * TotalCost is the sum of the costs cost of
7346 // - the generated runtime checks, i.e. RtC
7347 // - performing any additional work in the vector.early.exit block for
7348 // loops with uncountable early exits.
7349 // - the middle block, if ExpectedTC <= VF.Width.
7350 // * VecC is the cost of a single vector iteration.
7351 // * TC is the actual trip count of the loop
7352 // * VF is the vectorization factor
7353 // * EpiCost is the cost of the generated epilogue, including the cost
7354 // of the remaining scalar operations.
7355 //
7356 // Vectorization is profitable once the total vector cost is less than the
7357 // total scalar cost:
7358 // TotalCost + VecC * (TC / VF) + EpiC < ScalarC * TC
7359 //
7360 // Now we can compute the minimum required trip count TC as
7361 // VF * (TotalCost + EpiC) / (ScalarC * VF - VecC) < TC
7362 //
7363 // For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
7364 // the computations are performed on doubles, not integers and the result
7365 // is rounded up, hence we get an upper estimate of the TC.
7366 unsigned IntVF = estimateElementCount(VF: VF.Width, VScale);
7367 uint64_t Div = ScalarC * IntVF - VF.Cost.getValue();
7368 uint64_t MinTC1 =
7369 Div == 0 ? 0 : divideCeil(Numerator: TotalCost.getValue() * IntVF, Denominator: Div);
7370
7371 // Second, compute a minimum iteration count so that the cost of the
7372 // runtime checks is only a fraction of the total scalar loop cost. This
7373 // adds a loop-dependent bound on the overhead incurred if the runtime
7374 // checks fail. In case the runtime checks fail, the cost is RtC + ScalarC
7375 // * TC. To bound the runtime check to be a fraction 1/X of the scalar
7376 // cost, compute
7377 // RtC < ScalarC * TC * (1 / X) ==> RtC * X / ScalarC < TC
7378 uint64_t MinTC2 = divideCeil(Numerator: RtC.getValue() * 10, Denominator: ScalarC);
7379
7380 // Now pick the larger minimum. If it is not a multiple of VF and an epilogue
7381 // is allowed, choose the next closest multiple of VF. This should partly
7382 // compensate for ignoring the epilogue cost.
7383 uint64_t MinTC = std::max(a: MinTC1, b: MinTC2);
7384 if (SEL == CM_EpilogueAllowed)
7385 MinTC = alignTo(Value: MinTC, Align: IntVF);
7386 VF.MinProfitableTripCount = ElementCount::getFixed(MinVal: MinTC);
7387
7388 LLVM_DEBUG(
7389 dbgs() << "LV: Minimum required TC for runtime checks to be profitable:"
7390 << VF.MinProfitableTripCount << "\n");
7391
7392 // Skip vectorization if the expected trip count is less than the minimum
7393 // required trip count.
7394 if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
7395 if (ElementCount::isKnownLT(LHS: *ExpectedTC, RHS: VF.MinProfitableTripCount)) {
7396 LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
7397 "trip count < minimum profitable VF ("
7398 << *ExpectedTC << " < " << VF.MinProfitableTripCount
7399 << ")\n");
7400
7401 return false;
7402 }
7403 }
7404 return true;
7405}
7406
7407LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
7408 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
7409 !EnableLoopInterleaving),
7410 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
7411 !EnableLoopVectorization) {}
7412
7413/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
7414/// vectorization.
7415static SmallVector<VPInstruction *>
7416preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
7417 using namespace VPlanPatternMatch;
7418 // When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
7419 // introduce multiple uses of undef/poison. If the reduction start value may
7420 // be undef or poison it needs to be frozen and the frozen start has to be
7421 // used when computing the reduction result. We also need to use the frozen
7422 // value in the resume phi generated by the main vector loop, as this is also
7423 // used to compute the reduction result after the epilogue vector loop.
7424 auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
7425 bool UpdateResumePhis) {
7426 VPBuilder Builder(Plan.getEntry());
7427 for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
7428 auto *VPI = dyn_cast<VPInstruction>(Val: &R);
7429 if (!VPI)
7430 continue;
7431 VPValue *OrigStart;
7432 if (!matchFindIVResult(VPI, ReducedIV: m_VPValue(), Start: m_VPValue(V&: OrigStart)))
7433 continue;
7434 if (isGuaranteedNotToBeUndefOrPoison(V: OrigStart->getLiveInIRValue()))
7435 continue;
7436 VPInstruction *Freeze =
7437 Builder.createNaryOp(Opcode: Instruction::Freeze, Operands: {OrigStart}, DL: {}, Name: "fr");
7438 VPI->setOperand(I: 2, New: Freeze);
7439 if (UpdateResumePhis)
7440 OrigStart->replaceUsesWithIf(New: Freeze, ShouldReplace: [Freeze](VPUser &U, unsigned) {
7441 return Freeze != &U && isa<VPPhi>(Val: &U);
7442 });
7443 }
7444 };
7445 AddFreezeForFindLastIVReductions(MainPlan, true);
7446 AddFreezeForFindLastIVReductions(EpiPlan, false);
7447
7448 VPValue *VectorTC = nullptr;
7449 auto *Term =
7450 MainPlan.getVectorLoopRegion()->getExitingBasicBlock()->getTerminator();
7451 [[maybe_unused]] bool MatchedTC =
7452 match(V: Term, P: m_BranchOnCount(Op0: m_VPValue(), Op1: m_VPValue(V&: VectorTC)));
7453 assert(MatchedTC && "must match vector trip count");
7454
7455 // If there is a suitable resume value for the canonical induction in the
7456 // scalar (which will become vector) epilogue loop, use it and move it to the
7457 // beginning of the scalar preheader. Otherwise create it below.
7458 VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader();
7459 auto ResumePhiIter =
7460 find_if(Range: MainScalarPH->phis(), P: [VectorTC](VPRecipeBase &R) {
7461 return match(V: &R, P: m_VPInstruction<Instruction::PHI>(Ops: m_Specific(VPV: VectorTC),
7462 Ops: m_ZeroInt()));
7463 });
7464 VPPhi *ResumePhi = nullptr;
7465 if (ResumePhiIter == MainScalarPH->phis().end()) {
7466 assert(MainPlan.getVectorLoopRegion()->getCanonicalIV() &&
7467 "canonical IV must exist");
7468 Type *Ty = VectorTC->getScalarType();
7469 VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
7470 ResumePhi = ScalarPHBuilder.createScalarPhi(
7471 IncomingValues: {VectorTC, MainPlan.getZero(Ty)}, DL: {}, Name: "vec.epilog.resume.val");
7472 } else {
7473 ResumePhi = cast<VPPhi>(Val: &*ResumePhiIter);
7474 ResumePhi->setName("vec.epilog.resume.val");
7475 if (&MainScalarPH->front() != ResumePhi)
7476 ResumePhi->moveBefore(BB&: *MainScalarPH, I: MainScalarPH->begin());
7477 }
7478
7479 // Create a ResumeForEpilogue for the canonical IV resume and its bypass value
7480 // as the first non-phi, to keep them alive for the epilogue.
7481 VPBuilder ResumeBuilder(MainScalarPH);
7482 ResumeBuilder.createNaryOp(Opcode: VPInstruction::ResumeForEpilogue,
7483 Operands: {ResumePhi, ResumePhi->getOperand(N: 1)});
7484
7485 // Create ResumeForEpilogue instructions for the resume phis of the
7486 // VPIRPhis and their bypass values in the scalar header of the main plan and
7487 // return them so they can be used as resume values when vectorizing the
7488 // epilogue.
7489 return to_vector(
7490 Range: map_range(C: MainPlan.getScalarHeader()->phis(), F: [&](VPRecipeBase &R) {
7491 assert(isa<VPIRPhi>(R) &&
7492 "only VPIRPhis expected in the scalar header");
7493 VPValue *MainResumePhi = R.getOperand(N: 0);
7494 VPValue *Bypass = MainResumePhi->getDefiningRecipe()->getOperand(N: 1);
7495 return ResumeBuilder.createNaryOp(Opcode: VPInstruction::ResumeForEpilogue,
7496 Operands: {MainResumePhi, Bypass});
7497 }));
7498}
7499
7500/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
7501/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes. Some
7502/// reductions require creating new instructions to compute the resume values.
7503/// They are collected in a vector and returned. They must be moved to the
7504/// preheader of the vector epilogue loop, after created by the execution of \p
7505/// Plan.
7506static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
7507 VPlan &MainPlan, VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs,
7508 EpilogueLoopVectorizationInfo &EPI, LoopVectorizationPlanner &LVP,
7509 VFSelectionContext &Config, ScalarEvolution &SE,
7510 ArrayRef<VPInstruction *> ResumeValues) {
7511 // Build a map from the scalar-header PHI to the ResumeForEpilogue markers
7512 // from the main plan.
7513 // TODO: Replace the IR PHI key.
7514 DenseMap<PHINode *, VPInstruction *> IRPhiToResumeForEpi;
7515 for (auto [HeaderPhi, ResumeForEpi] :
7516 zip_equal(t: MainPlan.getScalarHeader()->phis(), u&: ResumeValues))
7517 IRPhiToResumeForEpi[&cast<VPIRPhi>(Val&: HeaderPhi).getIRPhi()] = ResumeForEpi;
7518 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
7519 VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
7520 Header->setName("vec.epilog.vector.body");
7521
7522 VPValue *IV = VectorLoop->getCanonicalIV();
7523 // When vectorizing the epilogue loop, the canonical induction needs to start
7524 // at the resume value from the main vector loop. Find the resume value
7525 // created during execution of the main VPlan. Add this resume value as an
7526 // offset to the canonical IV of the epilogue loop.
7527 using namespace llvm::PatternMatch;
7528 VPInstruction *ResumeForEpilogue =
7529 cast<VPInstruction>(Val: &*MainPlan.getScalarPreheader()->getFirstNonPhi());
7530 Value *EPResumeVal = ResumeForEpilogue->getUnderlyingValue();
7531 if (auto *ResumePhi = dyn_cast<PHINode>(Val: EPResumeVal)) {
7532 for (Value *Inc : ResumePhi->incoming_values()) {
7533 if (match(V: Inc, P: m_SpecificInt(V: 0)))
7534 continue;
7535 assert(!EPI.VectorTripCount &&
7536 "Must only have a single non-zero incoming value");
7537 EPI.VectorTripCount = Inc;
7538 }
7539 // If we didn't find a non-zero vector trip count, all incoming values
7540 // must be zero, which also means the vector trip count is zero.
7541 if (!EPI.VectorTripCount) {
7542 assert(ResumePhi->getNumIncomingValues() > 0 &&
7543 all_of(ResumePhi->incoming_values(), match_fn(m_SpecificInt(0))) &&
7544 "all incoming values must be 0");
7545 EPI.VectorTripCount = ResumePhi->getIncomingValue(i: 0);
7546 }
7547 } else {
7548 EPI.VectorTripCount = EPResumeVal;
7549 }
7550 VPValue *VPV = Plan.getOrAddLiveIn(V: EPResumeVal);
7551 assert(all_of(IV->users(),
7552 [](const VPUser *U) {
7553 if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe>(U))
7554 return true;
7555 unsigned Opc = cast<VPInstruction>(U)->getOpcode();
7556 return Instruction::isCast(Opc) || Opc == Instruction::Add;
7557 }) &&
7558 "the canonical IV should only be used by its increment or "
7559 "ScalarIVSteps when resetting the start value");
7560 VPBuilder Builder(Header, Header->getFirstNonPhi());
7561 VPInstruction *Add = Builder.createAdd(LHS: IV, RHS: VPV);
7562 // Replace all users of the canonical IV and its increment with the offset
7563 // version, except for the Add itself and the canonical IV increment.
7564 auto *Increment = vputils::findCanonicalIVIncrement(Plan);
7565 assert(Increment && "Must have a canonical IV increment at this point");
7566 IV->replaceUsesWithIf(New: Add, ShouldReplace: [Add, Increment](VPUser &U, unsigned) {
7567 return &U != Add && &U != Increment;
7568 });
7569 VPInstruction *OffsetIVInc =
7570 VPBuilder::getToInsertAfter(R: Increment).createAdd(LHS: Increment, RHS: VPV);
7571 Increment->replaceAllUsesWith(New: OffsetIVInc);
7572 OffsetIVInc->setOperand(I: 0, New: Increment);
7573
7574 DenseMap<Value *, Value *> ToFrozen;
7575 SmallVector<Instruction *> InstsToMove;
7576 // Ensure that the start values for all header phi recipes are updated before
7577 // vectorizing the epilogue loop.
7578 for (VPRecipeBase &R : Header->phis()) {
7579 Value *ResumeV = nullptr;
7580 // TODO: Move setting of resume values to prepareToExecute.
7581 if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(Val: &R)) {
7582 // Find the reduction result by searching users of the phi or its backedge
7583 // value.
7584 auto IsReductionResult = [](VPRecipeBase *R) {
7585 auto *VPI = dyn_cast<VPInstruction>(Val: R);
7586 return VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult;
7587 };
7588 auto *RdxResult = cast<VPInstruction>(
7589 Val: vputils::findRecipe(Start: ReductionPhi->getBackedgeValue(), Pred: IsReductionResult));
7590 assert(RdxResult && "expected to find reduction result");
7591
7592 VPInstruction *ResumeForEpi = IRPhiToResumeForEpi.at(
7593 Val: cast<PHINode>(Val: ReductionPhi->getUnderlyingInstr()));
7594 ResumeV = ResumeForEpi->getUnderlyingValue();
7595
7596 // Check for FindIV pattern by looking for icmp user of RdxResult.
7597 // The pattern is: select(icmp ne RdxResult, Sentinel), RdxResult, Start
7598 using namespace VPlanPatternMatch;
7599 VPValue *SentinelVPV = nullptr;
7600 bool IsFindIV = any_of(Range: RdxResult->users(), P: [&](VPUser *U) {
7601 return match(U, P: VPlanPatternMatch::m_SpecificICmp(
7602 MatchPred: ICmpInst::ICMP_NE, Op0: m_Specific(VPV: RdxResult),
7603 Op1: m_VPValue(V&: SentinelVPV)));
7604 });
7605
7606 RecurKind RK = ReductionPhi->getRecurrenceKind();
7607 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK) || IsFindIV) {
7608 auto *ResumePhi = cast<PHINode>(Val: ResumeV);
7609 VPValue *BypassOp = ResumeForEpi->getOperand(N: 1);
7610 assert((isa<VPIRValue>(BypassOp) ||
7611 VPlanPatternMatch::match(
7612 BypassOp,
7613 m_VPInstruction<Instruction::Freeze>(m_VPValue()))) &&
7614 "expected live-in or Freeze");
7615 Value *StartV = BypassOp->getUnderlyingValue();
7616 IRBuilder<> Builder(ResumePhi->getParent(),
7617 ResumePhi->getParent()->getFirstNonPHIIt());
7618
7619 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK)) {
7620 // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
7621 // start value; compare the final value from the main vector loop
7622 // to the start value.
7623 ResumeV = Builder.CreateICmpNE(LHS: ResumeV, RHS: StartV);
7624 if (auto *I = dyn_cast<Instruction>(Val: ResumeV))
7625 InstsToMove.push_back(Elt: I);
7626 } else {
7627 assert(SentinelVPV && "expected to find icmp using RdxResult");
7628 if (auto *FreezeI = dyn_cast<FreezeInst>(Val: StartV))
7629 ToFrozen[FreezeI->getOperand(i_nocapture: 0)] = StartV;
7630
7631 // Adjust resume: select(icmp eq ResumeV, StartV), Sentinel, ResumeV
7632 Value *Cmp = Builder.CreateICmpEQ(LHS: ResumeV, RHS: StartV);
7633 if (auto *I = dyn_cast<Instruction>(Val: Cmp))
7634 InstsToMove.push_back(Elt: I);
7635 ResumeV = Builder.CreateSelect(C: Cmp, True: SentinelVPV->getLiveInIRValue(),
7636 False: ResumeV);
7637 if (auto *I = dyn_cast<Instruction>(Val: ResumeV))
7638 InstsToMove.push_back(Elt: I);
7639 }
7640 } else {
7641 VPValue *StartVal = Plan.getOrAddLiveIn(V: ResumeV);
7642 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
7643 if (auto *VPI = dyn_cast<VPInstruction>(Val: PhiR->getStartValue())) {
7644 assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
7645 "unexpected start value");
7646 // Partial sub-reductions always start at 0 and account for the
7647 // reduction start value in a final subtraction. Update it to use the
7648 // resume value from the main vector loop.
7649 if (PhiR->getVFScaleFactor() > 1 &&
7650 RecurrenceDescriptor::isSubRecurrenceKind(
7651 Kind: PhiR->getRecurrenceKind())) {
7652 auto *Sub = cast<VPInstruction>(Val: RdxResult->getSingleUser());
7653 assert((Sub->getOpcode() == Instruction::Sub ||
7654 Sub->getOpcode() == Instruction::FSub) &&
7655 "Unexpected opcode");
7656 assert(isa<VPIRValue>(Sub->getOperand(0)) &&
7657 "Expected operand to match the original start value of the "
7658 "reduction");
7659 // For integer sub-reductions, verify start value is zero.
7660 // For FP sub-reductions, verify start value is negative zero.
7661 [[maybe_unused]] auto StartValueIsIdentity = [&] {
7662 Value *IdentityValue = getRecurrenceIdentity(
7663 K: PhiR->getRecurrenceKind(), Tp: ResumeV->getType(),
7664 FMF: PhiR->getFastMathFlagsOrNone());
7665 auto *StartValue = dyn_cast<VPIRValue>(Val: VPI->getOperand(N: 0));
7666 return StartValue && StartValue->getValue() == IdentityValue;
7667 };
7668 assert(StartValueIsIdentity() &&
7669 "Expected start value for partial sub-reduction to be zero "
7670 "(or negative zero)");
7671
7672 Sub->setOperand(I: 0, New: StartVal);
7673 } else
7674 VPI->setOperand(I: 0, New: StartVal);
7675 continue;
7676 }
7677 }
7678 } else {
7679 // Retrieve the induction resume value via ResumeForEpilogue.
7680 PHINode *IndPhi = cast<VPWidenInductionRecipe>(Val: &R)->getPHINode();
7681 ResumeV = IRPhiToResumeForEpi.at(Val: IndPhi)->getUnderlyingValue();
7682 }
7683 assert(ResumeV && "Must have a resume value");
7684 VPValue *StartVal = Plan.getOrAddLiveIn(V: ResumeV);
7685 cast<VPHeaderPHIRecipe>(Val: &R)->setStartValue(StartVal);
7686 }
7687
7688 // For some VPValues in the epilogue plan we must re-use the generated IR
7689 // values from the main plan. Replace them with live-in VPValues.
7690 // TODO: This is a workaround needed for epilogue vectorization and it
7691 // should be removed once induction resume value creation is done
7692 // directly in VPlan.
7693 for (auto &R : make_early_inc_range(Range&: *Plan.getEntry())) {
7694 // Re-use frozen values from the main plan for Freeze VPInstructions in the
7695 // epilogue plan. This ensures all users use the same frozen value.
7696 auto *VPI = dyn_cast<VPInstruction>(Val: &R);
7697 if (VPI && VPI->getOpcode() == Instruction::Freeze) {
7698 VPI->replaceAllUsesWith(New: Plan.getOrAddLiveIn(
7699 V: ToFrozen.lookup(Val: VPI->getOperand(N: 0)->getLiveInIRValue())));
7700 continue;
7701 }
7702
7703 // Re-use the trip count and steps expanded for the main loop, as
7704 // skeleton creation needs it as a value that dominates both the scalar
7705 // and vector epilogue loops
7706 auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(Val: &R);
7707 if (!ExpandR)
7708 continue;
7709 assert(ExpandedSCEVs.contains(ExpandR->getSCEV()) &&
7710 "Epilogue plan needs a SCEV not expanded for the main loop");
7711 VPValue *ExpandedVal =
7712 Plan.getOrAddLiveIn(V: ExpandedSCEVs.lookup(Val: ExpandR->getSCEV()));
7713 ExpandR->replaceAllUsesWith(New: ExpandedVal);
7714 if (Plan.getTripCount() == ExpandR)
7715 Plan.resetTripCount(NewTripCount: ExpandedVal);
7716 ExpandR->eraseFromParent();
7717 }
7718
7719 auto VScale = Config.getVScaleForTuning();
7720 unsigned MainLoopStep =
7721 estimateElementCount(VF: EPI.MainLoopVF * EPI.MainLoopUF, VScale);
7722 unsigned EpilogueLoopStep =
7723 estimateElementCount(VF: EPI.EpilogueVF * EPI.EpilogueUF, VScale);
7724 RUN_VPLAN_PASS(VPlanTransforms::addMinimumVectorEpilogueIterationCheck, Plan,
7725 EPI.VectorTripCount, Plan.requiresScalarEpilogue(),
7726 EPI.EpilogueVF, EPI.EpilogueUF, MainLoopStep, EpilogueLoopStep,
7727 SE);
7728
7729 return InstsToMove;
7730}
7731
7732static void
7733fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
7734 VPlan &BestEpiPlan,
7735 ArrayRef<VPInstruction *> ResumeValues) {
7736 // Fix resume values from the additional bypass block.
7737 BasicBlock *PH = L->getLoopPreheader();
7738 for (auto *Pred : predecessors(BB: PH)) {
7739 for (PHINode &Phi : PH->phis()) {
7740 if (Phi.getBasicBlockIndex(BB: Pred) != -1)
7741 continue;
7742 Phi.addIncoming(V: Phi.getIncomingValueForBlock(BB: BypassBlock), BB: Pred);
7743 }
7744 }
7745 auto *ScalarPH = cast<VPIRBasicBlock>(Val: BestEpiPlan.getScalarPreheader());
7746 if (ScalarPH->hasPredecessors()) {
7747 // Fix resume values for inductions and reductions from the additional
7748 // bypass block using the incoming values from the main loop's resume phis.
7749 // ResumeValues correspond 1:1 with the scalar loop header phis.
7750 for (auto [ResumeV, HeaderPhi] :
7751 zip(t&: ResumeValues, u: BestEpiPlan.getScalarHeader()->phis())) {
7752 auto *HeaderPhiR = cast<VPIRPhi>(Val: &HeaderPhi);
7753 auto *EpiResumePhi =
7754 cast<PHINode>(Val: HeaderPhiR->getIRPhi().getIncomingValueForBlock(BB: PH));
7755 if (EpiResumePhi->getBasicBlockIndex(BB: BypassBlock) == -1)
7756 continue;
7757 auto *MainResumePhi = cast<PHINode>(Val: ResumeV->getUnderlyingValue());
7758 EpiResumePhi->setIncomingValueForBlock(
7759 BB: BypassBlock, V: MainResumePhi->getIncomingValueForBlock(BB: BypassBlock));
7760 }
7761 }
7762}
7763
7764/// Connect the epilogue vector loop generated for \p EpiPlan to the main vector
7765/// loop, after both plans have executed, updating branches from the iteration
7766/// and runtime checks of the main loop, as well as updating various phis. \p
7767/// InstsToMove contains instructions that need to be moved to the preheader of
7768/// the epilogue vector loop.
7769static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
7770 EpilogueLoopVectorizationInfo &EPI,
7771 DominatorTree *DT,
7772 GeneratedRTChecks &Checks,
7773 ArrayRef<Instruction *> InstsToMove,
7774 ArrayRef<VPInstruction *> ResumeValues) {
7775 BasicBlock *VecEpilogueIterationCountCheck =
7776 cast<VPIRBasicBlock>(Val: EpiPlan.getEntry())->getIRBasicBlock();
7777
7778 BasicBlock *VecEpiloguePreHeader =
7779 cast<CondBrInst>(Val: VecEpilogueIterationCountCheck->getTerminator())
7780 ->getSuccessor(i: 1);
7781 // Adjust the control flow taking the state info from the main loop
7782 // vectorization into account.
7783 assert(EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck &&
7784 "expected this to be saved from the previous pass.");
7785 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
7786
7787 // Helper to redirect an edge from \p BB to \p VecEpilogueIterationCountCheck
7788 // to \p NewSucc instead, updating the DomTree.
7789 auto RedirectEdge = [&](BasicBlock *BB, BasicBlock *NewSucc) {
7790 BB->getTerminator()->replaceUsesOfWith(From: VecEpilogueIterationCountCheck,
7791 To: NewSucc);
7792 DTU.applyUpdates(
7793 Updates: {{DominatorTree::Delete, BB, VecEpilogueIterationCountCheck},
7794 {DominatorTree::Insert, BB, NewSucc}});
7795 };
7796
7797 RedirectEdge(EPI.MainLoopIterationCountCheck, VecEpiloguePreHeader);
7798
7799 BasicBlock *ScalarPH =
7800 cast<VPIRBasicBlock>(Val: EpiPlan.getScalarPreheader())->getIRBasicBlock();
7801 RedirectEdge(EPI.EpilogueIterationCountCheck, ScalarPH);
7802
7803 // Adjust the terminators of runtime check blocks and phis using them.
7804 BasicBlock *SCEVCheckBlock = Checks.getSCEVChecks().second;
7805 BasicBlock *MemCheckBlock = Checks.getMemRuntimeChecks().second;
7806 if (SCEVCheckBlock)
7807 RedirectEdge(SCEVCheckBlock, ScalarPH);
7808 if (MemCheckBlock)
7809 RedirectEdge(MemCheckBlock, ScalarPH);
7810
7811 // The vec.epilog.iter.check block may contain Phi nodes from inductions
7812 // or reductions which merge control-flow from the latch block and the
7813 // middle block. Update the incoming values here and move the Phi into the
7814 // preheader.
7815 SmallVector<PHINode *, 4> PhisInBlock(
7816 llvm::make_pointer_range(Range: VecEpilogueIterationCountCheck->phis()));
7817
7818 for (PHINode *Phi : PhisInBlock) {
7819 Phi->moveBefore(InsertPos: VecEpiloguePreHeader->getFirstNonPHIIt());
7820 Phi->replaceIncomingBlockWith(
7821 Old: VecEpilogueIterationCountCheck->getSinglePredecessor(),
7822 New: VecEpilogueIterationCountCheck);
7823
7824 // If the phi doesn't have an incoming value from the
7825 // EpilogueIterationCountCheck, we are done. Otherwise remove the
7826 // incoming value and also those from other check blocks. This is needed
7827 // for reduction phis only.
7828 if (none_of(Range: Phi->blocks(), P: [&](BasicBlock *IncB) {
7829 return EPI.EpilogueIterationCountCheck == IncB;
7830 }))
7831 continue;
7832 for (BasicBlock *BB :
7833 {EPI.EpilogueIterationCountCheck, SCEVCheckBlock, MemCheckBlock}) {
7834 if (BB)
7835 Phi->removeIncomingValue(BB);
7836 }
7837 }
7838
7839 auto IP = VecEpiloguePreHeader->getFirstNonPHIIt();
7840 for (auto *I : InstsToMove)
7841 I->moveBefore(InsertPos: IP);
7842
7843 // VecEpilogueIterationCountCheck conditionally skips over the epilogue loop
7844 // after executing the main loop. We need to update the resume values of
7845 // inductions and reductions during epilogue vectorization.
7846 fixScalarResumeValuesFromBypass(BypassBlock: VecEpilogueIterationCountCheck, L, BestEpiPlan&: EpiPlan,
7847 ResumeValues);
7848
7849 // Remove dead phis that were moved to the epilogue preheader but are unused
7850 // (e.g., resume phis for inductions not widened in the epilogue vector loop).
7851 for (PHINode &Phi : make_early_inc_range(Range: VecEpiloguePreHeader->phis()))
7852 if (Phi.use_empty())
7853 Phi.eraseFromParent();
7854}
7855
7856bool LoopVectorizePass::processLoop(Loop *L) {
7857 assert((EnableVPlanNativePath || L->isInnermost()) &&
7858 "VPlan-native path is not enabled. Only process inner loops.");
7859
7860 LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in '"
7861 << L->getHeader()->getParent()->getName() << "' from "
7862 << L->getLocStr() << "\n");
7863
7864 LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE, TTI);
7865
7866 LLVM_DEBUG(
7867 dbgs() << "LV: Loop hints:"
7868 << " force="
7869 << (Hints.getForce() == LoopVectorizeHints::FK_Disabled
7870 ? "disabled"
7871 : (Hints.getForce() == LoopVectorizeHints::FK_Enabled
7872 ? "enabled"
7873 : "?"))
7874 << " width=" << Hints.getWidth()
7875 << " interleave=" << Hints.getInterleave() << "\n");
7876
7877 // Function containing loop
7878 Function *F = L->getHeader()->getParent();
7879
7880 // Looking at the diagnostic output is the only way to determine if a loop
7881 // was vectorized (other than looking at the IR or machine code), so it
7882 // is important to generate an optimization remark for each loop. Most of
7883 // these messages are generated as OptimizationRemarkAnalysis. Remarks
7884 // generated as OptimizationRemark and OptimizationRemarkMissed are
7885 // less verbose reporting vectorized loops and unvectorized loops that may
7886 // benefit from vectorization, respectively.
7887
7888 if (!Hints.allowVectorization(F, L, VectorizeOnlyWhenForced)) {
7889 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
7890 return false;
7891 }
7892
7893 PredicatedScalarEvolution PSE(*SE, *L);
7894
7895 // Query this against the original loop and save it here because the profile
7896 // of the original loop header may change as the transformation happens.
7897 bool OptForSize = llvm::shouldOptimizeForSize(
7898 BB: L->getHeader(), PSI,
7899 BFI: PSI && PSI->hasProfileSummary() ? &GetBFI() : nullptr,
7900 QueryType: PGSOQueryType::IRPass);
7901
7902 // Check if it is legal to vectorize the loop.
7903 LoopVectorizationRequirements Requirements;
7904 LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE,
7905 &Requirements, &Hints, DB, AC,
7906 /*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
7907 if (!LVL.canVectorize(UseVPlanNativePath: EnableVPlanNativePath)) {
7908 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
7909 Hints.emitRemarkWithHints();
7910 return false;
7911 }
7912
7913 bool IsInnerLoop = L->isInnermost();
7914
7915 // Outer loops require a computable trip count.
7916 if (!IsInnerLoop && isa<SCEVCouldNotCompute>(Val: PSE.getBackedgeTakenCount())) {
7917 LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n");
7918 return false;
7919 }
7920
7921 if (LVL.hasUncountableEarlyExit()) {
7922 if (!EnableEarlyExitVectorization) {
7923 reportVectorizationFailure(DebugMsg: "Auto-vectorization of loops with uncountable "
7924 "early exit is not enabled",
7925 ORETag: "UncountableEarlyExitLoopsDisabled", ORE, TheLoop: L);
7926 return false;
7927 }
7928 if (LVL.hasUncountableExitWithSideEffects() &&
7929 !EnableEarlyExitVectorizationWithSideEffects) {
7930 reportVectorizationFailure(DebugMsg: "Auto-vectorization of loops with uncountable "
7931 "early exit and side effects is not enabled",
7932 ORETag: "UncountableEarlyExitSideEffectLoopsDisabled",
7933 ORE, TheLoop: L);
7934 return false;
7935 }
7936 }
7937
7938 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
7939 bool UseInterleaved =
7940 IsInnerLoop && TTI->enableInterleavedAccessVectorization();
7941
7942 // If an override option has been passed in for interleaved accesses, use it.
7943 if (EnableInterleavedMemAccesses.getNumOccurrences() > 0)
7944 UseInterleaved = IsInnerLoop && EnableInterleavedMemAccesses;
7945
7946 // Analyze interleaved memory accesses.
7947 if (UseInterleaved)
7948 IAI.analyzeInterleaving(EnableMaskedInterleavedGroup: useMaskedInterleavedAccesses(TTI: *TTI));
7949
7950 if (LVL.hasUncountableEarlyExit()) {
7951 BasicBlock *LoopLatch = L->getLoopLatch();
7952 if (IAI.requiresScalarEpilogue() ||
7953 any_of(Range: LVL.getCountableExitingBlocks(), P: not_equal_to(Arg&: LoopLatch))) {
7954 reportVectorizationFailure(DebugMsg: "Auto-vectorization of early exit loops "
7955 "requiring a scalar epilogue is unsupported",
7956 ORETag: "UncountableEarlyExitUnsupported", ORE, TheLoop: L);
7957 return false;
7958 }
7959 }
7960
7961 // Check the function attributes and profiles to find out if this function
7962 // should be optimized for size.
7963 EpilogueLowering SEL =
7964 getEpilogueLowering(F, L, Hints, OptForSize, TTI, TLI, LVL, IAI: &IAI);
7965
7966 // Check the loop for a trip count threshold: vectorize loops with a tiny trip
7967 // count by optimizing for size, to minimize overheads.
7968 auto ExpectedTC = getSmallBestKnownTC(PSE, L);
7969 if (ExpectedTC && ExpectedTC->isFixed() &&
7970 ExpectedTC->getFixedValue() < TinyTripCountVectorThreshold) {
7971 LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
7972 << "This loop is worth vectorizing only if no scalar "
7973 << "iteration overheads are incurred.");
7974 if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
7975 LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
7976 else {
7977 LLVM_DEBUG(dbgs() << "\n");
7978 // Tail-folded loops are efficient even when the loop
7979 // iteration count is low. However, setting the epilogue policy to
7980 // `CM_EpilogueNotAllowedLowTripLoop` prevents vectorizing loops
7981 // with runtime checks. It's more effective to let
7982 // `isOutsideLoopWorkProfitable` determine if vectorization is
7983 // beneficial for the loop.
7984 if (SEL != CM_EpilogueNotNeededFoldTail)
7985 SEL = CM_EpilogueNotAllowedLowTripLoop;
7986 }
7987 }
7988
7989 // Check the function attributes to see if implicit floats or vectors are
7990 // allowed.
7991 if (F->hasFnAttribute(Kind: Attribute::NoImplicitFloat)) {
7992 reportVectorizationFailure(
7993 DebugMsg: "Can't vectorize when the NoImplicitFloat attribute is used",
7994 OREMsg: "loop not vectorized due to NoImplicitFloat attribute",
7995 ORETag: "NoImplicitFloat", ORE, TheLoop: L);
7996 Hints.emitRemarkWithHints();
7997 return false;
7998 }
7999
8000 // Check if the target supports potentially unsafe FP vectorization.
8001 // FIXME: Add a check for the type of safety issue (denormal, signaling)
8002 // for the target we're vectorizing for, to make sure none of the
8003 // additional fp-math flags can help.
8004 if (Hints.isPotentiallyUnsafe() &&
8005 TTI->isFPVectorizationPotentiallyUnsafe()) {
8006 reportVectorizationFailure(
8007 DebugMsg: "Potentially unsafe FP op prevents vectorization",
8008 OREMsg: "loop not vectorized due to unsafe FP support.", ORETag: "UnsafeFP", ORE, TheLoop: L);
8009 Hints.emitRemarkWithHints();
8010 return false;
8011 }
8012
8013 bool AllowOrderedReductions;
8014 // If the flag is set, use that instead and override the TTI behaviour.
8015 if (ForceOrderedReductions.getNumOccurrences() > 0)
8016 AllowOrderedReductions = ForceOrderedReductions;
8017 else
8018 AllowOrderedReductions = TTI->enableOrderedReductions();
8019 if (!LVL.canVectorizeFPMath(EnableStrictReductions: AllowOrderedReductions)) {
8020 ORE->emit(RemarkBuilder: [&]() {
8021 auto *ExactFPMathInst = Requirements.getExactFPInst();
8022 return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE, "CantReorderFPOps",
8023 ExactFPMathInst->getDebugLoc(),
8024 ExactFPMathInst->getParent())
8025 << "loop not vectorized: cannot prove it is safe to reorder "
8026 "floating-point operations";
8027 });
8028 LLVM_DEBUG(dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
8029 "reorder floating-point operations\n");
8030 Hints.emitRemarkWithHints();
8031 return false;
8032 }
8033
8034 // Use the cost model.
8035 VFSelectionContext Config(*TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
8036 OptForSize);
8037 // Use the planner for vectorization.
8038 LoopVectorizationPlanner LVP(
8039 L, LI, DT, TLI, *TTI, &LVL,
8040 std::make_unique<LoopVectorizationCostModel>(
8041 args&: SEL, args&: L, args&: PSE, args&: LI, args: &LVL, args&: *TTI, args&: TLI, args&: AC, args&: ORE, args&: GetBFI, args&: F, args&: IAI, args&: Config),
8042 Config, IAI, PSE, ORE);
8043
8044 EpilogueLowering EpilogueTailLoweringStatus =
8045 getEpilogueTailLowering(MainCM: LVP.getCostModel(), L, ORE, LVL, Hints);
8046 if (EpilogueTailLoweringStatus ==
8047 EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
8048 // TODO: Apply tail-folding on the vectorized epilogue loop.
8049 LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
8050 reportVectorizationInfo(
8051 Msg: "The epilogue-tail-folding policy prefer-fold-tail is not supported "
8052 "yet, fall back to a normal epilogue",
8053 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
8054 }
8055
8056 // Get user vectorization factor and interleave count.
8057 ElementCount UserVF = Hints.getWidth();
8058 unsigned UserIC = Hints.getInterleave();
8059 // Outer loops don't have LoopAccessInfo, so skip the safety check and reset
8060 // UserIC (interleaving is not supported for outer loops).
8061 if (!IsInnerLoop)
8062 UserIC = 0;
8063 else if (UserIC > 1 && !LVL.isSafeForAnyVectorWidth())
8064 UserIC = 1;
8065
8066 // Plan how to best vectorize.
8067 LVP.plan(UserVF, UserIC);
8068 auto [VF, BestPlanPtr] = LVP.computeBestVF();
8069 unsigned IC = 1;
8070
8071 // For VPlan build stress testing of outer loops, bail after plan
8072 // construction.
8073 if (!IsInnerLoop && VPlanBuildOuterloopStressTest)
8074 return false;
8075
8076 if (IsInnerLoop && ORE->allowExtraAnalysis(LV_NAME))
8077 LVP.emitInvalidCostRemarks(ORE);
8078
8079 assert((IsInnerLoop || !LVP.getCostModel().maskPartialAliasing()) &&
8080 "Did not expect to alias-mask outer loop");
8081
8082 GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
8083 LVP.getCostModel().maskPartialAliasing());
8084 if (IsInnerLoop && LVP.hasPlanWithVF(VF: VF.Width)) {
8085 // Select the interleave count.
8086 IC = LVP.selectInterleaveCount(Plan&: *BestPlanPtr, VF: VF.Width, LoopCost: VF.Cost);
8087
8088 unsigned SelectedIC = std::max(a: IC, b: UserIC);
8089 // Optimistically generate runtime checks if they are needed. Drop them if
8090 // they turn out to not be profitable.
8091 if (VF.Width.isVector() || SelectedIC > 1) {
8092 Checks.create(L, LAI: *LVL.getLAI(), UnionPred: PSE.getPredicate(), VF: VF.Width, IC: SelectedIC,
8093 ORE&: *ORE);
8094
8095 // Bail out early if either the SCEV or memory runtime checks are known to
8096 // fail. In that case, the vector loop would never execute.
8097 using namespace llvm::PatternMatch;
8098 if (Checks.getSCEVChecks().first &&
8099 match(V: Checks.getSCEVChecks().first, P: m_One()))
8100 return false;
8101 if (Checks.getMemRuntimeChecks().first &&
8102 match(V: Checks.getMemRuntimeChecks().first, P: m_One()))
8103 return false;
8104 }
8105
8106 // Check if it is profitable to vectorize with runtime checks.
8107 bool ForceVectorization =
8108 Hints.getForce() == LoopVectorizeHints::FK_Enabled;
8109 VPCostContext CostCtx(*TLI, *BestPlanPtr, LVP.getCostModel(), Config,
8110 /*ReusePrintingSlotTracker=*/true);
8111 if (!ForceVectorization &&
8112 !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx, Plan&: *BestPlanPtr,
8113 SEL, VScale: Config.getVScaleForTuning())) {
8114 ORE->emit(RemarkBuilder: [&]() {
8115 return OptimizationRemarkAnalysisAliasing(
8116 DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
8117 L->getHeader())
8118 << "loop not vectorized: cannot prove it is safe to reorder "
8119 "memory operations";
8120 });
8121 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
8122 Hints.emitRemarkWithHints();
8123 return false;
8124 }
8125 }
8126
8127 // Identify the diagnostic messages that should be produced.
8128 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
8129 bool VectorizeLoop = true, InterleaveLoop = true;
8130 if (VF.Width.isScalar()) {
8131 LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n");
8132 VecDiagMsg = {
8133 "VectorizationNotBeneficial",
8134 "the cost-model indicates that vectorization is not beneficial"};
8135 VectorizeLoop = false;
8136 }
8137
8138 if (UserIC == 1 && Hints.getInterleave() > 1) {
8139 assert(!LVL.isSafeForAnyVectorWidth() &&
8140 "UserIC should only be ignored due to unsafe dependencies");
8141 LLVM_DEBUG(dbgs() << "LV: Ignoring user-specified interleave count.\n");
8142 IntDiagMsg = {"InterleavingUnsafe",
8143 "Ignoring user-specified interleave count due to possibly "
8144 "unsafe dependencies in the loop."};
8145 InterleaveLoop = false;
8146 } else if (!LVP.hasPlanWithVF(VF: VF.Width) && UserIC > 1) {
8147 // Tell the user interleaving was avoided up-front, despite being explicitly
8148 // requested.
8149 LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
8150 "interleaving should be avoided up front\n");
8151 IntDiagMsg = {"InterleavingAvoided",
8152 "Ignoring UserIC, because interleaving was avoided up front"};
8153 InterleaveLoop = false;
8154 } else if (IC == 1 && UserIC <= 1) {
8155 // Tell the user interleaving is not beneficial.
8156 LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
8157 IntDiagMsg = {
8158 "InterleavingNotBeneficial",
8159 "the cost-model indicates that interleaving is not beneficial"};
8160 InterleaveLoop = false;
8161 if (UserIC == 1) {
8162 IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
8163 IntDiagMsg.second +=
8164 " and is explicitly disabled or interleave count is set to 1";
8165 }
8166 } else if (IC > 1 && UserIC == 1) {
8167 // Tell the user interleaving is beneficial, but it explicitly disabled.
8168 LLVM_DEBUG(dbgs() << "LV: Interleaving is beneficial but is explicitly "
8169 "disabled.\n");
8170 IntDiagMsg = {"InterleavingBeneficialButDisabled",
8171 "the cost-model indicates that interleaving is beneficial "
8172 "but is explicitly disabled or interleave count is set to 1"};
8173 InterleaveLoop = false;
8174 }
8175
8176 // If there is a histogram in the loop, do not just interleave without
8177 // vectorizing. The order of operations will be incorrect without the
8178 // histogram intrinsics, which are only used for recipes with VF > 1.
8179 if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
8180 LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
8181 << "to histogram operations.\n");
8182 IntDiagMsg = {
8183 "HistogramPreventsScalarInterleaving",
8184 "Unable to interleave without vectorization due to constraints on "
8185 "the order of histogram operations"};
8186 InterleaveLoop = false;
8187 }
8188
8189 // Override IC if user provided an interleave count.
8190 IC = UserIC > 0 ? UserIC : IC;
8191
8192 if (LVP.getCostModel().maskPartialAliasing()) {
8193 LLVM_DEBUG(
8194 dbgs()
8195 << "LV: Not interleaving due to partial aliasing vectorization.\n");
8196 IntDiagMsg = {
8197 "PartialAliasingVectorization",
8198 "Unable to interleave due to partial aliasing vectorization."};
8199 InterleaveLoop = false;
8200 IC = 1;
8201 }
8202
8203 // FIXME: Enable interleaving for EE-with-side-effects.
8204 if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) {
8205 LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n");
8206 IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving",
8207 "Unable to interleave due to early exit with side effects."};
8208 InterleaveLoop = false;
8209 IC = 1;
8210 }
8211
8212 // Emit diagnostic messages, if any.
8213 if (!VectorizeLoop && !InterleaveLoop) {
8214 // Do not vectorize or interleaving the loop.
8215 ORE->emit(RemarkBuilder: [&]() {
8216 return OptimizationRemarkMissed(LV_NAME, VecDiagMsg.first,
8217 L->getStartLoc(), L->getHeader())
8218 << VecDiagMsg.second;
8219 });
8220 ORE->emit(RemarkBuilder: [&]() {
8221 return OptimizationRemarkMissed(LV_NAME, IntDiagMsg.first,
8222 L->getStartLoc(), L->getHeader())
8223 << IntDiagMsg.second;
8224 });
8225 return false;
8226 }
8227
8228 if (!VectorizeLoop && InterleaveLoop) {
8229 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
8230 ORE->emit(RemarkBuilder: [&]() {
8231 return OptimizationRemarkAnalysis(LV_NAME, VecDiagMsg.first,
8232 L->getStartLoc(), L->getHeader())
8233 << VecDiagMsg.second;
8234 });
8235 } else if (VectorizeLoop && !InterleaveLoop) {
8236 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
8237 << ") in " << L->getLocStr() << '\n');
8238 ORE->emit(RemarkBuilder: [&]() {
8239 return OptimizationRemarkAnalysis(LV_NAME, IntDiagMsg.first,
8240 L->getStartLoc(), L->getHeader())
8241 << IntDiagMsg.second;
8242 });
8243 } else if (VectorizeLoop && InterleaveLoop) {
8244 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
8245 << ") in " << L->getLocStr() << '\n');
8246 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
8247 }
8248
8249 // Report the vectorization decision.
8250 if (VF.Width.isScalar()) {
8251 using namespace ore;
8252 assert(IC > 1);
8253 ORE->emit(RemarkBuilder: [&]() {
8254 return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(),
8255 L->getHeader())
8256 << "interleaved loop (interleaved count: "
8257 << NV("InterleaveCount", IC) << ")";
8258 });
8259 } else {
8260 // Report the vectorization decision.
8261 reportVectorization(ORE, TheLoop: L, VFWidth: VF.Width, IC);
8262 }
8263 if (ORE->allowExtraAnalysis(LV_NAME))
8264 checkMixedPrecision(L, ORE);
8265
8266 // If we decided that it is *legal* to interleave or vectorize the loop, then
8267 // do it.
8268
8269 // Whether a scalar epilogue may be created is decided by the epilogue
8270 // lowering policy.
8271 // TODO: Also move check to be based on VPlan.
8272 bool ScalarEpilogueAllowed = LVP.getCostModel().isEpilogueAllowed();
8273
8274 // Destroy the cost model before executing any plan, so that code generation
8275 // cannot rely on cost-modeling decisions.
8276 LVP.clearCostModel();
8277
8278 VPlan &BestPlan = *BestPlanPtr;
8279 // Consider vectorizing the epilogue too if it's profitable.
8280 std::unique_ptr<VPlan> EpiPlan =
8281 LVP.selectBestEpiloguePlan(MainPlan&: BestPlan, MainLoopVF: VF.Width, IC, ScalarEpilogueAllowed);
8282 bool HasBranchWeights =
8283 hasBranchWeightMD(I: *L->getLoopLatch()->getTerminator());
8284 if (EpiPlan) {
8285 VPlan &BestEpiPlan = *EpiPlan;
8286 VPlan &BestMainPlan = BestPlan;
8287 ElementCount EpilogueVF = BestEpiPlan.getSingleVF();
8288
8289 // The first pass vectorizes the main loop and creates a scalar epilogue
8290 // to be vectorized by executing the plan (potentially with a different
8291 // factor) again shortly afterwards.
8292 BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
8293 BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
8294 SmallVector<VPInstruction *> ResumeValues =
8295 preparePlanForMainVectorLoop(MainPlan&: BestMainPlan, EpiPlan&: BestEpiPlan);
8296 EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF, 1, BestEpiPlan);
8297
8298 // Add minimum iteration check for the epilogue plan, followed by runtime
8299 // checks for the main plan.
8300 LVP.addMinimumIterationCheck(Plan&: BestMainPlan, VF: EPI.EpilogueVF, UF: EPI.EpilogueUF,
8301 MinProfitableTripCount: ElementCount::getFixed(MinVal: 0));
8302 LVP.attachRuntimeChecks(Plan&: BestMainPlan, RTChecks&: Checks, HasBranchWeights);
8303 RUN_VPLAN_PASS(
8304 VPlanTransforms::addIterationCountCheckBlock, BestMainPlan,
8305 EPI.MainLoopVF, EPI.MainLoopUF, BestMainPlan.requiresScalarEpilogue(),
8306 L, HasBranchWeights ? MinItersBypassWeights : nullptr,
8307 L->getLoopPredecessor()->getTerminator()->getDebugLoc(), PSE);
8308
8309 EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TTI, AC, EPI, Checks,
8310 BestMainPlan);
8311 auto ExpandedSCEVs = LVP.executePlan(
8312 BestVF: EPI.MainLoopVF, BestUF: EPI.MainLoopUF, BestVPlan&: BestMainPlan, ILV&: MainILV, DT,
8313 EpilogueVecKind: LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
8314 ++LoopsVectorized;
8315
8316 // Derive EPI fields from VPlan-generated IR.
8317 BasicBlock *EntryBB =
8318 cast<VPIRBasicBlock>(Val: BestMainPlan.getEntry())->getIRBasicBlock();
8319 EntryBB->setName("iter.check");
8320 EPI.EpilogueIterationCountCheck = EntryBB;
8321 // The check chain is: Entry -> [SCEV] -> [Mem] -> MainCheck -> VecPH.
8322 // MainCheck is the non-bypass successor of the last runtime check block
8323 // (or Entry if there are no runtime checks).
8324 BasicBlock *LastCheck = EntryBB;
8325 if (BasicBlock *MemBB = Checks.getMemRuntimeChecks().second)
8326 LastCheck = MemBB;
8327 else if (BasicBlock *SCEVBB = Checks.getSCEVChecks().second)
8328 LastCheck = SCEVBB;
8329 BasicBlock *ScalarPH = L->getLoopPreheader();
8330 auto *BI = cast<CondBrInst>(Val: LastCheck->getTerminator());
8331 EPI.MainLoopIterationCountCheck =
8332 BI->getSuccessor(i: BI->getSuccessor(i: 0) == ScalarPH);
8333
8334 // Second pass vectorizes the epilogue and adjusts the control flow
8335 // edges from the first pass.
8336 EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TTI, AC, EPI,
8337 Checks, BestEpiPlan);
8338 SmallVector<Instruction *> InstsToMove = preparePlanForEpilogueVectorLoop(
8339 MainPlan&: BestMainPlan, Plan&: BestEpiPlan, L, ExpandedSCEVs, EPI, LVP, Config,
8340 SE&: *PSE.getSE(), ResumeValues);
8341 RUN_VPLAN_PASS(VPlanTransforms::simplifyLiveInsWithSCEV, BestEpiPlan, PSE);
8342 LVP.executePlan(
8343 BestVF: EPI.EpilogueVF, BestUF: EPI.EpilogueUF, BestVPlan&: BestEpiPlan, ILV&: EpilogILV, DT,
8344 EpilogueVecKind: LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
8345 connectEpilogueVectorLoop(EpiPlan&: BestEpiPlan, L, EPI, DT, Checks, InstsToMove,
8346 ResumeValues);
8347 ++LoopsEpilogueVectorized;
8348 } else {
8349 InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
8350 BestPlan);
8351 LVP.addMinimumIterationCheck(Plan&: BestPlan, VF: VF.Width, UF: IC,
8352 MinProfitableTripCount: VF.MinProfitableTripCount);
8353 LVP.attachRuntimeChecks(Plan&: BestPlan, RTChecks&: Checks, HasBranchWeights);
8354
8355 if (!IsInnerLoop)
8356 LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" << F->getName()
8357 << "\"\n");
8358 LVP.executePlan(BestVF: VF.Width, BestUF: IC, BestVPlan&: BestPlan, ILV&: LB, DT);
8359 ++LoopsVectorized;
8360 }
8361
8362 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
8363 "DT not preserved correctly");
8364
8365 return true;
8366}
8367
8368LoopVectorizeResult LoopVectorizePass::runImpl(Function &F) {
8369
8370 // Don't attempt if
8371 // 1. the target claims to have no vector registers, and
8372 // 2. interleaving won't help ILP.
8373 //
8374 // The second condition is necessary because, even if the target has no
8375 // vector registers, loop vectorization may still enable scalar
8376 // interleaving.
8377 if (!TTI->getNumberOfRegisters(ClassID: TTI->getRegisterClassForType(Vector: true)) &&
8378 (TTI->getMaxInterleaveFactor(VF: ElementCount::getFixed(MinVal: 1), HasUnorderedReductions: false) < 2 ||
8379 TTI->getMaxInterleaveFactor(VF: ElementCount::getFixed(MinVal: 1), HasUnorderedReductions: true) < 2))
8380 return LoopVectorizeResult(false, false);
8381
8382 bool Changed = false, CFGChanged = false;
8383
8384 // The vectorizer requires loops to be in simplified form.
8385 // Since simplification may add new inner loops, it has to run before the
8386 // legality and profitability checks. This means running the loop vectorizer
8387 // will simplify all loops, regardless of whether anything end up being
8388 // vectorized.
8389 for (const auto &L : *LI)
8390 Changed |= CFGChanged |=
8391 simplifyLoop(L, DT, LI, SE, AC, MSSAU: nullptr, PreserveLCSSA: false /* PreserveLCSSA */);
8392
8393 // Build up a worklist of inner-loops to vectorize. This is necessary as
8394 // the act of vectorizing or partially unrolling a loop creates new loops
8395 // and can invalidate iterators across the loops.
8396 SmallVector<Loop *, 8> Worklist;
8397
8398 for (Loop *L : *LI)
8399 collectSupportedLoops(L&: *L, LI, ORE, V&: Worklist);
8400
8401 LoopsAnalyzed += Worklist.size();
8402
8403 // Now walk the identified inner loops.
8404 while (!Worklist.empty()) {
8405 Loop *L = Worklist.pop_back_val();
8406
8407 // For the inner loops we actually process, form LCSSA to simplify the
8408 // transform.
8409 Changed |= formLCSSARecursively(L&: *L, DT: *DT, LI, SE);
8410
8411 Changed |= CFGChanged |= processLoop(L);
8412
8413 if (Changed) {
8414 LAIs->clear();
8415
8416 // If CycleAnalysis was cached by a prior pass (e.g. DSE), it now holds
8417 // stale pointers to blocks that may have been deleted during
8418 // vectorization. Clear it so that BlockFrequencyAnalysis (if requested
8419 // for a later loop) recomputes it fresh.
8420 if (FAM->getCachedResult<CycleAnalysis>(IR&: F))
8421 FAM->clearAnalysis<CycleAnalysis>(IR&: F);
8422
8423#ifndef NDEBUG
8424 if (VerifySCEV)
8425 SE->verify();
8426#endif
8427 }
8428 }
8429
8430 // Verify once per function rather than once per processed loop, which would
8431 // make the pass quadratic in the number of loops.
8432 assert((!Changed || !verifyFunction(F, &dbgs())) &&
8433 "Invalid IR produced by LoopVectorize");
8434
8435 // Process each loop nest in the function.
8436 return LoopVectorizeResult(Changed, CFGChanged);
8437}
8438
8439PreservedAnalyses LoopVectorizePass::run(Function &F,
8440 FunctionAnalysisManager &AM) {
8441 LI = &AM.getResult<LoopAnalysis>(IR&: F);
8442 // There are no loops in the function. Return before computing other
8443 // expensive analyses.
8444 if (LI->empty())
8445 return PreservedAnalyses::all();
8446 SE = &AM.getResult<ScalarEvolutionAnalysis>(IR&: F);
8447 TTI = &AM.getResult<TargetIRAnalysis>(IR&: F);
8448 DT = &AM.getResult<DominatorTreeAnalysis>(IR&: F);
8449 TLI = &AM.getResult<TargetLibraryAnalysis>(IR&: F);
8450 AC = &AM.getResult<AssumptionAnalysis>(IR&: F);
8451 DB = &AM.getResult<DemandedBitsAnalysis>(IR&: F);
8452 ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F);
8453 LAIs = &AM.getResult<LoopAccessAnalysis>(IR&: F);
8454 AA = &AM.getResult<AAManager>(IR&: F);
8455
8456 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
8457 PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(IR&: *F.getParent());
8458 FAM = &AM;
8459 GetBFI = [&AM, &F]() -> BlockFrequencyInfo & {
8460 return AM.getResult<BlockFrequencyAnalysis>(IR&: F);
8461 };
8462 LoopVectorizeResult Result = runImpl(F);
8463 if (!Result.MadeAnyChange)
8464 return PreservedAnalyses::all();
8465 PreservedAnalyses PA;
8466
8467 if (isAssignmentTrackingEnabled(M: *F.getParent())) {
8468 for (auto &BB : F)
8469 RemoveRedundantDbgInstrs(BB: &BB);
8470 }
8471
8472 PA.preserve<LoopAnalysis>();
8473 PA.preserve<DominatorTreeAnalysis>();
8474 PA.preserve<ScalarEvolutionAnalysis>();
8475 PA.preserve<LoopAccessAnalysis>();
8476
8477 if (Result.MadeCFGChange) {
8478 // Making CFG changes likely means a loop got vectorized. Indicate that
8479 // extra simplification passes should be run.
8480 // TODO: MadeCFGChanges is not a prefect proxy. Extra passes should only
8481 // be run if runtime checks have been added.
8482 AM.getResult<ShouldRunExtraVectorPasses>(IR&: F);
8483 PA.preserve<ShouldRunExtraVectorPasses>();
8484 } else {
8485 PA.preserveSet<CFGAnalyses>();
8486 }
8487 return PA;
8488}
8489
8490void LoopVectorizePass::printPipeline(
8491 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
8492 static_cast<PassInfoMixin<LoopVectorizePass> *>(this)->printPipeline(
8493 OS, MapClassName2PassName);
8494
8495 OS << '<';
8496 OS << (InterleaveOnlyWhenForced ? "" : "no-") << "interleave-forced-only;";
8497 OS << (VectorizeOnlyWhenForced ? "" : "no-") << "vectorize-forced-only;";
8498 OS << '>';
8499}
8500