1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
56#include "llvm/Transforms/Vectorize/LoopVectorize.h"
57#include "LoopVectorizationPlanner.h"
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanAnalysis.h"
61#include "VPlanCFG.h"
62#include "VPlanHelpers.h"
63#include "VPlanPatternMatch.h"
64#include "VPlanTransforms.h"
65#include "VPlanUtils.h"
66#include "VPlanVerifier.h"
67#include "llvm/ADT/APInt.h"
68#include "llvm/ADT/ArrayRef.h"
69#include "llvm/ADT/DenseMap.h"
70#include "llvm/ADT/Hashing.h"
71#include "llvm/ADT/MapVector.h"
72#include "llvm/ADT/STLExtras.h"
73#include "llvm/ADT/SmallPtrSet.h"
74#include "llvm/ADT/SmallVector.h"
75#include "llvm/ADT/Statistic.h"
76#include "llvm/ADT/StringRef.h"
77#include "llvm/ADT/Twine.h"
78#include "llvm/ADT/TypeSwitch.h"
79#include "llvm/ADT/iterator_range.h"
80#include "llvm/Analysis/AssumptionCache.h"
81#include "llvm/Analysis/BasicAliasAnalysis.h"
82#include "llvm/Analysis/BlockFrequencyInfo.h"
83#include "llvm/Analysis/BranchProbabilityInfo.h"
84#include "llvm/Analysis/CFG.h"
85#include "llvm/Analysis/CodeMetrics.h"
86#include "llvm/Analysis/CycleAnalysis.h"
87#include "llvm/Analysis/DemandedBits.h"
88#include "llvm/Analysis/GlobalsModRef.h"
89#include "llvm/Analysis/LoopAccessAnalysis.h"
90#include "llvm/Analysis/LoopAnalysisManager.h"
91#include "llvm/Analysis/LoopInfo.h"
92#include "llvm/Analysis/LoopIterator.h"
93#include "llvm/Analysis/OptimizationRemarkEmitter.h"
94#include "llvm/Analysis/ProfileSummaryInfo.h"
95#include "llvm/Analysis/ScalarEvolution.h"
96#include "llvm/Analysis/ScalarEvolutionExpressions.h"
97#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
98#include "llvm/Analysis/TargetLibraryInfo.h"
99#include "llvm/Analysis/TargetTransformInfo.h"
100#include "llvm/Analysis/ValueTracking.h"
101#include "llvm/Analysis/VectorUtils.h"
102#include "llvm/IR/Attributes.h"
103#include "llvm/IR/BasicBlock.h"
104#include "llvm/IR/CFG.h"
105#include "llvm/IR/Constant.h"
106#include "llvm/IR/Constants.h"
107#include "llvm/IR/DataLayout.h"
108#include "llvm/IR/DebugInfo.h"
109#include "llvm/IR/DebugLoc.h"
110#include "llvm/IR/DerivedTypes.h"
111#include "llvm/IR/DiagnosticInfo.h"
112#include "llvm/IR/Dominators.h"
113#include "llvm/IR/Function.h"
114#include "llvm/IR/IRBuilder.h"
115#include "llvm/IR/InstrTypes.h"
116#include "llvm/IR/Instruction.h"
117#include "llvm/IR/Instructions.h"
118#include "llvm/IR/IntrinsicInst.h"
119#include "llvm/IR/Intrinsics.h"
120#include "llvm/IR/MDBuilder.h"
121#include "llvm/IR/Metadata.h"
122#include "llvm/IR/Module.h"
123#include "llvm/IR/Operator.h"
124#include "llvm/IR/PatternMatch.h"
125#include "llvm/IR/ProfDataUtils.h"
126#include "llvm/IR/Type.h"
127#include "llvm/IR/Use.h"
128#include "llvm/IR/User.h"
129#include "llvm/IR/Value.h"
130#include "llvm/IR/Verifier.h"
131#include "llvm/Support/Casting.h"
132#include "llvm/Support/CommandLine.h"
133#include "llvm/Support/Debug.h"
134#include "llvm/Support/ErrorHandling.h"
135#include "llvm/Support/InstructionCost.h"
136#include "llvm/Support/MathExtras.h"
137#include "llvm/Support/NativeFormatting.h"
138#include "llvm/Support/raw_ostream.h"
139#include "llvm/Transforms/Utils/BasicBlockUtils.h"
140#include "llvm/Transforms/Utils/InjectTLIMappings.h"
141#include "llvm/Transforms/Utils/Local.h"
142#include "llvm/Transforms/Utils/LoopSimplify.h"
143#include "llvm/Transforms/Utils/LoopUtils.h"
144#include "llvm/Transforms/Utils/LoopVersioning.h"
145#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
146#include "llvm/Transforms/Utils/SizeOpts.h"
147#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
148#include <algorithm>
149#include <cassert>
150#include <cmath>
151#include <cstdint>
152#include <functional>
153#include <iterator>
154#include <memory>
155#include <string>
156#include <tuple>
157#include <utility>
158
159using namespace llvm;
160using namespace SCEVPatternMatch;
161using namespace LoopVectorizationUtils;
162
163#define LV_NAME "loop-vectorize"
164#define DEBUG_TYPE LV_NAME
165
166#ifndef NDEBUG
167const char VerboseDebug[] = DEBUG_TYPE "-verbose";
168#endif
169
170STATISTIC(LoopsVectorized, "Number of loops vectorized");
171STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
172STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
173STATISTIC(LoopsEarlyExitVectorized, "Number of early exit loops vectorized");
174STATISTIC(LoopsPartialAliasVectorized,
175 "Number of partial aliasing loops vectorized");
176
177static cl::opt<bool> EnableEpilogueVectorization(
178 "enable-epilogue-vectorization", cl::init(Val: true), cl::Hidden,
179 cl::desc("Enable vectorization of epilogue loops."));
180
181static cl::opt<ElementCount> EpilogueVectorizationForceVF(
182 "epilogue-vectorization-force-VF", cl::init(Val: ElementCount::getFixed(MinVal: 1)),
183 cl::Hidden,
184 cl::desc("When epilogue vectorization is enabled, and a value greater than "
185 "1 is specified, forces the given VF for all applicable epilogue "
186 "loops. Note: This allows all scalable VFs >= vscale x 1."));
187
188static cl::opt<unsigned> EpilogueVectorizationMinVF(
189 "epilogue-vectorization-minimum-VF", cl::Hidden,
190 cl::desc("Only loops with vectorization factor equal to or larger than "
191 "the specified value are considered for epilogue vectorization."));
192
193/// Loops with a known constant trip count below this number are vectorized only
194/// if no scalar iteration overheads are incurred.
195static cl::opt<unsigned> TinyTripCountVectorThreshold(
196 "vectorizer-min-trip-count", cl::init(Val: 16), cl::Hidden,
197 cl::desc("Loops with a constant trip count that is smaller than this "
198 "value are vectorized only if no scalar iteration overheads "
199 "are incurred."));
200
201static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
202 "vectorize-memory-check-threshold", cl::init(Val: 128), cl::Hidden,
203 cl::desc("The maximum allowed number of runtime memory checks"));
204
205static cl::opt<bool> ForcePartialAliasingVectorization(
206 "force-partial-aliasing-vectorization", cl::init(Val: false), cl::Hidden,
207 cl::desc("Replace pointer diff checks with alias masks."));
208
209/// Option tail-folding-policy controls the tail-folding strategy and lists all
210/// available options. The vectorizer will attempt to fold the tail-loop into
211/// the vector loop (main/epilogue loops) and predicate the instructions
212/// accordingly. If tail-folding fails, there are different fallback strategies
213/// depending on these values:
214enum class TailFoldingPolicyTy { None = 0, PreferFoldTail, MustFoldTail };
215
216static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
217 "tail-folding-policy", cl::init(Val: TailFoldingPolicyTy::None), cl::Hidden,
218 cl::desc("Tail-folding preferences over creating an epilogue loop."),
219 cl::values(
220 clEnumValN(TailFoldingPolicyTy::None, "dont-fold-tail",
221 "Don't tail-fold loops."),
222 clEnumValN(TailFoldingPolicyTy::PreferFoldTail, "prefer-fold-tail",
223 "prefer tail-folding, otherwise create an epilogue when "
224 "appropriate."),
225 clEnumValN(TailFoldingPolicyTy::MustFoldTail, "must-fold-tail",
226 "always tail-fold, don't attempt vectorization if "
227 "tail-folding fails.")));
228
229static cl::opt<TailFoldingPolicyTy> EpilogueTailFoldingPolicy(
230 "epilogue-tail-folding-policy", cl::Hidden,
231 cl::desc(
232 "Epilogue-tail-folding preferences over creating an epilogue loop."),
233 cl::values(
234 clEnumValN(TailFoldingPolicyTy::None, "dont-fold-tail",
235 "Don't tail-fold loops."),
236 clEnumValN(TailFoldingPolicyTy::PreferFoldTail, "prefer-fold-tail",
237 "prefer tail-folding, otherwise create an epilogue when "
238 "appropriate.")));
239
240static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
241 "force-tail-folding-style", cl::desc("Force the tail folding style"),
242 cl::init(Val: TailFoldingStyle::None),
243 cl::values(
244 clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"),
245 clEnumValN(
246 TailFoldingStyle::Data, "data",
247 "Create lane mask for data only, using active.lane.mask intrinsic"),
248 clEnumValN(TailFoldingStyle::DataWithoutLaneMask,
249 "data-without-lane-mask",
250 "Create lane mask with compare/stepvector"),
251 clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control",
252 "Create lane mask using active.lane.mask intrinsic, and use "
253 "it for both data and control flow"),
254 clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl",
255 "Use predicated EVL instructions for tail folding. If EVL "
256 "is unsupported, fallback to data-without-lane-mask.")));
257
258static cl::opt<bool> EnableInterleavedMemAccesses(
259 "enable-interleaved-mem-accesses", cl::init(Val: false), cl::Hidden,
260 cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
261
262/// An interleave-group may need masking if it resides in a block that needs
263/// predication, or in order to mask away gaps.
264static cl::opt<bool> EnableMaskedInterleavedMemAccesses(
265 "enable-masked-interleaved-mem-accesses", cl::init(Val: false), cl::Hidden,
266 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));
267
268static cl::opt<unsigned> ForceTargetNumScalarRegs(
269 "force-target-num-scalar-regs", cl::init(Val: 0), cl::Hidden,
270 cl::desc("A flag that overrides the target's number of scalar registers."));
271
272static cl::opt<unsigned> ForceTargetNumVectorRegs(
273 "force-target-num-vector-regs", cl::init(Val: 0), cl::Hidden,
274 cl::desc("A flag that overrides the target's number of vector registers."));
275
276static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor(
277 "force-target-max-scalar-interleave", cl::init(Val: 0), cl::Hidden,
278 cl::desc("A flag that overrides the target's max interleave factor for "
279 "scalar loops."));
280
281static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(
282 "force-target-max-vector-interleave", cl::init(Val: 0), cl::Hidden,
283 cl::desc("A flag that overrides the target's max interleave factor for "
284 "vectorized loops."));
285
286static cl::opt<unsigned> SmallLoopCost(
287 "small-loop-cost", cl::init(Val: 20), cl::Hidden,
288 cl::desc(
289 "The cost of a loop that is considered 'small' by the interleaver."));
290
291static cl::opt<bool> LoopVectorizeWithBlockFrequency(
292 "loop-vectorize-with-block-frequency", cl::init(Val: true), cl::Hidden,
293 cl::desc("Enable the use of the block frequency analysis to access PGO "
294 "heuristics minimizing code growth in cold regions and being more "
295 "aggressive in hot regions."));
296
297// Runtime interleave loops for load/store throughput.
298static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
299 "enable-loadstore-runtime-interleave", cl::init(Val: true), cl::Hidden,
300 cl::desc(
301 "Enable runtime interleaving until load/store ports are saturated"));
302
303// TODO: Move size-based thresholds out of legality checking, make cost based
304// decisions instead of hard thresholds.
305static cl::opt<unsigned> VectorizeSCEVCheckThreshold(
306 "vectorize-scev-check-threshold", cl::init(Val: 16), cl::Hidden,
307 cl::desc("The maximum number of SCEV checks allowed."));
308
309static cl::opt<unsigned> PragmaVectorizeSCEVCheckThreshold(
310 "pragma-vectorize-scev-check-threshold", cl::init(Val: 128), cl::Hidden,
311 cl::desc("The maximum number of SCEV checks allowed with a "
312 "vectorize(enable) pragma"));
313
314static cl::opt<bool> EnableIndVarRegisterHeur(
315 "enable-ind-var-reg-heur", cl::init(Val: true), cl::Hidden,
316 cl::desc("Count the induction variable only once when interleaving"));
317
318static cl::opt<unsigned> MaxNestedScalarReductionIC(
319 "max-nested-scalar-reduction-interleave", cl::init(Val: 2), cl::Hidden,
320 cl::desc("The maximum interleave count to use when interleaving a scalar "
321 "reduction in a nested loop."));
322
323static cl::opt<bool> ForceOrderedReductions(
324 "force-ordered-reductions", cl::init(Val: false), cl::Hidden,
325 cl::desc("Enable the vectorisation of loops with in-order (strict) "
326 "FP reductions"));
327
328static cl::opt<bool> PreferPredicatedReductionSelect(
329 "prefer-predicated-reduction-select", cl::init(Val: false), cl::Hidden,
330 cl::desc(
331 "Prefer predicating a reduction operation over an after loop select."));
332
333static cl::opt<bool> EnableVPlanNativePath(
334 "enable-vplan-native-path", cl::Hidden,
335 cl::desc("Enable VPlan-native vectorization path with "
336 "support for outer loop vectorization."));
337
338cl::opt<bool>
339 llvm::VerifyEachVPlan("vplan-verify-each",
340#ifdef EXPENSIVE_CHECKS
341 cl::init(true),
342#else
343 cl::init(Val: false),
344#endif
345 cl::Hidden,
346 cl::desc("Verify VPlans after VPlan transforms."));
347
348#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
349cl::opt<bool> llvm::VPlanPrintBeforeAll(
350 "vplan-print-before-all", cl::init(false), cl::Hidden,
351 cl::desc("Print VPlans before all VPlan transformations."));
352
353cl::opt<bool> llvm::VPlanPrintAfterAll(
354 "vplan-print-after-all", cl::init(false), cl::Hidden,
355 cl::desc("Print VPlans after all VPlan transformations."));
356
357cl::list<std::string> llvm::VPlanPrintBeforePasses(
358 "vplan-print-before", cl::Hidden,
359 cl::desc("Print VPlans before specified VPlan transformations (regexp)."));
360
361cl::list<std::string> llvm::VPlanPrintAfterPasses(
362 "vplan-print-after", cl::Hidden,
363 cl::desc("Print VPlans after specified VPlan transformations (regexp)."));
364
365cl::opt<bool> llvm::VPlanPrintVectorRegionScope(
366 "vplan-print-vector-region-scope", cl::init(false), cl::Hidden,
367 cl::desc("Limit VPlan printing to vector loop region in "
368 "`-vplan-print-after*` if the plan has one."));
369#endif
370
371cl::opt<bool> llvm::EnableLoopInterleaving(
372 "interleave-loops", cl::init(Val: true), cl::Hidden,
373 cl::desc("Enable loop interleaving in Loop vectorization passes"));
374cl::opt<bool> llvm::EnableLoopVectorization(
375 "vectorize-loops", cl::init(Val: true), cl::Hidden,
376 cl::desc("Run the Loop vectorization passes"));
377
378namespace llvm {
379cl::opt<unsigned> ForceTargetInstructionCost(
380 "force-target-instruction-cost", cl::init(Val: 0), cl::Hidden,
381 cl::desc("A flag that overrides the target's expected cost for "
382 "an instruction to a single constant value. Mostly "
383 "useful for getting consistent testing."));
384
385/// The number of stores in a loop that are allowed to need predication.
386cl::opt<unsigned> NumberOfStoresToPredicate(
387 "vectorize-num-stores-pred", cl::init(Val: 1), cl::Hidden,
388 cl::desc("Max number of stores to be predicated behind an if."));
389
390// This flag enables the stress testing of the VPlan H-CFG construction in the
391// VPlan-native vectorization path. It must be used in conjuction with
392// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
393// verification of the H-CFGs built.
394cl::opt<bool> VPlanBuildOuterloopStressTest(
395 "vplan-build-outerloop-stress-test", cl::init(Val: false), cl::Hidden,
396 cl::desc(
397 "Build VPlan for every supported loop nest in the function and bail "
398 "out right after the build (stress test the VPlan H-CFG construction "
399 "in the VPlan-native vectorization path)."));
400} // namespace llvm
401
402static cl::opt<cl::boolOrDefault>
403 ForceMaskedDivRem("force-widen-divrem-via-masked-intrinsic", cl::Hidden,
404 cl::desc("Override cost based masked intrinsic widening "
405 "for div/rem instructions"));
406
407static cl::opt<bool> EnableEarlyExitVectorization(
408 "enable-early-exit-vectorization", cl::init(Val: true), cl::Hidden,
409 cl::desc(
410 "Enable vectorization of early exit loops with uncountable exits."));
411
412static cl::opt<bool> EnableEarlyExitVectorizationWithSideEffects(
413 "enable-early-exit-vectorization-with-side-effects", cl::init(Val: false),
414 cl::Hidden,
415 cl::desc("Enable vectorization of early exit loops with uncountable exits "
416 "and side effects"));
417
418// Returns true if the epilogue VF has been set to a non-zero value other than
419// VF=1 (scalar).
420static bool hasForcedEpilogueVF() {
421 return EpilogueVectorizationForceVF.isNonZero() &&
422 EpilogueVectorizationForceVF != ElementCount::getFixed(MinVal: 1);
423}
424
425// Likelyhood of bypassing the vectorized loop because there are zero trips left
426// after prolog. See `emitIterationCountCheck`.
427static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
428
429/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
430/// ElementCount to include loops whose trip count is a function of vscale.
431static ElementCount getSmallConstantTripCount(ScalarEvolution *SE,
432 const Loop *L) {
433 if (unsigned ExpectedTC = SE->getSmallConstantTripCount(L))
434 return ElementCount::getFixed(MinVal: ExpectedTC);
435
436 const SCEV *BTC = SE->getBackedgeTakenCount(L);
437 if (isa<SCEVCouldNotCompute>(Val: BTC))
438 return ElementCount::getFixed(MinVal: 0);
439
440 const SCEV *ExitCount = SE->getTripCountFromExitCount(ExitCount: BTC, EvalTy: BTC->getType(), L);
441 if (isa<SCEVVScale>(Val: ExitCount))
442 return ElementCount::getScalable(MinVal: 1);
443
444 const APInt *Scale;
445 if (match(S: ExitCount, P: m_scev_Mul(Op0: m_scev_APInt(C&: Scale), Op1: m_SCEVVScale())))
446 if (cast<SCEVMulExpr>(Val: ExitCount)->hasNoUnsignedWrap())
447 if (Scale->getActiveBits() <= 32)
448 return ElementCount::getScalable(MinVal: Scale->getZExtValue());
449
450 return ElementCount::getFixed(MinVal: 0);
451}
452
453/// Get the maximum trip count for \p L from the SCEV unsigned range, excluding
454/// zero from the range. Only valid when not folding the tail, as the minimum
455/// iteration count check guards against a zero trip count. Returns 0 if
456/// unknown.
457static unsigned getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE,
458 Loop *L) {
459 const SCEV *BTC = PSE.getBackedgeTakenCount();
460 if (isa<SCEVCouldNotCompute>(Val: BTC))
461 return 0;
462 ScalarEvolution *SE = PSE.getSE();
463 const SCEV *TripCount = SE->getTripCountFromExitCount(ExitCount: BTC, EvalTy: BTC->getType(), L);
464 ConstantRange TCRange = SE->getUnsignedRange(S: TripCount);
465 APInt MaxTCFromRange = TCRange.getUnsignedMax();
466 if (!MaxTCFromRange.isZero() && MaxTCFromRange.getActiveBits() <= 32)
467 return MaxTCFromRange.getZExtValue();
468 return 0;
469}
470
471/// Returns "best known" trip count, which is either a valid positive trip count
472/// or std::nullopt when an estimate cannot be made (including when the trip
473/// count would overflow), for the specified loop \p L as defined by the
474/// following procedure:
475/// 1) Returns exact trip count if it is known.
476/// 2) Returns expected trip count according to profile data if any.
477/// 3) Returns upper bound estimate if known, if \p CanUseConstantMax, and
478/// if \p ComputeUpperBoundOnly is false.
479/// 4) Returns the maximum trip count from the SCEV range excluding zero,
480/// if \p CanUseConstantMax and \p CanExcludeZeroTrips.
481/// 5) Returns std::nullopt if all of the above failed.
482static std::optional<ElementCount> getSmallBestKnownTC(
483 PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax = true,
484 bool CanExcludeZeroTrips = false, bool ComputeUpperBoundOnly = false) {
485 // Check if exact trip count is known.
486 if (auto ExpectedTC = getSmallConstantTripCount(SE: PSE.getSE(), L))
487 return ExpectedTC;
488
489 // Check if there is an expected trip count available from profile data.
490 // An estimate of zero means the loop is estimated not to be entered; it is
491 // not a usable trip count for the profitability decisions below (and would
492 // e.g. divide by zero when scaling runtime check cost), so treat it as
493 // unknown.
494 if (LoopVectorizeWithBlockFrequency && !ComputeUpperBoundOnly)
495 if (unsigned EstimatedTC = getLoopEstimatedTripCount(L).value_or(u: 0))
496 return ElementCount::getFixed(MinVal: EstimatedTC);
497
498 if (!CanUseConstantMax)
499 return std::nullopt;
500
501 // Check if upper bound estimate is known.
502 if (unsigned ExpectedTC = PSE.getSmallConstantMaxTripCount())
503 return ElementCount::getFixed(MinVal: ExpectedTC);
504
505 // Get the maximum trip count from the SCEV range excluding zero. This is
506 // only safe when not folding the tail, as the minimum iteration count check
507 // prevents entering the vector loop with a zero trip count.
508 if (CanUseConstantMax && CanExcludeZeroTrips)
509 if (unsigned RefinedTC = getMaxTCFromNonZeroRange(PSE, L))
510 return ElementCount::getFixed(MinVal: RefinedTC);
511
512 return std::nullopt;
513}
514
515namespace {
516// Forward declare GeneratedRTChecks.
517class GeneratedRTChecks;
518
519using SCEV2ValueTy = DenseMap<const SCEV *, Value *>;
520} // namespace
521
522namespace llvm {
523
524AnalysisKey ShouldRunExtraVectorPasses::Key;
525
526/// InnerLoopVectorizer vectorizes loops which contain only one basic
527/// block to a specified vectorization factor (VF).
528/// This class performs the widening of scalars into vectors, or multiple
529/// scalars. This class also implements the following features:
530/// * It inserts an epilogue loop for handling loops that don't have iteration
531/// counts that are known to be a multiple of the vectorization factor.
532/// * It handles the code generation for reduction variables.
533/// * Scalarization (implementation using scalars) of un-vectorizable
534/// instructions.
535/// InnerLoopVectorizer does not perform any vectorization-legality
536/// checks, and relies on the caller to check for the different legality
537/// aspects. The InnerLoopVectorizer relies on the
538/// LoopVectorizationLegality class to provide information about the induction
539/// and reduction variables that were found to a given vectorization factor.
540class InnerLoopVectorizer {
541public:
542 InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
543 LoopInfo *LI, DominatorTree *DT,
544 const TargetTransformInfo *TTI, AssumptionCache *AC,
545 ElementCount VecWidth, unsigned UnrollFactor,
546 GeneratedRTChecks &RTChecks, VPlan &Plan)
547 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TTI(TTI), AC(AC),
548 VF(VecWidth), UF(UnrollFactor), Builder(PSE.getSE()->getContext()),
549 RTChecks(RTChecks), Plan(Plan),
550 VectorPHVPBB(cast<VPBasicBlock>(
551 Val: Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
552
553 virtual ~InnerLoopVectorizer() = default;
554
555 /// Creates a basic block for the scalar preheader. Both
556 /// EpilogueVectorizerMainLoop and EpilogueVectorizerEpilogueLoop overwrite
557 /// the method to create additional blocks and checks needed for epilogue
558 /// vectorization.
559 virtual BasicBlock *createVectorizedLoopSkeleton();
560
561 /// Fix the vectorized code, taking care of header phi's, and more.
562 void fixVectorizedLoop(VPTransformState &State);
563
564protected:
565 friend class LoopVectorizationPlanner;
566
567 /// Create and return a new IR basic block for the scalar preheader whose name
568 /// is prefixed with \p Prefix.
569 BasicBlock *createScalarPreheader(StringRef Prefix);
570
571 /// Allow subclasses to override and print debug traces before/after vplan
572 /// execution, when trace information is requested.
573 virtual void printDebugTracesAtStart() {}
574 virtual void printDebugTracesAtEnd() {}
575
576 /// The original loop.
577 Loop *OrigLoop;
578
579 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
580 /// dynamic knowledge to simplify SCEV expressions and converts them to a
581 /// more usable form.
582 PredicatedScalarEvolution &PSE;
583
584 /// Loop Info.
585 LoopInfo *LI;
586
587 /// Dominator Tree.
588 DominatorTree *DT;
589
590 /// Target Transform Info.
591 const TargetTransformInfo *TTI;
592
593 /// Assumption Cache.
594 AssumptionCache *AC;
595
596 /// The vectorization SIMD factor to use. Each vector will have this many
597 /// vector elements.
598 ElementCount VF;
599
600 /// The vectorization unroll factor to use. Each scalar is vectorized to this
601 /// many different vector instructions.
602 unsigned UF;
603
604 /// The builder that we use
605 IRBuilder<> Builder;
606
607 // --- Vectorization state ---
608
609 /// Structure to hold information about generated runtime checks, responsible
610 /// for cleaning the checks, if vectorization turns out unprofitable.
611 GeneratedRTChecks &RTChecks;
612
613 VPlan &Plan;
614
615 /// The vector preheader block of \p Plan, used as target for check blocks
616 /// introduced during skeleton creation.
617 VPBasicBlock *VectorPHVPBB;
618};
619
620/// Encapsulate information regarding vectorization of a loop and its epilogue.
621/// This information is meant to be updated and used across two stages of
622/// epilogue vectorization.
623struct EpilogueLoopVectorizationInfo {
624 ElementCount MainLoopVF = ElementCount::getFixed(MinVal: 0);
625 unsigned MainLoopUF = 0;
626 ElementCount EpilogueVF = ElementCount::getFixed(MinVal: 0);
627 unsigned EpilogueUF = 0;
628 BasicBlock *MainLoopIterationCountCheck = nullptr;
629 BasicBlock *EpilogueIterationCountCheck = nullptr;
630 Value *VectorTripCount = nullptr;
631
632 EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF,
633 ElementCount EVF, unsigned EUF)
634 : MainLoopVF(MVF), MainLoopUF(MUF), EpilogueVF(EVF), EpilogueUF(EUF) {
635 assert(EUF == 1 &&
636 "A high UF for the epilogue loop is likely not beneficial.");
637 }
638};
639
640/// An extension of the inner loop vectorizer that creates a skeleton for a
641/// vectorized loop that has its epilogue (residual) also vectorized.
642/// The idea is to run the vplan on a given loop twice, firstly to setup the
643/// skeleton and vectorize the main loop, and secondly to complete the skeleton
644/// from the first step and vectorize the epilogue. This is achieved by
645/// deriving two concrete strategy classes from this base class and invoking
646/// them in succession from the loop vectorizer planner.
647class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer {
648public:
649 InnerLoopAndEpilogueVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
650 LoopInfo *LI, DominatorTree *DT,
651 const TargetTransformInfo *TTI,
652 AssumptionCache *AC,
653 EpilogueLoopVectorizationInfo &EPI,
654 GeneratedRTChecks &Checks, VPlan &Plan,
655 ElementCount VecWidth, unsigned UnrollFactor)
656 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, VecWidth,
657 UnrollFactor, Checks, Plan),
658 EPI(EPI) {}
659
660 /// Holds and updates state information required to vectorize the main loop
661 /// and its epilogue in two separate passes. This setup helps us avoid
662 /// regenerating and recomputing runtime safety checks. It also helps us to
663 /// shorten the iteration-count-check path length for the cases where the
664 /// iteration count of the loop is so small that the main vector loop is
665 /// completely skipped.
666 EpilogueLoopVectorizationInfo &EPI;
667};
668
669/// A specialized derived class of inner loop vectorizer that performs
670/// vectorization of *main* loops in the process of vectorizing loops and their
671/// epilogues.
672class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
673public:
674 EpilogueVectorizerMainLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
675 LoopInfo *LI, DominatorTree *DT,
676 const TargetTransformInfo *TTI,
677 AssumptionCache *AC,
678 EpilogueLoopVectorizationInfo &EPI,
679 GeneratedRTChecks &Check, VPlan &Plan)
680 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, EPI,
681 Check, Plan, EPI.MainLoopVF,
682 EPI.MainLoopUF) {}
683
684protected:
685 void printDebugTracesAtStart() override;
686 void printDebugTracesAtEnd() override;
687};
688
689// A specialized derived class of inner loop vectorizer that performs
690// vectorization of *epilogue* loops in the process of vectorizing loops and
691// their epilogues.
692class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
693public:
694 EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
695 LoopInfo *LI, DominatorTree *DT,
696 const TargetTransformInfo *TTI,
697 AssumptionCache *AC,
698 EpilogueLoopVectorizationInfo &EPI,
699 GeneratedRTChecks &Checks, VPlan &Plan)
700 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, EPI,
701 Checks, Plan, EPI.EpilogueVF,
702 EPI.EpilogueUF) {}
703 /// Implements the interface for creating a vectorized skeleton using the
704 /// *epilogue loop* strategy (i.e., the second pass of VPlan execution).
705 BasicBlock *createVectorizedLoopSkeleton() final;
706
707protected:
708 void printDebugTracesAtStart() override;
709 void printDebugTracesAtEnd() override;
710};
711} // end namespace llvm
712
713/// Look for a meaningful debug location on the instruction or its operands.
714static DebugLoc getDebugLocFromInstOrOperands(Instruction *I) {
715 if (!I)
716 return DebugLoc::getUnknown();
717
718 DebugLoc Empty;
719 if (I->getDebugLoc() != Empty)
720 return I->getDebugLoc();
721
722 for (Use &Op : I->operands()) {
723 if (Instruction *OpInst = dyn_cast<Instruction>(Val&: Op))
724 if (OpInst->getDebugLoc() != Empty)
725 return OpInst->getDebugLoc();
726 }
727
728 return I->getDebugLoc();
729}
730
731namespace llvm {
732
733/// Return the runtime value for VF.
734Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF) {
735 return B.CreateElementCount(Ty, EC: VF);
736}
737
738} // end namespace llvm
739
740namespace llvm {
741
742// Loop vectorization cost-model hints how the epilogue/tail loop should be
743// lowered.
744enum EpilogueLowering {
745
746 // The default: allowing epilogues.
747 CM_EpilogueAllowed,
748
749 // Vectorization with OptForSize: don't allow epilogues.
750 CM_EpilogueNotAllowedOptSize,
751
752 // A special case of vectorisation with OptForSize: loops with a very small
753 // trip count are considered for vectorization under OptForSize, thereby
754 // making sure the cost of their loop body is dominant, free of runtime
755 // guards and scalar iteration overheads.
756 CM_EpilogueNotAllowedLowTripLoop,
757
758 // Loop hint indicating an epilogue is undesired, apply tail folding.
759 CM_EpilogueNotNeededFoldTail,
760
761 // Directive indicating we must either fold the epilogue/tail or not vectorize
762 CM_EpilogueNotAllowedFoldTail
763};
764
765enum class AliasMaskingStatus { NotDecided, Disabled, Enabled };
766
767/// LoopVectorizationCostModel - estimates the expected speedups due to
768/// vectorization.
769/// In many cases vectorization is not profitable. This can happen because of
770/// a number of reasons. In this class we mainly attempt to predict the
771/// expected speedup/slowdowns due to the supported instruction set. We use the
772/// TargetTransformInfo to query the different backends for the cost of
773/// different operations.
774class LoopVectorizationCostModel {
775 friend class LoopVectorizationPlanner;
776
777public:
778 LoopVectorizationCostModel(EpilogueLowering SEL, Loop *L,
779 PredicatedScalarEvolution &PSE, LoopInfo *LI,
780 LoopVectorizationLegality *Legal,
781 const TargetTransformInfo &TTI,
782 const TargetLibraryInfo *TLI, AssumptionCache *AC,
783 OptimizationRemarkEmitter *ORE,
784 std::function<BlockFrequencyInfo &()> GetBFI,
785 const Function *F, InterleavedAccessInfo &IAI,
786 VFSelectionContext &Config)
787 : Config(Config), EpilogueLoweringStatus(SEL), TheLoop(L), PSE(PSE),
788 LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), AC(AC), ORE(ORE),
789 GetBFI(GetBFI), TheFunction(F), InterleaveInfo(IAI) {}
790
791 /// \return An upper bound for the vectorization factors (both fixed and
792 /// scalable). If the factors are 0, vectorization and interleaving should be
793 /// avoided up front.
794 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
795
796 /// Memory access instruction may be vectorized in more than one way.
797 /// Form of instruction after vectorization depends on cost.
798 /// This function takes cost-based decisions for Load/Store instructions
799 /// and collects them in a map. This decisions map is used for building
800 /// the lists of loop-uniform and loop-scalar instructions.
801 /// The calculated cost is saved with widening decision in order to
802 /// avoid redundant calculations.
803 void setCostBasedWideningDecision(ElementCount VF);
804
805 /// Collect values we want to ignore in the cost model.
806 void collectValuesToIgnore();
807
808 /// \returns True if it is more profitable to scalarize instruction \p I for
809 /// vectorization factor \p VF.
810 bool isProfitableToScalarize(Instruction *I, ElementCount VF) const {
811 assert(VF.isVector() &&
812 "Profitable to scalarize relevant only for VF > 1.");
813 assert(
814 TheLoop->isInnermost() &&
815 "cost-model should not be used for outer loops (in VPlan-native path)");
816
817 auto Scalars = InstsToScalarize.find(Key: VF);
818 assert(Scalars != InstsToScalarize.end() &&
819 "VF not yet analyzed for scalarization profitability");
820 return Scalars->second.contains(Key: I);
821 }
822
823 /// Returns true if \p I is known to be uniform after vectorization.
824 bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const {
825 assert(
826 TheLoop->isInnermost() &&
827 "cost-model should not be used for outer loops (in VPlan-native path)");
828
829 // If VF is scalar, then all instructions are trivially uniform.
830 if (VF.isScalar())
831 return true;
832
833 // Pseudo probes must be duplicated per vector lane so that the
834 // profiled loop trip count is not undercounted.
835 if (isa<PseudoProbeInst>(Val: I))
836 return false;
837
838 auto UniformsPerVF = Uniforms.find(Val: VF);
839 assert(UniformsPerVF != Uniforms.end() &&
840 "VF not yet analyzed for uniformity");
841 return UniformsPerVF->second.count(Ptr: I);
842 }
843
844 /// Returns true if \p I is known to be scalar after vectorization.
845 bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const {
846 assert(
847 TheLoop->isInnermost() &&
848 "cost-model should not be used for outer loops (in VPlan-native path)");
849 if (VF.isScalar())
850 return true;
851
852 auto ScalarsPerVF = Scalars.find(Val: VF);
853 assert(ScalarsPerVF != Scalars.end() &&
854 "Scalar values are not calculated for VF");
855 return ScalarsPerVF->second.count(Ptr: I);
856 }
857
858 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
859 /// for vectorization factor \p VF.
860 bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const {
861 const auto &MinBWs = Config.getMinimalBitwidths();
862 // Truncs must truncate at most to their destination type.
863 if (isa_and_nonnull<TruncInst>(Val: I) && MinBWs.contains(Key: I) &&
864 I->getType()->getScalarSizeInBits() < MinBWs.lookup(Key: I))
865 return false;
866 return VF.isVector() && MinBWs.contains(Key: I) &&
867 !isProfitableToScalarize(I, VF) &&
868 !isScalarAfterVectorization(I, VF);
869 }
870
871 /// Decision that was taken during cost calculation for memory instruction.
872 enum InstWidening {
873 CM_Unknown,
874 CM_Widen, // For consecutive accesses with stride +1.
875 CM_Widen_Reverse, // For consecutive accesses with stride -1.
876 CM_Interleave,
877 CM_GatherScatter,
878 CM_Scalarize,
879 /// A widening decision that has been invalidated after replacing the
880 /// corresponding recipe during VPlan transforms.
881 /// TODO: Remove once the legacy exit cost computation is retired.
882 CM_InvalidatedDecision
883 };
884
885 /// Save vectorization decision \p W and \p Cost taken by the cost model for
886 /// instruction \p I and vector width \p VF.
887 void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W,
888 InstructionCost Cost) {
889 assert(VF.isVector() && "Expected VF >=2");
890 WideningDecisions[{I, VF}] = {W, Cost};
891 }
892
893 /// Save vectorization decision \p W and \p Cost taken by the cost model for
894 /// interleaving group \p Grp and vector width \p VF.
895 void setWideningDecision(const InterleaveGroup<Instruction> *Grp,
896 ElementCount VF, InstWidening W,
897 InstructionCost Cost) {
898 assert(VF.isVector() && "Expected VF >=2");
899 /// Broadcast this decicion to all instructions inside the group.
900 /// When interleaving, the cost will only be assigned one instruction, the
901 /// insert position. For other cases, add the appropriate fraction of the
902 /// total cost to each instruction. This ensures accurate costs are used,
903 /// even if the insert position instruction is not used.
904 InstructionCost InsertPosCost = Cost;
905 InstructionCost OtherMemberCost = 0;
906 if (W != CM_Interleave)
907 OtherMemberCost = InsertPosCost = Cost / Grp->getNumMembers();
908 ;
909 for (auto *I : Grp->members()) {
910 if (Grp->getInsertPos() == I)
911 WideningDecisions[{I, VF}] = {W, InsertPosCost};
912 else
913 WideningDecisions[{I, VF}] = {W, OtherMemberCost};
914 }
915 }
916
917 /// Return the cost model decision for the given instruction \p I and vector
918 /// width \p VF. Return CM_Unknown if this instruction did not pass
919 /// through the cost modeling.
920 InstWidening getWideningDecision(Instruction *I, ElementCount VF) const {
921 assert(VF.isVector() && "Expected VF to be a vector VF");
922 assert(
923 TheLoop->isInnermost() &&
924 "cost-model should not be used for outer loops (in VPlan-native path)");
925
926 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
927 auto Itr = WideningDecisions.find(Val: InstOnVF);
928 if (Itr == WideningDecisions.end())
929 return CM_Unknown;
930 return Itr->second.first;
931 }
932
933 /// Return the vectorization cost for the given instruction \p I and vector
934 /// width \p VF.
935 InstructionCost getWideningCost(Instruction *I, ElementCount VF) {
936 assert(VF.isVector() && "Expected VF >=2");
937 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
938 assert(WideningDecisions.contains(InstOnVF) &&
939 "The cost is not calculated");
940 return WideningDecisions[InstOnVF].second;
941 }
942
943 /// Return True if instruction \p I is an optimizable truncate whose operand
944 /// is an induction variable. Such a truncate will be removed by adding a new
945 /// induction variable with the destination type.
946 bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) {
947 // If the instruction is not a truncate, return false.
948 auto *Trunc = dyn_cast<TruncInst>(Val: I);
949 if (!Trunc)
950 return false;
951
952 // Get the source and destination types of the truncate.
953 Type *SrcTy = toVectorTy(Scalar: Trunc->getSrcTy(), EC: VF);
954 Type *DestTy = toVectorTy(Scalar: Trunc->getDestTy(), EC: VF);
955
956 // If the truncate is free for the given types, return false. Replacing a
957 // free truncate with an induction variable would add an induction variable
958 // update instruction to each iteration of the loop. We exclude from this
959 // check the primary induction variable since it will need an update
960 // instruction regardless.
961 Value *Op = Trunc->getOperand(i_nocapture: 0);
962 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(Ty1: SrcTy, Ty2: DestTy))
963 return false;
964
965 // If the truncated value is not an induction variable, return false.
966 return Legal->isInductionPhi(V: Op);
967 }
968
969 /// Collects the instructions to scalarize for each predicated instruction in
970 /// the loop.
971 void collectInstsToScalarize(ElementCount VF);
972
973 /// Collect values that will not be widened, including Uniforms, Scalars, and
974 /// Instructions to Scalarize for the given \p VF.
975 /// The sets depend on CM decision for Load/Store instructions
976 /// that may be vectorized as interleave, gather-scatter or scalarized.
977 /// Also make a decision on what to do about call instructions in the loop
978 /// at that VF -- scalarize, call a known vector routine, or call a
979 /// vector intrinsic.
980 void collectNonVectorizedAndSetWideningDecisions(ElementCount VF) {
981 // Do the analysis once.
982 if (VF.isScalar() || Uniforms.contains(Val: VF))
983 return;
984 setCostBasedWideningDecision(VF);
985 collectLoopUniforms(VF);
986 collectLoopScalars(VF);
987 collectInstsToScalarize(VF);
988 }
989
990 /// Given costs for both strategies, return true if the scalar predication
991 /// lowering should be used for div/rem. This incorporates an override
992 /// option so it is not simply a cost comparison.
993 bool isDivRemScalarWithPredication(InstructionCost ScalarCost,
994 InstructionCost MaskedCost) const {
995 switch (ForceMaskedDivRem) {
996 case cl::boolOrDefault::BOU_UNSET:
997 return ScalarCost < MaskedCost;
998 case cl::boolOrDefault::BOU_TRUE:
999 return false;
1000 case cl::boolOrDefault::BOU_FALSE:
1001 return true;
1002 }
1003 llvm_unreachable("impossible case value");
1004 }
1005
1006 /// Returns true if \p I is an instruction which requires predication and
1007 /// for which our chosen predication strategy is scalarization (i.e. we
1008 /// don't have an alternate strategy such as masking available).
1009 /// \p VF is the vectorization factor that will be used to vectorize \p I.
1010 bool isScalarWithPredication(Instruction *I, ElementCount VF);
1011
1012 /// Wrapper function for LoopVectorizationLegality::isMaskRequired,
1013 /// that passes the Instruction \p I and if we fold tail.
1014 bool isMaskRequired(Instruction *I) const;
1015
1016 /// Returns true if \p I is an instruction that needs to be predicated
1017 /// at runtime. The result is independent of the predication mechanism.
1018 /// Superset of instructions that return true for isScalarWithPredication.
1019 bool isPredicatedInst(Instruction *I) const;
1020
1021 /// A helper function that returns how much we should divide the cost of a
1022 /// predicated block by. Typically this is the reciprocal of the block
1023 /// probability, i.e. if we return X we are assuming the predicated block will
1024 /// execute once for every X iterations of the loop header so the block should
1025 /// only contribute 1/X of its cost to the total cost calculation, but when
1026 /// optimizing for code size it will just be 1 as code size costs don't depend
1027 /// on execution probabilities.
1028 ///
1029 /// Note that if a block wasn't originally predicated but was predicated due
1030 /// to tail folding, the divisor will still be 1 because it will execute for
1031 /// every iteration of the loop header.
1032 inline uint64_t
1033 getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind,
1034 const BasicBlock *BB);
1035
1036 /// Returns true if an artificially high cost for emulated masked memrefs
1037 /// should be used.
1038 bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF) const;
1039
1040 /// Return the costs for our two available strategies for lowering a
1041 /// div/rem operation which requires speculating at least one lane.
1042 /// First result is for scalarization (will be invalid for scalable
1043 /// vectors); second is for the masked intrinsic strategy.
1044 std::pair<InstructionCost, InstructionCost>
1045 getDivRemSpeculationCost(Instruction *I, ElementCount VF);
1046
1047 /// If \p I is a memory instruction with a consecutive pointer that can be
1048 /// widened, returns the widening kind (CM_Widen or CM_Widen_Reverse) and
1049 /// std::nullopt otherwise.
1050 std::optional<InstWidening> memoryInstructionCanBeWidened(Instruction *I,
1051 ElementCount VF);
1052
1053 /// Returns true if \p I is a memory instruction in an interleaved-group
1054 /// of memory accesses that can be vectorized with wide vector loads/stores
1055 /// and shuffles.
1056 bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const;
1057
1058 /// Returns true if the target machine supports masked loads or stores
1059 /// for \p I's data type and alignment. The caller must ensure the access is
1060 /// consecutive or part of an interleave group.
1061 bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const;
1062
1063 /// Returns true if the target machine supports gather or scatter for \p I's
1064 /// data type and alignment.
1065 bool isLegalGatherOrScatter(Instruction *I, ElementCount VF) const;
1066
1067 /// Check if \p Instr belongs to any interleaved access group.
1068 bool isAccessInterleaved(Instruction *Instr) const {
1069 return InterleaveInfo.isInterleaved(Instr);
1070 }
1071
1072 /// Get the interleaved access group that \p Instr belongs to.
1073 const InterleaveGroup<Instruction> *
1074 getInterleavedAccessGroup(Instruction *Instr) const {
1075 return InterleaveInfo.getInterleaveGroup(Instr);
1076 }
1077
1078 /// Returns true if we're required to use a scalar epilogue for at least
1079 /// the final iteration of the original loop.
1080 bool requiresScalarEpilogue(bool IsVectorizing) const {
1081 if (!isEpilogueAllowed()) {
1082 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1083 return false;
1084 }
1085 // If we might exit from anywhere but the latch and early exit vectorization
1086 // is disabled, we must run the exiting iteration in scalar form.
1087 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
1088 !(EnableEarlyExitVectorization && Legal->hasUncountableEarlyExit())) {
1089 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
1090 "from latch block\n");
1091 return true;
1092 }
1093 if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
1094 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
1095 "interleaved group requires scalar epilogue\n");
1096 return true;
1097 }
1098 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1099 return false;
1100 }
1101
1102 /// Returns true if an epilogue is allowed (e.g., not prevented by
1103 /// optsize or a loop hint annotation).
1104 bool isEpilogueAllowed() const {
1105 return EpilogueLoweringStatus == CM_EpilogueAllowed;
1106 }
1107
1108 /// Returns true if tail-folding is preferred over an epilogue.
1109 bool preferTailFoldedLoop() const {
1110 return EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail ||
1111 EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail;
1112 }
1113
1114 /// Returns the TailFoldingStyle that is best for the current loop.
1115 TailFoldingStyle getTailFoldingStyle() const {
1116 return ChosenTailFoldingStyle;
1117 }
1118
1119 /// Selects and saves TailFoldingStyle.
1120 /// \param IsScalableVF true if scalable vector factors enabled.
1121 /// \param UserIC User specific interleave count.
1122 void setTailFoldingStyle(bool IsScalableVF, unsigned UserIC) {
1123 assert(ChosenTailFoldingStyle == TailFoldingStyle::None &&
1124 "Tail folding must not be selected yet.");
1125 if (!Legal->canFoldTailByMasking()) {
1126 ChosenTailFoldingStyle = TailFoldingStyle::None;
1127 return;
1128 }
1129
1130 // Default to TTI preference, but allow command line override.
1131 ChosenTailFoldingStyle = TTI.getPreferredTailFoldingStyle();
1132 if (ForceTailFoldingStyle.getNumOccurrences())
1133 ChosenTailFoldingStyle = ForceTailFoldingStyle.getValue();
1134
1135 if (ChosenTailFoldingStyle != TailFoldingStyle::DataWithEVL)
1136 return;
1137 // Override EVL styles if needed.
1138 // FIXME: Investigate opportunity for fixed vector factor.
1139 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
1140 TTI.hasActiveVectorLength() && !EnableVPlanNativePath;
1141 if (EVLIsLegal)
1142 return;
1143 // If for some reason EVL mode is unsupported, fallback to an epilogue
1144 // if it's allowed, or DataWithoutLaneMask otherwise.
1145 if (EpilogueLoweringStatus == CM_EpilogueAllowed ||
1146 EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail)
1147 ChosenTailFoldingStyle = TailFoldingStyle::None;
1148 else
1149 ChosenTailFoldingStyle = TailFoldingStyle::DataWithoutLaneMask;
1150
1151 LLVM_DEBUG(
1152 dbgs() << "LV: Preference for VP intrinsics indicated. Will "
1153 "not try to generate VP Intrinsics "
1154 << (UserIC > 1
1155 ? "since interleave count specified is greater than 1.\n"
1156 : "due to non-interleaving reasons.\n"));
1157 }
1158
1159 /// Returns true if all loop blocks should be masked to fold tail loop.
1160 bool foldTailByMasking() const {
1161 return getTailFoldingStyle() != TailFoldingStyle::None;
1162 }
1163
1164 void tryToEnablePartialAliasMasking() {
1165 assert(foldTailByMasking() && "Expected tail folding to be enabled!");
1166 assert(!foldTailWithEVL() &&
1167 "Did not expect to enable alias masking with EVL!");
1168 assert(PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided);
1169
1170 // Assume we fail to enable alias masking (in case we early exit).
1171 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
1172
1173 // Note: FixedOrderRecurrences are not supported yet as we cannot handle
1174 // the required `splice.right` with the alias-mask.
1175 if (!ForcePartialAliasingVectorization ||
1176 !Legal->getFixedOrderRecurrences().empty())
1177 return;
1178
1179 const RuntimePointerChecking *Checks = Legal->getRuntimePointerChecking();
1180 if (!Checks)
1181 return;
1182
1183 auto DiffChecks = Checks->getDiffChecks();
1184 if (!DiffChecks || DiffChecks->empty())
1185 return;
1186
1187 [[maybe_unused]] auto HasPointerArgs = [](CallBase *CB) {
1188 return any_of(Range: CB->args(), P: [](Value const *Arg) {
1189 return Arg->getType()->isPointerTy();
1190 });
1191 };
1192
1193 for (BasicBlock *BB : TheLoop->blocks()) {
1194 for (Instruction &I : *BB) {
1195 if (!isa<LoadInst, StoreInst>(Val: I)) {
1196 [[maybe_unused]] auto *Call = dyn_cast<CallInst>(Val: &I);
1197 assert(
1198 (!I.mayReadOrWriteMemory() || (Call && !HasPointerArgs(Call))) &&
1199 "Skipped unexpected memory access");
1200 continue;
1201 }
1202
1203 Type *ScalarTy = getLoadStoreType(I: &I);
1204 Value *Ptr = getLoadStorePointerOperand(V: &I);
1205
1206 // Currently, we can't handle alias masking in reverse. Reversing the
1207 // alias mask is not correct (or necessary). When combined with
1208 // tail-folding the active lane mask should only be reversed where the
1209 // alias-mask is true.
1210 if (Legal->isConsecutivePtr(AccessTy: ScalarTy, Ptr) == -1)
1211 return;
1212 }
1213 }
1214
1215 PartialAliasMaskingStatus = AliasMaskingStatus::Enabled;
1216 }
1217
1218 /// Returns true if all loop blocks should have partial aliases masked.
1219 bool maskPartialAliasing() const {
1220 return PartialAliasMaskingStatus == AliasMaskingStatus::Enabled;
1221 }
1222
1223 /// Returns true if the instructions in this block requires predication
1224 /// for any reason, e.g. because tail folding now requires a predicate
1225 /// or because the block in the original loop was predicated.
1226 bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const {
1227 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
1228 }
1229
1230 /// Returns true if VP intrinsics with explicit vector length support should
1231 /// be generated in the tail folded loop.
1232 bool foldTailWithEVL() const {
1233 return getTailFoldingStyle() == TailFoldingStyle::DataWithEVL;
1234 }
1235
1236 /// Returns true if the predicated reduction select should be used to set the
1237 /// incoming value for the reduction phi.
1238 bool usePredicatedReductionSelect(RecurKind RecurrenceKind) const {
1239 // Force to use predicated reduction select since the EVL of the
1240 // second-to-last iteration might not be VF*UF.
1241 if (foldTailWithEVL())
1242 return true;
1243
1244 // Force a predicated select with alias-masking to avoid propagating poison
1245 // values to the header phi for lanes outside the alias-mask.
1246 if (maskPartialAliasing())
1247 return true;
1248
1249 // Note: For FindLast recurrences we prefer a predicated select to simplify
1250 // matching in handleFindLastReductions(), rather than handle multiple
1251 // cases.
1252 if (RecurrenceDescriptor::isFindLastRecurrenceKind(Kind: RecurrenceKind))
1253 return true;
1254
1255 return PreferPredicatedReductionSelect ||
1256 TTI.preferPredicatedReductionSelect();
1257 }
1258
1259 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
1260 /// with factor VF. Return the cost of the instruction, including
1261 /// scalarization overhead if it's needed.
1262 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;
1263
1264 /// Estimate cost of a call instruction CI if it were vectorized with factor
1265 /// VF. Return the cost of the instruction, including scalarization overhead
1266 /// if it's needed.
1267 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const;
1268
1269 /// Invalidates decisions already taken by the cost model.
1270 void invalidateCostModelingDecisions() {
1271 WideningDecisions.clear();
1272 Uniforms.clear();
1273 Scalars.clear();
1274 }
1275
1276 /// Returns the expected execution cost. The unit of the cost does
1277 /// not matter because we use the 'cost' units to compare different
1278 /// vector widths. The cost that is returned is *not* normalized by
1279 /// the factor width.
1280 InstructionCost expectedCost(ElementCount VF);
1281
1282 /// Returns the execution time cost of an instruction for a given vector
1283 /// width. Vector width of one means scalar.
1284 InstructionCost getInstructionCost(Instruction *I, ElementCount VF);
1285
1286 /// Returns true if \p Op should be considered invariant and if it is
1287 /// trivially hoistable.
1288 bool shouldConsiderInvariant(Value *Op);
1289
1290 /// Returns true if \p I has been forced to be scalarized at \p VF.
1291 bool isForcedScalar(Instruction *I, ElementCount VF) const {
1292 auto FS = ForcedScalars.find(Val: VF);
1293 return FS != ForcedScalars.end() && FS->second.contains(key: I);
1294 }
1295
1296private:
1297 unsigned NumPredStores = 0;
1298
1299 /// VF selection state independent of cost-modeling decisions.
1300 VFSelectionContext &Config;
1301
1302 /// Wrapper around LoopVectorizationLegality::isUniform() that takes into
1303 /// account if alias-masking is enabled. We consider the VF to be unknown when
1304 /// alias masking.
1305 bool isUniform(Value *V, ElementCount VF) const {
1306 // With alias-masking our runtime VF is [2, VF] (and not necessarily a
1307 // power-of-two). Something that is uniform for VF may not be for the full
1308 // range.
1309 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
1310 "alias-mask status must be decided already");
1311 return Legal->isUniform(V, VF: PartialAliasMaskingStatus ==
1312 AliasMaskingStatus::Disabled
1313 ? std::optional(VF)
1314 : std::nullopt);
1315 }
1316
1317 /// Wrapper around LoopVectorizationLegality::isUniformMemOp() that takes into
1318 /// account if alias-masking is enabled. We consider the VF to be unknown when
1319 /// alias masking.
1320 bool isUniformMemOp(Instruction &I, ElementCount VF) const {
1321 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
1322 "alias-mask status must be decided already");
1323 return Legal->isUniformMemOp(I, VF: PartialAliasMaskingStatus ==
1324 AliasMaskingStatus::Disabled
1325 ? std::optional(VF)
1326 : std::nullopt);
1327 }
1328
1329 /// Calculate vectorization cost of memory instruction \p I.
1330 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
1331
1332 /// The cost computation for scalarized memory instruction.
1333 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1334
1335 /// The cost computation for interleaving group of memory instructions.
1336 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF) const;
1337
1338 /// The cost computation for Gather/Scatter instruction.
1339 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF) const;
1340
1341 /// The cost computation for widening instruction \p I with consecutive
1342 /// memory access.
1343 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF,
1344 InstWidening Kind);
1345
1346 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1347 /// Load: scalar load + broadcast.
1348 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1349 /// element)
1350 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF) const;
1351
1352 /// Estimate the overhead of scalarizing an instruction. This is a
1353 /// convenience wrapper for the type-based getScalarizationOverhead API.
1354 InstructionCost getScalarizationOverhead(Instruction *I,
1355 ElementCount VF) const;
1356
1357 /// A type representing the costs for instructions if they were to be
1358 /// scalarized rather than vectorized. The entries are Instruction-Cost
1359 /// pairs.
1360 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
1361
1362 /// A set containing all BasicBlocks that are known to present after
1363 /// vectorization as a predicated block.
1364 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
1365 PredicatedBBsAfterVectorization;
1366
1367 /// Records whether it is allowed to have the original scalar loop execute at
1368 /// least once. This may be needed as a fallback loop in case runtime
1369 /// aliasing/dependence checks fail, or to handle the tail/remainder
1370 /// iterations when the trip count is unknown or doesn't divide by the VF,
1371 /// or as a peel-loop to handle gaps in interleave-groups.
1372 /// Under optsize and when the trip count is very small we don't allow any
1373 /// iterations to execute in the scalar loop.
1374 EpilogueLowering EpilogueLoweringStatus = CM_EpilogueAllowed;
1375
1376 /// Control finally chosen tail folding style.
1377 TailFoldingStyle ChosenTailFoldingStyle = TailFoldingStyle::None;
1378
1379 /// If partial alias masking is enabled/disabled or not decided.
1380 AliasMaskingStatus PartialAliasMaskingStatus = AliasMaskingStatus::NotDecided;
1381
1382 /// A map holding scalar costs for different vectorization factors. The
1383 /// presence of a cost for an instruction in the mapping indicates that the
1384 /// instruction will be scalarized when vectorizing with the associated
1385 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1386 MapVector<ElementCount, ScalarCostsTy> InstsToScalarize;
1387
1388 /// Holds the instructions known to be uniform after vectorization.
1389 /// The data is collected per VF.
1390 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1391
1392 /// Holds the instructions known to be scalar after vectorization.
1393 /// The data is collected per VF.
1394 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1395
1396 /// Holds the instructions (address computations) that are forced to be
1397 /// scalarized.
1398 DenseMap<ElementCount, SmallSetVector<Instruction *, 4>> ForcedScalars;
1399
1400 /// Returns the expected difference in cost from scalarizing the expression
1401 /// feeding a predicated instruction \p PredInst. The instructions to
1402 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1403 /// non-negative return value implies the expression will be scalarized.
1404 /// Currently, only single-use chains are considered for scalarization.
1405 InstructionCost computePredInstDiscount(Instruction *PredInst,
1406 ScalarCostsTy &ScalarCosts,
1407 ElementCount VF);
1408
1409 /// Collect the instructions that are uniform after vectorization. An
1410 /// instruction is uniform if we represent it with a single scalar value in
1411 /// the vectorized loop corresponding to each vector iteration. Examples of
1412 /// uniform instructions include pointer operands of consecutive or
1413 /// interleaved memory accesses. Note that although uniformity implies an
1414 /// instruction will be scalar, the reverse is not true. In general, a
1415 /// scalarized instruction will be represented by VF scalar values in the
1416 /// vectorized loop, each corresponding to an iteration of the original
1417 /// scalar loop.
1418 void collectLoopUniforms(ElementCount VF);
1419
1420 /// Collect the instructions that are scalar after vectorization. An
1421 /// instruction is scalar if it is known to be uniform or will be scalarized
1422 /// during vectorization. collectLoopScalars should only add non-uniform nodes
1423 /// to the list if they are used by a load/store instruction that is marked as
1424 /// CM_Scalarize. Non-uniform scalarized instructions will be represented by
1425 /// VF values in the vectorized loop, each corresponding to an iteration of
1426 /// the original scalar loop.
1427 void collectLoopScalars(ElementCount VF);
1428
1429 /// Keeps cost model vectorization decision and cost for instructions.
1430 /// Right now it is used for memory instructions only.
1431 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1432 std::pair<InstWidening, InstructionCost>>;
1433
1434 DecisionList WideningDecisions;
1435
1436 /// Returns true if \p V is expected to be vectorized and it needs to be
1437 /// extracted.
1438 bool needsExtract(Value *V, ElementCount VF) const {
1439 Instruction *I = dyn_cast<Instruction>(Val: V);
1440 if (VF.isScalar() || !I || !TheLoop->contains(Inst: I) ||
1441 TheLoop->isLoopInvariant(V: I) ||
1442 getWideningDecision(I, VF) == CM_Scalarize)
1443 return false;
1444
1445 // Assume we can vectorize V (and hence we need extraction) if the
1446 // scalars are not computed yet. This can happen, because it is called
1447 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1448 // the scalars are collected. That should be a safe assumption in most
1449 // cases, because we check if the operands have vectorizable types
1450 // beforehand in LoopVectorizationLegality.
1451 return !Scalars.contains(Val: VF) || !isScalarAfterVectorization(I, VF);
1452 };
1453
1454 /// Returns a range containing only operands needing to be extracted.
1455 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
1456 ElementCount VF) const {
1457
1458 SmallPtrSet<const Value *, 4> UniqueOperands;
1459 SmallVector<Value *, 4> Res;
1460 for (Value *Op : Ops) {
1461 if (isa<Constant>(Val: Op) || !UniqueOperands.insert(Ptr: Op).second ||
1462 !needsExtract(V: Op, VF))
1463 continue;
1464 Res.push_back(Elt: Op);
1465 }
1466 return Res;
1467 }
1468
1469public:
1470 /// The loop that we evaluate.
1471 Loop *TheLoop;
1472
1473 /// Predicated scalar evolution analysis.
1474 PredicatedScalarEvolution &PSE;
1475
1476 /// Loop Info analysis.
1477 LoopInfo *LI;
1478
1479 /// Vectorization legality.
1480 LoopVectorizationLegality *Legal;
1481
1482 /// Vector target information.
1483 const TargetTransformInfo &TTI;
1484
1485 /// Target Library Info.
1486 const TargetLibraryInfo *TLI;
1487
1488 /// Assumption cache.
1489 AssumptionCache *AC;
1490
1491 /// Interface to emit optimization remarks.
1492 OptimizationRemarkEmitter *ORE;
1493
1494 /// A function to lazily fetch BlockFrequencyInfo. This avoids computing it
1495 /// unless necessary, e.g. when the loop isn't legal to vectorize or when
1496 /// there is no predication.
1497 std::function<BlockFrequencyInfo &()> GetBFI;
1498 /// The BlockFrequencyInfo returned from GetBFI.
1499 BlockFrequencyInfo *BFI = nullptr;
1500 /// Returns the BlockFrequencyInfo for the function if cached, otherwise
1501 /// fetches it via GetBFI. Avoids an indirect call to the std::function.
1502 BlockFrequencyInfo &getBFI() {
1503 if (!BFI)
1504 BFI = &GetBFI();
1505 return *BFI;
1506 }
1507
1508 const Function *TheFunction;
1509
1510 /// The interleave access information contains groups of interleaved accesses
1511 /// with the same stride and close to each other.
1512 InterleavedAccessInfo &InterleaveInfo;
1513
1514 /// Values to ignore in the cost model.
1515 SmallPtrSet<const Value *, 16> ValuesToIgnore;
1516
1517 /// Values to ignore in the cost model when VF > 1.
1518 SmallPtrSet<const Value *, 16> VecValuesToIgnore;
1519};
1520} // end namespace llvm
1521
1522namespace {
1523/// Helper struct to manage generating runtime checks for vectorization.
1524///
1525/// The runtime checks are created up-front in temporary blocks to allow better
1526/// estimating the cost and un-linked from the existing IR. After deciding to
1527/// vectorize, the checks are moved back. If deciding not to vectorize, the
1528/// temporary blocks are completely removed.
1529class GeneratedRTChecks {
1530 /// Basic block which contains the generated SCEV checks, if any.
1531 BasicBlock *SCEVCheckBlock = nullptr;
1532
1533 /// The value representing the result of the generated SCEV checks. If it is
1534 /// nullptr no SCEV checks have been generated.
1535 Value *SCEVCheckCond = nullptr;
1536
1537 /// Basic block which contains the generated memory runtime checks, if any.
1538 BasicBlock *MemCheckBlock = nullptr;
1539
1540 /// The value representing the result of the generated memory runtime checks.
1541 /// If it is nullptr no memory runtime checks have been generated.
1542 Value *MemRuntimeCheckCond = nullptr;
1543
1544 DominatorTree *DT;
1545 LoopInfo *LI;
1546 TargetTransformInfo *TTI;
1547
1548 SCEVExpander SCEVExp;
1549 SCEVExpander MemCheckExp;
1550
1551 bool CostTooHigh = false;
1552
1553 Loop *OuterLoop = nullptr;
1554
1555 PredicatedScalarEvolution &PSE;
1556
1557 /// The kind of cost that we are calculating
1558 TTI::TargetCostKind CostKind;
1559
1560 /// True if the loop is alias-masked (which allows us to omit diff checks).
1561 bool LoopUsesPartialAliasMasking = false;
1562
1563public:
1564 GeneratedRTChecks(PredicatedScalarEvolution &PSE, DominatorTree *DT,
1565 LoopInfo *LI, TargetTransformInfo *TTI,
1566 TTI::TargetCostKind CostKind,
1567 bool LoopUsesPartialAliasMasking)
1568 : DT(DT), LI(LI), TTI(TTI),
1569 SCEVExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1570 MemCheckExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1571 PSE(PSE), CostKind(CostKind),
1572 LoopUsesPartialAliasMasking(LoopUsesPartialAliasMasking) {}
1573
1574 /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
1575 /// accurately estimate the cost of the runtime checks. The blocks are
1576 /// un-linked from the IR and are added back during vector code generation. If
1577 /// there is no vector code generation, the check blocks are removed
1578 /// completely.
1579 void create(Loop *L, const LoopAccessInfo &LAI,
1580 const SCEVPredicate &UnionPred, ElementCount VF, unsigned IC,
1581 OptimizationRemarkEmitter &ORE) {
1582
1583 // Hard cutoff to limit compile-time increase in case a very large number of
1584 // runtime checks needs to be generated.
1585 // TODO: Skip cutoff if the loop is guaranteed to execute, e.g. due to
1586 // profile info.
1587 CostTooHigh =
1588 LAI.getNumRuntimePointerChecks() > VectorizeMemoryCheckThreshold;
1589 if (CostTooHigh) {
1590 // Mark runtime checks as never succeeding when they exceed the threshold.
1591 MemRuntimeCheckCond = ConstantInt::getTrue(Context&: L->getHeader()->getContext());
1592 SCEVCheckCond = ConstantInt::getTrue(Context&: L->getHeader()->getContext());
1593 ORE.emit(RemarkBuilder: [&]() {
1594 return OptimizationRemarkAnalysisAliasing(
1595 DEBUG_TYPE, "TooManyMemoryRuntimeChecks", L->getStartLoc(),
1596 L->getHeader())
1597 << "loop not vectorized: too many memory checks needed";
1598 });
1599 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
1600 return;
1601 }
1602
1603 BasicBlock *LoopHeader = L->getHeader();
1604 BasicBlock *Preheader = L->getLoopPreheader();
1605
1606 // Use SplitBlock to create blocks for SCEV & memory runtime checks to
1607 // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
1608 // may be used by SCEVExpander. The blocks will be un-linked from their
1609 // predecessors and removed from LI & DT at the end of the function.
1610 if (!UnionPred.isAlwaysTrue()) {
1611 SCEVCheckBlock = SplitBlock(Old: Preheader, SplitPt: Preheader->getTerminator(), DT, LI,
1612 MSSAU: nullptr, BBName: "vector.scevcheck");
1613
1614 SCEVCheckCond = SCEVExp.expandCodeForPredicate(
1615 Pred: &UnionPred, Loc: SCEVCheckBlock->getTerminator());
1616 if (isa<Constant>(Val: SCEVCheckCond)) {
1617 // Clean up directly after expanding the predicate to a constant, to
1618 // avoid further expansions re-using anything left over from SCEVExp.
1619 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1620 SCEVCleaner.cleanup();
1621 }
1622 }
1623
1624 const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
1625 // TODO: We need to estimate the cost of alias-masking in
1626 // GeneratedRTChecks::getCost(). We can't check the MemCheckBlock as the
1627 // alias-mask is generated later in VPlan.
1628 if (RtPtrChecking.Need && !LoopUsesPartialAliasMasking) {
1629 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1630 MemCheckBlock = SplitBlock(Old: Pred, SplitPt: Pred->getTerminator(), DT, LI, MSSAU: nullptr,
1631 BBName: "vector.memcheck");
1632
1633 auto DiffChecks = RtPtrChecking.getDiffChecks();
1634 if (DiffChecks) {
1635 MemRuntimeCheckCond = addDiffRuntimeChecks(
1636 Loc: MemCheckBlock->getTerminator(), Checks: *DiffChecks, Expander&: MemCheckExp, VF, IC);
1637 } else {
1638 MemRuntimeCheckCond = addRuntimeChecks(
1639 Loc: MemCheckBlock->getTerminator(), TheLoop: L, PointerChecks: RtPtrChecking.getChecks(),
1640 Expander&: MemCheckExp, HoistRuntimeChecks: VectorizerParams::HoistRuntimeChecks);
1641 }
1642 assert(MemRuntimeCheckCond &&
1643 "no RT checks generated although RtPtrChecking "
1644 "claimed checks are required");
1645 }
1646
1647 SCEVExp.eraseDeadInstructions(Root: SCEVCheckCond);
1648
1649 if (!MemCheckBlock && !SCEVCheckBlock)
1650 return;
1651
1652 // Unhook the temporary block with the checks, update various places
1653 // accordingly.
1654 if (SCEVCheckBlock)
1655 SCEVCheckBlock->replaceAllUsesWith(V: Preheader);
1656 if (MemCheckBlock)
1657 MemCheckBlock->replaceAllUsesWith(V: Preheader);
1658
1659 if (SCEVCheckBlock) {
1660 SCEVCheckBlock->getTerminator()->moveBefore(
1661 InsertPos: Preheader->getTerminator()->getIterator());
1662 auto *UI = new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
1663 UI->setDebugLoc(DebugLoc::getTemporary());
1664 Preheader->getTerminator()->eraseFromParent();
1665 }
1666 if (MemCheckBlock) {
1667 MemCheckBlock->getTerminator()->moveBefore(
1668 InsertPos: Preheader->getTerminator()->getIterator());
1669 auto *UI = new UnreachableInst(Preheader->getContext(), MemCheckBlock);
1670 UI->setDebugLoc(DebugLoc::getTemporary());
1671 Preheader->getTerminator()->eraseFromParent();
1672 }
1673
1674 DT->changeImmediateDominator(BB: LoopHeader, NewBB: Preheader);
1675 if (MemCheckBlock) {
1676 DT->eraseNode(BB: MemCheckBlock);
1677 LI->removeBlock(BB: MemCheckBlock);
1678 }
1679 if (SCEVCheckBlock) {
1680 DT->eraseNode(BB: SCEVCheckBlock);
1681 LI->removeBlock(BB: SCEVCheckBlock);
1682 }
1683
1684 // Outer loop is used as part of the later cost calculations.
1685 OuterLoop = L->getParentLoop();
1686 }
1687
1688 InstructionCost getCost() {
1689 if (SCEVCheckBlock || MemCheckBlock)
1690 LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
1691
1692 if (CostTooHigh) {
1693 InstructionCost Cost;
1694 Cost.setInvalid();
1695 LLVM_DEBUG(dbgs() << " number of checks exceeded threshold\n");
1696 return Cost;
1697 }
1698
1699 InstructionCost RTCheckCost = 0;
1700 if (SCEVCheckBlock)
1701 for (Instruction &I : *SCEVCheckBlock) {
1702 if (SCEVCheckBlock->getTerminator() == &I)
1703 continue;
1704 InstructionCost C = TTI->getInstructionCost(U: &I, CostKind);
1705 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1706 RTCheckCost += C;
1707 }
1708 if (MemCheckBlock) {
1709 InstructionCost MemCheckCost = 0;
1710 for (Instruction &I : *MemCheckBlock) {
1711 if (MemCheckBlock->getTerminator() == &I)
1712 continue;
1713 InstructionCost C = TTI->getInstructionCost(U: &I, CostKind);
1714 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1715 MemCheckCost += C;
1716 }
1717
1718 // If the runtime memory checks are being created inside an outer loop
1719 // we should find out if these checks are outer loop invariant. If so,
1720 // the checks will likely be hoisted out and so the effective cost will
1721 // reduce according to the outer loop trip count.
1722 if (OuterLoop) {
1723 ScalarEvolution *SE = MemCheckExp.getSE();
1724 // TODO: If profitable, we could refine this further by analysing every
1725 // individual memory check, since there could be a mixture of loop
1726 // variant and invariant checks that mean the final condition is
1727 // variant.
1728 const SCEV *Cond = SE->getSCEV(V: MemRuntimeCheckCond);
1729 if (SE->isLoopInvariant(S: Cond, L: OuterLoop)) {
1730 // It seems reasonable to assume that we can reduce the effective
1731 // cost of the checks even when we know nothing about the trip
1732 // count. Assume that the outer loop executes at least twice.
1733 unsigned BestTripCount = 2;
1734
1735 // Get the best known TC estimate.
1736 if (auto EstimatedTC = getSmallBestKnownTC(
1737 PSE, L: OuterLoop, /* CanUseConstantMax = */ false))
1738 if (EstimatedTC->isFixed())
1739 BestTripCount = EstimatedTC->getFixedValue();
1740
1741 InstructionCost NewMemCheckCost = MemCheckCost / BestTripCount;
1742
1743 // Let's ensure the cost is always at least 1.
1744 NewMemCheckCost = std::max(a: NewMemCheckCost.getValue(),
1745 b: (InstructionCost::CostType)1);
1746
1747 if (BestTripCount > 1)
1748 LLVM_DEBUG(dbgs()
1749 << "We expect runtime memory checks to be hoisted "
1750 << "out of the outer loop. Cost reduced from "
1751 << MemCheckCost << " to " << NewMemCheckCost << '\n');
1752
1753 MemCheckCost = NewMemCheckCost;
1754 }
1755 }
1756
1757 RTCheckCost += MemCheckCost;
1758 }
1759
1760 if (SCEVCheckBlock || MemCheckBlock)
1761 LLVM_DEBUG(dbgs() << "Total cost of runtime checks: " << RTCheckCost
1762 << "\n");
1763
1764 return RTCheckCost;
1765 }
1766
1767 /// Remove the created SCEV & memory runtime check blocks & instructions, if
1768 /// unused.
1769 ~GeneratedRTChecks() {
1770 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1771 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1772 bool SCEVChecksUsed = !SCEVCheckBlock || !pred_empty(BB: SCEVCheckBlock);
1773 bool MemChecksUsed = !MemCheckBlock || !pred_empty(BB: MemCheckBlock);
1774 if (SCEVChecksUsed)
1775 SCEVCleaner.markResultUsed();
1776
1777 if (MemChecksUsed) {
1778 MemCheckCleaner.markResultUsed();
1779 } else {
1780 auto &SE = *MemCheckExp.getSE();
1781 // Memory runtime check generation creates compares that use expanded
1782 // values. Remove them before running the SCEVExpanderCleaners.
1783 for (auto &I : make_early_inc_range(Range: reverse(C&: *MemCheckBlock))) {
1784 if (MemCheckExp.isInsertedInstruction(I: &I))
1785 continue;
1786 SE.forgetValue(V: &I);
1787 I.eraseFromParent();
1788 }
1789 }
1790 MemCheckCleaner.cleanup();
1791 SCEVCleaner.cleanup();
1792
1793 if (!SCEVChecksUsed)
1794 SCEVCheckBlock->eraseFromParent();
1795 if (!MemChecksUsed)
1796 MemCheckBlock->eraseFromParent();
1797 }
1798
1799 /// Retrieves the SCEVCheckCond and SCEVCheckBlock that were generated as IR
1800 /// outside VPlan.
1801 std::pair<Value *, BasicBlock *> getSCEVChecks() const {
1802 using namespace llvm::PatternMatch;
1803 if (!SCEVCheckCond || match(V: SCEVCheckCond, P: m_ZeroInt()))
1804 return {nullptr, nullptr};
1805
1806 return {SCEVCheckCond, SCEVCheckBlock};
1807 }
1808
1809 /// Retrieves the MemCheckCond and MemCheckBlock that were generated as IR
1810 /// outside VPlan.
1811 std::pair<Value *, BasicBlock *> getMemRuntimeChecks() const {
1812 using namespace llvm::PatternMatch;
1813 if (MemRuntimeCheckCond && match(V: MemRuntimeCheckCond, P: m_ZeroInt()))
1814 return {nullptr, nullptr};
1815 return {MemRuntimeCheckCond, MemCheckBlock};
1816 }
1817
1818 /// Return true if any runtime checks have been added
1819 bool hasChecks() const {
1820 return getSCEVChecks().first || getMemRuntimeChecks().first;
1821 }
1822};
1823} // namespace
1824
1825static bool useActiveLaneMask(TailFoldingStyle Style) {
1826 return Style == TailFoldingStyle::Data ||
1827 Style == TailFoldingStyle::DataAndControlFlow;
1828}
1829
1830static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style) {
1831 return Style == TailFoldingStyle::DataAndControlFlow;
1832}
1833
1834// Return true if \p OuterLp is an outer loop annotated with hints for explicit
1835// vectorization. The loop needs to be annotated with #pragma omp simd
1836// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
1837// vector length information is not provided, vectorization is not considered
1838// explicit. Interleave hints are not allowed either. These limitations will be
1839// relaxed in the future.
1840// Please, note that we are currently forced to abuse the pragma 'clang
1841// vectorize' semantics. This pragma provides *auto-vectorization hints*
1842// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
1843// provides *explicit vectorization hints* (LV can bypass legal checks and
1844// assume that vectorization is legal). However, both hints are implemented
1845// using the same metadata (llvm.loop.vectorize, processed by
1846// LoopVectorizeHints). This will be fixed in the future when the native IR
1847// representation for pragma 'omp simd' is introduced.
1848static bool isExplicitVecOuterLoop(Loop *OuterLp,
1849 OptimizationRemarkEmitter *ORE) {
1850 assert(!OuterLp->isInnermost() && "This is not an outer loop");
1851 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
1852
1853 // Only outer loops with an explicit vectorization hint are supported.
1854 // Unannotated outer loops are ignored.
1855 if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
1856 return false;
1857
1858 Function *Fn = OuterLp->getHeader()->getParent();
1859 if (!Hints.allowVectorization(F: Fn, L: OuterLp,
1860 VectorizeOnlyWhenForced: true /*VectorizeOnlyWhenForced*/)) {
1861 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
1862 return false;
1863 }
1864
1865 if (Hints.getInterleave() > 1) {
1866 // TODO: Interleave support is future work.
1867 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
1868 "outer loops.\n");
1869 Hints.emitRemarkWithHints();
1870 return false;
1871 }
1872
1873 return true;
1874}
1875
1876static void collectSupportedLoops(Loop &L, LoopInfo *LI,
1877 OptimizationRemarkEmitter *ORE,
1878 SmallVectorImpl<Loop *> &V) {
1879 // Collect inner loops and outer loops without irreducible control flow. For
1880 // now, only collect outer loops that have explicit vectorization hints. If we
1881 // are stress testing the VPlan H-CFG construction, we collect the outermost
1882 // loop of every loop nest.
1883 if (L.isInnermost() || VPlanBuildOuterloopStressTest ||
1884 (EnableVPlanNativePath && isExplicitVecOuterLoop(OuterLp: &L, ORE))) {
1885 LoopBlocksRPO RPOT(&L);
1886 RPOT.perform(LI);
1887 if (!containsIrreducibleCFG<const BasicBlock *>(RPOTraversal&: RPOT, LI: *LI)) {
1888 V.push_back(Elt: &L);
1889 // TODO: Collect inner loops inside marked outer loops in case
1890 // vectorization fails for the outer loop. Do not invoke
1891 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
1892 // already known to be reducible. We can use an inherited attribute for
1893 // that.
1894 return;
1895 }
1896 }
1897 for (Loop *InnerL : L)
1898 collectSupportedLoops(L&: *InnerL, LI, ORE, V);
1899}
1900
1901//===----------------------------------------------------------------------===//
1902// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
1903// LoopVectorizationCostModel and LoopVectorizationPlanner.
1904//===----------------------------------------------------------------------===//
1905
1906/// For the given VF and UF and maximum trip count computed for the loop, return
1907/// whether the induction variable might overflow in the vectorized loop. If not,
1908/// then we know a runtime overflow check always evaluates to false and can be
1909/// removed.
1910static bool isIndvarOverflowCheckKnownFalse(
1911 const LoopVectorizationCostModel *Cost,
1912 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
1913 // Always be conservative if we don't know the exact unroll factor.
1914 uint64_t MaxUF = UF ? *UF
1915 : std::max(a: Cost->TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions: false),
1916 b: Cost->TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions: true));
1917
1918 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
1919 APInt MaxUIntTripCount = IdxTy->getMask();
1920
1921 // We know the runtime overflow check is known false iff the (max) trip-count
1922 // is known and (max) trip-count + (VF * UF) does not overflow in the type of
1923 // the vector loop induction variable.
1924 if (std::optional<ElementCount> TC = getSmallBestKnownTC(
1925 PSE&: Cost->PSE, L: Cost->TheLoop,
1926 /*CanUseConstantMax=*/true, /*CanExcludeZeroTrips=*/false,
1927 /*ComputeUpperBoundOnly=*/true)) {
1928 // Compute the maximum runtime values of VF and the trip count.
1929 std::optional<uint64_t> MaxStep =
1930 getMaxRuntimeElementCount(EC: VF * MaxUF, F: *Cost->TheFunction);
1931 std::optional<uint64_t> MaxTC =
1932 getMaxRuntimeElementCount(EC: *TC, F: *Cost->TheFunction);
1933 if (!MaxStep || !MaxTC)
1934 return false;
1935
1936 // Bail out if the maximum trip count is not representable in the induction
1937 // variable's type.
1938 if (MaxUIntTripCount.ult(RHS: *MaxTC))
1939 return false;
1940
1941 return (MaxUIntTripCount - *MaxTC).ugt(RHS: *MaxStep);
1942 }
1943
1944 return false;
1945}
1946
1947// Return whether we allow using masked interleave-groups (for dealing with
1948// strided loads/stores that reside in predicated blocks, or for dealing
1949// with gaps).
1950static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
1951 // If an override option has been passed in for interleaved accesses, use it.
1952 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
1953 return EnableMaskedInterleavedMemAccesses;
1954
1955 return TTI.enableMaskedInterleavedAccessVectorization();
1956}
1957
1958/// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
1959/// VPBB are moved to the end of the newly created VPIRBasicBlock. All
1960/// predecessors and successors of VPBB, if any, are rewired to the new
1961/// VPIRBasicBlock. If \p VPBB may be unreachable, \p Plan must be passed.
1962static VPIRBasicBlock *replaceVPBBWithIRVPBB(VPBasicBlock *VPBB,
1963 BasicBlock *IRBB,
1964 VPlan *Plan = nullptr) {
1965 if (!Plan)
1966 Plan = VPBB->getPlan();
1967 VPIRBasicBlock *IRVPBB = Plan->createVPIRBasicBlock(IRBB);
1968 auto IP = IRVPBB->begin();
1969 for (auto &R : make_early_inc_range(Range: VPBB->phis()))
1970 R.moveBefore(BB&: *IRVPBB, I: IP);
1971
1972 for (auto &R :
1973 make_early_inc_range(Range: make_range(x: VPBB->getFirstNonPhi(), y: VPBB->end())))
1974 R.moveBefore(BB&: *IRVPBB, I: IRVPBB->end());
1975
1976 VPBlockUtils::reassociateBlocks(Old: VPBB, New: IRVPBB);
1977 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
1978 return IRVPBB;
1979}
1980
1981BasicBlock *InnerLoopVectorizer::createScalarPreheader(StringRef Prefix) {
1982 BasicBlock *VectorPH = OrigLoop->getLoopPreheader();
1983 assert(VectorPH && "Invalid loop structure");
1984
1985 // NOTE: The Plan's scalar preheader VPBB isn't replaced with a VPIRBasicBlock
1986 // wrapping the newly created scalar preheader here at the moment, because the
1987 // Plan's scalar preheader may be unreachable at this point. Instead it is
1988 // replaced in executePlan.
1989 return SplitBlock(Old: VectorPH, SplitPt: VectorPH->getTerminator(), DT, LI, MSSAU: nullptr,
1990 BBName: Twine(Prefix) + "scalar.ph");
1991}
1992
1993/// Knowing that loop \p L executes a single vector iteration, add instructions
1994/// that will get simplified and thus should not have any cost to \p
1995/// InstsToIgnore.
1996static void addFullyUnrolledInstructionsToIgnore(
1997 Loop *L, const LoopVectorizationLegality::InductionList &IL,
1998 SmallPtrSetImpl<Instruction *> &InstsToIgnore) {
1999 auto *Cmp = L->getLatchCmpInst();
2000 if (Cmp)
2001 InstsToIgnore.insert(Ptr: Cmp);
2002 for (const auto &KV : IL) {
2003 // Extract the key by hand so that it can be used in the lambda below. Note
2004 // that captured structured bindings are a C++20 extension.
2005 PHINode *IV = KV.first;
2006
2007 // The induction is free: a widened induction generates a vector phi with
2008 // its start value and an increment that is dead without a backedge.
2009 InstsToIgnore.insert(Ptr: IV);
2010
2011 // Get next iteration value of the induction variable.
2012 Instruction *IVInst =
2013 cast<Instruction>(Val: IV->getIncomingValueForBlock(BB: L->getLoopLatch()));
2014 if (all_of(Range: IVInst->users(),
2015 P: [&](const User *U) { return U == IV || U == Cmp; }))
2016 InstsToIgnore.insert(Ptr: IVInst);
2017 }
2018}
2019
2020BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
2021 // Create a new IR basic block for the scalar preheader.
2022 BasicBlock *ScalarPH = createScalarPreheader(Prefix: "");
2023 return ScalarPH->getSinglePredecessor();
2024}
2025
2026namespace {
2027
2028struct CSEDenseMapInfo {
2029 static bool canHandle(const Instruction *I) {
2030 return isa<InsertElementInst>(Val: I) || isa<ExtractElementInst>(Val: I) ||
2031 isa<ShuffleVectorInst>(Val: I) || isa<GetElementPtrInst>(Val: I);
2032 }
2033
2034 static unsigned getHashValue(const Instruction *I) {
2035 assert(canHandle(I) && "Unknown instruction!");
2036 return hash_combine(args: I->getOpcode(),
2037 args: hash_combine_range(R: I->operand_values()));
2038 }
2039
2040 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
2041 return LHS->isIdenticalTo(I: RHS);
2042 }
2043};
2044
2045} // end anonymous namespace
2046
2047/// FIXME: This legacy common-subexpression-elimination routine is scheduled for
2048/// removal, in favor of the VPlan-based one.
2049static void legacyCSE(BasicBlock *BB) {
2050 // Perform simple cse.
2051 SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
2052 for (Instruction &In : llvm::make_early_inc_range(Range&: *BB)) {
2053 if (!CSEDenseMapInfo::canHandle(I: &In))
2054 continue;
2055
2056 // Check if we can replace this instruction with any of the
2057 // visited instructions.
2058 if (Instruction *V = CSEMap.lookup(Val: &In)) {
2059 In.replaceAllUsesWith(V);
2060 In.eraseFromParent();
2061 continue;
2062 }
2063
2064 CSEMap[&In] = &In;
2065 }
2066}
2067
2068/// This function attempts to return a value that represents the ElementCount
2069/// at runtime. For fixed-width VFs we know this precisely at compile
2070/// time, but for scalable VFs we calculate it based on an estimate of the
2071/// vscale value.
2072static unsigned estimateElementCount(ElementCount VF,
2073 std::optional<unsigned> VScale) {
2074 unsigned EstimatedVF = VF.getKnownMinValue();
2075 if (VF.isScalable())
2076 if (VScale)
2077 EstimatedVF *= *VScale;
2078 assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
2079 return EstimatedVF;
2080}
2081
2082/// Returns the vector library variant function of \p CI usable at \p VF,
2083/// respecting \p MaskRequired, or nullptr if none is found: a mapping with
2084/// matching VF, masked if required, whose vector function is declared in the
2085/// module.
2086static Function *getVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
2087 bool MaskRequired,
2088 const TargetLibraryInfo *TLI) {
2089 if (!TLI || CI.isNoBuiltin())
2090 return nullptr;
2091 for (const VFInfo &Info : VFDatabase::getMappings(CI))
2092 if (Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()))
2093 if (Function *F = CI.getModule()->getFunction(Name: Info.VectorName))
2094 return F;
2095 return nullptr;
2096}
2097
2098/// Returns true iff \p CI has a library vector variant usable at \p VF.
2099static bool hasVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
2100 bool MaskRequired,
2101 const TargetLibraryInfo *TLI) {
2102 return getVectorLibraryVariantFor(CI, VF, MaskRequired, TLI) != nullptr;
2103}
2104
2105InstructionCost
2106LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
2107 ElementCount VF) const {
2108 Type *RetTy = CI->getType();
2109 SmallVector<Type *, 4> Tys;
2110 for (auto &ArgOp : CI->args())
2111 Tys.push_back(Elt: ArgOp->getType());
2112
2113 InstructionCost ScalarCallCost = TTI.getCallInstrCost(
2114 F: CI->getCalledFunction(), RetTy, Tys, CostKind: Config.CostKind);
2115
2116 // Cost of the scalar call (scalar VF) or its scalarization (vector VF). The
2117 // scalarization cost is only meaningful for fixed VFs.
2118 InstructionCost Cost = VF.isScalable()
2119 ? InstructionCost::getInvalid()
2120 : ScalarCallCost * VF.getKnownMinValue() +
2121 getScalarizationOverhead(I: CI, VF);
2122
2123 // The call may be vectorized at this VF, via a vector intrinsic or a vector
2124 // library variant.
2125 if (getVectorIntrinsicIDForCall(CI, TLI))
2126 Cost = std::min(a: Cost, b: getVectorIntrinsicCost(CI, VF));
2127
2128 if (Function *Variant =
2129 getVectorLibraryVariantFor(CI: *CI, VF, MaskRequired: isMaskRequired(I: CI), TLI))
2130 Cost = std::min(a: Cost,
2131 b: TTI.getCallInstrCost(
2132 /*F=*/nullptr, RetTy: Variant->getReturnType(),
2133 Tys: Variant->getFunctionType()->params(), CostKind: Config.CostKind));
2134
2135 return Cost;
2136}
2137
2138static Type *maybeVectorizeType(Type *Ty, ElementCount VF) {
2139 if (VF.isScalar() || !canVectorizeTy(Ty))
2140 return Ty;
2141 return toVectorizedTy(Ty, EC: VF);
2142}
2143
2144InstructionCost
2145LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
2146 ElementCount VF) const {
2147 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2148 assert(ID && "Expected intrinsic call!");
2149 Type *RetTy = maybeVectorizeType(Ty: CI->getType(), VF);
2150 FastMathFlags FMF;
2151 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: CI))
2152 FMF = FPMO->getFastMathFlags();
2153
2154 SmallVector<const Value *> Arguments(CI->args());
2155 FunctionType *FTy = CI->getCalledFunction()->getFunctionType();
2156 SmallVector<Type *> ParamTys;
2157 std::transform(first: FTy->param_begin(), last: FTy->param_end(),
2158 result: std::back_inserter(x&: ParamTys),
2159 unary_op: [&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
2160
2161 IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
2162 dyn_cast<IntrinsicInst>(Val: CI),
2163 InstructionCost::getInvalid());
2164 return TTI.getIntrinsicInstrCost(ICA: CostAttrs, CostKind: Config.CostKind);
2165}
2166
2167void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
2168 // Don't apply optimizations below when no (vector) loop remains, as they all
2169 // require one at the moment.
2170 VPBasicBlock *HeaderVPBB =
2171 vputils::getFirstLoopHeader(Plan&: *State.Plan, VPDT&: State.VPDT);
2172 if (!HeaderVPBB)
2173 return;
2174
2175 BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
2176
2177 // Remove redundant induction instructions.
2178 legacyCSE(BB: HeaderBB);
2179}
2180
2181void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
2182 // We should not collect Scalars more than once per VF. Right now, this
2183 // function is called from collectUniformsAndScalars(), which already does
2184 // this check. Collecting Scalars for VF=1 does not make any sense.
2185 assert(VF.isVector() && !Scalars.contains(VF) &&
2186 "This function should not be visited twice for the same VF");
2187
2188 // This avoids any chances of creating a REPLICATE recipe during planning
2189 // since that would result in generation of scalarized code during execution,
2190 // which is not supported for scalable vectors.
2191 if (VF.isScalable()) {
2192 Scalars[VF].insert_range(R&: Uniforms[VF]);
2193 return;
2194 }
2195
2196 SmallSetVector<Instruction *, 8> Worklist;
2197
2198 // These sets are used to seed the analysis with pointers used by memory
2199 // accesses that will remain scalar.
2200 SmallSetVector<Instruction *, 8> ScalarPtrs;
2201 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
2202 auto *Latch = TheLoop->getLoopLatch();
2203
2204 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
2205 // The pointer operands of loads and stores will be scalar as long as the
2206 // memory access is not a gather/scatter or histogram operation. The value
2207 // operand of a store will remain scalar if the store is scalarized.
2208 auto IsScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
2209 InstWidening WideningDecision = getWideningDecision(I: MemAccess, VF);
2210 assert(WideningDecision != CM_Unknown &&
2211 "Widening decision should be ready at this moment");
2212 auto *Store = dyn_cast<StoreInst>(Val: MemAccess);
2213 if (Store && Ptr == Store->getValueOperand())
2214 return WideningDecision == CM_Scalarize;
2215 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&
2216 "Ptr is neither a value or pointer operand");
2217 return WideningDecision != CM_GatherScatter &&
2218 !(Store && Legal->getHistogramInfo(I: Store));
2219 };
2220
2221 // A helper that returns true if the given value is a getelementptr
2222 // instruction contained in the loop.
2223 auto IsLoopVaryingGEP = [&](Value *V) {
2224 return isa<GetElementPtrInst>(Val: V) && !TheLoop->isLoopInvariant(V);
2225 };
2226
2227 // A helper that evaluates a memory access's use of a pointer. If the use will
2228 // be a scalar use and the pointer is only used by memory accesses, we place
2229 // the pointer in ScalarPtrs. Otherwise, the pointer is placed in
2230 // PossibleNonScalarPtrs.
2231 auto EvaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
2232 // We only care about bitcast and getelementptr instructions contained in
2233 // the loop.
2234 if (!IsLoopVaryingGEP(Ptr))
2235 return;
2236
2237 // If the pointer has already been identified as scalar (e.g., if it was
2238 // also identified as uniform), there's nothing to do.
2239 auto *I = cast<Instruction>(Val: Ptr);
2240 if (Worklist.count(key: I))
2241 return;
2242
2243 // If the use of the pointer will be a scalar use, and all users of the
2244 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
2245 // place the pointer in PossibleNonScalarPtrs.
2246 if (IsScalarUse(MemAccess, Ptr) &&
2247 all_of(Range: I->users(), P: IsaPred<LoadInst, StoreInst>))
2248 ScalarPtrs.insert(X: I);
2249 else
2250 PossibleNonScalarPtrs.insert(Ptr: I);
2251 };
2252
2253 // We seed the scalars analysis with three classes of instructions: (1)
2254 // instructions marked uniform-after-vectorization and (2) bitcast,
2255 // getelementptr and (pointer) phi instructions used by memory accesses
2256 // requiring a scalar use.
2257 //
2258 // (1) Add to the worklist all instructions that have been identified as
2259 // uniform-after-vectorization.
2260 Worklist.insert_range(R&: Uniforms[VF]);
2261
2262 // (2) Add to the worklist all bitcast and getelementptr instructions used by
2263 // memory accesses requiring a scalar use. The pointer operands of loads and
2264 // stores will be scalar unless the operation is a gather or scatter.
2265 // The value operand of a store will remain scalar if the store is scalarized.
2266 for (auto *BB : TheLoop->blocks())
2267 for (auto &I : *BB) {
2268 if (auto *Load = dyn_cast<LoadInst>(Val: &I)) {
2269 EvaluatePtrUse(Load, Load->getPointerOperand());
2270 } else if (auto *Store = dyn_cast<StoreInst>(Val: &I)) {
2271 EvaluatePtrUse(Store, Store->getPointerOperand());
2272 EvaluatePtrUse(Store, Store->getValueOperand());
2273 }
2274 }
2275 for (auto *I : ScalarPtrs)
2276 if (!PossibleNonScalarPtrs.count(Ptr: I)) {
2277 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n");
2278 Worklist.insert(X: I);
2279 }
2280
2281 // Insert the forced scalars.
2282 // FIXME: Currently VPWidenPHIRecipe() often creates a dead vector
2283 // induction variable when the PHI user is scalarized.
2284 auto ForcedScalar = ForcedScalars.find(Val: VF);
2285 if (ForcedScalar != ForcedScalars.end())
2286 for (auto *I : ForcedScalar->second) {
2287 LLVM_DEBUG(dbgs() << "LV: Found (forced) scalar instruction: " << *I << "\n");
2288 Worklist.insert(X: I);
2289 }
2290
2291 // Expand the worklist by looking through any bitcasts and getelementptr
2292 // instructions we've already identified as scalar. This is similar to the
2293 // expansion step in collectLoopUniforms(); however, here we're only
2294 // expanding to include additional bitcasts and getelementptr instructions.
2295 unsigned Idx = 0;
2296 while (Idx != Worklist.size()) {
2297 Instruction *Dst = Worklist[Idx++];
2298 if (!IsLoopVaryingGEP(Dst->getOperand(i: 0)))
2299 continue;
2300 auto *Src = cast<Instruction>(Val: Dst->getOperand(i: 0));
2301 if (llvm::all_of(Range: Src->users(), P: [&](User *U) -> bool {
2302 auto *J = cast<Instruction>(Val: U);
2303 return !TheLoop->contains(Inst: J) || Worklist.count(key: J) ||
2304 ((isa<LoadInst>(Val: J) || isa<StoreInst>(Val: J)) &&
2305 IsScalarUse(J, Src));
2306 })) {
2307 Worklist.insert(X: Src);
2308 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n");
2309 }
2310 }
2311
2312 // An induction variable will remain scalar if all users of the induction
2313 // variable and induction variable update remain scalar.
2314 for (const auto &Induction : Legal->getInductionVars()) {
2315 auto *Ind = Induction.first;
2316 auto *IndUpdate = cast<Instruction>(Val: Ind->getIncomingValueForBlock(BB: Latch));
2317
2318 // If tail-folding is applied, the primary induction variable will be used
2319 // to feed a vector compare.
2320 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
2321 continue;
2322
2323 // Returns true if \p Indvar is a pointer induction that is used directly by
2324 // load/store instruction \p I.
2325 auto IsDirectLoadStoreFromPtrIndvar = [&](Instruction *Indvar,
2326 Instruction *I) {
2327 return Induction.second.getKind() ==
2328 InductionDescriptor::IK_PtrInduction &&
2329 (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I)) &&
2330 Indvar == getLoadStorePointerOperand(V: I) && IsScalarUse(I, Indvar);
2331 };
2332
2333 // Determine if all users of the induction variable are scalar after
2334 // vectorization.
2335 bool ScalarInd = all_of(Range: Ind->users(), P: [&](User *U) -> bool {
2336 auto *I = cast<Instruction>(Val: U);
2337 return I == IndUpdate || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2338 IsDirectLoadStoreFromPtrIndvar(Ind, I);
2339 });
2340 if (!ScalarInd)
2341 continue;
2342
2343 // If the induction variable update is a fixed-order recurrence, neither the
2344 // induction variable or its update should be marked scalar after
2345 // vectorization.
2346 auto *IndUpdatePhi = dyn_cast<PHINode>(Val: IndUpdate);
2347 if (IndUpdatePhi && Legal->isFixedOrderRecurrence(Phi: IndUpdatePhi))
2348 continue;
2349
2350 // Determine if all users of the induction variable update instruction are
2351 // scalar after vectorization.
2352 bool ScalarIndUpdate = all_of(Range: IndUpdate->users(), P: [&](User *U) -> bool {
2353 auto *I = cast<Instruction>(Val: U);
2354 return I == Ind || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2355 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
2356 });
2357 if (!ScalarIndUpdate)
2358 continue;
2359
2360 // The induction variable and its update instruction will remain scalar.
2361 Worklist.insert(X: Ind);
2362 Worklist.insert(X: IndUpdate);
2363 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n");
2364 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate
2365 << "\n");
2366 }
2367
2368 Scalars[VF].insert_range(R&: Worklist);
2369}
2370
2371bool LoopVectorizationCostModel::isLegalMaskedLoadOrStore(
2372 Instruction *I, ElementCount VF) const {
2373 assert(isa<LoadInst>(I) || isa<StoreInst>(I));
2374 return Config.isLegalMaskedLoadOrStore(IsLoad: isa<LoadInst>(Val: I), ScalarTy: getLoadStoreType(I),
2375 Alignment: getLoadStoreAlignment(I),
2376 AddressSpace: getLoadStoreAddressSpace(I));
2377}
2378
2379bool LoopVectorizationCostModel::isLegalGatherOrScatter(Instruction *I,
2380 ElementCount VF) const {
2381 assert((isa<LoadInst, StoreInst>(I)));
2382 return Config.isLegalGatherOrScatter(IsLoad: isa<LoadInst>(Val: I), ScalarTy: getLoadStoreType(I),
2383 Alignment: getLoadStoreAlignment(I), VF);
2384}
2385
2386bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
2387 ElementCount VF) {
2388 if (!isPredicatedInst(I))
2389 return false;
2390
2391 // Do we have a non-scalar lowering for this predicated
2392 // instruction? No - it is scalar with predication.
2393 switch(I->getOpcode()) {
2394 default:
2395 return true;
2396 case Instruction::Call: {
2397 if (VF.isScalar())
2398 return true;
2399 auto *CI = cast<CallInst>(Val: I);
2400 // A vector intrinsic or library variant lowering avoids scalarization.
2401 return !getVectorIntrinsicIDForCall(CI, TLI) &&
2402 !hasVectorLibraryVariantFor(CI: *CI, VF, MaskRequired: isMaskRequired(I: CI), TLI);
2403 }
2404 case Instruction::Load:
2405 case Instruction::Store: {
2406 bool IsConsecutive = Legal->isConsecutivePtr(AccessTy: getLoadStoreType(I),
2407 Ptr: getLoadStorePointerOperand(V: I));
2408 return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) &&
2409 !isLegalGatherOrScatter(I, VF);
2410 }
2411 case Instruction::UDiv:
2412 case Instruction::SDiv:
2413 case Instruction::SRem:
2414 case Instruction::URem: {
2415 // We have the option to use the llvm.masked.udiv intrinsics to avoid
2416 // predication. The cost based decision here will always select the masked
2417 // intrinsics for scalable vectors as scalarization isn't legal.
2418 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
2419 return isDivRemScalarWithPredication(ScalarCost, MaskedCost);
2420 }
2421 }
2422}
2423
2424bool LoopVectorizationCostModel::isMaskRequired(Instruction *I) const {
2425 return Legal->isMaskRequired(I, TailFolded: foldTailByMasking());
2426}
2427
2428// TODO: Fold into LoopVectorizationLegality::isMaskRequired.
2429bool LoopVectorizationCostModel::isPredicatedInst(Instruction *I) const {
2430 // TODO: We can use the loop-preheader as context point here and get
2431 // context sensitive reasoning for isSafeToSpeculativelyExecute.
2432 if (isSafeToSpeculativelyExecute(I) ||
2433 (isa<LoadInst, StoreInst, CallInst>(Val: I) && !isMaskRequired(I)) ||
2434 isa<UncondBrInst, CondBrInst, SwitchInst, PHINode, AllocaInst>(Val: I))
2435 return false;
2436
2437 // If the instruction was executed conditionally in the original scalar loop,
2438 // predication is needed with a mask whose lanes are all possibly inactive.
2439 if (Legal->blockNeedsPredication(BB: I->getParent()))
2440 return true;
2441
2442 // If we're not folding the tail by masking and not vectorizing a loop with
2443 // uncountable exits and side effects, predication is unnecessary.
2444 if (!foldTailByMasking() && !Legal->hasUncountableExitWithSideEffects())
2445 return false;
2446
2447 // All that remain are instructions with side-effects originally executed in
2448 // the loop unconditionally, but now execute under a tail-fold mask (only)
2449 // having at least one active lane (the first). If the side-effects of the
2450 // instruction are invariant, executing it w/o (the tail-folding) mask is safe
2451 // - it will cause the same side-effects as when masked.
2452 switch(I->getOpcode()) {
2453 default:
2454 llvm_unreachable(
2455 "instruction should have been considered by earlier checks");
2456 case Instruction::Call:
2457 // Side-effects of a Call are assumed to be non-invariant, needing a
2458 // (fold-tail) mask.
2459 assert(isMaskRequired(I) &&
2460 "should have returned earlier for calls not needing a mask");
2461 return true;
2462 case Instruction::Load:
2463 // If the address is loop invariant no predication is needed.
2464 return !Legal->isInvariant(V: getLoadStorePointerOperand(V: I));
2465 case Instruction::Store: {
2466 // For stores, we need to prove both speculation safety (which follows from
2467 // the same argument as loads), but also must prove the value being stored
2468 // is correct. The easiest form of the later is to require that all values
2469 // stored are the same.
2470 return !(Legal->isInvariant(V: getLoadStorePointerOperand(V: I)) &&
2471 TheLoop->isLoopInvariant(V: cast<StoreInst>(Val: I)->getValueOperand()));
2472 }
2473 case Instruction::UDiv:
2474 case Instruction::URem:
2475 // If the divisor is loop-invariant no predication is needed.
2476 return !Legal->isInvariant(V: I->getOperand(i: 1));
2477 case Instruction::SDiv:
2478 case Instruction::SRem:
2479 // Conservative for now, since masked-off lanes may be poison and could
2480 // trigger signed overflow.
2481 return true;
2482 }
2483}
2484
2485uint64_t LoopVectorizationCostModel::getPredBlockCostDivisor(
2486 TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) {
2487 if (CostKind == TTI::TCK_CodeSize)
2488 return 1;
2489 // If the block wasn't originally predicated then return early to avoid
2490 // computing BlockFrequencyInfo unnecessarily.
2491 if (!Legal->blockNeedsPredication(BB))
2492 return 1;
2493
2494 uint64_t HeaderFreq =
2495 getBFI().getBlockFreq(BB: TheLoop->getHeader()).getFrequency();
2496 uint64_t BBFreq = getBFI().getBlockFreq(BB).getFrequency();
2497 assert(HeaderFreq >= BBFreq &&
2498 "Header has smaller block freq than dominated BB?");
2499 return std::round(x: (double)HeaderFreq / BBFreq);
2500}
2501
2502static Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode) {
2503 switch (Opcode) {
2504 case Instruction::UDiv:
2505 return Intrinsic::masked_udiv;
2506 case Instruction::SDiv:
2507 return Intrinsic::masked_sdiv;
2508 case Instruction::URem:
2509 return Intrinsic::masked_urem;
2510 case Instruction::SRem:
2511 return Intrinsic::masked_srem;
2512 default:
2513 llvm_unreachable("Unexpected opcode");
2514 }
2515}
2516
2517std::pair<InstructionCost, InstructionCost>
2518LoopVectorizationCostModel::getDivRemSpeculationCost(Instruction *I,
2519 ElementCount VF) {
2520 assert(I->getOpcode() == Instruction::UDiv ||
2521 I->getOpcode() == Instruction::SDiv ||
2522 I->getOpcode() == Instruction::SRem ||
2523 I->getOpcode() == Instruction::URem);
2524 assert(!isSafeToSpeculativelyExecute(I));
2525
2526 // Scalarization isn't legal for scalable vector types
2527 InstructionCost ScalarizationCost = InstructionCost::getInvalid();
2528 if (!VF.isScalable()) {
2529 // Get the scalarization cost and scale this amount by the probability of
2530 // executing the predicated block. If the instruction is not predicated,
2531 // we fall through to the next case.
2532 ScalarizationCost = 0;
2533
2534 // These instructions have a non-void type, so account for the phi nodes
2535 // that we will create. This cost is likely to be zero. The phi node
2536 // cost, if any, should be scaled by the block probability because it
2537 // models a copy at the end of each predicated block.
2538 ScalarizationCost += VF.getFixedValue() *
2539 TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
2540
2541 // The cost of the non-predicated instruction.
2542 ScalarizationCost +=
2543 VF.getFixedValue() * TTI.getArithmeticInstrCost(
2544 Opcode: I->getOpcode(), Ty: I->getType(), CostKind: Config.CostKind);
2545
2546 // The cost of insertelement and extractelement instructions needed for
2547 // scalarization.
2548 ScalarizationCost += getScalarizationOverhead(I, VF);
2549
2550 // Scale the cost by the probability of executing the predicated blocks.
2551 // This assumes the predicated block for each vector lane is equally
2552 // likely.
2553 ScalarizationCost =
2554 ScalarizationCost /
2555 getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
2556 }
2557
2558 auto *VecTy = toVectorTy(Scalar: I->getType(), EC: VF);
2559 auto *MaskTy = toVectorTy(Scalar: Type::getInt1Ty(C&: I->getContext()), EC: VF);
2560 IntrinsicCostAttributes ICA(getMaskedDivRemIntrinsic(Opcode: I->getOpcode()), VecTy,
2561 {VecTy, VecTy, MaskTy});
2562 InstructionCost MaskedCost = TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind);
2563 return {ScalarizationCost, MaskedCost};
2564}
2565
2566bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
2567 Instruction *I, ElementCount VF) const {
2568 assert(isAccessInterleaved(I) && "Expecting interleaved access.");
2569 assert(getWideningDecision(I, VF) == CM_Unknown &&
2570 "Decision should not be set yet.");
2571 auto *Group = getInterleavedAccessGroup(Instr: I);
2572 assert(Group && "Must have a group.");
2573 unsigned InterleaveFactor = Group->getFactor();
2574
2575 // If the instruction's allocated size doesn't equal its type size, it
2576 // requires padding and will be scalarized.
2577 auto &DL = I->getDataLayout();
2578 auto *ScalarTy = getLoadStoreType(I);
2579 if (hasIrregularType(Ty: ScalarTy, DL))
2580 return false;
2581
2582 // For scalable vectors, the interleave factors must be <= 8 since we require
2583 // the (de)interleaveN intrinsics instead of shufflevectors.
2584 if (VF.isScalable() && InterleaveFactor > 8)
2585 return false;
2586
2587 // If the group involves a non-integral pointer, we may not be able to
2588 // losslessly cast all values to a common type.
2589 bool ScalarNI = DL.isNonIntegralPointerType(Ty: ScalarTy);
2590 for (Instruction *Member : Group->members()) {
2591 auto *MemberTy = getLoadStoreType(I: Member);
2592 bool MemberNI = DL.isNonIntegralPointerType(Ty: MemberTy);
2593 // Don't coerce non-integral pointers to integers or vice versa.
2594 if (MemberNI != ScalarNI)
2595 // TODO: Consider adding special nullptr value case here
2596 return false;
2597 if (MemberNI && ScalarNI &&
2598 ScalarTy->getPointerAddressSpace() !=
2599 MemberTy->getPointerAddressSpace())
2600 return false;
2601 }
2602
2603 // Check if masking is required.
2604 // A Group may need masking for one of two reasons: it resides in a block that
2605 // needs predication, or it was decided to use masking to deal with gaps
2606 // (either a gap at the end of a load-access that may result in a speculative
2607 // load, or any gaps in a store-access).
2608 bool PredicatedAccessRequiresMasking =
2609 blockNeedsPredicationForAnyReason(BB: I->getParent()) && isMaskRequired(I);
2610 bool LoadAccessWithGapsRequiresEpilogMasking =
2611 isa<LoadInst>(Val: I) && Group->requiresScalarEpilogue() &&
2612 !isEpilogueAllowed();
2613 bool StoreAccessWithGapsRequiresMasking =
2614 isa<StoreInst>(Val: I) && !Group->isFull();
2615 if (!PredicatedAccessRequiresMasking &&
2616 !LoadAccessWithGapsRequiresEpilogMasking &&
2617 !StoreAccessWithGapsRequiresMasking)
2618 return true;
2619
2620 // If masked interleaving is required, we expect that the user/target had
2621 // enabled it, because otherwise it either wouldn't have been created or
2622 // it should have been invalidated by the CostModel.
2623 assert(useMaskedInterleavedAccesses(TTI) &&
2624 "Masked interleave-groups for predicated accesses are not enabled.");
2625
2626 if (Group->isReverse())
2627 return false;
2628
2629 // TODO: Support interleaved access that requires a gap mask for scalable VFs.
2630 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2631 StoreAccessWithGapsRequiresMasking;
2632 if (VF.isScalable() && NeedsMaskForGaps)
2633 return false;
2634
2635 return isLegalMaskedLoadOrStore(I, VF);
2636}
2637
2638std::optional<LoopVectorizationCostModel::InstWidening>
2639LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
2640 ElementCount VF) {
2641 // Get and ensure we have a valid memory instruction.
2642 assert((isa<LoadInst, StoreInst>(I)) && "Invalid memory instruction");
2643
2644 auto *Ptr = getLoadStorePointerOperand(V: I);
2645 auto *ScalarTy = getLoadStoreType(I);
2646
2647 // In order to be widened, the pointer should be consecutive, first of all.
2648 int Stride = Legal->isConsecutivePtr(AccessTy: ScalarTy, Ptr);
2649 if (!Stride)
2650 return std::nullopt;
2651
2652 // If the instruction is a store located in a predicated block, it will be
2653 // scalarized.
2654 if (isScalarWithPredication(I, VF))
2655 return std::nullopt;
2656
2657 // If the instruction's allocated size doesn't equal it's type size, it
2658 // requires padding and will be scalarized.
2659 auto &DL = I->getDataLayout();
2660 if (hasIrregularType(Ty: ScalarTy, DL))
2661 return std::nullopt;
2662
2663 return Stride == 1 ? CM_Widen : CM_Widen_Reverse;
2664}
2665
2666void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
2667 // We should not collect Uniforms more than once per VF. Right now,
2668 // this function is called from collectUniformsAndScalars(), which
2669 // already does this check. Collecting Uniforms for VF=1 does not make any
2670 // sense.
2671
2672 assert(VF.isVector() && !Uniforms.contains(VF) &&
2673 "This function should not be visited twice for the same VF");
2674
2675 // Visit the list of Uniforms. If we find no uniform value, we won't
2676 // analyze again. Uniforms.count(VF) will return 1.
2677 Uniforms[VF].clear();
2678
2679 // Now we know that the loop is vectorizable!
2680 // Collect instructions inside the loop that will remain uniform after
2681 // vectorization.
2682
2683 // Global values, params and instructions outside of current loop are out of
2684 // scope.
2685 auto IsOutOfScope = [&](Value *V) -> bool {
2686 Instruction *I = dyn_cast<Instruction>(Val: V);
2687 return (!I || !TheLoop->contains(Inst: I));
2688 };
2689
2690 // Worklist containing uniform instructions demanding lane 0.
2691 SetVector<Instruction *> Worklist;
2692
2693 // Add uniform instructions demanding lane 0 to the worklist. Instructions
2694 // that require predication must not be considered uniform after
2695 // vectorization, because that would create an erroneous replicating region
2696 // where only a single instance out of VF should be formed.
2697 auto AddToWorklistIfAllowed = [&](Instruction *I) -> void {
2698 if (IsOutOfScope(I)) {
2699 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "
2700 << *I << "\n");
2701 return;
2702 }
2703 if (isPredicatedInst(I)) {
2704 LLVM_DEBUG(
2705 dbgs() << "LV: Found not uniform due to requiring predication: " << *I
2706 << "\n");
2707 return;
2708 }
2709 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n");
2710 Worklist.insert(X: I);
2711 };
2712
2713 // Start with the conditional branches exiting the loop. If the branch
2714 // condition is an instruction contained in the loop that is only used by the
2715 // branch, it is uniform. Note conditions from uncountable early exits are not
2716 // uniform.
2717 SmallVector<BasicBlock *> Exiting;
2718 TheLoop->getExitingBlocks(ExitingBlocks&: Exiting);
2719 for (BasicBlock *E : Exiting) {
2720 if (Legal->hasUncountableEarlyExit() && TheLoop->getLoopLatch() != E)
2721 continue;
2722 auto *Cmp = dyn_cast<Instruction>(Val: E->getTerminator()->getOperand(i: 0));
2723 if (Cmp && TheLoop->contains(Inst: Cmp) && Cmp->hasOneUse())
2724 AddToWorklistIfAllowed(Cmp);
2725 }
2726
2727 auto PrevVF = VF.divideCoefficientBy(RHS: 2);
2728 // Return true if all lanes perform the same memory operation, and we can
2729 // thus choose to execute only one.
2730 auto IsUniformMemOpUse = [&](Instruction *I) {
2731 // If the value was already known to not be uniform for the previous
2732 // (smaller VF), it cannot be uniform for the larger VF.
2733 if (PrevVF.isVector()) {
2734 auto Iter = Uniforms.find(Val: PrevVF);
2735 if (Iter != Uniforms.end() && !Iter->second.contains(Ptr: I))
2736 return false;
2737 }
2738 if (!isUniformMemOp(I&: *I, VF))
2739 return false;
2740 if (isa<LoadInst>(Val: I))
2741 // Loading the same address always produces the same result - at least
2742 // assuming aliasing and ordering which have already been checked.
2743 return true;
2744 // Storing the same value on every iteration.
2745 return TheLoop->isLoopInvariant(V: cast<StoreInst>(Val: I)->getValueOperand());
2746 };
2747
2748 auto IsUniformDecision = [&](Instruction *I, ElementCount VF) {
2749 InstWidening WideningDecision = getWideningDecision(I, VF);
2750 assert(WideningDecision != CM_Unknown &&
2751 "Widening decision should be ready at this moment");
2752
2753 if (IsUniformMemOpUse(I))
2754 return true;
2755
2756 return (WideningDecision == CM_Widen ||
2757 WideningDecision == CM_Widen_Reverse ||
2758 WideningDecision == CM_Interleave);
2759 };
2760
2761 // Returns true if Ptr is the pointer operand of a memory access instruction
2762 // I, I is known to not require scalarization, and the pointer is not also
2763 // stored.
2764 auto IsVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
2765 if (isa<StoreInst>(Val: I) && I->getOperand(i: 0) == Ptr)
2766 return false;
2767 return getLoadStorePointerOperand(V: I) == Ptr &&
2768 (IsUniformDecision(I, VF) || Legal->isInvariant(V: Ptr));
2769 };
2770
2771 // Holds a list of values which are known to have at least one uniform use.
2772 // Note that there may be other uses which aren't uniform. A "uniform use"
2773 // here is something which only demands lane 0 of the unrolled iterations;
2774 // it does not imply that all lanes produce the same value (e.g. this is not
2775 // the usual meaning of uniform)
2776 SetVector<Value *> HasUniformUse;
2777
2778 // Scan the loop for instructions which are either a) known to have only
2779 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
2780 for (auto *BB : TheLoop->blocks())
2781 for (auto &I : *BB) {
2782 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &I)) {
2783 switch (II->getIntrinsicID()) {
2784 case Intrinsic::sideeffect:
2785 case Intrinsic::experimental_noalias_scope_decl:
2786 case Intrinsic::assume:
2787 case Intrinsic::lifetime_start:
2788 case Intrinsic::lifetime_end:
2789 if (TheLoop->hasLoopInvariantOperands(I: &I))
2790 AddToWorklistIfAllowed(&I);
2791 break;
2792 default:
2793 break;
2794 }
2795 }
2796
2797 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: &I)) {
2798 if (IsOutOfScope(EVI->getAggregateOperand())) {
2799 AddToWorklistIfAllowed(EVI);
2800 continue;
2801 }
2802 // Only ExtractValue instructions where the aggregate value comes from a
2803 // call are allowed to be non-uniform.
2804 assert(isa<CallInst>(EVI->getAggregateOperand()) &&
2805 "Expected aggregate value to be call return value");
2806 }
2807
2808 // If there's no pointer operand, there's nothing to do.
2809 auto *Ptr = getLoadStorePointerOperand(V: &I);
2810 if (!Ptr)
2811 continue;
2812
2813 // If the pointer can be proven to be uniform, always add it to the
2814 // worklist.
2815 if (isa<Instruction>(Val: Ptr) && isUniform(V: Ptr, VF))
2816 AddToWorklistIfAllowed(cast<Instruction>(Val: Ptr));
2817
2818 if (IsUniformMemOpUse(&I))
2819 AddToWorklistIfAllowed(&I);
2820
2821 if (IsVectorizedMemAccessUse(&I, Ptr))
2822 HasUniformUse.insert(X: Ptr);
2823 }
2824
2825 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
2826 // demanding) users. Since loops are assumed to be in LCSSA form, this
2827 // disallows uses outside the loop as well.
2828 for (auto *V : HasUniformUse) {
2829 if (IsOutOfScope(V))
2830 continue;
2831 auto *I = cast<Instruction>(Val: V);
2832 bool UsersAreMemAccesses = all_of(Range: I->users(), P: [&](User *U) -> bool {
2833 auto *UI = cast<Instruction>(Val: U);
2834 return TheLoop->contains(Inst: UI) && IsVectorizedMemAccessUse(UI, V);
2835 });
2836 if (UsersAreMemAccesses)
2837 AddToWorklistIfAllowed(I);
2838 }
2839
2840 // Expand Worklist in topological order: whenever a new instruction
2841 // is added , its users should be already inside Worklist. It ensures
2842 // a uniform instruction will only be used by uniform instructions.
2843 unsigned Idx = 0;
2844 while (Idx != Worklist.size()) {
2845 Instruction *I = Worklist[Idx++];
2846
2847 for (auto *OV : I->operand_values()) {
2848 // isOutOfScope operands cannot be uniform instructions.
2849 if (IsOutOfScope(OV))
2850 continue;
2851 // First order recurrence Phi's should typically be considered
2852 // non-uniform.
2853 auto *OP = dyn_cast<PHINode>(Val: OV);
2854 if (OP && Legal->isFixedOrderRecurrence(Phi: OP))
2855 continue;
2856 // If all the users of the operand are uniform, then add the
2857 // operand into the uniform worklist.
2858 auto *OI = cast<Instruction>(Val: OV);
2859 if (llvm::all_of(Range: OI->users(), P: [&](User *U) -> bool {
2860 auto *J = cast<Instruction>(Val: U);
2861 return Worklist.count(key: J) || IsVectorizedMemAccessUse(J, OI);
2862 }))
2863 AddToWorklistIfAllowed(OI);
2864 }
2865 }
2866
2867 // For an instruction to be added into Worklist above, all its users inside
2868 // the loop should also be in Worklist. However, this condition cannot be
2869 // true for phi nodes that form a cyclic dependence. We must process phi
2870 // nodes separately. An induction variable will remain uniform if all users
2871 // of the induction variable and induction variable update remain uniform.
2872 // The code below handles both pointer and non-pointer induction variables.
2873 BasicBlock *Latch = TheLoop->getLoopLatch();
2874 for (const auto &Induction : Legal->getInductionVars()) {
2875 auto *Ind = Induction.first;
2876 auto *IndUpdate = cast<Instruction>(Val: Ind->getIncomingValueForBlock(BB: Latch));
2877
2878 // Determine if all users of the induction variable are uniform after
2879 // vectorization.
2880 bool UniformInd = all_of(Range: Ind->users(), P: [&](User *U) -> bool {
2881 auto *I = cast<Instruction>(Val: U);
2882 return I == IndUpdate || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2883 IsVectorizedMemAccessUse(I, Ind);
2884 });
2885 if (!UniformInd)
2886 continue;
2887
2888 // Determine if all users of the induction variable update instruction are
2889 // uniform after vectorization.
2890 bool UniformIndUpdate = all_of(Range: IndUpdate->users(), P: [&](User *U) -> bool {
2891 auto *I = cast<Instruction>(Val: U);
2892 return I == Ind || Worklist.count(key: I) ||
2893 IsVectorizedMemAccessUse(I, IndUpdate);
2894 });
2895 if (!UniformIndUpdate)
2896 continue;
2897
2898 // The induction variable and its update instruction will remain uniform.
2899 AddToWorklistIfAllowed(Ind);
2900 AddToWorklistIfAllowed(IndUpdate);
2901 }
2902
2903 Uniforms[VF].insert_range(R&: Worklist);
2904}
2905
2906FixedScalableVFPair
2907LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
2908 // Make sure once we return PartialAliasMaskingStatus is not "NotDecided".
2909 scope_exit EnsureAliasMaskingStatusIsDecidedOnReturn([this] {
2910 if (PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided)
2911 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
2912 });
2913
2914 // For outer loops, use simple type-based heuristic VF. No cost model or
2915 // memory dependence analysis is available.
2916 if (!TheLoop->isInnermost()) {
2917 return Config.computeVPlanOuterloopVF(UserVF);
2918 }
2919
2920 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
2921 // TODO: It may be useful to do since it's still likely to be dynamically
2922 // uniform if the target can skip.
2923 reportVectorizationFailure(
2924 DebugMsg: "Not inserting runtime ptr check for divergent target",
2925 OREMsg: "runtime pointer checks needed. Not enabled for divergent target",
2926 ORETag: "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
2927 return FixedScalableVFPair::getNone();
2928 }
2929
2930 ScalarEvolution *SE = PSE.getSE();
2931 ElementCount TC = getSmallConstantTripCount(SE, L: TheLoop);
2932 unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
2933 if (!MaxTC && EpilogueLoweringStatus == CM_EpilogueAllowed)
2934 MaxTC = getMaxTCFromNonZeroRange(PSE, L: TheLoop);
2935 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
2936 if (TC != ElementCount::getFixed(MinVal: MaxTC))
2937 LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
2938 if (TC.isScalar()) {
2939 reportVectorizationFailure(
2940 DebugMsg: "Single iteration (non) loop",
2941 OREMsg: "loop trip count is one, irrelevant for vectorization",
2942 ORETag: "SingleIterationLoop", ORE, TheLoop);
2943 return FixedScalableVFPair::getNone();
2944 }
2945
2946 // If BTC matches the widest induction type and is -1 then the trip count
2947 // computation will wrap to 0 and the vector trip count will be 0. Do not try
2948 // to vectorize.
2949 const SCEV *BTC = SE->getBackedgeTakenCount(L: TheLoop);
2950 if (!isa<SCEVCouldNotCompute>(Val: BTC) &&
2951 BTC->getType()->getScalarSizeInBits() >=
2952 Legal->getWidestInductionType()->getScalarSizeInBits() &&
2953 SE->isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: BTC,
2954 RHS: SE->getMinusOne(Ty: BTC->getType()))) {
2955 reportVectorizationFailure(
2956 DebugMsg: "Trip count computation wrapped",
2957 OREMsg: "backedge-taken count is -1, loop trip count wrapped to 0",
2958 ORETag: "TripCountWrapped", ORE, TheLoop);
2959 return FixedScalableVFPair::getNone();
2960 }
2961
2962 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
2963 "No cost-modeling decisions should have been taken at this point");
2964
2965 switch (EpilogueLoweringStatus) {
2966 case CM_EpilogueAllowed:
2967 return Config.computeFeasibleMaxVF(MaxTripCount: MaxTC, UserVF, UserIC, FoldTailByMasking: false,
2968 RequiresScalarEpilogue: requiresScalarEpilogue(IsVectorizing: true));
2969 case CM_EpilogueNotAllowedFoldTail:
2970 [[fallthrough]];
2971 case CM_EpilogueNotNeededFoldTail:
2972 LLVM_DEBUG(dbgs() << "LV: tail-folding hint/switch found.\n"
2973 << "LV: Not allowing epilogue, creating tail-folded "
2974 << "vector loop.\n");
2975 break;
2976 case CM_EpilogueNotAllowedLowTripLoop:
2977 // fallthrough as a special case of OptForSize
2978 case CM_EpilogueNotAllowedOptSize:
2979 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize)
2980 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
2981 else
2982 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
2983 << "count.\n");
2984
2985 // Bail if runtime checks are required, which are not good when optimising
2986 // for size.
2987 if (Config.runtimeChecksRequired())
2988 return FixedScalableVFPair::getNone();
2989
2990 break;
2991 }
2992
2993 // Now try the tail folding
2994
2995 // Invalidate interleave groups that require an epilogue if we can't mask
2996 // the interleave-group.
2997 if (!useMaskedInterleavedAccesses(TTI)) {
2998 // Note: There is no need to invalidate any cost modeling decisions here, as
2999 // none were taken so far (see assertion above).
3000 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
3001 }
3002
3003 FixedScalableVFPair MaxFactors = Config.computeFeasibleMaxVF(
3004 MaxTripCount: MaxTC, UserVF, UserIC, FoldTailByMasking: true, RequiresScalarEpilogue: requiresScalarEpilogue(IsVectorizing: true));
3005
3006 // Avoid tail folding if the trip count is known to be a multiple of any VF
3007 // we choose.
3008 std::optional<uint64_t> MaxPowerOf2RuntimeVF =
3009 MaxFactors.FixedVF.getFixedValue();
3010 if (MaxFactors.ScalableVF) {
3011 if (std::optional<uint64_t> MaxRuntimeScalableVF =
3012 getMaxRuntimeElementCount(EC: MaxFactors.ScalableVF, F: *TheFunction))
3013 MaxPowerOf2RuntimeVF =
3014 std::max(a: *MaxPowerOf2RuntimeVF, b: *MaxRuntimeScalableVF);
3015 else
3016 MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
3017 }
3018
3019 auto NoScalarEpilogueNeeded = [this, &UserIC](uint64_t MaxRuntimeVF) {
3020 // Return false if the loop is neither a single-latch-exit loop nor an
3021 // early-exit loop as tail-folding is not supported in that case.
3022 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
3023 !Legal->hasUncountableEarlyExit())
3024 return false;
3025 uint64_t MaxVFtimesIC = MaxRuntimeVF * std::max<uint64_t>(a: UserIC, b: 1);
3026 ScalarEvolution *SE = PSE.getSE();
3027 // Calling getSymbolicMaxBackedgeTakenCount enables support for loops
3028 // with uncountable exits. For countable loops, the symbolic maximum must
3029 // remain identical to the known back-edge taken count.
3030 const SCEV *BackedgeTakenCount = PSE.getSymbolicMaxBackedgeTakenCount();
3031 assert((Legal->hasUncountableEarlyExit() ||
3032 BackedgeTakenCount == PSE.getBackedgeTakenCount()) &&
3033 "Invalid loop count");
3034 const SCEV *ExitCount = SE->getAddExpr(
3035 LHS: BackedgeTakenCount, RHS: SE->getOne(Ty: BackedgeTakenCount->getType()));
3036 const SCEV *Rem = SE->getURemExpr(
3037 LHS: SE->applyLoopGuards(Expr: ExitCount, L: TheLoop),
3038 RHS: SE->getConstant(Ty: BackedgeTakenCount->getType(), V: MaxVFtimesIC));
3039 return Rem->isZero();
3040 };
3041
3042 if (MaxPowerOf2RuntimeVF > 0u) {
3043 assert((UserVF.isNonZero() || isPowerOf2_64(*MaxPowerOf2RuntimeVF)) &&
3044 "MaxFixedVF must be a power of 2");
3045 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
3046 // Accept MaxFixedVF if we do not have a tail.
3047 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
3048 return MaxFactors;
3049 }
3050 }
3051
3052 auto ExpectedTC = getSmallBestKnownTC(PSE, L: TheLoop);
3053 if (ExpectedTC && ExpectedTC->isFixed() &&
3054 ExpectedTC->getFixedValue() <=
3055 TTI.getMinTripCountTailFoldingThreshold()) {
3056 if (MaxPowerOf2RuntimeVF > 0u) {
3057 // If we have a low-trip-count, and the fixed-width VF is known to divide
3058 // the trip count but the scalable factor does not, use the fixed-width
3059 // factor in preference to allow the generation of a non-predicated loop.
3060 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
3061 NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
3062 LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
3063 "remain for any chosen VF.\n");
3064 MaxFactors.ScalableVF = ElementCount::getScalable(MinVal: 0);
3065 return MaxFactors;
3066 }
3067 }
3068
3069 reportVectorizationFailure(
3070 DebugMsg: "The trip count is below the minial threshold value.",
3071 OREMsg: "loop trip count is too low, avoiding vectorization", ORETag: "LowTripCount",
3072 ORE, TheLoop);
3073 return FixedScalableVFPair::getNone();
3074 }
3075
3076 // If we don't know the precise trip count, or if the trip count that we
3077 // found modulo the vectorization factor is not zero, try to fold the tail
3078 // by masking.
3079 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
3080 bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
3081 setTailFoldingStyle(IsScalableVF: ContainsScalableVF, UserIC);
3082 if (foldTailByMasking()) {
3083 if (foldTailWithEVL()) {
3084 LLVM_DEBUG(
3085 dbgs()
3086 << "LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
3087 "try to generate VP Intrinsics with scalable vector "
3088 "factors only.\n");
3089 // Tail folded loop using VP intrinsics restricts the VF to be scalable
3090 // for now.
3091 // TODO: extend it for fixed vectors, if required.
3092 assert(ContainsScalableVF && "Expected scalable vector factor.");
3093
3094 MaxFactors.FixedVF = ElementCount::getFixed(MinVal: 1);
3095 } else {
3096 tryToEnablePartialAliasMasking();
3097 }
3098 return MaxFactors;
3099 }
3100
3101 // If there was a tail-folding hint/switch, but we can't fold the tail by
3102 // masking, fallback to a vectorization with an epilogue.
3103 if (EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail) {
3104 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with an "
3105 "epilogue instead.\n");
3106 EpilogueLoweringStatus = CM_EpilogueAllowed;
3107 return MaxFactors;
3108 }
3109
3110 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail) {
3111 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
3112 return FixedScalableVFPair::getNone();
3113 }
3114
3115 if (TC.isZero()) {
3116 reportVectorizationFailure(
3117 DebugMsg: "unable to calculate the loop count due to complex control flow",
3118 ORETag: "UnknownLoopCountComplexCFG", ORE, TheLoop);
3119 return FixedScalableVFPair::getNone();
3120 }
3121
3122 reportVectorizationFailure(
3123 DebugMsg: "Cannot optimize for size and vectorize at the same time.",
3124 OREMsg: "cannot optimize for size and vectorize at the same time. "
3125 "Enable vectorization of this loop with '#pragma clang loop "
3126 "vectorize(enable)' when compiling with -Os/-Oz",
3127 ORETag: "NoTailLoopWithOptForSize", ORE, TheLoop);
3128 return FixedScalableVFPair::getNone();
3129}
3130
3131void LoopVectorizationPlanner::emitInvalidCostRemarks(
3132 OptimizationRemarkEmitter *ORE) {
3133 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
3134 SmallVector<RecipeVFPair> InvalidCosts;
3135 for (const auto &Plan : VPlans) {
3136 for (ElementCount VF : Plan->vectorFactors()) {
3137 // The VPlan-based cost model is designed for computing vector cost.
3138 // Querying VPlan-based cost model with a scarlar VF will cause some
3139 // errors because we expect the VF is vector for most of the widen
3140 // recipes.
3141 if (VF.isScalar())
3142 continue;
3143
3144 VPCostContext CostCtx(*TLI, *Plan, *CM, Config,
3145 /*ReusePrintingSlotTracker=*/true);
3146 precomputeCosts(Plan&: *Plan, VF, CostCtx);
3147 auto Iter = vp_depth_first_deep(G: Plan->getVectorLoopRegion()->getEntry());
3148 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: Iter)) {
3149 for (auto &R : *VPBB) {
3150 if (!R.cost(VF, Ctx&: CostCtx).isValid())
3151 InvalidCosts.emplace_back(Args: &R, Args&: VF);
3152 }
3153 }
3154 }
3155 }
3156 if (InvalidCosts.empty())
3157 return;
3158
3159 // Emit a report of VFs with invalid costs in the loop.
3160
3161 // Group the remarks per recipe, keeping the recipe order from InvalidCosts.
3162 DenseMap<VPRecipeBase *, unsigned> Numbering;
3163 unsigned I = 0;
3164 for (auto &Pair : InvalidCosts)
3165 if (Numbering.try_emplace(Key: Pair.first, Args&: I).second)
3166 ++I;
3167
3168 // Sort the list, first on recipe(number) then on VF.
3169 sort(C&: InvalidCosts, Comp: [&Numbering](RecipeVFPair &A, RecipeVFPair &B) {
3170 unsigned NA = Numbering[A.first];
3171 unsigned NB = Numbering[B.first];
3172 if (NA != NB)
3173 return NA < NB;
3174 return ElementCount::isKnownLT(LHS: A.second, RHS: B.second);
3175 });
3176
3177 // For a list of ordered recipe-VF pairs:
3178 // [(load, VF1), (load, VF2), (store, VF1)]
3179 // group the recipes together to emit separate remarks for:
3180 // load (VF1, VF2)
3181 // store (VF1)
3182 auto Tail = ArrayRef<RecipeVFPair>(InvalidCosts);
3183 auto Subset = ArrayRef<RecipeVFPair>();
3184 do {
3185 if (Subset.empty())
3186 Subset = Tail.take_front(N: 1);
3187
3188 VPRecipeBase *R = Subset.front().first;
3189
3190 unsigned Opcode =
3191 TypeSwitch<const VPRecipeBase *, unsigned>(R)
3192 .Case(caseFn: [](const VPHeaderPHIRecipe *R) { return Instruction::PHI; })
3193 .Case(
3194 caseFn: [](const VPWidenStoreRecipe *R) { return Instruction::Store; })
3195 .Case(caseFn: [](const VPWidenLoadRecipe *R) { return Instruction::Load; })
3196 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
3197 caseFn: [](const auto *R) { return Instruction::Call; })
3198 .Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
3199 VPWidenCastRecipe>(
3200 caseFn: [](const auto *R) { return R->getOpcode(); })
3201 .Case(caseFn: [](const VPInterleaveRecipe *R) {
3202 return R->getStoredValues().empty() ? Instruction::Load
3203 : Instruction::Store;
3204 })
3205 .Case(caseFn: [](const VPReductionRecipe *R) {
3206 return RecurrenceDescriptor::getOpcode(Kind: R->getRecurrenceKind());
3207 });
3208
3209 // If the next recipe is different, or if there are no other pairs,
3210 // emit a remark for the collated subset. e.g.
3211 // [(load, VF1), (load, VF2))]
3212 // to emit:
3213 // remark: invalid costs for 'load' at VF=(VF1, VF2)
3214 if (Subset == Tail || Tail[Subset.size()].first != R) {
3215 std::string OutString;
3216 raw_string_ostream OS(OutString);
3217 assert(!Subset.empty() && "Unexpected empty range");
3218 OS << "Recipe with invalid costs prevented vectorization at VF=(";
3219 for (const auto &Pair : Subset)
3220 OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second;
3221 OS << "):";
3222 if (Opcode == Instruction::Call) {
3223 StringRef Name = "";
3224 if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(Val: R)) {
3225 Name = Int->getIntrinsicName();
3226 } else {
3227 auto *WidenCall = dyn_cast<VPWidenCallRecipe>(Val: R);
3228 Function *CalledFn =
3229 WidenCall ? WidenCall->getCalledScalarFunction()
3230 : cast<Function>(Val: R->getOperand(N: R->getNumOperands() - 1)
3231 ->getLiveInIRValue());
3232 Name = CalledFn->getName();
3233 }
3234 OS << " call to " << Name;
3235 } else
3236 OS << " " << Instruction::getOpcodeName(Opcode);
3237 reportVectorizationInfo(Msg: OutString, ORETag: "InvalidCost", ORE, TheLoop: OrigLoop, I: nullptr,
3238 DL: R->getDebugLoc());
3239 Tail = Tail.drop_front(N: Subset.size());
3240 Subset = {};
3241 } else
3242 // Grow the subset by one element
3243 Subset = Tail.take_front(N: Subset.size() + 1);
3244 } while (!Tail.empty());
3245}
3246
3247/// Check if any recipe of \p Plan will generate a vector value, which will be
3248/// assigned a vector register.
3249static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
3250 const TargetTransformInfo &TTI) {
3251 assert(VF.isVector() && "Checking a scalar VF?");
3252 DenseSet<VPRecipeBase *> EphemeralRecipes;
3253 collectEphemeralRecipesForVPlan(Plan, EphRecipes&: EphemeralRecipes);
3254 // Set of already visited types.
3255 DenseSet<Type *> Visited;
3256 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3257 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
3258 for (VPRecipeBase &R : *VPBB) {
3259 if (EphemeralRecipes.contains(V: &R))
3260 continue;
3261 // Continue early if the recipe is considered to not produce a vector
3262 // result. Note that this includes VPInstruction where some opcodes may
3263 // produce a vector, to preserve existing behavior as VPInstructions model
3264 // aspects not directly mapped to existing IR instructions.
3265 switch (R.getVPRecipeID()) {
3266 case VPRecipeBase::VPDerivedIVSC:
3267 case VPRecipeBase::VPScalarIVStepsSC:
3268 case VPRecipeBase::VPReplicateSC:
3269 case VPRecipeBase::VPInstructionSC:
3270 case VPRecipeBase::VPCurrentIterationPHISC:
3271 case VPRecipeBase::VPVectorPointerSC:
3272 case VPRecipeBase::VPVectorEndPointerSC:
3273 case VPRecipeBase::VPExpandSCEVSC:
3274 case VPRecipeBase::VPPredInstPHISC:
3275 case VPRecipeBase::VPBranchOnMaskSC:
3276 continue;
3277 case VPRecipeBase::VPReductionSC:
3278 case VPRecipeBase::VPActiveLaneMaskPHISC:
3279 case VPRecipeBase::VPWidenCallSC:
3280 case VPRecipeBase::VPWidenCanonicalIVSC:
3281 case VPRecipeBase::VPWidenCastSC:
3282 case VPRecipeBase::VPWidenGEPSC:
3283 case VPRecipeBase::VPWidenIntrinsicSC:
3284 case VPRecipeBase::VPWidenMemIntrinsicSC:
3285 case VPRecipeBase::VPWidenSC:
3286 case VPRecipeBase::VPBlendSC:
3287 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
3288 case VPRecipeBase::VPHistogramSC:
3289 case VPRecipeBase::VPWidenPHISC:
3290 case VPRecipeBase::VPWidenIntOrFpInductionSC:
3291 case VPRecipeBase::VPWidenPointerInductionSC:
3292 case VPRecipeBase::VPReductionPHISC:
3293 case VPRecipeBase::VPInterleaveEVLSC:
3294 case VPRecipeBase::VPInterleaveSC:
3295 case VPRecipeBase::VPWidenLoadEVLSC:
3296 case VPRecipeBase::VPWidenLoadSC:
3297 case VPRecipeBase::VPWidenStoreEVLSC:
3298 case VPRecipeBase::VPWidenStoreSC:
3299 break;
3300 default:
3301 llvm_unreachable("unhandled recipe");
3302 }
3303
3304 auto WillGenerateTargetVectors = [&TTI, VF](Type *VectorTy) {
3305 unsigned NumLegalParts = TTI.getNumberOfParts(Tp: VectorTy);
3306 if (!NumLegalParts)
3307 return false;
3308 if (VF.isScalable()) {
3309 // <vscale x 1 x iN> is assumed to be profitable over iN because
3310 // scalable registers are a distinct register class from scalar
3311 // ones. If we ever find a target which wants to lower scalable
3312 // vectors back to scalars, we'll need to update this code to
3313 // explicitly ask TTI about the register class uses for each part.
3314 return NumLegalParts <= VF.getKnownMinValue();
3315 }
3316 // Two or more elements that share a register - are vectorized.
3317 return NumLegalParts < VF.getFixedValue();
3318 };
3319
3320 // If no def nor is a store, e.g., branches, continue - no value to check.
3321 if (R.getNumDefinedValues() == 0 &&
3322 !isa<VPWidenStoreRecipe, VPWidenStoreEVLRecipe, VPInterleaveBase>(Val: &R))
3323 continue;
3324 // For multi-def recipes, currently only interleaved loads, suffice to
3325 // check first def only.
3326 // For stores check their stored value; for interleaved stores suffice
3327 // the check first stored value only. In all cases this is the second
3328 // operand.
3329 VPValue *ToCheck =
3330 R.getNumDefinedValues() >= 1 ? R.getVPValue(I: 0) : R.getOperand(N: 1);
3331 Type *ScalarTy = ToCheck->getScalarType();
3332 if (!Visited.insert(V: {ScalarTy}).second)
3333 continue;
3334 Type *WideTy = toVectorizedTy(Ty: ScalarTy, EC: VF);
3335 if (any_of(Range: getContainedTypes(Ty: WideTy), P: WillGenerateTargetVectors))
3336 return true;
3337 }
3338 }
3339
3340 return false;
3341}
3342
3343static bool hasReplicatorRegion(VPlan &Plan) {
3344 return any_of(Range: VPBlockUtils::blocksOnly<VPRegionBlock>(Range: vp_depth_first_shallow(
3345 G: Plan.getVectorLoopRegion()->getEntry())),
3346 P: [](auto *VPRB) { return VPRB->isReplicator(); });
3347}
3348
3349/// Returns true if the VPlan contains a VPReductionPHIRecipe with
3350/// FindLast recurrence kind.
3351static bool hasFindLastReductionPhi(VPlan &Plan) {
3352 return any_of(Range: make_isa_range<VPReductionPHIRecipe>(
3353 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
3354 P: [](VPReductionPHIRecipe &RedPhi) {
3355 return RecurrenceDescriptor::isFindLastRecurrenceKind(
3356 Kind: RedPhi.getRecurrenceKind());
3357 });
3358}
3359
3360/// Determine how to lower the epilogue for the vector epilogue loop.
3361/// Check if there are any conflicts that prevent tail-folding the epilogue.
3362/// \return CM_EpilogueNotNeededFoldTail if epilogue tail-folding is possible,
3363/// otherwise CM_EpilogueAllowed.
3364static EpilogueLowering
3365getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
3366 OptimizationRemarkEmitter *ORE,
3367 LoopVectorizationLegality &LVL,
3368 LoopVectorizeHints &Hints) {
3369 // Epilogue TF is only enabled when explicitly requested via command line.
3370 if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
3371 EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
3372 return CM_EpilogueAllowed;
3373
3374 if (!EnableEpilogueVectorization) {
3375 reportVectorizationInfo(
3376 Msg: "Options conflict, epilogue vectorization is disallowed while "
3377 "epilogue tail-folding allowed!",
3378 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
3379 return CM_EpilogueAllowed;
3380 }
3381
3382 if (!Hints.getWidth() || !hasForcedEpilogueVF()) {
3383 reportVectorizationInfo(Msg: "For now, epilogue tail-folding can't be "
3384 "applied without forced main/epilogue loop VF",
3385 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
3386 return CM_EpilogueAllowed;
3387 }
3388
3389 if (ElementCount::isKnownLE(LHS: Hints.getWidth(), RHS: EpilogueVectorizationForceVF)) {
3390 reportVectorizationInfo(Msg: "For now, epilogue tail-folding can't be applied "
3391 "when VF of the main loop <= VF of the epilogue",
3392 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
3393 return CM_EpilogueAllowed;
3394 }
3395
3396 if (!L->isInnermost()) {
3397 reportVectorizationInfo(
3398 Msg: "Epilogue tail-folding is not supported for outer loop",
3399 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3400 return CM_EpilogueAllowed;
3401 }
3402
3403 // If scalar epilogue is explicitly required, we can't apply TF.
3404 if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
3405 reportVectorizationInfo(
3406 Msg: "Epilogue tail-folding can't be applied because scalar epilogue is "
3407 "required. Fall back to a normal epilogue",
3408 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3409 return CM_EpilogueAllowed;
3410 }
3411
3412 // If having epilogue is NOT allowed, then no epilogue to apply TF for.
3413 if (!MainCM.isEpilogueAllowed()) {
3414 reportVectorizationInfo(Msg: "Not applying tail-folding to the epilogue, since "
3415 "no epilogue is allowed.",
3416 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3417 return CM_EpilogueAllowed;
3418 }
3419
3420 if (L->getExitingBlock() != L->getLoopLatch() ||
3421 LVL.hasUncountableEarlyExit()) {
3422 reportVectorizationInfo(
3423 Msg: "Epilogue tail-folding is not supported yet for early-exit loops",
3424 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3425 return CM_EpilogueAllowed;
3426 }
3427
3428 // We can apply tail-folding on the vectorized epilogue loop.
3429 return CM_EpilogueNotNeededFoldTail;
3430}
3431
3432bool VFSelectionContext::isEpilogueVectorizationProfitable(
3433 const ElementCount VF, const unsigned IC) const {
3434 // FIXME: We need a much better cost-model to take different parameters such
3435 // as register pressure, code size increase and cost of extra branches into
3436 // account. For now we apply a very crude heuristic and only consider loops
3437 // with vectorization factors larger than a certain value.
3438
3439 // Allow the target to opt out.
3440 if (!TTI.preferEpilogueVectorization(Iters: VF * IC))
3441 return false;
3442
3443 unsigned MinVFThreshold = EpilogueVectorizationMinVF.getNumOccurrences() > 0
3444 ? EpilogueVectorizationMinVF
3445 : TTI.getEpilogueVectorizationMinVF();
3446 return estimateElementCount(VF: VF * IC, VScale: getVScaleForTuning()) >= MinVFThreshold;
3447}
3448
3449std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
3450 VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC,
3451 bool ScalarEpilogueAllowed) {
3452 if (!EnableEpilogueVectorization) {
3453 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
3454 return nullptr;
3455 }
3456
3457 if (!ScalarEpilogueAllowed) {
3458 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
3459 "epilogue is allowed.\n");
3460 return nullptr;
3461 }
3462
3463 if (vputils::findIncomingAliasMask(Plan: MainPlan)) {
3464 LLVM_DEBUG(
3465 dbgs()
3466 << "LEV: Epilogue vectorization not supported with alias masking.\n");
3467 return nullptr;
3468 }
3469
3470 // Not really a cost consideration, but check for unsupported cases here to
3471 // simplify the logic.
3472 if (!isCandidateForEpilogueVectorization(MainPlan)) {
3473 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
3474 "is not a supported candidate.\n");
3475 return nullptr;
3476 }
3477
3478 if (hasForcedEpilogueVF()) {
3479 if (estimateElementCount(VF: EpilogueVectorizationForceVF,
3480 VScale: Config.getVScaleForTuning()) >=
3481 IC * estimateElementCount(VF: MainLoopVF, VScale: Config.getVScaleForTuning())) {
3482 // Note that the main loop leaves IC * MainLoopVF iterations iff a scalar
3483 // epilogue is required, but then the epilogue loop also requires a scalar
3484 // epilogue.
3485 LLVM_DEBUG(dbgs() << "LEV: Forced epilogue VF results in dead epilogue "
3486 "vector loop, skipping vectorizing epilogue.\n");
3487 return nullptr;
3488 }
3489
3490 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
3491 if (hasPlanWithVF(VF: EpilogueVectorizationForceVF)) {
3492 std::unique_ptr<VPlan> Clone(
3493 getPlanFor(VF: EpilogueVectorizationForceVF).duplicate());
3494 Clone->setVF(EpilogueVectorizationForceVF);
3495 return Clone;
3496 }
3497
3498 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization forced factor is not "
3499 "viable.\n");
3500 return nullptr;
3501 }
3502
3503 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
3504 LLVM_DEBUG(
3505 dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n");
3506 return nullptr;
3507 }
3508
3509 if (!Config.isEpilogueVectorizationProfitable(VF: MainLoopVF, IC)) {
3510 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
3511 "this loop\n");
3512 return nullptr;
3513 }
3514
3515 // Check if a plan's vector loop processes fewer iterations than VF (e.g. when
3516 // interleave groups have been narrowed) narrowInterleaveGroups) and return
3517 // the adjusted, effective VF.
3518 using namespace VPlanPatternMatch;
3519 auto GetEffectiveVF = [](VPlan &Plan, ElementCount VF) -> ElementCount {
3520 auto *Exiting = Plan.getVectorLoopRegion()->getExitingBasicBlock();
3521 if (match(V: &Exiting->back(),
3522 P: m_BranchOnCount(Op0: m_Add(Op0: m_CanonicalIV(), Op1: m_Specific(VPV: &Plan.getUF())),
3523 Op1: m_VPValue())))
3524 return ElementCount::get(MinVal: 1, Scalable: VF.isScalable());
3525 return VF;
3526 };
3527
3528 // Check if the main loop processes fewer than MainLoopVF elements per
3529 // iteration (e.g. due to narrowing interleave groups). Adjust MainLoopVF
3530 // as needed.
3531 MainLoopVF = GetEffectiveVF(MainPlan, MainLoopVF);
3532
3533 // If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know
3534 // the main loop handles 8 lanes per iteration. We could still benefit from
3535 // vectorizing the epilogue loop with VF=4.
3536 ElementCount EstimatedRuntimeVF = ElementCount::getFixed(
3537 MinVal: estimateElementCount(VF: MainLoopVF, VScale: Config.getVScaleForTuning()));
3538
3539 Type *TCType = Legal->getWidestInductionType();
3540 const SCEV *RemainingIterations = nullptr;
3541 unsigned MaxTripCount = 0;
3542 const SCEV *TC = vputils::getSCEVExprForVPValue(V: MainPlan.getTripCount(), PSE);
3543 assert(!isa<SCEVCouldNotCompute>(TC) && "Trip count SCEV must be computable");
3544 const SCEV *KnownMinTC;
3545 bool ScalableTC = match(S: TC, P: m_scev_c_Mul(Op0: m_SCEV(V&: KnownMinTC), Op1: m_SCEVVScale()));
3546 bool ScalableRemIter = false;
3547 ScalarEvolution &SE = *PSE.getSE();
3548 // Use versions of TC and VF in which both are either scalable or fixed.
3549 if (ScalableTC == MainLoopVF.isScalable()) {
3550 ScalableRemIter = ScalableTC;
3551 RemainingIterations =
3552 SE.getURemExpr(LHS: TC, RHS: SE.getElementCount(Ty: TCType, EC: MainLoopVF * IC));
3553 } else if (ScalableTC) {
3554 const SCEV *EstimatedTC = SE.getMulExpr(
3555 LHS: KnownMinTC,
3556 RHS: SE.getConstant(Ty: TCType, V: Config.getVScaleForTuning().value_or(u: 1)));
3557 RemainingIterations = SE.getURemExpr(
3558 LHS: EstimatedTC, RHS: SE.getElementCount(Ty: TCType, EC: MainLoopVF * IC));
3559 } else
3560 RemainingIterations =
3561 SE.getURemExpr(LHS: TC, RHS: SE.getElementCount(Ty: TCType, EC: EstimatedRuntimeVF * IC));
3562
3563 // No iterations left to process in the epilogue.
3564 if (RemainingIterations->isZero())
3565 return nullptr;
3566
3567 if (MainLoopVF.isFixed()) {
3568 MaxTripCount = MainLoopVF.getFixedValue() * IC - 1;
3569 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_ULT, LHS: RemainingIterations,
3570 RHS: SE.getConstant(Ty: TCType, V: MaxTripCount))) {
3571 MaxTripCount = SE.getUnsignedRangeMax(S: RemainingIterations).getZExtValue();
3572 }
3573 LLVM_DEBUG(dbgs() << "LEV: Maximum Trip Count for Epilogue: "
3574 << MaxTripCount << "\n");
3575 }
3576
3577 auto SkipVF = [&](const SCEV *VF, const SCEV *RemIter) -> bool {
3578 return SE.isKnownPredicate(Pred: CmpInst::ICMP_UGT, LHS: VF, RHS: RemIter);
3579 };
3580 VectorizationFactor Result = VectorizationFactor::Disabled();
3581 VPlan *BestPlan = nullptr;
3582 for (auto &NextVF : ProfitableVFs) {
3583 // Skip candidate VFs without a corresponding VPlan.
3584 if (!hasPlanWithVF(VF: NextVF.Width))
3585 continue;
3586
3587 VPlan &CurrentPlan = getPlanFor(VF: NextVF.Width);
3588 ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
3589 // Skip fixed vector VFs > than the estimated runtime VF, or any VF > than
3590 // the VF of the main loop.
3591 if ((!EffectiveVF.isScalable() && MainLoopVF.isScalable() &&
3592 ElementCount::isKnownGT(LHS: EffectiveVF, RHS: EstimatedRuntimeVF)) ||
3593 ElementCount::isKnownGT(LHS: EffectiveVF, RHS: MainLoopVF))
3594 continue;
3595
3596 // If EffectiveVF is greater than the number of remaining iterations, the
3597 // epilogue loop would be dead. Skip such factors. If the epilogue plan
3598 // also has narrowed interleave groups, use the effective VF since
3599 // the epilogue step will be reduced to its IC.
3600 // TODO: We should also consider comparing against a scalable
3601 // RemainingIterations when SCEV be able to evaluate non-canonical
3602 // vscale-based expressions.
3603 if (!ScalableRemIter) {
3604 // Handle the case where EffectiveVF and RemainingIterations are in
3605 // different numerical spaces.
3606 if (EffectiveVF.isScalable())
3607 EffectiveVF = ElementCount::getFixed(
3608 MinVal: estimateElementCount(VF: EffectiveVF, VScale: Config.getVScaleForTuning()));
3609 if (SkipVF(SE.getElementCount(Ty: TCType, EC: EffectiveVF), RemainingIterations))
3610 continue;
3611 }
3612
3613 if (Result.Width.isScalar() ||
3614 isMoreProfitable(A: NextVF, B: Result, MaxTripCount,
3615 HasTail: !MainPlan.hasTailFolded(),
3616 /*IsEpilogue*/ true)) {
3617 Result = NextVF;
3618 BestPlan = &CurrentPlan;
3619 }
3620 }
3621
3622 if (!BestPlan)
3623 return nullptr;
3624
3625 LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
3626 << Result.Width << "\n");
3627 std::unique_ptr<VPlan> Clone(BestPlan->duplicate());
3628 Clone->setVF(Result.Width);
3629 return Clone;
3630}
3631
3632unsigned
3633LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
3634 InstructionCost LoopCost) {
3635 // -- The interleave heuristics --
3636 // We interleave the loop in order to expose ILP and reduce the loop overhead.
3637 // There are many micro-architectural considerations that we can't predict
3638 // at this level. For example, frontend pressure (on decode or fetch) due to
3639 // code size, or the number and capabilities of the execution ports.
3640 //
3641 // We use the following heuristics to select the interleave count:
3642 // 1. If the code has reductions, then we interleave to break the cross
3643 // iteration dependency.
3644 // 2. If the loop is really small, then we interleave to reduce the loop
3645 // overhead.
3646 // 3. We don't interleave if we think that we will spill registers to memory
3647 // due to the increased register pressure.
3648
3649 // Do not interleave tail-folded loops, as the overhead of multiple
3650 // instructions to calculate the predicate is likely not beneficial.
3651 // If an epilogue is not allowed for any other reason, do not interleave.
3652 if (!CM->isEpilogueAllowed())
3653 return 1;
3654
3655 if (any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3656 P: IsaPred<VPCurrentIterationPHIRecipe>)) {
3657 LLVM_DEBUG(dbgs() << "LV: Loop requires variable-length step. "
3658 "Unroll factor forced to be 1.\n");
3659 return 1;
3660 }
3661
3662 // We used the distance for the interleave count.
3663 if (!Legal->isSafeForAnyVectorWidth())
3664 return 1;
3665
3666 // We don't attempt to perform interleaving for loops with uncountable early
3667 // exits because the VPInstruction::AnyOf code cannot currently handle
3668 // multiple parts.
3669 if (Plan.hasEarlyExit())
3670 return 1;
3671
3672 const bool HasReductions =
3673 any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3674 P: IsaPred<VPReductionPHIRecipe>);
3675
3676 // FIXME: implement interleaving for FindLast transform correctly.
3677 if (hasFindLastReductionPhi(Plan))
3678 return 1;
3679
3680 VPRegisterUsage R = calculateRegisterUsageForPlan(Plan, VFs: {VF}, TTI)[0];
3681
3682 // If we did not calculate the cost for VF (because the user selected the VF)
3683 // then we calculate the cost of VF here.
3684 if (LoopCost == 0) {
3685 if (VF.isScalar())
3686 LoopCost = CM->expectedCost(VF);
3687 else
3688 LoopCost = cost(Plan, VF, RU: &R);
3689 assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
3690
3691 // Loop body is free and there is no need for interleaving.
3692 if (LoopCost == 0)
3693 return 1;
3694 }
3695
3696 // We divide by these constants so assume that we have at least one
3697 // instruction that uses at least one register.
3698 for (auto &Pair : R.MaxLocalUsers) {
3699 Pair.second = std::max(a: Pair.second, b: 1U);
3700 }
3701
3702 // We calculate the interleave count using the following formula.
3703 // Subtract the number of loop invariants from the number of available
3704 // registers. These registers are used by all of the interleaved instances.
3705 // Next, divide the remaining registers by the number of registers that is
3706 // required by the loop, in order to estimate how many parallel instances
3707 // fit without causing spills. All of this is rounded down if necessary to be
3708 // a power of two. We want power of two interleave count to simplify any
3709 // addressing operations or alignment considerations.
3710 // We also want power of two interleave counts to ensure that the induction
3711 // variable of the vector loop wraps to zero, when tail is folded by masking;
3712 // this currently happens when OptForSize, in which case IC is set to 1 above.
3713 unsigned IC = UINT_MAX;
3714
3715 for (const auto &Pair : R.MaxLocalUsers) {
3716 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(ClassID: Pair.first);
3717 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters
3718 << " registers of "
3719 << TTI.getRegisterClassName(Pair.first)
3720 << " register class\n");
3721 if (VF.isScalar()) {
3722 if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)
3723 TargetNumRegisters = ForceTargetNumScalarRegs;
3724 } else {
3725 if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)
3726 TargetNumRegisters = ForceTargetNumVectorRegs;
3727 }
3728 unsigned MaxLocalUsers = Pair.second;
3729 unsigned LoopInvariantRegs = 0;
3730 if (R.LoopInvariantRegs.contains(Key: Pair.first))
3731 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
3732
3733 unsigned TmpIC = llvm::bit_floor(Value: (TargetNumRegisters - LoopInvariantRegs) /
3734 MaxLocalUsers);
3735 // Don't count the induction variable as interleaved.
3736 if (EnableIndVarRegisterHeur) {
3737 TmpIC = llvm::bit_floor(Value: (TargetNumRegisters - LoopInvariantRegs - 1) /
3738 std::max(a: 1U, b: (MaxLocalUsers - 1)));
3739 }
3740
3741 IC = std::min(a: IC, b: TmpIC);
3742 }
3743
3744 // Clamp the interleave ranges to reasonable counts.
3745 bool HasUnorderedReductions =
3746 HasReductions &&
3747 !any_of(Range: make_isa_range<VPReductionPHIRecipe>(
3748 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
3749 P: [](VPReductionPHIRecipe &RedR) { return RedR.isOrdered(); });
3750 unsigned MaxInterleaveCount =
3751 TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions);
3752 LLVM_DEBUG(dbgs() << "LV: MaxInterleaveFactor for the target is "
3753 << MaxInterleaveCount << "\n");
3754
3755 // Check if the user has overridden the max.
3756 if (VF.isScalar()) {
3757 if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
3758 MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
3759 } else {
3760 if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)
3761 MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
3762 }
3763
3764 // Try to get the exact trip count, or an estimate based on profiling data or
3765 // ConstantMax from PSE, failing that.
3766 auto BestKnownTC =
3767 getSmallBestKnownTC(PSE, L: OrigLoop,
3768 /*CanUseConstantMax=*/true,
3769 /*CanExcludeZeroTrips=*/CM->isEpilogueAllowed());
3770
3771 // For fixed length VFs treat a scalable trip count as unknown.
3772 if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
3773 // Re-evaluate trip counts and VFs to be in the same numerical space.
3774 unsigned AvailableTC =
3775 estimateElementCount(VF: *BestKnownTC, VScale: Config.getVScaleForTuning());
3776 unsigned EstimatedVF =
3777 estimateElementCount(VF, VScale: Config.getVScaleForTuning());
3778
3779 // At least one iteration must be scalar when this constraint holds. So the
3780 // maximum available iterations for interleaving is one less.
3781 if (Plan.requiresScalarEpilogue())
3782 --AvailableTC;
3783
3784 unsigned InterleaveCountLB = bit_floor(Value: std::max(
3785 a: 1u, b: std::min(a: AvailableTC / (EstimatedVF * 2), b: MaxInterleaveCount)));
3786
3787 if (getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop).isNonZero()) {
3788 // If the best known trip count is exact, we select between two
3789 // prospective ICs, where
3790 //
3791 // 1) the aggressive IC is capped by the trip count divided by VF
3792 // 2) the conservative IC is capped by the trip count divided by (VF * 2)
3793 //
3794 // The final IC is selected in a way that the epilogue loop trip count is
3795 // minimized while maximizing the IC itself, so that we either run the
3796 // vector loop at least once if it generates a small epilogue loop, or
3797 // else we run the vector loop at least twice.
3798
3799 unsigned InterleaveCountUB = bit_floor(Value: std::max(
3800 a: 1u, b: std::min(a: AvailableTC / EstimatedVF, b: MaxInterleaveCount)));
3801 MaxInterleaveCount = InterleaveCountLB;
3802
3803 if (InterleaveCountUB != InterleaveCountLB) {
3804 unsigned TailTripCountUB =
3805 (AvailableTC % (EstimatedVF * InterleaveCountUB));
3806 unsigned TailTripCountLB =
3807 (AvailableTC % (EstimatedVF * InterleaveCountLB));
3808 // If both produce same scalar tail, maximize the IC to do the same work
3809 // in fewer vector loop iterations
3810 if (TailTripCountUB == TailTripCountLB)
3811 MaxInterleaveCount = InterleaveCountUB;
3812 }
3813 } else {
3814 // If trip count is an estimated compile time constant, limit the
3815 // IC to be capped by the trip count divided by VF * 2, such that the
3816 // vector loop runs at least twice to make interleaving seem profitable
3817 // when there is an epilogue loop present. Since exact Trip count is not
3818 // known we choose to be conservative in our IC estimate.
3819 MaxInterleaveCount = InterleaveCountLB;
3820 }
3821 }
3822
3823 assert(MaxInterleaveCount > 0 &&
3824 "Maximum interleave count must be greater than 0");
3825
3826 // Clamp the calculated IC to be between the 1 and the max interleave count
3827 // that the target and trip count allows.
3828 if (IC > MaxInterleaveCount)
3829 IC = MaxInterleaveCount;
3830 else
3831 // Make sure IC is greater than 0.
3832 IC = std::max(a: 1u, b: IC);
3833
3834 assert(IC > 0 && "Interleave count must be greater than 0.");
3835
3836 // Interleave if we vectorized this loop and there is a reduction that could
3837 // benefit from interleaving.
3838 if (VF.isVector() && HasReductions) {
3839 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
3840 return IC;
3841 }
3842
3843 // For any scalar loop that either requires runtime checks or tail-folding we
3844 // are better off leaving this to the unroller. Note that if we've already
3845 // vectorized the loop we will have done the runtime check and so interleaving
3846 // won't require further checks.
3847 bool ScalarInterleavingRequiresPredication =
3848 (VF.isScalar() && any_of(Range: OrigLoop->blocks(), P: [this](BasicBlock *BB) {
3849 return Legal->blockNeedsPredication(BB);
3850 }));
3851 bool ScalarInterleavingRequiresRuntimePointerCheck =
3852 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need);
3853
3854 // We want to interleave small loops in order to reduce the loop overhead and
3855 // potentially expose ILP opportunities.
3856 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
3857 << "LV: IC is " << IC << '\n'
3858 << "LV: VF is " << VF << '\n');
3859 const bool AggressivelyInterleave =
3860 TTI.enableAggressiveInterleaving(LoopHasReductions: HasReductions);
3861 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
3862 !ScalarInterleavingRequiresPredication && LoopCost < SmallLoopCost) {
3863 // We assume that the cost overhead is 1 and we use the cost model
3864 // to estimate the cost of the loop and interleave until the cost of the
3865 // loop overhead is about 5% of the cost of the loop.
3866 unsigned SmallIC = std::min(a: IC, b: (unsigned)llvm::bit_floor<uint64_t>(
3867 Value: SmallLoopCost / LoopCost.getValue()));
3868
3869 // Interleave until store/load ports (estimated by max interleave count) are
3870 // saturated.
3871 unsigned NumStores = 0;
3872 unsigned NumLoads = 0;
3873 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3874 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
3875 for (VPRecipeBase &R : *VPBB) {
3876 if (isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Val: &R)) {
3877 NumLoads++;
3878 continue;
3879 }
3880 if (isa<VPWidenStoreRecipe, VPWidenStoreEVLRecipe>(Val: &R)) {
3881 NumStores++;
3882 continue;
3883 }
3884
3885 if (auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(Val: &R)) {
3886 if (unsigned StoreOps = InterleaveR->getNumStoreOperands())
3887 NumStores += StoreOps;
3888 else
3889 NumLoads += InterleaveR->getNumDefinedValues();
3890 continue;
3891 }
3892 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
3893 NumLoads += isa<LoadInst>(Val: RepR->getUnderlyingInstr());
3894 NumStores += isa<StoreInst>(Val: RepR->getUnderlyingInstr());
3895 continue;
3896 }
3897 if (isa<VPHistogramRecipe>(Val: &R)) {
3898 NumLoads++;
3899 NumStores++;
3900 continue;
3901 }
3902 }
3903 }
3904 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
3905 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
3906
3907 // There is little point in interleaving for reductions containing selects
3908 // and compares when VF=1 since it may just create more overhead than it's
3909 // worth for loops with small trip counts. This is because we still have to
3910 // do the final reduction after the loop.
3911 bool HasSelectCmpReductions =
3912 HasReductions &&
3913 any_of(Range: make_isa_range<VPReductionPHIRecipe>(
3914 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
3915 P: [](VPReductionPHIRecipe &RedR) {
3916 return RecurrenceDescriptor::isAnyOfRecurrenceKind(
3917 Kind: RedR.getRecurrenceKind()) ||
3918 RecurrenceDescriptor::isFindIVRecurrenceKind(
3919 Kind: RedR.getRecurrenceKind());
3920 });
3921 if (HasSelectCmpReductions) {
3922 LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
3923 return 1;
3924 }
3925
3926 // If we have a scalar reduction (vector reductions are already dealt with
3927 // by this point), we can increase the critical path length if the loop
3928 // we're interleaving is inside another loop. For tree-wise reductions
3929 // set the limit to 2, and for ordered reductions it's best to disable
3930 // interleaving entirely.
3931 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
3932 bool HasOrderedReductions =
3933 any_of(Range: make_isa_range<VPReductionPHIRecipe>(
3934 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
3935 P: [](VPReductionPHIRecipe &RedR) { return RedR.isOrdered(); });
3936 if (HasOrderedReductions) {
3937 LLVM_DEBUG(
3938 dbgs() << "LV: Not interleaving scalar ordered reductions.\n");
3939 return 1;
3940 }
3941
3942 unsigned F = MaxNestedScalarReductionIC;
3943 SmallIC = std::min(a: SmallIC, b: F);
3944 StoresIC = std::min(a: StoresIC, b: F);
3945 LoadsIC = std::min(a: LoadsIC, b: F);
3946 }
3947
3948 if (EnableLoadStoreRuntimeInterleave &&
3949 std::max(a: StoresIC, b: LoadsIC) > SmallIC) {
3950 LLVM_DEBUG(
3951 dbgs() << "LV: Interleaving to saturate store or load ports.\n");
3952 return std::max(a: StoresIC, b: LoadsIC);
3953 }
3954
3955 // If there are scalar reductions and TTI has enabled aggressive
3956 // interleaving for reductions, we will interleave to expose ILP.
3957 if (VF.isScalar() && AggressivelyInterleave) {
3958 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3959 // Interleave no less than SmallIC but not as aggressive as the normal IC
3960 // to satisfy the rare situation when resources are too limited.
3961 return std::max(a: IC / 2, b: SmallIC);
3962 }
3963
3964 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
3965 return SmallIC;
3966 }
3967
3968 // Interleave if this is a large loop (small loops are already dealt with by
3969 // this point) that could benefit from interleaving.
3970 if (AggressivelyInterleave) {
3971 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3972 return IC;
3973 }
3974
3975 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n");
3976 return 1;
3977}
3978
3979bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(
3980 Instruction *I, ElementCount VF) const {
3981 // TODO: Cost model for emulated masked load/store is completely
3982 // broken. This hack guides the cost model to use an artificially
3983 // high enough value to practically disable vectorization with such
3984 // operations, except where previously deployed legality hack allowed
3985 // using very low cost values. This is to avoid regressions coming simply
3986 // from moving "masked load/store" check from legality to cost model.
3987 // Masked Load/Gather emulation was previously never allowed.
3988 // Limited number of Masked Store/Scatter emulation was allowed.
3989 assert((isPredicatedInst(I)) &&
3990 "Expecting a scalar emulated instruction");
3991 return isa<LoadInst>(Val: I) ||
3992 (isa<StoreInst>(Val: I) &&
3993 NumPredStores > NumberOfStoresToPredicate);
3994}
3995
3996void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
3997 assert(VF.isVector() && "Expected VF >= 2");
3998
3999 // If we've already collected the instructions to scalarize or the predicated
4000 // BBs after vectorization, there's nothing to do. Collection may already have
4001 // occurred if we have a user-selected VF and are now computing the expected
4002 // cost for interleaving.
4003 if (InstsToScalarize.contains(Key: VF) ||
4004 PredicatedBBsAfterVectorization.contains(Val: VF))
4005 return;
4006
4007 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's
4008 // not profitable to scalarize any instructions, the presence of VF in the
4009 // map will indicate that we've analyzed it already.
4010 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
4011
4012 // Find all the instructions that are scalar with predication in the loop and
4013 // determine if it would be better to not if-convert the blocks they are in.
4014 // If so, we also record the instructions to scalarize.
4015 for (BasicBlock *BB : TheLoop->blocks()) {
4016 if (!blockNeedsPredicationForAnyReason(BB))
4017 continue;
4018 for (Instruction &I : *BB)
4019 if (isScalarWithPredication(I: &I, VF)) {
4020 ScalarCostsTy ScalarCosts;
4021 // Do not apply discount logic for:
4022 // 1. Scalars after vectorization, as there will only be a single copy
4023 // of the instruction.
4024 // 2. Scalable VF, as that would lead to invalid scalarization costs.
4025 // 3. Emulated masked memrefs, if a hacked cost is needed.
4026 if (!isScalarAfterVectorization(I: &I, VF) && !VF.isScalable() &&
4027 !useEmulatedMaskMemRefHack(I: &I, VF) &&
4028 computePredInstDiscount(PredInst: &I, ScalarCosts, VF) >= 0) {
4029 for (const auto &[I, IC] : ScalarCosts)
4030 ScalarCostsVF.insert(KV: {I, IC});
4031 }
4032 // Remember that BB will remain after vectorization.
4033 PredicatedBBsAfterVectorization[VF].insert(Ptr: BB);
4034 for (auto *Pred : predecessors(BB)) {
4035 if (Pred->getSingleSuccessor() == BB)
4036 PredicatedBBsAfterVectorization[VF].insert(Ptr: Pred);
4037 }
4038 }
4039 }
4040}
4041
4042InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
4043 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) {
4044 assert(!isUniformAfterVectorization(PredInst, VF) &&
4045 "Instruction marked uniform-after-vectorization will be predicated");
4046
4047 // Initialize the discount to zero, meaning that the scalar version and the
4048 // vector version cost the same.
4049 InstructionCost Discount = 0;
4050
4051 // Holds instructions to analyze. The instructions we visit are mapped in
4052 // ScalarCosts. Those instructions are the ones that would be scalarized if
4053 // we find that the scalar version costs less.
4054 SmallVector<Instruction *, 8> Worklist;
4055
4056 // Returns true if the given instruction can be scalarized.
4057 auto CanBeScalarized = [&](Instruction *I) -> bool {
4058 // We only attempt to scalarize instructions forming a single-use chain
4059 // from the original predicated block that would otherwise be vectorized.
4060 // Although not strictly necessary, we give up on instructions we know will
4061 // already be scalar to avoid traversing chains that are unlikely to be
4062 // beneficial.
4063 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
4064 isScalarAfterVectorization(I, VF))
4065 return false;
4066
4067 // If the instruction is scalar with predication, it will be analyzed
4068 // separately. We ignore it within the context of PredInst.
4069 if (isScalarWithPredication(I, VF))
4070 return false;
4071
4072 // If any of the instruction's operands are uniform after vectorization,
4073 // the instruction cannot be scalarized. This prevents, for example, a
4074 // masked load from being scalarized.
4075 //
4076 // We assume we will only emit a value for lane zero of an instruction
4077 // marked uniform after vectorization, rather than VF identical values.
4078 // Thus, if we scalarize an instruction that uses a uniform, we would
4079 // create uses of values corresponding to the lanes we aren't emitting code
4080 // for. This behavior can be changed by allowing getScalarValue to clone
4081 // the lane zero values for uniforms rather than asserting.
4082 for (Use &U : I->operands())
4083 if (auto *J = dyn_cast<Instruction>(Val: U.get()))
4084 if (isUniformAfterVectorization(I: J, VF))
4085 return false;
4086
4087 // Otherwise, we can scalarize the instruction.
4088 return true;
4089 };
4090
4091 // Compute the expected cost discount from scalarizing the entire expression
4092 // feeding the predicated instruction. We currently only consider expressions
4093 // that are single-use instruction chains.
4094 Worklist.push_back(Elt: PredInst);
4095 while (!Worklist.empty()) {
4096 Instruction *I = Worklist.pop_back_val();
4097
4098 // If we've already analyzed the instruction, there's nothing to do.
4099 if (ScalarCosts.contains(Key: I))
4100 continue;
4101
4102 // Cannot scalarize fixed-order recurrence phis at the moment.
4103 if (isa<PHINode>(Val: I) && Legal->isFixedOrderRecurrence(Phi: cast<PHINode>(Val: I)))
4104 continue;
4105
4106 // Compute the cost of the vector instruction. Note that this cost already
4107 // includes the scalarization overhead of the predicated instruction.
4108 InstructionCost VectorCost = getInstructionCost(I, VF);
4109
4110 // Compute the cost of the scalarized instruction. This cost is the cost of
4111 // the instruction as if it wasn't if-converted and instead remained in the
4112 // predicated block. We will scale this cost by block probability after
4113 // computing the scalarization overhead.
4114 InstructionCost ScalarCost =
4115 VF.getFixedValue() * getInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1));
4116
4117 // Compute the scalarization overhead of needed insertelement instructions
4118 // and phi nodes.
4119 if (isScalarWithPredication(I, VF) && !I->getType()->isVoidTy()) {
4120 Type *WideTy = toVectorizedTy(Ty: I->getType(), EC: VF);
4121 for (Type *VectorTy : getContainedTypes(Ty: WideTy)) {
4122 ScalarCost += TTI.getScalarizationOverhead(
4123 Ty: cast<VectorType>(Val: VectorTy), DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4124 /*Insert=*/true,
4125 /*Extract=*/false, CostKind: Config.CostKind);
4126 }
4127 ScalarCost += VF.getFixedValue() *
4128 TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
4129 }
4130
4131 // Compute the scalarization overhead of needed extractelement
4132 // instructions. For each of the instruction's operands, if the operand can
4133 // be scalarized, add it to the worklist; otherwise, account for the
4134 // overhead.
4135 for (Use &U : I->operands())
4136 if (auto *J = dyn_cast<Instruction>(Val: U.get())) {
4137 assert(canVectorizeTy(J->getType()) &&
4138 "Instruction has non-scalar type");
4139 if (CanBeScalarized(J))
4140 Worklist.push_back(Elt: J);
4141 else if (needsExtract(V: J, VF)) {
4142 Type *WideTy = toVectorizedTy(Ty: J->getType(), EC: VF);
4143 for (Type *VectorTy : getContainedTypes(Ty: WideTy)) {
4144 ScalarCost += TTI.getScalarizationOverhead(
4145 Ty: cast<VectorType>(Val: VectorTy),
4146 DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()), /*Insert*/ false,
4147 /*Extract*/ true, CostKind: Config.CostKind);
4148 }
4149 }
4150 }
4151
4152 // Scale the total scalar cost by block probability.
4153 ScalarCost /= getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
4154
4155 // Compute the discount. A non-negative discount means the vector version
4156 // of the instruction costs more, and scalarizing would be beneficial.
4157 Discount += VectorCost - ScalarCost;
4158 ScalarCosts[I] = ScalarCost;
4159 }
4160
4161 return Discount;
4162}
4163
4164InstructionCost LoopVectorizationCostModel::expectedCost(ElementCount VF) {
4165 InstructionCost Cost;
4166 assert(VF.isScalar() && "must only be called for scalar VFs");
4167
4168 // For each block.
4169 for (BasicBlock *BB : TheLoop->blocks()) {
4170 InstructionCost BlockCost;
4171
4172 // For each instruction in the old loop.
4173 for (Instruction &I : *BB) {
4174 // Skip ignored values.
4175 if (ValuesToIgnore.count(Ptr: &I) ||
4176 (VF.isVector() && VecValuesToIgnore.count(Ptr: &I)))
4177 continue;
4178
4179 InstructionCost C = getInstructionCost(I: &I, VF);
4180
4181 // Check if we should override the cost.
4182 if (C.isValid() && ForceTargetInstructionCost.getNumOccurrences() > 0)
4183 C = InstructionCost(ForceTargetInstructionCost);
4184
4185 BlockCost += C;
4186 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C << " for VF "
4187 << VF << " For instruction: " << I << '\n');
4188 }
4189
4190 // In the scalar loop, we may not always execute the predicated block, if it
4191 // is an if-else block. Thus, scale the block's cost by the probability of
4192 // executing it. getPredBlockCostDivisor will return 1 for blocks that are
4193 // only predicated by the header mask when folding the tail.
4194 Cost += BlockCost / getPredBlockCostDivisor(CostKind: Config.CostKind, BB);
4195 }
4196
4197 return Cost;
4198}
4199
4200/// Gets the address access SCEV for Ptr, if it should be used for cost modeling
4201/// according to isAddressSCEVForCost.
4202///
4203/// This SCEV can be sent to the Target in order to estimate the address
4204/// calculation cost.
4205static const SCEV *getAddressAccessSCEV(
4206 Value *Ptr,
4207 PredicatedScalarEvolution &PSE,
4208 const Loop *TheLoop) {
4209 const SCEV *Addr = PSE.getSCEV(V: Ptr);
4210 return vputils::isAddressSCEVForCost(Addr, SE&: *PSE.getSE(), L: TheLoop) ? Addr
4211 : nullptr;
4212}
4213
4214InstructionCost
4215LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
4216 ElementCount VF) {
4217 assert(VF.isVector() &&
4218 "Scalarization cost of instruction implies vectorization.");
4219 if (VF.isScalable())
4220 return InstructionCost::getInvalid();
4221
4222 Type *ValTy = getLoadStoreType(I);
4223 auto *SE = PSE.getSE();
4224
4225 unsigned AS = getLoadStoreAddressSpace(I);
4226 Value *Ptr = getLoadStorePointerOperand(V: I);
4227 Type *PtrTy = toVectorTy(Scalar: Ptr->getType(), EC: VF);
4228 // NOTE: PtrTy is a vector to signal `TTI::getAddressComputationCost`
4229 // that it is being called from this specific place.
4230
4231 // Figure out whether the access is strided and get the stride value
4232 // if it's known in compile time
4233 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, PSE, TheLoop);
4234
4235 // Get the cost of the scalar memory instruction and address computation.
4236 InstructionCost Cost =
4237 VF.getFixedValue() *
4238 TTI.getAddressComputationCost(PtrTy, SE, Ptr: PtrSCEV, CostKind: Config.CostKind);
4239
4240 // Don't pass *I here, since it is scalar but will actually be part of a
4241 // vectorized loop where the user of it is a vectorized instruction.
4242 const Align Alignment = getLoadStoreAlignment(I);
4243 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4244 Cost += VF.getFixedValue() *
4245 TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: ValTy->getScalarType(), Alignment,
4246 AddressSpace: AS, CostKind: Config.CostKind, OpdInfo: OpInfo);
4247
4248 // Get the overhead of the extractelement and insertelement instructions
4249 // we might create due to scalarization.
4250 Cost += getScalarizationOverhead(I, VF);
4251
4252 // If we have a predicated load/store, it will need extra i1 extracts and
4253 // conditional branches, but may not be executed for each vector lane. Scale
4254 // the cost by the probability of executing the predicated block.
4255 if (isPredicatedInst(I)) {
4256 Cost /= getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
4257
4258 // Add the cost of an i1 extract and a branch
4259 auto *VecI1Ty =
4260 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: ValTy->getContext()), EC: VF);
4261 Cost += TTI.getScalarizationOverhead(
4262 Ty: VecI1Ty, DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4263 /*Insert=*/false, /*Extract=*/true, CostKind: Config.CostKind);
4264 Cost += TTI.getCFInstrCost(Opcode: Instruction::CondBr, CostKind: Config.CostKind);
4265
4266 if (useEmulatedMaskMemRefHack(I, VF))
4267 // Artificially setting to a high enough value to practically disable
4268 // vectorization with such operations.
4269 Cost = 3000000;
4270 }
4271
4272 return Cost;
4273}
4274
4275InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
4276 Instruction *I, ElementCount VF, InstWidening Kind) {
4277 assert((Kind == CM_Widen || Kind == CM_Widen_Reverse) &&
4278 "Expected a consecutive widening decision");
4279 Type *ValTy = getLoadStoreType(I);
4280 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4281 unsigned AS = getLoadStoreAddressSpace(I);
4282
4283 const Align Alignment = getLoadStoreAlignment(I);
4284 InstructionCost Cost = 0;
4285 if (isMaskRequired(I)) {
4286 unsigned IID = I->getOpcode() == Instruction::Load
4287 ? Intrinsic::masked_load
4288 : Intrinsic::masked_store;
4289 Cost += TTI.getMemIntrinsicInstrCost(
4290 MICA: MemIntrinsicCostAttributes(IID, VectorTy, Alignment, AS),
4291 CostKind: Config.CostKind);
4292 } else {
4293 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4294 Cost += TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: VectorTy, Alignment, AddressSpace: AS,
4295 CostKind: Config.CostKind, OpdInfo: OpInfo, I);
4296 }
4297
4298 if (Kind == CM_Widen_Reverse)
4299 Cost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Reverse, DstTy: VectorTy,
4300 SrcTy: VectorTy, CostKind: Config.CostKind, Mask: {}, Index: 0);
4301 return Cost;
4302}
4303
4304InstructionCost
4305LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
4306 ElementCount VF) const {
4307 assert(isUniformMemOp(*I, VF));
4308
4309 Type *ValTy = getLoadStoreType(I);
4310 Type *PtrTy = getLoadStorePointerOperand(V: I)->getType();
4311 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4312 const Align Alignment = getLoadStoreAlignment(I);
4313 unsigned AS = getLoadStoreAddressSpace(I);
4314 if (isa<LoadInst>(Val: I)) {
4315 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4316 CostKind: Config.CostKind) +
4317 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: ValTy, Alignment, AddressSpace: AS,
4318 CostKind: Config.CostKind) +
4319 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Broadcast, DstTy: VectorTy,
4320 SrcTy: VectorTy, CostKind: Config.CostKind);
4321 }
4322 StoreInst *SI = cast<StoreInst>(Val: I);
4323
4324 bool IsLoopInvariantStoreValue = Legal->isInvariant(V: SI->getValueOperand());
4325 // TODO: We have existing tests that request the cost of extracting element
4326 // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent
4327 // the actual generated code, which involves extracting the last element of
4328 // a scalable vector where the lane to extract is unknown at compile time.
4329 InstructionCost Cost =
4330 TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr, CostKind: Config.CostKind) +
4331 TTI.getMemoryOpCost(Opcode: Instruction::Store, Src: ValTy, Alignment, AddressSpace: AS,
4332 CostKind: Config.CostKind);
4333 if (!IsLoopInvariantStoreValue)
4334 Cost += TTI.getIndexedVectorInstrCostFromEnd(Opcode: Instruction::ExtractElement,
4335 Val: VectorTy, CostKind: Config.CostKind, Index: 0);
4336 return Cost;
4337}
4338
4339InstructionCost
4340LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
4341 ElementCount VF) const {
4342 Type *ValTy = getLoadStoreType(I);
4343 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4344 const Align Alignment = getLoadStoreAlignment(I);
4345 Value *Ptr = getLoadStorePointerOperand(V: I);
4346 Type *PtrTy = Ptr->getType();
4347
4348 if (!isUniform(V: Ptr, VF))
4349 PtrTy = toVectorTy(Scalar: PtrTy, EC: VF);
4350
4351 unsigned IID = I->getOpcode() == Instruction::Load
4352 ? Intrinsic::masked_gather
4353 : Intrinsic::masked_scatter;
4354 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4355 CostKind: Config.CostKind) +
4356 TTI.getMemIntrinsicInstrCost(
4357 MICA: MemIntrinsicCostAttributes(IID, VectorTy, Ptr, isMaskRequired(I),
4358 Alignment, I),
4359 CostKind: Config.CostKind);
4360}
4361
4362InstructionCost
4363LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
4364 ElementCount VF) const {
4365 const auto *Group = getInterleavedAccessGroup(Instr: I);
4366 assert(Group && "Fail to get an interleaved access group.");
4367
4368 Instruction *InsertPos = Group->getInsertPos();
4369 Type *ValTy = getLoadStoreType(I: InsertPos);
4370 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4371 unsigned AS = getLoadStoreAddressSpace(I: InsertPos);
4372
4373 unsigned InterleaveFactor = Group->getFactor();
4374 auto *WideVecTy = VectorType::get(ElementType: ValTy, EC: VF * InterleaveFactor);
4375
4376 // Holds the indices of existing members in the interleaved group.
4377 SmallVector<unsigned, 4> Indices;
4378 for (unsigned IF = 0; IF < InterleaveFactor; IF++)
4379 if (Group->getMember(Index: IF))
4380 Indices.push_back(Elt: IF);
4381
4382 // Calculate the cost of the whole interleaved group.
4383 bool UseMaskForGaps =
4384 (Group->requiresScalarEpilogue() && !isEpilogueAllowed()) ||
4385 (isa<StoreInst>(Val: I) && !Group->isFull());
4386 InstructionCost Cost = TTI.getInterleavedMemoryOpCost(
4387 Opcode: InsertPos->getOpcode(), VecTy: WideVecTy, Factor: Group->getFactor(), Indices,
4388 Alignment: Group->getAlign(), AddressSpace: AS, CostKind: Config.CostKind, UseMaskForCond: isMaskRequired(I),
4389 UseMaskForGaps);
4390
4391 if (Group->isReverse()) {
4392 // TODO: Add support for reversed masked interleaved access.
4393 assert(!isMaskRequired(I) &&
4394 "Reverse masked interleaved access not supported.");
4395 Cost += Group->getNumMembers() *
4396 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Reverse, DstTy: VectorTy,
4397 SrcTy: VectorTy, CostKind: Config.CostKind, Mask: {}, Index: 0);
4398 }
4399 return Cost;
4400}
4401
4402InstructionCost
4403LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
4404 ElementCount VF) {
4405 // Calculate scalar cost only. Vectorization cost should be ready at this
4406 // moment.
4407 if (VF.isScalar()) {
4408 Type *ValTy = getLoadStoreType(I);
4409 Type *PtrTy = getLoadStorePointerOperand(V: I)->getType();
4410 const Align Alignment = getLoadStoreAlignment(I);
4411 unsigned AS = getLoadStoreAddressSpace(I);
4412
4413 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4414 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4415 CostKind: Config.CostKind) +
4416 TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: ValTy, Alignment, AddressSpace: AS,
4417 CostKind: Config.CostKind, OpdInfo: OpInfo, I);
4418 }
4419 return getWideningCost(I, VF);
4420}
4421
4422InstructionCost
4423LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
4424 ElementCount VF) const {
4425
4426 // There is no mechanism yet to create a scalable scalarization loop,
4427 // so this is currently Invalid.
4428 if (VF.isScalable())
4429 return InstructionCost::getInvalid();
4430
4431 if (VF.isScalar())
4432 return 0;
4433
4434 InstructionCost Cost = 0;
4435 Type *RetTy = toVectorizedTy(Ty: I->getType(), EC: VF);
4436 if (!RetTy->isVoidTy() &&
4437 (!isa<LoadInst>(Val: I) || !TTI.supportsEfficientVectorElementLoadStore())) {
4438
4439 TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None;
4440 if (isa<LoadInst>(Val: I))
4441 VIC = TTI::VectorInstrContext::Load;
4442 else if (isa<StoreInst>(Val: I))
4443 VIC = TTI::VectorInstrContext::Store;
4444
4445 for (Type *VectorTy : getContainedTypes(Ty: RetTy)) {
4446 Cost += TTI.getScalarizationOverhead(
4447 Ty: cast<VectorType>(Val: VectorTy), DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4448 /*Insert=*/true, /*Extract=*/false, CostKind: Config.CostKind,
4449 /*ForPoisonSrc=*/true, VL: {}, VIC);
4450 }
4451 }
4452
4453 // Some targets keep addresses scalar.
4454 if (isa<LoadInst>(Val: I) && !TTI.prefersVectorizedAddressing())
4455 return Cost;
4456
4457 // Some targets support efficient element stores.
4458 if (isa<StoreInst>(Val: I) && TTI.supportsEfficientVectorElementLoadStore())
4459 return Cost;
4460
4461 // Collect operands to consider.
4462 CallInst *CI = dyn_cast<CallInst>(Val: I);
4463 Instruction::op_range Ops = CI ? CI->args() : I->operands();
4464
4465 // Skip operands that do not require extraction/scalarization and do not incur
4466 // any overhead.
4467 SmallVector<Type *> Tys;
4468 for (auto *V : filterExtractingOperands(Ops, VF))
4469 Tys.push_back(Elt: maybeVectorizeType(Ty: V->getType(), VF));
4470
4471 TTI::VectorInstrContext OperandVIC = isa<StoreInst>(Val: I)
4472 ? TTI::VectorInstrContext::Store
4473 : TTI::VectorInstrContext::None;
4474 return Cost +
4475 TTI.getOperandsScalarizationOverhead(Tys, CostKind: Config.CostKind, VIC: OperandVIC);
4476}
4477
4478void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
4479 if (VF.isScalar())
4480 return;
4481
4482 // TODO: We should generate better code and update the cost model for
4483 // predicated uniform stores. Today they are treated as any other
4484 // predicated store (see added test cases in
4485 // invariant-store-vectorization.ll).
4486 NumPredStores = 0;
4487 for (BasicBlock *BB : TheLoop->blocks())
4488 for (Instruction &I : *BB)
4489 if (isa<StoreInst>(Val: &I) && isScalarWithPredication(I: &I, VF))
4490 ++NumPredStores;
4491
4492 for (BasicBlock *BB : TheLoop->blocks()) {
4493 // For each instruction in the old loop.
4494 for (Instruction &I : *BB) {
4495 Value *Ptr = getLoadStorePointerOperand(V: &I);
4496 if (!Ptr)
4497 continue;
4498
4499 if (isUniformMemOp(I, VF)) {
4500 auto IsLegalToScalarize = [&]() {
4501 if (!VF.isScalable())
4502 // Scalarization of fixed length vectors "just works".
4503 return true;
4504
4505 // We have dedicated lowering for unpredicated uniform loads and
4506 // stores. Note that even with tail folding we know that at least
4507 // one lane is active (i.e. generalized predication is not possible
4508 // here), and the logic below depends on this fact.
4509 if (!foldTailByMasking())
4510 return true;
4511
4512 // For scalable vectors, a uniform memop load is always
4513 // uniform-by-parts and we know how to scalarize that.
4514 if (isa<LoadInst>(Val: I))
4515 return true;
4516
4517 // A uniform store isn't neccessarily uniform-by-part
4518 // and we can't assume scalarization.
4519 auto &SI = cast<StoreInst>(Val&: I);
4520 return TheLoop->isLoopInvariant(V: SI.getValueOperand());
4521 };
4522
4523 const InstructionCost GatherScatterCost =
4524 isLegalGatherOrScatter(I: &I, VF) ? getGatherScatterCost(I: &I, VF)
4525 : InstructionCost::getInvalid();
4526
4527 // Load: Scalar load + broadcast
4528 // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
4529 // FIXME: This cost is a significant under-estimate for tail folded
4530 // memory ops.
4531 const InstructionCost ScalarizationCost =
4532 IsLegalToScalarize() ? getUniformMemOpCost(I: &I, VF)
4533 : InstructionCost::getInvalid();
4534
4535 // Choose better solution for the current VF, Note that Invalid
4536 // costs compare as maximumal large. If both are invalid, we get
4537 // scalable invalid which signals a failure and a vectorization abort.
4538 if (GatherScatterCost < ScalarizationCost)
4539 setWideningDecision(I: &I, VF, W: CM_GatherScatter, Cost: GatherScatterCost);
4540 else
4541 setWideningDecision(I: &I, VF, W: CM_Scalarize, Cost: ScalarizationCost);
4542 continue;
4543 }
4544
4545 // We assume that widening is the best solution when possible.
4546 if (std::optional<InstWidening> Decision =
4547 memoryInstructionCanBeWidened(I: &I, VF)) {
4548 setWideningDecision(I: &I, VF, W: *Decision,
4549 Cost: getConsecutiveMemOpCost(I: &I, VF, Kind: *Decision));
4550 continue;
4551 }
4552
4553 // Choose between Interleaving, Gather/Scatter or Scalarization.
4554 InstructionCost InterleaveCost = InstructionCost::getInvalid();
4555 unsigned NumAccesses = 1;
4556 if (isAccessInterleaved(Instr: &I)) {
4557 const auto *Group = getInterleavedAccessGroup(Instr: &I);
4558 assert(Group && "Fail to get an interleaved access group.");
4559
4560 // Make one decision for the whole group.
4561 if (getWideningDecision(I: &I, VF) != CM_Unknown)
4562 continue;
4563
4564 NumAccesses = Group->getNumMembers();
4565 if (interleavedAccessCanBeWidened(I: &I, VF))
4566 InterleaveCost = getInterleaveGroupCost(I: &I, VF);
4567 }
4568
4569 InstructionCost GatherScatterCost =
4570 isLegalGatherOrScatter(I: &I, VF)
4571 ? getGatherScatterCost(I: &I, VF) * NumAccesses
4572 : InstructionCost::getInvalid();
4573
4574 InstructionCost ScalarizationCost =
4575 getMemInstScalarizationCost(I: &I, VF) * NumAccesses;
4576
4577 // Choose better solution for the current VF,
4578 // write down this decision and use it during vectorization.
4579 InstructionCost Cost;
4580 InstWidening Decision;
4581 if (InterleaveCost <= GatherScatterCost &&
4582 InterleaveCost < ScalarizationCost) {
4583 Decision = CM_Interleave;
4584 Cost = InterleaveCost;
4585 } else if (GatherScatterCost < ScalarizationCost) {
4586 Decision = CM_GatherScatter;
4587 Cost = GatherScatterCost;
4588 } else {
4589 Decision = CM_Scalarize;
4590 Cost = ScalarizationCost;
4591 }
4592 // If the instructions belongs to an interleave group, the whole group
4593 // receives the same decision. The whole group receives the cost, but
4594 // the cost will actually be assigned to one instruction.
4595 if (const auto *Group = getInterleavedAccessGroup(Instr: &I)) {
4596 if (Decision == CM_Scalarize) {
4597 for (Instruction *I : Group->members())
4598 setWideningDecision(I, VF, W: Decision,
4599 Cost: getMemInstScalarizationCost(I, VF));
4600 } else {
4601 setWideningDecision(Grp: Group, VF, W: Decision, Cost);
4602 }
4603 } else
4604 setWideningDecision(I: &I, VF, W: Decision, Cost);
4605 }
4606 }
4607
4608 // Make sure that any load of address and any other address computation
4609 // remains scalar unless there is gather/scatter support. This avoids
4610 // inevitable extracts into address registers, and also has the benefit of
4611 // activating LSR more, since that pass can't optimize vectorized
4612 // addresses.
4613 if (TTI.prefersVectorizedAddressing())
4614 return;
4615
4616 // Start with all scalar pointer uses.
4617 SmallSetVector<Instruction *, 8> AddrDefs;
4618 for (BasicBlock *BB : TheLoop->blocks())
4619 for (Instruction &I : *BB) {
4620 Instruction *PtrDef =
4621 dyn_cast_or_null<Instruction>(Val: getLoadStorePointerOperand(V: &I));
4622 if (PtrDef && TheLoop->contains(Inst: PtrDef) &&
4623 getWideningDecision(I: &I, VF) != CM_GatherScatter)
4624 AddrDefs.insert(X: PtrDef);
4625 }
4626
4627 // Add all instructions used to generate the addresses.
4628 SmallVector<Instruction *, 4> Worklist;
4629 append_range(C&: Worklist, R&: AddrDefs);
4630 while (!Worklist.empty()) {
4631 Instruction *I = Worklist.pop_back_val();
4632 for (auto &Op : I->operands())
4633 if (auto *InstOp = dyn_cast<Instruction>(Val&: Op))
4634 if (TheLoop->contains(Inst: InstOp) && !isa<PHINode>(Val: InstOp) &&
4635 AddrDefs.insert(X: InstOp))
4636 Worklist.push_back(Elt: InstOp);
4637 }
4638
4639 auto UpdateMemOpUserCost = [this, VF](LoadInst *LI) {
4640 // If there are direct memory op users of the newly scalarized load,
4641 // their cost may have changed because there's no scalarization
4642 // overhead for the operand. Update it.
4643 for (User *U : LI->users()) {
4644 if (!isa<LoadInst, StoreInst>(Val: U))
4645 continue;
4646 if (getWideningDecision(I: cast<Instruction>(Val: U), VF) != CM_Scalarize)
4647 continue;
4648 setWideningDecision(
4649 I: cast<Instruction>(Val: U), VF, W: CM_Scalarize,
4650 Cost: getMemInstScalarizationCost(I: cast<Instruction>(Val: U), VF));
4651 }
4652 };
4653 for (auto *I : AddrDefs) {
4654 if (isa<LoadInst>(Val: I)) {
4655 // Setting the desired widening decision should ideally be handled in
4656 // by cost functions, but since this involves the task of finding out
4657 // if the loaded register is involved in an address computation, it is
4658 // instead changed here when we know this is the case.
4659 InstWidening Decision = getWideningDecision(I, VF);
4660 if (!isPredicatedInst(I) &&
4661 (Decision == CM_Widen || Decision == CM_Widen_Reverse ||
4662 (!isUniformMemOp(I&: *I, VF) && Decision == CM_Scalarize))) {
4663 // Scalarize a widened load of address or update the cost of a scalar
4664 // load of an address.
4665 setWideningDecision(
4666 I, VF, W: CM_Scalarize,
4667 Cost: (VF.getKnownMinValue() *
4668 getMemoryInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1))));
4669 UpdateMemOpUserCost(cast<LoadInst>(Val: I));
4670 } else if (const auto *Group = getInterleavedAccessGroup(Instr: I)) {
4671 // Scalarize all members of this interleaved group when any member
4672 // is used as an address. The address-used load skips scalarization
4673 // overhead, other members include it.
4674 for (Instruction *Member : Group->members()) {
4675 InstructionCost Cost = AddrDefs.contains(key: Member)
4676 ? (VF.getKnownMinValue() *
4677 getMemoryInstructionCost(
4678 I: Member, VF: ElementCount::getFixed(MinVal: 1)))
4679 : getMemInstScalarizationCost(I: Member, VF);
4680 setWideningDecision(I: Member, VF, W: CM_Scalarize, Cost);
4681 UpdateMemOpUserCost(cast<LoadInst>(Val: Member));
4682 }
4683 }
4684 } else {
4685 // Cannot scalarize fixed-order recurrence phis at the moment.
4686 if (isa<PHINode>(Val: I) && Legal->isFixedOrderRecurrence(Phi: cast<PHINode>(Val: I)))
4687 continue;
4688
4689 // Make sure I gets scalarized and a cost estimate without
4690 // scalarization overhead.
4691 ForcedScalars[VF].insert(X: I);
4692 }
4693 }
4694}
4695
4696bool LoopVectorizationCostModel::shouldConsiderInvariant(Value *Op) {
4697 if (!Legal->isInvariant(V: Op))
4698 return false;
4699 // Consider Op invariant, if it or its operands aren't predicated
4700 // instruction in the loop. In that case, it is not trivially hoistable.
4701 auto *OpI = dyn_cast<Instruction>(Val: Op);
4702 return !OpI || !TheLoop->contains(Inst: OpI) ||
4703 (!isPredicatedInst(I: OpI) &&
4704 (!isa<PHINode>(Val: OpI) || OpI->getParent() != TheLoop->getHeader()) &&
4705 all_of(Range: OpI->operands(),
4706 P: [this](Value *Op) { return shouldConsiderInvariant(Op); }));
4707}
4708
4709InstructionCost
4710LoopVectorizationCostModel::getInstructionCost(Instruction *I,
4711 ElementCount VF) {
4712 // If we know that this instruction will remain uniform, check the cost of
4713 // the scalar version.
4714 if (isUniformAfterVectorization(I, VF))
4715 VF = ElementCount::getFixed(MinVal: 1);
4716
4717 if (VF.isVector() && isProfitableToScalarize(I, VF))
4718 return InstsToScalarize[VF][I];
4719
4720 // Forced scalars do not have any scalarization overhead.
4721 auto ForcedScalar = ForcedScalars.find(Val: VF);
4722 if (VF.isVector() && ForcedScalar != ForcedScalars.end()) {
4723 auto InstSet = ForcedScalar->second;
4724 if (InstSet.count(key: I))
4725 return getInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1)) *
4726 VF.getKnownMinValue();
4727 }
4728
4729 const auto &MinBWs = Config.getMinimalBitwidths();
4730 uint64_t InstrMinBWs = MinBWs.lookup(Key: I);
4731 Type *RetTy = I->getType();
4732 if (canTruncateToMinimalBitwidth(I, VF))
4733 RetTy = IntegerType::get(C&: RetTy->getContext(), NumBits: InstrMinBWs);
4734 auto *SE = PSE.getSE();
4735
4736 Type *VectorTy;
4737 if (isScalarAfterVectorization(I, VF)) {
4738 [[maybe_unused]] auto HasSingleCopyAfterVectorization =
4739 [this](Instruction *I, ElementCount VF) -> bool {
4740 if (VF.isScalar())
4741 return true;
4742
4743 auto Scalarized = InstsToScalarize.find(Key: VF);
4744 assert(Scalarized != InstsToScalarize.end() &&
4745 "VF not yet analyzed for scalarization profitability");
4746 return !Scalarized->second.count(Key: I) &&
4747 llvm::all_of(Range: I->users(), P: [&](User *U) {
4748 auto *UI = cast<Instruction>(Val: U);
4749 return !Scalarized->second.count(Key: UI);
4750 });
4751 };
4752
4753 // With the exception of GEPs and PHIs, after scalarization there should
4754 // only be one copy of the instruction generated in the loop. This is
4755 // because the VF is either 1, or any instructions that need scalarizing
4756 // have already been dealt with by the time we get here. As a result,
4757 // it means we don't have to multiply the instruction cost by VF.
4758 assert(I->getOpcode() == Instruction::GetElementPtr ||
4759 I->getOpcode() == Instruction::PHI ||
4760 (I->getOpcode() == Instruction::BitCast &&
4761 I->getType()->isPointerTy()) ||
4762 HasSingleCopyAfterVectorization(I, VF));
4763 VectorTy = RetTy;
4764 } else
4765 VectorTy = toVectorizedTy(Ty: RetTy, EC: VF);
4766
4767 if (VF.isVector() && VectorTy->isVectorTy() &&
4768 !TTI.getNumberOfParts(Tp: VectorTy))
4769 return InstructionCost::getInvalid();
4770
4771 // TODO: We need to estimate the cost of intrinsic calls.
4772 switch (I->getOpcode()) {
4773 case Instruction::GetElementPtr:
4774 // We mark this instruction as zero-cost because the cost of GEPs in
4775 // vectorized code depends on whether the corresponding memory instruction
4776 // is scalarized or not. Therefore, we handle GEPs with the memory
4777 // instruction cost.
4778 return 0;
4779 case Instruction::UncondBr:
4780 case Instruction::CondBr: {
4781 // In cases of scalarized and predicated instructions, there will be VF
4782 // predicated blocks in the vectorized loop. Each branch around these
4783 // blocks requires also an extract of its vector compare i1 element.
4784 // Note that the conditional branch from the loop latch will be replaced by
4785 // a single branch controlling the loop, so there is no extra overhead from
4786 // scalarization.
4787 bool ScalarPredicatedBB = false;
4788 CondBrInst *BI = dyn_cast<CondBrInst>(Val: I);
4789 if (VF.isVector() && BI &&
4790 (PredicatedBBsAfterVectorization[VF].count(Ptr: BI->getSuccessor(i: 0)) ||
4791 PredicatedBBsAfterVectorization[VF].count(Ptr: BI->getSuccessor(i: 1))) &&
4792 BI->getParent() != TheLoop->getLoopLatch())
4793 ScalarPredicatedBB = true;
4794
4795 if (ScalarPredicatedBB) {
4796 // Not possible to scalarize scalable vector with predicated instructions.
4797 if (VF.isScalable())
4798 return InstructionCost::getInvalid();
4799 // Return cost for branches around scalarized and predicated blocks.
4800 auto *VecI1Ty =
4801 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: RetTy->getContext()), EC: VF);
4802 return (TTI.getScalarizationOverhead(
4803 Ty: VecI1Ty, DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4804 /*Insert*/ false, /*Extract*/ true, CostKind: Config.CostKind) +
4805 (TTI.getCFInstrCost(Opcode: Instruction::CondBr, CostKind: Config.CostKind) *
4806 VF.getFixedValue()));
4807 }
4808
4809 if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar())
4810 // The back-edge branch will remain, as will all scalar branches.
4811 return TTI.getCFInstrCost(Opcode: Instruction::UncondBr, CostKind: Config.CostKind);
4812
4813 // This branch will be eliminated by if-conversion.
4814 return 0;
4815 // Note: We currently assume zero cost for an unconditional branch inside
4816 // a predicated block since it will become a fall-through, although we
4817 // may decide in the future to call TTI for all branches.
4818 }
4819 case Instruction::Switch: {
4820 if (VF.isScalar())
4821 return TTI.getCFInstrCost(Opcode: Instruction::Switch, CostKind: Config.CostKind);
4822 auto *Switch = cast<SwitchInst>(Val: I);
4823 return Switch->getNumCases() *
4824 TTI.getCmpSelInstrCost(
4825 Opcode: Instruction::ICmp,
4826 ValTy: toVectorTy(Scalar: Switch->getCondition()->getType(), EC: VF),
4827 CondTy: toVectorTy(Scalar: Type::getInt1Ty(C&: I->getContext()), EC: VF),
4828 VecPred: CmpInst::ICMP_EQ, CostKind: Config.CostKind);
4829 }
4830 case Instruction::PHI: {
4831 auto *Phi = cast<PHINode>(Val: I);
4832
4833 // First-order recurrences are replaced by vector shuffles inside the loop.
4834 if (VF.isVector() && Legal->isFixedOrderRecurrence(Phi)) {
4835 return TTI.getShuffleCost(
4836 Kind: TargetTransformInfo::SK_Splice, DstTy: cast<VectorType>(Val: VectorTy),
4837 SrcTy: cast<VectorType>(Val: VectorTy), CostKind: Config.CostKind, Mask: {}, Index: -1);
4838 }
4839
4840 // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
4841 // converted into select instructions. We require N - 1 selects per phi
4842 // node, where N is the number of incoming values.
4843 if (VF.isVector() && Phi->getParent() != TheLoop->getHeader()) {
4844 Type *ResultTy = Phi->getType();
4845
4846 // All instructions in an Any-of reduction chain are narrowed to bool.
4847 // Check if that is the case for this phi node.
4848 auto *HeaderUser = cast_if_present<PHINode>(
4849 Val: find_singleton<User>(Range: Phi->users(), P: [this](User *U, bool) -> User * {
4850 auto *Phi = dyn_cast<PHINode>(Val: U);
4851 if (Phi && Phi->getParent() == TheLoop->getHeader())
4852 return Phi;
4853 return nullptr;
4854 }));
4855 if (HeaderUser) {
4856 auto &ReductionVars = Legal->getReductionVars();
4857 auto Iter = ReductionVars.find(Key: HeaderUser);
4858 if (Iter != ReductionVars.end() &&
4859 RecurrenceDescriptor::isAnyOfRecurrenceKind(
4860 Kind: Iter->second.getRecurrenceKind()))
4861 ResultTy = Type::getInt1Ty(C&: Phi->getContext());
4862 }
4863 return (Phi->getNumIncomingValues() - 1) *
4864 TTI.getCmpSelInstrCost(
4865 Opcode: Instruction::Select, ValTy: toVectorTy(Scalar: ResultTy, EC: VF),
4866 CondTy: toVectorTy(Scalar: Type::getInt1Ty(C&: Phi->getContext()), EC: VF),
4867 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind: Config.CostKind);
4868 }
4869
4870 // When tail folding with EVL, if the phi is part of an out of loop
4871 // reduction then it will be transformed into a wide vp_merge.
4872 if (VF.isVector() && foldTailWithEVL() &&
4873 Legal->getReductionVars().contains(Key: Phi) &&
4874 !Config.isInLoopReduction(Phi)) {
4875 IntrinsicCostAttributes ICA(
4876 Intrinsic::vp_merge, toVectorTy(Scalar: Phi->getType(), EC: VF),
4877 {toVectorTy(Scalar: Type::getInt1Ty(C&: Phi->getContext()), EC: VF)});
4878 return TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind);
4879 }
4880
4881 return TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
4882 }
4883 case Instruction::UDiv:
4884 case Instruction::SDiv:
4885 case Instruction::URem:
4886 case Instruction::SRem:
4887 if (VF.isVector() && isPredicatedInst(I)) {
4888 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
4889 return isDivRemScalarWithPredication(ScalarCost, MaskedCost) ? ScalarCost
4890 : MaskedCost;
4891 }
4892 // We've proven all lanes safe to speculate, fall through.
4893 [[fallthrough]];
4894 case Instruction::Add:
4895 case Instruction::Sub: {
4896 auto Info = Legal->getHistogramInfo(I);
4897 if (Info && VF.isVector()) {
4898 const HistogramInfo *HGram = Info.value();
4899 // Assume that a non-constant update value (or a constant != 1) requires
4900 // a multiply, and add that into the cost.
4901 InstructionCost MulCost = TTI::TCC_Free;
4902 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: I->getOperand(i: 1));
4903 if (!RHS || RHS->getZExtValue() != 1)
4904 MulCost = TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: VectorTy,
4905 CostKind: Config.CostKind);
4906
4907 // Find the cost of the histogram operation itself.
4908 Type *PtrTy = VectorType::get(ElementType: HGram->Load->getPointerOperandType(), EC: VF);
4909 Type *ScalarTy = I->getType();
4910 Type *MaskTy = VectorType::get(ElementType: Type::getInt1Ty(C&: I->getContext()), EC: VF);
4911 IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
4912 Type::getVoidTy(C&: I->getContext()),
4913 {PtrTy, ScalarTy, MaskTy});
4914
4915 // Add the costs together with the add/sub operation.
4916 return TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind) + MulCost +
4917 TTI.getArithmeticInstrCost(Opcode: I->getOpcode(), Ty: VectorTy,
4918 CostKind: Config.CostKind);
4919 }
4920 [[fallthrough]];
4921 }
4922 case Instruction::FAdd:
4923 case Instruction::FSub:
4924 case Instruction::Mul:
4925 case Instruction::FMul:
4926 case Instruction::FDiv:
4927 case Instruction::FRem:
4928 case Instruction::Shl:
4929 case Instruction::LShr:
4930 case Instruction::AShr:
4931 case Instruction::And:
4932 case Instruction::Or:
4933 case Instruction::Xor: {
4934 // If we're speculating on the stride being 1, the multiplication may
4935 // fold away. We can generalize this for all operations using the notion
4936 // of neutral elements. (TODO)
4937 if (I->getOpcode() == Instruction::Mul &&
4938 ((TheLoop->isLoopInvariant(V: I->getOperand(i: 0)) &&
4939 PSE.getSCEV(V: I->getOperand(i: 0))->isOne()) ||
4940 (TheLoop->isLoopInvariant(V: I->getOperand(i: 1)) &&
4941 PSE.getSCEV(V: I->getOperand(i: 1))->isOne())))
4942 return 0;
4943
4944 // Certain instructions can be cheaper to vectorize if they have a constant
4945 // second vector operand. One example of this are shifts on x86.
4946 Value *Op2 = I->getOperand(i: 1);
4947 if (!isa<Constant>(Val: Op2) && TheLoop->isLoopInvariant(V: Op2) &&
4948 PSE.getSE()->isSCEVable(Ty: Op2->getType()) &&
4949 isa<SCEVConstant>(Val: PSE.getSCEV(V: Op2))) {
4950 Op2 = cast<SCEVConstant>(Val: PSE.getSCEV(V: Op2))->getValue();
4951 }
4952 auto Op2Info = TTI.getOperandInfo(V: Op2);
4953 if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue &&
4954 shouldConsiderInvariant(Op: Op2))
4955 Op2Info.Kind = TargetTransformInfo::OK_UniformValue;
4956
4957 SmallVector<const Value *, 4> Operands(I->operand_values());
4958 return TTI.getArithmeticInstrCost(
4959 Opcode: I->getOpcode(), Ty: VectorTy, CostKind: Config.CostKind,
4960 Opd1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
4961 Opd2Info: Op2Info, Args: Operands, CxtI: I, TLibInfo: TLI);
4962 }
4963 case Instruction::FNeg: {
4964 return TTI.getArithmeticInstrCost(
4965 Opcode: I->getOpcode(), Ty: VectorTy, CostKind: Config.CostKind,
4966 Opd1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
4967 Opd2Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
4968 Args: I->getOperand(i: 0), CxtI: I);
4969 }
4970 case Instruction::Select: {
4971 SelectInst *SI = cast<SelectInst>(Val: I);
4972 const SCEV *CondSCEV = SE->getSCEV(V: SI->getCondition());
4973 bool ScalarCond = (SE->isLoopInvariant(S: CondSCEV, L: TheLoop));
4974
4975 const Value *Op0, *Op1;
4976 using namespace llvm::PatternMatch;
4977 if (!ScalarCond && (match(V: I, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))) ||
4978 match(V: I, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))) {
4979 // select x, y, false --> x & y
4980 // select x, true, y --> x | y
4981 const auto [Op1VK, Op1VP] = TTI::getOperandInfo(V: Op0);
4982 const auto [Op2VK, Op2VP] = TTI::getOperandInfo(V: Op1);
4983 assert(Op0->getType()->getScalarSizeInBits() == 1 &&
4984 Op1->getType()->getScalarSizeInBits() == 1);
4985
4986 return TTI.getArithmeticInstrCost(
4987 Opcode: match(V: I, P: m_LogicalOr()) ? Instruction::Or : Instruction::And,
4988 Ty: VectorTy, CostKind: Config.CostKind, Opd1Info: {.Kind: Op1VK, .Properties: Op1VP}, Opd2Info: {.Kind: Op2VK, .Properties: Op2VP}, Args: {Op0, Op1},
4989 CxtI: I);
4990 }
4991
4992 Type *CondTy = SI->getCondition()->getType();
4993 if (!ScalarCond)
4994 CondTy = VectorType::get(ElementType: CondTy, EC: VF);
4995
4996 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
4997 if (auto *Cmp = dyn_cast<CmpInst>(Val: SI->getCondition()))
4998 Pred = Cmp->getPredicate();
4999 return TTI.getCmpSelInstrCost(
5000 Opcode: I->getOpcode(), ValTy: VectorTy, CondTy, VecPred: Pred, CostKind: Config.CostKind,
5001 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
5002 }
5003 case Instruction::ICmp:
5004 case Instruction::FCmp: {
5005 Type *ValTy = I->getOperand(i: 0)->getType();
5006
5007 if (canTruncateToMinimalBitwidth(I, VF)) {
5008 [[maybe_unused]] Instruction *Op0AsInstruction =
5009 dyn_cast<Instruction>(Val: I->getOperand(i: 0));
5010 assert((!canTruncateToMinimalBitwidth(Op0AsInstruction, VF) ||
5011 InstrMinBWs == MinBWs.lookup(Op0AsInstruction)) &&
5012 "if both the operand and the compare are marked for "
5013 "truncation, they must have the same bitwidth");
5014 ValTy = IntegerType::get(C&: ValTy->getContext(), NumBits: InstrMinBWs);
5015 }
5016
5017 VectorTy = toVectorTy(Scalar: ValTy, EC: VF);
5018 return TTI.getCmpSelInstrCost(
5019 Opcode: I->getOpcode(), ValTy: VectorTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VectorTy),
5020 VecPred: cast<CmpInst>(Val: I)->getPredicate(), CostKind: Config.CostKind,
5021 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
5022 }
5023 case Instruction::Store:
5024 case Instruction::Load: {
5025 ElementCount Width = VF;
5026 if (Width.isVector()) {
5027 InstWidening Decision = getWideningDecision(I, VF: Width);
5028 assert(Decision != CM_Unknown &&
5029 "CM decision should be taken at this point");
5030 if (getWideningCost(I, VF) == InstructionCost::getInvalid())
5031 return InstructionCost::getInvalid();
5032 if (Decision == CM_Scalarize)
5033 Width = ElementCount::getFixed(MinVal: 1);
5034 }
5035 VectorTy = toVectorTy(Scalar: getLoadStoreType(I), EC: Width);
5036 return getMemoryInstructionCost(I, VF);
5037 }
5038 case Instruction::BitCast:
5039 if (I->getType()->isPointerTy())
5040 return 0;
5041 [[fallthrough]];
5042 case Instruction::ZExt:
5043 case Instruction::SExt:
5044 case Instruction::FPToUI:
5045 case Instruction::FPToSI:
5046 case Instruction::FPExt:
5047 case Instruction::PtrToInt:
5048 case Instruction::IntToPtr:
5049 case Instruction::SIToFP:
5050 case Instruction::UIToFP:
5051 case Instruction::Trunc:
5052 case Instruction::FPTrunc: {
5053 // Computes the CastContextHint from a Load/Store instruction.
5054 auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
5055 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
5056 "Expected a load or a store!");
5057
5058 if (VF.isScalar() || !TheLoop->contains(Inst: I))
5059 return TTI::CastContextHint::Normal;
5060
5061 switch (getWideningDecision(I, VF)) {
5062 case LoopVectorizationCostModel::CM_GatherScatter:
5063 return TTI::CastContextHint::GatherScatter;
5064 case LoopVectorizationCostModel::CM_Interleave:
5065 return TTI::CastContextHint::Interleave;
5066 case LoopVectorizationCostModel::CM_Scalarize:
5067 case LoopVectorizationCostModel::CM_Widen:
5068 return isPredicatedInst(I) ? TTI::CastContextHint::Masked
5069 : TTI::CastContextHint::Normal;
5070 case LoopVectorizationCostModel::CM_Widen_Reverse:
5071 return TTI::CastContextHint::Reversed;
5072 case LoopVectorizationCostModel::CM_Unknown:
5073 llvm_unreachable("Instr did not go through cost modelling?");
5074 case LoopVectorizationCostModel::CM_InvalidatedDecision:
5075 return TTI::CastContextHint::None;
5076 }
5077
5078 llvm_unreachable("Unhandled case!");
5079 };
5080
5081 unsigned Opcode = I->getOpcode();
5082 TTI::CastContextHint CCH = TTI::CastContextHint::None;
5083 // For Trunc, the context is the only user, which must be a StoreInst.
5084 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
5085 if (I->hasOneUse())
5086 if (StoreInst *Store = dyn_cast<StoreInst>(Val: *I->user_begin()))
5087 CCH = ComputeCCH(Store);
5088 }
5089 // For Z/Sext, the context is the operand, which must be a LoadInst.
5090 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
5091 Opcode == Instruction::FPExt) {
5092 if (LoadInst *Load = dyn_cast<LoadInst>(Val: I->getOperand(i: 0)))
5093 CCH = ComputeCCH(Load);
5094 }
5095
5096 // We optimize the truncation of induction variables having constant
5097 // integer steps. The cost of these truncations is the same as the scalar
5098 // operation.
5099 if (isOptimizableIVTruncate(I, VF)) {
5100 auto *Trunc = cast<TruncInst>(Val: I);
5101 return TTI.getCastInstrCost(Opcode: Instruction::Trunc, Dst: Trunc->getDestTy(),
5102 Src: Trunc->getSrcTy(), CCH, CostKind: Config.CostKind,
5103 I: Trunc);
5104 }
5105
5106 Type *SrcScalarTy = I->getOperand(i: 0)->getType();
5107 Instruction *Op0AsInstruction = dyn_cast<Instruction>(Val: I->getOperand(i: 0));
5108 if (canTruncateToMinimalBitwidth(I: Op0AsInstruction, VF))
5109 SrcScalarTy = IntegerType::get(C&: SrcScalarTy->getContext(),
5110 NumBits: MinBWs.lookup(Key: Op0AsInstruction));
5111 Type *SrcVecTy =
5112 VectorTy->isVectorTy() ? toVectorTy(Scalar: SrcScalarTy, EC: VF) : SrcScalarTy;
5113
5114 if (canTruncateToMinimalBitwidth(I, VF)) {
5115 // If the result type is <= the source type, there will be no extend
5116 // after truncating the users to the minimal required bitwidth.
5117 if (VectorTy->getScalarSizeInBits() <= SrcVecTy->getScalarSizeInBits() &&
5118 (I->getOpcode() == Instruction::ZExt ||
5119 I->getOpcode() == Instruction::SExt))
5120 return 0;
5121 }
5122
5123 return TTI.getCastInstrCost(Opcode, Dst: VectorTy, Src: SrcVecTy, CCH,
5124 CostKind: Config.CostKind, I);
5125 }
5126 case Instruction::Call:
5127 return getVectorCallCost(CI: cast<CallInst>(Val: I), VF);
5128 case Instruction::ExtractValue:
5129 return TTI.getInstructionCost(U: I, CostKind: Config.CostKind);
5130 case Instruction::Alloca:
5131 // We cannot easily widen alloca to a scalable alloca, as
5132 // the result would need to be a vector of pointers.
5133 if (VF.isScalable())
5134 return InstructionCost::getInvalid();
5135 return TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: RetTy, CostKind: Config.CostKind);
5136 case Instruction::Freeze:
5137 return TTI::TCC_Free;
5138 default:
5139 // This opcode is unknown. Assume that it is the same as 'mul'.
5140 return TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: VectorTy,
5141 CostKind: Config.CostKind);
5142 } // end of switch.
5143}
5144
5145void LoopVectorizationCostModel::collectValuesToIgnore() {
5146 // Ignore ephemeral values.
5147 CodeMetrics::collectEphemeralValues(L: TheLoop, AC, EphValues&: ValuesToIgnore);
5148
5149 SmallVector<Value *, 4> DeadInterleavePointerOps;
5150 SmallVector<Value *, 4> DeadOps;
5151
5152 // If a scalar epilogue is required, users outside the loop won't use
5153 // live-outs from the vector loop but from the scalar epilogue. Ignore them if
5154 // that is the case.
5155 bool RequiresScalarEpilogue = requiresScalarEpilogue(IsVectorizing: true);
5156 auto IsLiveOutDead = [this, RequiresScalarEpilogue](User *U) {
5157 return RequiresScalarEpilogue &&
5158 !TheLoop->contains(BB: cast<Instruction>(Val: U)->getParent());
5159 };
5160
5161 LoopBlocksDFS DFS(TheLoop);
5162 DFS.perform(LI);
5163 for (BasicBlock *BB : reverse(C: make_range(x: DFS.beginRPO(), y: DFS.endRPO())))
5164 for (Instruction &I : reverse(C&: *BB)) {
5165 if (VecValuesToIgnore.contains(Ptr: &I) || ValuesToIgnore.contains(Ptr: &I))
5166 continue;
5167
5168 // Add instructions that would be trivially dead and are only used by
5169 // values already ignored to DeadOps to seed worklist.
5170 if (wouldInstructionBeTriviallyDead(I: &I, TLI) &&
5171 all_of(Range: I.users(), P: [this, IsLiveOutDead](User *U) {
5172 return VecValuesToIgnore.contains(Ptr: U) ||
5173 ValuesToIgnore.contains(Ptr: U) || IsLiveOutDead(U);
5174 }))
5175 DeadOps.push_back(Elt: &I);
5176
5177 // For interleave groups, we only create a pointer for the start of the
5178 // interleave group. Queue up addresses of group members except the insert
5179 // position for further processing.
5180 if (isAccessInterleaved(Instr: &I)) {
5181 auto *Group = getInterleavedAccessGroup(Instr: &I);
5182 if (Group->getInsertPos() == &I)
5183 continue;
5184 Value *PointerOp = getLoadStorePointerOperand(V: &I);
5185 DeadInterleavePointerOps.push_back(Elt: PointerOp);
5186 }
5187
5188 // Queue branches for analysis. They are dead, if their successors only
5189 // contain dead instructions.
5190 if (isa<CondBrInst>(Val: &I))
5191 DeadOps.push_back(Elt: &I);
5192 }
5193
5194 // Mark ops feeding interleave group members as free, if they are only used
5195 // by other dead computations.
5196 for (unsigned I = 0; I != DeadInterleavePointerOps.size(); ++I) {
5197 auto *Op = dyn_cast<Instruction>(Val: DeadInterleavePointerOps[I]);
5198 if (!Op || !TheLoop->contains(Inst: Op) || any_of(Range: Op->users(), P: [this](User *U) {
5199 Instruction *UI = cast<Instruction>(Val: U);
5200 return !VecValuesToIgnore.contains(Ptr: U) &&
5201 (!isAccessInterleaved(Instr: UI) ||
5202 getInterleavedAccessGroup(Instr: UI)->getInsertPos() == UI);
5203 }))
5204 continue;
5205 VecValuesToIgnore.insert(Ptr: Op);
5206 append_range(C&: DeadInterleavePointerOps, R: Op->operands());
5207 }
5208
5209 // Mark ops that would be trivially dead and are only used by ignored
5210 // instructions as free.
5211 BasicBlock *Header = TheLoop->getHeader();
5212
5213 // Returns true if the block contains only dead instructions. Such blocks will
5214 // be removed by VPlan-to-VPlan transforms and won't be considered by the
5215 // VPlan-based cost model, so skip them in the legacy cost-model as well.
5216 auto IsEmptyBlock = [this](BasicBlock *BB) {
5217 return all_of(Range&: *BB, P: [this](Instruction &I) {
5218 return ValuesToIgnore.contains(Ptr: &I) || VecValuesToIgnore.contains(Ptr: &I) ||
5219 isa<UncondBrInst>(Val: &I);
5220 });
5221 };
5222 for (unsigned I = 0; I != DeadOps.size(); ++I) {
5223 auto *Op = dyn_cast<Instruction>(Val: DeadOps[I]);
5224
5225 // Check if the branch should be considered dead.
5226 if (auto *Br = dyn_cast_or_null<CondBrInst>(Val: Op)) {
5227 BasicBlock *ThenBB = Br->getSuccessor(i: 0);
5228 BasicBlock *ElseBB = Br->getSuccessor(i: 1);
5229 // Don't considers branches leaving the loop for simplification.
5230 if (!TheLoop->contains(BB: ThenBB) || !TheLoop->contains(BB: ElseBB))
5231 continue;
5232 bool ThenEmpty = IsEmptyBlock(ThenBB);
5233 bool ElseEmpty = IsEmptyBlock(ElseBB);
5234 if ((ThenEmpty && ElseEmpty) ||
5235 (ThenEmpty && ThenBB->getSingleSuccessor() == ElseBB &&
5236 ElseBB->phis().empty()) ||
5237 (ElseEmpty && ElseBB->getSingleSuccessor() == ThenBB &&
5238 ThenBB->phis().empty())) {
5239 VecValuesToIgnore.insert(Ptr: Br);
5240 DeadOps.push_back(Elt: Br->getCondition());
5241 }
5242 continue;
5243 }
5244
5245 // Skip any op that shouldn't be considered dead.
5246 if (!Op || !TheLoop->contains(Inst: Op) ||
5247 (isa<PHINode>(Val: Op) && Op->getParent() == Header) ||
5248 !wouldInstructionBeTriviallyDead(I: Op, TLI) ||
5249 any_of(Range: Op->users(), P: [this, IsLiveOutDead](User *U) {
5250 return !VecValuesToIgnore.contains(Ptr: U) &&
5251 !ValuesToIgnore.contains(Ptr: U) && !IsLiveOutDead(U);
5252 }))
5253 continue;
5254
5255 // If all of Op's users are in ValuesToIgnore, add it to ValuesToIgnore
5256 // which applies for both scalar and vector versions. Otherwise it is only
5257 // dead in vector versions, so only add it to VecValuesToIgnore.
5258 if (all_of(Range: Op->users(),
5259 P: [this](User *U) { return ValuesToIgnore.contains(Ptr: U); }))
5260 ValuesToIgnore.insert(Ptr: Op);
5261
5262 VecValuesToIgnore.insert(Ptr: Op);
5263 append_range(C&: DeadOps, R: Op->operands());
5264 }
5265
5266 // Ignore type-promoting instructions we identified during reduction
5267 // detection.
5268 for (const auto &Reduction : Legal->getReductionVars()) {
5269 const RecurrenceDescriptor &RedDes = Reduction.second;
5270 const SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
5271 VecValuesToIgnore.insert_range(R: Casts);
5272 }
5273 // Ignore type-casting instructions we identified during induction
5274 // detection.
5275 for (const auto &Induction : Legal->getInductionVars()) {
5276 const InductionDescriptor &IndDes = Induction.second;
5277 VecValuesToIgnore.insert_range(R: IndDes.getCastInsts());
5278 }
5279}
5280
5281void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
5282 CM->collectValuesToIgnore();
5283 Config.collectElementTypesForWidening(ValuesToIgnore: &CM->ValuesToIgnore);
5284
5285 FixedScalableVFPair MaxFactors = CM->computeMaxVF(UserVF, UserIC);
5286 if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
5287 return;
5288
5289 Config.collectInLoopReductions();
5290 // Cases that may be vectorized may be optimized by unit stride predicates.
5291 // TODO: Currently unit stride predicates are added unconditionally, even if
5292 // they are not used for the selected VF (e.g. when only interleaving).
5293 if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
5294 Legal->collectUnitStridePredicates();
5295
5296 auto VPlan1 = tryToBuildVPlan1();
5297 if (!VPlan1)
5298 return;
5299
5300 if (!OrigLoop->isInnermost()) {
5301 // For outer loops, computeMaxVF returns a single non-scalar VF; build a
5302 // plan for that VF only.
5303 ElementCount VF =
5304 MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
5305 buildVPlans(VPlan1&: *VPlan1, MinVF: VF, MaxVF: VF);
5306 LLVM_DEBUG(printPlans(dbgs()));
5307 return;
5308 }
5309
5310 // Compute the minimal bitwidths required for integer operations in the loop
5311 // for later use by the cost model.
5312 Config.computeMinimalBitwidths();
5313
5314 // Invalidate interleave groups if all blocks of loop will be predicated.
5315 if (CM->blockNeedsPredicationForAnyReason(BB: OrigLoop->getHeader()) &&
5316 !useMaskedInterleavedAccesses(TTI)) {
5317 LLVM_DEBUG(
5318 dbgs()
5319 << "LV: Invalidate all interleaved groups due to fold-tail by masking "
5320 "which requires masked-interleaved support.\n");
5321 if (CM->InterleaveInfo.invalidateGroups())
5322 // Invalidating interleave groups also requires invalidating all decisions
5323 // based on them, which includes widening decisions and uniform and scalar
5324 // values.
5325 CM->invalidateCostModelingDecisions();
5326 }
5327
5328 if (CM->foldTailByMasking())
5329 Legal->prepareToFoldTailByMasking();
5330
5331 ElementCount MaxUserVF =
5332 UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
5333 if (UserVF) {
5334 if (!ElementCount::isKnownLE(LHS: UserVF, RHS: MaxUserVF)) {
5335 reportVectorizationInfo(
5336 Msg: "UserVF ignored because it may be larger than the maximal safe VF",
5337 ORETag: "InvalidUserVF", ORE, TheLoop: OrigLoop);
5338 } else {
5339 assert(isPowerOf2_32(UserVF.getKnownMinValue()) &&
5340 "VF needs to be a power of two");
5341 // Collect the instructions (and their associated costs) that will be more
5342 // profitable to scalarize.
5343 CM->collectNonVectorizedAndSetWideningDecisions(VF: UserVF);
5344 buildVPlans(VPlan1&: *VPlan1, MinVF: UserVF, MaxVF: UserVF);
5345 ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
5346 if (EpilogueUserVF.isVector() &&
5347 ElementCount::isKnownLT(LHS: EpilogueUserVF, RHS: UserVF)) {
5348 CM->collectNonVectorizedAndSetWideningDecisions(VF: EpilogueUserVF);
5349 buildVPlans(VPlan1&: *VPlan1, MinVF: EpilogueUserVF, MaxVF: EpilogueUserVF);
5350 }
5351 if (!VPlans.empty() && VPlans.front()->getSingleVF() == UserVF) {
5352 // For scalar VF, skip VPlan cost check as VPlan cost is designed for
5353 // vector VFs only.
5354 if (UserVF.isScalar() ||
5355 cost(Plan&: *VPlans.front(), VF: UserVF, /*RU=*/nullptr).isValid()) {
5356 LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
5357 LLVM_DEBUG(printPlans(dbgs()));
5358 return;
5359 }
5360 }
5361 VPlans.clear();
5362 reportVectorizationInfo(Msg: "UserVF ignored because of invalid costs.",
5363 ORETag: "InvalidCost", ORE, TheLoop: OrigLoop);
5364 }
5365 }
5366
5367 // Collect the Vectorization Factor Candidates.
5368 SmallVector<ElementCount> VFCandidates;
5369 for (auto VF = ElementCount::getFixed(MinVal: 1);
5370 ElementCount::isKnownLE(LHS: VF, RHS: MaxFactors.FixedVF); VF *= 2)
5371 VFCandidates.push_back(Elt: VF);
5372 for (auto VF = ElementCount::getScalable(MinVal: 1);
5373 ElementCount::isKnownLE(LHS: VF, RHS: MaxFactors.ScalableVF); VF *= 2)
5374 VFCandidates.push_back(Elt: VF);
5375
5376 for (const auto &VF : VFCandidates) {
5377 // Collect Uniform and Scalar instructions after vectorization with VF.
5378 CM->collectNonVectorizedAndSetWideningDecisions(VF);
5379 }
5380
5381 buildVPlans(VPlan1&: *VPlan1, MinVF: ElementCount::getFixed(MinVal: 1), MaxVF: MaxFactors.FixedVF);
5382 buildVPlans(VPlan1&: *VPlan1, MinVF: ElementCount::getScalable(MinVal: 1), MaxVF: MaxFactors.ScalableVF);
5383
5384 LLVM_DEBUG(printPlans(dbgs()));
5385}
5386
5387VPCostContext::VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan,
5388 LoopVectorizationCostModel &CM,
5389 VFSelectionContext &Config,
5390 bool ReusePrintingSlotTracker)
5391 : TTI(Config.getTTI()), TLI(TLI), LLVMCtx(Plan.getContext()), CM(CM),
5392 Config(Config), CostKind(Config.CostKind), PSE(Config.getPSE()),
5393 L(Config.getLoop()) {
5394#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5395 if (ReusePrintingSlotTracker)
5396 PlanForSlotTracker = &Plan;
5397#endif
5398}
5399
5400InstructionCost VPCostContext::getLegacyCost(Instruction *UI,
5401 ElementCount VF) const {
5402 InstructionCost Cost = CM.getInstructionCost(I: UI, VF);
5403 if (Cost.isValid() && ForceTargetInstructionCost.getNumOccurrences())
5404 return InstructionCost(ForceTargetInstructionCost);
5405 return Cost;
5406}
5407
5408bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const {
5409 return CM.ValuesToIgnore.contains(Ptr: UI) ||
5410 (IsVector && CM.VecValuesToIgnore.contains(Ptr: UI)) ||
5411 SkipCostComputation.contains(Ptr: UI);
5412}
5413
5414void VPCostContext::invalidateWideningDecision(Instruction *I,
5415 ElementCount VF) {
5416 CM.setWideningDecision(I, VF,
5417 W: LoopVectorizationCostModel::CM_InvalidatedDecision, Cost: 0);
5418}
5419
5420bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const {
5421 return CM.isScalarWithPredication(I, VF) ||
5422 CM.isUniformAfterVectorization(I, VF) || CM.isForcedScalar(I, VF) ||
5423 (VF.isVector() && CM.isProfitableToScalarize(I, VF));
5424}
5425
5426bool VPCostContext::isMaskRequired(Instruction *I) const {
5427 return CM.isMaskRequired(I);
5428}
5429
5430bool VPCostContext::executesAtMostOnce(const VPlan &Plan, ElementCount VF) {
5431 auto *TC = dyn_cast_if_present<ConstantInt>(
5432 Val: Plan.getTripCount()->getUnderlyingValue());
5433 return TC && TC->getValue().ule(RHS: VF.getKnownMinValue());
5434}
5435
5436InstructionCost
5437LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
5438 VPCostContext &CostCtx) const {
5439 InstructionCost Cost;
5440
5441 // If the vector loop gets executed exactly once with the given VF, ignore the
5442 // costs of comparison and induction instructions, as they'll get simplified
5443 // away.
5444 // TODO: Remove this code after stepping away from the legacy cost model and
5445 // adding code to simplify VPlans before calculating their costs.
5446 auto TC = getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop);
5447 if (TC == VF && !Plan.hasTailFolded())
5448 addFullyUnrolledInstructionsToIgnore(L: OrigLoop, IL: Legal->getInductionVars(),
5449 InstsToIgnore&: CostCtx.SkipCostComputation);
5450
5451 // Pre-compute the costs for branches except for the backedge, as the number
5452 // of replicate regions in a VPlan may not directly match the number of
5453 // branches, which would lead to different decisions.
5454 // TODO: Compute cost of branches for each replicate region in the VPlan,
5455 // which is more accurate than the legacy cost model.
5456 for (BasicBlock *BB : OrigLoop->blocks()) {
5457 if (CostCtx.skipCostComputation(UI: BB->getTerminator(), IsVector: VF.isVector()))
5458 continue;
5459 CostCtx.SkipCostComputation.insert(Ptr: BB->getTerminator());
5460 if (BB == OrigLoop->getLoopLatch())
5461 continue;
5462 auto BranchCost = CostCtx.getLegacyCost(UI: BB->getTerminator(), VF);
5463 Cost += BranchCost;
5464 }
5465
5466 // Don't apply special costs when instruction cost is forced to make sure the
5467 // forced cost is used for each recipe.
5468 if (ForceTargetInstructionCost.getNumOccurrences())
5469 return Cost;
5470
5471 // Pre-compute costs for instructions that are forced-scalar or profitable to
5472 // scalarize. For most such instructions, their scalarization costs are
5473 // accounted for here using the legacy cost model. However, some opcodes
5474 // are excluded from these precomputed scalarization costs and are instead
5475 // modeled later by the VPlan cost model (see UseVPlanCostModel below).
5476 for (Instruction *ForcedScalar : CostCtx.CM.ForcedScalars[VF]) {
5477 if (CostCtx.skipCostComputation(UI: ForcedScalar, IsVector: VF.isVector()))
5478 continue;
5479 CostCtx.SkipCostComputation.insert(Ptr: ForcedScalar);
5480 InstructionCost ForcedCost = CostCtx.getLegacyCost(UI: ForcedScalar, VF);
5481 LLVM_DEBUG({
5482 dbgs() << "Cost of " << ForcedCost << " for VF " << VF
5483 << ": forced scalar " << *ForcedScalar << "\n";
5484 });
5485 Cost += ForcedCost;
5486 }
5487
5488 // Don't apply legacy scalarization costs if nothing remains scalar &
5489 // predicated.
5490 if (!hasReplicatorRegion(Plan))
5491 return Cost;
5492
5493 auto UseVPlanCostModel = [](Instruction *I) -> bool {
5494 switch (I->getOpcode()) {
5495 case Instruction::SDiv:
5496 case Instruction::UDiv:
5497 case Instruction::SRem:
5498 case Instruction::URem:
5499 return true;
5500 default:
5501 return false;
5502 }
5503 };
5504 for (const auto &[Scalarized, ScalarCost] : CostCtx.CM.InstsToScalarize[VF]) {
5505 if (UseVPlanCostModel(Scalarized) ||
5506 CostCtx.skipCostComputation(UI: Scalarized, IsVector: VF.isVector()))
5507 continue;
5508 CostCtx.SkipCostComputation.insert(Ptr: Scalarized);
5509 LLVM_DEBUG({
5510 dbgs() << "Cost of " << ScalarCost << " for VF " << VF
5511 << ": profitable to scalarize " << *Scalarized << "\n";
5512 });
5513 Cost += ScalarCost;
5514 }
5515
5516 return Cost;
5517}
5518
5519InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
5520 VPRegisterUsage *RU) const {
5521 VPCostContext CostCtx(*TLI, Plan, *CM, Config,
5522 /*ReusePrintingSlotTracker=*/true);
5523 InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
5524
5525 // Now compute and add the VPlan-based cost.
5526 Cost += Plan.cost(VF, Ctx&: CostCtx);
5527
5528 // Add the cost of spills due to excess register usage
5529 if (RU && Config.shouldConsiderRegPressureForVF(VF))
5530 Cost += RU->spillCost(TTI, CostKind: Config.CostKind, OverrideMaxNumRegs: ForceTargetNumVectorRegs);
5531
5532#ifndef NDEBUG
5533 unsigned EstimatedWidth =
5534 estimateElementCount(VF, Config.getVScaleForTuning());
5535 LLVM_DEBUG(dbgs() << "Cost for VF " << VF << ": " << Cost
5536 << " (Estimated cost per lane: ");
5537 if (Cost.isValid()) {
5538 APFloat CostPerLane(APFloat::IEEEdouble());
5539 APFloat EstimatedWidthAsAPFloat(APFloat::IEEEdouble());
5540 (void)CostPerLane.convertFromAPInt(APInt(64, (uint64_t)Cost.getValue()),
5541 false, APFloat::rmTowardZero);
5542 (void)EstimatedWidthAsAPFloat.convertFromAPInt(
5543 APInt(64, (uint64_t)EstimatedWidth), false, APFloat::rmTowardZero);
5544 (void)CostPerLane.divide(EstimatedWidthAsAPFloat, APFloat::rmTowardZero);
5545
5546 SmallString<16> Str;
5547 CostPerLane.toString(Str, 3);
5548 LLVM_DEBUG(dbgs() << Str);
5549 } else /* No point dividing an invalid cost - it will still be invalid */
5550 LLVM_DEBUG(dbgs() << "Invalid");
5551 LLVM_DEBUG(dbgs() << ")\n");
5552#endif
5553 return Cost;
5554}
5555
5556std::pair<VectorizationFactor, VPlan *>
5557LoopVectorizationPlanner::computeBestVF() {
5558 if (VPlans.empty())
5559 return {VectorizationFactor::Disabled(), nullptr};
5560 // If there is a single VPlan with a single VF, return it directly.
5561 VPlan &FirstPlan = *VPlans[0];
5562
5563 ElementCount UserVF = Config.getHints().getWidth();
5564 if (VPlans.size() == 1) {
5565 // For outer loops, the plan has a single vector VF determined by the
5566 // heuristic.
5567 assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
5568 FirstPlan.isOuterLoop()) &&
5569 "must have a single scalar VF, UserVF or an outer loop");
5570 return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
5571 }
5572
5573 if (hasPlanWithVF(VF: UserVF) && hasForcedEpilogueVF() && VPlans.size() == 2) {
5574 assert(VPlans[0]->getSingleVF() == UserVF &&
5575 "expected second plan to be for the forced UserVF");
5576 assert(VPlans[1]->getSingleVF() == EpilogueVectorizationForceVF &&
5577 "expected first plan to be for the forced epilogue VF");
5578 return {VectorizationFactor(UserVF, 0, 0), VPlans[0].get()};
5579 }
5580
5581 LLVM_DEBUG(dbgs() << "LV: Computing best VF using cost kind: "
5582 << (Config.CostKind == TTI::TCK_RecipThroughput
5583 ? "Reciprocal Throughput\n"
5584 : Config.CostKind == TTI::TCK_Latency
5585 ? "Instruction Latency\n"
5586 : Config.CostKind == TTI::TCK_CodeSize ? "Code Size\n"
5587 : Config.CostKind == TTI::TCK_SizeAndLatency
5588 ? "Code Size and Latency\n"
5589 : "Unknown\n"));
5590
5591 ElementCount ScalarVF = ElementCount::getFixed(MinVal: 1);
5592 assert(FirstPlan.hasVF(ScalarVF) &&
5593 "More than a single plan/VF w/o any plan having scalar VF");
5594
5595 // TODO: Compute scalar cost using VPlan-based cost model.
5596 InstructionCost ScalarCost = CM->expectedCost(VF: ScalarVF);
5597 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
5598 VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
5599 VectorizationFactor BestFactor = ScalarFactor;
5600
5601 bool ForceVectorization =
5602 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
5603 if (ForceVectorization) {
5604 // Ignore scalar width, because the user explicitly wants vectorization.
5605 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
5606 // evaluation.
5607 BestFactor.Cost = InstructionCost::getMax();
5608 }
5609
5610 VPlan *PlanForBestVF = &FirstPlan;
5611
5612 for (auto &P : VPlans) {
5613 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
5614 P->vectorFactors().end());
5615
5616 SmallVector<VPRegisterUsage, 8> RUs;
5617 bool ConsiderRegPressure = any_of(Range&: VFs, P: [this](ElementCount VF) {
5618 return Config.shouldConsiderRegPressureForVF(VF);
5619 });
5620 if (ConsiderRegPressure)
5621 RUs = calculateRegisterUsageForPlan(Plan&: *P, VFs, TTI);
5622
5623 for (unsigned I = 0; I < VFs.size(); I++) {
5624 ElementCount VF = VFs[I];
5625 if (VF.isScalar())
5626 continue;
5627 if (!ForceVectorization && !willGenerateVectors(Plan&: *P, VF, TTI)) {
5628 LLVM_DEBUG(
5629 dbgs()
5630 << "LV: Not considering vector loop of width " << VF
5631 << " because it will not generate any vector instructions.\n");
5632 continue;
5633 }
5634 if (Config.OptForSize && !ForceVectorization && hasReplicatorRegion(Plan&: *P)) {
5635 LLVM_DEBUG(
5636 dbgs()
5637 << "LV: Not considering vector loop of width " << VF
5638 << " because it would cause replicated blocks to be generated,"
5639 << " which isn't allowed when optimizing for size.\n");
5640 continue;
5641 }
5642
5643 InstructionCost Cost =
5644 cost(Plan&: *P, VF, RU: ConsiderRegPressure ? &RUs[I] : nullptr);
5645 VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
5646
5647 if (isMoreProfitable(A: CurrentFactor, B: BestFactor, HasTail: P->hasScalarTail())) {
5648 BestFactor = CurrentFactor;
5649 PlanForBestVF = P.get();
5650 }
5651
5652 // If profitable add it to ProfitableVF list.
5653 if (isMoreProfitable(A: CurrentFactor, B: ScalarFactor, HasTail: P->hasScalarTail()))
5654 ProfitableVFs.push_back(Elt: CurrentFactor);
5655 }
5656 }
5657
5658 VPlan &BestPlan = *PlanForBestVF;
5659
5660 assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
5661 "when vectorizing, the scalar cost must be computed.");
5662
5663 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << BestFactor.Width << ".\n");
5664 return {BestFactor, &BestPlan};
5665}
5666
5667LoopVectorizationPlanner::LoopVectorizationPlanner(
5668 Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
5669 const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
5670 std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
5671 InterleavedAccessInfo &IAI, PredicatedScalarEvolution &PSE,
5672 OptimizationRemarkEmitter *ORE,
5673 std::function<const BranchProbabilityInfo &()> GetBPI)
5674 : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal),
5675 CM(std::move(CM)), Config(Config), IAI(IAI), PSE(PSE), ORE(ORE),
5676 GetBPI(GetBPI) {}
5677
5678LoopVectorizationPlanner::~LoopVectorizationPlanner() = default;
5679
5680void LoopVectorizationPlanner::clearCostModel() { CM.reset(); }
5681
5682DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
5683 ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
5684 InnerLoopVectorizer &ILV, DominatorTree *DT,
5685 EpilogueVectorizationKind EpilogueVecKind) {
5686 assert(BestVPlan.hasVF(BestVF) &&
5687 "Trying to execute plan with unsupported VF");
5688 assert(BestVPlan.hasUF(BestUF) &&
5689 "Trying to execute plan with unsupported UF");
5690 if (BestVPlan.hasEarlyExit())
5691 ++LoopsEarlyExitVectorized;
5692
5693 RUN_VPLAN_PASS(VPlanTransforms::replaceWideCanonicalIVWithWideIV, BestVPlan,
5694 *PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF);
5695 // TODO: Move to VPlan transform stage once the transition to the VPlan-based
5696 // cost model is complete for better cost estimates.
5697 RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
5698 RUN_VPLAN_PASS(VPlanTransforms::materializePacksAndUnpacks, BestVPlan);
5699 RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
5700 RUN_VPLAN_PASS(VPlanTransforms::replicateByVF, BestVPlan, BestVF);
5701 bool HasBranchWeights =
5702 hasBranchWeightMD(I: *OrigLoop->getLoopLatch()->getTerminator());
5703 if (HasBranchWeights) {
5704 std::optional<unsigned> VScale = Config.getVScaleForTuning();
5705 RUN_VPLAN_PASS(VPlanTransforms::addBranchWeightToMiddleTerminator,
5706 BestVPlan, BestVF, VScale);
5707 }
5708
5709 if (vputils::findIncomingAliasMask(Plan: BestVPlan)) {
5710 assert(BestVPlan.hasTailFolded() && "Expected tail folding to be enabled");
5711 RUN_VPLAN_PASS(VPlanTransforms::materializeAliasMaskCheckBlock, BestVPlan,
5712 *Legal->getRuntimePointerChecking()->getDiffChecks(),
5713 HasBranchWeights);
5714 ++LoopsPartialAliasVectorized;
5715 }
5716
5717 // Retrieving VectorPH now when it's easier while VPlan still has Regions.
5718 VPBasicBlock *VectorPH = cast<VPBasicBlock>(Val: BestVPlan.getVectorPreheader());
5719
5720 RUN_VPLAN_PASS(VPlanTransforms::materializeConstantVectorTripCount, BestVPlan,
5721 BestVF, BestUF, PSE);
5722 RUN_VPLAN_PASS(VPlanTransforms::optimizeForVFAndUF, BestVPlan, BestVF, BestUF,
5723 PSE);
5724 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
5725 // Check if scalar epilogue is required, before simplifying constant branches.
5726 const bool RequiresScalarEpilogue = BestVPlan.requiresScalarEpilogue();
5727 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5728 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5729 /*OnlyLatches=*/false);
5730 if (BestVPlan.getEntry()->getSingleSuccessor() ==
5731 BestVPlan.getScalarPreheader()) {
5732 // TODO: The vector loop would be dead, should not even try to vectorize.
5733 ORE->emit(RemarkBuilder: [&]() {
5734 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationDead",
5735 OrigLoop->getStartLoc(),
5736 OrigLoop->getHeader())
5737 << "Created vector loop never executes due to insufficient trip "
5738 "count.";
5739 });
5740 return DenseMap<const SCEV *, Value *>();
5741 }
5742
5743 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, BestVPlan);
5744
5745 RUN_VPLAN_PASS(VPlanTransforms::convertToConcreteRecipes, BestVPlan);
5746 // Convert the exit condition to AVLNext == 0 for EVL tail folded loops.
5747 RUN_VPLAN_PASS(VPlanTransforms::convertEVLExitCond, BestVPlan);
5748 // Regions are dissolved after optimizing for VF and UF, which completely
5749 // removes unneeded loop regions first.
5750 const bool HasTailFolded = BestVPlan.hasTailFolded();
5751 RUN_VPLAN_PASS(VPlanTransforms::dissolveLoopRegions, BestVPlan);
5752 // Expand BranchOnTwoConds after dissolution, when latch has direct access to
5753 // its successors.
5754 RUN_VPLAN_PASS(VPlanTransforms::expandBranchOnTwoConds, BestVPlan);
5755 // Convert loops with variable-length stepping after regions are dissolved.
5756 RUN_VPLAN_PASS(VPlanTransforms::convertToVariableLengthStep, BestVPlan);
5757 // Remove dead back-edges for single-iteration loops with BranchOnCond(true).
5758 // Only process loop latches to avoid removing edges from the middle block,
5759 // which may be needed for epilogue vectorization.
5760 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5761 /*OnlyLatches=*/true);
5762 RUN_VPLAN_PASS(VPlanTransforms::materializeBackedgeTakenCount, BestVPlan,
5763 VectorPH);
5764 std::optional<uint64_t> MaxRuntimeStep = getMaxRuntimeElementCount(
5765 EC: BestVF * BestUF, F: *OrigLoop->getHeader()->getParent());
5766
5767 assert((LI->getUniqueLatchExitBlock(*OrigLoop) || RequiresScalarEpilogue) &&
5768 "loops not exiting via the latch without required epilogue?");
5769 RUN_VPLAN_PASS(VPlanTransforms::materializeVectorTripCount, BestVPlan,
5770 VectorPH, HasTailFolded, RequiresScalarEpilogue,
5771 &BestVPlan.getVFxUF(), MaxRuntimeStep);
5772 RUN_VPLAN_PASS(VPlanTransforms::materializeFactors, BestVPlan, VectorPH,
5773 BestVF);
5774 // Limit expansions to VPInstruction to when not vectorizing the epilogue.
5775 // Currently this code path still relies on code re-using SCEVs expanded
5776 // directly to IR instructions.
5777 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5778 RUN_VPLAN_PASS(VPlanTransforms::expandSCEVsToVPInstructions, BestVPlan,
5779 *PSE.getSE());
5780 RUN_VPLAN_PASS(VPlanTransforms::cse, BestVPlan);
5781 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
5782 // Removing branches and incoming values may expose additional simplification
5783 // opportunities.
5784 if (RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5785 /*OnlyLatches=*/EpilogueVecKind !=
5786 EpilogueVectorizationKind::None))
5787 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
5788 RUN_VPLAN_PASS(VPlanTransforms::simplifyKnownEVL, BestVPlan, BestVF, PSE);
5789
5790 // 0. Generate SCEV-dependent code in the entry, including TripCount, before
5791 // making any changes to the CFG.
5792 DenseMap<const SCEV *, Value *> ExpandedSCEVs =
5793 RUN_VPLAN_PASS(VPlanTransforms::expandSCEVs, BestVPlan, *PSE.getSE());
5794
5795 // Perform the actual loop transformation.
5796 VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
5797 OrigLoop->getParentLoop());
5798
5799#ifdef EXPENSIVE_CHECKS
5800 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
5801#endif
5802
5803 // 1. Set up the skeleton for vectorization, including vector pre-header and
5804 // middle block. The vector loop is created during VPlan execution.
5805 State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
5806 if (VPBasicBlock *ScalarPH = BestVPlan.getScalarPreheader())
5807 replaceVPBBWithIRVPBB(VPBB: ScalarPH, IRBB: State.CFG.PrevBB->getSingleSuccessor(),
5808 Plan: &BestVPlan);
5809 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, BestVPlan);
5810
5811 assert(verifyVPlanIsValid(BestVPlan) && "final VPlan is invalid");
5812
5813 // After vectorization, the exit blocks of the original loop will have
5814 // additional predecessors. Invalidate SCEVs for the exit phis in case SE
5815 // looked through single-entry phis.
5816 ScalarEvolution &SE = *PSE.getSE();
5817 for (VPIRBasicBlock *Exit : BestVPlan.getExitBlocks()) {
5818 if (!Exit->hasPredecessors())
5819 continue;
5820 for (VPRecipeBase &PhiR : Exit->phis())
5821 SE.forgetLcssaPhiWithNewPredecessor(L: OrigLoop,
5822 V: &cast<VPIRPhi>(Val&: PhiR).getIRPhi());
5823 }
5824
5825 // Query whether the target wants loops it vectorizes to remain eligible for
5826 // runtime unrolling. Do this here, on the original loop and before its SCEV
5827 // is forgotten below.
5828 TargetTransformInfo::UnrollingPreferences UP;
5829 TTI.getUnrollingPreferences(L: OrigLoop, SE, UP, ORE);
5830 bool UnrollVectorizedLoop = UP.UnrollVectorizedLoop;
5831
5832 // Forget the original loop and block dispositions.
5833 SE.forgetLoop(L: OrigLoop);
5834 SE.forgetBlockAndLoopDispositions();
5835
5836 ILV.printDebugTracesAtStart();
5837
5838 //===------------------------------------------------===//
5839 //
5840 // Notice: any optimization or new instruction that go
5841 // into the code below should also be implemented in
5842 // the cost-model.
5843 //
5844 //===------------------------------------------------===//
5845
5846 // Retrieve loop information before executing the plan, which may remove the
5847 // original loop, if it becomes unreachable.
5848 MDNode *LID = OrigLoop->getLoopID();
5849 unsigned OrigLoopInvocationWeight = 0;
5850 std::optional<unsigned> OrigAverageTripCount =
5851 getLoopEstimatedTripCount(L: OrigLoop, EstimatedLoopInvocationWeight: &OrigLoopInvocationWeight);
5852
5853 BestVPlan.execute(State: &State);
5854
5855 // 2.6. Maintain Loop Hints
5856 // Keep all loop hints from the original loop on the vector loop (we'll
5857 // replace the vectorizer-specific hints below).
5858 VPBasicBlock *HeaderVPBB = vputils::getFirstLoopHeader(Plan&: BestVPlan, VPDT&: State.VPDT);
5859 // Add metadata to disable runtime unrolling a scalar loop when there
5860 // are no runtime checks about strides and memory. A scalar loop that is
5861 // rarely used is not worth unrolling.
5862 bool DisableRuntimeUnroll = !ILV.RTChecks.hasChecks() && !BestVF.isScalar();
5863 updateLoopMetadataAndProfileInfo(
5864 VectorLoop: HeaderVPBB ? LI->getLoopFor(BB: State.CFG.VPBB2IRBB.lookup(Val: HeaderVPBB))
5865 : nullptr,
5866 HeaderVPBB, Plan: BestVPlan,
5867 VectorizingEpilogue: EpilogueVecKind == EpilogueVectorizationKind::Epilogue, OrigLoopID: LID,
5868 OrigAverageTripCount, OrigLoopInvocationWeight,
5869 EstimatedVFxUF: estimateElementCount(VF: BestVF * BestUF, VScale: Config.getVScaleForTuning()),
5870 DisableRuntimeUnroll, UnrollVectorizedLoop);
5871
5872 // 3. Fix the vectorized code: take care of header phi's, live-outs,
5873 // predication, updating analyses.
5874 ILV.fixVectorizedLoop(State);
5875
5876 ILV.printDebugTracesAtEnd();
5877
5878 return ExpandedSCEVs;
5879}
5880
5881//===--------------------------------------------------------------------===//
5882// EpilogueVectorizerMainLoop
5883//===--------------------------------------------------------------------===//
5884
5885void EpilogueVectorizerMainLoop::printDebugTracesAtStart() {
5886 LLVM_DEBUG({
5887 dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
5888 << "Main Loop VF:" << EPI.MainLoopVF
5889 << ", Main Loop UF:" << EPI.MainLoopUF
5890 << ", Epilogue Loop VF:" << EPI.EpilogueVF
5891 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
5892 });
5893}
5894
5895void EpilogueVectorizerMainLoop::printDebugTracesAtEnd() {
5896 DEBUG_WITH_TYPE(VerboseDebug, {
5897 dbgs() << "intermediate fn:\n"
5898 << *OrigLoop->getHeader()->getParent() << "\n";
5899 });
5900}
5901
5902//===--------------------------------------------------------------------===//
5903// EpilogueVectorizerEpilogueLoop
5904//===--------------------------------------------------------------------===//
5905
5906/// This function creates a new scalar preheader, using the previous one as
5907/// entry block to the epilogue VPlan. The minimum iteration check is being
5908/// represented in VPlan.
5909BasicBlock *EpilogueVectorizerEpilogueLoop::createVectorizedLoopSkeleton() {
5910 BasicBlock *NewScalarPH = createScalarPreheader(Prefix: "vec.epilog.");
5911 BasicBlock *OriginalScalarPH = NewScalarPH->getSinglePredecessor();
5912 OriginalScalarPH->setName("vec.epilog.iter.check");
5913 VPIRBasicBlock *NewEntry = Plan.createVPIRBasicBlock(IRBB: OriginalScalarPH);
5914 VPBasicBlock *OldEntry = Plan.getEntry();
5915 for (auto &R : make_early_inc_range(Range&: *OldEntry)) {
5916 // Skip moving VPIRInstructions (including VPIRPhis), which are unmovable by
5917 // defining.
5918 if (isa<VPIRInstruction>(Val: &R))
5919 continue;
5920 R.moveBefore(BB&: *NewEntry, I: NewEntry->end());
5921 }
5922
5923 VPBlockUtils::reassociateBlocks(Old: OldEntry, New: NewEntry);
5924 Plan.setEntry(NewEntry);
5925 // OldEntry is now dead and will be cleaned up when the plan gets destroyed.
5926
5927 return OriginalScalarPH;
5928}
5929
5930void EpilogueVectorizerEpilogueLoop::printDebugTracesAtStart() {
5931 LLVM_DEBUG({
5932 dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
5933 << "Epilogue Loop VF:" << EPI.EpilogueVF
5934 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
5935 });
5936}
5937
5938void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
5939 DEBUG_WITH_TYPE(VerboseDebug, {
5940 dbgs() << "final fn:\n" << *OrigLoop->getHeader()->getParent() << "\n";
5941 });
5942}
5943
5944bool VPRecipeBuilder::isPredicatedInst(Instruction *I) const {
5945 return CM.isPredicatedInst(I);
5946}
5947
5948bool VPRecipeBuilder::prefersVectorizedAddressing() const {
5949 return CM.TTI.prefersVectorizedAddressing();
5950}
5951
5952VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
5953 VFRange &Range) {
5954 assert((VPI->getOpcode() == Instruction::Load ||
5955 VPI->getOpcode() == Instruction::Store) &&
5956 "Must be called with either a load or store");
5957 Instruction *I = VPI->getUnderlyingInstr();
5958
5959 auto WillWiden = [&](ElementCount VF) -> bool {
5960 LoopVectorizationCostModel::InstWidening Decision =
5961 CM.getWideningDecision(I, VF);
5962 assert(Decision != LoopVectorizationCostModel::CM_Unknown &&
5963 "CM decision should be taken at this point.");
5964 if (Decision == LoopVectorizationCostModel::CM_Interleave)
5965 return true;
5966 if (CM.isScalarAfterVectorization(I, VF) ||
5967 CM.isProfitableToScalarize(I, VF))
5968 return false;
5969 return Decision != LoopVectorizationCostModel::CM_Scalarize;
5970 };
5971
5972 if (!LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: WillWiden, Range))
5973 return nullptr;
5974
5975 // If a mask is not required, drop it - use unmasked version for safe loads.
5976 // TODO: Determine if mask is needed in VPlan.
5977 VPValue *Mask = CM.isMaskRequired(I) ? VPI->getMask() : nullptr;
5978
5979 // Determine if the pointer operand of the access is either consecutive or
5980 // reverse consecutive.
5981 LoopVectorizationCostModel::InstWidening Decision =
5982 CM.getWideningDecision(I, VF: Range.Start);
5983 bool Reverse = Decision == LoopVectorizationCostModel::CM_Widen_Reverse;
5984 bool Consecutive =
5985 Reverse || Decision == LoopVectorizationCostModel::CM_Widen;
5986
5987 VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(N: 0)
5988 : VPI->getOperand(N: 1);
5989 Builder.setInsertPoint(VPI);
5990 if (Consecutive) {
5991 Ptr = Builder.createConsecutiveVectorPointer(Ptr, SourceElementTy: getLoadStoreType(I),
5992 Reverse, DL: VPI->getDebugLoc());
5993 }
5994
5995 if (Reverse && Mask)
5996 Mask = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Mask, DL: I->getDebugLoc());
5997
5998 if (VPI->getOpcode() == Instruction::Load) {
5999 auto *Load = cast<LoadInst>(Val: I);
6000 auto *LoadR = Builder.createWidenLoad(Load&: *Load, Addr: Ptr, Mask, Consecutive, Metadata: *VPI,
6001 DL: Load->getDebugLoc());
6002 if (Reverse)
6003 return Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: LoadR,
6004 DL: LoadR->getDebugLoc());
6005 return LoadR;
6006 }
6007
6008 StoreInst *Store = cast<StoreInst>(Val: I);
6009 VPValue *StoredVal = VPI->getOperand(N: 0);
6010 if (Reverse)
6011 StoredVal = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: StoredVal,
6012 DL: Store->getDebugLoc());
6013 return Builder.createWidenStore(Store&: *Store, Addr: Ptr, StoredVal, Mask, Consecutive,
6014 Metadata: *VPI, DL: Store->getDebugLoc());
6015}
6016
6017VPWidenIntOrFpInductionRecipe *
6018VPRecipeBuilder::tryToOptimizeInductionTruncate(VPInstruction *VPI,
6019 VFRange &Range) {
6020 auto *I = cast<TruncInst>(Val: VPI->getUnderlyingInstr());
6021 // Optimize the special case where the source is a constant integer
6022 // induction variable. Notice that we can only optimize the 'trunc' case
6023 // because (a) FP conversions lose precision, (b) sext/zext may wrap, and
6024 // (c) other casts depend on pointer size.
6025
6026 // Determine whether \p K is a truncation based on an induction variable that
6027 // can be optimized.
6028 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
6029 Predicate: bind_front(Fn: &LoopVectorizationCostModel::isOptimizableIVTruncate, BindArgs&: CM,
6030 BindArgs&: I),
6031 Range))
6032 return nullptr;
6033
6034 auto *WidenIV = cast<VPWidenIntOrFpInductionRecipe>(
6035 Val: VPI->getOperand(N: 0)->getDefiningRecipe());
6036 PHINode *Phi = WidenIV->getPHINode();
6037 VPValue *Start = WidenIV->getStartValue();
6038 const InductionDescriptor &IndDesc = WidenIV->getInductionDescriptor();
6039
6040 // Wrap flags from the original induction do not apply to the truncated type,
6041 // so do not propagate them.
6042 VPIRFlags Flags = VPIRFlags::WrapFlagsTy(false, false);
6043 VPValue *Step =
6044 vputils::getOrCreateVPValueForSCEVExpr(Plan, Expr: IndDesc.getStep());
6045 return new VPWidenIntOrFpInductionRecipe(
6046 Phi, Start, Step, &Plan.getVF(), IndDesc, I, Flags, VPI->getDebugLoc());
6047}
6048
6049bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
6050 assert((!isa<UncondBrInst, CondBrInst, PHINode, LoadInst, StoreInst>(I)) &&
6051 "Instruction should have been handled earlier");
6052 // Instruction should be widened, unless it is scalar after vectorization,
6053 // scalarization is profitable or it is predicated.
6054 auto WillScalarize = [this, I](ElementCount VF) -> bool {
6055 return CM.isScalarAfterVectorization(I, VF) ||
6056 CM.isProfitableToScalarize(I, VF) ||
6057 CM.isScalarWithPredication(I, VF);
6058 };
6059 return !LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: WillScalarize,
6060 Range);
6061}
6062
6063VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
6064 auto *I = VPI->getUnderlyingInstr();
6065 switch (VPI->getOpcode()) {
6066 default:
6067 return nullptr;
6068 case Instruction::SDiv:
6069 case Instruction::UDiv:
6070 case Instruction::SRem:
6071 case Instruction::URem:
6072 // If not provably safe, use a masked intrinsic.
6073 if (CM.isPredicatedInst(I))
6074 return new VPWidenIntrinsicRecipe(
6075 getMaskedDivRemIntrinsic(Opcode: VPI->getOpcode()), VPI->operands(),
6076 I->getType(), {}, {}, VPI->getDebugLoc());
6077 [[fallthrough]];
6078 case Instruction::Add:
6079 case Instruction::And:
6080 case Instruction::AShr:
6081 case Instruction::FAdd:
6082 case Instruction::FCmp:
6083 case Instruction::FDiv:
6084 case Instruction::FMul:
6085 case Instruction::FNeg:
6086 case Instruction::FRem:
6087 case Instruction::FSub:
6088 case Instruction::ICmp:
6089 case Instruction::LShr:
6090 case Instruction::Mul:
6091 case Instruction::Or:
6092 case Instruction::Select:
6093 case Instruction::Shl:
6094 case Instruction::Sub:
6095 case Instruction::Xor:
6096 case Instruction::Freeze:
6097 return new VPWidenRecipe(*I, VPI->operandsWithoutMask(), *VPI, *VPI,
6098 VPI->getDebugLoc());
6099 case Instruction::ExtractValue: {
6100 SmallVector<VPValue *> NewOps(VPI->operandsWithoutMask());
6101 auto *EVI = cast<ExtractValueInst>(Val: I);
6102 assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
6103 unsigned Idx = EVI->getIndices()[0];
6104 NewOps.push_back(Elt: Plan.getConstantInt(BitWidth: 32, Val: Idx));
6105 return new VPWidenRecipe(*I, NewOps, *VPI, *VPI, VPI->getDebugLoc());
6106 }
6107 };
6108}
6109
6110VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
6111 if (VPI->getOpcode() != Instruction::Store)
6112 return nullptr;
6113
6114 auto HistInfo =
6115 Legal->getHistogramInfo(I: cast<StoreInst>(Val: VPI->getUnderlyingInstr()));
6116 if (!HistInfo)
6117 return nullptr;
6118
6119 const HistogramInfo *HI = *HistInfo;
6120 // FIXME: Support other operations.
6121 unsigned Opcode = HI->Update->getOpcode();
6122 assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
6123 "Histogram update operation must be an Add or Sub");
6124
6125 SmallVector<VPValue *, 3> HGramOps;
6126 // Bucket address.
6127 HGramOps.push_back(Elt: VPI->getOperand(N: 1));
6128 // Increment value.
6129 HGramOps.push_back(Elt: Plan.getOrAddLiveIn(V: HI->Update->getOperand(i: 1)));
6130
6131 // In case of predicated execution (due to tail-folding, or conditional
6132 // execution, or both), pass the relevant mask.
6133 if (CM.isMaskRequired(I: HI->Store))
6134 HGramOps.push_back(Elt: VPI->getMask());
6135
6136 return new VPHistogramRecipe(Opcode, HGramOps, cast<VPIRMetadata>(Val&: *VPI),
6137 VPI->getDebugLoc());
6138}
6139
6140bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
6141 VPInstruction *VPI, VPBuilder &FinalRedStoresBuilder) {
6142 StoreInst *SI;
6143 if ((SI = dyn_cast<StoreInst>(Val: VPI->getUnderlyingInstr())) &&
6144 Legal->isInvariantAddressOfReduction(V: SI->getPointerOperand())) {
6145 // Only create recipe for the final invariant store of the reduction.
6146 if (Legal->isInvariantStoreOfReduction(SI)) {
6147 VPValue *Val = VPI->getOperand(N: 0);
6148 VPValue *Addr = VPI->getOperand(N: 1);
6149 // We need to store the exiting value of the reduction, so use the blend
6150 // if tail folded.
6151 if (auto *Blend = VPlanPatternMatch::findUserOf<VPBlendRecipe>(V: Val))
6152 Val = Blend;
6153 [[maybe_unused]] auto *Rdx =
6154 VPlanPatternMatch::findUserOf<VPReductionPHIRecipe>(V: Val);
6155 assert((isa<VPIRValue>(Val) || !Rdx || Rdx->getBackedgeValue() == Val) &&
6156 "Store of reduction thats not the backedge value?");
6157 auto *Recipe = new VPReplicateRecipe(
6158 SI, {Val, Addr}, true /* IsUniform */, nullptr /*Mask*/, *VPI, *VPI,
6159 VPI->getDebugLoc());
6160 FinalRedStoresBuilder.insert(R: Recipe);
6161 }
6162 VPI->eraseFromParent();
6163 return true;
6164 }
6165
6166 return false;
6167}
6168
6169VPSingleDefRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
6170 VFRange &Range) {
6171 auto *I = VPI->getUnderlyingInstr();
6172 bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
6173 Predicate: [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
6174 Range);
6175
6176 bool IsPredicated = CM.isPredicatedInst(I);
6177
6178 // Even if the instruction is not marked as uniform, there are certain
6179 // intrinsic calls that can be effectively treated as such, so we check for
6180 // them here. Conservatively, we only do this for scalable vectors, since
6181 // for fixed-width VFs we can always fall back on full scalarization.
6182 if (!IsUniform && Range.Start.isScalable() && isa<IntrinsicInst>(Val: I)) {
6183 switch (cast<IntrinsicInst>(Val: I)->getIntrinsicID()) {
6184 case Intrinsic::assume:
6185 case Intrinsic::lifetime_start:
6186 case Intrinsic::lifetime_end:
6187 // For scalable vectors if one of the operands is variant then we still
6188 // want to mark as uniform, which will generate one instruction for just
6189 // the first lane of the vector. We can't scalarize the call in the same
6190 // way as for fixed-width vectors because we don't know how many lanes
6191 // there are.
6192 //
6193 // The reasons for doing it this way for scalable vectors are:
6194 // 1. For the assume intrinsic generating the instruction for the first
6195 // lane is still be better than not generating any at all. For
6196 // example, the input may be a splat across all lanes.
6197 // 2. For the lifetime start/end intrinsics the pointer operand only
6198 // does anything useful when the input comes from a stack object,
6199 // which suggests it should always be uniform. For non-stack objects
6200 // the effect is to poison the object, which still allows us to
6201 // remove the call.
6202 IsUniform = true;
6203 break;
6204 default:
6205 break;
6206 }
6207 }
6208 VPValue *BlockInMask = nullptr;
6209 if (!IsPredicated) {
6210 // Finalize the recipe for Instr, first if it is not predicated.
6211 LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n");
6212 } else {
6213 LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n");
6214 // Instructions marked for predication are replicated and a mask operand is
6215 // added initially. Masked replicate recipes will later be placed under an
6216 // if-then construct to prevent side-effects. Generate recipes to compute
6217 // the block mask for this region.
6218 BlockInMask = VPI->getMask();
6219 }
6220
6221 // Note that there is some custom logic to mark some intrinsics as uniform
6222 // manually above for scalable vectors, which this assert needs to account for
6223 // as well.
6224 assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
6225 (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
6226 "Should not predicate a uniform recipe");
6227 if (IsUniform) {
6228 return VPBuilder::createSingleScalarOp(
6229 Opcode: VPI->getOpcode(), Operands: VPI->operandsWithoutMask(), Mask: BlockInMask, Flags: *VPI, Metadata: *VPI,
6230 DL: VPI->getDebugLoc(), UV: I);
6231 }
6232 auto *Recipe = new VPReplicateRecipe(I, VPI->operandsWithoutMask(),
6233 /*IsSingleScalar=*/false, BlockInMask,
6234 *VPI, *VPI, VPI->getDebugLoc());
6235 return Recipe;
6236}
6237
6238VPRecipeBase *
6239VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
6240 VFRange &Range) {
6241 assert(!R->isPhi() && "phis must be handled earlier");
6242 // First, check for specific widening recipes that deal with optimizing
6243 // truncates and memory operations.
6244 auto *VPI = cast<VPInstruction>(Val: R);
6245 assert(VPI->getOpcode() != Instruction::Call &&
6246 "Call should have been handled by makeCallWideningDecisions");
6247
6248 VPRecipeBase *Recipe;
6249 if (VPI->getOpcode() == Instruction::Trunc &&
6250 (Recipe = tryToOptimizeInductionTruncate(VPI, Range)))
6251 return Recipe;
6252
6253 // All widen recipes below deal only with VF > 1.
6254 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6255 Predicate: [&](ElementCount VF) { return VF.isScalar(); }, Range))
6256 return nullptr;
6257
6258 Instruction *Instr = R->getUnderlyingInstr();
6259 assert(!is_contained({Instruction::Load, Instruction::Store},
6260 VPI->getOpcode()) &&
6261 "Should have been handled prior to this!");
6262
6263 // We can only replicate an extractvalue if its operand generates per lane in
6264 // the same block, otherwise we would need to extract a lane from its struct
6265 // operand which is invalid.
6266 if (VPI->getOpcode() == Instruction::ExtractValue &&
6267 !vputils::isSingleScalar(VPV: VPI->getOperand(N: 0)))
6268 if (VPRecipeBase *OpR = VPI->getOperand(N: 0)->getDefiningRecipe())
6269 if (!vputils::doesGeneratePerAllLanes(R: OpR) ||
6270 OpR->getParent() != VPI->getParent())
6271 return tryToWiden(VPI);
6272
6273 if (!shouldWiden(I: Instr, Range))
6274 return nullptr;
6275
6276 if (VPI->getOpcode() == Instruction::GetElementPtr) {
6277 auto *GEP = cast<GetElementPtrInst>(Val: Instr);
6278 return new VPWidenGEPRecipe(GEP->getSourceElementType(),
6279 VPI->operandsWithoutMask(), *VPI,
6280 VPI->getDebugLoc(), GEP);
6281 }
6282
6283 if (Instruction::isCast(Opcode: VPI->getOpcode())) {
6284 auto *CI = cast<CastInst>(Val: Instr);
6285 return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(N: 0),
6286 VPI->getScalarType(), CI, *VPI, *VPI,
6287 VPI->getDebugLoc());
6288 }
6289
6290 return tryToWiden(VPI);
6291}
6292
6293// To allow RUN_VPLAN_PASS to print the VPlan after VF/UF independent
6294// optimizations.
6295static void printOptimizedVPlan(VPlan &) {}
6296
6297#ifndef NDEBUG
6298/// Cross-check the execution frequencies recorded in \p Plan against
6299/// BlockFrequencyInfo for the blocks of \p OrigLoop.
6300/// FIXME: Temporary verification aid, to be removed.
6301static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
6302 LoopInfo *LI,
6303 LoopVectorizationCostModel &CM) {
6304 // Limited to loops with the latch as only exiting block
6305 if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
6306 return true;
6307
6308 // Visit the loop body in the same order as recordExecutionFrequencies. Both
6309 // are reverse post-orders of the same CFG, so indices correspond.
6310 VPBasicBlock *Header = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan).first;
6311 SmallVector<VPBasicBlock *> Blocks = vp_rpo_plain_cfg_loop_body(Header);
6312 assert(Blocks.size() == OrigLoop->getNumBlocks() &&
6313 "loop body and original loop must have the same blocks");
6314
6315 LoopBlocksRPO OrigRPO(OrigLoop);
6316 OrigRPO.perform(LI);
6317
6318 // Only request the expensive BFI once the cheap bail-outs are past.
6319 BlockFrequencyInfo &BFI = CM.getBFI();
6320 uint64_t HeaderFreq = BFI.getBlockFreq(OrigLoop->getHeader()).getFrequency();
6321 if (HeaderFreq == 0)
6322 return true;
6323
6324 // BFI's fixed-point mass propagation loses up to 1 ULP per edge, so bound the
6325 // error by the number of edges in the region.
6326 uint64_t Edges = 0;
6327 for (const VPBasicBlock *VPBB : Blocks)
6328 Edges += VPBB->getNumSuccessors();
6329 uint64_t Tolerance = Edges + BranchProbability::getDenominator() / HeaderFreq;
6330
6331 for (const auto &[VPBB, BB] :
6332 zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
6333 // All recipes of a block share the same recorded frequency; empty blocks
6334 // and blocks that always or never execute carry none.
6335 if (VPBB->empty())
6336 continue;
6337 std::optional<VPExecutionFrequency> Freq =
6338 cast<VPInstruction>(&VPBB->front())->getExecutionFrequency();
6339 if (!Freq)
6340 continue;
6341 BranchProbability Computed = vputils::getExecutionProbability(Freq->Freq);
6342
6343 // Clamp to the header's frequency, which BFI's rounding may exceed.
6344 uint64_t BBFreq = BFI.getBlockFreq(BB).getFrequency();
6345 BranchProbability Expected = BranchProbability::getBranchProbability(
6346 std::min(BBFreq, HeaderFreq), HeaderFreq);
6347 if (AbsoluteDifference(Computed.getNumerator(), Expected.getNumerator()) <=
6348 Tolerance)
6349 continue;
6350
6351 errs() << "Block frequency mismatch for " << VPBB->getName() << ": VPlan "
6352 << Computed << ", BlockFrequencyInfo " << Expected << "\n";
6353 return false;
6354 }
6355 return true;
6356}
6357#endif
6358
6359VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
6360 bool IsInnerLoop = OrigLoop->isInnermost();
6361
6362 // Set up loop versioning for inner loops with memory runtime checks.
6363 // Outer loops don't have LoopAccessInfo since canVectorizeMemory() is not
6364 // called for them.
6365 std::optional<LoopVersioning> LVer;
6366 if (IsInnerLoop) {
6367 const LoopAccessInfo *LAI = Legal->getLAI();
6368 LVer.emplace(args: *LAI, args: LAI->getRuntimePointerChecking()->getChecks(), args&: OrigLoop,
6369 args&: LI, args&: DT, args: PSE.getSE());
6370 if (!LAI->getRuntimePointerChecking()->getChecks().empty() &&
6371 !LAI->getRuntimePointerChecking()->getDiffChecks()) {
6372 // Only use noalias metadata when using memory checks guaranteeing no
6373 // overlap across all iterations.
6374 LVer->prepareNoAliasMetadata();
6375 }
6376 }
6377
6378 // Create initial base VPlan0, to serve as common starting point for all
6379 // candidates built later for specific VF ranges.
6380 auto VPlan0 = VPlanTransforms::buildVPlan0(
6381 TheLoop: OrigLoop, LI&: *LI, InductionTy: Legal->getWidestInductionType(), PSE,
6382 LVer: LVer ? &*LVer : nullptr, GetBPI);
6383
6384 VPDominatorTree VPDT(*VPlan0);
6385 if (const LoopAccessInfo *LAI = Legal->getLAI())
6386 RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
6387 LAI->getSymbolicStrides(), VPDT);
6388 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, *VPlan0);
6389 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
6390 if (IsInnerLoop) {
6391 RUN_VPLAN_PASS(VPlanTransforms::recordExecutionFrequencies, *VPlan0);
6392 assert(verifyExecutionFrequenciesMatchBFI(*VPlan0, OrigLoop, LI, *CM) &&
6393 "execution frequencies do not match the loop's block frequencies");
6394 }
6395
6396 // Create recipes for header phis. For outer loops, reductions, recurrences
6397 // and in-loop reductions are empty since legality doesn't detect them.
6398 if (!RUN_VPLAN_PASS(
6399 VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE, *OrigLoop,
6400 VPDT, Legal->getInductionVars(), Legal->getReductionVars(),
6401 Legal->getFixedOrderRecurrences(), Config.getInLoopReductions(),
6402 Config.getHints().allowReordering())) {
6403 return nullptr;
6404 }
6405
6406 if (const LoopAccessInfo *LAI = Legal->getLAI())
6407 RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
6408 LAI->getSymbolicStrides(), VPDT);
6409
6410 // Add surviving induction predicates to PSE and check constraints.
6411 bool ForceVectorization =
6412 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
6413 bool OptForSize =
6414 !ForceVectorization &&
6415 (CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
6416 CM->EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
6417 unsigned SCEVCheckThreshold = ForceVectorization
6418 ? PragmaVectorizeSCEVCheckThreshold
6419 : VectorizeSCEVCheckThreshold;
6420 if (!RUN_VPLAN_PASS(VPlanTransforms::finalizeSCEVPredicates, *VPlan0, PSE,
6421 OptForSize, SCEVCheckThreshold, ORE, OrigLoop))
6422 return nullptr;
6423
6424 RUN_VPLAN_PASS(VPlanTransforms::addMiddleCheck, *VPlan0);
6425
6426 // If we're vectorizing a loop with an uncountable exit, make sure that the
6427 // recipes are safe to handle.
6428 // TODO: Remove this once we can properly check the VPlan itself for both
6429 // the presence of an uncountable exit and the presence of stores in
6430 // the loop inside handleUncountableEarlyExits itself.
6431 if (Legal->hasUncountableEarlyExit()) {
6432 // TODO: Check target preference for style.
6433 UncountableExitStyle EEStyle =
6434 Legal->hasUncountableExitWithSideEffects()
6435 ? UncountableExitStyle::MaskedHandleExitInScalarLoop
6436 : UncountableExitStyle::ReadOnly;
6437 if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0,
6438 OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
6439 EEStyle))
6440 return nullptr;
6441 } else {
6442 RUN_VPLAN_PASS(VPlanTransforms::handleCountableEarlyExits, *VPlan0);
6443 }
6444
6445 RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
6446 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
6447 if (CM->foldTailByMasking())
6448 RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
6449
6450 RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
6451
6452 return VPlan0;
6453}
6454
6455void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
6456 ElementCount MaxVF) {
6457 if (ElementCount::isKnownGT(LHS: MinVF, RHS: MaxVF))
6458 return;
6459
6460 auto MaxVFTimes2 = MaxVF * 2;
6461 for (ElementCount VF = MinVF; ElementCount::isKnownLT(LHS: VF, RHS: MaxVFTimes2);) {
6462 VFRange SubRange = {VF, MaxVFTimes2};
6463 auto Plan =
6464 tryToBuildVPlan(InitialPlan: std::unique_ptr<VPlan>(VPlan1.duplicate()), Range&: SubRange);
6465 VF = SubRange.End;
6466
6467 if (!Plan)
6468 continue;
6469
6470 // Now optimize the initial VPlan.
6471 RUN_VPLAN_PASS(VPlanTransforms::hoistPredicatedLoads, *Plan, PSE, OrigLoop);
6472 RUN_VPLAN_PASS(VPlanTransforms::sinkPredicatedStores, *Plan, PSE, OrigLoop);
6473 RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
6474 Config.getMinimalBitwidths());
6475 RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
6476 // TODO: try to put addExplicitVectorLength close to addActiveLaneMask
6477 if (CM->foldTailWithEVL()) {
6478 RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
6479 Config.getMaxSafeElements());
6480 RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
6481 }
6482
6483 if (auto P =
6484 RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
6485 VPlans.push_back(Elt: std::move(P));
6486
6487 TailFoldingStyle Style = CM->getTailFoldingStyle();
6488 RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
6489 useActiveLaneMask(Style),
6490 useActiveLaneMaskForControlFlow(Style));
6491
6492 RUN_VPLAN_PASS_NO_VERIFY(printOptimizedVPlan, *Plan);
6493 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6494 VPlans.push_back(Elt: std::move(Plan));
6495 }
6496}
6497
6498VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
6499 VFRange &Range) {
6500
6501 // For outer loops, the plan only needs basic recipe conversion and induction
6502 // live-out optimization; the full inner-loop recipe building below does not
6503 // apply (no widening decisions, interleave groups, reductions, etc.).
6504 if (Plan->isOuterLoop()) {
6505 for (ElementCount VF : Range)
6506 Plan->addVF(VF);
6507 if (!RUN_VPLAN_PASS(VPlanTransforms::tryToConvertVPInstructionsToVPRecipes,
6508 *Plan, *TLI, PSE, OrigLoop))
6509 return nullptr;
6510 RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
6511 OrigLoop);
6512 return Plan;
6513 }
6514
6515 using namespace llvm::VPlanPatternMatch;
6516 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
6517
6518 // ---------------------------------------------------------------------------
6519 // Build initial VPlan: Scan the body of the loop in a topological order to
6520 // visit each basic block after having visited its predecessor basic blocks.
6521 // ---------------------------------------------------------------------------
6522
6523 bool RequiresScalarEpilogueCheck =
6524 LoopVectorizationPlanner::getDecisionAndClampRange(
6525 Predicate: [this](ElementCount VF) {
6526 return !CM->requiresScalarEpilogue(IsVectorizing: VF.isVector());
6527 },
6528 Range);
6529 // Update the branch in the middle block if a scalar epilogue is required.
6530 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6531 if (!RequiresScalarEpilogueCheck && MiddleVPBB->getNumSuccessors() == 2) {
6532 auto *BranchOnCond = cast<VPInstruction>(Val: MiddleVPBB->getTerminator());
6533 assert(MiddleVPBB->getSuccessors()[1] == Plan->getScalarPreheader() &&
6534 "second successor must be scalar preheader");
6535 BranchOnCond->setOperand(I: 0, New: Plan->getFalse());
6536 }
6537
6538 // Don't use getDecisionAndClampRange here, because we don't know the UF
6539 // so this function is better to be conservative, rather than to split
6540 // it up into different VPlans.
6541 // TODO: Consider using getDecisionAndClampRange here to split up VPlans.
6542 bool IVUpdateMayOverflow = false;
6543 for (ElementCount VF : Range)
6544 IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(Cost: CM.get(), VF);
6545
6546 TailFoldingStyle Style = CM->getTailFoldingStyle();
6547 // Use NUW for the induction increment if we proved that it won't overflow in
6548 // the vector loop or when not folding the tail. In the later case, we know
6549 // that the canonical induction increment will not overflow as the vector trip
6550 // count is >= increment and a multiple of the increment.
6551 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
6552 bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
6553 if (!HasNUW) {
6554 auto *IVInc =
6555 LoopRegion->getExitingBasicBlock()->getTerminator()->getOperand(N: 0);
6556 assert(match(IVInc,
6557 m_VPInstruction<Instruction::Add>(
6558 m_Specific(LoopRegion->getCanonicalIV()), m_VPValue())) &&
6559 "Did not find the canonical IV increment");
6560 LoopRegion->clearCanonicalIVNUW(Increment: cast<VPInstruction>(Val: IVInc));
6561 }
6562
6563 // ---------------------------------------------------------------------------
6564 // Pre-construction: record ingredients whose recipes we'll need to further
6565 // process after constructing the initial VPlan.
6566 // ---------------------------------------------------------------------------
6567
6568 // For each interleave group which is relevant for this (possibly trimmed)
6569 // Range, add it to the set of groups to be later applied to the VPlan and add
6570 // placeholders for its members' Recipes which we'll be replacing with a
6571 // single VPInterleaveRecipe.
6572 for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
6573 auto ApplyIG = [IG, this](ElementCount VF) -> bool {
6574 bool Result = (VF.isVector() && // Query is illegal for VF == 1
6575 CM->getWideningDecision(I: IG->getInsertPos(), VF) ==
6576 LoopVectorizationCostModel::CM_Interleave);
6577 // For scalable vectors, the interleave factors must be <= 8 since we
6578 // require the (de)interleaveN intrinsics instead of shufflevectors.
6579 assert((!Result || !VF.isScalable() || IG->getFactor() <= 8) &&
6580 "Unsupported interleave factor for scalable vectors");
6581 return Result;
6582 };
6583 if (!getDecisionAndClampRange(Predicate: ApplyIG, Range))
6584 continue;
6585 InterleaveGroups.insert(Ptr: IG);
6586 }
6587
6588 // ---------------------------------------------------------------------------
6589 // Construct wide recipes and apply predication for original scalar
6590 // VPInstructions in the loop.
6591 // ---------------------------------------------------------------------------
6592 VPRecipeBuilder RecipeBuilder(*Plan, Legal, *CM, Builder);
6593
6594 // Scan the body of the loop in a topological order to visit each basic block
6595 // after having visited its predecessor basic blocks.
6596 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
6597 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
6598 HeaderVPBB);
6599
6600 RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
6601 Range.Start);
6602
6603 VPCostContext CostCtx(*TLI, *Plan, *CM, Config);
6604
6605 RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
6606 RecipeBuilder, CostCtx);
6607
6608 RUN_VPLAN_PASS(VPlanTransforms::makeScalarizationDecisions, *Plan, Range);
6609
6610 RUN_VPLAN_PASS(VPlanTransforms::makeCallWideningDecisions, *Plan, Range,
6611 RecipeBuilder, CostCtx);
6612
6613 // Now process all other blocks and instructions.
6614 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
6615 // Convert input VPInstructions to widened recipes.
6616 for (VPRecipeBase &R : make_early_inc_range(
6617 Range: make_range(x: VPBB->getFirstNonPhi(), y: VPBB->end()))) {
6618 // Skip recipes that do not need transforming or have already been
6619 // transformed.
6620 if (isa<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe,
6621 VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
6622 VPWidenCallRecipe, VPWidenIntrinsicRecipe, VPVectorPointerRecipe,
6623 VPVectorEndPointerRecipe, VPHistogramRecipe>(Val: &R) ||
6624 (Instruction::isCast(Opcode: cast<VPInstruction>(Val&: R).getOpcode()) &&
6625 vputils::onlyFirstLaneUsed(Def: R.getVPSingleValue())))
6626 continue;
6627 auto *VPI = cast<VPInstruction>(Val: &R);
6628 if (!VPI->getUnderlyingValue())
6629 continue;
6630
6631 // TODO: Gradually replace uses of underlying instruction by analyses on
6632 // VPlan. Migrate code relying on the underlying instruction from VPlan0
6633 // to construct recipes below to not use the underlying instruction.
6634 Instruction *Instr = cast<Instruction>(Val: VPI->getUnderlyingValue());
6635 Builder.setInsertPoint(VPI);
6636
6637 VPRecipeBase *Recipe =
6638 RecipeBuilder.tryToCreateWidenNonPhiRecipe(R: VPI, Range);
6639 if (!Recipe)
6640 Recipe =
6641 RecipeBuilder.handleReplication(VPI: cast<VPInstruction>(Val: VPI), Range);
6642
6643 if (isa<VPWidenIntOrFpInductionRecipe>(Val: Recipe) && isa<TruncInst>(Val: Instr)) {
6644 // Optimized a truncate to VPWidenIntOrFpInductionRecipe. It needs to be
6645 // moved to the phi section in the header.
6646 Recipe->insertBefore(BB&: *HeaderVPBB, IP: HeaderVPBB->getFirstNonPhi());
6647 } else {
6648 Builder.insert(R: Recipe);
6649 }
6650 if (Recipe->getNumDefinedValues() == 1) {
6651 VPI->replaceAllUsesWith(New: Recipe->getVPSingleValue());
6652 } else {
6653 assert(Recipe->getNumDefinedValues() == 0 &&
6654 "Unexpected multidef recipe");
6655 }
6656 R.eraseFromParent();
6657 }
6658 }
6659
6660 assert(isa<VPRegionBlock>(LoopRegion) &&
6661 !LoopRegion->getEntryBasicBlock()->empty() &&
6662 "entry block must be set to a VPRegionBlock having a non-empty entry "
6663 "VPBasicBlock");
6664
6665 RUN_VPLAN_PASS(VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers, *Plan,
6666 Range);
6667
6668 // ---------------------------------------------------------------------------
6669 // Transform initial VPlan: Apply previously taken decisions, in order, to
6670 // bring the VPlan to its final state.
6671 // ---------------------------------------------------------------------------
6672
6673 addReductionResultComputation(Plan, MinVF: Range.Start);
6674
6675 // Optimize FindIV reductions to use sentinel-based approach when possible.
6676 RUN_VPLAN_PASS(VPlanTransforms::optimizeFindIVReductions, *Plan, PSE,
6677 *OrigLoop);
6678 RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
6679 OrigLoop);
6680
6681 // Apply mandatory transformation to handle reductions with multiple in-loop
6682 // uses if possible, bail out otherwise.
6683 if (!RUN_VPLAN_PASS(VPlanTransforms::handleMultiUseReductions, *Plan, ORE,
6684 OrigLoop))
6685 return nullptr;
6686 // Apply mandatory transformation to handle FP maxnum/minnum reduction with
6687 // NaNs if possible, bail out otherwise.
6688 if (!RUN_VPLAN_PASS(VPlanTransforms::handleMaxMinNumReductions, *Plan))
6689 return nullptr;
6690
6691 // Create whole-vector selects for find-last recurrences.
6692 if (!RUN_VPLAN_PASS(VPlanTransforms::handleFindLastReductions, *Plan))
6693 return nullptr;
6694
6695 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, *Plan, false);
6696
6697 // Create partial reduction recipes for scaled reductions and transform
6698 // recipes to abstract recipes if it is legal and beneficial and clamp the
6699 // range for better cost estimation.
6700 RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
6701 Range);
6702 RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
6703 Range);
6704
6705 // Interleave memory: for each Interleave Group we marked earlier as relevant
6706 // for this VPlan, replace the Recipes widening its memory instructions with a
6707 // single VPInterleaveRecipe at its insertion point.
6708 RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
6709 InterleaveGroups, CM->isEpilogueAllowed());
6710
6711 // Convert memory recipes to strided access recipes if the strided access is
6712 // legal and profitable.
6713 RUN_VPLAN_PASS(VPlanTransforms::convertToStridedAccesses, *Plan, PSE,
6714 *OrigLoop, CostCtx, Range);
6715
6716 // Ensure scalar VF plans only contain VF=1, as required by hasScalarVFOnly.
6717 if (Range.Start.isScalar())
6718 Range.End = Range.Start * 2;
6719
6720 for (ElementCount VF : Range)
6721 Plan->addVF(VF);
6722 Plan->setName("Initial VPlan");
6723
6724 RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
6725
6726 if (CM->maskPartialAliasing())
6727 RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
6728
6729 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6730 return Plan;
6731}
6732
6733void LoopVectorizationPlanner::addReductionResultComputation(
6734 VPlanPtr &Plan, ElementCount MinVF) {
6735 using namespace VPlanPatternMatch;
6736 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
6737 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6738 VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
6739 Builder.setInsertPoint(&*std::prev(x: std::prev(x: LatchVPBB->end())));
6740 VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
6741 VPValue *HeaderMask = Plan->getVectorLoopRegion()->getHeaderMask();
6742 for (VPRecipeBase &R : make_early_inc_range(
6743 Range: Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
6744 VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
6745 if (!PhiR)
6746 continue;
6747
6748 // Clean up reductions that have become invariant.
6749 if (PhiR->getBackedgeValue() == PhiR) {
6750 PhiR->replaceAllUsesWith(New: PhiR->getStartValue());
6751 PhiR->eraseFromParent();
6752 continue;
6753 }
6754
6755 RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
6756 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
6757 PN: cast<PHINode>(Val: PhiR->getUnderlyingInstr()));
6758 Type *PhiTy = PhiR->getScalarType();
6759
6760 // Convert a VPBlendRecipe backedge to a select.
6761 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: PhiR->getBackedgeValue())) {
6762 if (Blend->getNumIncomingValues() == 2 &&
6763 Blend->getMask(Idx: 0) == HeaderMask) {
6764 auto *Sel = VPBuilder(Blend).createSelect(
6765 Cond: Blend->getMask(Idx: 0), TrueVal: Blend->getIncomingValue(Idx: 0),
6766 FalseVal: Blend->getIncomingValue(Idx: 1), DL: {}, Name: "", Flags: *Blend);
6767 Blend->replaceAllUsesWith(New: Sel);
6768 Blend->eraseFromParent();
6769 }
6770 }
6771
6772 auto *OrigExitingVPV = PhiR->getBackedgeValue();
6773 auto *NewExitingVPV = OrigExitingVPV;
6774
6775 // Remove the predicated select if the target doesn't want it.
6776 VPValue *V;
6777 if (!CM->usePredicatedReductionSelect(RecurrenceKind) &&
6778 match(V: PhiR->getBackedgeValue(),
6779 P: m_Select(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue(V), Op2: m_Specific(VPV: PhiR))))
6780 PhiR->setBackedgeValue(V);
6781
6782 // We want code in the middle block to appear to execute on the location of
6783 // the scalar loop's latch terminator because: (a) it is all compiler
6784 // generated, (b) these instructions are always executed after evaluating
6785 // the latch conditional branch, and (c) other passes may add new
6786 // predecessors which terminate on this line. This is the easiest way to
6787 // ensure we don't accidentally cause an extra step back into the loop while
6788 // debugging.
6789 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
6790
6791 // TODO: At the moment ComputeReductionResult also drives creation of the
6792 // bc.merge.rdx phi nodes, hence it needs to be created unconditionally here
6793 // even for in-loop reductions, until the reduction resume value handling is
6794 // also modeled in VPlan.
6795 VPInstruction *FinalReductionResult;
6796 VPBuilder::InsertPointGuard Guard(Builder);
6797 Builder.setInsertPoint(TheBB: MiddleVPBB, IP);
6798 // For AnyOf reductions, find the select among PhiR's users and convert
6799 // the reduction phi to operate on bools before creating the final
6800 // reduction result.
6801 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RecurrenceKind)) {
6802 auto *AnyOfSelect = cast<VPSingleDefRecipe>(
6803 Val: findUserOf(V: PhiR, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())));
6804 VPValue *Start = PhiR->getStartValue();
6805 bool TrueValIsPhi = AnyOfSelect->getOperand(N: 1) == PhiR;
6806 // NewVal is the non-phi operand of the select.
6807 VPValue *NewVal = TrueValIsPhi ? AnyOfSelect->getOperand(N: 2)
6808 : AnyOfSelect->getOperand(N: 1);
6809
6810 // Adjust AnyOf reductions; replace the reduction phi for the selected
6811 // value with a boolean reduction phi node to check if the condition is
6812 // true in any iteration. The final value is selected by the final
6813 // ComputeReductionResult.
6814 VPValue *Cmp = AnyOfSelect->getOperand(N: 0);
6815 // If the compare is checking the reduction PHI node, adjust it to check
6816 // the start value.
6817 if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
6818 CmpR->replaceUsesOfWith(From: PhiR, To: PhiR->getStartValue());
6819 Builder.setInsertPoint(AnyOfSelect);
6820
6821 // If the true value of the select is the reduction phi, the new value
6822 // is selected if the negated condition is true in any iteration.
6823 if (TrueValIsPhi)
6824 Cmp = Builder.createNot(Operand: Cmp);
6825
6826 // Build a fresh i1 chain (phi, or, and i1 versions of any blend/select
6827 // the exiting value flows through).
6828 auto *NewPhiR =
6829 PhiR->cloneWithOperands(Start: Plan->getFalse(), BackedgeValue: Plan->getFalse());
6830 NewPhiR->insertBefore(InsertPos: PhiR);
6831 VPValue *NewExiting = Builder.createOr(LHS: NewPhiR, RHS: Cmp);
6832
6833 // The exiting value may flow through a chain of VPBlendRecipes and
6834 // select recipes (VPInstruction, VPWidenRecipe or VPReplicateRecipe with
6835 // Select opcode) before reaching OrigExitingVPV. Clone each chain link
6836 // in topological order so each clone refers to the already-rewritten i1
6837 // operands via Substitutions.
6838 DenseMap<VPValue *, VPValue *> Substitutions = {{AnyOfSelect, NewExiting},
6839 {PhiR, NewPhiR}};
6840 std::function<void(VPSingleDefRecipe *)> CloneChain =
6841 [&](VPSingleDefRecipe *Old) {
6842 if (Substitutions.contains(Val: Old))
6843 return;
6844 SmallVector<VPValue *> NewOps;
6845 for (VPValue *Op : Old->operands()) {
6846 if (isa<VPBlendRecipe>(Val: Op) ||
6847 match(V: Op, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())))
6848 CloneChain(cast<VPSingleDefRecipe>(Val: Op));
6849 NewOps.push_back(Elt: Substitutions.lookup_or(Val: Op, Default&: Op));
6850 }
6851 VPSingleDefRecipe *New;
6852 if (auto *B = dyn_cast<VPBlendRecipe>(Val: Old))
6853 New = B->cloneWithOperands(NewOperands: NewOps);
6854 else if (auto *W = dyn_cast<VPWidenRecipe>(Val: Old))
6855 New = W->cloneWithOperands(NewOperands: NewOps);
6856 else if (auto *Rep = dyn_cast<VPReplicateRecipe>(Val: Old))
6857 New = Rep->cloneWithOperands(NewOperands: NewOps);
6858 else
6859 New = cast<VPInstruction>(Val: Old)->cloneWithOperands(NewOperands: NewOps);
6860 New->insertBefore(InsertPos: Old);
6861 Substitutions[Old] = New;
6862 };
6863
6864 if (OrigExitingVPV != AnyOfSelect) {
6865 CloneChain(cast<VPSingleDefRecipe>(Val: OrigExitingVPV));
6866 NewExiting = Substitutions.lookup(Val: OrigExitingVPV);
6867 }
6868 NewPhiR->setOperand(I: 1, New: NewExiting);
6869 PhiR->replaceAllUsesWith(New: Plan->getPoison(Ty: PhiR->getScalarType()));
6870
6871 Builder.setInsertPoint(TheBB: MiddleVPBB, IP);
6872 FinalReductionResult =
6873 Builder.createAnyOfReduction(ChainOp: NewExiting, TrueVal: NewVal, FalseVal: Start, DL: ExitDL);
6874 } else {
6875 // If the vector reduction can be performed in a smaller type, we
6876 // truncate then extend the loop exit value to enable InstCombine to
6877 // evaluate the entire expression in the smaller type.
6878 VPValue *ReductionOp = NewExitingVPV;
6879 Instruction::CastOps ExtendOpc = Instruction::CastOpsEnd;
6880 if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
6881 assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
6882 assert(!RecurrenceDescriptor::isMinMaxRecurrenceKind(RecurrenceKind) &&
6883 "Unexpected truncated min-max recurrence!");
6884 Type *RdxTy = RdxDesc.getRecurrenceType();
6885 ExtendOpc = RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
6886 {
6887 VPBuilder::InsertPointGuard Guard(Builder);
6888 Builder.setInsertPoint(
6889 TheBB: NewExitingVPV->getDefiningRecipe()->getParent(),
6890 IP: std::next(x: NewExitingVPV->getDefiningRecipe()->getIterator()));
6891 ReductionOp =
6892 Builder.createWidenCast(Opcode: Instruction::Trunc, Op: NewExitingVPV, ResultTy: RdxTy);
6893 VPWidenCastRecipe *Extnd =
6894 Builder.createWidenCast(Opcode: ExtendOpc, Op: ReductionOp, ResultTy: PhiTy);
6895 if (PhiR->getOperand(N: 1) == NewExitingVPV)
6896 PhiR->setOperand(I: 1, New: Extnd);
6897 }
6898 }
6899
6900 VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
6901 PhiR->getFastMathFlagsOrNone());
6902 FinalReductionResult = Builder.createNaryOp(
6903 Opcode: VPInstruction::ComputeReductionResult, Operands: {ReductionOp}, Flags, DL: ExitDL);
6904 if (ExtendOpc != Instruction::CastOpsEnd)
6905 FinalReductionResult = Builder.createScalarCast(
6906 Opcode: ExtendOpc, Op: FinalReductionResult, ResultTy: PhiTy, DL: {});
6907 }
6908
6909 // Update all users outside the vector region. Also replace redundant
6910 // extracts.
6911 for (auto *U : to_vector(Range: OrigExitingVPV->users())) {
6912 auto *Parent = cast<VPRecipeBase>(Val: U)->getParent();
6913 if (FinalReductionResult == U || Parent->getParent())
6914 continue;
6915 // Skip ComputeReductionResult and FindIV reductions when they are not the
6916 // final result.
6917 if (match(U, P: m_VPInstruction<VPInstruction::ComputeReductionResult>()) ||
6918 (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind: RecurrenceKind) &&
6919 match(U, P: m_VPInstruction<Instruction::ICmp>())))
6920 continue;
6921 U->replaceUsesOfWith(From: OrigExitingVPV, To: FinalReductionResult);
6922
6923 // Look through ExtractLastPart.
6924 if (match(U, P: m_ExtractLastPart(Op0: m_VPValue())))
6925 U = cast<VPInstruction>(Val: U)->getSingleUser();
6926
6927 if (match(U, P: m_CombineOr(Ps: m_ExtractLane(Op0: m_VPValue(), Op1: m_VPValue()),
6928 Ps: m_ExtractLastLane(Op0: m_VPValue()))))
6929 cast<VPInstruction>(Val: U)->replaceAllUsesWith(New: FinalReductionResult);
6930 }
6931
6932 RecurKind RK = PhiR->getRecurrenceKind();
6933 if ((!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK) &&
6934 !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind: RK) &&
6935 !RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind: RK) &&
6936 !RecurrenceDescriptor::isFindLastRecurrenceKind(Kind: RK))) {
6937 VPBuilder PHBuilder(Plan->getVectorPreheader());
6938 VPValue *Iden = Plan->getOrAddLiveIn(
6939 V: getRecurrenceIdentity(K: RK, Tp: PhiTy, FMF: PhiR->getFastMathFlagsOrNone()));
6940 auto *ScaleFactorVPV = Plan->getConstantInt(BitWidth: 32, Val: 1);
6941 VPValue *StartV = PHBuilder.createNaryOp(
6942 Opcode: VPInstruction::ReductionStartVector,
6943 Operands: {PhiR->getStartValue(), Iden, ScaleFactorVPV}, Flags: *PhiR);
6944 PhiR->setOperand(I: 0, New: StartV);
6945 }
6946 }
6947
6948 RUN_VPLAN_PASS(VPlanTransforms::clearReductionWrapFlags, *Plan);
6949}
6950
6951void LoopVectorizationPlanner::attachRuntimeChecks(
6952 VPlan &Plan, GeneratedRTChecks &RTChecks, bool HasBranchWeights) const {
6953 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
6954 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(N: 0)) {
6955 assert((!Config.OptForSize ||
6956 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled) &&
6957 "Cannot SCEV check stride or overflow when optimizing for size");
6958 RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan, SCEVCheckCond,
6959 SCEVCheckBlock, HasBranchWeights);
6960 }
6961 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
6962 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(N: 0)) {
6963 // VPlan-native path does not do any analysis for runtime checks
6964 // currently.
6965 assert((!EnableVPlanNativePath || !Plan.isOuterLoop()) &&
6966 "Runtime checks are not supported for outer loops yet");
6967
6968 if (Config.OptForSize) {
6969 assert(
6970 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled &&
6971 "Cannot emit memory checks when optimizing for size, unless forced "
6972 "to vectorize.");
6973 ORE->emit(RemarkBuilder: [&]() {
6974 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
6975 OrigLoop->getStartLoc(),
6976 OrigLoop->getHeader())
6977 << "Code-size may be reduced by not forcing "
6978 "vectorization, or by source-code modifications "
6979 "eliminating the need for runtime checks "
6980 "(e.g., adding 'restrict').";
6981 });
6982 }
6983 RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan, MemCheckCond,
6984 MemCheckBlock, HasBranchWeights);
6985 }
6986}
6987
6988void LoopVectorizationPlanner::addMinimumIterationCheck(
6989 VPlan &Plan, ElementCount VF, unsigned UF,
6990 ElementCount MinProfitableTripCount) const {
6991 const uint32_t *BranchWeights =
6992 hasBranchWeightMD(I: *OrigLoop->getLoopLatch()->getTerminator())
6993 ? &MinItersBypassWeights[0]
6994 : nullptr;
6995 RUN_VPLAN_PASS(VPlanTransforms::addMinimumIterationCheck, Plan, VF, UF,
6996 MinProfitableTripCount, Plan.requiresScalarEpilogue(),
6997 Plan.hasTailFolded(), OrigLoop, BranchWeights,
6998 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
6999 PSE, Plan.getEntry());
7000}
7001
7002// Determine how to lower the epilogue, which depends on 1) optimising
7003// for minimum code-size, 2) tail-folding compiler options, 3) loop
7004// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
7005// is suitable for tail-folding.
7006// This function determines epilogue lowering for the main vector loop while
7007// epilogue lowering for the tail-folded epilogue path will be handled
7008// separately in getEpilogueTailLowering.
7009static EpilogueLowering
7010getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
7011 bool OptForSize, TargetTransformInfo *TTI,
7012 TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL,
7013 InterleavedAccessInfo *IAI) {
7014 // 1) OptSize takes precedence over all other options, i.e. if this is set,
7015 // don't look at hints or options, and don't request an epilogue.
7016 if (F->hasOptSize() ||
7017 (OptForSize && Hints.getForce() != LoopVectorizeHints::FK_Enabled))
7018 return CM_EpilogueNotAllowedOptSize;
7019
7020 // 2) If set, obey the directives
7021 if (TailFoldingPolicy.getNumOccurrences()) {
7022 switch (TailFoldingPolicy) {
7023 case TailFoldingPolicyTy::None:
7024 return CM_EpilogueAllowed;
7025 case TailFoldingPolicyTy::PreferFoldTail:
7026 return CM_EpilogueNotNeededFoldTail;
7027 case TailFoldingPolicyTy::MustFoldTail:
7028 return CM_EpilogueNotAllowedFoldTail;
7029 };
7030 }
7031
7032 // 3) If set, obey the hints
7033 switch (Hints.getPredicate()) {
7034 case LoopVectorizeHints::FK_Enabled:
7035 return CM_EpilogueNotNeededFoldTail;
7036 case LoopVectorizeHints::FK_Disabled:
7037 return CM_EpilogueAllowed;
7038 };
7039
7040 // 4) if the TTI hook indicates this is profitable, request tail-folding.
7041 TailFoldingInfo TFI(TLI, &LVL, IAI);
7042 if (TTI->preferTailFoldingOverEpilogue(TFI: &TFI))
7043 return CM_EpilogueNotNeededFoldTail;
7044
7045 return CM_EpilogueAllowed;
7046}
7047
7048// Emit a remark if there are stores to floats that required a floating point
7049// extension. If the vectorized loop was generated with floating point there
7050// will be a performance penalty from the conversion overhead and the change in
7051// the vector width.
7052static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
7053 SmallVector<Instruction *, 4> Worklist;
7054 for (BasicBlock *BB : L->getBlocks()) {
7055 for (Instruction &Inst : *BB) {
7056 if (auto *S = dyn_cast<StoreInst>(Val: &Inst)) {
7057 if (S->getValueOperand()->getType()->isFloatTy())
7058 Worklist.push_back(Elt: S);
7059 }
7060 }
7061 }
7062
7063 // Traverse the floating point stores upwards searching, for floating point
7064 // conversions.
7065 SmallPtrSet<const Instruction *, 4> Visited;
7066 SmallPtrSet<const Instruction *, 4> EmittedRemark;
7067 while (!Worklist.empty()) {
7068 auto *I = Worklist.pop_back_val();
7069 if (!L->contains(Inst: I))
7070 continue;
7071 if (!Visited.insert(Ptr: I).second)
7072 continue;
7073
7074 // Emit a remark if the floating point store required a floating
7075 // point conversion.
7076 // TODO: More work could be done to identify the root cause such as a
7077 // constant or a function return type and point the user to it.
7078 if (isa<FPExtInst>(Val: I) && EmittedRemark.insert(Ptr: I).second)
7079 ORE->emit(RemarkBuilder: [&]() {
7080 return OptimizationRemarkAnalysis(LV_NAME, "VectorMixedPrecision",
7081 I->getDebugLoc(), L->getHeader())
7082 << "floating point conversion changes vector width. "
7083 << "Mixed floating point precision requires an up/down "
7084 << "cast that will negatively impact performance.";
7085 });
7086
7087 for (Use &Op : I->operands())
7088 if (auto *OpI = dyn_cast<Instruction>(Val&: Op))
7089 Worklist.push_back(Elt: OpI);
7090 }
7091}
7092
7093/// For loops with uncountable early exits, find the cost of doing work when
7094/// exiting the loop early, such as calculating the final exit values of
7095/// variables used outside the loop.
7096/// TODO: This is currently overly pessimistic because the loop may not take
7097/// the early exit, but better to keep this conservative for now. In future,
7098/// it might be possible to relax this by using branch probabilities.
7099static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx,
7100 VPlan &Plan, ElementCount VF) {
7101 InstructionCost Cost = 0;
7102 for (auto *ExitVPBB : Plan.getExitBlocks()) {
7103 for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
7104 // If the predecessor is not the middle.block, then it must be the
7105 // vector.early.exit block, which may contain work to calculate the exit
7106 // values of variables used outside the loop.
7107 if (PredVPBB != Plan.getMiddleBlock()) {
7108 LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
7109 << PredVPBB->getName() << ":\n");
7110 Cost += PredVPBB->cost(VF, Ctx&: CostCtx);
7111 }
7112 }
7113 }
7114 return Cost;
7115}
7116
7117/// This function determines whether or not it's still profitable to vectorize
7118/// the loop given the extra work we have to do outside of the loop:
7119/// 1. Perform the runtime checks before entering the loop to ensure it's safe
7120/// to vectorize.
7121/// 2. In the case of loops with uncountable early exits, we may have to do
7122/// extra work when exiting the loop early, such as calculating the final
7123/// exit values of variables used outside the loop.
7124/// 3. The middle block.
7125static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
7126 VectorizationFactor &VF, Loop *L,
7127 PredicatedScalarEvolution &PSE,
7128 VPCostContext &CostCtx, VPlan &Plan,
7129 EpilogueLowering SEL,
7130 std::optional<unsigned> VScale) {
7131 InstructionCost RtC = Checks.getCost();
7132 if (!RtC.isValid())
7133 return false;
7134
7135 // When interleaving only scalar and vector cost will be equal, which in turn
7136 // would lead to a divide by 0. Fall back to hard threshold.
7137 if (VF.Width.isScalar()) {
7138 // TODO: Should we rename VectorizeMemoryCheckThreshold?
7139 if (RtC > VectorizeMemoryCheckThreshold) {
7140 LLVM_DEBUG(
7141 dbgs()
7142 << "LV: Interleaving only is not profitable due to runtime checks\n");
7143 return false;
7144 }
7145 return true;
7146 }
7147
7148 // The scalar cost should only be 0 when vectorizing with a user specified
7149 // VF/IC. In those cases, runtime checks should always be generated.
7150 uint64_t ScalarC = VF.ScalarCost.getValue();
7151 if (ScalarC == 0)
7152 return true;
7153
7154 InstructionCost TotalCost = RtC;
7155 // Add on the cost of any work required in the vector early exit block, if
7156 // one exists.
7157 TotalCost += calculateEarlyExitCost(CostCtx, Plan, VF: VF.Width);
7158 TotalCost += Plan.getMiddleBlock()->cost(VF: VF.Width, Ctx&: CostCtx);
7159
7160 // First, compute the minimum iteration count required so that the vector
7161 // loop outperforms the scalar loop.
7162 // The total cost of the scalar loop is
7163 // ScalarC * TC
7164 // where
7165 // * TC is the actual trip count of the loop.
7166 // * ScalarC is the cost of a single scalar iteration.
7167 //
7168 // The total cost of the vector loop is
7169 // TotalCost + VecC * (TC / VF) + EpiC
7170 // where
7171 // * TotalCost is the sum of the costs cost of
7172 // - the generated runtime checks, i.e. RtC
7173 // - performing any additional work in the vector.early.exit block for
7174 // loops with uncountable early exits.
7175 // - the middle block, if ExpectedTC <= VF.Width.
7176 // * VecC is the cost of a single vector iteration.
7177 // * TC is the actual trip count of the loop
7178 // * VF is the vectorization factor
7179 // * EpiCost is the cost of the generated epilogue, including the cost
7180 // of the remaining scalar operations.
7181 //
7182 // Vectorization is profitable once the total vector cost is less than the
7183 // total scalar cost:
7184 // TotalCost + VecC * (TC / VF) + EpiC < ScalarC * TC
7185 //
7186 // Now we can compute the minimum required trip count TC as
7187 // VF * (TotalCost + EpiC) / (ScalarC * VF - VecC) < TC
7188 //
7189 // For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
7190 // the computations are performed on doubles, not integers and the result
7191 // is rounded up, hence we get an upper estimate of the TC.
7192 unsigned IntVF = estimateElementCount(VF: VF.Width, VScale);
7193 uint64_t Div = ScalarC * IntVF - VF.Cost.getValue();
7194 uint64_t MinTC1 =
7195 Div == 0 ? 0 : divideCeil(Numerator: TotalCost.getValue() * IntVF, Denominator: Div);
7196
7197 // Second, compute a minimum iteration count so that the cost of the
7198 // runtime checks is only a fraction of the total scalar loop cost. This
7199 // adds a loop-dependent bound on the overhead incurred if the runtime
7200 // checks fail. In case the runtime checks fail, the cost is RtC + ScalarC
7201 // * TC. To bound the runtime check to be a fraction 1/X of the scalar
7202 // cost, compute
7203 // RtC < ScalarC * TC * (1 / X) ==> RtC * X / ScalarC < TC
7204 uint64_t MinTC2 = divideCeil(Numerator: RtC.getValue() * 10, Denominator: ScalarC);
7205
7206 // Now pick the larger minimum. If it is not a multiple of VF and an epilogue
7207 // is allowed, choose the next closest multiple of VF. This should partly
7208 // compensate for ignoring the epilogue cost.
7209 uint64_t MinTC = std::max(a: MinTC1, b: MinTC2);
7210 if (SEL == CM_EpilogueAllowed)
7211 MinTC = alignTo(Value: MinTC, Align: IntVF);
7212 VF.MinProfitableTripCount = ElementCount::getFixed(MinVal: MinTC);
7213
7214 LLVM_DEBUG(
7215 dbgs() << "LV: Minimum required TC for runtime checks to be profitable:"
7216 << VF.MinProfitableTripCount << "\n");
7217
7218 // Skip vectorization if the expected trip count is less than the minimum
7219 // required trip count.
7220 if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
7221 if (ElementCount::isKnownLT(LHS: *ExpectedTC, RHS: VF.MinProfitableTripCount)) {
7222 LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
7223 "trip count < minimum profitable VF ("
7224 << *ExpectedTC << " < " << VF.MinProfitableTripCount
7225 << ")\n");
7226
7227 return false;
7228 }
7229 }
7230 return true;
7231}
7232
7233LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
7234 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
7235 !EnableLoopInterleaving),
7236 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
7237 !EnableLoopVectorization) {}
7238
7239/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
7240/// vectorization.
7241static SmallVector<VPInstruction *>
7242preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
7243 using namespace VPlanPatternMatch;
7244 // When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
7245 // introduce multiple uses of undef/poison. If the reduction start value may
7246 // be undef or poison it needs to be frozen and the frozen start has to be
7247 // used when computing the reduction result. We also need to use the frozen
7248 // value in the resume phi generated by the main vector loop, as this is also
7249 // used to compute the reduction result after the epilogue vector loop.
7250 auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
7251 bool UpdateResumePhis) {
7252 VPBuilder Builder(Plan.getEntry());
7253 for (VPInstruction &VPI :
7254 make_isa_range<VPInstruction>(Range&: *Plan.getMiddleBlock())) {
7255 VPValue *OrigStart;
7256 if (!matchFindIVResult(VPI: &VPI, ReducedIV: m_VPValue(), Start: m_VPValue(V&: OrigStart)))
7257 continue;
7258 if (isGuaranteedNotToBeUndefOrPoison(V: OrigStart->getLiveInIRValue()))
7259 continue;
7260 VPInstruction *Freeze = Builder.createFreeze(Op: OrigStart, DL: {}, Name: "fr");
7261 VPI.setOperand(I: 2, New: Freeze);
7262 if (UpdateResumePhis)
7263 OrigStart->replaceUsesWithIf(New: Freeze, ShouldReplace: [Freeze](VPUser &U, unsigned) {
7264 return Freeze != &U && isa<VPPhi>(Val: &U);
7265 });
7266 }
7267 };
7268 AddFreezeForFindLastIVReductions(MainPlan, true);
7269 AddFreezeForFindLastIVReductions(EpiPlan, false);
7270
7271 VPValue *VectorTC = nullptr;
7272 auto *Term =
7273 MainPlan.getVectorLoopRegion()->getExitingBasicBlock()->getTerminator();
7274 [[maybe_unused]] bool MatchedTC =
7275 match(V: Term, P: m_BranchOnCount(Op0: m_VPValue(), Op1: m_VPValue(V&: VectorTC)));
7276 assert(MatchedTC && "must match vector trip count");
7277
7278 // If there is a suitable resume value for the canonical induction in the
7279 // scalar (which will become vector) epilogue loop, use it and move it to the
7280 // beginning of the scalar preheader. Otherwise create it below.
7281 VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader();
7282 auto ResumePhiIter =
7283 find_if(Range: MainScalarPH->phis(), P: [VectorTC](VPRecipeBase &R) {
7284 return match(V: &R, P: m_VPInstruction<Instruction::PHI>(Ops: m_Specific(VPV: VectorTC),
7285 Ops: m_ZeroInt()));
7286 });
7287 VPPhi *ResumePhi = nullptr;
7288 if (ResumePhiIter == MainScalarPH->phis().end()) {
7289 assert(MainPlan.getVectorLoopRegion()->getCanonicalIV() &&
7290 "canonical IV must exist");
7291 Type *Ty = VectorTC->getScalarType();
7292 VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
7293 ResumePhi = ScalarPHBuilder.createScalarPhi(
7294 IncomingValues: {VectorTC, MainPlan.getZero(Ty)}, DL: {}, Name: "vec.epilog.resume.val");
7295 } else {
7296 ResumePhi = cast<VPPhi>(Val: &*ResumePhiIter);
7297 ResumePhi->setName("vec.epilog.resume.val");
7298 if (&MainScalarPH->front() != ResumePhi)
7299 ResumePhi->moveBefore(BB&: *MainScalarPH, I: MainScalarPH->begin());
7300 }
7301
7302 // Create a ResumeForEpilogue for the canonical IV resume and its bypass value
7303 // as the first non-phi, to keep them alive for the epilogue.
7304 VPBuilder ResumeBuilder(MainScalarPH);
7305 ResumeBuilder.createNaryOp(Opcode: VPInstruction::ResumeForEpilogue,
7306 Operands: {ResumePhi, ResumePhi->getOperand(N: 1)});
7307
7308 // Create ResumeForEpilogue instructions for the resume phis of the
7309 // VPIRPhis and their bypass values in the scalar header of the main plan and
7310 // return them so they can be used as resume values when vectorizing the
7311 // epilogue.
7312 return to_vector(
7313 Range: map_range(C: MainPlan.getScalarHeader()->phis(), F: [&](VPRecipeBase &R) {
7314 assert(isa<VPIRPhi>(R) &&
7315 "only VPIRPhis expected in the scalar header");
7316 VPValue *MainResumePhi = R.getOperand(N: 0);
7317 VPValue *Bypass = MainResumePhi->getDefiningRecipe()->getOperand(N: 1);
7318 return ResumeBuilder.createNaryOp(Opcode: VPInstruction::ResumeForEpilogue,
7319 Operands: {MainResumePhi, Bypass});
7320 }));
7321}
7322
7323/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
7324/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes. Some
7325/// reductions require creating new instructions to compute the resume values.
7326/// They are collected in a vector and returned. They must be moved to the
7327/// preheader of the vector epilogue loop, after created by the execution of \p
7328/// Plan.
7329static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
7330 VPlan &MainPlan, VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs,
7331 EpilogueLoopVectorizationInfo &EPI, LoopVectorizationPlanner &LVP,
7332 VFSelectionContext &Config, ScalarEvolution &SE,
7333 ArrayRef<VPInstruction *> ResumeValues) {
7334 // Build a map from the scalar-header PHI to the ResumeForEpilogue markers
7335 // from the main plan.
7336 // TODO: Replace the IR PHI key.
7337 DenseMap<PHINode *, VPInstruction *> IRPhiToResumeForEpi;
7338 for (auto [HeaderPhi, ResumeForEpi] :
7339 zip_equal(t: MainPlan.getScalarHeader()->phis(), u&: ResumeValues))
7340 IRPhiToResumeForEpi[&cast<VPIRPhi>(Val&: HeaderPhi).getIRPhi()] = ResumeForEpi;
7341 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
7342 VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
7343 Header->setName("vec.epilog.vector.body");
7344
7345 VPValue *IV = VectorLoop->getCanonicalIV();
7346 // When vectorizing the epilogue loop, the canonical induction needs to start
7347 // at the resume value from the main vector loop. Find the resume value
7348 // created during execution of the main VPlan. Add this resume value as an
7349 // offset to the canonical IV of the epilogue loop.
7350 using namespace llvm::PatternMatch;
7351 VPInstruction *ResumeForEpilogue =
7352 cast<VPInstruction>(Val: &*MainPlan.getScalarPreheader()->getFirstNonPhi());
7353 Value *EPResumeVal = ResumeForEpilogue->getUnderlyingValue();
7354 if (auto *ResumePhi = dyn_cast<PHINode>(Val: EPResumeVal)) {
7355 for (Value *Inc : ResumePhi->incoming_values()) {
7356 if (match(V: Inc, P: m_SpecificInt(V: 0)))
7357 continue;
7358 assert(!EPI.VectorTripCount &&
7359 "Must only have a single non-zero incoming value");
7360 EPI.VectorTripCount = Inc;
7361 }
7362 // If we didn't find a non-zero vector trip count, all incoming values
7363 // must be zero, which also means the vector trip count is zero.
7364 if (!EPI.VectorTripCount) {
7365 assert(ResumePhi->getNumIncomingValues() > 0 &&
7366 all_of(ResumePhi->incoming_values(), match_fn(m_SpecificInt(0))) &&
7367 "all incoming values must be 0");
7368 EPI.VectorTripCount = ResumePhi->getIncomingValue(i: 0);
7369 }
7370 } else {
7371 EPI.VectorTripCount = EPResumeVal;
7372 }
7373 VPValue *VPV = Plan.getOrAddLiveIn(V: EPResumeVal);
7374 assert(all_of(IV->users(),
7375 [](const VPUser *U) {
7376 if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe>(U))
7377 return true;
7378 unsigned Opc = cast<VPInstruction>(U)->getOpcode();
7379 return Instruction::isCast(Opc) || Opc == Instruction::Add;
7380 }) &&
7381 "the canonical IV should only be used by its increment or "
7382 "ScalarIVSteps when resetting the start value");
7383 VPBuilder Builder(Header, Header->getFirstNonPhi());
7384 VPInstruction *Add = Builder.createAdd(LHS: IV, RHS: VPV);
7385 // Replace all users of the canonical IV and its increment with the offset
7386 // version, except for the Add itself and the canonical IV increment.
7387 auto *Increment = vputils::findCanonicalIVIncrement(Plan);
7388 assert(Increment && "Must have a canonical IV increment at this point");
7389 IV->replaceUsesWithIf(New: Add, ShouldReplace: [Add, Increment](VPUser &U, unsigned) {
7390 return &U != Add && &U != Increment;
7391 });
7392 VPInstruction *OffsetIVInc =
7393 VPBuilder::getToInsertAfter(R: Increment).createAdd(LHS: Increment, RHS: VPV);
7394 Increment->replaceAllUsesWith(New: OffsetIVInc);
7395 OffsetIVInc->setOperand(I: 0, New: Increment);
7396
7397 DenseMap<Value *, Value *> ToFrozen;
7398 SmallVector<Instruction *> InstsToMove;
7399 // Ensure that the start values for all header phi recipes are updated before
7400 // vectorizing the epilogue loop.
7401 for (VPRecipeBase &R : Header->phis()) {
7402 Value *ResumeV = nullptr;
7403 // TODO: Move setting of resume values to prepareToExecute.
7404 if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(Val: &R)) {
7405 // Find the reduction result by searching users of the phi or its backedge
7406 // value.
7407 auto IsReductionResult = [](VPRecipeBase *R) {
7408 auto *VPI = dyn_cast<VPInstruction>(Val: R);
7409 return VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult;
7410 };
7411 auto *RdxResult = cast<VPInstruction>(
7412 Val: vputils::findRecipe(Start: ReductionPhi->getBackedgeValue(), Pred: IsReductionResult));
7413 assert(RdxResult && "expected to find reduction result");
7414
7415 VPInstruction *ResumeForEpi = IRPhiToResumeForEpi.at(
7416 Val: cast<PHINode>(Val: ReductionPhi->getUnderlyingInstr()));
7417 ResumeV = ResumeForEpi->getUnderlyingValue();
7418
7419 // Check for FindIV pattern by looking for icmp user of RdxResult.
7420 // The pattern is: select(icmp ne RdxResult, Sentinel), RdxResult, Start
7421 using namespace VPlanPatternMatch;
7422 VPValue *SentinelVPV = nullptr;
7423 bool IsFindIV = any_of(Range: RdxResult->users(), P: [&](VPUser *U) {
7424 return match(U, P: VPlanPatternMatch::m_SpecificICmp(
7425 MatchPred: ICmpInst::ICMP_NE, Op0: m_Specific(VPV: RdxResult),
7426 Op1: m_VPValue(V&: SentinelVPV)));
7427 });
7428
7429 RecurKind RK = ReductionPhi->getRecurrenceKind();
7430 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK) || IsFindIV) {
7431 auto *ResumePhi = cast<PHINode>(Val: ResumeV);
7432 VPValue *BypassOp = ResumeForEpi->getOperand(N: 1);
7433 assert((isa<VPIRValue>(BypassOp) ||
7434 VPlanPatternMatch::match(
7435 BypassOp,
7436 m_VPInstruction<Instruction::Freeze>(m_VPValue()))) &&
7437 "expected live-in or Freeze");
7438 Value *StartV = BypassOp->getUnderlyingValue();
7439 IRBuilder<> Builder(ResumePhi->getParent(),
7440 ResumePhi->getParent()->getFirstNonPHIIt());
7441
7442 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK)) {
7443 // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
7444 // start value; compare the final value from the main vector loop
7445 // to the start value.
7446 ResumeV = Builder.CreateICmpNE(LHS: ResumeV, RHS: StartV);
7447 if (auto *I = dyn_cast<Instruction>(Val: ResumeV))
7448 InstsToMove.push_back(Elt: I);
7449 } else {
7450 assert(SentinelVPV && "expected to find icmp using RdxResult");
7451 if (auto *FreezeI = dyn_cast<FreezeInst>(Val: StartV))
7452 ToFrozen[FreezeI->getOperand(i_nocapture: 0)] = StartV;
7453
7454 // Adjust resume: select(icmp eq ResumeV, StartV), Sentinel, ResumeV
7455 Value *Cmp = Builder.CreateICmpEQ(LHS: ResumeV, RHS: StartV);
7456 if (auto *I = dyn_cast<Instruction>(Val: Cmp))
7457 InstsToMove.push_back(Elt: I);
7458 ResumeV = Builder.CreateSelect(C: Cmp, True: SentinelVPV->getLiveInIRValue(),
7459 False: ResumeV);
7460 if (auto *I = dyn_cast<Instruction>(Val: ResumeV))
7461 InstsToMove.push_back(Elt: I);
7462 }
7463 } else {
7464 VPValue *StartVal = Plan.getOrAddLiveIn(V: ResumeV);
7465 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
7466 if (auto *VPI = dyn_cast<VPInstruction>(Val: PhiR->getStartValue())) {
7467 assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
7468 "unexpected start value");
7469 // Partial sub-reductions always start at 0 and account for the
7470 // reduction start value in a final subtraction. Update it to use the
7471 // resume value from the main vector loop.
7472 if (PhiR->getVFScaleFactor() > 1 &&
7473 RecurrenceDescriptor::isSubRecurrenceKind(
7474 Kind: PhiR->getRecurrenceKind())) {
7475 auto *Sub = cast<VPInstruction>(Val: RdxResult->getSingleUser());
7476 assert((Sub->getOpcode() == Instruction::Sub ||
7477 Sub->getOpcode() == Instruction::FSub) &&
7478 "Unexpected opcode");
7479 assert(isa<VPIRValue>(Sub->getOperand(0)) &&
7480 "Expected operand to match the original start value of the "
7481 "reduction");
7482 // For integer sub-reductions, verify start value is zero.
7483 // For FP sub-reductions, verify start value is negative zero.
7484 [[maybe_unused]] auto StartValueIsIdentity = [&] {
7485 Value *IdentityValue = getRecurrenceIdentity(
7486 K: PhiR->getRecurrenceKind(), Tp: ResumeV->getType(),
7487 FMF: PhiR->getFastMathFlagsOrNone());
7488 auto *StartValue = dyn_cast<VPIRValue>(Val: VPI->getOperand(N: 0));
7489 return StartValue && StartValue->getValue() == IdentityValue;
7490 };
7491 assert(StartValueIsIdentity() &&
7492 "Expected start value for partial sub-reduction to be zero "
7493 "(or negative zero)");
7494
7495 Sub->setOperand(I: 0, New: StartVal);
7496 } else
7497 VPI->setOperand(I: 0, New: StartVal);
7498 continue;
7499 }
7500 }
7501 } else {
7502 // Retrieve the induction resume value via ResumeForEpilogue.
7503 PHINode *IndPhi = cast<VPWidenInductionRecipe>(Val: &R)->getPHINode();
7504 ResumeV = IRPhiToResumeForEpi.at(Val: IndPhi)->getUnderlyingValue();
7505 }
7506 assert(ResumeV && "Must have a resume value");
7507 VPValue *StartVal = Plan.getOrAddLiveIn(V: ResumeV);
7508 cast<VPHeaderPHIRecipe>(Val: &R)->setStartValue(StartVal);
7509 }
7510
7511 // For some VPValues in the epilogue plan we must re-use the generated IR
7512 // values from the main plan. Replace them with live-in VPValues.
7513 // TODO: This is a workaround needed for epilogue vectorization and it
7514 // should be removed once induction resume value creation is done
7515 // directly in VPlan.
7516 for (auto &R : make_early_inc_range(Range&: *Plan.getEntry())) {
7517 // Re-use frozen values from the main plan for Freeze VPInstructions in the
7518 // epilogue plan. This ensures all users use the same frozen value.
7519 auto *VPI = dyn_cast<VPInstruction>(Val: &R);
7520 if (VPI && VPI->getOpcode() == Instruction::Freeze) {
7521 VPI->replaceAllUsesWith(New: Plan.getOrAddLiveIn(
7522 V: ToFrozen.lookup(Val: VPI->getOperand(N: 0)->getLiveInIRValue())));
7523 continue;
7524 }
7525
7526 // Re-use the trip count and steps expanded for the main loop, as
7527 // skeleton creation needs it as a value that dominates both the scalar
7528 // and vector epilogue loops
7529 auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(Val: &R);
7530 if (!ExpandR)
7531 continue;
7532 assert(ExpandedSCEVs.contains(ExpandR->getSCEV()) &&
7533 "Epilogue plan needs a SCEV not expanded for the main loop");
7534 VPValue *ExpandedVal =
7535 Plan.getOrAddLiveIn(V: ExpandedSCEVs.lookup(Val: ExpandR->getSCEV()));
7536 ExpandR->replaceAllUsesWith(New: ExpandedVal);
7537 if (Plan.getTripCount() == ExpandR)
7538 Plan.resetTripCount(NewTripCount: ExpandedVal);
7539 ExpandR->eraseFromParent();
7540 }
7541
7542 auto VScale = Config.getVScaleForTuning();
7543 unsigned MainLoopStep =
7544 estimateElementCount(VF: EPI.MainLoopVF * EPI.MainLoopUF, VScale);
7545 unsigned EpilogueLoopStep =
7546 estimateElementCount(VF: EPI.EpilogueVF * EPI.EpilogueUF, VScale);
7547 RUN_VPLAN_PASS(VPlanTransforms::addMinimumVectorEpilogueIterationCheck, Plan,
7548 EPI.VectorTripCount, Plan.requiresScalarEpilogue(),
7549 EPI.EpilogueVF, EPI.EpilogueUF, MainLoopStep, EpilogueLoopStep,
7550 SE);
7551
7552 return InstsToMove;
7553}
7554
7555static void
7556fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
7557 VPlan &BestEpiPlan,
7558 ArrayRef<VPInstruction *> ResumeValues) {
7559 // Fix resume values from the additional bypass block.
7560 BasicBlock *PH = L->getLoopPreheader();
7561 for (auto *Pred : predecessors(BB: PH)) {
7562 for (PHINode &Phi : PH->phis()) {
7563 if (Phi.getBasicBlockIndex(BB: Pred) != -1)
7564 continue;
7565 Phi.addIncoming(V: Phi.getIncomingValueForBlock(BB: BypassBlock), BB: Pred);
7566 }
7567 }
7568 auto *ScalarPH = cast<VPIRBasicBlock>(Val: BestEpiPlan.getScalarPreheader());
7569 if (ScalarPH->hasPredecessors()) {
7570 // Fix resume values for inductions and reductions from the additional
7571 // bypass block using the incoming values from the main loop's resume phis.
7572 // ResumeValues correspond 1:1 with the scalar loop header phis.
7573 for (auto [ResumeV, HeaderPhi] :
7574 zip(t&: ResumeValues, u: BestEpiPlan.getScalarHeader()->phis())) {
7575 auto *HeaderPhiR = cast<VPIRPhi>(Val: &HeaderPhi);
7576 auto *EpiResumePhi =
7577 cast<PHINode>(Val: HeaderPhiR->getIRPhi().getIncomingValueForBlock(BB: PH));
7578 if (EpiResumePhi->getBasicBlockIndex(BB: BypassBlock) == -1)
7579 continue;
7580 auto *MainResumePhi = cast<PHINode>(Val: ResumeV->getUnderlyingValue());
7581 EpiResumePhi->setIncomingValueForBlock(
7582 BB: BypassBlock, V: MainResumePhi->getIncomingValueForBlock(BB: BypassBlock));
7583 }
7584 }
7585}
7586
7587/// Connect the epilogue vector loop generated for \p EpiPlan to the main vector
7588/// loop, after both plans have executed, updating branches from the iteration
7589/// and runtime checks of the main loop, as well as updating various phis. \p
7590/// InstsToMove contains instructions that need to be moved to the preheader of
7591/// the epilogue vector loop.
7592static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
7593 EpilogueLoopVectorizationInfo &EPI,
7594 DominatorTree *DT,
7595 GeneratedRTChecks &Checks,
7596 ArrayRef<Instruction *> InstsToMove,
7597 ArrayRef<VPInstruction *> ResumeValues) {
7598 BasicBlock *VecEpilogueIterationCountCheck =
7599 cast<VPIRBasicBlock>(Val: EpiPlan.getEntry())->getIRBasicBlock();
7600
7601 BasicBlock *VecEpiloguePreHeader =
7602 cast<CondBrInst>(Val: VecEpilogueIterationCountCheck->getTerminator())
7603 ->getSuccessor(i: 1);
7604 // Adjust the control flow taking the state info from the main loop
7605 // vectorization into account.
7606 assert(EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck &&
7607 "expected this to be saved from the previous pass.");
7608 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
7609
7610 // Helper to redirect an edge from \p BB to \p VecEpilogueIterationCountCheck
7611 // to \p NewSucc instead, updating the DomTree.
7612 auto RedirectEdge = [&](BasicBlock *BB, BasicBlock *NewSucc) {
7613 BB->getTerminator()->replaceUsesOfWith(From: VecEpilogueIterationCountCheck,
7614 To: NewSucc);
7615 DTU.applyUpdates(
7616 Updates: {{DominatorTree::Delete, BB, VecEpilogueIterationCountCheck},
7617 {DominatorTree::Insert, BB, NewSucc}});
7618 };
7619
7620 RedirectEdge(EPI.MainLoopIterationCountCheck, VecEpiloguePreHeader);
7621
7622 BasicBlock *ScalarPH =
7623 cast<VPIRBasicBlock>(Val: EpiPlan.getScalarPreheader())->getIRBasicBlock();
7624 RedirectEdge(EPI.EpilogueIterationCountCheck, ScalarPH);
7625
7626 // Adjust the terminators of runtime check blocks and phis using them.
7627 BasicBlock *SCEVCheckBlock = Checks.getSCEVChecks().second;
7628 BasicBlock *MemCheckBlock = Checks.getMemRuntimeChecks().second;
7629 if (SCEVCheckBlock)
7630 RedirectEdge(SCEVCheckBlock, ScalarPH);
7631 if (MemCheckBlock)
7632 RedirectEdge(MemCheckBlock, ScalarPH);
7633
7634 // The vec.epilog.iter.check block may contain Phi nodes from inductions
7635 // or reductions which merge control-flow from the latch block and the
7636 // middle block. Update the incoming values here and move the Phi into the
7637 // preheader.
7638 SmallVector<PHINode *, 4> PhisInBlock(
7639 llvm::make_pointer_range(Range: VecEpilogueIterationCountCheck->phis()));
7640
7641 for (PHINode *Phi : PhisInBlock) {
7642 Phi->moveBefore(InsertPos: VecEpiloguePreHeader->getFirstNonPHIIt());
7643 Phi->replaceIncomingBlockWith(
7644 Old: VecEpilogueIterationCountCheck->getSinglePredecessor(),
7645 New: VecEpilogueIterationCountCheck);
7646
7647 // If the phi doesn't have an incoming value from the
7648 // EpilogueIterationCountCheck, we are done. Otherwise remove the
7649 // incoming value and also those from other check blocks. This is needed
7650 // for reduction phis only.
7651 if (none_of(Range: Phi->blocks(), P: [&](BasicBlock *IncB) {
7652 return EPI.EpilogueIterationCountCheck == IncB;
7653 }))
7654 continue;
7655 for (BasicBlock *BB :
7656 {EPI.EpilogueIterationCountCheck, SCEVCheckBlock, MemCheckBlock}) {
7657 if (BB)
7658 Phi->removeIncomingValue(BB);
7659 }
7660 }
7661
7662 auto IP = VecEpiloguePreHeader->getFirstNonPHIIt();
7663 for (auto *I : InstsToMove)
7664 I->moveBefore(InsertPos: IP);
7665
7666 // VecEpilogueIterationCountCheck conditionally skips over the epilogue loop
7667 // after executing the main loop. We need to update the resume values of
7668 // inductions and reductions during epilogue vectorization.
7669 fixScalarResumeValuesFromBypass(BypassBlock: VecEpilogueIterationCountCheck, L, BestEpiPlan&: EpiPlan,
7670 ResumeValues);
7671
7672 // Remove dead phis that were moved to the epilogue preheader but are unused
7673 // (e.g., resume phis for inductions not widened in the epilogue vector loop).
7674 for (PHINode &Phi : make_early_inc_range(Range: VecEpiloguePreHeader->phis()))
7675 if (Phi.use_empty())
7676 Phi.eraseFromParent();
7677}
7678
7679bool LoopVectorizePass::processLoop(Loop *L) {
7680 assert((EnableVPlanNativePath || L->isInnermost()) &&
7681 "VPlan-native path is not enabled. Only process inner loops.");
7682
7683 LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in '"
7684 << L->getHeader()->getParent()->getName() << "' from "
7685 << L->getLocStr() << "\n");
7686
7687 LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE, TTI);
7688
7689 LLVM_DEBUG(
7690 dbgs() << "LV: Loop hints:"
7691 << " force="
7692 << (Hints.getForce() == LoopVectorizeHints::FK_Disabled
7693 ? "disabled"
7694 : (Hints.getForce() == LoopVectorizeHints::FK_Enabled
7695 ? "enabled"
7696 : "?"))
7697 << " width=" << Hints.getWidth()
7698 << " interleave=" << Hints.getInterleave() << "\n");
7699
7700 // Function containing loop
7701 Function *F = L->getHeader()->getParent();
7702
7703 // Looking at the diagnostic output is the only way to determine if a loop
7704 // was vectorized (other than looking at the IR or machine code), so it
7705 // is important to generate an optimization remark for each loop. Most of
7706 // these messages are generated as OptimizationRemarkAnalysis. Remarks
7707 // generated as OptimizationRemark and OptimizationRemarkMissed are
7708 // less verbose reporting vectorized loops and unvectorized loops that may
7709 // benefit from vectorization, respectively.
7710
7711 if (!Hints.allowVectorization(F, L, VectorizeOnlyWhenForced)) {
7712 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
7713 return false;
7714 }
7715
7716 PredicatedScalarEvolution PSE(*SE, *L);
7717
7718 // Query this against the original loop and save it here because the profile
7719 // of the original loop header may change as the transformation happens.
7720 bool OptForSize = llvm::shouldOptimizeForSize(
7721 BB: L->getHeader(), PSI,
7722 BFI: PSI && PSI->hasProfileSummary() ? &GetBFI() : nullptr,
7723 QueryType: PGSOQueryType::IRPass);
7724
7725 // Check if it is legal to vectorize the loop.
7726 LoopVectorizationRequirements Requirements;
7727 LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE,
7728 &Requirements, &Hints, DB, AC,
7729 /*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
7730 if (!LVL.canVectorize(UseVPlanNativePath: EnableVPlanNativePath)) {
7731 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
7732 Hints.emitRemarkWithHints();
7733 return false;
7734 }
7735
7736 bool IsInnerLoop = L->isInnermost();
7737
7738 // Outer loops require a computable trip count.
7739 if (!IsInnerLoop && isa<SCEVCouldNotCompute>(Val: PSE.getBackedgeTakenCount())) {
7740 LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n");
7741 return false;
7742 }
7743
7744 if (LVL.hasUncountableEarlyExit()) {
7745 if (!EnableEarlyExitVectorization) {
7746 reportVectorizationFailure(DebugMsg: "Auto-vectorization of loops with uncountable "
7747 "early exit is not enabled",
7748 ORETag: "UncountableEarlyExitLoopsDisabled", ORE, TheLoop: L);
7749 return false;
7750 }
7751 if (LVL.hasUncountableExitWithSideEffects() &&
7752 !EnableEarlyExitVectorizationWithSideEffects) {
7753 reportVectorizationFailure(DebugMsg: "Auto-vectorization of loops with uncountable "
7754 "early exit and side effects is not enabled",
7755 ORETag: "UncountableEarlyExitSideEffectLoopsDisabled",
7756 ORE, TheLoop: L);
7757 return false;
7758 }
7759 }
7760
7761 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
7762 bool UseInterleaved =
7763 IsInnerLoop && TTI->enableInterleavedAccessVectorization();
7764
7765 // If an override option has been passed in for interleaved accesses, use it.
7766 if (EnableInterleavedMemAccesses.getNumOccurrences() > 0)
7767 UseInterleaved = IsInnerLoop && EnableInterleavedMemAccesses;
7768
7769 // Analyze interleaved memory accesses.
7770 if (UseInterleaved)
7771 IAI.analyzeInterleaving(EnableMaskedInterleavedGroup: useMaskedInterleavedAccesses(TTI: *TTI));
7772
7773 if (LVL.hasUncountableEarlyExit()) {
7774 BasicBlock *LoopLatch = L->getLoopLatch();
7775 if (IAI.requiresScalarEpilogue() ||
7776 any_of(Range: LVL.getCountableExitingBlocks(), P: not_equal_to(Arg&: LoopLatch))) {
7777 reportVectorizationFailure(DebugMsg: "Auto-vectorization of early exit loops "
7778 "requiring a scalar epilogue is unsupported",
7779 ORETag: "UncountableEarlyExitUnsupported", ORE, TheLoop: L);
7780 return false;
7781 }
7782 }
7783
7784 // Check the function attributes and profiles to find out if this function
7785 // should be optimized for size.
7786 EpilogueLowering SEL =
7787 getEpilogueLowering(F, L, Hints, OptForSize, TTI, TLI, LVL, IAI: &IAI);
7788
7789 // Check the loop for a trip count threshold: vectorize loops with a tiny trip
7790 // count by optimizing for size, to minimize overheads.
7791 auto ExpectedTC = getSmallBestKnownTC(PSE, L);
7792 if (ExpectedTC && ExpectedTC->isFixed() &&
7793 ExpectedTC->getFixedValue() < TinyTripCountVectorThreshold) {
7794 LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
7795 << "This loop is worth vectorizing only if no scalar "
7796 << "iteration overheads are incurred.");
7797 if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
7798 LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
7799 else {
7800 LLVM_DEBUG(dbgs() << "\n");
7801 // Tail-folded loops are efficient even when the loop
7802 // iteration count is low. However, setting the epilogue policy to
7803 // `CM_EpilogueNotAllowedLowTripLoop` prevents vectorizing loops
7804 // with runtime checks. It's more effective to let
7805 // `isOutsideLoopWorkProfitable` determine if vectorization is
7806 // beneficial for the loop.
7807 if (SEL != CM_EpilogueNotNeededFoldTail)
7808 SEL = CM_EpilogueNotAllowedLowTripLoop;
7809 }
7810 }
7811
7812 // Check the function attributes to see if implicit floats or vectors are
7813 // allowed.
7814 if (F->hasFnAttribute(Kind: Attribute::NoImplicitFloat)) {
7815 reportVectorizationFailure(
7816 DebugMsg: "Can't vectorize when the NoImplicitFloat attribute is used",
7817 OREMsg: "loop not vectorized due to NoImplicitFloat attribute",
7818 ORETag: "NoImplicitFloat", ORE, TheLoop: L);
7819 Hints.emitRemarkWithHints();
7820 return false;
7821 }
7822
7823 // Check if the target supports potentially unsafe FP vectorization.
7824 // FIXME: Add a check for the type of safety issue (denormal, signaling)
7825 // for the target we're vectorizing for, to make sure none of the
7826 // additional fp-math flags can help.
7827 if (Hints.isPotentiallyUnsafe() &&
7828 TTI->isFPVectorizationPotentiallyUnsafe()) {
7829 reportVectorizationFailure(
7830 DebugMsg: "Potentially unsafe FP op prevents vectorization",
7831 OREMsg: "loop not vectorized due to unsafe FP support.", ORETag: "UnsafeFP", ORE, TheLoop: L);
7832 Hints.emitRemarkWithHints();
7833 return false;
7834 }
7835
7836 bool AllowOrderedReductions;
7837 // If the flag is set, use that instead and override the TTI behaviour.
7838 if (ForceOrderedReductions.getNumOccurrences() > 0)
7839 AllowOrderedReductions = ForceOrderedReductions;
7840 else
7841 AllowOrderedReductions = TTI->enableOrderedReductions();
7842 if (!LVL.canVectorizeFPMath(EnableStrictReductions: AllowOrderedReductions)) {
7843 ORE->emit(RemarkBuilder: [&]() {
7844 auto *ExactFPMathInst = Requirements.getExactFPInst();
7845 return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE, "CantReorderFPOps",
7846 ExactFPMathInst->getDebugLoc(),
7847 ExactFPMathInst->getParent())
7848 << "loop not vectorized: cannot prove it is safe to reorder "
7849 "floating-point operations";
7850 });
7851 LLVM_DEBUG(dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
7852 "reorder floating-point operations\n");
7853 Hints.emitRemarkWithHints();
7854 return false;
7855 }
7856
7857 // Use the cost model.
7858 VFSelectionContext Config(*TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
7859 OptForSize);
7860 // Use the planner for vectorization.
7861 LoopVectorizationPlanner LVP(
7862 L, LI, DT, TLI, *TTI, &LVL,
7863 std::make_unique<LoopVectorizationCostModel>(
7864 args&: SEL, args&: L, args&: PSE, args&: LI, args: &LVL, args&: *TTI, args&: TLI, args&: AC, args&: ORE, args&: GetBFI, args&: F, args&: IAI, args&: Config),
7865 Config, IAI, PSE, ORE, GetBPI);
7866
7867 EpilogueLowering EpilogueTailLoweringStatus =
7868 getEpilogueTailLowering(MainCM: LVP.getCostModel(), L, ORE, LVL, Hints);
7869 if (EpilogueTailLoweringStatus ==
7870 EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
7871 // TODO: Apply tail-folding on the vectorized epilogue loop.
7872 LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
7873 reportVectorizationInfo(
7874 Msg: "The epilogue-tail-folding policy prefer-fold-tail is not supported "
7875 "yet, fall back to a normal epilogue",
7876 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
7877 }
7878
7879 // Get user vectorization factor and interleave count.
7880 ElementCount UserVF = Hints.getWidth();
7881 unsigned UserIC = Hints.getInterleave();
7882 // Outer loops don't have LoopAccessInfo, so skip the safety check and reset
7883 // UserIC (interleaving is not supported for outer loops).
7884 if (!IsInnerLoop)
7885 UserIC = 0;
7886 else if (UserIC > 1 && !LVL.isSafeForAnyVectorWidth())
7887 UserIC = 1;
7888
7889 // Plan how to best vectorize.
7890 LVP.plan(UserVF, UserIC);
7891 auto [VF, BestPlanPtr] = LVP.computeBestVF();
7892 unsigned IC = 1;
7893
7894 // For VPlan build stress testing of outer loops, bail after plan
7895 // construction.
7896 if (!IsInnerLoop && VPlanBuildOuterloopStressTest)
7897 return false;
7898
7899 if (IsInnerLoop && ORE->allowExtraAnalysis(LV_NAME))
7900 LVP.emitInvalidCostRemarks(ORE);
7901
7902 assert((IsInnerLoop || !LVP.getCostModel().maskPartialAliasing()) &&
7903 "Did not expect to alias-mask outer loop");
7904
7905 GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
7906 LVP.getCostModel().maskPartialAliasing());
7907 if (IsInnerLoop && LVP.hasPlanWithVF(VF: VF.Width)) {
7908 // Select the interleave count.
7909 IC = LVP.selectInterleaveCount(Plan&: *BestPlanPtr, VF: VF.Width, LoopCost: VF.Cost);
7910
7911 unsigned SelectedIC = std::max(a: IC, b: UserIC);
7912 // Optimistically generate runtime checks if they are needed. Drop them if
7913 // they turn out to not be profitable.
7914 if (VF.Width.isVector() || SelectedIC > 1) {
7915 Checks.create(L, LAI: *LVL.getLAI(), UnionPred: PSE.getPredicate(), VF: VF.Width, IC: SelectedIC,
7916 ORE&: *ORE);
7917
7918 // Bail out early if either the SCEV or memory runtime checks are known to
7919 // fail. In that case, the vector loop would never execute.
7920 using namespace llvm::PatternMatch;
7921 if (Checks.getSCEVChecks().first &&
7922 match(V: Checks.getSCEVChecks().first, P: m_One()))
7923 return false;
7924 if (Checks.getMemRuntimeChecks().first &&
7925 match(V: Checks.getMemRuntimeChecks().first, P: m_One()))
7926 return false;
7927 }
7928
7929 // Check if it is profitable to vectorize with runtime checks.
7930 bool ForceVectorization =
7931 Hints.getForce() == LoopVectorizeHints::FK_Enabled;
7932 VPCostContext CostCtx(*TLI, *BestPlanPtr, LVP.getCostModel(), Config,
7933 /*ReusePrintingSlotTracker=*/true);
7934 if (!ForceVectorization &&
7935 !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx, Plan&: *BestPlanPtr,
7936 SEL, VScale: Config.getVScaleForTuning())) {
7937 ORE->emit(RemarkBuilder: [&]() {
7938 return OptimizationRemarkAnalysisAliasing(
7939 DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
7940 L->getHeader())
7941 << "loop not vectorized: cannot prove it is safe to reorder "
7942 "memory operations";
7943 });
7944 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
7945 Hints.emitRemarkWithHints();
7946 return false;
7947 }
7948 }
7949
7950 // Identify the diagnostic messages that should be produced.
7951 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
7952 bool VectorizeLoop = true, InterleaveLoop = true;
7953 if (VF.Width.isScalar()) {
7954 LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n");
7955 VecDiagMsg = {
7956 "VectorizationNotBeneficial",
7957 "the cost-model indicates that vectorization is not beneficial"};
7958 VectorizeLoop = false;
7959 }
7960
7961 if (UserIC == 1 && Hints.getInterleave() > 1) {
7962 assert(!LVL.isSafeForAnyVectorWidth() &&
7963 "UserIC should only be ignored due to unsafe dependencies");
7964 LLVM_DEBUG(dbgs() << "LV: Ignoring user-specified interleave count.\n");
7965 IntDiagMsg = {"InterleavingUnsafe",
7966 "Ignoring user-specified interleave count due to possibly "
7967 "unsafe dependencies in the loop."};
7968 InterleaveLoop = false;
7969 } else if (!LVP.hasPlanWithVF(VF: VF.Width) && UserIC > 1) {
7970 // Tell the user interleaving was avoided up-front, despite being explicitly
7971 // requested.
7972 LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
7973 "interleaving should be avoided up front\n");
7974 IntDiagMsg = {"InterleavingAvoided",
7975 "Ignoring UserIC, because interleaving was avoided up front"};
7976 InterleaveLoop = false;
7977 } else if (IC == 1 && UserIC <= 1) {
7978 // Tell the user interleaving is not beneficial.
7979 LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
7980 IntDiagMsg = {
7981 "InterleavingNotBeneficial",
7982 "the cost-model indicates that interleaving is not beneficial"};
7983 InterleaveLoop = false;
7984 if (UserIC == 1) {
7985 IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
7986 IntDiagMsg.second +=
7987 " and is explicitly disabled or interleave count is set to 1";
7988 }
7989 } else if (IC > 1 && UserIC == 1) {
7990 // Tell the user interleaving is beneficial, but it explicitly disabled.
7991 LLVM_DEBUG(dbgs() << "LV: Interleaving is beneficial but is explicitly "
7992 "disabled.\n");
7993 IntDiagMsg = {"InterleavingBeneficialButDisabled",
7994 "the cost-model indicates that interleaving is beneficial "
7995 "but is explicitly disabled or interleave count is set to 1"};
7996 InterleaveLoop = false;
7997 }
7998
7999 // If there is a histogram in the loop, do not just interleave without
8000 // vectorizing. The order of operations will be incorrect without the
8001 // histogram intrinsics, which are only used for recipes with VF > 1.
8002 if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
8003 LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
8004 << "to histogram operations.\n");
8005 IntDiagMsg = {
8006 "HistogramPreventsScalarInterleaving",
8007 "Unable to interleave without vectorization due to constraints on "
8008 "the order of histogram operations"};
8009 InterleaveLoop = false;
8010 }
8011
8012 // Override IC if user provided an interleave count.
8013 IC = UserIC > 0 ? UserIC : IC;
8014
8015 if (LVP.getCostModel().maskPartialAliasing()) {
8016 LLVM_DEBUG(
8017 dbgs()
8018 << "LV: Not interleaving due to partial aliasing vectorization.\n");
8019 IntDiagMsg = {
8020 "PartialAliasingVectorization",
8021 "Unable to interleave due to partial aliasing vectorization."};
8022 InterleaveLoop = false;
8023 IC = 1;
8024 }
8025
8026 // FIXME: Enable interleaving for EE-with-side-effects.
8027 if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) {
8028 LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n");
8029 IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving",
8030 "Unable to interleave due to early exit with side effects."};
8031 InterleaveLoop = false;
8032 IC = 1;
8033 }
8034
8035 // Emit diagnostic messages, if any.
8036 if (!VectorizeLoop && !InterleaveLoop) {
8037 // Do not vectorize or interleaving the loop.
8038 ORE->emit(RemarkBuilder: [&]() {
8039 return OptimizationRemarkMissed(LV_NAME, VecDiagMsg.first,
8040 L->getStartLoc(), L->getHeader())
8041 << VecDiagMsg.second;
8042 });
8043 ORE->emit(RemarkBuilder: [&]() {
8044 return OptimizationRemarkMissed(LV_NAME, IntDiagMsg.first,
8045 L->getStartLoc(), L->getHeader())
8046 << IntDiagMsg.second;
8047 });
8048 return false;
8049 }
8050
8051 if (!VectorizeLoop && InterleaveLoop) {
8052 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
8053 ORE->emit(RemarkBuilder: [&]() {
8054 return OptimizationRemarkAnalysis(LV_NAME, VecDiagMsg.first,
8055 L->getStartLoc(), L->getHeader())
8056 << VecDiagMsg.second;
8057 });
8058 } else if (VectorizeLoop && !InterleaveLoop) {
8059 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
8060 << ") in " << L->getLocStr() << '\n');
8061 ORE->emit(RemarkBuilder: [&]() {
8062 return OptimizationRemarkAnalysis(LV_NAME, IntDiagMsg.first,
8063 L->getStartLoc(), L->getHeader())
8064 << IntDiagMsg.second;
8065 });
8066 } else if (VectorizeLoop && InterleaveLoop) {
8067 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
8068 << ") in " << L->getLocStr() << '\n');
8069 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
8070 }
8071
8072 // Report the vectorization decision.
8073 if (VF.Width.isScalar()) {
8074 using namespace ore;
8075 assert(IC > 1);
8076 ORE->emit(RemarkBuilder: [&]() {
8077 return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(),
8078 L->getHeader())
8079 << "interleaved loop (interleaved count: "
8080 << NV("InterleaveCount", IC) << ")";
8081 });
8082 } else {
8083 // Report the vectorization decision.
8084 reportVectorization(ORE, TheLoop: L, VFWidth: VF.Width, IC);
8085 }
8086 if (ORE->allowExtraAnalysis(LV_NAME))
8087 checkMixedPrecision(L, ORE);
8088
8089 // If we decided that it is *legal* to interleave or vectorize the loop, then
8090 // do it.
8091
8092 // Whether a scalar epilogue may be created is decided by the epilogue
8093 // lowering policy.
8094 // TODO: Also move check to be based on VPlan.
8095 bool ScalarEpilogueAllowed = LVP.getCostModel().isEpilogueAllowed();
8096
8097 // Destroy the cost model before executing any plan, so that code generation
8098 // cannot rely on cost-modeling decisions.
8099 LVP.clearCostModel();
8100
8101 VPlan &BestPlan = *BestPlanPtr;
8102 // Consider vectorizing the epilogue too if it's profitable.
8103 std::unique_ptr<VPlan> EpiPlan =
8104 LVP.selectBestEpiloguePlan(MainPlan&: BestPlan, MainLoopVF: VF.Width, IC, ScalarEpilogueAllowed);
8105 bool HasBranchWeights =
8106 hasBranchWeightMD(I: *L->getLoopLatch()->getTerminator());
8107 if (EpiPlan) {
8108 VPlan &BestEpiPlan = *EpiPlan;
8109 VPlan &BestMainPlan = BestPlan;
8110 ElementCount EpilogueVF = BestEpiPlan.getSingleVF();
8111
8112 // The first pass vectorizes the main loop and creates a scalar epilogue
8113 // to be vectorized by executing the plan (potentially with a different
8114 // factor) again shortly afterwards.
8115 BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
8116 BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
8117 SmallVector<VPInstruction *> ResumeValues =
8118 preparePlanForMainVectorLoop(MainPlan&: BestMainPlan, EpiPlan&: BestEpiPlan);
8119 EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF, 1);
8120
8121 // Add minimum iteration check for the epilogue plan, followed by runtime
8122 // checks for the main plan.
8123 LVP.addMinimumIterationCheck(Plan&: BestMainPlan, VF: EPI.EpilogueVF, UF: EPI.EpilogueUF,
8124 MinProfitableTripCount: ElementCount::getFixed(MinVal: 0));
8125 LVP.attachRuntimeChecks(Plan&: BestMainPlan, RTChecks&: Checks, HasBranchWeights);
8126 RUN_VPLAN_PASS(
8127 VPlanTransforms::addIterationCountCheckBlock, BestMainPlan,
8128 EPI.MainLoopVF, EPI.MainLoopUF, BestMainPlan.requiresScalarEpilogue(),
8129 L, HasBranchWeights ? MinItersBypassWeights : nullptr,
8130 L->getLoopPredecessor()->getTerminator()->getDebugLoc(), PSE);
8131
8132 EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TTI, AC, EPI, Checks,
8133 BestMainPlan);
8134 auto ExpandedSCEVs = LVP.executePlan(
8135 BestVF: EPI.MainLoopVF, BestUF: EPI.MainLoopUF, BestVPlan&: BestMainPlan, ILV&: MainILV, DT,
8136 EpilogueVecKind: LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
8137 ++LoopsVectorized;
8138
8139 // Derive EPI fields from VPlan-generated IR.
8140 BasicBlock *EntryBB =
8141 cast<VPIRBasicBlock>(Val: BestMainPlan.getEntry())->getIRBasicBlock();
8142 EntryBB->setName("iter.check");
8143 EPI.EpilogueIterationCountCheck = EntryBB;
8144 // The check chain is: Entry -> [SCEV] -> [Mem] -> MainCheck -> VecPH.
8145 // MainCheck is the non-bypass successor of the last runtime check block
8146 // (or Entry if there are no runtime checks).
8147 BasicBlock *LastCheck = EntryBB;
8148 if (BasicBlock *MemBB = Checks.getMemRuntimeChecks().second)
8149 LastCheck = MemBB;
8150 else if (BasicBlock *SCEVBB = Checks.getSCEVChecks().second)
8151 LastCheck = SCEVBB;
8152 BasicBlock *ScalarPH = L->getLoopPreheader();
8153 auto *BI = cast<CondBrInst>(Val: LastCheck->getTerminator());
8154 EPI.MainLoopIterationCountCheck =
8155 BI->getSuccessor(i: BI->getSuccessor(i: 0) == ScalarPH);
8156
8157 // Second pass vectorizes the epilogue and adjusts the control flow
8158 // edges from the first pass.
8159 EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TTI, AC, EPI,
8160 Checks, BestEpiPlan);
8161 SmallVector<Instruction *> InstsToMove = preparePlanForEpilogueVectorLoop(
8162 MainPlan&: BestMainPlan, Plan&: BestEpiPlan, L, ExpandedSCEVs, EPI, LVP, Config,
8163 SE&: *PSE.getSE(), ResumeValues);
8164 RUN_VPLAN_PASS(VPlanTransforms::simplifyLiveInsWithSCEV, BestEpiPlan, PSE);
8165 LVP.executePlan(
8166 BestVF: EPI.EpilogueVF, BestUF: EPI.EpilogueUF, BestVPlan&: BestEpiPlan, ILV&: EpilogILV, DT,
8167 EpilogueVecKind: LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
8168 connectEpilogueVectorLoop(EpiPlan&: BestEpiPlan, L, EPI, DT, Checks, InstsToMove,
8169 ResumeValues);
8170 ++LoopsEpilogueVectorized;
8171 } else {
8172 InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, Checks,
8173 BestPlan);
8174 LVP.addMinimumIterationCheck(Plan&: BestPlan, VF: VF.Width, UF: IC,
8175 MinProfitableTripCount: VF.MinProfitableTripCount);
8176 LVP.attachRuntimeChecks(Plan&: BestPlan, RTChecks&: Checks, HasBranchWeights);
8177
8178 if (!IsInnerLoop)
8179 LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" << F->getName()
8180 << "\"\n");
8181 LVP.executePlan(BestVF: VF.Width, BestUF: IC, BestVPlan&: BestPlan, ILV&: LB, DT);
8182 ++LoopsVectorized;
8183 }
8184
8185 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
8186 "DT not preserved correctly");
8187
8188 return true;
8189}
8190
8191LoopVectorizeResult LoopVectorizePass::runImpl(Function &F) {
8192 CFGChanged = false;
8193
8194 // Don't attempt if
8195 // 1. the target claims to have no vector registers, and
8196 // 2. interleaving won't help ILP.
8197 //
8198 // The second condition is necessary because, even if the target has no
8199 // vector registers, loop vectorization may still enable scalar
8200 // interleaving.
8201 if (!TTI->getNumberOfRegisters(ClassID: TTI->getRegisterClassForType(Vector: true)) &&
8202 (TTI->getMaxInterleaveFactor(VF: ElementCount::getFixed(MinVal: 1), HasUnorderedReductions: false) < 2 ||
8203 TTI->getMaxInterleaveFactor(VF: ElementCount::getFixed(MinVal: 1), HasUnorderedReductions: true) < 2))
8204 return LoopVectorizeResult(false, false);
8205
8206 bool Changed = false;
8207
8208 // The vectorizer requires loops to be in simplified form.
8209 // Since simplification may add new inner loops, it has to run before the
8210 // legality and profitability checks. This means running the loop vectorizer
8211 // will simplify all loops, regardless of whether anything end up being
8212 // vectorized.
8213 for (const auto &L : *LI)
8214 Changed |= CFGChanged |=
8215 simplifyLoop(L, DT, LI, SE, AC, MSSAU: nullptr, PreserveLCSSA: false /* PreserveLCSSA */);
8216
8217 // Build up a worklist of inner-loops to vectorize. This is necessary as
8218 // the act of vectorizing or partially unrolling a loop creates new loops
8219 // and can invalidate iterators across the loops.
8220 SmallVector<Loop *, 8> Worklist;
8221
8222 for (Loop *L : *LI)
8223 collectSupportedLoops(L&: *L, LI, ORE, V&: Worklist);
8224
8225 LoopsAnalyzed += Worklist.size();
8226
8227 // Now walk the identified inner loops.
8228 while (!Worklist.empty()) {
8229 Loop *L = Worklist.pop_back_val();
8230
8231 // For the inner loops we actually process, form LCSSA to simplify the
8232 // transform.
8233 Changed |= formLCSSARecursively(L&: *L, DT: *DT, LI, SE);
8234
8235 Changed |= CFGChanged |= processLoop(L);
8236
8237 if (Changed) {
8238 LAIs->clear();
8239
8240#ifndef NDEBUG
8241 if (VerifySCEV)
8242 SE->verify();
8243#endif
8244 }
8245 }
8246
8247 // Verify once per function rather than once per processed loop, which would
8248 // make the pass quadratic in the number of loops.
8249 assert((!Changed || !verifyFunction(F, &dbgs())) &&
8250 "Invalid IR produced by LoopVectorize");
8251
8252 // Process each loop nest in the function.
8253 return LoopVectorizeResult(Changed, CFGChanged);
8254}
8255
8256PreservedAnalyses LoopVectorizePass::run(Function &F,
8257 FunctionAnalysisManager &AM) {
8258 LI = &AM.getResult<LoopAnalysis>(IR&: F);
8259 // There are no loops in the function. Return before computing other
8260 // expensive analyses.
8261 if (LI->empty())
8262 return PreservedAnalyses::all();
8263 SE = &AM.getResult<ScalarEvolutionAnalysis>(IR&: F);
8264 TTI = &AM.getResult<TargetIRAnalysis>(IR&: F);
8265 DT = &AM.getResult<DominatorTreeAnalysis>(IR&: F);
8266 TLI = &AM.getResult<TargetLibraryAnalysis>(IR&: F);
8267 AC = &AM.getResult<AssumptionAnalysis>(IR&: F);
8268 DB = &AM.getResult<DemandedBitsAnalysis>(IR&: F);
8269 ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F);
8270 LAIs = &AM.getResult<LoopAccessAnalysis>(IR&: F);
8271 AA = &AM.getResult<AAManager>(IR&: F);
8272
8273 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
8274 PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(IR&: *F.getParent());
8275 // CycleInfo cached by an earlier pass is invalidated when the CFG changes.
8276 // Both BlockFrequencyAnalysis and BranchProbabilityAnalysis depend on it, so
8277 // drop the stale result before either is (re-)computed.
8278 auto ClearStaleCycleInfo = [this, &AM, &F] {
8279 if (CFGChanged && AM.getCachedResult<CycleAnalysis>(IR&: F))
8280 AM.clearAnalysis<CycleAnalysis>(IR&: F);
8281 };
8282 GetBFI = [&AM, &F, ClearStaleCycleInfo]() -> BlockFrequencyInfo & {
8283 ClearStaleCycleInfo();
8284 return AM.getResult<BlockFrequencyAnalysis>(IR&: F);
8285 };
8286 GetBPI = [&AM, &F, ClearStaleCycleInfo]() -> const BranchProbabilityInfo & {
8287 ClearStaleCycleInfo();
8288 return AM.getResult<BranchProbabilityAnalysis>(IR&: F);
8289 };
8290 LoopVectorizeResult Result = runImpl(F);
8291 if (!Result.MadeAnyChange)
8292 return PreservedAnalyses::all();
8293 PreservedAnalyses PA;
8294
8295 if (isAssignmentTrackingEnabled(M: *F.getParent())) {
8296 for (auto &BB : F)
8297 RemoveRedundantDbgInstrs(BB: &BB);
8298 }
8299
8300 PA.preserve<LoopAnalysis>();
8301 PA.preserve<DominatorTreeAnalysis>();
8302 PA.preserve<ScalarEvolutionAnalysis>();
8303 PA.preserve<LoopAccessAnalysis>();
8304
8305 if (Result.MadeCFGChange) {
8306 // Making CFG changes likely means a loop got vectorized. Indicate that
8307 // extra simplification passes should be run.
8308 // TODO: MadeCFGChanges is not a prefect proxy. Extra passes should only
8309 // be run if runtime checks have been added.
8310 AM.getResult<ShouldRunExtraVectorPasses>(IR&: F);
8311 PA.preserve<ShouldRunExtraVectorPasses>();
8312 } else {
8313 PA.preserveSet<CFGAnalyses>();
8314 }
8315 return PA;
8316}
8317
8318void LoopVectorizePass::printPipeline(
8319 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
8320 static_cast<PassInfoMixin<LoopVectorizePass> *>(this)->printPipeline(
8321 OS, MapClassName2PassName);
8322
8323 OS << '<';
8324 OS << (InterleaveOnlyWhenForced ? "" : "no-") << "interleave-forced-only;";
8325 OS << (VectorizeOnlyWhenForced ? "" : "no-") << "vectorize-forced-only;";
8326 OS << '>';
8327}
8328