1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has three parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. InnerLoopVectorizer - A unit that performs the actual
23// widening of instructions.
24// 4. LoopVectorizationCostModel - A unit that checks for the profitability
25// of vectorization. It decides on the optimal vector width, which
26// can be one, if vectorization is not profitable.
27//
28// There is a development effort going on to migrate loop vectorizer to the
29// VPlan infrastructure and to introduce outer loop vectorization support (see
30// docs/VectorizationPlan.rst and
31// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
32// purpose, we temporarily introduced the VPlan-native vectorization path: an
33// alternative vectorization path that is natively implemented on top of the
34// VPlan infrastructure. See EnableVPlanNativePath for enabling.
35//
36//===----------------------------------------------------------------------===//
37//
38// The reduction-variable vectorization is based on the paper:
39// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
40//
41// Variable uniformity checks are inspired by:
42// Karrenberg, R. and Hack, S. Whole Function Vectorization.
43//
44// The interleaved access vectorization is based on the paper:
45// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
46// Data for SIMD
47//
48// Other ideas/concepts are from:
49// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
50//
51// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
52// Vectorizing Compilers.
53//
54//===----------------------------------------------------------------------===//
55
56#include "llvm/Transforms/Vectorize/LoopVectorize.h"
57#include "LoopVectorizationPlanner.h"
58#include "VPRecipeBuilder.h"
59#include "VPlan.h"
60#include "VPlanAnalysis.h"
61#include "VPlanCFG.h"
62#include "VPlanHelpers.h"
63#include "VPlanPatternMatch.h"
64#include "VPlanTransforms.h"
65#include "VPlanUtils.h"
66#include "VPlanVerifier.h"
67#include "llvm/ADT/APInt.h"
68#include "llvm/ADT/ArrayRef.h"
69#include "llvm/ADT/DenseMap.h"
70#include "llvm/ADT/DenseMapInfo.h"
71#include "llvm/ADT/Hashing.h"
72#include "llvm/ADT/MapVector.h"
73#include "llvm/ADT/STLExtras.h"
74#include "llvm/ADT/SmallPtrSet.h"
75#include "llvm/ADT/SmallVector.h"
76#include "llvm/ADT/Statistic.h"
77#include "llvm/ADT/StringRef.h"
78#include "llvm/ADT/Twine.h"
79#include "llvm/ADT/TypeSwitch.h"
80#include "llvm/ADT/iterator_range.h"
81#include "llvm/Analysis/AssumptionCache.h"
82#include "llvm/Analysis/BasicAliasAnalysis.h"
83#include "llvm/Analysis/BlockFrequencyInfo.h"
84#include "llvm/Analysis/CFG.h"
85#include "llvm/Analysis/CodeMetrics.h"
86#include "llvm/Analysis/DemandedBits.h"
87#include "llvm/Analysis/GlobalsModRef.h"
88#include "llvm/Analysis/LoopAccessAnalysis.h"
89#include "llvm/Analysis/LoopAnalysisManager.h"
90#include "llvm/Analysis/LoopInfo.h"
91#include "llvm/Analysis/LoopIterator.h"
92#include "llvm/Analysis/OptimizationRemarkEmitter.h"
93#include "llvm/Analysis/ProfileSummaryInfo.h"
94#include "llvm/Analysis/ScalarEvolution.h"
95#include "llvm/Analysis/ScalarEvolutionExpressions.h"
96#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
97#include "llvm/Analysis/TargetLibraryInfo.h"
98#include "llvm/Analysis/TargetTransformInfo.h"
99#include "llvm/Analysis/ValueTracking.h"
100#include "llvm/Analysis/VectorUtils.h"
101#include "llvm/IR/Attributes.h"
102#include "llvm/IR/BasicBlock.h"
103#include "llvm/IR/CFG.h"
104#include "llvm/IR/Constant.h"
105#include "llvm/IR/Constants.h"
106#include "llvm/IR/DataLayout.h"
107#include "llvm/IR/DebugInfo.h"
108#include "llvm/IR/DebugLoc.h"
109#include "llvm/IR/DerivedTypes.h"
110#include "llvm/IR/DiagnosticInfo.h"
111#include "llvm/IR/Dominators.h"
112#include "llvm/IR/Function.h"
113#include "llvm/IR/IRBuilder.h"
114#include "llvm/IR/InstrTypes.h"
115#include "llvm/IR/Instruction.h"
116#include "llvm/IR/Instructions.h"
117#include "llvm/IR/IntrinsicInst.h"
118#include "llvm/IR/Intrinsics.h"
119#include "llvm/IR/MDBuilder.h"
120#include "llvm/IR/Metadata.h"
121#include "llvm/IR/Module.h"
122#include "llvm/IR/Operator.h"
123#include "llvm/IR/PatternMatch.h"
124#include "llvm/IR/ProfDataUtils.h"
125#include "llvm/IR/Type.h"
126#include "llvm/IR/Use.h"
127#include "llvm/IR/User.h"
128#include "llvm/IR/Value.h"
129#include "llvm/IR/Verifier.h"
130#include "llvm/Support/Casting.h"
131#include "llvm/Support/CommandLine.h"
132#include "llvm/Support/Debug.h"
133#include "llvm/Support/ErrorHandling.h"
134#include "llvm/Support/InstructionCost.h"
135#include "llvm/Support/MathExtras.h"
136#include "llvm/Support/NativeFormatting.h"
137#include "llvm/Support/raw_ostream.h"
138#include "llvm/Transforms/Utils/BasicBlockUtils.h"
139#include "llvm/Transforms/Utils/InjectTLIMappings.h"
140#include "llvm/Transforms/Utils/Local.h"
141#include "llvm/Transforms/Utils/LoopSimplify.h"
142#include "llvm/Transforms/Utils/LoopUtils.h"
143#include "llvm/Transforms/Utils/LoopVersioning.h"
144#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
145#include "llvm/Transforms/Utils/SizeOpts.h"
146#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
147#include <algorithm>
148#include <cassert>
149#include <cmath>
150#include <cstdint>
151#include <functional>
152#include <iterator>
153#include <limits>
154#include <memory>
155#include <string>
156#include <tuple>
157#include <utility>
158
159using namespace llvm;
160using namespace SCEVPatternMatch;
161using namespace LoopVectorizationUtils;
162
163#define LV_NAME "loop-vectorize"
164#define DEBUG_TYPE LV_NAME
165
166#ifndef NDEBUG
167const char VerboseDebug[] = DEBUG_TYPE "-verbose";
168#endif
169
170STATISTIC(LoopsVectorized, "Number of loops vectorized");
171STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
172STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
173STATISTIC(LoopsEarlyExitVectorized, "Number of early exit loops vectorized");
174STATISTIC(LoopsPartialAliasVectorized,
175 "Number of partial aliasing loops vectorized");
176
177static cl::opt<bool> EnableEpilogueVectorization(
178 "enable-epilogue-vectorization", cl::init(Val: true), cl::Hidden,
179 cl::desc("Enable vectorization of epilogue loops."));
180
181static cl::opt<ElementCount> EpilogueVectorizationForceVF(
182 "epilogue-vectorization-force-VF", cl::init(Val: ElementCount::getFixed(MinVal: 1)),
183 cl::Hidden,
184 cl::desc("When epilogue vectorization is enabled, and a value greater than "
185 "1 is specified, forces the given VF for all applicable epilogue "
186 "loops. Note: This allows all scalable VFs >= vscale x 1."));
187
188static cl::opt<unsigned> EpilogueVectorizationMinVF(
189 "epilogue-vectorization-minimum-VF", cl::Hidden,
190 cl::desc("Only loops with vectorization factor equal to or larger than "
191 "the specified value are considered for epilogue vectorization."));
192
193/// Loops with a known constant trip count below this number are vectorized only
194/// if no scalar iteration overheads are incurred.
195static cl::opt<unsigned> TinyTripCountVectorThreshold(
196 "vectorizer-min-trip-count", cl::init(Val: 16), cl::Hidden,
197 cl::desc("Loops with a constant trip count that is smaller than this "
198 "value are vectorized only if no scalar iteration overheads "
199 "are incurred."));
200
201static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
202 "vectorize-memory-check-threshold", cl::init(Val: 128), cl::Hidden,
203 cl::desc("The maximum allowed number of runtime memory checks"));
204
205static cl::opt<bool> ForcePartialAliasingVectorization(
206 "force-partial-aliasing-vectorization", cl::init(Val: false), cl::Hidden,
207 cl::desc("Replace pointer diff checks with alias masks."));
208
209/// Option tail-folding-policy controls the tail-folding strategy and lists all
210/// available options. The vectorizer will attempt to fold the tail-loop into
211/// the vector loop (main/epilogue loops) and predicate the instructions
212/// accordingly. If tail-folding fails, there are different fallback strategies
213/// depending on these values:
214enum class TailFoldingPolicyTy { None = 0, PreferFoldTail, MustFoldTail };
215
216static cl::opt<TailFoldingPolicyTy> TailFoldingPolicy(
217 "tail-folding-policy", cl::init(Val: TailFoldingPolicyTy::None), cl::Hidden,
218 cl::desc("Tail-folding preferences over creating an epilogue loop."),
219 cl::values(
220 clEnumValN(TailFoldingPolicyTy::None, "dont-fold-tail",
221 "Don't tail-fold loops."),
222 clEnumValN(TailFoldingPolicyTy::PreferFoldTail, "prefer-fold-tail",
223 "prefer tail-folding, otherwise create an epilogue when "
224 "appropriate."),
225 clEnumValN(TailFoldingPolicyTy::MustFoldTail, "must-fold-tail",
226 "always tail-fold, don't attempt vectorization if "
227 "tail-folding fails.")));
228
229static cl::opt<TailFoldingPolicyTy> EpilogueTailFoldingPolicy(
230 "epilogue-tail-folding-policy", cl::Hidden,
231 cl::desc(
232 "Epilogue-tail-folding preferences over creating an epilogue loop."),
233 cl::values(
234 clEnumValN(TailFoldingPolicyTy::None, "dont-fold-tail",
235 "Don't tail-fold loops."),
236 clEnumValN(TailFoldingPolicyTy::PreferFoldTail, "prefer-fold-tail",
237 "prefer tail-folding, otherwise create an epilogue when "
238 "appropriate.")));
239
240static cl::opt<TailFoldingStyle> ForceTailFoldingStyle(
241 "force-tail-folding-style", cl::desc("Force the tail folding style"),
242 cl::init(Val: TailFoldingStyle::None),
243 cl::values(
244 clEnumValN(TailFoldingStyle::None, "none", "Disable tail folding"),
245 clEnumValN(
246 TailFoldingStyle::Data, "data",
247 "Create lane mask for data only, using active.lane.mask intrinsic"),
248 clEnumValN(TailFoldingStyle::DataWithoutLaneMask,
249 "data-without-lane-mask",
250 "Create lane mask with compare/stepvector"),
251 clEnumValN(TailFoldingStyle::DataAndControlFlow, "data-and-control",
252 "Create lane mask using active.lane.mask intrinsic, and use "
253 "it for both data and control flow"),
254 clEnumValN(TailFoldingStyle::DataWithEVL, "data-with-evl",
255 "Use predicated EVL instructions for tail folding. If EVL "
256 "is unsupported, fallback to data-without-lane-mask.")));
257
258cl::opt<bool> llvm::EnableWideActiveLaneMask(
259 "enable-wide-lane-mask", cl::init(Val: false), cl::Hidden,
260 cl::desc("Enable use of wide lane masks when used for control flow in "
261 "tail-folded loops"));
262
263static cl::opt<bool> EnableInterleavedMemAccesses(
264 "enable-interleaved-mem-accesses", cl::init(Val: false), cl::Hidden,
265 cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
266
267/// An interleave-group may need masking if it resides in a block that needs
268/// predication, or in order to mask away gaps.
269static cl::opt<bool> EnableMaskedInterleavedMemAccesses(
270 "enable-masked-interleaved-mem-accesses", cl::init(Val: false), cl::Hidden,
271 cl::desc("Enable vectorization on masked interleaved memory accesses in a loop"));
272
273static cl::opt<unsigned> ForceTargetNumScalarRegs(
274 "force-target-num-scalar-regs", cl::init(Val: 0), cl::Hidden,
275 cl::desc("A flag that overrides the target's number of scalar registers."));
276
277static cl::opt<unsigned> ForceTargetNumVectorRegs(
278 "force-target-num-vector-regs", cl::init(Val: 0), cl::Hidden,
279 cl::desc("A flag that overrides the target's number of vector registers."));
280
281static cl::opt<unsigned> ForceTargetMaxScalarInterleaveFactor(
282 "force-target-max-scalar-interleave", cl::init(Val: 0), cl::Hidden,
283 cl::desc("A flag that overrides the target's max interleave factor for "
284 "scalar loops."));
285
286static cl::opt<unsigned> ForceTargetMaxVectorInterleaveFactor(
287 "force-target-max-vector-interleave", cl::init(Val: 0), cl::Hidden,
288 cl::desc("A flag that overrides the target's max interleave factor for "
289 "vectorized loops."));
290
291cl::opt<unsigned> llvm::ForceTargetInstructionCost(
292 "force-target-instruction-cost", cl::init(Val: 0), cl::Hidden,
293 cl::desc("A flag that overrides the target's expected cost for "
294 "an instruction to a single constant value. Mostly "
295 "useful for getting consistent testing."));
296
297static cl::opt<unsigned> SmallLoopCost(
298 "small-loop-cost", cl::init(Val: 20), cl::Hidden,
299 cl::desc(
300 "The cost of a loop that is considered 'small' by the interleaver."));
301
302static cl::opt<bool> LoopVectorizeWithBlockFrequency(
303 "loop-vectorize-with-block-frequency", cl::init(Val: true), cl::Hidden,
304 cl::desc("Enable the use of the block frequency analysis to access PGO "
305 "heuristics minimizing code growth in cold regions and being more "
306 "aggressive in hot regions."));
307
308// Runtime interleave loops for load/store throughput.
309static cl::opt<bool> EnableLoadStoreRuntimeInterleave(
310 "enable-loadstore-runtime-interleave", cl::init(Val: true), cl::Hidden,
311 cl::desc(
312 "Enable runtime interleaving until load/store ports are saturated"));
313
314/// The number of stores in a loop that are allowed to need predication.
315cl::opt<unsigned> NumberOfStoresToPredicate(
316 "vectorize-num-stores-pred", cl::init(Val: 1), cl::Hidden,
317 cl::desc("Max number of stores to be predicated behind an if."));
318
319// TODO: Move size-based thresholds out of legality checking, make cost based
320// decisions instead of hard thresholds.
321static cl::opt<unsigned> VectorizeSCEVCheckThreshold(
322 "vectorize-scev-check-threshold", cl::init(Val: 16), cl::Hidden,
323 cl::desc("The maximum number of SCEV checks allowed."));
324
325static cl::opt<unsigned> PragmaVectorizeSCEVCheckThreshold(
326 "pragma-vectorize-scev-check-threshold", cl::init(Val: 128), cl::Hidden,
327 cl::desc("The maximum number of SCEV checks allowed with a "
328 "vectorize(enable) pragma"));
329
330static cl::opt<bool> EnableIndVarRegisterHeur(
331 "enable-ind-var-reg-heur", cl::init(Val: true), cl::Hidden,
332 cl::desc("Count the induction variable only once when interleaving"));
333
334static cl::opt<unsigned> MaxNestedScalarReductionIC(
335 "max-nested-scalar-reduction-interleave", cl::init(Val: 2), cl::Hidden,
336 cl::desc("The maximum interleave count to use when interleaving a scalar "
337 "reduction in a nested loop."));
338
339static cl::opt<bool> ForceOrderedReductions(
340 "force-ordered-reductions", cl::init(Val: false), cl::Hidden,
341 cl::desc("Enable the vectorisation of loops with in-order (strict) "
342 "FP reductions"));
343
344static cl::opt<bool> PreferPredicatedReductionSelect(
345 "prefer-predicated-reduction-select", cl::init(Val: false), cl::Hidden,
346 cl::desc(
347 "Prefer predicating a reduction operation over an after loop select."));
348
349cl::opt<bool> llvm::EnableVPlanNativePath(
350 "enable-vplan-native-path", cl::Hidden,
351 cl::desc("Enable VPlan-native vectorization path with "
352 "support for outer loop vectorization."));
353
354cl::opt<bool>
355 llvm::VerifyEachVPlan("vplan-verify-each",
356#ifdef EXPENSIVE_CHECKS
357 cl::init(true),
358#else
359 cl::init(Val: false),
360#endif
361 cl::Hidden,
362 cl::desc("Verify VPlans after VPlan transforms."));
363
364#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
365cl::opt<bool> llvm::VPlanPrintBeforeAll(
366 "vplan-print-before-all", cl::init(false), cl::Hidden,
367 cl::desc("Print VPlans before all VPlan transformations."));
368
369cl::opt<bool> llvm::VPlanPrintAfterAll(
370 "vplan-print-after-all", cl::init(false), cl::Hidden,
371 cl::desc("Print VPlans after all VPlan transformations."));
372
373cl::list<std::string> llvm::VPlanPrintBeforePasses(
374 "vplan-print-before", cl::Hidden,
375 cl::desc("Print VPlans before specified VPlan transformations (regexp)."));
376
377cl::list<std::string> llvm::VPlanPrintAfterPasses(
378 "vplan-print-after", cl::Hidden,
379 cl::desc("Print VPlans after specified VPlan transformations (regexp)."));
380
381cl::opt<bool> llvm::VPlanPrintVectorRegionScope(
382 "vplan-print-vector-region-scope", cl::init(false), cl::Hidden,
383 cl::desc("Limit VPlan printing to vector loop region in "
384 "`-vplan-print-after*` if the plan has one."));
385#endif
386
387// This flag enables the stress testing of the VPlan H-CFG construction in the
388// VPlan-native vectorization path. It must be used in conjuction with
389// -enable-vplan-native-path. -vplan-verify-hcfg can also be used to enable the
390// verification of the H-CFGs built.
391cl::opt<bool> VPlanBuildOuterloopStressTest(
392 "vplan-build-outerloop-stress-test", cl::init(Val: false), cl::Hidden,
393 cl::desc(
394 "Build VPlan for every supported loop nest in the function and bail "
395 "out right after the build (stress test the VPlan H-CFG construction "
396 "in the VPlan-native vectorization path)."));
397
398cl::opt<bool> llvm::EnableLoopInterleaving(
399 "interleave-loops", cl::init(Val: true), cl::Hidden,
400 cl::desc("Enable loop interleaving in Loop vectorization passes"));
401cl::opt<bool> llvm::EnableLoopVectorization(
402 "vectorize-loops", cl::init(Val: true), cl::Hidden,
403 cl::desc("Run the Loop vectorization passes"));
404
405static cl::opt<cl::boolOrDefault>
406 ForceMaskedDivRem("force-widen-divrem-via-masked-intrinsic", cl::Hidden,
407 cl::desc("Override cost based masked intrinsic widening "
408 "for div/rem instructions"));
409
410static cl::opt<bool> EnableEarlyExitVectorization(
411 "enable-early-exit-vectorization", cl::init(Val: true), cl::Hidden,
412 cl::desc(
413 "Enable vectorization of early exit loops with uncountable exits."));
414
415static cl::opt<bool> EnableEarlyExitVectorizationWithSideEffects(
416 "enable-early-exit-vectorization-with-side-effects", cl::init(Val: false),
417 cl::Hidden,
418 cl::desc("Enable vectorization of early exit loops with uncountable exits "
419 "and side effects"));
420
421// Returns true if the epilogue VF has been set to a non-zero value other than
422// VF=1 (scalar).
423static bool hasForcedEpilogueVF() {
424 return EpilogueVectorizationForceVF.isNonZero() &&
425 EpilogueVectorizationForceVF != ElementCount::getFixed(MinVal: 1);
426}
427
428// Likelyhood of bypassing the vectorized loop because there are zero trips left
429// after prolog. See `emitIterationCountCheck`.
430static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
431
432/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
433/// ElementCount to include loops whose trip count is a function of vscale.
434static ElementCount getSmallConstantTripCount(ScalarEvolution *SE,
435 const Loop *L) {
436 if (unsigned ExpectedTC = SE->getSmallConstantTripCount(L))
437 return ElementCount::getFixed(MinVal: ExpectedTC);
438
439 const SCEV *BTC = SE->getBackedgeTakenCount(L);
440 if (isa<SCEVCouldNotCompute>(Val: BTC))
441 return ElementCount::getFixed(MinVal: 0);
442
443 const SCEV *ExitCount = SE->getTripCountFromExitCount(ExitCount: BTC, EvalTy: BTC->getType(), L);
444 if (isa<SCEVVScale>(Val: ExitCount))
445 return ElementCount::getScalable(MinVal: 1);
446
447 const APInt *Scale;
448 if (match(S: ExitCount, P: m_scev_Mul(Op0: m_scev_APInt(C&: Scale), Op1: m_SCEVVScale())))
449 if (cast<SCEVMulExpr>(Val: ExitCount)->hasNoUnsignedWrap())
450 if (Scale->getActiveBits() <= 32)
451 return ElementCount::getScalable(MinVal: Scale->getZExtValue());
452
453 return ElementCount::getFixed(MinVal: 0);
454}
455
456/// Get the maximum trip count for \p L from the SCEV unsigned range, excluding
457/// zero from the range. Only valid when not folding the tail, as the minimum
458/// iteration count check guards against a zero trip count. Returns 0 if
459/// unknown.
460static unsigned getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE,
461 Loop *L) {
462 const SCEV *BTC = PSE.getBackedgeTakenCount();
463 if (isa<SCEVCouldNotCompute>(Val: BTC))
464 return 0;
465 ScalarEvolution *SE = PSE.getSE();
466 const SCEV *TripCount = SE->getTripCountFromExitCount(ExitCount: BTC, EvalTy: BTC->getType(), L);
467 ConstantRange TCRange = SE->getUnsignedRange(S: TripCount);
468 APInt MaxTCFromRange = TCRange.getUnsignedMax();
469 if (!MaxTCFromRange.isZero() && MaxTCFromRange.getActiveBits() <= 32)
470 return MaxTCFromRange.getZExtValue();
471 return 0;
472}
473
474/// Returns "best known" trip count, which is either a valid positive trip count
475/// or std::nullopt when an estimate cannot be made (including when the trip
476/// count would overflow), for the specified loop \p L as defined by the
477/// following procedure:
478/// 1) Returns exact trip count if it is known.
479/// 2) Returns expected trip count according to profile data if any.
480/// 3) Returns upper bound estimate if known, if \p CanUseConstantMax, and
481/// if \p ComputeUpperBoundOnly is false.
482/// 4) Returns the maximum trip count from the SCEV range excluding zero,
483/// if \p CanUseConstantMax and \p CanExcludeZeroTrips.
484/// 5) Returns std::nullopt if all of the above failed.
485static std::optional<ElementCount> getSmallBestKnownTC(
486 PredicatedScalarEvolution &PSE, Loop *L, bool CanUseConstantMax = true,
487 bool CanExcludeZeroTrips = false, bool ComputeUpperBoundOnly = false) {
488 // Check if exact trip count is known.
489 if (auto ExpectedTC = getSmallConstantTripCount(SE: PSE.getSE(), L))
490 return ExpectedTC;
491
492 // Check if there is an expected trip count available from profile data.
493 if (LoopVectorizeWithBlockFrequency && !ComputeUpperBoundOnly)
494 if (auto EstimatedTC = getLoopEstimatedTripCount(L))
495 return ElementCount::getFixed(MinVal: *EstimatedTC);
496
497 if (!CanUseConstantMax)
498 return std::nullopt;
499
500 // Check if upper bound estimate is known.
501 if (unsigned ExpectedTC = PSE.getSmallConstantMaxTripCount())
502 return ElementCount::getFixed(MinVal: ExpectedTC);
503
504 // Get the maximum trip count from the SCEV range excluding zero. This is
505 // only safe when not folding the tail, as the minimum iteration count check
506 // prevents entering the vector loop with a zero trip count.
507 if (CanUseConstantMax && CanExcludeZeroTrips)
508 if (unsigned RefinedTC = getMaxTCFromNonZeroRange(PSE, L))
509 return ElementCount::getFixed(MinVal: RefinedTC);
510
511 return std::nullopt;
512}
513
514namespace {
515// Forward declare GeneratedRTChecks.
516class GeneratedRTChecks;
517
518using SCEV2ValueTy = DenseMap<const SCEV *, Value *>;
519} // namespace
520
521namespace llvm {
522
523AnalysisKey ShouldRunExtraVectorPasses::Key;
524
525/// InnerLoopVectorizer vectorizes loops which contain only one basic
526/// block to a specified vectorization factor (VF).
527/// This class performs the widening of scalars into vectors, or multiple
528/// scalars. This class also implements the following features:
529/// * It inserts an epilogue loop for handling loops that don't have iteration
530/// counts that are known to be a multiple of the vectorization factor.
531/// * It handles the code generation for reduction variables.
532/// * Scalarization (implementation using scalars) of un-vectorizable
533/// instructions.
534/// InnerLoopVectorizer does not perform any vectorization-legality
535/// checks, and relies on the caller to check for the different legality
536/// aspects. The InnerLoopVectorizer relies on the
537/// LoopVectorizationLegality class to provide information about the induction
538/// and reduction variables that were found to a given vectorization factor.
539class InnerLoopVectorizer {
540public:
541 InnerLoopVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
542 LoopInfo *LI, DominatorTree *DT,
543 const TargetTransformInfo *TTI, AssumptionCache *AC,
544 ElementCount VecWidth, unsigned UnrollFactor,
545 LoopVectorizationCostModel *CM,
546 GeneratedRTChecks &RTChecks, VPlan &Plan)
547 : OrigLoop(OrigLoop), PSE(PSE), LI(LI), DT(DT), TTI(TTI), AC(AC),
548 VF(VecWidth), UF(UnrollFactor), Builder(PSE.getSE()->getContext()),
549 Cost(CM), RTChecks(RTChecks), Plan(Plan),
550 VectorPHVPBB(cast<VPBasicBlock>(
551 Val: Plan.getVectorLoopRegion()->getSinglePredecessor())) {}
552
553 virtual ~InnerLoopVectorizer() = default;
554
555 /// Creates a basic block for the scalar preheader. Both
556 /// EpilogueVectorizerMainLoop and EpilogueVectorizerEpilogueLoop overwrite
557 /// the method to create additional blocks and checks needed for epilogue
558 /// vectorization.
559 virtual BasicBlock *createVectorizedLoopSkeleton();
560
561 /// Fix the vectorized code, taking care of header phi's, and more.
562 void fixVectorizedLoop(VPTransformState &State);
563
564protected:
565 friend class LoopVectorizationPlanner;
566
567 /// Create and return a new IR basic block for the scalar preheader whose name
568 /// is prefixed with \p Prefix.
569 BasicBlock *createScalarPreheader(StringRef Prefix);
570
571 /// Allow subclasses to override and print debug traces before/after vplan
572 /// execution, when trace information is requested.
573 virtual void printDebugTracesAtStart() {}
574 virtual void printDebugTracesAtEnd() {}
575
576 /// The original loop.
577 Loop *OrigLoop;
578
579 /// A wrapper around ScalarEvolution used to add runtime SCEV checks. Applies
580 /// dynamic knowledge to simplify SCEV expressions and converts them to a
581 /// more usable form.
582 PredicatedScalarEvolution &PSE;
583
584 /// Loop Info.
585 LoopInfo *LI;
586
587 /// Dominator Tree.
588 DominatorTree *DT;
589
590 /// Target Transform Info.
591 const TargetTransformInfo *TTI;
592
593 /// Assumption Cache.
594 AssumptionCache *AC;
595
596 /// The vectorization SIMD factor to use. Each vector will have this many
597 /// vector elements.
598 ElementCount VF;
599
600 /// The vectorization unroll factor to use. Each scalar is vectorized to this
601 /// many different vector instructions.
602 unsigned UF;
603
604 /// The builder that we use
605 IRBuilder<> Builder;
606
607 // --- Vectorization state ---
608
609 /// The profitablity analysis.
610 LoopVectorizationCostModel *Cost;
611
612 /// Structure to hold information about generated runtime checks, responsible
613 /// for cleaning the checks, if vectorization turns out unprofitable.
614 GeneratedRTChecks &RTChecks;
615
616 VPlan &Plan;
617
618 /// The vector preheader block of \p Plan, used as target for check blocks
619 /// introduced during skeleton creation.
620 VPBasicBlock *VectorPHVPBB;
621};
622
623/// Encapsulate information regarding vectorization of a loop and its epilogue.
624/// This information is meant to be updated and used across two stages of
625/// epilogue vectorization.
626struct EpilogueLoopVectorizationInfo {
627 ElementCount MainLoopVF = ElementCount::getFixed(MinVal: 0);
628 unsigned MainLoopUF = 0;
629 ElementCount EpilogueVF = ElementCount::getFixed(MinVal: 0);
630 unsigned EpilogueUF = 0;
631 BasicBlock *MainLoopIterationCountCheck = nullptr;
632 BasicBlock *EpilogueIterationCountCheck = nullptr;
633 Value *VectorTripCount = nullptr;
634 VPlan &EpiloguePlan;
635
636 EpilogueLoopVectorizationInfo(ElementCount MVF, unsigned MUF,
637 ElementCount EVF, unsigned EUF,
638 VPlan &EpiloguePlan)
639 : MainLoopVF(MVF), MainLoopUF(MUF), EpilogueVF(EVF), EpilogueUF(EUF),
640 EpiloguePlan(EpiloguePlan) {
641 assert(EUF == 1 &&
642 "A high UF for the epilogue loop is likely not beneficial.");
643 }
644};
645
646/// An extension of the inner loop vectorizer that creates a skeleton for a
647/// vectorized loop that has its epilogue (residual) also vectorized.
648/// The idea is to run the vplan on a given loop twice, firstly to setup the
649/// skeleton and vectorize the main loop, and secondly to complete the skeleton
650/// from the first step and vectorize the epilogue. This is achieved by
651/// deriving two concrete strategy classes from this base class and invoking
652/// them in succession from the loop vectorizer planner.
653class InnerLoopAndEpilogueVectorizer : public InnerLoopVectorizer {
654public:
655 InnerLoopAndEpilogueVectorizer(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
656 LoopInfo *LI, DominatorTree *DT,
657 const TargetTransformInfo *TTI,
658 AssumptionCache *AC,
659 EpilogueLoopVectorizationInfo &EPI,
660 LoopVectorizationCostModel *CM,
661 GeneratedRTChecks &Checks, VPlan &Plan,
662 ElementCount VecWidth, unsigned UnrollFactor)
663 : InnerLoopVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, VecWidth,
664 UnrollFactor, CM, Checks, Plan),
665 EPI(EPI) {}
666
667 /// Holds and updates state information required to vectorize the main loop
668 /// and its epilogue in two separate passes. This setup helps us avoid
669 /// regenerating and recomputing runtime safety checks. It also helps us to
670 /// shorten the iteration-count-check path length for the cases where the
671 /// iteration count of the loop is so small that the main vector loop is
672 /// completely skipped.
673 EpilogueLoopVectorizationInfo &EPI;
674};
675
676/// A specialized derived class of inner loop vectorizer that performs
677/// vectorization of *main* loops in the process of vectorizing loops and their
678/// epilogues.
679class EpilogueVectorizerMainLoop : public InnerLoopAndEpilogueVectorizer {
680public:
681 EpilogueVectorizerMainLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
682 LoopInfo *LI, DominatorTree *DT,
683 const TargetTransformInfo *TTI,
684 AssumptionCache *AC,
685 EpilogueLoopVectorizationInfo &EPI,
686 LoopVectorizationCostModel *CM,
687 GeneratedRTChecks &Check, VPlan &Plan)
688 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, EPI, CM,
689 Check, Plan, EPI.MainLoopVF,
690 EPI.MainLoopUF) {}
691
692protected:
693 void printDebugTracesAtStart() override;
694 void printDebugTracesAtEnd() override;
695};
696
697// A specialized derived class of inner loop vectorizer that performs
698// vectorization of *epilogue* loops in the process of vectorizing loops and
699// their epilogues.
700class EpilogueVectorizerEpilogueLoop : public InnerLoopAndEpilogueVectorizer {
701public:
702 EpilogueVectorizerEpilogueLoop(Loop *OrigLoop, PredicatedScalarEvolution &PSE,
703 LoopInfo *LI, DominatorTree *DT,
704 const TargetTransformInfo *TTI,
705 AssumptionCache *AC,
706 EpilogueLoopVectorizationInfo &EPI,
707 LoopVectorizationCostModel *CM,
708 GeneratedRTChecks &Checks, VPlan &Plan)
709 : InnerLoopAndEpilogueVectorizer(OrigLoop, PSE, LI, DT, TTI, AC, EPI, CM,
710 Checks, Plan, EPI.EpilogueVF,
711 EPI.EpilogueUF) {}
712 /// Implements the interface for creating a vectorized skeleton using the
713 /// *epilogue loop* strategy (i.e., the second pass of VPlan execution).
714 BasicBlock *createVectorizedLoopSkeleton() final;
715
716protected:
717 void printDebugTracesAtStart() override;
718 void printDebugTracesAtEnd() override;
719};
720} // end namespace llvm
721
722/// Look for a meaningful debug location on the instruction or its operands.
723static DebugLoc getDebugLocFromInstOrOperands(Instruction *I) {
724 if (!I)
725 return DebugLoc::getUnknown();
726
727 DebugLoc Empty;
728 if (I->getDebugLoc() != Empty)
729 return I->getDebugLoc();
730
731 for (Use &Op : I->operands()) {
732 if (Instruction *OpInst = dyn_cast<Instruction>(Val&: Op))
733 if (OpInst->getDebugLoc() != Empty)
734 return OpInst->getDebugLoc();
735 }
736
737 return I->getDebugLoc();
738}
739
740namespace llvm {
741
742/// Return the runtime value for VF.
743Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF) {
744 return B.CreateElementCount(Ty, EC: VF);
745}
746
747} // end namespace llvm
748
749namespace llvm {
750
751// Loop vectorization cost-model hints how the epilogue/tail loop should be
752// lowered.
753enum EpilogueLowering {
754
755 // The default: allowing epilogues.
756 CM_EpilogueAllowed,
757
758 // Vectorization with OptForSize: don't allow epilogues.
759 CM_EpilogueNotAllowedOptSize,
760
761 // A special case of vectorisation with OptForSize: loops with a very small
762 // trip count are considered for vectorization under OptForSize, thereby
763 // making sure the cost of their loop body is dominant, free of runtime
764 // guards and scalar iteration overheads.
765 CM_EpilogueNotAllowedLowTripLoop,
766
767 // Loop hint indicating an epilogue is undesired, apply tail folding.
768 CM_EpilogueNotNeededFoldTail,
769
770 // Directive indicating we must either fold the epilogue/tail or not vectorize
771 CM_EpilogueNotAllowedFoldTail
772};
773
774enum class AliasMaskingStatus { NotDecided, Disabled, Enabled };
775
776/// LoopVectorizationCostModel - estimates the expected speedups due to
777/// vectorization.
778/// In many cases vectorization is not profitable. This can happen because of
779/// a number of reasons. In this class we mainly attempt to predict the
780/// expected speedup/slowdowns due to the supported instruction set. We use the
781/// TargetTransformInfo to query the different backends for the cost of
782/// different operations.
783class LoopVectorizationCostModel {
784 friend class LoopVectorizationPlanner;
785
786public:
787 LoopVectorizationCostModel(EpilogueLowering SEL, Loop *L,
788 PredicatedScalarEvolution &PSE, LoopInfo *LI,
789 LoopVectorizationLegality *Legal,
790 const TargetTransformInfo &TTI,
791 const TargetLibraryInfo *TLI, AssumptionCache *AC,
792 OptimizationRemarkEmitter *ORE,
793 std::function<BlockFrequencyInfo &()> GetBFI,
794 const Function *F, const LoopVectorizeHints *Hints,
795 InterleavedAccessInfo &IAI,
796 VFSelectionContext &Config)
797 : Config(Config), EpilogueLoweringStatus(SEL), TheLoop(L), PSE(PSE),
798 LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), AC(AC), ORE(ORE),
799 GetBFI(GetBFI), TheFunction(F), Hints(Hints), InterleaveInfo(IAI) {}
800
801 /// \return An upper bound for the vectorization factors (both fixed and
802 /// scalable). If the factors are 0, vectorization and interleaving should be
803 /// avoided up front.
804 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
805
806 /// Memory access instruction may be vectorized in more than one way.
807 /// Form of instruction after vectorization depends on cost.
808 /// This function takes cost-based decisions for Load/Store instructions
809 /// and collects them in a map. This decisions map is used for building
810 /// the lists of loop-uniform and loop-scalar instructions.
811 /// The calculated cost is saved with widening decision in order to
812 /// avoid redundant calculations.
813 void setCostBasedWideningDecision(ElementCount VF);
814
815 /// Collect values we want to ignore in the cost model.
816 void collectValuesToIgnore();
817
818 /// \returns True if it is more profitable to scalarize instruction \p I for
819 /// vectorization factor \p VF.
820 bool isProfitableToScalarize(Instruction *I, ElementCount VF) const {
821 assert(VF.isVector() &&
822 "Profitable to scalarize relevant only for VF > 1.");
823 assert(
824 TheLoop->isInnermost() &&
825 "cost-model should not be used for outer loops (in VPlan-native path)");
826
827 auto Scalars = InstsToScalarize.find(Key: VF);
828 assert(Scalars != InstsToScalarize.end() &&
829 "VF not yet analyzed for scalarization profitability");
830 return Scalars->second.contains(Key: I);
831 }
832
833 /// Returns true if \p I is known to be uniform after vectorization.
834 bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const {
835 assert(
836 TheLoop->isInnermost() &&
837 "cost-model should not be used for outer loops (in VPlan-native path)");
838
839 // If VF is scalar, then all instructions are trivially uniform.
840 if (VF.isScalar())
841 return true;
842
843 // Pseudo probes must be duplicated per vector lane so that the
844 // profiled loop trip count is not undercounted.
845 if (isa<PseudoProbeInst>(Val: I))
846 return false;
847
848 auto UniformsPerVF = Uniforms.find(Val: VF);
849 assert(UniformsPerVF != Uniforms.end() &&
850 "VF not yet analyzed for uniformity");
851 return UniformsPerVF->second.count(Ptr: I);
852 }
853
854 /// Returns true if \p I is known to be scalar after vectorization.
855 bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const {
856 assert(
857 TheLoop->isInnermost() &&
858 "cost-model should not be used for outer loops (in VPlan-native path)");
859 if (VF.isScalar())
860 return true;
861
862 auto ScalarsPerVF = Scalars.find(Val: VF);
863 assert(ScalarsPerVF != Scalars.end() &&
864 "Scalar values are not calculated for VF");
865 return ScalarsPerVF->second.count(Ptr: I);
866 }
867
868 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
869 /// for vectorization factor \p VF.
870 bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const {
871 const auto &MinBWs = Config.getMinimalBitwidths();
872 // Truncs must truncate at most to their destination type.
873 if (isa_and_nonnull<TruncInst>(Val: I) && MinBWs.contains(Key: I) &&
874 I->getType()->getScalarSizeInBits() < MinBWs.lookup(Key: I))
875 return false;
876 return VF.isVector() && MinBWs.contains(Key: I) &&
877 !isProfitableToScalarize(I, VF) &&
878 !isScalarAfterVectorization(I, VF);
879 }
880
881 /// Decision that was taken during cost calculation for memory instruction.
882 enum InstWidening {
883 CM_Unknown,
884 CM_Widen, // For consecutive accesses with stride +1.
885 CM_Widen_Reverse, // For consecutive accesses with stride -1.
886 CM_Interleave,
887 CM_GatherScatter,
888 CM_Scalarize,
889 /// A widening decision that has been invalidated after replacing the
890 /// corresponding recipe during VPlan transforms.
891 /// TODO: Remove once the legacy exit cost computation is retired.
892 CM_InvalidatedDecision
893 };
894
895 /// Save vectorization decision \p W and \p Cost taken by the cost model for
896 /// instruction \p I and vector width \p VF.
897 void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W,
898 InstructionCost Cost) {
899 assert(VF.isVector() && "Expected VF >=2");
900 WideningDecisions[{I, VF}] = {W, Cost};
901 }
902
903 /// Save vectorization decision \p W and \p Cost taken by the cost model for
904 /// interleaving group \p Grp and vector width \p VF.
905 void setWideningDecision(const InterleaveGroup<Instruction> *Grp,
906 ElementCount VF, InstWidening W,
907 InstructionCost Cost) {
908 assert(VF.isVector() && "Expected VF >=2");
909 /// Broadcast this decicion to all instructions inside the group.
910 /// When interleaving, the cost will only be assigned one instruction, the
911 /// insert position. For other cases, add the appropriate fraction of the
912 /// total cost to each instruction. This ensures accurate costs are used,
913 /// even if the insert position instruction is not used.
914 InstructionCost InsertPosCost = Cost;
915 InstructionCost OtherMemberCost = 0;
916 if (W != CM_Interleave)
917 OtherMemberCost = InsertPosCost = Cost / Grp->getNumMembers();
918 ;
919 for (auto *I : Grp->members()) {
920 if (Grp->getInsertPos() == I)
921 WideningDecisions[{I, VF}] = {W, InsertPosCost};
922 else
923 WideningDecisions[{I, VF}] = {W, OtherMemberCost};
924 }
925 }
926
927 /// Return the cost model decision for the given instruction \p I and vector
928 /// width \p VF. Return CM_Unknown if this instruction did not pass
929 /// through the cost modeling.
930 InstWidening getWideningDecision(Instruction *I, ElementCount VF) const {
931 assert(VF.isVector() && "Expected VF to be a vector VF");
932 assert(
933 TheLoop->isInnermost() &&
934 "cost-model should not be used for outer loops (in VPlan-native path)");
935
936 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
937 auto Itr = WideningDecisions.find(Val: InstOnVF);
938 if (Itr == WideningDecisions.end())
939 return CM_Unknown;
940 return Itr->second.first;
941 }
942
943 /// Return the vectorization cost for the given instruction \p I and vector
944 /// width \p VF.
945 InstructionCost getWideningCost(Instruction *I, ElementCount VF) {
946 assert(VF.isVector() && "Expected VF >=2");
947 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
948 assert(WideningDecisions.contains(InstOnVF) &&
949 "The cost is not calculated");
950 return WideningDecisions[InstOnVF].second;
951 }
952
953 /// Return True if instruction \p I is an optimizable truncate whose operand
954 /// is an induction variable. Such a truncate will be removed by adding a new
955 /// induction variable with the destination type.
956 bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) {
957 // If the instruction is not a truncate, return false.
958 auto *Trunc = dyn_cast<TruncInst>(Val: I);
959 if (!Trunc)
960 return false;
961
962 // Get the source and destination types of the truncate.
963 Type *SrcTy = toVectorTy(Scalar: Trunc->getSrcTy(), EC: VF);
964 Type *DestTy = toVectorTy(Scalar: Trunc->getDestTy(), EC: VF);
965
966 // If the truncate is free for the given types, return false. Replacing a
967 // free truncate with an induction variable would add an induction variable
968 // update instruction to each iteration of the loop. We exclude from this
969 // check the primary induction variable since it will need an update
970 // instruction regardless.
971 Value *Op = Trunc->getOperand(i_nocapture: 0);
972 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(Ty1: SrcTy, Ty2: DestTy))
973 return false;
974
975 // If the truncated value is not an induction variable, return false.
976 return Legal->isInductionPhi(V: Op);
977 }
978
979 /// Collects the instructions to scalarize for each predicated instruction in
980 /// the loop.
981 void collectInstsToScalarize(ElementCount VF);
982
983 /// Collect values that will not be widened, including Uniforms, Scalars, and
984 /// Instructions to Scalarize for the given \p VF.
985 /// The sets depend on CM decision for Load/Store instructions
986 /// that may be vectorized as interleave, gather-scatter or scalarized.
987 /// Also make a decision on what to do about call instructions in the loop
988 /// at that VF -- scalarize, call a known vector routine, or call a
989 /// vector intrinsic.
990 void collectNonVectorizedAndSetWideningDecisions(ElementCount VF) {
991 // Do the analysis once.
992 if (VF.isScalar() || Uniforms.contains(Val: VF))
993 return;
994 setCostBasedWideningDecision(VF);
995 collectLoopUniforms(VF);
996 collectLoopScalars(VF);
997 collectInstsToScalarize(VF);
998 }
999
1000 /// Given costs for both strategies, return true if the scalar predication
1001 /// lowering should be used for div/rem. This incorporates an override
1002 /// option so it is not simply a cost comparison.
1003 bool isDivRemScalarWithPredication(InstructionCost ScalarCost,
1004 InstructionCost MaskedCost) const {
1005 switch (ForceMaskedDivRem) {
1006 case cl::boolOrDefault::BOU_UNSET:
1007 return ScalarCost < MaskedCost;
1008 case cl::boolOrDefault::BOU_TRUE:
1009 return false;
1010 case cl::boolOrDefault::BOU_FALSE:
1011 return true;
1012 }
1013 llvm_unreachable("impossible case value");
1014 }
1015
1016 /// Returns true if \p I is an instruction which requires predication and
1017 /// for which our chosen predication strategy is scalarization (i.e. we
1018 /// don't have an alternate strategy such as masking available).
1019 /// \p VF is the vectorization factor that will be used to vectorize \p I.
1020 bool isScalarWithPredication(Instruction *I, ElementCount VF);
1021
1022 /// Wrapper function for LoopVectorizationLegality::isMaskRequired,
1023 /// that passes the Instruction \p I and if we fold tail.
1024 bool isMaskRequired(Instruction *I) const;
1025
1026 /// Returns true if \p I is an instruction that needs to be predicated
1027 /// at runtime. The result is independent of the predication mechanism.
1028 /// Superset of instructions that return true for isScalarWithPredication.
1029 bool isPredicatedInst(Instruction *I) const;
1030
1031 /// A helper function that returns how much we should divide the cost of a
1032 /// predicated block by. Typically this is the reciprocal of the block
1033 /// probability, i.e. if we return X we are assuming the predicated block will
1034 /// execute once for every X iterations of the loop header so the block should
1035 /// only contribute 1/X of its cost to the total cost calculation, but when
1036 /// optimizing for code size it will just be 1 as code size costs don't depend
1037 /// on execution probabilities.
1038 ///
1039 /// Note that if a block wasn't originally predicated but was predicated due
1040 /// to tail folding, the divisor will still be 1 because it will execute for
1041 /// every iteration of the loop header.
1042 inline uint64_t
1043 getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind,
1044 const BasicBlock *BB);
1045
1046 /// Returns true if an artificially high cost for emulated masked memrefs
1047 /// should be used.
1048 bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF);
1049
1050 /// Return the costs for our two available strategies for lowering a
1051 /// div/rem operation which requires speculating at least one lane.
1052 /// First result is for scalarization (will be invalid for scalable
1053 /// vectors); second is for the masked intrinsic strategy.
1054 std::pair<InstructionCost, InstructionCost>
1055 getDivRemSpeculationCost(Instruction *I, ElementCount VF);
1056
1057 /// If \p I is a memory instruction with a consecutive pointer that can be
1058 /// widened, returns the widening kind (CM_Widen or CM_Widen_Reverse) and
1059 /// std::nullopt otherwise.
1060 std::optional<InstWidening> memoryInstructionCanBeWidened(Instruction *I,
1061 ElementCount VF);
1062
1063 /// Returns true if \p I is a memory instruction in an interleaved-group
1064 /// of memory accesses that can be vectorized with wide vector loads/stores
1065 /// and shuffles.
1066 bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const;
1067
1068 /// Check if \p Instr belongs to any interleaved access group.
1069 bool isAccessInterleaved(Instruction *Instr) const {
1070 return InterleaveInfo.isInterleaved(Instr);
1071 }
1072
1073 /// Get the interleaved access group that \p Instr belongs to.
1074 const InterleaveGroup<Instruction> *
1075 getInterleavedAccessGroup(Instruction *Instr) const {
1076 return InterleaveInfo.getInterleaveGroup(Instr);
1077 }
1078
1079 /// Returns true if we're required to use a scalar epilogue for at least
1080 /// the final iteration of the original loop.
1081 bool requiresScalarEpilogue(bool IsVectorizing) const {
1082 if (!isEpilogueAllowed()) {
1083 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1084 return false;
1085 }
1086 // If we might exit from anywhere but the latch and early exit vectorization
1087 // is disabled, we must run the exiting iteration in scalar form.
1088 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
1089 !(EnableEarlyExitVectorization && Legal->hasUncountableEarlyExit())) {
1090 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
1091 "from latch block\n");
1092 return true;
1093 }
1094 if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
1095 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
1096 "interleaved group requires scalar epilogue\n");
1097 return true;
1098 }
1099 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
1100 return false;
1101 }
1102
1103 /// Returns true if an epilogue is allowed (e.g., not prevented by
1104 /// optsize or a loop hint annotation).
1105 bool isEpilogueAllowed() const {
1106 return EpilogueLoweringStatus == CM_EpilogueAllowed;
1107 }
1108
1109 /// Returns true if tail-folding is preferred over an epilogue.
1110 bool preferTailFoldedLoop() const {
1111 return EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail ||
1112 EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail;
1113 }
1114
1115 /// Returns the TailFoldingStyle that is best for the current loop.
1116 TailFoldingStyle getTailFoldingStyle() const {
1117 return ChosenTailFoldingStyle;
1118 }
1119
1120 /// Selects and saves TailFoldingStyle.
1121 /// \param IsScalableVF true if scalable vector factors enabled.
1122 /// \param UserIC User specific interleave count.
1123 void setTailFoldingStyle(bool IsScalableVF, unsigned UserIC) {
1124 assert(ChosenTailFoldingStyle == TailFoldingStyle::None &&
1125 "Tail folding must not be selected yet.");
1126 if (!Legal->canFoldTailByMasking()) {
1127 ChosenTailFoldingStyle = TailFoldingStyle::None;
1128 return;
1129 }
1130
1131 // Default to TTI preference, but allow command line override.
1132 ChosenTailFoldingStyle = TTI.getPreferredTailFoldingStyle();
1133 if (ForceTailFoldingStyle.getNumOccurrences())
1134 ChosenTailFoldingStyle = ForceTailFoldingStyle.getValue();
1135
1136 if (ChosenTailFoldingStyle != TailFoldingStyle::DataWithEVL)
1137 return;
1138 // Override EVL styles if needed.
1139 // FIXME: Investigate opportunity for fixed vector factor.
1140 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
1141 TTI.hasActiveVectorLength() && !EnableVPlanNativePath;
1142 if (EVLIsLegal)
1143 return;
1144 // If for some reason EVL mode is unsupported, fallback to an epilogue
1145 // if it's allowed, or DataWithoutLaneMask otherwise.
1146 if (EpilogueLoweringStatus == CM_EpilogueAllowed ||
1147 EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail)
1148 ChosenTailFoldingStyle = TailFoldingStyle::None;
1149 else
1150 ChosenTailFoldingStyle = TailFoldingStyle::DataWithoutLaneMask;
1151
1152 LLVM_DEBUG(
1153 dbgs() << "LV: Preference for VP intrinsics indicated. Will "
1154 "not try to generate VP Intrinsics "
1155 << (UserIC > 1
1156 ? "since interleave count specified is greater than 1.\n"
1157 : "due to non-interleaving reasons.\n"));
1158 }
1159
1160 /// Returns true if all loop blocks should be masked to fold tail loop.
1161 bool foldTailByMasking() const {
1162 return getTailFoldingStyle() != TailFoldingStyle::None;
1163 }
1164
1165 void tryToEnablePartialAliasMasking() {
1166 assert(foldTailByMasking() && "Expected tail folding to be enabled!");
1167 assert(!foldTailWithEVL() &&
1168 "Did not expect to enable alias masking with EVL!");
1169 assert(PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided);
1170
1171 // Assume we fail to enable alias masking (in case we early exit).
1172 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
1173
1174 // Note: FixedOrderRecurrences are not supported yet as we cannot handle
1175 // the required `splice.right` with the alias-mask.
1176 if (!ForcePartialAliasingVectorization ||
1177 !Legal->getFixedOrderRecurrences().empty())
1178 return;
1179
1180 const RuntimePointerChecking *Checks = Legal->getRuntimePointerChecking();
1181 if (!Checks)
1182 return;
1183
1184 auto DiffChecks = Checks->getDiffChecks();
1185 if (!DiffChecks || DiffChecks->empty())
1186 return;
1187
1188 [[maybe_unused]] auto HasPointerArgs = [](CallBase *CB) {
1189 return any_of(Range: CB->args(), P: [](Value const *Arg) {
1190 return Arg->getType()->isPointerTy();
1191 });
1192 };
1193
1194 for (BasicBlock *BB : TheLoop->blocks()) {
1195 for (Instruction &I : *BB) {
1196 if (!isa<LoadInst, StoreInst>(Val: I)) {
1197 [[maybe_unused]] auto *Call = dyn_cast<CallInst>(Val: &I);
1198 assert(
1199 (!I.mayReadOrWriteMemory() || (Call && !HasPointerArgs(Call))) &&
1200 "Skipped unexpected memory access");
1201 continue;
1202 }
1203
1204 Type *ScalarTy = getLoadStoreType(I: &I);
1205 Value *Ptr = getLoadStorePointerOperand(V: &I);
1206
1207 // Currently, we can't handle alias masking in reverse. Reversing the
1208 // alias mask is not correct (or necessary). When combined with
1209 // tail-folding the active lane mask should only be reversed where the
1210 // alias-mask is true.
1211 if (Legal->isConsecutivePtr(AccessTy: ScalarTy, Ptr) == -1)
1212 return;
1213 }
1214 }
1215
1216 PartialAliasMaskingStatus = AliasMaskingStatus::Enabled;
1217 }
1218
1219 /// Returns true if all loop blocks should have partial aliases masked.
1220 bool maskPartialAliasing() const {
1221 return PartialAliasMaskingStatus == AliasMaskingStatus::Enabled;
1222 }
1223
1224 /// Returns true if the use of wide lane masks is requested and the loop is
1225 /// using tail-folding with a lane mask for control flow.
1226 bool useWideActiveLaneMask() const {
1227 if (!EnableWideActiveLaneMask)
1228 return false;
1229
1230 return getTailFoldingStyle() == TailFoldingStyle::DataAndControlFlow;
1231 }
1232
1233 /// Returns true if the instructions in this block requires predication
1234 /// for any reason, e.g. because tail folding now requires a predicate
1235 /// or because the block in the original loop was predicated.
1236 bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const {
1237 return foldTailByMasking() || Legal->blockNeedsPredication(BB);
1238 }
1239
1240 /// Returns true if VP intrinsics with explicit vector length support should
1241 /// be generated in the tail folded loop.
1242 bool foldTailWithEVL() const {
1243 return getTailFoldingStyle() == TailFoldingStyle::DataWithEVL;
1244 }
1245
1246 /// Returns true if the predicated reduction select should be used to set the
1247 /// incoming value for the reduction phi.
1248 bool usePredicatedReductionSelect(RecurKind RecurrenceKind) const {
1249 // Force to use predicated reduction select since the EVL of the
1250 // second-to-last iteration might not be VF*UF.
1251 if (foldTailWithEVL())
1252 return true;
1253
1254 // Force a predicated select with alias-masking to avoid propagating poison
1255 // values to the header phi for lanes outside the alias-mask.
1256 if (maskPartialAliasing())
1257 return true;
1258
1259 // Note: For FindLast recurrences we prefer a predicated select to simplify
1260 // matching in handleFindLastReductions(), rather than handle multiple
1261 // cases.
1262 if (RecurrenceDescriptor::isFindLastRecurrenceKind(Kind: RecurrenceKind))
1263 return true;
1264
1265 return PreferPredicatedReductionSelect ||
1266 TTI.preferPredicatedReductionSelect();
1267 }
1268
1269 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
1270 /// with factor VF. Return the cost of the instruction, including
1271 /// scalarization overhead if it's needed.
1272 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;
1273
1274 /// Estimate cost of a call instruction CI if it were vectorized with factor
1275 /// VF. Return the cost of the instruction, including scalarization overhead
1276 /// if it's needed.
1277 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const;
1278
1279 /// Invalidates decisions already taken by the cost model.
1280 void invalidateCostModelingDecisions() {
1281 WideningDecisions.clear();
1282 Uniforms.clear();
1283 Scalars.clear();
1284 }
1285
1286 /// Returns the expected execution cost. The unit of the cost does
1287 /// not matter because we use the 'cost' units to compare different
1288 /// vector widths. The cost that is returned is *not* normalized by
1289 /// the factor width.
1290 InstructionCost expectedCost(ElementCount VF);
1291
1292 /// Returns true if epilogue vectorization is considered profitable, and
1293 /// false otherwise.
1294 /// \p VF is the vectorization factor chosen for the original loop.
1295 /// \p Multiplier is an aditional scaling factor applied to VF before
1296 /// comparing to EpilogueVectorizationMinVF.
1297 bool isEpilogueVectorizationProfitable(const ElementCount VF,
1298 const unsigned IC) const;
1299
1300 /// Returns the execution time cost of an instruction for a given vector
1301 /// width. Vector width of one means scalar.
1302 InstructionCost getInstructionCost(Instruction *I, ElementCount VF);
1303
1304 /// Return the cost of instructions in an inloop reduction pattern, if I is
1305 /// part of that pattern.
1306 std::optional<InstructionCost> getReductionPatternCost(Instruction *I,
1307 ElementCount VF,
1308 Type *VectorTy) const;
1309
1310 /// Returns true if \p Op should be considered invariant and if it is
1311 /// trivially hoistable.
1312 bool shouldConsiderInvariant(Value *Op);
1313
1314 /// Returns true if \p I has been forced to be scalarized at \p VF.
1315 bool isForcedScalar(Instruction *I, ElementCount VF) const {
1316 auto FS = ForcedScalars.find(Val: VF);
1317 return FS != ForcedScalars.end() && FS->second.contains(Ptr: I);
1318 }
1319
1320private:
1321 unsigned NumPredStores = 0;
1322
1323 /// VF selection state independent of cost-modeling decisions.
1324 VFSelectionContext &Config;
1325
1326 /// Wrapper around LoopVectorizationLegality::isUniform() that takes into
1327 /// account if alias-masking is enabled. We consider the VF to be unknown when
1328 /// alias masking.
1329 bool isUniform(Value *V, ElementCount VF) const {
1330 // With alias-masking our runtime VF is [2, VF] (and not necessarily a
1331 // power-of-two). Something that is uniform for VF may not be for the full
1332 // range.
1333 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
1334 "alias-mask status must be decided already");
1335 return Legal->isUniform(V, VF: PartialAliasMaskingStatus ==
1336 AliasMaskingStatus::Disabled
1337 ? std::optional(VF)
1338 : std::nullopt);
1339 }
1340
1341 /// Wrapper around LoopVectorizationLegality::isUniformMemOp() that takes into
1342 /// account if alias-masking is enabled. We consider the VF to be unknown when
1343 /// alias masking.
1344 bool isUniformMemOp(Instruction &I, ElementCount VF) const {
1345 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
1346 "alias-mask status must be decided already");
1347 return Legal->isUniformMemOp(I, VF: PartialAliasMaskingStatus ==
1348 AliasMaskingStatus::Disabled
1349 ? std::optional(VF)
1350 : std::nullopt);
1351 }
1352
1353 /// Calculate vectorization cost of memory instruction \p I.
1354 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
1355
1356 /// The cost computation for scalarized memory instruction.
1357 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
1358
1359 /// The cost computation for interleaving group of memory instructions.
1360 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF);
1361
1362 /// The cost computation for Gather/Scatter instruction.
1363 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF);
1364
1365 /// The cost computation for widening instruction \p I with consecutive
1366 /// memory access.
1367 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF,
1368 InstWidening Kind);
1369
1370 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
1371 /// Load: scalar load + broadcast.
1372 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
1373 /// element)
1374 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF);
1375
1376 /// Estimate the overhead of scalarizing an instruction. This is a
1377 /// convenience wrapper for the type-based getScalarizationOverhead API.
1378 InstructionCost getScalarizationOverhead(Instruction *I,
1379 ElementCount VF) const;
1380
1381 /// A type representing the costs for instructions if they were to be
1382 /// scalarized rather than vectorized. The entries are Instruction-Cost
1383 /// pairs.
1384 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
1385
1386 /// A set containing all BasicBlocks that are known to present after
1387 /// vectorization as a predicated block.
1388 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
1389 PredicatedBBsAfterVectorization;
1390
1391 /// Records whether it is allowed to have the original scalar loop execute at
1392 /// least once. This may be needed as a fallback loop in case runtime
1393 /// aliasing/dependence checks fail, or to handle the tail/remainder
1394 /// iterations when the trip count is unknown or doesn't divide by the VF,
1395 /// or as a peel-loop to handle gaps in interleave-groups.
1396 /// Under optsize and when the trip count is very small we don't allow any
1397 /// iterations to execute in the scalar loop.
1398 EpilogueLowering EpilogueLoweringStatus = CM_EpilogueAllowed;
1399
1400 /// Control finally chosen tail folding style.
1401 TailFoldingStyle ChosenTailFoldingStyle = TailFoldingStyle::None;
1402
1403 /// If partial alias masking is enabled/disabled or not decided.
1404 AliasMaskingStatus PartialAliasMaskingStatus = AliasMaskingStatus::NotDecided;
1405
1406 /// A map holding scalar costs for different vectorization factors. The
1407 /// presence of a cost for an instruction in the mapping indicates that the
1408 /// instruction will be scalarized when vectorizing with the associated
1409 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
1410 MapVector<ElementCount, ScalarCostsTy> InstsToScalarize;
1411
1412 /// Holds the instructions known to be uniform after vectorization.
1413 /// The data is collected per VF.
1414 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
1415
1416 /// Holds the instructions known to be scalar after vectorization.
1417 /// The data is collected per VF.
1418 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
1419
1420 /// Holds the instructions (address computations) that are forced to be
1421 /// scalarized.
1422 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> ForcedScalars;
1423
1424 /// Returns the expected difference in cost from scalarizing the expression
1425 /// feeding a predicated instruction \p PredInst. The instructions to
1426 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
1427 /// non-negative return value implies the expression will be scalarized.
1428 /// Currently, only single-use chains are considered for scalarization.
1429 InstructionCost computePredInstDiscount(Instruction *PredInst,
1430 ScalarCostsTy &ScalarCosts,
1431 ElementCount VF);
1432
1433 /// Collect the instructions that are uniform after vectorization. An
1434 /// instruction is uniform if we represent it with a single scalar value in
1435 /// the vectorized loop corresponding to each vector iteration. Examples of
1436 /// uniform instructions include pointer operands of consecutive or
1437 /// interleaved memory accesses. Note that although uniformity implies an
1438 /// instruction will be scalar, the reverse is not true. In general, a
1439 /// scalarized instruction will be represented by VF scalar values in the
1440 /// vectorized loop, each corresponding to an iteration of the original
1441 /// scalar loop.
1442 void collectLoopUniforms(ElementCount VF);
1443
1444 /// Collect the instructions that are scalar after vectorization. An
1445 /// instruction is scalar if it is known to be uniform or will be scalarized
1446 /// during vectorization. collectLoopScalars should only add non-uniform nodes
1447 /// to the list if they are used by a load/store instruction that is marked as
1448 /// CM_Scalarize. Non-uniform scalarized instructions will be represented by
1449 /// VF values in the vectorized loop, each corresponding to an iteration of
1450 /// the original scalar loop.
1451 void collectLoopScalars(ElementCount VF);
1452
1453 /// Keeps cost model vectorization decision and cost for instructions.
1454 /// Right now it is used for memory instructions only.
1455 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1456 std::pair<InstWidening, InstructionCost>>;
1457
1458 DecisionList WideningDecisions;
1459
1460 /// Returns true if \p V is expected to be vectorized and it needs to be
1461 /// extracted.
1462 bool needsExtract(Value *V, ElementCount VF) const {
1463 Instruction *I = dyn_cast<Instruction>(Val: V);
1464 if (VF.isScalar() || !I || !TheLoop->contains(Inst: I) ||
1465 TheLoop->isLoopInvariant(V: I) ||
1466 getWideningDecision(I, VF) == CM_Scalarize)
1467 return false;
1468
1469 // Assume we can vectorize V (and hence we need extraction) if the
1470 // scalars are not computed yet. This can happen, because it is called
1471 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1472 // the scalars are collected. That should be a safe assumption in most
1473 // cases, because we check if the operands have vectorizable types
1474 // beforehand in LoopVectorizationLegality.
1475 return !Scalars.contains(Val: VF) || !isScalarAfterVectorization(I, VF);
1476 };
1477
1478 /// Returns a range containing only operands needing to be extracted.
1479 SmallVector<Value *, 4> filterExtractingOperands(Instruction::op_range Ops,
1480 ElementCount VF) const {
1481
1482 SmallPtrSet<const Value *, 4> UniqueOperands;
1483 SmallVector<Value *, 4> Res;
1484 for (Value *Op : Ops) {
1485 if (isa<Constant>(Val: Op) || !UniqueOperands.insert(Ptr: Op).second ||
1486 !needsExtract(V: Op, VF))
1487 continue;
1488 Res.push_back(Elt: Op);
1489 }
1490 return Res;
1491 }
1492
1493public:
1494 /// The loop that we evaluate.
1495 Loop *TheLoop;
1496
1497 /// Predicated scalar evolution analysis.
1498 PredicatedScalarEvolution &PSE;
1499
1500 /// Loop Info analysis.
1501 LoopInfo *LI;
1502
1503 /// Vectorization legality.
1504 LoopVectorizationLegality *Legal;
1505
1506 /// Vector target information.
1507 const TargetTransformInfo &TTI;
1508
1509 /// Target Library Info.
1510 const TargetLibraryInfo *TLI;
1511
1512 /// Assumption cache.
1513 AssumptionCache *AC;
1514
1515 /// Interface to emit optimization remarks.
1516 OptimizationRemarkEmitter *ORE;
1517
1518 /// A function to lazily fetch BlockFrequencyInfo. This avoids computing it
1519 /// unless necessary, e.g. when the loop isn't legal to vectorize or when
1520 /// there is no predication.
1521 std::function<BlockFrequencyInfo &()> GetBFI;
1522 /// The BlockFrequencyInfo returned from GetBFI.
1523 BlockFrequencyInfo *BFI = nullptr;
1524 /// Returns the BlockFrequencyInfo for the function if cached, otherwise
1525 /// fetches it via GetBFI. Avoids an indirect call to the std::function.
1526 BlockFrequencyInfo &getBFI() {
1527 if (!BFI)
1528 BFI = &GetBFI();
1529 return *BFI;
1530 }
1531
1532 const Function *TheFunction;
1533
1534 /// Loop Vectorize Hint.
1535 const LoopVectorizeHints *Hints;
1536
1537 /// The interleave access information contains groups of interleaved accesses
1538 /// with the same stride and close to each other.
1539 InterleavedAccessInfo &InterleaveInfo;
1540
1541 /// Values to ignore in the cost model.
1542 SmallPtrSet<const Value *, 16> ValuesToIgnore;
1543
1544 /// Values to ignore in the cost model when VF > 1.
1545 SmallPtrSet<const Value *, 16> VecValuesToIgnore;
1546};
1547} // end namespace llvm
1548
1549namespace {
1550/// Helper struct to manage generating runtime checks for vectorization.
1551///
1552/// The runtime checks are created up-front in temporary blocks to allow better
1553/// estimating the cost and un-linked from the existing IR. After deciding to
1554/// vectorize, the checks are moved back. If deciding not to vectorize, the
1555/// temporary blocks are completely removed.
1556class GeneratedRTChecks {
1557 /// Basic block which contains the generated SCEV checks, if any.
1558 BasicBlock *SCEVCheckBlock = nullptr;
1559
1560 /// The value representing the result of the generated SCEV checks. If it is
1561 /// nullptr no SCEV checks have been generated.
1562 Value *SCEVCheckCond = nullptr;
1563
1564 /// Basic block which contains the generated memory runtime checks, if any.
1565 BasicBlock *MemCheckBlock = nullptr;
1566
1567 /// The value representing the result of the generated memory runtime checks.
1568 /// If it is nullptr no memory runtime checks have been generated.
1569 Value *MemRuntimeCheckCond = nullptr;
1570
1571 DominatorTree *DT;
1572 LoopInfo *LI;
1573 TargetTransformInfo *TTI;
1574
1575 SCEVExpander SCEVExp;
1576 SCEVExpander MemCheckExp;
1577
1578 bool CostTooHigh = false;
1579
1580 Loop *OuterLoop = nullptr;
1581
1582 PredicatedScalarEvolution &PSE;
1583
1584 /// The kind of cost that we are calculating
1585 TTI::TargetCostKind CostKind;
1586
1587 /// True if the loop is alias-masked (which allows us to omit diff checks).
1588 bool LoopUsesPartialAliasMasking = false;
1589
1590public:
1591 GeneratedRTChecks(PredicatedScalarEvolution &PSE, DominatorTree *DT,
1592 LoopInfo *LI, TargetTransformInfo *TTI,
1593 TTI::TargetCostKind CostKind,
1594 bool LoopUsesPartialAliasMasking)
1595 : DT(DT), LI(LI), TTI(TTI),
1596 SCEVExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1597 MemCheckExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1598 PSE(PSE), CostKind(CostKind),
1599 LoopUsesPartialAliasMasking(LoopUsesPartialAliasMasking) {}
1600
1601 /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
1602 /// accurately estimate the cost of the runtime checks. The blocks are
1603 /// un-linked from the IR and are added back during vector code generation. If
1604 /// there is no vector code generation, the check blocks are removed
1605 /// completely.
1606 void create(Loop *L, const LoopAccessInfo &LAI,
1607 const SCEVPredicate &UnionPred, ElementCount VF, unsigned IC,
1608 OptimizationRemarkEmitter &ORE) {
1609
1610 // Hard cutoff to limit compile-time increase in case a very large number of
1611 // runtime checks needs to be generated.
1612 // TODO: Skip cutoff if the loop is guaranteed to execute, e.g. due to
1613 // profile info.
1614 CostTooHigh =
1615 LAI.getNumRuntimePointerChecks() > VectorizeMemoryCheckThreshold;
1616 if (CostTooHigh) {
1617 // Mark runtime checks as never succeeding when they exceed the threshold.
1618 MemRuntimeCheckCond = ConstantInt::getTrue(Context&: L->getHeader()->getContext());
1619 SCEVCheckCond = ConstantInt::getTrue(Context&: L->getHeader()->getContext());
1620 ORE.emit(RemarkBuilder: [&]() {
1621 return OptimizationRemarkAnalysisAliasing(
1622 DEBUG_TYPE, "TooManyMemoryRuntimeChecks", L->getStartLoc(),
1623 L->getHeader())
1624 << "loop not vectorized: too many memory checks needed";
1625 });
1626 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
1627 return;
1628 }
1629
1630 BasicBlock *LoopHeader = L->getHeader();
1631 BasicBlock *Preheader = L->getLoopPreheader();
1632
1633 // Use SplitBlock to create blocks for SCEV & memory runtime checks to
1634 // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
1635 // may be used by SCEVExpander. The blocks will be un-linked from their
1636 // predecessors and removed from LI & DT at the end of the function.
1637 if (!UnionPred.isAlwaysTrue()) {
1638 SCEVCheckBlock = SplitBlock(Old: Preheader, SplitPt: Preheader->getTerminator(), DT, LI,
1639 MSSAU: nullptr, BBName: "vector.scevcheck");
1640
1641 SCEVCheckCond = SCEVExp.expandCodeForPredicate(
1642 Pred: &UnionPred, Loc: SCEVCheckBlock->getTerminator());
1643 if (isa<Constant>(Val: SCEVCheckCond)) {
1644 // Clean up directly after expanding the predicate to a constant, to
1645 // avoid further expansions re-using anything left over from SCEVExp.
1646 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1647 SCEVCleaner.cleanup();
1648 }
1649 }
1650
1651 const auto &RtPtrChecking = *LAI.getRuntimePointerChecking();
1652 // TODO: We need to estimate the cost of alias-masking in
1653 // GeneratedRTChecks::getCost(). We can't check the MemCheckBlock as the
1654 // alias-mask is generated later in VPlan.
1655 if (RtPtrChecking.Need && !LoopUsesPartialAliasMasking) {
1656 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1657 MemCheckBlock = SplitBlock(Old: Pred, SplitPt: Pred->getTerminator(), DT, LI, MSSAU: nullptr,
1658 BBName: "vector.memcheck");
1659
1660 auto DiffChecks = RtPtrChecking.getDiffChecks();
1661 if (DiffChecks) {
1662 Value *RuntimeVF = nullptr;
1663 MemRuntimeCheckCond = addDiffRuntimeChecks(
1664 Loc: MemCheckBlock->getTerminator(), Checks: *DiffChecks, Expander&: MemCheckExp,
1665 GetVF: [VF, &RuntimeVF](IRBuilderBase &B, unsigned Bits) {
1666 if (!RuntimeVF)
1667 RuntimeVF = getRuntimeVF(B, Ty: B.getIntNTy(N: Bits), VF);
1668 return RuntimeVF;
1669 },
1670 IC);
1671 } else {
1672 MemRuntimeCheckCond = addRuntimeChecks(
1673 Loc: MemCheckBlock->getTerminator(), TheLoop: L, PointerChecks: RtPtrChecking.getChecks(),
1674 Expander&: MemCheckExp, HoistRuntimeChecks: VectorizerParams::HoistRuntimeChecks);
1675 }
1676 assert(MemRuntimeCheckCond &&
1677 "no RT checks generated although RtPtrChecking "
1678 "claimed checks are required");
1679 }
1680
1681 SCEVExp.eraseDeadInstructions(Root: SCEVCheckCond);
1682
1683 if (!MemCheckBlock && !SCEVCheckBlock)
1684 return;
1685
1686 // Unhook the temporary block with the checks, update various places
1687 // accordingly.
1688 if (SCEVCheckBlock)
1689 SCEVCheckBlock->replaceAllUsesWith(V: Preheader);
1690 if (MemCheckBlock)
1691 MemCheckBlock->replaceAllUsesWith(V: Preheader);
1692
1693 if (SCEVCheckBlock) {
1694 SCEVCheckBlock->getTerminator()->moveBefore(
1695 InsertPos: Preheader->getTerminator()->getIterator());
1696 auto *UI = new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
1697 UI->setDebugLoc(DebugLoc::getTemporary());
1698 Preheader->getTerminator()->eraseFromParent();
1699 }
1700 if (MemCheckBlock) {
1701 MemCheckBlock->getTerminator()->moveBefore(
1702 InsertPos: Preheader->getTerminator()->getIterator());
1703 auto *UI = new UnreachableInst(Preheader->getContext(), MemCheckBlock);
1704 UI->setDebugLoc(DebugLoc::getTemporary());
1705 Preheader->getTerminator()->eraseFromParent();
1706 }
1707
1708 DT->changeImmediateDominator(BB: LoopHeader, NewBB: Preheader);
1709 if (MemCheckBlock) {
1710 DT->eraseNode(BB: MemCheckBlock);
1711 LI->removeBlock(BB: MemCheckBlock);
1712 }
1713 if (SCEVCheckBlock) {
1714 DT->eraseNode(BB: SCEVCheckBlock);
1715 LI->removeBlock(BB: SCEVCheckBlock);
1716 }
1717
1718 // Outer loop is used as part of the later cost calculations.
1719 OuterLoop = L->getParentLoop();
1720 }
1721
1722 InstructionCost getCost() {
1723 if (SCEVCheckBlock || MemCheckBlock)
1724 LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
1725
1726 if (CostTooHigh) {
1727 InstructionCost Cost;
1728 Cost.setInvalid();
1729 LLVM_DEBUG(dbgs() << " number of checks exceeded threshold\n");
1730 return Cost;
1731 }
1732
1733 InstructionCost RTCheckCost = 0;
1734 if (SCEVCheckBlock)
1735 for (Instruction &I : *SCEVCheckBlock) {
1736 if (SCEVCheckBlock->getTerminator() == &I)
1737 continue;
1738 InstructionCost C = TTI->getInstructionCost(U: &I, CostKind);
1739 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1740 RTCheckCost += C;
1741 }
1742 if (MemCheckBlock) {
1743 InstructionCost MemCheckCost = 0;
1744 for (Instruction &I : *MemCheckBlock) {
1745 if (MemCheckBlock->getTerminator() == &I)
1746 continue;
1747 InstructionCost C = TTI->getInstructionCost(U: &I, CostKind);
1748 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1749 MemCheckCost += C;
1750 }
1751
1752 // If the runtime memory checks are being created inside an outer loop
1753 // we should find out if these checks are outer loop invariant. If so,
1754 // the checks will likely be hoisted out and so the effective cost will
1755 // reduce according to the outer loop trip count.
1756 if (OuterLoop) {
1757 ScalarEvolution *SE = MemCheckExp.getSE();
1758 // TODO: If profitable, we could refine this further by analysing every
1759 // individual memory check, since there could be a mixture of loop
1760 // variant and invariant checks that mean the final condition is
1761 // variant.
1762 const SCEV *Cond = SE->getSCEV(V: MemRuntimeCheckCond);
1763 if (SE->isLoopInvariant(S: Cond, L: OuterLoop)) {
1764 // It seems reasonable to assume that we can reduce the effective
1765 // cost of the checks even when we know nothing about the trip
1766 // count. Assume that the outer loop executes at least twice.
1767 unsigned BestTripCount = 2;
1768
1769 // Get the best known TC estimate.
1770 if (auto EstimatedTC = getSmallBestKnownTC(
1771 PSE, L: OuterLoop, /* CanUseConstantMax = */ false))
1772 if (EstimatedTC->isFixed())
1773 BestTripCount = EstimatedTC->getFixedValue();
1774
1775 InstructionCost NewMemCheckCost = MemCheckCost / BestTripCount;
1776
1777 // Let's ensure the cost is always at least 1.
1778 NewMemCheckCost = std::max(a: NewMemCheckCost.getValue(),
1779 b: (InstructionCost::CostType)1);
1780
1781 if (BestTripCount > 1)
1782 LLVM_DEBUG(dbgs()
1783 << "We expect runtime memory checks to be hoisted "
1784 << "out of the outer loop. Cost reduced from "
1785 << MemCheckCost << " to " << NewMemCheckCost << '\n');
1786
1787 MemCheckCost = NewMemCheckCost;
1788 }
1789 }
1790
1791 RTCheckCost += MemCheckCost;
1792 }
1793
1794 if (SCEVCheckBlock || MemCheckBlock)
1795 LLVM_DEBUG(dbgs() << "Total cost of runtime checks: " << RTCheckCost
1796 << "\n");
1797
1798 return RTCheckCost;
1799 }
1800
1801 /// Remove the created SCEV & memory runtime check blocks & instructions, if
1802 /// unused.
1803 ~GeneratedRTChecks() {
1804 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1805 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1806 bool SCEVChecksUsed = !SCEVCheckBlock || !pred_empty(BB: SCEVCheckBlock);
1807 bool MemChecksUsed = !MemCheckBlock || !pred_empty(BB: MemCheckBlock);
1808 if (SCEVChecksUsed)
1809 SCEVCleaner.markResultUsed();
1810
1811 if (MemChecksUsed) {
1812 MemCheckCleaner.markResultUsed();
1813 } else {
1814 auto &SE = *MemCheckExp.getSE();
1815 // Memory runtime check generation creates compares that use expanded
1816 // values. Remove them before running the SCEVExpanderCleaners.
1817 for (auto &I : make_early_inc_range(Range: reverse(C&: *MemCheckBlock))) {
1818 if (MemCheckExp.isInsertedInstruction(I: &I))
1819 continue;
1820 SE.forgetValue(V: &I);
1821 I.eraseFromParent();
1822 }
1823 }
1824 MemCheckCleaner.cleanup();
1825 SCEVCleaner.cleanup();
1826
1827 if (!SCEVChecksUsed)
1828 SCEVCheckBlock->eraseFromParent();
1829 if (!MemChecksUsed)
1830 MemCheckBlock->eraseFromParent();
1831 }
1832
1833 /// Retrieves the SCEVCheckCond and SCEVCheckBlock that were generated as IR
1834 /// outside VPlan.
1835 std::pair<Value *, BasicBlock *> getSCEVChecks() const {
1836 using namespace llvm::PatternMatch;
1837 if (!SCEVCheckCond || match(V: SCEVCheckCond, P: m_ZeroInt()))
1838 return {nullptr, nullptr};
1839
1840 return {SCEVCheckCond, SCEVCheckBlock};
1841 }
1842
1843 /// Retrieves the MemCheckCond and MemCheckBlock that were generated as IR
1844 /// outside VPlan.
1845 std::pair<Value *, BasicBlock *> getMemRuntimeChecks() const {
1846 using namespace llvm::PatternMatch;
1847 if (MemRuntimeCheckCond && match(V: MemRuntimeCheckCond, P: m_ZeroInt()))
1848 return {nullptr, nullptr};
1849 return {MemRuntimeCheckCond, MemCheckBlock};
1850 }
1851
1852 /// Return true if any runtime checks have been added
1853 bool hasChecks() const {
1854 return getSCEVChecks().first || getMemRuntimeChecks().first;
1855 }
1856};
1857} // namespace
1858
1859static bool useActiveLaneMask(TailFoldingStyle Style) {
1860 return Style == TailFoldingStyle::Data ||
1861 Style == TailFoldingStyle::DataAndControlFlow;
1862}
1863
1864static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style) {
1865 return Style == TailFoldingStyle::DataAndControlFlow;
1866}
1867
1868// Return true if \p OuterLp is an outer loop annotated with hints for explicit
1869// vectorization. The loop needs to be annotated with #pragma omp simd
1870// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
1871// vector length information is not provided, vectorization is not considered
1872// explicit. Interleave hints are not allowed either. These limitations will be
1873// relaxed in the future.
1874// Please, note that we are currently forced to abuse the pragma 'clang
1875// vectorize' semantics. This pragma provides *auto-vectorization hints*
1876// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
1877// provides *explicit vectorization hints* (LV can bypass legal checks and
1878// assume that vectorization is legal). However, both hints are implemented
1879// using the same metadata (llvm.loop.vectorize, processed by
1880// LoopVectorizeHints). This will be fixed in the future when the native IR
1881// representation for pragma 'omp simd' is introduced.
1882static bool isExplicitVecOuterLoop(Loop *OuterLp,
1883 OptimizationRemarkEmitter *ORE) {
1884 assert(!OuterLp->isInnermost() && "This is not an outer loop");
1885 LoopVectorizeHints Hints(OuterLp, true /*DisableInterleaving*/, *ORE);
1886
1887 // Only outer loops with an explicit vectorization hint are supported.
1888 // Unannotated outer loops are ignored.
1889 if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
1890 return false;
1891
1892 Function *Fn = OuterLp->getHeader()->getParent();
1893 if (!Hints.allowVectorization(F: Fn, L: OuterLp,
1894 VectorizeOnlyWhenForced: true /*VectorizeOnlyWhenForced*/)) {
1895 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
1896 return false;
1897 }
1898
1899 if (Hints.getInterleave() > 1) {
1900 // TODO: Interleave support is future work.
1901 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
1902 "outer loops.\n");
1903 Hints.emitRemarkWithHints();
1904 return false;
1905 }
1906
1907 return true;
1908}
1909
1910static void collectSupportedLoops(Loop &L, LoopInfo *LI,
1911 OptimizationRemarkEmitter *ORE,
1912 SmallVectorImpl<Loop *> &V) {
1913 // Collect inner loops and outer loops without irreducible control flow. For
1914 // now, only collect outer loops that have explicit vectorization hints. If we
1915 // are stress testing the VPlan H-CFG construction, we collect the outermost
1916 // loop of every loop nest.
1917 if (L.isInnermost() || VPlanBuildOuterloopStressTest ||
1918 (EnableVPlanNativePath && isExplicitVecOuterLoop(OuterLp: &L, ORE))) {
1919 LoopBlocksRPO RPOT(&L);
1920 RPOT.perform(LI);
1921 if (!containsIrreducibleCFG<const BasicBlock *>(RPOTraversal&: RPOT, LI: *LI)) {
1922 V.push_back(Elt: &L);
1923 // TODO: Collect inner loops inside marked outer loops in case
1924 // vectorization fails for the outer loop. Do not invoke
1925 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
1926 // already known to be reducible. We can use an inherited attribute for
1927 // that.
1928 return;
1929 }
1930 }
1931 for (Loop *InnerL : L)
1932 collectSupportedLoops(L&: *InnerL, LI, ORE, V);
1933}
1934
1935//===----------------------------------------------------------------------===//
1936// Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
1937// LoopVectorizationCostModel and LoopVectorizationPlanner.
1938//===----------------------------------------------------------------------===//
1939
1940/// For the given VF and UF and maximum trip count computed for the loop, return
1941/// whether the induction variable might overflow in the vectorized loop. If not,
1942/// then we know a runtime overflow check always evaluates to false and can be
1943/// removed.
1944static bool isIndvarOverflowCheckKnownFalse(
1945 const LoopVectorizationCostModel *Cost,
1946 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
1947 // Always be conservative if we don't know the exact unroll factor.
1948 unsigned MaxUF = UF ? *UF
1949 : std::max(a: Cost->TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions: false),
1950 b: Cost->TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions: true));
1951
1952 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
1953 APInt MaxUIntTripCount = IdxTy->getMask();
1954
1955 // We know the runtime overflow check is known false iff the (max) trip-count
1956 // is known and (max) trip-count + (VF * UF) does not overflow in the type of
1957 // the vector loop induction variable.
1958 if (std::optional<ElementCount> TC = getSmallBestKnownTC(
1959 PSE&: Cost->PSE, L: Cost->TheLoop,
1960 /*CanUseConstantMax=*/true, /*CanExcludeZeroTrips=*/false,
1961 /*ComputeUpperBoundOnly=*/true)) {
1962 unsigned MaxVF = VF.getKnownMinValue();
1963 unsigned MaxTC = TC->getKnownMinValue();
1964 if (VF.isScalable() || TC->isScalable()) {
1965 std::optional<unsigned> MaxVScale =
1966 getMaxVScale(F: *Cost->TheFunction, TTI: Cost->TTI);
1967 if (!MaxVScale)
1968 return false;
1969 if (VF.isScalable())
1970 MaxVF *= *MaxVScale;
1971 if (TC->isScalable()) {
1972 bool Overflow;
1973 MaxTC = SaturatingMultiply(X: MaxTC, Y: *MaxVScale, ResultOverflowed: &Overflow);
1974 if (Overflow)
1975 return false;
1976 }
1977 }
1978
1979 return (MaxUIntTripCount - MaxTC).ugt(RHS: MaxVF * MaxUF);
1980 }
1981
1982 return false;
1983}
1984
1985// Return whether we allow using masked interleave-groups (for dealing with
1986// strided loads/stores that reside in predicated blocks, or for dealing
1987// with gaps).
1988static bool useMaskedInterleavedAccesses(const TargetTransformInfo &TTI) {
1989 // If an override option has been passed in for interleaved accesses, use it.
1990 if (EnableMaskedInterleavedMemAccesses.getNumOccurrences() > 0)
1991 return EnableMaskedInterleavedMemAccesses;
1992
1993 return TTI.enableMaskedInterleavedAccessVectorization();
1994}
1995
1996/// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
1997/// VPBB are moved to the end of the newly created VPIRBasicBlock. All
1998/// predecessors and successors of VPBB, if any, are rewired to the new
1999/// VPIRBasicBlock. If \p VPBB may be unreachable, \p Plan must be passed.
2000static VPIRBasicBlock *replaceVPBBWithIRVPBB(VPBasicBlock *VPBB,
2001 BasicBlock *IRBB,
2002 VPlan *Plan = nullptr) {
2003 if (!Plan)
2004 Plan = VPBB->getPlan();
2005 VPIRBasicBlock *IRVPBB = Plan->createVPIRBasicBlock(IRBB);
2006 auto IP = IRVPBB->begin();
2007 for (auto &R : make_early_inc_range(Range: VPBB->phis()))
2008 R.moveBefore(BB&: *IRVPBB, I: IP);
2009
2010 for (auto &R :
2011 make_early_inc_range(Range: make_range(x: VPBB->getFirstNonPhi(), y: VPBB->end())))
2012 R.moveBefore(BB&: *IRVPBB, I: IRVPBB->end());
2013
2014 VPBlockUtils::reassociateBlocks(Old: VPBB, New: IRVPBB);
2015 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
2016 return IRVPBB;
2017}
2018
2019BasicBlock *InnerLoopVectorizer::createScalarPreheader(StringRef Prefix) {
2020 BasicBlock *VectorPH = OrigLoop->getLoopPreheader();
2021 assert(VectorPH && "Invalid loop structure");
2022
2023 // NOTE: The Plan's scalar preheader VPBB isn't replaced with a VPIRBasicBlock
2024 // wrapping the newly created scalar preheader here at the moment, because the
2025 // Plan's scalar preheader may be unreachable at this point. Instead it is
2026 // replaced in executePlan.
2027 return SplitBlock(Old: VectorPH, SplitPt: VectorPH->getTerminator(), DT, LI, MSSAU: nullptr,
2028 BBName: Twine(Prefix) + "scalar.ph");
2029}
2030
2031/// Knowing that loop \p L executes a single vector iteration, add instructions
2032/// that will get simplified and thus should not have any cost to \p
2033/// InstsToIgnore.
2034static void addFullyUnrolledInstructionsToIgnore(
2035 Loop *L, const LoopVectorizationLegality::InductionList &IL,
2036 SmallPtrSetImpl<Instruction *> &InstsToIgnore) {
2037 auto *Cmp = L->getLatchCmpInst();
2038 if (Cmp)
2039 InstsToIgnore.insert(Ptr: Cmp);
2040 for (const auto &KV : IL) {
2041 // Extract the key by hand so that it can be used in the lambda below. Note
2042 // that captured structured bindings are a C++20 extension.
2043 const PHINode *IV = KV.first;
2044
2045 // Get next iteration value of the induction variable.
2046 Instruction *IVInst =
2047 cast<Instruction>(Val: IV->getIncomingValueForBlock(BB: L->getLoopLatch()));
2048 if (all_of(Range: IVInst->users(),
2049 P: [&](const User *U) { return U == IV || U == Cmp; }))
2050 InstsToIgnore.insert(Ptr: IVInst);
2051 }
2052}
2053
2054BasicBlock *InnerLoopVectorizer::createVectorizedLoopSkeleton() {
2055 // Create a new IR basic block for the scalar preheader.
2056 BasicBlock *ScalarPH = createScalarPreheader(Prefix: "");
2057 return ScalarPH->getSinglePredecessor();
2058}
2059
2060namespace {
2061
2062struct CSEDenseMapInfo {
2063 static bool canHandle(const Instruction *I) {
2064 return isa<InsertElementInst>(Val: I) || isa<ExtractElementInst>(Val: I) ||
2065 isa<ShuffleVectorInst>(Val: I) || isa<GetElementPtrInst>(Val: I);
2066 }
2067
2068 static unsigned getHashValue(const Instruction *I) {
2069 assert(canHandle(I) && "Unknown instruction!");
2070 return hash_combine(args: I->getOpcode(),
2071 args: hash_combine_range(R: I->operand_values()));
2072 }
2073
2074 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
2075 return LHS->isIdenticalTo(I: RHS);
2076 }
2077};
2078
2079} // end anonymous namespace
2080
2081/// FIXME: This legacy common-subexpression-elimination routine is scheduled for
2082/// removal, in favor of the VPlan-based one.
2083static void legacyCSE(BasicBlock *BB) {
2084 // Perform simple cse.
2085 SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
2086 for (Instruction &In : llvm::make_early_inc_range(Range&: *BB)) {
2087 if (!CSEDenseMapInfo::canHandle(I: &In))
2088 continue;
2089
2090 // Check if we can replace this instruction with any of the
2091 // visited instructions.
2092 if (Instruction *V = CSEMap.lookup(Val: &In)) {
2093 In.replaceAllUsesWith(V);
2094 In.eraseFromParent();
2095 continue;
2096 }
2097
2098 CSEMap[&In] = &In;
2099 }
2100}
2101
2102/// This function attempts to return a value that represents the ElementCount
2103/// at runtime. For fixed-width VFs we know this precisely at compile
2104/// time, but for scalable VFs we calculate it based on an estimate of the
2105/// vscale value.
2106static unsigned estimateElementCount(ElementCount VF,
2107 std::optional<unsigned> VScale) {
2108 unsigned EstimatedVF = VF.getKnownMinValue();
2109 if (VF.isScalable())
2110 if (VScale)
2111 EstimatedVF *= *VScale;
2112 assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
2113 return EstimatedVF;
2114}
2115
2116/// Returns the vector library variant function of \p CI usable at \p VF,
2117/// respecting \p MaskRequired, or nullptr if none is found: a mapping with
2118/// matching VF, masked if required, whose vector function is declared in the
2119/// module.
2120static Function *getVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
2121 bool MaskRequired,
2122 const TargetLibraryInfo *TLI) {
2123 if (!TLI || CI.isNoBuiltin())
2124 return nullptr;
2125 for (const VFInfo &Info : VFDatabase::getMappings(CI))
2126 if (Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()))
2127 if (Function *F = CI.getModule()->getFunction(Name: Info.VectorName))
2128 return F;
2129 return nullptr;
2130}
2131
2132/// Returns true iff \p CI has a library vector variant usable at \p VF.
2133static bool hasVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
2134 bool MaskRequired,
2135 const TargetLibraryInfo *TLI) {
2136 return getVectorLibraryVariantFor(CI, VF, MaskRequired, TLI) != nullptr;
2137}
2138
2139InstructionCost
2140LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
2141 ElementCount VF) const {
2142 Type *RetTy = CI->getType();
2143 SmallVector<Type *, 4> Tys;
2144 for (auto &ArgOp : CI->args())
2145 Tys.push_back(Elt: ArgOp->getType());
2146
2147 InstructionCost ScalarCallCost = TTI.getCallInstrCost(
2148 F: CI->getCalledFunction(), RetTy, Tys, CostKind: Config.CostKind);
2149
2150 // Cost of the scalar call (scalar VF) or its scalarization (vector VF). The
2151 // scalarization cost is only meaningful for fixed VFs.
2152 InstructionCost Cost = VF.isScalable()
2153 ? InstructionCost::getInvalid()
2154 : ScalarCallCost * VF.getKnownMinValue() +
2155 getScalarizationOverhead(I: CI, VF);
2156
2157 // The call may be vectorized at this VF, via a vector intrinsic or a vector
2158 // library variant.
2159 if (getVectorIntrinsicIDForCall(CI, TLI))
2160 Cost = std::min(a: Cost, b: getVectorIntrinsicCost(CI, VF));
2161
2162 if (Function *Variant =
2163 getVectorLibraryVariantFor(CI: *CI, VF, MaskRequired: isMaskRequired(I: CI), TLI))
2164 Cost = std::min(a: Cost,
2165 b: TTI.getCallInstrCost(
2166 /*F=*/nullptr, RetTy: Variant->getReturnType(),
2167 Tys: Variant->getFunctionType()->params(), CostKind: Config.CostKind));
2168
2169 return Cost;
2170}
2171
2172static Type *maybeVectorizeType(Type *Ty, ElementCount VF) {
2173 if (VF.isScalar() || !canVectorizeTy(Ty))
2174 return Ty;
2175 return toVectorizedTy(Ty, EC: VF);
2176}
2177
2178InstructionCost
2179LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
2180 ElementCount VF) const {
2181 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
2182 assert(ID && "Expected intrinsic call!");
2183 Type *RetTy = maybeVectorizeType(Ty: CI->getType(), VF);
2184 FastMathFlags FMF;
2185 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: CI))
2186 FMF = FPMO->getFastMathFlags();
2187
2188 SmallVector<const Value *> Arguments(CI->args());
2189 FunctionType *FTy = CI->getCalledFunction()->getFunctionType();
2190 SmallVector<Type *> ParamTys;
2191 std::transform(first: FTy->param_begin(), last: FTy->param_end(),
2192 result: std::back_inserter(x&: ParamTys),
2193 unary_op: [&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
2194
2195 IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
2196 dyn_cast<IntrinsicInst>(Val: CI),
2197 InstructionCost::getInvalid());
2198 return TTI.getIntrinsicInstrCost(ICA: CostAttrs, CostKind: Config.CostKind);
2199}
2200
2201void InnerLoopVectorizer::fixVectorizedLoop(VPTransformState &State) {
2202 // Don't apply optimizations below when no (vector) loop remains, as they all
2203 // require one at the moment.
2204 VPBasicBlock *HeaderVPBB =
2205 vputils::getFirstLoopHeader(Plan&: *State.Plan, VPDT&: State.VPDT);
2206 if (!HeaderVPBB)
2207 return;
2208
2209 BasicBlock *HeaderBB = State.CFG.VPBB2IRBB[HeaderVPBB];
2210
2211 // Remove redundant induction instructions.
2212 legacyCSE(BB: HeaderBB);
2213}
2214
2215void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
2216 // We should not collect Scalars more than once per VF. Right now, this
2217 // function is called from collectUniformsAndScalars(), which already does
2218 // this check. Collecting Scalars for VF=1 does not make any sense.
2219 assert(VF.isVector() && !Scalars.contains(VF) &&
2220 "This function should not be visited twice for the same VF");
2221
2222 // This avoids any chances of creating a REPLICATE recipe during planning
2223 // since that would result in generation of scalarized code during execution,
2224 // which is not supported for scalable vectors.
2225 if (VF.isScalable()) {
2226 Scalars[VF].insert_range(R&: Uniforms[VF]);
2227 return;
2228 }
2229
2230 SmallSetVector<Instruction *, 8> Worklist;
2231
2232 // These sets are used to seed the analysis with pointers used by memory
2233 // accesses that will remain scalar.
2234 SmallSetVector<Instruction *, 8> ScalarPtrs;
2235 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
2236 auto *Latch = TheLoop->getLoopLatch();
2237
2238 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
2239 // The pointer operands of loads and stores will be scalar as long as the
2240 // memory access is not a gather or scatter operation. The value operand of a
2241 // store will remain scalar if the store is scalarized.
2242 auto IsScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
2243 InstWidening WideningDecision = getWideningDecision(I: MemAccess, VF);
2244 assert(WideningDecision != CM_Unknown &&
2245 "Widening decision should be ready at this moment");
2246 if (auto *Store = dyn_cast<StoreInst>(Val: MemAccess))
2247 if (Ptr == Store->getValueOperand())
2248 return WideningDecision == CM_Scalarize;
2249 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&
2250 "Ptr is neither a value or pointer operand");
2251 return WideningDecision != CM_GatherScatter;
2252 };
2253
2254 // A helper that returns true if the given value is a getelementptr
2255 // instruction contained in the loop.
2256 auto IsLoopVaryingGEP = [&](Value *V) {
2257 return isa<GetElementPtrInst>(Val: V) && !TheLoop->isLoopInvariant(V);
2258 };
2259
2260 // A helper that evaluates a memory access's use of a pointer. If the use will
2261 // be a scalar use and the pointer is only used by memory accesses, we place
2262 // the pointer in ScalarPtrs. Otherwise, the pointer is placed in
2263 // PossibleNonScalarPtrs.
2264 auto EvaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
2265 // We only care about bitcast and getelementptr instructions contained in
2266 // the loop.
2267 if (!IsLoopVaryingGEP(Ptr))
2268 return;
2269
2270 // If the pointer has already been identified as scalar (e.g., if it was
2271 // also identified as uniform), there's nothing to do.
2272 auto *I = cast<Instruction>(Val: Ptr);
2273 if (Worklist.count(key: I))
2274 return;
2275
2276 // If the use of the pointer will be a scalar use, and all users of the
2277 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
2278 // place the pointer in PossibleNonScalarPtrs.
2279 if (IsScalarUse(MemAccess, Ptr) &&
2280 all_of(Range: I->users(), P: IsaPred<LoadInst, StoreInst>))
2281 ScalarPtrs.insert(X: I);
2282 else
2283 PossibleNonScalarPtrs.insert(Ptr: I);
2284 };
2285
2286 // We seed the scalars analysis with three classes of instructions: (1)
2287 // instructions marked uniform-after-vectorization and (2) bitcast,
2288 // getelementptr and (pointer) phi instructions used by memory accesses
2289 // requiring a scalar use.
2290 //
2291 // (1) Add to the worklist all instructions that have been identified as
2292 // uniform-after-vectorization.
2293 Worklist.insert_range(R&: Uniforms[VF]);
2294
2295 // (2) Add to the worklist all bitcast and getelementptr instructions used by
2296 // memory accesses requiring a scalar use. The pointer operands of loads and
2297 // stores will be scalar unless the operation is a gather or scatter.
2298 // The value operand of a store will remain scalar if the store is scalarized.
2299 for (auto *BB : TheLoop->blocks())
2300 for (auto &I : *BB) {
2301 if (auto *Load = dyn_cast<LoadInst>(Val: &I)) {
2302 EvaluatePtrUse(Load, Load->getPointerOperand());
2303 } else if (auto *Store = dyn_cast<StoreInst>(Val: &I)) {
2304 EvaluatePtrUse(Store, Store->getPointerOperand());
2305 EvaluatePtrUse(Store, Store->getValueOperand());
2306 }
2307 }
2308 for (auto *I : ScalarPtrs)
2309 if (!PossibleNonScalarPtrs.count(Ptr: I)) {
2310 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n");
2311 Worklist.insert(X: I);
2312 }
2313
2314 // Insert the forced scalars.
2315 // FIXME: Currently VPWidenPHIRecipe() often creates a dead vector
2316 // induction variable when the PHI user is scalarized.
2317 auto ForcedScalar = ForcedScalars.find(Val: VF);
2318 if (ForcedScalar != ForcedScalars.end())
2319 for (auto *I : ForcedScalar->second) {
2320 LLVM_DEBUG(dbgs() << "LV: Found (forced) scalar instruction: " << *I << "\n");
2321 Worklist.insert(X: I);
2322 }
2323
2324 // Expand the worklist by looking through any bitcasts and getelementptr
2325 // instructions we've already identified as scalar. This is similar to the
2326 // expansion step in collectLoopUniforms(); however, here we're only
2327 // expanding to include additional bitcasts and getelementptr instructions.
2328 unsigned Idx = 0;
2329 while (Idx != Worklist.size()) {
2330 Instruction *Dst = Worklist[Idx++];
2331 if (!IsLoopVaryingGEP(Dst->getOperand(i: 0)))
2332 continue;
2333 auto *Src = cast<Instruction>(Val: Dst->getOperand(i: 0));
2334 if (llvm::all_of(Range: Src->users(), P: [&](User *U) -> bool {
2335 auto *J = cast<Instruction>(Val: U);
2336 return !TheLoop->contains(Inst: J) || Worklist.count(key: J) ||
2337 ((isa<LoadInst>(Val: J) || isa<StoreInst>(Val: J)) &&
2338 IsScalarUse(J, Src));
2339 })) {
2340 Worklist.insert(X: Src);
2341 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n");
2342 }
2343 }
2344
2345 // An induction variable will remain scalar if all users of the induction
2346 // variable and induction variable update remain scalar.
2347 for (const auto &Induction : Legal->getInductionVars()) {
2348 auto *Ind = Induction.first;
2349 auto *IndUpdate = cast<Instruction>(Val: Ind->getIncomingValueForBlock(BB: Latch));
2350
2351 // If tail-folding is applied, the primary induction variable will be used
2352 // to feed a vector compare.
2353 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
2354 continue;
2355
2356 // Returns true if \p Indvar is a pointer induction that is used directly by
2357 // load/store instruction \p I.
2358 auto IsDirectLoadStoreFromPtrIndvar = [&](Instruction *Indvar,
2359 Instruction *I) {
2360 return Induction.second.getKind() ==
2361 InductionDescriptor::IK_PtrInduction &&
2362 (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I)) &&
2363 Indvar == getLoadStorePointerOperand(V: I) && IsScalarUse(I, Indvar);
2364 };
2365
2366 // Determine if all users of the induction variable are scalar after
2367 // vectorization.
2368 bool ScalarInd = all_of(Range: Ind->users(), P: [&](User *U) -> bool {
2369 auto *I = cast<Instruction>(Val: U);
2370 return I == IndUpdate || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2371 IsDirectLoadStoreFromPtrIndvar(Ind, I);
2372 });
2373 if (!ScalarInd)
2374 continue;
2375
2376 // If the induction variable update is a fixed-order recurrence, neither the
2377 // induction variable or its update should be marked scalar after
2378 // vectorization.
2379 auto *IndUpdatePhi = dyn_cast<PHINode>(Val: IndUpdate);
2380 if (IndUpdatePhi && Legal->isFixedOrderRecurrence(Phi: IndUpdatePhi))
2381 continue;
2382
2383 // Determine if all users of the induction variable update instruction are
2384 // scalar after vectorization.
2385 bool ScalarIndUpdate = all_of(Range: IndUpdate->users(), P: [&](User *U) -> bool {
2386 auto *I = cast<Instruction>(Val: U);
2387 return I == Ind || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2388 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
2389 });
2390 if (!ScalarIndUpdate)
2391 continue;
2392
2393 // The induction variable and its update instruction will remain scalar.
2394 Worklist.insert(X: Ind);
2395 Worklist.insert(X: IndUpdate);
2396 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n");
2397 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate
2398 << "\n");
2399 }
2400
2401 Scalars[VF].insert_range(R&: Worklist);
2402}
2403
2404bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
2405 ElementCount VF) {
2406 if (!isPredicatedInst(I))
2407 return false;
2408
2409 // Do we have a non-scalar lowering for this predicated
2410 // instruction? No - it is scalar with predication.
2411 switch(I->getOpcode()) {
2412 default:
2413 return true;
2414 case Instruction::Call: {
2415 if (VF.isScalar())
2416 return true;
2417 auto *CI = cast<CallInst>(Val: I);
2418 // A vector intrinsic or library variant lowering avoids scalarization.
2419 return !getVectorIntrinsicIDForCall(CI, TLI) &&
2420 !hasVectorLibraryVariantFor(CI: *CI, VF, MaskRequired: isMaskRequired(I: CI), TLI);
2421 }
2422 case Instruction::Load:
2423 case Instruction::Store: {
2424 bool IsConsecutive = Legal->isConsecutivePtr(AccessTy: getLoadStoreType(I),
2425 Ptr: getLoadStorePointerOperand(V: I));
2426 return !(IsConsecutive && Config.isLegalMaskedLoadOrStore(I, VF)) &&
2427 !Config.isLegalGatherOrScatter(V: I, VF);
2428 }
2429 case Instruction::UDiv:
2430 case Instruction::SDiv:
2431 case Instruction::SRem:
2432 case Instruction::URem: {
2433 // We have the option to use the llvm.masked.udiv intrinsics to avoid
2434 // predication. The cost based decision here will always select the masked
2435 // intrinsics for scalable vectors as scalarization isn't legal.
2436 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
2437 return isDivRemScalarWithPredication(ScalarCost, MaskedCost);
2438 }
2439 }
2440}
2441
2442bool LoopVectorizationCostModel::isMaskRequired(Instruction *I) const {
2443 return Legal->isMaskRequired(I, TailFolded: foldTailByMasking());
2444}
2445
2446// TODO: Fold into LoopVectorizationLegality::isMaskRequired.
2447bool LoopVectorizationCostModel::isPredicatedInst(Instruction *I) const {
2448 // TODO: We can use the loop-preheader as context point here and get
2449 // context sensitive reasoning for isSafeToSpeculativelyExecute.
2450 if (isSafeToSpeculativelyExecute(I) ||
2451 (isa<LoadInst, StoreInst, CallInst>(Val: I) && !isMaskRequired(I)) ||
2452 isa<UncondBrInst, CondBrInst, SwitchInst, PHINode, AllocaInst>(Val: I))
2453 return false;
2454
2455 // If the instruction was executed conditionally in the original scalar loop,
2456 // predication is needed with a mask whose lanes are all possibly inactive.
2457 if (Legal->blockNeedsPredication(BB: I->getParent()))
2458 return true;
2459
2460 // If we're not folding the tail by masking and not vectorizing a loop with
2461 // uncountable exits and side effects, predication is unnecessary.
2462 if (!foldTailByMasking() && !Legal->hasUncountableExitWithSideEffects())
2463 return false;
2464
2465 // All that remain are instructions with side-effects originally executed in
2466 // the loop unconditionally, but now execute under a tail-fold mask (only)
2467 // having at least one active lane (the first). If the side-effects of the
2468 // instruction are invariant, executing it w/o (the tail-folding) mask is safe
2469 // - it will cause the same side-effects as when masked.
2470 switch(I->getOpcode()) {
2471 default:
2472 llvm_unreachable(
2473 "instruction should have been considered by earlier checks");
2474 case Instruction::Call:
2475 // Side-effects of a Call are assumed to be non-invariant, needing a
2476 // (fold-tail) mask.
2477 assert(isMaskRequired(I) &&
2478 "should have returned earlier for calls not needing a mask");
2479 return true;
2480 case Instruction::Load:
2481 // If the address is loop invariant no predication is needed.
2482 return !Legal->isInvariant(V: getLoadStorePointerOperand(V: I));
2483 case Instruction::Store: {
2484 // For stores, we need to prove both speculation safety (which follows from
2485 // the same argument as loads), but also must prove the value being stored
2486 // is correct. The easiest form of the later is to require that all values
2487 // stored are the same.
2488 return !(Legal->isInvariant(V: getLoadStorePointerOperand(V: I)) &&
2489 TheLoop->isLoopInvariant(V: cast<StoreInst>(Val: I)->getValueOperand()));
2490 }
2491 case Instruction::UDiv:
2492 case Instruction::URem:
2493 // If the divisor is loop-invariant no predication is needed.
2494 return !Legal->isInvariant(V: I->getOperand(i: 1));
2495 case Instruction::SDiv:
2496 case Instruction::SRem:
2497 // Conservative for now, since masked-off lanes may be poison and could
2498 // trigger signed overflow.
2499 return true;
2500 }
2501}
2502
2503uint64_t LoopVectorizationCostModel::getPredBlockCostDivisor(
2504 TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) {
2505 if (CostKind == TTI::TCK_CodeSize)
2506 return 1;
2507 // If the block wasn't originally predicated then return early to avoid
2508 // computing BlockFrequencyInfo unnecessarily.
2509 if (!Legal->blockNeedsPredication(BB))
2510 return 1;
2511
2512 uint64_t HeaderFreq =
2513 getBFI().getBlockFreq(BB: TheLoop->getHeader()).getFrequency();
2514 uint64_t BBFreq = getBFI().getBlockFreq(BB).getFrequency();
2515 assert(HeaderFreq >= BBFreq &&
2516 "Header has smaller block freq than dominated BB?");
2517 return std::round(x: (double)HeaderFreq / BBFreq);
2518}
2519
2520static Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode) {
2521 switch (Opcode) {
2522 case Instruction::UDiv:
2523 return Intrinsic::masked_udiv;
2524 case Instruction::SDiv:
2525 return Intrinsic::masked_sdiv;
2526 case Instruction::URem:
2527 return Intrinsic::masked_urem;
2528 case Instruction::SRem:
2529 return Intrinsic::masked_srem;
2530 default:
2531 llvm_unreachable("Unexpected opcode");
2532 }
2533}
2534
2535std::pair<InstructionCost, InstructionCost>
2536LoopVectorizationCostModel::getDivRemSpeculationCost(Instruction *I,
2537 ElementCount VF) {
2538 assert(I->getOpcode() == Instruction::UDiv ||
2539 I->getOpcode() == Instruction::SDiv ||
2540 I->getOpcode() == Instruction::SRem ||
2541 I->getOpcode() == Instruction::URem);
2542 assert(!isSafeToSpeculativelyExecute(I));
2543
2544 // Scalarization isn't legal for scalable vector types
2545 InstructionCost ScalarizationCost = InstructionCost::getInvalid();
2546 if (!VF.isScalable()) {
2547 // Get the scalarization cost and scale this amount by the probability of
2548 // executing the predicated block. If the instruction is not predicated,
2549 // we fall through to the next case.
2550 ScalarizationCost = 0;
2551
2552 // These instructions have a non-void type, so account for the phi nodes
2553 // that we will create. This cost is likely to be zero. The phi node
2554 // cost, if any, should be scaled by the block probability because it
2555 // models a copy at the end of each predicated block.
2556 ScalarizationCost += VF.getFixedValue() *
2557 TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
2558
2559 // The cost of the non-predicated instruction.
2560 ScalarizationCost +=
2561 VF.getFixedValue() * TTI.getArithmeticInstrCost(
2562 Opcode: I->getOpcode(), Ty: I->getType(), CostKind: Config.CostKind);
2563
2564 // The cost of insertelement and extractelement instructions needed for
2565 // scalarization.
2566 ScalarizationCost += getScalarizationOverhead(I, VF);
2567
2568 // Scale the cost by the probability of executing the predicated blocks.
2569 // This assumes the predicated block for each vector lane is equally
2570 // likely.
2571 ScalarizationCost =
2572 ScalarizationCost /
2573 getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
2574 }
2575
2576 auto *VecTy = toVectorTy(Scalar: I->getType(), EC: VF);
2577 auto *MaskTy = toVectorTy(Scalar: Type::getInt1Ty(C&: I->getContext()), EC: VF);
2578 IntrinsicCostAttributes ICA(getMaskedDivRemIntrinsic(Opcode: I->getOpcode()), VecTy,
2579 {VecTy, VecTy, MaskTy});
2580 InstructionCost MaskedCost = TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind);
2581 return {ScalarizationCost, MaskedCost};
2582}
2583
2584bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
2585 Instruction *I, ElementCount VF) const {
2586 assert(isAccessInterleaved(I) && "Expecting interleaved access.");
2587 assert(getWideningDecision(I, VF) == CM_Unknown &&
2588 "Decision should not be set yet.");
2589 auto *Group = getInterleavedAccessGroup(Instr: I);
2590 assert(Group && "Must have a group.");
2591 unsigned InterleaveFactor = Group->getFactor();
2592
2593 // If the instruction's allocated size doesn't equal its type size, it
2594 // requires padding and will be scalarized.
2595 auto &DL = I->getDataLayout();
2596 auto *ScalarTy = getLoadStoreType(I);
2597 if (hasIrregularType(Ty: ScalarTy, DL))
2598 return false;
2599
2600 // For scalable vectors, the interleave factors must be <= 8 since we require
2601 // the (de)interleaveN intrinsics instead of shufflevectors.
2602 if (VF.isScalable() && InterleaveFactor > 8)
2603 return false;
2604
2605 // If the group involves a non-integral pointer, we may not be able to
2606 // losslessly cast all values to a common type.
2607 bool ScalarNI = DL.isNonIntegralPointerType(Ty: ScalarTy);
2608 for (Instruction *Member : Group->members()) {
2609 auto *MemberTy = getLoadStoreType(I: Member);
2610 bool MemberNI = DL.isNonIntegralPointerType(Ty: MemberTy);
2611 // Don't coerce non-integral pointers to integers or vice versa.
2612 if (MemberNI != ScalarNI)
2613 // TODO: Consider adding special nullptr value case here
2614 return false;
2615 if (MemberNI && ScalarNI &&
2616 ScalarTy->getPointerAddressSpace() !=
2617 MemberTy->getPointerAddressSpace())
2618 return false;
2619 }
2620
2621 // Check if masking is required.
2622 // A Group may need masking for one of two reasons: it resides in a block that
2623 // needs predication, or it was decided to use masking to deal with gaps
2624 // (either a gap at the end of a load-access that may result in a speculative
2625 // load, or any gaps in a store-access).
2626 bool PredicatedAccessRequiresMasking =
2627 blockNeedsPredicationForAnyReason(BB: I->getParent()) && isMaskRequired(I);
2628 bool LoadAccessWithGapsRequiresEpilogMasking =
2629 isa<LoadInst>(Val: I) && Group->requiresScalarEpilogue() &&
2630 !isEpilogueAllowed();
2631 bool StoreAccessWithGapsRequiresMasking =
2632 isa<StoreInst>(Val: I) && !Group->isFull();
2633 if (!PredicatedAccessRequiresMasking &&
2634 !LoadAccessWithGapsRequiresEpilogMasking &&
2635 !StoreAccessWithGapsRequiresMasking)
2636 return true;
2637
2638 // If masked interleaving is required, we expect that the user/target had
2639 // enabled it, because otherwise it either wouldn't have been created or
2640 // it should have been invalidated by the CostModel.
2641 assert(useMaskedInterleavedAccesses(TTI) &&
2642 "Masked interleave-groups for predicated accesses are not enabled.");
2643
2644 if (Group->isReverse())
2645 return false;
2646
2647 // TODO: Support interleaved access that requires a gap mask for scalable VFs.
2648 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2649 StoreAccessWithGapsRequiresMasking;
2650 if (VF.isScalable() && NeedsMaskForGaps)
2651 return false;
2652
2653 return Config.isLegalMaskedLoadOrStore(I, VF);
2654}
2655
2656std::optional<LoopVectorizationCostModel::InstWidening>
2657LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
2658 ElementCount VF) {
2659 // Get and ensure we have a valid memory instruction.
2660 assert((isa<LoadInst, StoreInst>(I)) && "Invalid memory instruction");
2661
2662 auto *Ptr = getLoadStorePointerOperand(V: I);
2663 auto *ScalarTy = getLoadStoreType(I);
2664
2665 // In order to be widened, the pointer should be consecutive, first of all.
2666 int Stride = Legal->isConsecutivePtr(AccessTy: ScalarTy, Ptr);
2667 if (!Stride)
2668 return std::nullopt;
2669
2670 // If the instruction is a store located in a predicated block, it will be
2671 // scalarized.
2672 if (isScalarWithPredication(I, VF))
2673 return std::nullopt;
2674
2675 // If the instruction's allocated size doesn't equal it's type size, it
2676 // requires padding and will be scalarized.
2677 auto &DL = I->getDataLayout();
2678 if (hasIrregularType(Ty: ScalarTy, DL))
2679 return std::nullopt;
2680
2681 return Stride == 1 ? CM_Widen : CM_Widen_Reverse;
2682}
2683
2684void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
2685 // We should not collect Uniforms more than once per VF. Right now,
2686 // this function is called from collectUniformsAndScalars(), which
2687 // already does this check. Collecting Uniforms for VF=1 does not make any
2688 // sense.
2689
2690 assert(VF.isVector() && !Uniforms.contains(VF) &&
2691 "This function should not be visited twice for the same VF");
2692
2693 // Visit the list of Uniforms. If we find no uniform value, we won't
2694 // analyze again. Uniforms.count(VF) will return 1.
2695 Uniforms[VF].clear();
2696
2697 // Now we know that the loop is vectorizable!
2698 // Collect instructions inside the loop that will remain uniform after
2699 // vectorization.
2700
2701 // Global values, params and instructions outside of current loop are out of
2702 // scope.
2703 auto IsOutOfScope = [&](Value *V) -> bool {
2704 Instruction *I = dyn_cast<Instruction>(Val: V);
2705 return (!I || !TheLoop->contains(Inst: I));
2706 };
2707
2708 // Worklist containing uniform instructions demanding lane 0.
2709 SetVector<Instruction *> Worklist;
2710
2711 // Add uniform instructions demanding lane 0 to the worklist. Instructions
2712 // that require predication must not be considered uniform after
2713 // vectorization, because that would create an erroneous replicating region
2714 // where only a single instance out of VF should be formed.
2715 auto AddToWorklistIfAllowed = [&](Instruction *I) -> void {
2716 if (IsOutOfScope(I)) {
2717 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "
2718 << *I << "\n");
2719 return;
2720 }
2721 if (isPredicatedInst(I)) {
2722 LLVM_DEBUG(
2723 dbgs() << "LV: Found not uniform due to requiring predication: " << *I
2724 << "\n");
2725 return;
2726 }
2727 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n");
2728 Worklist.insert(X: I);
2729 };
2730
2731 // Start with the conditional branches exiting the loop. If the branch
2732 // condition is an instruction contained in the loop that is only used by the
2733 // branch, it is uniform. Note conditions from uncountable early exits are not
2734 // uniform.
2735 SmallVector<BasicBlock *> Exiting;
2736 TheLoop->getExitingBlocks(ExitingBlocks&: Exiting);
2737 for (BasicBlock *E : Exiting) {
2738 if (Legal->hasUncountableEarlyExit() && TheLoop->getLoopLatch() != E)
2739 continue;
2740 auto *Cmp = dyn_cast<Instruction>(Val: E->getTerminator()->getOperand(i: 0));
2741 if (Cmp && TheLoop->contains(Inst: Cmp) && Cmp->hasOneUse())
2742 AddToWorklistIfAllowed(Cmp);
2743 }
2744
2745 auto PrevVF = VF.divideCoefficientBy(RHS: 2);
2746 // Return true if all lanes perform the same memory operation, and we can
2747 // thus choose to execute only one.
2748 auto IsUniformMemOpUse = [&](Instruction *I) {
2749 // If the value was already known to not be uniform for the previous
2750 // (smaller VF), it cannot be uniform for the larger VF.
2751 if (PrevVF.isVector()) {
2752 auto Iter = Uniforms.find(Val: PrevVF);
2753 if (Iter != Uniforms.end() && !Iter->second.contains(Ptr: I))
2754 return false;
2755 }
2756 if (!isUniformMemOp(I&: *I, VF))
2757 return false;
2758 if (isa<LoadInst>(Val: I))
2759 // Loading the same address always produces the same result - at least
2760 // assuming aliasing and ordering which have already been checked.
2761 return true;
2762 // Storing the same value on every iteration.
2763 return TheLoop->isLoopInvariant(V: cast<StoreInst>(Val: I)->getValueOperand());
2764 };
2765
2766 auto IsUniformDecision = [&](Instruction *I, ElementCount VF) {
2767 InstWidening WideningDecision = getWideningDecision(I, VF);
2768 assert(WideningDecision != CM_Unknown &&
2769 "Widening decision should be ready at this moment");
2770
2771 if (IsUniformMemOpUse(I))
2772 return true;
2773
2774 return (WideningDecision == CM_Widen ||
2775 WideningDecision == CM_Widen_Reverse ||
2776 WideningDecision == CM_Interleave);
2777 };
2778
2779 // Returns true if Ptr is the pointer operand of a memory access instruction
2780 // I, I is known to not require scalarization, and the pointer is not also
2781 // stored.
2782 auto IsVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
2783 if (isa<StoreInst>(Val: I) && I->getOperand(i: 0) == Ptr)
2784 return false;
2785 return getLoadStorePointerOperand(V: I) == Ptr &&
2786 (IsUniformDecision(I, VF) || Legal->isInvariant(V: Ptr));
2787 };
2788
2789 // Holds a list of values which are known to have at least one uniform use.
2790 // Note that there may be other uses which aren't uniform. A "uniform use"
2791 // here is something which only demands lane 0 of the unrolled iterations;
2792 // it does not imply that all lanes produce the same value (e.g. this is not
2793 // the usual meaning of uniform)
2794 SetVector<Value *> HasUniformUse;
2795
2796 // Scan the loop for instructions which are either a) known to have only
2797 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
2798 for (auto *BB : TheLoop->blocks())
2799 for (auto &I : *BB) {
2800 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &I)) {
2801 switch (II->getIntrinsicID()) {
2802 case Intrinsic::sideeffect:
2803 case Intrinsic::experimental_noalias_scope_decl:
2804 case Intrinsic::assume:
2805 case Intrinsic::lifetime_start:
2806 case Intrinsic::lifetime_end:
2807 if (TheLoop->hasLoopInvariantOperands(I: &I))
2808 AddToWorklistIfAllowed(&I);
2809 break;
2810 default:
2811 break;
2812 }
2813 }
2814
2815 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: &I)) {
2816 if (IsOutOfScope(EVI->getAggregateOperand())) {
2817 AddToWorklistIfAllowed(EVI);
2818 continue;
2819 }
2820 // Only ExtractValue instructions where the aggregate value comes from a
2821 // call are allowed to be non-uniform.
2822 assert(isa<CallInst>(EVI->getAggregateOperand()) &&
2823 "Expected aggregate value to be call return value");
2824 }
2825
2826 // If there's no pointer operand, there's nothing to do.
2827 auto *Ptr = getLoadStorePointerOperand(V: &I);
2828 if (!Ptr)
2829 continue;
2830
2831 // If the pointer can be proven to be uniform, always add it to the
2832 // worklist.
2833 if (isa<Instruction>(Val: Ptr) && isUniform(V: Ptr, VF))
2834 AddToWorklistIfAllowed(cast<Instruction>(Val: Ptr));
2835
2836 if (IsUniformMemOpUse(&I))
2837 AddToWorklistIfAllowed(&I);
2838
2839 if (IsVectorizedMemAccessUse(&I, Ptr))
2840 HasUniformUse.insert(X: Ptr);
2841 }
2842
2843 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
2844 // demanding) users. Since loops are assumed to be in LCSSA form, this
2845 // disallows uses outside the loop as well.
2846 for (auto *V : HasUniformUse) {
2847 if (IsOutOfScope(V))
2848 continue;
2849 auto *I = cast<Instruction>(Val: V);
2850 bool UsersAreMemAccesses = all_of(Range: I->users(), P: [&](User *U) -> bool {
2851 auto *UI = cast<Instruction>(Val: U);
2852 return TheLoop->contains(Inst: UI) && IsVectorizedMemAccessUse(UI, V);
2853 });
2854 if (UsersAreMemAccesses)
2855 AddToWorklistIfAllowed(I);
2856 }
2857
2858 // Expand Worklist in topological order: whenever a new instruction
2859 // is added , its users should be already inside Worklist. It ensures
2860 // a uniform instruction will only be used by uniform instructions.
2861 unsigned Idx = 0;
2862 while (Idx != Worklist.size()) {
2863 Instruction *I = Worklist[Idx++];
2864
2865 for (auto *OV : I->operand_values()) {
2866 // isOutOfScope operands cannot be uniform instructions.
2867 if (IsOutOfScope(OV))
2868 continue;
2869 // First order recurrence Phi's should typically be considered
2870 // non-uniform.
2871 auto *OP = dyn_cast<PHINode>(Val: OV);
2872 if (OP && Legal->isFixedOrderRecurrence(Phi: OP))
2873 continue;
2874 // If all the users of the operand are uniform, then add the
2875 // operand into the uniform worklist.
2876 auto *OI = cast<Instruction>(Val: OV);
2877 if (llvm::all_of(Range: OI->users(), P: [&](User *U) -> bool {
2878 auto *J = cast<Instruction>(Val: U);
2879 return Worklist.count(key: J) || IsVectorizedMemAccessUse(J, OI);
2880 }))
2881 AddToWorklistIfAllowed(OI);
2882 }
2883 }
2884
2885 // For an instruction to be added into Worklist above, all its users inside
2886 // the loop should also be in Worklist. However, this condition cannot be
2887 // true for phi nodes that form a cyclic dependence. We must process phi
2888 // nodes separately. An induction variable will remain uniform if all users
2889 // of the induction variable and induction variable update remain uniform.
2890 // The code below handles both pointer and non-pointer induction variables.
2891 BasicBlock *Latch = TheLoop->getLoopLatch();
2892 for (const auto &Induction : Legal->getInductionVars()) {
2893 auto *Ind = Induction.first;
2894 auto *IndUpdate = cast<Instruction>(Val: Ind->getIncomingValueForBlock(BB: Latch));
2895
2896 // Determine if all users of the induction variable are uniform after
2897 // vectorization.
2898 bool UniformInd = all_of(Range: Ind->users(), P: [&](User *U) -> bool {
2899 auto *I = cast<Instruction>(Val: U);
2900 return I == IndUpdate || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2901 IsVectorizedMemAccessUse(I, Ind);
2902 });
2903 if (!UniformInd)
2904 continue;
2905
2906 // Determine if all users of the induction variable update instruction are
2907 // uniform after vectorization.
2908 bool UniformIndUpdate = all_of(Range: IndUpdate->users(), P: [&](User *U) -> bool {
2909 auto *I = cast<Instruction>(Val: U);
2910 return I == Ind || Worklist.count(key: I) ||
2911 IsVectorizedMemAccessUse(I, IndUpdate);
2912 });
2913 if (!UniformIndUpdate)
2914 continue;
2915
2916 // The induction variable and its update instruction will remain uniform.
2917 AddToWorklistIfAllowed(Ind);
2918 AddToWorklistIfAllowed(IndUpdate);
2919 }
2920
2921 Uniforms[VF].insert_range(R&: Worklist);
2922}
2923
2924FixedScalableVFPair
2925LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
2926 // Make sure once we return PartialAliasMaskingStatus is not "NotDecided".
2927 scope_exit EnsureAliasMaskingStatusIsDecidedOnReturn([this] {
2928 if (PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided)
2929 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
2930 });
2931
2932 // For outer loops, use simple type-based heuristic VF. No cost model or
2933 // memory dependence analysis is available.
2934 if (!TheLoop->isInnermost()) {
2935 return Config.computeVPlanOuterloopVF(UserVF);
2936 }
2937
2938 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
2939 // TODO: It may be useful to do since it's still likely to be dynamically
2940 // uniform if the target can skip.
2941 reportVectorizationFailure(
2942 DebugMsg: "Not inserting runtime ptr check for divergent target",
2943 OREMsg: "runtime pointer checks needed. Not enabled for divergent target",
2944 ORETag: "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
2945 return FixedScalableVFPair::getNone();
2946 }
2947
2948 ScalarEvolution *SE = PSE.getSE();
2949 ElementCount TC = getSmallConstantTripCount(SE, L: TheLoop);
2950 unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
2951 if (!MaxTC && EpilogueLoweringStatus == CM_EpilogueAllowed)
2952 MaxTC = getMaxTCFromNonZeroRange(PSE, L: TheLoop);
2953 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
2954 if (TC != ElementCount::getFixed(MinVal: MaxTC))
2955 LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
2956 if (TC.isScalar()) {
2957 reportVectorizationFailure(
2958 DebugMsg: "Single iteration (non) loop",
2959 OREMsg: "loop trip count is one, irrelevant for vectorization",
2960 ORETag: "SingleIterationLoop", ORE, TheLoop);
2961 return FixedScalableVFPair::getNone();
2962 }
2963
2964 // If BTC matches the widest induction type and is -1 then the trip count
2965 // computation will wrap to 0 and the vector trip count will be 0. Do not try
2966 // to vectorize.
2967 const SCEV *BTC = SE->getBackedgeTakenCount(L: TheLoop);
2968 if (!isa<SCEVCouldNotCompute>(Val: BTC) &&
2969 BTC->getType()->getScalarSizeInBits() >=
2970 Legal->getWidestInductionType()->getScalarSizeInBits() &&
2971 SE->isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: BTC,
2972 RHS: SE->getMinusOne(Ty: BTC->getType()))) {
2973 reportVectorizationFailure(
2974 DebugMsg: "Trip count computation wrapped",
2975 OREMsg: "backedge-taken count is -1, loop trip count wrapped to 0",
2976 ORETag: "TripCountWrapped", ORE, TheLoop);
2977 return FixedScalableVFPair::getNone();
2978 }
2979
2980 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
2981 "No cost-modeling decisions should have been taken at this point");
2982
2983 switch (EpilogueLoweringStatus) {
2984 case CM_EpilogueAllowed:
2985 return Config.computeFeasibleMaxVF(MaxTripCount: MaxTC, UserVF, UserIC, FoldTailByMasking: false,
2986 RequiresScalarEpilogue: requiresScalarEpilogue(IsVectorizing: true));
2987 case CM_EpilogueNotAllowedFoldTail:
2988 [[fallthrough]];
2989 case CM_EpilogueNotNeededFoldTail:
2990 LLVM_DEBUG(dbgs() << "LV: tail-folding hint/switch found.\n"
2991 << "LV: Not allowing epilogue, creating tail-folded "
2992 << "vector loop.\n");
2993 break;
2994 case CM_EpilogueNotAllowedLowTripLoop:
2995 // fallthrough as a special case of OptForSize
2996 case CM_EpilogueNotAllowedOptSize:
2997 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize)
2998 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
2999 else
3000 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
3001 << "count.\n");
3002
3003 // Bail if runtime checks are required, which are not good when optimising
3004 // for size.
3005 if (Config.runtimeChecksRequired())
3006 return FixedScalableVFPair::getNone();
3007
3008 break;
3009 }
3010
3011 // Now try the tail folding
3012
3013 // Invalidate interleave groups that require an epilogue if we can't mask
3014 // the interleave-group.
3015 if (!useMaskedInterleavedAccesses(TTI)) {
3016 // Note: There is no need to invalidate any cost modeling decisions here, as
3017 // none were taken so far (see assertion above).
3018 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
3019 }
3020
3021 FixedScalableVFPair MaxFactors = Config.computeFeasibleMaxVF(
3022 MaxTripCount: MaxTC, UserVF, UserIC, FoldTailByMasking: true, RequiresScalarEpilogue: requiresScalarEpilogue(IsVectorizing: true));
3023
3024 // Avoid tail folding if the trip count is known to be a multiple of any VF
3025 // we choose.
3026 std::optional<unsigned> MaxPowerOf2RuntimeVF =
3027 MaxFactors.FixedVF.getFixedValue();
3028 if (MaxFactors.ScalableVF) {
3029 std::optional<unsigned> MaxVScale = getMaxVScale(F: *TheFunction, TTI);
3030 if (MaxVScale) {
3031 MaxPowerOf2RuntimeVF = std::max<unsigned>(
3032 a: *MaxPowerOf2RuntimeVF,
3033 b: *MaxVScale * MaxFactors.ScalableVF.getKnownMinValue());
3034 } else
3035 MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
3036 }
3037
3038 auto NoScalarEpilogueNeeded = [this, &UserIC](unsigned MaxVF) {
3039 // Return false if the loop is neither a single-latch-exit loop nor an
3040 // early-exit loop as tail-folding is not supported in that case.
3041 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
3042 !Legal->hasUncountableEarlyExit())
3043 return false;
3044 unsigned MaxVFtimesIC = UserIC ? MaxVF * UserIC : MaxVF;
3045 ScalarEvolution *SE = PSE.getSE();
3046 // Calling getSymbolicMaxBackedgeTakenCount enables support for loops
3047 // with uncountable exits. For countable loops, the symbolic maximum must
3048 // remain identical to the known back-edge taken count.
3049 const SCEV *BackedgeTakenCount = PSE.getSymbolicMaxBackedgeTakenCount();
3050 assert((Legal->hasUncountableEarlyExit() ||
3051 BackedgeTakenCount == PSE.getBackedgeTakenCount()) &&
3052 "Invalid loop count");
3053 const SCEV *ExitCount = SE->getAddExpr(
3054 LHS: BackedgeTakenCount, RHS: SE->getOne(Ty: BackedgeTakenCount->getType()));
3055 const SCEV *Rem = SE->getURemExpr(
3056 LHS: SE->applyLoopGuards(Expr: ExitCount, L: TheLoop),
3057 RHS: SE->getConstant(Ty: BackedgeTakenCount->getType(), V: MaxVFtimesIC));
3058 return Rem->isZero();
3059 };
3060
3061 if (MaxPowerOf2RuntimeVF > 0u) {
3062 assert((UserVF.isNonZero() || isPowerOf2_32(*MaxPowerOf2RuntimeVF)) &&
3063 "MaxFixedVF must be a power of 2");
3064 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
3065 // Accept MaxFixedVF if we do not have a tail.
3066 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
3067 return MaxFactors;
3068 }
3069 }
3070
3071 auto ExpectedTC = getSmallBestKnownTC(PSE, L: TheLoop);
3072 if (ExpectedTC && ExpectedTC->isFixed() &&
3073 ExpectedTC->getFixedValue() <=
3074 TTI.getMinTripCountTailFoldingThreshold()) {
3075 if (MaxPowerOf2RuntimeVF > 0u) {
3076 // If we have a low-trip-count, and the fixed-width VF is known to divide
3077 // the trip count but the scalable factor does not, use the fixed-width
3078 // factor in preference to allow the generation of a non-predicated loop.
3079 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop &&
3080 NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
3081 LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
3082 "remain for any chosen VF.\n");
3083 MaxFactors.ScalableVF = ElementCount::getScalable(MinVal: 0);
3084 return MaxFactors;
3085 }
3086 }
3087
3088 reportVectorizationFailure(
3089 DebugMsg: "The trip count is below the minial threshold value.",
3090 OREMsg: "loop trip count is too low, avoiding vectorization", ORETag: "LowTripCount",
3091 ORE, TheLoop);
3092 return FixedScalableVFPair::getNone();
3093 }
3094
3095 // If we don't know the precise trip count, or if the trip count that we
3096 // found modulo the vectorization factor is not zero, try to fold the tail
3097 // by masking.
3098 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
3099 bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
3100 setTailFoldingStyle(IsScalableVF: ContainsScalableVF, UserIC);
3101 if (foldTailByMasking()) {
3102 if (foldTailWithEVL()) {
3103 LLVM_DEBUG(
3104 dbgs()
3105 << "LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
3106 "try to generate VP Intrinsics with scalable vector "
3107 "factors only.\n");
3108 // Tail folded loop using VP intrinsics restricts the VF to be scalable
3109 // for now.
3110 // TODO: extend it for fixed vectors, if required.
3111 assert(ContainsScalableVF && "Expected scalable vector factor.");
3112
3113 MaxFactors.FixedVF = ElementCount::getFixed(MinVal: 1);
3114 } else {
3115 tryToEnablePartialAliasMasking();
3116 }
3117 return MaxFactors;
3118 }
3119
3120 // If there was a tail-folding hint/switch, but we can't fold the tail by
3121 // masking, fallback to a vectorization with an epilogue.
3122 if (EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail) {
3123 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with an "
3124 "epilogue instead.\n");
3125 EpilogueLoweringStatus = CM_EpilogueAllowed;
3126 return MaxFactors;
3127 }
3128
3129 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail) {
3130 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
3131 return FixedScalableVFPair::getNone();
3132 }
3133
3134 if (TC.isZero()) {
3135 reportVectorizationFailure(
3136 DebugMsg: "unable to calculate the loop count due to complex control flow",
3137 ORETag: "UnknownLoopCountComplexCFG", ORE, TheLoop);
3138 return FixedScalableVFPair::getNone();
3139 }
3140
3141 reportVectorizationFailure(
3142 DebugMsg: "Cannot optimize for size and vectorize at the same time.",
3143 OREMsg: "cannot optimize for size and vectorize at the same time. "
3144 "Enable vectorization of this loop with '#pragma clang loop "
3145 "vectorize(enable)' when compiling with -Os/-Oz",
3146 ORETag: "NoTailLoopWithOptForSize", ORE, TheLoop);
3147 return FixedScalableVFPair::getNone();
3148}
3149
3150void LoopVectorizationPlanner::emitInvalidCostRemarks(
3151 OptimizationRemarkEmitter *ORE) {
3152 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
3153 SmallVector<RecipeVFPair> InvalidCosts;
3154 for (const auto &Plan : VPlans) {
3155 for (ElementCount VF : Plan->vectorFactors()) {
3156 // The VPlan-based cost model is designed for computing vector cost.
3157 // Querying VPlan-based cost model with a scarlar VF will cause some
3158 // errors because we expect the VF is vector for most of the widen
3159 // recipes.
3160 if (VF.isScalar())
3161 continue;
3162
3163 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, Config.CostKind, CM.PSE,
3164 OrigLoop);
3165 precomputeCosts(Plan&: *Plan, VF, CostCtx);
3166 auto Iter = vp_depth_first_deep(G: Plan->getVectorLoopRegion()->getEntry());
3167 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: Iter)) {
3168 for (auto &R : *VPBB) {
3169 if (!R.cost(VF, Ctx&: CostCtx).isValid())
3170 InvalidCosts.emplace_back(Args: &R, Args&: VF);
3171 }
3172 }
3173 }
3174 }
3175 if (InvalidCosts.empty())
3176 return;
3177
3178 // Emit a report of VFs with invalid costs in the loop.
3179
3180 // Group the remarks per recipe, keeping the recipe order from InvalidCosts.
3181 DenseMap<VPRecipeBase *, unsigned> Numbering;
3182 unsigned I = 0;
3183 for (auto &Pair : InvalidCosts)
3184 if (Numbering.try_emplace(Key: Pair.first, Args&: I).second)
3185 ++I;
3186
3187 // Sort the list, first on recipe(number) then on VF.
3188 sort(C&: InvalidCosts, Comp: [&Numbering](RecipeVFPair &A, RecipeVFPair &B) {
3189 unsigned NA = Numbering[A.first];
3190 unsigned NB = Numbering[B.first];
3191 if (NA != NB)
3192 return NA < NB;
3193 return ElementCount::isKnownLT(LHS: A.second, RHS: B.second);
3194 });
3195
3196 // For a list of ordered recipe-VF pairs:
3197 // [(load, VF1), (load, VF2), (store, VF1)]
3198 // group the recipes together to emit separate remarks for:
3199 // load (VF1, VF2)
3200 // store (VF1)
3201 auto Tail = ArrayRef<RecipeVFPair>(InvalidCosts);
3202 auto Subset = ArrayRef<RecipeVFPair>();
3203 do {
3204 if (Subset.empty())
3205 Subset = Tail.take_front(N: 1);
3206
3207 VPRecipeBase *R = Subset.front().first;
3208
3209 unsigned Opcode =
3210 TypeSwitch<const VPRecipeBase *, unsigned>(R)
3211 .Case(caseFn: [](const VPHeaderPHIRecipe *R) { return Instruction::PHI; })
3212 .Case(
3213 caseFn: [](const VPWidenStoreRecipe *R) { return Instruction::Store; })
3214 .Case(caseFn: [](const VPWidenLoadRecipe *R) { return Instruction::Load; })
3215 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
3216 caseFn: [](const auto *R) { return Instruction::Call; })
3217 .Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
3218 VPWidenCastRecipe>(
3219 caseFn: [](const auto *R) { return R->getOpcode(); })
3220 .Case(caseFn: [](const VPInterleaveRecipe *R) {
3221 return R->getStoredValues().empty() ? Instruction::Load
3222 : Instruction::Store;
3223 })
3224 .Case(caseFn: [](const VPReductionRecipe *R) {
3225 return RecurrenceDescriptor::getOpcode(Kind: R->getRecurrenceKind());
3226 });
3227
3228 // If the next recipe is different, or if there are no other pairs,
3229 // emit a remark for the collated subset. e.g.
3230 // [(load, VF1), (load, VF2))]
3231 // to emit:
3232 // remark: invalid costs for 'load' at VF=(VF1, VF2)
3233 if (Subset == Tail || Tail[Subset.size()].first != R) {
3234 std::string OutString;
3235 raw_string_ostream OS(OutString);
3236 assert(!Subset.empty() && "Unexpected empty range");
3237 OS << "Recipe with invalid costs prevented vectorization at VF=(";
3238 for (const auto &Pair : Subset)
3239 OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second;
3240 OS << "):";
3241 if (Opcode == Instruction::Call) {
3242 StringRef Name = "";
3243 if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(Val: R)) {
3244 Name = Int->getIntrinsicName();
3245 } else {
3246 auto *WidenCall = dyn_cast<VPWidenCallRecipe>(Val: R);
3247 Function *CalledFn =
3248 WidenCall ? WidenCall->getCalledScalarFunction()
3249 : cast<Function>(Val: R->getOperand(N: R->getNumOperands() - 1)
3250 ->getLiveInIRValue());
3251 Name = CalledFn->getName();
3252 }
3253 OS << " call to " << Name;
3254 } else
3255 OS << " " << Instruction::getOpcodeName(Opcode);
3256 reportVectorizationInfo(Msg: OutString, ORETag: "InvalidCost", ORE, TheLoop: OrigLoop, I: nullptr,
3257 DL: R->getDebugLoc());
3258 Tail = Tail.drop_front(N: Subset.size());
3259 Subset = {};
3260 } else
3261 // Grow the subset by one element
3262 Subset = Tail.take_front(N: Subset.size() + 1);
3263 } while (!Tail.empty());
3264}
3265
3266/// Check if any recipe of \p Plan will generate a vector value, which will be
3267/// assigned a vector register.
3268static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
3269 const TargetTransformInfo &TTI) {
3270 assert(VF.isVector() && "Checking a scalar VF?");
3271 DenseSet<VPRecipeBase *> EphemeralRecipes;
3272 collectEphemeralRecipesForVPlan(Plan, EphRecipes&: EphemeralRecipes);
3273 // Set of already visited types.
3274 DenseSet<Type *> Visited;
3275 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3276 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
3277 for (VPRecipeBase &R : *VPBB) {
3278 if (EphemeralRecipes.contains(V: &R))
3279 continue;
3280 // Continue early if the recipe is considered to not produce a vector
3281 // result. Note that this includes VPInstruction where some opcodes may
3282 // produce a vector, to preserve existing behavior as VPInstructions model
3283 // aspects not directly mapped to existing IR instructions.
3284 switch (R.getVPRecipeID()) {
3285 case VPRecipeBase::VPDerivedIVSC:
3286 case VPRecipeBase::VPScalarIVStepsSC:
3287 case VPRecipeBase::VPReplicateSC:
3288 case VPRecipeBase::VPInstructionSC:
3289 case VPRecipeBase::VPCurrentIterationPHISC:
3290 case VPRecipeBase::VPVectorPointerSC:
3291 case VPRecipeBase::VPVectorEndPointerSC:
3292 case VPRecipeBase::VPExpandSCEVSC:
3293 case VPRecipeBase::VPPredInstPHISC:
3294 case VPRecipeBase::VPBranchOnMaskSC:
3295 continue;
3296 case VPRecipeBase::VPReductionSC:
3297 case VPRecipeBase::VPActiveLaneMaskPHISC:
3298 case VPRecipeBase::VPWidenCallSC:
3299 case VPRecipeBase::VPWidenCanonicalIVSC:
3300 case VPRecipeBase::VPWidenCastSC:
3301 case VPRecipeBase::VPWidenGEPSC:
3302 case VPRecipeBase::VPWidenIntrinsicSC:
3303 case VPRecipeBase::VPWidenMemIntrinsicSC:
3304 case VPRecipeBase::VPWidenSC:
3305 case VPRecipeBase::VPBlendSC:
3306 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
3307 case VPRecipeBase::VPHistogramSC:
3308 case VPRecipeBase::VPWidenPHISC:
3309 case VPRecipeBase::VPWidenIntOrFpInductionSC:
3310 case VPRecipeBase::VPWidenPointerInductionSC:
3311 case VPRecipeBase::VPReductionPHISC:
3312 case VPRecipeBase::VPInterleaveEVLSC:
3313 case VPRecipeBase::VPInterleaveSC:
3314 case VPRecipeBase::VPWidenLoadEVLSC:
3315 case VPRecipeBase::VPWidenLoadSC:
3316 case VPRecipeBase::VPWidenStoreEVLSC:
3317 case VPRecipeBase::VPWidenStoreSC:
3318 break;
3319 default:
3320 llvm_unreachable("unhandled recipe");
3321 }
3322
3323 auto WillGenerateTargetVectors = [&TTI, VF](Type *VectorTy) {
3324 unsigned NumLegalParts = TTI.getNumberOfParts(Tp: VectorTy);
3325 if (!NumLegalParts)
3326 return false;
3327 if (VF.isScalable()) {
3328 // <vscale x 1 x iN> is assumed to be profitable over iN because
3329 // scalable registers are a distinct register class from scalar
3330 // ones. If we ever find a target which wants to lower scalable
3331 // vectors back to scalars, we'll need to update this code to
3332 // explicitly ask TTI about the register class uses for each part.
3333 return NumLegalParts <= VF.getKnownMinValue();
3334 }
3335 // Two or more elements that share a register - are vectorized.
3336 return NumLegalParts < VF.getFixedValue();
3337 };
3338
3339 // If no def nor is a store, e.g., branches, continue - no value to check.
3340 if (R.getNumDefinedValues() == 0 &&
3341 !isa<VPWidenStoreRecipe, VPWidenStoreEVLRecipe, VPInterleaveBase>(Val: &R))
3342 continue;
3343 // For multi-def recipes, currently only interleaved loads, suffice to
3344 // check first def only.
3345 // For stores check their stored value; for interleaved stores suffice
3346 // the check first stored value only. In all cases this is the second
3347 // operand.
3348 VPValue *ToCheck =
3349 R.getNumDefinedValues() >= 1 ? R.getVPValue(I: 0) : R.getOperand(N: 1);
3350 Type *ScalarTy = ToCheck->getScalarType();
3351 if (!Visited.insert(V: {ScalarTy}).second)
3352 continue;
3353 Type *WideTy = toVectorizedTy(Ty: ScalarTy, EC: VF);
3354 if (any_of(Range: getContainedTypes(Ty: WideTy), P: WillGenerateTargetVectors))
3355 return true;
3356 }
3357 }
3358
3359 return false;
3360}
3361
3362static bool hasReplicatorRegion(VPlan &Plan) {
3363 return any_of(Range: VPBlockUtils::blocksOnly<VPRegionBlock>(Range: vp_depth_first_shallow(
3364 G: Plan.getVectorLoopRegion()->getEntry())),
3365 P: [](auto *VPRB) { return VPRB->isReplicator(); });
3366}
3367
3368/// Returns true if the VPlan contains a VPReductionPHIRecipe with
3369/// FindLast recurrence kind.
3370static bool hasFindLastReductionPhi(VPlan &Plan) {
3371 return any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3372 P: [](VPRecipeBase &R) {
3373 auto *RedPhi = dyn_cast<VPReductionPHIRecipe>(Val: &R);
3374 return RedPhi &&
3375 RecurrenceDescriptor::isFindLastRecurrenceKind(
3376 Kind: RedPhi->getRecurrenceKind());
3377 });
3378}
3379
3380/// Returns true if the VPlan contains header phi recipes that are not currently
3381/// supported for epilogue vectorization.
3382static bool hasUnsupportedHeaderPhiRecipe(VPlan &Plan) {
3383 return any_of(
3384 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3385 P: [](VPRecipeBase &R) {
3386 switch (R.getVPRecipeID()) {
3387 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
3388 // TODO: Add support for fixed-order recurrences.
3389 return true;
3390 case VPRecipeBase::VPWidenIntOrFpInductionSC:
3391 return !cast<VPWidenIntOrFpInductionRecipe>(Val: &R)->getPHINode();
3392 case VPRecipeBase::VPReductionPHISC: {
3393 auto *RedPhi = cast<VPReductionPHIRecipe>(Val: &R);
3394 // TODO: Support FMinNum/FMaxNum, FindLast reductions, and reductions
3395 // without underlying values.
3396 RecurKind Kind = RedPhi->getRecurrenceKind();
3397 if (RecurrenceDescriptor::isFPMinMaxNumRecurrenceKind(Kind) ||
3398 RecurrenceDescriptor::isFindLastRecurrenceKind(Kind) ||
3399 !RedPhi->getUnderlyingValue())
3400 return true;
3401 // TODO: Add support for FindIV reductions with sunk expressions: the
3402 // resume value from the main loop is in expression domain (e.g.,
3403 // mul(ReducedIV, 3)), but the epilogue tracks raw IV values. A sunk
3404 // expression is identified by a non-VPInstruction user of
3405 // ComputeReductionResult.
3406 if (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind)) {
3407 auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhi);
3408 assert(RdxResult &&
3409 "FindIV reduction must have ComputeReductionResult");
3410 return any_of(Range: RdxResult->users(),
3411 P: std::not_fn(fn: IsaPred<VPInstruction>));
3412 }
3413 return false;
3414 }
3415 default:
3416 return false;
3417 };
3418 });
3419}
3420
3421bool LoopVectorizationPlanner::isCandidateForEpilogueVectorization(
3422 VPlan &MainPlan) const {
3423 // Bail out if the plan contains header phi recipes not yet supported
3424 // for epilogue vectorization.
3425 if (hasUnsupportedHeaderPhiRecipe(Plan&: MainPlan))
3426 return false;
3427
3428 // Epilogue vectorization code has not been auditted to ensure it handles
3429 // non-latch exits properly. It may be fine, but it needs auditted and
3430 // tested.
3431 // TODO: Add support for loops with an early exit.
3432 if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
3433 return false;
3434
3435 return true;
3436}
3437
3438bool LoopVectorizationCostModel::isEpilogueVectorizationProfitable(
3439 const ElementCount VF, const unsigned IC) const {
3440 // FIXME: We need a much better cost-model to take different parameters such
3441 // as register pressure, code size increase and cost of extra branches into
3442 // account. For now we apply a very crude heuristic and only consider loops
3443 // with vectorization factors larger than a certain value.
3444
3445 // Allow the target to opt out.
3446 if (!TTI.preferEpilogueVectorization(Iters: VF * IC))
3447 return false;
3448
3449 unsigned MinVFThreshold = EpilogueVectorizationMinVF.getNumOccurrences() > 0
3450 ? EpilogueVectorizationMinVF
3451 : TTI.getEpilogueVectorizationMinVF();
3452 return estimateElementCount(VF: VF * IC, VScale: Config.getVScaleForTuning()) >=
3453 MinVFThreshold;
3454}
3455
3456std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
3457 VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC) {
3458 if (!EnableEpilogueVectorization) {
3459 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
3460 return nullptr;
3461 }
3462
3463 if (!CM.isEpilogueAllowed()) {
3464 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
3465 "epilogue is allowed.\n");
3466 return nullptr;
3467 }
3468
3469 if (CM.maskPartialAliasing()) {
3470 LLVM_DEBUG(
3471 dbgs()
3472 << "LEV: Epilogue vectorization not supported with alias masking.\n");
3473 return nullptr;
3474 }
3475
3476 // Not really a cost consideration, but check for unsupported cases here to
3477 // simplify the logic.
3478 if (!isCandidateForEpilogueVectorization(MainPlan)) {
3479 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
3480 "is not a supported candidate.\n");
3481 return nullptr;
3482 }
3483
3484 if (hasForcedEpilogueVF()) {
3485 if (estimateElementCount(VF: EpilogueVectorizationForceVF,
3486 VScale: Config.getVScaleForTuning()) >=
3487 IC * estimateElementCount(VF: MainLoopVF, VScale: Config.getVScaleForTuning())) {
3488 // Note that the main loop leaves IC * MainLoopVF iterations iff a scalar
3489 // epilogue is required, but then the epilogue loop also requires a scalar
3490 // epilogue.
3491 LLVM_DEBUG(dbgs() << "LEV: Forced epilogue VF results in dead epilogue "
3492 "vector loop, skipping vectorizing epilogue.\n");
3493 return nullptr;
3494 }
3495
3496 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
3497 if (hasPlanWithVF(VF: EpilogueVectorizationForceVF)) {
3498 std::unique_ptr<VPlan> Clone(
3499 getPlanFor(VF: EpilogueVectorizationForceVF).duplicate());
3500 Clone->setVF(EpilogueVectorizationForceVF);
3501 return Clone;
3502 }
3503
3504 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization forced factor is not "
3505 "viable.\n");
3506 return nullptr;
3507 }
3508
3509 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
3510 LLVM_DEBUG(
3511 dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n");
3512 return nullptr;
3513 }
3514
3515 if (!CM.isEpilogueVectorizationProfitable(VF: MainLoopVF, IC)) {
3516 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
3517 "this loop\n");
3518 return nullptr;
3519 }
3520
3521 // Check if a plan's vector loop processes fewer iterations than VF (e.g. when
3522 // interleave groups have been narrowed) narrowInterleaveGroups) and return
3523 // the adjusted, effective VF.
3524 using namespace VPlanPatternMatch;
3525 auto GetEffectiveVF = [](VPlan &Plan, ElementCount VF) -> ElementCount {
3526 auto *Exiting = Plan.getVectorLoopRegion()->getExitingBasicBlock();
3527 if (match(V: &Exiting->back(),
3528 P: m_BranchOnCount(Op0: m_Add(Op0: m_CanonicalIV(), Op1: m_Specific(VPV: &Plan.getUF())),
3529 Op1: m_VPValue())))
3530 return ElementCount::get(MinVal: 1, Scalable: VF.isScalable());
3531 return VF;
3532 };
3533
3534 // Check if the main loop processes fewer than MainLoopVF elements per
3535 // iteration (e.g. due to narrowing interleave groups). Adjust MainLoopVF
3536 // as needed.
3537 MainLoopVF = GetEffectiveVF(MainPlan, MainLoopVF);
3538
3539 // If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know
3540 // the main loop handles 8 lanes per iteration. We could still benefit from
3541 // vectorizing the epilogue loop with VF=4.
3542 ElementCount EstimatedRuntimeVF = ElementCount::getFixed(
3543 MinVal: estimateElementCount(VF: MainLoopVF, VScale: Config.getVScaleForTuning()));
3544
3545 Type *TCType = Legal->getWidestInductionType();
3546 const SCEV *RemainingIterations = nullptr;
3547 unsigned MaxTripCount = 0;
3548 const SCEV *TC = vputils::getSCEVExprForVPValue(V: MainPlan.getTripCount(), PSE);
3549 assert(!isa<SCEVCouldNotCompute>(TC) && "Trip count SCEV must be computable");
3550 const SCEV *KnownMinTC;
3551 bool ScalableTC = match(S: TC, P: m_scev_c_Mul(Op0: m_SCEV(V&: KnownMinTC), Op1: m_SCEVVScale()));
3552 bool ScalableRemIter = false;
3553 ScalarEvolution &SE = *PSE.getSE();
3554 // Use versions of TC and VF in which both are either scalable or fixed.
3555 if (ScalableTC == MainLoopVF.isScalable()) {
3556 ScalableRemIter = ScalableTC;
3557 RemainingIterations =
3558 SE.getURemExpr(LHS: TC, RHS: SE.getElementCount(Ty: TCType, EC: MainLoopVF * IC));
3559 } else if (ScalableTC) {
3560 const SCEV *EstimatedTC = SE.getMulExpr(
3561 LHS: KnownMinTC,
3562 RHS: SE.getConstant(Ty: TCType, V: Config.getVScaleForTuning().value_or(u: 1)));
3563 RemainingIterations = SE.getURemExpr(
3564 LHS: EstimatedTC, RHS: SE.getElementCount(Ty: TCType, EC: MainLoopVF * IC));
3565 } else
3566 RemainingIterations =
3567 SE.getURemExpr(LHS: TC, RHS: SE.getElementCount(Ty: TCType, EC: EstimatedRuntimeVF * IC));
3568
3569 // No iterations left to process in the epilogue.
3570 if (RemainingIterations->isZero())
3571 return nullptr;
3572
3573 if (MainLoopVF.isFixed()) {
3574 MaxTripCount = MainLoopVF.getFixedValue() * IC - 1;
3575 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_ULT, LHS: RemainingIterations,
3576 RHS: SE.getConstant(Ty: TCType, V: MaxTripCount))) {
3577 MaxTripCount = SE.getUnsignedRangeMax(S: RemainingIterations).getZExtValue();
3578 }
3579 LLVM_DEBUG(dbgs() << "LEV: Maximum Trip Count for Epilogue: "
3580 << MaxTripCount << "\n");
3581 }
3582
3583 auto SkipVF = [&](const SCEV *VF, const SCEV *RemIter) -> bool {
3584 return SE.isKnownPredicate(Pred: CmpInst::ICMP_UGT, LHS: VF, RHS: RemIter);
3585 };
3586 VectorizationFactor Result = VectorizationFactor::Disabled();
3587 VPlan *BestPlan = nullptr;
3588 for (auto &NextVF : ProfitableVFs) {
3589 // Skip candidate VFs without a corresponding VPlan.
3590 if (!hasPlanWithVF(VF: NextVF.Width))
3591 continue;
3592
3593 VPlan &CurrentPlan = getPlanFor(VF: NextVF.Width);
3594 ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
3595 // Skip fixed vector VFs > than the estimated runtime VF, or any VF > than
3596 // the VF of the main loop.
3597 if ((!EffectiveVF.isScalable() && MainLoopVF.isScalable() &&
3598 ElementCount::isKnownGT(LHS: EffectiveVF, RHS: EstimatedRuntimeVF)) ||
3599 ElementCount::isKnownGT(LHS: EffectiveVF, RHS: MainLoopVF))
3600 continue;
3601
3602 // If EffectiveVF is greater than the number of remaining iterations, the
3603 // epilogue loop would be dead. Skip such factors. If the epilogue plan
3604 // also has narrowed interleave groups, use the effective VF since
3605 // the epilogue step will be reduced to its IC.
3606 // TODO: We should also consider comparing against a scalable
3607 // RemainingIterations when SCEV be able to evaluate non-canonical
3608 // vscale-based expressions.
3609 if (!ScalableRemIter) {
3610 // Handle the case where EffectiveVF and RemainingIterations are in
3611 // different numerical spaces.
3612 if (EffectiveVF.isScalable())
3613 EffectiveVF = ElementCount::getFixed(
3614 MinVal: estimateElementCount(VF: EffectiveVF, VScale: Config.getVScaleForTuning()));
3615 if (SkipVF(SE.getElementCount(Ty: TCType, EC: EffectiveVF), RemainingIterations))
3616 continue;
3617 }
3618
3619 if (Result.Width.isScalar() ||
3620 isMoreProfitable(A: NextVF, B: Result, MaxTripCount,
3621 HasTail: !MainPlan.hasTailFolded(),
3622 /*IsEpilogue*/ true)) {
3623 Result = NextVF;
3624 BestPlan = &CurrentPlan;
3625 }
3626 }
3627
3628 if (!BestPlan)
3629 return nullptr;
3630
3631 LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
3632 << Result.Width << "\n");
3633 std::unique_ptr<VPlan> Clone(BestPlan->duplicate());
3634 Clone->setVF(Result.Width);
3635 return Clone;
3636}
3637
3638unsigned
3639LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
3640 InstructionCost LoopCost) {
3641 // -- The interleave heuristics --
3642 // We interleave the loop in order to expose ILP and reduce the loop overhead.
3643 // There are many micro-architectural considerations that we can't predict
3644 // at this level. For example, frontend pressure (on decode or fetch) due to
3645 // code size, or the number and capabilities of the execution ports.
3646 //
3647 // We use the following heuristics to select the interleave count:
3648 // 1. If the code has reductions, then we interleave to break the cross
3649 // iteration dependency.
3650 // 2. If the loop is really small, then we interleave to reduce the loop
3651 // overhead.
3652 // 3. We don't interleave if we think that we will spill registers to memory
3653 // due to the increased register pressure.
3654
3655 // Only interleave tail-folded loops if wide lane masks are requested, as the
3656 // overhead of multiple instructions to calculate the predicate is likely
3657 // not beneficial. If an epilogue is not allowed for any other reason,
3658 // do not interleave.
3659 if (!CM.isEpilogueAllowed() &&
3660 !(CM.preferTailFoldedLoop() && CM.useWideActiveLaneMask()))
3661 return 1;
3662
3663 if (any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3664 P: IsaPred<VPCurrentIterationPHIRecipe>)) {
3665 LLVM_DEBUG(dbgs() << "LV: Loop requires variable-length step. "
3666 "Unroll factor forced to be 1.\n");
3667 return 1;
3668 }
3669
3670 // We used the distance for the interleave count.
3671 if (!Legal->isSafeForAnyVectorWidth())
3672 return 1;
3673
3674 // We don't attempt to perform interleaving for loops with uncountable early
3675 // exits because the VPInstruction::AnyOf code cannot currently handle
3676 // multiple parts.
3677 if (Plan.hasEarlyExit())
3678 return 1;
3679
3680 const bool HasReductions =
3681 any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3682 P: IsaPred<VPReductionPHIRecipe>);
3683
3684 // FIXME: implement interleaving for FindLast transform correctly.
3685 if (hasFindLastReductionPhi(Plan))
3686 return 1;
3687
3688 VPRegisterUsage R =
3689 calculateRegisterUsageForPlan(Plan, VFs: {VF}, TTI, ValuesToIgnore: CM.ValuesToIgnore)[0];
3690
3691 // If we did not calculate the cost for VF (because the user selected the VF)
3692 // then we calculate the cost of VF here.
3693 if (LoopCost == 0) {
3694 if (VF.isScalar())
3695 LoopCost = CM.expectedCost(VF);
3696 else
3697 LoopCost = cost(Plan, VF, RU: &R);
3698 assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
3699
3700 // Loop body is free and there is no need for interleaving.
3701 if (LoopCost == 0)
3702 return 1;
3703 }
3704
3705 // We divide by these constants so assume that we have at least one
3706 // instruction that uses at least one register.
3707 for (auto &Pair : R.MaxLocalUsers) {
3708 Pair.second = std::max(a: Pair.second, b: 1U);
3709 }
3710
3711 // We calculate the interleave count using the following formula.
3712 // Subtract the number of loop invariants from the number of available
3713 // registers. These registers are used by all of the interleaved instances.
3714 // Next, divide the remaining registers by the number of registers that is
3715 // required by the loop, in order to estimate how many parallel instances
3716 // fit without causing spills. All of this is rounded down if necessary to be
3717 // a power of two. We want power of two interleave count to simplify any
3718 // addressing operations or alignment considerations.
3719 // We also want power of two interleave counts to ensure that the induction
3720 // variable of the vector loop wraps to zero, when tail is folded by masking;
3721 // this currently happens when OptForSize, in which case IC is set to 1 above.
3722 unsigned IC = UINT_MAX;
3723
3724 for (const auto &Pair : R.MaxLocalUsers) {
3725 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(ClassID: Pair.first);
3726 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters
3727 << " registers of "
3728 << TTI.getRegisterClassName(Pair.first)
3729 << " register class\n");
3730 if (VF.isScalar()) {
3731 if (ForceTargetNumScalarRegs.getNumOccurrences() > 0)
3732 TargetNumRegisters = ForceTargetNumScalarRegs;
3733 } else {
3734 if (ForceTargetNumVectorRegs.getNumOccurrences() > 0)
3735 TargetNumRegisters = ForceTargetNumVectorRegs;
3736 }
3737 unsigned MaxLocalUsers = Pair.second;
3738 unsigned LoopInvariantRegs = 0;
3739 if (R.LoopInvariantRegs.contains(Key: Pair.first))
3740 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
3741
3742 unsigned TmpIC = llvm::bit_floor(Value: (TargetNumRegisters - LoopInvariantRegs) /
3743 MaxLocalUsers);
3744 // Don't count the induction variable as interleaved.
3745 if (EnableIndVarRegisterHeur) {
3746 TmpIC = llvm::bit_floor(Value: (TargetNumRegisters - LoopInvariantRegs - 1) /
3747 std::max(a: 1U, b: (MaxLocalUsers - 1)));
3748 }
3749
3750 IC = std::min(a: IC, b: TmpIC);
3751 }
3752
3753 // Clamp the interleave ranges to reasonable counts.
3754 bool HasUnorderedReductions =
3755 HasReductions &&
3756 !any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3757 P: [](VPRecipeBase &R) {
3758 auto *RedR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
3759 return RedR && RedR->isOrdered();
3760 });
3761 unsigned MaxInterleaveCount =
3762 TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions);
3763 LLVM_DEBUG(dbgs() << "LV: MaxInterleaveFactor for the target is "
3764 << MaxInterleaveCount << "\n");
3765
3766 // Check if the user has overridden the max.
3767 if (VF.isScalar()) {
3768 if (ForceTargetMaxScalarInterleaveFactor.getNumOccurrences() > 0)
3769 MaxInterleaveCount = ForceTargetMaxScalarInterleaveFactor;
3770 } else {
3771 if (ForceTargetMaxVectorInterleaveFactor.getNumOccurrences() > 0)
3772 MaxInterleaveCount = ForceTargetMaxVectorInterleaveFactor;
3773 }
3774
3775 // Try to get the exact trip count, or an estimate based on profiling data or
3776 // ConstantMax from PSE, failing that.
3777 auto BestKnownTC =
3778 getSmallBestKnownTC(PSE, L: OrigLoop,
3779 /*CanUseConstantMax=*/true,
3780 /*CanExcludeZeroTrips=*/CM.isEpilogueAllowed());
3781
3782 // For fixed length VFs treat a scalable trip count as unknown.
3783 if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
3784 // Re-evaluate trip counts and VFs to be in the same numerical space.
3785 unsigned AvailableTC =
3786 estimateElementCount(VF: *BestKnownTC, VScale: Config.getVScaleForTuning());
3787 unsigned EstimatedVF =
3788 estimateElementCount(VF, VScale: Config.getVScaleForTuning());
3789
3790 // At least one iteration must be scalar when this constraint holds. So the
3791 // maximum available iterations for interleaving is one less.
3792 if (requiresScalarEpilogue(Plan, VF))
3793 --AvailableTC;
3794
3795 unsigned InterleaveCountLB = bit_floor(Value: std::max(
3796 a: 1u, b: std::min(a: AvailableTC / (EstimatedVF * 2), b: MaxInterleaveCount)));
3797
3798 if (getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop).isNonZero()) {
3799 // If the best known trip count is exact, we select between two
3800 // prospective ICs, where
3801 //
3802 // 1) the aggressive IC is capped by the trip count divided by VF
3803 // 2) the conservative IC is capped by the trip count divided by (VF * 2)
3804 //
3805 // The final IC is selected in a way that the epilogue loop trip count is
3806 // minimized while maximizing the IC itself, so that we either run the
3807 // vector loop at least once if it generates a small epilogue loop, or
3808 // else we run the vector loop at least twice.
3809
3810 unsigned InterleaveCountUB = bit_floor(Value: std::max(
3811 a: 1u, b: std::min(a: AvailableTC / EstimatedVF, b: MaxInterleaveCount)));
3812 MaxInterleaveCount = InterleaveCountLB;
3813
3814 if (InterleaveCountUB != InterleaveCountLB) {
3815 unsigned TailTripCountUB =
3816 (AvailableTC % (EstimatedVF * InterleaveCountUB));
3817 unsigned TailTripCountLB =
3818 (AvailableTC % (EstimatedVF * InterleaveCountLB));
3819 // If both produce same scalar tail, maximize the IC to do the same work
3820 // in fewer vector loop iterations
3821 if (TailTripCountUB == TailTripCountLB)
3822 MaxInterleaveCount = InterleaveCountUB;
3823 }
3824 } else {
3825 // If trip count is an estimated compile time constant, limit the
3826 // IC to be capped by the trip count divided by VF * 2, such that the
3827 // vector loop runs at least twice to make interleaving seem profitable
3828 // when there is an epilogue loop present. Since exact Trip count is not
3829 // known we choose to be conservative in our IC estimate.
3830 MaxInterleaveCount = InterleaveCountLB;
3831 }
3832 }
3833
3834 assert(MaxInterleaveCount > 0 &&
3835 "Maximum interleave count must be greater than 0");
3836
3837 // Clamp the calculated IC to be between the 1 and the max interleave count
3838 // that the target and trip count allows.
3839 if (IC > MaxInterleaveCount)
3840 IC = MaxInterleaveCount;
3841 else
3842 // Make sure IC is greater than 0.
3843 IC = std::max(a: 1u, b: IC);
3844
3845 assert(IC > 0 && "Interleave count must be greater than 0.");
3846
3847 // Interleave if we vectorized this loop and there is a reduction that could
3848 // benefit from interleaving.
3849 if (VF.isVector() && HasReductions) {
3850 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
3851 return IC;
3852 }
3853
3854 // For any scalar loop that either requires runtime checks or tail-folding we
3855 // are better off leaving this to the unroller. Note that if we've already
3856 // vectorized the loop we will have done the runtime check and so interleaving
3857 // won't require further checks.
3858 bool ScalarInterleavingRequiresPredication =
3859 (VF.isScalar() && any_of(Range: OrigLoop->blocks(), P: [this](BasicBlock *BB) {
3860 return Legal->blockNeedsPredication(BB);
3861 }));
3862 bool ScalarInterleavingRequiresRuntimePointerCheck =
3863 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need);
3864
3865 // We want to interleave small loops in order to reduce the loop overhead and
3866 // potentially expose ILP opportunities.
3867 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
3868 << "LV: IC is " << IC << '\n'
3869 << "LV: VF is " << VF << '\n');
3870 const bool AggressivelyInterleave =
3871 TTI.enableAggressiveInterleaving(LoopHasReductions: HasReductions);
3872 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
3873 !ScalarInterleavingRequiresPredication && LoopCost < SmallLoopCost) {
3874 // We assume that the cost overhead is 1 and we use the cost model
3875 // to estimate the cost of the loop and interleave until the cost of the
3876 // loop overhead is about 5% of the cost of the loop.
3877 unsigned SmallIC = std::min(a: IC, b: (unsigned)llvm::bit_floor<uint64_t>(
3878 Value: SmallLoopCost / LoopCost.getValue()));
3879
3880 // Interleave until store/load ports (estimated by max interleave count) are
3881 // saturated.
3882 unsigned NumStores = 0;
3883 unsigned NumLoads = 0;
3884 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3885 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
3886 for (VPRecipeBase &R : *VPBB) {
3887 if (isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Val: &R)) {
3888 NumLoads++;
3889 continue;
3890 }
3891 if (isa<VPWidenStoreRecipe, VPWidenStoreEVLRecipe>(Val: &R)) {
3892 NumStores++;
3893 continue;
3894 }
3895
3896 if (auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(Val: &R)) {
3897 if (unsigned StoreOps = InterleaveR->getNumStoreOperands())
3898 NumStores += StoreOps;
3899 else
3900 NumLoads += InterleaveR->getNumDefinedValues();
3901 continue;
3902 }
3903 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
3904 NumLoads += isa<LoadInst>(Val: RepR->getUnderlyingInstr());
3905 NumStores += isa<StoreInst>(Val: RepR->getUnderlyingInstr());
3906 continue;
3907 }
3908 if (isa<VPHistogramRecipe>(Val: &R)) {
3909 NumLoads++;
3910 NumStores++;
3911 continue;
3912 }
3913 }
3914 }
3915 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
3916 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
3917
3918 // There is little point in interleaving for reductions containing selects
3919 // and compares when VF=1 since it may just create more overhead than it's
3920 // worth for loops with small trip counts. This is because we still have to
3921 // do the final reduction after the loop.
3922 bool HasSelectCmpReductions =
3923 HasReductions &&
3924 any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3925 P: [](VPRecipeBase &R) {
3926 auto *RedR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
3927 return RedR && (RecurrenceDescriptor::isAnyOfRecurrenceKind(
3928 Kind: RedR->getRecurrenceKind()) ||
3929 RecurrenceDescriptor::isFindIVRecurrenceKind(
3930 Kind: RedR->getRecurrenceKind()));
3931 });
3932 if (HasSelectCmpReductions) {
3933 LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
3934 return 1;
3935 }
3936
3937 // If we have a scalar reduction (vector reductions are already dealt with
3938 // by this point), we can increase the critical path length if the loop
3939 // we're interleaving is inside another loop. For tree-wise reductions
3940 // set the limit to 2, and for ordered reductions it's best to disable
3941 // interleaving entirely.
3942 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
3943 bool HasOrderedReductions =
3944 any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3945 P: [](VPRecipeBase &R) {
3946 auto *RedR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
3947
3948 return RedR && RedR->isOrdered();
3949 });
3950 if (HasOrderedReductions) {
3951 LLVM_DEBUG(
3952 dbgs() << "LV: Not interleaving scalar ordered reductions.\n");
3953 return 1;
3954 }
3955
3956 unsigned F = MaxNestedScalarReductionIC;
3957 SmallIC = std::min(a: SmallIC, b: F);
3958 StoresIC = std::min(a: StoresIC, b: F);
3959 LoadsIC = std::min(a: LoadsIC, b: F);
3960 }
3961
3962 if (EnableLoadStoreRuntimeInterleave &&
3963 std::max(a: StoresIC, b: LoadsIC) > SmallIC) {
3964 LLVM_DEBUG(
3965 dbgs() << "LV: Interleaving to saturate store or load ports.\n");
3966 return std::max(a: StoresIC, b: LoadsIC);
3967 }
3968
3969 // If there are scalar reductions and TTI has enabled aggressive
3970 // interleaving for reductions, we will interleave to expose ILP.
3971 if (VF.isScalar() && AggressivelyInterleave) {
3972 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3973 // Interleave no less than SmallIC but not as aggressive as the normal IC
3974 // to satisfy the rare situation when resources are too limited.
3975 return std::max(a: IC / 2, b: SmallIC);
3976 }
3977
3978 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
3979 return SmallIC;
3980 }
3981
3982 // Interleave if this is a large loop (small loops are already dealt with by
3983 // this point) that could benefit from interleaving.
3984 if (AggressivelyInterleave) {
3985 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3986 return IC;
3987 }
3988
3989 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n");
3990 return 1;
3991}
3992
3993bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(Instruction *I,
3994 ElementCount VF) {
3995 // TODO: Cost model for emulated masked load/store is completely
3996 // broken. This hack guides the cost model to use an artificially
3997 // high enough value to practically disable vectorization with such
3998 // operations, except where previously deployed legality hack allowed
3999 // using very low cost values. This is to avoid regressions coming simply
4000 // from moving "masked load/store" check from legality to cost model.
4001 // Masked Load/Gather emulation was previously never allowed.
4002 // Limited number of Masked Store/Scatter emulation was allowed.
4003 assert((isPredicatedInst(I)) &&
4004 "Expecting a scalar emulated instruction");
4005 return isa<LoadInst>(Val: I) ||
4006 (isa<StoreInst>(Val: I) &&
4007 NumPredStores > NumberOfStoresToPredicate);
4008}
4009
4010void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
4011 assert(VF.isVector() && "Expected VF >= 2");
4012
4013 // If we've already collected the instructions to scalarize or the predicated
4014 // BBs after vectorization, there's nothing to do. Collection may already have
4015 // occurred if we have a user-selected VF and are now computing the expected
4016 // cost for interleaving.
4017 if (InstsToScalarize.contains(Key: VF) ||
4018 PredicatedBBsAfterVectorization.contains(Val: VF))
4019 return;
4020
4021 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's
4022 // not profitable to scalarize any instructions, the presence of VF in the
4023 // map will indicate that we've analyzed it already.
4024 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
4025
4026 // Find all the instructions that are scalar with predication in the loop and
4027 // determine if it would be better to not if-convert the blocks they are in.
4028 // If so, we also record the instructions to scalarize.
4029 for (BasicBlock *BB : TheLoop->blocks()) {
4030 if (!blockNeedsPredicationForAnyReason(BB))
4031 continue;
4032 for (Instruction &I : *BB)
4033 if (isScalarWithPredication(I: &I, VF)) {
4034 ScalarCostsTy ScalarCosts;
4035 // Do not apply discount logic for:
4036 // 1. Scalars after vectorization, as there will only be a single copy
4037 // of the instruction.
4038 // 2. Scalable VF, as that would lead to invalid scalarization costs.
4039 // 3. Emulated masked memrefs, if a hacked cost is needed.
4040 if (!isScalarAfterVectorization(I: &I, VF) && !VF.isScalable() &&
4041 !useEmulatedMaskMemRefHack(I: &I, VF) &&
4042 computePredInstDiscount(PredInst: &I, ScalarCosts, VF) >= 0) {
4043 for (const auto &[I, IC] : ScalarCosts)
4044 ScalarCostsVF.insert(KV: {I, IC});
4045 }
4046 // Remember that BB will remain after vectorization.
4047 PredicatedBBsAfterVectorization[VF].insert(Ptr: BB);
4048 for (auto *Pred : predecessors(BB)) {
4049 if (Pred->getSingleSuccessor() == BB)
4050 PredicatedBBsAfterVectorization[VF].insert(Ptr: Pred);
4051 }
4052 }
4053 }
4054}
4055
4056InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
4057 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) {
4058 assert(!isUniformAfterVectorization(PredInst, VF) &&
4059 "Instruction marked uniform-after-vectorization will be predicated");
4060
4061 // Initialize the discount to zero, meaning that the scalar version and the
4062 // vector version cost the same.
4063 InstructionCost Discount = 0;
4064
4065 // Holds instructions to analyze. The instructions we visit are mapped in
4066 // ScalarCosts. Those instructions are the ones that would be scalarized if
4067 // we find that the scalar version costs less.
4068 SmallVector<Instruction *, 8> Worklist;
4069
4070 // Returns true if the given instruction can be scalarized.
4071 auto CanBeScalarized = [&](Instruction *I) -> bool {
4072 // We only attempt to scalarize instructions forming a single-use chain
4073 // from the original predicated block that would otherwise be vectorized.
4074 // Although not strictly necessary, we give up on instructions we know will
4075 // already be scalar to avoid traversing chains that are unlikely to be
4076 // beneficial.
4077 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
4078 isScalarAfterVectorization(I, VF))
4079 return false;
4080
4081 // If the instruction is scalar with predication, it will be analyzed
4082 // separately. We ignore it within the context of PredInst.
4083 if (isScalarWithPredication(I, VF))
4084 return false;
4085
4086 // If any of the instruction's operands are uniform after vectorization,
4087 // the instruction cannot be scalarized. This prevents, for example, a
4088 // masked load from being scalarized.
4089 //
4090 // We assume we will only emit a value for lane zero of an instruction
4091 // marked uniform after vectorization, rather than VF identical values.
4092 // Thus, if we scalarize an instruction that uses a uniform, we would
4093 // create uses of values corresponding to the lanes we aren't emitting code
4094 // for. This behavior can be changed by allowing getScalarValue to clone
4095 // the lane zero values for uniforms rather than asserting.
4096 for (Use &U : I->operands())
4097 if (auto *J = dyn_cast<Instruction>(Val: U.get()))
4098 if (isUniformAfterVectorization(I: J, VF))
4099 return false;
4100
4101 // Otherwise, we can scalarize the instruction.
4102 return true;
4103 };
4104
4105 // Compute the expected cost discount from scalarizing the entire expression
4106 // feeding the predicated instruction. We currently only consider expressions
4107 // that are single-use instruction chains.
4108 Worklist.push_back(Elt: PredInst);
4109 while (!Worklist.empty()) {
4110 Instruction *I = Worklist.pop_back_val();
4111
4112 // If we've already analyzed the instruction, there's nothing to do.
4113 if (ScalarCosts.contains(Key: I))
4114 continue;
4115
4116 // Cannot scalarize fixed-order recurrence phis at the moment.
4117 if (isa<PHINode>(Val: I) && Legal->isFixedOrderRecurrence(Phi: cast<PHINode>(Val: I)))
4118 continue;
4119
4120 // Compute the cost of the vector instruction. Note that this cost already
4121 // includes the scalarization overhead of the predicated instruction.
4122 InstructionCost VectorCost = getInstructionCost(I, VF);
4123
4124 // Compute the cost of the scalarized instruction. This cost is the cost of
4125 // the instruction as if it wasn't if-converted and instead remained in the
4126 // predicated block. We will scale this cost by block probability after
4127 // computing the scalarization overhead.
4128 InstructionCost ScalarCost =
4129 VF.getFixedValue() * getInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1));
4130
4131 // Compute the scalarization overhead of needed insertelement instructions
4132 // and phi nodes.
4133 if (isScalarWithPredication(I, VF) && !I->getType()->isVoidTy()) {
4134 Type *WideTy = toVectorizedTy(Ty: I->getType(), EC: VF);
4135 for (Type *VectorTy : getContainedTypes(Ty: WideTy)) {
4136 ScalarCost += TTI.getScalarizationOverhead(
4137 Ty: cast<VectorType>(Val: VectorTy), DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4138 /*Insert=*/true,
4139 /*Extract=*/false, CostKind: Config.CostKind);
4140 }
4141 ScalarCost += VF.getFixedValue() *
4142 TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
4143 }
4144
4145 // Compute the scalarization overhead of needed extractelement
4146 // instructions. For each of the instruction's operands, if the operand can
4147 // be scalarized, add it to the worklist; otherwise, account for the
4148 // overhead.
4149 for (Use &U : I->operands())
4150 if (auto *J = dyn_cast<Instruction>(Val: U.get())) {
4151 assert(canVectorizeTy(J->getType()) &&
4152 "Instruction has non-scalar type");
4153 if (CanBeScalarized(J))
4154 Worklist.push_back(Elt: J);
4155 else if (needsExtract(V: J, VF)) {
4156 Type *WideTy = toVectorizedTy(Ty: J->getType(), EC: VF);
4157 for (Type *VectorTy : getContainedTypes(Ty: WideTy)) {
4158 ScalarCost += TTI.getScalarizationOverhead(
4159 Ty: cast<VectorType>(Val: VectorTy),
4160 DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()), /*Insert*/ false,
4161 /*Extract*/ true, CostKind: Config.CostKind);
4162 }
4163 }
4164 }
4165
4166 // Scale the total scalar cost by block probability.
4167 ScalarCost /= getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
4168
4169 // Compute the discount. A non-negative discount means the vector version
4170 // of the instruction costs more, and scalarizing would be beneficial.
4171 Discount += VectorCost - ScalarCost;
4172 ScalarCosts[I] = ScalarCost;
4173 }
4174
4175 return Discount;
4176}
4177
4178InstructionCost LoopVectorizationCostModel::expectedCost(ElementCount VF) {
4179 InstructionCost Cost;
4180 assert(VF.isScalar() && "must only be called for scalar VFs");
4181
4182 // For each block.
4183 for (BasicBlock *BB : TheLoop->blocks()) {
4184 InstructionCost BlockCost;
4185
4186 // For each instruction in the old loop.
4187 for (Instruction &I : *BB) {
4188 // Skip ignored values.
4189 if (ValuesToIgnore.count(Ptr: &I) ||
4190 (VF.isVector() && VecValuesToIgnore.count(Ptr: &I)))
4191 continue;
4192
4193 InstructionCost C = getInstructionCost(I: &I, VF);
4194
4195 // Check if we should override the cost.
4196 if (C.isValid() && ForceTargetInstructionCost.getNumOccurrences() > 0)
4197 C = InstructionCost(ForceTargetInstructionCost);
4198
4199 BlockCost += C;
4200 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C << " for VF "
4201 << VF << " For instruction: " << I << '\n');
4202 }
4203
4204 // In the scalar loop, we may not always execute the predicated block, if it
4205 // is an if-else block. Thus, scale the block's cost by the probability of
4206 // executing it. getPredBlockCostDivisor will return 1 for blocks that are
4207 // only predicated by the header mask when folding the tail.
4208 Cost += BlockCost / getPredBlockCostDivisor(CostKind: Config.CostKind, BB);
4209 }
4210
4211 return Cost;
4212}
4213
4214/// Gets the address access SCEV for Ptr, if it should be used for cost modeling
4215/// according to isAddressSCEVForCost.
4216///
4217/// This SCEV can be sent to the Target in order to estimate the address
4218/// calculation cost.
4219static const SCEV *getAddressAccessSCEV(
4220 Value *Ptr,
4221 PredicatedScalarEvolution &PSE,
4222 const Loop *TheLoop) {
4223 const SCEV *Addr = PSE.getSCEV(V: Ptr);
4224 return vputils::isAddressSCEVForCost(Addr, SE&: *PSE.getSE(), L: TheLoop) ? Addr
4225 : nullptr;
4226}
4227
4228InstructionCost
4229LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
4230 ElementCount VF) {
4231 assert(VF.isVector() &&
4232 "Scalarization cost of instruction implies vectorization.");
4233 if (VF.isScalable())
4234 return InstructionCost::getInvalid();
4235
4236 Type *ValTy = getLoadStoreType(I);
4237 auto *SE = PSE.getSE();
4238
4239 unsigned AS = getLoadStoreAddressSpace(I);
4240 Value *Ptr = getLoadStorePointerOperand(V: I);
4241 Type *PtrTy = toVectorTy(Scalar: Ptr->getType(), EC: VF);
4242 // NOTE: PtrTy is a vector to signal `TTI::getAddressComputationCost`
4243 // that it is being called from this specific place.
4244
4245 // Figure out whether the access is strided and get the stride value
4246 // if it's known in compile time
4247 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, PSE, TheLoop);
4248
4249 // Get the cost of the scalar memory instruction and address computation.
4250 InstructionCost Cost =
4251 VF.getFixedValue() *
4252 TTI.getAddressComputationCost(PtrTy, SE, Ptr: PtrSCEV, CostKind: Config.CostKind);
4253
4254 // Don't pass *I here, since it is scalar but will actually be part of a
4255 // vectorized loop where the user of it is a vectorized instruction.
4256 const Align Alignment = getLoadStoreAlignment(I);
4257 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4258 Cost += VF.getFixedValue() *
4259 TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: ValTy->getScalarType(), Alignment,
4260 AddressSpace: AS, CostKind: Config.CostKind, OpdInfo: OpInfo);
4261
4262 // Get the overhead of the extractelement and insertelement instructions
4263 // we might create due to scalarization.
4264 Cost += getScalarizationOverhead(I, VF);
4265
4266 // If we have a predicated load/store, it will need extra i1 extracts and
4267 // conditional branches, but may not be executed for each vector lane. Scale
4268 // the cost by the probability of executing the predicated block.
4269 if (isPredicatedInst(I)) {
4270 Cost /= getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
4271
4272 // Add the cost of an i1 extract and a branch
4273 auto *VecI1Ty =
4274 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: ValTy->getContext()), EC: VF);
4275 Cost += TTI.getScalarizationOverhead(
4276 Ty: VecI1Ty, DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4277 /*Insert=*/false, /*Extract=*/true, CostKind: Config.CostKind);
4278 Cost += TTI.getCFInstrCost(Opcode: Instruction::CondBr, CostKind: Config.CostKind);
4279
4280 if (useEmulatedMaskMemRefHack(I, VF))
4281 // Artificially setting to a high enough value to practically disable
4282 // vectorization with such operations.
4283 Cost = 3000000;
4284 }
4285
4286 return Cost;
4287}
4288
4289InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
4290 Instruction *I, ElementCount VF, InstWidening Kind) {
4291 assert((Kind == CM_Widen || Kind == CM_Widen_Reverse) &&
4292 "Expected a consecutive widening decision");
4293 Type *ValTy = getLoadStoreType(I);
4294 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4295 unsigned AS = getLoadStoreAddressSpace(I);
4296
4297 const Align Alignment = getLoadStoreAlignment(I);
4298 InstructionCost Cost = 0;
4299 if (isMaskRequired(I)) {
4300 unsigned IID = I->getOpcode() == Instruction::Load
4301 ? Intrinsic::masked_load
4302 : Intrinsic::masked_store;
4303 Cost += TTI.getMemIntrinsicInstrCost(
4304 MICA: MemIntrinsicCostAttributes(IID, VectorTy, Alignment, AS),
4305 CostKind: Config.CostKind);
4306 } else {
4307 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4308 Cost += TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: VectorTy, Alignment, AddressSpace: AS,
4309 CostKind: Config.CostKind, OpdInfo: OpInfo, I);
4310 }
4311
4312 if (Kind == CM_Widen_Reverse)
4313 Cost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Reverse, DstTy: VectorTy,
4314 SrcTy: VectorTy, Mask: {}, CostKind: Config.CostKind, Index: 0);
4315 return Cost;
4316}
4317
4318InstructionCost
4319LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
4320 ElementCount VF) {
4321 assert(isUniformMemOp(*I, VF));
4322
4323 Type *ValTy = getLoadStoreType(I);
4324 Type *PtrTy = getLoadStorePointerOperand(V: I)->getType();
4325 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4326 const Align Alignment = getLoadStoreAlignment(I);
4327 unsigned AS = getLoadStoreAddressSpace(I);
4328 if (isa<LoadInst>(Val: I)) {
4329 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4330 CostKind: Config.CostKind) +
4331 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: ValTy, Alignment, AddressSpace: AS,
4332 CostKind: Config.CostKind) +
4333 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Broadcast, DstTy: VectorTy,
4334 SrcTy: VectorTy, Mask: {}, CostKind: Config.CostKind);
4335 }
4336 StoreInst *SI = cast<StoreInst>(Val: I);
4337
4338 bool IsLoopInvariantStoreValue = Legal->isInvariant(V: SI->getValueOperand());
4339 // TODO: We have existing tests that request the cost of extracting element
4340 // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent
4341 // the actual generated code, which involves extracting the last element of
4342 // a scalable vector where the lane to extract is unknown at compile time.
4343 InstructionCost Cost =
4344 TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr, CostKind: Config.CostKind) +
4345 TTI.getMemoryOpCost(Opcode: Instruction::Store, Src: ValTy, Alignment, AddressSpace: AS,
4346 CostKind: Config.CostKind);
4347 if (!IsLoopInvariantStoreValue)
4348 Cost += TTI.getIndexedVectorInstrCostFromEnd(Opcode: Instruction::ExtractElement,
4349 Val: VectorTy, CostKind: Config.CostKind, Index: 0);
4350 return Cost;
4351}
4352
4353InstructionCost
4354LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
4355 ElementCount VF) {
4356 Type *ValTy = getLoadStoreType(I);
4357 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4358 const Align Alignment = getLoadStoreAlignment(I);
4359 Value *Ptr = getLoadStorePointerOperand(V: I);
4360 Type *PtrTy = Ptr->getType();
4361
4362 if (!isUniform(V: Ptr, VF))
4363 PtrTy = toVectorTy(Scalar: PtrTy, EC: VF);
4364
4365 unsigned IID = I->getOpcode() == Instruction::Load
4366 ? Intrinsic::masked_gather
4367 : Intrinsic::masked_scatter;
4368 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4369 CostKind: Config.CostKind) +
4370 TTI.getMemIntrinsicInstrCost(
4371 MICA: MemIntrinsicCostAttributes(IID, VectorTy, Ptr, isMaskRequired(I),
4372 Alignment, I),
4373 CostKind: Config.CostKind);
4374}
4375
4376InstructionCost
4377LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
4378 ElementCount VF) {
4379 const auto *Group = getInterleavedAccessGroup(Instr: I);
4380 assert(Group && "Fail to get an interleaved access group.");
4381
4382 Instruction *InsertPos = Group->getInsertPos();
4383 Type *ValTy = getLoadStoreType(I: InsertPos);
4384 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4385 unsigned AS = getLoadStoreAddressSpace(I: InsertPos);
4386
4387 unsigned InterleaveFactor = Group->getFactor();
4388 auto *WideVecTy = VectorType::get(ElementType: ValTy, EC: VF * InterleaveFactor);
4389
4390 // Holds the indices of existing members in the interleaved group.
4391 SmallVector<unsigned, 4> Indices;
4392 for (unsigned IF = 0; IF < InterleaveFactor; IF++)
4393 if (Group->getMember(Index: IF))
4394 Indices.push_back(Elt: IF);
4395
4396 // Calculate the cost of the whole interleaved group.
4397 bool UseMaskForGaps =
4398 (Group->requiresScalarEpilogue() && !isEpilogueAllowed()) ||
4399 (isa<StoreInst>(Val: I) && !Group->isFull());
4400 InstructionCost Cost = TTI.getInterleavedMemoryOpCost(
4401 Opcode: InsertPos->getOpcode(), VecTy: WideVecTy, Factor: Group->getFactor(), Indices,
4402 Alignment: Group->getAlign(), AddressSpace: AS, CostKind: Config.CostKind, UseMaskForCond: isMaskRequired(I),
4403 UseMaskForGaps);
4404
4405 if (Group->isReverse()) {
4406 // TODO: Add support for reversed masked interleaved access.
4407 assert(!isMaskRequired(I) &&
4408 "Reverse masked interleaved access not supported.");
4409 Cost += Group->getNumMembers() *
4410 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Reverse, DstTy: VectorTy,
4411 SrcTy: VectorTy, Mask: {}, CostKind: Config.CostKind, Index: 0);
4412 }
4413 return Cost;
4414}
4415
4416std::optional<InstructionCost>
4417LoopVectorizationCostModel::getReductionPatternCost(Instruction *I,
4418 ElementCount VF,
4419 Type *Ty) const {
4420 using namespace llvm::PatternMatch;
4421 // Early exit for no inloop reductions
4422 if (Config.getInLoopReductions().empty() || VF.isScalar() ||
4423 !isa<VectorType>(Val: Ty))
4424 return std::nullopt;
4425 auto *VectorTy = cast<VectorType>(Val: Ty);
4426
4427 // We are looking for a pattern of, and finding the minimal acceptable cost:
4428 // reduce(mul(ext(A), ext(B))) or
4429 // reduce(mul(A, B)) or
4430 // reduce(ext(A)) or
4431 // reduce(A).
4432 // The basic idea is that we walk down the tree to do that, finding the root
4433 // reduction instruction in InLoopReductionImmediateChains. From there we find
4434 // the pattern of mul/ext and test the cost of the entire pattern vs the cost
4435 // of the components. If the reduction cost is lower then we return it for the
4436 // reduction instruction and 0 for the other instructions in the pattern. If
4437 // it is not we return an invalid cost specifying the orignal cost method
4438 // should be used.
4439 Instruction *RetI = I;
4440 if (match(V: RetI, P: m_ZExtOrSExt(Op: m_Value()))) {
4441 if (!RetI->hasOneUser())
4442 return std::nullopt;
4443 RetI = RetI->user_back();
4444 }
4445
4446 if (match(V: RetI, P: m_OneUse(SubPattern: m_Mul(L: m_Value(), R: m_Value()))) &&
4447 RetI->user_back()->getOpcode() == Instruction::Add) {
4448 RetI = RetI->user_back();
4449 }
4450
4451 // Test if the found instruction is a reduction, and if not return an invalid
4452 // cost specifying the parent to use the original cost modelling.
4453 Instruction *LastChain = Config.getInLoopReductionImmediateChain(I: RetI);
4454 if (!LastChain)
4455 return std::nullopt;
4456
4457 // Find the reduction this chain is a part of and calculate the basic cost of
4458 // the reduction on its own.
4459 Instruction *ReductionPhi = LastChain;
4460 while (!isa<PHINode>(Val: ReductionPhi))
4461 ReductionPhi = Config.getInLoopReductionImmediateChain(I: ReductionPhi);
4462
4463 const RecurrenceDescriptor &RdxDesc =
4464 Legal->getRecurrenceDescriptor(PN: cast<PHINode>(Val: ReductionPhi));
4465
4466 InstructionCost BaseCost;
4467 RecurKind RK = RdxDesc.getRecurrenceKind();
4468 if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind: RK)) {
4469 Intrinsic::ID MinMaxID = getMinMaxReductionIntrinsicOp(RK);
4470 BaseCost = TTI.getMinMaxReductionCost(
4471 IID: MinMaxID, Ty: VectorTy, FMF: RdxDesc.getFastMathFlags(), CostKind: Config.CostKind);
4472 } else {
4473 BaseCost = TTI.getArithmeticReductionCost(Opcode: RdxDesc.getOpcode(), Ty: VectorTy,
4474 FMF: RdxDesc.getFastMathFlags(),
4475 CostKind: Config.CostKind);
4476 }
4477
4478 // For a call to the llvm.fmuladd intrinsic we need to add the cost of a
4479 // normal fmul instruction to the cost of the fadd reduction.
4480 if (RK == RecurKind::FMulAdd)
4481 BaseCost += TTI.getArithmeticInstrCost(Opcode: Instruction::FMul, Ty: VectorTy,
4482 CostKind: Config.CostKind);
4483
4484 // If we're using ordered reductions then we can just return the base cost
4485 // here, since getArithmeticReductionCost calculates the full ordered
4486 // reduction cost when FP reassociation is not allowed.
4487 if (Config.useOrderedReductions(RdxDesc))
4488 return BaseCost;
4489
4490 // Get the operand that was not the reduction chain and match it to one of the
4491 // patterns, returning the better cost if it is found.
4492 Instruction *RedOp = RetI->getOperand(i: 1) == LastChain
4493 ? dyn_cast<Instruction>(Val: RetI->getOperand(i: 0))
4494 : dyn_cast<Instruction>(Val: RetI->getOperand(i: 1));
4495
4496 VectorTy = VectorType::get(ElementType: I->getOperand(i: 0)->getType(), Other: VectorTy);
4497
4498 Instruction *Op0, *Op1;
4499 if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
4500 match(V: RedOp,
4501 P: m_ZExtOrSExt(Op: m_Mul(L: m_Instruction(I&: Op0), R: m_Instruction(I&: Op1)))) &&
4502 match(V: Op0, P: m_ZExtOrSExt(Op: m_Value())) &&
4503 Op0->getOpcode() == Op1->getOpcode() &&
4504 Op0->getOperand(i: 0)->getType() == Op1->getOperand(i: 0)->getType() &&
4505 !TheLoop->isLoopInvariant(V: Op0) && !TheLoop->isLoopInvariant(V: Op1) &&
4506 (Op0->getOpcode() == RedOp->getOpcode() || Op0 == Op1)) {
4507
4508 // Matched reduce.add(ext(mul(ext(A), ext(B)))
4509 // Note that the extend opcodes need to all match, or if A==B they will have
4510 // been converted to zext(mul(sext(A), sext(A))) as it is known positive,
4511 // which is equally fine.
4512 bool IsUnsigned = isa<ZExtInst>(Val: Op0);
4513 auto *ExtType = VectorType::get(ElementType: Op0->getOperand(i: 0)->getType(), Other: VectorTy);
4514 auto *MulType = VectorType::get(ElementType: Op0->getType(), Other: VectorTy);
4515
4516 InstructionCost ExtCost =
4517 TTI.getCastInstrCost(Opcode: Op0->getOpcode(), Dst: MulType, Src: ExtType,
4518 CCH: TTI::CastContextHint::None, CostKind: Config.CostKind, I: Op0);
4519 InstructionCost MulCost =
4520 TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: MulType, CostKind: Config.CostKind);
4521 InstructionCost Ext2Cost = TTI.getCastInstrCost(
4522 Opcode: RedOp->getOpcode(), Dst: VectorTy, Src: MulType, CCH: TTI::CastContextHint::None,
4523 CostKind: Config.CostKind, I: RedOp);
4524
4525 InstructionCost RedCost = TTI.getMulAccReductionCost(
4526 IsUnsigned, RedOpcode: RdxDesc.getOpcode(), ResTy: RdxDesc.getRecurrenceType(), Ty: ExtType,
4527 CostKind: Config.CostKind);
4528
4529 if (RedCost.isValid() &&
4530 RedCost < ExtCost * 2 + MulCost + Ext2Cost + BaseCost)
4531 return I == RetI ? RedCost : 0;
4532 } else if (RedOp && match(V: RedOp, P: m_ZExtOrSExt(Op: m_Value())) &&
4533 !TheLoop->isLoopInvariant(V: RedOp)) {
4534 // Matched reduce(ext(A))
4535 bool IsUnsigned = isa<ZExtInst>(Val: RedOp);
4536 auto *ExtType = VectorType::get(ElementType: RedOp->getOperand(i: 0)->getType(), Other: VectorTy);
4537 InstructionCost RedCost = TTI.getExtendedReductionCost(
4538 Opcode: RdxDesc.getOpcode(), IsUnsigned, ResTy: RdxDesc.getRecurrenceType(), Ty: ExtType,
4539 FMF: RdxDesc.getFastMathFlags(), CostKind: Config.CostKind);
4540
4541 InstructionCost ExtCost = TTI.getCastInstrCost(
4542 Opcode: RedOp->getOpcode(), Dst: VectorTy, Src: ExtType, CCH: TTI::CastContextHint::None,
4543 CostKind: Config.CostKind, I: RedOp);
4544 if (RedCost.isValid() && RedCost < BaseCost + ExtCost)
4545 return I == RetI ? RedCost : 0;
4546 } else if (RedOp && RdxDesc.getOpcode() == Instruction::Add &&
4547 match(V: RedOp, P: m_Mul(L: m_Instruction(I&: Op0), R: m_Instruction(I&: Op1)))) {
4548 if (match(V: Op0, P: m_ZExtOrSExt(Op: m_Value())) &&
4549 Op0->getOpcode() == Op1->getOpcode() &&
4550 !TheLoop->isLoopInvariant(V: Op0) && !TheLoop->isLoopInvariant(V: Op1)) {
4551 bool IsUnsigned = isa<ZExtInst>(Val: Op0);
4552 Type *Op0Ty = Op0->getOperand(i: 0)->getType();
4553 Type *Op1Ty = Op1->getOperand(i: 0)->getType();
4554 Type *LargestOpTy =
4555 Op0Ty->getIntegerBitWidth() < Op1Ty->getIntegerBitWidth() ? Op1Ty
4556 : Op0Ty;
4557 auto *ExtType = VectorType::get(ElementType: LargestOpTy, Other: VectorTy);
4558
4559 // Matched reduce.add(mul(ext(A), ext(B))), where the two ext may be of
4560 // different sizes. We take the largest type as the ext to reduce, and add
4561 // the remaining cost as, for example reduce(mul(ext(ext(A)), ext(B))).
4562 InstructionCost ExtCost0 = TTI.getCastInstrCost(
4563 Opcode: Op0->getOpcode(), Dst: VectorTy, Src: VectorType::get(ElementType: Op0Ty, Other: VectorTy),
4564 CCH: TTI::CastContextHint::None, CostKind: Config.CostKind, I: Op0);
4565 InstructionCost ExtCost1 = TTI.getCastInstrCost(
4566 Opcode: Op1->getOpcode(), Dst: VectorTy, Src: VectorType::get(ElementType: Op1Ty, Other: VectorTy),
4567 CCH: TTI::CastContextHint::None, CostKind: Config.CostKind, I: Op1);
4568 InstructionCost MulCost = TTI.getArithmeticInstrCost(
4569 Opcode: Instruction::Mul, Ty: VectorTy, CostKind: Config.CostKind);
4570
4571 InstructionCost RedCost = TTI.getMulAccReductionCost(
4572 IsUnsigned, RedOpcode: RdxDesc.getOpcode(), ResTy: RdxDesc.getRecurrenceType(), Ty: ExtType,
4573 CostKind: Config.CostKind);
4574 InstructionCost ExtraExtCost = 0;
4575 if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
4576 Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
4577 ExtraExtCost = TTI.getCastInstrCost(
4578 Opcode: ExtraExtOp->getOpcode(), Dst: ExtType,
4579 Src: VectorType::get(ElementType: ExtraExtOp->getOperand(i: 0)->getType(), Other: VectorTy),
4580 CCH: TTI::CastContextHint::None, CostKind: Config.CostKind, I: ExtraExtOp);
4581 }
4582
4583 if (RedCost.isValid() &&
4584 (RedCost + ExtraExtCost) < (ExtCost0 + ExtCost1 + MulCost + BaseCost))
4585 return I == RetI ? RedCost : 0;
4586 } else if (!match(V: I, P: m_ZExtOrSExt(Op: m_Value()))) {
4587 // Matched reduce.add(mul())
4588 InstructionCost MulCost = TTI.getArithmeticInstrCost(
4589 Opcode: Instruction::Mul, Ty: VectorTy, CostKind: Config.CostKind);
4590
4591 InstructionCost RedCost = TTI.getMulAccReductionCost(
4592 IsUnsigned: true, RedOpcode: RdxDesc.getOpcode(), ResTy: RdxDesc.getRecurrenceType(), Ty: VectorTy,
4593 CostKind: Config.CostKind);
4594
4595 if (RedCost.isValid() && RedCost < MulCost + BaseCost)
4596 return I == RetI ? RedCost : 0;
4597 }
4598 }
4599
4600 return I == RetI ? std::optional<InstructionCost>(BaseCost) : std::nullopt;
4601}
4602
4603InstructionCost
4604LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
4605 ElementCount VF) {
4606 // Calculate scalar cost only. Vectorization cost should be ready at this
4607 // moment.
4608 if (VF.isScalar()) {
4609 Type *ValTy = getLoadStoreType(I);
4610 Type *PtrTy = getLoadStorePointerOperand(V: I)->getType();
4611 const Align Alignment = getLoadStoreAlignment(I);
4612 unsigned AS = getLoadStoreAddressSpace(I);
4613
4614 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4615 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4616 CostKind: Config.CostKind) +
4617 TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: ValTy, Alignment, AddressSpace: AS,
4618 CostKind: Config.CostKind, OpdInfo: OpInfo, I);
4619 }
4620 return getWideningCost(I, VF);
4621}
4622
4623InstructionCost
4624LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
4625 ElementCount VF) const {
4626
4627 // There is no mechanism yet to create a scalable scalarization loop,
4628 // so this is currently Invalid.
4629 if (VF.isScalable())
4630 return InstructionCost::getInvalid();
4631
4632 if (VF.isScalar())
4633 return 0;
4634
4635 InstructionCost Cost = 0;
4636 Type *RetTy = toVectorizedTy(Ty: I->getType(), EC: VF);
4637 if (!RetTy->isVoidTy() &&
4638 (!isa<LoadInst>(Val: I) || !TTI.supportsEfficientVectorElementLoadStore())) {
4639
4640 TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None;
4641 if (isa<LoadInst>(Val: I))
4642 VIC = TTI::VectorInstrContext::Load;
4643 else if (isa<StoreInst>(Val: I))
4644 VIC = TTI::VectorInstrContext::Store;
4645
4646 for (Type *VectorTy : getContainedTypes(Ty: RetTy)) {
4647 Cost += TTI.getScalarizationOverhead(
4648 Ty: cast<VectorType>(Val: VectorTy), DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4649 /*Insert=*/true, /*Extract=*/false, CostKind: Config.CostKind,
4650 /*ForPoisonSrc=*/true, VL: {}, VIC);
4651 }
4652 }
4653
4654 // Some targets keep addresses scalar.
4655 if (isa<LoadInst>(Val: I) && !TTI.prefersVectorizedAddressing())
4656 return Cost;
4657
4658 // Some targets support efficient element stores.
4659 if (isa<StoreInst>(Val: I) && TTI.supportsEfficientVectorElementLoadStore())
4660 return Cost;
4661
4662 // Collect operands to consider.
4663 CallInst *CI = dyn_cast<CallInst>(Val: I);
4664 Instruction::op_range Ops = CI ? CI->args() : I->operands();
4665
4666 // Skip operands that do not require extraction/scalarization and do not incur
4667 // any overhead.
4668 SmallVector<Type *> Tys;
4669 for (auto *V : filterExtractingOperands(Ops, VF))
4670 Tys.push_back(Elt: maybeVectorizeType(Ty: V->getType(), VF));
4671
4672 TTI::VectorInstrContext OperandVIC = isa<StoreInst>(Val: I)
4673 ? TTI::VectorInstrContext::Store
4674 : TTI::VectorInstrContext::None;
4675 return Cost +
4676 TTI.getOperandsScalarizationOverhead(Tys, CostKind: Config.CostKind, VIC: OperandVIC);
4677}
4678
4679void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
4680 if (VF.isScalar())
4681 return;
4682
4683 // TODO: We should generate better code and update the cost model for
4684 // predicated uniform stores. Today they are treated as any other
4685 // predicated store (see added test cases in
4686 // invariant-store-vectorization.ll).
4687 NumPredStores = 0;
4688 for (BasicBlock *BB : TheLoop->blocks())
4689 for (Instruction &I : *BB)
4690 if (isa<StoreInst>(Val: &I) && isScalarWithPredication(I: &I, VF))
4691 ++NumPredStores;
4692
4693 for (BasicBlock *BB : TheLoop->blocks()) {
4694 // For each instruction in the old loop.
4695 for (Instruction &I : *BB) {
4696 Value *Ptr = getLoadStorePointerOperand(V: &I);
4697 if (!Ptr)
4698 continue;
4699
4700 if (isUniformMemOp(I, VF)) {
4701 auto IsLegalToScalarize = [&]() {
4702 if (!VF.isScalable())
4703 // Scalarization of fixed length vectors "just works".
4704 return true;
4705
4706 // We have dedicated lowering for unpredicated uniform loads and
4707 // stores. Note that even with tail folding we know that at least
4708 // one lane is active (i.e. generalized predication is not possible
4709 // here), and the logic below depends on this fact.
4710 if (!foldTailByMasking())
4711 return true;
4712
4713 // For scalable vectors, a uniform memop load is always
4714 // uniform-by-parts and we know how to scalarize that.
4715 if (isa<LoadInst>(Val: I))
4716 return true;
4717
4718 // A uniform store isn't neccessarily uniform-by-part
4719 // and we can't assume scalarization.
4720 auto &SI = cast<StoreInst>(Val&: I);
4721 return TheLoop->isLoopInvariant(V: SI.getValueOperand());
4722 };
4723
4724 const InstructionCost GatherScatterCost =
4725 Config.isLegalGatherOrScatter(V: &I, VF)
4726 ? getGatherScatterCost(I: &I, VF)
4727 : InstructionCost::getInvalid();
4728
4729 // Load: Scalar load + broadcast
4730 // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
4731 // FIXME: This cost is a significant under-estimate for tail folded
4732 // memory ops.
4733 const InstructionCost ScalarizationCost =
4734 IsLegalToScalarize() ? getUniformMemOpCost(I: &I, VF)
4735 : InstructionCost::getInvalid();
4736
4737 // Choose better solution for the current VF, Note that Invalid
4738 // costs compare as maximumal large. If both are invalid, we get
4739 // scalable invalid which signals a failure and a vectorization abort.
4740 if (GatherScatterCost < ScalarizationCost)
4741 setWideningDecision(I: &I, VF, W: CM_GatherScatter, Cost: GatherScatterCost);
4742 else
4743 setWideningDecision(I: &I, VF, W: CM_Scalarize, Cost: ScalarizationCost);
4744 continue;
4745 }
4746
4747 // We assume that widening is the best solution when possible.
4748 if (std::optional<InstWidening> Decision =
4749 memoryInstructionCanBeWidened(I: &I, VF)) {
4750 setWideningDecision(I: &I, VF, W: *Decision,
4751 Cost: getConsecutiveMemOpCost(I: &I, VF, Kind: *Decision));
4752 continue;
4753 }
4754
4755 // Choose between Interleaving, Gather/Scatter or Scalarization.
4756 InstructionCost InterleaveCost = InstructionCost::getInvalid();
4757 unsigned NumAccesses = 1;
4758 if (isAccessInterleaved(Instr: &I)) {
4759 const auto *Group = getInterleavedAccessGroup(Instr: &I);
4760 assert(Group && "Fail to get an interleaved access group.");
4761
4762 // Make one decision for the whole group.
4763 if (getWideningDecision(I: &I, VF) != CM_Unknown)
4764 continue;
4765
4766 NumAccesses = Group->getNumMembers();
4767 if (interleavedAccessCanBeWidened(I: &I, VF))
4768 InterleaveCost = getInterleaveGroupCost(I: &I, VF);
4769 }
4770
4771 InstructionCost GatherScatterCost =
4772 Config.isLegalGatherOrScatter(V: &I, VF)
4773 ? getGatherScatterCost(I: &I, VF) * NumAccesses
4774 : InstructionCost::getInvalid();
4775
4776 InstructionCost ScalarizationCost =
4777 getMemInstScalarizationCost(I: &I, VF) * NumAccesses;
4778
4779 // Choose better solution for the current VF,
4780 // write down this decision and use it during vectorization.
4781 InstructionCost Cost;
4782 InstWidening Decision;
4783 if (InterleaveCost <= GatherScatterCost &&
4784 InterleaveCost < ScalarizationCost) {
4785 Decision = CM_Interleave;
4786 Cost = InterleaveCost;
4787 } else if (GatherScatterCost < ScalarizationCost) {
4788 Decision = CM_GatherScatter;
4789 Cost = GatherScatterCost;
4790 } else {
4791 Decision = CM_Scalarize;
4792 Cost = ScalarizationCost;
4793 }
4794 // If the instructions belongs to an interleave group, the whole group
4795 // receives the same decision. The whole group receives the cost, but
4796 // the cost will actually be assigned to one instruction.
4797 if (const auto *Group = getInterleavedAccessGroup(Instr: &I)) {
4798 if (Decision == CM_Scalarize) {
4799 for (Instruction *I : Group->members())
4800 setWideningDecision(I, VF, W: Decision,
4801 Cost: getMemInstScalarizationCost(I, VF));
4802 } else {
4803 setWideningDecision(Grp: Group, VF, W: Decision, Cost);
4804 }
4805 } else
4806 setWideningDecision(I: &I, VF, W: Decision, Cost);
4807 }
4808 }
4809
4810 // Make sure that any load of address and any other address computation
4811 // remains scalar unless there is gather/scatter support. This avoids
4812 // inevitable extracts into address registers, and also has the benefit of
4813 // activating LSR more, since that pass can't optimize vectorized
4814 // addresses.
4815 if (TTI.prefersVectorizedAddressing())
4816 return;
4817
4818 // Start with all scalar pointer uses.
4819 SmallSetVector<Instruction *, 8> AddrDefs;
4820 for (BasicBlock *BB : TheLoop->blocks())
4821 for (Instruction &I : *BB) {
4822 Instruction *PtrDef =
4823 dyn_cast_or_null<Instruction>(Val: getLoadStorePointerOperand(V: &I));
4824 if (PtrDef && TheLoop->contains(Inst: PtrDef) &&
4825 getWideningDecision(I: &I, VF) != CM_GatherScatter)
4826 AddrDefs.insert(X: PtrDef);
4827 }
4828
4829 // Add all instructions used to generate the addresses.
4830 SmallVector<Instruction *, 4> Worklist;
4831 append_range(C&: Worklist, R&: AddrDefs);
4832 while (!Worklist.empty()) {
4833 Instruction *I = Worklist.pop_back_val();
4834 for (auto &Op : I->operands())
4835 if (auto *InstOp = dyn_cast<Instruction>(Val&: Op))
4836 if (TheLoop->contains(Inst: InstOp) && !isa<PHINode>(Val: InstOp) &&
4837 AddrDefs.insert(X: InstOp))
4838 Worklist.push_back(Elt: InstOp);
4839 }
4840
4841 auto UpdateMemOpUserCost = [this, VF](LoadInst *LI) {
4842 // If there are direct memory op users of the newly scalarized load,
4843 // their cost may have changed because there's no scalarization
4844 // overhead for the operand. Update it.
4845 for (User *U : LI->users()) {
4846 if (!isa<LoadInst, StoreInst>(Val: U))
4847 continue;
4848 if (getWideningDecision(I: cast<Instruction>(Val: U), VF) != CM_Scalarize)
4849 continue;
4850 setWideningDecision(
4851 I: cast<Instruction>(Val: U), VF, W: CM_Scalarize,
4852 Cost: getMemInstScalarizationCost(I: cast<Instruction>(Val: U), VF));
4853 }
4854 };
4855 for (auto *I : AddrDefs) {
4856 if (isa<LoadInst>(Val: I)) {
4857 // Setting the desired widening decision should ideally be handled in
4858 // by cost functions, but since this involves the task of finding out
4859 // if the loaded register is involved in an address computation, it is
4860 // instead changed here when we know this is the case.
4861 InstWidening Decision = getWideningDecision(I, VF);
4862 if (!isPredicatedInst(I) &&
4863 (Decision == CM_Widen || Decision == CM_Widen_Reverse ||
4864 (!isUniformMemOp(I&: *I, VF) && Decision == CM_Scalarize))) {
4865 // Scalarize a widened load of address or update the cost of a scalar
4866 // load of an address.
4867 setWideningDecision(
4868 I, VF, W: CM_Scalarize,
4869 Cost: (VF.getKnownMinValue() *
4870 getMemoryInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1))));
4871 UpdateMemOpUserCost(cast<LoadInst>(Val: I));
4872 } else if (const auto *Group = getInterleavedAccessGroup(Instr: I)) {
4873 // Scalarize all members of this interleaved group when any member
4874 // is used as an address. The address-used load skips scalarization
4875 // overhead, other members include it.
4876 for (Instruction *Member : Group->members()) {
4877 InstructionCost Cost = AddrDefs.contains(key: Member)
4878 ? (VF.getKnownMinValue() *
4879 getMemoryInstructionCost(
4880 I: Member, VF: ElementCount::getFixed(MinVal: 1)))
4881 : getMemInstScalarizationCost(I: Member, VF);
4882 setWideningDecision(I: Member, VF, W: CM_Scalarize, Cost);
4883 UpdateMemOpUserCost(cast<LoadInst>(Val: Member));
4884 }
4885 }
4886 } else {
4887 // Cannot scalarize fixed-order recurrence phis at the moment.
4888 if (isa<PHINode>(Val: I) && Legal->isFixedOrderRecurrence(Phi: cast<PHINode>(Val: I)))
4889 continue;
4890
4891 // Make sure I gets scalarized and a cost estimate without
4892 // scalarization overhead.
4893 ForcedScalars[VF].insert(Ptr: I);
4894 }
4895 }
4896}
4897
4898bool LoopVectorizationCostModel::shouldConsiderInvariant(Value *Op) {
4899 if (!Legal->isInvariant(V: Op))
4900 return false;
4901 // Consider Op invariant, if it or its operands aren't predicated
4902 // instruction in the loop. In that case, it is not trivially hoistable.
4903 auto *OpI = dyn_cast<Instruction>(Val: Op);
4904 return !OpI || !TheLoop->contains(Inst: OpI) ||
4905 (!isPredicatedInst(I: OpI) &&
4906 (!isa<PHINode>(Val: OpI) || OpI->getParent() != TheLoop->getHeader()) &&
4907 all_of(Range: OpI->operands(),
4908 P: [this](Value *Op) { return shouldConsiderInvariant(Op); }));
4909}
4910
4911InstructionCost
4912LoopVectorizationCostModel::getInstructionCost(Instruction *I,
4913 ElementCount VF) {
4914 // If we know that this instruction will remain uniform, check the cost of
4915 // the scalar version.
4916 if (isUniformAfterVectorization(I, VF))
4917 VF = ElementCount::getFixed(MinVal: 1);
4918
4919 if (VF.isVector() && isProfitableToScalarize(I, VF))
4920 return InstsToScalarize[VF][I];
4921
4922 // Forced scalars do not have any scalarization overhead.
4923 auto ForcedScalar = ForcedScalars.find(Val: VF);
4924 if (VF.isVector() && ForcedScalar != ForcedScalars.end()) {
4925 auto InstSet = ForcedScalar->second;
4926 if (InstSet.count(Ptr: I))
4927 return getInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1)) *
4928 VF.getKnownMinValue();
4929 }
4930
4931 const auto &MinBWs = Config.getMinimalBitwidths();
4932 uint64_t InstrMinBWs = MinBWs.lookup(Key: I);
4933 Type *RetTy = I->getType();
4934 if (canTruncateToMinimalBitwidth(I, VF))
4935 RetTy = IntegerType::get(C&: RetTy->getContext(), NumBits: InstrMinBWs);
4936 auto *SE = PSE.getSE();
4937
4938 Type *VectorTy;
4939 if (isScalarAfterVectorization(I, VF)) {
4940 [[maybe_unused]] auto HasSingleCopyAfterVectorization =
4941 [this](Instruction *I, ElementCount VF) -> bool {
4942 if (VF.isScalar())
4943 return true;
4944
4945 auto Scalarized = InstsToScalarize.find(Key: VF);
4946 assert(Scalarized != InstsToScalarize.end() &&
4947 "VF not yet analyzed for scalarization profitability");
4948 return !Scalarized->second.count(Key: I) &&
4949 llvm::all_of(Range: I->users(), P: [&](User *U) {
4950 auto *UI = cast<Instruction>(Val: U);
4951 return !Scalarized->second.count(Key: UI);
4952 });
4953 };
4954
4955 // With the exception of GEPs and PHIs, after scalarization there should
4956 // only be one copy of the instruction generated in the loop. This is
4957 // because the VF is either 1, or any instructions that need scalarizing
4958 // have already been dealt with by the time we get here. As a result,
4959 // it means we don't have to multiply the instruction cost by VF.
4960 assert(I->getOpcode() == Instruction::GetElementPtr ||
4961 I->getOpcode() == Instruction::PHI ||
4962 (I->getOpcode() == Instruction::BitCast &&
4963 I->getType()->isPointerTy()) ||
4964 HasSingleCopyAfterVectorization(I, VF));
4965 VectorTy = RetTy;
4966 } else
4967 VectorTy = toVectorizedTy(Ty: RetTy, EC: VF);
4968
4969 if (VF.isVector() && VectorTy->isVectorTy() &&
4970 !TTI.getNumberOfParts(Tp: VectorTy))
4971 return InstructionCost::getInvalid();
4972
4973 // TODO: We need to estimate the cost of intrinsic calls.
4974 switch (I->getOpcode()) {
4975 case Instruction::GetElementPtr:
4976 // We mark this instruction as zero-cost because the cost of GEPs in
4977 // vectorized code depends on whether the corresponding memory instruction
4978 // is scalarized or not. Therefore, we handle GEPs with the memory
4979 // instruction cost.
4980 return 0;
4981 case Instruction::UncondBr:
4982 case Instruction::CondBr: {
4983 // In cases of scalarized and predicated instructions, there will be VF
4984 // predicated blocks in the vectorized loop. Each branch around these
4985 // blocks requires also an extract of its vector compare i1 element.
4986 // Note that the conditional branch from the loop latch will be replaced by
4987 // a single branch controlling the loop, so there is no extra overhead from
4988 // scalarization.
4989 bool ScalarPredicatedBB = false;
4990 CondBrInst *BI = dyn_cast<CondBrInst>(Val: I);
4991 if (VF.isVector() && BI &&
4992 (PredicatedBBsAfterVectorization[VF].count(Ptr: BI->getSuccessor(i: 0)) ||
4993 PredicatedBBsAfterVectorization[VF].count(Ptr: BI->getSuccessor(i: 1))) &&
4994 BI->getParent() != TheLoop->getLoopLatch())
4995 ScalarPredicatedBB = true;
4996
4997 if (ScalarPredicatedBB) {
4998 // Not possible to scalarize scalable vector with predicated instructions.
4999 if (VF.isScalable())
5000 return InstructionCost::getInvalid();
5001 // Return cost for branches around scalarized and predicated blocks.
5002 auto *VecI1Ty =
5003 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: RetTy->getContext()), EC: VF);
5004 return (TTI.getScalarizationOverhead(
5005 Ty: VecI1Ty, DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
5006 /*Insert*/ false, /*Extract*/ true, CostKind: Config.CostKind) +
5007 (TTI.getCFInstrCost(Opcode: Instruction::CondBr, CostKind: Config.CostKind) *
5008 VF.getFixedValue()));
5009 }
5010
5011 if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar())
5012 // The back-edge branch will remain, as will all scalar branches.
5013 return TTI.getCFInstrCost(Opcode: Instruction::UncondBr, CostKind: Config.CostKind);
5014
5015 // This branch will be eliminated by if-conversion.
5016 return 0;
5017 // Note: We currently assume zero cost for an unconditional branch inside
5018 // a predicated block since it will become a fall-through, although we
5019 // may decide in the future to call TTI for all branches.
5020 }
5021 case Instruction::Switch: {
5022 if (VF.isScalar())
5023 return TTI.getCFInstrCost(Opcode: Instruction::Switch, CostKind: Config.CostKind);
5024 auto *Switch = cast<SwitchInst>(Val: I);
5025 return Switch->getNumCases() *
5026 TTI.getCmpSelInstrCost(
5027 Opcode: Instruction::ICmp,
5028 ValTy: toVectorTy(Scalar: Switch->getCondition()->getType(), EC: VF),
5029 CondTy: toVectorTy(Scalar: Type::getInt1Ty(C&: I->getContext()), EC: VF),
5030 VecPred: CmpInst::ICMP_EQ, CostKind: Config.CostKind);
5031 }
5032 case Instruction::PHI: {
5033 auto *Phi = cast<PHINode>(Val: I);
5034
5035 // First-order recurrences are replaced by vector shuffles inside the loop.
5036 if (VF.isVector() && Legal->isFixedOrderRecurrence(Phi)) {
5037 return TTI.getShuffleCost(
5038 Kind: TargetTransformInfo::SK_Splice, DstTy: cast<VectorType>(Val: VectorTy),
5039 SrcTy: cast<VectorType>(Val: VectorTy), Mask: {}, CostKind: Config.CostKind, Index: -1);
5040 }
5041
5042 // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
5043 // converted into select instructions. We require N - 1 selects per phi
5044 // node, where N is the number of incoming values.
5045 if (VF.isVector() && Phi->getParent() != TheLoop->getHeader()) {
5046 Type *ResultTy = Phi->getType();
5047
5048 // All instructions in an Any-of reduction chain are narrowed to bool.
5049 // Check if that is the case for this phi node.
5050 auto *HeaderUser = cast_if_present<PHINode>(
5051 Val: find_singleton<User>(Range: Phi->users(), P: [this](User *U, bool) -> User * {
5052 auto *Phi = dyn_cast<PHINode>(Val: U);
5053 if (Phi && Phi->getParent() == TheLoop->getHeader())
5054 return Phi;
5055 return nullptr;
5056 }));
5057 if (HeaderUser) {
5058 auto &ReductionVars = Legal->getReductionVars();
5059 auto Iter = ReductionVars.find(Key: HeaderUser);
5060 if (Iter != ReductionVars.end() &&
5061 RecurrenceDescriptor::isAnyOfRecurrenceKind(
5062 Kind: Iter->second.getRecurrenceKind()))
5063 ResultTy = Type::getInt1Ty(C&: Phi->getContext());
5064 }
5065 return (Phi->getNumIncomingValues() - 1) *
5066 TTI.getCmpSelInstrCost(
5067 Opcode: Instruction::Select, ValTy: toVectorTy(Scalar: ResultTy, EC: VF),
5068 CondTy: toVectorTy(Scalar: Type::getInt1Ty(C&: Phi->getContext()), EC: VF),
5069 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind: Config.CostKind);
5070 }
5071
5072 // When tail folding with EVL, if the phi is part of an out of loop
5073 // reduction then it will be transformed into a wide vp_merge.
5074 if (VF.isVector() && foldTailWithEVL() &&
5075 Legal->getReductionVars().contains(Key: Phi) &&
5076 !Config.isInLoopReduction(Phi)) {
5077 IntrinsicCostAttributes ICA(
5078 Intrinsic::vp_merge, toVectorTy(Scalar: Phi->getType(), EC: VF),
5079 {toVectorTy(Scalar: Type::getInt1Ty(C&: Phi->getContext()), EC: VF)});
5080 return TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind);
5081 }
5082
5083 return TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
5084 }
5085 case Instruction::UDiv:
5086 case Instruction::SDiv:
5087 case Instruction::URem:
5088 case Instruction::SRem:
5089 if (VF.isVector() && isPredicatedInst(I)) {
5090 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
5091 return isDivRemScalarWithPredication(ScalarCost, MaskedCost) ? ScalarCost
5092 : MaskedCost;
5093 }
5094 // We've proven all lanes safe to speculate, fall through.
5095 [[fallthrough]];
5096 case Instruction::Add:
5097 case Instruction::Sub: {
5098 auto Info = Legal->getHistogramInfo(I);
5099 if (Info && VF.isVector()) {
5100 const HistogramInfo *HGram = Info.value();
5101 // Assume that a non-constant update value (or a constant != 1) requires
5102 // a multiply, and add that into the cost.
5103 InstructionCost MulCost = TTI::TCC_Free;
5104 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: I->getOperand(i: 1));
5105 if (!RHS || RHS->getZExtValue() != 1)
5106 MulCost = TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: VectorTy,
5107 CostKind: Config.CostKind);
5108
5109 // Find the cost of the histogram operation itself.
5110 Type *PtrTy = VectorType::get(ElementType: HGram->Load->getPointerOperandType(), EC: VF);
5111 Type *ScalarTy = I->getType();
5112 Type *MaskTy = VectorType::get(ElementType: Type::getInt1Ty(C&: I->getContext()), EC: VF);
5113 IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
5114 Type::getVoidTy(C&: I->getContext()),
5115 {PtrTy, ScalarTy, MaskTy});
5116
5117 // Add the costs together with the add/sub operation.
5118 return TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind) + MulCost +
5119 TTI.getArithmeticInstrCost(Opcode: I->getOpcode(), Ty: VectorTy,
5120 CostKind: Config.CostKind);
5121 }
5122 [[fallthrough]];
5123 }
5124 case Instruction::FAdd:
5125 case Instruction::FSub:
5126 case Instruction::Mul:
5127 case Instruction::FMul:
5128 case Instruction::FDiv:
5129 case Instruction::FRem:
5130 case Instruction::Shl:
5131 case Instruction::LShr:
5132 case Instruction::AShr:
5133 case Instruction::And:
5134 case Instruction::Or:
5135 case Instruction::Xor: {
5136 // If we're speculating on the stride being 1, the multiplication may
5137 // fold away. We can generalize this for all operations using the notion
5138 // of neutral elements. (TODO)
5139 if (I->getOpcode() == Instruction::Mul &&
5140 ((TheLoop->isLoopInvariant(V: I->getOperand(i: 0)) &&
5141 PSE.getSCEV(V: I->getOperand(i: 0))->isOne()) ||
5142 (TheLoop->isLoopInvariant(V: I->getOperand(i: 1)) &&
5143 PSE.getSCEV(V: I->getOperand(i: 1))->isOne())))
5144 return 0;
5145
5146 // Detect reduction patterns
5147 if (auto RedCost = getReductionPatternCost(I, VF, Ty: VectorTy))
5148 return *RedCost;
5149
5150 // Certain instructions can be cheaper to vectorize if they have a constant
5151 // second vector operand. One example of this are shifts on x86.
5152 Value *Op2 = I->getOperand(i: 1);
5153 if (!isa<Constant>(Val: Op2) && TheLoop->isLoopInvariant(V: Op2) &&
5154 PSE.getSE()->isSCEVable(Ty: Op2->getType()) &&
5155 isa<SCEVConstant>(Val: PSE.getSCEV(V: Op2))) {
5156 Op2 = cast<SCEVConstant>(Val: PSE.getSCEV(V: Op2))->getValue();
5157 }
5158 auto Op2Info = TTI.getOperandInfo(V: Op2);
5159 if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue &&
5160 shouldConsiderInvariant(Op: Op2))
5161 Op2Info.Kind = TargetTransformInfo::OK_UniformValue;
5162
5163 SmallVector<const Value *, 4> Operands(I->operand_values());
5164 return TTI.getArithmeticInstrCost(
5165 Opcode: I->getOpcode(), Ty: VectorTy, CostKind: Config.CostKind,
5166 Opd1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
5167 Opd2Info: Op2Info, Args: Operands, CxtI: I, TLibInfo: TLI);
5168 }
5169 case Instruction::FNeg: {
5170 return TTI.getArithmeticInstrCost(
5171 Opcode: I->getOpcode(), Ty: VectorTy, CostKind: Config.CostKind,
5172 Opd1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
5173 Opd2Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
5174 Args: I->getOperand(i: 0), CxtI: I);
5175 }
5176 case Instruction::Select: {
5177 SelectInst *SI = cast<SelectInst>(Val: I);
5178 const SCEV *CondSCEV = SE->getSCEV(V: SI->getCondition());
5179 bool ScalarCond = (SE->isLoopInvariant(S: CondSCEV, L: TheLoop));
5180
5181 const Value *Op0, *Op1;
5182 using namespace llvm::PatternMatch;
5183 if (!ScalarCond && (match(V: I, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))) ||
5184 match(V: I, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))) {
5185 // select x, y, false --> x & y
5186 // select x, true, y --> x | y
5187 const auto [Op1VK, Op1VP] = TTI::getOperandInfo(V: Op0);
5188 const auto [Op2VK, Op2VP] = TTI::getOperandInfo(V: Op1);
5189 assert(Op0->getType()->getScalarSizeInBits() == 1 &&
5190 Op1->getType()->getScalarSizeInBits() == 1);
5191
5192 return TTI.getArithmeticInstrCost(
5193 Opcode: match(V: I, P: m_LogicalOr()) ? Instruction::Or : Instruction::And,
5194 Ty: VectorTy, CostKind: Config.CostKind, Opd1Info: {.Kind: Op1VK, .Properties: Op1VP}, Opd2Info: {.Kind: Op2VK, .Properties: Op2VP}, Args: {Op0, Op1},
5195 CxtI: I);
5196 }
5197
5198 Type *CondTy = SI->getCondition()->getType();
5199 if (!ScalarCond)
5200 CondTy = VectorType::get(ElementType: CondTy, EC: VF);
5201
5202 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
5203 if (auto *Cmp = dyn_cast<CmpInst>(Val: SI->getCondition()))
5204 Pred = Cmp->getPredicate();
5205 return TTI.getCmpSelInstrCost(
5206 Opcode: I->getOpcode(), ValTy: VectorTy, CondTy, VecPred: Pred, CostKind: Config.CostKind,
5207 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
5208 }
5209 case Instruction::ICmp:
5210 case Instruction::FCmp: {
5211 Type *ValTy = I->getOperand(i: 0)->getType();
5212
5213 if (canTruncateToMinimalBitwidth(I, VF)) {
5214 [[maybe_unused]] Instruction *Op0AsInstruction =
5215 dyn_cast<Instruction>(Val: I->getOperand(i: 0));
5216 assert((!canTruncateToMinimalBitwidth(Op0AsInstruction, VF) ||
5217 InstrMinBWs == MinBWs.lookup(Op0AsInstruction)) &&
5218 "if both the operand and the compare are marked for "
5219 "truncation, they must have the same bitwidth");
5220 ValTy = IntegerType::get(C&: ValTy->getContext(), NumBits: InstrMinBWs);
5221 }
5222
5223 VectorTy = toVectorTy(Scalar: ValTy, EC: VF);
5224 return TTI.getCmpSelInstrCost(
5225 Opcode: I->getOpcode(), ValTy: VectorTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VectorTy),
5226 VecPred: cast<CmpInst>(Val: I)->getPredicate(), CostKind: Config.CostKind,
5227 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
5228 }
5229 case Instruction::Store:
5230 case Instruction::Load: {
5231 ElementCount Width = VF;
5232 if (Width.isVector()) {
5233 InstWidening Decision = getWideningDecision(I, VF: Width);
5234 assert(Decision != CM_Unknown &&
5235 "CM decision should be taken at this point");
5236 if (getWideningCost(I, VF) == InstructionCost::getInvalid())
5237 return InstructionCost::getInvalid();
5238 if (Decision == CM_Scalarize)
5239 Width = ElementCount::getFixed(MinVal: 1);
5240 }
5241 VectorTy = toVectorTy(Scalar: getLoadStoreType(I), EC: Width);
5242 return getMemoryInstructionCost(I, VF);
5243 }
5244 case Instruction::BitCast:
5245 if (I->getType()->isPointerTy())
5246 return 0;
5247 [[fallthrough]];
5248 case Instruction::ZExt:
5249 case Instruction::SExt:
5250 case Instruction::FPToUI:
5251 case Instruction::FPToSI:
5252 case Instruction::FPExt:
5253 case Instruction::PtrToInt:
5254 case Instruction::IntToPtr:
5255 case Instruction::SIToFP:
5256 case Instruction::UIToFP:
5257 case Instruction::Trunc:
5258 case Instruction::FPTrunc: {
5259 // Computes the CastContextHint from a Load/Store instruction.
5260 auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
5261 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
5262 "Expected a load or a store!");
5263
5264 if (VF.isScalar() || !TheLoop->contains(Inst: I))
5265 return TTI::CastContextHint::Normal;
5266
5267 switch (getWideningDecision(I, VF)) {
5268 case LoopVectorizationCostModel::CM_GatherScatter:
5269 return TTI::CastContextHint::GatherScatter;
5270 case LoopVectorizationCostModel::CM_Interleave:
5271 return TTI::CastContextHint::Interleave;
5272 case LoopVectorizationCostModel::CM_Scalarize:
5273 case LoopVectorizationCostModel::CM_Widen:
5274 return isPredicatedInst(I) ? TTI::CastContextHint::Masked
5275 : TTI::CastContextHint::Normal;
5276 case LoopVectorizationCostModel::CM_Widen_Reverse:
5277 return TTI::CastContextHint::Reversed;
5278 case LoopVectorizationCostModel::CM_Unknown:
5279 llvm_unreachable("Instr did not go through cost modelling?");
5280 case LoopVectorizationCostModel::CM_InvalidatedDecision:
5281 return TTI::CastContextHint::None;
5282 }
5283
5284 llvm_unreachable("Unhandled case!");
5285 };
5286
5287 unsigned Opcode = I->getOpcode();
5288 TTI::CastContextHint CCH = TTI::CastContextHint::None;
5289 // For Trunc, the context is the only user, which must be a StoreInst.
5290 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
5291 if (I->hasOneUse())
5292 if (StoreInst *Store = dyn_cast<StoreInst>(Val: *I->user_begin()))
5293 CCH = ComputeCCH(Store);
5294 }
5295 // For Z/Sext, the context is the operand, which must be a LoadInst.
5296 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
5297 Opcode == Instruction::FPExt) {
5298 if (LoadInst *Load = dyn_cast<LoadInst>(Val: I->getOperand(i: 0)))
5299 CCH = ComputeCCH(Load);
5300 }
5301
5302 // We optimize the truncation of induction variables having constant
5303 // integer steps. The cost of these truncations is the same as the scalar
5304 // operation.
5305 if (isOptimizableIVTruncate(I, VF)) {
5306 auto *Trunc = cast<TruncInst>(Val: I);
5307 return TTI.getCastInstrCost(Opcode: Instruction::Trunc, Dst: Trunc->getDestTy(),
5308 Src: Trunc->getSrcTy(), CCH, CostKind: Config.CostKind,
5309 I: Trunc);
5310 }
5311
5312 // Detect reduction patterns
5313 if (auto RedCost = getReductionPatternCost(I, VF, Ty: VectorTy))
5314 return *RedCost;
5315
5316 Type *SrcScalarTy = I->getOperand(i: 0)->getType();
5317 Instruction *Op0AsInstruction = dyn_cast<Instruction>(Val: I->getOperand(i: 0));
5318 if (canTruncateToMinimalBitwidth(I: Op0AsInstruction, VF))
5319 SrcScalarTy = IntegerType::get(C&: SrcScalarTy->getContext(),
5320 NumBits: MinBWs.lookup(Key: Op0AsInstruction));
5321 Type *SrcVecTy =
5322 VectorTy->isVectorTy() ? toVectorTy(Scalar: SrcScalarTy, EC: VF) : SrcScalarTy;
5323
5324 if (canTruncateToMinimalBitwidth(I, VF)) {
5325 // If the result type is <= the source type, there will be no extend
5326 // after truncating the users to the minimal required bitwidth.
5327 if (VectorTy->getScalarSizeInBits() <= SrcVecTy->getScalarSizeInBits() &&
5328 (I->getOpcode() == Instruction::ZExt ||
5329 I->getOpcode() == Instruction::SExt))
5330 return 0;
5331 }
5332
5333 return TTI.getCastInstrCost(Opcode, Dst: VectorTy, Src: SrcVecTy, CCH,
5334 CostKind: Config.CostKind, I);
5335 }
5336 case Instruction::Call:
5337 return getVectorCallCost(CI: cast<CallInst>(Val: I), VF);
5338 case Instruction::ExtractValue:
5339 return TTI.getInstructionCost(U: I, CostKind: Config.CostKind);
5340 case Instruction::Alloca:
5341 // We cannot easily widen alloca to a scalable alloca, as
5342 // the result would need to be a vector of pointers.
5343 if (VF.isScalable())
5344 return InstructionCost::getInvalid();
5345 return TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: RetTy, CostKind: Config.CostKind);
5346 case Instruction::Freeze:
5347 return TTI::TCC_Free;
5348 default:
5349 // This opcode is unknown. Assume that it is the same as 'mul'.
5350 return TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: VectorTy,
5351 CostKind: Config.CostKind);
5352 } // end of switch.
5353}
5354
5355void LoopVectorizationCostModel::collectValuesToIgnore() {
5356 // Ignore ephemeral values.
5357 CodeMetrics::collectEphemeralValues(L: TheLoop, AC, EphValues&: ValuesToIgnore);
5358
5359 SmallVector<Value *, 4> DeadInterleavePointerOps;
5360 SmallVector<Value *, 4> DeadOps;
5361
5362 // If a scalar epilogue is required, users outside the loop won't use
5363 // live-outs from the vector loop but from the scalar epilogue. Ignore them if
5364 // that is the case.
5365 bool RequiresScalarEpilogue = requiresScalarEpilogue(IsVectorizing: true);
5366 auto IsLiveOutDead = [this, RequiresScalarEpilogue](User *U) {
5367 return RequiresScalarEpilogue &&
5368 !TheLoop->contains(BB: cast<Instruction>(Val: U)->getParent());
5369 };
5370
5371 LoopBlocksDFS DFS(TheLoop);
5372 DFS.perform(LI);
5373 for (BasicBlock *BB : reverse(C: make_range(x: DFS.beginRPO(), y: DFS.endRPO())))
5374 for (Instruction &I : reverse(C&: *BB)) {
5375 if (VecValuesToIgnore.contains(Ptr: &I) || ValuesToIgnore.contains(Ptr: &I))
5376 continue;
5377
5378 // Add instructions that would be trivially dead and are only used by
5379 // values already ignored to DeadOps to seed worklist.
5380 if (wouldInstructionBeTriviallyDead(I: &I, TLI) &&
5381 all_of(Range: I.users(), P: [this, IsLiveOutDead](User *U) {
5382 return VecValuesToIgnore.contains(Ptr: U) ||
5383 ValuesToIgnore.contains(Ptr: U) || IsLiveOutDead(U);
5384 }))
5385 DeadOps.push_back(Elt: &I);
5386
5387 // For interleave groups, we only create a pointer for the start of the
5388 // interleave group. Queue up addresses of group members except the insert
5389 // position for further processing.
5390 if (isAccessInterleaved(Instr: &I)) {
5391 auto *Group = getInterleavedAccessGroup(Instr: &I);
5392 if (Group->getInsertPos() == &I)
5393 continue;
5394 Value *PointerOp = getLoadStorePointerOperand(V: &I);
5395 DeadInterleavePointerOps.push_back(Elt: PointerOp);
5396 }
5397
5398 // Queue branches for analysis. They are dead, if their successors only
5399 // contain dead instructions.
5400 if (isa<CondBrInst>(Val: &I))
5401 DeadOps.push_back(Elt: &I);
5402 }
5403
5404 // Mark ops feeding interleave group members as free, if they are only used
5405 // by other dead computations.
5406 for (unsigned I = 0; I != DeadInterleavePointerOps.size(); ++I) {
5407 auto *Op = dyn_cast<Instruction>(Val: DeadInterleavePointerOps[I]);
5408 if (!Op || !TheLoop->contains(Inst: Op) || any_of(Range: Op->users(), P: [this](User *U) {
5409 Instruction *UI = cast<Instruction>(Val: U);
5410 return !VecValuesToIgnore.contains(Ptr: U) &&
5411 (!isAccessInterleaved(Instr: UI) ||
5412 getInterleavedAccessGroup(Instr: UI)->getInsertPos() == UI);
5413 }))
5414 continue;
5415 VecValuesToIgnore.insert(Ptr: Op);
5416 append_range(C&: DeadInterleavePointerOps, R: Op->operands());
5417 }
5418
5419 // Mark ops that would be trivially dead and are only used by ignored
5420 // instructions as free.
5421 BasicBlock *Header = TheLoop->getHeader();
5422
5423 // Returns true if the block contains only dead instructions. Such blocks will
5424 // be removed by VPlan-to-VPlan transforms and won't be considered by the
5425 // VPlan-based cost model, so skip them in the legacy cost-model as well.
5426 auto IsEmptyBlock = [this](BasicBlock *BB) {
5427 return all_of(Range&: *BB, P: [this](Instruction &I) {
5428 return ValuesToIgnore.contains(Ptr: &I) || VecValuesToIgnore.contains(Ptr: &I) ||
5429 isa<UncondBrInst>(Val: &I);
5430 });
5431 };
5432 for (unsigned I = 0; I != DeadOps.size(); ++I) {
5433 auto *Op = dyn_cast<Instruction>(Val: DeadOps[I]);
5434
5435 // Check if the branch should be considered dead.
5436 if (auto *Br = dyn_cast_or_null<CondBrInst>(Val: Op)) {
5437 BasicBlock *ThenBB = Br->getSuccessor(i: 0);
5438 BasicBlock *ElseBB = Br->getSuccessor(i: 1);
5439 // Don't considers branches leaving the loop for simplification.
5440 if (!TheLoop->contains(BB: ThenBB) || !TheLoop->contains(BB: ElseBB))
5441 continue;
5442 bool ThenEmpty = IsEmptyBlock(ThenBB);
5443 bool ElseEmpty = IsEmptyBlock(ElseBB);
5444 if ((ThenEmpty && ElseEmpty) ||
5445 (ThenEmpty && ThenBB->getSingleSuccessor() == ElseBB &&
5446 ElseBB->phis().empty()) ||
5447 (ElseEmpty && ElseBB->getSingleSuccessor() == ThenBB &&
5448 ThenBB->phis().empty())) {
5449 VecValuesToIgnore.insert(Ptr: Br);
5450 DeadOps.push_back(Elt: Br->getCondition());
5451 }
5452 continue;
5453 }
5454
5455 // Skip any op that shouldn't be considered dead.
5456 if (!Op || !TheLoop->contains(Inst: Op) ||
5457 (isa<PHINode>(Val: Op) && Op->getParent() == Header) ||
5458 !wouldInstructionBeTriviallyDead(I: Op, TLI) ||
5459 any_of(Range: Op->users(), P: [this, IsLiveOutDead](User *U) {
5460 return !VecValuesToIgnore.contains(Ptr: U) &&
5461 !ValuesToIgnore.contains(Ptr: U) && !IsLiveOutDead(U);
5462 }))
5463 continue;
5464
5465 // If all of Op's users are in ValuesToIgnore, add it to ValuesToIgnore
5466 // which applies for both scalar and vector versions. Otherwise it is only
5467 // dead in vector versions, so only add it to VecValuesToIgnore.
5468 if (all_of(Range: Op->users(),
5469 P: [this](User *U) { return ValuesToIgnore.contains(Ptr: U); }))
5470 ValuesToIgnore.insert(Ptr: Op);
5471
5472 VecValuesToIgnore.insert(Ptr: Op);
5473 append_range(C&: DeadOps, R: Op->operands());
5474 }
5475
5476 // Ignore type-promoting instructions we identified during reduction
5477 // detection.
5478 for (const auto &Reduction : Legal->getReductionVars()) {
5479 const RecurrenceDescriptor &RedDes = Reduction.second;
5480 const SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
5481 VecValuesToIgnore.insert_range(R: Casts);
5482 }
5483 // Ignore type-casting instructions we identified during induction
5484 // detection.
5485 for (const auto &Induction : Legal->getInductionVars()) {
5486 const InductionDescriptor &IndDes = Induction.second;
5487 VecValuesToIgnore.insert_range(R: IndDes.getCastInsts());
5488 }
5489}
5490
5491void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
5492 CM.collectValuesToIgnore();
5493 Config.collectElementTypesForWidening(ValuesToIgnore: &CM.ValuesToIgnore);
5494
5495 FixedScalableVFPair MaxFactors = CM.computeMaxVF(UserVF, UserIC);
5496 if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
5497 return;
5498
5499 Config.collectInLoopReductions();
5500 // Cases that may be vectorized may be optimized by unit stride predicates.
5501 // TODO: Currently unit stride predicates are added unconditionally, even if
5502 // they are not used for the selected VF (e.g. when only interleaving).
5503 if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
5504 Legal->collectUnitStridePredicates();
5505
5506 auto VPlan1 = tryToBuildVPlan1();
5507 if (!VPlan1)
5508 return;
5509
5510 if (!OrigLoop->isInnermost()) {
5511 // For outer loops, computeMaxVF returns a single non-scalar VF; build a
5512 // plan for that VF only.
5513 ElementCount VF =
5514 MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
5515 buildVPlans(VPlan1&: *VPlan1, MinVF: VF, MaxVF: VF);
5516 LLVM_DEBUG(printPlans(dbgs()));
5517 return;
5518 }
5519
5520 // Compute the minimal bitwidths required for integer operations in the loop
5521 // for later use by the cost model.
5522 Config.computeMinimalBitwidths();
5523
5524 // Invalidate interleave groups if all blocks of loop will be predicated.
5525 if (CM.blockNeedsPredicationForAnyReason(BB: OrigLoop->getHeader()) &&
5526 !useMaskedInterleavedAccesses(TTI)) {
5527 LLVM_DEBUG(
5528 dbgs()
5529 << "LV: Invalidate all interleaved groups due to fold-tail by masking "
5530 "which requires masked-interleaved support.\n");
5531 if (CM.InterleaveInfo.invalidateGroups())
5532 // Invalidating interleave groups also requires invalidating all decisions
5533 // based on them, which includes widening decisions and uniform and scalar
5534 // values.
5535 CM.invalidateCostModelingDecisions();
5536 }
5537
5538 if (CM.foldTailByMasking())
5539 Legal->prepareToFoldTailByMasking();
5540
5541 ElementCount MaxUserVF =
5542 UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
5543 if (UserVF) {
5544 if (!ElementCount::isKnownLE(LHS: UserVF, RHS: MaxUserVF)) {
5545 reportVectorizationInfo(
5546 Msg: "UserVF ignored because it may be larger than the maximal safe VF",
5547 ORETag: "InvalidUserVF", ORE, TheLoop: OrigLoop);
5548 } else {
5549 assert(isPowerOf2_32(UserVF.getKnownMinValue()) &&
5550 "VF needs to be a power of two");
5551 // Collect the instructions (and their associated costs) that will be more
5552 // profitable to scalarize.
5553 CM.collectNonVectorizedAndSetWideningDecisions(VF: UserVF);
5554 ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
5555 if (EpilogueUserVF.isVector() &&
5556 ElementCount::isKnownLT(LHS: EpilogueUserVF, RHS: UserVF)) {
5557 CM.collectNonVectorizedAndSetWideningDecisions(VF: EpilogueUserVF);
5558 buildVPlans(VPlan1&: *VPlan1, MinVF: EpilogueUserVF, MaxVF: EpilogueUserVF);
5559 }
5560 buildVPlans(VPlan1&: *VPlan1, MinVF: UserVF, MaxVF: UserVF);
5561 if (!VPlans.empty() && VPlans.back()->getSingleVF() == UserVF) {
5562 // For scalar VF, skip VPlan cost check as VPlan cost is designed for
5563 // vector VFs only.
5564 if (UserVF.isScalar() ||
5565 cost(Plan&: *VPlans.back(), VF: UserVF, /*RU=*/nullptr).isValid()) {
5566 LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
5567 LLVM_DEBUG(printPlans(dbgs()));
5568 return;
5569 }
5570 }
5571 VPlans.clear();
5572 reportVectorizationInfo(Msg: "UserVF ignored because of invalid costs.",
5573 ORETag: "InvalidCost", ORE, TheLoop: OrigLoop);
5574 }
5575 }
5576
5577 // Collect the Vectorization Factor Candidates.
5578 SmallVector<ElementCount> VFCandidates;
5579 for (auto VF = ElementCount::getFixed(MinVal: 1);
5580 ElementCount::isKnownLE(LHS: VF, RHS: MaxFactors.FixedVF); VF *= 2)
5581 VFCandidates.push_back(Elt: VF);
5582 for (auto VF = ElementCount::getScalable(MinVal: 1);
5583 ElementCount::isKnownLE(LHS: VF, RHS: MaxFactors.ScalableVF); VF *= 2)
5584 VFCandidates.push_back(Elt: VF);
5585
5586 for (const auto &VF : VFCandidates) {
5587 // Collect Uniform and Scalar instructions after vectorization with VF.
5588 CM.collectNonVectorizedAndSetWideningDecisions(VF);
5589 }
5590
5591 buildVPlans(VPlan1&: *VPlan1, MinVF: ElementCount::getFixed(MinVal: 1), MaxVF: MaxFactors.FixedVF);
5592 buildVPlans(VPlan1&: *VPlan1, MinVF: ElementCount::getScalable(MinVal: 1), MaxVF: MaxFactors.ScalableVF);
5593
5594 LLVM_DEBUG(printPlans(dbgs()));
5595}
5596
5597InstructionCost VPCostContext::getLegacyCost(Instruction *UI,
5598 ElementCount VF) const {
5599 InstructionCost Cost = CM.getInstructionCost(I: UI, VF);
5600 if (Cost.isValid() && ForceTargetInstructionCost.getNumOccurrences())
5601 return InstructionCost(ForceTargetInstructionCost);
5602 return Cost;
5603}
5604
5605bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const {
5606 return CM.ValuesToIgnore.contains(Ptr: UI) ||
5607 (IsVector && CM.VecValuesToIgnore.contains(Ptr: UI)) ||
5608 SkipCostComputation.contains(Ptr: UI);
5609}
5610
5611void VPCostContext::invalidateWideningDecision(Instruction *I,
5612 ElementCount VF) {
5613 CM.setWideningDecision(I, VF,
5614 W: LoopVectorizationCostModel::CM_InvalidatedDecision, Cost: 0);
5615}
5616
5617uint64_t VPCostContext::getPredBlockCostDivisor(BasicBlock *BB) const {
5618 return CM.getPredBlockCostDivisor(CostKind, BB);
5619}
5620
5621bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const {
5622 return CM.isScalarWithPredication(I, VF) ||
5623 CM.isUniformAfterVectorization(I, VF) || CM.isForcedScalar(I, VF) ||
5624 (VF.isVector() && CM.isProfitableToScalarize(I, VF));
5625}
5626
5627bool VPCostContext::isMaskRequired(Instruction *I) const {
5628 return CM.isMaskRequired(I);
5629}
5630
5631InstructionCost
5632LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
5633 VPCostContext &CostCtx) const {
5634 InstructionCost Cost;
5635 // Cost modeling for inductions is inaccurate in the legacy cost model
5636 // compared to the recipes that are generated. To match here initially during
5637 // VPlan cost model bring up directly use the induction costs from the legacy
5638 // cost model. Note that we do this as pre-processing; the VPlan may not have
5639 // any recipes associated with the original induction increment instruction
5640 // and may replace truncates with VPWidenIntOrFpInductionRecipe. We precompute
5641 // the cost of induction phis and increments (both that are represented by
5642 // recipes and those that are not), to avoid distinguishing between them here,
5643 // and skip all recipes that represent induction phis and increments (the
5644 // former case) later on, if they exist, to avoid counting them twice.
5645 // Similarly we pre-compute the cost of any optimized truncates.
5646 // TODO: Switch to more accurate costing based on VPlan.
5647 for (const auto &[IV, IndDesc] : Legal->getInductionVars()) {
5648 Instruction *IVInc = cast<Instruction>(
5649 Val: IV->getIncomingValueForBlock(BB: OrigLoop->getLoopLatch()));
5650 SmallVector<Instruction *> IVInsts = {IVInc};
5651 for (unsigned I = 0; I != IVInsts.size(); I++) {
5652 for (Value *Op : IVInsts[I]->operands()) {
5653 auto *OpI = dyn_cast<Instruction>(Val: Op);
5654 if (Op == IV || !OpI || !OrigLoop->contains(Inst: OpI) || !Op->hasOneUse())
5655 continue;
5656 IVInsts.push_back(Elt: OpI);
5657 }
5658 }
5659 IVInsts.push_back(Elt: IV);
5660 for (User *U : IV->users()) {
5661 auto *CI = cast<Instruction>(Val: U);
5662 if (!CostCtx.CM.isOptimizableIVTruncate(I: CI, VF))
5663 continue;
5664 IVInsts.push_back(Elt: CI);
5665 }
5666
5667 // If the vector loop gets executed exactly once with the given VF, ignore
5668 // the costs of comparison and induction instructions, as they'll get
5669 // simplified away.
5670 // TODO: Remove this code after stepping away from the legacy cost model and
5671 // adding code to simplify VPlans before calculating their costs.
5672 auto TC = getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop);
5673 if (TC == VF && !Plan.hasTailFolded())
5674 addFullyUnrolledInstructionsToIgnore(L: OrigLoop, IL: Legal->getInductionVars(),
5675 InstsToIgnore&: CostCtx.SkipCostComputation);
5676
5677 for (Instruction *IVInst : IVInsts) {
5678 if (CostCtx.skipCostComputation(UI: IVInst, IsVector: VF.isVector()))
5679 continue;
5680 InstructionCost InductionCost = CostCtx.getLegacyCost(UI: IVInst, VF);
5681 LLVM_DEBUG({
5682 dbgs() << "Cost of " << InductionCost << " for VF " << VF
5683 << ": induction instruction " << *IVInst << "\n";
5684 });
5685 Cost += InductionCost;
5686 CostCtx.SkipCostComputation.insert(Ptr: IVInst);
5687 }
5688 }
5689
5690 // Pre-compute the costs for branches except for the backedge, as the number
5691 // of replicate regions in a VPlan may not directly match the number of
5692 // branches, which would lead to different decisions.
5693 // TODO: Compute cost of branches for each replicate region in the VPlan,
5694 // which is more accurate than the legacy cost model.
5695 for (BasicBlock *BB : OrigLoop->blocks()) {
5696 if (CostCtx.skipCostComputation(UI: BB->getTerminator(), IsVector: VF.isVector()))
5697 continue;
5698 CostCtx.SkipCostComputation.insert(Ptr: BB->getTerminator());
5699 if (BB == OrigLoop->getLoopLatch())
5700 continue;
5701 auto BranchCost = CostCtx.getLegacyCost(UI: BB->getTerminator(), VF);
5702 Cost += BranchCost;
5703 }
5704
5705 // Don't apply special costs when instruction cost is forced to make sure the
5706 // forced cost is used for each recipe.
5707 if (ForceTargetInstructionCost.getNumOccurrences())
5708 return Cost;
5709
5710 // Pre-compute costs for instructions that are forced-scalar or profitable to
5711 // scalarize. For most such instructions, their scalarization costs are
5712 // accounted for here using the legacy cost model. However, some opcodes
5713 // are excluded from these precomputed scalarization costs and are instead
5714 // modeled later by the VPlan cost model (see UseVPlanCostModel below).
5715 for (Instruction *ForcedScalar : CM.ForcedScalars[VF]) {
5716 if (CostCtx.skipCostComputation(UI: ForcedScalar, IsVector: VF.isVector()))
5717 continue;
5718 CostCtx.SkipCostComputation.insert(Ptr: ForcedScalar);
5719 InstructionCost ForcedCost = CostCtx.getLegacyCost(UI: ForcedScalar, VF);
5720 LLVM_DEBUG({
5721 dbgs() << "Cost of " << ForcedCost << " for VF " << VF
5722 << ": forced scalar " << *ForcedScalar << "\n";
5723 });
5724 Cost += ForcedCost;
5725 }
5726
5727 auto UseVPlanCostModel = [](Instruction *I) -> bool {
5728 switch (I->getOpcode()) {
5729 case Instruction::SDiv:
5730 case Instruction::UDiv:
5731 case Instruction::SRem:
5732 case Instruction::URem:
5733 return true;
5734 default:
5735 return false;
5736 }
5737 };
5738 for (const auto &[Scalarized, ScalarCost] : CM.InstsToScalarize[VF]) {
5739 if (UseVPlanCostModel(Scalarized) ||
5740 CostCtx.skipCostComputation(UI: Scalarized, IsVector: VF.isVector()))
5741 continue;
5742 CostCtx.SkipCostComputation.insert(Ptr: Scalarized);
5743 LLVM_DEBUG({
5744 dbgs() << "Cost of " << ScalarCost << " for VF " << VF
5745 << ": profitable to scalarize " << *Scalarized << "\n";
5746 });
5747 Cost += ScalarCost;
5748 }
5749
5750 return Cost;
5751}
5752
5753InstructionCost LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
5754 VPRegisterUsage *RU) const {
5755 VPCostContext CostCtx(CM.TTI, *CM.TLI, Plan, CM, Config.CostKind, PSE,
5756 OrigLoop);
5757 InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
5758
5759 // Now compute and add the VPlan-based cost.
5760 Cost += Plan.cost(VF, Ctx&: CostCtx);
5761
5762 // Add the cost of spills due to excess register usage
5763 if (RU && Config.shouldConsiderRegPressureForVF(VF))
5764 Cost += RU->spillCost(TTI: CM.TTI, CostKind: Config.CostKind, OverrideMaxNumRegs: ForceTargetNumVectorRegs);
5765
5766#ifndef NDEBUG
5767 unsigned EstimatedWidth =
5768 estimateElementCount(VF, Config.getVScaleForTuning());
5769 LLVM_DEBUG(dbgs() << "Cost for VF " << VF << ": " << Cost
5770 << " (Estimated cost per lane: ");
5771 if (Cost.isValid()) {
5772 APFloat CostPerLane(APFloat::IEEEdouble());
5773 APFloat EstimatedWidthAsAPFloat(APFloat::IEEEdouble());
5774 (void)CostPerLane.convertFromAPInt(APInt(64, (uint64_t)Cost.getValue()),
5775 false, APFloat::rmTowardZero);
5776 (void)EstimatedWidthAsAPFloat.convertFromAPInt(
5777 APInt(64, (uint64_t)EstimatedWidth), false, APFloat::rmTowardZero);
5778 (void)CostPerLane.divide(EstimatedWidthAsAPFloat, APFloat::rmTowardZero);
5779
5780 SmallString<16> Str;
5781 CostPerLane.toString(Str, 3);
5782 LLVM_DEBUG(dbgs() << Str);
5783 } else /* No point dividing an invalid cost - it will still be invalid */
5784 LLVM_DEBUG(dbgs() << "Invalid");
5785 LLVM_DEBUG(dbgs() << ")\n");
5786#endif
5787 return Cost;
5788}
5789
5790std::pair<VectorizationFactor, VPlan *>
5791LoopVectorizationPlanner::computeBestVF() {
5792 if (VPlans.empty())
5793 return {VectorizationFactor::Disabled(), nullptr};
5794 // If there is a single VPlan with a single VF, return it directly.
5795 VPlan &FirstPlan = *VPlans[0];
5796
5797 ElementCount UserVF = Hints.getWidth();
5798 if (VPlans.size() == 1) {
5799 // For outer loops, the plan has a single vector VF determined by the
5800 // heuristic.
5801 assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
5802 FirstPlan.isOuterLoop()) &&
5803 "must have a single scalar VF, UserVF or an outer loop");
5804 return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
5805 }
5806
5807 if (hasPlanWithVF(VF: UserVF) && hasForcedEpilogueVF()) {
5808 assert(VPlans.size() == 2 && "Must have exactly 2 VPlans built");
5809 assert(VPlans[0]->getSingleVF() == EpilogueVectorizationForceVF &&
5810 "expected first plan to be for the forced epilogue VF");
5811 assert(VPlans[1]->getSingleVF() == UserVF &&
5812 "expected second plan to be for the forced UserVF");
5813 return {VectorizationFactor(UserVF, 0, 0), VPlans[1].get()};
5814 }
5815
5816 LLVM_DEBUG(dbgs() << "LV: Computing best VF using cost kind: "
5817 << (Config.CostKind == TTI::TCK_RecipThroughput
5818 ? "Reciprocal Throughput\n"
5819 : Config.CostKind == TTI::TCK_Latency
5820 ? "Instruction Latency\n"
5821 : Config.CostKind == TTI::TCK_CodeSize ? "Code Size\n"
5822 : Config.CostKind == TTI::TCK_SizeAndLatency
5823 ? "Code Size and Latency\n"
5824 : "Unknown\n"));
5825
5826 ElementCount ScalarVF = ElementCount::getFixed(MinVal: 1);
5827 assert(FirstPlan.hasVF(ScalarVF) &&
5828 "More than a single plan/VF w/o any plan having scalar VF");
5829
5830 // TODO: Compute scalar cost using VPlan-based cost model.
5831 InstructionCost ScalarCost = CM.expectedCost(VF: ScalarVF);
5832 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
5833 VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
5834 VectorizationFactor BestFactor = ScalarFactor;
5835
5836 bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
5837 if (ForceVectorization) {
5838 // Ignore scalar width, because the user explicitly wants vectorization.
5839 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
5840 // evaluation.
5841 BestFactor.Cost = InstructionCost::getMax();
5842 }
5843
5844 VPlan *PlanForBestVF = &FirstPlan;
5845
5846 for (auto &P : VPlans) {
5847 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
5848 P->vectorFactors().end());
5849
5850 SmallVector<VPRegisterUsage, 8> RUs;
5851 bool ConsiderRegPressure = any_of(Range&: VFs, P: [this](ElementCount VF) {
5852 return Config.shouldConsiderRegPressureForVF(VF);
5853 });
5854 if (ConsiderRegPressure)
5855 RUs = calculateRegisterUsageForPlan(Plan&: *P, VFs, TTI, ValuesToIgnore: CM.ValuesToIgnore);
5856
5857 for (unsigned I = 0; I < VFs.size(); I++) {
5858 ElementCount VF = VFs[I];
5859 if (VF.isScalar())
5860 continue;
5861 if (!ForceVectorization && !willGenerateVectors(Plan&: *P, VF, TTI)) {
5862 LLVM_DEBUG(
5863 dbgs()
5864 << "LV: Not considering vector loop of width " << VF
5865 << " because it will not generate any vector instructions.\n");
5866 continue;
5867 }
5868 if (Config.OptForSize && !ForceVectorization && hasReplicatorRegion(Plan&: *P)) {
5869 LLVM_DEBUG(
5870 dbgs()
5871 << "LV: Not considering vector loop of width " << VF
5872 << " because it would cause replicated blocks to be generated,"
5873 << " which isn't allowed when optimizing for size.\n");
5874 continue;
5875 }
5876
5877 InstructionCost Cost =
5878 cost(Plan&: *P, VF, RU: ConsiderRegPressure ? &RUs[I] : nullptr);
5879 VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
5880
5881 if (isMoreProfitable(A: CurrentFactor, B: BestFactor, HasTail: P->hasScalarTail())) {
5882 BestFactor = CurrentFactor;
5883 PlanForBestVF = P.get();
5884 }
5885
5886 // If profitable add it to ProfitableVF list.
5887 if (isMoreProfitable(A: CurrentFactor, B: ScalarFactor, HasTail: P->hasScalarTail()))
5888 ProfitableVFs.push_back(Elt: CurrentFactor);
5889 }
5890 }
5891
5892 VPlan &BestPlan = *PlanForBestVF;
5893
5894 assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
5895 "when vectorizing, the scalar cost must be computed.");
5896
5897 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << BestFactor.Width << ".\n");
5898 return {BestFactor, &BestPlan};
5899}
5900
5901DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
5902 ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
5903 InnerLoopVectorizer &ILV, DominatorTree *DT,
5904 EpilogueVectorizationKind EpilogueVecKind) {
5905 assert(BestVPlan.hasVF(BestVF) &&
5906 "Trying to execute plan with unsupported VF");
5907 assert(BestVPlan.hasUF(BestUF) &&
5908 "Trying to execute plan with unsupported UF");
5909 if (BestVPlan.hasEarlyExit())
5910 ++LoopsEarlyExitVectorized;
5911
5912 RUN_VPLAN_PASS(VPlanTransforms::replaceWideCanonicalIVWithWideIV, BestVPlan,
5913 *PSE.getSE(), CM.TTI, Config.CostKind, BestVF, BestUF,
5914 CM.ValuesToIgnore);
5915 // TODO: Move to VPlan transform stage once the transition to the VPlan-based
5916 // cost model is complete for better cost estimates.
5917 RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
5918 RUN_VPLAN_PASS(VPlanTransforms::materializePacksAndUnpacks, BestVPlan);
5919 RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
5920 RUN_VPLAN_PASS(VPlanTransforms::replicateByVF, BestVPlan, BestVF);
5921 bool HasBranchWeights =
5922 hasBranchWeightMD(I: *OrigLoop->getLoopLatch()->getTerminator());
5923 if (HasBranchWeights) {
5924 std::optional<unsigned> VScale = Config.getVScaleForTuning();
5925 RUN_VPLAN_PASS(VPlanTransforms::addBranchWeightToMiddleTerminator,
5926 BestVPlan, BestVF, VScale);
5927 }
5928
5929 if (CM.maskPartialAliasing()) {
5930 assert(BestVPlan.hasTailFolded() && "Expected tail folding to be enabled");
5931 RUN_VPLAN_PASS(VPlanTransforms::materializeAliasMaskCheckBlock, BestVPlan,
5932 *CM.Legal->getRuntimePointerChecking()->getDiffChecks(),
5933 HasBranchWeights);
5934 ++LoopsPartialAliasVectorized;
5935 }
5936
5937 // Retrieving VectorPH now when it's easier while VPlan still has Regions.
5938 VPBasicBlock *VectorPH = cast<VPBasicBlock>(Val: BestVPlan.getVectorPreheader());
5939
5940 RUN_VPLAN_PASS(VPlanTransforms::materializeConstantVectorTripCount, BestVPlan,
5941 BestVF, BestUF, PSE);
5942 RUN_VPLAN_PASS(VPlanTransforms::optimizeForVFAndUF, BestVPlan, BestVF, BestUF,
5943 PSE);
5944 RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, BestVPlan);
5945 // Check if scalar epilogue is required, before simplifying constant branches.
5946 const bool RequiresScalarEpilogue = requiresScalarEpilogue(Plan&: BestVPlan, VF: BestVF);
5947 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5948 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5949 /*OnlyLatches=*/false);
5950 if (BestVPlan.getEntry()->getSingleSuccessor() ==
5951 BestVPlan.getScalarPreheader()) {
5952 // TODO: The vector loop would be dead, should not even try to vectorize.
5953 ORE->emit(RemarkBuilder: [&]() {
5954 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationDead",
5955 OrigLoop->getStartLoc(),
5956 OrigLoop->getHeader())
5957 << "Created vector loop never executes due to insufficient trip "
5958 "count.";
5959 });
5960 return DenseMap<const SCEV *, Value *>();
5961 }
5962
5963 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, BestVPlan);
5964
5965 RUN_VPLAN_PASS(VPlanTransforms::convertToConcreteRecipes, BestVPlan);
5966 // Convert the exit condition to AVLNext == 0 for EVL tail folded loops.
5967 RUN_VPLAN_PASS(VPlanTransforms::convertEVLExitCond, BestVPlan);
5968 // Regions are dissolved after optimizing for VF and UF, which completely
5969 // removes unneeded loop regions first.
5970 const bool HasTailFolded = BestVPlan.hasTailFolded();
5971 RUN_VPLAN_PASS(VPlanTransforms::dissolveLoopRegions, BestVPlan);
5972 // Expand BranchOnTwoConds after dissolution, when latch has direct access to
5973 // its successors.
5974 RUN_VPLAN_PASS(VPlanTransforms::expandBranchOnTwoConds, BestVPlan);
5975 // Convert loops with variable-length stepping after regions are dissolved.
5976 RUN_VPLAN_PASS(VPlanTransforms::convertToVariableLengthStep, BestVPlan);
5977 // Remove dead back-edges for single-iteration loops with BranchOnCond(true).
5978 // Only process loop latches to avoid removing edges from the middle block,
5979 // which may be needed for epilogue vectorization.
5980 VPlanTransforms::removeBranchOnConst(Plan&: BestVPlan, /*OnlyLatches=*/true);
5981 VPlanTransforms::materializeBackedgeTakenCount(Plan&: BestVPlan, VectorPH);
5982 std::optional<uint64_t> MaxRuntimeStep;
5983 if (auto MaxVScale = getMaxVScale(F: *CM.TheFunction, TTI: CM.TTI))
5984 MaxRuntimeStep = uint64_t(*MaxVScale) * BestVF.getKnownMinValue() * BestUF;
5985 assert((OrigLoop->getUniqueLatchExitBlock() || RequiresScalarEpilogue) &&
5986 "loops not exiting via the latch without required epilogue?");
5987 VPlanTransforms::materializeVectorTripCount(
5988 Plan&: BestVPlan, VectorPHVPBB: VectorPH, TailByMasking: HasTailFolded, RequiresScalarEpilogue,
5989 Step: &BestVPlan.getVFxUF(), MaxRuntimeStep);
5990 VPlanTransforms::materializeFactors(Plan&: BestVPlan, VectorPH, VF: BestVF);
5991 // Limit expansions to VPInstruction to when not vectorizing the epilogue.
5992 // Currently this code path still relies on code re-using SCEVs expanded
5993 // directly to IR instructions.
5994 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5995 VPlanTransforms::expandSCEVsToVPInstructions(Plan&: BestVPlan, SE&: *PSE.getSE());
5996 VPlanTransforms::cse(Plan&: BestVPlan);
5997 VPlanTransforms::simplifyRecipes(Plan&: BestVPlan);
5998 // Removing branches and incoming values may expose additional simplification
5999 // opportunities.
6000 if (VPlanTransforms::removeBranchOnConst(Plan&: BestVPlan,
6001 /*OnlyLatches=*/EpilogueVecKind !=
6002 EpilogueVectorizationKind::None))
6003 VPlanTransforms::simplifyRecipes(Plan&: BestVPlan);
6004 VPlanTransforms::simplifyKnownEVL(Plan&: BestVPlan, VF: BestVF, PSE);
6005
6006 // 0. Generate SCEV-dependent code in the entry, including TripCount, before
6007 // making any changes to the CFG.
6008 DenseMap<const SCEV *, Value *> ExpandedSCEVs =
6009 VPlanTransforms::expandSCEVs(Plan&: BestVPlan, SE&: *PSE.getSE());
6010
6011 // Perform the actual loop transformation.
6012 VPTransformState State(&TTI, BestVF, LI, DT, ILV.AC, ILV.Builder, &BestVPlan,
6013 OrigLoop->getParentLoop());
6014
6015#ifdef EXPENSIVE_CHECKS
6016 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
6017#endif
6018
6019 // 1. Set up the skeleton for vectorization, including vector pre-header and
6020 // middle block. The vector loop is created during VPlan execution.
6021 State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
6022 if (VPBasicBlock *ScalarPH = BestVPlan.getScalarPreheader())
6023 replaceVPBBWithIRVPBB(VPBB: ScalarPH, IRBB: State.CFG.PrevBB->getSingleSuccessor(),
6024 Plan: &BestVPlan);
6025 VPlanTransforms::removeDeadRecipes(Plan&: BestVPlan);
6026
6027 assert(verifyVPlanIsValid(BestVPlan) && "final VPlan is invalid");
6028
6029 // After vectorization, the exit blocks of the original loop will have
6030 // additional predecessors. Invalidate SCEVs for the exit phis in case SE
6031 // looked through single-entry phis.
6032 ScalarEvolution &SE = *PSE.getSE();
6033 for (VPIRBasicBlock *Exit : BestVPlan.getExitBlocks()) {
6034 if (!Exit->hasPredecessors())
6035 continue;
6036 for (VPRecipeBase &PhiR : Exit->phis())
6037 SE.forgetLcssaPhiWithNewPredecessor(L: OrigLoop,
6038 V: &cast<VPIRPhi>(Val&: PhiR).getIRPhi());
6039 }
6040 // Forget the original loop and block dispositions.
6041 SE.forgetLoop(L: OrigLoop);
6042 SE.forgetBlockAndLoopDispositions();
6043
6044 ILV.printDebugTracesAtStart();
6045
6046 //===------------------------------------------------===//
6047 //
6048 // Notice: any optimization or new instruction that go
6049 // into the code below should also be implemented in
6050 // the cost-model.
6051 //
6052 //===------------------------------------------------===//
6053
6054 // Retrieve loop information before executing the plan, which may remove the
6055 // original loop, if it becomes unreachable.
6056 MDNode *LID = OrigLoop->getLoopID();
6057 unsigned OrigLoopInvocationWeight = 0;
6058 std::optional<unsigned> OrigAverageTripCount =
6059 getLoopEstimatedTripCount(L: OrigLoop, EstimatedLoopInvocationWeight: &OrigLoopInvocationWeight);
6060
6061 BestVPlan.execute(State: &State);
6062
6063 // 2.6. Maintain Loop Hints
6064 // Keep all loop hints from the original loop on the vector loop (we'll
6065 // replace the vectorizer-specific hints below).
6066 VPBasicBlock *HeaderVPBB = vputils::getFirstLoopHeader(Plan&: BestVPlan, VPDT&: State.VPDT);
6067 // Add metadata to disable runtime unrolling a scalar loop when there
6068 // are no runtime checks about strides and memory. A scalar loop that is
6069 // rarely used is not worth unrolling.
6070 bool DisableRuntimeUnroll = !ILV.RTChecks.hasChecks() && !BestVF.isScalar();
6071 updateLoopMetadataAndProfileInfo(
6072 VectorLoop: HeaderVPBB ? LI->getLoopFor(BB: State.CFG.VPBB2IRBB.lookup(Val: HeaderVPBB))
6073 : nullptr,
6074 HeaderVPBB, Plan: BestVPlan,
6075 VectorizingEpilogue: EpilogueVecKind == EpilogueVectorizationKind::Epilogue, OrigLoopID: LID,
6076 OrigAverageTripCount, OrigLoopInvocationWeight,
6077 EstimatedVFxUF: estimateElementCount(VF: BestVF * BestUF, VScale: Config.getVScaleForTuning()),
6078 DisableRuntimeUnroll);
6079
6080 // 3. Fix the vectorized code: take care of header phi's, live-outs,
6081 // predication, updating analyses.
6082 ILV.fixVectorizedLoop(State);
6083
6084 ILV.printDebugTracesAtEnd();
6085
6086 return ExpandedSCEVs;
6087}
6088
6089//===--------------------------------------------------------------------===//
6090// EpilogueVectorizerMainLoop
6091//===--------------------------------------------------------------------===//
6092
6093void EpilogueVectorizerMainLoop::printDebugTracesAtStart() {
6094 LLVM_DEBUG({
6095 dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
6096 << "Main Loop VF:" << EPI.MainLoopVF
6097 << ", Main Loop UF:" << EPI.MainLoopUF
6098 << ", Epilogue Loop VF:" << EPI.EpilogueVF
6099 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
6100 });
6101}
6102
6103void EpilogueVectorizerMainLoop::printDebugTracesAtEnd() {
6104 DEBUG_WITH_TYPE(VerboseDebug, {
6105 dbgs() << "intermediate fn:\n"
6106 << *OrigLoop->getHeader()->getParent() << "\n";
6107 });
6108}
6109
6110//===--------------------------------------------------------------------===//
6111// EpilogueVectorizerEpilogueLoop
6112//===--------------------------------------------------------------------===//
6113
6114/// This function creates a new scalar preheader, using the previous one as
6115/// entry block to the epilogue VPlan. The minimum iteration check is being
6116/// represented in VPlan.
6117BasicBlock *EpilogueVectorizerEpilogueLoop::createVectorizedLoopSkeleton() {
6118 BasicBlock *NewScalarPH = createScalarPreheader(Prefix: "vec.epilog.");
6119 BasicBlock *OriginalScalarPH = NewScalarPH->getSinglePredecessor();
6120 OriginalScalarPH->setName("vec.epilog.iter.check");
6121 VPIRBasicBlock *NewEntry = Plan.createVPIRBasicBlock(IRBB: OriginalScalarPH);
6122 VPBasicBlock *OldEntry = Plan.getEntry();
6123 for (auto &R : make_early_inc_range(Range&: *OldEntry)) {
6124 // Skip moving VPIRInstructions (including VPIRPhis), which are unmovable by
6125 // defining.
6126 if (isa<VPIRInstruction>(Val: &R))
6127 continue;
6128 R.moveBefore(BB&: *NewEntry, I: NewEntry->end());
6129 }
6130
6131 VPBlockUtils::reassociateBlocks(Old: OldEntry, New: NewEntry);
6132 Plan.setEntry(NewEntry);
6133 // OldEntry is now dead and will be cleaned up when the plan gets destroyed.
6134
6135 return OriginalScalarPH;
6136}
6137
6138void EpilogueVectorizerEpilogueLoop::printDebugTracesAtStart() {
6139 LLVM_DEBUG({
6140 dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
6141 << "Epilogue Loop VF:" << EPI.EpilogueVF
6142 << ", Epilogue Loop UF:" << EPI.EpilogueUF << "\n";
6143 });
6144}
6145
6146void EpilogueVectorizerEpilogueLoop::printDebugTracesAtEnd() {
6147 DEBUG_WITH_TYPE(VerboseDebug, {
6148 dbgs() << "final fn:\n" << *OrigLoop->getHeader()->getParent() << "\n";
6149 });
6150}
6151
6152bool VPRecipeBuilder::isPredicatedInst(Instruction *I) const {
6153 return CM.isPredicatedInst(I);
6154}
6155
6156bool VPRecipeBuilder::prefersVectorizedAddressing() const {
6157 return CM.TTI.prefersVectorizedAddressing();
6158}
6159
6160VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
6161 VFRange &Range) {
6162 assert((VPI->getOpcode() == Instruction::Load ||
6163 VPI->getOpcode() == Instruction::Store) &&
6164 "Must be called with either a load or store");
6165 Instruction *I = VPI->getUnderlyingInstr();
6166
6167 auto WillWiden = [&](ElementCount VF) -> bool {
6168 LoopVectorizationCostModel::InstWidening Decision =
6169 CM.getWideningDecision(I, VF);
6170 assert(Decision != LoopVectorizationCostModel::CM_Unknown &&
6171 "CM decision should be taken at this point.");
6172 if (Decision == LoopVectorizationCostModel::CM_Interleave)
6173 return true;
6174 if (CM.isScalarAfterVectorization(I, VF) ||
6175 CM.isProfitableToScalarize(I, VF))
6176 return false;
6177 return Decision != LoopVectorizationCostModel::CM_Scalarize;
6178 };
6179
6180 if (!LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: WillWiden, Range))
6181 return nullptr;
6182
6183 // If a mask is not required, drop it - use unmasked version for safe loads.
6184 // TODO: Determine if mask is needed in VPlan.
6185 VPValue *Mask = CM.isMaskRequired(I) ? VPI->getMask() : nullptr;
6186
6187 // Determine if the pointer operand of the access is either consecutive or
6188 // reverse consecutive.
6189 LoopVectorizationCostModel::InstWidening Decision =
6190 CM.getWideningDecision(I, VF: Range.Start);
6191 bool Reverse = Decision == LoopVectorizationCostModel::CM_Widen_Reverse;
6192 bool Consecutive =
6193 Reverse || Decision == LoopVectorizationCostModel::CM_Widen;
6194
6195 VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(N: 0)
6196 : VPI->getOperand(N: 1);
6197 if (Consecutive) {
6198 Builder.setInsertPoint(VPI);
6199 Ptr = Builder.createConsecutiveVectorPointer(Ptr, SourceElementTy: getLoadStoreType(I),
6200 Reverse, DL: VPI->getDebugLoc());
6201 }
6202
6203 if (Reverse && Mask)
6204 Mask = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Mask, DL: I->getDebugLoc());
6205
6206 if (VPI->getOpcode() == Instruction::Load) {
6207 auto *Load = cast<LoadInst>(Val: I);
6208 auto *LoadR = new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, *VPI,
6209 Load->getDebugLoc());
6210 if (Reverse) {
6211 Builder.insert(R: LoadR);
6212 return new VPInstruction(VPInstruction::Reverse, LoadR, {}, {},
6213 LoadR->getDebugLoc());
6214 }
6215 return LoadR;
6216 }
6217
6218 StoreInst *Store = cast<StoreInst>(Val: I);
6219 VPValue *StoredVal = VPI->getOperand(N: 0);
6220 if (Reverse)
6221 StoredVal = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: StoredVal,
6222 DL: Store->getDebugLoc());
6223 return new VPWidenStoreRecipe(*Store, Ptr, StoredVal, Mask, Consecutive, *VPI,
6224 Store->getDebugLoc());
6225}
6226
6227VPWidenIntOrFpInductionRecipe *
6228VPRecipeBuilder::tryToOptimizeInductionTruncate(VPInstruction *VPI,
6229 VFRange &Range) {
6230 auto *I = cast<TruncInst>(Val: VPI->getUnderlyingInstr());
6231 // Optimize the special case where the source is a constant integer
6232 // induction variable. Notice that we can only optimize the 'trunc' case
6233 // because (a) FP conversions lose precision, (b) sext/zext may wrap, and
6234 // (c) other casts depend on pointer size.
6235
6236 // Determine whether \p K is a truncation based on an induction variable that
6237 // can be optimized.
6238 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
6239 Predicate: bind_front(Fn: &LoopVectorizationCostModel::isOptimizableIVTruncate, BindArgs&: CM,
6240 BindArgs&: I),
6241 Range))
6242 return nullptr;
6243
6244 auto *WidenIV = cast<VPWidenIntOrFpInductionRecipe>(
6245 Val: VPI->getOperand(N: 0)->getDefiningRecipe());
6246 PHINode *Phi = WidenIV->getPHINode();
6247 VPIRValue *Start = WidenIV->getStartValue();
6248 const InductionDescriptor &IndDesc = WidenIV->getInductionDescriptor();
6249
6250 // Wrap flags from the original induction do not apply to the truncated type,
6251 // so do not propagate them.
6252 VPIRFlags Flags = VPIRFlags::WrapFlagsTy(false, false);
6253 VPValue *Step =
6254 vputils::getOrCreateVPValueForSCEVExpr(Plan, Expr: IndDesc.getStep());
6255 return new VPWidenIntOrFpInductionRecipe(
6256 Phi, Start, Step, &Plan.getVF(), IndDesc, I, Flags, VPI->getDebugLoc());
6257}
6258
6259bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
6260 assert((!isa<UncondBrInst, CondBrInst, PHINode, LoadInst, StoreInst>(I)) &&
6261 "Instruction should have been handled earlier");
6262 // Instruction should be widened, unless it is scalar after vectorization,
6263 // scalarization is profitable or it is predicated.
6264 auto WillScalarize = [this, I](ElementCount VF) -> bool {
6265 return CM.isScalarAfterVectorization(I, VF) ||
6266 CM.isProfitableToScalarize(I, VF) ||
6267 CM.isScalarWithPredication(I, VF);
6268 };
6269 return !LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: WillScalarize,
6270 Range);
6271}
6272
6273VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
6274 auto *I = VPI->getUnderlyingInstr();
6275 switch (VPI->getOpcode()) {
6276 default:
6277 return nullptr;
6278 case Instruction::SDiv:
6279 case Instruction::UDiv:
6280 case Instruction::SRem:
6281 case Instruction::URem:
6282 // If not provably safe, use a masked intrinsic.
6283 if (CM.isPredicatedInst(I))
6284 return new VPWidenIntrinsicRecipe(
6285 getMaskedDivRemIntrinsic(Opcode: VPI->getOpcode()), VPI->operands(),
6286 I->getType(), {}, {}, VPI->getDebugLoc());
6287 [[fallthrough]];
6288 case Instruction::Add:
6289 case Instruction::And:
6290 case Instruction::AShr:
6291 case Instruction::FAdd:
6292 case Instruction::FCmp:
6293 case Instruction::FDiv:
6294 case Instruction::FMul:
6295 case Instruction::FNeg:
6296 case Instruction::FRem:
6297 case Instruction::FSub:
6298 case Instruction::ICmp:
6299 case Instruction::LShr:
6300 case Instruction::Mul:
6301 case Instruction::Or:
6302 case Instruction::Select:
6303 case Instruction::Shl:
6304 case Instruction::Sub:
6305 case Instruction::Xor:
6306 case Instruction::Freeze:
6307 return new VPWidenRecipe(*I, VPI->operandsWithoutMask(), *VPI, *VPI,
6308 VPI->getDebugLoc());
6309 case Instruction::ExtractValue: {
6310 SmallVector<VPValue *> NewOps(VPI->operandsWithoutMask());
6311 auto *EVI = cast<ExtractValueInst>(Val: I);
6312 assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
6313 unsigned Idx = EVI->getIndices()[0];
6314 NewOps.push_back(Elt: Plan.getConstantInt(BitWidth: 32, Val: Idx));
6315 return new VPWidenRecipe(*I, NewOps, *VPI, *VPI, VPI->getDebugLoc());
6316 }
6317 };
6318}
6319
6320VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
6321 if (VPI->getOpcode() != Instruction::Store)
6322 return nullptr;
6323
6324 auto HistInfo =
6325 Legal->getHistogramInfo(I: cast<StoreInst>(Val: VPI->getUnderlyingInstr()));
6326 if (!HistInfo)
6327 return nullptr;
6328
6329 const HistogramInfo *HI = *HistInfo;
6330 // FIXME: Support other operations.
6331 unsigned Opcode = HI->Update->getOpcode();
6332 assert((Opcode == Instruction::Add || Opcode == Instruction::Sub) &&
6333 "Histogram update operation must be an Add or Sub");
6334
6335 SmallVector<VPValue *, 3> HGramOps;
6336 // Bucket address.
6337 HGramOps.push_back(Elt: VPI->getOperand(N: 1));
6338 // Increment value.
6339 HGramOps.push_back(Elt: Plan.getOrAddLiveIn(V: HI->Update->getOperand(i: 1)));
6340
6341 // In case of predicated execution (due to tail-folding, or conditional
6342 // execution, or both), pass the relevant mask.
6343 if (CM.isMaskRequired(I: HI->Store))
6344 HGramOps.push_back(Elt: VPI->getMask());
6345
6346 return new VPHistogramRecipe(Opcode, HGramOps, cast<VPIRMetadata>(Val&: *VPI),
6347 VPI->getDebugLoc());
6348}
6349
6350bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
6351 VPInstruction *VPI, VPBuilder &FinalRedStoresBuilder) {
6352 StoreInst *SI;
6353 if ((SI = dyn_cast<StoreInst>(Val: VPI->getUnderlyingInstr())) &&
6354 Legal->isInvariantAddressOfReduction(V: SI->getPointerOperand())) {
6355 // Only create recipe for the final invariant store of the reduction.
6356 if (Legal->isInvariantStoreOfReduction(SI)) {
6357 VPValue *Val = VPI->getOperand(N: 0);
6358 VPValue *Addr = VPI->getOperand(N: 1);
6359 // We need to store the exiting value of the reduction, so use the blend
6360 // if tail folded.
6361 if (auto *Blend = VPlanPatternMatch::findUserOf<VPBlendRecipe>(V: Val))
6362 Val = Blend;
6363 [[maybe_unused]] auto *Rdx =
6364 VPlanPatternMatch::findUserOf<VPReductionPHIRecipe>(V: Val);
6365 assert((!Rdx || Rdx->getBackedgeValue() == Val) &&
6366 "Store of reduction thats not the backedge value?");
6367 auto *Recipe = new VPReplicateRecipe(
6368 SI, {Val, Addr}, true /* IsUniform */, nullptr /*Mask*/, *VPI, *VPI,
6369 VPI->getDebugLoc());
6370 FinalRedStoresBuilder.insert(R: Recipe);
6371 }
6372 VPI->eraseFromParent();
6373 return true;
6374 }
6375
6376 return false;
6377}
6378
6379VPSingleDefRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
6380 VFRange &Range) {
6381 auto *I = VPI->getUnderlyingInstr();
6382 bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
6383 Predicate: [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
6384 Range);
6385
6386 bool IsPredicated = CM.isPredicatedInst(I);
6387
6388 // Even if the instruction is not marked as uniform, there are certain
6389 // intrinsic calls that can be effectively treated as such, so we check for
6390 // them here. Conservatively, we only do this for scalable vectors, since
6391 // for fixed-width VFs we can always fall back on full scalarization.
6392 if (!IsUniform && Range.Start.isScalable() && isa<IntrinsicInst>(Val: I)) {
6393 switch (cast<IntrinsicInst>(Val: I)->getIntrinsicID()) {
6394 case Intrinsic::assume:
6395 case Intrinsic::lifetime_start:
6396 case Intrinsic::lifetime_end:
6397 // For scalable vectors if one of the operands is variant then we still
6398 // want to mark as uniform, which will generate one instruction for just
6399 // the first lane of the vector. We can't scalarize the call in the same
6400 // way as for fixed-width vectors because we don't know how many lanes
6401 // there are.
6402 //
6403 // The reasons for doing it this way for scalable vectors are:
6404 // 1. For the assume intrinsic generating the instruction for the first
6405 // lane is still be better than not generating any at all. For
6406 // example, the input may be a splat across all lanes.
6407 // 2. For the lifetime start/end intrinsics the pointer operand only
6408 // does anything useful when the input comes from a stack object,
6409 // which suggests it should always be uniform. For non-stack objects
6410 // the effect is to poison the object, which still allows us to
6411 // remove the call.
6412 IsUniform = true;
6413 break;
6414 default:
6415 break;
6416 }
6417 }
6418 VPValue *BlockInMask = nullptr;
6419 if (!IsPredicated) {
6420 // Finalize the recipe for Instr, first if it is not predicated.
6421 LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n");
6422 } else {
6423 LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n");
6424 // Instructions marked for predication are replicated and a mask operand is
6425 // added initially. Masked replicate recipes will later be placed under an
6426 // if-then construct to prevent side-effects. Generate recipes to compute
6427 // the block mask for this region.
6428 BlockInMask = VPI->getMask();
6429 }
6430
6431 // Note that there is some custom logic to mark some intrinsics as uniform
6432 // manually above for scalable vectors, which this assert needs to account for
6433 // as well.
6434 assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
6435 (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
6436 "Should not predicate a uniform recipe");
6437 if (IsUniform) {
6438 return VPBuilder::createSingleScalarOp(
6439 Opcode: VPI->getOpcode(), Operands: VPI->operandsWithoutMask(), Mask: BlockInMask, Flags: *VPI, Metadata: *VPI,
6440 DL: VPI->getDebugLoc(), UV: I);
6441 }
6442 auto *Recipe = new VPReplicateRecipe(I, VPI->operandsWithoutMask(),
6443 /*IsSingleScalar=*/false, BlockInMask,
6444 *VPI, *VPI, VPI->getDebugLoc());
6445 return Recipe;
6446}
6447
6448VPRecipeBase *
6449VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
6450 VFRange &Range) {
6451 assert(!R->isPhi() && "phis must be handled earlier");
6452 // First, check for specific widening recipes that deal with optimizing
6453 // truncates and memory operations.
6454 auto *VPI = cast<VPInstruction>(Val: R);
6455 assert(VPI->getOpcode() != Instruction::Call &&
6456 "Call should have been handled by makeCallWideningDecisions");
6457
6458 VPRecipeBase *Recipe;
6459 if (VPI->getOpcode() == Instruction::Trunc &&
6460 (Recipe = tryToOptimizeInductionTruncate(VPI, Range)))
6461 return Recipe;
6462
6463 // All widen recipes below deal only with VF > 1.
6464 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6465 Predicate: [&](ElementCount VF) { return VF.isScalar(); }, Range))
6466 return nullptr;
6467
6468 Instruction *Instr = R->getUnderlyingInstr();
6469 assert(!is_contained({Instruction::Load, Instruction::Store},
6470 VPI->getOpcode()) &&
6471 "Should have been handled prior to this!");
6472
6473 if (!shouldWiden(I: Instr, Range))
6474 return nullptr;
6475
6476 if (VPI->getOpcode() == Instruction::GetElementPtr) {
6477 auto *GEP = cast<GetElementPtrInst>(Val: Instr);
6478 return new VPWidenGEPRecipe(GEP->getSourceElementType(),
6479 VPI->operandsWithoutMask(), *VPI,
6480 VPI->getDebugLoc(), GEP);
6481 }
6482
6483 if (Instruction::isCast(Opcode: VPI->getOpcode())) {
6484 auto *CI = cast<CastInst>(Val: Instr);
6485 auto *CastR = cast<VPInstructionWithType>(Val: VPI);
6486 return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(N: 0),
6487 CastR->getResultType(), CI, *VPI, *VPI,
6488 VPI->getDebugLoc());
6489 }
6490
6491 return tryToWiden(VPI);
6492}
6493
6494// To allow RUN_VPLAN_PASS to print the VPlan after VF/UF independent
6495// optimizations.
6496static void printOptimizedVPlan(VPlan &) {}
6497
6498VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan1() {
6499 bool IsInnerLoop = OrigLoop->isInnermost();
6500
6501 // Set up loop versioning for inner loops with memory runtime checks.
6502 // Outer loops don't have LoopAccessInfo since canVectorizeMemory() is not
6503 // called for them.
6504 std::optional<LoopVersioning> LVer;
6505 if (IsInnerLoop) {
6506 const LoopAccessInfo *LAI = Legal->getLAI();
6507 LVer.emplace(args: *LAI, args: LAI->getRuntimePointerChecking()->getChecks(), args&: OrigLoop,
6508 args&: LI, args&: DT, args: PSE.getSE());
6509 if (!LAI->getRuntimePointerChecking()->getChecks().empty() &&
6510 !LAI->getRuntimePointerChecking()->getDiffChecks()) {
6511 // Only use noalias metadata when using memory checks guaranteeing no
6512 // overlap across all iterations.
6513 LVer->prepareNoAliasMetadata();
6514 }
6515 }
6516
6517 // Create initial base VPlan0, to serve as common starting point for all
6518 // candidates built later for specific VF ranges.
6519 auto VPlan0 = VPlanTransforms::buildVPlan0(TheLoop: OrigLoop, LI&: *LI,
6520 InductionTy: Legal->getWidestInductionType(),
6521 PSE, LVer: LVer ? &*LVer : nullptr);
6522
6523 VPDominatorTree VPDT(*VPlan0);
6524 if (const LoopAccessInfo *LAI = Legal->getLAI())
6525 RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
6526 LAI->getSymbolicStrides(), VPDT);
6527 RUN_VPLAN_PASS(VPlanTransforms::simplifyRecipes, *VPlan0);
6528 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
6529
6530 // Create recipes for header phis. For outer loops, reductions, recurrences
6531 // and in-loop reductions are empty since legality doesn't detect them.
6532 if (!RUN_VPLAN_PASS(VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE,
6533 *OrigLoop, VPDT, Legal->getInductionVars(),
6534 Legal->getReductionVars(),
6535 Legal->getFixedOrderRecurrences(),
6536 Config.getInLoopReductions(), Hints.allowReordering())) {
6537 return nullptr;
6538 }
6539
6540 if (const LoopAccessInfo *LAI = Legal->getLAI())
6541 RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
6542 LAI->getSymbolicStrides(), VPDT);
6543
6544 // Add surviving induction predicates to PSE and check constraints.
6545 bool ForceVectorization = Hints.getForce() == LoopVectorizeHints::FK_Enabled;
6546 bool OptForSize =
6547 !ForceVectorization &&
6548 (CM.EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize ||
6549 CM.EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop);
6550 unsigned SCEVCheckThreshold = ForceVectorization
6551 ? PragmaVectorizeSCEVCheckThreshold
6552 : VectorizeSCEVCheckThreshold;
6553 if (!RUN_VPLAN_PASS(VPlanTransforms::finalizeSCEVPredicates, *VPlan0, PSE,
6554 OptForSize, SCEVCheckThreshold, ORE, OrigLoop))
6555 return nullptr;
6556
6557 RUN_VPLAN_PASS(VPlanTransforms::addMiddleCheck, *VPlan0);
6558
6559 // If we're vectorizing a loop with an uncountable exit, make sure that the
6560 // recipes are safe to handle.
6561 // TODO: Remove this once we can properly check the VPlan itself for both
6562 // the presence of an uncountable exit and the presence of stores in
6563 // the loop inside handleEarlyExits itself.
6564 UncountableExitStyle EEStyle = UncountableExitStyle::NoUncountableExit;
6565 if (Legal->hasUncountableEarlyExit())
6566 EEStyle = Legal->hasUncountableExitWithSideEffects()
6567 ? UncountableExitStyle::MaskedHandleExitInScalarLoop
6568 : UncountableExitStyle::ReadOnly;
6569
6570 if (!RUN_VPLAN_PASS(VPlanTransforms::handleEarlyExits, *VPlan0, EEStyle,
6571 OrigLoop, PSE, *DT, Legal->getAssumptionCache())) {
6572 return nullptr;
6573 }
6574
6575 RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
6576 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
6577 if (CM.foldTailByMasking())
6578 RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
6579 RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
6580
6581 return VPlan0;
6582}
6583
6584void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
6585 ElementCount MaxVF) {
6586 if (ElementCount::isKnownGT(LHS: MinVF, RHS: MaxVF))
6587 return;
6588
6589 auto MaxVFTimes2 = MaxVF * 2;
6590 for (ElementCount VF = MinVF; ElementCount::isKnownLT(LHS: VF, RHS: MaxVFTimes2);) {
6591 VFRange SubRange = {VF, MaxVFTimes2};
6592 auto Plan =
6593 tryToBuildVPlan(InitialPlan: std::unique_ptr<VPlan>(VPlan1.duplicate()), Range&: SubRange);
6594 VF = SubRange.End;
6595
6596 if (!Plan)
6597 continue;
6598
6599 // Now optimize the initial VPlan.
6600 RUN_VPLAN_PASS(VPlanTransforms::hoistPredicatedLoads, *Plan, PSE, OrigLoop);
6601 RUN_VPLAN_PASS(VPlanTransforms::sinkPredicatedStores, *Plan, PSE, OrigLoop);
6602 RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
6603 Config.getMinimalBitwidths());
6604 RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
6605 // TODO: try to put addExplicitVectorLength close to addActiveLaneMask
6606 if (CM.foldTailWithEVL()) {
6607 RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
6608 Config.getMaxSafeElements());
6609 RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
6610 }
6611
6612 if (auto P =
6613 RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
6614 VPlans.push_back(Elt: std::move(P));
6615
6616 TailFoldingStyle Style = CM.getTailFoldingStyle();
6617 RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
6618 useActiveLaneMask(Style),
6619 useActiveLaneMaskForControlFlow(Style));
6620
6621 RUN_VPLAN_PASS_NO_VERIFY(printOptimizedVPlan, *Plan);
6622 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6623 VPlans.push_back(Elt: std::move(Plan));
6624 }
6625}
6626
6627VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan,
6628 VFRange &Range) {
6629
6630 // For outer loops, the plan only needs basic recipe conversion and induction
6631 // live-out optimization; the full inner-loop recipe building below does not
6632 // apply (no widening decisions, interleave groups, reductions, etc.).
6633 if (Plan->isOuterLoop()) {
6634 for (ElementCount VF : Range)
6635 Plan->addVF(VF);
6636 if (!RUN_VPLAN_PASS(VPlanTransforms::tryToConvertVPInstructionsToVPRecipes,
6637 *Plan, *TLI))
6638 return nullptr;
6639 RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE);
6640 return Plan;
6641 }
6642
6643 using namespace llvm::VPlanPatternMatch;
6644 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
6645
6646 // ---------------------------------------------------------------------------
6647 // Build initial VPlan: Scan the body of the loop in a topological order to
6648 // visit each basic block after having visited its predecessor basic blocks.
6649 // ---------------------------------------------------------------------------
6650
6651 bool RequiresScalarEpilogueCheck =
6652 LoopVectorizationPlanner::getDecisionAndClampRange(
6653 Predicate: [this](ElementCount VF) {
6654 return !CM.requiresScalarEpilogue(IsVectorizing: VF.isVector());
6655 },
6656 Range);
6657 // Update the branch in the middle block if a scalar epilogue is required.
6658 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6659 if (!RequiresScalarEpilogueCheck && MiddleVPBB->getNumSuccessors() == 2) {
6660 auto *BranchOnCond = cast<VPInstruction>(Val: MiddleVPBB->getTerminator());
6661 assert(MiddleVPBB->getSuccessors()[1] == Plan->getScalarPreheader() &&
6662 "second successor must be scalar preheader");
6663 BranchOnCond->setOperand(I: 0, New: Plan->getFalse());
6664 }
6665
6666 // Don't use getDecisionAndClampRange here, because we don't know the UF
6667 // so this function is better to be conservative, rather than to split
6668 // it up into different VPlans.
6669 // TODO: Consider using getDecisionAndClampRange here to split up VPlans.
6670 bool IVUpdateMayOverflow = false;
6671 for (ElementCount VF : Range)
6672 IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(Cost: &CM, VF);
6673
6674 TailFoldingStyle Style = CM.getTailFoldingStyle();
6675 // Use NUW for the induction increment if we proved that it won't overflow in
6676 // the vector loop or when not folding the tail. In the later case, we know
6677 // that the canonical induction increment will not overflow as the vector trip
6678 // count is >= increment and a multiple of the increment.
6679 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
6680 bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
6681 if (!HasNUW) {
6682 auto *IVInc =
6683 LoopRegion->getExitingBasicBlock()->getTerminator()->getOperand(N: 0);
6684 assert(match(IVInc,
6685 m_VPInstruction<Instruction::Add>(
6686 m_Specific(LoopRegion->getCanonicalIV()), m_VPValue())) &&
6687 "Did not find the canonical IV increment");
6688 LoopRegion->clearCanonicalIVNUW(Increment: cast<VPInstruction>(Val: IVInc));
6689 }
6690
6691 // ---------------------------------------------------------------------------
6692 // Pre-construction: record ingredients whose recipes we'll need to further
6693 // process after constructing the initial VPlan.
6694 // ---------------------------------------------------------------------------
6695
6696 // For each interleave group which is relevant for this (possibly trimmed)
6697 // Range, add it to the set of groups to be later applied to the VPlan and add
6698 // placeholders for its members' Recipes which we'll be replacing with a
6699 // single VPInterleaveRecipe.
6700 for (InterleaveGroup<Instruction> *IG : IAI.getInterleaveGroups()) {
6701 auto ApplyIG = [IG, this](ElementCount VF) -> bool {
6702 bool Result = (VF.isVector() && // Query is illegal for VF == 1
6703 CM.getWideningDecision(I: IG->getInsertPos(), VF) ==
6704 LoopVectorizationCostModel::CM_Interleave);
6705 // For scalable vectors, the interleave factors must be <= 8 since we
6706 // require the (de)interleaveN intrinsics instead of shufflevectors.
6707 assert((!Result || !VF.isScalable() || IG->getFactor() <= 8) &&
6708 "Unsupported interleave factor for scalable vectors");
6709 return Result;
6710 };
6711 if (!getDecisionAndClampRange(Predicate: ApplyIG, Range))
6712 continue;
6713 InterleaveGroups.insert(Ptr: IG);
6714 }
6715
6716 // ---------------------------------------------------------------------------
6717 // Construct wide recipes and apply predication for original scalar
6718 // VPInstructions in the loop.
6719 // ---------------------------------------------------------------------------
6720 VPRecipeBuilder RecipeBuilder(*Plan, Legal, CM, Builder);
6721
6722 // Scan the body of the loop in a topological order to visit each basic block
6723 // after having visited its predecessor basic blocks.
6724 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
6725 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
6726 HeaderVPBB);
6727
6728 RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
6729 Range.Start);
6730
6731 VPCostContext CostCtx(CM.TTI, *CM.TLI, *Plan, CM, Config.CostKind, CM.PSE,
6732 OrigLoop);
6733
6734 RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
6735 RecipeBuilder, CostCtx);
6736
6737 RUN_VPLAN_PASS(VPlanTransforms::makeScalarizationDecisions, *Plan, Range);
6738
6739 RUN_VPLAN_PASS(VPlanTransforms::makeCallWideningDecisions, *Plan, Range,
6740 RecipeBuilder, CostCtx);
6741
6742 // Now process all other blocks and instructions.
6743 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
6744 // Convert input VPInstructions to widened recipes.
6745 for (VPRecipeBase &R : make_early_inc_range(
6746 Range: make_range(x: VPBB->getFirstNonPhi(), y: VPBB->end()))) {
6747 // Skip recipes that do not need transforming or have already been
6748 // transformed.
6749 if (isa<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe,
6750 VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
6751 VPWidenCallRecipe, VPWidenIntrinsicRecipe, VPVectorPointerRecipe,
6752 VPVectorEndPointerRecipe, VPHistogramRecipe>(Val: &R) ||
6753 (isa<VPInstructionWithType>(Val: R) &&
6754 Instruction::isCast(Opcode: cast<VPInstructionWithType>(Val&: R).getOpcode()) &&
6755 vputils::onlyFirstLaneUsed(Def: R.getVPSingleValue())))
6756 continue;
6757 auto *VPI = cast<VPInstruction>(Val: &R);
6758 if (!VPI->getUnderlyingValue())
6759 continue;
6760
6761 // TODO: Gradually replace uses of underlying instruction by analyses on
6762 // VPlan. Migrate code relying on the underlying instruction from VPlan0
6763 // to construct recipes below to not use the underlying instruction.
6764 Instruction *Instr = cast<Instruction>(Val: VPI->getUnderlyingValue());
6765 Builder.setInsertPoint(VPI);
6766
6767 VPRecipeBase *Recipe =
6768 RecipeBuilder.tryToCreateWidenNonPhiRecipe(R: VPI, Range);
6769 if (!Recipe)
6770 Recipe =
6771 RecipeBuilder.handleReplication(VPI: cast<VPInstruction>(Val: VPI), Range);
6772
6773 if (isa<VPWidenIntOrFpInductionRecipe>(Val: Recipe) && isa<TruncInst>(Val: Instr)) {
6774 // Optimized a truncate to VPWidenIntOrFpInductionRecipe. It needs to be
6775 // moved to the phi section in the header.
6776 Recipe->insertBefore(BB&: *HeaderVPBB, IP: HeaderVPBB->getFirstNonPhi());
6777 } else {
6778 Builder.insert(R: Recipe);
6779 }
6780 if (Recipe->getNumDefinedValues() == 1) {
6781 VPI->replaceAllUsesWith(New: Recipe->getVPSingleValue());
6782 } else {
6783 assert(Recipe->getNumDefinedValues() == 0 &&
6784 "Unexpected multidef recipe");
6785 }
6786 R.eraseFromParent();
6787 }
6788 }
6789
6790 assert(isa<VPRegionBlock>(LoopRegion) &&
6791 !LoopRegion->getEntryBasicBlock()->empty() &&
6792 "entry block must be set to a VPRegionBlock having a non-empty entry "
6793 "VPBasicBlock");
6794
6795 RUN_VPLAN_PASS(VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers, *Plan,
6796 Range);
6797
6798 // ---------------------------------------------------------------------------
6799 // Transform initial VPlan: Apply previously taken decisions, in order, to
6800 // bring the VPlan to its final state.
6801 // ---------------------------------------------------------------------------
6802
6803 addReductionResultComputation(Plan, RecipeBuilder, MinVF: Range.Start);
6804
6805 // Optimize FindIV reductions to use sentinel-based approach when possible.
6806 RUN_VPLAN_PASS(VPlanTransforms::optimizeFindIVReductions, *Plan, PSE,
6807 *OrigLoop);
6808 RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE);
6809
6810 // Apply mandatory transformation to handle reductions with multiple in-loop
6811 // uses if possible, bail out otherwise.
6812 if (!RUN_VPLAN_PASS(VPlanTransforms::handleMultiUseReductions, *Plan, ORE,
6813 OrigLoop))
6814 return nullptr;
6815 // Apply mandatory transformation to handle FP maxnum/minnum reduction with
6816 // NaNs if possible, bail out otherwise.
6817 if (!RUN_VPLAN_PASS(VPlanTransforms::handleMaxMinNumReductions, *Plan))
6818 return nullptr;
6819
6820 // Create whole-vector selects for find-last recurrences.
6821 if (!RUN_VPLAN_PASS(VPlanTransforms::handleFindLastReductions, *Plan))
6822 return nullptr;
6823
6824 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, *Plan, false);
6825
6826 // Create partial reduction recipes for scaled reductions and transform
6827 // recipes to abstract recipes if it is legal and beneficial and clamp the
6828 // range for better cost estimation.
6829 // TODO: Enable following transform when the EVL-version of extended-reduction
6830 // and mulacc-reduction are implemented.
6831 if (!CM.foldTailWithEVL()) {
6832 RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
6833 Range);
6834 RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
6835 Range);
6836 }
6837
6838 // Interleave memory: for each Interleave Group we marked earlier as relevant
6839 // for this VPlan, replace the Recipes widening its memory instructions with a
6840 // single VPInterleaveRecipe at its insertion point.
6841 RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
6842 InterleaveGroups, CM.isEpilogueAllowed());
6843
6844 // Convert memory recipes to strided access recipes if the strided access is
6845 // legal and profitable.
6846 RUN_VPLAN_PASS(VPlanTransforms::convertToStridedAccesses, *Plan, PSE,
6847 *OrigLoop, CostCtx, Range);
6848
6849 // Ensure scalar VF plans only contain VF=1, as required by hasScalarVFOnly.
6850 if (Range.Start.isScalar())
6851 Range.End = Range.Start * 2;
6852
6853 for (ElementCount VF : Range)
6854 Plan->addVF(VF);
6855 Plan->setName("Initial VPlan");
6856
6857 RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
6858
6859 if (CM.maskPartialAliasing())
6860 RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
6861
6862 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6863 return Plan;
6864}
6865
6866void LoopVectorizationPlanner::addReductionResultComputation(
6867 VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
6868 using namespace VPlanPatternMatch;
6869 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
6870 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6871 VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
6872 Builder.setInsertPoint(&*std::prev(x: std::prev(x: LatchVPBB->end())));
6873 VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
6874 VPValue *HeaderMask = Plan->getVectorLoopRegion()->getHeaderMask();
6875 for (VPRecipeBase &R :
6876 Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
6877 VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
6878 if (!PhiR)
6879 continue;
6880
6881 RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
6882 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
6883 PN: cast<PHINode>(Val: PhiR->getUnderlyingInstr()));
6884 Type *PhiTy = PhiR->getScalarType();
6885
6886 // Convert a VPBlendRecipe backedge to a select.
6887 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: PhiR->getBackedgeValue())) {
6888 if (Blend->getNumIncomingValues() == 2 &&
6889 Blend->getMask(Idx: 0) == HeaderMask) {
6890 auto *Sel = VPBuilder(Blend).createSelect(
6891 Cond: Blend->getMask(Idx: 0), TrueVal: Blend->getIncomingValue(Idx: 0),
6892 FalseVal: Blend->getIncomingValue(Idx: 1), DL: {}, Name: "", Flags: *Blend);
6893 Blend->replaceAllUsesWith(New: Sel);
6894 Blend->eraseFromParent();
6895 }
6896 }
6897
6898 auto *OrigExitingVPV = PhiR->getBackedgeValue();
6899 auto *NewExitingVPV = OrigExitingVPV;
6900
6901 // Remove the predicated select if the target doesn't want it.
6902 VPValue *V;
6903 if (!CM.usePredicatedReductionSelect(RecurrenceKind) &&
6904 match(V: PhiR->getBackedgeValue(),
6905 P: m_Select(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue(V), Op2: m_Specific(VPV: PhiR))))
6906 PhiR->setBackedgeValue(V);
6907
6908 // We want code in the middle block to appear to execute on the location of
6909 // the scalar loop's latch terminator because: (a) it is all compiler
6910 // generated, (b) these instructions are always executed after evaluating
6911 // the latch conditional branch, and (c) other passes may add new
6912 // predecessors which terminate on this line. This is the easiest way to
6913 // ensure we don't accidentally cause an extra step back into the loop while
6914 // debugging.
6915 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
6916
6917 // TODO: At the moment ComputeReductionResult also drives creation of the
6918 // bc.merge.rdx phi nodes, hence it needs to be created unconditionally here
6919 // even for in-loop reductions, until the reduction resume value handling is
6920 // also modeled in VPlan.
6921 VPInstruction *FinalReductionResult;
6922 VPBuilder::InsertPointGuard Guard(Builder);
6923 Builder.setInsertPoint(TheBB: MiddleVPBB, IP);
6924 // For AnyOf reductions, find the select among PhiR's users and convert
6925 // the reduction phi to operate on bools before creating the final
6926 // reduction result.
6927 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RecurrenceKind)) {
6928 auto *AnyOfSelect = cast<VPSingleDefRecipe>(
6929 Val: findUserOf(V: PhiR, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())));
6930 VPValue *Start = PhiR->getStartValue();
6931 bool TrueValIsPhi = AnyOfSelect->getOperand(N: 1) == PhiR;
6932 // NewVal is the non-phi operand of the select.
6933 VPValue *NewVal = TrueValIsPhi ? AnyOfSelect->getOperand(N: 2)
6934 : AnyOfSelect->getOperand(N: 1);
6935
6936 // Adjust AnyOf reductions; replace the reduction phi for the selected
6937 // value with a boolean reduction phi node to check if the condition is
6938 // true in any iteration. The final value is selected by the final
6939 // ComputeReductionResult.
6940 VPValue *Cmp = AnyOfSelect->getOperand(N: 0);
6941 // If the compare is checking the reduction PHI node, adjust it to check
6942 // the start value.
6943 if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
6944 CmpR->replaceUsesOfWith(From: PhiR, To: PhiR->getStartValue());
6945 Builder.setInsertPoint(AnyOfSelect);
6946
6947 // If the true value of the select is the reduction phi, the new value
6948 // is selected if the negated condition is true in any iteration.
6949 if (TrueValIsPhi)
6950 Cmp = Builder.createNot(Operand: Cmp);
6951
6952 // Build a fresh i1 chain (phi, or, and i1 versions of any blend/select
6953 // the exiting value flows through).
6954 auto *NewPhiR =
6955 PhiR->cloneWithOperands(Start: Plan->getFalse(), BackedgeValue: Plan->getFalse());
6956 NewPhiR->insertBefore(InsertPos: PhiR);
6957 VPValue *NewExiting = Builder.createOr(LHS: NewPhiR, RHS: Cmp);
6958
6959 // The exiting value may flow through a chain of VPBlendRecipes and
6960 // select recipes (VPInstruction, VPWidenRecipe or VPReplicateRecipe with
6961 // Select opcode) before reaching OrigExitingVPV. Clone each chain link
6962 // in topological order so each clone refers to the already-rewritten i1
6963 // operands via Substitutions.
6964 DenseMap<VPValue *, VPValue *> Substitutions = {{AnyOfSelect, NewExiting},
6965 {PhiR, NewPhiR}};
6966 std::function<void(VPSingleDefRecipe *)> CloneChain =
6967 [&](VPSingleDefRecipe *Old) {
6968 if (Substitutions.contains(Val: Old))
6969 return;
6970 SmallVector<VPValue *> NewOps;
6971 for (VPValue *Op : Old->operands()) {
6972 if (isa<VPBlendRecipe>(Val: Op) ||
6973 match(V: Op, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())))
6974 CloneChain(cast<VPSingleDefRecipe>(Val: Op));
6975 NewOps.push_back(Elt: Substitutions.lookup_or(Val: Op, Default&: Op));
6976 }
6977 VPSingleDefRecipe *New;
6978 if (auto *B = dyn_cast<VPBlendRecipe>(Val: Old))
6979 New = B->cloneWithOperands(NewOperands: NewOps);
6980 else if (auto *W = dyn_cast<VPWidenRecipe>(Val: Old))
6981 New = W->cloneWithOperands(NewOperands: NewOps);
6982 else if (auto *Rep = dyn_cast<VPReplicateRecipe>(Val: Old))
6983 New = Rep->cloneWithOperands(NewOperands: NewOps);
6984 else
6985 New = cast<VPInstruction>(Val: Old)->cloneWithOperands(NewOperands: NewOps);
6986 New->insertBefore(InsertPos: Old);
6987 Substitutions[Old] = New;
6988 };
6989
6990 if (OrigExitingVPV != AnyOfSelect) {
6991 CloneChain(cast<VPSingleDefRecipe>(Val: OrigExitingVPV));
6992 NewExiting = Substitutions.lookup(Val: OrigExitingVPV);
6993 }
6994 NewPhiR->setOperand(I: 1, New: NewExiting);
6995 PhiR->replaceAllUsesWith(New: Plan->getPoison(Ty: PhiR->getScalarType()));
6996
6997 Builder.setInsertPoint(TheBB: MiddleVPBB, IP);
6998 FinalReductionResult =
6999 Builder.createAnyOfReduction(ChainOp: NewExiting, TrueVal: NewVal, FalseVal: Start, DL: ExitDL);
7000 } else {
7001 // If the vector reduction can be performed in a smaller type, we
7002 // truncate then extend the loop exit value to enable InstCombine to
7003 // evaluate the entire expression in the smaller type.
7004 VPValue *ReductionOp = NewExitingVPV;
7005 Instruction::CastOps ExtendOpc = Instruction::CastOpsEnd;
7006 if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
7007 assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
7008 assert(!RecurrenceDescriptor::isMinMaxRecurrenceKind(RecurrenceKind) &&
7009 "Unexpected truncated min-max recurrence!");
7010 Type *RdxTy = RdxDesc.getRecurrenceType();
7011 ExtendOpc = RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
7012 {
7013 VPBuilder::InsertPointGuard Guard(Builder);
7014 Builder.setInsertPoint(
7015 TheBB: NewExitingVPV->getDefiningRecipe()->getParent(),
7016 IP: std::next(x: NewExitingVPV->getDefiningRecipe()->getIterator()));
7017 ReductionOp =
7018 Builder.createWidenCast(Opcode: Instruction::Trunc, Op: NewExitingVPV, ResultTy: RdxTy);
7019 VPWidenCastRecipe *Extnd =
7020 Builder.createWidenCast(Opcode: ExtendOpc, Op: ReductionOp, ResultTy: PhiTy);
7021 if (PhiR->getOperand(N: 1) == NewExitingVPV)
7022 PhiR->setOperand(I: 1, New: Extnd);
7023 }
7024 }
7025
7026 VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
7027 PhiR->getFastMathFlagsOrNone());
7028 FinalReductionResult = Builder.createNaryOp(
7029 Opcode: VPInstruction::ComputeReductionResult, Operands: {ReductionOp}, Flags, DL: ExitDL);
7030 if (ExtendOpc != Instruction::CastOpsEnd)
7031 FinalReductionResult = Builder.createScalarCast(
7032 Opcode: ExtendOpc, Op: FinalReductionResult, ResultTy: PhiTy, DL: {});
7033 }
7034
7035 // Update all users outside the vector region. Also replace redundant
7036 // extracts.
7037 for (auto *U : to_vector(Range: OrigExitingVPV->users())) {
7038 auto *Parent = cast<VPRecipeBase>(Val: U)->getParent();
7039 if (FinalReductionResult == U || Parent->getParent())
7040 continue;
7041 // Skip ComputeReductionResult and FindIV reductions when they are not the
7042 // final result.
7043 if (match(U, P: m_VPInstruction<VPInstruction::ComputeReductionResult>()) ||
7044 (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind: RecurrenceKind) &&
7045 match(U, P: m_VPInstruction<Instruction::ICmp>())))
7046 continue;
7047 U->replaceUsesOfWith(From: OrigExitingVPV, To: FinalReductionResult);
7048
7049 // Look through ExtractLastPart.
7050 if (match(U, P: m_ExtractLastPart(Op0: m_VPValue())))
7051 U = cast<VPInstruction>(Val: U)->getSingleUser();
7052
7053 if (match(U, P: m_CombineOr(Ps: m_ExtractLane(Op0: m_VPValue(), Op1: m_VPValue()),
7054 Ps: m_ExtractLastLane(Op0: m_VPValue()))))
7055 cast<VPInstruction>(Val: U)->replaceAllUsesWith(New: FinalReductionResult);
7056 }
7057
7058 RecurKind RK = PhiR->getRecurrenceKind();
7059 if ((!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK) &&
7060 !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind: RK) &&
7061 !RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind: RK) &&
7062 !RecurrenceDescriptor::isFindLastRecurrenceKind(Kind: RK))) {
7063 VPBuilder PHBuilder(Plan->getVectorPreheader());
7064 VPValue *Iden = Plan->getOrAddLiveIn(
7065 V: getRecurrenceIdentity(K: RK, Tp: PhiTy, FMF: PhiR->getFastMathFlagsOrNone()));
7066 auto *ScaleFactorVPV = Plan->getConstantInt(BitWidth: 32, Val: 1);
7067 VPValue *StartV = PHBuilder.createNaryOp(
7068 Opcode: VPInstruction::ReductionStartVector,
7069 Operands: {PhiR->getStartValue(), Iden, ScaleFactorVPV}, Flags: *PhiR);
7070 PhiR->setOperand(I: 0, New: StartV);
7071 }
7072 }
7073
7074 RUN_VPLAN_PASS(VPlanTransforms::clearReductionWrapFlags, *Plan);
7075}
7076
7077void LoopVectorizationPlanner::attachRuntimeChecks(
7078 VPlan &Plan, GeneratedRTChecks &RTChecks, bool HasBranchWeights) const {
7079 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
7080 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(N: 0)) {
7081 assert((!Config.OptForSize ||
7082 CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled) &&
7083 "Cannot SCEV check stride or overflow when optimizing for size");
7084 RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan, SCEVCheckCond,
7085 SCEVCheckBlock, HasBranchWeights);
7086 }
7087 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
7088 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(N: 0)) {
7089 // VPlan-native path does not do any analysis for runtime checks
7090 // currently.
7091 assert((!EnableVPlanNativePath || !Plan.isOuterLoop()) &&
7092 "Runtime checks are not supported for outer loops yet");
7093
7094 if (Config.OptForSize) {
7095 assert(
7096 CM.Hints->getForce() == LoopVectorizeHints::FK_Enabled &&
7097 "Cannot emit memory checks when optimizing for size, unless forced "
7098 "to vectorize.");
7099 ORE->emit(RemarkBuilder: [&]() {
7100 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
7101 OrigLoop->getStartLoc(),
7102 OrigLoop->getHeader())
7103 << "Code-size may be reduced by not forcing "
7104 "vectorization, or by source-code modifications "
7105 "eliminating the need for runtime checks "
7106 "(e.g., adding 'restrict').";
7107 });
7108 }
7109 RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan, MemCheckCond,
7110 MemCheckBlock, HasBranchWeights);
7111 }
7112}
7113
7114bool LoopVectorizationPlanner::requiresScalarEpilogue(VPlan &Plan,
7115 ElementCount VF) const {
7116 // A scalar epilogue is required, if we unconditionally execute the scalar
7117 // loop. Must be called before removeBranchOnConst.
7118 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
7119 bool Result = MiddleVPBB->getSingleSuccessor() == Plan.getScalarPreheader();
7120 assert(CM.requiresScalarEpilogue(VF.isVector()) == Result &&
7121 "CM.requiresScalarEpilogue and the VPlan-based check must agree");
7122 return Result;
7123}
7124
7125void LoopVectorizationPlanner::addMinimumIterationCheck(
7126 VPlan &Plan, ElementCount VF, unsigned UF,
7127 ElementCount MinProfitableTripCount) const {
7128 const uint32_t *BranchWeights =
7129 hasBranchWeightMD(I: *OrigLoop->getLoopLatch()->getTerminator())
7130 ? &MinItersBypassWeights[0]
7131 : nullptr;
7132 RUN_VPLAN_PASS(VPlanTransforms::addMinimumIterationCheck, Plan, VF, UF,
7133 MinProfitableTripCount, requiresScalarEpilogue(Plan, VF),
7134 Plan.hasTailFolded(), OrigLoop, BranchWeights,
7135 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
7136 PSE, Plan.getEntry());
7137}
7138
7139// Determine how to lower the epilogue, which depends on 1) optimising
7140// for minimum code-size, 2) tail-folding compiler options, 3) loop
7141// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
7142// is suitable for tail-folding.
7143// This function determines epilogue lowering for the main vector loop while
7144// epilogue lowering for the tail-folded epilogue path will be handled
7145// separately in getEpilogueTailLowering.
7146static EpilogueLowering
7147getEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
7148 bool OptForSize, TargetTransformInfo *TTI,
7149 TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL,
7150 InterleavedAccessInfo *IAI) {
7151 // 1) OptSize takes precedence over all other options, i.e. if this is set,
7152 // don't look at hints or options, and don't request an epilogue.
7153 if (F->hasOptSize() ||
7154 (OptForSize && Hints.getForce() != LoopVectorizeHints::FK_Enabled))
7155 return CM_EpilogueNotAllowedOptSize;
7156
7157 // 2) If set, obey the directives
7158 if (TailFoldingPolicy.getNumOccurrences()) {
7159 switch (TailFoldingPolicy) {
7160 case TailFoldingPolicyTy::None:
7161 return CM_EpilogueAllowed;
7162 case TailFoldingPolicyTy::PreferFoldTail:
7163 return CM_EpilogueNotNeededFoldTail;
7164 case TailFoldingPolicyTy::MustFoldTail:
7165 return CM_EpilogueNotAllowedFoldTail;
7166 };
7167 }
7168
7169 // 3) If set, obey the hints
7170 switch (Hints.getPredicate()) {
7171 case LoopVectorizeHints::FK_Enabled:
7172 return CM_EpilogueNotNeededFoldTail;
7173 case LoopVectorizeHints::FK_Disabled:
7174 return CM_EpilogueAllowed;
7175 };
7176
7177 // 4) if the TTI hook indicates this is profitable, request tail-folding.
7178 TailFoldingInfo TFI(TLI, &LVL, IAI);
7179 if (TTI->preferTailFoldingOverEpilogue(TFI: &TFI))
7180 return CM_EpilogueNotNeededFoldTail;
7181
7182 return CM_EpilogueAllowed;
7183}
7184
7185/// Determine how to lower the epilogue for the vector epilogue loop.
7186/// Check if there are any conflicts that prevent tail-folding the epilogue.
7187/// \return CM_EpilogueNotNeededFoldTail if epilogue tail-folding is possible,
7188/// otherwise CM_EpilogueAllowed.
7189static EpilogueLowering
7190getEpilogueTailLowering(const LoopVectorizationCostModel &MainCM, const Loop *L,
7191 OptimizationRemarkEmitter *ORE) {
7192 // Epilogue TF is only enabled when explicitly requested via command line.
7193 if (!EpilogueTailFoldingPolicy.getNumOccurrences() ||
7194 EpilogueTailFoldingPolicy != TailFoldingPolicyTy::PreferFoldTail)
7195 return CM_EpilogueAllowed;
7196
7197 if (!EnableEpilogueVectorization) {
7198 reportVectorizationInfo(
7199 Msg: "Options conflict, epilogue vectorization is disallowed while "
7200 "epilogue tail-folding allowed!\n",
7201 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
7202 return CM_EpilogueAllowed;
7203 }
7204
7205 // If scalar epilogue is explicitly required, we can't apply TF.
7206 if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
7207 LLVM_DEBUG(dbgs() << "LV: Epilogue tail-folding can't be applied because "
7208 "scalar epilogue is required\n"
7209 "LV: Fall back to a normal epilogue\n");
7210 return CM_EpilogueAllowed;
7211 }
7212
7213 // If having epilogue is NOT allowed, then no epilogue to apply TF for.
7214 if (!MainCM.isEpilogueAllowed()) {
7215 LLVM_DEBUG(dbgs() << "LV: No epilogue to apply tail-folding for.\n"
7216 "LV: Fall back to a normal epilogue\n");
7217 return CM_EpilogueAllowed;
7218 }
7219
7220 // We can apply tail-folding on the vectorized epilogue loop.
7221 return CM_EpilogueNotNeededFoldTail;
7222}
7223
7224// Emit a remark if there are stores to floats that required a floating point
7225// extension. If the vectorized loop was generated with floating point there
7226// will be a performance penalty from the conversion overhead and the change in
7227// the vector width.
7228static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
7229 SmallVector<Instruction *, 4> Worklist;
7230 for (BasicBlock *BB : L->getBlocks()) {
7231 for (Instruction &Inst : *BB) {
7232 if (auto *S = dyn_cast<StoreInst>(Val: &Inst)) {
7233 if (S->getValueOperand()->getType()->isFloatTy())
7234 Worklist.push_back(Elt: S);
7235 }
7236 }
7237 }
7238
7239 // Traverse the floating point stores upwards searching, for floating point
7240 // conversions.
7241 SmallPtrSet<const Instruction *, 4> Visited;
7242 SmallPtrSet<const Instruction *, 4> EmittedRemark;
7243 while (!Worklist.empty()) {
7244 auto *I = Worklist.pop_back_val();
7245 if (!L->contains(Inst: I))
7246 continue;
7247 if (!Visited.insert(Ptr: I).second)
7248 continue;
7249
7250 // Emit a remark if the floating point store required a floating
7251 // point conversion.
7252 // TODO: More work could be done to identify the root cause such as a
7253 // constant or a function return type and point the user to it.
7254 if (isa<FPExtInst>(Val: I) && EmittedRemark.insert(Ptr: I).second)
7255 ORE->emit(RemarkBuilder: [&]() {
7256 return OptimizationRemarkAnalysis(LV_NAME, "VectorMixedPrecision",
7257 I->getDebugLoc(), L->getHeader())
7258 << "floating point conversion changes vector width. "
7259 << "Mixed floating point precision requires an up/down "
7260 << "cast that will negatively impact performance.";
7261 });
7262
7263 for (Use &Op : I->operands())
7264 if (auto *OpI = dyn_cast<Instruction>(Val&: Op))
7265 Worklist.push_back(Elt: OpI);
7266 }
7267}
7268
7269/// For loops with uncountable early exits, find the cost of doing work when
7270/// exiting the loop early, such as calculating the final exit values of
7271/// variables used outside the loop.
7272/// TODO: This is currently overly pessimistic because the loop may not take
7273/// the early exit, but better to keep this conservative for now. In future,
7274/// it might be possible to relax this by using branch probabilities.
7275static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx,
7276 VPlan &Plan, ElementCount VF) {
7277 InstructionCost Cost = 0;
7278 for (auto *ExitVPBB : Plan.getExitBlocks()) {
7279 for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
7280 // If the predecessor is not the middle.block, then it must be the
7281 // vector.early.exit block, which may contain work to calculate the exit
7282 // values of variables used outside the loop.
7283 if (PredVPBB != Plan.getMiddleBlock()) {
7284 LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
7285 << PredVPBB->getName() << ":\n");
7286 Cost += PredVPBB->cost(VF, Ctx&: CostCtx);
7287 }
7288 }
7289 }
7290 return Cost;
7291}
7292
7293/// This function determines whether or not it's still profitable to vectorize
7294/// the loop given the extra work we have to do outside of the loop:
7295/// 1. Perform the runtime checks before entering the loop to ensure it's safe
7296/// to vectorize.
7297/// 2. In the case of loops with uncountable early exits, we may have to do
7298/// extra work when exiting the loop early, such as calculating the final
7299/// exit values of variables used outside the loop.
7300/// 3. The middle block.
7301static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
7302 VectorizationFactor &VF, Loop *L,
7303 PredicatedScalarEvolution &PSE,
7304 VPCostContext &CostCtx, VPlan &Plan,
7305 EpilogueLowering SEL,
7306 std::optional<unsigned> VScale) {
7307 InstructionCost RtC = Checks.getCost();
7308 if (!RtC.isValid())
7309 return false;
7310
7311 // When interleaving only scalar and vector cost will be equal, which in turn
7312 // would lead to a divide by 0. Fall back to hard threshold.
7313 if (VF.Width.isScalar()) {
7314 // TODO: Should we rename VectorizeMemoryCheckThreshold?
7315 if (RtC > VectorizeMemoryCheckThreshold) {
7316 LLVM_DEBUG(
7317 dbgs()
7318 << "LV: Interleaving only is not profitable due to runtime checks\n");
7319 return false;
7320 }
7321 return true;
7322 }
7323
7324 // The scalar cost should only be 0 when vectorizing with a user specified
7325 // VF/IC. In those cases, runtime checks should always be generated.
7326 uint64_t ScalarC = VF.ScalarCost.getValue();
7327 if (ScalarC == 0)
7328 return true;
7329
7330 InstructionCost TotalCost = RtC;
7331 // Add on the cost of any work required in the vector early exit block, if
7332 // one exists.
7333 TotalCost += calculateEarlyExitCost(CostCtx, Plan, VF: VF.Width);
7334 TotalCost += Plan.getMiddleBlock()->cost(VF: VF.Width, Ctx&: CostCtx);
7335
7336 // First, compute the minimum iteration count required so that the vector
7337 // loop outperforms the scalar loop.
7338 // The total cost of the scalar loop is
7339 // ScalarC * TC
7340 // where
7341 // * TC is the actual trip count of the loop.
7342 // * ScalarC is the cost of a single scalar iteration.
7343 //
7344 // The total cost of the vector loop is
7345 // TotalCost + VecC * (TC / VF) + EpiC
7346 // where
7347 // * TotalCost is the sum of the costs cost of
7348 // - the generated runtime checks, i.e. RtC
7349 // - performing any additional work in the vector.early.exit block for
7350 // loops with uncountable early exits.
7351 // - the middle block, if ExpectedTC <= VF.Width.
7352 // * VecC is the cost of a single vector iteration.
7353 // * TC is the actual trip count of the loop
7354 // * VF is the vectorization factor
7355 // * EpiCost is the cost of the generated epilogue, including the cost
7356 // of the remaining scalar operations.
7357 //
7358 // Vectorization is profitable once the total vector cost is less than the
7359 // total scalar cost:
7360 // TotalCost + VecC * (TC / VF) + EpiC < ScalarC * TC
7361 //
7362 // Now we can compute the minimum required trip count TC as
7363 // VF * (TotalCost + EpiC) / (ScalarC * VF - VecC) < TC
7364 //
7365 // For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
7366 // the computations are performed on doubles, not integers and the result
7367 // is rounded up, hence we get an upper estimate of the TC.
7368 unsigned IntVF = estimateElementCount(VF: VF.Width, VScale);
7369 uint64_t Div = ScalarC * IntVF - VF.Cost.getValue();
7370 uint64_t MinTC1 =
7371 Div == 0 ? 0 : divideCeil(Numerator: TotalCost.getValue() * IntVF, Denominator: Div);
7372
7373 // Second, compute a minimum iteration count so that the cost of the
7374 // runtime checks is only a fraction of the total scalar loop cost. This
7375 // adds a loop-dependent bound on the overhead incurred if the runtime
7376 // checks fail. In case the runtime checks fail, the cost is RtC + ScalarC
7377 // * TC. To bound the runtime check to be a fraction 1/X of the scalar
7378 // cost, compute
7379 // RtC < ScalarC * TC * (1 / X) ==> RtC * X / ScalarC < TC
7380 uint64_t MinTC2 = divideCeil(Numerator: RtC.getValue() * 10, Denominator: ScalarC);
7381
7382 // Now pick the larger minimum. If it is not a multiple of VF and an epilogue
7383 // is allowed, choose the next closest multiple of VF. This should partly
7384 // compensate for ignoring the epilogue cost.
7385 uint64_t MinTC = std::max(a: MinTC1, b: MinTC2);
7386 if (SEL == CM_EpilogueAllowed)
7387 MinTC = alignTo(Value: MinTC, Align: IntVF);
7388 VF.MinProfitableTripCount = ElementCount::getFixed(MinVal: MinTC);
7389
7390 LLVM_DEBUG(
7391 dbgs() << "LV: Minimum required TC for runtime checks to be profitable:"
7392 << VF.MinProfitableTripCount << "\n");
7393
7394 // Skip vectorization if the expected trip count is less than the minimum
7395 // required trip count.
7396 if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
7397 if (ElementCount::isKnownLT(LHS: *ExpectedTC, RHS: VF.MinProfitableTripCount)) {
7398 LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
7399 "trip count < minimum profitable VF ("
7400 << *ExpectedTC << " < " << VF.MinProfitableTripCount
7401 << ")\n");
7402
7403 return false;
7404 }
7405 }
7406 return true;
7407}
7408
7409LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
7410 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced ||
7411 !EnableLoopInterleaving),
7412 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced ||
7413 !EnableLoopVectorization) {}
7414
7415/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
7416/// vectorization.
7417static SmallVector<VPInstruction *>
7418preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
7419 using namespace VPlanPatternMatch;
7420 // When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
7421 // introduce multiple uses of undef/poison. If the reduction start value may
7422 // be undef or poison it needs to be frozen and the frozen start has to be
7423 // used when computing the reduction result. We also need to use the frozen
7424 // value in the resume phi generated by the main vector loop, as this is also
7425 // used to compute the reduction result after the epilogue vector loop.
7426 auto AddFreezeForFindLastIVReductions = [](VPlan &Plan,
7427 bool UpdateResumePhis) {
7428 VPBuilder Builder(Plan.getEntry());
7429 for (VPRecipeBase &R : *Plan.getMiddleBlock()) {
7430 auto *VPI = dyn_cast<VPInstruction>(Val: &R);
7431 if (!VPI)
7432 continue;
7433 VPValue *OrigStart;
7434 if (!matchFindIVResult(VPI, ReducedIV: m_VPValue(), Start: m_VPValue(V&: OrigStart)))
7435 continue;
7436 if (isGuaranteedNotToBeUndefOrPoison(V: OrigStart->getLiveInIRValue()))
7437 continue;
7438 VPInstruction *Freeze =
7439 Builder.createNaryOp(Opcode: Instruction::Freeze, Operands: {OrigStart}, DL: {}, Name: "fr");
7440 VPI->setOperand(I: 2, New: Freeze);
7441 if (UpdateResumePhis)
7442 OrigStart->replaceUsesWithIf(New: Freeze, ShouldReplace: [Freeze](VPUser &U, unsigned) {
7443 return Freeze != &U && isa<VPPhi>(Val: &U);
7444 });
7445 }
7446 };
7447 AddFreezeForFindLastIVReductions(MainPlan, true);
7448 AddFreezeForFindLastIVReductions(EpiPlan, false);
7449
7450 VPValue *VectorTC = nullptr;
7451 auto *Term =
7452 MainPlan.getVectorLoopRegion()->getExitingBasicBlock()->getTerminator();
7453 [[maybe_unused]] bool MatchedTC =
7454 match(V: Term, P: m_BranchOnCount(Op0: m_VPValue(), Op1: m_VPValue(V&: VectorTC)));
7455 assert(MatchedTC && "must match vector trip count");
7456
7457 // If there is a suitable resume value for the canonical induction in the
7458 // scalar (which will become vector) epilogue loop, use it and move it to the
7459 // beginning of the scalar preheader. Otherwise create it below.
7460 VPBasicBlock *MainScalarPH = MainPlan.getScalarPreheader();
7461 auto ResumePhiIter =
7462 find_if(Range: MainScalarPH->phis(), P: [VectorTC](VPRecipeBase &R) {
7463 return match(V: &R, P: m_VPInstruction<Instruction::PHI>(Ops: m_Specific(VPV: VectorTC),
7464 Ops: m_ZeroInt()));
7465 });
7466 VPPhi *ResumePhi = nullptr;
7467 if (ResumePhiIter == MainScalarPH->phis().end()) {
7468 assert(MainPlan.getVectorLoopRegion()->getCanonicalIV() &&
7469 "canonical IV must exist");
7470 Type *Ty = VectorTC->getScalarType();
7471 VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
7472 ResumePhi = ScalarPHBuilder.createScalarPhi(
7473 IncomingValues: {VectorTC, MainPlan.getZero(Ty)}, DL: {}, Name: "vec.epilog.resume.val");
7474 } else {
7475 ResumePhi = cast<VPPhi>(Val: &*ResumePhiIter);
7476 ResumePhi->setName("vec.epilog.resume.val");
7477 if (&MainScalarPH->front() != ResumePhi)
7478 ResumePhi->moveBefore(BB&: *MainScalarPH, I: MainScalarPH->begin());
7479 }
7480
7481 // Create a ResumeForEpilogue for the canonical IV resume and its bypass value
7482 // as the first non-phi, to keep them alive for the epilogue.
7483 VPBuilder ResumeBuilder(MainScalarPH);
7484 ResumeBuilder.createNaryOp(Opcode: VPInstruction::ResumeForEpilogue,
7485 Operands: {ResumePhi, ResumePhi->getOperand(N: 1)});
7486
7487 // Create ResumeForEpilogue instructions for the resume phis of the
7488 // VPIRPhis and their bypass values in the scalar header of the main plan and
7489 // return them so they can be used as resume values when vectorizing the
7490 // epilogue.
7491 return to_vector(
7492 Range: map_range(C: MainPlan.getScalarHeader()->phis(), F: [&](VPRecipeBase &R) {
7493 assert(isa<VPIRPhi>(R) &&
7494 "only VPIRPhis expected in the scalar header");
7495 VPValue *MainResumePhi = R.getOperand(N: 0);
7496 VPValue *Bypass = MainResumePhi->getDefiningRecipe()->getOperand(N: 1);
7497 return ResumeBuilder.createNaryOp(Opcode: VPInstruction::ResumeForEpilogue,
7498 Operands: {MainResumePhi, Bypass});
7499 }));
7500}
7501
7502/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
7503/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes. Some
7504/// reductions require creating new instructions to compute the resume values.
7505/// They are collected in a vector and returned. They must be moved to the
7506/// preheader of the vector epilogue loop, after created by the execution of \p
7507/// Plan.
7508static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
7509 VPlan &MainPlan, VPlan &Plan, Loop *L, const SCEV2ValueTy &ExpandedSCEVs,
7510 EpilogueLoopVectorizationInfo &EPI, LoopVectorizationPlanner &LVP,
7511 VFSelectionContext &Config, ScalarEvolution &SE,
7512 ArrayRef<VPInstruction *> ResumeValues) {
7513 // Build a map from the scalar-header PHI to the ResumeForEpilogue markers
7514 // from the main plan.
7515 // TODO: Replace the IR PHI key.
7516 DenseMap<PHINode *, VPInstruction *> IRPhiToResumeForEpi;
7517 for (auto [HeaderPhi, ResumeForEpi] :
7518 zip_equal(t: MainPlan.getScalarHeader()->phis(), u&: ResumeValues))
7519 IRPhiToResumeForEpi[&cast<VPIRPhi>(Val&: HeaderPhi).getIRPhi()] = ResumeForEpi;
7520 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
7521 VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
7522 Header->setName("vec.epilog.vector.body");
7523
7524 VPValue *IV = VectorLoop->getCanonicalIV();
7525 // When vectorizing the epilogue loop, the canonical induction needs to start
7526 // at the resume value from the main vector loop. Find the resume value
7527 // created during execution of the main VPlan. Add this resume value as an
7528 // offset to the canonical IV of the epilogue loop.
7529 using namespace llvm::PatternMatch;
7530 VPInstruction *ResumeForEpilogue =
7531 cast<VPInstruction>(Val: &*MainPlan.getScalarPreheader()->getFirstNonPhi());
7532 Value *EPResumeVal = ResumeForEpilogue->getUnderlyingValue();
7533 if (auto *ResumePhi = dyn_cast<PHINode>(Val: EPResumeVal)) {
7534 for (Value *Inc : ResumePhi->incoming_values()) {
7535 if (match(V: Inc, P: m_SpecificInt(V: 0)))
7536 continue;
7537 assert(!EPI.VectorTripCount &&
7538 "Must only have a single non-zero incoming value");
7539 EPI.VectorTripCount = Inc;
7540 }
7541 // If we didn't find a non-zero vector trip count, all incoming values
7542 // must be zero, which also means the vector trip count is zero.
7543 if (!EPI.VectorTripCount) {
7544 assert(ResumePhi->getNumIncomingValues() > 0 &&
7545 all_of(ResumePhi->incoming_values(), match_fn(m_SpecificInt(0))) &&
7546 "all incoming values must be 0");
7547 EPI.VectorTripCount = ResumePhi->getIncomingValue(i: 0);
7548 }
7549 } else {
7550 EPI.VectorTripCount = EPResumeVal;
7551 }
7552 VPValue *VPV = Plan.getOrAddLiveIn(V: EPResumeVal);
7553 assert(all_of(IV->users(),
7554 [](const VPUser *U) {
7555 if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe>(U))
7556 return true;
7557 unsigned Opc = cast<VPInstruction>(U)->getOpcode();
7558 return Instruction::isCast(Opc) || Opc == Instruction::Add;
7559 }) &&
7560 "the canonical IV should only be used by its increment or "
7561 "ScalarIVSteps when resetting the start value");
7562 VPBuilder Builder(Header, Header->getFirstNonPhi());
7563 VPInstruction *Add = Builder.createAdd(LHS: IV, RHS: VPV);
7564 // Replace all users of the canonical IV and its increment with the offset
7565 // version, except for the Add itself and the canonical IV increment.
7566 auto *Increment = vputils::findCanonicalIVIncrement(Plan);
7567 assert(Increment && "Must have a canonical IV increment at this point");
7568 IV->replaceUsesWithIf(New: Add, ShouldReplace: [Add, Increment](VPUser &U, unsigned) {
7569 return &U != Add && &U != Increment;
7570 });
7571 VPInstruction *OffsetIVInc =
7572 VPBuilder::getToInsertAfter(R: Increment).createAdd(LHS: Increment, RHS: VPV);
7573 Increment->replaceAllUsesWith(New: OffsetIVInc);
7574 OffsetIVInc->setOperand(I: 0, New: Increment);
7575
7576 DenseMap<Value *, Value *> ToFrozen;
7577 SmallVector<Instruction *> InstsToMove;
7578 // Ensure that the start values for all header phi recipes are updated before
7579 // vectorizing the epilogue loop.
7580 for (VPRecipeBase &R : Header->phis()) {
7581 Value *ResumeV = nullptr;
7582 // TODO: Move setting of resume values to prepareToExecute.
7583 if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(Val: &R)) {
7584 // Find the reduction result by searching users of the phi or its backedge
7585 // value.
7586 auto IsReductionResult = [](VPRecipeBase *R) {
7587 auto *VPI = dyn_cast<VPInstruction>(Val: R);
7588 return VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult;
7589 };
7590 auto *RdxResult = cast<VPInstruction>(
7591 Val: vputils::findRecipe(Start: ReductionPhi->getBackedgeValue(), Pred: IsReductionResult));
7592 assert(RdxResult && "expected to find reduction result");
7593
7594 VPInstruction *ResumeForEpi = IRPhiToResumeForEpi.at(
7595 Val: cast<PHINode>(Val: ReductionPhi->getUnderlyingInstr()));
7596 ResumeV = ResumeForEpi->getUnderlyingValue();
7597
7598 // Check for FindIV pattern by looking for icmp user of RdxResult.
7599 // The pattern is: select(icmp ne RdxResult, Sentinel), RdxResult, Start
7600 using namespace VPlanPatternMatch;
7601 VPValue *SentinelVPV = nullptr;
7602 bool IsFindIV = any_of(Range: RdxResult->users(), P: [&](VPUser *U) {
7603 return match(U, P: VPlanPatternMatch::m_SpecificICmp(
7604 MatchPred: ICmpInst::ICMP_NE, Op0: m_Specific(VPV: RdxResult),
7605 Op1: m_VPValue(V&: SentinelVPV)));
7606 });
7607
7608 RecurKind RK = ReductionPhi->getRecurrenceKind();
7609 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK) || IsFindIV) {
7610 auto *ResumePhi = cast<PHINode>(Val: ResumeV);
7611 VPValue *BypassOp = ResumeForEpi->getOperand(N: 1);
7612 assert((isa<VPIRValue>(BypassOp) ||
7613 VPlanPatternMatch::match(
7614 BypassOp,
7615 m_VPInstruction<Instruction::Freeze>(m_VPValue()))) &&
7616 "expected live-in or Freeze");
7617 Value *StartV = BypassOp->getUnderlyingValue();
7618 IRBuilder<> Builder(ResumePhi->getParent(),
7619 ResumePhi->getParent()->getFirstNonPHIIt());
7620
7621 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK)) {
7622 // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
7623 // start value; compare the final value from the main vector loop
7624 // to the start value.
7625 ResumeV = Builder.CreateICmpNE(LHS: ResumeV, RHS: StartV);
7626 if (auto *I = dyn_cast<Instruction>(Val: ResumeV))
7627 InstsToMove.push_back(Elt: I);
7628 } else {
7629 assert(SentinelVPV && "expected to find icmp using RdxResult");
7630 if (auto *FreezeI = dyn_cast<FreezeInst>(Val: StartV))
7631 ToFrozen[FreezeI->getOperand(i_nocapture: 0)] = StartV;
7632
7633 // Adjust resume: select(icmp eq ResumeV, StartV), Sentinel, ResumeV
7634 Value *Cmp = Builder.CreateICmpEQ(LHS: ResumeV, RHS: StartV);
7635 if (auto *I = dyn_cast<Instruction>(Val: Cmp))
7636 InstsToMove.push_back(Elt: I);
7637 ResumeV = Builder.CreateSelect(C: Cmp, True: SentinelVPV->getLiveInIRValue(),
7638 False: ResumeV);
7639 if (auto *I = dyn_cast<Instruction>(Val: ResumeV))
7640 InstsToMove.push_back(Elt: I);
7641 }
7642 } else {
7643 VPValue *StartVal = Plan.getOrAddLiveIn(V: ResumeV);
7644 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
7645 if (auto *VPI = dyn_cast<VPInstruction>(Val: PhiR->getStartValue())) {
7646 assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
7647 "unexpected start value");
7648 // Partial sub-reductions always start at 0 and account for the
7649 // reduction start value in a final subtraction. Update it to use the
7650 // resume value from the main vector loop.
7651 if (PhiR->getVFScaleFactor() > 1 &&
7652 RecurrenceDescriptor::isSubRecurrenceKind(
7653 Kind: PhiR->getRecurrenceKind())) {
7654 auto *Sub = cast<VPInstruction>(Val: RdxResult->getSingleUser());
7655 assert((Sub->getOpcode() == Instruction::Sub ||
7656 Sub->getOpcode() == Instruction::FSub) &&
7657 "Unexpected opcode");
7658 assert(isa<VPIRValue>(Sub->getOperand(0)) &&
7659 "Expected operand to match the original start value of the "
7660 "reduction");
7661 // For integer sub-reductions, verify start value is zero.
7662 // For FP sub-reductions, verify start value is negative zero.
7663 [[maybe_unused]] auto StartValueIsIdentity = [&] {
7664 Value *IdentityValue = getRecurrenceIdentity(
7665 K: PhiR->getRecurrenceKind(), Tp: ResumeV->getType(),
7666 FMF: PhiR->getFastMathFlagsOrNone());
7667 auto *StartValue = dyn_cast<VPIRValue>(Val: VPI->getOperand(N: 0));
7668 return StartValue && StartValue->getValue() == IdentityValue;
7669 };
7670 assert(StartValueIsIdentity() &&
7671 "Expected start value for partial sub-reduction to be zero "
7672 "(or negative zero)");
7673
7674 Sub->setOperand(I: 0, New: StartVal);
7675 } else
7676 VPI->setOperand(I: 0, New: StartVal);
7677 continue;
7678 }
7679 }
7680 } else {
7681 // Retrieve the induction resume value via ResumeForEpilogue.
7682 PHINode *IndPhi = cast<VPWidenInductionRecipe>(Val: &R)->getPHINode();
7683 ResumeV = IRPhiToResumeForEpi.at(Val: IndPhi)->getUnderlyingValue();
7684 }
7685 assert(ResumeV && "Must have a resume value");
7686 VPValue *StartVal = Plan.getOrAddLiveIn(V: ResumeV);
7687 cast<VPHeaderPHIRecipe>(Val: &R)->setStartValue(StartVal);
7688 }
7689
7690 // For some VPValues in the epilogue plan we must re-use the generated IR
7691 // values from the main plan. Replace them with live-in VPValues.
7692 // TODO: This is a workaround needed for epilogue vectorization and it
7693 // should be removed once induction resume value creation is done
7694 // directly in VPlan.
7695 for (auto &R : make_early_inc_range(Range&: *Plan.getEntry())) {
7696 // Re-use frozen values from the main plan for Freeze VPInstructions in the
7697 // epilogue plan. This ensures all users use the same frozen value.
7698 auto *VPI = dyn_cast<VPInstruction>(Val: &R);
7699 if (VPI && VPI->getOpcode() == Instruction::Freeze) {
7700 VPI->replaceAllUsesWith(New: Plan.getOrAddLiveIn(
7701 V: ToFrozen.lookup(Val: VPI->getOperand(N: 0)->getLiveInIRValue())));
7702 continue;
7703 }
7704
7705 // Re-use the trip count and steps expanded for the main loop, as
7706 // skeleton creation needs it as a value that dominates both the scalar
7707 // and vector epilogue loops
7708 auto *ExpandR = dyn_cast<VPExpandSCEVRecipe>(Val: &R);
7709 if (!ExpandR)
7710 continue;
7711 VPValue *ExpandedVal =
7712 Plan.getOrAddLiveIn(V: ExpandedSCEVs.lookup(Val: ExpandR->getSCEV()));
7713 ExpandR->replaceAllUsesWith(New: ExpandedVal);
7714 if (Plan.getTripCount() == ExpandR)
7715 Plan.resetTripCount(NewTripCount: ExpandedVal);
7716 ExpandR->eraseFromParent();
7717 }
7718
7719 auto VScale = Config.getVScaleForTuning();
7720 unsigned MainLoopStep =
7721 estimateElementCount(VF: EPI.MainLoopVF * EPI.MainLoopUF, VScale);
7722 unsigned EpilogueLoopStep =
7723 estimateElementCount(VF: EPI.EpilogueVF * EPI.EpilogueUF, VScale);
7724 RUN_VPLAN_PASS(
7725 VPlanTransforms::addMinimumVectorEpilogueIterationCheck, Plan,
7726 EPI.VectorTripCount, LVP.requiresScalarEpilogue(Plan, EPI.EpilogueVF),
7727 EPI.EpilogueVF, EPI.EpilogueUF, MainLoopStep, EpilogueLoopStep, SE);
7728
7729 return InstsToMove;
7730}
7731
7732static void
7733fixScalarResumeValuesFromBypass(BasicBlock *BypassBlock, Loop *L,
7734 VPlan &BestEpiPlan,
7735 ArrayRef<VPInstruction *> ResumeValues) {
7736 // Fix resume values from the additional bypass block.
7737 BasicBlock *PH = L->getLoopPreheader();
7738 for (auto *Pred : predecessors(BB: PH)) {
7739 for (PHINode &Phi : PH->phis()) {
7740 if (Phi.getBasicBlockIndex(BB: Pred) != -1)
7741 continue;
7742 Phi.addIncoming(V: Phi.getIncomingValueForBlock(BB: BypassBlock), BB: Pred);
7743 }
7744 }
7745 auto *ScalarPH = cast<VPIRBasicBlock>(Val: BestEpiPlan.getScalarPreheader());
7746 if (ScalarPH->hasPredecessors()) {
7747 // Fix resume values for inductions and reductions from the additional
7748 // bypass block using the incoming values from the main loop's resume phis.
7749 // ResumeValues correspond 1:1 with the scalar loop header phis.
7750 for (auto [ResumeV, HeaderPhi] :
7751 zip(t&: ResumeValues, u: BestEpiPlan.getScalarHeader()->phis())) {
7752 auto *HeaderPhiR = cast<VPIRPhi>(Val: &HeaderPhi);
7753 auto *EpiResumePhi =
7754 cast<PHINode>(Val: HeaderPhiR->getIRPhi().getIncomingValueForBlock(BB: PH));
7755 if (EpiResumePhi->getBasicBlockIndex(BB: BypassBlock) == -1)
7756 continue;
7757 auto *MainResumePhi = cast<PHINode>(Val: ResumeV->getUnderlyingValue());
7758 EpiResumePhi->setIncomingValueForBlock(
7759 BB: BypassBlock, V: MainResumePhi->getIncomingValueForBlock(BB: BypassBlock));
7760 }
7761 }
7762}
7763
7764/// Connect the epilogue vector loop generated for \p EpiPlan to the main vector
7765/// loop, after both plans have executed, updating branches from the iteration
7766/// and runtime checks of the main loop, as well as updating various phis. \p
7767/// InstsToMove contains instructions that need to be moved to the preheader of
7768/// the epilogue vector loop.
7769static void connectEpilogueVectorLoop(VPlan &EpiPlan, Loop *L,
7770 EpilogueLoopVectorizationInfo &EPI,
7771 DominatorTree *DT,
7772 GeneratedRTChecks &Checks,
7773 ArrayRef<Instruction *> InstsToMove,
7774 ArrayRef<VPInstruction *> ResumeValues) {
7775 BasicBlock *VecEpilogueIterationCountCheck =
7776 cast<VPIRBasicBlock>(Val: EpiPlan.getEntry())->getIRBasicBlock();
7777
7778 BasicBlock *VecEpiloguePreHeader =
7779 cast<CondBrInst>(Val: VecEpilogueIterationCountCheck->getTerminator())
7780 ->getSuccessor(i: 1);
7781 // Adjust the control flow taking the state info from the main loop
7782 // vectorization into account.
7783 assert(EPI.MainLoopIterationCountCheck && EPI.EpilogueIterationCountCheck &&
7784 "expected this to be saved from the previous pass.");
7785 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
7786
7787 // Helper to redirect an edge from \p BB to \p VecEpilogueIterationCountCheck
7788 // to \p NewSucc instead, updating the DomTree.
7789 auto RedirectEdge = [&](BasicBlock *BB, BasicBlock *NewSucc) {
7790 BB->getTerminator()->replaceUsesOfWith(From: VecEpilogueIterationCountCheck,
7791 To: NewSucc);
7792 DTU.applyUpdates(
7793 Updates: {{DominatorTree::Delete, BB, VecEpilogueIterationCountCheck},
7794 {DominatorTree::Insert, BB, NewSucc}});
7795 };
7796
7797 RedirectEdge(EPI.MainLoopIterationCountCheck, VecEpiloguePreHeader);
7798
7799 BasicBlock *ScalarPH =
7800 cast<VPIRBasicBlock>(Val: EpiPlan.getScalarPreheader())->getIRBasicBlock();
7801 RedirectEdge(EPI.EpilogueIterationCountCheck, ScalarPH);
7802
7803 // Adjust the terminators of runtime check blocks and phis using them.
7804 BasicBlock *SCEVCheckBlock = Checks.getSCEVChecks().second;
7805 BasicBlock *MemCheckBlock = Checks.getMemRuntimeChecks().second;
7806 if (SCEVCheckBlock)
7807 RedirectEdge(SCEVCheckBlock, ScalarPH);
7808 if (MemCheckBlock)
7809 RedirectEdge(MemCheckBlock, ScalarPH);
7810
7811 // The vec.epilog.iter.check block may contain Phi nodes from inductions
7812 // or reductions which merge control-flow from the latch block and the
7813 // middle block. Update the incoming values here and move the Phi into the
7814 // preheader.
7815 SmallVector<PHINode *, 4> PhisInBlock(
7816 llvm::make_pointer_range(Range: VecEpilogueIterationCountCheck->phis()));
7817
7818 for (PHINode *Phi : PhisInBlock) {
7819 Phi->moveBefore(InsertPos: VecEpiloguePreHeader->getFirstNonPHIIt());
7820 Phi->replaceIncomingBlockWith(
7821 Old: VecEpilogueIterationCountCheck->getSinglePredecessor(),
7822 New: VecEpilogueIterationCountCheck);
7823
7824 // If the phi doesn't have an incoming value from the
7825 // EpilogueIterationCountCheck, we are done. Otherwise remove the
7826 // incoming value and also those from other check blocks. This is needed
7827 // for reduction phis only.
7828 if (none_of(Range: Phi->blocks(), P: [&](BasicBlock *IncB) {
7829 return EPI.EpilogueIterationCountCheck == IncB;
7830 }))
7831 continue;
7832 for (BasicBlock *BB :
7833 {EPI.EpilogueIterationCountCheck, SCEVCheckBlock, MemCheckBlock}) {
7834 if (BB)
7835 Phi->removeIncomingValue(BB);
7836 }
7837 }
7838
7839 auto IP = VecEpiloguePreHeader->getFirstNonPHIIt();
7840 for (auto *I : InstsToMove)
7841 I->moveBefore(InsertPos: IP);
7842
7843 // VecEpilogueIterationCountCheck conditionally skips over the epilogue loop
7844 // after executing the main loop. We need to update the resume values of
7845 // inductions and reductions during epilogue vectorization.
7846 fixScalarResumeValuesFromBypass(BypassBlock: VecEpilogueIterationCountCheck, L, BestEpiPlan&: EpiPlan,
7847 ResumeValues);
7848
7849 // Remove dead phis that were moved to the epilogue preheader but are unused
7850 // (e.g., resume phis for inductions not widened in the epilogue vector loop).
7851 for (PHINode &Phi : make_early_inc_range(Range: VecEpiloguePreHeader->phis()))
7852 if (Phi.use_empty())
7853 Phi.eraseFromParent();
7854}
7855
7856bool LoopVectorizePass::processLoop(Loop *L) {
7857 assert((EnableVPlanNativePath || L->isInnermost()) &&
7858 "VPlan-native path is not enabled. Only process inner loops.");
7859
7860 LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in '"
7861 << L->getHeader()->getParent()->getName() << "' from "
7862 << L->getLocStr() << "\n");
7863
7864 LoopVectorizeHints Hints(L, InterleaveOnlyWhenForced, *ORE, TTI);
7865
7866 LLVM_DEBUG(
7867 dbgs() << "LV: Loop hints:"
7868 << " force="
7869 << (Hints.getForce() == LoopVectorizeHints::FK_Disabled
7870 ? "disabled"
7871 : (Hints.getForce() == LoopVectorizeHints::FK_Enabled
7872 ? "enabled"
7873 : "?"))
7874 << " width=" << Hints.getWidth()
7875 << " interleave=" << Hints.getInterleave() << "\n");
7876
7877 // Function containing loop
7878 Function *F = L->getHeader()->getParent();
7879
7880 // Looking at the diagnostic output is the only way to determine if a loop
7881 // was vectorized (other than looking at the IR or machine code), so it
7882 // is important to generate an optimization remark for each loop. Most of
7883 // these messages are generated as OptimizationRemarkAnalysis. Remarks
7884 // generated as OptimizationRemark and OptimizationRemarkMissed are
7885 // less verbose reporting vectorized loops and unvectorized loops that may
7886 // benefit from vectorization, respectively.
7887
7888 if (!Hints.allowVectorization(F, L, VectorizeOnlyWhenForced)) {
7889 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
7890 return false;
7891 }
7892
7893 PredicatedScalarEvolution PSE(*SE, *L);
7894
7895 // Query this against the original loop and save it here because the profile
7896 // of the original loop header may change as the transformation happens.
7897 bool OptForSize = llvm::shouldOptimizeForSize(
7898 BB: L->getHeader(), PSI,
7899 BFI: PSI && PSI->hasProfileSummary() ? &GetBFI() : nullptr,
7900 QueryType: PGSOQueryType::IRPass);
7901
7902 // Check if it is legal to vectorize the loop.
7903 LoopVectorizationRequirements Requirements;
7904 LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE,
7905 &Requirements, &Hints, DB, AC,
7906 /*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
7907 if (!LVL.canVectorize(UseVPlanNativePath: EnableVPlanNativePath)) {
7908 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
7909 Hints.emitRemarkWithHints();
7910 return false;
7911 }
7912
7913 bool IsInnerLoop = L->isInnermost();
7914
7915 // Outer loops require a computable trip count.
7916 if (!IsInnerLoop && isa<SCEVCouldNotCompute>(Val: PSE.getBackedgeTakenCount())) {
7917 LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n");
7918 return false;
7919 }
7920
7921 if (LVL.hasUncountableEarlyExit()) {
7922 if (!EnableEarlyExitVectorization) {
7923 reportVectorizationFailure(DebugMsg: "Auto-vectorization of loops with uncountable "
7924 "early exit is not enabled",
7925 ORETag: "UncountableEarlyExitLoopsDisabled", ORE, TheLoop: L);
7926 return false;
7927 }
7928 if (LVL.hasUncountableExitWithSideEffects() &&
7929 !EnableEarlyExitVectorizationWithSideEffects) {
7930 reportVectorizationFailure(DebugMsg: "Auto-vectorization of loops with uncountable "
7931 "early exit and side effects is not enabled",
7932 ORETag: "UncountableEarlyExitSideEffectLoopsDisabled",
7933 ORE, TheLoop: L);
7934 return false;
7935 }
7936 }
7937
7938 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
7939 bool UseInterleaved =
7940 IsInnerLoop && TTI->enableInterleavedAccessVectorization();
7941
7942 // If an override option has been passed in for interleaved accesses, use it.
7943 if (EnableInterleavedMemAccesses.getNumOccurrences() > 0)
7944 UseInterleaved = IsInnerLoop && EnableInterleavedMemAccesses;
7945
7946 // Analyze interleaved memory accesses.
7947 if (UseInterleaved)
7948 IAI.analyzeInterleaving(EnableMaskedInterleavedGroup: useMaskedInterleavedAccesses(TTI: *TTI));
7949
7950 if (LVL.hasUncountableEarlyExit()) {
7951 BasicBlock *LoopLatch = L->getLoopLatch();
7952 if (IAI.requiresScalarEpilogue() ||
7953 any_of(Range: LVL.getCountableExitingBlocks(), P: not_equal_to(Arg&: LoopLatch))) {
7954 reportVectorizationFailure(DebugMsg: "Auto-vectorization of early exit loops "
7955 "requiring a scalar epilogue is unsupported",
7956 ORETag: "UncountableEarlyExitUnsupported", ORE, TheLoop: L);
7957 return false;
7958 }
7959 }
7960
7961 // Check the function attributes and profiles to find out if this function
7962 // should be optimized for size.
7963 EpilogueLowering SEL =
7964 getEpilogueLowering(F, L, Hints, OptForSize, TTI, TLI, LVL, IAI: &IAI);
7965
7966 // Check the loop for a trip count threshold: vectorize loops with a tiny trip
7967 // count by optimizing for size, to minimize overheads.
7968 auto ExpectedTC = getSmallBestKnownTC(PSE, L);
7969 if (ExpectedTC && ExpectedTC->isFixed() &&
7970 ExpectedTC->getFixedValue() < TinyTripCountVectorThreshold) {
7971 LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
7972 << "This loop is worth vectorizing only if no scalar "
7973 << "iteration overheads are incurred.");
7974 if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
7975 LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
7976 else {
7977 LLVM_DEBUG(dbgs() << "\n");
7978 // Tail-folded loops are efficient even when the loop
7979 // iteration count is low. However, setting the epilogue policy to
7980 // `CM_EpilogueNotAllowedLowTripLoop` prevents vectorizing loops
7981 // with runtime checks. It's more effective to let
7982 // `isOutsideLoopWorkProfitable` determine if vectorization is
7983 // beneficial for the loop.
7984 if (SEL != CM_EpilogueNotNeededFoldTail)
7985 SEL = CM_EpilogueNotAllowedLowTripLoop;
7986 }
7987 }
7988
7989 // Check the function attributes to see if implicit floats or vectors are
7990 // allowed.
7991 if (F->hasFnAttribute(Kind: Attribute::NoImplicitFloat)) {
7992 reportVectorizationFailure(
7993 DebugMsg: "Can't vectorize when the NoImplicitFloat attribute is used",
7994 OREMsg: "loop not vectorized due to NoImplicitFloat attribute",
7995 ORETag: "NoImplicitFloat", ORE, TheLoop: L);
7996 Hints.emitRemarkWithHints();
7997 return false;
7998 }
7999
8000 // Check if the target supports potentially unsafe FP vectorization.
8001 // FIXME: Add a check for the type of safety issue (denormal, signaling)
8002 // for the target we're vectorizing for, to make sure none of the
8003 // additional fp-math flags can help.
8004 if (Hints.isPotentiallyUnsafe() &&
8005 TTI->isFPVectorizationPotentiallyUnsafe()) {
8006 reportVectorizationFailure(
8007 DebugMsg: "Potentially unsafe FP op prevents vectorization",
8008 OREMsg: "loop not vectorized due to unsafe FP support.", ORETag: "UnsafeFP", ORE, TheLoop: L);
8009 Hints.emitRemarkWithHints();
8010 return false;
8011 }
8012
8013 bool AllowOrderedReductions;
8014 // If the flag is set, use that instead and override the TTI behaviour.
8015 if (ForceOrderedReductions.getNumOccurrences() > 0)
8016 AllowOrderedReductions = ForceOrderedReductions;
8017 else
8018 AllowOrderedReductions = TTI->enableOrderedReductions();
8019 if (!LVL.canVectorizeFPMath(EnableStrictReductions: AllowOrderedReductions)) {
8020 ORE->emit(RemarkBuilder: [&]() {
8021 auto *ExactFPMathInst = Requirements.getExactFPInst();
8022 return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE, "CantReorderFPOps",
8023 ExactFPMathInst->getDebugLoc(),
8024 ExactFPMathInst->getParent())
8025 << "loop not vectorized: cannot prove it is safe to reorder "
8026 "floating-point operations";
8027 });
8028 LLVM_DEBUG(dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
8029 "reorder floating-point operations\n");
8030 Hints.emitRemarkWithHints();
8031 return false;
8032 }
8033
8034 // Use the cost model.
8035 VFSelectionContext Config(*TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
8036 OptForSize);
8037 LoopVectorizationCostModel CM(SEL, L, PSE, LI, &LVL, *TTI, TLI, AC, ORE,
8038 GetBFI, F, &Hints, IAI, Config);
8039 // Use the planner for vectorization.
8040 LoopVectorizationPlanner LVP(L, LI, DT, TLI, *TTI, &LVL, CM, Config, IAI, PSE,
8041 Hints, ORE);
8042
8043 EpilogueLowering EpilogueTailLoweringStatus =
8044 getEpilogueTailLowering(MainCM: CM, L, ORE);
8045 if (EpilogueTailLoweringStatus ==
8046 EpilogueLowering::CM_EpilogueNotNeededFoldTail) {
8047 // TODO: Apply tail-folding on the vectorized epilogue loop.
8048 LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is not supported yet\n");
8049 reportVectorizationInfo(
8050 Msg: "The epilogue-tail-folding policy prefer-fold-tail is not supported "
8051 "yet, fall back to a normal epilogue",
8052 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
8053 }
8054
8055 // Get user vectorization factor and interleave count.
8056 ElementCount UserVF = Hints.getWidth();
8057 unsigned UserIC = Hints.getInterleave();
8058 // Outer loops don't have LoopAccessInfo, so skip the safety check and reset
8059 // UserIC (interleaving is not supported for outer loops).
8060 if (!IsInnerLoop)
8061 UserIC = 0;
8062 else if (UserIC > 1 && !LVL.isSafeForAnyVectorWidth())
8063 UserIC = 1;
8064
8065 // Plan how to best vectorize.
8066 LVP.plan(UserVF, UserIC);
8067 auto [VF, BestPlanPtr] = LVP.computeBestVF();
8068 unsigned IC = 1;
8069
8070 // For VPlan build stress testing of outer loops, bail after plan
8071 // construction.
8072 if (!IsInnerLoop && VPlanBuildOuterloopStressTest)
8073 return false;
8074
8075 if (IsInnerLoop && ORE->allowExtraAnalysis(LV_NAME))
8076 LVP.emitInvalidCostRemarks(ORE);
8077
8078 assert((IsInnerLoop || !CM.maskPartialAliasing()) &&
8079 "Did not expect to alias-mask outer loop");
8080
8081 GeneratedRTChecks Checks(PSE, DT, LI, TTI, Config.CostKind,
8082 CM.maskPartialAliasing());
8083 if (IsInnerLoop && LVP.hasPlanWithVF(VF: VF.Width)) {
8084 // Select the interleave count.
8085 IC = LVP.selectInterleaveCount(Plan&: *BestPlanPtr, VF: VF.Width, LoopCost: VF.Cost);
8086
8087 unsigned SelectedIC = std::max(a: IC, b: UserIC);
8088 // Optimistically generate runtime checks if they are needed. Drop them if
8089 // they turn out to not be profitable.
8090 if (VF.Width.isVector() || SelectedIC > 1) {
8091 Checks.create(L, LAI: *LVL.getLAI(), UnionPred: PSE.getPredicate(), VF: VF.Width, IC: SelectedIC,
8092 ORE&: *ORE);
8093
8094 // Bail out early if either the SCEV or memory runtime checks are known to
8095 // fail. In that case, the vector loop would never execute.
8096 using namespace llvm::PatternMatch;
8097 if (Checks.getSCEVChecks().first &&
8098 match(V: Checks.getSCEVChecks().first, P: m_One()))
8099 return false;
8100 if (Checks.getMemRuntimeChecks().first &&
8101 match(V: Checks.getMemRuntimeChecks().first, P: m_One()))
8102 return false;
8103 }
8104
8105 // Check if it is profitable to vectorize with runtime checks.
8106 bool ForceVectorization =
8107 Hints.getForce() == LoopVectorizeHints::FK_Enabled;
8108 VPCostContext CostCtx(CM.TTI, *CM.TLI, *BestPlanPtr, CM, Config.CostKind,
8109 CM.PSE, L);
8110 if (!ForceVectorization &&
8111 !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx, Plan&: *BestPlanPtr,
8112 SEL, VScale: Config.getVScaleForTuning())) {
8113 ORE->emit(RemarkBuilder: [&]() {
8114 return OptimizationRemarkAnalysisAliasing(
8115 DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
8116 L->getHeader())
8117 << "loop not vectorized: cannot prove it is safe to reorder "
8118 "memory operations";
8119 });
8120 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
8121 Hints.emitRemarkWithHints();
8122 return false;
8123 }
8124 }
8125
8126 // Identify the diagnostic messages that should be produced.
8127 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
8128 bool VectorizeLoop = true, InterleaveLoop = true;
8129 if (VF.Width.isScalar()) {
8130 LLVM_DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n");
8131 VecDiagMsg = {
8132 "VectorizationNotBeneficial",
8133 "the cost-model indicates that vectorization is not beneficial"};
8134 VectorizeLoop = false;
8135 }
8136
8137 if (UserIC == 1 && Hints.getInterleave() > 1) {
8138 assert(!LVL.isSafeForAnyVectorWidth() &&
8139 "UserIC should only be ignored due to unsafe dependencies");
8140 LLVM_DEBUG(dbgs() << "LV: Ignoring user-specified interleave count.\n");
8141 IntDiagMsg = {"InterleavingUnsafe",
8142 "Ignoring user-specified interleave count due to possibly "
8143 "unsafe dependencies in the loop."};
8144 InterleaveLoop = false;
8145 } else if (!LVP.hasPlanWithVF(VF: VF.Width) && UserIC > 1) {
8146 // Tell the user interleaving was avoided up-front, despite being explicitly
8147 // requested.
8148 LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
8149 "interleaving should be avoided up front\n");
8150 IntDiagMsg = {"InterleavingAvoided",
8151 "Ignoring UserIC, because interleaving was avoided up front"};
8152 InterleaveLoop = false;
8153 } else if (IC == 1 && UserIC <= 1) {
8154 // Tell the user interleaving is not beneficial.
8155 LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
8156 IntDiagMsg = {
8157 "InterleavingNotBeneficial",
8158 "the cost-model indicates that interleaving is not beneficial"};
8159 InterleaveLoop = false;
8160 if (UserIC == 1) {
8161 IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
8162 IntDiagMsg.second +=
8163 " and is explicitly disabled or interleave count is set to 1";
8164 }
8165 } else if (IC > 1 && UserIC == 1) {
8166 // Tell the user interleaving is beneficial, but it explicitly disabled.
8167 LLVM_DEBUG(dbgs() << "LV: Interleaving is beneficial but is explicitly "
8168 "disabled.\n");
8169 IntDiagMsg = {"InterleavingBeneficialButDisabled",
8170 "the cost-model indicates that interleaving is beneficial "
8171 "but is explicitly disabled or interleave count is set to 1"};
8172 InterleaveLoop = false;
8173 }
8174
8175 // If there is a histogram in the loop, do not just interleave without
8176 // vectorizing. The order of operations will be incorrect without the
8177 // histogram intrinsics, which are only used for recipes with VF > 1.
8178 if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
8179 LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
8180 << "to histogram operations.\n");
8181 IntDiagMsg = {
8182 "HistogramPreventsScalarInterleaving",
8183 "Unable to interleave without vectorization due to constraints on "
8184 "the order of histogram operations"};
8185 InterleaveLoop = false;
8186 }
8187
8188 // Override IC if user provided an interleave count.
8189 IC = UserIC > 0 ? UserIC : IC;
8190
8191 if (CM.maskPartialAliasing()) {
8192 LLVM_DEBUG(
8193 dbgs()
8194 << "LV: Not interleaving due to partial aliasing vectorization.\n");
8195 IntDiagMsg = {
8196 "PartialAliasingVectorization",
8197 "Unable to interleave due to partial aliasing vectorization."};
8198 InterleaveLoop = false;
8199 IC = 1;
8200 }
8201
8202 // FIXME: Enable interleaving for EE-with-side-effects.
8203 if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) {
8204 LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n");
8205 IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving",
8206 "Unable to interleave due to early exit with side effects."};
8207 InterleaveLoop = false;
8208 IC = 1;
8209 }
8210
8211 // Emit diagnostic messages, if any.
8212 if (!VectorizeLoop && !InterleaveLoop) {
8213 // Do not vectorize or interleaving the loop.
8214 ORE->emit(RemarkBuilder: [&]() {
8215 return OptimizationRemarkMissed(LV_NAME, VecDiagMsg.first,
8216 L->getStartLoc(), L->getHeader())
8217 << VecDiagMsg.second;
8218 });
8219 ORE->emit(RemarkBuilder: [&]() {
8220 return OptimizationRemarkMissed(LV_NAME, IntDiagMsg.first,
8221 L->getStartLoc(), L->getHeader())
8222 << IntDiagMsg.second;
8223 });
8224 return false;
8225 }
8226
8227 if (!VectorizeLoop && InterleaveLoop) {
8228 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
8229 ORE->emit(RemarkBuilder: [&]() {
8230 return OptimizationRemarkAnalysis(LV_NAME, VecDiagMsg.first,
8231 L->getStartLoc(), L->getHeader())
8232 << VecDiagMsg.second;
8233 });
8234 } else if (VectorizeLoop && !InterleaveLoop) {
8235 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
8236 << ") in " << L->getLocStr() << '\n');
8237 ORE->emit(RemarkBuilder: [&]() {
8238 return OptimizationRemarkAnalysis(LV_NAME, IntDiagMsg.first,
8239 L->getStartLoc(), L->getHeader())
8240 << IntDiagMsg.second;
8241 });
8242 } else if (VectorizeLoop && InterleaveLoop) {
8243 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
8244 << ") in " << L->getLocStr() << '\n');
8245 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
8246 }
8247
8248 // Report the vectorization decision.
8249 if (VF.Width.isScalar()) {
8250 using namespace ore;
8251 assert(IC > 1);
8252 ORE->emit(RemarkBuilder: [&]() {
8253 return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(),
8254 L->getHeader())
8255 << "interleaved loop (interleaved count: "
8256 << NV("InterleaveCount", IC) << ")";
8257 });
8258 } else {
8259 // Report the vectorization decision.
8260 reportVectorization(ORE, TheLoop: L, VFWidth: VF.Width, IC);
8261 }
8262 if (ORE->allowExtraAnalysis(LV_NAME))
8263 checkMixedPrecision(L, ORE);
8264
8265 // If we decided that it is *legal* to interleave or vectorize the loop, then
8266 // do it.
8267
8268 VPlan &BestPlan = *BestPlanPtr;
8269 // Consider vectorizing the epilogue too if it's profitable.
8270 std::unique_ptr<VPlan> EpiPlan =
8271 LVP.selectBestEpiloguePlan(MainPlan&: BestPlan, MainLoopVF: VF.Width, IC);
8272 bool HasBranchWeights =
8273 hasBranchWeightMD(I: *L->getLoopLatch()->getTerminator());
8274 if (EpiPlan) {
8275 VPlan &BestEpiPlan = *EpiPlan;
8276 VPlan &BestMainPlan = BestPlan;
8277 ElementCount EpilogueVF = BestEpiPlan.getSingleVF();
8278
8279 // The first pass vectorizes the main loop and creates a scalar epilogue
8280 // to be vectorized by executing the plan (potentially with a different
8281 // factor) again shortly afterwards.
8282 BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
8283 BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
8284 SmallVector<VPInstruction *> ResumeValues =
8285 preparePlanForMainVectorLoop(MainPlan&: BestMainPlan, EpiPlan&: BestEpiPlan);
8286 EpilogueLoopVectorizationInfo EPI(VF.Width, IC, EpilogueVF, 1, BestEpiPlan);
8287
8288 // Add minimum iteration check for the epilogue plan, followed by runtime
8289 // checks for the main plan.
8290 LVP.addMinimumIterationCheck(Plan&: BestMainPlan, VF: EPI.EpilogueVF, UF: EPI.EpilogueUF,
8291 MinProfitableTripCount: ElementCount::getFixed(MinVal: 0));
8292 LVP.attachRuntimeChecks(Plan&: BestMainPlan, RTChecks&: Checks, HasBranchWeights);
8293 RUN_VPLAN_PASS(VPlanTransforms::addIterationCountCheckBlock, BestMainPlan,
8294 EPI.MainLoopVF, EPI.MainLoopUF,
8295 LVP.requiresScalarEpilogue(BestMainPlan, EPI.MainLoopVF), L,
8296 HasBranchWeights ? MinItersBypassWeights : nullptr,
8297 L->getLoopPredecessor()->getTerminator()->getDebugLoc(),
8298 PSE);
8299
8300 EpilogueVectorizerMainLoop MainILV(L, PSE, LI, DT, TTI, AC, EPI, &CM,
8301 Checks, BestMainPlan);
8302 auto ExpandedSCEVs = LVP.executePlan(
8303 BestVF: EPI.MainLoopVF, BestUF: EPI.MainLoopUF, BestVPlan&: BestMainPlan, ILV&: MainILV, DT,
8304 EpilogueVecKind: LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop);
8305 ++LoopsVectorized;
8306
8307 // Derive EPI fields from VPlan-generated IR.
8308 BasicBlock *EntryBB =
8309 cast<VPIRBasicBlock>(Val: BestMainPlan.getEntry())->getIRBasicBlock();
8310 EntryBB->setName("iter.check");
8311 EPI.EpilogueIterationCountCheck = EntryBB;
8312 // The check chain is: Entry -> [SCEV] -> [Mem] -> MainCheck -> VecPH.
8313 // MainCheck is the non-bypass successor of the last runtime check block
8314 // (or Entry if there are no runtime checks).
8315 BasicBlock *LastCheck = EntryBB;
8316 if (BasicBlock *MemBB = Checks.getMemRuntimeChecks().second)
8317 LastCheck = MemBB;
8318 else if (BasicBlock *SCEVBB = Checks.getSCEVChecks().second)
8319 LastCheck = SCEVBB;
8320 BasicBlock *ScalarPH = L->getLoopPreheader();
8321 auto *BI = cast<CondBrInst>(Val: LastCheck->getTerminator());
8322 EPI.MainLoopIterationCountCheck =
8323 BI->getSuccessor(i: BI->getSuccessor(i: 0) == ScalarPH);
8324
8325 // Second pass vectorizes the epilogue and adjusts the control flow
8326 // edges from the first pass.
8327 EpilogueVectorizerEpilogueLoop EpilogILV(L, PSE, LI, DT, TTI, AC, EPI, &CM,
8328 Checks, BestEpiPlan);
8329 SmallVector<Instruction *> InstsToMove = preparePlanForEpilogueVectorLoop(
8330 MainPlan&: BestMainPlan, Plan&: BestEpiPlan, L, ExpandedSCEVs, EPI, LVP, Config,
8331 SE&: *PSE.getSE(), ResumeValues);
8332 LVP.attachRuntimeChecks(Plan&: BestEpiPlan, RTChecks&: Checks, HasBranchWeights);
8333 LVP.executePlan(
8334 BestVF: EPI.EpilogueVF, BestUF: EPI.EpilogueUF, BestVPlan&: BestEpiPlan, ILV&: EpilogILV, DT,
8335 EpilogueVecKind: LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
8336 connectEpilogueVectorLoop(EpiPlan&: BestEpiPlan, L, EPI, DT, Checks, InstsToMove,
8337 ResumeValues);
8338 ++LoopsEpilogueVectorized;
8339 } else {
8340 InnerLoopVectorizer LB(L, PSE, LI, DT, TTI, AC, VF.Width, IC, &CM, Checks,
8341 BestPlan);
8342 LVP.addMinimumIterationCheck(Plan&: BestPlan, VF: VF.Width, UF: IC,
8343 MinProfitableTripCount: VF.MinProfitableTripCount);
8344 LVP.attachRuntimeChecks(Plan&: BestPlan, RTChecks&: Checks, HasBranchWeights);
8345
8346 if (!IsInnerLoop)
8347 LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" << F->getName()
8348 << "\"\n");
8349 LVP.executePlan(BestVF: VF.Width, BestUF: IC, BestVPlan&: BestPlan, ILV&: LB, DT);
8350 ++LoopsVectorized;
8351 }
8352
8353 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
8354 "DT not preserved correctly");
8355 assert(!verifyFunction(*F, &dbgs()));
8356
8357 return true;
8358}
8359
8360LoopVectorizeResult LoopVectorizePass::runImpl(Function &F) {
8361
8362 // Don't attempt if
8363 // 1. the target claims to have no vector registers, and
8364 // 2. interleaving won't help ILP.
8365 //
8366 // The second condition is necessary because, even if the target has no
8367 // vector registers, loop vectorization may still enable scalar
8368 // interleaving.
8369 if (!TTI->getNumberOfRegisters(ClassID: TTI->getRegisterClassForType(Vector: true)) &&
8370 (TTI->getMaxInterleaveFactor(VF: ElementCount::getFixed(MinVal: 1), HasUnorderedReductions: false) < 2 ||
8371 TTI->getMaxInterleaveFactor(VF: ElementCount::getFixed(MinVal: 1), HasUnorderedReductions: true) < 2))
8372 return LoopVectorizeResult(false, false);
8373
8374 bool Changed = false, CFGChanged = false;
8375
8376 // The vectorizer requires loops to be in simplified form.
8377 // Since simplification may add new inner loops, it has to run before the
8378 // legality and profitability checks. This means running the loop vectorizer
8379 // will simplify all loops, regardless of whether anything end up being
8380 // vectorized.
8381 for (const auto &L : *LI)
8382 Changed |= CFGChanged |=
8383 simplifyLoop(L, DT, LI, SE, AC, MSSAU: nullptr, PreserveLCSSA: false /* PreserveLCSSA */);
8384
8385 // Build up a worklist of inner-loops to vectorize. This is necessary as
8386 // the act of vectorizing or partially unrolling a loop creates new loops
8387 // and can invalidate iterators across the loops.
8388 SmallVector<Loop *, 8> Worklist;
8389
8390 for (Loop *L : *LI)
8391 collectSupportedLoops(L&: *L, LI, ORE, V&: Worklist);
8392
8393 LoopsAnalyzed += Worklist.size();
8394
8395 // Now walk the identified inner loops.
8396 while (!Worklist.empty()) {
8397 Loop *L = Worklist.pop_back_val();
8398
8399 // For the inner loops we actually process, form LCSSA to simplify the
8400 // transform.
8401 Changed |= formLCSSARecursively(L&: *L, DT: *DT, LI, SE);
8402
8403 Changed |= CFGChanged |= processLoop(L);
8404
8405 if (Changed) {
8406 LAIs->clear();
8407
8408#ifndef NDEBUG
8409 if (VerifySCEV)
8410 SE->verify();
8411#endif
8412 }
8413 }
8414
8415 // Process each loop nest in the function.
8416 return LoopVectorizeResult(Changed, CFGChanged);
8417}
8418
8419PreservedAnalyses LoopVectorizePass::run(Function &F,
8420 FunctionAnalysisManager &AM) {
8421 LI = &AM.getResult<LoopAnalysis>(IR&: F);
8422 // There are no loops in the function. Return before computing other
8423 // expensive analyses.
8424 if (LI->empty())
8425 return PreservedAnalyses::all();
8426 SE = &AM.getResult<ScalarEvolutionAnalysis>(IR&: F);
8427 TTI = &AM.getResult<TargetIRAnalysis>(IR&: F);
8428 DT = &AM.getResult<DominatorTreeAnalysis>(IR&: F);
8429 TLI = &AM.getResult<TargetLibraryAnalysis>(IR&: F);
8430 AC = &AM.getResult<AssumptionAnalysis>(IR&: F);
8431 DB = &AM.getResult<DemandedBitsAnalysis>(IR&: F);
8432 ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F);
8433 LAIs = &AM.getResult<LoopAccessAnalysis>(IR&: F);
8434 AA = &AM.getResult<AAManager>(IR&: F);
8435
8436 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
8437 PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(IR&: *F.getParent());
8438 GetBFI = [&AM, &F]() -> BlockFrequencyInfo & {
8439 return AM.getResult<BlockFrequencyAnalysis>(IR&: F);
8440 };
8441 LoopVectorizeResult Result = runImpl(F);
8442 if (!Result.MadeAnyChange)
8443 return PreservedAnalyses::all();
8444 PreservedAnalyses PA;
8445
8446 if (isAssignmentTrackingEnabled(M: *F.getParent())) {
8447 for (auto &BB : F)
8448 RemoveRedundantDbgInstrs(BB: &BB);
8449 }
8450
8451 PA.preserve<LoopAnalysis>();
8452 PA.preserve<DominatorTreeAnalysis>();
8453 PA.preserve<ScalarEvolutionAnalysis>();
8454 PA.preserve<LoopAccessAnalysis>();
8455
8456 if (Result.MadeCFGChange) {
8457 // Making CFG changes likely means a loop got vectorized. Indicate that
8458 // extra simplification passes should be run.
8459 // TODO: MadeCFGChanges is not a prefect proxy. Extra passes should only
8460 // be run if runtime checks have been added.
8461 AM.getResult<ShouldRunExtraVectorPasses>(IR&: F);
8462 PA.preserve<ShouldRunExtraVectorPasses>();
8463 } else {
8464 PA.preserveSet<CFGAnalyses>();
8465 }
8466 return PA;
8467}
8468
8469void LoopVectorizePass::printPipeline(
8470 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
8471 static_cast<PassInfoMixin<LoopVectorizePass> *>(this)->printPipeline(
8472 OS, MapClassName2PassName);
8473
8474 OS << '<';
8475 OS << (InterleaveOnlyWhenForced ? "" : "no-") << "interleave-forced-only;";
8476 OS << (VectorizeOnlyWhenForced ? "" : "no-") << "vectorize-forced-only;";
8477 OS << '>';
8478}
8479