1//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
10// and generates target-independent LLVM-IR.
11// The vectorizer uses the TargetTransformInfo analysis to estimate the costs
12// of instructions in order to estimate the profitability of vectorization.
13//
14// The loop vectorizer combines consecutive loop iterations into a single
15// 'wide' iteration. After this transformation the index is incremented
16// by the SIMD vector width, and not by one.
17//
18// This pass has four parts:
19// 1. The main loop pass that drives the different parts.
20// 2. LoopVectorizationLegality - A unit that checks for the legality
21// of the vectorization.
22// 3. LoopVectorizationPlanner - A unit that builds VPlans for the candidate
23// vectorization factors, selects the best one and executes it to generate
24// the vector code.
25// 4. LoopVectorizationCostModel - A unit that checks for the profitability
26// of vectorization. It decides on the optimal vector width, which
27// can be one, if vectorization is not profitable.
28//
29// There is a development effort going on to migrate loop vectorizer to the
30// VPlan infrastructure and to introduce outer loop vectorization support (see
31// docs/VectorizationPlan.rst and
32// http://lists.llvm.org/pipermail/llvm-dev/2017-December/119523.html). For this
33// purpose, we temporarily introduced the VPlan-native vectorization path: an
34// alternative vectorization path that is natively implemented on top of the
35// VPlan infrastructure. See EnableVPlanNativePath for enabling.
36//
37//===----------------------------------------------------------------------===//
38//
39// The reduction-variable vectorization is based on the paper:
40// D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
41//
42// Variable uniformity checks are inspired by:
43// Karrenberg, R. and Hack, S. Whole Function Vectorization.
44//
45// The interleaved access vectorization is based on the paper:
46// Dorit Nuzman, Ira Rosen and Ayal Zaks. Auto-Vectorization of Interleaved
47// Data for SIMD
48//
49// Other ideas/concepts are from:
50// A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
51//
52// S. Maleki, Y. Gao, M. Garzaran, T. Wong and D. Padua. An Evaluation of
53// Vectorizing Compilers.
54//
55//===----------------------------------------------------------------------===//
56
57#include "llvm/Transforms/Vectorize/LoopVectorize.h"
58#include "LoopVectorizationPlanner.h"
59#include "VPRecipeBuilder.h"
60#include "VPlan.h"
61#include "VPlanAnalysis.h"
62#include "VPlanCFG.h"
63#include "VPlanHelpers.h"
64#include "VPlanPatternMatch.h"
65#include "VPlanTransforms.h"
66#include "VPlanUtils.h"
67#include "VPlanVerifier.h"
68#include "VectorizeOptions.h"
69#include "llvm/ADT/APInt.h"
70#include "llvm/ADT/ArrayRef.h"
71#include "llvm/ADT/DenseMap.h"
72#include "llvm/ADT/Hashing.h"
73#include "llvm/ADT/MapVector.h"
74#include "llvm/ADT/STLExtras.h"
75#include "llvm/ADT/SmallPtrSet.h"
76#include "llvm/ADT/SmallVector.h"
77#include "llvm/ADT/Statistic.h"
78#include "llvm/ADT/StringRef.h"
79#include "llvm/ADT/Twine.h"
80#include "llvm/ADT/TypeSwitch.h"
81#include "llvm/ADT/iterator_range.h"
82#include "llvm/Analysis/AssumptionCache.h"
83#include "llvm/Analysis/BasicAliasAnalysis.h"
84#include "llvm/Analysis/BlockFrequencyInfo.h"
85#include "llvm/Analysis/BranchProbabilityInfo.h"
86#include "llvm/Analysis/CFG.h"
87#include "llvm/Analysis/CodeMetrics.h"
88#include "llvm/Analysis/CycleAnalysis.h"
89#include "llvm/Analysis/DemandedBits.h"
90#include "llvm/Analysis/GlobalsModRef.h"
91#include "llvm/Analysis/LoopAccessAnalysis.h"
92#include "llvm/Analysis/LoopAnalysisManager.h"
93#include "llvm/Analysis/LoopInfo.h"
94#include "llvm/Analysis/LoopIterator.h"
95#include "llvm/Analysis/OptimizationRemarkEmitter.h"
96#include "llvm/Analysis/ProfileSummaryInfo.h"
97#include "llvm/Analysis/ScalarEvolution.h"
98#include "llvm/Analysis/ScalarEvolutionExpressions.h"
99#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
100#include "llvm/Analysis/TargetLibraryInfo.h"
101#include "llvm/Analysis/TargetTransformInfo.h"
102#include "llvm/Analysis/ValueTracking.h"
103#include "llvm/Analysis/VectorUtils.h"
104#include "llvm/IR/Attributes.h"
105#include "llvm/IR/BasicBlock.h"
106#include "llvm/IR/CFG.h"
107#include "llvm/IR/Constant.h"
108#include "llvm/IR/Constants.h"
109#include "llvm/IR/DataLayout.h"
110#include "llvm/IR/DebugInfo.h"
111#include "llvm/IR/DebugLoc.h"
112#include "llvm/IR/DerivedTypes.h"
113#include "llvm/IR/DiagnosticInfo.h"
114#include "llvm/IR/Dominators.h"
115#include "llvm/IR/Function.h"
116#include "llvm/IR/IRBuilder.h"
117#include "llvm/IR/InstrTypes.h"
118#include "llvm/IR/Instruction.h"
119#include "llvm/IR/Instructions.h"
120#include "llvm/IR/IntrinsicInst.h"
121#include "llvm/IR/Intrinsics.h"
122#include "llvm/IR/MDBuilder.h"
123#include "llvm/IR/Metadata.h"
124#include "llvm/IR/Module.h"
125#include "llvm/IR/Operator.h"
126#include "llvm/IR/PatternMatch.h"
127#include "llvm/IR/ProfDataUtils.h"
128#include "llvm/IR/Type.h"
129#include "llvm/IR/Use.h"
130#include "llvm/IR/User.h"
131#include "llvm/IR/Value.h"
132#include "llvm/IR/Verifier.h"
133#include "llvm/Support/Casting.h"
134#include "llvm/Support/CommandLine.h"
135#include "llvm/Support/Debug.h"
136#include "llvm/Support/ErrorHandling.h"
137#include "llvm/Support/InstructionCost.h"
138#include "llvm/Support/MathExtras.h"
139#include "llvm/Support/NativeFormatting.h"
140#include "llvm/Support/raw_ostream.h"
141#include "llvm/Transforms/Utils/BasicBlockUtils.h"
142#include "llvm/Transforms/Utils/InjectTLIMappings.h"
143#include "llvm/Transforms/Utils/Local.h"
144#include "llvm/Transforms/Utils/LoopSimplify.h"
145#include "llvm/Transforms/Utils/LoopUtils.h"
146#include "llvm/Transforms/Utils/LoopVersioning.h"
147#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
148#include "llvm/Transforms/Utils/SizeOpts.h"
149#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
150#include <algorithm>
151#include <cassert>
152#include <cmath>
153#include <cstdint>
154#include <functional>
155#include <iterator>
156#include <memory>
157#include <string>
158#include <tuple>
159#include <utility>
160
161using namespace llvm;
162using namespace SCEVPatternMatch;
163using namespace LoopVectorizationUtils;
164
165#define LV_NAME "loop-vectorize"
166#define DEBUG_TYPE LV_NAME
167
168#ifndef NDEBUG
169const char VerboseDebug[] = DEBUG_TYPE "-verbose";
170#endif
171
172STATISTIC(LoopsVectorized, "Number of loops vectorized");
173STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
174STATISTIC(LoopsEpilogueVectorized, "Number of epilogues vectorized");
175STATISTIC(LoopsEarlyExitVectorized, "Number of early exit loops vectorized");
176STATISTIC(LoopsPartialAliasVectorized,
177 "Number of partial aliasing loops vectorized");
178
179static cl::opt<ElementCount> EpilogueVectorizationForceVF(
180 "epilogue-vectorization-force-VF", cl::init(Val: ElementCount::getFixed(MinVal: 1)),
181 cl::Hidden,
182 cl::desc("When epilogue vectorization is enabled, and a value greater than "
183 "1 is specified, forces the given VF for all applicable epilogue "
184 "loops. Note: This allows all scalable VFs >= vscale x 1."));
185
186// Returns true if the epilogue VF has been set to a non-zero value other than
187// VF=1 (scalar).
188static bool hasForcedEpilogueVF() {
189 return EpilogueVectorizationForceVF.isNonZero() &&
190 EpilogueVectorizationForceVF != ElementCount::getFixed(MinVal: 1);
191}
192
193// Likelyhood of bypassing the vectorized loop because there are zero trips left
194// after prolog. See `emitIterationCountCheck`.
195static constexpr uint32_t MinItersBypassWeights[] = {1, 127};
196
197/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
198/// ElementCount to include loops whose trip count is a function of vscale.
199static ElementCount getSmallConstantTripCount(ScalarEvolution *SE,
200 const Loop *L) {
201 if (unsigned ExpectedTC = SE->getSmallConstantTripCount(L))
202 return ElementCount::getFixed(MinVal: ExpectedTC);
203
204 const SCEV *BTC = SE->getBackedgeTakenCount(L);
205 if (isa<SCEVCouldNotCompute>(Val: BTC))
206 return ElementCount::getFixed(MinVal: 0);
207
208 const SCEV *ExitCount = SE->getTripCountFromExitCount(ExitCount: BTC, EvalTy: BTC->getType(), L);
209 if (isa<SCEVVScale>(Val: ExitCount))
210 return ElementCount::getScalable(MinVal: 1);
211
212 const APInt *Scale;
213 if (match(S: ExitCount, P: m_scev_Mul(Op0: m_scev_APInt(C&: Scale), Op1: m_SCEVVScale())))
214 if (cast<SCEVMulExpr>(Val: ExitCount)->hasNoUnsignedWrap())
215 if (Scale->getActiveBits() <= 32)
216 return ElementCount::getScalable(MinVal: Scale->getZExtValue());
217
218 return ElementCount::getFixed(MinVal: 0);
219}
220
221/// Get the maximum trip count for \p L from the SCEV unsigned range, excluding
222/// zero from the range. Only valid when not folding the tail, as the minimum
223/// iteration count check guards against a zero trip count. Returns 0 if
224/// unknown.
225static unsigned getMaxTCFromNonZeroRange(PredicatedScalarEvolution &PSE,
226 Loop *L) {
227 const SCEV *BTC = PSE.getBackedgeTakenCount();
228 if (isa<SCEVCouldNotCompute>(Val: BTC))
229 return 0;
230 ScalarEvolution *SE = PSE.getSE();
231 const SCEV *TripCount = SE->getTripCountFromExitCount(ExitCount: BTC, EvalTy: BTC->getType(), L);
232 ConstantRange TCRange =
233 SE->getUnsignedRange(S: SE->applyLoopGuards(Expr: TripCount, L));
234 APInt MaxTCFromRange = TCRange.getUnsignedMax();
235 if (!MaxTCFromRange.isZero() && MaxTCFromRange.getActiveBits() <= 32)
236 return MaxTCFromRange.getZExtValue();
237 return 0;
238}
239
240/// Returns "best known" trip count, which is either a valid positive trip count
241/// or std::nullopt when an estimate cannot be made (including when the trip
242/// count would overflow), for the specified loop \p L as defined by the
243/// following procedure:
244/// 1) Returns exact trip count if it is known.
245/// 2) Returns expected trip count according to profile data if any.
246/// 3) Returns upper bound estimate if known, if \p CanUseConstantMax, and
247/// if \p ComputeUpperBoundOnly is false.
248/// 4) Returns the maximum trip count from the SCEV range excluding zero,
249/// if \p CanUseConstantMax and \p CanExcludeZeroTrips.
250/// 5) Returns std::nullopt if all of the above failed.
251static std::optional<ElementCount> getSmallBestKnownTC(
252 const VectorizeOptions &Opts, PredicatedScalarEvolution &PSE, Loop *L,
253 bool CanUseConstantMax = true, bool CanExcludeZeroTrips = false,
254 bool ComputeUpperBoundOnly = false) {
255 // Check if exact trip count is known.
256 if (auto ExpectedTC = getSmallConstantTripCount(SE: PSE.getSE(), L))
257 return ExpectedTC;
258
259 // Check if there is an expected trip count available from profile data.
260 // An estimate of zero means the loop is estimated not to be entered; it is
261 // not a usable trip count for the profitability decisions below (and would
262 // e.g. divide by zero when scaling runtime check cost), so treat it as
263 // unknown.
264 if (Opts.loop_vectorize_with_block_frequency && !ComputeUpperBoundOnly)
265 if (unsigned EstimatedTC = getLoopEstimatedTripCount(L).value_or(u: 0))
266 return ElementCount::getFixed(MinVal: EstimatedTC);
267
268 if (!CanUseConstantMax)
269 return std::nullopt;
270
271 // Check if upper bound estimate is known.
272 if (unsigned ExpectedTC = PSE.getSmallConstantMaxTripCount())
273 return ElementCount::getFixed(MinVal: ExpectedTC);
274
275 // Get the maximum trip count from the SCEV range excluding zero. This is
276 // only safe when not folding the tail, as the minimum iteration count check
277 // prevents entering the vector loop with a zero trip count.
278 if (CanUseConstantMax && CanExcludeZeroTrips)
279 if (unsigned RefinedTC = getMaxTCFromNonZeroRange(PSE, L))
280 return ElementCount::getFixed(MinVal: RefinedTC);
281
282 return std::nullopt;
283}
284
285namespace {
286using SCEV2ValueTy = DenseMap<const SCEV *, Value *>;
287} // namespace
288
289namespace llvm {
290
291AnalysisKey ShouldRunExtraVectorPasses::Key;
292
293} // end namespace llvm
294
295/// Look for a meaningful debug location on the instruction or its operands.
296static DebugLoc getDebugLocFromInstOrOperands(Instruction *I) {
297 if (!I)
298 return DebugLoc::getUnknown();
299
300 DebugLoc Empty;
301 if (I->getDebugLoc() != Empty)
302 return I->getDebugLoc();
303
304 for (Use &Op : I->operands()) {
305 if (Instruction *OpInst = dyn_cast<Instruction>(Val&: Op))
306 if (OpInst->getDebugLoc() != Empty)
307 return OpInst->getDebugLoc();
308 }
309
310 return I->getDebugLoc();
311}
312
313namespace llvm {
314
315/// Return the runtime value for VF.
316Value *getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF) {
317 return B.CreateElementCount(Ty, EC: VF);
318}
319
320} // end namespace llvm
321
322namespace llvm {
323
324// Loop vectorization cost-model hints how the epilogue/tail loop should be
325// lowered.
326enum EpilogueLowering {
327
328 // The default: allowing epilogues.
329 CM_EpilogueAllowed,
330
331 // Vectorization with OptForSize: don't allow epilogues.
332 CM_EpilogueNotAllowedOptSize,
333
334 // A special case for loops with a very small trip count: they are vectorized
335 // only if no scalar epilogue is required, ensuring the vectorized loop body
336 // is the dominant cost without scalar epilogue overheads.
337 CM_EpilogueNotAllowedLowTripLoop,
338
339 // Loop hint indicating an epilogue is undesired, apply tail folding.
340 CM_EpilogueNotNeededFoldTail,
341
342 // Directive indicating we must either fold the epilogue/tail or not vectorize
343 CM_EpilogueNotAllowedFoldTail
344};
345
346enum class AliasMaskingStatus { NotDecided, Disabled, Enabled };
347
348/// LoopVectorizationCostModel - estimates the expected speedups due to
349/// vectorization.
350/// In many cases vectorization is not profitable. This can happen because of
351/// a number of reasons. In this class we mainly attempt to predict the
352/// expected speedup/slowdowns due to the supported instruction set. We use the
353/// TargetTransformInfo to query the different backends for the cost of
354/// different operations.
355class LoopVectorizationCostModel {
356 friend class LoopVectorizationPlanner;
357
358public:
359 LoopVectorizationCostModel(EpilogueLowering SEL, Loop *L,
360 PredicatedScalarEvolution &PSE, LoopInfo *LI,
361 LoopVectorizationLegality *Legal,
362 const TargetTransformInfo &TTI,
363 const TargetLibraryInfo *TLI, AssumptionCache *AC,
364 OptimizationRemarkEmitter *ORE,
365 std::function<BlockFrequencyInfo &()> GetBFI,
366 const Function *F, InterleavedAccessInfo &IAI,
367 VFSelectionContext &Config)
368 : Opts(Config.getOpts()), Config(Config), EpilogueLoweringStatus(SEL),
369 TheLoop(L), PSE(PSE), LI(LI), Legal(Legal), TTI(TTI), TLI(TLI), AC(AC),
370 ORE(ORE), GetBFI(GetBFI), TheFunction(F), InterleaveInfo(IAI) {}
371
372 /// \return An upper bound for the vectorization factors (both fixed and
373 /// scalable). If the factors are 0, vectorization and interleaving should be
374 /// avoided up front.
375 FixedScalableVFPair computeMaxVF(ElementCount UserVF, unsigned UserIC);
376
377 /// Memory access instruction may be vectorized in more than one way.
378 /// Form of instruction after vectorization depends on cost.
379 /// This function takes cost-based decisions for Load/Store instructions
380 /// and collects them in a map. This decisions map is used for building
381 /// the lists of loop-uniform and loop-scalar instructions.
382 /// The calculated cost is saved with widening decision in order to
383 /// avoid redundant calculations.
384 void setCostBasedWideningDecision(ElementCount VF);
385
386 /// Collect values we want to ignore in the cost model.
387 void collectValuesToIgnore();
388
389 /// \returns True if it is more profitable to scalarize instruction \p I for
390 /// vectorization factor \p VF.
391 bool isProfitableToScalarize(Instruction *I, ElementCount VF) const {
392 assert(VF.isVector() &&
393 "Profitable to scalarize relevant only for VF > 1.");
394 assert(
395 TheLoop->isInnermost() &&
396 "cost-model should not be used for outer loops (in VPlan-native path)");
397
398 auto Scalars = InstsToScalarize.find(Val: VF);
399 assert(Scalars != InstsToScalarize.end() &&
400 "VF not yet analyzed for scalarization profitability");
401 return Scalars->second.contains(Key: I);
402 }
403
404 /// Returns true if \p I is known to be uniform after vectorization.
405 bool isUniformAfterVectorization(Instruction *I, ElementCount VF) const {
406 assert(
407 TheLoop->isInnermost() &&
408 "cost-model should not be used for outer loops (in VPlan-native path)");
409
410 // If VF is scalar, then all instructions are trivially uniform.
411 if (VF.isScalar())
412 return true;
413
414 // Pseudo probes must be duplicated per vector lane so that the
415 // profiled loop trip count is not undercounted.
416 if (isa<PseudoProbeInst>(Val: I))
417 return false;
418
419 auto UniformsPerVF = Uniforms.find(Val: VF);
420 assert(UniformsPerVF != Uniforms.end() &&
421 "VF not yet analyzed for uniformity");
422 return UniformsPerVF->second.count(Ptr: I);
423 }
424
425 /// Returns true if \p I is known to be scalar after vectorization.
426 bool isScalarAfterVectorization(Instruction *I, ElementCount VF) const {
427 assert(
428 TheLoop->isInnermost() &&
429 "cost-model should not be used for outer loops (in VPlan-native path)");
430 if (VF.isScalar())
431 return true;
432
433 auto ScalarsPerVF = Scalars.find(Val: VF);
434 assert(ScalarsPerVF != Scalars.end() &&
435 "Scalar values are not calculated for VF");
436 return ScalarsPerVF->second.count(Ptr: I);
437 }
438
439 /// \returns True if instruction \p I can be truncated to a smaller bitwidth
440 /// for vectorization factor \p VF.
441 bool canTruncateToMinimalBitwidth(Instruction *I, ElementCount VF) const {
442 const auto &MinBWs = Config.getMinimalBitwidths();
443 // Truncs must truncate at most to their destination type.
444 if (isa_and_nonnull<TruncInst>(Val: I) && MinBWs.contains(Key: I) &&
445 I->getType()->getScalarSizeInBits() < MinBWs.lookup(Key: I))
446 return false;
447 return VF.isVector() && MinBWs.contains(Key: I) &&
448 !isProfitableToScalarize(I, VF) &&
449 !isScalarAfterVectorization(I, VF);
450 }
451
452 /// Decision that was taken during cost calculation for memory instruction.
453 enum InstWidening {
454 CM_Unknown,
455 CM_Widen, // For consecutive accesses with stride +1.
456 CM_Widen_Reverse, // For consecutive accesses with stride -1.
457 CM_Interleave,
458 CM_GatherScatter,
459 CM_Scalarize,
460 /// A widening decision that has been invalidated after replacing the
461 /// corresponding recipe during VPlan transforms.
462 /// TODO: Remove once the legacy exit cost computation is retired.
463 CM_InvalidatedDecision
464 };
465
466#ifndef NDEBUG
467 static constexpr StringLiteral getInstWideningStr(InstWidening W) {
468 constexpr StringLiteral WideningStr[] = {
469 "Unknown", "Widen", "Widen_Reverse", "Interleave",
470 "GatherScatter", "Scalarize", "InvalidatedDecision"};
471 return WideningStr[W];
472 }
473#endif
474
475 /// Save vectorization decision \p W and \p Cost taken by the cost model for
476 /// instruction \p I and vector width \p VF.
477 void setWideningDecision(Instruction *I, ElementCount VF, InstWidening W,
478 InstructionCost Cost) {
479 assert(VF.isVector() && "Expected VF >=2");
480 LLVM_DEBUG(dbgs() << "LV: Setting widening decision to "
481 << getInstWideningStr(W) << " for VF " << VF
482 << " and instruction: " << *I << '\n');
483 WideningDecisions[{I, VF}] = {W, Cost};
484 }
485
486 /// Save vectorization decision \p W and \p Cost taken by the cost model for
487 /// interleaving group \p Grp and vector width \p VF.
488 void setWideningDecision(const InterleaveGroup<Instruction> *Grp,
489 ElementCount VF, InstWidening W,
490 InstructionCost Cost) {
491 assert(VF.isVector() && "Expected VF >=2");
492 /// Broadcast this decicion to all instructions inside the group.
493 /// When interleaving, the cost will only be assigned one instruction, the
494 /// insert position. For other cases, add the appropriate fraction of the
495 /// total cost to each instruction. This ensures accurate costs are used,
496 /// even if the insert position instruction is not used.
497 InstructionCost InsertPosCost = Cost;
498 InstructionCost OtherMemberCost = 0;
499 if (W != CM_Interleave)
500 OtherMemberCost = InsertPosCost = Cost / Grp->getNumMembers();
501 ;
502 for (auto *I : Grp->members()) {
503 LLVM_DEBUG(dbgs() << "LV: Setting widening decision to "
504 << getInstWideningStr(W) << " for VF " << VF
505 << " and instruction: " << *I << '\n');
506 if (Grp->getInsertPos() == I)
507 WideningDecisions[{I, VF}] = {W, InsertPosCost};
508 else
509 WideningDecisions[{I, VF}] = {W, OtherMemberCost};
510 }
511 }
512
513 /// Return the cost model decision for the given instruction \p I and vector
514 /// width \p VF. Return CM_Unknown if this instruction did not pass
515 /// through the cost modeling.
516 InstWidening getWideningDecision(Instruction *I, ElementCount VF) const {
517 assert(VF.isVector() && "Expected VF to be a vector VF");
518 assert(
519 TheLoop->isInnermost() &&
520 "cost-model should not be used for outer loops (in VPlan-native path)");
521
522 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
523 auto Itr = WideningDecisions.find(Val: InstOnVF);
524 if (Itr == WideningDecisions.end())
525 return CM_Unknown;
526 return Itr->second.first;
527 }
528
529 /// Return the vectorization cost for the given instruction \p I and vector
530 /// width \p VF.
531 InstructionCost getWideningCost(Instruction *I, ElementCount VF) {
532 assert(VF.isVector() && "Expected VF >=2");
533 std::pair<Instruction *, ElementCount> InstOnVF(I, VF);
534 assert(WideningDecisions.contains(InstOnVF) &&
535 "The cost is not calculated");
536 return WideningDecisions[InstOnVF].second;
537 }
538
539 /// Return True if instruction \p I is an optimizable truncate whose operand
540 /// is an induction variable. Such a truncate will be removed by adding a new
541 /// induction variable with the destination type.
542 bool isOptimizableIVTruncate(Instruction *I, ElementCount VF) {
543 // If the instruction is not a truncate, return false.
544 auto *Trunc = dyn_cast<TruncInst>(Val: I);
545 if (!Trunc)
546 return false;
547
548 // Get the source and destination types of the truncate.
549 Type *SrcTy = toVectorTy(Scalar: Trunc->getSrcTy(), EC: VF);
550 Type *DestTy = toVectorTy(Scalar: Trunc->getDestTy(), EC: VF);
551
552 // If the truncate is free for the given types, return false. Replacing a
553 // free truncate with an induction variable would add an induction variable
554 // update instruction to each iteration of the loop. We exclude from this
555 // check the primary induction variable since it will need an update
556 // instruction regardless.
557 Value *Op = Trunc->getOperand(i_nocapture: 0);
558 if (Op != Legal->getPrimaryInduction() && TTI.isTruncateFree(Ty1: SrcTy, Ty2: DestTy))
559 return false;
560
561 // If the truncated value is not an induction variable, return false.
562 return Legal->isInductionPhi(V: Op);
563 }
564
565 /// Collects the instructions to scalarize for each predicated instruction in
566 /// the loop.
567 void collectInstsToScalarize(ElementCount VF);
568
569 /// Collect values that will not be widened, including Uniforms, Scalars, and
570 /// Instructions to Scalarize for the given \p VF.
571 /// The sets depend on CM decision for Load/Store instructions
572 /// that may be vectorized as interleave, gather-scatter or scalarized.
573 /// Also make a decision on what to do about call instructions in the loop
574 /// at that VF -- scalarize, call a known vector routine, or call a
575 /// vector intrinsic.
576 void collectNonVectorizedAndSetWideningDecisions(ElementCount VF) {
577 // Do the analysis once.
578 if (VF.isScalar() || Uniforms.contains(Val: VF))
579 return;
580 setCostBasedWideningDecision(VF);
581 collectLoopUniforms(VF);
582 collectLoopScalars(VF);
583 collectInstsToScalarize(VF);
584 }
585
586 /// Given costs for both strategies, return true if the scalar predication
587 /// lowering should be used for div/rem. This incorporates an override
588 /// option so it is not simply a cost comparison.
589 bool isDivRemScalarWithPredication(InstructionCost ScalarCost,
590 InstructionCost MaskedCost) const {
591 return !valueOr(X: Opts.force_widen_divrem_via_masked_intrinsic,
592 Default: !(ScalarCost < MaskedCost));
593 }
594
595 /// Returns true if \p I is an instruction which requires predication and
596 /// for which our chosen predication strategy is scalarization (i.e. we
597 /// don't have an alternate strategy such as masking available).
598 /// \p VF is the vectorization factor that will be used to vectorize \p I.
599 bool isScalarWithPredication(Instruction *I, ElementCount VF);
600
601 /// Wrapper function for LoopVectorizationLegality::isMaskRequired,
602 /// that passes the Instruction \p I and if we fold tail.
603 bool isMaskRequired(Instruction *I) const;
604
605 /// Returns true if \p I is an instruction that needs to be predicated
606 /// at runtime. The result is independent of the predication mechanism.
607 /// Superset of instructions that return true for isScalarWithPredication.
608 bool isPredicatedInst(Instruction *I) const;
609
610 /// A helper function that returns how much we should divide the cost of a
611 /// predicated block by. Typically this is the reciprocal of the block
612 /// probability, i.e. if we return X we are assuming the predicated block will
613 /// execute once for every X iterations of the loop header so the block should
614 /// only contribute 1/X of its cost to the total cost calculation, but when
615 /// optimizing for code size it will just be 1 as code size costs don't depend
616 /// on execution probabilities.
617 ///
618 /// Note that if a block wasn't originally predicated but was predicated due
619 /// to tail folding, the divisor will still be 1 because it will execute for
620 /// every iteration of the loop header.
621 inline uint64_t
622 getPredBlockCostDivisor(TargetTransformInfo::TargetCostKind CostKind,
623 const BasicBlock *BB);
624
625 /// Returns true if an artificially high cost for emulated masked memrefs
626 /// should be used.
627 bool useEmulatedMaskMemRefHack(Instruction *I, ElementCount VF) const;
628
629 /// Return the costs for our two available strategies for lowering a
630 /// div/rem operation which requires speculating at least one lane.
631 /// First result is for scalarization (will be invalid for scalable
632 /// vectors); second is for the masked intrinsic strategy.
633 std::pair<InstructionCost, InstructionCost>
634 getDivRemSpeculationCost(Instruction *I, ElementCount VF);
635
636 /// If \p I is a memory instruction with a consecutive pointer that can be
637 /// widened, returns the widening kind (CM_Widen or CM_Widen_Reverse) and
638 /// std::nullopt otherwise.
639 std::optional<InstWidening> memoryInstructionCanBeWidened(Instruction *I,
640 ElementCount VF);
641
642 /// Returns true if \p I is a memory instruction in an interleaved-group
643 /// of memory accesses that can be vectorized with wide vector loads/stores
644 /// and shuffles.
645 bool interleavedAccessCanBeWidened(Instruction *I, ElementCount VF) const;
646
647 /// Returns true if the target machine supports masked loads or stores
648 /// for \p I's data type and alignment. The caller must ensure the access is
649 /// consecutive or part of an interleave group.
650 bool isLegalMaskedLoadOrStore(Instruction *I, ElementCount VF) const;
651
652 /// Returns true if the target machine supports gather or scatter for \p I's
653 /// data type and alignment.
654 bool isLegalGatherOrScatter(Instruction *I, ElementCount VF) const;
655
656 /// Check if \p Instr belongs to any interleaved access group.
657 bool isAccessInterleaved(Instruction *Instr) const {
658 return InterleaveInfo.isInterleaved(Instr);
659 }
660
661 /// Get the interleaved access group that \p Instr belongs to.
662 const InterleaveGroup<Instruction> *
663 getInterleavedAccessGroup(Instruction *Instr) const {
664 return InterleaveInfo.getInterleaveGroup(Instr);
665 }
666
667 /// Returns true if we're required to use a scalar epilogue for at least
668 /// the final iteration of the original loop.
669 bool requiresScalarEpilogue(bool IsVectorizing) const {
670 if (!isEpilogueAllowed()) {
671 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
672 return false;
673 }
674 // If we might exit from anywhere but the latch and early exit vectorization
675 // is disabled, we must run the exiting iteration in scalar form.
676 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
677 !(Opts.enable_early_exit_vectorization &&
678 Legal->hasUncountableEarlyExit())) {
679 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
680 "from latch block\n");
681 return true;
682 }
683 if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
684 LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
685 "interleaved group requires scalar epilogue\n");
686 return true;
687 }
688 LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
689 return false;
690 }
691
692 /// Returns true if an epilogue is allowed (e.g., not prevented by
693 /// optsize or a loop hint annotation).
694 bool isEpilogueAllowed() const {
695 return EpilogueLoweringStatus == CM_EpilogueAllowed;
696 }
697
698 /// Returns the TailFoldingStyle that is best for the current loop.
699 TailFoldingStyle getTailFoldingStyle() const {
700 return ChosenTailFoldingStyle;
701 }
702
703 /// Selects and saves TailFoldingStyle.
704 /// \param IsScalableVF true if scalable vector factors enabled.
705 /// \param UserIC User specific interleave count.
706 void setTailFoldingStyle(bool IsScalableVF, unsigned UserIC) {
707 assert(ChosenTailFoldingStyle == TailFoldingStyle::None &&
708 "Tail folding must not be selected yet.");
709 if (!Legal->canFoldTailByMasking()) {
710 ChosenTailFoldingStyle = TailFoldingStyle::None;
711 return;
712 }
713
714 // Default to TTI preference, but allow command line override.
715 ChosenTailFoldingStyle = Opts.force_tail_folding_style.value_or(
716 u: TTI.getPreferredTailFoldingStyle());
717
718 if (ChosenTailFoldingStyle != TailFoldingStyle::DataWithEVL)
719 return;
720 // Override EVL styles if needed.
721 // FIXME: Investigate opportunity for fixed vector factor.
722 bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
723 TTI.hasActiveVectorLength() &&
724 !Opts.enable_vplan_native_path;
725 if (EVLIsLegal)
726 return;
727 // If for some reason EVL mode is unsupported, fallback to an epilogue
728 // if it's allowed, or DataWithoutLaneMask otherwise.
729 if (EpilogueLoweringStatus == CM_EpilogueAllowed ||
730 EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail)
731 ChosenTailFoldingStyle = TailFoldingStyle::None;
732 else
733 ChosenTailFoldingStyle = TailFoldingStyle::DataWithoutLaneMask;
734
735 LLVM_DEBUG(
736 dbgs() << "LV: Preference for VP intrinsics indicated. Will "
737 "not try to generate VP Intrinsics "
738 << (UserIC > 1
739 ? "since interleave count specified is greater than 1.\n"
740 : "due to non-interleaving reasons.\n"));
741 }
742
743 /// Returns true if all loop blocks should be masked to fold tail loop.
744 bool foldTailByMasking() const {
745 return getTailFoldingStyle() != TailFoldingStyle::None;
746 }
747
748 void tryToEnablePartialAliasMasking() {
749 assert(foldTailByMasking() && "Expected tail folding to be enabled!");
750 assert(!foldTailWithEVL() &&
751 "Did not expect to enable alias masking with EVL!");
752 assert(PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided);
753
754 // Assume we fail to enable alias masking (in case we early exit).
755 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
756
757 // Note: FixedOrderRecurrences are not supported yet as we cannot handle
758 // the required `splice.right` with the alias-mask.
759 if (!Opts.force_partial_aliasing_vectorization ||
760 !Legal->getFixedOrderRecurrences().empty())
761 return;
762
763 const RuntimePointerChecking *Checks = Legal->getRuntimePointerChecking();
764 if (!Checks)
765 return;
766
767 auto DiffChecks = Checks->getDiffChecks();
768 if (!DiffChecks || DiffChecks->empty())
769 return;
770
771 [[maybe_unused]] auto HasPointerArgs = [](CallBase *CB) {
772 return any_of(Range: CB->args(), P: [](Value const *Arg) {
773 return Arg->getType()->isPointerTy();
774 });
775 };
776
777 for (BasicBlock *BB : TheLoop->blocks()) {
778 for (Instruction &I : *BB) {
779 if (!isa<LoadInst, StoreInst>(Val: I)) {
780 [[maybe_unused]] auto *Call = dyn_cast<CallInst>(Val: &I);
781 assert(
782 (!I.mayReadOrWriteMemory() || (Call && !HasPointerArgs(Call))) &&
783 "Skipped unexpected memory access");
784 continue;
785 }
786
787 Type *ScalarTy = getLoadStoreType(I: &I);
788 Value *Ptr = getLoadStorePointerOperand(V: &I);
789
790 // Currently, we can't handle alias masking in reverse. Reversing the
791 // alias mask is not correct (or necessary). When combined with
792 // tail-folding the active lane mask should only be reversed where the
793 // alias-mask is true.
794 if (Legal->isConsecutivePtr(AccessTy: ScalarTy, Ptr) == -1)
795 return;
796 }
797 }
798
799 PartialAliasMaskingStatus = AliasMaskingStatus::Enabled;
800 }
801
802 /// Returns true if all loop blocks should have partial aliases masked.
803 bool maskPartialAliasing() const {
804 return PartialAliasMaskingStatus == AliasMaskingStatus::Enabled;
805 }
806
807 /// Returns true if the instructions in this block requires predication
808 /// for any reason, e.g. because tail folding now requires a predicate,
809 /// because memory accesses are masked due to an uncountable exit with side
810 /// effects, or because the block in the original loop was predicated.
811 bool blockNeedsPredicationForAnyReason(BasicBlock *BB) const {
812 return foldTailByMasking() || Legal->hasUncountableExitWithSideEffects() ||
813 Legal->blockNeedsPredication(BB);
814 }
815
816 /// Returns true if VP intrinsics with explicit vector length support should
817 /// be generated in the tail folded loop.
818 bool foldTailWithEVL() const {
819 return getTailFoldingStyle() == TailFoldingStyle::DataWithEVL;
820 }
821
822 /// Returns true if the predicated reduction select should be used to set the
823 /// incoming value for the reduction phi.
824 bool usePredicatedReductionSelect(RecurKind RecurrenceKind,
825 bool HasUsesOutsideReductionChain) const {
826 // Force to use predicated reduction select since the EVL of the
827 // second-to-last iteration might not be VF*UF.
828 if (foldTailWithEVL())
829 return true;
830
831 // Force a predicated select with alias-masking to avoid propagating poison
832 // values to the header phi for lanes outside the alias-mask.
833 if (maskPartialAliasing())
834 return true;
835
836 // Note: For FindLast recurrences and multi-use reductions we prefer a
837 // predicated select to simplify matching in handleFindLastReductions() and
838 // handleMultiUseReductions() respectively, rather than handle multiple
839 // cases.
840 if (RecurrenceDescriptor::isFindLastRecurrenceKind(Kind: RecurrenceKind) ||
841 HasUsesOutsideReductionChain)
842 return true;
843
844 return Opts.prefer_predicated_reduction_select ||
845 TTI.preferPredicatedReductionSelect();
846 }
847
848 /// Estimate cost of an intrinsic call instruction CI if it were vectorized
849 /// with factor VF. Return the cost of the instruction, including
850 /// scalarization overhead if it's needed.
851 InstructionCost getVectorIntrinsicCost(CallInst *CI, ElementCount VF) const;
852
853 /// Estimate cost of a call instruction CI if it were vectorized with factor
854 /// VF. Return the cost of the instruction, including scalarization overhead
855 /// if it's needed.
856 InstructionCost getVectorCallCost(CallInst *CI, ElementCount VF) const;
857
858 /// Invalidates decisions already taken by the cost model.
859 void invalidateCostModelingDecisions() {
860 WideningDecisions.clear();
861 Uniforms.clear();
862 Scalars.clear();
863 }
864
865 /// Returns the expected execution cost. The unit of the cost does
866 /// not matter because we use the 'cost' units to compare different
867 /// vector widths. The cost that is returned is *not* normalized by
868 /// the factor width.
869 InstructionCost expectedCost(ElementCount VF);
870
871 /// Returns the execution time cost of an instruction for a given vector
872 /// width. Vector width of one means scalar.
873 InstructionCost getInstructionCost(Instruction *I, ElementCount VF);
874
875 /// Returns true if \p Op should be considered invariant and if it is
876 /// trivially hoistable.
877 bool shouldConsiderInvariant(Value *Op);
878
879 /// Returns true if \p I has been forced to be scalarized at \p VF.
880 bool isForcedScalar(Instruction *I, ElementCount VF) const {
881 auto FS = ForcedScalars.find(Val: VF);
882 return FS != ForcedScalars.end() && FS->second.contains(key: I);
883 }
884
885private:
886 const VectorizeOptions &Opts;
887
888 unsigned NumPredStores = 0;
889
890 /// VF selection state independent of cost-modeling decisions.
891 VFSelectionContext &Config;
892
893 /// Wrapper around LoopVectorizationLegality::isUniform() that takes into
894 /// account if alias-masking is enabled. We consider the VF to be unknown when
895 /// alias masking.
896 bool isUniform(Value *V, ElementCount VF) const {
897 // With alias-masking our runtime VF is [2, VF] (and not necessarily a
898 // power-of-two). Something that is uniform for VF may not be for the full
899 // range.
900 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
901 "alias-mask status must be decided already");
902 return Legal->isUniform(V, VF: PartialAliasMaskingStatus ==
903 AliasMaskingStatus::Disabled
904 ? std::optional(VF)
905 : std::nullopt);
906 }
907
908 /// Wrapper around LoopVectorizationLegality::isUniformMemOp() that takes into
909 /// account if alias-masking is enabled. We consider the VF to be unknown when
910 /// alias masking.
911 bool isUniformMemOp(Instruction &I, ElementCount VF) const {
912 assert(PartialAliasMaskingStatus != AliasMaskingStatus::NotDecided &&
913 "alias-mask status must be decided already");
914 return Legal->isUniformMemOp(I, VF: PartialAliasMaskingStatus ==
915 AliasMaskingStatus::Disabled
916 ? std::optional(VF)
917 : std::nullopt);
918 }
919
920 /// Calculate vectorization cost of memory instruction \p I.
921 InstructionCost getMemoryInstructionCost(Instruction *I, ElementCount VF);
922
923 /// The cost computation for scalarized memory instruction.
924 InstructionCost getMemInstScalarizationCost(Instruction *I, ElementCount VF);
925
926 /// The cost computation for interleaving group of memory instructions.
927 InstructionCost getInterleaveGroupCost(Instruction *I, ElementCount VF) const;
928
929 /// The cost computation for Gather/Scatter instruction.
930 InstructionCost getGatherScatterCost(Instruction *I, ElementCount VF) const;
931
932 /// The cost computation for widening instruction \p I with consecutive
933 /// memory access.
934 InstructionCost getConsecutiveMemOpCost(Instruction *I, ElementCount VF,
935 InstWidening Kind);
936
937 /// The cost calculation for Load/Store instruction \p I with uniform pointer -
938 /// Load: scalar load + broadcast.
939 /// Store: scalar store + (loop invariant value stored? 0 : extract of last
940 /// element)
941 InstructionCost getUniformMemOpCost(Instruction *I, ElementCount VF) const;
942
943 /// Estimate the overhead of scalarizing an instruction. This is a
944 /// convenience wrapper for the type-based getScalarizationOverhead API.
945 InstructionCost getScalarizationOverhead(Instruction *I,
946 ElementCount VF) const;
947
948 /// A type representing the costs for instructions if they were to be
949 /// scalarized rather than vectorized. The entries are Instruction-Cost
950 /// pairs.
951 using ScalarCostsTy = MapVector<Instruction *, InstructionCost>;
952
953 /// A set containing all BasicBlocks that are known to present after
954 /// vectorization as a predicated block.
955 DenseMap<ElementCount, SmallPtrSet<BasicBlock *, 4>>
956 PredicatedBBsAfterVectorization;
957
958 /// Records whether it is allowed to have the original scalar loop execute at
959 /// least once. This may be needed as a fallback loop in case runtime
960 /// aliasing/dependence checks fail, or to handle the tail/remainder
961 /// iterations when the trip count is unknown or doesn't divide by the VF,
962 /// or as a peel-loop to handle gaps in interleave-groups.
963 /// Under optsize and when the trip count is very small we don't allow any
964 /// iterations to execute in the scalar loop.
965 EpilogueLowering EpilogueLoweringStatus = CM_EpilogueAllowed;
966
967 /// Control finally chosen tail folding style.
968 TailFoldingStyle ChosenTailFoldingStyle = TailFoldingStyle::None;
969
970 /// If partial alias masking is enabled/disabled or not decided.
971 AliasMaskingStatus PartialAliasMaskingStatus = AliasMaskingStatus::NotDecided;
972
973 /// A map holding scalar costs for different vectorization factors. The
974 /// presence of a cost for an instruction in the mapping indicates that the
975 /// instruction will be scalarized when vectorizing with the associated
976 /// vectorization factor. The entries are VF-ScalarCostTy pairs.
977 DenseMap<ElementCount, ScalarCostsTy> InstsToScalarize;
978
979 /// Holds the instructions known to be uniform after vectorization.
980 /// The data is collected per VF.
981 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Uniforms;
982
983 /// Holds the instructions known to be scalar after vectorization.
984 /// The data is collected per VF.
985 DenseMap<ElementCount, SmallPtrSet<Instruction *, 4>> Scalars;
986
987 /// Holds the instructions (address computations) that are forced to be
988 /// scalarized.
989 DenseMap<ElementCount, SmallSetVector<Instruction *, 4>> ForcedScalars;
990
991 /// Returns the expected difference in cost from scalarizing the expression
992 /// feeding a predicated instruction \p PredInst. The instructions to
993 /// scalarize and their scalar costs are collected in \p ScalarCosts. A
994 /// non-negative return value implies the expression will be scalarized.
995 /// Currently, only single-use chains are considered for scalarization.
996 InstructionCost computePredInstDiscount(Instruction *PredInst,
997 ScalarCostsTy &ScalarCosts,
998 ElementCount VF);
999
1000 /// Collect the instructions that are uniform after vectorization. An
1001 /// instruction is uniform if we represent it with a single scalar value in
1002 /// the vectorized loop corresponding to each vector iteration. Examples of
1003 /// uniform instructions include pointer operands of consecutive or
1004 /// interleaved memory accesses. Note that although uniformity implies an
1005 /// instruction will be scalar, the reverse is not true. In general, a
1006 /// scalarized instruction will be represented by VF scalar values in the
1007 /// vectorized loop, each corresponding to an iteration of the original
1008 /// scalar loop.
1009 void collectLoopUniforms(ElementCount VF);
1010
1011 /// Collect the instructions that are scalar after vectorization. An
1012 /// instruction is scalar if it is known to be uniform or will be scalarized
1013 /// during vectorization. collectLoopScalars should only add non-uniform nodes
1014 /// to the list if they are used by a load/store instruction that is marked as
1015 /// CM_Scalarize. Non-uniform scalarized instructions will be represented by
1016 /// VF values in the vectorized loop, each corresponding to an iteration of
1017 /// the original scalar loop.
1018 void collectLoopScalars(ElementCount VF);
1019
1020 /// Keeps cost model vectorization decision and cost for instructions.
1021 /// Right now it is used for memory instructions only.
1022 using DecisionList = DenseMap<std::pair<Instruction *, ElementCount>,
1023 std::pair<InstWidening, InstructionCost>>;
1024
1025 DecisionList WideningDecisions;
1026
1027 /// Returns true if \p V is expected to be vectorized and it needs to be
1028 /// extracted.
1029 bool needsExtract(Value *V, ElementCount VF) const {
1030 Instruction *I = dyn_cast<Instruction>(Val: V);
1031 if (VF.isScalar() || !I || !TheLoop->contains(Inst: I) ||
1032 TheLoop->isLoopInvariant(V: I) ||
1033 getWideningDecision(I, VF) == CM_Scalarize)
1034 return false;
1035
1036 // Assume we can vectorize V (and hence we need extraction) if the
1037 // scalars are not computed yet. This can happen, because it is called
1038 // via getScalarizationOverhead from setCostBasedWideningDecision, before
1039 // the scalars are collected. That should be a safe assumption in most
1040 // cases, because we check if the operands have vectorizable types
1041 // beforehand in LoopVectorizationLegality.
1042 return !Scalars.contains(Val: VF) || !isScalarAfterVectorization(I, VF);
1043 };
1044
1045public:
1046 /// The loop that we evaluate.
1047 Loop *TheLoop;
1048
1049 /// Predicated scalar evolution analysis.
1050 PredicatedScalarEvolution &PSE;
1051
1052 /// Loop Info analysis.
1053 LoopInfo *LI;
1054
1055 /// Vectorization legality.
1056 LoopVectorizationLegality *Legal;
1057
1058 /// Vector target information.
1059 const TargetTransformInfo &TTI;
1060
1061 /// Target Library Info.
1062 const TargetLibraryInfo *TLI;
1063
1064 /// Assumption cache.
1065 AssumptionCache *AC;
1066
1067 /// Interface to emit optimization remarks.
1068 OptimizationRemarkEmitter *ORE;
1069
1070 /// A function to lazily fetch BlockFrequencyInfo. This avoids computing it
1071 /// unless necessary, e.g. when the loop isn't legal to vectorize or when
1072 /// there is no predication.
1073 std::function<BlockFrequencyInfo &()> GetBFI;
1074 /// The BlockFrequencyInfo returned from GetBFI.
1075 BlockFrequencyInfo *BFI = nullptr;
1076 /// Returns the BlockFrequencyInfo for the function if cached, otherwise
1077 /// fetches it via GetBFI. Avoids an indirect call to the std::function.
1078 BlockFrequencyInfo &getBFI() {
1079 if (!BFI)
1080 BFI = &GetBFI();
1081 return *BFI;
1082 }
1083
1084 const Function *TheFunction;
1085
1086 /// The interleave access information contains groups of interleaved accesses
1087 /// with the same stride and close to each other.
1088 InterleavedAccessInfo &InterleaveInfo;
1089
1090 /// Values to ignore in the cost model.
1091 SmallPtrSet<const Value *, 16> ValuesToIgnore;
1092
1093 /// Values to ignore in the cost model when VF > 1.
1094 SmallPtrSet<const Value *, 16> VecValuesToIgnore;
1095};
1096} // end namespace llvm
1097
1098namespace {
1099/// Helper struct to manage generating runtime checks for vectorization.
1100///
1101/// The runtime checks are created up-front in temporary blocks to allow better
1102/// estimating the cost and un-linked from the existing IR. After deciding to
1103/// vectorize, the checks are attached to VPlan as IR or recipes. If deciding
1104/// not to vectorize, the temporary blocks are completely removed.
1105class GeneratedRTChecks {
1106 const VectorizeOptions &Opts;
1107
1108 /// Basic block which contains the generated SCEV checks, if any.
1109 BasicBlock *SCEVCheckBlock = nullptr;
1110
1111 /// The value representing the result of the generated SCEV checks. If it is
1112 /// nullptr no SCEV checks have been generated.
1113 Value *SCEVCheckCond = nullptr;
1114
1115 /// Basic block which contains the generated memory runtime checks, if any.
1116 BasicBlock *MemCheckBlock = nullptr;
1117
1118 /// The value representing the result of the generated memory runtime checks.
1119 /// If it is nullptr no memory runtime checks have been generated.
1120 Value *MemRuntimeCheckCond = nullptr;
1121
1122 /// Whether checks were generated, retained after their IR is replaced or
1123 /// removed during VPlan execution.
1124 bool HasChecks = false;
1125
1126 DominatorTree *DT;
1127 LoopInfo *LI;
1128 TargetTransformInfo *TTI;
1129
1130 SCEVExpander SCEVExp;
1131 SCEVExpander MemCheckExp;
1132
1133 bool CostTooHigh = false;
1134
1135 Loop *OuterLoop = nullptr;
1136
1137 PredicatedScalarEvolution &PSE;
1138
1139 /// The kind of cost that we are calculating
1140 TTI::TargetCostKind CostKind;
1141
1142 /// True if the loop is alias-masked (which allows us to omit diff checks).
1143 bool LoopUsesPartialAliasMasking = false;
1144
1145public:
1146 GeneratedRTChecks(const VectorizeOptions &Opts,
1147 PredicatedScalarEvolution &PSE, DominatorTree *DT,
1148 LoopInfo *LI, TargetTransformInfo *TTI,
1149 TTI::TargetCostKind CostKind,
1150 bool LoopUsesPartialAliasMasking)
1151 : Opts(Opts), DT(DT), LI(LI), TTI(TTI),
1152 SCEVExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1153 MemCheckExp(*PSE.getSE(), "scev.check", /*PreserveLCSSA=*/false),
1154 PSE(PSE), CostKind(CostKind),
1155 LoopUsesPartialAliasMasking(LoopUsesPartialAliasMasking) {}
1156
1157 /// Generate runtime checks in SCEVCheckBlock and MemCheckBlock, so we can
1158 /// accurately estimate the cost of the runtime checks. The blocks are
1159 /// un-linked from the IR and attached to VPlan as IR or recipes if
1160 /// profitable. Otherwise, the check blocks are removed completely. If \p LAI
1161 /// is nullptr (outer loops have no LoopAccessInfo), no memory runtime checks
1162 /// are generated.
1163 void create(Loop *L, const LoopAccessInfo *LAI,
1164 const SCEVPredicate &UnionPred, ElementCount VF, unsigned IC,
1165 OptimizationRemarkEmitter &ORE) {
1166
1167 // Hard cutoff to limit compile-time increase in case a very large number of
1168 // runtime checks needs to be generated.
1169 // TODO: Skip cutoff if the loop is guaranteed to execute, e.g. due to
1170 // profile info.
1171 CostTooHigh = LAI && LAI->getNumRuntimePointerChecks() >
1172 VectorizerParams::VectorizeMemoryCheckThreshold;
1173 if (CostTooHigh) {
1174 // Mark runtime checks as never succeeding when they exceed the threshold.
1175 MemRuntimeCheckCond = ConstantInt::getTrue(Context&: L->getHeader()->getContext());
1176 SCEVCheckCond = ConstantInt::getTrue(Context&: L->getHeader()->getContext());
1177 ORE.emit(RemarkBuilder: [&]() {
1178 return OptimizationRemarkAnalysisAliasing(
1179 DEBUG_TYPE, "TooManyMemoryRuntimeChecks", L->getStartLoc(),
1180 L->getHeader())
1181 << "loop not vectorized: too many memory checks needed";
1182 });
1183 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
1184 return;
1185 }
1186
1187 BasicBlock *LoopHeader = L->getHeader();
1188 BasicBlock *Preheader = L->getLoopPreheader();
1189
1190 // Use SplitBlock to create blocks for SCEV & memory runtime checks to
1191 // ensure the blocks are properly added to LoopInfo & DominatorTree. Those
1192 // may be used by SCEVExpander. The blocks will be un-linked from their
1193 // predecessors and removed from LI & DT at the end of the function.
1194 if (!UnionPred.isAlwaysTrue()) {
1195 SCEVCheckBlock = SplitBlock(Old: Preheader, SplitPt: Preheader->getTerminator(), DT, LI,
1196 MSSAU: nullptr, BBName: "vector.scevcheck");
1197
1198 SCEVCheckCond = SCEVExp.expandCodeForPredicate(
1199 Pred: &UnionPred, Loc: SCEVCheckBlock->getTerminator());
1200 if (isa<Constant>(Val: SCEVCheckCond)) {
1201 // Clean up directly after expanding the predicate to a constant, to
1202 // avoid further expansions re-using anything left over from SCEVExp.
1203 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1204 SCEVCleaner.cleanup();
1205 }
1206 }
1207
1208 // TODO: We need to estimate the cost of alias-masking in
1209 // GeneratedRTChecks::getCost(). We can't check the MemCheckBlock as the
1210 // alias-mask is generated later in VPlan.
1211 if (LAI && LAI->getRuntimePointerChecking()->Need &&
1212 !LoopUsesPartialAliasMasking) {
1213 const auto &RtPtrChecking = *LAI->getRuntimePointerChecking();
1214 auto *Pred = SCEVCheckBlock ? SCEVCheckBlock : Preheader;
1215 MemCheckBlock = SplitBlock(Old: Pred, SplitPt: Pred->getTerminator(), DT, LI, MSSAU: nullptr,
1216 BBName: "vector.memcheck");
1217
1218 auto DiffChecks = RtPtrChecking.getDiffChecks();
1219 if (DiffChecks) {
1220 MemRuntimeCheckCond = addDiffRuntimeChecks(
1221 Loc: MemCheckBlock->getTerminator(), Checks: *DiffChecks, Expander&: MemCheckExp, VF, IC);
1222 } else {
1223 MemRuntimeCheckCond = addRuntimeChecks(
1224 Loc: MemCheckBlock->getTerminator(), TheLoop: L, PointerChecks: RtPtrChecking.getChecks(),
1225 Expander&: MemCheckExp, HoistRuntimeChecks: VectorizerParams::HoistRuntimeChecks);
1226 }
1227 assert(MemRuntimeCheckCond &&
1228 "no RT checks generated although RtPtrChecking "
1229 "claimed checks are required");
1230 }
1231
1232 SCEVExp.eraseDeadInstructions(Root: SCEVCheckCond);
1233 HasChecks = getSCEVChecks().first || getMemRuntimeChecks().first;
1234
1235 if (!MemCheckBlock && !SCEVCheckBlock)
1236 return;
1237
1238 // Unhook the temporary block with the checks, update various places
1239 // accordingly.
1240 if (SCEVCheckBlock)
1241 SCEVCheckBlock->replaceAllUsesWith(V: Preheader);
1242 if (MemCheckBlock)
1243 MemCheckBlock->replaceAllUsesWith(V: Preheader);
1244
1245 if (SCEVCheckBlock) {
1246 SCEVCheckBlock->getTerminator()->moveBefore(
1247 InsertPos: Preheader->getTerminator()->getIterator());
1248 auto *UI = new UnreachableInst(Preheader->getContext(), SCEVCheckBlock);
1249 UI->setDebugLoc(DebugLoc::getTemporary());
1250 Preheader->getTerminator()->eraseFromParent();
1251 }
1252 if (MemCheckBlock) {
1253 MemCheckBlock->getTerminator()->moveBefore(
1254 InsertPos: Preheader->getTerminator()->getIterator());
1255 auto *UI = new UnreachableInst(Preheader->getContext(), MemCheckBlock);
1256 UI->setDebugLoc(DebugLoc::getTemporary());
1257 Preheader->getTerminator()->eraseFromParent();
1258 }
1259
1260 DT->changeImmediateDominator(BB: LoopHeader, NewBB: Preheader);
1261 if (MemCheckBlock) {
1262 DT->eraseNode(BB: MemCheckBlock);
1263 LI->removeBlock(BB: MemCheckBlock);
1264 }
1265 if (SCEVCheckBlock) {
1266 DT->eraseNode(BB: SCEVCheckBlock);
1267 LI->removeBlock(BB: SCEVCheckBlock);
1268 }
1269
1270 // Clean up directly if the memory checks folded to false, to avoid further
1271 // expansions re-using anything left over from MemCheckExp.
1272 if (MemCheckBlock && !getMemRuntimeChecks().first)
1273 eraseMemCheckBlock();
1274
1275 // Outer loop is used as part of the later cost calculations.
1276 OuterLoop = L->getParentLoop();
1277 }
1278
1279 InstructionCost getCost() {
1280 if (SCEVCheckBlock || MemCheckBlock)
1281 LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
1282
1283 if (CostTooHigh) {
1284 InstructionCost Cost;
1285 Cost.setInvalid();
1286 LLVM_DEBUG(dbgs() << " number of checks exceeded threshold\n");
1287 return Cost;
1288 }
1289
1290 InstructionCost RTCheckCost = 0;
1291 if (SCEVCheckBlock)
1292 for (Instruction &I : *SCEVCheckBlock) {
1293 if (SCEVCheckBlock->getTerminator() == &I)
1294 continue;
1295 InstructionCost C = TTI->getInstructionCost(U: &I, CostKind);
1296 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1297 RTCheckCost += C;
1298 }
1299 if (MemCheckBlock) {
1300 InstructionCost MemCheckCost = 0;
1301 for (Instruction &I : *MemCheckBlock) {
1302 if (MemCheckBlock->getTerminator() == &I)
1303 continue;
1304 InstructionCost C = TTI->getInstructionCost(U: &I, CostKind);
1305 LLVM_DEBUG(dbgs() << " " << C << " for " << I << "\n");
1306 MemCheckCost += C;
1307 }
1308
1309 // If the runtime memory checks are being created inside an outer loop
1310 // we should find out if these checks are outer loop invariant. If so,
1311 // the checks will likely be hoisted out and so the effective cost will
1312 // reduce according to the outer loop trip count.
1313 if (OuterLoop) {
1314 ScalarEvolution *SE = MemCheckExp.getSE();
1315 // TODO: If profitable, we could refine this further by analysing every
1316 // individual memory check, since there could be a mixture of loop
1317 // variant and invariant checks that mean the final condition is
1318 // variant.
1319 const SCEV *Cond = SE->getSCEV(V: MemRuntimeCheckCond);
1320 if (SE->isLoopInvariant(S: Cond, L: OuterLoop)) {
1321 // It seems reasonable to assume that we can reduce the effective
1322 // cost of the checks even when we know nothing about the trip
1323 // count. Assume that the outer loop executes at least twice.
1324 unsigned BestTripCount = 2;
1325
1326 // Get the best known TC estimate.
1327 if (auto EstimatedTC = getSmallBestKnownTC(
1328 Opts, PSE, L: OuterLoop, /* CanUseConstantMax = */ false))
1329 if (EstimatedTC->isFixed())
1330 BestTripCount = EstimatedTC->getFixedValue();
1331
1332 InstructionCost NewMemCheckCost = MemCheckCost / BestTripCount;
1333
1334 // Let's ensure the cost is always at least 1.
1335 NewMemCheckCost = std::max(a: NewMemCheckCost.getValue(),
1336 b: (InstructionCost::CostType)1);
1337
1338 if (BestTripCount > 1)
1339 LLVM_DEBUG(dbgs()
1340 << "We expect runtime memory checks to be hoisted "
1341 << "out of the outer loop. Cost reduced from "
1342 << MemCheckCost << " to " << NewMemCheckCost << '\n');
1343
1344 MemCheckCost = NewMemCheckCost;
1345 }
1346 }
1347
1348 RTCheckCost += MemCheckCost;
1349 }
1350
1351 if (SCEVCheckBlock || MemCheckBlock)
1352 LLVM_DEBUG(dbgs() << "Total cost of runtime checks: " << RTCheckCost
1353 << "\n");
1354
1355 return RTCheckCost;
1356 }
1357
1358 /// Remove the created SCEV & memory runtime check blocks & instructions, if
1359 /// unused.
1360 ~GeneratedRTChecks() {
1361 SCEVExpanderCleaner SCEVCleaner(SCEVExp);
1362 bool SCEVChecksUsed = !SCEVCheckBlock || !pred_empty(BB: SCEVCheckBlock);
1363 if (SCEVChecksUsed)
1364 SCEVCleaner.markResultUsed();
1365
1366 if (MemCheckBlock && pred_empty(BB: MemCheckBlock))
1367 eraseMemCheckBlock();
1368
1369 SCEVCleaner.cleanup();
1370
1371 if (!SCEVChecksUsed)
1372 SCEVCheckBlock->eraseFromParent();
1373 }
1374
1375 /// Retrieves the SCEVCheckCond and SCEVCheckBlock that were generated as IR
1376 /// outside VPlan.
1377 std::pair<Value *, BasicBlock *> getSCEVChecks() const {
1378 using namespace llvm::PatternMatch;
1379 if (!SCEVCheckCond || match(V: SCEVCheckCond, P: m_ZeroInt()))
1380 return {nullptr, nullptr};
1381
1382 return {SCEVCheckCond, SCEVCheckBlock};
1383 }
1384
1385 /// Retrieves the MemCheckCond and MemCheckBlock that were generated as IR
1386 /// outside VPlan.
1387 std::pair<Value *, BasicBlock *> getMemRuntimeChecks() const {
1388 using namespace llvm::PatternMatch;
1389 if (MemRuntimeCheckCond && match(V: MemRuntimeCheckCond, P: m_ZeroInt()))
1390 return {nullptr, nullptr};
1391 return {MemRuntimeCheckCond, MemCheckBlock};
1392 }
1393
1394 /// Return true if any runtime checks have been added
1395 bool hasChecks() const { return HasChecks; }
1396
1397 /// Erase the memory check block, its instructions and their SCEV expansions.
1398 void eraseMemCheckBlock() {
1399 SCEVExpanderCleaner MemCheckCleaner(MemCheckExp);
1400 auto &SE = *MemCheckExp.getSE();
1401 // Memory runtime check generation creates compares that use expanded
1402 // values. Remove them before running the SCEVExpanderCleaner.
1403 for (auto &I : make_early_inc_range(Range: reverse(C&: *MemCheckBlock))) {
1404 if (MemCheckExp.isInsertedInstruction(I: &I))
1405 continue;
1406 SE.forgetValue(V: &I);
1407 I.eraseFromParent();
1408 }
1409 MemCheckCleaner.cleanup();
1410 MemCheckBlock->eraseFromParent();
1411 MemCheckBlock = nullptr;
1412 MemRuntimeCheckCond = nullptr;
1413 }
1414};
1415} // namespace
1416
1417static bool useActiveLaneMask(TailFoldingStyle Style) {
1418 return Style == TailFoldingStyle::Data ||
1419 Style == TailFoldingStyle::DataAndControlFlow;
1420}
1421
1422static bool useActiveLaneMaskForControlFlow(TailFoldingStyle Style) {
1423 return Style == TailFoldingStyle::DataAndControlFlow;
1424}
1425
1426// Return true if \p OuterLp is an outer loop annotated with hints for explicit
1427// vectorization. The loop needs to be annotated with #pragma omp simd
1428// simdlen(#) or #pragma clang vectorize(enable) vectorize_width(#). If the
1429// vector length information is not provided, vectorization is not considered
1430// explicit. Interleave hints are not allowed either. These limitations will be
1431// relaxed in the future.
1432// Please, note that we are currently forced to abuse the pragma 'clang
1433// vectorize' semantics. This pragma provides *auto-vectorization hints*
1434// (i.e., LV must check that vectorization is legal) whereas pragma 'omp simd'
1435// provides *explicit vectorization hints* (LV can bypass legal checks and
1436// assume that vectorization is legal). However, both hints are implemented
1437// using the same metadata (llvm.loop.vectorize, processed by
1438// LoopVectorizeHints). This will be fixed in the future when the native IR
1439// representation for pragma 'omp simd' is introduced.
1440static bool isExplicitVecOuterLoop(const VectorizeOptions &Opts, Loop *OuterLp,
1441 OptimizationRemarkEmitter *ORE) {
1442 assert(!OuterLp->isInnermost() && "This is not an outer loop");
1443 LoopVectorizeHints Hints(Opts, OuterLp, true /*DisableInterleaving*/, *ORE);
1444
1445 // Only outer loops with an explicit vectorization hint are supported.
1446 // Unannotated outer loops are ignored.
1447 if (Hints.getForce() == LoopVectorizeHints::FK_Undefined)
1448 return false;
1449
1450 if (!Hints.allowVectorization(L: OuterLp, VectorizeOnlyWhenForced: true /*VectorizeOnlyWhenForced*/)) {
1451 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent outer loop vectorization.\n");
1452 return false;
1453 }
1454
1455 if (Hints.getInterleave() > 1) {
1456 // TODO: Interleave support is future work.
1457 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Interleave is not supported for "
1458 "outer loops.\n");
1459 Hints.emitRemarkWithHints();
1460 return false;
1461 }
1462
1463 return true;
1464}
1465
1466static void collectSupportedLoops(const VectorizeOptions &Opts, Loop &L,
1467 LoopInfo *LI, OptimizationRemarkEmitter *ORE,
1468 SmallVectorImpl<Loop *> &V) {
1469 // Collect inner loops and outer loops without irreducible control flow. For
1470 // now, only collect outer loops that have explicit vectorization hints. If we
1471 // are stress testing the VPlan H-CFG construction, we collect the outermost
1472 // loop of every loop nest.
1473 if (L.isInnermost() || Opts.vplan_build_outerloop_stress_test ||
1474 (Opts.enable_vplan_native_path &&
1475 isExplicitVecOuterLoop(Opts, OuterLp: &L, ORE))) {
1476 LoopBlocksRPO RPOT(&L);
1477 RPOT.perform(LI);
1478 if (!containsIrreducibleCFG<const BasicBlock *>(RPOTraversal&: RPOT, LI: *LI)) {
1479 V.push_back(Elt: &L);
1480 // TODO: Collect inner loops inside marked outer loops in case
1481 // vectorization fails for the outer loop. Do not invoke
1482 // 'containsIrreducibleCFG' again for inner loops when the outer loop is
1483 // already known to be reducible. We can use an inherited attribute for
1484 // that.
1485 return;
1486 }
1487 }
1488 for (Loop *InnerL : L)
1489 collectSupportedLoops(Opts, L&: *InnerL, LI, ORE, V);
1490}
1491
1492//===----------------------------------------------------------------------===//
1493// Implementation of LoopVectorizationCostModel and LoopVectorizationPlanner.
1494//===----------------------------------------------------------------------===//
1495
1496/// For the given VF and UF and maximum trip count computed for the loop, return
1497/// whether the induction variable might overflow in the vectorized loop. If not,
1498/// then we know a runtime overflow check always evaluates to false and can be
1499/// removed.
1500static bool isIndvarOverflowCheckKnownFalse(
1501 const VectorizeOptions &Opts, const LoopVectorizationCostModel *Cost,
1502 ElementCount VF, std::optional<unsigned> UF = std::nullopt) {
1503 // Always be conservative if we don't know the exact unroll factor.
1504 uint64_t MaxUF = UF ? *UF
1505 : std::max(a: Cost->TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions: false),
1506 b: Cost->TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions: true));
1507
1508 IntegerType *IdxTy = Cost->Legal->getWidestInductionType();
1509 APInt MaxUIntTripCount = IdxTy->getMask();
1510
1511 // We know the runtime overflow check is known false iff the (max) trip-count
1512 // is known and (max) trip-count + (VF * UF) does not overflow in the type of
1513 // the vector loop induction variable.
1514 if (std::optional<ElementCount> TC = getSmallBestKnownTC(
1515 Opts, PSE&: Cost->PSE, L: Cost->TheLoop,
1516 /*CanUseConstantMax=*/true, /*CanExcludeZeroTrips=*/false,
1517 /*ComputeUpperBoundOnly=*/true)) {
1518 // Compute the maximum runtime values of VF and the trip count.
1519 std::optional<uint64_t> MaxStep =
1520 getMaxRuntimeElementCount(EC: VF * MaxUF, F: *Cost->TheFunction);
1521 std::optional<uint64_t> MaxTC =
1522 getMaxRuntimeElementCount(EC: *TC, F: *Cost->TheFunction);
1523 if (!MaxStep || !MaxTC)
1524 return false;
1525
1526 // Bail out if the maximum trip count is not representable in the induction
1527 // variable's type.
1528 if (MaxUIntTripCount.ult(RHS: *MaxTC))
1529 return false;
1530
1531 return (MaxUIntTripCount - *MaxTC).ugt(RHS: *MaxStep);
1532 }
1533
1534 return false;
1535}
1536
1537// Return whether we allow using masked interleave-groups (for dealing with
1538// strided loads/stores that reside in predicated blocks, or for dealing
1539// with gaps).
1540static bool useMaskedInterleavedAccesses(const VectorizeOptions &Opts,
1541 const TargetTransformInfo &TTI) {
1542 // If an override option has been passed in for interleaved accesses, use it.
1543 return valueOr(X: Opts.enable_masked_interleaved_mem_accesses,
1544 Default: TTI.enableMaskedInterleavedAccessVectorization());
1545}
1546
1547/// Replace \p VPBB with a VPIRBasicBlock wrapping \p IRBB. All recipes from \p
1548/// VPBB are moved to the end of the newly created VPIRBasicBlock. All
1549/// predecessors and successors of VPBB, if any, are rewired to the new
1550/// VPIRBasicBlock. If \p VPBB may be unreachable, \p Plan must be passed.
1551static VPIRBasicBlock *replaceVPBBWithIRVPBB(VPBasicBlock *VPBB,
1552 BasicBlock *IRBB,
1553 VPlan *Plan = nullptr) {
1554 if (!Plan)
1555 Plan = VPBB->getPlan();
1556 VPIRBasicBlock *IRVPBB = Plan->createEmptyVPIRBasicBlock(IRBB);
1557 auto IP = IRVPBB->begin();
1558 for (auto &R : make_early_inc_range(Range: VPBB->phis()))
1559 R.moveBefore(I: IP);
1560
1561 for (auto &R :
1562 make_early_inc_range(Range: make_range(x: VPBB->getFirstNonPhi(), y: VPBB->end())))
1563 R.moveBefore(I: IRVPBB->end());
1564
1565 VPBlockUtils::reassociateBlocks(Old: VPBB, New: IRVPBB);
1566 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
1567 return IRVPBB;
1568}
1569
1570/// Knowing that loop \p L executes a single vector iteration, add instructions
1571/// that will get simplified and thus should not have any cost to \p
1572/// InstsToIgnore.
1573static void addFullyUnrolledInstructionsToIgnore(
1574 Loop *L, const LoopVectorizationLegality::InductionList &IL,
1575 SmallPtrSetImpl<Instruction *> &InstsToIgnore) {
1576 auto *Cmp = L->getLatchCmpInst();
1577 if (Cmp)
1578 InstsToIgnore.insert(Ptr: Cmp);
1579 for (PHINode *IV : IL.keys()) {
1580 // The induction is free: a widened induction generates a vector phi with
1581 // its start value and an increment that is dead without a backedge.
1582 InstsToIgnore.insert(Ptr: IV);
1583
1584 // Get next iteration value of the induction variable.
1585 Instruction *IVInst =
1586 cast<Instruction>(Val: IV->getIncomingValueForBlock(BB: L->getLoopLatch()));
1587 if (all_of(Range: IVInst->users(),
1588 P: [&](const User *U) { return U == IV || U == Cmp; }))
1589 InstsToIgnore.insert(Ptr: IVInst);
1590 }
1591}
1592
1593namespace {
1594
1595struct CSEDenseMapInfo {
1596 static bool canHandle(const Instruction *I) {
1597 return isa<InsertElementInst>(Val: I) || isa<ExtractElementInst>(Val: I) ||
1598 isa<ShuffleVectorInst>(Val: I) || isa<GetElementPtrInst>(Val: I);
1599 }
1600
1601 static unsigned getHashValue(const Instruction *I) {
1602 assert(canHandle(I) && "Unknown instruction!");
1603 return hash_combine(args: I->getOpcode(),
1604 args: hash_combine_range(R: I->operand_values()));
1605 }
1606
1607 static bool isEqual(const Instruction *LHS, const Instruction *RHS) {
1608 return LHS->isIdenticalTo(I: RHS);
1609 }
1610};
1611
1612} // end anonymous namespace
1613
1614/// FIXME: This legacy common-subexpression-elimination routine is scheduled for
1615/// removal, in favor of the VPlan-based one.
1616static void legacyCSE(BasicBlock *BB) {
1617 // Perform simple cse.
1618 SmallDenseMap<Instruction *, Instruction *, 4, CSEDenseMapInfo> CSEMap;
1619 for (Instruction &In : llvm::make_early_inc_range(Range&: *BB)) {
1620 if (!CSEDenseMapInfo::canHandle(I: &In))
1621 continue;
1622
1623 // Check if we can replace this instruction with any of the
1624 // visited instructions.
1625 if (Instruction *V = CSEMap.lookup(Val: &In)) {
1626 In.replaceAllUsesWith(V);
1627 In.eraseFromParent();
1628 continue;
1629 }
1630
1631 CSEMap[&In] = &In;
1632 }
1633}
1634
1635/// This function attempts to return a value that represents the ElementCount
1636/// at runtime. For fixed-width VFs we know this precisely at compile
1637/// time, but for scalable VFs we calculate it based on an estimate of the
1638/// vscale value.
1639static unsigned estimateElementCount(ElementCount VF,
1640 std::optional<unsigned> VScale) {
1641 unsigned EstimatedVF = VF.getKnownMinValue();
1642 if (VF.isScalable())
1643 if (VScale)
1644 EstimatedVF *= *VScale;
1645 assert(EstimatedVF >= 1 && "Estimated VF shouldn't be less than 1");
1646 return EstimatedVF;
1647}
1648
1649/// Returns the vector library variant function of \p CI usable at \p VF,
1650/// respecting \p MaskRequired, or nullptr if none is found: a mapping with
1651/// matching VF, masked if required, whose vector function is declared in the
1652/// module.
1653static Function *getVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
1654 bool MaskRequired,
1655 const TargetLibraryInfo *TLI) {
1656 if (!TLI || CI.isNoBuiltin())
1657 return nullptr;
1658 for (const VFInfo &Info : VFDatabase::getMappings(CI))
1659 if (Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()))
1660 if (Function *F = CI.getModule()->getFunction(Name: Info.VectorName))
1661 return F;
1662 return nullptr;
1663}
1664
1665/// Returns true iff \p CI has a library vector variant usable at \p VF.
1666static bool hasVectorLibraryVariantFor(const CallInst &CI, ElementCount VF,
1667 bool MaskRequired,
1668 const TargetLibraryInfo *TLI) {
1669 return getVectorLibraryVariantFor(CI, VF, MaskRequired, TLI) != nullptr;
1670}
1671
1672InstructionCost
1673LoopVectorizationCostModel::getVectorCallCost(CallInst *CI,
1674 ElementCount VF) const {
1675 Type *RetTy = CI->getType();
1676 SmallVector<Type *, 4> Tys;
1677 for (auto &ArgOp : CI->args())
1678 Tys.push_back(Elt: ArgOp->getType());
1679
1680 InstructionCost ScalarCallCost = TTI.getCallInstrCost(
1681 F: CI->getCalledFunction(), RetTy, Tys, CostKind: Config.CostKind);
1682
1683 // Cost of the scalar call (scalar VF) or its scalarization (vector VF). The
1684 // scalarization cost is only meaningful for fixed VFs.
1685 InstructionCost Cost = VF.isScalable()
1686 ? InstructionCost::getInvalid()
1687 : ScalarCallCost * VF.getKnownMinValue() +
1688 getScalarizationOverhead(I: CI, VF);
1689
1690 // The call may be vectorized at this VF, via a vector intrinsic or a vector
1691 // library variant.
1692 if (getVectorIntrinsicIDForCall(CI, TLI))
1693 Cost = std::min(a: Cost, b: getVectorIntrinsicCost(CI, VF));
1694
1695 if (Function *Variant =
1696 getVectorLibraryVariantFor(CI: *CI, VF, MaskRequired: isMaskRequired(I: CI), TLI))
1697 Cost = std::min(a: Cost,
1698 b: TTI.getCallInstrCost(
1699 /*F=*/nullptr, RetTy: Variant->getReturnType(),
1700 Tys: Variant->getFunctionType()->params(), CostKind: Config.CostKind));
1701
1702 return Cost;
1703}
1704
1705static Type *maybeVectorizeType(Type *Ty, ElementCount VF) {
1706 if (VF.isScalar() || !canVectorizeTy(Ty))
1707 return Ty;
1708 return toVectorizedTy(Ty, EC: VF);
1709}
1710
1711InstructionCost
1712LoopVectorizationCostModel::getVectorIntrinsicCost(CallInst *CI,
1713 ElementCount VF) const {
1714 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI);
1715 assert(ID && "Expected intrinsic call!");
1716 Type *RetTy = maybeVectorizeType(Ty: CI->getType(), VF);
1717 FastMathFlags FMF;
1718 if (auto *FPMO = dyn_cast<FPMathOperator>(Val: CI))
1719 FMF = FPMO->getFastMathFlags();
1720
1721 SmallVector<const Value *> Arguments(CI->args());
1722 FunctionType *FTy = CI->getCalledFunction()->getFunctionType();
1723 SmallVector<Type *> ParamTys;
1724 std::transform(first: FTy->param_begin(), last: FTy->param_end(),
1725 result: std::back_inserter(x&: ParamTys),
1726 unary_op: [&](Type *Ty) { return maybeVectorizeType(Ty, VF); });
1727
1728 IntrinsicCostAttributes CostAttrs(ID, RetTy, Arguments, ParamTys, FMF,
1729 dyn_cast<IntrinsicInst>(Val: CI),
1730 InstructionCost::getInvalid());
1731 return TTI.getIntrinsicInstrCost(ICA: CostAttrs, CostKind: Config.CostKind);
1732}
1733
1734void LoopVectorizationCostModel::collectLoopScalars(ElementCount VF) {
1735 // We should not collect Scalars more than once per VF. Right now, this
1736 // function is called from collectUniformsAndScalars(), which already does
1737 // this check. Collecting Scalars for VF=1 does not make any sense.
1738 assert(VF.isVector() && !Scalars.contains(VF) &&
1739 "This function should not be visited twice for the same VF");
1740
1741 // This avoids any chances of creating a REPLICATE recipe during planning
1742 // since that would result in generation of scalarized code during execution,
1743 // which is not supported for scalable vectors.
1744 if (VF.isScalable()) {
1745 Scalars[VF].insert_range(R&: Uniforms[VF]);
1746 return;
1747 }
1748
1749 SmallSetVector<Instruction *, 8> Worklist;
1750
1751 // These sets are used to seed the analysis with pointers used by memory
1752 // accesses that will remain scalar.
1753 SmallSetVector<Instruction *, 8> ScalarPtrs;
1754 SmallPtrSet<Instruction *, 8> PossibleNonScalarPtrs;
1755 auto *Latch = TheLoop->getLoopLatch();
1756
1757 // A helper that returns true if the use of Ptr by MemAccess will be scalar.
1758 // The pointer operands of loads and stores will be scalar as long as the
1759 // memory access is not a gather/scatter or histogram operation. The value
1760 // operand of a store will remain scalar if the store is scalarized.
1761 auto IsScalarUse = [&](Instruction *MemAccess, Value *Ptr) {
1762 InstWidening WideningDecision = getWideningDecision(I: MemAccess, VF);
1763 assert(WideningDecision != CM_Unknown &&
1764 "Widening decision should be ready at this moment");
1765 auto *Store = dyn_cast<StoreInst>(Val: MemAccess);
1766 if (Store && Ptr == Store->getValueOperand())
1767 return WideningDecision == CM_Scalarize;
1768 assert(Ptr == getLoadStorePointerOperand(MemAccess) &&
1769 "Ptr is neither a value or pointer operand");
1770 return WideningDecision != CM_GatherScatter &&
1771 !(Store && Legal->getHistogramInfo(I: Store));
1772 };
1773
1774 // A helper that returns true if the given value is a getelementptr
1775 // instruction contained in the loop.
1776 auto IsLoopVaryingGEP = [&](Value *V) {
1777 return isa<GetElementPtrInst>(Val: V) && !TheLoop->isLoopInvariant(V);
1778 };
1779
1780 // A helper that evaluates a memory access's use of a pointer. If the use will
1781 // be a scalar use and the pointer is only used by memory accesses, we place
1782 // the pointer in ScalarPtrs. Otherwise, the pointer is placed in
1783 // PossibleNonScalarPtrs.
1784 auto EvaluatePtrUse = [&](Instruction *MemAccess, Value *Ptr) {
1785 // We only care about bitcast and getelementptr instructions contained in
1786 // the loop.
1787 if (!IsLoopVaryingGEP(Ptr))
1788 return;
1789
1790 // If the pointer has already been identified as scalar (e.g., if it was
1791 // also identified as uniform), there's nothing to do.
1792 auto *I = cast<Instruction>(Val: Ptr);
1793 if (Worklist.count(key: I))
1794 return;
1795
1796 // If the use of the pointer will be a scalar use, and all users of the
1797 // pointer are memory accesses, place the pointer in ScalarPtrs. Otherwise,
1798 // place the pointer in PossibleNonScalarPtrs.
1799 if (IsScalarUse(MemAccess, Ptr) &&
1800 all_of(Range: I->users(), P: IsaPred<LoadInst, StoreInst>))
1801 ScalarPtrs.insert(X: I);
1802 else
1803 PossibleNonScalarPtrs.insert(Ptr: I);
1804 };
1805
1806 // We seed the scalars analysis with three classes of instructions: (1)
1807 // instructions marked uniform-after-vectorization and (2) bitcast,
1808 // getelementptr and (pointer) phi instructions used by memory accesses
1809 // requiring a scalar use.
1810 //
1811 // (1) Add to the worklist all instructions that have been identified as
1812 // uniform-after-vectorization.
1813 Worklist.insert_range(R&: Uniforms[VF]);
1814
1815 // (2) Add to the worklist all bitcast and getelementptr instructions used by
1816 // memory accesses requiring a scalar use. The pointer operands of loads and
1817 // stores will be scalar unless the operation is a gather or scatter.
1818 // The value operand of a store will remain scalar if the store is scalarized.
1819 for (auto *BB : TheLoop->blocks())
1820 for (auto &I : *BB) {
1821 if (auto *Load = dyn_cast<LoadInst>(Val: &I)) {
1822 EvaluatePtrUse(Load, Load->getPointerOperand());
1823 } else if (auto *Store = dyn_cast<StoreInst>(Val: &I)) {
1824 EvaluatePtrUse(Store, Store->getPointerOperand());
1825 EvaluatePtrUse(Store, Store->getValueOperand());
1826 }
1827 }
1828 for (auto *I : ScalarPtrs)
1829 if (!PossibleNonScalarPtrs.count(Ptr: I)) {
1830 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *I << "\n");
1831 Worklist.insert(X: I);
1832 }
1833
1834 // Insert the forced scalars.
1835 // FIXME: Currently VPWidenPHIRecipe() often creates a dead vector
1836 // induction variable when the PHI user is scalarized.
1837 auto ForcedScalar = ForcedScalars.find(Val: VF);
1838 if (ForcedScalar != ForcedScalars.end())
1839 for (auto *I : ForcedScalar->second) {
1840 LLVM_DEBUG(dbgs() << "LV: Found (forced) scalar instruction: " << *I << "\n");
1841 Worklist.insert(X: I);
1842 }
1843
1844 // Expand the worklist by looking through any bitcasts and getelementptr
1845 // instructions we've already identified as scalar. This is similar to the
1846 // expansion step in collectLoopUniforms(); however, here we're only
1847 // expanding to include additional bitcasts and getelementptr instructions.
1848 unsigned Idx = 0;
1849 while (Idx != Worklist.size()) {
1850 Instruction *Dst = Worklist[Idx++];
1851 if (!IsLoopVaryingGEP(Dst->getOperand(i: 0)))
1852 continue;
1853 auto *Src = cast<Instruction>(Val: Dst->getOperand(i: 0));
1854 if (llvm::all_of(Range: Src->users(), P: [&](User *U) -> bool {
1855 auto *J = cast<Instruction>(Val: U);
1856 return !TheLoop->contains(Inst: J) || Worklist.count(key: J) ||
1857 ((isa<LoadInst>(Val: J) || isa<StoreInst>(Val: J)) &&
1858 IsScalarUse(J, Src));
1859 })) {
1860 Worklist.insert(X: Src);
1861 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Src << "\n");
1862 }
1863 }
1864
1865 // An induction variable will remain scalar if all users of the induction
1866 // variable and induction variable update remain scalar.
1867 for (const auto &Induction : Legal->getInductionVars()) {
1868 auto *Ind = Induction.first;
1869 auto *IndUpdate = cast<Instruction>(Val: Ind->getIncomingValueForBlock(BB: Latch));
1870
1871 // If tail-folding is applied, the primary induction variable will be used
1872 // to feed a vector compare.
1873 if (Ind == Legal->getPrimaryInduction() && foldTailByMasking())
1874 continue;
1875
1876 // Returns true if \p Indvar is a pointer induction that is used directly by
1877 // load/store instruction \p I.
1878 auto IsDirectLoadStoreFromPtrIndvar = [&](Instruction *Indvar,
1879 Instruction *I) {
1880 return Induction.second.getKind() ==
1881 InductionDescriptor::IK_PtrInduction &&
1882 (isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I)) &&
1883 Indvar == getLoadStorePointerOperand(V: I) && IsScalarUse(I, Indvar);
1884 };
1885
1886 // Determine if all users of the induction variable are scalar after
1887 // vectorization.
1888 bool ScalarInd = all_of(Range: Ind->users(), P: [&](User *U) -> bool {
1889 auto *I = cast<Instruction>(Val: U);
1890 return I == IndUpdate || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
1891 IsDirectLoadStoreFromPtrIndvar(Ind, I);
1892 });
1893 if (!ScalarInd)
1894 continue;
1895
1896 // If the induction variable update is a fixed-order recurrence, neither the
1897 // induction variable or its update should be marked scalar after
1898 // vectorization.
1899 auto *IndUpdatePhi = dyn_cast<PHINode>(Val: IndUpdate);
1900 if (IndUpdatePhi && Legal->isFixedOrderRecurrence(Phi: IndUpdatePhi))
1901 continue;
1902
1903 // Determine if all users of the induction variable update instruction are
1904 // scalar after vectorization.
1905 bool ScalarIndUpdate = all_of(Range: IndUpdate->users(), P: [&](User *U) -> bool {
1906 auto *I = cast<Instruction>(Val: U);
1907 return I == Ind || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
1908 IsDirectLoadStoreFromPtrIndvar(IndUpdate, I);
1909 });
1910 if (!ScalarIndUpdate)
1911 continue;
1912
1913 // The induction variable and its update instruction will remain scalar.
1914 Worklist.insert(X: Ind);
1915 Worklist.insert(X: IndUpdate);
1916 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *Ind << "\n");
1917 LLVM_DEBUG(dbgs() << "LV: Found scalar instruction: " << *IndUpdate
1918 << "\n");
1919 }
1920
1921 Scalars[VF].insert_range(R&: Worklist);
1922}
1923
1924bool LoopVectorizationCostModel::isLegalMaskedLoadOrStore(
1925 Instruction *I, ElementCount VF) const {
1926 assert(isa<LoadInst>(I) || isa<StoreInst>(I));
1927 return Config.isLegalMaskedLoadOrStore(IsLoad: isa<LoadInst>(Val: I), ScalarTy: getLoadStoreType(I),
1928 Alignment: getLoadStoreAlignment(I),
1929 AddressSpace: getLoadStoreAddressSpace(I));
1930}
1931
1932bool LoopVectorizationCostModel::isLegalGatherOrScatter(Instruction *I,
1933 ElementCount VF) const {
1934 assert((isa<LoadInst, StoreInst>(I)));
1935 return Config.isLegalGatherOrScatter(IsLoad: isa<LoadInst>(Val: I), ScalarTy: getLoadStoreType(I),
1936 Alignment: getLoadStoreAlignment(I), VF);
1937}
1938
1939bool LoopVectorizationCostModel::isScalarWithPredication(Instruction *I,
1940 ElementCount VF) {
1941 if (!isPredicatedInst(I))
1942 return false;
1943
1944 // Do we have a non-scalar lowering for this predicated
1945 // instruction? No - it is scalar with predication.
1946 switch(I->getOpcode()) {
1947 default:
1948 return true;
1949 case Instruction::Call: {
1950 if (VF.isScalar())
1951 return true;
1952 auto *CI = cast<CallInst>(Val: I);
1953 // A vector intrinsic or library variant lowering avoids scalarization.
1954 return !getVectorIntrinsicIDForCall(CI, TLI) &&
1955 !hasVectorLibraryVariantFor(CI: *CI, VF, MaskRequired: isMaskRequired(I: CI), TLI);
1956 }
1957 case Instruction::Load:
1958 case Instruction::Store: {
1959 Type *ScalarTy = getLoadStoreType(I);
1960 Value *Ptr = getLoadStorePointerOperand(V: I);
1961 if (Legal->isCompressedLoadOrStore(I))
1962 return false; // We can't scalarize compressed loads/stores.
1963 bool IsConsecutive = Legal->isConsecutivePtr(AccessTy: ScalarTy, Ptr);
1964 return !(IsConsecutive && isLegalMaskedLoadOrStore(I, VF)) &&
1965 !isLegalGatherOrScatter(I, VF);
1966 }
1967 case Instruction::UDiv:
1968 case Instruction::SDiv:
1969 case Instruction::SRem:
1970 case Instruction::URem: {
1971 // We have the option to use the llvm.masked.udiv intrinsics to avoid
1972 // predication. The cost based decision here will always select the masked
1973 // intrinsics for scalable vectors as scalarization isn't legal.
1974 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
1975 return isDivRemScalarWithPredication(ScalarCost, MaskedCost);
1976 }
1977 }
1978}
1979
1980bool LoopVectorizationCostModel::isMaskRequired(Instruction *I) const {
1981 return Legal->isMaskRequired(I, TailFolded: foldTailByMasking());
1982}
1983
1984// TODO: Fold into LoopVectorizationLegality::isMaskRequired.
1985bool LoopVectorizationCostModel::isPredicatedInst(Instruction *I) const {
1986 // TODO: We can use the loop-preheader as context point here and get
1987 // context sensitive reasoning for isSafeToSpeculativelyExecute.
1988 if (isSafeToSpeculativelyExecute(I) ||
1989 (isa<LoadInst, StoreInst, CallInst>(Val: I) && !isMaskRequired(I)) ||
1990 isa<UncondBrInst, CondBrInst, SwitchInst, PHINode, AllocaInst>(Val: I))
1991 return false;
1992
1993 // If the instruction was executed conditionally in the original scalar loop,
1994 // predication is needed with a mask whose lanes are all possibly inactive.
1995 if (Legal->blockNeedsPredication(BB: I->getParent()))
1996 return true;
1997
1998 // If we're not folding the tail by masking and not vectorizing a loop with
1999 // uncountable exits and side effects, predication is unnecessary.
2000 if (!foldTailByMasking() && !Legal->hasUncountableExitWithSideEffects())
2001 return false;
2002
2003 // All that remain are instructions with side-effects originally executed in
2004 // the loop unconditionally, but now execute under a tail-fold mask (only)
2005 // having at least one active lane (the first). If the side-effects of the
2006 // instruction are invariant, executing it w/o (the tail-folding) mask is safe
2007 // - it will cause the same side-effects as when masked.
2008 switch(I->getOpcode()) {
2009 default:
2010 llvm_unreachable(
2011 "instruction should have been considered by earlier checks");
2012 case Instruction::Call:
2013 // Side-effects of a Call are assumed to be non-invariant, needing a
2014 // (fold-tail) mask.
2015 assert(isMaskRequired(I) &&
2016 "should have returned earlier for calls not needing a mask");
2017 return true;
2018 case Instruction::Load:
2019 // If the address is loop invariant no predication is needed.
2020 return !Legal->isInvariant(V: getLoadStorePointerOperand(V: I));
2021 case Instruction::Store: {
2022 // For stores, we need to prove both speculation safety (which follows from
2023 // the same argument as loads), but also must prove the value being stored
2024 // is correct. The easiest form of the later is to require that all values
2025 // stored are the same.
2026 return !(Legal->isInvariant(V: getLoadStorePointerOperand(V: I)) &&
2027 TheLoop->isLoopInvariant(V: cast<StoreInst>(Val: I)->getValueOperand()));
2028 }
2029 case Instruction::UDiv:
2030 case Instruction::URem:
2031 // If the divisor is loop-invariant no predication is needed.
2032 return !Legal->isInvariant(V: I->getOperand(i: 1));
2033 case Instruction::SDiv:
2034 case Instruction::SRem:
2035 // Conservative for now, since masked-off lanes may be poison and could
2036 // trigger signed overflow.
2037 return true;
2038 }
2039}
2040
2041uint64_t LoopVectorizationCostModel::getPredBlockCostDivisor(
2042 TargetTransformInfo::TargetCostKind CostKind, const BasicBlock *BB) {
2043 if (CostKind == TTI::TCK_CodeSize)
2044 return 1;
2045 // If the block wasn't originally predicated then return early to avoid
2046 // computing BlockFrequencyInfo unnecessarily.
2047 if (!Legal->blockNeedsPredication(BB))
2048 return 1;
2049
2050 uint64_t HeaderFreq =
2051 getBFI().getBlockFreq(BB: TheLoop->getHeader()).getFrequency();
2052 uint64_t BBFreq = getBFI().getBlockFreq(BB).getFrequency();
2053 assert(HeaderFreq >= BBFreq &&
2054 "Header has smaller block freq than dominated BB?");
2055 return std::round(x: (double)HeaderFreq / BBFreq);
2056}
2057
2058static Intrinsic::ID getMaskedDivRemIntrinsic(unsigned Opcode) {
2059 switch (Opcode) {
2060 case Instruction::UDiv:
2061 return Intrinsic::masked_udiv;
2062 case Instruction::SDiv:
2063 return Intrinsic::masked_sdiv;
2064 case Instruction::URem:
2065 return Intrinsic::masked_urem;
2066 case Instruction::SRem:
2067 return Intrinsic::masked_srem;
2068 default:
2069 llvm_unreachable("Unexpected opcode");
2070 }
2071}
2072
2073std::pair<InstructionCost, InstructionCost>
2074LoopVectorizationCostModel::getDivRemSpeculationCost(Instruction *I,
2075 ElementCount VF) {
2076 assert(I->getOpcode() == Instruction::UDiv ||
2077 I->getOpcode() == Instruction::SDiv ||
2078 I->getOpcode() == Instruction::SRem ||
2079 I->getOpcode() == Instruction::URem);
2080 assert(!isSafeToSpeculativelyExecute(I));
2081
2082 // Scalarization isn't legal for scalable vector types
2083 InstructionCost ScalarizationCost = InstructionCost::getInvalid();
2084 if (!VF.isScalable()) {
2085 // Get the scalarization cost and scale this amount by the probability of
2086 // executing the predicated block. If the instruction is not predicated,
2087 // we fall through to the next case.
2088 ScalarizationCost = 0;
2089
2090 // These instructions have a non-void type, so account for the phi nodes
2091 // that we will create. This cost is likely to be zero. The phi node
2092 // cost, if any, should be scaled by the block probability because it
2093 // models a copy at the end of each predicated block.
2094 ScalarizationCost += VF.getFixedValue() *
2095 TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
2096
2097 // The cost of the non-predicated instruction.
2098 ScalarizationCost +=
2099 VF.getFixedValue() * TTI.getArithmeticInstrCost(
2100 Opcode: I->getOpcode(), Ty: I->getType(), CostKind: Config.CostKind);
2101
2102 // The cost of insertelement and extractelement instructions needed for
2103 // scalarization.
2104 ScalarizationCost += getScalarizationOverhead(I, VF);
2105
2106 // Scale the cost by the probability of executing the predicated blocks.
2107 // This assumes the predicated block for each vector lane is equally
2108 // likely.
2109 ScalarizationCost =
2110 ScalarizationCost /
2111 getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
2112 }
2113
2114 auto *VecTy = toVectorTy(Scalar: I->getType(), EC: VF);
2115 auto *MaskTy = toVectorTy(Scalar: Type::getInt1Ty(C&: I->getContext()), EC: VF);
2116 IntrinsicCostAttributes ICA(getMaskedDivRemIntrinsic(Opcode: I->getOpcode()), VecTy,
2117 {VecTy, VecTy, MaskTy});
2118 InstructionCost MaskedCost = TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind);
2119 return {ScalarizationCost, MaskedCost};
2120}
2121
2122bool LoopVectorizationCostModel::interleavedAccessCanBeWidened(
2123 Instruction *I, ElementCount VF) const {
2124 assert(isAccessInterleaved(I) && "Expecting interleaved access.");
2125 assert(getWideningDecision(I, VF) == CM_Unknown &&
2126 "Decision should not be set yet.");
2127 auto *Group = getInterleavedAccessGroup(Instr: I);
2128 assert(Group && "Must have a group.");
2129 unsigned InterleaveFactor = Group->getFactor();
2130
2131 // If the instruction's allocated size doesn't equal its type size, it
2132 // requires padding and will be scalarized.
2133 auto &DL = I->getDataLayout();
2134 auto *ScalarTy = getLoadStoreType(I);
2135 if (hasIrregularType(Ty: ScalarTy, DL))
2136 return false;
2137
2138 // For scalable vectors, the interleave factors must be <= 8 since we require
2139 // the (de)interleaveN intrinsics instead of shufflevectors.
2140 if (VF.isScalable() && InterleaveFactor > 8)
2141 return false;
2142
2143 // If the group involves a non-integral pointer, we may not be able to
2144 // losslessly cast all values to a common type.
2145 bool ScalarNI = DL.isNonIntegralPointerType(Ty: ScalarTy);
2146 for (Instruction *Member : Group->members()) {
2147 auto *MemberTy = getLoadStoreType(I: Member);
2148 bool MemberNI = DL.isNonIntegralPointerType(Ty: MemberTy);
2149 // Don't coerce non-integral pointers to integers or vice versa.
2150 if (MemberNI != ScalarNI)
2151 // TODO: Consider adding special nullptr value case here
2152 return false;
2153 if (MemberNI && ScalarNI &&
2154 ScalarTy->getPointerAddressSpace() !=
2155 MemberTy->getPointerAddressSpace())
2156 return false;
2157 }
2158
2159 // Check if masking is required.
2160 // A Group may need masking for one of two reasons: it resides in a block that
2161 // needs predication, or it was decided to use masking to deal with gaps
2162 // (either a gap at the end of a load-access that may result in a speculative
2163 // load, or any gaps in a store-access).
2164 bool PredicatedAccessRequiresMasking =
2165 blockNeedsPredicationForAnyReason(BB: I->getParent()) && isMaskRequired(I);
2166 bool LoadAccessWithGapsRequiresEpilogMasking =
2167 isa<LoadInst>(Val: I) && Group->requiresScalarEpilogue() &&
2168 !isEpilogueAllowed();
2169 bool StoreAccessWithGapsRequiresMasking =
2170 isa<StoreInst>(Val: I) && !Group->isFull();
2171 if (!PredicatedAccessRequiresMasking &&
2172 !LoadAccessWithGapsRequiresEpilogMasking &&
2173 !StoreAccessWithGapsRequiresMasking)
2174 return true;
2175
2176 // If masked interleaving is required, we expect that the user/target had
2177 // enabled it, because otherwise it either wouldn't have been created or
2178 // it should have been invalidated by the CostModel.
2179 assert(useMaskedInterleavedAccesses(Opts, TTI) &&
2180 "Masked interleave-groups for predicated accesses are not enabled.");
2181
2182 if (Group->isReverse())
2183 return false;
2184
2185 // TODO: Support interleaved access that requires a gap mask for scalable VFs.
2186 bool NeedsMaskForGaps = LoadAccessWithGapsRequiresEpilogMasking ||
2187 StoreAccessWithGapsRequiresMasking;
2188 if (VF.isScalable() && NeedsMaskForGaps)
2189 return false;
2190
2191 return isLegalMaskedLoadOrStore(I, VF);
2192}
2193
2194std::optional<LoopVectorizationCostModel::InstWidening>
2195LoopVectorizationCostModel::memoryInstructionCanBeWidened(Instruction *I,
2196 ElementCount VF) {
2197 // Get and ensure we have a valid memory instruction.
2198 assert((isa<LoadInst, StoreInst>(I)) && "Invalid memory instruction");
2199
2200 auto *Ptr = getLoadStorePointerOperand(V: I);
2201 auto *ScalarTy = getLoadStoreType(I);
2202
2203 // In order to be widened, the pointer should be consecutive or compressed.
2204 int Stride = Legal->isConsecutivePtr(AccessTy: ScalarTy, Ptr);
2205
2206 // Check if the pointer is derived from a conditional induction PHI, which
2207 // can be widened as a compressstore/expandload.
2208 if (std::optional<CompressedPtrInfo> PtrInfo =
2209 Legal->getCompressedPtrInfo(Ptr)) {
2210 Stride = getStrideFromAddRec(AR: PtrInfo->PtrSCEV, Lp: TheLoop, AccessTy: ScalarTy, Ptr, PSE)
2211 .value_or(u: 0);
2212 assert(Stride == 1 && "Compressed memory ops must be consecutive");
2213 }
2214
2215 if (!Stride)
2216 return std::nullopt;
2217
2218 // If the instruction is a store located in a predicated block, it will be
2219 // scalarized.
2220 if (isScalarWithPredication(I, VF))
2221 return std::nullopt;
2222
2223 // If the instruction's allocated size doesn't equal it's type size, it
2224 // requires padding and will be scalarized.
2225 auto &DL = I->getDataLayout();
2226 if (hasIrregularType(Ty: ScalarTy, DL))
2227 return std::nullopt;
2228
2229 return Stride == 1 ? CM_Widen : CM_Widen_Reverse;
2230}
2231
2232void LoopVectorizationCostModel::collectLoopUniforms(ElementCount VF) {
2233 // We should not collect Uniforms more than once per VF. Right now,
2234 // this function is called from collectUniformsAndScalars(), which
2235 // already does this check. Collecting Uniforms for VF=1 does not make any
2236 // sense.
2237
2238 assert(VF.isVector() && !Uniforms.contains(VF) &&
2239 "This function should not be visited twice for the same VF");
2240
2241 // Visit the list of Uniforms. If we find no uniform value, we won't
2242 // analyze again. Uniforms.count(VF) will return 1.
2243 Uniforms[VF].clear();
2244
2245 // Now we know that the loop is vectorizable!
2246 // Collect instructions inside the loop that will remain uniform after
2247 // vectorization.
2248
2249 // Global values, params and instructions outside of current loop are out of
2250 // scope.
2251 auto IsOutOfScope = [&](Value *V) -> bool {
2252 Instruction *I = dyn_cast<Instruction>(Val: V);
2253 return (!I || !TheLoop->contains(Inst: I));
2254 };
2255
2256 // Worklist containing uniform instructions demanding lane 0.
2257 SetVector<Instruction *> Worklist;
2258
2259 // Add uniform instructions demanding lane 0 to the worklist. Instructions
2260 // that require predication must not be considered uniform after
2261 // vectorization, because that would create an erroneous replicating region
2262 // where only a single instance out of VF should be formed.
2263 auto AddToWorklistIfAllowed = [&](Instruction *I) -> void {
2264 if (IsOutOfScope(I)) {
2265 LLVM_DEBUG(dbgs() << "LV: Found not uniform due to scope: "
2266 << *I << "\n");
2267 return;
2268 }
2269 if (isPredicatedInst(I)) {
2270 LLVM_DEBUG(
2271 dbgs() << "LV: Found not uniform due to requiring predication: " << *I
2272 << "\n");
2273 return;
2274 }
2275 LLVM_DEBUG(dbgs() << "LV: Found uniform instruction: " << *I << "\n");
2276 Worklist.insert(X: I);
2277 };
2278
2279 // Start with the conditional branches exiting the loop. If the branch
2280 // condition is an instruction contained in the loop that is only used by the
2281 // branch, it is uniform. Note conditions from uncountable early exits are not
2282 // uniform.
2283 SmallVector<BasicBlock *> Exiting;
2284 TheLoop->getExitingBlocks(ExitingBlocks&: Exiting);
2285 for (BasicBlock *E : Exiting) {
2286 if (Legal->hasUncountableEarlyExit() && TheLoop->getLoopLatch() != E)
2287 continue;
2288 auto *Cmp = dyn_cast<Instruction>(Val: E->getTerminator()->getOperand(i: 0));
2289 if (!Cmp || !TheLoop->contains(Inst: Cmp) || !Cmp->hasOneUse())
2290 continue;
2291
2292 // If we have an exit condition that is actually two conditions (one
2293 // countable and the other uncountable) combined via an or, only add the
2294 // countable comparison as a uniform value.
2295 if (Legal->hasUncountableExitWithSideEffects() &&
2296 TheLoop->getLoopLatch() == E) {
2297 if (Instruction *Countable =
2298 Legal->findCountableComparisonInCombinedCondition(Cond: Cmp)) {
2299 if (Countable->hasOneUse())
2300 AddToWorklistIfAllowed(Countable);
2301 continue;
2302 }
2303 }
2304
2305 // Normal exit comparisons are uniform.
2306 AddToWorklistIfAllowed(Cmp);
2307 }
2308
2309 auto PrevVF = VF.divideCoefficientBy(RHS: 2);
2310 // Return true if all lanes perform the same memory operation, and we can
2311 // thus choose to execute only one.
2312 auto IsUniformMemOpUse = [&](Instruction *I) {
2313 // If the value was already known to not be uniform for the previous
2314 // (smaller VF), it cannot be uniform for the larger VF.
2315 if (PrevVF.isVector()) {
2316 auto Iter = Uniforms.find(Val: PrevVF);
2317 if (Iter != Uniforms.end() && !Iter->second.contains(Ptr: I))
2318 return false;
2319 }
2320 if (!isUniformMemOp(I&: *I, VF))
2321 return false;
2322 if (isa<LoadInst>(Val: I))
2323 // Loading the same address always produces the same result - at least
2324 // assuming aliasing and ordering which have already been checked.
2325 return true;
2326 // Storing the same value on every iteration.
2327 return TheLoop->isLoopInvariant(V: cast<StoreInst>(Val: I)->getValueOperand());
2328 };
2329
2330 auto IsUniformDecision = [&](Instruction *I, ElementCount VF) {
2331 InstWidening WideningDecision = getWideningDecision(I, VF);
2332 assert(WideningDecision != CM_Unknown &&
2333 "Widening decision should be ready at this moment");
2334
2335 if (IsUniformMemOpUse(I))
2336 return true;
2337
2338 return (WideningDecision == CM_Widen ||
2339 WideningDecision == CM_Widen_Reverse ||
2340 WideningDecision == CM_Interleave);
2341 };
2342
2343 // Returns true if Ptr is the pointer operand of a memory access instruction
2344 // I, I is known to not require scalarization, and the pointer is not also
2345 // stored.
2346 auto IsVectorizedMemAccessUse = [&](Instruction *I, Value *Ptr) -> bool {
2347 if (isa<StoreInst>(Val: I) && I->getOperand(i: 0) == Ptr)
2348 return false;
2349 return getLoadStorePointerOperand(V: I) == Ptr &&
2350 (IsUniformDecision(I, VF) || Legal->isInvariant(V: Ptr));
2351 };
2352
2353 // Holds a list of values which are known to have at least one uniform use.
2354 // Note that there may be other uses which aren't uniform. A "uniform use"
2355 // here is something which only demands lane 0 of the unrolled iterations;
2356 // it does not imply that all lanes produce the same value (e.g. this is not
2357 // the usual meaning of uniform)
2358 SetVector<Value *> HasUniformUse;
2359
2360 // Scan the loop for instructions which are either a) known to have only
2361 // lane 0 demanded or b) are uses which demand only lane 0 of their operand.
2362 for (auto *BB : TheLoop->blocks())
2363 for (auto &I : *BB) {
2364 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: &I)) {
2365 switch (II->getIntrinsicID()) {
2366 case Intrinsic::sideeffect:
2367 case Intrinsic::experimental_noalias_scope_decl:
2368 case Intrinsic::assume:
2369 case Intrinsic::lifetime_start:
2370 case Intrinsic::lifetime_end:
2371 if (TheLoop->hasLoopInvariantOperands(I: &I))
2372 AddToWorklistIfAllowed(&I);
2373 break;
2374 default:
2375 break;
2376 }
2377 }
2378
2379 if (auto *EVI = dyn_cast<ExtractValueInst>(Val: &I)) {
2380 if (IsOutOfScope(EVI->getAggregateOperand())) {
2381 AddToWorklistIfAllowed(EVI);
2382 continue;
2383 }
2384 // Only ExtractValue instructions where the aggregate value comes from a
2385 // call are allowed to be non-uniform.
2386 assert(isa<CallInst>(EVI->getAggregateOperand()) &&
2387 "Expected aggregate value to be call return value");
2388 }
2389
2390 // If there's no pointer operand, there's nothing to do.
2391 auto *Ptr = getLoadStorePointerOperand(V: &I);
2392 if (!Ptr)
2393 continue;
2394
2395 // If the pointer can be proven to be uniform, always add it to the
2396 // worklist.
2397 if (isa<Instruction>(Val: Ptr) && isUniform(V: Ptr, VF))
2398 AddToWorklistIfAllowed(cast<Instruction>(Val: Ptr));
2399
2400 if (IsUniformMemOpUse(&I))
2401 AddToWorklistIfAllowed(&I);
2402
2403 if (IsVectorizedMemAccessUse(&I, Ptr))
2404 HasUniformUse.insert(X: Ptr);
2405 }
2406
2407 // Add to the worklist any operands which have *only* uniform (e.g. lane 0
2408 // demanding) users. Since loops are assumed to be in LCSSA form, this
2409 // disallows uses outside the loop as well.
2410 for (auto *V : HasUniformUse) {
2411 if (IsOutOfScope(V))
2412 continue;
2413 auto *I = cast<Instruction>(Val: V);
2414 bool UsersAreMemAccesses = all_of(Range: I->users(), P: [&](User *U) -> bool {
2415 auto *UI = cast<Instruction>(Val: U);
2416 return TheLoop->contains(Inst: UI) && IsVectorizedMemAccessUse(UI, V);
2417 });
2418 if (UsersAreMemAccesses)
2419 AddToWorklistIfAllowed(I);
2420 }
2421
2422 // Expand Worklist in topological order: whenever a new instruction
2423 // is added , its users should be already inside Worklist. It ensures
2424 // a uniform instruction will only be used by uniform instructions.
2425 unsigned Idx = 0;
2426 while (Idx != Worklist.size()) {
2427 Instruction *I = Worklist[Idx++];
2428
2429 for (auto *OV : I->operand_values()) {
2430 // isOutOfScope operands cannot be uniform instructions.
2431 if (IsOutOfScope(OV))
2432 continue;
2433 // First order recurrence Phi's should typically be considered
2434 // non-uniform.
2435 auto *OP = dyn_cast<PHINode>(Val: OV);
2436 if (OP && Legal->isFixedOrderRecurrence(Phi: OP))
2437 continue;
2438 // If all the users of the operand are uniform, then add the
2439 // operand into the uniform worklist.
2440 auto *OI = cast<Instruction>(Val: OV);
2441 if (llvm::all_of(Range: OI->users(), P: [&](User *U) -> bool {
2442 auto *J = cast<Instruction>(Val: U);
2443 return Worklist.count(key: J) || IsVectorizedMemAccessUse(J, OI);
2444 }))
2445 AddToWorklistIfAllowed(OI);
2446 }
2447 }
2448
2449 // For an instruction to be added into Worklist above, all its users inside
2450 // the loop should also be in Worklist. However, this condition cannot be
2451 // true for phi nodes that form a cyclic dependence. We must process phi
2452 // nodes separately. An induction variable will remain uniform if all users
2453 // of the induction variable and induction variable update remain uniform.
2454 // The code below handles both pointer and non-pointer induction variables.
2455 BasicBlock *Latch = TheLoop->getLoopLatch();
2456 for (PHINode *Ind : Legal->getInductionVars().keys()) {
2457 auto *IndUpdate = cast<Instruction>(Val: Ind->getIncomingValueForBlock(BB: Latch));
2458
2459 // Determine if all users of the induction variable are uniform after
2460 // vectorization.
2461 bool UniformInd = all_of(Range: Ind->users(), P: [&](User *U) -> bool {
2462 auto *I = cast<Instruction>(Val: U);
2463 return I == IndUpdate || !TheLoop->contains(Inst: I) || Worklist.count(key: I) ||
2464 IsVectorizedMemAccessUse(I, Ind);
2465 });
2466 if (!UniformInd)
2467 continue;
2468
2469 // Determine if all users of the induction variable update instruction are
2470 // uniform after vectorization.
2471 bool UniformIndUpdate = all_of(Range: IndUpdate->users(), P: [&](User *U) -> bool {
2472 auto *I = cast<Instruction>(Val: U);
2473 return I == Ind || Worklist.count(key: I) ||
2474 IsVectorizedMemAccessUse(I, IndUpdate);
2475 });
2476 if (!UniformIndUpdate)
2477 continue;
2478
2479 // The induction variable and its update instruction will remain uniform.
2480 AddToWorklistIfAllowed(Ind);
2481 AddToWorklistIfAllowed(IndUpdate);
2482 }
2483
2484 Uniforms[VF].insert_range(R&: Worklist);
2485}
2486
2487FixedScalableVFPair
2488LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
2489 // Make sure once we return PartialAliasMaskingStatus is not "NotDecided".
2490 scope_exit EnsureAliasMaskingStatusIsDecidedOnReturn([this] {
2491 if (PartialAliasMaskingStatus == AliasMaskingStatus::NotDecided)
2492 PartialAliasMaskingStatus = AliasMaskingStatus::Disabled;
2493 });
2494
2495 // For outer loops, use simple type-based heuristic VF. No cost model or
2496 // memory dependence analysis is available.
2497 if (!TheLoop->isInnermost()) {
2498 return Config.computeVPlanOuterloopVF(UserVF);
2499 }
2500
2501 if (Legal->getRuntimePointerChecking()->Need && TTI.hasBranchDivergence()) {
2502 // TODO: It may be useful to do since it's still likely to be dynamically
2503 // uniform if the target can skip.
2504 reportVectorizationFailure(
2505 DebugMsg: "Not inserting runtime ptr check for divergent target",
2506 OREMsg: "runtime pointer checks needed. Not enabled for divergent target",
2507 ORETag: "CantVersionLoopWithDivergentTarget", ORE, TheLoop);
2508 return FixedScalableVFPair::getNone();
2509 }
2510
2511 ScalarEvolution *SE = PSE.getSE();
2512 ElementCount TC = getSmallConstantTripCount(SE, L: TheLoop);
2513 unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
2514 if (!MaxTC && EpilogueLoweringStatus == CM_EpilogueAllowed)
2515 MaxTC = getMaxTCFromNonZeroRange(PSE, L: TheLoop);
2516 LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
2517 if (TC != ElementCount::getFixed(MinVal: MaxTC))
2518 LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
2519 if (TC.isScalar()) {
2520 reportVectorizationFailure(
2521 DebugMsg: "Single iteration (non) loop",
2522 OREMsg: "loop trip count is one, irrelevant for vectorization",
2523 ORETag: "SingleIterationLoop", ORE, TheLoop);
2524 return FixedScalableVFPair::getNone();
2525 }
2526
2527 // If BTC matches the widest induction type and is -1 then the trip count
2528 // computation will wrap to 0 and the vector trip count will be 0. Do not try
2529 // to vectorize.
2530 const SCEV *BTC = SE->getBackedgeTakenCount(L: TheLoop);
2531 if (!isa<SCEVCouldNotCompute>(Val: BTC) &&
2532 BTC->getType()->getScalarSizeInBits() >=
2533 Legal->getWidestInductionType()->getScalarSizeInBits() &&
2534 SE->isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: BTC,
2535 RHS: SE->getMinusOne(Ty: BTC->getType()))) {
2536 reportVectorizationFailure(
2537 DebugMsg: "Trip count computation wrapped",
2538 OREMsg: "backedge-taken count is -1, loop trip count wrapped to 0",
2539 ORETag: "TripCountWrapped", ORE, TheLoop);
2540 return FixedScalableVFPair::getNone();
2541 }
2542
2543 assert(WideningDecisions.empty() && Uniforms.empty() && Scalars.empty() &&
2544 "No cost-modeling decisions should have been taken at this point");
2545
2546 switch (EpilogueLoweringStatus) {
2547 case CM_EpilogueAllowed:
2548 return Config.computeFeasibleMaxVF(MaxTripCount: MaxTC, UserVF, UserIC, FoldTailByMasking: false,
2549 RequiresScalarEpilogue: requiresScalarEpilogue(IsVectorizing: true));
2550 case CM_EpilogueNotAllowedFoldTail:
2551 [[fallthrough]];
2552 case CM_EpilogueNotNeededFoldTail:
2553 LLVM_DEBUG(dbgs() << "LV: tail-folding hint/switch found.\n"
2554 << "LV: Not allowing epilogue, creating tail-folded "
2555 << "vector loop.\n");
2556 break;
2557 case CM_EpilogueNotAllowedLowTripLoop:
2558 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to low trip "
2559 << "count.\n");
2560 break;
2561 case CM_EpilogueNotAllowedOptSize:
2562 LLVM_DEBUG(dbgs() << "LV: Not allowing epilogue due to -Os/-Oz.\n");
2563
2564 // Bail if runtime checks are required, which are not good when optimising
2565 // for size.
2566 if (Config.runtimeChecksRequired())
2567 return FixedScalableVFPair::getNone();
2568
2569 break;
2570 }
2571
2572 // Now try the tail folding
2573
2574 // Invalidate interleave groups that require an epilogue if we can't mask
2575 // the interleave-group.
2576 if (!useMaskedInterleavedAccesses(Opts, TTI)) {
2577 // Note: There is no need to invalidate any cost modeling decisions here, as
2578 // none were taken so far (see assertion above).
2579 InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
2580 }
2581
2582 FixedScalableVFPair MaxFactors = Config.computeFeasibleMaxVF(
2583 MaxTripCount: MaxTC, UserVF, UserIC, FoldTailByMasking: true, RequiresScalarEpilogue: requiresScalarEpilogue(IsVectorizing: true));
2584
2585 // Avoid tail folding if the trip count is known to be a multiple of any VF
2586 // we choose.
2587 std::optional<uint64_t> MaxPowerOf2RuntimeVF =
2588 MaxFactors.FixedVF.getFixedValue();
2589 if (MaxFactors.ScalableVF) {
2590 if (std::optional<uint64_t> MaxRuntimeScalableVF =
2591 getMaxRuntimeElementCount(EC: MaxFactors.ScalableVF, F: *TheFunction))
2592 MaxPowerOf2RuntimeVF =
2593 std::max(a: *MaxPowerOf2RuntimeVF, b: *MaxRuntimeScalableVF);
2594 else
2595 MaxPowerOf2RuntimeVF = std::nullopt; // Stick with tail-folding for now.
2596 }
2597
2598 // TODO: Make NoScalarEpilogueNeeded lambda a separate function to be used
2599 // only for main loop VF not also epilogueVF. Using it for epilogueVF against
2600 // full TC is inaccurate.
2601 auto NoScalarEpilogueNeeded = [this, &UserIC](uint64_t MaxRuntimeVF) {
2602 // Return false if the loop is neither a single-latch-exit loop nor an
2603 // early-exit loop as tail-folding is not supported in that case.
2604 if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch() &&
2605 !Legal->hasUncountableEarlyExit())
2606 return false;
2607 uint64_t MaxVFtimesIC = MaxRuntimeVF * std::max<uint64_t>(a: UserIC, b: 1);
2608 ScalarEvolution *SE = PSE.getSE();
2609 // Calling getSymbolicMaxBackedgeTakenCount enables support for loops
2610 // with uncountable exits. For countable loops, the symbolic maximum must
2611 // remain identical to the known back-edge taken count.
2612 const SCEV *BackedgeTakenCount = PSE.getSymbolicMaxBackedgeTakenCount();
2613 assert((Legal->hasUncountableEarlyExit() ||
2614 BackedgeTakenCount == PSE.getBackedgeTakenCount()) &&
2615 "Invalid loop count");
2616 const SCEV *ExitCount = SE->getAddExpr(
2617 LHS: BackedgeTakenCount, RHS: SE->getOne(Ty: BackedgeTakenCount->getType()));
2618 const SCEV *Rem = SE->getURemExpr(
2619 LHS: SE->applyLoopGuards(Expr: ExitCount, L: TheLoop),
2620 RHS: SE->getConstant(Ty: BackedgeTakenCount->getType(), V: MaxVFtimesIC));
2621 return Rem->isZero();
2622 };
2623
2624 if (MaxPowerOf2RuntimeVF > 0u) {
2625 assert((UserVF.isNonZero() || isPowerOf2_64(*MaxPowerOf2RuntimeVF)) &&
2626 "MaxFixedVF must be a power of 2");
2627 if (NoScalarEpilogueNeeded(*MaxPowerOf2RuntimeVF)) {
2628 // Accept MaxFixedVF if we do not have a tail.
2629 LLVM_DEBUG(dbgs() << "LV: No tail will remain for any chosen VF.\n");
2630 return MaxFactors;
2631 }
2632 }
2633
2634 auto ExpectedTC = getSmallBestKnownTC(Opts, PSE, L: TheLoop);
2635 if (ExpectedTC && ExpectedTC->isFixed() &&
2636 ExpectedTC->getFixedValue() <=
2637 TTI.getMinTripCountTailFoldingThreshold()) {
2638 // If we have a low-trip-count, or optimize for code size, and the
2639 // fixed-width VF is known to divide the trip count the fixed-width factor
2640 // in preference to allow the generation of a non-predicated loop.
2641 if ((EpilogueLoweringStatus == CM_EpilogueNotAllowedLowTripLoop ||
2642 EpilogueLoweringStatus == CM_EpilogueNotAllowedOptSize) &&
2643 NoScalarEpilogueNeeded(MaxFactors.FixedVF.getFixedValue())) {
2644 LLVM_DEBUG(dbgs() << "LV: Picking a fixed-width so that no tail will "
2645 "remain for any chosen VF.\n");
2646 MaxFactors.ScalableVF = ElementCount::getScalable(MinVal: 0);
2647 return MaxFactors;
2648 }
2649
2650 // Allow cases where the ExactTC == (VF * IC) + 1.
2651 //
2652 // This produces 1 vector iteration, and 1 scalar iteration with no
2653 // remainder. Later passes will eliminate the loop and leave straight-line
2654 // code as the both iteration counts are statically known.
2655 //
2656 // If a function is marked as minsize/optsize or OptForSize is set, do not
2657 // allow this form of transformation as this will increase CodeSize.
2658 //
2659 // For loops with small bodies, the cost model is not currently reliable
2660 // enough to accurately determine if vectorization is beneficial.
2661 unsigned EffectiveIC = UserIC > 0 ? UserIC : 1;
2662 unsigned MaxVFForTC = llvm::bit_floor(Value: TC.getFixedValue());
2663 if (TC.getFixedValue() - MaxVFForTC == 1 && MaxVFForTC / EffectiveIC > 1 &&
2664 MaxVFForTC <= (MaxFactors.FixedVF.getFixedValue() * EffectiveIC) &&
2665 !Config.OptForSize) {
2666 unsigned NumOfInstructions = llvm::sum_of(
2667 Range: llvm::map_range(C: TheLoop->blocks(),
2668 F: [](BasicBlock *BB) { return BB->size(); }),
2669 Init: unsigned(0));
2670 if (NumOfInstructions > Opts.low_trip_count_loop_body_size_limit) {
2671 unsigned VF = MaxVFForTC / EffectiveIC;
2672 LLVM_DEBUG(dbgs() << "LV: Picking MaxVF=" << VF
2673 << " with 1 scalar iteration remaining.\n");
2674 MaxFactors.FixedVF = ElementCount::getFixed(MinVal: VF);
2675 MaxFactors.ScalableVF = ElementCount::getScalable(MinVal: 0);
2676 return MaxFactors;
2677 }
2678 }
2679
2680 reportVectorizationFailure(
2681 DebugMsg: "The trip count is below the minial threshold value.",
2682 OREMsg: "loop trip count is too low, avoiding vectorization", ORETag: "LowTripCount",
2683 ORE, TheLoop);
2684 return FixedScalableVFPair::getNone();
2685 }
2686
2687 // If we don't know the precise trip count, or if the trip count that we
2688 // found modulo the vectorization factor is not zero, try to fold the tail
2689 // by masking.
2690 // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
2691 bool ContainsScalableVF = MaxFactors.ScalableVF.isNonZero();
2692 setTailFoldingStyle(IsScalableVF: ContainsScalableVF, UserIC);
2693 if (foldTailByMasking()) {
2694 if (foldTailWithEVL()) {
2695 LLVM_DEBUG(
2696 dbgs()
2697 << "LV: tail is folded with EVL, forcing unroll factor to be 1. Will "
2698 "try to generate VP Intrinsics with scalable vector "
2699 "factors only.\n");
2700 // Tail folded loop using VP intrinsics restricts the VF to be scalable
2701 // for now.
2702 // TODO: extend it for fixed vectors, if required.
2703 assert(ContainsScalableVF && "Expected scalable vector factor.");
2704
2705 MaxFactors.FixedVF = ElementCount::getFixed(MinVal: 1);
2706 } else {
2707 tryToEnablePartialAliasMasking();
2708 }
2709 return MaxFactors;
2710 }
2711
2712 // If there was a tail-folding hint/switch, but we can't fold the tail by
2713 // masking, fallback to a vectorization with an epilogue.
2714 if (EpilogueLoweringStatus == CM_EpilogueNotNeededFoldTail) {
2715 LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with an "
2716 "epilogue instead.\n");
2717 EpilogueLoweringStatus = CM_EpilogueAllowed;
2718 return MaxFactors;
2719 }
2720
2721 if (EpilogueLoweringStatus == CM_EpilogueNotAllowedFoldTail) {
2722 LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
2723 return FixedScalableVFPair::getNone();
2724 }
2725
2726 if (TC.isZero()) {
2727 reportVectorizationFailure(
2728 DebugMsg: "unable to calculate the loop count due to complex control flow",
2729 ORETag: "UnknownLoopCountComplexCFG", ORE, TheLoop);
2730 return FixedScalableVFPair::getNone();
2731 }
2732
2733 reportVectorizationFailure(
2734 DebugMsg: "Cannot optimize for size and vectorize at the same time.",
2735 OREMsg: "cannot optimize for size and vectorize at the same time. "
2736 "Enable vectorization of this loop with '#pragma clang loop "
2737 "vectorize(enable)' when compiling with -Os/-Oz",
2738 ORETag: "NoTailLoopWithOptForSize", ORE, TheLoop);
2739 return FixedScalableVFPair::getNone();
2740}
2741
2742void LoopVectorizationPlanner::emitInvalidCostRemarks(
2743 OptimizationRemarkEmitter *ORE) {
2744 using RecipeVFPair = std::pair<VPRecipeBase *, ElementCount>;
2745 SmallVector<RecipeVFPair> InvalidCosts;
2746 for (const auto &Plan : VPlans) {
2747 // Skip cost remarks when Plan is not compatible with the CM.
2748 // Specifically for the case of epilogue tail-folded Plans.
2749 if (Plan->hasTailFolded() != CM->foldTailByMasking())
2750 continue;
2751 for (ElementCount VF : Plan->vectorFactors()) {
2752 // The VPlan-based cost model is designed for computing vector cost.
2753 // Querying VPlan-based cost model with a scalar VF will cause some
2754 // errors because we expect the VF is vector for most of the widen
2755 // recipes.
2756 if (VF.isScalar())
2757 continue;
2758
2759 VPCostContext CostCtx(*TLI, *Plan, *CM, Config,
2760 /*ReusePrintingSlotTracker=*/true);
2761 precomputeCosts(Plan&: *Plan, VF, CostCtx);
2762 auto Iter = vp_depth_first_deep(G: Plan->getVectorLoopRegion()->getEntry());
2763 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: Iter)) {
2764 for (auto &R : *VPBB) {
2765 if (!R.cost(VF, Ctx&: CostCtx).isValid())
2766 InvalidCosts.emplace_back(Args: &R, Args&: VF);
2767 }
2768 }
2769 }
2770 }
2771 if (InvalidCosts.empty())
2772 return;
2773
2774 // Emit a report of VFs with invalid costs in the loop.
2775
2776 // Group the remarks per recipe, keeping the recipe order from InvalidCosts.
2777 DenseMap<VPRecipeBase *, unsigned> Numbering;
2778 unsigned I = 0;
2779 for (auto &Pair : InvalidCosts)
2780 if (Numbering.try_emplace(Key: Pair.first, Args&: I).second)
2781 ++I;
2782
2783 // Sort the list, first on recipe(number) then on VF.
2784 sort(C&: InvalidCosts, Comp: [&Numbering](RecipeVFPair &A, RecipeVFPair &B) {
2785 unsigned NA = Numbering[A.first];
2786 unsigned NB = Numbering[B.first];
2787 if (NA != NB)
2788 return NA < NB;
2789 return ElementCount::isKnownLT(LHS: A.second, RHS: B.second);
2790 });
2791
2792 // For a list of ordered recipe-VF pairs:
2793 // [(load, VF1), (load, VF2), (store, VF1)]
2794 // group the recipes together to emit separate remarks for:
2795 // load (VF1, VF2)
2796 // store (VF1)
2797 auto Tail = ArrayRef<RecipeVFPair>(InvalidCosts);
2798 auto Subset = ArrayRef<RecipeVFPair>();
2799 do {
2800 if (Subset.empty())
2801 Subset = Tail.take_front(N: 1);
2802
2803 VPRecipeBase *R = Subset.front().first;
2804
2805 unsigned Opcode =
2806 TypeSwitch<const VPRecipeBase *, unsigned>(R)
2807 .Case(caseFn: [](const VPHeaderPHIRecipe *R) { return Instruction::PHI; })
2808 .Case(
2809 caseFn: [](const VPWidenStoreRecipe *R) { return Instruction::Store; })
2810 .Case(caseFn: [](const VPWidenLoadRecipe *R) { return Instruction::Load; })
2811 .Case(caseFn: [](const VPWidenStoreEVLRecipe *R) {
2812 return Instruction::Store;
2813 })
2814 .Case(
2815 caseFn: [](const VPWidenLoadEVLRecipe *R) { return Instruction::Load; })
2816 .Case<VPWidenCallRecipe, VPWidenIntrinsicRecipe>(
2817 caseFn: [](const auto *R) { return Instruction::Call; })
2818 .Case<VPInstruction, VPWidenRecipe, VPReplicateRecipe,
2819 VPWidenCastRecipe>(
2820 caseFn: [](const auto *R) { return R->getOpcode(); })
2821 .Case(caseFn: [](const VPInterleaveRecipe *R) {
2822 return R->getStoredValues().empty() ? Instruction::Load
2823 : Instruction::Store;
2824 })
2825 .Case(caseFn: [](const VPReductionRecipe *R) {
2826 return RecurrenceDescriptor::getOpcode(Kind: R->getRecurrenceKind());
2827 })
2828 .Case(caseFn: [](const VPBranchOnMaskRecipe *) {
2829 return Instruction::CondBr;
2830 });
2831
2832 // If the next recipe is different, or if there are no other pairs,
2833 // emit a remark for the collated subset. e.g.
2834 // [(load, VF1), (load, VF2))]
2835 // to emit:
2836 // remark: invalid costs for 'load' at VF=(VF1, VF2)
2837 if (Subset == Tail || Tail[Subset.size()].first != R) {
2838 std::string OutString;
2839 raw_string_ostream OS(OutString);
2840 assert(!Subset.empty() && "Unexpected empty range");
2841 OS << "Recipe with invalid costs prevented vectorization at VF=(";
2842 for (const auto &Pair : Subset)
2843 OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second;
2844 OS << "):";
2845 if (Opcode == Instruction::Call) {
2846 StringRef Name = "";
2847 if (auto *Int = dyn_cast<VPWidenIntrinsicRecipe>(Val: R)) {
2848 Name = Int->getIntrinsicName();
2849 } else {
2850 auto *WidenCall = dyn_cast<VPWidenCallRecipe>(Val: R);
2851 Function *CalledFn =
2852 WidenCall
2853 ? WidenCall->getCalledScalarFunction()
2854 : cast<Function>(Val: R->getLastOperand()->getLiveInIRValue());
2855 Name = CalledFn->getName();
2856 }
2857 OS << " call to " << Name;
2858 } else
2859 OS << " " << Instruction::getOpcodeName(Opcode);
2860 reportVectorizationInfo(Msg: OutString, ORETag: "InvalidCost", ORE, TheLoop: OrigLoop, I: nullptr,
2861 DL: R->getDebugLoc());
2862 Tail = Tail.drop_front(N: Subset.size());
2863 Subset = {};
2864 } else
2865 // Grow the subset by one element
2866 Subset = Tail.take_front(N: Subset.size() + 1);
2867 } while (!Tail.empty());
2868}
2869
2870/// Check if any recipe of \p Plan will generate a vector value, which will be
2871/// assigned a vector register.
2872static bool willGenerateVectors(VPlan &Plan, ElementCount VF,
2873 const TargetTransformInfo &TTI) {
2874 assert(VF.isVector() && "Checking a scalar VF?");
2875 DenseSet<VPRecipeBase *> EphemeralRecipes;
2876 collectEphemeralRecipesForVPlan(Plan, EphRecipes&: EphemeralRecipes);
2877 // Set of already visited types.
2878 DenseSet<Type *> Visited;
2879 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2880 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
2881 for (VPRecipeBase &R : *VPBB) {
2882 if (EphemeralRecipes.contains(V: &R))
2883 continue;
2884 // Continue early if the recipe is considered to not produce a vector
2885 // result. Note that this includes VPInstruction where some opcodes may
2886 // produce a vector, to preserve existing behavior as VPInstructions model
2887 // aspects not directly mapped to existing IR instructions.
2888 switch (R.getVPRecipeID()) {
2889 case VPRecipeBase::VPDerivedIVSC:
2890 case VPRecipeBase::VPScalarIVStepsSC:
2891 case VPRecipeBase::VPReplicateSC:
2892 case VPRecipeBase::VPInstructionSC:
2893 case VPRecipeBase::VPCurrentIterationPHISC:
2894 case VPRecipeBase::VPVectorPointerSC:
2895 case VPRecipeBase::VPVectorEndPointerSC:
2896 case VPRecipeBase::VPExpandSCEVSC:
2897 case VPRecipeBase::VPPredInstPHISC:
2898 case VPRecipeBase::VPBranchOnMaskSC:
2899 case VPRecipeBase::VPConditionalInductionPHISC:
2900 continue;
2901 case VPRecipeBase::VPReductionSC:
2902 case VPRecipeBase::VPActiveLaneMaskPHISC:
2903 case VPRecipeBase::VPWidenCallSC:
2904 case VPRecipeBase::VPWidenCanonicalIVSC:
2905 case VPRecipeBase::VPWidenCastSC:
2906 case VPRecipeBase::VPWidenGEPSC:
2907 case VPRecipeBase::VPWidenIntrinsicSC:
2908 case VPRecipeBase::VPWidenMemIntrinsicSC:
2909 case VPRecipeBase::VPWidenSC:
2910 case VPRecipeBase::VPBlendSC:
2911 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
2912 case VPRecipeBase::VPHistogramSC:
2913 case VPRecipeBase::VPWidenPHISC:
2914 case VPRecipeBase::VPWidenIntOrFpInductionSC:
2915 case VPRecipeBase::VPWidenPointerInductionSC:
2916 case VPRecipeBase::VPReductionPHISC:
2917 case VPRecipeBase::VPInterleaveEVLSC:
2918 case VPRecipeBase::VPInterleaveSC:
2919 case VPRecipeBase::VPWidenLoadEVLSC:
2920 case VPRecipeBase::VPWidenLoadSC:
2921 case VPRecipeBase::VPWidenStoreEVLSC:
2922 case VPRecipeBase::VPWidenStoreSC:
2923 break;
2924 default:
2925 llvm_unreachable("unhandled recipe");
2926 }
2927
2928 auto WillGenerateTargetVectors = [&TTI, VF](Type *VectorTy) {
2929 unsigned NumLegalParts = TTI.getNumberOfParts(Tp: VectorTy);
2930 if (!NumLegalParts)
2931 return false;
2932 if (VF.isScalable()) {
2933 // <vscale x 1 x iN> is assumed to be profitable over iN because
2934 // scalable registers are a distinct register class from scalar
2935 // ones. If we ever find a target which wants to lower scalable
2936 // vectors back to scalars, we'll need to update this code to
2937 // explicitly ask TTI about the register class uses for each part.
2938 return NumLegalParts <= VF.getKnownMinValue();
2939 }
2940 // Two or more elements that share a register - are vectorized.
2941 return NumLegalParts < VF.getFixedValue();
2942 };
2943
2944 // If no def nor is a store, e.g., branches, continue - no value to check.
2945 if (R.getNumDefinedValues() == 0 &&
2946 !isa<VPWidenStoreRecipe, VPWidenStoreEVLRecipe, VPInterleaveBase>(Val: &R))
2947 continue;
2948 // For multi-def recipes, currently only interleaved loads, suffice to
2949 // check first def only.
2950 // For stores check their stored value; for interleaved stores suffice
2951 // the check first stored value only. In all cases this is the second
2952 // operand.
2953 VPValue *ToCheck =
2954 R.getNumDefinedValues() >= 1 ? R.getVPValue(I: 0) : R.getOperand(N: 1);
2955 Type *ScalarTy = ToCheck->getScalarType();
2956 if (!Visited.insert(V: {ScalarTy}).second)
2957 continue;
2958 Type *WideTy = toVectorizedTy(Ty: ScalarTy, EC: VF);
2959 if (any_of(Range: getContainedTypes(Ty: WideTy), P: WillGenerateTargetVectors))
2960 return true;
2961 }
2962 }
2963
2964 return false;
2965}
2966
2967static bool hasReplicatorRegion(VPlan &Plan) {
2968 return any_of(Range: VPBlockUtils::blocksOnly<VPRegionBlock>(Range: vp_depth_first_shallow(
2969 G: Plan.getVectorLoopRegion()->getEntry())),
2970 P: [](auto *VPRB) { return VPRB->isReplicator(); });
2971}
2972
2973/// Returns true if the VPlan contains a VPReductionPHIRecipe with
2974/// FindLast recurrence kind.
2975static bool hasFindLastReductionPhi(VPlan &Plan) {
2976 return any_of(Range: make_isa_range<VPReductionPHIRecipe>(
2977 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
2978 P: [](VPReductionPHIRecipe &RedPhi) {
2979 return RecurrenceDescriptor::isFindLastRecurrenceKind(
2980 Kind: RedPhi.getRecurrenceKind());
2981 });
2982}
2983
2984/// Returns true if the VPlan contains a VPConditionalInductionPHIRecipe.
2985static bool hasConditionalInductionPhi(VPlan &Plan) {
2986 return any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
2987 P: IsaPred<VPConditionalInductionPHIRecipe>);
2988}
2989
2990/// Determine how to lower the epilogue for the vector epilogue loop.
2991/// Check if there are any conflicts that prevent tail-folding the epilogue.
2992/// \return CM_EpilogueNotNeededFoldTail if epilogue tail-folding is possible,
2993/// otherwise CM_EpilogueAllowed.
2994static EpilogueLowering getEpilogueTailLowering(
2995 const VectorizeOptions &Opts, const LoopVectorizationCostModel &MainCM,
2996 const Loop *L, OptimizationRemarkEmitter *ORE,
2997 LoopVectorizationLegality &LVL, const LoopVectorizeHints &Hints,
2998 const TargetTransformInfo *TTI) {
2999 // Epilogue TF is only enabled when explicitly requested via command line.
3000 if (Opts.epilogue_tail_folding_policy != TailFoldingPolicyTy::PreferFoldTail)
3001 return CM_EpilogueAllowed;
3002
3003 if (!Opts.enable_epilogue_vectorization) {
3004 reportVectorizationInfo(
3005 Msg: "Options conflict, epilogue vectorization is disallowed while "
3006 "epilogue tail-folding allowed!",
3007 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
3008 return CM_EpilogueAllowed;
3009 }
3010
3011 if (!Hints.getWidth() || !hasForcedEpilogueVF()) {
3012 reportVectorizationInfo(Msg: "For now, epilogue tail-folding can't be "
3013 "applied without forced main/epilogue loop VF",
3014 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
3015 return CM_EpilogueAllowed;
3016 }
3017
3018 if (ElementCount::isKnownLE(LHS: Hints.getWidth(), RHS: EpilogueVectorizationForceVF)) {
3019 reportVectorizationInfo(Msg: "For now, epilogue tail-folding can't be applied "
3020 "when VF of the main loop <= VF of the epilogue",
3021 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
3022 return CM_EpilogueAllowed;
3023 }
3024
3025 if (!L->isInnermost()) {
3026 reportVectorizationInfo(
3027 Msg: "Epilogue tail-folding is not supported for outer loop",
3028 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3029 return CM_EpilogueAllowed;
3030 }
3031
3032 // If scalar epilogue is explicitly required, we can't apply TF.
3033 if (MainCM.requiresScalarEpilogue(/*IsVectorizing*/ true)) {
3034 reportVectorizationInfo(
3035 Msg: "Epilogue tail-folding can't be applied because scalar epilogue is "
3036 "required. Fall back to a normal epilogue",
3037 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3038 return CM_EpilogueAllowed;
3039 }
3040
3041 // If having epilogue is NOT allowed, then no epilogue to apply TF for.
3042 if (!MainCM.isEpilogueAllowed()) {
3043 reportVectorizationInfo(Msg: "Not applying tail-folding to the epilogue, since "
3044 "no epilogue is allowed.",
3045 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3046 return CM_EpilogueAllowed;
3047 }
3048
3049 if (L->getExitingBlock() != L->getLoopLatch() ||
3050 LVL.hasUncountableEarlyExit()) {
3051 reportVectorizationInfo(
3052 Msg: "Epilogue tail-folding is not supported yet for early-exit loops",
3053 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3054 return CM_EpilogueAllowed;
3055 }
3056
3057 // The epilogue reuses the main loop's interleave groups, so it can't be
3058 // tail-folded if the target can't mask interleaved accesses.
3059 // TODO: Add support once the epilogue has its own IAI, separate from the main
3060 // loop's.
3061 if (MainCM.InterleaveInfo.hasGroups() &&
3062 !useMaskedInterleavedAccesses(Opts, TTI: *TTI)) {
3063 reportVectorizationInfo(
3064 Msg: "Epilogue tail-folding is not supported with interleaved accesses "
3065 "when masking them isn't supported",
3066 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3067 return CM_EpilogueAllowed;
3068 }
3069
3070 if (Opts.force_partial_aliasing_vectorization) {
3071 reportVectorizationInfo(
3072 Msg: "Epilogue tail-folding is not supported with alias masking",
3073 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3074 return CM_EpilogueAllowed;
3075 }
3076
3077 if (!LVL.getReductionVars().empty()) {
3078 reportVectorizationInfo(
3079 Msg: "Epilogue tail-folding is not supported with reductions",
3080 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3081 return CM_EpilogueAllowed;
3082 }
3083
3084 if (!LVL.getFixedOrderRecurrences().empty()) {
3085 reportVectorizationInfo(
3086 Msg: "Epilogue tail-folding is not supported with fixed-order recurrence",
3087 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: L);
3088 return CM_EpilogueAllowed;
3089 }
3090
3091 // TODO: This is conservative: it rejects any target that prefers EVL, even
3092 // for fixed-width epilogue VFs where EVL won't be chosen. Move this check to
3093 // where the epilogue's TF style is known once epilogue TF is supported.
3094 TailFoldingStyle TFStyle = Opts.force_tail_folding_style.value_or(
3095 u: TTI->getPreferredTailFoldingStyle());
3096 // TODO: Remove once EVL recipes support cloning.
3097 if (TFStyle == TailFoldingStyle::DataWithEVL) {
3098 reportVectorizationInfo(Msg: "Epilogue tail-folding is not supported yet with "
3099 "EVL-based tail-folding",
3100 ORETag: "UnsupportedEpilogueTailFoldingPolicy", ORE, TheLoop: L);
3101 return CM_EpilogueAllowed;
3102 }
3103
3104 // We can apply tail-folding on the vectorized epilogue loop.
3105 return CM_EpilogueNotNeededFoldTail;
3106}
3107
3108bool VFSelectionContext::isEpilogueVectorizationProfitable(
3109 const ElementCount VF, const unsigned IC) const {
3110 // FIXME: We need a much better cost-model to take different parameters such
3111 // as register pressure, code size increase and cost of extra branches into
3112 // account. For now we apply a very crude heuristic and only consider loops
3113 // with vectorization factors larger than a certain value.
3114
3115 // Allow the target to opt out.
3116 if (!TTI.preferEpilogueVectorization(Iters: VF * IC))
3117 return false;
3118
3119 unsigned MinVFThreshold = Opts.epilogue_vectorization_minimum_VF.value_or(
3120 u: TTI.getEpilogueVectorizationMinVF());
3121 return estimateElementCount(VF: VF * IC, VScale: getVScaleForTuning()) >= MinVFThreshold;
3122}
3123
3124std::unique_ptr<VPlan> LoopVectorizationPlanner::selectBestEpiloguePlan(
3125 VPlan &MainPlan, ElementCount MainLoopVF, unsigned IC,
3126 bool ScalarEpilogueAllowed) {
3127 if (!Opts.enable_epilogue_vectorization) {
3128 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is disabled.\n");
3129 return nullptr;
3130 }
3131
3132 if (!ScalarEpilogueAllowed) {
3133 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because no "
3134 "epilogue is allowed.\n");
3135 return nullptr;
3136 }
3137
3138 if (vputils::findIncomingAliasMask(Plan: MainPlan)) {
3139 LLVM_DEBUG(
3140 dbgs()
3141 << "LEV: Epilogue vectorization not supported with alias masking.\n");
3142 return nullptr;
3143 }
3144
3145 // Not really a cost consideration, but check for unsupported cases here to
3146 // simplify the logic.
3147 if (!isCandidateForEpilogueVectorization(MainPlan)) {
3148 LLVM_DEBUG(dbgs() << "LEV: Unable to vectorize epilogue because the loop "
3149 "is not a supported candidate.\n");
3150 return nullptr;
3151 }
3152
3153 if (hasForcedEpilogueVF()) {
3154 if (estimateElementCount(VF: EpilogueVectorizationForceVF,
3155 VScale: Config.getVScaleForTuning()) >=
3156 IC * estimateElementCount(VF: MainLoopVF, VScale: Config.getVScaleForTuning())) {
3157 // Note that the main loop leaves IC * MainLoopVF iterations iff a scalar
3158 // epilogue is required, but then the epilogue loop also requires a scalar
3159 // epilogue.
3160 LLVM_DEBUG(dbgs() << "LEV: Forced epilogue VF results in dead epilogue "
3161 "vector loop, skipping vectorizing epilogue.\n");
3162 return nullptr;
3163 }
3164
3165 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization factor is forced.\n");
3166 if (hasPlanWithVF(VF: EpilogueVectorizationForceVF)) {
3167 std::unique_ptr<VPlan> Clone(
3168 getPlanFor(VF: EpilogueVectorizationForceVF).duplicate());
3169 Clone->setVF(EpilogueVectorizationForceVF);
3170 return Clone;
3171 }
3172
3173 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization forced factor is not "
3174 "viable.\n");
3175 return nullptr;
3176 }
3177
3178 if (OrigLoop->getHeader()->getParent()->hasOptSize()) {
3179 LLVM_DEBUG(
3180 dbgs() << "LEV: Epilogue vectorization skipped due to opt for size.\n");
3181 return nullptr;
3182 }
3183
3184 if (!Config.isEpilogueVectorizationProfitable(VF: MainLoopVF, IC)) {
3185 LLVM_DEBUG(dbgs() << "LEV: Epilogue vectorization is not profitable for "
3186 "this loop\n");
3187 return nullptr;
3188 }
3189
3190 // Check if a plan's vector loop processes fewer iterations than VF (e.g. when
3191 // interleave groups have been narrowed) narrowInterleaveGroups) and return
3192 // the adjusted, effective VF.
3193 using namespace VPlanPatternMatch;
3194 auto GetEffectiveVF = [](VPlan &Plan, ElementCount VF) -> ElementCount {
3195 auto *Exiting = Plan.getVectorLoopRegion()->getExitingBasicBlock();
3196 if (match(V: &Exiting->back(),
3197 P: m_BranchOnCount(Op0: m_Add(Op0: m_CanonicalIV(), Op1: m_Specific(VPV: &Plan.getUF())),
3198 Op1: m_VPValue())))
3199 return ElementCount::get(MinVal: 1, Scalable: VF.isScalable());
3200 return VF;
3201 };
3202
3203 // Check if the main loop processes fewer than MainLoopVF elements per
3204 // iteration (e.g. due to narrowing interleave groups). Adjust MainLoopVF
3205 // as needed.
3206 MainLoopVF = GetEffectiveVF(MainPlan, MainLoopVF);
3207
3208 // If MainLoopVF = vscale x 2, and vscale is expected to be 4, then we know
3209 // the main loop handles 8 lanes per iteration. We could still benefit from
3210 // vectorizing the epilogue loop with VF=4.
3211 ElementCount EstimatedRuntimeVF = ElementCount::getFixed(
3212 MinVal: estimateElementCount(VF: MainLoopVF, VScale: Config.getVScaleForTuning()));
3213
3214 Type *TCType = Legal->getWidestInductionType();
3215 const SCEV *RemainingIterations = nullptr;
3216 unsigned MaxTripCount = 0;
3217 const SCEV *TC = vputils::getSCEVExprForVPValue(V: MainPlan.getTripCount(), PSE);
3218 assert(!isa<SCEVCouldNotCompute>(TC) && "Trip count SCEV must be computable");
3219 const SCEV *KnownMinTC;
3220 bool ScalableTC = match(S: TC, P: m_scev_c_Mul(Op0: m_SCEV(V&: KnownMinTC), Op1: m_SCEVVScale()));
3221 bool ScalableRemIter = false;
3222 ScalarEvolution &SE = *PSE.getSE();
3223 // Use versions of TC and VF in which both are either scalable or fixed.
3224 if (ScalableTC == MainLoopVF.isScalable()) {
3225 ScalableRemIter = ScalableTC;
3226 RemainingIterations =
3227 SE.getURemExpr(LHS: TC, RHS: SE.getElementCount(Ty: TCType, EC: MainLoopVF * IC));
3228 } else if (ScalableTC) {
3229 const SCEV *EstimatedTC = SE.getMulExpr(
3230 LHS: KnownMinTC,
3231 RHS: SE.getConstant(Ty: TCType, V: Config.getVScaleForTuning().value_or(u: 1)));
3232 RemainingIterations = SE.getURemExpr(
3233 LHS: EstimatedTC, RHS: SE.getElementCount(Ty: TCType, EC: MainLoopVF * IC));
3234 } else
3235 RemainingIterations =
3236 SE.getURemExpr(LHS: TC, RHS: SE.getElementCount(Ty: TCType, EC: EstimatedRuntimeVF * IC));
3237
3238 // No iterations left to process in the epilogue.
3239 if (RemainingIterations->isZero())
3240 return nullptr;
3241
3242 if (MainLoopVF.isFixed()) {
3243 MaxTripCount = MainLoopVF.getFixedValue() * IC - 1;
3244 if (SE.isKnownPredicate(Pred: CmpInst::ICMP_ULT, LHS: RemainingIterations,
3245 RHS: SE.getConstant(Ty: TCType, V: MaxTripCount))) {
3246 MaxTripCount = SE.getUnsignedRangeMax(S: RemainingIterations).getZExtValue();
3247 }
3248 LLVM_DEBUG(dbgs() << "LEV: Maximum Trip Count for Epilogue: "
3249 << MaxTripCount << "\n");
3250 }
3251
3252 auto SkipVF = [&](const SCEV *VF, const SCEV *RemIter) -> bool {
3253 return SE.isKnownPredicate(Pred: CmpInst::ICMP_UGT, LHS: VF, RHS: RemIter);
3254 };
3255 VectorizationFactor Result = VectorizationFactor::Disabled();
3256 VPlan *BestPlan = nullptr;
3257 for (auto &NextVF : ProfitableVFs) {
3258 // Skip candidate VFs without a corresponding VPlan.
3259 if (!hasPlanWithVF(VF: NextVF.Width))
3260 continue;
3261
3262 VPlan &CurrentPlan = getPlanFor(VF: NextVF.Width);
3263 ElementCount EffectiveVF = GetEffectiveVF(CurrentPlan, NextVF.Width);
3264 // Skip fixed vector VFs > than the estimated runtime VF, or any VF > than
3265 // the VF of the main loop.
3266 if ((!EffectiveVF.isScalable() && MainLoopVF.isScalable() &&
3267 ElementCount::isKnownGT(LHS: EffectiveVF, RHS: EstimatedRuntimeVF)) ||
3268 ElementCount::isKnownGT(LHS: EffectiveVF, RHS: MainLoopVF))
3269 continue;
3270
3271 // If EffectiveVF is greater than the number of remaining iterations, the
3272 // epilogue loop would be dead. Skip such factors. If the epilogue plan
3273 // also has narrowed interleave groups, use the effective VF since
3274 // the epilogue step will be reduced to its IC.
3275 // TODO: We should also consider comparing against a scalable
3276 // RemainingIterations when SCEV be able to evaluate non-canonical
3277 // vscale-based expressions.
3278 if (!ScalableRemIter) {
3279 // Handle the case where EffectiveVF and RemainingIterations are in
3280 // different numerical spaces.
3281 if (EffectiveVF.isScalable())
3282 EffectiveVF = ElementCount::getFixed(
3283 MinVal: estimateElementCount(VF: EffectiveVF, VScale: Config.getVScaleForTuning()));
3284 if (SkipVF(SE.getElementCount(Ty: TCType, EC: EffectiveVF), RemainingIterations))
3285 continue;
3286 }
3287
3288 if (Result.Width.isScalar() ||
3289 isMoreProfitable(A: NextVF, B: Result, MaxTripCount,
3290 HasTail: !MainPlan.hasTailFolded(),
3291 /*IsEpilogue*/ true)) {
3292 Result = NextVF;
3293 BestPlan = &CurrentPlan;
3294 }
3295 }
3296
3297 if (!BestPlan)
3298 return nullptr;
3299
3300 LLVM_DEBUG(dbgs() << "LEV: Vectorizing epilogue loop with VF = "
3301 << Result.Width << "\n");
3302 std::unique_ptr<VPlan> Clone(BestPlan->duplicate());
3303 Clone->setVF(Result.Width);
3304 return Clone;
3305}
3306
3307unsigned
3308LoopVectorizationPlanner::selectInterleaveCount(VPlan &Plan, ElementCount VF,
3309 InstructionCost LoopCost) {
3310 // -- The interleave heuristics --
3311 // We interleave the loop in order to expose ILP and reduce the loop overhead.
3312 // There are many micro-architectural considerations that we can't predict
3313 // at this level. For example, frontend pressure (on decode or fetch) due to
3314 // code size, or the number and capabilities of the execution ports.
3315 //
3316 // We use the following heuristics to select the interleave count:
3317 // 1. If the code has reductions, then we interleave to break the cross
3318 // iteration dependency.
3319 // 2. If the loop is really small, then we interleave to reduce the loop
3320 // overhead.
3321 // 3. We don't interleave if we think that we will spill registers to memory
3322 // due to the increased register pressure.
3323
3324 // Do not interleave tail-folded loops, as the overhead of multiple
3325 // instructions to calculate the predicate is likely not beneficial.
3326 // If an epilogue is not allowed for any other reason, do not interleave.
3327 if (!CM->isEpilogueAllowed())
3328 return 1;
3329
3330 if (any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3331 P: IsaPred<VPCurrentIterationPHIRecipe>)) {
3332 LLVM_DEBUG(dbgs() << "LV: Loop requires variable-length step. "
3333 "Unroll factor forced to be 1.\n");
3334 return 1;
3335 }
3336
3337 // We used the distance for the interleave count.
3338 if (!Legal->isSafeForAnyVectorWidth())
3339 return 1;
3340
3341 // We don't attempt to perform interleaving for loops with uncountable early
3342 // exits because the VPInstruction::AnyOf code cannot currently handle
3343 // multiple parts.
3344 if (Plan.hasEarlyExit())
3345 return 1;
3346
3347 // TODO: Conditional inductions don't support interleaving yet.
3348 if (hasConditionalInductionPhi(Plan))
3349 return 1;
3350
3351 const bool HasReductions =
3352 any_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
3353 P: IsaPred<VPReductionPHIRecipe>);
3354
3355 // FIXME: implement interleaving for FindLast transform correctly.
3356 if (hasFindLastReductionPhi(Plan))
3357 return 1;
3358
3359 VPRegisterUsage R = calculateRegisterUsageForPlan(Plan, VFs: {VF}, TTI)[0];
3360
3361 // If we did not calculate the cost for VF (because the user selected the VF)
3362 // then we calculate the cost of VF here.
3363 if (LoopCost == 0) {
3364 if (VF.isScalar())
3365 LoopCost = CM->expectedCost(VF);
3366 else
3367 LoopCost = cost(Plan, VF, RU: &R, PlanCM&: *CM);
3368 assert(LoopCost.isValid() && "Expected to have chosen a VF with valid cost");
3369
3370 // Loop body is free and there is no need for interleaving.
3371 if (LoopCost == 0)
3372 return 1;
3373 }
3374
3375 // We divide by these constants so assume that we have at least one
3376 // instruction that uses at least one register.
3377 for (auto &Pair : R.MaxLocalUsers) {
3378 Pair.second = std::max(a: Pair.second, b: 1U);
3379 }
3380
3381 // We calculate the interleave count using the following formula.
3382 // Subtract the number of loop invariants from the number of available
3383 // registers. These registers are used by all of the interleaved instances.
3384 // Next, divide the remaining registers by the number of registers that is
3385 // required by the loop, in order to estimate how many parallel instances
3386 // fit without causing spills. All of this is rounded down if necessary to be
3387 // a power of two. We want power of two interleave count to simplify any
3388 // addressing operations or alignment considerations.
3389 // We also want power of two interleave counts to ensure that the induction
3390 // variable of the vector loop wraps to zero, when tail is folded by masking;
3391 // this currently happens when OptForSize, in which case IC is set to 1 above.
3392 unsigned IC = UINT_MAX;
3393
3394 for (const auto &Pair : R.MaxLocalUsers) {
3395 unsigned TargetNumRegisters = TTI.getNumberOfRegisters(ClassID: Pair.first);
3396 LLVM_DEBUG(dbgs() << "LV: The target has " << TargetNumRegisters
3397 << " registers of "
3398 << TTI.getRegisterClassName(Pair.first)
3399 << " register class\n");
3400 TargetNumRegisters = (VF.isScalar() ? Opts.force_target_num_scalar_regs
3401 : Opts.force_target_num_vector_regs)
3402 .value_or(u&: TargetNumRegisters);
3403 unsigned MaxLocalUsers = Pair.second;
3404 unsigned LoopInvariantRegs = 0;
3405 if (R.LoopInvariantRegs.contains(Key: Pair.first))
3406 LoopInvariantRegs = R.LoopInvariantRegs[Pair.first];
3407
3408 unsigned TmpIC = llvm::bit_floor(Value: (TargetNumRegisters - LoopInvariantRegs) /
3409 MaxLocalUsers);
3410 // Don't count the induction variable as interleaved.
3411 if (Opts.enable_ind_var_reg_heur) {
3412 TmpIC = llvm::bit_floor(Value: (TargetNumRegisters - LoopInvariantRegs - 1) /
3413 std::max(a: 1U, b: (MaxLocalUsers - 1)));
3414 }
3415
3416 IC = std::min(a: IC, b: TmpIC);
3417 }
3418
3419 // Clamp the interleave ranges to reasonable counts.
3420 bool HasUnorderedReductions =
3421 HasReductions &&
3422 !any_of(Range: make_isa_range<VPReductionPHIRecipe>(
3423 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
3424 P: [](VPReductionPHIRecipe &RedR) { return RedR.isOrdered(); });
3425 unsigned MaxInterleaveCount =
3426 TTI.getMaxInterleaveFactor(VF, HasUnorderedReductions);
3427 LLVM_DEBUG(dbgs() << "LV: MaxInterleaveFactor for the target is "
3428 << MaxInterleaveCount << "\n");
3429
3430 // Check if the user has overridden the max.
3431 MaxInterleaveCount = (VF.isScalar() ? Opts.force_target_max_scalar_interleave
3432 : Opts.force_target_max_vector_interleave)
3433 .value_or(u&: MaxInterleaveCount);
3434
3435 // Try to get the exact trip count, or an estimate based on profiling data or
3436 // ConstantMax from PSE, failing that.
3437 auto BestKnownTC =
3438 getSmallBestKnownTC(Opts, PSE, L: OrigLoop,
3439 /*CanUseConstantMax=*/true,
3440 /*CanExcludeZeroTrips=*/CM->isEpilogueAllowed());
3441
3442 // For fixed length VFs treat a scalable trip count as unknown.
3443 if (BestKnownTC && (BestKnownTC->isFixed() || VF.isScalable())) {
3444 // Re-evaluate trip counts and VFs to be in the same numerical space.
3445 unsigned AvailableTC =
3446 estimateElementCount(VF: *BestKnownTC, VScale: Config.getVScaleForTuning());
3447 unsigned EstimatedVF =
3448 estimateElementCount(VF, VScale: Config.getVScaleForTuning());
3449
3450 // At least one iteration must be scalar when this constraint holds. So the
3451 // maximum available iterations for interleaving is one less.
3452 if (Plan.requiresScalarEpilogue())
3453 --AvailableTC;
3454
3455 unsigned InterleaveCountLB = bit_floor(Value: std::max(
3456 a: 1u, b: std::min(a: AvailableTC / (EstimatedVF * 2), b: MaxInterleaveCount)));
3457
3458 if (getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop).isNonZero()) {
3459 // If the best known trip count is exact, we select between two
3460 // prospective ICs, where
3461 //
3462 // 1) the aggressive IC is capped by the trip count divided by VF
3463 // 2) the conservative IC is capped by the trip count divided by (VF * 2)
3464 //
3465 // The final IC is selected in a way that the epilogue loop trip count is
3466 // minimized while maximizing the IC itself, so that we either run the
3467 // vector loop at least once if it generates a small epilogue loop, or
3468 // else we run the vector loop at least twice.
3469
3470 unsigned InterleaveCountUB = bit_floor(Value: std::max(
3471 a: 1u, b: std::min(a: AvailableTC / EstimatedVF, b: MaxInterleaveCount)));
3472 MaxInterleaveCount = InterleaveCountLB;
3473
3474 if (InterleaveCountUB != InterleaveCountLB) {
3475 unsigned TailTripCountUB =
3476 (AvailableTC % (EstimatedVF * InterleaveCountUB));
3477 unsigned TailTripCountLB =
3478 (AvailableTC % (EstimatedVF * InterleaveCountLB));
3479 // If both produce same scalar tail, maximize the IC to do the same work
3480 // in fewer vector loop iterations
3481 if (TailTripCountUB == TailTripCountLB)
3482 MaxInterleaveCount = InterleaveCountUB;
3483 }
3484 } else {
3485 // If trip count is an estimated compile time constant, limit the
3486 // IC to be capped by the trip count divided by VF * 2, such that the
3487 // vector loop runs at least twice to make interleaving seem profitable
3488 // when there is an epilogue loop present. Since exact Trip count is not
3489 // known we choose to be conservative in our IC estimate.
3490 MaxInterleaveCount = InterleaveCountLB;
3491 }
3492 }
3493
3494 assert(MaxInterleaveCount > 0 &&
3495 "Maximum interleave count must be greater than 0");
3496
3497 // Clamp the calculated IC to be between the 1 and the max interleave count
3498 // that the target and trip count allows.
3499 if (IC > MaxInterleaveCount)
3500 IC = MaxInterleaveCount;
3501 else
3502 // Make sure IC is greater than 0.
3503 IC = std::max(a: 1u, b: IC);
3504
3505 assert(IC > 0 && "Interleave count must be greater than 0.");
3506
3507 // FIXME: VF is required to be a vector only because there are several recipes
3508 // whose computeCost function asserts that the VF is a vector.
3509 if (Opts.enable_interleave_to_reduce_stalls && VF.isVector()) {
3510 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce stall cycles due to "
3511 "instruction latency.\n");
3512 // Interleaving a replicator region means we get extra copies of the blocks
3513 // in the region. Vector values used in those blocks are expected to later
3514 // be sunk to just before them, negating any reduction in stalls. Therefore
3515 // there's no gain to be had from interleaving.
3516 if (hasReplicatorRegion(Plan)) {
3517 LLVM_DEBUG(dbgs() << "LV: Not interleaving due to replicator region.\n");
3518 return 1;
3519 }
3520 VPCostContext LatencyCtx(*TLI, Plan, *CM, Config,
3521 /*ReusePrintingSlotTracker=*/true);
3522 LatencyCtx.CostKind = TTI::TCK_Latency;
3523 VPCostContext ThroughputCtx(*TLI, Plan, *CM, Config,
3524 /*ReusePrintingSlotTracker=*/true);
3525 ThroughputCtx.CostKind = TTI::TCK_RecipThroughput;
3526 // Find the largest interleave count that will help with reducing stalls.
3527 unsigned StallsIC = 1;
3528 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3529 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
3530 for (VPRecipeBase &R : *VPBB) {
3531 // Assuming that each value will be needed as soon as it's generated the
3532 // number of stall cycles is one less than the latency.
3533 InstructionCost StallCycles = R.cost(VF, Ctx&: LatencyCtx) - 1;
3534 if (Opts.force_target_load_latency > 0 && R.mayReadFromMemory())
3535 StallCycles = Opts.force_target_load_latency - 1;
3536 // Each interleaving above 1 will reduce the stalls by RecipThroughput,
3537 // so pick the interleaving that will reduce stalls to zero.
3538 InstructionCost RecipThroughput = R.cost(VF, Ctx&: ThroughputCtx);
3539 if (StallCycles.isValid() && RecipThroughput.isValid() &&
3540 StallCycles > 0 && RecipThroughput > 0) {
3541 unsigned ThisIC = bit_floor<uint64_t>(
3542 Value: 1 + (StallCycles / RecipThroughput).getValue());
3543 StallsIC = std::max(a: StallsIC, b: ThisIC);
3544 }
3545 }
3546 }
3547 if (StallsIC <= 1) {
3548 LLVM_DEBUG(
3549 dbgs() << "LV: Not interleaving as it wouldn't reduce stalls.\n");
3550 return 1;
3551 }
3552 IC = std::min(a: IC, b: StallsIC);
3553 return IC;
3554 }
3555
3556 // Interleave if we vectorized this loop and there is a reduction that could
3557 // benefit from interleaving.
3558 if (VF.isVector() && HasReductions) {
3559 LLVM_DEBUG(dbgs() << "LV: Interleaving because of reductions.\n");
3560 return IC;
3561 }
3562
3563 // For any scalar loop that either requires runtime checks or tail-folding we
3564 // are better off leaving this to the unroller. Note that if we've already
3565 // vectorized the loop we will have done the runtime check and so interleaving
3566 // won't require further checks.
3567 bool ScalarInterleavingRequiresPredication =
3568 (VF.isScalar() && any_of(Range: OrigLoop->blocks(), P: [this](BasicBlock *BB) {
3569 return Legal->blockNeedsPredication(BB);
3570 }));
3571 bool ScalarInterleavingRequiresRuntimePointerCheck =
3572 (VF.isScalar() && Legal->getRuntimePointerChecking()->Need);
3573
3574 // We want to interleave small loops in order to reduce the loop overhead and
3575 // potentially expose ILP opportunities.
3576 LLVM_DEBUG(dbgs() << "LV: Loop cost is " << LoopCost << '\n'
3577 << "LV: IC is " << IC << '\n'
3578 << "LV: VF is " << VF << '\n');
3579 const bool AggressivelyInterleave =
3580 TTI.enableAggressiveInterleaving(LoopHasReductions: HasReductions);
3581 if (!ScalarInterleavingRequiresRuntimePointerCheck &&
3582 !ScalarInterleavingRequiresPredication &&
3583 LoopCost < Opts.small_loop_cost) {
3584 // We assume that the cost overhead is 1 and we use the cost model
3585 // to estimate the cost of the loop and interleave until the cost of the
3586 // loop overhead is about 5% of the cost of the loop.
3587 unsigned SmallIC =
3588 std::min(a: IC, b: (unsigned)llvm::bit_floor<uint64_t>(Value: Opts.small_loop_cost /
3589 LoopCost.getValue()));
3590
3591 // Interleave until store/load ports (estimated by max interleave count) are
3592 // saturated.
3593 unsigned NumStores = 0;
3594 unsigned NumLoads = 0;
3595 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3596 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
3597 for (VPRecipeBase &R : *VPBB) {
3598 if (isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Val: &R)) {
3599 NumLoads++;
3600 continue;
3601 }
3602 if (isa<VPWidenStoreRecipe, VPWidenStoreEVLRecipe>(Val: &R)) {
3603 NumStores++;
3604 continue;
3605 }
3606
3607 if (auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(Val: &R)) {
3608 if (unsigned StoreOps = InterleaveR->getNumStoreOperands())
3609 NumStores += StoreOps;
3610 else
3611 NumLoads += InterleaveR->getNumDefinedValues();
3612 continue;
3613 }
3614 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
3615 NumLoads += isa<LoadInst>(Val: RepR->getUnderlyingInstr());
3616 NumStores += isa<StoreInst>(Val: RepR->getUnderlyingInstr());
3617 continue;
3618 }
3619 if (isa<VPHistogramRecipe>(Val: &R)) {
3620 NumLoads++;
3621 NumStores++;
3622 continue;
3623 }
3624 }
3625 }
3626 unsigned StoresIC = IC / (NumStores ? NumStores : 1);
3627 unsigned LoadsIC = IC / (NumLoads ? NumLoads : 1);
3628
3629 // There is little point in interleaving for reductions containing selects
3630 // and compares when VF=1 since it may just create more overhead than it's
3631 // worth for loops with small trip counts. This is because we still have to
3632 // do the final reduction after the loop.
3633 bool HasSelectCmpReductions =
3634 HasReductions &&
3635 any_of(Range: make_isa_range<VPReductionPHIRecipe>(
3636 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
3637 P: [](VPReductionPHIRecipe &RedR) {
3638 return RecurrenceDescriptor::isAnyOfRecurrenceKind(
3639 Kind: RedR.getRecurrenceKind()) ||
3640 RecurrenceDescriptor::isFindIVRecurrenceKind(
3641 Kind: RedR.getRecurrenceKind());
3642 });
3643 if (HasSelectCmpReductions) {
3644 LLVM_DEBUG(dbgs() << "LV: Not interleaving select-cmp reductions.\n");
3645 return 1;
3646 }
3647
3648 // If we have a scalar reduction (vector reductions are already dealt with
3649 // by this point), we can increase the critical path length if the loop
3650 // we're interleaving is inside another loop. For tree-wise reductions
3651 // set the limit to 2, and for ordered reductions it's best to disable
3652 // interleaving entirely.
3653 if (HasReductions && OrigLoop->getLoopDepth() > 1) {
3654 bool HasOrderedReductions =
3655 any_of(Range: make_isa_range<VPReductionPHIRecipe>(
3656 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()),
3657 P: [](VPReductionPHIRecipe &RedR) { return RedR.isOrdered(); });
3658 if (HasOrderedReductions) {
3659 LLVM_DEBUG(
3660 dbgs() << "LV: Not interleaving scalar ordered reductions.\n");
3661 return 1;
3662 }
3663
3664 unsigned F = Opts.max_nested_scalar_reduction_interleave;
3665 SmallIC = std::min(a: SmallIC, b: F);
3666 StoresIC = std::min(a: StoresIC, b: F);
3667 LoadsIC = std::min(a: LoadsIC, b: F);
3668 }
3669
3670 if (Opts.enable_loadstore_runtime_interleave &&
3671 std::max(a: StoresIC, b: LoadsIC) > SmallIC) {
3672 LLVM_DEBUG(
3673 dbgs() << "LV: Interleaving to saturate store or load ports.\n");
3674 return std::max(a: StoresIC, b: LoadsIC);
3675 }
3676
3677 // If there are scalar reductions and TTI has enabled aggressive
3678 // interleaving for reductions, we will interleave to expose ILP.
3679 if (VF.isScalar() && AggressivelyInterleave) {
3680 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3681 // Interleave no less than SmallIC but not as aggressive as the normal IC
3682 // to satisfy the rare situation when resources are too limited.
3683 return std::max(a: IC / 2, b: SmallIC);
3684 }
3685
3686 LLVM_DEBUG(dbgs() << "LV: Interleaving to reduce branch cost.\n");
3687 return SmallIC;
3688 }
3689
3690 // Interleave if this is a large loop (small loops are already dealt with by
3691 // this point) that could benefit from interleaving.
3692 if (AggressivelyInterleave) {
3693 LLVM_DEBUG(dbgs() << "LV: Interleaving to expose ILP.\n");
3694 return IC;
3695 }
3696
3697 LLVM_DEBUG(dbgs() << "LV: Not Interleaving.\n");
3698 return 1;
3699}
3700
3701bool LoopVectorizationCostModel::useEmulatedMaskMemRefHack(
3702 Instruction *I, ElementCount VF) const {
3703 // TODO: Cost model for emulated masked load/store is completely
3704 // broken. This hack guides the cost model to use an artificially
3705 // high enough value to practically disable vectorization with such
3706 // operations, except where previously deployed legality hack allowed
3707 // using very low cost values. This is to avoid regressions coming simply
3708 // from moving "masked load/store" check from legality to cost model.
3709 // Masked Load/Gather emulation was previously never allowed.
3710 // Limited number of Masked Store/Scatter emulation was allowed.
3711 assert((isPredicatedInst(I)) &&
3712 "Expecting a scalar emulated instruction");
3713 return isa<LoadInst>(Val: I) ||
3714 (isa<StoreInst>(Val: I) && NumPredStores > Opts.vectorize_num_stores_pred);
3715}
3716
3717void LoopVectorizationCostModel::collectInstsToScalarize(ElementCount VF) {
3718 assert(VF.isVector() && "Expected VF >= 2");
3719 assert(!InstsToScalarize.contains(VF) &&
3720 !PredicatedBBsAfterVectorization.contains(VF) &&
3721 "Routine expected to be called once per VF");
3722
3723 // Initialize a mapping for VF in InstsToScalalarize. If we find that it's
3724 // not profitable to scalarize any instructions, the presence of VF in the
3725 // map will indicate that we've analyzed it already.
3726 ScalarCostsTy &ScalarCostsVF = InstsToScalarize[VF];
3727
3728 // Find all the instructions that are scalar with predication in the loop and
3729 // determine if it would be better to not if-convert the blocks they are in.
3730 // If so, we also record the instructions to scalarize.
3731 for (BasicBlock *BB : TheLoop->blocks()) {
3732 if (!blockNeedsPredicationForAnyReason(BB))
3733 continue;
3734 for (Instruction &I : *BB)
3735 if (isScalarWithPredication(I: &I, VF)) {
3736 ScalarCostsTy ScalarCosts;
3737 // Do not apply discount logic for:
3738 // 1. Scalars after vectorization, as there will only be a single copy
3739 // of the instruction.
3740 // 2. Scalable VF, as that would lead to invalid scalarization costs.
3741 // 3. Emulated masked memrefs, if a hacked cost is needed.
3742 // 4. Compressed loads/stores (which do not support scalarization)
3743 if (!isScalarAfterVectorization(I: &I, VF) && !VF.isScalable() &&
3744 !useEmulatedMaskMemRefHack(I: &I, VF) &&
3745 !Legal->isCompressedLoadOrStore(I: &I) &&
3746 computePredInstDiscount(PredInst: &I, ScalarCosts, VF) >= 0) {
3747 for (const auto &[I, IC] : ScalarCosts)
3748 ScalarCostsVF.insert(KV: {I, IC});
3749 }
3750 // Remember that BB will remain after vectorization.
3751 PredicatedBBsAfterVectorization[VF].insert(Ptr: BB);
3752 for (auto *Pred : predecessors(BB)) {
3753 if (Pred->getSingleSuccessor() == BB)
3754 PredicatedBBsAfterVectorization[VF].insert(Ptr: Pred);
3755 }
3756 }
3757 }
3758}
3759
3760InstructionCost LoopVectorizationCostModel::computePredInstDiscount(
3761 Instruction *PredInst, ScalarCostsTy &ScalarCosts, ElementCount VF) {
3762 assert(!isUniformAfterVectorization(PredInst, VF) &&
3763 "Instruction marked uniform-after-vectorization will be predicated");
3764
3765 // Initialize the discount to zero, meaning that the scalar version and the
3766 // vector version cost the same.
3767 InstructionCost Discount = 0;
3768
3769 // Holds instructions to analyze. The instructions we visit are mapped in
3770 // ScalarCosts. Those instructions are the ones that would be scalarized if
3771 // we find that the scalar version costs less.
3772 SmallVector<Instruction *, 8> Worklist;
3773
3774 // Returns true if the given instruction can be scalarized.
3775 auto CanBeScalarized = [&](Instruction *I) -> bool {
3776 // We only attempt to scalarize instructions forming a single-use chain
3777 // from the original predicated block that would otherwise be vectorized.
3778 // Although not strictly necessary, we give up on instructions we know will
3779 // already be scalar to avoid traversing chains that are unlikely to be
3780 // beneficial.
3781 if (!I->hasOneUse() || PredInst->getParent() != I->getParent() ||
3782 isScalarAfterVectorization(I, VF))
3783 return false;
3784
3785 // If the instruction is scalar with predication, it will be analyzed
3786 // separately. We ignore it within the context of PredInst.
3787 if (isScalarWithPredication(I, VF))
3788 return false;
3789
3790 // If any of the instruction's operands are uniform after vectorization,
3791 // the instruction cannot be scalarized. This prevents, for example, a
3792 // masked load from being scalarized.
3793 //
3794 // We assume we will only emit a value for lane zero of an instruction
3795 // marked uniform after vectorization, rather than VF identical values.
3796 // Thus, if we scalarize an instruction that uses a uniform, we would
3797 // create uses of values corresponding to the lanes we aren't emitting code
3798 // for. This behavior can be changed by allowing getScalarValue to clone
3799 // the lane zero values for uniforms rather than asserting.
3800 for (Use &U : I->operands())
3801 if (auto *J = dyn_cast<Instruction>(Val: U.get()))
3802 if (isUniformAfterVectorization(I: J, VF))
3803 return false;
3804
3805 // Otherwise, we can scalarize the instruction.
3806 return true;
3807 };
3808
3809 // Compute the expected cost discount from scalarizing the entire expression
3810 // feeding the predicated instruction. We currently only consider expressions
3811 // that are single-use instruction chains.
3812 Worklist.push_back(Elt: PredInst);
3813 while (!Worklist.empty()) {
3814 Instruction *I = Worklist.pop_back_val();
3815
3816 // If we've already analyzed the instruction, there's nothing to do.
3817 if (ScalarCosts.contains(Key: I))
3818 continue;
3819
3820 // Cannot scalarize fixed-order recurrence phis at the moment.
3821 if (isa<PHINode>(Val: I) && Legal->isFixedOrderRecurrence(Phi: cast<PHINode>(Val: I)))
3822 continue;
3823
3824 // Compute the cost of the vector instruction. Note that this cost already
3825 // includes the scalarization overhead of the predicated instruction.
3826 InstructionCost VectorCost = getInstructionCost(I, VF);
3827
3828 // Compute the cost of the scalarized instruction. This cost is the cost of
3829 // the instruction as if it wasn't if-converted and instead remained in the
3830 // predicated block. We will scale this cost by block probability after
3831 // computing the scalarization overhead.
3832 InstructionCost ScalarCost =
3833 VF.getFixedValue() * getInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1));
3834
3835 // Compute the scalarization overhead of needed insertelement instructions
3836 // and phi nodes.
3837 if (isScalarWithPredication(I, VF) && !I->getType()->isVoidTy()) {
3838 Type *WideTy = toVectorizedTy(Ty: I->getType(), EC: VF);
3839 for (Type *VectorTy : getContainedTypes(Ty: WideTy)) {
3840 ScalarCost += TTI.getScalarizationOverhead(
3841 Ty: cast<VectorType>(Val: VectorTy), DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
3842 /*Insert=*/true,
3843 /*Extract=*/false, CostKind: Config.CostKind);
3844 }
3845 ScalarCost += VF.getFixedValue() *
3846 TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
3847 }
3848
3849 // Compute the scalarization overhead of needed extractelement
3850 // instructions. For each of the instruction's operands, if the operand can
3851 // be scalarized, add it to the worklist; otherwise, account for the
3852 // overhead.
3853 for (Use &U : I->operands())
3854 if (auto *J = dyn_cast<Instruction>(Val: U.get())) {
3855 assert(canVectorizeTy(J->getType()) &&
3856 "Instruction has non-scalar type");
3857 if (CanBeScalarized(J))
3858 Worklist.push_back(Elt: J);
3859 else if (needsExtract(V: J, VF)) {
3860 Type *WideTy = toVectorizedTy(Ty: J->getType(), EC: VF);
3861 for (Type *VectorTy : getContainedTypes(Ty: WideTy)) {
3862 ScalarCost += TTI.getScalarizationOverhead(
3863 Ty: cast<VectorType>(Val: VectorTy),
3864 DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()), /*Insert*/ false,
3865 /*Extract*/ true, CostKind: Config.CostKind);
3866 }
3867 }
3868 }
3869
3870 // Scale the total scalar cost by block probability.
3871 ScalarCost /= getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
3872
3873 // Compute the discount. A non-negative discount means the vector version
3874 // of the instruction costs more, and scalarizing would be beneficial.
3875 Discount += VectorCost - ScalarCost;
3876 ScalarCosts[I] = ScalarCost;
3877 }
3878
3879 return Discount;
3880}
3881
3882InstructionCost LoopVectorizationCostModel::expectedCost(ElementCount VF) {
3883 InstructionCost Cost;
3884 assert(VF.isScalar() && "must only be called for scalar VFs");
3885
3886 // For each block.
3887 for (BasicBlock *BB : TheLoop->blocks()) {
3888 InstructionCost BlockCost;
3889
3890 // For each instruction in the old loop.
3891 for (Instruction &I : *BB) {
3892 // Skip ignored values.
3893 if (ValuesToIgnore.count(Ptr: &I) ||
3894 (VF.isVector() && VecValuesToIgnore.count(Ptr: &I)))
3895 continue;
3896
3897 InstructionCost C = getInstructionCost(I: &I, VF);
3898
3899 // Check if we should override the cost.
3900 if (C.isValid() && Opts.force_target_instruction_cost)
3901 C = InstructionCost(*Opts.force_target_instruction_cost);
3902
3903 BlockCost += C;
3904 LLVM_DEBUG(dbgs() << "LV: Found an estimated cost of " << C << " for VF "
3905 << VF << " For instruction: " << I << '\n');
3906 }
3907
3908 // In the scalar loop, we may not always execute the predicated block, if it
3909 // is an if-else block. Thus, scale the block's cost by the probability of
3910 // executing it. getPredBlockCostDivisor will return 1 for blocks that are
3911 // only predicated by the header mask when folding the tail.
3912 Cost += BlockCost / getPredBlockCostDivisor(CostKind: Config.CostKind, BB);
3913 }
3914
3915 return Cost;
3916}
3917
3918/// Gets the address access SCEV for Ptr, if it should be used for cost modeling
3919/// according to isAddressSCEVForCost.
3920///
3921/// This SCEV can be sent to the Target in order to estimate the address
3922/// calculation cost.
3923static const SCEV *getAddressAccessSCEV(
3924 Value *Ptr,
3925 PredicatedScalarEvolution &PSE,
3926 const Loop *TheLoop) {
3927 const SCEV *Addr = PSE.getSCEV(V: Ptr);
3928 return vputils::isAddressSCEVForCost(Addr, SE&: *PSE.getSE(), L: TheLoop) ? Addr
3929 : nullptr;
3930}
3931
3932InstructionCost
3933LoopVectorizationCostModel::getMemInstScalarizationCost(Instruction *I,
3934 ElementCount VF) {
3935 assert(VF.isVector() &&
3936 "Scalarization cost of instruction implies vectorization.");
3937 if (VF.isScalable())
3938 return InstructionCost::getInvalid();
3939
3940 Type *ValTy = getLoadStoreType(I);
3941 auto *SE = PSE.getSE();
3942
3943 unsigned AS = getLoadStoreAddressSpace(I);
3944 Value *Ptr = getLoadStorePointerOperand(V: I);
3945 Type *PtrTy = toVectorTy(Scalar: Ptr->getType(), EC: VF);
3946 // NOTE: PtrTy is a vector to signal `TTI::getAddressComputationCost`
3947 // that it is being called from this specific place.
3948
3949 // Figure out whether the access is strided and get the stride value
3950 // if it's known in compile time
3951 const SCEV *PtrSCEV = getAddressAccessSCEV(Ptr, PSE, TheLoop);
3952
3953 // Get the cost of the scalar memory instruction and address computation.
3954 InstructionCost Cost =
3955 VF.getFixedValue() *
3956 TTI.getAddressComputationCost(PtrTy, SE, Ptr: PtrSCEV, CostKind: Config.CostKind);
3957
3958 // Don't pass *I here, since it is scalar but will actually be part of a
3959 // vectorized loop where the user of it is a vectorized instruction.
3960 const Align Alignment = getLoadStoreAlignment(I);
3961 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
3962 Cost += VF.getFixedValue() *
3963 TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: ValTy->getScalarType(), Alignment,
3964 AddressSpace: AS, CostKind: Config.CostKind, OpdInfo: OpInfo);
3965
3966 // Get the overhead of the extractelement and insertelement instructions
3967 // we might create due to scalarization.
3968 Cost += getScalarizationOverhead(I, VF);
3969
3970 // If we have a predicated load/store, it will need extra i1 extracts and
3971 // conditional branches, but may not be executed for each vector lane. Scale
3972 // the cost by the probability of executing the predicated block.
3973 if (isPredicatedInst(I)) {
3974 Cost /= getPredBlockCostDivisor(CostKind: Config.CostKind, BB: I->getParent());
3975
3976 // Add the cost of an i1 extract and a branch
3977 auto *VecI1Ty =
3978 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: ValTy->getContext()), EC: VF);
3979 Cost += TTI.getScalarizationOverhead(
3980 Ty: VecI1Ty, DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
3981 /*Insert=*/false, /*Extract=*/true, CostKind: Config.CostKind);
3982 Cost += TTI.getCFInstrCost(Opcode: Instruction::CondBr, CostKind: Config.CostKind);
3983
3984 if (useEmulatedMaskMemRefHack(I, VF))
3985 // Artificially setting to a high enough value to practically disable
3986 // vectorization with such operations.
3987 Cost = 3000000;
3988 }
3989
3990 return Cost;
3991}
3992
3993InstructionCost LoopVectorizationCostModel::getConsecutiveMemOpCost(
3994 Instruction *I, ElementCount VF, InstWidening Kind) {
3995 assert((Kind == CM_Widen || Kind == CM_Widen_Reverse) &&
3996 "Expected a consecutive widening decision");
3997 Type *ValTy = getLoadStoreType(I);
3998 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
3999 unsigned AS = getLoadStoreAddressSpace(I);
4000
4001 const Align Alignment = getLoadStoreAlignment(I);
4002 InstructionCost Cost = 0;
4003 if (isMaskRequired(I)) {
4004 Intrinsic::ID LoadIID = Intrinsic::masked_load;
4005 Intrinsic::ID StoreIID = Intrinsic::masked_store;
4006 if (Legal->isCompressedLoadOrStore(I)) {
4007 LoadIID = Intrinsic::masked_expandload;
4008 StoreIID = Intrinsic::masked_compressstore;
4009 }
4010
4011 unsigned IID = I->getOpcode() == Instruction::Load ? LoadIID : StoreIID;
4012 Cost += TTI.getMemIntrinsicInstrCost(
4013 MICA: MemIntrinsicCostAttributes(IID, VectorTy, Alignment, AS),
4014 CostKind: Config.CostKind);
4015 } else {
4016 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4017 Cost += TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: VectorTy, Alignment, AddressSpace: AS,
4018 CostKind: Config.CostKind, OpdInfo: OpInfo, I);
4019 }
4020
4021 if (Kind == CM_Widen_Reverse)
4022 Cost += TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Reverse, DstTy: VectorTy,
4023 SrcTy: VectorTy, CostKind: Config.CostKind, Mask: {}, Index: 0);
4024 return Cost;
4025}
4026
4027InstructionCost
4028LoopVectorizationCostModel::getUniformMemOpCost(Instruction *I,
4029 ElementCount VF) const {
4030 assert(isUniformMemOp(*I, VF));
4031
4032 Type *ValTy = getLoadStoreType(I);
4033 Type *PtrTy = getLoadStorePointerOperand(V: I)->getType();
4034 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4035 const Align Alignment = getLoadStoreAlignment(I);
4036 unsigned AS = getLoadStoreAddressSpace(I);
4037 if (isa<LoadInst>(Val: I)) {
4038 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4039 CostKind: Config.CostKind) +
4040 TTI.getMemoryOpCost(Opcode: Instruction::Load, Src: ValTy, Alignment, AddressSpace: AS,
4041 CostKind: Config.CostKind) +
4042 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Broadcast, DstTy: VectorTy,
4043 SrcTy: VectorTy, CostKind: Config.CostKind);
4044 }
4045 StoreInst *SI = cast<StoreInst>(Val: I);
4046
4047 bool IsLoopInvariantStoreValue = Legal->isInvariant(V: SI->getValueOperand());
4048 // TODO: We have existing tests that request the cost of extracting element
4049 // VF.getKnownMinValue() - 1 from a scalable vector. This does not represent
4050 // the actual generated code, which involves extracting the last element of
4051 // a scalable vector where the lane to extract is unknown at compile time.
4052 InstructionCost Cost =
4053 TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr, CostKind: Config.CostKind) +
4054 TTI.getMemoryOpCost(Opcode: Instruction::Store, Src: ValTy, Alignment, AddressSpace: AS,
4055 CostKind: Config.CostKind);
4056 if (!IsLoopInvariantStoreValue)
4057 Cost += TTI.getIndexedVectorInstrCostFromEnd(Opcode: Instruction::ExtractElement,
4058 Val: VectorTy, CostKind: Config.CostKind, Index: 0);
4059 return Cost;
4060}
4061
4062InstructionCost
4063LoopVectorizationCostModel::getGatherScatterCost(Instruction *I,
4064 ElementCount VF) const {
4065 Type *ValTy = getLoadStoreType(I);
4066 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4067 const Align Alignment = getLoadStoreAlignment(I);
4068 Value *Ptr = getLoadStorePointerOperand(V: I);
4069 Type *PtrTy = Ptr->getType();
4070
4071 if (!isUniform(V: Ptr, VF))
4072 PtrTy = toVectorTy(Scalar: PtrTy, EC: VF);
4073
4074 unsigned IID = I->getOpcode() == Instruction::Load
4075 ? Intrinsic::masked_gather
4076 : Intrinsic::masked_scatter;
4077 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4078 CostKind: Config.CostKind) +
4079 TTI.getMemIntrinsicInstrCost(
4080 MICA: MemIntrinsicCostAttributes(IID, VectorTy, Ptr, isMaskRequired(I),
4081 Alignment, I),
4082 CostKind: Config.CostKind);
4083}
4084
4085InstructionCost
4086LoopVectorizationCostModel::getInterleaveGroupCost(Instruction *I,
4087 ElementCount VF) const {
4088 const auto *Group = getInterleavedAccessGroup(Instr: I);
4089 assert(Group && "Fail to get an interleaved access group.");
4090
4091 Instruction *InsertPos = Group->getInsertPos();
4092 Type *ValTy = getLoadStoreType(I: InsertPos);
4093 auto *VectorTy = cast<VectorType>(Val: toVectorTy(Scalar: ValTy, EC: VF));
4094 unsigned AS = getLoadStoreAddressSpace(I: InsertPos);
4095
4096 unsigned InterleaveFactor = Group->getFactor();
4097 auto *WideVecTy = VectorType::get(ElementType: ValTy, EC: VF * InterleaveFactor);
4098
4099 // Holds the indices of existing members in the interleaved group.
4100 SmallVector<unsigned, 4> Indices;
4101 for (unsigned IF = 0; IF < InterleaveFactor; IF++)
4102 if (Group->getMember(Index: IF))
4103 Indices.push_back(Elt: IF);
4104
4105 // Calculate the cost of the whole interleaved group.
4106 bool UseMaskForGaps =
4107 (Group->requiresScalarEpilogue() && !isEpilogueAllowed()) ||
4108 (isa<StoreInst>(Val: I) && !Group->isFull());
4109 InstructionCost Cost = TTI.getInterleavedMemoryOpCost(
4110 Opcode: InsertPos->getOpcode(), VecTy: WideVecTy, Factor: Group->getFactor(), Indices,
4111 Alignment: Group->getAlign(), AddressSpace: AS, CostKind: Config.CostKind, UseMaskForCond: isMaskRequired(I),
4112 UseMaskForGaps);
4113
4114 if (Group->isReverse()) {
4115 // TODO: Add support for reversed masked interleaved access.
4116 assert(!isMaskRequired(I) &&
4117 "Reverse masked interleaved access not supported.");
4118 Cost += Group->getNumMembers() *
4119 TTI.getShuffleCost(Kind: TargetTransformInfo::SK_Reverse, DstTy: VectorTy,
4120 SrcTy: VectorTy, CostKind: Config.CostKind, Mask: {}, Index: 0);
4121 }
4122 return Cost;
4123}
4124
4125InstructionCost
4126LoopVectorizationCostModel::getMemoryInstructionCost(Instruction *I,
4127 ElementCount VF) {
4128 // Calculate scalar cost only. Vectorization cost should be ready at this
4129 // moment.
4130 if (VF.isScalar()) {
4131 Type *ValTy = getLoadStoreType(I);
4132 Type *PtrTy = getLoadStorePointerOperand(V: I)->getType();
4133 const Align Alignment = getLoadStoreAlignment(I);
4134 unsigned AS = getLoadStoreAddressSpace(I);
4135
4136 TTI::OperandValueInfo OpInfo = TTI::getOperandInfo(V: I->getOperand(i: 0));
4137 return TTI.getAddressComputationCost(PtrTy, SE: nullptr, Ptr: nullptr,
4138 CostKind: Config.CostKind) +
4139 TTI.getMemoryOpCost(Opcode: I->getOpcode(), Src: ValTy, Alignment, AddressSpace: AS,
4140 CostKind: Config.CostKind, OpdInfo: OpInfo, I);
4141 }
4142 return getWideningCost(I, VF);
4143}
4144
4145InstructionCost
4146LoopVectorizationCostModel::getScalarizationOverhead(Instruction *I,
4147 ElementCount VF) const {
4148
4149 // There is no mechanism yet to create a scalable scalarization loop,
4150 // so this is currently Invalid.
4151 if (VF.isScalable())
4152 return InstructionCost::getInvalid();
4153
4154 if (VF.isScalar())
4155 return 0;
4156
4157 InstructionCost Cost = 0;
4158 Type *RetTy = toVectorizedTy(Ty: I->getType(), EC: VF);
4159 if (!RetTy->isVoidTy() &&
4160 (!isa<LoadInst>(Val: I) || !TTI.supportsEfficientVectorElementLoadStore())) {
4161
4162 TTI::VectorInstrContext VIC = TTI::VectorInstrContext::None;
4163 if (isa<LoadInst>(Val: I))
4164 VIC = TTI::VectorInstrContext::Load;
4165 else if (isa<StoreInst>(Val: I))
4166 VIC = TTI::VectorInstrContext::Store;
4167
4168 for (Type *VectorTy : getContainedTypes(Ty: RetTy)) {
4169 Cost += TTI.getScalarizationOverhead(
4170 Ty: cast<VectorType>(Val: VectorTy), DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4171 /*Insert=*/true, /*Extract=*/false, CostKind: Config.CostKind,
4172 /*ForPoisonSrc=*/true, VL: {}, VIC);
4173 }
4174 }
4175
4176 // Some targets keep addresses scalar.
4177 if (isa<LoadInst>(Val: I) && !TTI.prefersVectorizedAddressing())
4178 return Cost;
4179
4180 // Some targets support efficient element stores.
4181 if (isa<StoreInst>(Val: I) && TTI.supportsEfficientVectorElementLoadStore())
4182 return Cost;
4183
4184 // Collect operands to consider.
4185 CallInst *CI = dyn_cast<CallInst>(Val: I);
4186 Instruction::op_range Ops = CI ? CI->args() : I->operands();
4187
4188 // Skip operands that do not require extraction/scalarization and do not incur
4189 // any overhead.
4190 SmallVector<Type *> Tys;
4191 SmallPtrSet<Value *, 4> SeenOps;
4192 for (Value *Op : Ops)
4193 if (needsExtract(V: Op, VF) && SeenOps.insert(Ptr: Op).second)
4194 Tys.push_back(Elt: maybeVectorizeType(Ty: Op->getType(), VF));
4195
4196 TTI::VectorInstrContext OperandVIC = isa<StoreInst>(Val: I)
4197 ? TTI::VectorInstrContext::Store
4198 : TTI::VectorInstrContext::None;
4199 return Cost +
4200 TTI.getOperandsScalarizationOverhead(Tys, CostKind: Config.CostKind, VIC: OperandVIC);
4201}
4202
4203void LoopVectorizationCostModel::setCostBasedWideningDecision(ElementCount VF) {
4204 if (VF.isScalar())
4205 return;
4206
4207 // TODO: We should generate better code and update the cost model for
4208 // predicated uniform stores. Today they are treated as any other
4209 // predicated store (see added test cases in
4210 // invariant-store-vectorization.ll).
4211 NumPredStores = 0;
4212 for (BasicBlock *BB : TheLoop->blocks())
4213 for (Instruction &I : *BB)
4214 if (isa<StoreInst>(Val: &I) && isScalarWithPredication(I: &I, VF))
4215 ++NumPredStores;
4216
4217 for (BasicBlock *BB : TheLoop->blocks()) {
4218 // For each instruction in the old loop.
4219 for (Instruction &I : *BB) {
4220 Value *Ptr = getLoadStorePointerOperand(V: &I);
4221
4222 // Ignore if no pointer found or a widening decision has already been
4223 // made.
4224 if (!Ptr || (getWideningDecision(I: &I, VF) != CM_Unknown))
4225 continue;
4226
4227 LLVM_DEBUG(dbgs() << "LV: Memory widening: calculating best strategy for "
4228 << I << '\n');
4229 if (isUniformMemOp(I, VF)) {
4230 auto IsLegalToScalarize = [&]() {
4231 if (!VF.isScalable())
4232 // Scalarization of fixed length vectors "just works".
4233 return true;
4234
4235 // We have dedicated lowering for unpredicated uniform loads and
4236 // stores. Note that even with tail folding we know that at least
4237 // one lane is active (i.e. generalized predication is not possible
4238 // here), and the logic below depends on this fact.
4239 if (!foldTailByMasking())
4240 return true;
4241
4242 // For scalable vectors, a uniform memop load is always
4243 // uniform-by-parts and we know how to scalarize that.
4244 if (isa<LoadInst>(Val: I))
4245 return true;
4246
4247 // A uniform store isn't neccessarily uniform-by-part
4248 // and we can't assume scalarization.
4249 auto &SI = cast<StoreInst>(Val&: I);
4250 return TheLoop->isLoopInvariant(V: SI.getValueOperand());
4251 };
4252
4253 const InstructionCost GatherScatterCost =
4254 isLegalGatherOrScatter(I: &I, VF) ? getGatherScatterCost(I: &I, VF)
4255 : InstructionCost::getInvalid();
4256
4257 // Load: Scalar load + broadcast
4258 // Store: Scalar store + isLoopInvariantStoreValue ? 0 : extract
4259 // FIXME: This cost is a significant under-estimate for tail folded
4260 // memory ops.
4261 const InstructionCost ScalarizationCost =
4262 IsLegalToScalarize() ? getUniformMemOpCost(I: &I, VF)
4263 : InstructionCost::getInvalid();
4264
4265 // Choose better solution for the current VF, Note that Invalid
4266 // costs compare as maximumal large. If both are invalid, we get
4267 // scalable invalid which signals a failure and a vectorization abort.
4268 LLVM_DEBUG(dbgs() << "LV: Memory widening: uniform memory op has "
4269 "GatherScatterCost = "
4270 << GatherScatterCost << ", ScalarizationCost = "
4271 << ScalarizationCost << '\n');
4272 if (GatherScatterCost < ScalarizationCost)
4273 setWideningDecision(I: &I, VF, W: CM_GatherScatter, Cost: GatherScatterCost);
4274 else
4275 setWideningDecision(I: &I, VF, W: CM_Scalarize, Cost: ScalarizationCost);
4276 continue;
4277 }
4278
4279 // We assume that widening is the best solution when possible.
4280 if (std::optional<InstWidening> Decision =
4281 memoryInstructionCanBeWidened(I: &I, VF)) {
4282 InstructionCost WidenCost = getConsecutiveMemOpCost(I: &I, VF, Kind: *Decision);
4283 LLVM_DEBUG(
4284 dbgs() << "LV: Memory widening: can be widened normally with cost "
4285 << WidenCost << '\n');
4286 setWideningDecision(I: &I, VF, W: *Decision, Cost: WidenCost);
4287 continue;
4288 }
4289
4290 assert(!Legal->isCompressedLoadOrStore(&I) &&
4291 "compressed loads/stores should be handled in "
4292 "memoryInstructionCanBeWidened");
4293
4294 // Choose between Interleaving, Gather/Scatter or Scalarization.
4295 InstructionCost InterleaveCost = InstructionCost::getInvalid();
4296 unsigned NumAccesses = 1;
4297 if (isAccessInterleaved(Instr: &I)) {
4298 const auto *Group = getInterleavedAccessGroup(Instr: &I);
4299 assert(Group && "Fail to get an interleaved access group.");
4300
4301 NumAccesses = Group->getNumMembers();
4302 if (interleavedAccessCanBeWidened(I: &I, VF))
4303 InterleaveCost = getInterleaveGroupCost(I: &I, VF);
4304 }
4305
4306 InstructionCost GatherScatterCost =
4307 isLegalGatherOrScatter(I: &I, VF)
4308 ? getGatherScatterCost(I: &I, VF) * NumAccesses
4309 : InstructionCost::getInvalid();
4310
4311 InstructionCost ScalarizationCost =
4312 getMemInstScalarizationCost(I: &I, VF) * NumAccesses;
4313
4314 // Choose better solution for the current VF,
4315 // write down this decision and use it during vectorization.
4316 InstructionCost Cost;
4317 InstWidening Decision;
4318 if (InterleaveCost <= GatherScatterCost &&
4319 InterleaveCost < ScalarizationCost) {
4320 Decision = CM_Interleave;
4321 Cost = InterleaveCost;
4322 } else if (GatherScatterCost < ScalarizationCost) {
4323 Decision = CM_GatherScatter;
4324 Cost = GatherScatterCost;
4325 } else {
4326 Decision = CM_Scalarize;
4327 Cost = ScalarizationCost;
4328 }
4329 LLVM_DEBUG(
4330 dbgs() << "LV: Memory widening: InterleaveCost = " << InterleaveCost
4331 << ", GatherScatterCost = " << GatherScatterCost
4332 << ", ScalarizationCost = " << ScalarizationCost << '\n');
4333
4334 // If the instructions belongs to an interleave group, the whole group
4335 // receives the same decision. The whole group receives the cost, but
4336 // the cost will actually be assigned to one instruction.
4337 if (const auto *Group = getInterleavedAccessGroup(Instr: &I)) {
4338 if (Decision == CM_Scalarize) {
4339 for (Instruction *I : Group->members())
4340 setWideningDecision(I, VF, W: Decision,
4341 Cost: getMemInstScalarizationCost(I, VF));
4342 } else {
4343 setWideningDecision(Grp: Group, VF, W: Decision, Cost);
4344 }
4345 } else
4346 setWideningDecision(I: &I, VF, W: Decision, Cost);
4347 }
4348 }
4349
4350 // Make sure that any load of address and any other address computation
4351 // remains scalar unless there is gather/scatter support. This avoids
4352 // inevitable extracts into address registers, and also has the benefit of
4353 // activating LSR more, since that pass can't optimize vectorized
4354 // addresses.
4355 if (TTI.prefersVectorizedAddressing())
4356 return;
4357
4358 // Start with all scalar pointer uses.
4359 SmallSetVector<Instruction *, 8> AddrDefs;
4360 for (BasicBlock *BB : TheLoop->blocks())
4361 for (Instruction &I : *BB) {
4362 Instruction *PtrDef =
4363 dyn_cast_or_null<Instruction>(Val: getLoadStorePointerOperand(V: &I));
4364 if (PtrDef && TheLoop->contains(Inst: PtrDef) &&
4365 getWideningDecision(I: &I, VF) != CM_GatherScatter)
4366 AddrDefs.insert(X: PtrDef);
4367 }
4368
4369 // Add all instructions used to generate the addresses.
4370 SmallVector<Instruction *, 4> Worklist;
4371 append_range(C&: Worklist, R&: AddrDefs);
4372 while (!Worklist.empty()) {
4373 Instruction *I = Worklist.pop_back_val();
4374 for (auto &Op : I->operands())
4375 if (auto *InstOp = dyn_cast<Instruction>(Val&: Op))
4376 if (TheLoop->contains(Inst: InstOp) && !isa<PHINode>(Val: InstOp) &&
4377 AddrDefs.insert(X: InstOp))
4378 Worklist.push_back(Elt: InstOp);
4379 }
4380
4381 auto UpdateMemOpUserCost = [this, VF](LoadInst *LI) {
4382 // If there are direct memory op users of the newly scalarized load,
4383 // their cost may have changed because there's no scalarization
4384 // overhead for the operand. Update it.
4385 for (User *U : LI->users()) {
4386 if (!isa<LoadInst, StoreInst>(Val: U))
4387 continue;
4388 if (getWideningDecision(I: cast<Instruction>(Val: U), VF) != CM_Scalarize)
4389 continue;
4390 auto UI = cast<Instruction>(Val: U);
4391 LLVM_DEBUG(
4392 dbgs() << "LV: Memory widening: updating decision for load user "
4393 << *UI << '\n');
4394 setWideningDecision(
4395 I: UI, VF, W: CM_Scalarize,
4396 Cost: getMemInstScalarizationCost(I: cast<Instruction>(Val: U), VF));
4397 }
4398 };
4399 for (auto *I : AddrDefs) {
4400 if (isa<LoadInst>(Val: I)) {
4401 // Setting the desired widening decision should ideally be handled in
4402 // by cost functions, but since this involves the task of finding out
4403 // if the loaded register is involved in an address computation, it is
4404 // instead changed here when we know this is the case.
4405 InstWidening Decision = getWideningDecision(I, VF);
4406 if (!isPredicatedInst(I) &&
4407 (Decision == CM_Widen || Decision == CM_Widen_Reverse ||
4408 (!isUniformMemOp(I&: *I, VF) && Decision == CM_Scalarize))) {
4409 // Scalarize a widened load of address or update the cost of a scalar
4410 // load of an address.
4411 LLVM_DEBUG(dbgs() << "LV: Memory widening: updating decision for load "
4412 << *I << '\n');
4413 setWideningDecision(
4414 I, VF, W: CM_Scalarize,
4415 Cost: (VF.getKnownMinValue() *
4416 getMemoryInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1))));
4417 UpdateMemOpUserCost(cast<LoadInst>(Val: I));
4418 } else if (const auto *Group = getInterleavedAccessGroup(Instr: I)) {
4419 // Scalarize all members of this interleaved group when any member
4420 // is used as an address. The address-used load skips scalarization
4421 // overhead, other members include it.
4422 for (Instruction *Member : Group->members()) {
4423 InstructionCost Cost = AddrDefs.contains(key: Member)
4424 ? (VF.getKnownMinValue() *
4425 getMemoryInstructionCost(
4426 I: Member, VF: ElementCount::getFixed(MinVal: 1)))
4427 : getMemInstScalarizationCost(I: Member, VF);
4428 LLVM_DEBUG(
4429 dbgs()
4430 << "LV: Memory widening: updating decision for interleave member "
4431 << *Member << '\n');
4432 setWideningDecision(I: Member, VF, W: CM_Scalarize, Cost);
4433 UpdateMemOpUserCost(cast<LoadInst>(Val: Member));
4434 }
4435 }
4436 } else {
4437 // Cannot scalarize fixed-order recurrence phis at the moment.
4438 if (isa<PHINode>(Val: I) && Legal->isFixedOrderRecurrence(Phi: cast<PHINode>(Val: I)))
4439 continue;
4440
4441 // Make sure I gets scalarized and a cost estimate without
4442 // scalarization overhead.
4443 ForcedScalars[VF].insert(X: I);
4444 }
4445 }
4446}
4447
4448bool LoopVectorizationCostModel::shouldConsiderInvariant(Value *Op) {
4449 if (!Legal->isInvariant(V: Op))
4450 return false;
4451 // Consider Op invariant, if it or its operands aren't predicated
4452 // instruction in the loop. In that case, it is not trivially hoistable.
4453 auto *OpI = dyn_cast<Instruction>(Val: Op);
4454 return !OpI || !TheLoop->contains(Inst: OpI) ||
4455 (!isPredicatedInst(I: OpI) &&
4456 (!isa<PHINode>(Val: OpI) || OpI->getParent() != TheLoop->getHeader()) &&
4457 all_of(Range: OpI->operands(),
4458 P: [this](Value *Op) { return shouldConsiderInvariant(Op); }));
4459}
4460
4461InstructionCost
4462LoopVectorizationCostModel::getInstructionCost(Instruction *I,
4463 ElementCount VF) {
4464 // If we know that this instruction will remain uniform, check the cost of
4465 // the scalar version.
4466 if (isUniformAfterVectorization(I, VF))
4467 VF = ElementCount::getFixed(MinVal: 1);
4468
4469 if (VF.isVector() && isProfitableToScalarize(I, VF))
4470 return InstsToScalarize[VF][I];
4471
4472 // Forced scalars do not have any scalarization overhead.
4473 auto ForcedScalar = ForcedScalars.find(Val: VF);
4474 if (VF.isVector() && ForcedScalar != ForcedScalars.end()) {
4475 auto InstSet = ForcedScalar->second;
4476 if (InstSet.count(key: I))
4477 return getInstructionCost(I, VF: ElementCount::getFixed(MinVal: 1)) *
4478 VF.getKnownMinValue();
4479 }
4480
4481 const auto &MinBWs = Config.getMinimalBitwidths();
4482 uint64_t InstrMinBWs = MinBWs.lookup(Key: I);
4483 Type *RetTy = I->getType();
4484 if (canTruncateToMinimalBitwidth(I, VF))
4485 RetTy = IntegerType::get(C&: RetTy->getContext(), NumBits: InstrMinBWs);
4486 auto *SE = PSE.getSE();
4487
4488 Type *VectorTy;
4489 if (isScalarAfterVectorization(I, VF)) {
4490 [[maybe_unused]] auto HasSingleCopyAfterVectorization =
4491 [this](Instruction *I, ElementCount VF) -> bool {
4492 if (VF.isScalar())
4493 return true;
4494
4495 auto Scalarized = InstsToScalarize.find(Val: VF);
4496 assert(Scalarized != InstsToScalarize.end() &&
4497 "VF not yet analyzed for scalarization profitability");
4498 return !Scalarized->second.count(Key: I) &&
4499 llvm::all_of(Range: I->users(), P: [&](User *U) {
4500 auto *UI = cast<Instruction>(Val: U);
4501 return !Scalarized->second.count(Key: UI);
4502 });
4503 };
4504
4505 // With the exception of GEPs and PHIs, after scalarization there should
4506 // only be one copy of the instruction generated in the loop. This is
4507 // because the VF is either 1, or any instructions that need scalarizing
4508 // have already been dealt with by the time we get here. As a result,
4509 // it means we don't have to multiply the instruction cost by VF.
4510 assert(I->getOpcode() == Instruction::GetElementPtr ||
4511 I->getOpcode() == Instruction::PHI ||
4512 (I->getOpcode() == Instruction::BitCast &&
4513 I->getType()->isPointerTy()) ||
4514 HasSingleCopyAfterVectorization(I, VF));
4515 VectorTy = RetTy;
4516 } else
4517 VectorTy = toVectorizedTy(Ty: RetTy, EC: VF);
4518
4519 if (VF.isVector() && VectorTy->isVectorTy() &&
4520 !TTI.getNumberOfParts(Tp: VectorTy))
4521 return InstructionCost::getInvalid();
4522
4523 // TODO: We need to estimate the cost of intrinsic calls.
4524 switch (I->getOpcode()) {
4525 case Instruction::GetElementPtr:
4526 // We mark this instruction as zero-cost because the cost of GEPs in
4527 // vectorized code depends on whether the corresponding memory instruction
4528 // is scalarized or not. Therefore, we handle GEPs with the memory
4529 // instruction cost.
4530 return 0;
4531 case Instruction::UncondBr:
4532 case Instruction::CondBr: {
4533 // In cases of scalarized and predicated instructions, there will be VF
4534 // predicated blocks in the vectorized loop. Each branch around these
4535 // blocks requires also an extract of its vector compare i1 element.
4536 // Note that the conditional branch from the loop latch will be replaced by
4537 // a single branch controlling the loop, so there is no extra overhead from
4538 // scalarization.
4539 bool ScalarPredicatedBB = false;
4540 CondBrInst *BI = dyn_cast<CondBrInst>(Val: I);
4541 if (VF.isVector() && BI &&
4542 (PredicatedBBsAfterVectorization[VF].count(Ptr: BI->getSuccessor(i: 0)) ||
4543 PredicatedBBsAfterVectorization[VF].count(Ptr: BI->getSuccessor(i: 1))) &&
4544 BI->getParent() != TheLoop->getLoopLatch())
4545 ScalarPredicatedBB = true;
4546
4547 if (ScalarPredicatedBB) {
4548 // Not possible to scalarize scalable vector with predicated instructions.
4549 if (VF.isScalable())
4550 return InstructionCost::getInvalid();
4551 // Return cost for branches around scalarized and predicated blocks.
4552 auto *VecI1Ty =
4553 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: RetTy->getContext()), EC: VF);
4554 return (TTI.getScalarizationOverhead(
4555 Ty: VecI1Ty, DemandedElts: APInt::getAllOnes(numBits: VF.getFixedValue()),
4556 /*Insert*/ false, /*Extract*/ true, CostKind: Config.CostKind) +
4557 (TTI.getCFInstrCost(Opcode: Instruction::CondBr, CostKind: Config.CostKind) *
4558 VF.getFixedValue()));
4559 }
4560
4561 if (I->getParent() == TheLoop->getLoopLatch() || VF.isScalar())
4562 // The back-edge branch will remain, as will all scalar branches.
4563 return TTI.getCFInstrCost(Opcode: Instruction::UncondBr, CostKind: Config.CostKind);
4564
4565 // This branch will be eliminated by if-conversion.
4566 return 0;
4567 // Note: We currently assume zero cost for an unconditional branch inside
4568 // a predicated block since it will become a fall-through, although we
4569 // may decide in the future to call TTI for all branches.
4570 }
4571 case Instruction::Switch: {
4572 if (VF.isScalar())
4573 return TTI.getCFInstrCost(Opcode: Instruction::Switch, CostKind: Config.CostKind);
4574 auto *Switch = cast<SwitchInst>(Val: I);
4575 return Switch->getNumCases() *
4576 TTI.getCmpSelInstrCost(
4577 Opcode: Instruction::ICmp,
4578 ValTy: toVectorTy(Scalar: Switch->getCondition()->getType(), EC: VF),
4579 CondTy: toVectorTy(Scalar: Type::getInt1Ty(C&: I->getContext()), EC: VF),
4580 VecPred: CmpInst::ICMP_EQ, CostKind: Config.CostKind);
4581 }
4582 case Instruction::PHI: {
4583 auto *Phi = cast<PHINode>(Val: I);
4584
4585 // First-order recurrences are replaced by vector shuffles inside the loop.
4586 if (VF.isVector() && Legal->isFixedOrderRecurrence(Phi)) {
4587 return TTI.getShuffleCost(
4588 Kind: TargetTransformInfo::SK_Splice, DstTy: cast<VectorType>(Val: VectorTy),
4589 SrcTy: cast<VectorType>(Val: VectorTy), CostKind: Config.CostKind, Mask: {}, Index: -1);
4590 }
4591
4592 // Phi nodes in non-header blocks (not inductions, reductions, etc.) are
4593 // converted into select instructions. We require N - 1 selects per phi
4594 // node, where N is the number of incoming values.
4595 if (VF.isVector() && Phi->getParent() != TheLoop->getHeader()) {
4596 Type *ResultTy = Phi->getType();
4597
4598 // All instructions in an Any-of reduction chain are narrowed to bool.
4599 // Check if that is the case for this phi node.
4600 auto *HeaderUser = find_singleton<PHINode>(
4601 Range: Phi->users(), P: [this](User *U, bool) -> PHINode * {
4602 auto *Phi = dyn_cast<PHINode>(Val: U);
4603 if (Phi && Phi->getParent() == TheLoop->getHeader())
4604 return Phi;
4605 return nullptr;
4606 });
4607 if (HeaderUser) {
4608 auto &ReductionVars = Legal->getReductionVars();
4609 auto Iter = ReductionVars.find(Key: HeaderUser);
4610 if (Iter != ReductionVars.end() &&
4611 RecurrenceDescriptor::isAnyOfRecurrenceKind(
4612 Kind: Iter->second.getRecurrenceKind()))
4613 ResultTy = Type::getInt1Ty(C&: Phi->getContext());
4614 }
4615 return (Phi->getNumIncomingValues() - 1) *
4616 TTI.getCmpSelInstrCost(
4617 Opcode: Instruction::Select, ValTy: toVectorTy(Scalar: ResultTy, EC: VF),
4618 CondTy: toVectorTy(Scalar: Type::getInt1Ty(C&: Phi->getContext()), EC: VF),
4619 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind: Config.CostKind);
4620 }
4621
4622 // When tail folding with EVL, if the phi is part of an out of loop
4623 // reduction then it will be transformed into a wide vp_merge.
4624 if (VF.isVector() && foldTailWithEVL() &&
4625 Legal->getReductionVars().contains(Key: Phi) &&
4626 !Config.isInLoopReduction(Phi)) {
4627 IntrinsicCostAttributes ICA(
4628 Intrinsic::vp_merge, toVectorTy(Scalar: Phi->getType(), EC: VF),
4629 {toVectorTy(Scalar: Type::getInt1Ty(C&: Phi->getContext()), EC: VF)});
4630 return TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind);
4631 }
4632
4633 return TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind: Config.CostKind);
4634 }
4635 case Instruction::UDiv:
4636 case Instruction::SDiv:
4637 case Instruction::URem:
4638 case Instruction::SRem:
4639 if (VF.isVector() && isPredicatedInst(I)) {
4640 const auto [ScalarCost, MaskedCost] = getDivRemSpeculationCost(I, VF);
4641 return isDivRemScalarWithPredication(ScalarCost, MaskedCost) ? ScalarCost
4642 : MaskedCost;
4643 }
4644 // We've proven all lanes safe to speculate, fall through.
4645 [[fallthrough]];
4646 case Instruction::Add:
4647 case Instruction::Sub: {
4648 const HistogramInfo *HGram = Legal->getHistogramInfo(I);
4649 if (HGram && VF.isVector()) {
4650 // Assume that a non-constant update value (or a constant != 1) requires
4651 // a multiply, and add that into the cost.
4652 InstructionCost MulCost = TTI::TCC_Free;
4653 ConstantInt *RHS = dyn_cast<ConstantInt>(Val: I->getOperand(i: 1));
4654 if (!RHS || RHS->getZExtValue() != 1)
4655 MulCost = TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: VectorTy,
4656 CostKind: Config.CostKind);
4657
4658 // Find the cost of the histogram operation itself.
4659 Type *PtrTy = VectorType::get(ElementType: HGram->Load->getPointerOperandType(), EC: VF);
4660 Type *ScalarTy = I->getType();
4661 Type *MaskTy = VectorType::get(ElementType: Type::getInt1Ty(C&: I->getContext()), EC: VF);
4662 IntrinsicCostAttributes ICA(Intrinsic::experimental_vector_histogram_add,
4663 Type::getVoidTy(C&: I->getContext()),
4664 {PtrTy, ScalarTy, MaskTy});
4665
4666 // Add the costs together with the add/sub operation.
4667 return TTI.getIntrinsicInstrCost(ICA, CostKind: Config.CostKind) + MulCost +
4668 TTI.getArithmeticInstrCost(Opcode: I->getOpcode(), Ty: VectorTy,
4669 CostKind: Config.CostKind);
4670 }
4671 [[fallthrough]];
4672 }
4673 case Instruction::FAdd:
4674 case Instruction::FSub:
4675 case Instruction::Mul:
4676 case Instruction::FMul:
4677 case Instruction::FDiv:
4678 case Instruction::FRem:
4679 case Instruction::Shl:
4680 case Instruction::LShr:
4681 case Instruction::AShr:
4682 case Instruction::And:
4683 case Instruction::Or:
4684 case Instruction::Xor: {
4685 // If we're speculating on the stride being 1, the multiplication may
4686 // fold away. We can generalize this for all operations using the notion
4687 // of neutral elements. (TODO)
4688 if (I->getOpcode() == Instruction::Mul &&
4689 ((TheLoop->isLoopInvariant(V: I->getOperand(i: 0)) &&
4690 PSE.getSCEV(V: I->getOperand(i: 0))->isOne()) ||
4691 (TheLoop->isLoopInvariant(V: I->getOperand(i: 1)) &&
4692 PSE.getSCEV(V: I->getOperand(i: 1))->isOne())))
4693 return 0;
4694
4695 // Certain instructions can be cheaper to vectorize if they have a constant
4696 // second vector operand. One example of this are shifts on x86.
4697 Value *Op2 = I->getOperand(i: 1);
4698 if (!isa<Constant>(Val: Op2) && TheLoop->isLoopInvariant(V: Op2) &&
4699 PSE.getSE()->isSCEVable(Ty: Op2->getType()) &&
4700 isa<SCEVConstant>(Val: PSE.getSCEV(V: Op2))) {
4701 Op2 = cast<SCEVConstant>(Val: PSE.getSCEV(V: Op2))->getValue();
4702 }
4703 auto Op2Info = TTI.getOperandInfo(V: Op2);
4704 if (Op2Info.Kind == TargetTransformInfo::OK_AnyValue &&
4705 shouldConsiderInvariant(Op: Op2))
4706 Op2Info.Kind = TargetTransformInfo::OK_UniformValue;
4707
4708 SmallVector<const Value *, 4> Operands(I->operand_values());
4709 return TTI.getArithmeticInstrCost(
4710 Opcode: I->getOpcode(), Ty: VectorTy, CostKind: Config.CostKind,
4711 Opd1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
4712 Opd2Info: Op2Info, Args: Operands, CtxI: I, TLibInfo: TLI);
4713 }
4714 case Instruction::FNeg: {
4715 return TTI.getArithmeticInstrCost(
4716 Opcode: I->getOpcode(), Ty: VectorTy, CostKind: Config.CostKind,
4717 Opd1Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
4718 Opd2Info: {.Kind: TargetTransformInfo::OK_AnyValue, .Properties: TargetTransformInfo::OP_None},
4719 Args: I->getOperand(i: 0), CtxI: I);
4720 }
4721 case Instruction::Select: {
4722 SelectInst *SI = cast<SelectInst>(Val: I);
4723 const SCEV *CondSCEV = SE->getSCEV(V: SI->getCondition());
4724 bool ScalarCond = (SE->isLoopInvariant(S: CondSCEV, L: TheLoop));
4725
4726 const Value *Op0, *Op1;
4727 using namespace llvm::PatternMatch;
4728 if (!ScalarCond && (match(V: I, P: m_LogicalAnd(L: m_Value(V&: Op0), R: m_Value(V&: Op1))) ||
4729 match(V: I, P: m_LogicalOr(L: m_Value(V&: Op0), R: m_Value(V&: Op1))))) {
4730 // select x, y, false --> x & y
4731 // select x, true, y --> x | y
4732 const auto [Op1VK, Op1VP] = TTI::getOperandInfo(V: Op0);
4733 const auto [Op2VK, Op2VP] = TTI::getOperandInfo(V: Op1);
4734 assert(Op0->getType()->getScalarSizeInBits() == 1 &&
4735 Op1->getType()->getScalarSizeInBits() == 1);
4736
4737 return TTI.getArithmeticInstrCost(
4738 Opcode: match(V: I, P: m_LogicalOr()) ? Instruction::Or : Instruction::And,
4739 Ty: VectorTy, CostKind: Config.CostKind, Opd1Info: {.Kind: Op1VK, .Properties: Op1VP}, Opd2Info: {.Kind: Op2VK, .Properties: Op2VP}, Args: {Op0, Op1},
4740 CtxI: I);
4741 }
4742
4743 Type *CondTy = SI->getCondition()->getType();
4744 if (!ScalarCond)
4745 CondTy = VectorType::get(ElementType: CondTy, EC: VF);
4746
4747 CmpInst::Predicate Pred = CmpInst::BAD_ICMP_PREDICATE;
4748 if (auto *Cmp = dyn_cast<CmpInst>(Val: SI->getCondition()))
4749 Pred = Cmp->getPredicate();
4750 return TTI.getCmpSelInstrCost(
4751 Opcode: I->getOpcode(), ValTy: VectorTy, CondTy, VecPred: Pred, CostKind: Config.CostKind,
4752 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
4753 }
4754 case Instruction::ICmp:
4755 case Instruction::FCmp: {
4756 Type *ValTy = I->getOperand(i: 0)->getType();
4757
4758 if (canTruncateToMinimalBitwidth(I, VF)) {
4759 [[maybe_unused]] Instruction *Op0AsInstruction =
4760 dyn_cast<Instruction>(Val: I->getOperand(i: 0));
4761 assert((!canTruncateToMinimalBitwidth(Op0AsInstruction, VF) ||
4762 InstrMinBWs == MinBWs.lookup(Op0AsInstruction)) &&
4763 "if both the operand and the compare are marked for "
4764 "truncation, they must have the same bitwidth");
4765 ValTy = IntegerType::get(C&: ValTy->getContext(), NumBits: InstrMinBWs);
4766 }
4767
4768 VectorTy = toVectorTy(Scalar: ValTy, EC: VF);
4769 return TTI.getCmpSelInstrCost(
4770 Opcode: I->getOpcode(), ValTy: VectorTy, CondTy: CmpInst::makeCmpResultType(opnd_type: VectorTy),
4771 VecPred: cast<CmpInst>(Val: I)->getPredicate(), CostKind: Config.CostKind,
4772 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
4773 }
4774 case Instruction::Store:
4775 case Instruction::Load: {
4776 ElementCount Width = VF;
4777 if (Width.isVector()) {
4778 InstWidening Decision = getWideningDecision(I, VF: Width);
4779 assert(Decision != CM_Unknown &&
4780 "CM decision should be taken at this point");
4781 if (getWideningCost(I, VF) == InstructionCost::getInvalid())
4782 return InstructionCost::getInvalid();
4783 if (Decision == CM_Scalarize)
4784 Width = ElementCount::getFixed(MinVal: 1);
4785 }
4786 VectorTy = toVectorTy(Scalar: getLoadStoreType(I), EC: Width);
4787 return getMemoryInstructionCost(I, VF);
4788 }
4789 case Instruction::BitCast:
4790 if (I->getType()->isPointerTy())
4791 return 0;
4792 [[fallthrough]];
4793 case Instruction::ZExt:
4794 case Instruction::SExt:
4795 case Instruction::FPToUI:
4796 case Instruction::FPToSI:
4797 case Instruction::FPExt:
4798 case Instruction::PtrToInt:
4799 case Instruction::IntToPtr:
4800 case Instruction::SIToFP:
4801 case Instruction::UIToFP:
4802 case Instruction::Trunc:
4803 case Instruction::FPTrunc: {
4804 // Computes the CastContextHint from a Load/Store instruction.
4805 auto ComputeCCH = [&](Instruction *I) -> TTI::CastContextHint {
4806 assert((isa<LoadInst>(I) || isa<StoreInst>(I)) &&
4807 "Expected a load or a store!");
4808
4809 if (VF.isScalar() || !TheLoop->contains(Inst: I))
4810 return TTI::CastContextHint::Normal;
4811
4812 switch (getWideningDecision(I, VF)) {
4813 case LoopVectorizationCostModel::CM_GatherScatter:
4814 return TTI::CastContextHint::GatherScatter;
4815 case LoopVectorizationCostModel::CM_Interleave:
4816 return TTI::CastContextHint::Interleave;
4817 case LoopVectorizationCostModel::CM_Scalarize:
4818 case LoopVectorizationCostModel::CM_Widen:
4819 // TODO: Add 'Compressed' hint (not needed for any targets yet).
4820 if (Legal->isCompressedLoadOrStore(I))
4821 return TTI::CastContextHint::None;
4822 return isPredicatedInst(I) ? TTI::CastContextHint::Masked
4823 : TTI::CastContextHint::Normal;
4824 case LoopVectorizationCostModel::CM_Widen_Reverse:
4825 return TTI::CastContextHint::Reversed;
4826 case LoopVectorizationCostModel::CM_Unknown:
4827 llvm_unreachable("Instr did not go through cost modelling?");
4828 case LoopVectorizationCostModel::CM_InvalidatedDecision:
4829 return TTI::CastContextHint::None;
4830 }
4831
4832 llvm_unreachable("Unhandled case!");
4833 };
4834
4835 unsigned Opcode = I->getOpcode();
4836 TTI::CastContextHint CCH = TTI::CastContextHint::None;
4837 // For Trunc, the context is the only user, which must be a StoreInst.
4838 if (Opcode == Instruction::Trunc || Opcode == Instruction::FPTrunc) {
4839 if (I->hasOneUse())
4840 if (StoreInst *Store = dyn_cast<StoreInst>(Val: *I->user_begin()))
4841 CCH = ComputeCCH(Store);
4842 }
4843 // For Z/Sext, the context is the operand, which must be a LoadInst.
4844 else if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt ||
4845 Opcode == Instruction::FPExt) {
4846 if (LoadInst *Load = dyn_cast<LoadInst>(Val: I->getOperand(i: 0)))
4847 CCH = ComputeCCH(Load);
4848 }
4849
4850 // We optimize the truncation of induction variables having constant
4851 // integer steps. The cost of these truncations is the same as the scalar
4852 // operation.
4853 if (isOptimizableIVTruncate(I, VF)) {
4854 auto *Trunc = cast<TruncInst>(Val: I);
4855 return TTI.getCastInstrCost(Opcode: Instruction::Trunc, Dst: Trunc->getDestTy(),
4856 Src: Trunc->getSrcTy(), CCH, CostKind: Config.CostKind,
4857 I: Trunc);
4858 }
4859
4860 Type *SrcScalarTy = I->getOperand(i: 0)->getType();
4861 Instruction *Op0AsInstruction = dyn_cast<Instruction>(Val: I->getOperand(i: 0));
4862 if (canTruncateToMinimalBitwidth(I: Op0AsInstruction, VF))
4863 SrcScalarTy = IntegerType::get(C&: SrcScalarTy->getContext(),
4864 NumBits: MinBWs.lookup(Key: Op0AsInstruction));
4865 Type *SrcVecTy =
4866 VectorTy->isVectorTy() ? toVectorTy(Scalar: SrcScalarTy, EC: VF) : SrcScalarTy;
4867
4868 if (canTruncateToMinimalBitwidth(I, VF)) {
4869 // If the result type is <= the source type, there will be no extend
4870 // after truncating the users to the minimal required bitwidth.
4871 if (VectorTy->getScalarSizeInBits() <= SrcVecTy->getScalarSizeInBits() &&
4872 (I->getOpcode() == Instruction::ZExt ||
4873 I->getOpcode() == Instruction::SExt))
4874 return 0;
4875 }
4876
4877 return TTI.getCastInstrCost(Opcode, Dst: VectorTy, Src: SrcVecTy, CCH,
4878 CostKind: Config.CostKind, I);
4879 }
4880 case Instruction::Call:
4881 return getVectorCallCost(CI: cast<CallInst>(Val: I), VF);
4882 case Instruction::ExtractValue:
4883 return TTI.getInstructionCost(U: I, CostKind: Config.CostKind);
4884 case Instruction::Alloca:
4885 // We cannot easily widen alloca to a scalable alloca, as
4886 // the result would need to be a vector of pointers.
4887 if (VF.isScalable())
4888 return InstructionCost::getInvalid();
4889 return TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: RetTy, CostKind: Config.CostKind);
4890 case Instruction::Freeze:
4891 return TTI::TCC_Free;
4892 default:
4893 // This opcode is unknown. Assume that it is the same as 'mul'.
4894 return TTI.getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: VectorTy,
4895 CostKind: Config.CostKind);
4896 } // end of switch.
4897}
4898
4899void LoopVectorizationCostModel::collectValuesToIgnore() {
4900 // Ignore ephemeral values.
4901 CodeMetrics::collectEphemeralValues(L: TheLoop, AC, EphValues&: ValuesToIgnore);
4902
4903 SmallVector<Value *, 4> DeadInterleavePointerOps;
4904 SmallVector<Value *, 4> DeadOps;
4905
4906 // If a scalar epilogue is required, users outside the loop won't use
4907 // live-outs from the vector loop but from the scalar epilogue. Ignore them if
4908 // that is the case.
4909 bool RequiresScalarEpilogue = requiresScalarEpilogue(IsVectorizing: true);
4910 auto IsLiveOutDead = [this, RequiresScalarEpilogue](User *U) {
4911 return RequiresScalarEpilogue &&
4912 !TheLoop->contains(BB: cast<Instruction>(Val: U)->getParent());
4913 };
4914
4915 // Returns true if the address of interleaved access I is used by its group.
4916 // VPlan uses the address of member 0 for store groups. For load groups, it
4917 // may use the address of the insert position instead.
4918 auto IsGroupAddress = [this](Instruction *I) {
4919 auto *Group = getInterleavedAccessGroup(Instr: I);
4920 return isa<StoreInst>(Val: I) ? Group->getMember(Index: 0) == I
4921 : Group->getInsertPos() == I;
4922 };
4923
4924 LoopBlocksDFS DFS(TheLoop);
4925 DFS.perform(LI);
4926 for (BasicBlock *BB : reverse(C: make_range(x: DFS.beginRPO(), y: DFS.endRPO())))
4927 for (Instruction &I : reverse(C&: *BB)) {
4928 if (VecValuesToIgnore.contains(Ptr: &I) || ValuesToIgnore.contains(Ptr: &I))
4929 continue;
4930
4931 // Add instructions that would be trivially dead and are only used by
4932 // values already ignored to DeadOps to seed worklist.
4933 if (wouldInstructionBeTriviallyDead(I: &I, TLI) &&
4934 all_of(Range: I.users(), P: [this, IsLiveOutDead](User *U) {
4935 return VecValuesToIgnore.contains(Ptr: U) ||
4936 ValuesToIgnore.contains(Ptr: U) || IsLiveOutDead(U);
4937 }))
4938 DeadOps.push_back(Elt: &I);
4939
4940 // For interleave groups, we only create a pointer for the start of the
4941 // interleave group. Queue up addresses of group members not used by the
4942 // group for further processing.
4943 if (isAccessInterleaved(Instr: &I)) {
4944 if (IsGroupAddress(&I))
4945 continue;
4946 Value *PointerOp = getLoadStorePointerOperand(V: &I);
4947 DeadInterleavePointerOps.push_back(Elt: PointerOp);
4948 }
4949
4950 // Queue branches for analysis. They are dead, if their successors only
4951 // contain dead instructions.
4952 if (isa<CondBrInst>(Val: &I))
4953 DeadOps.push_back(Elt: &I);
4954 }
4955
4956 // Mark ops feeding interleave group members as free, if they are only used
4957 // by other dead computations.
4958 for (unsigned I = 0; I != DeadInterleavePointerOps.size(); ++I) {
4959 auto *Op = dyn_cast<Instruction>(Val: DeadInterleavePointerOps[I]);
4960 if (!Op || !TheLoop->contains(Inst: Op) ||
4961 any_of(Range: Op->users(), P: [this, &IsGroupAddress](User *U) {
4962 Instruction *UI = cast<Instruction>(Val: U);
4963 return !VecValuesToIgnore.contains(Ptr: U) &&
4964 (!isAccessInterleaved(Instr: UI) || IsGroupAddress(UI));
4965 }))
4966 continue;
4967 VecValuesToIgnore.insert(Ptr: Op);
4968 append_range(C&: DeadInterleavePointerOps, R: Op->operands());
4969 }
4970
4971 // Mark ops that would be trivially dead and are only used by ignored
4972 // instructions as free.
4973 BasicBlock *Header = TheLoop->getHeader();
4974
4975 // Returns true if the block contains only dead instructions. Such blocks will
4976 // be removed by VPlan-to-VPlan transforms and won't be considered by the
4977 // VPlan-based cost model, so skip them in the legacy cost-model as well.
4978 auto IsEmptyBlock = [this](BasicBlock *BB) {
4979 return all_of(Range&: *BB, P: [this](Instruction &I) {
4980 return ValuesToIgnore.contains(Ptr: &I) || VecValuesToIgnore.contains(Ptr: &I) ||
4981 isa<UncondBrInst>(Val: &I);
4982 });
4983 };
4984 SmallPtrSet<Instruction *, 16> ProcessedDeadOps;
4985 for (unsigned I = 0; I != DeadOps.size(); ++I) {
4986 auto *Op = dyn_cast<Instruction>(Val: DeadOps[I]);
4987
4988 // Check if the branch should be considered dead.
4989 if (auto *Br = dyn_cast_or_null<CondBrInst>(Val: Op)) {
4990 BasicBlock *ThenBB = Br->getSuccessor(i: 0);
4991 BasicBlock *ElseBB = Br->getSuccessor(i: 1);
4992 // Don't considers branches leaving the loop for simplification.
4993 if (!TheLoop->contains(BB: ThenBB) || !TheLoop->contains(BB: ElseBB))
4994 continue;
4995 bool ThenEmpty = IsEmptyBlock(ThenBB);
4996 bool ElseEmpty = IsEmptyBlock(ElseBB);
4997 if ((ThenEmpty && ElseEmpty) ||
4998 (ThenEmpty && ThenBB->getSingleSuccessor() == ElseBB &&
4999 ElseBB->phis().empty()) ||
5000 (ElseEmpty && ElseBB->getSingleSuccessor() == ThenBB &&
5001 ThenBB->phis().empty())) {
5002 VecValuesToIgnore.insert(Ptr: Br);
5003 DeadOps.push_back(Elt: Br->getCondition());
5004 }
5005 continue;
5006 }
5007
5008 // Skip any op that shouldn't be considered dead.
5009 if (!Op || !TheLoop->contains(Inst: Op) ||
5010 (isa<PHINode>(Val: Op) && Op->getParent() == Header) ||
5011 !wouldInstructionBeTriviallyDead(I: Op, TLI) ||
5012 any_of(Range: Op->users(), P: [this, IsLiveOutDead](User *U) {
5013 return !VecValuesToIgnore.contains(Ptr: U) &&
5014 !ValuesToIgnore.contains(Ptr: U) && !IsLiveOutDead(U);
5015 }))
5016 continue;
5017
5018 // If all of Op's users are in ValuesToIgnore, add it to ValuesToIgnore
5019 // which applies for both scalar and vector versions. Otherwise it is only
5020 // dead in vector versions, so only add it to VecValuesToIgnore.
5021 bool BecameScalarDead = false;
5022 if (all_of(Range: Op->users(),
5023 P: [this](User *U) { return ValuesToIgnore.contains(Ptr: U); }))
5024 BecameScalarDead = ValuesToIgnore.insert(Ptr: Op).second;
5025
5026 VecValuesToIgnore.insert(Ptr: Op);
5027 // Shared operands may be queued more than once. Propagate deadness once,
5028 // and again if an instruction previously dead only in the vector loop
5029 // becomes dead in the scalar loop too.
5030 if (ProcessedDeadOps.insert(Ptr: Op).second || BecameScalarDead)
5031 append_range(C&: DeadOps, R: Op->operands());
5032 }
5033
5034 // Ignore type-promoting instructions we identified during reduction
5035 // detection.
5036 for (const RecurrenceDescriptor &RedDes :
5037 Legal->getReductionVars().values()) {
5038 VecValuesToIgnore.insert_range(R: RedDes.getCastInsts());
5039 }
5040 // Ignore type-casting instructions we identified during induction
5041 // detection.
5042 for (const InductionDescriptor &IndDes : Legal->getInductionVars().values())
5043 VecValuesToIgnore.insert_range(R: IndDes.getCastInsts());
5044}
5045
5046void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
5047 CM->collectValuesToIgnore();
5048 Config.collectElementTypesForWidening(ValuesToIgnore: &CM->ValuesToIgnore);
5049
5050 FixedScalableVFPair MaxFactors = CM->computeMaxVF(UserVF, UserIC);
5051 if (!MaxFactors) // Cases that should not to be vectorized nor interleaved.
5052 return;
5053
5054 Config.collectInLoopReductions();
5055 // Cases that may be vectorized may be optimized by unit stride predicates.
5056 // TODO: Currently unit stride predicates are added unconditionally, even if
5057 // they are not used for the selected VF (e.g. when only interleaving).
5058 if (MaxFactors.FixedVF.isVector() || MaxFactors.ScalableVF.isVector())
5059 Legal->collectUnitStridePredicates();
5060
5061 auto VPlan1 = tryToBuildVPlan1(PlanCM&: *CM);
5062 if (!VPlan1)
5063 return;
5064
5065 LLVM_DEBUG(dbgs() << "LV: VPlan created successfully. Loop can be "
5066 "vectorized.\n");
5067
5068 if (!OrigLoop->isInnermost()) {
5069 // For outer loops, computeMaxVF returns a single non-scalar VF; build a
5070 // plan for that VF only.
5071 ElementCount VF =
5072 MaxFactors.FixedVF ? MaxFactors.FixedVF : MaxFactors.ScalableVF;
5073 buildVPlans(VPlan1&: *VPlan1, MinVF: VF, MaxVF: VF, PlanCM&: *CM);
5074 LLVM_DEBUG(printPlans(dbgs()));
5075 return;
5076 }
5077
5078 // Compute the minimal bitwidths required for integer operations in the loop
5079 // for later use by the cost model.
5080 Config.computeMinimalBitwidths();
5081
5082 // Invalidate interleave groups if all blocks of loop will be predicated.
5083 if (CM->blockNeedsPredicationForAnyReason(BB: OrigLoop->getHeader()) &&
5084 !useMaskedInterleavedAccesses(Opts, TTI)) {
5085 LLVM_DEBUG(dbgs() << "LV: Invalidate all interleaved groups due to "
5086 "predication which requires masked-interleaved "
5087 "support.\n");
5088 if (CM->InterleaveInfo.invalidateGroups())
5089 // Invalidating interleave groups also requires invalidating all decisions
5090 // based on them, which includes widening decisions and uniform and scalar
5091 // values.
5092 CM->invalidateCostModelingDecisions();
5093 }
5094
5095 if (CM->foldTailByMasking())
5096 Legal->prepareToFoldTailByMasking();
5097
5098 ElementCount MaxUserVF =
5099 UserVF.isScalable() ? MaxFactors.ScalableVF : MaxFactors.FixedVF;
5100 if (UserVF) {
5101 if (!ElementCount::isKnownLE(LHS: UserVF, RHS: MaxUserVF)) {
5102 reportVectorizationInfo(
5103 Msg: "UserVF ignored because it may be larger than the maximal safe VF",
5104 ORETag: "InvalidUserVF", ORE, TheLoop: OrigLoop);
5105 } else {
5106 assert(UserVF.isPowerOf2() && "VF needs to be a power of two");
5107 // Collect the instructions (and their associated costs) that will be more
5108 // profitable to scalarize.
5109 CM->collectNonVectorizedAndSetWideningDecisions(VF: UserVF);
5110 buildVPlans(VPlan1&: *VPlan1, MinVF: UserVF, MaxVF: UserVF, PlanCM&: *CM);
5111
5112 // For scalar VF, skip VPlan cost check as VPlan cost is designed for
5113 // vector VFs only.
5114 if (!VPlans.empty() && (VPlans.front()->getSingleVF() == UserVF) &&
5115 (UserVF.isScalar() ||
5116 cost(Plan&: *VPlans.front(), VF: UserVF, /*RU=*/nullptr, PlanCM&: *CM).isValid())) {
5117 // Plan for epilogue only if we succeeded in building the Vplan for the
5118 // main loop.
5119 // Try to build a tail-folded epilogue Vplan. Otherwise build a regular
5120 // plan for the epilogue:
5121 if (!planForEpilogueTF()) {
5122 ElementCount EpilogueUserVF = EpilogueVectorizationForceVF;
5123 if (EpilogueUserVF.isVector() &&
5124 ElementCount::isKnownLT(LHS: EpilogueUserVF, RHS: UserVF)) {
5125 CM->collectNonVectorizedAndSetWideningDecisions(VF: EpilogueUserVF);
5126 buildVPlans(VPlan1&: *VPlan1, MinVF: EpilogueUserVF, MaxVF: EpilogueUserVF, PlanCM&: *CM);
5127 }
5128 }
5129 LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
5130 LLVM_DEBUG(printPlans(dbgs()));
5131 return;
5132 }
5133 VPlans.clear();
5134 reportVectorizationInfo(Msg: "UserVF ignored because of invalid costs.",
5135 ORETag: "InvalidCost", ORE, TheLoop: OrigLoop);
5136 }
5137 }
5138
5139 // Collect the Vectorization Factor Candidates.
5140 SmallVector<ElementCount> VFCandidates;
5141 for (auto VF = ElementCount::getFixed(MinVal: 1);
5142 ElementCount::isKnownLE(LHS: VF, RHS: MaxFactors.FixedVF); VF *= 2)
5143 VFCandidates.push_back(Elt: VF);
5144 for (auto VF = ElementCount::getScalable(MinVal: 1);
5145 ElementCount::isKnownLE(LHS: VF, RHS: MaxFactors.ScalableVF); VF *= 2)
5146 VFCandidates.push_back(Elt: VF);
5147
5148 for (const auto &VF : VFCandidates) {
5149 // Collect Uniform and Scalar instructions after vectorization with VF.
5150 CM->collectNonVectorizedAndSetWideningDecisions(VF);
5151 }
5152
5153 buildVPlans(VPlan1&: *VPlan1, MinVF: ElementCount::getFixed(MinVal: 1), MaxVF: MaxFactors.FixedVF, PlanCM&: *CM);
5154 buildVPlans(VPlan1&: *VPlan1, MinVF: ElementCount::getScalable(MinVal: 1), MaxVF: MaxFactors.ScalableVF,
5155 PlanCM&: *CM);
5156
5157 LLVM_DEBUG(printPlans(dbgs()));
5158}
5159
5160bool LoopVectorizationPlanner::planForEpilogueTF() {
5161 EpilogueLowering EpilogueTailLoweringStatus = getEpilogueTailLowering(
5162 Opts, MainCM: *CM, L: OrigLoop, ORE, LVL&: *Legal, Hints: Config.getHints(), TTI: &TTI);
5163 if (EpilogueTailLoweringStatus !=
5164 EpilogueLowering::CM_EpilogueNotNeededFoldTail)
5165 return false;
5166 LLVM_DEBUG(dbgs() << "LV: epilogue tail-folding is enabled\n");
5167
5168 LoopVectorizationCostModel EpilogueTfCM(
5169 EpilogueTailLoweringStatus, OrigLoop, PSE, LI, Legal, TTI, TLI, CM->AC,
5170 ORE, CM->GetBFI, CM->TheFunction, CM->InterleaveInfo, Config);
5171
5172 assert(!EpilogueTfCM.isEpilogueAllowed() &&
5173 "Epilogue tail-folding is expected to be enabled");
5174
5175 LLVM_DEBUG(dbgs() << "LV: plan for tail-folded epilogue\n");
5176
5177 EpilogueTfCM.ValuesToIgnore.insert_range(R&: CM->ValuesToIgnore);
5178 EpilogueTfCM.VecValuesToIgnore.insert_range(R&: CM->VecValuesToIgnore);
5179
5180 FixedScalableVFPair MaxFactors =
5181 EpilogueTfCM.computeMaxVF(UserVF: EpilogueVectorizationForceVF, /*UserIC*/ 1);
5182 // TODO: Validate the epilogue VF against MaxFactors.
5183 // That validation is ignored for now because computeMaxVF returns 0 for
5184 // max safe scalable VF when UserVF is not scalable, and at that case having
5185 // scalable epilogue VF is considered unsafe which is inaccurate.
5186 if (!MaxFactors || !EpilogueTfCM.foldTailByMasking()) {
5187 // Cases that should not be vectorized or tail-folded.
5188 reportVectorizationInfo(Msg: "This case of epilogue loop can't be tail-folded",
5189 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: OrigLoop);
5190 return false;
5191 }
5192
5193 if (EpilogueTfCM.getTailFoldingStyle() !=
5194 TailFoldingStyle::DataAndControlFlow) {
5195 reportVectorizationInfo(Msg: "Right now tail-folded epilogue is supported only "
5196 "for DataAndControlFlow tail-folding style.",
5197 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: OrigLoop);
5198 return false;
5199 }
5200
5201 auto VPlan1 = tryToBuildVPlan1(PlanCM&: EpilogueTfCM);
5202
5203 if (!VPlan1) {
5204 reportVectorizationInfo(
5205 Msg: "Failed to build initial tail-folded epilogue VPlan",
5206 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: OrigLoop);
5207 return false;
5208 }
5209
5210 Legal->prepareToFoldTailByMasking();
5211
5212 // Collect the instructions (and their associated costs) that will be more
5213 // profitable to scalarize.
5214 EpilogueTfCM.collectNonVectorizedAndSetWideningDecisions(
5215 VF: EpilogueVectorizationForceVF);
5216
5217 [[maybe_unused]] size_t NumPlansBefore = VPlans.size();
5218 buildVPlans(VPlan1&: *VPlan1, MinVF: EpilogueVectorizationForceVF,
5219 MaxVF: EpilogueVectorizationForceVF, PlanCM&: EpilogueTfCM);
5220
5221 // Check that a Vplan is successfully built:
5222 assert(VPlans.size() == NumPlansBefore + 1 &&
5223 "Expected to build a single tail-folded VPlan");
5224 VPlan &Plan = *VPlans.back();
5225 assert(Plan.getSingleVF() == EpilogueVectorizationForceVF &&
5226 Plan.hasTailFolded() &&
5227 "Expected to build a valid tail-folded VPlan with epilogue VF");
5228
5229 // Validate that VPlan has a mask to make sure that the epilogue loop
5230 // terminates correctly:
5231 if (none_of(Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis(),
5232 P: IsaPred<VPActiveLaneMaskPHIRecipe>)) {
5233 reportVectorizationInfo(
5234 Msg: "Tail-folded epilogue has no active-lane-mask exit, so it may not "
5235 "terminate, can't apply tail-folded epilogue",
5236 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: OrigLoop);
5237 VPlans.pop_back();
5238 return false;
5239 }
5240
5241 if (!cost(Plan, VF: EpilogueVectorizationForceVF, /*RU=*/nullptr, PlanCM&: EpilogueTfCM)
5242 .isValid()) {
5243 VPlans.pop_back();
5244 reportVectorizationInfo(Msg: "This case of epilogue loop can't be tail-folded "
5245 "- Invalid costs",
5246 ORETag: "InvalidTailFoldedEpilogue", ORE, TheLoop: OrigLoop);
5247 return false;
5248 }
5249 LLVM_DEBUG(dbgs() << "LV: Tail-folded epilogue VPlan is created\n");
5250 return true;
5251}
5252
5253VPCostContext::VPCostContext(const TargetLibraryInfo &TLI, const VPlan &Plan,
5254 LoopVectorizationCostModel &CM,
5255 VFSelectionContext &Config,
5256 bool ReusePrintingSlotTracker)
5257 : Opts(Config.getOpts()), TTI(Config.getTTI()), TLI(TLI),
5258 LLVMCtx(Plan.getContext()), CM(CM), Config(Config),
5259 CostKind(Config.CostKind), PSE(Config.getPSE()), L(Config.getLoop()) {
5260#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
5261 if (ReusePrintingSlotTracker)
5262 PlanForSlotTracker = &Plan;
5263#endif
5264}
5265
5266InstructionCost VPCostContext::getLegacyCost(Instruction *UI,
5267 ElementCount VF) const {
5268 InstructionCost Cost = CM.getInstructionCost(I: UI, VF);
5269 if (Cost.isValid() && Opts.force_target_instruction_cost)
5270 return InstructionCost(*Opts.force_target_instruction_cost);
5271 return Cost;
5272}
5273
5274bool VPCostContext::skipCostComputation(Instruction *UI, bool IsVector) const {
5275 return CM.ValuesToIgnore.contains(Ptr: UI) ||
5276 (IsVector && CM.VecValuesToIgnore.contains(Ptr: UI)) ||
5277 SkipCostComputation.contains(Ptr: UI);
5278}
5279
5280void VPCostContext::invalidateWideningDecision(Instruction *I,
5281 ElementCount VF) {
5282 CM.setWideningDecision(I, VF,
5283 W: LoopVectorizationCostModel::CM_InvalidatedDecision, Cost: 0);
5284}
5285
5286bool VPCostContext::willBeScalarized(Instruction *I, ElementCount VF) const {
5287 return CM.isScalarWithPredication(I, VF) ||
5288 CM.isUniformAfterVectorization(I, VF) || CM.isForcedScalar(I, VF) ||
5289 (VF.isVector() && CM.isProfitableToScalarize(I, VF));
5290}
5291
5292bool VPCostContext::isMaskRequired(Instruction *I) const {
5293 return CM.isMaskRequired(I);
5294}
5295
5296bool VPCostContext::executesAtMostOnce(const VPlan &Plan, ElementCount VF) {
5297 auto *TC = dyn_cast_if_present<ConstantInt>(
5298 Val: Plan.getTripCount()->getUnderlyingValue());
5299 return TC && TC->getValue().ule(RHS: VF.getKnownMinValue());
5300}
5301
5302InstructionCost
5303LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
5304 VPCostContext &CostCtx) const {
5305 InstructionCost Cost;
5306
5307 // If the vector loop gets executed exactly once with the given VF, ignore the
5308 // costs of comparison and induction instructions, as they'll get simplified
5309 // away.
5310 // TODO: Remove this code after stepping away from the legacy cost model and
5311 // adding code to simplify VPlans before calculating their costs.
5312 auto TC = getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop);
5313 if (TC == VF && !Plan.hasTailFolded())
5314 addFullyUnrolledInstructionsToIgnore(L: OrigLoop, IL: Legal->getInductionVars(),
5315 InstsToIgnore&: CostCtx.SkipCostComputation);
5316
5317 // Don't apply special costs when instruction cost is forced to make sure the
5318 // forced cost is used for each recipe.
5319 if (Opts.force_target_instruction_cost)
5320 return Cost;
5321
5322 // Pre-compute costs for instructions that are forced-scalar or profitable to
5323 // scalarize. For most such instructions, their scalarization costs are
5324 // accounted for here using the legacy cost model. However, some opcodes
5325 // are excluded from these precomputed scalarization costs and are instead
5326 // modeled later by the VPlan cost model (see UseVPlanCostModel below).
5327 for (Instruction *ForcedScalar : CostCtx.CM.ForcedScalars[VF]) {
5328 if (CostCtx.skipCostComputation(UI: ForcedScalar, IsVector: VF.isVector()))
5329 continue;
5330 CostCtx.SkipCostComputation.insert(Ptr: ForcedScalar);
5331 InstructionCost ForcedCost = CostCtx.getLegacyCost(UI: ForcedScalar, VF);
5332 LLVM_DEBUG({
5333 dbgs() << "Cost of " << ForcedCost << " for VF " << VF
5334 << ": forced scalar " << *ForcedScalar << "\n";
5335 });
5336 Cost += ForcedCost;
5337 }
5338
5339 // Don't apply legacy scalarization costs if nothing remains scalar &
5340 // predicated.
5341 if (!hasReplicatorRegion(Plan))
5342 return Cost;
5343
5344 auto UseVPlanCostModel = [](Instruction *I) -> bool {
5345 switch (I->getOpcode()) {
5346 case Instruction::SDiv:
5347 case Instruction::UDiv:
5348 case Instruction::SRem:
5349 case Instruction::URem:
5350 return true;
5351 default:
5352 return false;
5353 }
5354 };
5355 // Only charge legacy scalarization costs for instructions VPlan replicates.
5356 SmallPtrSet<Instruction *, 8> Replicated;
5357 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
5358 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry())))
5359 for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(Range&: *VPBB))
5360 Replicated.insert(Ptr: RepR.getUnderlyingInstr());
5361 for (const auto &[Scalarized, ScalarCost] : CostCtx.CM.InstsToScalarize[VF]) {
5362 if (!Replicated.contains(Ptr: Scalarized) || UseVPlanCostModel(Scalarized) ||
5363 CostCtx.skipCostComputation(UI: Scalarized, IsVector: VF.isVector()))
5364 continue;
5365 CostCtx.SkipCostComputation.insert(Ptr: Scalarized);
5366 LLVM_DEBUG({
5367 dbgs() << "Cost of " << ScalarCost << " for VF " << VF
5368 << ": profitable to scalarize " << *Scalarized << "\n";
5369 });
5370 Cost += ScalarCost;
5371 }
5372
5373 return Cost;
5374}
5375
5376#ifndef NDEBUG
5377/// Returns the frequency with which \p VPBB executes, as recorded on its
5378/// recipes. All recipes of a block share the same frequency.
5379static std::optional<VPExecutionFrequency>
5380getRecordedExecutionFrequency(const VPBasicBlock *VPBB) {
5381 if (VPBB->empty())
5382 return std::nullopt;
5383 return cast<VPInstruction>(&VPBB->front())->getExecutionFrequency();
5384}
5385#endif
5386
5387InstructionCost
5388LoopVectorizationPlanner::cost(VPlan &Plan, ElementCount VF,
5389 VPRegisterUsage *RU,
5390 LoopVectorizationCostModel &PlanCM) const {
5391 VPCostContext CostCtx(*TLI, Plan, PlanCM, Config,
5392 /*ReusePrintingSlotTracker=*/true);
5393 InstructionCost Cost = precomputeCosts(Plan, VF, CostCtx);
5394 LLVM_DEBUG(dbgs() << "Precomputed costs for VF " << VF << ": " << Cost
5395 << '\n');
5396
5397 // Now compute and add the VPlan-based cost.
5398 Cost += Plan.cost(VF, Ctx&: CostCtx);
5399
5400 // Add the cost of spills due to excess register usage
5401 if (RU && Config.shouldConsiderRegPressureForVF(VF)) {
5402 InstructionCost SpillCost = RU->spillCost(
5403 TTI, CostKind: Config.CostKind, OverrideMaxNumRegs: Opts.force_target_num_vector_regs.value_or(u: 0));
5404 LLVM_DEBUG(dbgs() << "Spill costs for VF " << VF << ": " << SpillCost
5405 << '\n');
5406 Cost += SpillCost;
5407 }
5408
5409#ifndef NDEBUG
5410 unsigned EstimatedWidth =
5411 estimateElementCount(VF, Config.getVScaleForTuning());
5412 LLVM_DEBUG(dbgs() << "Cost for VF " << VF << ": " << Cost
5413 << " (Estimated cost per lane: ");
5414 if (Cost.isValid()) {
5415 APFloat CostPerLane(APFloat::IEEEdouble());
5416 APFloat EstimatedWidthAsAPFloat(APFloat::IEEEdouble());
5417 (void)CostPerLane.convertFromAPInt(APInt(64, (uint64_t)Cost.getValue()),
5418 false, APFloat::rmTowardZero);
5419 (void)EstimatedWidthAsAPFloat.convertFromAPInt(
5420 APInt(64, (uint64_t)EstimatedWidth), false, APFloat::rmTowardZero);
5421 (void)CostPerLane.divide(EstimatedWidthAsAPFloat, APFloat::rmTowardZero);
5422
5423 SmallString<16> Str;
5424 CostPerLane.toString(Str, 3);
5425 LLVM_DEBUG(dbgs() << Str);
5426 } else /* No point dividing an invalid cost - it will still be invalid */
5427 LLVM_DEBUG(dbgs() << "Invalid");
5428 LLVM_DEBUG(dbgs() << ")\n");
5429#endif
5430 return Cost;
5431}
5432
5433std::pair<VectorizationFactor, VPlan *>
5434LoopVectorizationPlanner::computeBestVF() {
5435 if (VPlans.empty())
5436 return {VectorizationFactor::Disabled(), nullptr};
5437 // If there is a single VPlan with a single VF, return it directly.
5438 VPlan &FirstPlan = *VPlans[0];
5439
5440 ElementCount UserVF = Config.getHints().getWidth();
5441 if (VPlans.size() == 1) {
5442 // For outer loops, the plan has a single vector VF determined by the
5443 // heuristic.
5444 assert((FirstPlan.hasScalarVFOnly() || hasPlanWithVF(UserVF) ||
5445 FirstPlan.isOuterLoop()) &&
5446 "must have a single scalar VF, UserVF or an outer loop");
5447 return {VectorizationFactor(FirstPlan.getSingleVF(), 0, 0), &FirstPlan};
5448 }
5449
5450 if (hasPlanWithVF(VF: UserVF) && hasForcedEpilogueVF() && VPlans.size() == 2) {
5451 assert(VPlans[0]->getSingleVF() == UserVF &&
5452 "expected second plan to be for the forced UserVF");
5453 assert(VPlans[1]->getSingleVF() == EpilogueVectorizationForceVF &&
5454 "expected first plan to be for the forced epilogue VF");
5455 return {VectorizationFactor(UserVF, 0, 0), VPlans[0].get()};
5456 }
5457
5458 LLVM_DEBUG(dbgs() << "LV: Computing best VF using cost kind: "
5459 << (Config.CostKind == TTI::TCK_RecipThroughput
5460 ? "Reciprocal Throughput\n"
5461 : Config.CostKind == TTI::TCK_Latency
5462 ? "Instruction Latency\n"
5463 : Config.CostKind == TTI::TCK_CodeSize ? "Code Size\n"
5464 : Config.CostKind == TTI::TCK_SizeAndLatency
5465 ? "Code Size and Latency\n"
5466 : "Unknown\n"));
5467
5468 ElementCount ScalarVF = ElementCount::getFixed(MinVal: 1);
5469 assert(FirstPlan.hasVF(ScalarVF) &&
5470 "More than a single plan/VF w/o any plan having scalar VF");
5471
5472 // TODO: Compute scalar cost using VPlan-based cost model.
5473 InstructionCost ScalarCost = CM->expectedCost(VF: ScalarVF);
5474 LLVM_DEBUG(dbgs() << "LV: Scalar loop costs: " << ScalarCost << ".\n");
5475 VectorizationFactor ScalarFactor(ScalarVF, ScalarCost, ScalarCost);
5476 VectorizationFactor BestFactor = ScalarFactor;
5477
5478 bool ForceVectorization =
5479 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
5480 if (ForceVectorization) {
5481 // Ignore scalar width, because the user explicitly wants vectorization.
5482 // Initialize cost to max so that VF = 2 is, at least, chosen during cost
5483 // evaluation.
5484 BestFactor.Cost = InstructionCost::getMax();
5485 }
5486
5487 VPlan *PlanForBestVF = &FirstPlan;
5488 ElementCount ExactTC = getSmallConstantTripCount(SE: PSE.getSE(), L: OrigLoop);
5489
5490 for (auto &P : VPlans) {
5491 ArrayRef<ElementCount> VFs(P->vectorFactors().begin(),
5492 P->vectorFactors().end());
5493
5494 // For loops where the Trip Count is below the TailFoldingThreshold, only
5495 // consider the largest VF to result in at most one vector iteration, and at
5496 // most one scalar iteration.
5497 // FIXME: Encode this decision directly in LVPlanner.
5498 if (!ForceVectorization && P->hasScalarTail() && ExactTC.isFixed() &&
5499 ExactTC.getFixedValue() > 0 &&
5500 ExactTC.getFixedValue() <= TTI.getMinTripCountTailFoldingThreshold()) {
5501 VFs = VFs.take_back(N: 1);
5502 }
5503
5504 SmallVector<VPRegisterUsage, 8> RUs;
5505 bool ConsiderRegPressure = any_of(Range&: VFs, P: [this](ElementCount VF) {
5506 return Config.shouldConsiderRegPressureForVF(VF);
5507 });
5508 if (ConsiderRegPressure)
5509 RUs = calculateRegisterUsageForPlan(Plan&: *P, VFs, TTI);
5510
5511 for (unsigned I = 0; I < VFs.size(); I++) {
5512 ElementCount VF = VFs[I];
5513 if (VF.isScalar())
5514 continue;
5515 if (!ForceVectorization && !willGenerateVectors(Plan&: *P, VF, TTI)) {
5516 LLVM_DEBUG(
5517 dbgs()
5518 << "LV: Not considering vector loop of width " << VF
5519 << " because it will not generate any vector instructions.\n");
5520 continue;
5521 }
5522 if (Config.OptForSize && !ForceVectorization && hasReplicatorRegion(Plan&: *P)) {
5523 LLVM_DEBUG(
5524 dbgs()
5525 << "LV: Not considering vector loop of width " << VF
5526 << " because it would cause replicated blocks to be generated,"
5527 << " which isn't allowed when optimizing for size.\n");
5528 continue;
5529 }
5530
5531 InstructionCost Cost =
5532 cost(Plan&: *P, VF, RU: ConsiderRegPressure ? &RUs[I] : nullptr, PlanCM&: *CM);
5533 VectorizationFactor CurrentFactor(VF, Cost, ScalarCost);
5534
5535 if (isMoreProfitable(A: CurrentFactor, B: BestFactor, HasTail: P->hasScalarTail())) {
5536 BestFactor = CurrentFactor;
5537 PlanForBestVF = P.get();
5538 }
5539
5540 // If profitable add it to ProfitableVF list.
5541 if (isMoreProfitable(A: CurrentFactor, B: ScalarFactor, HasTail: P->hasScalarTail()))
5542 ProfitableVFs.push_back(Elt: CurrentFactor);
5543 }
5544 }
5545
5546 VPlan &BestPlan = *PlanForBestVF;
5547
5548 assert((BestFactor.Width.isScalar() || BestFactor.ScalarCost > 0) &&
5549 "when vectorizing, the scalar cost must be computed.");
5550
5551 LLVM_DEBUG(dbgs() << "LV: Selecting VF: " << BestFactor.Width << ".\n");
5552 return {BestFactor, &BestPlan};
5553}
5554
5555LoopVectorizationPlanner::LoopVectorizationPlanner(
5556 Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
5557 const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
5558 std::unique_ptr<LoopVectorizationCostModel> CM, VFSelectionContext &Config,
5559 PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE,
5560 std::function<const BranchProbabilityInfo &()> GetBPI)
5561 : Opts(Config.getOpts()), OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI),
5562 Legal(Legal), CM(std::move(CM)), Config(Config), PSE(PSE), ORE(ORE),
5563 GetBPI(GetBPI) {}
5564
5565LoopVectorizationPlanner::~LoopVectorizationPlanner() = default;
5566
5567void LoopVectorizationPlanner::clearCostModel() { CM.reset(); }
5568
5569DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
5570 ElementCount BestVF, unsigned BestUF, VPlan &BestVPlan,
5571 bool HasRuntimeChecks, EpilogueVectorizationKind EpilogueVecKind,
5572 MutableArrayRef<VPBasicBlock *> TrackedBlocks) {
5573 assert(BestVPlan.hasVF(BestVF) &&
5574 "Trying to execute plan with unsupported VF");
5575 assert(BestVPlan.hasUF(BestUF) &&
5576 "Trying to execute plan with unsupported UF");
5577 if (BestVPlan.hasEarlyExit())
5578 ++LoopsEarlyExitVectorized;
5579
5580 RUN_VPLAN_PASS(VPlanTransforms::replaceWideCanonicalIVWithWideIV, BestVPlan,
5581 *PSE.getSE(), TTI, Config.CostKind, BestVF, BestUF);
5582 if (TTI.hasMultiVectorLoadStore(
5583 NumVectors: BestUF, Mask: BestVPlan.hasTailFolded()
5584 ? TargetTransformInfo::MaskSource::ActiveLaneMask
5585 : TargetTransformInfo::MaskSource::None))
5586 RUN_VPLAN_PASS(VPlanTransforms::widenMemoryAccessesByUF, BestVPlan, BestVF,
5587 BestUF, TTI);
5588 // TODO: Move to VPlan transform stage once the transition to the VPlan-based
5589 // cost model is complete for better cost estimates.
5590 RUN_VPLAN_PASS(VPlanTransforms::unrollByUF, BestVPlan, BestUF);
5591 RUN_VPLAN_PASS(VPlanTransforms::materializeUnpacks, BestVPlan);
5592 RUN_VPLAN_PASS(VPlanTransforms::materializeBroadcasts, BestVPlan);
5593 RUN_VPLAN_PASS(VPlanTransforms::replicateByVF, BestVPlan, BestVF);
5594 bool HasBranchWeights =
5595 hasBranchWeightMD(I: *OrigLoop->getLoopLatch()->getTerminator());
5596 if (HasBranchWeights) {
5597 std::optional<unsigned> VScale = Config.getVScaleForTuning();
5598 RUN_VPLAN_PASS(VPlanTransforms::addBranchWeightToMiddleTerminator,
5599 BestVPlan, BestVF, VScale);
5600 }
5601
5602 if (vputils::findIncomingAliasMask(Plan: BestVPlan)) {
5603 assert(BestVPlan.hasTailFolded() && "Expected tail folding to be enabled");
5604 RUN_VPLAN_PASS(VPlanTransforms::materializeAliasMaskCheckBlock, BestVPlan,
5605 *Legal->getRuntimePointerChecking()->getDiffChecks(),
5606 *PSE.getSE(), HasBranchWeights);
5607 ++LoopsPartialAliasVectorized;
5608 }
5609
5610 // Retrieving VectorPH now when it's easier while VPlan still has Regions.
5611 VPBasicBlock *VectorPH = cast<VPBasicBlock>(Val: BestVPlan.getVectorPreheader());
5612
5613 RUN_VPLAN_PASS(VPlanTransforms::materializeConstantVectorTripCount, BestVPlan,
5614 BestVF, BestUF, PSE);
5615 RUN_VPLAN_PASS(VPlanTransforms::optimizeForVFAndUF, BestVPlan, BestVF, BestUF,
5616 PSE);
5617 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
5618 // Check if scalar epilogue is required, before simplifying constant branches.
5619 const bool RequiresScalarEpilogue = BestVPlan.requiresScalarEpilogue();
5620 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5621 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5622 /*OnlyLatches=*/false);
5623 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, BestVPlan);
5624
5625 RUN_VPLAN_PASS(VPlanTransforms::convertToConcreteRecipes, BestVPlan);
5626 // Convert the exit condition to AVLNext == 0 for EVL tail folded loops.
5627 RUN_VPLAN_PASS(VPlanTransforms::convertEVLExitCond, BestVPlan);
5628 // Regions are dissolved after optimizing for VF and UF, which completely
5629 // removes unneeded loop regions first.
5630 const bool HasTailFolded = BestVPlan.hasTailFolded();
5631 RUN_VPLAN_PASS(VPlanTransforms::dissolveLoopRegions, BestVPlan);
5632 // Expand BranchOnTwoConds after dissolution, when latch has direct access to
5633 // its successors.
5634 RUN_VPLAN_PASS(VPlanTransforms::expandBranchOnTwoConds, BestVPlan);
5635 // Convert loops with variable-length stepping after regions are dissolved.
5636 RUN_VPLAN_PASS(VPlanTransforms::convertToVariableLengthStep, BestVPlan);
5637 // Remove dead back-edges for single-iteration loops with BranchOnCond(true).
5638 // Only process loop latches to avoid removing edges from the middle block,
5639 // which may be needed for epilogue vectorization.
5640 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5641 /*OnlyLatches=*/true);
5642 std::optional<uint64_t> MaxRuntimeStep = getMaxRuntimeElementCount(
5643 EC: BestVF * BestUF, F: *OrigLoop->getHeader()->getParent());
5644
5645 assert((LI->getUniqueLatchExitBlock(*OrigLoop) || RequiresScalarEpilogue) &&
5646 "loops not exiting via the latch without required epilogue?");
5647 RUN_VPLAN_PASS(VPlanTransforms::materializeVectorTripCount, BestVPlan,
5648 VectorPH, HasTailFolded, RequiresScalarEpilogue,
5649 &BestVPlan.getVFxUF(), MaxRuntimeStep);
5650 RUN_VPLAN_PASS(VPlanTransforms::materializeFactors, BestVPlan, VectorPH,
5651 BestVF);
5652 // Limit expansions to VPInstruction to when not vectorizing the epilogue.
5653 // Currently this code path still relies on code re-using SCEVs expanded
5654 // directly to IR instructions.
5655 if (EpilogueVecKind == EpilogueVectorizationKind::None)
5656 RUN_VPLAN_PASS(VPlanTransforms::expandSCEVsToVPInstructions, BestVPlan,
5657 *PSE.getSE());
5658 RUN_VPLAN_PASS(VPlanTransforms::cse, BestVPlan, VPDominatorTree(BestVPlan));
5659 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
5660 // Removing branches and incoming values may expose additional simplification
5661 // opportunities.
5662 if (RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, BestVPlan,
5663 /*OnlyLatches=*/EpilogueVecKind !=
5664 EpilogueVectorizationKind::None))
5665 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, BestVPlan);
5666 RUN_VPLAN_PASS(VPlanTransforms::simplifyKnownEVL, BestVPlan, BestVF, PSE);
5667 if (BestVPlan.getEntry()->getSingleSuccessor() ==
5668 BestVPlan.getScalarPreheader()) {
5669 // TODO: The vector loop would be dead, should not even try to vectorize.
5670 ORE->emit(RemarkBuilder: [&]() {
5671 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationDead",
5672 OrigLoop->getStartLoc(),
5673 OrigLoop->getHeader())
5674 << "Created vector loop never executes due to insufficient trip "
5675 "count.";
5676 });
5677 return DenseMap<const SCEV *, Value *>();
5678 }
5679
5680 // 0. Generate SCEV-dependent code in the entry, including TripCount, before
5681 // making any changes to the CFG.
5682 DenseMap<const SCEV *, Value *> ExpandedSCEVs =
5683 RUN_VPLAN_PASS(VPlanTransforms::expandSCEVs, BestVPlan, *PSE.getSE());
5684
5685 // Perform the actual loop transformation.
5686 IRBuilder<> Builder(PSE.getSE()->getModule());
5687 VPTransformState State(&TTI, BestVF, LI, DT, Legal->getAssumptionCache(),
5688 Builder, &BestVPlan, OrigLoop->getParentLoop());
5689
5690#ifdef EXPENSIVE_CHECKS
5691 assert(DT->verify(DominatorTree::VerificationLevel::Fast));
5692#endif
5693
5694 // 1. Create the scalar preheader; all other blocks are created during VPlan
5695 // execution.
5696 State.CFG.PrevBB = OrigLoop->getLoopPreheader();
5697 assert(State.CFG.PrevBB && "Invalid loop structure");
5698 BasicBlock *ScalarPHBB = SplitBlock(
5699 Old: State.CFG.PrevBB, SplitPt: State.CFG.PrevBB->getTerminator(), DT, LI, MSSAU: nullptr,
5700 BBName: EpilogueVecKind == EpilogueVectorizationKind::Epilogue
5701 ? "vec.epilog.scalar.ph"
5702 : "scalar.ph");
5703 if (VPBasicBlock *ScalarPH = BestVPlan.getScalarPreheader())
5704 replaceVPBBWithIRVPBB(VPBB: ScalarPH, IRBB: ScalarPHBB, Plan: &BestVPlan);
5705 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, BestVPlan);
5706
5707 assert(verifyVPlanIsValid(BestVPlan) && "final VPlan is invalid");
5708
5709 // After vectorization, the exit blocks of the original loop will have
5710 // additional predecessors. Invalidate SCEVs for the exit phis in case SE
5711 // looked through single-entry phis.
5712 ScalarEvolution &SE = *PSE.getSE();
5713 for (VPIRBasicBlock *Exit : BestVPlan.getExitBlocks()) {
5714 if (!Exit->hasPredecessors())
5715 continue;
5716 for (VPRecipeBase &PhiR : Exit->phis())
5717 SE.forgetLcssaPhiWithNewPredecessor(L: OrigLoop,
5718 V: &cast<VPIRPhi>(Val&: PhiR).getIRPhi());
5719 }
5720
5721 // Query whether the target wants loops it vectorizes to remain eligible for
5722 // runtime unrolling. Do this here, on the original loop and before its SCEV
5723 // is forgotten below.
5724 TargetTransformInfo::UnrollingPreferences UP;
5725 TTI.getUnrollingPreferences(L: OrigLoop, SE, UP, ORE);
5726 bool UnrollVectorizedLoop = UP.UnrollVectorizedLoop;
5727
5728 // Forget the original loop and block dispositions.
5729 SE.forgetLoop(L: OrigLoop);
5730 SE.forgetBlockAndLoopDispositions();
5731
5732 //===------------------------------------------------===//
5733 //
5734 // Notice: any optimization or new instruction that go
5735 // into the code below should also be implemented in
5736 // the cost-model.
5737 //
5738 //===------------------------------------------------===//
5739
5740 // Retrieve loop information before executing the plan, which may remove the
5741 // original loop, if it becomes unreachable.
5742 MDNode *LID = OrigLoop->getLoopID();
5743 unsigned OrigLoopInvocationWeight = 0;
5744 std::optional<unsigned> OrigAverageTripCount =
5745 getLoopEstimatedTripCount(L: OrigLoop, EstimatedLoopInvocationWeight: &OrigLoopInvocationWeight);
5746
5747 BestVPlan.execute(State: &State);
5748
5749 // 2.6. Maintain Loop Hints
5750 // Keep all loop hints from the original loop on the vector loop (we'll
5751 // replace the vectorizer-specific hints below).
5752 VPBasicBlock *HeaderVPBB = vputils::getFirstLoopHeader(Plan&: BestVPlan, VPDT&: State.VPDT);
5753 // Add metadata to disable runtime unrolling a scalar loop when there
5754 // are no runtime checks about strides and memory. A scalar loop that is
5755 // rarely used is not worth unrolling.
5756 bool DisableRuntimeUnroll = !HasRuntimeChecks && !BestVF.isScalar();
5757 updateLoopMetadataAndProfileInfo(
5758 VectorLoop: HeaderVPBB ? LI->getLoopFor(BB: State.CFG.VPBB2IRBB.lookup(Val: HeaderVPBB))
5759 : nullptr,
5760 HeaderVPBB, Plan: BestVPlan,
5761 VectorizingEpilogue: EpilogueVecKind == EpilogueVectorizationKind::Epilogue, OrigLoopID: LID,
5762 OrigAverageTripCount, OrigLoopInvocationWeight,
5763 EstimatedVFxUF: estimateElementCount(VF: BestVF * BestUF, VScale: Config.getVScaleForTuning()),
5764 DisableRuntimeUnroll, UnrollVectorizedLoop);
5765
5766 // 3. Remove redundant induction instructions, if a vector loop remains.
5767 if (HeaderVPBB)
5768 legacyCSE(BB: State.CFG.VPBB2IRBB.lookup(Val: HeaderVPBB));
5769
5770 // Wrap the generated blocks in VPIRBasicBlocks, so they can be used in the
5771 // epilogue plan.
5772 if (EpilogueVecKind == EpilogueVectorizationKind::MainLoop)
5773 for (VPBasicBlock *VPBB : to_vector(Range: VPBlockUtils::blocksAs<VPBasicBlock>(
5774 Range: vp_depth_first_shallow(G: BestVPlan.getEntry()))))
5775 if (!isa<VPIRBasicBlock>(Val: VPBB)) {
5776 VPBasicBlock *IRVPBB = replaceVPBBWithIRVPBB(
5777 VPBB, IRBB: State.CFG.VPBB2IRBB.at(Val: VPBB), Plan: &BestVPlan);
5778 replace(Range&: TrackedBlocks, OldValue: VPBB, NewValue: IRVPBB);
5779 }
5780
5781 return ExpandedSCEVs;
5782}
5783
5784bool VPRecipeBuilder::isPredicatedInst(Instruction *I) const {
5785 return CM.isPredicatedInst(I);
5786}
5787
5788bool VPRecipeBuilder::prefersVectorizedAddressing() const {
5789 return CM.TTI.prefersVectorizedAddressing();
5790}
5791
5792VPRecipeBase *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
5793 VFRange &Range) {
5794 assert((VPI->getOpcode() == Instruction::Load ||
5795 VPI->getOpcode() == Instruction::Store) &&
5796 "Must be called with either a load or store");
5797 Instruction *I = VPI->getUnderlyingInstr();
5798
5799 auto WillWiden = [&](ElementCount VF) -> bool {
5800 LoopVectorizationCostModel::InstWidening Decision =
5801 CM.getWideningDecision(I, VF);
5802 assert(Decision != LoopVectorizationCostModel::CM_Unknown &&
5803 "CM decision should be taken at this point.");
5804 if (Decision == LoopVectorizationCostModel::CM_Interleave)
5805 return true;
5806 if (CM.isScalarAfterVectorization(I, VF) ||
5807 CM.isProfitableToScalarize(I, VF))
5808 return false;
5809 return Decision != LoopVectorizationCostModel::CM_Scalarize;
5810 };
5811
5812 if (!LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: WillWiden, Range))
5813 return nullptr;
5814
5815 // If a mask is not required, drop it - use unmasked version for safe loads.
5816 // TODO: Determine if mask is needed in VPlan.
5817 VPValue *Mask = CM.isMaskRequired(I) ? VPI->getMask() : nullptr;
5818
5819 // Determine if the pointer operand of the access is either consecutive or
5820 // reverse consecutive.
5821 LoopVectorizationCostModel::InstWidening Decision =
5822 CM.getWideningDecision(I, VF: Range.Start);
5823 bool Reverse = Decision == LoopVectorizationCostModel::CM_Widen_Reverse;
5824 bool Consecutive =
5825 Reverse || Decision == LoopVectorizationCostModel::CM_Widen;
5826
5827 VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(N: 0)
5828 : VPI->getOperand(N: 1);
5829 Builder.setInsertPoint(VPI);
5830 if (Consecutive) {
5831 Ptr = Builder.createConsecutiveVectorPointer(Ptr, SourceElementTy: getLoadStoreType(I),
5832 Reverse, DL: VPI->getDebugLoc());
5833 }
5834
5835 if (Reverse && Mask)
5836 Mask = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Mask, DL: I->getDebugLoc());
5837
5838 if (VPI->getOpcode() == Instruction::Load) {
5839 auto *Load = cast<LoadInst>(Val: I);
5840 auto *LoadR =
5841 Builder.createWidenLoad(Load&: *Load, Addr: Ptr, Mask, Consecutive, Alignment: Load->getAlign(),
5842 Metadata: *VPI, DL: Load->getDebugLoc());
5843 if (Reverse)
5844 return Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: LoadR,
5845 DL: LoadR->getDebugLoc());
5846 return LoadR;
5847 }
5848
5849 StoreInst *Store = cast<StoreInst>(Val: I);
5850 VPValue *StoredVal = VPI->getOperand(N: 0);
5851 if (Reverse)
5852 StoredVal = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: StoredVal,
5853 DL: Store->getDebugLoc());
5854 return Builder.createWidenStore(Store&: *Store, Addr: Ptr, StoredVal, Mask, Consecutive,
5855 Alignment: Store->getAlign(), Metadata: *VPI,
5856 DL: Store->getDebugLoc());
5857}
5858
5859bool VPRecipeBuilder::shouldWiden(Instruction *I, VFRange &Range) const {
5860 assert((!isa<UncondBrInst, CondBrInst, PHINode, LoadInst, StoreInst>(I)) &&
5861 "Instruction should have been handled earlier");
5862 // Instruction should be widened, unless it is scalar after vectorization,
5863 // scalarization is profitable or it is predicated.
5864 auto WillScalarize = [this, I](ElementCount VF) -> bool {
5865 return CM.isScalarAfterVectorization(I, VF) ||
5866 CM.isProfitableToScalarize(I, VF) ||
5867 CM.isScalarWithPredication(I, VF);
5868 };
5869 return !LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: WillScalarize,
5870 Range);
5871}
5872
5873VPRecipeWithIRFlags *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
5874 auto *I = VPI->getUnderlyingInstr();
5875 switch (VPI->getOpcode()) {
5876 default:
5877 return nullptr;
5878 case Instruction::SDiv:
5879 case Instruction::UDiv:
5880 case Instruction::SRem:
5881 case Instruction::URem:
5882 // If not provably safe, use a masked intrinsic.
5883 if (CM.isPredicatedInst(I))
5884 return new VPWidenIntrinsicRecipe(
5885 getMaskedDivRemIntrinsic(Opcode: VPI->getOpcode()), VPI->operands(),
5886 I->getType(), {}, {}, VPI->getDebugLoc());
5887 [[fallthrough]];
5888 case Instruction::Add:
5889 case Instruction::And:
5890 case Instruction::AShr:
5891 case Instruction::FAdd:
5892 case Instruction::FCmp:
5893 case Instruction::FDiv:
5894 case Instruction::FMul:
5895 case Instruction::FNeg:
5896 case Instruction::FRem:
5897 case Instruction::FSub:
5898 case Instruction::ICmp:
5899 case Instruction::LShr:
5900 case Instruction::Mul:
5901 case Instruction::Or:
5902 case Instruction::Select:
5903 case Instruction::Shl:
5904 case Instruction::Sub:
5905 case Instruction::Xor:
5906 case Instruction::Freeze:
5907 case Instruction::ExtractValue:
5908 return new VPWidenRecipe(*I, VPI->operandsWithoutMask(), *VPI, *VPI,
5909 VPI->getDebugLoc());
5910 };
5911}
5912
5913VPHistogramRecipe *VPRecipeBuilder::widenIfHistogram(VPInstruction *VPI) {
5914 if (VPI->getOpcode() != Instruction::Store)
5915 return nullptr;
5916
5917 const HistogramInfo *HistInfo =
5918 Legal->getHistogramInfo(I: cast<StoreInst>(Val: VPI->getUnderlyingInstr()));
5919 if (!HistInfo)
5920 return nullptr;
5921
5922 SmallVector<VPValue *, 3> HGramOps;
5923 // Bucket address.
5924 HGramOps.push_back(Elt: VPI->getOperand(N: 1));
5925 // Increment value.
5926 HGramOps.push_back(Elt: Plan.getOrAddLiveIn(V: HistInfo->Update->getOperand(i: 1)));
5927
5928 // In case of predicated execution (due to tail-folding, or conditional
5929 // execution, or both), pass the relevant mask.
5930 if (VPValue *Mask = VPI->getMask())
5931 HGramOps.push_back(Elt: Mask);
5932
5933 return new VPHistogramRecipe(HistInfo->UpdateKind, HGramOps,
5934 cast<VPIRMetadata>(Val&: *VPI), VPI->getDebugLoc());
5935}
5936
5937std::optional<CompressedPtrInfo>
5938VPRecipeBuilder::getCompressedPtrInfo(VPInstruction *VPI) {
5939 if (!is_contained(Set: {Instruction::Load, Instruction::Store}, Element: VPI->getOpcode()))
5940 return std::nullopt;
5941 Instruction *I = VPI->getUnderlyingInstr();
5942 return Legal->getCompressedPtrInfo(Ptr: getLoadStorePointerOperand(V: I));
5943}
5944
5945VPWidenMemIntrinsicRecipe *
5946VPRecipeBuilder::widenCompressedLoadOrStore(VPInstruction *VPI) {
5947 assert(getCompressedPtrInfo(VPI) && "Expected a compressed load/store");
5948
5949 VPBuilder::InsertPointGuard Guard(Builder);
5950 Builder.setInsertPoint(VPI);
5951
5952 VPValue *Mask = VPI->getMask();
5953 Type *AccessTy = vputils::getLoadStoreValueType(R: VPI);
5954 Align Alignment = getLoadStoreAlignment(I: VPI->getUnderlyingInstr());
5955
5956 VPValue *Ptr = VPI->getOpcode() == Instruction::Load ? VPI->getOperand(N: 0)
5957 : VPI->getOperand(N: 1);
5958 Ptr = Builder.createConsecutiveVectorPointer(Ptr, SourceElementTy: AccessTy,
5959 /*Reverse=*/false,
5960 DL: VPI->getDebugLoc());
5961
5962 // Note: For expandload we fill masked-off lanes with passthru poison.
5963 if (VPI->getOpcode() == Instruction::Load)
5964 return new VPWidenMemIntrinsicRecipe(
5965 Intrinsic::masked_expandload, {Ptr, Mask, Plan.getPoison(Ty: AccessTy)},
5966 AccessTy, Alignment, *VPI, VPI->getDebugLoc());
5967
5968 VPValue *StoredValue = VPI->getOperand(N: 0);
5969 return new VPWidenMemIntrinsicRecipe(Intrinsic::masked_compressstore,
5970 {StoredValue, Ptr, Mask}, AccessTy,
5971 Alignment, *VPI, VPI->getDebugLoc());
5972}
5973
5974bool VPRecipeBuilder::replaceWithFinalIfReductionStore(
5975 VPInstruction *VPI, VPBuilder &FinalRedStoresBuilder) {
5976 StoreInst *SI;
5977 if ((SI = dyn_cast<StoreInst>(Val: VPI->getUnderlyingInstr())) &&
5978 Legal->isInvariantAddressOfReduction(V: SI->getPointerOperand())) {
5979 // Only create recipe for the final invariant store of the reduction.
5980 if (Legal->isInvariantStoreOfReduction(SI)) {
5981 VPValue *Val = VPI->getOperand(N: 0);
5982 VPValue *Addr = VPI->getOperand(N: 1);
5983 // We need to store the exiting value of the reduction, so use the blend
5984 // if tail folded.
5985 if (auto *Blend = VPlanPatternMatch::findSingleUser<VPBlendRecipe>(V: Val))
5986 Val = Blend;
5987 [[maybe_unused]] auto *Rdx =
5988 VPlanPatternMatch::findSingleUser<VPReductionPHIRecipe>(V: Val);
5989 assert((isa<VPIRValue>(Val) || !Rdx || Rdx->getBackedgeValue() == Val) &&
5990 "Store of reduction thats not the backedge value?");
5991 auto *Recipe = new VPReplicateRecipe(
5992 SI, {Val, Addr}, true /* IsUniform */, nullptr /*Mask*/, *VPI, *VPI,
5993 VPI->getDebugLoc());
5994 FinalRedStoresBuilder.insert(R: Recipe);
5995 }
5996 VPI->eraseFromParent();
5997 return true;
5998 }
5999
6000 return false;
6001}
6002
6003VPSingleDefRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
6004 VFRange &Range) {
6005 auto *I = VPI->getUnderlyingInstr();
6006 bool IsUniform = LoopVectorizationPlanner::getDecisionAndClampRange(
6007 Predicate: [&](ElementCount VF) { return CM.isUniformAfterVectorization(I, VF); },
6008 Range);
6009
6010 bool IsPredicated = CM.isPredicatedInst(I);
6011
6012 // Even if the instruction is not marked as uniform, there are certain
6013 // intrinsic calls that can be effectively treated as such, so we check for
6014 // them here. Conservatively, we only do this for scalable vectors, since
6015 // for fixed-width VFs we can always fall back on full scalarization.
6016 if (!IsUniform && Range.Start.isScalable() && isa<IntrinsicInst>(Val: I)) {
6017 switch (cast<IntrinsicInst>(Val: I)->getIntrinsicID()) {
6018 case Intrinsic::assume:
6019 case Intrinsic::lifetime_start:
6020 case Intrinsic::lifetime_end:
6021 // For scalable vectors if one of the operands is variant then we still
6022 // want to mark as uniform, which will generate one instruction for just
6023 // the first lane of the vector. We can't scalarize the call in the same
6024 // way as for fixed-width vectors because we don't know how many lanes
6025 // there are.
6026 //
6027 // The reasons for doing it this way for scalable vectors are:
6028 // 1. For the assume intrinsic generating the instruction for the first
6029 // lane is still be better than not generating any at all. For
6030 // example, the input may be a splat across all lanes.
6031 // 2. For the lifetime start/end intrinsics the pointer operand only
6032 // does anything useful when the input comes from a stack object,
6033 // which suggests it should always be uniform. For non-stack objects
6034 // the effect is to poison the object, which still allows us to
6035 // remove the call.
6036 IsUniform = true;
6037 break;
6038 default:
6039 break;
6040 }
6041 }
6042 VPValue *BlockInMask = nullptr;
6043 if (!IsPredicated) {
6044 // Finalize the recipe for Instr, first if it is not predicated.
6045 LLVM_DEBUG(dbgs() << "LV: Scalarizing:" << *I << "\n");
6046 } else {
6047 LLVM_DEBUG(dbgs() << "LV: Scalarizing and predicating:" << *I << "\n");
6048 // Instructions marked for predication are replicated and a mask operand is
6049 // added initially. Masked replicate recipes will later be placed under an
6050 // if-then construct to prevent side-effects. Generate recipes to compute
6051 // the block mask for this region.
6052 BlockInMask = VPI->getMask();
6053 }
6054
6055 // Note that there is some custom logic to mark some intrinsics as uniform
6056 // manually above for scalable vectors, which this assert needs to account for
6057 // as well.
6058 assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
6059 (Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
6060 "Should not predicate a uniform recipe");
6061 if (IsUniform) {
6062 return VPBuilder::createSingleScalarOp(
6063 Opcode: VPI->getOpcode(), Operands: VPI->operandsWithoutMask(), Mask: BlockInMask, Flags: *VPI, Metadata: *VPI,
6064 DL: VPI->getDebugLoc(), ResultTy: VPI->getScalarType(), UV: I);
6065 }
6066 auto *Recipe = new VPReplicateRecipe(I, VPI->operandsWithoutMask(),
6067 /*IsSingleScalar=*/false, BlockInMask,
6068 *VPI, *VPI, VPI->getDebugLoc());
6069 return Recipe;
6070}
6071
6072VPRecipeBase *
6073VPRecipeBuilder::tryToCreateWidenNonPhiRecipe(VPSingleDefRecipe *R,
6074 VFRange &Range) {
6075 assert(!R->isPhi() && "phis must be handled earlier");
6076 auto *VPI = cast<VPInstruction>(Val: R);
6077 assert(VPI->getOpcode() != Instruction::Call &&
6078 "Call should have been handled by makeCallWideningDecisions");
6079
6080 // All widen recipes below deal only with VF > 1.
6081 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6082 Predicate: [&](ElementCount VF) { return VF.isScalar(); }, Range))
6083 return nullptr;
6084
6085 Instruction *Instr = R->getUnderlyingInstr();
6086 assert(!is_contained({Instruction::Load, Instruction::Store},
6087 VPI->getOpcode()) &&
6088 "Should have been handled prior to this!");
6089
6090 // An extractvalue takes the shape of its operand: we cannot extract a lane
6091 // from a widened struct, nor build a struct from replicated lanes.
6092 if (VPI->getOpcode() == Instruction::ExtractValue &&
6093 !vputils::isSingleScalar(VPV: VPI->getOperand(N: 0)))
6094 if (VPRecipeBase *OpR = VPI->getOperand(N: 0)->getDefiningRecipe())
6095 if (!vputils::doesGeneratePerAllLanes(R: OpR))
6096 return tryToWiden(VPI);
6097 if (VPI->getOpcode() == Instruction::ExtractValue)
6098 return new VPReplicateRecipe(Instr, VPI->operandsWithoutMask(),
6099 vputils::isSingleScalar(VPV: VPI->getOperand(N: 0)),
6100 /*Mask=*/nullptr, *VPI, *VPI,
6101 VPI->getDebugLoc());
6102
6103 if (!shouldWiden(I: Instr, Range))
6104 return nullptr;
6105
6106 if (VPI->getOpcode() == Instruction::GetElementPtr) {
6107 auto *GEP = cast<GetElementPtrInst>(Val: Instr);
6108 return new VPWidenGEPRecipe(GEP->getSourceElementType(),
6109 VPI->operandsWithoutMask(), *VPI,
6110 VPI->getDebugLoc(), GEP);
6111 }
6112
6113 if (Instruction::isCast(Opcode: VPI->getOpcode())) {
6114 auto *CI = cast<CastInst>(Val: Instr);
6115 return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(N: 0),
6116 VPI->getScalarType(), CI, *VPI, *VPI,
6117 VPI->getDebugLoc());
6118 }
6119
6120 return tryToWiden(VPI);
6121}
6122
6123// To allow RUN_VPLAN_PASS to print the VPlan after VF/UF independent
6124// optimizations.
6125static void printOptimizedVPlan(VPlan &) {}
6126
6127#ifndef NDEBUG
6128/// Cross-check the execution frequencies recorded in \p Plan against
6129/// BlockFrequencyInfo for the blocks of \p OrigLoop.
6130/// FIXME: Temporary verification aid, to be removed.
6131static bool verifyExecutionFrequenciesMatchBFI(VPlan &Plan, Loop *OrigLoop,
6132 LoopInfo *LI,
6133 LoopVectorizationCostModel &CM) {
6134 // Limited to loops with the latch as only exiting block
6135 if (OrigLoop->getExitingBlock() != OrigLoop->getLoopLatch())
6136 return true;
6137
6138 // Visit the loop body in the same order as recordExecutionFrequencies. Both
6139 // are reverse post-orders of the same CFG, so indices correspond.
6140 VPBasicBlock *Header = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan).first;
6141 SmallVector<VPBasicBlock *> Blocks = vp_rpo_plain_cfg_loop_body(Header);
6142 assert(Blocks.size() == OrigLoop->getNumBlocks() &&
6143 "loop body and original loop must have the same blocks");
6144
6145 LoopBlocksRPO OrigRPO(OrigLoop);
6146 OrigRPO.perform(LI);
6147
6148 // Only request the expensive BFI once the cheap bail-outs are past.
6149 BlockFrequencyInfo &BFI = CM.getBFI();
6150 uint64_t HeaderFreq = BFI.getBlockFreq(OrigLoop->getHeader()).getFrequency();
6151 if (HeaderFreq == 0)
6152 return true;
6153
6154 // BFI's fixed-point mass propagation loses up to 1 ULP per edge, so bound the
6155 // error by the number of edges in the region.
6156 uint64_t Edges = 0;
6157 for (const VPBasicBlock *VPBB : Blocks)
6158 Edges += VPBB->getNumSuccessors();
6159 uint64_t Tolerance = Edges + BranchProbability::getDenominator() / HeaderFreq;
6160
6161 for (const auto &[VPBB, BB] :
6162 zip_equal(drop_begin(Blocks), drop_begin(OrigRPO))) {
6163 // Nothing to check for blocks without a recorded frequency.
6164 std::optional<VPExecutionFrequency> Freq =
6165 getRecordedExecutionFrequency(VPBB);
6166 if (!Freq)
6167 continue;
6168 BranchProbability Computed = vputils::getExecutionProbability(Freq->Freq);
6169
6170 // Clamp to the header's frequency, which BFI's rounding may exceed.
6171 uint64_t BBFreq = BFI.getBlockFreq(BB).getFrequency();
6172 BranchProbability Expected = BranchProbability::getBranchProbability(
6173 std::min(BBFreq, HeaderFreq), HeaderFreq);
6174 if (AbsoluteDifference(Computed.getNumerator(), Expected.getNumerator()) <=
6175 Tolerance)
6176 continue;
6177
6178 errs() << "Block frequency mismatch for " << VPBB->getName() << ": VPlan "
6179 << Computed << ", BlockFrequencyInfo " << Expected << "\n";
6180 return false;
6181 }
6182 return true;
6183}
6184#endif
6185
6186VPlanPtr
6187LoopVectorizationPlanner::tryToBuildVPlan1(LoopVectorizationCostModel &PlanCM) {
6188 bool IsInnerLoop = OrigLoop->isInnermost();
6189
6190 // Set up loop versioning for inner loops with memory runtime checks.
6191 // Outer loops don't have LoopAccessInfo since canVectorizeMemory() is not
6192 // called for them.
6193 std::optional<LoopVersioning> LVer;
6194 if (IsInnerLoop) {
6195 const LoopAccessInfo *LAI = Legal->getLAI();
6196 LVer.emplace(args: *LAI, args: LAI->getRuntimePointerChecking()->getChecks(), args&: OrigLoop,
6197 args&: LI, args&: DT, args: PSE.getSE());
6198 if (!LAI->getRuntimePointerChecking()->getChecks().empty() &&
6199 !LAI->getRuntimePointerChecking()->getDiffChecks()) {
6200 // Only use noalias metadata when using memory checks guaranteeing no
6201 // overlap across all iterations.
6202 LVer->prepareNoAliasMetadata();
6203 }
6204 }
6205
6206 // Create initial base VPlan0, to serve as common starting point for all
6207 // candidates built later for specific VF ranges.
6208 auto VPlan0 = VPlanTransforms::buildVPlan0(
6209 TheLoop: OrigLoop, LI&: *LI, InductionTy: Legal->getWidestInductionType(), PSE,
6210 LVer: LVer ? &*LVer : nullptr, GetBPI);
6211
6212 VPDominatorTree VPDT(*VPlan0);
6213 if (const LoopAccessInfo *LAI = Legal->getLAI())
6214 RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
6215 LAI->getSymbolicStrides(), VPDT);
6216 RUN_VPLAN_PASS(VPlanTransforms::combineRecipes, *VPlan0);
6217 RUN_VPLAN_PASS(VPlanTransforms::removeDeadRecipes, *VPlan0);
6218 if (IsInnerLoop) {
6219 RUN_VPLAN_PASS(VPlanTransforms::recordExecutionFrequencies, *VPlan0);
6220 assert(verifyExecutionFrequenciesMatchBFI(*VPlan0, OrigLoop, LI, PlanCM) &&
6221 "execution frequencies do not match the loop's block frequencies");
6222 }
6223
6224 // Create recipes for header phis. For outer loops, reductions, recurrences
6225 // and in-loop reductions are empty since legality doesn't detect them.
6226 if (!RUN_VPLAN_PASS(
6227 VPlanTransforms::createHeaderPhiRecipes, *VPlan0, PSE, *OrigLoop, ORE,
6228 VPDT, Legal->getInductionVars(), Legal->getReductionVars(),
6229 Legal->getConditionalInductions(), Legal->getFixedOrderRecurrences(),
6230 Config.getInLoopReductions(), Config.getHints().allowReordering())) {
6231 return nullptr;
6232 }
6233
6234 if (const LoopAccessInfo *LAI = Legal->getLAI())
6235 RUN_VPLAN_PASS(VPlanTransforms::replaceSymbolicStrides, *VPlan0, PSE,
6236 LAI->getSymbolicStrides(), VPDT);
6237
6238 // Without -vplan-outer-loop-legality-checks, trust the explicit vectorization
6239 // pragma that vectorizing the outer loop is safe.
6240 if (!IsInnerLoop && Opts.vplan_outer_loop_legality_checks &&
6241 !proveOuterLoopMemorySafety(Plan&: *VPlan0, PSE, AA&: *Legal->getAA(), VPDT,
6242 OuterLoop: OrigLoop)) {
6243 reportVectorizationFailure(
6244 DebugMsg: "Cannot prove outer loop memory accesses safe",
6245 OREMsg: "cannot prove memory accesses in outer loop are safe to vectorize",
6246 ORETag: "CantProveOuterLoopMemSafety", ORE, TheLoop: OrigLoop);
6247 return nullptr;
6248 }
6249
6250 // Add surviving induction predicates to PSE and check constraints.
6251 bool ForceVectorization =
6252 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled;
6253 bool OptForSize = !ForceVectorization && PlanCM.EpilogueLoweringStatus ==
6254 CM_EpilogueNotAllowedOptSize;
6255 unsigned SCEVCheckThreshold = ForceVectorization
6256 ? Opts.pragma_vectorize_scev_check_threshold
6257 : Opts.vectorize_scev_check_threshold;
6258 if (!RUN_VPLAN_PASS(VPlanTransforms::finalizeSCEVPredicates, *VPlan0, PSE,
6259 OptForSize, SCEVCheckThreshold, ORE, OrigLoop))
6260 return nullptr;
6261
6262 RUN_VPLAN_PASS(VPlanTransforms::addMiddleCheck, *VPlan0);
6263
6264 // If we're vectorizing a loop with an uncountable exit, make sure that the
6265 // recipes are safe to handle.
6266 // TODO: Remove this once we can properly check the VPlan itself for the
6267 // presence of an uncountable exit.
6268 if (Legal->hasUncountableEarlyExit()) {
6269 if (!RUN_VPLAN_PASS(VPlanTransforms::splitCombinedExits, *VPlan0, PSE,
6270 OrigLoop))
6271 return nullptr;
6272
6273 // TODO: Check target preference for style.
6274 UncountableExitStyle EEStyle = UncountableExitStyle::ReadOnly;
6275 if (!RUN_VPLAN_PASS(VPlanTransforms::handleUncountableEarlyExits, *VPlan0,
6276 ORE, OrigLoop, PSE, *DT, Legal->getAssumptionCache(),
6277 EEStyle)) {
6278 return nullptr;
6279 }
6280 } else {
6281 RUN_VPLAN_PASS(VPlanTransforms::handleCountableEarlyExits, *VPlan0);
6282 }
6283
6284 RUN_VPLAN_PASS(VPlanTransforms::createLoopRegions, *VPlan0,
6285 getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()));
6286 if (PlanCM.foldTailByMasking())
6287 RUN_VPLAN_PASS(VPlanTransforms::foldTailByMasking, *VPlan0);
6288
6289 RUN_VPLAN_PASS(VPlanTransforms::introduceMasksAndLinearize, *VPlan0);
6290
6291 return VPlan0;
6292}
6293
6294void LoopVectorizationPlanner::buildVPlans(VPlan &VPlan1, ElementCount MinVF,
6295 ElementCount MaxVF,
6296 LoopVectorizationCostModel &PlanCM) {
6297 if (ElementCount::isKnownGT(LHS: MinVF, RHS: MaxVF))
6298 return;
6299
6300 auto MaxVFTimes2 = MaxVF * 2;
6301 for (ElementCount VF = MinVF; ElementCount::isKnownLT(LHS: VF, RHS: MaxVFTimes2);) {
6302 VFRange SubRange = {VF, MaxVFTimes2};
6303 auto Plan = tryToBuildVPlan(InitialPlan: std::unique_ptr<VPlan>(VPlan1.duplicate()),
6304 Range&: SubRange, PlanCM);
6305 VF = SubRange.End;
6306
6307 if (!Plan)
6308 continue;
6309
6310 // Now optimize the initial VPlan.
6311 RUN_VPLAN_PASS(VPlanTransforms::hoistPredicatedLoads, *Plan, PSE, OrigLoop);
6312 RUN_VPLAN_PASS(VPlanTransforms::sinkPredicatedStores, *Plan, PSE, OrigLoop);
6313 RUN_VPLAN_PASS(VPlanTransforms::truncateToMinimalBitwidths, *Plan,
6314 Config.getMinimalBitwidths());
6315 RUN_VPLAN_PASS(VPlanTransforms::optimize, *Plan);
6316 // TODO: try to put addExplicitVectorLength close to addActiveLaneMask
6317 if (PlanCM.foldTailWithEVL()) {
6318 RUN_VPLAN_PASS(VPlanTransforms::addExplicitVectorLength, *Plan,
6319 Config.getMaxSafeElements());
6320 RUN_VPLAN_PASS(VPlanTransforms::optimizeEVLMasks, *Plan);
6321 VPCostContext CostCtx(*TLI, *Plan, *CM, Config);
6322 RUN_VPLAN_PASS(VPlanTransforms::foldPredicateMerge, *Plan, CostCtx);
6323 }
6324
6325 if (auto P =
6326 RUN_VPLAN_PASS(VPlanTransforms::narrowInterleaveGroups, *Plan, TTI))
6327 VPlans.push_back(Elt: std::move(P));
6328
6329 TailFoldingStyle Style = PlanCM.getTailFoldingStyle();
6330 RUN_VPLAN_PASS(VPlanTransforms::materializeHeaderMask, *Plan,
6331 useActiveLaneMask(Style),
6332 useActiveLaneMaskForControlFlow(Style));
6333
6334 RUN_VPLAN_PASS_NO_VERIFY(printOptimizedVPlan, *Plan);
6335 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6336 VPlans.push_back(Elt: std::move(Plan));
6337 }
6338}
6339
6340VPlanPtr
6341LoopVectorizationPlanner::tryToBuildVPlan(VPlanPtr Plan, VFRange &Range,
6342 LoopVectorizationCostModel &PlanCM) {
6343
6344 // For outer loops, the plan only needs basic recipe conversion and induction
6345 // live-out optimization; the full inner-loop recipe building below does not
6346 // apply (no widening decisions, interleave groups, reductions, etc.).
6347 if (Plan->isOuterLoop()) {
6348 for (ElementCount VF : Range)
6349 Plan->addVF(VF);
6350 if (!RUN_VPLAN_PASS(VPlanTransforms::tryToConvertVPInstructionsToVPRecipes,
6351 *Plan, *TLI, PSE, OrigLoop))
6352 return nullptr;
6353 RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
6354 OrigLoop);
6355 return Plan;
6356 }
6357
6358 using namespace llvm::VPlanPatternMatch;
6359 SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
6360
6361 // ---------------------------------------------------------------------------
6362 // Build initial VPlan: Scan the body of the loop in a topological order to
6363 // visit each basic block after having visited its predecessor basic blocks.
6364 // ---------------------------------------------------------------------------
6365
6366 bool RequiresScalarEpilogueCheck =
6367 LoopVectorizationPlanner::getDecisionAndClampRange(
6368 Predicate: [&PlanCM](ElementCount VF) {
6369 return !PlanCM.requiresScalarEpilogue(IsVectorizing: VF.isVector());
6370 },
6371 Range);
6372 // Update the branch in the middle block if a scalar epilogue is required.
6373 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6374 if (!RequiresScalarEpilogueCheck && MiddleVPBB->getNumSuccessors() == 2) {
6375 auto *BranchOnCond = cast<VPInstruction>(Val: MiddleVPBB->getTerminator());
6376 assert(MiddleVPBB->getSuccessors()[1] == Plan->getScalarPreheader() &&
6377 "second successor must be scalar preheader");
6378 BranchOnCond->setOperand(I: 0, New: Plan->getFalse());
6379 }
6380
6381 // Don't use getDecisionAndClampRange here, because we don't know the UF
6382 // so this function is better to be conservative, rather than to split
6383 // it up into different VPlans.
6384 // TODO: Consider using getDecisionAndClampRange here to split up VPlans.
6385 bool IVUpdateMayOverflow = false;
6386 for (ElementCount VF : Range)
6387 IVUpdateMayOverflow |= !isIndvarOverflowCheckKnownFalse(Opts, Cost: &PlanCM, VF);
6388
6389 TailFoldingStyle Style = PlanCM.getTailFoldingStyle();
6390 // Use NUW for the induction increment if we proved that it won't overflow in
6391 // the vector loop or when not folding the tail. In the later case, we know
6392 // that the canonical induction increment will not overflow as the vector trip
6393 // count is >= increment and a multiple of the increment.
6394 VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
6395 bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
6396 if (!HasNUW) {
6397 auto *IVInc =
6398 LoopRegion->getExitingBasicBlock()->getTerminator()->getOperand(N: 0);
6399 assert(match(IVInc,
6400 m_VPInstruction<Instruction::Add>(
6401 m_Specific(LoopRegion->getCanonicalIV()), m_VPValue())) &&
6402 "Did not find the canonical IV increment");
6403 LoopRegion->clearCanonicalIVNUW(Increment: cast<VPInstruction>(Val: IVInc));
6404 }
6405
6406 // ---------------------------------------------------------------------------
6407 // Pre-construction: record ingredients whose recipes we'll need to further
6408 // process after constructing the initial VPlan.
6409 // ---------------------------------------------------------------------------
6410
6411 // For each interleave group which is relevant for this (possibly trimmed)
6412 // Range, add it to the set of groups to be later applied to the VPlan and add
6413 // placeholders for its members' Recipes which we'll be replacing with a
6414 // single VPInterleaveRecipe.
6415 for (InterleaveGroup<Instruction> *IG :
6416 PlanCM.InterleaveInfo.getInterleaveGroups()) {
6417 auto ApplyIG = [IG, &PlanCM](ElementCount VF) -> bool {
6418 bool Result = (VF.isVector() && // Query is illegal for VF == 1
6419 PlanCM.getWideningDecision(I: IG->getInsertPos(), VF) ==
6420 LoopVectorizationCostModel::CM_Interleave);
6421 // For scalable vectors, the interleave factors must be <= 8 since we
6422 // require the (de)interleaveN intrinsics instead of shufflevectors.
6423 assert((!Result || !VF.isScalable() || IG->getFactor() <= 8) &&
6424 "Unsupported interleave factor for scalable vectors");
6425 return Result;
6426 };
6427 if (!getDecisionAndClampRange(Predicate: ApplyIG, Range))
6428 continue;
6429 InterleaveGroups.insert(Ptr: IG);
6430 }
6431
6432 // ---------------------------------------------------------------------------
6433 // Construct wide recipes and apply predication for original scalar
6434 // VPInstructions in the loop.
6435 // ---------------------------------------------------------------------------
6436 VPRecipeBuilder RecipeBuilder(*Plan, Legal, PlanCM, Builder);
6437
6438 if (!RUN_VPLAN_PASS(VPlanTransforms::widenCompressingPatterns, *Plan,
6439 RecipeBuilder, *PSE.getSE()))
6440 return nullptr;
6441
6442 RUN_VPLAN_PASS(VPlanTransforms::createInLoopReductionRecipes, *Plan,
6443 Range.Start);
6444
6445 VPCostContext CostCtx(*TLI, *Plan, PlanCM, Config);
6446
6447 RUN_VPLAN_PASS(VPlanTransforms::makeMemOpWideningDecisions, *Plan, Range,
6448 RecipeBuilder, CostCtx);
6449
6450 RUN_VPLAN_PASS(VPlanTransforms::makeScalarizationDecisions, *Plan, Range);
6451
6452 // Widened calls may only use the first lane of some operands.
6453 if (RUN_VPLAN_PASS(VPlanTransforms::makeCallWideningDecisions, *Plan, Range,
6454 RecipeBuilder, CostCtx))
6455 RUN_VPLAN_PASS(VPlanTransforms::makeScalarizationDecisions, *Plan, Range);
6456
6457 RUN_VPLAN_PASS(VPlanTransforms::narrowInductionTruncates, *Plan, Range, TTI,
6458 PSE);
6459
6460 // Convert remaining VPInstructions to widen or replicate recipes.
6461 // TODO: This legacy code should eventually be migrated to VPlan.
6462 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
6463 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6464 Range: vp_depth_first_shallow(G: HeaderVPBB))) {
6465 // All types but VPInstructions are already widened and don't need extra
6466 // processing. We process VPInstructions below.
6467 assert(
6468 all_of(
6469 make_range(VPBB->getFirstNonPhi(), VPBB->end()),
6470 IsaPred<VPWidenCanonicalIVRecipe, VPBlendRecipe, VPReductionRecipe,
6471 VPReplicateRecipe, VPWidenLoadRecipe, VPWidenStoreRecipe,
6472 VPWidenCallRecipe, VPWidenIntrinsicRecipe,
6473 VPVectorPointerRecipe, VPVectorEndPointerRecipe,
6474 VPHistogramRecipe, VPInstruction>) &&
6475 "Unexpected recipe");
6476 for (VPInstruction &VPI :
6477 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range&: *VPBB))) {
6478 // We represent single-scalar casts directly as VPInstructions.
6479 if (Instruction::isCast(Opcode: VPI.getOpcode()) &&
6480 vputils::onlyFirstLaneUsed(Def: &VPI))
6481 continue;
6482
6483 // Only VPInstructions with an underlying value need to be processed.
6484 if (!VPI.getUnderlyingValue())
6485 continue;
6486
6487 Builder.setInsertPoint(&VPI);
6488
6489 VPRecipeBase *Recipe =
6490 RecipeBuilder.tryToCreateWidenNonPhiRecipe(R: &VPI, Range);
6491 if (!Recipe)
6492 Recipe = RecipeBuilder.handleReplication(VPI: &VPI, Range);
6493 Builder.insert(R: Recipe);
6494
6495 if (Recipe->getNumDefinedValues() == 1) {
6496 VPI.replaceAllUsesWith(New: Recipe->getVPSingleValue());
6497 } else {
6498 assert(Recipe->getNumDefinedValues() == 0 &&
6499 "Unexpected multidef recipe");
6500 }
6501 VPI.eraseFromParent();
6502 }
6503 }
6504
6505 assert(isa<VPRegionBlock>(LoopRegion) &&
6506 !LoopRegion->getEntryBasicBlock()->empty() &&
6507 "entry block must be set to a VPRegionBlock having a non-empty entry "
6508 "VPBasicBlock");
6509
6510 RUN_VPLAN_PASS(VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers, *Plan,
6511 Range);
6512
6513 // ---------------------------------------------------------------------------
6514 // Transform initial VPlan: Apply previously taken decisions, in order, to
6515 // bring the VPlan to its final state.
6516 // ---------------------------------------------------------------------------
6517
6518 addReductionResultComputation(Plan, MinVF: Range.Start, PlanCM);
6519
6520 // Optimize FindIV reductions to use sentinel-based approach when possible.
6521 RUN_VPLAN_PASS(VPlanTransforms::optimizeFindIVReductions, *Plan, PSE,
6522 *OrigLoop);
6523 RUN_VPLAN_PASS(VPlanTransforms::optimizeInductionLiveOutUsers, *Plan, PSE,
6524 OrigLoop);
6525
6526 // Apply mandatory transformation to handle reductions with multiple in-loop
6527 // uses if possible, bail out otherwise.
6528 if (!RUN_VPLAN_PASS(VPlanTransforms::handleMultiUseReductions, *Plan, ORE,
6529 OrigLoop))
6530 return nullptr;
6531 // Apply mandatory transformation to handle FP maxnum/minnum reduction with
6532 // NaNs if possible, bail out otherwise.
6533 if (!RUN_VPLAN_PASS(VPlanTransforms::handleMaxMinNumReductions, *Plan))
6534 return nullptr;
6535
6536 // Create whole-vector selects for find-last recurrences.
6537 if (!RUN_VPLAN_PASS(VPlanTransforms::handleFindLastReductions, *Plan))
6538 return nullptr;
6539
6540 RUN_VPLAN_PASS(VPlanTransforms::removeBranchOnConst, *Plan, false);
6541
6542 // Create partial reduction recipes for scaled reductions and transform
6543 // recipes to abstract recipes if it is legal and beneficial and clamp the
6544 // range for better cost estimation.
6545 RUN_VPLAN_PASS(VPlanTransforms::createPartialReductions, *Plan, CostCtx,
6546 Range);
6547 RUN_VPLAN_PASS(VPlanTransforms::convertToAbstractRecipes, *Plan, CostCtx,
6548 Range);
6549
6550 // Interleave memory: for each Interleave Group we marked earlier as relevant
6551 // for this VPlan, replace the Recipes widening its memory instructions with a
6552 // single VPInterleaveRecipe at its insertion point.
6553 RUN_VPLAN_PASS(VPlanTransforms::createInterleaveGroups, *Plan,
6554 InterleaveGroups, PlanCM.isEpilogueAllowed());
6555
6556 // Convert memory recipes to strided access recipes if the strided access is
6557 // legal and profitable.
6558 RUN_VPLAN_PASS(VPlanTransforms::convertToStridedAccesses, *Plan, PSE,
6559 *OrigLoop, CostCtx, Range);
6560
6561 if (Legal->hasUncountableExitWithSideEffects())
6562 RUN_VPLAN_PASS(VPlanTransforms::expandMaskBeforeFirst, *Plan, CostCtx,
6563 Range);
6564
6565 // Ensure scalar VF plans only contain VF=1, as required by hasScalarVFOnly.
6566 if (Range.Start.isScalar())
6567 Range.End = Range.Start * 2;
6568
6569 for (ElementCount VF : Range)
6570 Plan->addVF(VF);
6571 Plan->setName("Initial VPlan");
6572
6573 RUN_VPLAN_PASS(VPlanTransforms::dropPoisonGeneratingRecipes, *Plan);
6574
6575 if (PlanCM.maskPartialAliasing())
6576 RUN_VPLAN_PASS(VPlanTransforms::attachAliasMaskToHeaderMask, *Plan);
6577
6578 assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid");
6579 return Plan;
6580}
6581
6582void LoopVectorizationPlanner::addReductionResultComputation(
6583 VPlanPtr &Plan, ElementCount MinVF, LoopVectorizationCostModel &PlanCM) {
6584 using namespace VPlanPatternMatch;
6585 VPRegionBlock *VectorLoopRegion = Plan->getVectorLoopRegion();
6586 VPBasicBlock *MiddleVPBB = Plan->getMiddleBlock();
6587 VPBasicBlock *LatchVPBB = VectorLoopRegion->getExitingBasicBlock();
6588 Builder.setInsertPoint(&*std::prev(x: std::prev(x: LatchVPBB->end())));
6589 VPBasicBlock::iterator IP = MiddleVPBB->getFirstNonPhi();
6590 VPValue *HeaderMask = Plan->getVectorLoopRegion()->getHeaderMask();
6591 for (VPRecipeBase &R : make_early_inc_range(
6592 Range: Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
6593 VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
6594 if (!PhiR)
6595 continue;
6596
6597 // Clean up reductions that have become invariant.
6598 if (PhiR->getBackedgeValue() == PhiR) {
6599 PhiR->replaceAllUsesWith(New: PhiR->getStartValue());
6600 PhiR->eraseFromParent();
6601 continue;
6602 }
6603
6604 RecurKind RecurrenceKind = PhiR->getRecurrenceKind();
6605 const RecurrenceDescriptor &RdxDesc = Legal->getRecurrenceDescriptor(
6606 PN: cast<PHINode>(Val: PhiR->getUnderlyingInstr()));
6607 Type *PhiTy = PhiR->getScalarType();
6608
6609 // Convert a VPBlendRecipe backedge to a select.
6610 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: PhiR->getBackedgeValue())) {
6611 if (Blend->getNumIncomingValues() == 2 &&
6612 Blend->getMask(Idx: 0) == HeaderMask) {
6613 auto *Sel = VPBuilder(Blend).createSelect(
6614 Cond: Blend->getMask(Idx: 0), TrueVal: Blend->getIncomingValue(Idx: 0),
6615 FalseVal: Blend->getIncomingValue(Idx: 1), DL: {}, Name: "", Flags: *Blend);
6616 Blend->replaceAllUsesWith(New: Sel);
6617 Blend->eraseFromParent();
6618 }
6619 }
6620
6621 auto *OrigExitingVPV = PhiR->getBackedgeValue();
6622 auto *NewExitingVPV = OrigExitingVPV;
6623
6624 // Remove the predicated select if the target doesn't want it.
6625 VPValue *V;
6626 if (!PlanCM.usePredicatedReductionSelect(
6627 RecurrenceKind, HasUsesOutsideReductionChain: PhiR->hasUsesOutsideReductionChain()) &&
6628 match(V: PhiR->getBackedgeValue(),
6629 P: m_Select(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue(V), Op2: m_Specific(VPV: PhiR))))
6630 PhiR->setBackedgeValue(V);
6631
6632 // We want code in the middle block to appear to execute on the location of
6633 // the scalar loop's latch terminator because: (a) it is all compiler
6634 // generated, (b) these instructions are always executed after evaluating
6635 // the latch conditional branch, and (c) other passes may add new
6636 // predecessors which terminate on this line. This is the easiest way to
6637 // ensure we don't accidentally cause an extra step back into the loop while
6638 // debugging.
6639 DebugLoc ExitDL = OrigLoop->getLoopLatch()->getTerminator()->getDebugLoc();
6640
6641 // TODO: At the moment ComputeReductionResult also drives creation of the
6642 // bc.merge.rdx phi nodes, hence it needs to be created unconditionally here
6643 // even for in-loop reductions, until the reduction resume value handling is
6644 // also modeled in VPlan.
6645 VPInstruction *FinalReductionResult;
6646 VPBuilder::InsertPointGuard Guard(Builder);
6647 Builder.setInsertPoint(IP);
6648 // For AnyOf reductions, find the select among PhiR's users and convert
6649 // the reduction phi to operate on bools before creating the final
6650 // reduction result.
6651 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RecurrenceKind)) {
6652 auto *AnyOfSelect = findSingleUser<VPSingleDefRecipe>(
6653 V: PhiR,
6654 P: m_Select(Op0: m_Unless(P: m_Specific(VPV: HeaderMask)), Op1: m_VPValue(), Op2: m_VPValue()));
6655 VPValue *Start = PhiR->getStartValue();
6656 bool TrueValIsPhi = AnyOfSelect->getOperand(N: 1) == PhiR;
6657 // NewVal is the non-phi operand of the select.
6658 VPValue *NewVal = TrueValIsPhi ? AnyOfSelect->getOperand(N: 2)
6659 : AnyOfSelect->getOperand(N: 1);
6660
6661 // Adjust AnyOf reductions; replace the reduction phi for the selected
6662 // value with a boolean reduction phi node to check if the condition is
6663 // true in any iteration. The final value is selected by the final
6664 // ComputeReductionResult.
6665 VPValue *Cmp = AnyOfSelect->getOperand(N: 0);
6666 // If the compare is checking the reduction PHI node, adjust it to check
6667 // the start value.
6668 if (VPRecipeBase *CmpR = Cmp->getDefiningRecipe())
6669 CmpR->replaceUsesOfWith(From: PhiR, To: PhiR->getStartValue());
6670 Builder.setInsertPoint(AnyOfSelect);
6671
6672 // If the true value of the select is the reduction phi, the new value
6673 // is selected if the negated condition is true in any iteration.
6674 if (TrueValIsPhi)
6675 Cmp = Builder.createNot(Operand: Cmp);
6676
6677 // Build a fresh i1 chain (phi, or, and i1 versions of any blend/select
6678 // the exiting value flows through).
6679 auto *NewPhiR =
6680 PhiR->cloneWithOperands(Start: Plan->getFalse(), BackedgeValue: Plan->getFalse());
6681 NewPhiR->insertBefore(InsertPos: PhiR);
6682 VPValue *NewExiting = Builder.createOr(LHS: NewPhiR, RHS: Cmp);
6683
6684 // The exiting value may flow through a chain of VPBlendRecipes and
6685 // select recipes (VPInstruction, VPWidenRecipe or VPReplicateRecipe with
6686 // Select opcode) before reaching OrigExitingVPV. Clone each chain link
6687 // in topological order so each clone refers to the already-rewritten i1
6688 // operands via Substitutions.
6689 DenseMap<VPValue *, VPValue *> Substitutions = {{AnyOfSelect, NewExiting},
6690 {PhiR, NewPhiR}};
6691 std::function<void(VPSingleDefRecipe *)> CloneChain =
6692 [&](VPSingleDefRecipe *Old) {
6693 if (Substitutions.contains(Val: Old))
6694 return;
6695 SmallVector<VPValue *> NewOps;
6696 for (VPValue *Op : Old->operands()) {
6697 if (isa<VPBlendRecipe>(Val: Op) ||
6698 match(V: Op, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())))
6699 CloneChain(cast<VPSingleDefRecipe>(Val: Op));
6700 NewOps.push_back(Elt: Substitutions.lookup_or(Val: Op, Default&: Op));
6701 }
6702 VPSingleDefRecipe *New;
6703 if (auto *B = dyn_cast<VPBlendRecipe>(Val: Old))
6704 New = B->cloneWithOperands(NewOperands: NewOps);
6705 else if (auto *W = dyn_cast<VPWidenRecipe>(Val: Old))
6706 New = W->cloneWithOperands(NewOperands: NewOps);
6707 else if (auto *Rep = dyn_cast<VPReplicateRecipe>(Val: Old))
6708 New = Rep->cloneWithOperands(NewOperands: NewOps);
6709 else
6710 New = cast<VPInstruction>(Val: Old)->cloneWithOperands(NewOperands: NewOps);
6711 New->insertBefore(InsertPos: Old);
6712 Substitutions[Old] = New;
6713 };
6714
6715 if (OrigExitingVPV != AnyOfSelect) {
6716 CloneChain(cast<VPSingleDefRecipe>(Val: OrigExitingVPV));
6717 NewExiting = Substitutions.lookup(Val: OrigExitingVPV);
6718 }
6719 NewPhiR->setOperand(I: 1, New: NewExiting);
6720 PhiR->replaceAllUsesWith(New: Plan->getPoison(Ty: PhiR->getScalarType()));
6721
6722 Builder.setInsertPoint(IP);
6723 FinalReductionResult =
6724 Builder.createAnyOfReduction(ChainOp: NewExiting, TrueVal: NewVal, FalseVal: Start, DL: ExitDL);
6725 } else {
6726 // If the vector reduction can be performed in a smaller type, we
6727 // truncate then extend the loop exit value to enable InstCombine to
6728 // evaluate the entire expression in the smaller type.
6729 VPValue *ReductionOp = NewExitingVPV;
6730 Instruction::CastOps ExtendOpc = Instruction::CastOpsEnd;
6731 if (MinVF.isVector() && PhiTy != RdxDesc.getRecurrenceType()) {
6732 assert(!PhiR->isInLoop() && "Unexpected truncated inloop reduction!");
6733 assert(!RecurrenceDescriptor::isMinMaxRecurrenceKind(RecurrenceKind) &&
6734 "Unexpected truncated min-max recurrence!");
6735 Type *RdxTy = RdxDesc.getRecurrenceType();
6736 ExtendOpc = RdxDesc.isSigned() ? Instruction::SExt : Instruction::ZExt;
6737 {
6738 VPBuilder::InsertPointGuard Guard(Builder);
6739 Builder.setInsertPoint(
6740 std::next(x: NewExitingVPV->getDefiningRecipe()->getIterator()));
6741 ReductionOp =
6742 Builder.createWidenCast(Opcode: Instruction::Trunc, Op: NewExitingVPV, ResultTy: RdxTy);
6743 VPWidenCastRecipe *Extnd =
6744 Builder.createWidenCast(Opcode: ExtendOpc, Op: ReductionOp, ResultTy: PhiTy);
6745 if (PhiR->getOperand(N: 1) == NewExitingVPV)
6746 PhiR->setOperand(I: 1, New: Extnd);
6747 }
6748 }
6749
6750 VPIRFlags Flags(RecurrenceKind, PhiR->isOrdered(), PhiR->isInLoop(),
6751 PhiR->getFastMathFlagsOrNone());
6752 FinalReductionResult = Builder.createNaryOp(
6753 Opcode: VPInstruction::ComputeReductionResult, Operands: {ReductionOp}, Flags, DL: ExitDL);
6754 if (ExtendOpc != Instruction::CastOpsEnd)
6755 FinalReductionResult = Builder.createScalarCast(
6756 Opcode: ExtendOpc, Op: FinalReductionResult, ResultTy: PhiTy, DL: {});
6757 }
6758
6759 // Update all users outside the vector region. Also replace redundant
6760 // extracts.
6761 for (auto *U : to_vector(Range: OrigExitingVPV->users())) {
6762 auto *Parent = cast<VPRecipeBase>(Val: U)->getParent();
6763 if (FinalReductionResult == U || Parent->getParent())
6764 continue;
6765 // Skip ComputeReductionResult and FindIV reductions when they are not the
6766 // final result.
6767 if (match(U, P: m_VPInstruction<VPInstruction::ComputeReductionResult>()) ||
6768 (RecurrenceDescriptor::isFindIVRecurrenceKind(Kind: RecurrenceKind) &&
6769 match(U, P: m_VPInstruction<Instruction::ICmp>())))
6770 continue;
6771 U->replaceUsesOfWith(From: OrigExitingVPV, To: FinalReductionResult);
6772
6773 // Look through ExtractLastPart.
6774 if (match(U, P: m_ExtractLastPart(Op0: m_VPValue())))
6775 U = cast<VPInstruction>(Val: U)->getSingleUser();
6776
6777 if (match(U, P: m_CombineOr(Ps: m_ExtractLane(Op0: m_VPValue(), Op1: m_VPValue()),
6778 Ps: m_ExtractLastLane(Op0: m_VPValue()))))
6779 cast<VPInstruction>(Val: U)->replaceAllUsesWith(New: FinalReductionResult);
6780 }
6781
6782 RecurKind RK = PhiR->getRecurrenceKind();
6783 if ((!RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK) &&
6784 !RecurrenceDescriptor::isFindIVRecurrenceKind(Kind: RK) &&
6785 !RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind: RK) &&
6786 !RecurrenceDescriptor::isFindLastRecurrenceKind(Kind: RK))) {
6787 VPBuilder PHBuilder(Plan->getVectorPreheader());
6788 VPValue *Iden = Plan->getOrAddLiveIn(
6789 V: getRecurrenceIdentity(K: RK, Tp: PhiTy, FMF: PhiR->getFastMathFlagsOrNone()));
6790 auto *ScaleFactorVPV = Plan->getConstantInt(BitWidth: 32, Val: 1);
6791 VPValue *StartV = PHBuilder.createNaryOp(
6792 Opcode: VPInstruction::ReductionStartVector,
6793 Operands: {PhiR->getStartValue(), Iden, ScaleFactorVPV}, Flags: *PhiR);
6794 PhiR->setOperand(I: 0, New: StartV);
6795 }
6796 }
6797
6798 RUN_VPLAN_PASS(VPlanTransforms::clearReductionWrapFlags, *Plan);
6799}
6800
6801void LoopVectorizationPlanner::attachRuntimeChecks(
6802 VPlan &Plan, GeneratedRTChecks &RTChecks, ElementCount VF, unsigned UF,
6803 bool HasBranchWeights) const {
6804 const auto &[SCEVCheckCond, SCEVCheckBlock] = RTChecks.getSCEVChecks();
6805 if (SCEVCheckBlock && SCEVCheckBlock->hasNPredecessors(N: 0)) {
6806 assert((!Config.OptForSize ||
6807 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled) &&
6808 "Cannot SCEV check stride or overflow when optimizing for size");
6809 RUN_VPLAN_PASS(VPlanTransforms::attachCheckBlock, Plan, SCEVCheckCond,
6810 SCEVCheckBlock, HasBranchWeights);
6811 }
6812 const auto &[MemCheckCond, MemCheckBlock] = RTChecks.getMemRuntimeChecks();
6813 if (MemCheckBlock && MemCheckBlock->hasNPredecessors(N: 0)) {
6814 // VPlan-native path does not do any analysis for memory runtime checks
6815 // currently.
6816 assert((!Opts.enable_vplan_native_path || !Plan.isOuterLoop()) &&
6817 "Memory runtime checks are not supported for outer loops yet");
6818
6819 if (Config.OptForSize) {
6820 assert(
6821 Config.getHints().getForce() == LoopVectorizeHints::FK_Enabled &&
6822 "Cannot emit memory checks when optimizing for size, unless forced "
6823 "to vectorize.");
6824 ORE->emit(RemarkBuilder: [&]() {
6825 return OptimizationRemarkAnalysis(DEBUG_TYPE, "VectorizationCodeSize",
6826 OrigLoop->getStartLoc(),
6827 OrigLoop->getHeader())
6828 << "Code-size may be reduced by not forcing "
6829 "vectorization, or by source-code modifications "
6830 "eliminating the need for runtime checks "
6831 "(e.g., adding 'restrict').";
6832 });
6833 }
6834 const auto &RtPtrChecking = *Legal->getRuntimePointerChecking();
6835 // Erase the temporary IR before recipe expansion can reuse its values.
6836 RTChecks.eraseMemCheckBlock();
6837 RUN_VPLAN_PASS(VPlanTransforms::attachMemoryChecks, Plan, RtPtrChecking,
6838 OrigLoop, *PSE.getSE(), VF, UF, HasBranchWeights);
6839 }
6840}
6841
6842void LoopVectorizationPlanner::addMinimumIterationCheck(
6843 VPlan &Plan, ElementCount VF, unsigned UF,
6844 ElementCount MinProfitableTripCount, bool TailFoldedEpilogue) const {
6845 const uint32_t *BranchWeights =
6846 hasBranchWeightMD(I: *OrigLoop->getLoopLatch()->getTerminator())
6847 ? &MinItersBypassWeights[0]
6848 : nullptr;
6849 // If either the main vector loop or the epilogue vector loop is tail-folded,
6850 // it can handle any trip count, so the min iter check never needs
6851 // to branch to the scalar loop.
6852 RUN_VPLAN_PASS(VPlanTransforms::addMinimumIterationCheck, Plan, VF, UF,
6853 MinProfitableTripCount, Plan.requiresScalarEpilogue(),
6854 Plan.hasTailFolded() || TailFoldedEpilogue, OrigLoop,
6855 BranchWeights,
6856 OrigLoop->getLoopPredecessor()->getTerminator()->getDebugLoc(),
6857 PSE, Plan.getEntry());
6858}
6859
6860// Determine how to lower the epilogue, which depends on 1) optimising
6861// for minimum code-size, 2) tail-folding compiler options, 3) loop
6862// hints forcing tail-folding, and 4) a TTI hook that analyses whether the loop
6863// is suitable for tail-folding.
6864// This function determines epilogue lowering for the main vector loop while
6865// epilogue lowering for the tail-folded epilogue path will be handled
6866// separately in getEpilogueTailLowering.
6867static EpilogueLowering
6868getEpilogueLowering(const VectorizeOptions &Opts, LoopVectorizeHints &Hints,
6869 bool OptForSize, TargetTransformInfo *TTI,
6870 TargetLibraryInfo *TLI, LoopVectorizationLegality &LVL,
6871 InterleavedAccessInfo *IAI) {
6872 // 1) OptSize takes precedence over all other options, i.e. if this is set,
6873 // don't look at hints or options, and don't request an epilogue.
6874 if (OptForSize)
6875 return CM_EpilogueNotAllowedOptSize;
6876
6877 // 2) If set, obey the directives
6878 if (Opts.tail_folding_policy) {
6879 switch (*Opts.tail_folding_policy) {
6880 case TailFoldingPolicyTy::None:
6881 return CM_EpilogueAllowed;
6882 case TailFoldingPolicyTy::PreferFoldTail:
6883 return CM_EpilogueNotNeededFoldTail;
6884 case TailFoldingPolicyTy::MustFoldTail:
6885 return CM_EpilogueNotAllowedFoldTail;
6886 };
6887 }
6888
6889 // 3) If set, obey the hints
6890 switch (Hints.getPredicate()) {
6891 case LoopVectorizeHints::FK_Enabled:
6892 return CM_EpilogueNotNeededFoldTail;
6893 case LoopVectorizeHints::FK_Disabled:
6894 return CM_EpilogueAllowed;
6895 };
6896
6897 // 4) if the TTI hook indicates this is profitable, request tail-folding.
6898 TailFoldingInfo TFI(TLI, &LVL, IAI);
6899 if (TTI->preferTailFoldingOverEpilogue(TFI: &TFI))
6900 return CM_EpilogueNotNeededFoldTail;
6901
6902 return CM_EpilogueAllowed;
6903}
6904
6905// Emit a remark if there are stores to floats that required a floating point
6906// extension. If the vectorized loop was generated with floating point there
6907// will be a performance penalty from the conversion overhead and the change in
6908// the vector width.
6909static void checkMixedPrecision(Loop *L, OptimizationRemarkEmitter *ORE) {
6910 SmallVector<Instruction *, 4> Worklist;
6911 for (BasicBlock *BB : L->getBlocks()) {
6912 for (Instruction &Inst : *BB) {
6913 if (auto *S = dyn_cast<StoreInst>(Val: &Inst)) {
6914 if (S->getValueOperand()->getType()->isFloatTy())
6915 Worklist.push_back(Elt: S);
6916 }
6917 }
6918 }
6919
6920 // Traverse the floating point stores upwards searching, for floating point
6921 // conversions.
6922 SmallPtrSet<const Instruction *, 4> Visited;
6923 SmallPtrSet<const Instruction *, 4> EmittedRemark;
6924 while (!Worklist.empty()) {
6925 auto *I = Worklist.pop_back_val();
6926 if (!L->contains(Inst: I))
6927 continue;
6928 if (!Visited.insert(Ptr: I).second)
6929 continue;
6930
6931 // Emit a remark if the floating point store required a floating
6932 // point conversion.
6933 // TODO: More work could be done to identify the root cause such as a
6934 // constant or a function return type and point the user to it.
6935 if (isa<FPExtInst>(Val: I) && EmittedRemark.insert(Ptr: I).second)
6936 ORE->emit(RemarkBuilder: [&]() {
6937 return OptimizationRemarkAnalysis(LV_NAME, "VectorMixedPrecision",
6938 I->getDebugLoc(), L->getHeader())
6939 << "floating point conversion changes vector width. "
6940 << "Mixed floating point precision requires an up/down "
6941 << "cast that will negatively impact performance.";
6942 });
6943
6944 for (Use &Op : I->operands())
6945 if (auto *OpI = dyn_cast<Instruction>(Val&: Op))
6946 Worklist.push_back(Elt: OpI);
6947 }
6948}
6949
6950/// For loops with uncountable early exits, find the cost of doing work when
6951/// exiting the loop early, such as calculating the final exit values of
6952/// variables used outside the loop. The loop leaves through at most one of the
6953/// vector.early.exit blocks, so return the maximum of their costs.
6954/// TODO: This is currently overly pessimistic because the loop may not take
6955/// the early exit, but better to keep this conservative for now. In future,
6956/// it might be possible to relax this by using branch probabilities.
6957static InstructionCost calculateEarlyExitCost(VPCostContext &CostCtx,
6958 VPlan &Plan, ElementCount VF) {
6959 InstructionCost Cost = 0;
6960 for (auto *ExitVPBB : Plan.getExitBlocks()) {
6961 for (auto *PredVPBB : ExitVPBB->getPredecessors()) {
6962 // If the predecessor is not the middle.block, then it must be the
6963 // vector.early.exit block, which may contain work to calculate the exit
6964 // values of variables used outside the loop.
6965 if (PredVPBB != Plan.getMiddleBlock()) {
6966 LLVM_DEBUG(dbgs() << "Calculating cost of work in exit block "
6967 << PredVPBB->getName() << ":\n");
6968 Cost = std::max(a: Cost, b: PredVPBB->cost(VF, Ctx&: CostCtx));
6969 }
6970 }
6971 }
6972 return Cost;
6973}
6974
6975/// This function determines whether or not it's still profitable to vectorize
6976/// the loop given the extra work we have to do outside of the loop:
6977/// 1. Perform the runtime checks before entering the loop to ensure it's safe
6978/// to vectorize.
6979/// 2. In the case of loops with uncountable early exits, we may have to do
6980/// extra work when exiting the loop early, such as calculating the final
6981/// exit values of variables used outside the loop.
6982/// 3. The middle block.
6983static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
6984 VectorizationFactor &VF, Loop *L,
6985 PredicatedScalarEvolution &PSE,
6986 VPCostContext &CostCtx, VPlan &Plan,
6987 EpilogueLowering SEL,
6988 std::optional<unsigned> VScale) {
6989 InstructionCost RtC = Checks.getCost();
6990 if (!RtC.isValid())
6991 return false;
6992
6993 // When interleaving only scalar and vector cost will be equal, which in turn
6994 // would lead to a divide by 0. Fall back to hard threshold.
6995 if (VF.Width.isScalar()) {
6996 // TODO: Should we rename VectorizeMemoryCheckThreshold?
6997 if (RtC > VectorizerParams::VectorizeMemoryCheckThreshold) {
6998 LLVM_DEBUG(
6999 dbgs()
7000 << "LV: Interleaving only is not profitable due to runtime checks\n");
7001 return false;
7002 }
7003 return true;
7004 }
7005
7006 // The scalar cost should only be 0 when vectorizing with a user specified
7007 // VF/IC. In those cases, runtime checks should always be generated.
7008 uint64_t ScalarC = VF.ScalarCost.getValue();
7009 if (ScalarC == 0)
7010 return true;
7011
7012 // The vector loop is left through either a vector.early.exit block or the
7013 // middle block, so add the cost of the more expensive of the two.
7014 InstructionCost EarlyExitCost =
7015 calculateEarlyExitCost(CostCtx, Plan, VF: VF.Width);
7016 InstructionCost MiddleCost = Plan.getMiddleBlock()->cost(VF: VF.Width, Ctx&: CostCtx);
7017 InstructionCost TotalCost = RtC + std::max(a: EarlyExitCost, b: MiddleCost);
7018
7019 // First, compute the minimum iteration count required so that the vector
7020 // loop outperforms the scalar loop.
7021 // The total cost of the scalar loop is
7022 // ScalarC * TC
7023 // where
7024 // * TC is the actual trip count of the loop.
7025 // * ScalarC is the cost of a single scalar iteration.
7026 //
7027 // The total cost of the vector loop is
7028 // TotalCost + VecC * (TC / VF) + EpiC
7029 // where
7030 // * TotalCost is the sum of the costs of
7031 // - the generated runtime checks, i.e. RtC
7032 // - the most expensive way of leaving the vector loop, i.e. the maximum
7033 // of the costs of the middle block and, for loops with uncountable
7034 // early exits, the vector.early.exit blocks.
7035 // * VecC is the cost of a single vector iteration.
7036 // * TC is the actual trip count of the loop
7037 // * VF is the vectorization factor
7038 // * EpiCost is the cost of the generated epilogue, including the cost
7039 // of the remaining scalar operations.
7040 //
7041 // Vectorization is profitable once the total vector cost is less than the
7042 // total scalar cost:
7043 // TotalCost + VecC * (TC / VF) + EpiC < ScalarC * TC
7044 //
7045 // Now we can compute the minimum required trip count TC as
7046 // VF * (TotalCost + EpiC) / (ScalarC * VF - VecC) < TC
7047 //
7048 // For now we assume the epilogue cost EpiC = 0 for simplicity. Note that
7049 // the computations are performed on doubles, not integers and the result
7050 // is rounded up, hence we get an upper estimate of the TC.
7051 unsigned IntVF = estimateElementCount(VF: VF.Width, VScale);
7052 uint64_t Div = ScalarC * IntVF - VF.Cost.getValue();
7053 uint64_t MinTC1 =
7054 Div == 0 ? 0 : divideCeil(Numerator: TotalCost.getValue() * IntVF, Denominator: Div);
7055
7056 // Second, compute a minimum iteration count so that the cost of the
7057 // runtime checks is only a fraction of the total scalar loop cost. This
7058 // adds a loop-dependent bound on the overhead incurred if the runtime
7059 // checks fail. In case the runtime checks fail, the cost is RtC + ScalarC
7060 // * TC. To bound the runtime check to be a fraction 1/X of the scalar
7061 // cost, compute
7062 // RtC < ScalarC * TC * (1 / X) ==> RtC * X / ScalarC < TC
7063 uint64_t MinTC2 = divideCeil(Numerator: RtC.getValue() * 10, Denominator: ScalarC);
7064
7065 // Now pick the larger minimum. If it is not a multiple of VF and an epilogue
7066 // is allowed, choose the next closest multiple of VF. This should partly
7067 // compensate for ignoring the epilogue cost.
7068 uint64_t MinTC = std::max(a: MinTC1, b: MinTC2);
7069 if (SEL == CM_EpilogueAllowed)
7070 MinTC = alignTo(Value: MinTC, Align: IntVF);
7071 VF.MinProfitableTripCount = ElementCount::getFixed(MinVal: MinTC);
7072
7073 LLVM_DEBUG(
7074 dbgs() << "LV: Minimum required TC for runtime checks to be profitable:"
7075 << VF.MinProfitableTripCount << "\n");
7076
7077 // Skip vectorization if the expected trip count is less than the minimum
7078 // required trip count.
7079 if (auto ExpectedTC = getSmallBestKnownTC(Opts: CostCtx.Opts, PSE, L)) {
7080 if (ElementCount::isKnownLT(LHS: *ExpectedTC, RHS: VF.MinProfitableTripCount)) {
7081 LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
7082 "trip count < minimum profitable VF ("
7083 << *ExpectedTC << " < " << VF.MinProfitableTripCount
7084 << ")\n");
7085
7086 return false;
7087 }
7088 }
7089 return true;
7090}
7091
7092LoopVectorizePass::LoopVectorizePass(LoopVectorizeOptions Opts)
7093 : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced),
7094 VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced) {}
7095
7096/// ResumeForEpilogue markers in the main plan, used by the epilogue plan.
7097struct MainPlanResumeMarkers {
7098 VPInstruction *VectorTC;
7099 SmallVector<VPInstruction *> ResumeValues;
7100};
7101
7102/// Prepare \p MainPlan for vectorizing the main vector loop during epilogue
7103/// vectorization.
7104static MainPlanResumeMarkers preparePlanForMainVectorLoop(VPlan &MainPlan) {
7105 using namespace VPlanPatternMatch;
7106 // When vectorizing the epilogue, FindFirstIV & FindLastIV reductions can
7107 // introduce multiple uses of undef/poison. If the reduction start value may
7108 // be undef or poison it needs to be frozen and the frozen start has to be
7109 // used when computing the reduction result. We also need to use the frozen
7110 // value in the resume phi generated by the main vector loop, as this is also
7111 // used to compute the reduction result after the epilogue vector loop. The
7112 // epilogue plan re-uses the frozen values.
7113 VPBuilder Builder(MainPlan.getEntry());
7114 for (VPInstruction &VPI :
7115 make_isa_range<VPInstruction>(Range&: *MainPlan.getMiddleBlock())) {
7116 VPValue *OrigStart;
7117 if (!matchFindIVResult(VPI: &VPI, ReducedIV: m_VPValue(), Start: m_VPValue(V&: OrigStart)))
7118 continue;
7119 if (isGuaranteedNotToBeUndefOrPoison(V: OrigStart->getLiveInIRValue()))
7120 continue;
7121 VPInstruction *Freeze = Builder.createFreeze(Op: OrigStart, DL: {}, Name: "fr");
7122 VPI.setOperand(I: 2, New: Freeze);
7123 OrigStart->replaceUsesWithIf(New: Freeze, ShouldReplace: IsaPred<VPPhi>);
7124 }
7125
7126 VPValue *VectorTC = nullptr;
7127 auto *Term =
7128 MainPlan.getVectorLoopRegion()->getExitingBasicBlock()->getTerminator();
7129 [[maybe_unused]] bool MatchedTC =
7130 match(V: Term, P: m_BranchOnCount(Op0: m_VPValue(), Op1: m_VPValue(V&: VectorTC)));
7131 assert(MatchedTC && "must match vector trip count");
7132
7133 VPBasicBlock *MiddleVPBB = MainPlan.getMiddleBlock();
7134 VPBuilder MiddleBuilder(MiddleVPBB->getFirstNonPhi());
7135 VPInstruction *VectorTCMarker = MiddleBuilder.createNaryOp(
7136 Opcode: VPInstruction::ResumeForEpilogue,
7137 Operands: {VectorTC, MainPlan.getZero(Ty: VectorTC->getScalarType())});
7138
7139 // Replace the resume phis of the VPIRPhis in the scalar header of the main
7140 // plan by ResumeForEpilogue instructions for their incoming values and return
7141 // them so they can be used as resume values when vectorizing the epilogue.
7142 // The epilogue plan creates the resume phis after the main plan executed.
7143 VPRecipeBase *MiddleTerm = MiddleVPBB->getTerminator();
7144 MiddleBuilder.setInsertPoint(MiddleTerm ? MiddleTerm->getIterator()
7145 : MiddleVPBB->end());
7146 auto ResumeValues = to_vector(
7147 Range: map_range(C: MainPlan.getScalarHeader()->phis(), F: [&](VPRecipeBase &R) {
7148 auto *ResumePhi = cast<VPPhi>(Val: R.getOperand(N: 0));
7149 assert(ResumePhi->getNumIncoming() == 2 &&
7150 ResumePhi->getIncomingBlock(0) == MiddleVPBB &&
7151 "expected resume phi merging middle block and bypass value");
7152 return MiddleBuilder.createNaryOp(Opcode: VPInstruction::ResumeForEpilogue,
7153 Operands: ResumePhi->operands());
7154 }));
7155 for (VPRecipeBase &R :
7156 make_early_inc_range(Range: MainPlan.getScalarPreheader()->phis())) {
7157 auto *ResumePhi = cast<VPPhi>(Val: &R);
7158 ResumePhi->replaceAllUsesWith(New: ResumePhi->getOperand(N: 1));
7159 ResumePhi->eraseFromParent();
7160 }
7161 return {.VectorTC: VectorTCMarker, .ResumeValues: std::move(ResumeValues)};
7162}
7163
7164/// Prepare \p Plan for vectorizing the epilogue loop. That is, re-use expanded
7165/// SCEVs from \p ExpandedSCEVs and set resume values for header recipes.
7166static void preparePlanForEpilogueVectorLoop(
7167 VPlan &MainPlan, VPlan &Plan, const SCEV2ValueTy &ExpandedSCEVs,
7168 ElementCount MainLoopVF, unsigned MainLoopUF, ElementCount EpilogueVF,
7169 VFSelectionContext &Config, ScalarEvolution &SE,
7170 const MainPlanResumeMarkers &Markers, VPBasicBlock *MainCheck) {
7171 // Build a map from the scalar-header PHI to the ResumeForEpilogue markers
7172 // from the main plan.
7173 // TODO: Replace the IR PHI key.
7174 DenseMap<PHINode *, VPInstruction *> IRPhiToResumeForEpi;
7175 for (auto [HeaderPhi, ResumeForEpi] :
7176 zip_equal(t: MainPlan.getScalarHeader()->phis(), u: Markers.ResumeValues))
7177 IRPhiToResumeForEpi[&cast<VPIRPhi>(Val&: HeaderPhi).getIRPhi()] = ResumeForEpi;
7178 VPIRBasicBlock *EpilogueCheck = RUN_VPLAN_PASS(
7179 VPlanTransforms::modelGeneratedMainLoopBlocks, Plan, MainPlan, MainCheck);
7180 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
7181 VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
7182 Header->setName("vec.epilog.vector.body");
7183
7184 // When skipping the epilogue vector loop, the scalar loop resumes at the
7185 // values computed by the main vector loop.
7186 for (auto [R, ResumeForEpi] :
7187 zip_equal(t: Plan.getScalarHeader()->phis(), u: Markers.ResumeValues)) {
7188 // A tail-folded epilogue's middle block never branches to the scalar
7189 // preheader, so it has no resume phis and the scalar loop starts at the
7190 // original start values. The epilogue check never skips a tail-folded
7191 // epilogue either, so there is no resume value to set.
7192 auto *ResumePhi = dyn_cast<VPPhi>(Val: R.getOperand(N: 0));
7193 if (!ResumePhi) {
7194 assert(Plan.hasTailFolded() &&
7195 "expected resume phis unless the epilogue is tail-folded");
7196 continue;
7197 }
7198 ResumePhi->setIncomingValueForBlock(
7199 VPBB: EpilogueCheck, V: Plan.getOrAddLiveIn(V: ResumeForEpi->getUnderlyingValue()));
7200 }
7201
7202 VPBasicBlock *VectorPH = Plan.getVectorPreheader();
7203 assert(VectorPH->getNumPredecessors() == 2 &&
7204 VectorPH->getPredecessors()[0] == EpilogueCheck &&
7205 "vector preheader must be entered from the epilogue check first");
7206 // Returns the value to resume at for \p ResumeForEpi: a phi in the vector
7207 // preheader merging the value computed by the main vector loop with the
7208 // value used when the main vector loop has been bypassed, or the former if
7209 // both are equal.
7210 auto GetResumeValue = [&](VPInstruction *ResumeForEpi,
7211 const Twine &Name) -> VPValue * {
7212 VPValue *MainV = Plan.getOrAddLiveIn(V: ResumeForEpi->getUnderlyingValue());
7213 VPValue *BypassV =
7214 Plan.getOrAddLiveIn(V: ResumeForEpi->getOperand(N: 1)->getUnderlyingValue());
7215 if (MainV == BypassV)
7216 return MainV;
7217 return VPBuilder(VectorPH->getFirstNonPhi())
7218 .createScalarPhi(IncomingValues: {MainV, BypassV}, DL: {}, Name);
7219 };
7220
7221 VPValue *IV = VectorLoop->getCanonicalIV();
7222 // When vectorizing the epilogue loop, the canonical induction needs to start
7223 // at the resume value from the main vector loop. Find the resume value
7224 // created during execution of the main VPlan. Add this resume value as an
7225 // offset to the canonical IV of the epilogue loop.
7226 VPValue *VPV = GetResumeValue(Markers.VectorTC, "vec.epilog.resume.val");
7227 assert(all_of(IV->users(),
7228 [](const VPUser *U) {
7229 if (isa<VPScalarIVStepsRecipe, VPDerivedIVRecipe>(U))
7230 return true;
7231 auto *VPI = dyn_cast<VPInstruction>(U);
7232 if (!VPI)
7233 return false;
7234 // Only users linear in the canonical IV (e.g. scaling it by
7235 // a stride) stay correct when offsetting it below; others,
7236 // like header masks, may need an epilogue-relative IV.
7237 // The active-lane-masks of a tail-folded epilogue compare
7238 // against the full trip count, so they expect the offset
7239 // IV, like the active-lane-mask phi's entry value.
7240 unsigned Opc = VPI->getOpcode();
7241 return Instruction::isCast(Opc) ||
7242 is_contained({Instruction::Add, Instruction::Mul,
7243 Instruction::Shl},
7244 Opc) ||
7245 Opc == VPInstruction::WideActiveLaneMask;
7246 }) &&
7247 "the canonical IV should only be used by its increment, casts, "
7248 "stride scaling, ScalarIVSteps or DerivedIV when resetting the start "
7249 "value");
7250 VPBuilder Builder(Header->getFirstNonPhi());
7251 VPInstruction *Add = Builder.createAdd(LHS: IV, RHS: VPV);
7252 // Replace all users of the canonical IV and its increment with the offset
7253 // version, except for the Add itself and the canonical IV increment.
7254 auto *Increment = vputils::findCanonicalIVIncrement(Plan);
7255 assert(Increment && "Must have a canonical IV increment at this point");
7256 IV->replaceUsesWithIf(
7257 New: Add, ShouldReplace: [Add, Increment](VPUser *U) { return U != Add && U != Increment; });
7258 VPInstruction *OffsetIVInc =
7259 VPBuilder::getToInsertAfter(R: Increment).createAdd(LHS: Increment, RHS: VPV);
7260 Increment->replaceAllUsesWith(New: OffsetIVInc);
7261 OffsetIVInc->setOperand(I: 0, New: Increment);
7262
7263 // Resume values must be created in the vector preheader.
7264 VPBuilder PHBuilder(VectorPH->getFirstNonPhi());
7265
7266 // Ensure that the start values for all header phi recipes are updated before
7267 // vectorizing the epilogue loop.
7268 for (VPRecipeBase &R : Header->phis()) {
7269 VPValue *ResumeVPV = nullptr;
7270 // TODO: Move setting of resume values to prepareToExecute.
7271 if (auto *ReductionPhi = dyn_cast<VPReductionPHIRecipe>(Val: &R)) {
7272 // Find the reduction result by searching users of the phi or its backedge
7273 // value.
7274 auto IsReductionResult = [](VPRecipeBase *R) {
7275 auto *VPI = dyn_cast<VPInstruction>(Val: R);
7276 return VPI && VPI->getOpcode() == VPInstruction::ComputeReductionResult;
7277 };
7278 auto *RdxResult = cast<VPInstruction>(
7279 Val: vputils::findRecipe(Start: ReductionPhi->getBackedgeValue(), Pred: IsReductionResult));
7280 assert(RdxResult && "expected to find reduction result");
7281
7282 VPInstruction *ResumeForEpi = IRPhiToResumeForEpi.at(
7283 Val: cast<PHINode>(Val: ReductionPhi->getUnderlyingInstr()));
7284
7285 // Check for FindIV pattern by looking for icmp user of RdxResult.
7286 // The pattern is: select(icmp ne RdxResult, Sentinel), RdxResult, Start
7287 using namespace VPlanPatternMatch;
7288 VPValue *SentinelVPV = nullptr;
7289 bool IsFindIV = any_of(Range: RdxResult->users(), P: [&](VPUser *U) {
7290 return match(U, P: VPlanPatternMatch::m_SpecificICmp(
7291 MatchPred: ICmpInst::ICMP_NE, Op0: m_Specific(VPV: RdxResult),
7292 Op1: m_VPValue(V&: SentinelVPV)));
7293 });
7294
7295 RecurKind RK = ReductionPhi->getRecurrenceKind();
7296 ResumeVPV = GetResumeValue(ResumeForEpi, "bc.merge.rdx");
7297 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK) || IsFindIV) {
7298 VPValue *BypassOp = ResumeForEpi->getOperand(N: 1);
7299 assert((isa<VPIRValue>(BypassOp) ||
7300 VPlanPatternMatch::match(
7301 BypassOp,
7302 m_VPInstruction<Instruction::Freeze>(m_VPValue()))) &&
7303 "expected live-in or Freeze");
7304 VPValue *StartV = Plan.getOrAddLiveIn(V: BypassOp->getUnderlyingValue());
7305 if (RecurrenceDescriptor::isAnyOfRecurrenceKind(Kind: RK)) {
7306 // VPReductionPHIRecipes for AnyOf reductions expect a boolean as
7307 // start value; compare the final value from the main vector loop
7308 // to the start value.
7309 ResumeVPV = PHBuilder.createICmp(Pred: CmpInst::ICMP_NE, A: ResumeVPV, B: StartV);
7310 } else {
7311 assert(isa<VPIRValue>(SentinelVPV) &&
7312 "sentinel must be a live-in to be used in the preheader");
7313 // Use the start value frozen for the main plan, if any, when
7314 // computing the reduction result.
7315 if (match(V: BypassOp, P: m_Freeze(Op0: m_VPValue())))
7316 for (VPUser *U : RdxResult->users()) {
7317 auto *VPI = dyn_cast<VPInstruction>(Val: U);
7318 if (VPI && matchFindIVResult(VPI, ReducedIV: m_VPValue(), Start: m_VPValue()))
7319 VPI->setOperand(I: 2, New: StartV);
7320 }
7321
7322 // Adjust resume: select(icmp eq ResumeVPV, StartV), Sentinel,
7323 // ResumeVPV
7324 VPValue *Cmp =
7325 PHBuilder.createICmp(Pred: CmpInst::ICMP_EQ, A: ResumeVPV, B: StartV);
7326 ResumeVPV = PHBuilder.createSelect(Cond: Cmp, TrueVal: SentinelVPV, FalseVal: ResumeVPV);
7327 }
7328 // TODO: materializeBroadcasts does not cover values in the vector
7329 // preheader.
7330 ReductionPhi->setStartValue(
7331 PHBuilder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: ResumeVPV));
7332 continue;
7333 }
7334 if (auto *VPI = dyn_cast<VPInstruction>(Val: ReductionPhi->getStartValue())) {
7335 assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
7336 "unexpected start value");
7337 // Partial sub-reductions always start at 0 and account for the
7338 // reduction start value in a final subtraction. Update it to use the
7339 // resume value from the main vector loop.
7340 if (ReductionPhi->getVFScaleFactor() > 1 &&
7341 RecurrenceDescriptor::isSubRecurrenceKind(Kind: RK)) {
7342 auto *Sub = cast<VPInstruction>(Val: RdxResult->getSingleUser());
7343 assert((Sub->getOpcode() == Instruction::Sub ||
7344 Sub->getOpcode() == Instruction::FSub) &&
7345 "Unexpected opcode");
7346 assert(isa<VPIRValue>(Sub->getOperand(0)) &&
7347 "Expected operand to match the original start value of the "
7348 "reduction");
7349 // For integer sub-reductions, verify start value is zero.
7350 // For FP sub-reductions, verify start value is negative zero.
7351 [[maybe_unused]] auto StartValueIsIdentity = [&] {
7352 Value *IdentityValue =
7353 getRecurrenceIdentity(K: RK, Tp: ResumeVPV->getScalarType(),
7354 FMF: ReductionPhi->getFastMathFlagsOrNone());
7355 auto *StartValue = dyn_cast<VPIRValue>(Val: VPI->getOperand(N: 0));
7356 return StartValue && StartValue->getValue() == IdentityValue;
7357 };
7358 assert(StartValueIsIdentity() &&
7359 "Expected start value for partial sub-reduction to be zero "
7360 "(or negative zero)");
7361
7362 Sub->setOperand(I: 0, New: ResumeVPV);
7363 } else {
7364 // The start vector may be shared by multiple reductions, so create
7365 // a new one using the resume.
7366 auto *NewStart = cast<VPInstruction>(Val: VPI->clone());
7367 NewStart->setOperand(I: 0, New: ResumeVPV);
7368 NewStart->insertBefore(InsertPos: VPI);
7369 ReductionPhi->setStartValue(NewStart);
7370 }
7371 continue;
7372 }
7373 } else if (isa<VPActiveLaneMaskPHIRecipe>(Val: &R)) {
7374 // The active-lane-mask phi's entry value was computed assuming the
7375 // epilogue vector loop starts at 0. Rebuild it using the resume value
7376 // instead, so the mask reflects how many elements the main vector loop
7377 // already processed.
7378 VPBuilder EntryBuilder(Plan.getVectorPreheader());
7379 auto *EntryALM = EntryBuilder.createNaryOp(
7380 Opcode: VPInstruction::ActiveLaneMask, Operands: {VPV, Plan.getTripCount()},
7381 DL: R.getDebugLoc(), Name: "active.lane.mask.entry");
7382 cast<VPHeaderPHIRecipe>(Val: &R)->setStartValue(EntryALM);
7383 continue;
7384 } else {
7385 // Retrieve the induction resume value via ResumeForEpilogue.
7386 PHINode *IndPhi =
7387 isa<VPWidenInductionRecipe>(Val: &R)
7388 ? cast<VPWidenInductionRecipe>(Val: &R)->getPHINode()
7389 : cast<VPConditionalInductionPHIRecipe>(Val: &R)->getPHINode();
7390 ResumeVPV =
7391 GetResumeValue(IRPhiToResumeForEpi.at(Val: IndPhi), "bc.resume.val");
7392 }
7393 assert(ResumeVPV && "Must have a resume value");
7394 cast<VPHeaderPHIRecipe>(Val: &R)->setStartValue(ResumeVPV);
7395 }
7396
7397 // Re-use the SCEVs expanded for the main loop instead of expanding them
7398 // again, which also lets simplifyLiveInsWithSCEV fold their users.
7399 // TODO: Remove once the epilogue plan uses the main plan's VPValues directly.
7400 for (VPExpandSCEVRecipe &ExpandR : make_early_inc_range(
7401 Range: make_isa_range<VPExpandSCEVRecipe>(Range&: *Plan.getEntry()))) {
7402 assert(ExpandedSCEVs.contains(ExpandR.getSCEV()) &&
7403 "Epilogue plan needs a SCEV not expanded for the main loop");
7404 VPValue *ExpandedVal =
7405 Plan.getOrAddLiveIn(V: ExpandedSCEVs.lookup(Val: ExpandR.getSCEV()));
7406 ExpandR.replaceAllUsesWith(New: ExpandedVal);
7407 if (Plan.getTripCount() == &ExpandR)
7408 Plan.resetTripCount(NewTripCount: ExpandedVal);
7409 ExpandR.eraseFromParent();
7410 }
7411
7412 auto VScale = Config.getVScaleForTuning();
7413 unsigned MainLoopStep = estimateElementCount(VF: MainLoopVF * MainLoopUF, VScale);
7414 unsigned EpilogueLoopStep = estimateElementCount(VF: EpilogueVF, VScale);
7415 RUN_VPLAN_PASS(VPlanTransforms::addMinimumVectorEpilogueIterationCheck, Plan,
7416 Plan.getOrAddLiveIn(Markers.VectorTC->getUnderlyingValue()),
7417 Plan.requiresScalarEpilogue(), EpilogueVF, MainLoopStep,
7418 EpilogueLoopStep, SE);
7419}
7420
7421bool LoopVectorizePass::processLoop(Loop *L) {
7422 const VectorizeOptions &Opts = getVectorizeOptions(L: *L);
7423 assert((Opts.enable_vplan_native_path || L->isInnermost()) &&
7424 "VPlan-native path is not enabled. Only process inner loops.");
7425
7426 LLVM_DEBUG(dbgs() << "\nLV: Checking a loop in '"
7427 << L->getHeader()->getParent()->getName() << "' from "
7428 << L->getLocStr() << "\n");
7429
7430 LoopVectorizeHints Hints(
7431 Opts, L, InterleaveOnlyWhenForced || !Opts.interleave_loops, *ORE, TTI);
7432
7433 LLVM_DEBUG(
7434 dbgs() << "LV: Loop hints:"
7435 << " force="
7436 << (Hints.getForce() == LoopVectorizeHints::FK_Disabled
7437 ? "disabled"
7438 : (Hints.getForce() == LoopVectorizeHints::FK_Enabled
7439 ? "enabled"
7440 : "?"))
7441 << " width=" << Hints.getWidth()
7442 << " interleave=" << Hints.getInterleave() << "\n");
7443
7444 // Function containing loop
7445 Function *F = L->getHeader()->getParent();
7446
7447 // Looking at the diagnostic output is the only way to determine if a loop
7448 // was vectorized (other than looking at the IR or machine code), so it
7449 // is important to generate an optimization remark for each loop. Most of
7450 // these messages are generated as OptimizationRemarkAnalysis. Remarks
7451 // generated as OptimizationRemark and OptimizationRemarkMissed are
7452 // less verbose reporting vectorized loops and unvectorized loops that may
7453 // benefit from vectorization, respectively.
7454
7455 if (!Hints.allowVectorization(L, VectorizeOnlyWhenForced: VectorizeOnlyWhenForced ||
7456 !Opts.vectorize_loops)) {
7457 LLVM_DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
7458 return false;
7459 }
7460
7461 PredicatedScalarEvolution PSE(*SE, *L);
7462
7463 // Query this against the original loop and save it here because the profile
7464 // of the original loop header may change as the transformation happens.
7465 bool OptForSize = llvm::shouldOptimizeForSize(
7466 BB: L->getHeader(), PSI,
7467 BFI: PSI && PSI->hasProfileSummary() ? &GetBFI() : nullptr,
7468 QueryType: PGSOQueryType::IRPass,
7469 UseProfilingInfo: Hints.getForce() != LoopVectorizeHints::FK_Enabled);
7470
7471 // Check if it is legal to vectorize the loop.
7472 LoopVectorizationRequirements Requirements;
7473 LoopVectorizationLegality LVL(Opts, L, PSE, DT, TTI, TLI, F, *LAIs, LI, ORE,
7474 &Requirements, &Hints, DB, AC,
7475 /*AllowRuntimeSCEVChecks=*/!OptForSize, AA);
7476 if (!LVL.canVectorize(UseVPlanNativePath: Opts.enable_vplan_native_path)) {
7477 LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
7478 Hints.emitRemarkWithHints();
7479 return false;
7480 }
7481
7482 bool IsInnerLoop = L->isInnermost();
7483
7484 // Outer loops require a computable trip count.
7485 if (!IsInnerLoop && isa<SCEVCouldNotCompute>(Val: PSE.getBackedgeTakenCount())) {
7486 LLVM_DEBUG(dbgs() << "LV: cannot compute the outer-loop trip count\n");
7487 return false;
7488 }
7489
7490 if (LVL.hasUncountableEarlyExit()) {
7491 if (!Opts.enable_early_exit_vectorization) {
7492 reportVectorizationFailure(DebugMsg: "Auto-vectorization of loops with uncountable "
7493 "early exit is not enabled",
7494 ORETag: "UncountableEarlyExitLoopsDisabled", ORE, TheLoop: L);
7495 return false;
7496 }
7497 if (LVL.hasUncountableExitWithSideEffects() &&
7498 !Opts.enable_early_exit_vectorization_with_side_effects) {
7499 reportVectorizationFailure(DebugMsg: "Auto-vectorization of loops with uncountable "
7500 "early exit and side effects is not enabled",
7501 ORETag: "UncountableEarlyExitSideEffectLoopsDisabled",
7502 ORE, TheLoop: L);
7503 return false;
7504 }
7505 }
7506
7507 InterleavedAccessInfo IAI(PSE, L, DT, LI, LVL.getLAI(), OptForSize);
7508 // If an override option has been passed in for interleaved accesses, use it.
7509 bool UseInterleaved =
7510 IsInnerLoop && valueOr(X: Opts.enable_interleaved_mem_accesses,
7511 Default: TTI->enableInterleavedAccessVectorization());
7512
7513 // Analyze interleaved memory accesses.
7514 if (UseInterleaved) {
7515 IAI.analyzeInterleaving(EnableMaskedInterleavedGroup: useMaskedInterleavedAccesses(Opts, TTI: *TTI));
7516 // If the loop exits from a block other than the latch, the scalar
7517 // iteration reserved for the exit may leave the loop before reaching the
7518 // grouped accesses.
7519 if (L->getExitingBlock() != L->getLoopLatch() &&
7520 !LVL.hasUncountableEarlyExit())
7521 IAI.invalidateGroupsRequiringScalarEpilogue();
7522 }
7523
7524 if (LVL.hasUncountableEarlyExit()) {
7525 BasicBlock *LoopLatch = L->getLoopLatch();
7526 if (IAI.requiresScalarEpilogue() ||
7527 any_of(Range: LVL.getCountableExitingBlocks(), P: not_equal_to(Arg&: LoopLatch))) {
7528 reportVectorizationFailure(DebugMsg: "Auto-vectorization of early exit loops "
7529 "requiring a scalar epilogue is unsupported",
7530 ORETag: "UncountableEarlyExitUnsupported", ORE, TheLoop: L);
7531 return false;
7532 }
7533 }
7534
7535 EpilogueLowering SEL =
7536 getEpilogueLowering(Opts, Hints, OptForSize, TTI, TLI, LVL, IAI: &IAI);
7537
7538 // Check the loop for a trip count threshold: vectorize loops with a tiny trip
7539 // count by optimizing for size, to minimize overheads.
7540 auto ExpectedTC = getSmallBestKnownTC(Opts, PSE, L);
7541 if (ExpectedTC && ExpectedTC->isFixed() &&
7542 ExpectedTC->getFixedValue() < Opts.vectorizer_min_trip_count) {
7543 LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
7544 << "This loop is worth vectorizing only if no scalar "
7545 << "epilogue is required.");
7546 if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
7547 LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
7548 else {
7549 LLVM_DEBUG(dbgs() << "\n");
7550 // For loops with very small trip counts, require that no scalar epilogue
7551 // is needed. Tail-folded loops are still allowed since they avoid
7552 // epilogues, unless the trip count is below the target's minimum for
7553 // tail-folding. In that case the tail cannot be folded, so the loop
7554 // should be treated like any other low trip count loop.
7555 // TODO: Remove the check for CM_EpilogueNotAllowedOptSize here. The
7556 // EpilogueLowering enum currently conflates several pieces of
7557 // information:
7558 // * Whether a scalar epilogue is permitted
7559 // * Whether tail folding is preferred or required
7560 // * Whether optimizing for code size
7561 // Overwriting it here can drop OptForSize, breaking the
7562 // runtimeChecksRequired() bail-out in computeMaxVF. Split
7563 // EpilogueLowering into separate fields instead.
7564 if (SEL != CM_EpilogueNotAllowedOptSize &&
7565 (SEL != CM_EpilogueNotNeededFoldTail ||
7566 ExpectedTC->getFixedValue() <=
7567 TTI->getMinTripCountTailFoldingThreshold()))
7568 SEL = CM_EpilogueNotAllowedLowTripLoop;
7569 }
7570 }
7571
7572 // Check the function attributes to see if implicit floats or vectors are
7573 // allowed.
7574 if (F->hasFnAttribute(Kind: Attribute::NoImplicitFloat)) {
7575 reportVectorizationFailure(
7576 DebugMsg: "Can't vectorize when the NoImplicitFloat attribute is used",
7577 OREMsg: "loop not vectorized due to NoImplicitFloat attribute",
7578 ORETag: "NoImplicitFloat", ORE, TheLoop: L);
7579 Hints.emitRemarkWithHints();
7580 return false;
7581 }
7582
7583 // Check if the target supports potentially unsafe FP vectorization.
7584 // FIXME: Add a check for the type of safety issue (denormal, signaling)
7585 // for the target we're vectorizing for, to make sure none of the
7586 // additional fp-math flags can help.
7587 if (Hints.isPotentiallyUnsafe() &&
7588 TTI->isFPVectorizationPotentiallyUnsafe()) {
7589 reportVectorizationFailure(
7590 DebugMsg: "Potentially unsafe FP op prevents vectorization",
7591 OREMsg: "loop not vectorized due to unsafe FP support.", ORETag: "UnsafeFP", ORE, TheLoop: L);
7592 Hints.emitRemarkWithHints();
7593 return false;
7594 }
7595
7596 // If the flag is set, use that instead and override the TTI behaviour.
7597 bool AllowOrderedReductions =
7598 valueOr(X: Opts.force_ordered_reductions, Default: TTI->enableOrderedReductions());
7599 if (!LVL.canVectorizeFPMath(EnableStrictReductions: AllowOrderedReductions)) {
7600 ORE->emit(RemarkBuilder: [&]() {
7601 auto *ExactFPMathInst = Requirements.getExactFPInst();
7602 return OptimizationRemarkAnalysisFPCommute(DEBUG_TYPE, "CantReorderFPOps",
7603 ExactFPMathInst->getDebugLoc(),
7604 ExactFPMathInst->getParent())
7605 << "loop not vectorized: cannot prove it is safe to reorder "
7606 "floating-point operations";
7607 });
7608 LLVM_DEBUG(dbgs() << "LV: loop not vectorized: cannot prove it is safe to "
7609 "reorder floating-point operations\n");
7610 Hints.emitRemarkWithHints();
7611 return false;
7612 }
7613
7614 // Use the cost model.
7615 VFSelectionContext Config(Opts, *TTI, &LVL, L, *F, PSE, DB, ORE, &Hints,
7616 OptForSize);
7617 // Use the planner for vectorization.
7618 LoopVectorizationPlanner LVP(
7619 L, LI, DT, TLI, *TTI, &LVL,
7620 std::make_unique<LoopVectorizationCostModel>(
7621 args&: SEL, args&: L, args&: PSE, args&: LI, args: &LVL, args&: *TTI, args&: TLI, args&: AC, args&: ORE, args&: GetBFI, args&: F, args&: IAI, args&: Config),
7622 Config, PSE, ORE, GetBPI);
7623
7624 // Get user vectorization factor and interleave count.
7625 ElementCount UserVF = Hints.getWidth();
7626 unsigned UserIC = Hints.getInterleave();
7627 // Outer loops don't have LoopAccessInfo, so skip the safety check and reset
7628 // UserIC (interleaving is not supported for outer loops).
7629 if (!IsInnerLoop)
7630 UserIC = 0;
7631 else if (UserIC > 1 && !LVL.isSafeForAnyVectorWidth())
7632 UserIC = 1;
7633
7634 // Plan how to best vectorize.
7635 LVP.plan(UserVF, UserIC);
7636 auto [VF, BestPlanPtr] = LVP.computeBestVF();
7637 unsigned IC = 1;
7638
7639 // For VPlan build stress testing of outer loops, bail after plan
7640 // construction.
7641 if (!IsInnerLoop && Opts.vplan_build_outerloop_stress_test)
7642 return false;
7643
7644 if (IsInnerLoop && ORE->allowExtraAnalysis(LV_NAME))
7645 LVP.emitInvalidCostRemarks(ORE);
7646
7647 assert((IsInnerLoop || !LVP.getCostModel().maskPartialAliasing()) &&
7648 "Did not expect to alias-mask outer loop");
7649
7650 GeneratedRTChecks Checks(Opts, PSE, DT, LI, TTI, Config.CostKind,
7651 LVP.getCostModel().maskPartialAliasing());
7652 if (LVP.hasPlanWithVF(VF: VF.Width)) {
7653 // Select the interleave count. Outer loops are not interleaved.
7654 if (IsInnerLoop)
7655 IC = LVP.selectInterleaveCount(Plan&: *BestPlanPtr, VF: VF.Width, LoopCost: VF.Cost);
7656
7657 unsigned SelectedIC = UserIC > 0 ? UserIC : IC;
7658 // Optimistically generate runtime checks if they are needed. Drop them if
7659 // they turn out to not be profitable.
7660 if (VF.Width.isVector() || SelectedIC > 1) {
7661 Checks.create(L, LAI: LVL.getLAI(), UnionPred: PSE.getPredicate(), VF: VF.Width, IC: SelectedIC,
7662 ORE&: *ORE);
7663
7664 // Bail out early if either the SCEV or memory runtime checks are known to
7665 // fail. In that case, the vector loop would never execute.
7666 using namespace llvm::PatternMatch;
7667 if ((Checks.getSCEVChecks().first &&
7668 match(V: Checks.getSCEVChecks().first, P: m_One())) ||
7669 (Checks.getMemRuntimeChecks().first &&
7670 match(V: Checks.getMemRuntimeChecks().first, P: m_One()))) {
7671 reportVectorizationFailure(
7672 DebugMsg: "runtime checks are known to fail, so we will never enter the "
7673 "vector loop",
7674 ORETag: "RuntimeChecksNeverEnterVectorLoop", ORE, TheLoop: L);
7675 return false;
7676 }
7677 }
7678
7679 // Check if it is profitable to vectorize with runtime checks.
7680 bool ForceVectorization =
7681 Hints.getForce() == LoopVectorizeHints::FK_Enabled;
7682 assert((IsInnerLoop || ForceVectorization) &&
7683 "Outer loops must be forced to vectorize");
7684 VPCostContext CostCtx(*TLI, *BestPlanPtr, LVP.getCostModel(), Config,
7685 /*ReusePrintingSlotTracker=*/true);
7686 if (!ForceVectorization &&
7687 !isOutsideLoopWorkProfitable(Checks, VF, L, PSE, CostCtx, Plan&: *BestPlanPtr,
7688 SEL, VScale: Config.getVScaleForTuning())) {
7689 ORE->emit(RemarkBuilder: [&]() {
7690 return OptimizationRemarkAnalysisAliasing(
7691 DEBUG_TYPE, "CantReorderMemOps", L->getStartLoc(),
7692 L->getHeader())
7693 << "loop not vectorized: cannot prove it is safe to reorder "
7694 "memory operations";
7695 });
7696 LLVM_DEBUG(dbgs() << "LV: Too many memory checks needed.\n");
7697 Hints.emitRemarkWithHints();
7698 return false;
7699 }
7700 }
7701
7702 // Identify the diagnostic messages that should be produced.
7703 std::pair<StringRef, std::string> VecDiagMsg, IntDiagMsg;
7704 bool VectorizeLoop = true, InterleaveLoop = true;
7705 if (VF.Width.isScalar()) {
7706 if (LVP.hasVectorPlan()) {
7707 LLVM_DEBUG(
7708 dbgs() << "LV: Vectorization is possible but not beneficial.\n");
7709 VecDiagMsg = {
7710 "VectorizationNotBeneficial",
7711 "the cost-model indicates that vectorization is not beneficial"};
7712 } else {
7713 LLVM_DEBUG(dbgs() << "LV: Vectorization is not possible. Failed to "
7714 "create any vector VPlans.\n");
7715 VecDiagMsg = {"VectorizationNotPossible",
7716 "vectorization is not possible"};
7717 }
7718 VectorizeLoop = false;
7719 }
7720
7721 if (UserIC == 1 && Hints.getInterleave() > 1) {
7722 assert(!LVL.isSafeForAnyVectorWidth() &&
7723 "UserIC should only be ignored due to unsafe dependencies");
7724 LLVM_DEBUG(dbgs() << "LV: Ignoring user-specified interleave count.\n");
7725 IntDiagMsg = {"InterleavingUnsafe",
7726 "Ignoring user-specified interleave count due to possibly "
7727 "unsafe dependencies in the loop."};
7728 InterleaveLoop = false;
7729 } else if (!LVP.hasPlanWithVF(VF: VF.Width) && UserIC > 1) {
7730 // Tell the user interleaving was avoided up-front, despite being explicitly
7731 // requested.
7732 LLVM_DEBUG(dbgs() << "LV: Ignoring UserIC, because vectorization and "
7733 "interleaving should be avoided up front\n");
7734 IntDiagMsg = {"InterleavingAvoided",
7735 "Ignoring UserIC, because interleaving was avoided up front"};
7736 InterleaveLoop = false;
7737 } else if (IC == 1 && UserIC <= 1) {
7738 // Tell the user interleaving is not beneficial.
7739 if (LVP.hasAPlan()) {
7740 LLVM_DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
7741 IntDiagMsg = {
7742 "InterleavingNotBeneficial",
7743 "the cost-model indicates that interleaving is not beneficial"};
7744 if (UserIC == 1) {
7745 IntDiagMsg.first = "InterleavingNotBeneficialAndDisabled";
7746 IntDiagMsg.second +=
7747 " and is explicitly disabled or interleave count is set to 1";
7748 }
7749 } else {
7750 LLVM_DEBUG(dbgs() << "LV: Interleaving is not possible. Failed to create"
7751 << " any vplans\n");
7752 IntDiagMsg = {"InterleavingNotPossible", "interleaving is not possible"};
7753 }
7754 InterleaveLoop = false;
7755 } else if (IC > 1 && UserIC == 1) {
7756 // Tell the user interleaving is beneficial, but it explicitly disabled.
7757 LLVM_DEBUG(dbgs() << "LV: Interleaving is beneficial but is explicitly "
7758 "disabled.\n");
7759 IntDiagMsg = {"InterleavingBeneficialButDisabled",
7760 "the cost-model indicates that interleaving is beneficial "
7761 "but is explicitly disabled or interleave count is set to 1"};
7762 InterleaveLoop = false;
7763 }
7764
7765 // If there is a histogram in the loop, do not just interleave without
7766 // vectorizing. The order of operations will be incorrect without the
7767 // histogram intrinsics, which are only used for recipes with VF > 1.
7768 if (!VectorizeLoop && InterleaveLoop && LVL.hasHistograms()) {
7769 LLVM_DEBUG(dbgs() << "LV: Not interleaving without vectorization due "
7770 << "to histogram operations.\n");
7771 IntDiagMsg = {
7772 "HistogramPreventsScalarInterleaving",
7773 "Unable to interleave without vectorization due to constraints on "
7774 "the order of histogram operations"};
7775 InterleaveLoop = false;
7776 }
7777
7778 // Override IC if user provided an interleave count.
7779 IC = UserIC > 0 ? UserIC : IC;
7780
7781 if (LVP.getCostModel().maskPartialAliasing()) {
7782 LLVM_DEBUG(
7783 dbgs()
7784 << "LV: Not interleaving due to partial aliasing vectorization.\n");
7785 IntDiagMsg = {
7786 "PartialAliasingVectorization",
7787 "Unable to interleave due to partial aliasing vectorization."};
7788 InterleaveLoop = false;
7789 IC = 1;
7790 }
7791
7792 // FIXME: Enable interleaving for EE-with-side-effects.
7793 if (InterleaveLoop && LVL.hasUncountableExitWithSideEffects()) {
7794 LLVM_DEBUG(dbgs() << "LV: Not interleaving due to EE with side effects.\n");
7795 IntDiagMsg = {"EEWithSideEffectsPreventsInterleaving",
7796 "Unable to interleave due to early exit with side effects."};
7797 InterleaveLoop = false;
7798 IC = 1;
7799 }
7800
7801 if (InterleaveLoop && LVL.hasConditionalInductions()) {
7802 LLVM_DEBUG(
7803 dbgs() << "LV: Not interleaving due to conditional inductions.\n");
7804 IntDiagMsg = {"ConditionalInductionsPreventInterleaving",
7805 "Unable to interleave due to conditional inductions."};
7806 InterleaveLoop = false;
7807 IC = 1;
7808 }
7809
7810 // Emit diagnostic messages, if any.
7811 if (!VectorizeLoop && !InterleaveLoop) {
7812 // Do not vectorize or interleaving the loop.
7813 ORE->emit(RemarkBuilder: [&]() {
7814 return OptimizationRemarkMissed(LV_NAME, VecDiagMsg.first,
7815 L->getStartLoc(), L->getHeader())
7816 << VecDiagMsg.second;
7817 });
7818 ORE->emit(RemarkBuilder: [&]() {
7819 return OptimizationRemarkMissed(LV_NAME, IntDiagMsg.first,
7820 L->getStartLoc(), L->getHeader())
7821 << IntDiagMsg.second;
7822 });
7823 return false;
7824 }
7825
7826 if (!VectorizeLoop && InterleaveLoop) {
7827 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
7828 ORE->emit(RemarkBuilder: [&]() {
7829 return OptimizationRemarkAnalysis(LV_NAME, VecDiagMsg.first,
7830 L->getStartLoc(), L->getHeader())
7831 << VecDiagMsg.second;
7832 });
7833 } else if (VectorizeLoop && !InterleaveLoop) {
7834 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
7835 << ") in " << L->getLocStr() << '\n');
7836 ORE->emit(RemarkBuilder: [&]() {
7837 return OptimizationRemarkAnalysis(LV_NAME, IntDiagMsg.first,
7838 L->getStartLoc(), L->getHeader())
7839 << IntDiagMsg.second;
7840 });
7841 } else if (VectorizeLoop && InterleaveLoop) {
7842 LLVM_DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width
7843 << ") in " << L->getLocStr() << '\n');
7844 LLVM_DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
7845 }
7846
7847 // Report the vectorization decision.
7848 if (VF.Width.isScalar()) {
7849 using namespace ore;
7850 assert(IC > 1);
7851 ORE->emit(RemarkBuilder: [&]() {
7852 return OptimizationRemark(LV_NAME, "Interleaved", L->getStartLoc(),
7853 L->getHeader())
7854 << "interleaved loop (interleaved count: "
7855 << NV("InterleaveCount", IC) << ")";
7856 });
7857 } else {
7858 // Report the vectorization decision.
7859 reportVectorization(ORE, TheLoop: L, VFWidth: VF.Width, IC);
7860 }
7861 if (ORE->allowExtraAnalysis(LV_NAME))
7862 checkMixedPrecision(L, ORE);
7863
7864 // If we decided that it is *legal* to interleave or vectorize the loop, then
7865 // do it.
7866
7867 // Whether a scalar epilogue may be created is decided by the epilogue
7868 // lowering policy.
7869 // TODO: Also move check to be based on VPlan.
7870 bool ScalarEpilogueAllowed = LVP.getCostModel().isEpilogueAllowed();
7871
7872 // Destroy the cost model before executing any plan, so that code generation
7873 // cannot rely on cost-modeling decisions.
7874 LVP.clearCostModel();
7875
7876 VPlan &BestPlan = *BestPlanPtr;
7877 // Consider vectorizing the epilogue too if it's profitable.
7878 std::unique_ptr<VPlan> EpiPlan =
7879 LVP.selectBestEpiloguePlan(MainPlan&: BestPlan, MainLoopVF: VF.Width, IC, ScalarEpilogueAllowed);
7880 bool HasBranchWeights =
7881 hasBranchWeightMD(I: *L->getLoopLatch()->getTerminator());
7882 if (EpiPlan) {
7883 VPlan &BestEpiPlan = *EpiPlan;
7884 VPlan &BestMainPlan = BestPlan;
7885 ElementCount EpilogueVF = BestEpiPlan.getSingleVF();
7886
7887 // The first pass vectorizes the main loop and creates a scalar epilogue
7888 // to be vectorized by executing the plan (potentially with a different
7889 // factor) again shortly afterwards.
7890 BestEpiPlan.getMiddleBlock()->setName("vec.epilog.middle.block");
7891 BestEpiPlan.getVectorPreheader()->setName("vec.epilog.ph");
7892 MainPlanResumeMarkers Markers = preparePlanForMainVectorLoop(MainPlan&: BestMainPlan);
7893
7894 // Add minimum iteration check for the epilogue plan, followed by runtime
7895 // checks for the main plan.
7896 LVP.addMinimumIterationCheck(Plan&: BestMainPlan, VF: EpilogueVF, /*UF=*/1,
7897 MinProfitableTripCount: ElementCount::getFixed(MinVal: 0),
7898 TailFoldedEpilogue: BestEpiPlan.hasTailFolded());
7899 LVP.attachRuntimeChecks(Plan&: BestMainPlan, RTChecks&: Checks, VF: VF.Width, UF: IC,
7900 HasBranchWeights);
7901 RUN_VPLAN_PASS(VPlanTransforms::addIterationCountCheckBlock, BestMainPlan,
7902 VF.Width, IC, BestMainPlan.requiresScalarEpilogue(), L,
7903 HasBranchWeights ? MinItersBypassWeights : nullptr,
7904 L->getLoopPredecessor()->getTerminator()->getDebugLoc(),
7905 PSE);
7906
7907 LLVM_DEBUG({
7908 dbgs() << "Create Skeleton for epilogue vectorized loop (first pass)\n"
7909 << "Main Loop VF:" << VF.Width << ", Main Loop UF:" << IC
7910 << ", Epilogue Loop VF:" << EpilogueVF << ", Epilogue Loop UF:1\n";
7911 });
7912 // Keep track of the iteration count check of the main vector loop.
7913 VPBasicBlock *MainBlocks[] = {cast<VPBasicBlock>(
7914 Val: BestMainPlan.getVectorPreheader()->getSinglePredecessor())};
7915 auto ExpandedSCEVs = LVP.executePlan(
7916 BestVF: VF.Width, BestUF: IC, BestVPlan&: BestMainPlan, HasRuntimeChecks: Checks.hasChecks(),
7917 EpilogueVecKind: LoopVectorizationPlanner::EpilogueVectorizationKind::MainLoop,
7918 TrackedBlocks: MainBlocks);
7919 VPBasicBlock *MainCheck = MainBlocks[0];
7920 ++LoopsVectorized;
7921 DEBUG_WITH_TYPE(VerboseDebug, {
7922 dbgs() << "intermediate fn:\n" << *L->getHeader()->getParent() << "\n";
7923 });
7924
7925 BasicBlock *EntryBB =
7926 cast<VPIRBasicBlock>(Val: BestMainPlan.getEntry())->getIRBasicBlock();
7927 EntryBB->setName("iter.check");
7928 cast<VPIRBasicBlock>(Val: BestMainPlan.getScalarPreheader())
7929 ->getIRBasicBlock()
7930 ->setName("vec.epilog.iter.check");
7931
7932 // Second pass vectorizes the epilogue and adjusts the control flow
7933 // edges from the first pass.
7934 preparePlanForEpilogueVectorLoop(MainPlan&: BestMainPlan, Plan&: BestEpiPlan, ExpandedSCEVs,
7935 MainLoopVF: VF.Width, MainLoopUF: IC, EpilogueVF, Config,
7936 SE&: *PSE.getSE(), Markers, MainCheck);
7937 RUN_VPLAN_PASS(VPlanTransforms::simplifyLiveInsWithSCEV, BestEpiPlan, PSE);
7938 LLVM_DEBUG({
7939 dbgs() << "Create Skeleton for epilogue vectorized loop (second pass)\n"
7940 << "Epilogue Loop VF:" << EpilogueVF << ", Epilogue Loop UF:1\n";
7941 });
7942 LVP.executePlan(
7943 BestVF: EpilogueVF, /*BestUF=*/1, BestVPlan&: BestEpiPlan, HasRuntimeChecks: Checks.hasChecks(),
7944 EpilogueVecKind: LoopVectorizationPlanner::EpilogueVectorizationKind::Epilogue);
7945 DEBUG_WITH_TYPE(VerboseDebug, {
7946 dbgs() << "final fn:\n" << *L->getHeader()->getParent() << "\n";
7947 });
7948 ++LoopsEpilogueVectorized;
7949 } else {
7950 LVP.addMinimumIterationCheck(Plan&: BestPlan, VF: VF.Width, UF: IC,
7951 MinProfitableTripCount: VF.MinProfitableTripCount,
7952 /*TailFoldedEpilogue=*/false);
7953 LVP.attachRuntimeChecks(Plan&: BestPlan, RTChecks&: Checks, VF: VF.Width, UF: IC, HasBranchWeights);
7954
7955 if (!IsInnerLoop)
7956 LLVM_DEBUG(dbgs() << "Vectorizing outer loop in \"" << F->getName()
7957 << "\"\n");
7958 LVP.executePlan(BestVF: VF.Width, BestUF: IC, BestVPlan&: BestPlan, HasRuntimeChecks: Checks.hasChecks());
7959 ++LoopsVectorized;
7960 }
7961
7962#ifdef EXPENSIVE_CHECKS
7963 assert(DT->verify(DominatorTree::VerificationLevel::Fast) &&
7964 "DT not preserved correctly");
7965#endif
7966
7967 return true;
7968}
7969
7970LoopVectorizeResult LoopVectorizePass::runImpl(Function &F) {
7971 CFGChanged = false;
7972
7973 // Don't attempt if
7974 // 1. the target claims to have no vector registers, and
7975 // 2. interleaving won't help ILP.
7976 //
7977 // The second condition is necessary because, even if the target has no
7978 // vector registers, loop vectorization may still enable scalar
7979 // interleaving.
7980 if (!TTI->getNumberOfRegisters(ClassID: TTI->getRegisterClassForType(Vector: true)) &&
7981 (TTI->getMaxInterleaveFactor(VF: ElementCount::getFixed(MinVal: 1), HasUnorderedReductions: false) < 2 ||
7982 TTI->getMaxInterleaveFactor(VF: ElementCount::getFixed(MinVal: 1), HasUnorderedReductions: true) < 2))
7983 return LoopVectorizeResult(false, false);
7984
7985 bool Changed = false;
7986
7987 // The vectorizer requires loops to be in simplified form.
7988 // Since simplification may add new inner loops, it has to run before the
7989 // legality and profitability checks. This means running the loop vectorizer
7990 // will simplify all loops, regardless of whether anything end up being
7991 // vectorized.
7992 for (const auto &L : *LI)
7993 Changed |= CFGChanged |= simplifyLoop(L, DT, LI, SE, AC, MSSAU: nullptr,
7994 PreserveLCSSA: false /* PreserveLCSSA */, TTI);
7995
7996 // Build up a worklist of inner-loops to vectorize. This is necessary as
7997 // the act of vectorizing or partially unrolling a loop creates new loops
7998 // and can invalidate iterators across the loops.
7999 SmallVector<Loop *, 8> Worklist;
8000
8001 const VectorizeOptions &Opts = getVectorizeOptions(F);
8002 for (Loop *L : *LI)
8003 collectSupportedLoops(Opts, L&: *L, LI, ORE, V&: Worklist);
8004
8005 LoopsAnalyzed += Worklist.size();
8006
8007 // Now walk the identified inner loops.
8008 while (!Worklist.empty()) {
8009 Loop *L = Worklist.pop_back_val();
8010
8011 // For the inner loops we actually process, form LCSSA to simplify the
8012 // transform.
8013 Changed |= formLCSSARecursively(L&: *L, DT: *DT, LI, SE);
8014
8015 Changed |= CFGChanged |= processLoop(L);
8016
8017 if (Changed) {
8018 LAIs->clear();
8019
8020#ifndef NDEBUG
8021 if (VerifySCEV)
8022 SE->verify();
8023#endif
8024 }
8025 }
8026
8027 // Verify once per function rather than once per processed loop, which would
8028 // make the pass quadratic in the number of loops.
8029 assert((!Changed || DT->verify(DominatorTree::VerificationLevel::Fast)) &&
8030 "DT not preserved correctly");
8031 assert((!Changed || !verifyFunction(F, &dbgs())) &&
8032 "Invalid IR produced by LoopVectorize");
8033
8034 // Process each loop nest in the function.
8035 return LoopVectorizeResult(Changed, CFGChanged);
8036}
8037
8038PreservedAnalyses LoopVectorizePass::run(Function &F,
8039 FunctionAnalysisManager &AM) {
8040 LI = &AM.getResult<LoopAnalysis>(IR&: F);
8041 // There are no loops in the function. Return before computing other
8042 // expensive analyses.
8043 if (LI->empty())
8044 return PreservedAnalyses::all();
8045 SE = &AM.getResult<ScalarEvolutionAnalysis>(IR&: F);
8046 TTI = &AM.getResult<TargetIRAnalysis>(IR&: F);
8047 DT = &AM.getResult<DominatorTreeAnalysis>(IR&: F);
8048 TLI = &AM.getResult<TargetLibraryAnalysis>(IR&: F);
8049 AC = &AM.getResult<AssumptionAnalysis>(IR&: F);
8050 DB = &AM.getResult<DemandedBitsAnalysis>(IR&: F);
8051 ORE = &AM.getResult<OptimizationRemarkEmitterAnalysis>(IR&: F);
8052 LAIs = &AM.getResult<LoopAccessAnalysis>(IR&: F);
8053 AA = &AM.getResult<AAManager>(IR&: F);
8054
8055 auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(IR&: F);
8056 PSI = MAMProxy.getCachedResult<ProfileSummaryAnalysis>(IR&: *F.getParent());
8057 // CycleInfo cached by an earlier pass is invalidated when the CFG changes.
8058 // Both BlockFrequencyAnalysis and BranchProbabilityAnalysis depend on it, so
8059 // drop the stale result before either is (re-)computed.
8060 auto ClearStaleCycleInfo = [this, &AM, &F] {
8061 if (CFGChanged && AM.getCachedResult<CycleAnalysis>(IR&: F))
8062 AM.clearAnalysis<CycleAnalysis>(IR&: F);
8063 };
8064 GetBFI = [&AM, &F, ClearStaleCycleInfo]() -> BlockFrequencyInfo & {
8065 ClearStaleCycleInfo();
8066 return AM.getResult<BlockFrequencyAnalysis>(IR&: F);
8067 };
8068 GetBPI = [&AM, &F, ClearStaleCycleInfo]() -> const BranchProbabilityInfo & {
8069 ClearStaleCycleInfo();
8070 return AM.getResult<BranchProbabilityAnalysis>(IR&: F);
8071 };
8072 LoopVectorizeResult Result = runImpl(F);
8073 if (!Result.MadeAnyChange)
8074 return PreservedAnalyses::all();
8075 PreservedAnalyses PA;
8076
8077 if (isAssignmentTrackingEnabled(M: *F.getParent())) {
8078 for (auto &BB : F)
8079 RemoveRedundantDbgInstrs(BB: &BB);
8080 }
8081
8082 PA.preserve<LoopAnalysis>();
8083 PA.preserve<DominatorTreeAnalysis>();
8084 PA.preserve<ScalarEvolutionAnalysis>();
8085 PA.preserve<LoopAccessAnalysis>();
8086
8087 if (Result.MadeCFGChange) {
8088 // Making CFG changes likely means a loop got vectorized. Indicate that
8089 // extra simplification passes should be run.
8090 // TODO: MadeCFGChanges is not a prefect proxy. Extra passes should only
8091 // be run if runtime checks have been added.
8092 AM.getResult<ShouldRunExtraVectorPasses>(IR&: F);
8093 PA.preserve<ShouldRunExtraVectorPasses>();
8094 } else {
8095 PA.preserveSet<CFGAnalyses>();
8096 }
8097 return PA;
8098}
8099
8100void LoopVectorizePass::printPipeline(
8101 raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
8102 static_cast<PassInfoMixin<LoopVectorizePass> *>(this)->printPipeline(
8103 OS, MapClassName2PassName);
8104
8105 OS << '<';
8106 OS << (InterleaveOnlyWhenForced ? "" : "no-") << "interleave-forced-only;";
8107 OS << (VectorizeOnlyWhenForced ? "" : "no-") << "vectorize-forced-only;";
8108 OS << '>';
8109}
8110