1//===- LoopVectorizationPlanner.h - Planner for LoopVectorization ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file provides a LoopVectorizationPlanner class.
11/// InnerLoopVectorizer vectorizes loops which contain only one basic
12/// LoopVectorizationPlanner - drives the vectorization process after having
13/// passed Legality checks.
14/// The planner builds and optimizes the Vectorization Plans which record the
15/// decisions how to vectorize the given loop. In particular, represent the
16/// control-flow of the vectorized version, the replication of instructions that
17/// are to be scalarized, and interleave access groups.
18///
19/// Also provides a VPlan-based builder utility analogous to IRBuilder.
20/// It provides an instruction-level API for generating VPInstructions while
21/// abstracting away the Recipe manipulation details.
22//===----------------------------------------------------------------------===//
23
24#ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H
25#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H
26
27#include "VPlan.h"
28#include "llvm/ADT/SmallSet.h"
29#include "llvm/Support/InstructionCost.h"
30
31namespace llvm {
32
33class LoopInfo;
34class DominatorTree;
35class LoopVectorizationLegality;
36class LoopVectorizationCostModel;
37class PredicatedScalarEvolution;
38class LoopVectorizeHints;
39class LoopVersioning;
40class OptimizationRemarkEmitter;
41class TargetTransformInfo;
42class TargetLibraryInfo;
43class VPRecipeBuilder;
44struct VFRange;
45
46extern cl::opt<bool> EnableVPlanNativePath;
47extern cl::opt<unsigned> ForceTargetInstructionCost;
48
49/// VPlan-based builder utility analogous to IRBuilder.
50class VPBuilder {
51 VPBasicBlock *BB = nullptr;
52 VPBasicBlock::iterator InsertPt = VPBasicBlock::iterator();
53
54 /// Insert \p VPI in BB at InsertPt if BB is set.
55 template <typename T> T *tryInsertInstruction(T *R) {
56 if (BB)
57 BB->insert(Recipe: R, InsertPt);
58 return R;
59 }
60
61 VPInstruction *createInstruction(unsigned Opcode,
62 ArrayRef<VPValue *> Operands, DebugLoc DL,
63 const Twine &Name = "") {
64 return tryInsertInstruction(R: new VPInstruction(Opcode, Operands, DL, Name));
65 }
66
67 VPInstruction *createInstruction(unsigned Opcode,
68 std::initializer_list<VPValue *> Operands,
69 DebugLoc DL, const Twine &Name = "") {
70 return createInstruction(Opcode, Operands: ArrayRef<VPValue *>(Operands), DL, Name);
71 }
72
73public:
74 VPBuilder() = default;
75 VPBuilder(VPBasicBlock *InsertBB) { setInsertPoint(InsertBB); }
76 VPBuilder(VPRecipeBase *InsertPt) { setInsertPoint(InsertPt); }
77 VPBuilder(VPBasicBlock *TheBB, VPBasicBlock::iterator IP) {
78 setInsertPoint(TheBB, IP);
79 }
80
81 /// Clear the insertion point: created instructions will not be inserted into
82 /// a block.
83 void clearInsertionPoint() {
84 BB = nullptr;
85 InsertPt = VPBasicBlock::iterator();
86 }
87
88 VPBasicBlock *getInsertBlock() const { return BB; }
89 VPBasicBlock::iterator getInsertPoint() const { return InsertPt; }
90
91 /// Create a VPBuilder to insert after \p R.
92 static VPBuilder getToInsertAfter(VPRecipeBase *R) {
93 VPBuilder B;
94 B.setInsertPoint(TheBB: R->getParent(), IP: std::next(x: R->getIterator()));
95 return B;
96 }
97
98 /// InsertPoint - A saved insertion point.
99 class VPInsertPoint {
100 VPBasicBlock *Block = nullptr;
101 VPBasicBlock::iterator Point;
102
103 public:
104 /// Creates a new insertion point which doesn't point to anything.
105 VPInsertPoint() = default;
106
107 /// Creates a new insertion point at the given location.
108 VPInsertPoint(VPBasicBlock *InsertBlock, VPBasicBlock::iterator InsertPoint)
109 : Block(InsertBlock), Point(InsertPoint) {}
110
111 /// Returns true if this insert point is set.
112 bool isSet() const { return Block != nullptr; }
113
114 VPBasicBlock *getBlock() const { return Block; }
115 VPBasicBlock::iterator getPoint() const { return Point; }
116 };
117
118 /// Sets the current insert point to a previously-saved location.
119 void restoreIP(VPInsertPoint IP) {
120 if (IP.isSet())
121 setInsertPoint(TheBB: IP.getBlock(), IP: IP.getPoint());
122 else
123 clearInsertionPoint();
124 }
125
126 /// This specifies that created VPInstructions should be appended to the end
127 /// of the specified block.
128 void setInsertPoint(VPBasicBlock *TheBB) {
129 assert(TheBB && "Attempting to set a null insert point");
130 BB = TheBB;
131 InsertPt = BB->end();
132 }
133
134 /// This specifies that created instructions should be inserted at the
135 /// specified point.
136 void setInsertPoint(VPBasicBlock *TheBB, VPBasicBlock::iterator IP) {
137 BB = TheBB;
138 InsertPt = IP;
139 }
140
141 /// This specifies that created instructions should be inserted at the
142 /// specified point.
143 void setInsertPoint(VPRecipeBase *IP) {
144 BB = IP->getParent();
145 InsertPt = IP->getIterator();
146 }
147
148 /// Insert \p R at the current insertion point.
149 void insert(VPRecipeBase *R) { BB->insert(Recipe: R, InsertPt); }
150
151 /// Create an N-ary operation with \p Opcode, \p Operands and set \p Inst as
152 /// its underlying Instruction.
153 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
154 Instruction *Inst = nullptr,
155 const Twine &Name = "") {
156 DebugLoc DL = DebugLoc::getUnknown();
157 if (Inst)
158 DL = Inst->getDebugLoc();
159 VPInstruction *NewVPInst = createInstruction(Opcode, Operands, DL, Name);
160 NewVPInst->setUnderlyingValue(Inst);
161 return NewVPInst;
162 }
163 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
164 DebugLoc DL, const Twine &Name = "") {
165 return createInstruction(Opcode, Operands, DL, Name);
166 }
167 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
168 const VPIRFlags &Flags,
169 DebugLoc DL = DebugLoc::getUnknown(),
170 const Twine &Name = "") {
171 return tryInsertInstruction(
172 R: new VPInstruction(Opcode, Operands, Flags, DL, Name));
173 }
174
175 VPInstruction *createNaryOp(unsigned Opcode,
176 std::initializer_list<VPValue *> Operands,
177 Type *ResultTy, const VPIRFlags &Flags = {},
178 DebugLoc DL = DebugLoc::getUnknown(),
179 const Twine &Name = "") {
180 return tryInsertInstruction(
181 R: new VPInstructionWithType(Opcode, Operands, ResultTy, Flags, DL, Name));
182 }
183
184 VPInstruction *createOverflowingOp(unsigned Opcode,
185 std::initializer_list<VPValue *> Operands,
186 VPRecipeWithIRFlags::WrapFlagsTy WrapFlags,
187 DebugLoc DL = DebugLoc::getUnknown(),
188 const Twine &Name = "") {
189 return tryInsertInstruction(
190 R: new VPInstruction(Opcode, Operands, WrapFlags, DL, Name));
191 }
192
193 VPInstruction *createNot(VPValue *Operand,
194 DebugLoc DL = DebugLoc::getUnknown(),
195 const Twine &Name = "") {
196 return createInstruction(Opcode: VPInstruction::Not, Operands: {Operand}, DL, Name);
197 }
198
199 VPInstruction *createAnd(VPValue *LHS, VPValue *RHS,
200 DebugLoc DL = DebugLoc::getUnknown(),
201 const Twine &Name = "") {
202 return createInstruction(Opcode: Instruction::BinaryOps::And, Operands: {LHS, RHS}, DL, Name);
203 }
204
205 VPInstruction *createOr(VPValue *LHS, VPValue *RHS,
206 DebugLoc DL = DebugLoc::getUnknown(),
207 const Twine &Name = "") {
208
209 return tryInsertInstruction(R: new VPInstruction(
210 Instruction::BinaryOps::Or, {LHS, RHS},
211 VPRecipeWithIRFlags::DisjointFlagsTy(false), DL, Name));
212 }
213
214 VPInstruction *createLogicalAnd(VPValue *LHS, VPValue *RHS,
215 DebugLoc DL = DebugLoc::getUnknown(),
216 const Twine &Name = "") {
217 return tryInsertInstruction(
218 R: new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name));
219 }
220
221 VPInstruction *
222 createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
223 DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "",
224 std::optional<FastMathFlags> FMFs = std::nullopt) {
225 auto *Select =
226 FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
227 *FMFs, DL, Name)
228 : new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
229 DL, Name);
230 return tryInsertInstruction(R: Select);
231 }
232
233 /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
234 /// and \p B.
235 /// TODO: add createFCmp when needed.
236 VPInstruction *createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
237 DebugLoc DL = DebugLoc::getUnknown(),
238 const Twine &Name = "") {
239 assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE &&
240 Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate");
241 return tryInsertInstruction(
242 R: new VPInstruction(Instruction::ICmp, {A, B}, Pred, DL, Name));
243 }
244
245 VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset,
246 DebugLoc DL = DebugLoc::getUnknown(),
247 const Twine &Name = "") {
248 return tryInsertInstruction(
249 R: new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset},
250 GEPNoWrapFlags::none(), DL, Name));
251 }
252 VPInstruction *createInBoundsPtrAdd(VPValue *Ptr, VPValue *Offset,
253 DebugLoc DL = DebugLoc::getUnknown(),
254 const Twine &Name = "") {
255 return tryInsertInstruction(
256 R: new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset},
257 GEPNoWrapFlags::inBounds(), DL, Name));
258 }
259
260 VPPhi *createScalarPhi(ArrayRef<VPValue *> IncomingValues, DebugLoc DL,
261 const Twine &Name = "") {
262 return tryInsertInstruction(R: new VPPhi(IncomingValues, DL, Name));
263 }
264
265 /// Convert the input value \p Current to the corresponding value of an
266 /// induction with \p Start and \p Step values, using \p Start + \p Current *
267 /// \p Step.
268 VPDerivedIVRecipe *createDerivedIV(InductionDescriptor::InductionKind Kind,
269 FPMathOperator *FPBinOp, VPValue *Start,
270 VPValue *Current, VPValue *Step,
271 const Twine &Name = "") {
272 return tryInsertInstruction(
273 R: new VPDerivedIVRecipe(Kind, FPBinOp, Start, Current, Step, Name));
274 }
275
276 VPInstruction *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
277 Type *ResultTy, DebugLoc DL) {
278 return tryInsertInstruction(
279 R: new VPInstructionWithType(Opcode, Op, ResultTy, {}, DL));
280 }
281
282 VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy,
283 DebugLoc DL) {
284 if (ResultTy == SrcTy)
285 return Op;
286 Instruction::CastOps CastOp =
287 ResultTy->getScalarSizeInBits() < SrcTy->getScalarSizeInBits()
288 ? Instruction::Trunc
289 : Instruction::ZExt;
290 return createScalarCast(Opcode: CastOp, Op, ResultTy, DL);
291 }
292
293 VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,
294 Type *ResultTy) {
295 return tryInsertInstruction(R: new VPWidenCastRecipe(Opcode, Op, ResultTy));
296 }
297
298 VPScalarIVStepsRecipe *
299 createScalarIVSteps(Instruction::BinaryOps InductionOpcode,
300 FPMathOperator *FPBinOp, VPValue *IV, VPValue *Step,
301 VPValue *VF, DebugLoc DL) {
302 return tryInsertInstruction(R: new VPScalarIVStepsRecipe(
303 IV, Step, VF, InductionOpcode,
304 FPBinOp ? FPBinOp->getFastMathFlags() : FastMathFlags(), DL));
305 }
306
307 //===--------------------------------------------------------------------===//
308 // RAII helpers.
309 //===--------------------------------------------------------------------===//
310
311 /// RAII object that stores the current insertion point and restores it when
312 /// the object is destroyed.
313 class InsertPointGuard {
314 VPBuilder &Builder;
315 VPBasicBlock *Block;
316 VPBasicBlock::iterator Point;
317
318 public:
319 InsertPointGuard(VPBuilder &B)
320 : Builder(B), Block(B.getInsertBlock()), Point(B.getInsertPoint()) {}
321
322 InsertPointGuard(const InsertPointGuard &) = delete;
323 InsertPointGuard &operator=(const InsertPointGuard &) = delete;
324
325 ~InsertPointGuard() { Builder.restoreIP(IP: VPInsertPoint(Block, Point)); }
326 };
327};
328
329/// TODO: The following VectorizationFactor was pulled out of
330/// LoopVectorizationCostModel class. LV also deals with
331/// VectorizerParams::VectorizationFactor.
332/// We need to streamline them.
333
334/// Information about vectorization costs.
335struct VectorizationFactor {
336 /// Vector width with best cost.
337 ElementCount Width;
338
339 /// Cost of the loop with that width.
340 InstructionCost Cost;
341
342 /// Cost of the scalar loop.
343 InstructionCost ScalarCost;
344
345 /// The minimum trip count required to make vectorization profitable, e.g. due
346 /// to runtime checks.
347 ElementCount MinProfitableTripCount;
348
349 VectorizationFactor(ElementCount Width, InstructionCost Cost,
350 InstructionCost ScalarCost)
351 : Width(Width), Cost(Cost), ScalarCost(ScalarCost) {}
352
353 /// Width 1 means no vectorization, cost 0 means uncomputed cost.
354 static VectorizationFactor Disabled() {
355 return {ElementCount::getFixed(MinVal: 1), 0, 0};
356 }
357
358 bool operator==(const VectorizationFactor &rhs) const {
359 return Width == rhs.Width && Cost == rhs.Cost;
360 }
361
362 bool operator!=(const VectorizationFactor &rhs) const {
363 return !(*this == rhs);
364 }
365};
366
367/// A class that represents two vectorization factors (initialized with 0 by
368/// default). One for fixed-width vectorization and one for scalable
369/// vectorization. This can be used by the vectorizer to choose from a range of
370/// fixed and/or scalable VFs in order to find the most cost-effective VF to
371/// vectorize with.
372struct FixedScalableVFPair {
373 ElementCount FixedVF;
374 ElementCount ScalableVF;
375
376 FixedScalableVFPair()
377 : FixedVF(ElementCount::getFixed(MinVal: 0)),
378 ScalableVF(ElementCount::getScalable(MinVal: 0)) {}
379 FixedScalableVFPair(const ElementCount &Max) : FixedScalableVFPair() {
380 *(Max.isScalable() ? &ScalableVF : &FixedVF) = Max;
381 }
382 FixedScalableVFPair(const ElementCount &FixedVF,
383 const ElementCount &ScalableVF)
384 : FixedVF(FixedVF), ScalableVF(ScalableVF) {
385 assert(!FixedVF.isScalable() && ScalableVF.isScalable() &&
386 "Invalid scalable properties");
387 }
388
389 static FixedScalableVFPair getNone() { return FixedScalableVFPair(); }
390
391 /// \return true if either fixed- or scalable VF is non-zero.
392 explicit operator bool() const { return FixedVF || ScalableVF; }
393
394 /// \return true if either fixed- or scalable VF is a valid vector VF.
395 bool hasVector() const { return FixedVF.isVector() || ScalableVF.isVector(); }
396};
397
398/// Planner drives the vectorization process after having passed
399/// Legality checks.
400class LoopVectorizationPlanner {
401 /// The loop that we evaluate.
402 Loop *OrigLoop;
403
404 /// Loop Info analysis.
405 LoopInfo *LI;
406
407 /// The dominator tree.
408 DominatorTree *DT;
409
410 /// Target Library Info.
411 const TargetLibraryInfo *TLI;
412
413 /// Target Transform Info.
414 const TargetTransformInfo &TTI;
415
416 /// The legality analysis.
417 LoopVectorizationLegality *Legal;
418
419 /// The profitability analysis.
420 LoopVectorizationCostModel &CM;
421
422 /// The interleaved access analysis.
423 InterleavedAccessInfo &IAI;
424
425 PredicatedScalarEvolution &PSE;
426
427 const LoopVectorizeHints &Hints;
428
429 OptimizationRemarkEmitter *ORE;
430
431 SmallVector<VPlanPtr, 4> VPlans;
432
433 /// Profitable vector factors.
434 SmallVector<VectorizationFactor, 8> ProfitableVFs;
435
436 /// A builder used to construct the current plan.
437 VPBuilder Builder;
438
439 /// Computes the cost of \p Plan for vectorization factor \p VF.
440 ///
441 /// The current implementation requires access to the
442 /// LoopVectorizationLegality to handle inductions and reductions, which is
443 /// why it is kept separate from the VPlan-only cost infrastructure.
444 ///
445 /// TODO: Move to VPlan::cost once the use of LoopVectorizationLegality has
446 /// been retired.
447 InstructionCost cost(VPlan &Plan, ElementCount VF) const;
448
449 /// Precompute costs for certain instructions using the legacy cost model. The
450 /// function is used to bring up the VPlan-based cost model to initially avoid
451 /// taking different decisions due to inaccuracies in the legacy cost model.
452 InstructionCost precomputeCosts(VPlan &Plan, ElementCount VF,
453 VPCostContext &CostCtx) const;
454
455public:
456 LoopVectorizationPlanner(
457 Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
458 const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
459 LoopVectorizationCostModel &CM, InterleavedAccessInfo &IAI,
460 PredicatedScalarEvolution &PSE, const LoopVectorizeHints &Hints,
461 OptimizationRemarkEmitter *ORE)
462 : OrigLoop(L), LI(LI), DT(DT), TLI(TLI), TTI(TTI), Legal(Legal), CM(CM),
463 IAI(IAI), PSE(PSE), Hints(Hints), ORE(ORE) {}
464
465 /// Build VPlans for the specified \p UserVF and \p UserIC if they are
466 /// non-zero or all applicable candidate VFs otherwise. If vectorization and
467 /// interleaving should be avoided up-front, no plans are generated.
468 void plan(ElementCount UserVF, unsigned UserIC);
469
470 /// Use the VPlan-native path to plan how to best vectorize, return the best
471 /// VF and its cost.
472 VectorizationFactor planInVPlanNativePath(ElementCount UserVF);
473
474 /// Return the VPlan for \p VF. At the moment, there is always a single VPlan
475 /// for each VF.
476 VPlan &getPlanFor(ElementCount VF) const;
477
478 /// Compute and return the most profitable vectorization factor. Also collect
479 /// all profitable VFs in ProfitableVFs.
480 VectorizationFactor computeBestVF();
481
482 /// Generate the IR code for the vectorized loop captured in VPlan \p BestPlan
483 /// according to the best selected \p VF and \p UF.
484 ///
485 /// TODO: \p VectorizingEpilogue indicates if the executed VPlan is for the
486 /// epilogue vector loop. It should be removed once the re-use issue has been
487 /// fixed.
488 ///
489 /// Returns a mapping of SCEVs to their expanded IR values.
490 /// Note that this is a temporary workaround needed due to the current
491 /// epilogue handling.
492 DenseMap<const SCEV *, Value *> executePlan(ElementCount VF, unsigned UF,
493 VPlan &BestPlan,
494 InnerLoopVectorizer &LB,
495 DominatorTree *DT,
496 bool VectorizingEpilogue);
497
498#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
499 void printPlans(raw_ostream &O);
500#endif
501
502 /// Look through the existing plans and return true if we have one with
503 /// vectorization factor \p VF.
504 bool hasPlanWithVF(ElementCount VF) const {
505 return any_of(Range: VPlans,
506 P: [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); });
507 }
508
509 /// Test a \p Predicate on a \p Range of VF's. Return the value of applying
510 /// \p Predicate on Range.Start, possibly decreasing Range.End such that the
511 /// returned value holds for the entire \p Range.
512 static bool
513 getDecisionAndClampRange(const std::function<bool(ElementCount)> &Predicate,
514 VFRange &Range);
515
516 /// \return The most profitable vectorization factor and the cost of that VF
517 /// for vectorizing the epilogue. Returns VectorizationFactor::Disabled if
518 /// epilogue vectorization is not supported for the loop.
519 VectorizationFactor
520 selectEpilogueVectorizationFactor(const ElementCount MaxVF, unsigned IC);
521
522 /// Emit remarks for recipes with invalid costs in the available VPlans.
523 void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE);
524
525protected:
526 /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
527 /// according to the information gathered by Legal when it checked if it is
528 /// legal to vectorize the loop.
529 void buildVPlans(ElementCount MinVF, ElementCount MaxVF);
530
531private:
532 /// Build a VPlan according to the information gathered by Legal. \return a
533 /// VPlan for vectorization factors \p Range.Start and up to \p Range.End
534 /// exclusive, possibly decreasing \p Range.End. If no VPlan can be built for
535 /// the input range, set the largest included VF to the maximum VF for which
536 /// no plan could be built.
537 VPlanPtr tryToBuildVPlan(VFRange &Range);
538
539 /// Build a VPlan using VPRecipes according to the information gather by
540 /// Legal. This method is only used for the legacy inner loop vectorizer.
541 /// \p Range's largest included VF is restricted to the maximum VF the
542 /// returned VPlan is valid for. If no VPlan can be built for the input range,
543 /// set the largest included VF to the maximum VF for which no plan could be
544 /// built. Each VPlan is built starting from a copy of \p InitialPlan, which
545 /// is a plain CFG VPlan wrapping the original scalar loop.
546 VPlanPtr tryToBuildVPlanWithVPRecipes(VPlanPtr InitialPlan, VFRange &Range,
547 LoopVersioning *LVer);
548
549 /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
550 /// according to the information gathered by Legal when it checked if it is
551 /// legal to vectorize the loop. This method creates VPlans using VPRecipes.
552 void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF);
553
554 // Adjust the recipes for reductions. For in-loop reductions the chain of
555 // instructions leading from the loop exit instr to the phi need to be
556 // converted to reductions, with one operand being vector and the other being
557 // the scalar reduction chain. For other reductions, a select is introduced
558 // between the phi and users outside the vector region when folding the tail.
559 void adjustRecipesForReductions(VPlanPtr &Plan,
560 VPRecipeBuilder &RecipeBuilder,
561 ElementCount MinVF);
562
563#ifndef NDEBUG
564 /// \return The most profitable vectorization factor for the available VPlans
565 /// and the cost of that VF.
566 /// This is now only used to verify the decisions by the new VPlan-based
567 /// cost-model and will be retired once the VPlan-based cost-model is
568 /// stabilized.
569 VectorizationFactor selectVectorizationFactor();
570#endif
571
572 /// Returns true if the per-lane cost of VectorizationFactor A is lower than
573 /// that of B.
574 bool isMoreProfitable(const VectorizationFactor &A,
575 const VectorizationFactor &B, bool HasTail) const;
576
577 /// Returns true if the per-lane cost of VectorizationFactor A is lower than
578 /// that of B in the context of vectorizing a loop with known \p MaxTripCount.
579 bool isMoreProfitable(const VectorizationFactor &A,
580 const VectorizationFactor &B,
581 const unsigned MaxTripCount, bool HasTail) const;
582
583 /// Determines if we have the infrastructure to vectorize the loop and its
584 /// epilogue, assuming the main loop is vectorized by \p VF.
585 bool isCandidateForEpilogueVectorization(const ElementCount VF) const;
586};
587
588} // namespace llvm
589
590#endif // LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H
591