1//===- LoopVectorizationPlanner.h - Planner for LoopVectorization ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file provides a LoopVectorizationPlanner class.
11/// InnerLoopVectorizer vectorizes loops which contain only one basic
12/// LoopVectorizationPlanner - drives the vectorization process after having
13/// passed Legality checks.
14/// The planner builds and optimizes the Vectorization Plans which record the
15/// decisions how to vectorize the given loop. In particular, represent the
16/// control-flow of the vectorized version, the replication of instructions that
17/// are to be scalarized, and interleave access groups.
18///
19/// Also provides a VPlan-based builder utility analogous to IRBuilder.
20/// It provides an instruction-level API for generating VPInstructions while
21/// abstracting away the Recipe manipulation details.
22//===----------------------------------------------------------------------===//
23
24#ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H
25#define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H
26
27#include "VPlan.h"
28#include "llvm/ADT/SmallSet.h"
29#include "llvm/Analysis/TargetTransformInfo.h"
30#include "llvm/Support/InstructionCost.h"
31#include <optional>
32
33namespace {
34class GeneratedRTChecks;
35}
36
37namespace llvm {
38
39class LoopInfo;
40class DominatorTree;
41class LoopVectorizationLegality;
42class LoopVectorizationCostModel;
43class PredicatedScalarEvolution;
44class LoopVectorizeHints;
45class RecurrenceDescriptor;
46class LoopVersioning;
47class OptimizationRemarkEmitter;
48class TargetLibraryInfo;
49class VPRecipeBuilder;
50struct VPRegisterUsage;
51struct VFRange;
52
53extern cl::opt<bool> EnableVPlanNativePath;
54extern cl::opt<unsigned> ForceTargetInstructionCost;
55extern cl::opt<bool> PreferInLoopReductions;
56
57/// \return An upper bound for vscale based on TTI or the vscale_range
58/// attribute.
59std::optional<unsigned> getMaxVScale(const Function &F,
60 const TargetTransformInfo &TTI);
61
62// Utility functions that are used by different vectorization classes
63namespace LoopVectorizationUtils {
64
65/// Reports a vectorization failure: print \p DebugMsg for debugging
66/// purposes along with the corresponding optimization remark \p RemarkName.
67/// If \p I is passed, it is an instruction that prevents vectorization.
68/// Otherwise, the loop \p TheLoop is used for the location of the remark.
69void reportVectorizationFailure(const StringRef DebugMsg,
70 const StringRef OREMsg, const StringRef ORETag,
71 OptimizationRemarkEmitter *ORE,
72 const Loop *TheLoop, Instruction *I = nullptr);
73
74/// Same as above, but the debug message and optimization remark are identical
75inline void reportVectorizationFailure(const StringRef DebugMsg,
76 const StringRef ORETag,
77 OptimizationRemarkEmitter *ORE,
78 const Loop *TheLoop,
79 Instruction *I = nullptr) {
80 reportVectorizationFailure(DebugMsg, OREMsg: DebugMsg, ORETag, ORE, TheLoop, I);
81}
82
83/// Reports an informative message: print \p Msg for debugging purposes as well
84/// as an optimization remark. Uses either \p I as location of the remark, or
85/// otherwise \p TheLoop. If \p DL is passed, use it as debug location for the
86/// remark.
87void reportVectorizationInfo(const StringRef Msg, const StringRef ORETag,
88 OptimizationRemarkEmitter *ORE,
89 const Loop *TheLoop, Instruction *I = nullptr,
90 DebugLoc DL = {});
91
92/// Report successful vectorization of the loop. In case an outer loop is
93/// vectorized, prepend "outer" to the vectorization remark.
94void reportVectorization(OptimizationRemarkEmitter *ORE, Loop *TheLoop,
95 ElementCount VFWidth, unsigned IC);
96
97} // namespace LoopVectorizationUtils
98
99/// VPlan-based builder utility analogous to IRBuilder.
100class VPBuilder {
101private:
102 class VPInsertPoint {
103 VPBasicBlock *Block = nullptr;
104 VPBasicBlock::iterator Point;
105
106 public:
107 /// Creates a new insertion point which doesn't point to anything.
108 VPInsertPoint() = default;
109
110 /// Creates a new insertion point to insert at \p Point in \p Block.
111 VPInsertPoint(VPBasicBlock *Block, VPBasicBlock::iterator Point)
112 : Block(Block), Point(Point) {}
113
114 /// Creates a new insertion point to insert before \p R.
115 VPInsertPoint(VPRecipeBase *R)
116 : Block(R->getParent()), Point(R->getIterator()) {}
117
118 /// Creates a new insertion point to insert at the end of \p Block.
119 VPInsertPoint(VPBasicBlock *Block) : Block(Block), Point(Block->end()) {}
120
121 /// Returns true if this insert point is set.
122 operator bool() const { return Block; }
123
124 VPBasicBlock *getBlock() const { return Block; }
125
126 operator VPRecipeBase *() const {
127 return Point == Block->end() ? nullptr : &*Point;
128 }
129
130 template <typename T> void insert(T &R) { return Block->insert(Recipe: R, InsertPt: Point); }
131 };
132
133 VPInsertPoint InsertPt;
134
135 /// Insert \p VPI in BB at InsertPt if BB is set.
136 template <typename T> T *tryInsertInstruction(T *R) {
137 if (InsertPt)
138 InsertPt.insert(R);
139 return R;
140 }
141
142 VPInstruction *createInstruction(unsigned Opcode,
143 ArrayRef<VPValue *> Operands,
144 const VPIRMetadata &MD, DebugLoc DL,
145 const Twine &Name = "") {
146 return tryInsertInstruction(
147 R: new VPInstruction(Opcode, Operands, {}, MD, DL, Name));
148 }
149
150public:
151 VPlan &getPlan() const {
152 assert(InsertPt && "Insert block must be set");
153 return *InsertPt.getBlock()->getPlan();
154 }
155
156 VPBuilder() = default;
157 VPBuilder(const VPInsertPoint &IP) : InsertPt(IP) {}
158 VPBuilder(VPBasicBlock *TheBB, VPBasicBlock::iterator IP)
159 : InsertPt(TheBB, IP) {}
160
161 /// Get the recipe at the current insert point or nullptr if the insert point
162 /// is the end of the block.
163 VPRecipeBase *getRecipeAtInsertPoint() const { return InsertPt; }
164
165 /// Create a VPBuilder to insert after \p R.
166 static VPBuilder getToInsertAfter(VPRecipeBase *R) {
167 return {R->getParent(), std::next(x: R->getIterator())};
168 }
169
170 /// Sets the current insert point to a previously-saved location.
171 void restoreIP(VPInsertPoint IP) { InsertPt = IP; }
172
173 /// Set the current insert point.
174 void setInsertPoint(const VPInsertPoint &IP) {
175 assert(IP && "Attempting to set a null insert point");
176 InsertPt = IP;
177 }
178 void setInsertPoint(VPBasicBlock *TheBB, VPBasicBlock::iterator IP) {
179 assert(TheBB && "Attempting to set a null insert point");
180 InsertPt = VPInsertPoint(TheBB, IP);
181 }
182
183 /// Insert \p R at the current insertion point. Returns \p R unchanged.
184 template <typename T> [[maybe_unused]] T *insert(T *R) {
185 InsertPt.insert(R);
186 return R;
187 }
188
189 /// Create an N-ary operation with \p Opcode, \p Operands and set \p Inst as
190 /// its underlying Instruction.
191 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
192 Instruction *Inst = nullptr,
193 const VPIRFlags &Flags = {},
194 const VPIRMetadata &MD = {},
195 DebugLoc DL = DebugLoc::getUnknown(),
196 const Twine &Name = "",
197 Type *ResultTy = nullptr) {
198 VPInstruction *NewVPInst = tryInsertInstruction(
199 R: new VPInstruction(Opcode, Operands, Flags, MD, DL, Name, ResultTy));
200 NewVPInst->setUnderlyingValue(Inst);
201 return NewVPInst;
202 }
203 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
204 DebugLoc DL, const Twine &Name = "") {
205 return createInstruction(Opcode, Operands, MD: {}, DL, Name);
206 }
207 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
208 const VPIRFlags &Flags,
209 DebugLoc DL = DebugLoc::getUnknown(),
210 const Twine &Name = "") {
211 return tryInsertInstruction(
212 R: new VPInstruction(Opcode, Operands, Flags, {}, DL, Name));
213 }
214
215 VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
216 Type *ResultTy, const VPIRFlags &Flags = {},
217 DebugLoc DL = DebugLoc::getUnknown(),
218 const Twine &Name = "") {
219 return tryInsertInstruction(R: new VPInstructionWithType(
220 Opcode, Operands, ResultTy, Flags, {}, DL, Name));
221 }
222
223 VPInstruction *createFirstActiveLane(ArrayRef<VPValue *> Masks,
224 DebugLoc DL = DebugLoc::getUnknown(),
225 const Twine &Name = "") {
226 // Assume that the maximum possible number of elements in a vector fits
227 // within the index type for the default address space.
228 VPlan &Plan = getPlan();
229 Type *IndexTy = Plan.getDataLayout().getIndexType(C&: Plan.getContext(), AddressSpace: 0);
230 return tryInsertInstruction(R: new VPInstruction(
231 VPInstruction::FirstActiveLane, Masks, {}, {}, DL, Name, IndexTy));
232 }
233
234 VPInstruction *createLastActiveLane(ArrayRef<VPValue *> Masks,
235 DebugLoc DL = DebugLoc::getUnknown(),
236 const Twine &Name = "") {
237 // Assume that the maximum possible number of elements in a vector fits
238 // within the index type for the default address space.
239 VPlan &Plan = getPlan();
240 Type *IndexTy = Plan.getDataLayout().getIndexType(C&: Plan.getContext(), AddressSpace: 0);
241 return tryInsertInstruction(R: new VPInstruction(
242 VPInstruction::LastActiveLane, Masks, {}, {}, DL, Name, IndexTy));
243 }
244
245 VPInstruction *createOverflowingOp(
246 unsigned Opcode, ArrayRef<VPValue *> Operands,
247 VPRecipeWithIRFlags::WrapFlagsTy WrapFlags = {false, false},
248 DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") {
249 return tryInsertInstruction(
250 R: new VPInstruction(Opcode, Operands, WrapFlags, {}, DL, Name));
251 }
252
253 VPInstruction *createNot(VPValue *Operand,
254 DebugLoc DL = DebugLoc::getUnknown(),
255 const Twine &Name = "") {
256 return createInstruction(Opcode: VPInstruction::Not, Operands: {Operand}, MD: {}, DL, Name);
257 }
258
259 VPInstruction *createAnd(VPValue *LHS, VPValue *RHS,
260 DebugLoc DL = DebugLoc::getUnknown(),
261 const Twine &Name = "") {
262 return createInstruction(Opcode: Instruction::BinaryOps::And, Operands: {LHS, RHS}, MD: {}, DL,
263 Name);
264 }
265
266 VPInstruction *createOr(VPValue *LHS, VPValue *RHS,
267 DebugLoc DL = DebugLoc::getUnknown(),
268 const Twine &Name = "") {
269
270 return tryInsertInstruction(R: new VPInstruction(
271 Instruction::BinaryOps::Or, {LHS, RHS},
272 VPRecipeWithIRFlags::DisjointFlagsTy(false), {}, DL, Name));
273 }
274
275 VPInstruction *
276 createAdd(VPValue *LHS, VPValue *RHS, DebugLoc DL = DebugLoc::getUnknown(),
277 const Twine &Name = "",
278 VPRecipeWithIRFlags::WrapFlagsTy WrapFlags = {false, false}) {
279 return createOverflowingOp(Opcode: Instruction::Add, Operands: {LHS, RHS}, WrapFlags, DL,
280 Name);
281 }
282
283 VPInstruction *
284 createSub(VPValue *LHS, VPValue *RHS, DebugLoc DL = DebugLoc::getUnknown(),
285 const Twine &Name = "",
286 VPRecipeWithIRFlags::WrapFlagsTy WrapFlags = {false, false}) {
287 return createOverflowingOp(Opcode: Instruction::Sub, Operands: {LHS, RHS}, WrapFlags, DL,
288 Name);
289 }
290
291 VPInstruction *createLogicalAnd(VPValue *LHS, VPValue *RHS,
292 DebugLoc DL = DebugLoc::getUnknown(),
293 const Twine &Name = "") {
294 return createNaryOp(Opcode: VPInstruction::LogicalAnd, Operands: {LHS, RHS}, DL, Name);
295 }
296
297 VPInstruction *createLogicalOr(VPValue *LHS, VPValue *RHS,
298 DebugLoc DL = DebugLoc::getUnknown(),
299 const Twine &Name = "") {
300 return createNaryOp(Opcode: VPInstruction::LogicalOr, Operands: {LHS, RHS}, DL, Name);
301 }
302
303 /// Create a select of \p TrueVal and \p FalseVal based on \p Cond, using the
304 /// default flags for the result type, unless \p Flags is set.
305 VPInstruction *createSelect(VPValue *Cond, VPValue *TrueVal,
306 VPValue *FalseVal,
307 DebugLoc DL = DebugLoc::getUnknown(),
308 const Twine &Name = "",
309 std::optional<VPIRFlags> Flags = std::nullopt) {
310 return tryInsertInstruction(
311 R: new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
312 Flags.value_or(u: VPIRFlags::getDefaultFlags(
313 Opcode: Instruction::Select, ResultTy: TrueVal->getScalarType())),
314 {}, DL, Name));
315 }
316
317 /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
318 /// and \p B.
319 VPInstruction *createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
320 DebugLoc DL = DebugLoc::getUnknown(),
321 const Twine &Name = "") {
322 assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE &&
323 Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate");
324 return tryInsertInstruction(
325 R: new VPInstruction(Instruction::ICmp, {A, B}, Pred, {}, DL, Name));
326 }
327
328 /// Create a new FCmp VPInstruction with predicate \p Pred and operands \p A
329 /// and \p B.
330 VPInstruction *createFCmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
331 DebugLoc DL = DebugLoc::getUnknown(),
332 const Twine &Name = "") {
333 assert(Pred >= CmpInst::FIRST_FCMP_PREDICATE &&
334 Pred <= CmpInst::LAST_FCMP_PREDICATE && "invalid predicate");
335 return tryInsertInstruction(
336 R: new VPInstruction(Instruction::FCmp, {A, B},
337 VPIRFlags(Pred, FastMathFlags()), {}, DL, Name));
338 }
339
340 /// Create an AnyOf reduction pattern: or-reduce \p ChainOp, freeze the
341 /// result, then select between \p TrueVal and \p FalseVal.
342 VPInstruction *createAnyOfReduction(VPValue *ChainOp, VPValue *TrueVal,
343 VPValue *FalseVal,
344 DebugLoc DL = DebugLoc::getUnknown());
345
346 VPInstruction *createPtrAdd(VPValue *Ptr, VPValue *Offset,
347 DebugLoc DL = DebugLoc::getUnknown(),
348 const Twine &Name = "") {
349 return createNoWrapPtrAdd(Ptr, Offset, GEPFlags: GEPNoWrapFlags::none(), DL, Name);
350 }
351
352 VPInstruction *createNoWrapPtrAdd(VPValue *Ptr, VPValue *Offset,
353 GEPNoWrapFlags GEPFlags,
354 DebugLoc DL = DebugLoc::getUnknown(),
355 const Twine &Name = "") {
356 return tryInsertInstruction(R: new VPInstruction(
357 VPInstruction::PtrAdd, {Ptr, Offset}, GEPFlags, {}, DL, Name));
358 }
359
360 VPInstruction *createWidePtrAdd(VPValue *Ptr, VPValue *Offset,
361 DebugLoc DL = DebugLoc::getUnknown(),
362 const Twine &Name = "") {
363 return tryInsertInstruction(
364 R: new VPInstruction(VPInstruction::WidePtrAdd, {Ptr, Offset},
365 GEPNoWrapFlags::none(), {}, DL, Name));
366 }
367
368 /// Create a phi with \p IncomingValues, using the default flags for the
369 /// result type, unless \p Flags is set.
370 VPPhi *createScalarPhi(ArrayRef<VPValue *> IncomingValues,
371 DebugLoc DL = DebugLoc::getUnknown(),
372 const Twine &Name = "",
373 std::optional<VPIRFlags> Flags = std::nullopt,
374 Type *ResultTy = nullptr) {
375 Type *ScalarTy = ResultTy ? ResultTy : IncomingValues[0]->getScalarType();
376 return tryInsertInstruction(R: new VPPhi(
377 IncomingValues,
378 Flags.value_or(u: VPIRFlags::getDefaultFlags(Opcode: Instruction::PHI, ResultTy: ScalarTy)),
379 DL, Name, ResultTy));
380 }
381
382 VPWidenPHIRecipe *createWidenPhi(ArrayRef<VPValue *> IncomingValues,
383 DebugLoc DL = DebugLoc::getUnknown(),
384 const Twine &Name = "") {
385 return tryInsertInstruction(R: new VPWidenPHIRecipe(IncomingValues, DL, Name));
386 }
387
388 VPValue *createElementCount(Type *Ty, ElementCount EC) {
389 VPlan &Plan = getPlan();
390 unsigned MinEC = EC.getKnownMinValue();
391 if (EC.isScalable()) {
392 VPValue *VScale = createVScale(ResultTy: Ty);
393 if (MinEC == 1)
394 return VScale;
395 // TODO: Move this optimization into createOverflowingOp directly.
396 if (isPowerOf2_32(Value: MinEC)) {
397 VPValue *ShtAmt = Plan.getConstantInt(Ty, Val: Log2_32(Value: MinEC));
398 return createOverflowingOp(Opcode: Instruction::Shl, Operands: {VScale, ShtAmt},
399 WrapFlags: {true, false});
400 }
401 VPValue *MulAmt = Plan.getConstantInt(Ty, Val: MinEC);
402 return createOverflowingOp(Opcode: Instruction::Mul, Operands: {VScale, MulAmt},
403 WrapFlags: {true, false});
404 }
405 return Plan.getConstantInt(Ty, Val: MinEC);
406 }
407
408 /// Convert \p Current to \p Start + \p Current * \p Step.
409 VPDerivedIVRecipe *createDerivedIV(InductionDescriptor::InductionKind Kind,
410 FPMathOperator *FPBinOp, VPValue *Start,
411 VPValue *Current, VPValue *Step,
412 const VPIRFlags::WrapFlagsTy &Flags = {}) {
413 return tryInsertInstruction(
414 R: new VPDerivedIVRecipe(Kind, FPBinOp, Start, Current, Step, Flags));
415 }
416
417 VPInstructionWithType *createScalarLoad(Type *ResultTy, VPValue *Addr,
418 DebugLoc DL,
419 const VPIRMetadata &Metadata = {}) {
420 return tryInsertInstruction(R: new VPInstructionWithType(
421 Instruction::Load, Addr, ResultTy, {}, Metadata, DL));
422 }
423
424 VPInstruction *createScalarCast(Instruction::CastOps Opcode, VPValue *Op,
425 Type *ResultTy, DebugLoc DL,
426 std::optional<VPIRFlags> Flags = std::nullopt,
427 const VPIRMetadata &Metadata = {}) {
428 return tryInsertInstruction(R: new VPInstructionWithType(
429 Opcode, Op, ResultTy,
430 Flags.value_or(u: VPIRFlags::getDefaultFlags(Opcode)), Metadata, DL));
431 }
432
433 /// Create a scalar call to the intrinsic \p IntrinsicID with \p Operands, and
434 /// result type \p ResultTy
435 VPInstruction *createScalarIntrinsic(Intrinsic::ID IntrinsicID,
436 ArrayRef<VPValue *> Operands,
437 Type *ResultTy, DebugLoc DL) {
438 VPlan &Plan = getPlan();
439 SmallVector<VPValue *, 2> Ops(Operands);
440 Ops.push_back(Elt: Plan.getConstantInt(BitWidth: 8 * sizeof(IntrinsicID), Val: IntrinsicID));
441 return tryInsertInstruction(R: new VPInstructionWithType(
442 VPInstruction::Intrinsic, Ops, ResultTy, {}, {}, DL));
443 }
444
445 /// Create a scalar llvm.vscale call.
446 VPInstruction *createVScale(Type *ResultTy,
447 DebugLoc DL = DebugLoc::getUnknown()) {
448 return createScalarIntrinsic(IntrinsicID: Intrinsic::vscale, Operands: {}, ResultTy, DL);
449 }
450
451 VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, DebugLoc DL) {
452 Type *SrcTy = Op->getScalarType();
453 if (ResultTy == SrcTy)
454 return Op;
455 Instruction::CastOps CastOp =
456 ResultTy->getScalarSizeInBits() < SrcTy->getScalarSizeInBits()
457 ? Instruction::Trunc
458 : Instruction::ZExt;
459 return createScalarCast(Opcode: CastOp, Op, ResultTy, DL);
460 }
461
462 VPValue *createScalarSExtOrTrunc(VPValue *Op, Type *ResultTy, DebugLoc DL) {
463 Type *SrcTy = Op->getScalarType();
464 if (ResultTy == SrcTy)
465 return Op;
466 Instruction::CastOps CastOp =
467 ResultTy->getScalarSizeInBits() < SrcTy->getScalarSizeInBits()
468 ? Instruction::Trunc
469 : Instruction::SExt;
470 return createScalarCast(Opcode: CastOp, Op, ResultTy, DL);
471 }
472
473 VPValue *createScalarFreeze(VPValue *Op, DebugLoc DL) {
474 return tryInsertInstruction(
475 R: new VPInstruction(Instruction::Freeze, Op, {}, {}, DL));
476 }
477
478 VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,
479 Type *ResultTy) {
480 return tryInsertInstruction(R: new VPWidenCastRecipe(
481 Opcode, Op, ResultTy, nullptr, VPIRFlags::getDefaultFlags(Opcode)));
482 }
483
484 /// Create a single-scalar recipe with \p Opcode and \p Operands without
485 /// inserting it.
486 static VPSingleDefRecipe *createSingleScalarOp(unsigned Opcode,
487 ArrayRef<VPValue *> Operands,
488 VPValue *Mask,
489 const VPIRFlags &Flags,
490 const VPIRMetadata &Metadata,
491 DebugLoc DL, Instruction *UV) {
492 if (Instruction::isCast(Opcode)) {
493 assert(!Mask && "Cast cannot be predicated");
494 return new VPInstructionWithType(Opcode, Operands, UV->getType(), Flags,
495 Metadata, DL, UV->getName(), UV);
496 }
497 return new VPReplicateRecipe(UV, Operands, /*IsSingleScalar=*/true, Mask,
498 Flags, Metadata, DL);
499 }
500
501 VPScalarIVStepsRecipe *
502 createScalarIVSteps(Instruction::BinaryOps InductionOpcode,
503 FPMathOperator *FPBinOp, VPValue *IV, VPValue *Step,
504 VPValue *VF, DebugLoc DL) {
505 return tryInsertInstruction(R: new VPScalarIVStepsRecipe(
506 IV, Step, VF, InductionOpcode,
507 FPBinOp ? FPBinOp->getFastMathFlags() : FastMathFlags(), DL));
508 }
509
510 VPExpandSCEVRecipe *createExpandSCEV(const SCEV *Expr) {
511 return tryInsertInstruction(R: new VPExpandSCEVRecipe(Expr));
512 }
513
514 VPVectorPointerRecipe *
515 createVectorPointer(VPValue *Ptr, Type *SourceElementTy, VPValue *Stride,
516 GEPNoWrapFlags GEPFlags, DebugLoc DL) {
517 return tryInsertInstruction(
518 R: new VPVectorPointerRecipe(Ptr, SourceElementTy, Stride, GEPFlags, DL));
519 }
520
521 /// Create a vector pointer recipe for a consecutive memory access to \p Ptr
522 /// with element type \p SourceElementTy.
523 VPSingleDefRecipe *createConsecutiveVectorPointer(VPValue *Ptr,
524 Type *SourceElementTy,
525 bool Reverse, DebugLoc DL);
526
527 VPWidenMemIntrinsicRecipe *createWidenMemIntrinsic(
528 Intrinsic::ID VectorIntrinsicID, ArrayRef<VPValue *> CallArguments,
529 Type *Ty, Align Alignment, const VPIRMetadata &MD, DebugLoc DL) {
530 return tryInsertInstruction(R: new VPWidenMemIntrinsicRecipe(
531 VectorIntrinsicID, CallArguments, Ty, Alignment, MD, DL));
532 }
533
534 /// Create a recipe widening \p Load, loading from \p Addr with \p Mask (may
535 /// be null).
536 VPWidenLoadRecipe *createWidenLoad(LoadInst &Load, VPValue *Addr,
537 VPValue *Mask, bool Consecutive,
538 const VPIRMetadata &Metadata,
539 DebugLoc DL) {
540 return tryInsertInstruction(
541 R: new VPWidenLoadRecipe(Load, Addr, Mask, Consecutive, Metadata, DL));
542 }
543
544 /// Create a recipe widening \p Store, storing \p StoredVal to \p Addr with
545 /// \p Mask (may be null).
546 VPWidenStoreRecipe *createWidenStore(StoreInst &Store, VPValue *Addr,
547 VPValue *StoredVal, VPValue *Mask,
548 bool Consecutive,
549 const VPIRMetadata &Metadata,
550 DebugLoc DL) {
551 return tryInsertInstruction(R: new VPWidenStoreRecipe(
552 Store, Addr, StoredVal, Mask, Consecutive, Metadata, DL));
553 }
554
555 //===--------------------------------------------------------------------===//
556 // RAII helpers.
557 //===--------------------------------------------------------------------===//
558
559 /// RAII object that stores the current insertion point and restores it when
560 /// the object is destroyed.
561 class InsertPointGuard {
562 VPBuilder &Builder;
563 VPInsertPoint InsertPt;
564
565 public:
566 InsertPointGuard(VPBuilder &B) : Builder(B), InsertPt(B.InsertPt) {}
567
568 InsertPointGuard(const InsertPointGuard &) = delete;
569 InsertPointGuard &operator=(const InsertPointGuard &) = delete;
570
571 ~InsertPointGuard() { Builder.restoreIP(IP: InsertPt); }
572 };
573};
574
575/// TODO: The following VectorizationFactor was pulled out of
576/// LoopVectorizationCostModel class. LV also deals with
577/// VectorizerParams::VectorizationFactor.
578/// We need to streamline them.
579
580/// Information about vectorization costs.
581struct VectorizationFactor {
582 /// Vector width with best cost.
583 ElementCount Width;
584
585 /// Cost of the loop with that width.
586 InstructionCost Cost;
587
588 /// Cost of the scalar loop.
589 InstructionCost ScalarCost;
590
591 /// The minimum trip count required to make vectorization profitable, e.g. due
592 /// to runtime checks.
593 ElementCount MinProfitableTripCount;
594
595 VectorizationFactor(ElementCount Width, InstructionCost Cost,
596 InstructionCost ScalarCost)
597 : Width(Width), Cost(Cost), ScalarCost(ScalarCost) {}
598
599 /// Width 1 means no vectorization, cost 0 means uncomputed cost.
600 static VectorizationFactor Disabled() {
601 return {ElementCount::getFixed(MinVal: 1), 0, 0};
602 }
603
604 bool operator==(const VectorizationFactor &rhs) const {
605 return Width == rhs.Width && Cost == rhs.Cost;
606 }
607
608 bool operator!=(const VectorizationFactor &rhs) const {
609 return !(*this == rhs);
610 }
611};
612
613/// A class that represents two vectorization factors (initialized with 0 by
614/// default). One for fixed-width vectorization and one for scalable
615/// vectorization. This can be used by the vectorizer to choose from a range of
616/// fixed and/or scalable VFs in order to find the most cost-effective VF to
617/// vectorize with.
618struct FixedScalableVFPair {
619 ElementCount FixedVF;
620 ElementCount ScalableVF;
621
622 FixedScalableVFPair()
623 : FixedVF(ElementCount::getFixed(MinVal: 0)),
624 ScalableVF(ElementCount::getScalable(MinVal: 0)) {}
625 FixedScalableVFPair(const ElementCount &Max) : FixedScalableVFPair() {
626 *(Max.isScalable() ? &ScalableVF : &FixedVF) = Max;
627 }
628 FixedScalableVFPair(const ElementCount &FixedVF,
629 const ElementCount &ScalableVF)
630 : FixedVF(FixedVF), ScalableVF(ScalableVF) {
631 assert(!FixedVF.isScalable() && ScalableVF.isScalable() &&
632 "Invalid scalable properties");
633 }
634
635 static FixedScalableVFPair getNone() { return FixedScalableVFPair(); }
636
637 /// \return true if either fixed- or scalable VF is non-zero.
638 explicit operator bool() const { return FixedVF || ScalableVF; }
639
640 /// \return true if either fixed- or scalable VF is a valid vector VF.
641 bool hasVector() const { return FixedVF.isVector() || ScalableVF.isVector(); }
642};
643
644/// Holds state needed to make cost decisions before computing costs per-VF,
645/// including the maximum VFs.
646class VFSelectionContext {
647 /// \return True if maximizing vector bandwidth is enabled by the target or
648 /// user options, for the given register kind (scalable or fixed-width).
649 bool useMaxBandwidth(bool IsScalable) const;
650
651 /// \return the maximized element count based on the targets vector
652 /// registers and the loop trip-count, but limited to a maximum safe VF.
653 /// This is a helper function of computeFeasibleMaxVF.
654 ElementCount getMaximizedVFForTarget(unsigned MaxTripCount,
655 unsigned SmallestType,
656 unsigned WidestType,
657 ElementCount MaxSafeVF, unsigned UserIC,
658 bool FoldTailByMasking,
659 bool RequiresScalarEpilogue);
660
661 /// If \p VF * \p UserIC > MaxTripcount, clamps VF to the next lower VF
662 /// that results in VF * UserIC <= MaxTripCount.
663 ElementCount clampVFByMaxTripCount(ElementCount VF, unsigned MaxTripCount,
664 unsigned UserIC, bool FoldTailByMasking,
665 bool RequiresScalarEpilogue) const;
666
667 /// Checks if scalable vectorization is supported and enabled. Caches the
668 /// result to avoid repeated debug dumps for repeated queries.
669 bool isScalableVectorizationAllowed();
670
671 /// \return the maximum legal scalable VF, based on the safe max number
672 /// of elements.
673 ElementCount getMaxLegalScalableVF(unsigned MaxSafeElements);
674
675 /// Initializes the value of vscale used for tuning the cost model. If
676 /// vscale_range.min == vscale_range.max then return vscale_range.max, else
677 /// return the value returned by the corresponding TTI method.
678 void initializeVScaleForTuning();
679
680 const TargetTransformInfo &TTI;
681 const LoopVectorizationLegality *Legal;
682 const Loop *TheLoop;
683 const Function &F;
684 PredicatedScalarEvolution &PSE;
685 DemandedBits *DB;
686 OptimizationRemarkEmitter *ORE;
687 const LoopVectorizeHints *Hints;
688
689 /// Cached result of isScalableVectorizationAllowed.
690 std::optional<bool> IsScalableVectorizationAllowed;
691
692 /// Used to store the value of vscale used for tuning the cost model. It is
693 /// initialized during object construction.
694 std::optional<unsigned> VScaleForTuning;
695
696 /// The highest VF possible for this loop, without using MaxBandwidth.
697 FixedScalableVFPair MaxPermissibleVFWithoutMaxBW;
698
699 /// All element types found in the loop.
700 SmallPtrSet<Type *, 16> ElementTypesInLoop;
701
702 /// PHINodes of the reductions that should be expanded in-loop. Set by
703 /// collectInLoopReductions.
704 SmallPtrSet<PHINode *, 4> InLoopReductions;
705
706 /// A Map of inloop reduction operations and their immediate chain operand.
707 /// FIXME: This can be removed once reductions can be costed correctly in
708 /// VPlan. This was added to allow quick lookup of the inloop operations.
709 /// Set by collectInLoopReductions.
710 DenseMap<Instruction *, Instruction *> InLoopReductionImmediateChains;
711
712 /// Maximum safe number of elements to be processed per vector iteration,
713 /// which do not prevent store-load forwarding and are safe with regard to the
714 /// memory dependencies. Required for EVL-based vectorization, where this
715 /// value is used as the upper bound of the safe AVL. Set by
716 /// computeFeasibleMaxVF.
717 std::optional<unsigned> MaxSafeElements;
718
719 /// Map of scalar integer values to the smallest bitwidth they can be legally
720 /// represented as. The vector equivalents of these values should be truncated
721 /// to this type.
722 MapVector<Instruction *, uint64_t> MinBWs;
723
724public:
725 /// The kind of cost that we are calculating.
726 const TTI::TargetCostKind CostKind;
727
728 /// Whether this loop should be optimized for size based on function attribute
729 /// or profile information.
730 const bool OptForSize;
731
732 VFSelectionContext(const TargetTransformInfo &TTI,
733 const LoopVectorizationLegality *Legal,
734 const Loop *TheLoop, const Function &F,
735 PredicatedScalarEvolution &PSE, DemandedBits *DB,
736 OptimizationRemarkEmitter *ORE,
737 const LoopVectorizeHints *Hints, bool OptForSize)
738 : TTI(TTI), Legal(Legal), TheLoop(TheLoop), F(F), PSE(PSE), DB(DB),
739 ORE(ORE), Hints(Hints),
740 CostKind(F.hasMinSize() ? TTI::TCK_CodeSize : TTI::TCK_RecipThroughput),
741 OptForSize(OptForSize) {
742 initializeVScaleForTuning();
743 }
744
745 /// \return The vscale value used for tuning the cost model.
746 std::optional<unsigned> getVScaleForTuning() const { return VScaleForTuning; }
747
748 const TargetTransformInfo &getTTI() const { return TTI; }
749
750 PredicatedScalarEvolution &getPSE() const { return PSE; }
751
752 /// \return The loop being analyzed.
753 const Loop *getLoop() const { return TheLoop; }
754
755 /// \return The vectorization hints for the loop being analyzed.
756 const LoopVectorizeHints &getHints() const { return *Hints; }
757
758 /// Returns true if epilogue vectorization is considered profitable for a
759 /// main loop with vectorization factor \p VF and interleave count \p IC.
760 bool isEpilogueVectorizationProfitable(ElementCount VF, unsigned IC) const;
761
762 /// \return True if register pressure should be considered for the given VF.
763 bool shouldConsiderRegPressureForVF(ElementCount VF) const;
764
765 /// \return True if scalable vectors are supported by the target or forced.
766 bool supportsScalableVectors() const;
767
768 /// Collect element types in the loop that need widening.
769 void collectElementTypesForWidening(
770 const SmallPtrSetImpl<const Value *> *ValuesToIgnore = nullptr);
771
772 /// \return The size (in bits) of the smallest and widest types in the code
773 /// that need to be vectorized. We ignore values that remain scalar such as
774 /// 64 bit loop indices.
775 std::pair<unsigned, unsigned> getSmallestAndWidestTypes() const;
776
777 /// \return An upper bound for the vectorization factors for both
778 /// fixed and scalable vectorization, where the minimum-known number of
779 /// elements is a power-of-2 larger than zero. If scalable vectorization is
780 /// disabled or unsupported, then the scalable part will be equal to
781 /// ElementCount::getScalable(0). Also sets MaxSafeElements.
782 FixedScalableVFPair computeFeasibleMaxVF(unsigned MaxTripCount,
783 ElementCount UserVF, unsigned UserIC,
784 bool FoldTailByMasking,
785 bool RequiresScalarEpilogue);
786
787 /// Return maximum safe number of elements to be processed per vector
788 /// iteration, which do not prevent store-load forwarding and are safe with
789 /// regard to the memory dependencies. Required for EVL-based VPlans to
790 /// correctly calculate AVL (application vector length) as min(remaining AVL,
791 /// MaxSafeElements). Set by computeFeasibleMaxVF.
792 /// TODO: need to consider adjusting cost model to use this value as a
793 /// vectorization factor for EVL-based vectorization.
794 std::optional<unsigned> getMaxSafeElements() const { return MaxSafeElements; }
795
796 /// Returns true if we should use strict in-order reductions for the given
797 /// RdxDesc. This is true if the -enable-strict-reductions flag is passed,
798 /// the IsOrdered flag of RdxDesc is set and we do not allow reordering
799 /// of FP operations.
800 bool useOrderedReductions(const RecurrenceDescriptor &RdxDesc) const;
801
802 /// Returns true if the target machine supports a masked load (if \p IsLoad)
803 /// or masked store of scalar type \p ScalarTy with \p Alignment in address
804 /// space \p AddressSpace. The caller must ensure the access is consecutive or
805 /// part of an interleave group.
806 bool isLegalMaskedLoadOrStore(bool IsLoad, Type *ScalarTy, Align Alignment,
807 unsigned AddressSpace) const;
808
809 /// Returns true if the target machine can represent \p V as a masked gather
810 /// or scatter operation.
811 bool isLegalGatherOrScatter(Value *V, ElementCount VF) const;
812
813 /// Split reductions into those that happen in the loop, and those that
814 /// happen outside. In-loop reductions are collected into InLoopReductions.
815 /// InLoopReductionImmediateChains is filled with each in-loop reduction
816 /// operation and its immediate chain operand for use during cost modelling.
817 void collectInLoopReductions();
818
819 /// Returns true if the Phi is part of an inloop reduction.
820 bool isInLoopReduction(PHINode *Phi) const {
821 return InLoopReductions.contains(Ptr: Phi);
822 }
823
824 /// Returns the set of in-loop reduction PHIs.
825 const SmallPtrSetImpl<PHINode *> &getInLoopReductions() const {
826 return InLoopReductions;
827 }
828
829 /// Returns the immediate chain operand of in-loop reduction operation \p I,
830 /// or nullptr if \p I is not an in-loop reduction operation.
831 Instruction *getInLoopReductionImmediateChain(Instruction *I) const {
832 return InLoopReductionImmediateChains.lookup(Val: I);
833 }
834
835 /// Check whether vectorization would require runtime checks. When optimizing
836 /// for size, returning true here aborts vectorization.
837 bool runtimeChecksRequired();
838
839 /// Returns a scalable VF to use for outer-loop vectorization if the target
840 /// supports it and a fixed VF otherwise.
841 FixedScalableVFPair computeVPlanOuterloopVF(ElementCount UserVF);
842
843 /// Compute smallest bitwidth each instruction can be represented with.
844 /// The vector equivalents of these instructions should be truncated to this
845 /// type.
846 void computeMinimalBitwidths();
847
848 /// \returns The smallest bitwidth each instruction can be represented with.
849 const MapVector<Instruction *, uint64_t> &getMinimalBitwidths() const {
850 return MinBWs;
851 }
852};
853
854/// Planner drives the vectorization process after having passed
855/// Legality checks.
856class LoopVectorizationPlanner {
857 /// The loop that we evaluate.
858 Loop *OrigLoop;
859
860 /// Loop Info analysis.
861 LoopInfo *LI;
862
863 /// The dominator tree.
864 DominatorTree *DT;
865
866 /// Target Library Info.
867 const TargetLibraryInfo *TLI;
868
869 /// Target Transform Info.
870 const TargetTransformInfo &TTI;
871
872 /// The legality analysis.
873 LoopVectorizationLegality *Legal;
874
875 /// The profitability analysis. Cleared after making cost based decisions.
876 std::unique_ptr<LoopVectorizationCostModel> CM;
877
878 /// VF selection state independent of cost-modeling decisions.
879 VFSelectionContext &Config;
880
881 /// The interleaved access analysis.
882 InterleavedAccessInfo &IAI;
883
884 PredicatedScalarEvolution &PSE;
885
886 OptimizationRemarkEmitter *ORE;
887
888 SmallVector<VPlanPtr, 4> VPlans;
889
890 /// Profitable vector factors.
891 SmallVector<VectorizationFactor, 8> ProfitableVFs;
892
893 /// A builder used to construct the current plan.
894 VPBuilder Builder;
895
896 /// Computes the cost of \p Plan for vectorization factor \p VF.
897 ///
898 /// The current implementation requires access to the
899 /// LoopVectorizationLegality to handle inductions and reductions, which is
900 /// why it is kept separate from the VPlan-only cost infrastructure.
901 ///
902 /// TODO: Move to VPlan::cost once the use of LoopVectorizationLegality has
903 /// been retired.
904 InstructionCost cost(VPlan &Plan, ElementCount VF, VPRegisterUsage *RU) const;
905
906 /// Precompute costs for certain instructions using the legacy cost model. The
907 /// function is used to bring up the VPlan-based cost model to initially avoid
908 /// taking different decisions due to inaccuracies in the legacy cost model.
909 InstructionCost precomputeCosts(VPlan &Plan, ElementCount VF,
910 VPCostContext &CostCtx) const;
911
912public:
913 LoopVectorizationPlanner(
914 Loop *L, LoopInfo *LI, DominatorTree *DT, const TargetLibraryInfo *TLI,
915 const TargetTransformInfo &TTI, LoopVectorizationLegality *Legal,
916 std::unique_ptr<LoopVectorizationCostModel> CM,
917 VFSelectionContext &Config, InterleavedAccessInfo &IAI,
918 PredicatedScalarEvolution &PSE, OptimizationRemarkEmitter *ORE);
919
920 ~LoopVectorizationPlanner();
921
922 /// Return the cost model. Must not be called after clearCostModel().
923 LoopVectorizationCostModel &getCostModel() {
924 assert(CM && "Cost model has already been cleared");
925 return *CM;
926 }
927
928 /// Destroy the cost model.
929 void clearCostModel();
930
931 /// Build VPlans for the specified \p UserVF and \p UserIC if they are
932 /// non-zero or all applicable candidate VFs otherwise. If vectorization and
933 /// interleaving should be avoided up-front, no plans are generated.
934 void plan(ElementCount UserVF, unsigned UserIC);
935
936 /// Return the VPlan for \p VF. At the moment, there is always a single VPlan
937 /// for each VF.
938 VPlan &getPlanFor(ElementCount VF) const;
939
940 /// Compute and return the most profitable vectorization factor and the
941 /// corresponding best VPlan. Also collect all profitable VFs in
942 /// ProfitableVFs.
943 std::pair<VectorizationFactor, VPlan *> computeBestVF();
944
945 /// \return The desired interleave count.
946 /// If interleave count has been specified by metadata it will be returned.
947 /// Otherwise, the interleave count is computed and returned. VF and LoopCost
948 /// are the selected vectorization factor and the cost of the selected VF.
949 unsigned selectInterleaveCount(VPlan &Plan, ElementCount VF,
950 InstructionCost LoopCost);
951
952 /// Generate the IR code for the vectorized loop captured in VPlan \p BestPlan
953 /// according to the best selected \p VF and \p UF.
954 ///
955 /// TODO: \p EpilogueVecKind should be removed once the re-use issue has been
956 /// fixed.
957 ///
958 /// Returns a mapping of SCEVs to their expanded IR values.
959 /// Note that this is a temporary workaround needed due to the current
960 /// epilogue handling.
961 enum class EpilogueVectorizationKind {
962 None, ///< Not part of epilogue vectorization.
963 MainLoop, ///< Vectorizing the main loop of epilogue vectorization.
964 Epilogue ///< Vectorizing the epilogue loop.
965 };
966 DenseMap<const SCEV *, Value *>
967 executePlan(ElementCount VF, unsigned UF, VPlan &BestPlan,
968 InnerLoopVectorizer &LB, DominatorTree *DT,
969 EpilogueVectorizationKind EpilogueVecKind =
970 EpilogueVectorizationKind::None);
971
972#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
973 void printPlans(raw_ostream &O);
974#endif
975
976 /// Look through the existing plans and return true if we have one with
977 /// vectorization factor \p VF.
978 bool hasPlanWithVF(ElementCount VF) const {
979 return any_of(Range: VPlans,
980 P: [&](const VPlanPtr &Plan) { return Plan->hasVF(VF); });
981 }
982
983 /// Test a \p Predicate on a \p Range of VF's. Return the value of applying
984 /// \p Predicate on Range.Start, possibly decreasing Range.End such that the
985 /// returned value holds for the entire \p Range.
986 static bool
987 getDecisionAndClampRange(const std::function<bool(ElementCount)> &Predicate,
988 VFRange &Range);
989
990 /// \return A VPlan for the most profitable epilogue vectorization, with its
991 /// VF narrowed to the chosen factor. The returned plan is a duplicate.
992 /// Returns nullptr if epilogue vectorization is not supported or not
993 /// profitable for the loop. \p ScalarEpilogueAllowed indicates whether the
994 /// epilogue lowering policy permits creating a scalar epilogue at all.
995 std::unique_ptr<VPlan> selectBestEpiloguePlan(VPlan &MainPlan,
996 ElementCount MainLoopVF,
997 unsigned IC,
998 bool ScalarEpilogueAllowed);
999
1000 /// Emit remarks for recipes with invalid costs in the available VPlans.
1001 void emitInvalidCostRemarks(OptimizationRemarkEmitter *ORE);
1002
1003 /// Create a check to \p Plan to see if the vector loop should be executed
1004 /// based on its trip count.
1005 void addMinimumIterationCheck(VPlan &Plan, ElementCount VF, unsigned UF,
1006 ElementCount MinProfitableTripCount) const;
1007
1008 /// Attach the runtime checks of \p RTChecks to \p Plan.
1009 void attachRuntimeChecks(VPlan &Plan, GeneratedRTChecks &RTChecks,
1010 bool HasBranchWeights) const;
1011
1012 /// Update loop metadata and profile info for both the scalar remainder loop
1013 /// and \p VectorLoop, if it exists. Keeps all loop hints from the original
1014 /// loop on the vector loop and replaces vectorizer-specific metadata. The
1015 /// loop ID of the original loop \p OrigLoopID must be passed, together with
1016 /// the average trip count and invocation weight of the original loop (\p
1017 /// OrigAverageTripCount and \p OrigLoopInvocationWeight respectively). They
1018 /// cannot be retrieved after the plan has been executed, as the original loop
1019 /// may have been removed. \p UnrollVectorizedLoop indicates whether the
1020 /// target wants the vector loop left eligible for runtime unrolling.
1021 void updateLoopMetadataAndProfileInfo(
1022 Loop *VectorLoop, VPBasicBlock *HeaderVPBB, const VPlan &Plan,
1023 bool VectorizingEpilogue, MDNode *OrigLoopID,
1024 std::optional<unsigned> OrigAverageTripCount,
1025 unsigned OrigLoopInvocationWeight, unsigned EstimatedVFxUF,
1026 bool DisableRuntimeUnroll, bool UnrollVectorizedLoop);
1027
1028private:
1029 /// Build an initial VPlan, with HCFG wrapping the original scalar loop and
1030 /// scalar transformations applied. Returns null if an initial VPlan cannot
1031 /// be built.
1032 VPlanPtr tryToBuildVPlan1();
1033
1034 /// Build a VPlan using VPRecipes according to the information gathered by
1035 /// Legal and VPlan-based analysis. For outer loops, performs basic recipe
1036 /// conversion only. For inner loops, \p Range's largest included VF is
1037 /// restricted to the maximum VF the returned VPlan is valid for. If no VPlan
1038 /// can be built for the input range, set the largest included VF to the
1039 /// maximum VF for which no plan could be built. Each VPlan is built starting
1040 /// from a copy of \p InitialPlan, which is a plain CFG VPlan wrapping the
1041 /// original scalar loop.
1042 VPlanPtr tryToBuildVPlan(VPlanPtr InitialPlan, VFRange &Range);
1043
1044 /// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
1045 /// based on \p VPlan1 and according to the information gathered by Legal
1046 /// when it checked if it is legal to vectorize the loop.
1047 void buildVPlans(VPlan &VPlan1, ElementCount MinVF, ElementCount MaxVF);
1048
1049 /// Add ComputeReductionResult recipes to the middle block to compute the
1050 /// final reduction results. Add Select recipes to the latch block when
1051 /// folding tail, to feed ComputeReductionResult with the last or penultimate
1052 /// iteration values according to the header mask.
1053 void addReductionResultComputation(VPlanPtr &Plan,
1054 VPRecipeBuilder &RecipeBuilder,
1055 ElementCount MinVF);
1056
1057 /// Returns true if the per-lane cost of VectorizationFactor A is lower than
1058 /// that of B.
1059 bool isMoreProfitable(const VectorizationFactor &A,
1060 const VectorizationFactor &B, bool HasTail,
1061 bool IsEpilogue = false) const;
1062
1063 /// Returns true if the per-lane cost of VectorizationFactor A is lower than
1064 /// that of B in the context of vectorizing a loop with known \p MaxTripCount.
1065 bool isMoreProfitable(const VectorizationFactor &A,
1066 const VectorizationFactor &B,
1067 const unsigned MaxTripCount, bool HasTail,
1068 bool IsEpilogue = false) const;
1069
1070 /// Determines if we have the infrastructure to vectorize the loop and its
1071 /// epilogue, assuming the main loop is vectorized by \p MainPlan.
1072 bool isCandidateForEpilogueVectorization(VPlan &MainPlan) const;
1073};
1074
1075/// A helper function that returns true if the given type is irregular. The
1076/// type is irregular if its allocated size doesn't equal the store size of an
1077/// element of the corresponding vector type.
1078inline bool hasIrregularType(Type *Ty, const DataLayout &DL) {
1079 // Determine if an array of N elements of type Ty is "bitcast compatible"
1080 // with a <N x Ty> vector.
1081 // This is only true if there is no padding between the array elements.
1082 return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
1083}
1084
1085} // namespace llvm
1086
1087#endif // LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONPLANNER_H
1088