1//===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements a set of utility VPlan to VPlan transformations.
11///
12//===----------------------------------------------------------------------===//
13
14#include "VPlanTransforms.h"
15#include "VPRecipeBuilder.h"
16#include "VPlan.h"
17#include "VPlanAnalysis.h"
18#include "VPlanCFG.h"
19#include "VPlanDominatorTree.h"
20#include "VPlanHelpers.h"
21#include "VPlanPatternMatch.h"
22#include "VPlanUtils.h"
23#include "VPlanVerifier.h"
24#include "llvm/ADT/APInt.h"
25#include "llvm/ADT/PostOrderIterator.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SetVector.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/TypeSwitch.h"
30#include "llvm/Analysis/IVDescriptors.h"
31#include "llvm/Analysis/Loads.h"
32#include "llvm/Analysis/LoopAccessAnalysis.h"
33#include "llvm/Analysis/LoopInfo.h"
34#include "llvm/Analysis/MemoryLocation.h"
35#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
36#include "llvm/Analysis/ScopedNoAliasAA.h"
37#include "llvm/Analysis/VectorUtils.h"
38#include "llvm/IR/Intrinsics.h"
39#include "llvm/IR/MDBuilder.h"
40#include "llvm/IR/Metadata.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/TypeSize.h"
43#include "llvm/Transforms/Utils/LoopUtils.h"
44#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
45
46using namespace llvm;
47using namespace VPlanPatternMatch;
48using namespace SCEVPatternMatch;
49
50bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
51 VPlan &Plan, const TargetLibraryInfo &TLI) {
52
53 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
54 Plan.getVectorLoopRegion());
55 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
56 // Skip blocks outside region
57 if (!VPBB->getParent())
58 break;
59 VPRecipeBase *Term = VPBB->getTerminator();
60 auto EndIter = Term ? Term->getIterator() : VPBB->end();
61 // Introduce each ingredient into VPlan.
62 for (VPRecipeBase &Ingredient :
63 make_early_inc_range(Range: make_range(x: VPBB->begin(), y: EndIter))) {
64
65 VPValue *VPV = Ingredient.getVPSingleValue();
66 if (!VPV->getUnderlyingValue())
67 continue;
68
69 Instruction *Inst = cast<Instruction>(Val: VPV->getUnderlyingValue());
70
71 // Atomic accesses and fences have ordering/atomicity semantics that
72 // cannot be preserved by lane-wise widening.
73 if (isa<AtomicRMWInst, AtomicCmpXchgInst, FenceInst>(Val: Inst))
74 return false;
75
76 VPRecipeBase *NewRecipe = nullptr;
77 if (auto *PhiR = dyn_cast<VPPhi>(Val: &Ingredient)) {
78 auto *Phi = cast<PHINode>(Val: PhiR->getUnderlyingValue());
79 NewRecipe = new VPWidenPHIRecipe(PhiR->operands(), PhiR->getDebugLoc(),
80 Phi->getName());
81 } else if (auto *VPI = dyn_cast<VPInstruction>(Val: &Ingredient)) {
82 assert(!isa<PHINode>(Inst) && "phis should be handled above");
83 // Create VPWidenMemoryRecipe for loads and stores.
84 if (LoadInst *Load = dyn_cast<LoadInst>(Val: Inst)) {
85 NewRecipe = new VPWidenLoadRecipe(
86 *Load, Ingredient.getOperand(N: 0), nullptr /*Mask*/,
87 false /*Consecutive*/, *VPI, Ingredient.getDebugLoc());
88 } else if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst)) {
89 NewRecipe = new VPWidenStoreRecipe(
90 *Store, Ingredient.getOperand(N: 1), Ingredient.getOperand(N: 0),
91 nullptr /*Mask*/, false /*Consecutive*/, *VPI,
92 Ingredient.getDebugLoc());
93 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Inst)) {
94 NewRecipe = new VPWidenGEPRecipe(GEP->getSourceElementType(),
95 Ingredient.operands(), *VPI,
96 Ingredient.getDebugLoc(), GEP);
97 } else if (CallInst *CI = dyn_cast<CallInst>(Val: Inst)) {
98 Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, TLI: &TLI);
99 if (VectorID == Intrinsic::not_intrinsic)
100 return false;
101
102 // The noalias.scope.decl intrinsic declares a noalias scope that
103 // is valid for a single iteration. Emitting it as a single-scalar
104 // replicate would incorrectly extend the scope across multiple
105 // original iterations packed into one vector iteration.
106 // FIXME: If we want to vectorize this loop, then we have to drop
107 // all the associated !alias.scope and !noalias.
108 if (VectorID == Intrinsic::experimental_noalias_scope_decl)
109 return false;
110
111 // These intrinsics are recognized by getVectorIntrinsicIDForCall
112 // but are not widenable. Emit them as replicate instead of widening.
113 if (VectorID == Intrinsic::assume ||
114 VectorID == Intrinsic::lifetime_end ||
115 VectorID == Intrinsic::lifetime_start ||
116 VectorID == Intrinsic::sideeffect ||
117 VectorID == Intrinsic::pseudoprobe) {
118 // If the operand of llvm.assume holds before vectorization, it will
119 // also hold per lane.
120 // llvm.pseudoprobe requires to be duplicated per lane for accurate
121 // sample count.
122 const bool IsSingleScalar = VectorID != Intrinsic::assume &&
123 VectorID != Intrinsic::pseudoprobe;
124 NewRecipe = new VPReplicateRecipe(CI, Ingredient.operands(),
125 /*IsSingleScalar=*/IsSingleScalar,
126 /*Mask=*/nullptr, *VPI, *VPI,
127 Ingredient.getDebugLoc());
128 } else {
129 NewRecipe = new VPWidenIntrinsicRecipe(
130 *CI, VectorID, drop_end(RangeOrContainer: Ingredient.operands()), CI->getType(),
131 VPIRFlags(*CI), *VPI, CI->getDebugLoc());
132 }
133 } else if (auto *CI = dyn_cast<CastInst>(Val: Inst)) {
134 NewRecipe = new VPWidenCastRecipe(
135 CI->getOpcode(), Ingredient.getOperand(N: 0), CI->getType(), CI,
136 VPIRFlags(*CI), VPIRMetadata(*CI));
137 } else {
138 NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI,
139 *VPI, Ingredient.getDebugLoc());
140 }
141 } else {
142 assert(isa<VPWidenIntOrFpInductionRecipe>(&Ingredient) &&
143 "inductions must be created earlier");
144 continue;
145 }
146
147 NewRecipe->insertBefore(InsertPos: &Ingredient);
148 if (NewRecipe->getNumDefinedValues() == 1)
149 VPV->replaceAllUsesWith(New: NewRecipe->getVPSingleValue());
150 else
151 assert(NewRecipe->getNumDefinedValues() == 0 &&
152 "Only recpies with zero or one defined values expected");
153 Ingredient.eraseFromParent();
154 }
155 }
156 return true;
157}
158
159/// Helper for extra no-alias checks via known-safe recipe and SCEV.
160class SinkStoreInfo {
161 SmallPtrSet<VPReplicateRecipe *, 4> ExcludeRecipes;
162 VPReplicateRecipe &GroupLeader;
163 PredicatedScalarEvolution *PSE = nullptr;
164 const Loop *L = nullptr;
165
166 // Return true if \p A and \p B are known to not alias for all VFs in the
167 // plan, checked via the distance between the accesses
168 bool isNoAliasViaDistance(VPReplicateRecipe *A, VPReplicateRecipe *B) const {
169 if (A->getOpcode() != Instruction::Store ||
170 B->getOpcode() != Instruction::Store)
171 return false;
172
173 if (!PSE || !L)
174 return A == B;
175
176 VPValue *AddrA = A->getOperand(N: 1);
177 const SCEV *SCEVA = vputils::getSCEVExprForVPValue(V: AddrA, PSE&: *PSE, L);
178 VPValue *AddrB = B->getOperand(N: 1);
179 const SCEV *SCEVB = vputils::getSCEVExprForVPValue(V: AddrB, PSE&: *PSE, L);
180 if (isa<SCEVCouldNotCompute>(Val: SCEVA) || isa<SCEVCouldNotCompute>(Val: SCEVB))
181 return false;
182
183 const APInt *Distance;
184 ScalarEvolution &SE = *PSE->getSE();
185 if (!match(S: SE.getMinusSCEV(LHS: SCEVA, RHS: SCEVB), P: m_scev_APInt(C&: Distance)))
186 return false;
187
188 const DataLayout &DL = SE.getDataLayout();
189 Type *TyA = A->getOperand(N: 0)->getScalarType();
190 uint64_t SizeA = DL.getTypeStoreSize(Ty: TyA);
191 Type *TyB = B->getOperand(N: 0)->getScalarType();
192 uint64_t SizeB = DL.getTypeStoreSize(Ty: TyB);
193
194 // Use the maximum store size to ensure no overlap from either direction.
195 // Currently only handles fixed sizes, as it is only used for
196 // replicating VPReplicateRecipes.
197 uint64_t MaxStoreSize = std::max(a: SizeA, b: SizeB);
198
199 auto VFs = B->getParent()->getPlan()->vectorFactors();
200 ElementCount MaxVF = *max_element(Range&: VFs, C: ElementCount::isKnownLT);
201 if (MaxVF.isScalable())
202 return false;
203 return Distance->abs().uge(
204 RHS: MaxVF.multiplyCoefficientBy(RHS: MaxStoreSize).getFixedValue());
205 }
206
207public:
208 SinkStoreInfo(ArrayRef<VPReplicateRecipe *> ExcludeRecipes,
209 VPReplicateRecipe &GroupLeader, PredicatedScalarEvolution &PSE,
210 const Loop &L)
211 : ExcludeRecipes(ExcludeRecipes.begin(), ExcludeRecipes.end()),
212 GroupLeader(GroupLeader), PSE(&PSE), L(&L) {}
213
214 SinkStoreInfo(VPReplicateRecipe &GroupLeader) : GroupLeader(GroupLeader) {}
215
216 /// Return true if \p R should be skipped during alias checking, either
217 /// because it's in the exclude set or because no-alias can be proven via
218 /// SCEV.
219 bool shouldSkip(VPRecipeBase &R) const {
220 auto *Store = dyn_cast<VPReplicateRecipe>(Val: &R);
221 return ExcludeRecipes.contains(Ptr: Store) ||
222 (Store && isNoAliasViaDistance(A: Store, B: &GroupLeader));
223 }
224};
225
226/// Check if a memory operation doesn't alias with memory operations using
227/// scoped noalias metadata, in blocks in the single-successor chain between \p
228/// FirstBB and \p LastBB. If \p SinkInfo is std::nullopt, only recipes that may
229/// write to memory are checked (for load hoisting). Otherwise recipes that both
230/// read and write memory are checked, and SCEV is used to prove no-alias
231/// between the group leader and other replicate recipes (for store sinking).
232static bool
233canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
234 VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
235 std::optional<SinkStoreInfo> SinkInfo = {}) {
236 bool CheckReads = SinkInfo.has_value();
237 for (VPBasicBlock *VPBB :
238 VPBlockUtils::blocksInSingleSuccessorChainBetween(FirstBB, LastBB)) {
239 for (VPRecipeBase &R : *VPBB) {
240 if (SinkInfo && SinkInfo->shouldSkip(R))
241 continue;
242
243 // Skip recipes that don't need checking.
244 if (!R.mayWriteToMemory() && !(CheckReads && R.mayReadFromMemory()))
245 continue;
246
247 auto Loc = vputils::getMemoryLocation(R);
248 if (!Loc)
249 // Conservatively assume aliasing for memory operations without
250 // location.
251 return false;
252
253 if (ScopedNoAliasAAResult::alias(LocA: *Loc, LocB: MemLoc) != AliasResult::NoAlias)
254 return false;
255 }
256 }
257 return true;
258}
259
260/// Get the value type of the replicate load or store. \p IsLoad indicates
261/// whether it is a load.
262static Type *getLoadStoreValueType(VPReplicateRecipe *R, bool IsLoad) {
263 return (IsLoad ? R : R->getOperand(N: 0))->getScalarType();
264}
265
266/// Collect either replicated Loads or Stores grouped by their address SCEV and
267/// their load-store type, in a deep-traversal of the vector loop region in \p
268/// Plan.
269template <unsigned Opcode>
270static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
271collectGroupedReplicateMemOps(
272 VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L,
273 function_ref<bool(VPReplicateRecipe *)> FilterFn) {
274 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
275 "Only Load and Store opcodes supported");
276 constexpr bool IsLoad = (Opcode == Instruction::Load);
277 SmallDenseMap<std::pair<const SCEV *, const Type *>,
278 SmallVector<VPReplicateRecipe *, 4>>
279 RecipesByAddressAndType;
280 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
281 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
282 for (VPRecipeBase &R : *VPBB) {
283 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
284 if (!RepR || RepR->getOpcode() != Opcode || !FilterFn(RepR))
285 continue;
286
287 // For loads, operand 0 is address; for stores, operand 1 is address.
288 VPValue *Addr = RepR->getOperand(N: IsLoad ? 0 : 1);
289 const Type *LoadStoreTy = getLoadStoreValueType(R: RepR, IsLoad);
290 const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(V: Addr, PSE, L);
291 if (!isa<SCEVCouldNotCompute>(Val: AddrSCEV))
292 RecipesByAddressAndType[{AddrSCEV, LoadStoreTy}].push_back(Elt: RepR);
293 }
294 }
295 auto Groups = to_vector(Range: RecipesByAddressAndType.values());
296 VPDominatorTree VPDT(Plan);
297 for (auto &Group : Groups) {
298 // Sort mem ops by dominance order, with earliest (most dominating) first.
299 stable_sort(Group, [&VPDT](VPReplicateRecipe *A, VPReplicateRecipe *B) {
300 return VPDT.properlyDominates(A, B);
301 });
302 }
303 return Groups;
304}
305
306static bool sinkScalarOperands(VPlan &Plan) {
307 auto Iter = vp_depth_first_deep(G: Plan.getEntry());
308 bool ScalarVFOnly = Plan.hasScalarVFOnly();
309 bool Changed = false;
310
311 SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList;
312 auto InsertIfValidSinkCandidate = [ScalarVFOnly, &WorkList](
313 VPBasicBlock *SinkTo, VPValue *Op) {
314 auto *Candidate =
315 dyn_cast_or_null<VPSingleDefRecipe>(Val: Op->getDefiningRecipe());
316 if (!Candidate)
317 return;
318
319 // We only know how to sink VPReplicateRecipes and VPScalarIVStepsRecipes
320 // for now.
321 if (!isa<VPReplicateRecipe, VPScalarIVStepsRecipe>(Val: Candidate))
322 return;
323
324 if (Candidate->getParent() == SinkTo ||
325 vputils::cannotHoistOrSinkRecipe(R: *Candidate, /*Sinking=*/true))
326 return;
327
328 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: Candidate))
329 if (!ScalarVFOnly && RepR->isSingleScalar())
330 return;
331
332 WorkList.insert(X: {SinkTo, Candidate});
333 };
334
335 // First, collect the operands of all recipes in replicate blocks as seeds for
336 // sinking.
337 for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Range&: Iter)) {
338 VPBasicBlock *EntryVPBB = VPR->getEntryBasicBlock();
339 if (!VPR->isReplicator() || EntryVPBB->getSuccessors().size() != 2)
340 continue;
341 VPBasicBlock *VPBB = cast<VPBasicBlock>(Val: EntryVPBB->getSuccessors().front());
342 if (VPBB->getSingleSuccessor() != VPR->getExitingBasicBlock())
343 continue;
344 for (auto &Recipe : *VPBB)
345 for (VPValue *Op : Recipe.operands())
346 InsertIfValidSinkCandidate(VPBB, Op);
347 }
348
349 // Try to sink each replicate or scalar IV steps recipe in the worklist.
350 for (unsigned I = 0; I != WorkList.size(); ++I) {
351 VPBasicBlock *SinkTo;
352 VPSingleDefRecipe *SinkCandidate;
353 std::tie(args&: SinkTo, args&: SinkCandidate) = WorkList[I];
354
355 // All recipe users of SinkCandidate must be in the same block SinkTo or all
356 // users outside of SinkTo must only use the first lane of SinkCandidate. In
357 // the latter case, we need to duplicate SinkCandidate.
358 auto UsersOutsideSinkTo =
359 make_filter_range(Range: SinkCandidate->users(), Pred: [SinkTo](VPUser *U) {
360 return cast<VPRecipeBase>(Val: U)->getParent() != SinkTo;
361 });
362 if (any_of(Range&: UsersOutsideSinkTo, P: [SinkCandidate](VPUser *U) {
363 return !U->usesFirstLaneOnly(Op: SinkCandidate);
364 }))
365 continue;
366 bool NeedsDuplicating = !UsersOutsideSinkTo.empty();
367
368 if (NeedsDuplicating) {
369 if (ScalarVFOnly)
370 continue;
371 VPSingleDefRecipe *Clone;
372 if (auto *SinkCandidateRepR =
373 dyn_cast<VPReplicateRecipe>(Val: SinkCandidate)) {
374 // TODO: Handle converting to uniform recipes as separate transform,
375 // then cloning should be sufficient here.
376 Clone = VPBuilder::createSingleScalarOp(
377 Opcode: SinkCandidateRepR->getOpcode(), Operands: SinkCandidate->operands(),
378 /*Mask=*/nullptr, Flags: *SinkCandidateRepR, Metadata: *SinkCandidateRepR,
379 DL: SinkCandidate->getDebugLoc(), UV: SinkCandidate->getUnderlyingInstr());
380 // TODO: add ".cloned" suffix to name of Clone's VPValue.
381 } else {
382 Clone = SinkCandidate->clone();
383 }
384
385 Clone->insertBefore(InsertPos: SinkCandidate);
386 SinkCandidate->replaceUsesWithIf(New: Clone, ShouldReplace: [SinkTo](VPUser &U, unsigned) {
387 return cast<VPRecipeBase>(Val: &U)->getParent() != SinkTo;
388 });
389 }
390 SinkCandidate->moveBefore(BB&: *SinkTo, I: SinkTo->getFirstNonPhi());
391 for (VPValue *Op : SinkCandidate->operands())
392 InsertIfValidSinkCandidate(SinkTo, Op);
393 Changed = true;
394 }
395 return Changed;
396}
397
398/// If \p R is a region with a VPBranchOnMaskRecipe in the entry block, return
399/// the mask.
400static VPValue *getPredicatedMask(VPRegionBlock *R) {
401 auto *EntryBB = dyn_cast<VPBasicBlock>(Val: R->getEntry());
402 if (!EntryBB || EntryBB->size() != 1 ||
403 !isa<VPBranchOnMaskRecipe>(Val: EntryBB->begin()))
404 return nullptr;
405
406 return cast<VPBranchOnMaskRecipe>(Val: &*EntryBB->begin())->getOperand(N: 0);
407}
408
409/// If \p R is a triangle region, return the 'then' block of the triangle.
410static VPBasicBlock *getPredicatedThenBlock(VPRegionBlock *R) {
411 auto *EntryBB = cast<VPBasicBlock>(Val: R->getEntry());
412 if (EntryBB->getNumSuccessors() != 2)
413 return nullptr;
414
415 auto *Succ0 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[0]);
416 auto *Succ1 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[1]);
417 if (!Succ0 || !Succ1)
418 return nullptr;
419
420 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
421 return nullptr;
422 if (Succ0->getSingleSuccessor() == Succ1)
423 return Succ0;
424 if (Succ1->getSingleSuccessor() == Succ0)
425 return Succ1;
426 return nullptr;
427}
428
429// Merge replicate regions in their successor region, if a replicate region
430// is connected to a successor replicate region with the same predicate by a
431// single, empty VPBasicBlock.
432static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
433 SmallPtrSet<VPRegionBlock *, 4> TransformedRegions;
434
435 // Collect replicate regions followed by an empty block, followed by another
436 // replicate region with matching masks to process front. This is to avoid
437 // iterator invalidation issues while merging regions.
438 SmallVector<VPRegionBlock *, 8> WorkList;
439 for (VPRegionBlock *Region1 : VPBlockUtils::blocksOnly<VPRegionBlock>(
440 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
441 if (!Region1->isReplicator())
442 continue;
443 auto *MiddleBasicBlock =
444 dyn_cast_or_null<VPBasicBlock>(Val: Region1->getSingleSuccessor());
445 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
446 continue;
447
448 auto *Region2 =
449 dyn_cast_or_null<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
450 if (!Region2 || !Region2->isReplicator())
451 continue;
452
453 VPValue *Mask1 = getPredicatedMask(R: Region1);
454 VPValue *Mask2 = getPredicatedMask(R: Region2);
455 if (!Mask1 || Mask1 != Mask2)
456 continue;
457
458 assert(Mask1 && Mask2 && "both region must have conditions");
459 WorkList.push_back(Elt: Region1);
460 }
461
462 // Move recipes from Region1 to its successor region, if both are triangles.
463 for (VPRegionBlock *Region1 : WorkList) {
464 if (TransformedRegions.contains(Ptr: Region1))
465 continue;
466 auto *MiddleBasicBlock = cast<VPBasicBlock>(Val: Region1->getSingleSuccessor());
467 auto *Region2 = cast<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
468
469 VPBasicBlock *Then1 = getPredicatedThenBlock(R: Region1);
470 VPBasicBlock *Then2 = getPredicatedThenBlock(R: Region2);
471 if (!Then1 || !Then2)
472 continue;
473
474 // Note: No fusion-preventing memory dependencies are expected in either
475 // region. Such dependencies should be rejected during earlier dependence
476 // checks, which guarantee accesses can be re-ordered for vectorization.
477 //
478 // Move recipes to the successor region.
479 for (VPRecipeBase &ToMove : make_early_inc_range(Range: reverse(C&: *Then1)))
480 ToMove.moveBefore(BB&: *Then2, I: Then2->getFirstNonPhi());
481
482 auto *Merge1 = cast<VPBasicBlock>(Val: Then1->getSingleSuccessor());
483 auto *Merge2 = cast<VPBasicBlock>(Val: Then2->getSingleSuccessor());
484
485 // Move VPPredInstPHIRecipes from the merge block to the successor region's
486 // merge block. Update all users inside the successor region to use the
487 // original values.
488 for (VPRecipeBase &Phi1ToMove : make_early_inc_range(Range: reverse(C&: *Merge1))) {
489 VPValue *PredInst1 =
490 cast<VPPredInstPHIRecipe>(Val: &Phi1ToMove)->getOperand(N: 0);
491 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
492 Phi1ToMoveV->replaceUsesWithIf(New: PredInst1, ShouldReplace: [Then2](VPUser &U, unsigned) {
493 return cast<VPRecipeBase>(Val: &U)->getParent() == Then2;
494 });
495
496 // Remove phi recipes that are unused after merging the regions.
497 if (Phi1ToMove.getVPSingleValue()->user_empty()) {
498 Phi1ToMove.eraseFromParent();
499 continue;
500 }
501 Phi1ToMove.moveBefore(BB&: *Merge2, I: Merge2->begin());
502 }
503
504 // Remove the dead recipes in Region1's entry block.
505 for (VPRecipeBase &R :
506 make_early_inc_range(Range: reverse(C&: *Region1->getEntryBasicBlock())))
507 R.eraseFromParent();
508
509 // Finally, remove the first region.
510 for (VPBlockBase *Pred : make_early_inc_range(Range&: Region1->getPredecessors())) {
511 VPBlockUtils::disconnectBlocks(From: Pred, To: Region1);
512 VPBlockUtils::connectBlocks(From: Pred, To: MiddleBasicBlock);
513 }
514 VPBlockUtils::disconnectBlocks(From: Region1, To: MiddleBasicBlock);
515 TransformedRegions.insert(Ptr: Region1);
516 }
517
518 return !TransformedRegions.empty();
519}
520
521static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
522 VPRegionBlock *ParentRegion,
523 VPlan &Plan) {
524 Instruction *Instr = PredRecipe->getUnderlyingInstr();
525 // Build the triangular if-then region.
526 std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str();
527 assert(Instr->getParent() && "Predicated instruction not in any basic block");
528 auto *BlockInMask = PredRecipe->getMask();
529 auto *MaskDef = BlockInMask->getDefiningRecipe();
530 auto *BOMRecipe = new VPBranchOnMaskRecipe(
531 BlockInMask, MaskDef ? MaskDef->getDebugLoc() : DebugLoc::getUnknown());
532 auto *Entry =
533 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".entry", Recipe: BOMRecipe);
534
535 // Replace predicated replicate recipe with a replicate recipe without a
536 // mask but in the replicate region.
537 auto *RecipeWithoutMask = new VPReplicateRecipe(
538 PredRecipe->getUnderlyingInstr(), PredRecipe->operandsWithoutMask(),
539 PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
540 PredRecipe->getDebugLoc());
541 auto *Pred =
542 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".if", Recipe: RecipeWithoutMask);
543 auto *Exiting = Plan.createVPBasicBlock(Name: Twine(RegionName) + ".continue");
544 VPRegionBlock *Region =
545 Plan.createReplicateRegion(Entry, Exiting, Name: RegionName);
546
547 // Note: first set Entry as region entry and then connect successors starting
548 // from it in order, to propagate the "parent" of each VPBasicBlock.
549 Region->setParent(ParentRegion);
550 VPBlockUtils::insertTwoBlocksAfter(IfTrue: Pred, IfFalse: Exiting, BlockPtr: Entry);
551 VPBlockUtils::connectBlocks(From: Pred, To: Exiting);
552
553 if (!PredRecipe->user_empty()) {
554 auto *PHIRecipe = new VPPredInstPHIRecipe(RecipeWithoutMask,
555 RecipeWithoutMask->getDebugLoc());
556 Exiting->appendRecipe(Recipe: PHIRecipe);
557 PredRecipe->replaceAllUsesWith(New: PHIRecipe);
558 }
559 PredRecipe->eraseFromParent();
560 return Region;
561}
562
563static void addReplicateRegions(VPlan &Plan) {
564 SmallVector<VPReplicateRecipe *> WorkList;
565 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
566 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
567 for (VPRecipeBase &R : *VPBB)
568 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
569 if (RepR->isPredicated())
570 WorkList.push_back(Elt: RepR);
571 }
572 }
573
574 unsigned BBNum = 0;
575 for (VPReplicateRecipe *RepR : WorkList) {
576 VPBasicBlock *CurrentBlock = RepR->getParent();
577 VPBasicBlock *SplitBlock = CurrentBlock->splitAt(SplitAt: RepR->getIterator());
578
579 BasicBlock *OrigBB = RepR->getUnderlyingInstr()->getParent();
580 SplitBlock->setName(
581 OrigBB->hasName() ? OrigBB->getName() + "." + Twine(BBNum++) : "");
582 // Record predicated instructions for above packing optimizations.
583 VPRegionBlock *Region =
584 createReplicateRegion(PredRecipe: RepR, ParentRegion: CurrentBlock->getParent(), Plan);
585 VPBlockUtils::insertOnEdge(From: CurrentBlock, To: SplitBlock, BlockPtr: Region);
586
587 VPRegionBlock *ParentRegion = Region->getParent();
588 if (ParentRegion && ParentRegion->getExiting() == CurrentBlock)
589 ParentRegion->setExiting(SplitBlock);
590 }
591}
592
593bool VPlanTransforms::mergeBlocksIntoPredecessors(VPlan &Plan) {
594 SmallVector<VPBasicBlock *> WorkList;
595 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
596 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
597 // Don't fold the blocks in the skeleton of the Plan into their single
598 // predecessors for now.
599 // TODO: Remove restriction once more of the skeleton is modeled in VPlan.
600 if (!VPBB->getParent())
601 continue;
602 auto *PredVPBB =
603 dyn_cast_or_null<VPBasicBlock>(Val: VPBB->getSinglePredecessor());
604 if (!PredVPBB || PredVPBB->getNumSuccessors() != 1 ||
605 isa<VPIRBasicBlock>(Val: PredVPBB))
606 continue;
607 WorkList.push_back(Elt: VPBB);
608 }
609
610 for (VPBasicBlock *VPBB : WorkList) {
611 VPBasicBlock *PredVPBB = cast<VPBasicBlock>(Val: VPBB->getSinglePredecessor());
612 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
613 R.moveBefore(BB&: *PredVPBB, I: PredVPBB->end());
614 VPBlockUtils::disconnectBlocks(From: PredVPBB, To: VPBB);
615 auto *ParentRegion = VPBB->getParent();
616 if (ParentRegion && ParentRegion->getExiting() == VPBB)
617 ParentRegion->setExiting(PredVPBB);
618 VPBlockUtils::transferSuccessors(Old: VPBB, New: PredVPBB);
619 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
620 }
621 return !WorkList.empty();
622}
623
624void VPlanTransforms::createAndOptimizeReplicateRegions(VPlan &Plan) {
625 // Convert masked VPReplicateRecipes to if-then region blocks.
626 addReplicateRegions(Plan);
627
628 bool ShouldSimplify = true;
629 while (ShouldSimplify) {
630 ShouldSimplify = sinkScalarOperands(Plan);
631 ShouldSimplify |= mergeReplicateRegionsIntoSuccessors(Plan);
632 ShouldSimplify |= mergeBlocksIntoPredecessors(Plan);
633 }
634}
635
636/// Remove redundant casts of inductions.
637///
638/// Such redundant casts are casts of induction variables that can be ignored,
639/// because we already proved that the casted phi is equal to the uncasted phi
640/// in the vectorized loop. There is no need to vectorize the cast - the same
641/// value can be used for both the phi and casts in the vector loop.
642static void removeRedundantInductionCasts(VPlan &Plan) {
643 for (auto &Phi : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
644 auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: &Phi);
645 if (!IV || IV->getTruncInst())
646 continue;
647
648 // A sequence of IR Casts has potentially been recorded for IV, which
649 // *must be bypassed* when the IV is vectorized, because the vectorized IV
650 // will produce the desired casted value. This sequence forms a def-use
651 // chain and is provided in reverse order, ending with the cast that uses
652 // the IV phi. Search for the recipe of the last cast in the chain and
653 // replace it with the original IV. Note that only the final cast is
654 // expected to have users outside the cast-chain and the dead casts left
655 // over will be cleaned up later.
656 ArrayRef<Instruction *> Casts = IV->getInductionDescriptor().getCastInsts();
657 VPValue *FindMyCast = IV;
658 for (Instruction *IRCast : reverse(C&: Casts)) {
659 VPSingleDefRecipe *FoundUserCast = nullptr;
660 for (auto *U : FindMyCast->users()) {
661 auto *UserCast = dyn_cast<VPSingleDefRecipe>(Val: U);
662 if (UserCast && UserCast->getUnderlyingValue() == IRCast) {
663 FoundUserCast = UserCast;
664 break;
665 }
666 }
667 // A cast recipe in the chain may have been removed by earlier DCE.
668 if (!FoundUserCast)
669 break;
670 FindMyCast = FoundUserCast;
671 }
672 if (FindMyCast != IV)
673 FindMyCast->replaceAllUsesWith(New: IV);
674 }
675}
676
677static VPScalarIVStepsRecipe *
678createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
679 Instruction::BinaryOps InductionOpcode,
680 FPMathOperator *FPBinOp, Instruction *TruncI,
681 VPIRValue *StartV, VPValue *Step, DebugLoc DL,
682 VPBuilder &Builder) {
683 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
684 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
685 VPValue *CanonicalIV = LoopRegion->getCanonicalIV();
686 VPSingleDefRecipe *BaseIV =
687 Builder.createDerivedIV(Kind, FPBinOp, Start: StartV, Current: CanonicalIV, Step);
688
689 // Truncate base induction if needed.
690 Type *ResultTy = BaseIV->getScalarType();
691 if (TruncI) {
692 Type *TruncTy = TruncI->getType();
693 assert(ResultTy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits() &&
694 "Not truncating.");
695 assert(ResultTy->isIntegerTy() && "Truncation requires an integer type");
696 BaseIV = Builder.createScalarCast(Opcode: Instruction::Trunc, Op: BaseIV, ResultTy: TruncTy, DL);
697 ResultTy = TruncTy;
698 }
699
700 // Truncate step if needed.
701 Type *StepTy = Step->getScalarType();
702 if (ResultTy != StepTy) {
703 assert(StepTy->getScalarSizeInBits() > ResultTy->getScalarSizeInBits() &&
704 "Not truncating.");
705 assert(StepTy->isIntegerTy() && "Truncation requires an integer type");
706 auto *VecPreheader =
707 cast<VPBasicBlock>(Val: HeaderVPBB->getSingleHierarchicalPredecessor());
708 VPBuilder::InsertPointGuard Guard(Builder);
709 Builder.setInsertPoint(VecPreheader);
710 Step = Builder.createScalarCast(Opcode: Instruction::Trunc, Op: Step, ResultTy, DL);
711 }
712 return Builder.createScalarIVSteps(InductionOpcode, FPBinOp, IV: BaseIV, Step,
713 VF: &Plan.getVF(), DL);
714}
715
716void VPlanTransforms::replaceWideCanonicalIVWithWideIV(
717 VPlan &Plan, ScalarEvolution &SE, const TargetTransformInfo &TTI,
718 TargetTransformInfo::TargetCostKind CostKind, ElementCount VF, unsigned UF,
719 const SmallPtrSetImpl<const Value *> &ValuesToIgnore) {
720 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
721 if (!LoopRegion)
722 return;
723
724 auto *WideCanIV =
725 findUserOf<VPWidenCanonicalIVRecipe>(V: LoopRegion->getCanonicalIV());
726 if (!WideCanIV)
727 return;
728
729 Type *CanIVTy = LoopRegion->getCanonicalIVType();
730
731 // Replace the wide canonical IV with a scalar-iv-steps over the canonical
732 // IV.
733 if (Plan.hasScalarVFOnly() || vputils::onlyFirstLaneUsed(Def: WideCanIV)) {
734 VPBuilder Builder(WideCanIV);
735 WideCanIV->replaceAllUsesWith(New: createScalarIVSteps(
736 Plan, Kind: InductionDescriptor::IK_IntInduction, InductionOpcode: Instruction::Add, FPBinOp: nullptr,
737 TruncI: nullptr, StartV: Plan.getZero(Ty: CanIVTy), Step: Plan.getConstantInt(Ty: CanIVTy, Val: 1),
738 DL: WideCanIV->getDebugLoc(), Builder));
739 WideCanIV->eraseFromParent();
740 return;
741 }
742
743 if (vputils::onlyScalarValuesUsed(Def: WideCanIV))
744 return;
745
746 // If a canonical VPWidenIntOrFpInductionRecipe already produces vector lanes
747 // in the header, reuse it instead of introducing another wide induction phi.
748 VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
749 for (VPRecipeBase &Phi : Header->phis()) {
750 VPWidenIntOrFpInductionRecipe *WidenIV;
751 if (!match(V: &Phi, P: m_CanonicalWidenIV(V&: WidenIV)))
752 continue;
753 // The reused wide IV feeds the header mask, whose lanes may extend past
754 // the trip count; drop flags that only hold inside the scalar loop.
755 WidenIV->dropPoisonGeneratingFlags();
756 WideCanIV->replaceAllUsesWith(New: WidenIV);
757 WideCanIV->eraseFromParent();
758 return;
759 }
760
761 // Introduce a new VPWidenIntOrFpInductionRecipe if profitable.
762 auto *VecTy = VectorType::get(ElementType: CanIVTy, EC: VF);
763 InstructionCost BroadcastCost = TTI.getShuffleCost(
764 Kind: TargetTransformInfo::SK_Broadcast, DstTy: VecTy, SrcTy: VecTy, Mask: {}, CostKind);
765 InstructionCost PHICost = TTI.getCFInstrCost(Opcode: Instruction::PHI, CostKind);
766 if (PHICost > BroadcastCost)
767 return;
768
769 // Bail out if the additional wide induction phi increase the expected spill
770 // cost.
771 VPRegisterUsage UnrolledBase =
772 calculateRegisterUsageForPlan(Plan, VFs: VF, TTI, ValuesToIgnore)[0];
773 for (unsigned &NumUsers : make_second_range(c&: UnrolledBase.MaxLocalUsers))
774 NumUsers *= UF;
775 unsigned RegClass = TTI.getRegisterClassForType(/*Vector=*/true, Ty: VecTy);
776 VPRegisterUsage Projected = UnrolledBase;
777 Projected.MaxLocalUsers[RegClass] += TTI.getRegUsageForType(Ty: VecTy);
778 if (Projected.spillCost(TTI, CostKind) >
779 UnrolledBase.spillCost(TTI, CostKind))
780 return;
781
782 InductionDescriptor ID =
783 InductionDescriptor::getCanonicalIntInduction(Ty: CanIVTy, SE);
784 VPValue *StepV = Plan.getConstantInt(Ty: CanIVTy, Val: 1);
785 auto *NewWideIV = new VPWidenIntOrFpInductionRecipe(
786 /*IV=*/nullptr, Plan.getZero(Ty: CanIVTy), StepV, &Plan.getVF(), ID,
787 WideCanIV->getNoWrapFlags(), WideCanIV->getDebugLoc());
788 NewWideIV->insertBefore(InsertPos: &*Header->getFirstNonPhi());
789 WideCanIV->replaceAllUsesWith(New: NewWideIV);
790 WideCanIV->eraseFromParent();
791}
792
793void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
794 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
795 Plan.getEntry());
796 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
797 // The recipes in the block are processed in reverse order, to catch chains
798 // of dead recipes.
799 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
800 if (vputils::isDeadRecipe(R)) {
801 R.eraseFromParent();
802 continue;
803 }
804
805 // Check if R is a dead VPPhi <-> update cycle and remove it.
806 VPValue *Start, *Incoming;
807 if (!match(V: &R, P: m_VPPhi(Op0: m_VPValue(V&: Start), Op1: m_VPValue(V&: Incoming))))
808 continue;
809 auto *PhiR = cast<VPPhi>(Val: &R);
810 VPUser *PhiUser = PhiR->getSingleUser();
811 if (!PhiUser)
812 continue;
813 if (PhiUser != Incoming->getDefiningRecipe() ||
814 Incoming->getNumUsers() != 1)
815 continue;
816 PhiR->replaceAllUsesWith(New: Start);
817 PhiR->eraseFromParent();
818 Incoming->getDefiningRecipe()->eraseFromParent();
819 }
820 }
821}
822
823/// Scalarize a VPWidenPointerInductionRecipe by replacing it with a PtrAdd
824/// (IndStart, ScalarIVSteps (0, Step)). This is used when the recipe only
825/// generates scalar values.
826static VPValue *
827scalarizeVPWidenPointerInduction(VPWidenPointerInductionRecipe *PtrIV,
828 VPlan &Plan, VPBuilder &Builder) {
829 const InductionDescriptor &ID = PtrIV->getInductionDescriptor();
830 VPIRValue *StartV = Plan.getZero(Ty: ID.getStep()->getType());
831 VPValue *StepV = PtrIV->getOperand(N: 1);
832 VPScalarIVStepsRecipe *Steps = createScalarIVSteps(
833 Plan, Kind: InductionDescriptor::IK_IntInduction, InductionOpcode: Instruction::Add, FPBinOp: nullptr,
834 TruncI: nullptr, StartV, Step: StepV, DL: PtrIV->getDebugLoc(), Builder);
835
836 return Builder.createPtrAdd(Ptr: PtrIV->getStartValue(), Offset: Steps,
837 DL: PtrIV->getDebugLoc(), Name: "next.gep");
838}
839
840/// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd
841/// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as
842/// VPWidenPointerInductionRecipe will generate vectors only. If some users
843/// require vectors while other require scalars, the scalar uses need to extract
844/// the scalars from the generated vectors (Note that this is different to how
845/// int/fp inductions are handled). Legalize extract-from-ends using uniform
846/// VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so
847/// the correct end value is available. Also optimize
848/// VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by
849/// providing them scalar steps built on the canonical scalar IV and update the
850/// original IV's users. This is an optional optimization to reduce the needs of
851/// vector extracts.
852static void legalizeAndOptimizeInductions(VPlan &Plan) {
853 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
854 bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly();
855 VPBuilder Builder(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
856 for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
857 auto *PhiR = dyn_cast<VPWidenInductionRecipe>(Val: &Phi);
858 if (!PhiR)
859 continue;
860
861 // Try to narrow wide and replicating recipes to uniform recipes, based on
862 // VPlan analysis.
863 // TODO: Apply to all recipes in the future, to replace legacy uniformity
864 // analysis.
865 auto Users = vputils::collectUsersRecursively(V: PhiR);
866 for (VPUser *U : reverse(C&: Users)) {
867 auto *Def = dyn_cast<VPRecipeWithIRFlags>(Val: U);
868 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
869 // Skip recipes that shouldn't be narrowed.
870 if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Val: Def) ||
871 Def->user_empty() || !Def->getUnderlyingValue() ||
872 (RepR && (RepR->isSingleScalar() || RepR->isPredicated())))
873 continue;
874
875 // Skip recipes that may have other lanes than their first used.
876 if (!vputils::isSingleScalar(VPV: Def) && !vputils::onlyFirstLaneUsed(Def))
877 continue;
878
879 // TODO: Support scalarizing ExtractValue.
880 if (match(V: Def,
881 P: m_Binary<Instruction::ExtractValue>(Op0: m_VPValue(), Op1: m_VPValue())))
882 continue;
883
884 auto *Clone = VPBuilder::createSingleScalarOp(
885 Opcode: Def->getUnderlyingInstr()->getOpcode(), Operands: Def->operands(),
886 /*Mask=*/nullptr, Flags: *Def, Metadata: {}, DL: DebugLoc::getUnknown(),
887 UV: Def->getUnderlyingInstr());
888 Clone->insertAfter(InsertPos: Def);
889 Def->replaceAllUsesWith(New: Clone);
890 }
891
892 // Replace wide pointer inductions which have only their scalars used by
893 // PtrAdd(IndStart, ScalarIVSteps (0, Step)).
894 if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(Val: &Phi)) {
895 if (!Plan.hasScalarVFOnly() &&
896 !PtrIV->onlyScalarsGenerated(IsScalable: Plan.hasScalableVF()))
897 continue;
898
899 VPValue *PtrAdd = scalarizeVPWidenPointerInduction(PtrIV, Plan, Builder);
900 PtrIV->replaceAllUsesWith(New: PtrAdd);
901 continue;
902 }
903
904 // Replace widened induction with scalar steps for users that only use
905 // scalars.
906 auto *WideIV = cast<VPWidenIntOrFpInductionRecipe>(Val: &Phi);
907 if (HasOnlyVectorVFs && none_of(Range: WideIV->users(), P: [WideIV](VPUser *U) {
908 return U->usesScalars(Op: WideIV);
909 }))
910 continue;
911
912 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
913 VPScalarIVStepsRecipe *Steps = createScalarIVSteps(
914 Plan, Kind: ID.getKind(), InductionOpcode: ID.getInductionOpcode(),
915 FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
916 TruncI: WideIV->getTruncInst(), StartV: WideIV->getStartValue(), Step: WideIV->getStepValue(),
917 DL: WideIV->getDebugLoc(), Builder);
918
919 // Update scalar users of IV to use Step instead.
920 if (!HasOnlyVectorVFs) {
921 assert(!Plan.hasScalableVF() &&
922 "plans containing a scalar VF cannot also include scalable VFs");
923 WideIV->replaceAllUsesWith(New: Steps);
924 } else {
925 bool HasScalableVF = Plan.hasScalableVF();
926 WideIV->replaceUsesWithIf(New: Steps,
927 ShouldReplace: [WideIV, HasScalableVF](VPUser &U, unsigned) {
928 if (HasScalableVF)
929 return U.usesFirstLaneOnly(Op: WideIV);
930 return U.usesScalars(Op: WideIV);
931 });
932 }
933 }
934}
935
936/// Check if \p VPV is an untruncated wide induction, either before or after the
937/// increment. If so return the header IV (before the increment), otherwise
938/// return null.
939static VPWidenInductionRecipe *
940getOptimizableIVOf(VPValue *VPV, PredicatedScalarEvolution &PSE) {
941 auto *WideIV = dyn_cast<VPWidenInductionRecipe>(Val: VPV);
942 if (WideIV) {
943 // VPV itself is a wide induction, separately compute the end value for exit
944 // users if it is not a truncated IV.
945 auto *IntOrFpIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
946 return (IntOrFpIV && IntOrFpIV->getTruncInst()) ? nullptr : WideIV;
947 }
948
949 // Check if VPV is an optimizable induction increment.
950 VPRecipeBase *Def = VPV->getDefiningRecipe();
951 if (!Def || Def->getNumOperands() != 2)
952 return nullptr;
953 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 0));
954 if (!WideIV)
955 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 1));
956 if (!WideIV)
957 return nullptr;
958
959 auto IsWideIVInc = [&]() {
960 auto &ID = WideIV->getInductionDescriptor();
961
962 // Check if VPV increments the induction by the induction step.
963 VPValue *IVStep = WideIV->getStepValue();
964 switch (ID.getInductionOpcode()) {
965 case Instruction::Add:
966 return match(V: VPV, P: m_c_Add(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
967 case Instruction::FAdd:
968 return match(V: VPV, P: m_c_FAdd(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
969 case Instruction::FSub:
970 return match(V: VPV, P: m_Binary<Instruction::FSub>(Op0: m_Specific(VPV: WideIV),
971 Op1: m_Specific(VPV: IVStep)));
972 case Instruction::Sub: {
973 // IVStep will be the negated step of the subtraction. Check if Step == -1
974 // * IVStep.
975 VPValue *Step;
976 if (!match(V: VPV, P: m_Sub(Op0: m_VPValue(), Op1: m_VPValue(V&: Step))))
977 return false;
978 const SCEV *IVStepSCEV = vputils::getSCEVExprForVPValue(V: IVStep, PSE);
979 const SCEV *StepSCEV = vputils::getSCEVExprForVPValue(V: Step, PSE);
980 ScalarEvolution &SE = *PSE.getSE();
981 return !isa<SCEVCouldNotCompute>(Val: IVStepSCEV) &&
982 !isa<SCEVCouldNotCompute>(Val: StepSCEV) &&
983 IVStepSCEV == SE.getNegativeSCEV(V: StepSCEV);
984 }
985 default:
986 return ID.getKind() == InductionDescriptor::IK_PtrInduction &&
987 match(V: VPV, P: m_GetElementPtr(Op0: m_Specific(VPV: WideIV),
988 Op1: m_Specific(VPV: WideIV->getStepValue())));
989 }
990 llvm_unreachable("should have been covered by switch above");
991 };
992 return IsWideIVInc() ? WideIV : nullptr;
993}
994
995/// Attempts to optimize the induction variable exit values for users in the
996/// early exit block.
997static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan, VPValue *Op,
998 PredicatedScalarEvolution &PSE) {
999 VPValue *Incoming, *Mask;
1000 if (!match(V: Op, P: m_ExtractLane(Op0: m_FirstActiveLane(Op0: m_VPValue(V&: Mask)),
1001 Op1: m_VPValue(V&: Incoming))))
1002 return nullptr;
1003
1004 auto *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
1005 if (!WideIV)
1006 return nullptr;
1007
1008 auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
1009 if (WideIntOrFp && WideIntOrFp->getTruncInst())
1010 return nullptr;
1011
1012 // Calculate the final index.
1013 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
1014 auto *CanonicalIV = LoopRegion->getCanonicalIV();
1015 Type *CanonicalIVType = LoopRegion->getCanonicalIVType();
1016 auto *ExtractR = cast<VPInstruction>(Val: Op);
1017 VPBuilder B(ExtractR);
1018
1019 DebugLoc DL = ExtractR->getDebugLoc();
1020 VPValue *FirstActiveLane = B.createFirstActiveLane(Masks: Mask, DL);
1021 FirstActiveLane =
1022 B.createScalarZExtOrTrunc(Op: FirstActiveLane, ResultTy: CanonicalIVType, DL);
1023 VPValue *EndValue = B.createAdd(LHS: CanonicalIV, RHS: FirstActiveLane, DL);
1024
1025 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
1026 // changed it means the exit is using the incremented value, so we need to
1027 // add the step.
1028 if (Incoming != WideIV) {
1029 VPValue *One = Plan.getConstantInt(Ty: CanonicalIVType, Val: 1);
1030 EndValue = B.createAdd(LHS: EndValue, RHS: One, DL);
1031 }
1032
1033 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
1034 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
1035 VPIRValue *Start = WideIV->getStartValue();
1036 VPValue *Step = WideIV->getStepValue();
1037 EndValue = B.createDerivedIV(
1038 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
1039 Start, Current: EndValue, Step);
1040 }
1041
1042 return EndValue;
1043}
1044
1045/// Compute the end value for \p WideIV, unless it is truncated. Creates a
1046/// VPDerivedIVRecipe for non-canonical inductions.
1047static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV,
1048 VPBuilder &VectorPHBuilder,
1049 VPValue *VectorTC) {
1050 auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
1051 // Truncated wide inductions resume from the last lane of their vector value
1052 // in the last vector iteration which is handled elsewhere.
1053 if (WideIntOrFp && WideIntOrFp->getTruncInst())
1054 return nullptr;
1055
1056 VPIRValue *Start = WideIV->getStartValue();
1057 VPValue *Step = WideIV->getStepValue();
1058 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
1059 VPValue *EndValue = VectorTC;
1060 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
1061 EndValue = VectorPHBuilder.createDerivedIV(
1062 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
1063 Start, Current: VectorTC, Step);
1064 }
1065
1066 // EndValue is derived from the vector trip count (which has the same type as
1067 // the widest induction) and thus may be wider than the induction here.
1068 Type *ScalarTypeOfWideIV = WideIV->getScalarType();
1069 if (ScalarTypeOfWideIV != EndValue->getScalarType()) {
1070 EndValue = VectorPHBuilder.createScalarCast(Opcode: Instruction::Trunc, Op: EndValue,
1071 ResultTy: ScalarTypeOfWideIV,
1072 DL: WideIV->getDebugLoc());
1073 }
1074
1075 return EndValue;
1076}
1077
1078/// Attempts to optimize the induction variable exit values for users in the
1079/// exit block coming from the latch in the original scalar loop.
1080static VPValue *
1081optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
1082 DenseMap<VPValue *, VPValue *> &EndValues,
1083 PredicatedScalarEvolution &PSE) {
1084 VPValue *Incoming;
1085 if (!match(V: Op, P: m_CombineOr(Ps: m_ExtractLastLaneOfLastPart(Op0: m_VPValue(V&: Incoming)),
1086 Ps: m_ExtractLane(Op0: m_LastActiveLane(Op0: m_HeaderMask()),
1087 Op1: m_VPValue(V&: Incoming)))))
1088 return nullptr;
1089
1090 VPWidenInductionRecipe *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
1091 if (!WideIV)
1092 return nullptr;
1093
1094 VPValue *EndValue = EndValues.lookup(Val: WideIV);
1095 assert(EndValue && "Must have computed the end value up front");
1096
1097 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
1098 // changed it means the exit is using the incremented value, so we don't
1099 // need to subtract the step.
1100 if (Incoming != WideIV)
1101 return EndValue;
1102
1103 // Otherwise, subtract the step from the EndValue.
1104 auto *ExtractR = cast<VPInstruction>(Val: Op);
1105 VPBuilder B(ExtractR);
1106 VPValue *Step = WideIV->getStepValue();
1107 Type *ScalarTy = WideIV->getScalarType();
1108 if (ScalarTy->isIntegerTy())
1109 return B.createSub(LHS: EndValue, RHS: Step, DL: DebugLoc::getUnknown(), Name: "ind.escape");
1110 if (ScalarTy->isPointerTy()) {
1111 Type *StepTy = Step->getScalarType();
1112 auto *Zero = Plan.getZero(Ty: StepTy);
1113 return B.createPtrAdd(Ptr: EndValue, Offset: B.createSub(LHS: Zero, RHS: Step),
1114 DL: DebugLoc::getUnknown(), Name: "ind.escape");
1115 }
1116 if (ScalarTy->isFloatingPointTy()) {
1117 const auto &ID = WideIV->getInductionDescriptor();
1118 return B.createNaryOp(
1119 Opcode: ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
1120 ? Instruction::FSub
1121 : Instruction::FAdd,
1122 Operands: {EndValue, Step}, Flags: {ID.getInductionBinOp()->getFastMathFlags()});
1123 }
1124 llvm_unreachable("all possible induction types must be handled");
1125 return nullptr;
1126}
1127
1128void VPlanTransforms::optimizeInductionLiveOutUsers(
1129 VPlan &Plan, PredicatedScalarEvolution &PSE) {
1130 // Compute end values for all inductions.
1131 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
1132 auto *VectorPH = cast<VPBasicBlock>(Val: VectorRegion->getSinglePredecessor());
1133 VPBuilder VectorPHBuilder(VectorPH, VectorPH->begin());
1134 DenseMap<VPValue *, VPValue *> EndValues;
1135 VPValue *ResumeTC =
1136 Plan.hasTailFolded() ? Plan.getTripCount() : &Plan.getVectorTripCount();
1137 for (auto &Phi : VectorRegion->getEntryBasicBlock()->phis()) {
1138 auto *WideIV = dyn_cast<VPWidenInductionRecipe>(Val: &Phi);
1139 if (!WideIV)
1140 continue;
1141 if (VPValue *EndValue =
1142 tryToComputeEndValueForInduction(WideIV, VectorPHBuilder, VectorTC: ResumeTC))
1143 EndValues[WideIV] = EndValue;
1144 }
1145
1146 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
1147 for (VPRecipeBase &R : make_early_inc_range(Range&: *MiddleVPBB)) {
1148 VPValue *Op;
1149 if (!match(V: &R, P: m_ExitingIVValue(Op0: m_VPValue(V&: Op))))
1150 continue;
1151 auto *WideIV = cast<VPWidenInductionRecipe>(Val: Op);
1152 if (VPValue *EndValue = EndValues.lookup(Val: WideIV)) {
1153 R.getVPSingleValue()->replaceAllUsesWith(New: EndValue);
1154 R.eraseFromParent();
1155 }
1156 }
1157
1158 // Then, optimize exit block users.
1159 for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
1160 for (VPRecipeBase &R : ExitVPBB->phis()) {
1161 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
1162
1163 for (auto [Idx, PredVPBB] : enumerate(First&: ExitVPBB->getPredecessors())) {
1164 VPValue *Escape = nullptr;
1165 if (PredVPBB == MiddleVPBB)
1166 Escape = optimizeLatchExitInductionUser(
1167 Plan, Op: ExitIRI->getOperand(N: Idx), EndValues, PSE);
1168 else
1169 Escape = optimizeEarlyExitInductionUser(
1170 Plan, Op: ExitIRI->getOperand(N: Idx), PSE);
1171 if (Escape)
1172 ExitIRI->setOperand(I: Idx, New: Escape);
1173 }
1174 }
1175 }
1176}
1177
1178/// Remove redundant ExpandSCEVRecipes in \p Plan's entry block by replacing
1179/// them with already existing recipes expanding the same SCEV expression.
1180static void removeRedundantExpandSCEVRecipes(VPlan &Plan) {
1181 DenseMap<const SCEV *, VPValue *> SCEV2VPV;
1182
1183 for (VPRecipeBase &R :
1184 make_early_inc_range(Range&: *Plan.getEntry()->getEntryBasicBlock())) {
1185 auto *ExpR = dyn_cast<VPExpandSCEVRecipe>(Val: &R);
1186 if (!ExpR)
1187 continue;
1188
1189 const auto &[V, Inserted] = SCEV2VPV.try_emplace(Key: ExpR->getSCEV(), Args&: ExpR);
1190 if (Inserted)
1191 continue;
1192
1193 ExpR->replaceAllUsesWith(New: V->second);
1194 if (ExpR == Plan.getTripCount())
1195 Plan.resetTripCount(NewTripCount: V->second);
1196
1197 ExpR->eraseFromParent();
1198 }
1199}
1200
1201/// Try to simplify logical and bitwise recipes in \p Def.
1202static bool simplifyLogicalRecipe(VPSingleDefRecipe *Def, VPBuilder &Builder,
1203 bool CanCreateNewRecipe) {
1204 VPlan *Plan = Def->getParent()->getPlan();
1205
1206 // Simplify (X && Y) | (X && !Y) -> X.
1207 // TODO: Split up into simpler, modular combines: (X && Y) | (X && Z) into X
1208 // && (Y | Z) and (X | !X) into true. This requires queuing newly created
1209 // recipes to be visited during simplification.
1210 VPValue *X, *Y, *Z;
1211 if (match(R: Def,
1212 P: m_c_BinaryOr(Op0: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y)),
1213 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_Not(Op0: m_Deferred(V: Y)))))) {
1214 Def->replaceAllUsesWith(New: X);
1215 Def->eraseFromParent();
1216 return true;
1217 }
1218
1219 // x | AllOnes -> AllOnes
1220 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_AllOnes()))) {
1221 Def->replaceAllUsesWith(New: Plan->getAllOnesValue(Ty: Def->getScalarType()));
1222 return true;
1223 }
1224
1225 // x | 0 -> x
1226 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_ZeroInt()))) {
1227 Def->replaceAllUsesWith(New: X);
1228 return true;
1229 }
1230
1231 // x | !x -> AllOnes
1232 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X))))) {
1233 Def->replaceAllUsesWith(New: Plan->getAllOnesValue(Ty: Def->getScalarType()));
1234 return true;
1235 }
1236
1237 // x & 0 -> 0
1238 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_ZeroInt()))) {
1239 Def->replaceAllUsesWith(New: Plan->getZero(Ty: Def->getScalarType()));
1240 return true;
1241 }
1242
1243 // x & AllOnes -> x
1244 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_AllOnes()))) {
1245 Def->replaceAllUsesWith(New: X);
1246 return true;
1247 }
1248
1249 // x && false -> false
1250 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_False()))) {
1251 Def->replaceAllUsesWith(New: Plan->getFalse());
1252 return true;
1253 }
1254
1255 // x && true -> x
1256 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_True()))) {
1257 Def->replaceAllUsesWith(New: X);
1258 return true;
1259 }
1260
1261 // (x && y) | (x && z) -> x && (y | z)
1262 if (CanCreateNewRecipe &&
1263 match(R: Def, P: m_c_BinaryOr(Op0: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y)),
1264 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue(V&: Z)))) &&
1265 // Simplify only if one of the operands has one use to avoid creating an
1266 // extra recipe.
1267 (!Def->getOperand(N: 0)->hasMoreThanOneUniqueUser() ||
1268 !Def->getOperand(N: 1)->hasMoreThanOneUniqueUser())) {
1269 Def->replaceAllUsesWith(
1270 New: Builder.createLogicalAnd(LHS: X, RHS: Builder.createOr(LHS: Y, RHS: Z)));
1271 return true;
1272 }
1273
1274 // x && (x && y) -> x && y
1275 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1276 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue())))) {
1277 Def->replaceAllUsesWith(New: Def->getOperand(N: 1));
1278 return true;
1279 }
1280
1281 // x && (y && x) -> x && y
1282 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1283 Op1: m_LogicalAnd(Op0: m_VPValue(V&: Y), Op1: m_Deferred(V: X))))) {
1284 Def->replaceAllUsesWith(New: Builder.createLogicalAnd(LHS: X, RHS: Y));
1285 return true;
1286 }
1287
1288 // x && !x -> 0
1289 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X))))) {
1290 Def->replaceAllUsesWith(New: Plan->getFalse());
1291 return true;
1292 }
1293
1294 if (match(R: Def, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: X), Op2: m_Deferred(V: X)))) {
1295 Def->replaceAllUsesWith(New: X);
1296 return true;
1297 }
1298
1299 // select c, false, true -> not c
1300 VPValue *C;
1301 if (CanCreateNewRecipe &&
1302 match(R: Def, P: m_Select(Op0: m_VPValue(V&: C), Op1: m_False(), Op2: m_True()))) {
1303 Def->replaceAllUsesWith(New: Builder.createNot(Operand: C));
1304 return true;
1305 }
1306
1307 // select !c, x, y -> select c, y, x
1308 if (match(R: Def, P: m_Select(Op0: m_Not(Op0: m_VPValue(V&: C)), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1309 Def->setOperand(I: 0, New: C);
1310 Def->setOperand(I: 1, New: Y);
1311 Def->setOperand(I: 2, New: X);
1312 return true;
1313 }
1314
1315 // select x, (i1 y | z), y -> y | (x && z)
1316 if (CanCreateNewRecipe &&
1317 match(R: Def, P: m_Select(Op0: m_VPValue(V&: X),
1318 Op1: m_OneUse(SubPattern: m_c_BinaryOr(Op0: m_VPValue(V&: Y), Op1: m_VPValue(V&: Z))),
1319 Op2: m_Deferred(V: Y))) &&
1320 Y->getScalarType()->isIntegerTy(BitWidth: 1)) {
1321 Def->replaceAllUsesWith(
1322 New: Builder.createOr(LHS: Y, RHS: Builder.createLogicalAnd(LHS: X, RHS: Z)));
1323 return true;
1324 }
1325
1326 return false;
1327}
1328
1329/// Try to simplify VPSingleDefRecipe \p Def.
1330static void simplifyRecipe(VPSingleDefRecipe *Def) {
1331 VPlan *Plan = Def->getParent()->getPlan();
1332
1333 // Simplification of live-in IR values for SingleDef recipes using
1334 // InstSimplifyFolder.
1335 const DataLayout &DL = Plan->getDataLayout();
1336 if (VPValue *V = vputils::tryToFoldLiveIns(R&: *Def, Operands: Def->operands(), DL))
1337 return Def->replaceAllUsesWith(New: V);
1338
1339 // Fold PredPHI LiveIn -> LiveIn.
1340 if (auto *PredPHI = dyn_cast<VPPredInstPHIRecipe>(Val: Def)) {
1341 VPValue *Op = PredPHI->getOperand(N: 0);
1342 if (isa<VPIRValue>(Val: Op))
1343 PredPHI->replaceAllUsesWith(New: Op);
1344 }
1345
1346 // Drop the mask of a predicated store masked by the header mask (which is
1347 // guaranteed to be true at least for the first lane) and both the stored
1348 // value and the address are uniform across VF and UF. The header mask is
1349 // still the abstract region value here.
1350 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: Def);
1351 RepR && RepR->isPredicated() && RepR->getOpcode() == Instruction::Store &&
1352 all_of(Range: RepR->operandsWithoutMask(), P: vputils::isUniformAcrossVFsAndUFs) &&
1353 match(V: RepR->getMask(), P: m_HeaderMask())) {
1354 auto *Unmasked = new VPReplicateRecipe(
1355 RepR->getUnderlyingInstr(), RepR->operandsWithoutMask(),
1356 RepR->isSingleScalar(), /*Mask=*/nullptr, *RepR, *RepR,
1357 RepR->getDebugLoc());
1358 Unmasked->insertBefore(InsertPos: RepR);
1359 RepR->replaceAllUsesWith(New: Unmasked);
1360 RepR->eraseFromParent();
1361 return;
1362 }
1363
1364 VPBuilder Builder(Def);
1365
1366 // Avoid replacing VPInstructions with underlying values with new
1367 // VPInstructions, as we would fail to create widen/replicate recpes from the
1368 // new VPInstructions without an underlying value, and miss out on some
1369 // transformations that only apply to widened/replicated recipes later, by
1370 // doing so.
1371 // TODO: We should also not replace non-VPInstructions like VPWidenRecipe with
1372 // VPInstructions without underlying values, as those will get skipped during
1373 // cost computation.
1374 bool CanCreateNewRecipe =
1375 !isa<VPInstruction>(Val: Def) || !Def->getUnderlyingValue();
1376
1377 VPValue *A;
1378 if (match(R: Def, P: m_Trunc(Op0: m_ZExtOrSExt(Op0: m_VPValue(V&: A))))) {
1379 Type *TruncTy = Def->getScalarType();
1380 Type *ATy = A->getScalarType();
1381 if (TruncTy == ATy) {
1382 Def->replaceAllUsesWith(New: A);
1383 } else {
1384 // Don't replace a non-widened cast recipe with a widened cast.
1385 if (!isa<VPWidenCastRecipe>(Val: Def))
1386 return;
1387 if (ATy->getScalarSizeInBits() < TruncTy->getScalarSizeInBits()) {
1388
1389 unsigned ExtOpcode = match(V: Def->getOperand(N: 0), P: m_SExt(Op0: m_VPValue()))
1390 ? Instruction::SExt
1391 : Instruction::ZExt;
1392 auto *Ext = Builder.createWidenCast(Opcode: Instruction::CastOps(ExtOpcode), Op: A,
1393 ResultTy: TruncTy);
1394 if (auto *UnderlyingExt = Def->getOperand(N: 0)->getUnderlyingValue()) {
1395 // UnderlyingExt has distinct return type, used to retain legacy cost.
1396 Ext->setUnderlyingValue(UnderlyingExt);
1397 }
1398 Def->replaceAllUsesWith(New: Ext);
1399 } else if (ATy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) {
1400 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::Trunc, Op: A, ResultTy: TruncTy);
1401 Def->replaceAllUsesWith(New: Trunc);
1402 }
1403 }
1404 }
1405
1406 if (simplifyLogicalRecipe(Def, Builder, CanCreateNewRecipe))
1407 return;
1408
1409 VPValue *X, *Y, *C;
1410 if (match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))
1411 return Def->replaceAllUsesWith(New: A);
1412
1413 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_One())))
1414 return Def->replaceAllUsesWith(New: A);
1415
1416 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))
1417 return Def->replaceAllUsesWith(New: Plan->getZero(Ty: Def->getScalarType()));
1418
1419 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_AllOnes()))) {
1420 // Preserve nsw from the Mul on the new Sub.
1421 VPIRFlags::WrapFlagsTy NW = {
1422 false, cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap()};
1423 return Def->replaceAllUsesWith(New: Builder.createSub(
1424 LHS: Plan->getZero(Ty: A->getScalarType()), RHS: A, DL: Def->getDebugLoc(), Name: "", WrapFlags: NW));
1425 }
1426
1427 if (CanCreateNewRecipe &&
1428 match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: X), Op1: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Y))))) {
1429 // Preserve nsw from the Add and the Sub, if it's present on both, on the
1430 // new Sub.
1431 VPIRFlags::WrapFlagsTy NW = {
1432 false,
1433 cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap() &&
1434 cast<VPRecipeWithIRFlags>(Val: Def->getOperand(N: Def->getOperand(N: 0) == X))
1435 ->hasNoSignedWrap()};
1436 return Def->replaceAllUsesWith(
1437 New: Builder.createSub(LHS: X, RHS: Y, DL: Def->getDebugLoc(), Name: "", WrapFlags: NW));
1438 }
1439
1440 const APInt *APC;
1441 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_APInt(C&: APC))) &&
1442 APC->isPowerOf2()) {
1443 auto *MulR = cast<VPRecipeWithIRFlags>(Val: Def);
1444 unsigned ShiftAmt = APC->exactLogBase2();
1445 VPIRFlags::WrapFlagsTy NW(MulR->hasNoUnsignedWrap(),
1446 MulR->hasNoSignedWrap() &&
1447 ShiftAmt != APC->getBitWidth() - 1);
1448 return Def->replaceAllUsesWith(New: Builder.createNaryOp(
1449 Opcode: Instruction::Shl,
1450 Operands: {A, Plan->getConstantInt(BitWidth: APC->getBitWidth(), Val: ShiftAmt)}, Flags: NW,
1451 DL: Def->getDebugLoc()));
1452 }
1453
1454 if (CanCreateNewRecipe && match(R: Def, P: m_UDiv(Op0: m_VPValue(V&: A), Op1: m_APInt(C&: APC))) &&
1455 APC->isPowerOf2())
1456 return Def->replaceAllUsesWith(New: Builder.createNaryOp(
1457 Opcode: Instruction::LShr,
1458 Operands: {A, Plan->getConstantInt(BitWidth: APC->getBitWidth(), Val: APC->exactLogBase2())},
1459 Flags: *cast<VPRecipeWithIRFlags>(Val: Def), DL: Def->getDebugLoc()));
1460
1461 if (match(R: Def, P: m_Not(Op0: m_VPValue(V&: A)))) {
1462 if (match(V: A, P: m_Not(Op0: m_VPValue(V&: A))))
1463 return Def->replaceAllUsesWith(New: A);
1464
1465 // Try to fold Not into compares by adjusting the predicate in-place.
1466 CmpPredicate Pred;
1467 if (match(V: A, P: m_Cmp(Pred, Op0: m_VPValue(), Op1: m_VPValue()))) {
1468 auto *Cmp = cast<VPRecipeWithIRFlags>(Val: A);
1469 if (all_of(Range: Cmp->users(),
1470 P: match_fn(P: m_CombineOr(
1471 Ps: m_Not(Op0: m_Specific(VPV: Cmp)),
1472 Ps: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(), Op2: m_VPValue()))))) {
1473 Cmp->setPredicate(CmpInst::getInversePredicate(pred: Pred));
1474 for (VPUser *U : to_vector(Range: Cmp->users())) {
1475 auto *R = cast<VPSingleDefRecipe>(Val: U);
1476 if (match(R, P: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1477 // select (cmp pred), x, y -> select (cmp inv_pred), y, x
1478 R->setOperand(I: 1, New: Y);
1479 R->setOperand(I: 2, New: X);
1480 } else {
1481 // not (cmp pred) -> cmp inv_pred
1482 assert(match(R, m_Not(m_Specific(Cmp))) && "Unexpected user");
1483 R->replaceAllUsesWith(New: Cmp);
1484 }
1485 }
1486 // If Cmp doesn't have a debug location, use the one from the negation,
1487 // to preserve the location.
1488 if (!Cmp->getDebugLoc() && Def->getDebugLoc())
1489 Cmp->setDebugLoc(Def->getDebugLoc());
1490 }
1491 }
1492 }
1493
1494 // Fold any-of (fcmp uno %A, %A), (fcmp uno %B, %B), ... ->
1495 // any-of (fcmp uno %A, %B), ...
1496 if (match(R: Def, P: m_AnyOf())) {
1497 SmallVector<VPValue *, 4> NewOps;
1498 VPRecipeBase *UnpairedCmp = nullptr;
1499 for (VPValue *Op : Def->operands()) {
1500 VPValue *X;
1501 if (Op->getNumUsers() > 1 ||
1502 !match(V: Op, P: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X),
1503 Op1: m_Deferred(V: X)))) {
1504 NewOps.push_back(Elt: Op);
1505 } else if (!UnpairedCmp) {
1506 UnpairedCmp = Op->getDefiningRecipe();
1507 } else {
1508 NewOps.push_back(Elt: Builder.createFCmp(Pred: CmpInst::FCMP_UNO,
1509 A: UnpairedCmp->getOperand(N: 0), B: X));
1510 UnpairedCmp = nullptr;
1511 }
1512 }
1513
1514 if (UnpairedCmp)
1515 NewOps.push_back(Elt: UnpairedCmp->getVPSingleValue());
1516
1517 if (NewOps.size() < Def->getNumOperands()) {
1518 VPValue *NewAnyOf = Builder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: NewOps);
1519 return Def->replaceAllUsesWith(New: NewAnyOf);
1520 }
1521 }
1522
1523 // Fold (fcmp uno %X, %X) or (fcmp uno %Y, %Y) -> fcmp uno %X, %Y
1524 // This is useful for fmax/fmin without fast-math flags, where we need to
1525 // check if any operand is NaN.
1526 if (CanCreateNewRecipe &&
1527 match(R: Def, P: m_BinaryOr(Op0: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X),
1528 Op1: m_Deferred(V: X)),
1529 Op1: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: Y),
1530 Op1: m_Deferred(V: Y))))) {
1531 VPValue *NewCmp = Builder.createFCmp(Pred: CmpInst::FCMP_UNO, A: X, B: Y);
1532 return Def->replaceAllUsesWith(New: NewCmp);
1533 }
1534
1535 // Remove redundant DerviedIVs, that is 0 + A * 1 -> A and 0 + 0 * x -> 0.
1536 if ((match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_VPValue(V&: A), Op2: m_One())) ||
1537 match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_ZeroInt(), Op2: m_VPValue()))) &&
1538 Def->getOperand(N: 1)->getScalarType() == Def->getScalarType())
1539 return Def->replaceAllUsesWith(New: Def->getOperand(N: 1));
1540
1541 if (match(R: Def, P: m_VPInstruction<VPInstruction::WideIVStep>(Ops: m_VPValue(V&: X),
1542 Ops: m_One()))) {
1543 Type *WideStepTy = Def->getScalarType();
1544 if (X->getScalarType() != WideStepTy)
1545 X = Builder.createWidenCast(Opcode: Instruction::Trunc, Op: X, ResultTy: WideStepTy);
1546 Def->replaceAllUsesWith(New: X);
1547 return;
1548 }
1549
1550 // For i1 vp.merges produced by AnyOf reductions:
1551 // vp.merge true, (or x, y), x, evl -> vp.merge y, true, x, evl
1552 if (match(R: Def, P: m_Intrinsic<Intrinsic::vp_merge>(Ops: m_True(), Ops: m_VPValue(V&: A),
1553 Ops: m_VPValue(V&: X), Ops: m_VPValue())) &&
1554 match(V: A, P: m_c_BinaryOr(Op0: m_Specific(VPV: X), Op1: m_VPValue(V&: Y))) &&
1555 Def->getScalarType()->isIntegerTy(BitWidth: 1)) {
1556 Def->setOperand(I: 1, New: Def->getOperand(N: 0));
1557 Def->setOperand(I: 0, New: Y);
1558 return;
1559 }
1560
1561 // Simplify MaskedCond with no block mask to its single operand.
1562 if (match(R: Def, P: m_VPInstruction<VPInstruction::MaskedCond>()) &&
1563 !cast<VPInstruction>(Val: Def)->isMasked())
1564 return Def->replaceAllUsesWith(New: Def->getOperand(N: 0));
1565
1566 // Look through ExtractLastLane.
1567 if (match(R: Def, P: m_ExtractLastLane(Op0: m_VPValue(V&: A)))) {
1568 if (match(V: A, P: m_BuildVector())) {
1569 auto *BuildVector = cast<VPInstruction>(Val: A);
1570 Def->replaceAllUsesWith(
1571 New: BuildVector->getOperand(N: BuildVector->getNumOperands() - 1));
1572 return;
1573 }
1574
1575 if (match(V: A, P: m_Broadcast(Op0: m_VPValue(V&: X))))
1576 return Def->replaceAllUsesWith(New: X);
1577
1578 if (isa<VPInstruction, VPReplicateRecipe>(Val: A) && vputils::isSingleScalar(VPV: A))
1579 return Def->replaceAllUsesWith(New: A);
1580
1581 if (Plan->hasScalarVFOnly())
1582 return Def->replaceAllUsesWith(New: A);
1583 }
1584
1585 // Look through ExtractPenultimateElement (BuildVector ....).
1586 if (match(R: Def, P: m_ExtractPenultimateElement(Op0: m_BuildVector()))) {
1587 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1588 Def->replaceAllUsesWith(
1589 New: BuildVector->getOperand(N: BuildVector->getNumOperands() - 2));
1590 return;
1591 }
1592
1593 uint64_t Idx;
1594 if (match(R: Def, P: m_ExtractElement(Op0: m_BuildVector(), Op1: m_ConstantInt(C&: Idx)))) {
1595 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1596 Def->replaceAllUsesWith(New: BuildVector->getOperand(N: Idx));
1597 return;
1598 }
1599
1600 if (match(R: Def, P: m_BuildVector()) && all_equal(Range: Def->operands())) {
1601 Def->replaceAllUsesWith(
1602 New: Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: Def->getOperand(N: 0)));
1603 return;
1604 }
1605
1606 // Replace uses of a BuildVector by users that only use its first lane with
1607 // its first operand directly.
1608 if (match(R: Def, P: m_BuildVector())) {
1609 Def->replaceUsesWithIf(New: Def->getOperand(N: 0), ShouldReplace: [Def](VPUser &U, unsigned) {
1610 return U.usesFirstLaneOnly(Op: Def);
1611 });
1612 }
1613
1614 // Look through broadcast of single-scalar when used as select conditions; in
1615 // that case the scalar condition can be used directly.
1616 if (match(R: Def,
1617 P: m_Select(Op0: m_Broadcast(Op0: m_VPValue(V&: C)), Op1: m_VPValue(), Op2: m_VPValue()))) {
1618 assert(vputils::isSingleScalar(C) &&
1619 "broadcast operand must be single-scalar");
1620 Def->setOperand(I: 0, New: C);
1621 return;
1622 }
1623
1624 if (match(R: Def, P: m_Broadcast(Op0: m_VPValue(V&: X))))
1625 return Def->replaceUsesWithIf(
1626 New: X, ShouldReplace: [Def](const VPUser &U, unsigned) { return U.usesScalars(Op: Def); });
1627
1628 if (isa<VPPhi, VPWidenPHIRecipe, VPHeaderPHIRecipe>(Val: Def)) {
1629 if (Def->getNumOperands() == 1) {
1630 Def->replaceAllUsesWith(New: Def->getOperand(N: 0));
1631 return;
1632 }
1633 if (auto *Phi = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Val: Def)) {
1634 if (all_equal(Range: Phi->incoming_values()))
1635 Phi->replaceAllUsesWith(New: Phi->getOperand(N: 0));
1636 }
1637 return;
1638 }
1639
1640 VPIRValue *IRV;
1641 if (Def->getNumOperands() == 1 &&
1642 match(R: Def, P: m_ComputeReductionResult(Op0: m_VPIRValue(V&: IRV))))
1643 return Def->replaceAllUsesWith(New: IRV);
1644
1645 // Some simplifications can only be applied after unrolling. Perform them
1646 // below.
1647 if (!Plan->isUnrolled())
1648 return;
1649
1650 // After unrolling, extract-lane may be used to extract values from multiple
1651 // scalar sources. Only simplify when extracting from a single scalar source.
1652 VPValue *LaneToExtract;
1653 if (match(R: Def, P: m_ExtractLane(Op0: m_VPValue(V&: LaneToExtract), Op1: m_VPValue(V&: A)))) {
1654 // Simplify extract-lane(%lane_num, %scalar_val) -> %scalar_val.
1655 if (vputils::isSingleScalar(VPV: A))
1656 return Def->replaceAllUsesWith(New: A);
1657
1658 // Replace extract-lane(0, canonical-WIDEN-INDUCTION) with the region's
1659 // scalar canonical IV.
1660 VPWidenIntOrFpInductionRecipe *WidenIV;
1661 if (match(V: LaneToExtract, P: m_ZeroInt()) &&
1662 match(V: A, P: m_CanonicalWidenIV(V&: WidenIV)))
1663 return Def->replaceAllUsesWith(New: WidenIV->getRegion()->getCanonicalIV());
1664
1665 // Simplify extract-lane with single source to extract-element.
1666 Def->replaceAllUsesWith(New: Builder.createNaryOp(
1667 Opcode: Instruction::ExtractElement, Operands: {A, LaneToExtract}, DL: Def->getDebugLoc()));
1668 return;
1669 }
1670
1671 // Look for cycles where Def is of the form:
1672 // X = phi(0, IVInc) ; used only by IVInc, or by IVInc and Inc = X + Y
1673 // IVInc = X + Step ; used by X and Def
1674 // Def = IVInc + Y
1675 // Fold the increment Y into the phi's start value, replace Def with IVInc,
1676 // and if Inc exists, replace it with X.
1677 if (match(R: Def, P: m_Add(Op0: m_Add(Op0: m_VPValue(V&: X), Op1: m_VPValue()), Op1: m_VPValue(V&: Y))) &&
1678 isa<VPIRValue>(Val: Y) &&
1679 match(V: X, P: m_VPPhi(Op0: m_ZeroInt(), Op1: m_Specific(VPV: Def->getOperand(N: 0))))) {
1680 auto *Phi = cast<VPPhi>(Val: X);
1681 auto *IVInc = Def->getOperand(N: 0);
1682 if (IVInc->getNumUsers() == 2) {
1683 // If Phi has a second user (besides IVInc's defining recipe), it must
1684 // be Inc = Phi + Y for the fold to apply.
1685 auto *Inc = dyn_cast_or_null<VPSingleDefRecipe>(
1686 Val: findUserOf(V: Phi, P: m_Add(Op0: m_Specific(VPV: Phi), Op1: m_Specific(VPV: Y))));
1687 if (Phi->getNumUsers() == 1 || (Phi->getNumUsers() == 2 && Inc)) {
1688 Def->replaceAllUsesWith(New: IVInc);
1689 if (Inc)
1690 Inc->replaceAllUsesWith(New: Phi);
1691 Phi->setOperand(I: 0, New: Y);
1692 return;
1693 }
1694 }
1695 }
1696
1697 // Simplify unrolled VectorPointer without offset, or with zero offset, to
1698 // just the pointer operand.
1699 if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(Val: Def))
1700 if (!VPR->getVFxPart() || match(V: VPR->getVFxPart(), P: m_ZeroInt()))
1701 return VPR->replaceAllUsesWith(New: VPR->getOperand(N: 0));
1702
1703 // VPScalarIVSteps after unrolling can be replaced by their start value, if
1704 // the start index is zero and only the first lane 0 is demanded.
1705 if (auto *Steps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def)) {
1706 if (!Steps->getStartIndex() && vputils::onlyFirstLaneUsed(Def: Steps)) {
1707 Steps->replaceAllUsesWith(New: Steps->getOperand(N: 0));
1708 return;
1709 }
1710 }
1711 // Simplify redundant ReductionStartVector recipes after unrolling.
1712 VPValue *StartV;
1713 if (match(R: Def, P: m_VPInstruction<VPInstruction::ReductionStartVector>(
1714 Ops: m_VPValue(V&: StartV), Ops: m_VPValue(), Ops: m_VPValue()))) {
1715 Def->replaceUsesWithIf(New: StartV, ShouldReplace: [](const VPUser &U, unsigned Idx) {
1716 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &U);
1717 return PhiR && PhiR->isInLoop();
1718 });
1719 return;
1720 }
1721
1722 if (Plan->getConcreteUF() == 1 && match(R: Def, P: m_ExtractLastPart(Op0: m_VPValue(V&: A))))
1723 return Def->replaceAllUsesWith(New: A);
1724}
1725
1726void VPlanTransforms::simplifyRecipes(VPlan &Plan) {
1727 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
1728 Plan.getEntry());
1729 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
1730 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
1731 if (auto *Def = dyn_cast<VPSingleDefRecipe>(Val: &R))
1732 simplifyRecipe(Def);
1733 }
1734}
1735
1736void VPlanTransforms::simplifyReverses(VPlan &Plan) {
1737 // Pull out reverses from any elementwise op.
1738 // binop(reverse(x), reverse(y)) -> reverse(binop(x,y))
1739 vputils::pullOutPermutations(
1740 Plan, Perm: [](VPValue *&X) { return m_Reverse(Op0: m_VPValue(V&: X)); },
1741 Build: [](auto *X) { return new VPInstruction(VPInstruction::Reverse, X); });
1742
1743 // reverse(reverse(x)) -> x
1744 VPValue *X;
1745 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1746 Range: vp_depth_first_deep(G: Plan.getEntry())))
1747 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
1748 if (match(V: &R, P: m_Reverse(Op0: m_Reverse(Op0: m_VPValue(V&: X)))))
1749 R.getVPSingleValue()->replaceAllUsesWith(New: X);
1750}
1751
1752/// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the
1753/// header mask to be simplified further when tail folding, e.g. in
1754/// optimizeEVLMasks.
1755static void reassociateHeaderMask(VPlan &Plan) {
1756 VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
1757 if (!HeaderMask)
1758 return;
1759
1760 SmallVector<VPUser *> Worklist;
1761 for (VPUser *U : HeaderMask->users())
1762 if (match(U, P: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue())))
1763 append_range(C&: Worklist, R: cast<VPSingleDefRecipe>(Val: U)->users());
1764
1765 while (!Worklist.empty()) {
1766 auto *R = dyn_cast<VPSingleDefRecipe>(Val: Worklist.pop_back_val());
1767 VPValue *X, *Y;
1768 if (!R || !match(R, P: m_LogicalAnd(
1769 Op0: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue(V&: X)),
1770 Op1: m_VPValue(V&: Y))))
1771 continue;
1772 append_range(C&: Worklist, R: R->users());
1773 VPBuilder Builder(R);
1774 R->replaceAllUsesWith(
1775 New: Builder.createLogicalAnd(LHS: HeaderMask, RHS: Builder.createLogicalAnd(LHS: X, RHS: Y)));
1776 }
1777}
1778
1779static std::optional<Instruction::BinaryOps>
1780getUnmaskedDivRemOpcode(Intrinsic::ID ID) {
1781 switch (ID) {
1782 case Intrinsic::masked_udiv:
1783 return Instruction::UDiv;
1784 case Intrinsic::masked_sdiv:
1785 return Instruction::SDiv;
1786 case Intrinsic::masked_urem:
1787 return Instruction::URem;
1788 case Intrinsic::masked_srem:
1789 return Instruction::SRem;
1790 default:
1791 return {};
1792 }
1793}
1794
1795static void narrowToSingleScalarRecipes(VPlan &Plan) {
1796 if (Plan.hasScalarVFOnly())
1797 return;
1798
1799 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1800 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
1801 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
1802 if (!isa<VPWidenRecipe, VPWidenGEPRecipe, VPReplicateRecipe,
1803 VPWidenIntrinsicRecipe>(Val: &R))
1804 continue;
1805 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
1806 if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
1807 continue;
1808
1809 auto *RepOrWidenR = cast<VPRecipeWithIRFlags>(Val: &R);
1810 if (RepR && RepR->getOpcode() == Instruction::Store &&
1811 vputils::isSingleScalar(VPV: RepR->getOperand(N: 1))) {
1812 auto *Clone = new VPReplicateRecipe(
1813 RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
1814 true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
1815 *RepR /*Metadata*/, RepR->getDebugLoc());
1816 Clone->insertBefore(InsertPos: RepOrWidenR);
1817 VPBuilder Builder(Clone);
1818 VPValue *ExtractOp = Clone->getOperand(N: 0);
1819 if (vputils::isUniformAcrossVFsAndUFs(V: RepR->getOperand(N: 1)))
1820 ExtractOp =
1821 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastPart, Operands: ExtractOp);
1822 ExtractOp =
1823 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastLane, Operands: ExtractOp);
1824 Clone->setOperand(I: 0, New: ExtractOp);
1825 RepR->eraseFromParent();
1826 continue;
1827 }
1828
1829 // Narrow llvm.masked.{u,s}{div,rem} intrinsics with a safe divisor.
1830 if (auto *IntrR = dyn_cast<VPWidenIntrinsicRecipe>(Val: RepOrWidenR)) {
1831 if (!vputils::onlyFirstLaneUsed(Def: IntrR))
1832 continue;
1833 auto Opc = getUnmaskedDivRemOpcode(ID: IntrR->getVectorIntrinsicID());
1834 if (!Opc)
1835 continue;
1836 VPBuilder Builder(IntrR);
1837 VPValue *SafeDivisor = Builder.createSelect(
1838 Cond: IntrR->getOperand(N: 2), TrueVal: IntrR->getOperand(N: 1),
1839 FalseVal: Plan.getConstantInt(Ty: IntrR->getScalarType(), Val: 1));
1840 VPValue *Clone = Builder.createNaryOp(
1841 Opcode: *Opc, Operands: {IntrR->getOperand(N: 0), SafeDivisor},
1842 Flags: VPIRFlags::getDefaultFlags(Opcode: *Opc), DL: IntrR->getDebugLoc());
1843 IntrR->replaceAllUsesWith(New: Clone);
1844 IntrR->eraseFromParent();
1845 continue;
1846 }
1847
1848 // Skip recipes that aren't single scalars.
1849 if (!vputils::isSingleScalar(VPV: RepOrWidenR))
1850 continue;
1851
1852 // Predicate to check if a user of Op introduces extra broadcasts.
1853 auto IntroducesBCastOf = [](const VPValue *Op) {
1854 return [Op](const VPUser *U) {
1855 if (auto *VPI = dyn_cast<VPInstruction>(Val: U)) {
1856 if (is_contained(Set: {VPInstruction::ExtractLastLane,
1857 VPInstruction::ExtractLastPart,
1858 VPInstruction::ExtractPenultimateElement},
1859 Element: VPI->getOpcode()))
1860 return false;
1861 }
1862 return !U->usesScalars(Op);
1863 };
1864 };
1865
1866 if (any_of(Range: RepOrWidenR->users(), P: IntroducesBCastOf(RepOrWidenR)) &&
1867 none_of(Range: RepOrWidenR->operands(), P: [&](VPValue *Op) {
1868 if (any_of(
1869 Range: make_filter_range(Range: Op->users(), Pred: not_equal_to(Arg&: RepOrWidenR)),
1870 P: IntroducesBCastOf(Op)))
1871 return false;
1872 // Non-constant live-ins require broadcasts, while constants do not
1873 // need explicit broadcasts.
1874 bool LiveInNeedsBroadcast =
1875 isa<VPIRValue>(Val: Op) && !isa<VPConstant>(Val: Op);
1876 auto *OpR = dyn_cast<VPReplicateRecipe>(Val: Op);
1877 return LiveInNeedsBroadcast || (OpR && OpR->isSingleScalar());
1878 }))
1879 continue;
1880
1881 auto *Clone = VPBuilder::createSingleScalarOp(
1882 Opcode: vputils::getOpcode(V: RepOrWidenR), Operands: RepOrWidenR->operands(),
1883 /*Mask=*/nullptr, Flags: *RepOrWidenR, Metadata: {}, DL: DebugLoc::getUnknown(),
1884 UV: RepOrWidenR->getUnderlyingInstr());
1885 Clone->insertBefore(InsertPos: RepOrWidenR);
1886 RepOrWidenR->replaceAllUsesWith(New: Clone);
1887 if (vputils::isDeadRecipe(R&: *RepOrWidenR))
1888 RepOrWidenR->eraseFromParent();
1889 }
1890 }
1891}
1892
1893/// Try to see if all of \p Blend's masks share a common value logically and'ed
1894/// and remove it from the masks.
1895static void removeCommonBlendMask(VPBlendRecipe *Blend) {
1896 if (Blend->isNormalized())
1897 return;
1898 VPValue *CommonEdgeMask;
1899 if (!match(V: Blend->getMask(Idx: 0),
1900 P: m_LogicalAnd(Op0: m_VPValue(V&: CommonEdgeMask), Op1: m_VPValue())))
1901 return;
1902 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1903 if (!match(V: Blend->getMask(Idx: I),
1904 P: m_LogicalAnd(Op0: m_Specific(VPV: CommonEdgeMask), Op1: m_VPValue())))
1905 return;
1906 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1907 Blend->setMask(Idx: I, V: Blend->getMask(Idx: I)->getDefiningRecipe()->getOperand(N: 1));
1908}
1909
1910/// Normalize and simplify VPBlendRecipes. Should be run after simplifyRecipes
1911/// to make sure the masks are simplified.
1912static void simplifyBlends(VPlan &Plan) {
1913 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1914 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
1915 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
1916 auto *Blend = dyn_cast<VPBlendRecipe>(Val: &R);
1917 if (!Blend)
1918 continue;
1919
1920 removeCommonBlendMask(Blend);
1921
1922 // Try to remove redundant blend recipes.
1923 SmallPtrSet<VPValue *, 4> UniqueValues;
1924 if (Blend->isNormalized() || !match(V: Blend->getMask(Idx: 0), P: m_False()))
1925 UniqueValues.insert(Ptr: Blend->getIncomingValue(Idx: 0));
1926 for (unsigned I = 1; I != Blend->getNumIncomingValues(); ++I)
1927 if (!match(V: Blend->getMask(Idx: I), P: m_False()))
1928 UniqueValues.insert(Ptr: Blend->getIncomingValue(Idx: I));
1929
1930 if (UniqueValues.size() == 1) {
1931 Blend->replaceAllUsesWith(New: *UniqueValues.begin());
1932 Blend->eraseFromParent();
1933 continue;
1934 }
1935
1936 if (Blend->isNormalized())
1937 continue;
1938
1939 // Normalize the blend so its first incoming value is used as the initial
1940 // value with the others blended into it.
1941
1942 unsigned StartIndex = 0;
1943 for (unsigned I = 0; I != Blend->getNumIncomingValues(); ++I) {
1944 // If a value's mask is used only by the blend then is can be deadcoded.
1945 // TODO: Find the most expensive mask that can be deadcoded, or a mask
1946 // that's used by multiple blends where it can be removed from them all.
1947 VPValue *Mask = Blend->getMask(Idx: I);
1948 if (Mask->hasOneUse() && !match(V: Mask, P: m_False())) {
1949 StartIndex = I;
1950 break;
1951 }
1952 }
1953
1954 SmallVector<VPValue *, 4> OperandsWithMask;
1955 OperandsWithMask.push_back(Elt: Blend->getIncomingValue(Idx: StartIndex));
1956
1957 for (unsigned I = 0; I != Blend->getNumIncomingValues(); ++I) {
1958 if (I == StartIndex)
1959 continue;
1960 OperandsWithMask.push_back(Elt: Blend->getIncomingValue(Idx: I));
1961 OperandsWithMask.push_back(Elt: Blend->getMask(Idx: I));
1962 }
1963
1964 auto *NewBlend =
1965 new VPBlendRecipe(cast_or_null<PHINode>(Val: Blend->getUnderlyingValue()),
1966 OperandsWithMask, *Blend, Blend->getDebugLoc());
1967 NewBlend->insertBefore(InsertPos: &R);
1968
1969 VPValue *DeadMask = Blend->getMask(Idx: StartIndex);
1970 Blend->replaceAllUsesWith(New: NewBlend);
1971 Blend->eraseFromParent();
1972 vputils::recursivelyDeleteDeadRecipes(V: DeadMask);
1973
1974 /// Simplify BLEND %a, %b, Not(%mask) -> BLEND %b, %a, %mask.
1975 VPValue *NewMask;
1976 if (NewBlend->getNumOperands() == 3 &&
1977 match(V: NewBlend->getMask(Idx: 1), P: m_Not(Op0: m_VPValue(V&: NewMask)))) {
1978 VPValue *Inc0 = NewBlend->getOperand(N: 0);
1979 VPValue *Inc1 = NewBlend->getOperand(N: 1);
1980 VPValue *OldMask = NewBlend->getOperand(N: 2);
1981 NewBlend->setOperand(I: 0, New: Inc1);
1982 NewBlend->setOperand(I: 1, New: Inc0);
1983 NewBlend->setOperand(I: 2, New: NewMask);
1984 if (OldMask->user_empty())
1985 cast<VPInstruction>(Val: OldMask)->eraseFromParent();
1986 }
1987 }
1988 }
1989}
1990
1991/// Optimize the width of vector induction variables in \p Plan based on a known
1992/// constant Trip Count, \p BestVF and \p BestUF.
1993static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan,
1994 ElementCount BestVF,
1995 unsigned BestUF) {
1996 // Only proceed if we have not completely removed the vector region.
1997 if (!Plan.getVectorLoopRegion())
1998 return false;
1999
2000 const APInt *TC;
2001 if (!BestVF.isFixed() || !match(V: Plan.getTripCount(), P: m_APInt(C&: TC)))
2002 return false;
2003
2004 // Calculate the minimum power-of-2 bit width that can fit the known TC, VF
2005 // and UF. Returns at least 8.
2006 auto ComputeBitWidth = [](APInt TC, uint64_t Align) {
2007 APInt AlignedTC =
2008 Align * APIntOps::RoundingUDiv(A: TC, B: APInt(TC.getBitWidth(), Align),
2009 RM: APInt::Rounding::UP);
2010 APInt MaxVal = AlignedTC - 1;
2011 return std::max<unsigned>(a: PowerOf2Ceil(A: MaxVal.getActiveBits()), b: 8);
2012 };
2013 unsigned NewBitWidth =
2014 ComputeBitWidth(*TC, BestVF.getKnownMinValue() * BestUF);
2015
2016 LLVMContext &Ctx = Plan.getContext();
2017 auto *NewIVTy = IntegerType::get(C&: Ctx, NumBits: NewBitWidth);
2018
2019 bool MadeChange = false;
2020
2021 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
2022 for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
2023 // Currently only handle canonical IVs as it is trivial to replace the start
2024 // and stop values, and we currently only perform the optimization when the
2025 // IV has a single use.
2026 VPWidenIntOrFpInductionRecipe *WideIV;
2027 if (!match(V: &Phi, P: m_CanonicalWidenIV(V&: WideIV)))
2028 continue;
2029 if (WideIV->hasMoreThanOneUniqueUser() ||
2030 NewIVTy == WideIV->getScalarType())
2031 continue;
2032
2033 // Currently only handle cases where the single user is a header-mask
2034 // comparison with the backedge-taken-count.
2035 VPUser *SingleUser = WideIV->getSingleUser();
2036 if (!SingleUser ||
2037 !match(U: SingleUser,
2038 P: m_ICmp(Op0: m_Specific(VPV: WideIV),
2039 Op1: m_Broadcast(Op0: m_Specific(VPV: Plan.getBackedgeTakenCount())))))
2040 continue;
2041
2042 // Update IV operands and comparison bound to use new narrower type.
2043 assert(!WideIV->getTruncInst() &&
2044 "canonical IV is not expected to have a truncation");
2045 auto *NewWideIV = new VPWidenIntOrFpInductionRecipe(
2046 WideIV->getPHINode(), Plan.getZero(Ty: NewIVTy),
2047 Plan.getConstantInt(Ty: NewIVTy, Val: 1), WideIV->getVFValue(),
2048 WideIV->getInductionDescriptor(), *WideIV, WideIV->getDebugLoc());
2049 NewWideIV->insertBefore(InsertPos: WideIV);
2050
2051 auto *NewBTC = new VPWidenCastRecipe(
2052 Instruction::Trunc, Plan.getOrCreateBackedgeTakenCount(), NewIVTy,
2053 nullptr, VPIRFlags::getDefaultFlags(Opcode: Instruction::Trunc));
2054 Plan.getVectorPreheader()->appendRecipe(Recipe: NewBTC);
2055 auto *Cmp = cast<VPInstruction>(Val: WideIV->getSingleUser());
2056 Cmp->replaceAllUsesWith(
2057 New: VPBuilder(Cmp).createICmp(Pred: Cmp->getPredicate(), A: NewWideIV, B: NewBTC));
2058
2059 MadeChange = true;
2060 }
2061
2062 return MadeChange;
2063}
2064
2065/// Return true if \p Cond is known to be true for given \p BestVF and \p
2066/// BestUF.
2067static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
2068 ElementCount BestVF, unsigned BestUF,
2069 PredicatedScalarEvolution &PSE) {
2070 if (match(V: Cond, P: m_BinaryOr(Op0: m_VPValue(), Op1: m_VPValue())))
2071 return any_of(Range: Cond->getDefiningRecipe()->operands(), P: [&Plan, BestVF, BestUF,
2072 &PSE](VPValue *C) {
2073 return isConditionTrueViaVFAndUF(Cond: C, Plan, BestVF, BestUF, PSE);
2074 });
2075
2076 auto *CanIV = Plan.getVectorLoopRegion()->getCanonicalIV();
2077 if (!match(V: Cond, P: m_SpecificICmp(
2078 MatchPred: CmpInst::ICMP_EQ,
2079 Op0: m_c_Add(Op0: m_Specific(VPV: CanIV), Op1: m_Specific(VPV: &Plan.getVFxUF())),
2080 Op1: m_Specific(VPV: &Plan.getVectorTripCount()))))
2081 return false;
2082
2083 // The compare checks CanIV + VFxUF == vector trip count. The vector trip
2084 // count is not conveniently available as SCEV so far, so we compare directly
2085 // against the original trip count. This is stricter than necessary, as we
2086 // will only return true if the trip count == vector trip count.
2087 const SCEV *VectorTripCount =
2088 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2089 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2090 VectorTripCount = vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2091 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2092 "Trip count SCEV must be computable");
2093 ScalarEvolution &SE = *PSE.getSE();
2094 ElementCount NumElements = BestVF.multiplyCoefficientBy(RHS: BestUF);
2095 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2096 return SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: VectorTripCount, RHS: C);
2097}
2098
2099/// Try to replace multiple active lane masks used for control flow with
2100/// a single, wide active lane mask instruction followed by multiple
2101/// extract subvector intrinsics. This applies to the active lane mask
2102/// instructions both in the loop and in the preheader.
2103/// Incoming values of all ActiveLaneMaskPHIs are updated to use the
2104/// new extracts from the first active lane mask, which has it's last
2105/// operand (multiplier) set to UF.
2106static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF,
2107 unsigned UF) {
2108 if (!EnableWideActiveLaneMask || !VF.isVector() || UF == 1)
2109 return false;
2110
2111 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2112 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2113 auto *Term = &ExitingVPBB->back();
2114
2115 using namespace llvm::VPlanPatternMatch;
2116 if (!match(V: Term, P: m_BranchOnCond(Op0: m_Not(Op0: m_ActiveLaneMask(
2117 Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())))))
2118 return false;
2119
2120 auto *Header = cast<VPBasicBlock>(Val: VectorRegion->getEntry());
2121 LLVMContext &Ctx = Plan.getContext();
2122
2123 auto ExtractFromALM = [&](VPInstruction *ALM,
2124 SmallVectorImpl<VPValue *> &Extracts) {
2125 DebugLoc DL = ALM->getDebugLoc();
2126 for (unsigned Part = 0; Part < UF; ++Part) {
2127 SmallVector<VPValue *> Ops;
2128 Ops.append(IL: {ALM, Plan.getConstantInt(BitWidth: 64, Val: VF.getKnownMinValue() * Part)});
2129 auto *Ext =
2130 new VPWidenIntrinsicRecipe(Intrinsic::vector_extract, Ops,
2131 IntegerType::getInt1Ty(C&: Ctx), {}, {}, DL);
2132 Extracts[Part] = Ext;
2133 Ext->insertAfter(InsertPos: ALM);
2134 }
2135 };
2136
2137 // Create a list of each active lane mask phi, ordered by unroll part.
2138 SmallVector<VPActiveLaneMaskPHIRecipe *> Phis(UF, nullptr);
2139 for (VPRecipeBase &R : Header->phis()) {
2140 auto *Phi = dyn_cast<VPActiveLaneMaskPHIRecipe>(Val: &R);
2141 if (!Phi)
2142 continue;
2143 VPValue *Index = nullptr;
2144 match(V: Phi->getBackedgeValue(),
2145 P: m_ActiveLaneMask(Op0: m_VPValue(V&: Index), Op1: m_VPValue(), Op2: m_VPValue()));
2146 assert(Index && "Expected index from ActiveLaneMask instruction");
2147
2148 uint64_t Part;
2149 if (match(V: Index,
2150 P: m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>(
2151 Ops: m_VPValue(), Ops: m_Mul(Op0: m_VPValue(), Op1: m_ConstantInt(C&: Part)))))
2152 Phis[Part] = Phi;
2153 else {
2154 // Anything other than a CanonicalIVIncrementForPart is part 0
2155 assert(!match(
2156 Index,
2157 m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>()));
2158 Phis[0] = Phi;
2159 }
2160 }
2161
2162 assert(all_of(Phis, not_equal_to(nullptr)) &&
2163 "Expected one VPActiveLaneMaskPHIRecipe for each unroll part");
2164
2165 auto *EntryALM = cast<VPInstruction>(Val: Phis[0]->getStartValue());
2166 auto *LoopALM = cast<VPInstruction>(Val: Phis[0]->getBackedgeValue());
2167
2168 assert((EntryALM->getOpcode() == VPInstruction::ActiveLaneMask &&
2169 LoopALM->getOpcode() == VPInstruction::ActiveLaneMask) &&
2170 "Expected incoming values of Phi to be ActiveLaneMasks");
2171
2172 // When using wide lane masks, the return type of the get.active.lane.mask
2173 // intrinsic is VF x UF (last operand).
2174 VPValue *ALMMultiplier = Plan.getConstantInt(BitWidth: 64, Val: UF);
2175 EntryALM->setOperand(I: 2, New: ALMMultiplier);
2176 LoopALM->setOperand(I: 2, New: ALMMultiplier);
2177
2178 // Create UF x extract vectors and insert into preheader.
2179 SmallVector<VPValue *> EntryExtracts(UF);
2180 ExtractFromALM(EntryALM, EntryExtracts);
2181
2182 // Create UF x extract vectors and insert before the loop compare & branch,
2183 // updating the compare to use the first extract.
2184 SmallVector<VPValue *> LoopExtracts(UF);
2185 ExtractFromALM(LoopALM, LoopExtracts);
2186 VPInstruction *Not = cast<VPInstruction>(Val: Term->getOperand(N: 0));
2187 Not->setOperand(I: 0, New: LoopExtracts[0]);
2188
2189 // Update the incoming values of active lane mask phis.
2190 for (unsigned Part = 0; Part < UF; ++Part) {
2191 Phis[Part]->setStartValue(EntryExtracts[Part]);
2192 Phis[Part]->setBackedgeValue(LoopExtracts[Part]);
2193 }
2194
2195 return true;
2196}
2197
2198/// Try to simplify the branch condition of \p Plan. This may restrict the
2199/// resulting plan to \p BestVF and \p BestUF.
2200static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
2201 unsigned BestUF,
2202 PredicatedScalarEvolution &PSE) {
2203 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2204 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2205 auto *Term = &ExitingVPBB->back();
2206 VPValue *Cond;
2207 auto m_CanIVInc = m_Add(Op0: m_VPValue(), Op1: m_Specific(VPV: &Plan.getVFxUF()));
2208 // Check if the branch condition compares the canonical IV increment (for main
2209 // loop), or the canonical IV increment plus an offset (for epilog loop).
2210 if (match(V: Term, P: m_BranchOnCount(
2211 Op0: m_CombineOr(Ps: m_CanIVInc, Ps: m_c_Add(Op0: m_CanIVInc, Op1: m_LiveIn())),
2212 Op1: m_VPValue())) ||
2213 match(V: Term, P: m_BranchOnCond(Op0: m_Not(Op0: m_ActiveLaneMask(
2214 Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue()))))) {
2215 // Try to simplify the branch condition if VectorTC <= VF * UF when the
2216 // latch terminator is BranchOnCount or BranchOnCond(Not(ActiveLaneMask)).
2217 const SCEV *VectorTripCount =
2218 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2219 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2220 VectorTripCount =
2221 vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2222 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2223 "Trip count SCEV must be computable");
2224 ScalarEvolution &SE = *PSE.getSE();
2225 ElementCount NumElements = BestVF.multiplyCoefficientBy(RHS: BestUF);
2226 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2227 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_ULE, LHS: VectorTripCount, RHS: C))
2228 return false;
2229 } else if (match(V: Term, P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))) ||
2230 match(V: Term, P: m_BranchOnTwoConds(Op0: m_VPValue(), Op1: m_VPValue(V&: Cond)))) {
2231 // For BranchOnCond, check if we can prove the condition to be true using VF
2232 // and UF.
2233 if (!isConditionTrueViaVFAndUF(Cond, Plan, BestVF, BestUF, PSE))
2234 return false;
2235 } else {
2236 return false;
2237 }
2238
2239 // The vector loop region only executes once. Convert terminator of the
2240 // exiting block to exit in the first iteration.
2241 if (match(V: Term, P: m_BranchOnTwoConds())) {
2242 Term->setOperand(I: 1, New: Plan.getTrue());
2243 return true;
2244 }
2245
2246 auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, Plan.getTrue(), {},
2247 {}, Term->getDebugLoc());
2248 ExitingVPBB->appendRecipe(Recipe: BOC);
2249 Term->eraseFromParent();
2250
2251 return true;
2252}
2253
2254void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
2255 unsigned BestUF,
2256 PredicatedScalarEvolution &PSE) {
2257 assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
2258 assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
2259
2260 bool MadeChange = tryToReplaceALMWithWideALM(Plan, VF: BestVF, UF: BestUF);
2261 MadeChange |= simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
2262 MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
2263
2264 if (MadeChange) {
2265 Plan.setVF(BestVF);
2266 assert(Plan.getConcreteUF() == BestUF && "BestUF must match the Plan's UF");
2267 }
2268}
2269
2270void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
2271 for (VPRecipeBase &R :
2272 Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
2273 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
2274 if (!PhiR)
2275 continue;
2276 RecurKind RK = PhiR->getRecurrenceKind();
2277 if (RK != RecurKind::Add && RK != RecurKind::Mul && RK != RecurKind::Sub &&
2278 RK != RecurKind::AddChainWithSubs)
2279 continue;
2280
2281 for (VPUser *U : vputils::collectUsersRecursively(V: PhiR))
2282 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: U)) {
2283 RecWithFlags->dropPoisonGeneratingFlags();
2284 }
2285 }
2286}
2287
2288namespace {
2289struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
2290 /// If recipe \p R will lower to a GEP with a non-i8 source element type,
2291 /// return that source element type.
2292 static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) {
2293 // All VPInstructions that lower to GEPs must have the i8 source element
2294 // type (as they are PtrAdds), so we omit it.
2295 return TypeSwitch<const VPSingleDefRecipe *, Type *>(R)
2296 .Case(caseFn: [](const VPReplicateRecipe *I) -> Type * {
2297 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I->getUnderlyingValue()))
2298 return GEP->getSourceElementType();
2299 return nullptr;
2300 })
2301 .Case<VPVectorPointerRecipe, VPWidenGEPRecipe>(
2302 caseFn: [](auto *I) { return I->getSourceElementType(); })
2303 .Default(defaultFn: [](auto *) { return nullptr; });
2304 }
2305
2306 /// Returns true if recipe \p Def can be safely handed for CSE.
2307 static bool canHandle(const VPSingleDefRecipe *Def) {
2308 // We can extend the list of handled recipes in the future,
2309 // provided we account for the data embedded in them while checking for
2310 // equality or hashing.
2311 auto C = vputils::getOpcodeOrIntrinsicID(V: Def);
2312
2313 // The issue with (Insert|Extract)Value is that the index of the
2314 // insert/extract is not a proper operand in LLVM IR, and hence also not in
2315 // VPlan.
2316 if (!C || (!C->first && (C->second == Instruction::InsertValue ||
2317 C->second == Instruction::ExtractValue)))
2318 return false;
2319
2320 // During CSE, we can only handle non-memory recipes, as memory can alias.
2321 return !Def->mayReadOrWriteMemory();
2322 }
2323
2324 /// Hash the underlying data of \p Def.
2325 static unsigned getHashValue(const VPSingleDefRecipe *Def) {
2326 hash_code Result = hash_combine(
2327 args: Def->getVPRecipeID(), args: vputils::getOpcodeOrIntrinsicID(V: Def),
2328 args: getGEPSourceElementType(R: Def), args: Def->getScalarType(),
2329 args: vputils::isSingleScalar(VPV: Def), args: hash_combine_range(R: Def->operands()));
2330 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: Def))
2331 if (RFlags->hasPredicate())
2332 return hash_combine(args: Result, args: RFlags->getPredicate());
2333 if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def))
2334 return hash_combine(args: Result, args: SIVSteps->getInductionOpcode());
2335 return Result;
2336 }
2337
2338 /// Check equality of underlying data of \p L and \p R.
2339 static bool isEqual(const VPSingleDefRecipe *L, const VPSingleDefRecipe *R) {
2340 if (L->getVPRecipeID() != R->getVPRecipeID() ||
2341 vputils::getOpcodeOrIntrinsicID(V: L) !=
2342 vputils::getOpcodeOrIntrinsicID(V: R) ||
2343 getGEPSourceElementType(R: L) != getGEPSourceElementType(R) ||
2344 vputils::isSingleScalar(VPV: L) != vputils::isSingleScalar(VPV: R) ||
2345 !equal(LRange: L->operands(), RRange: R->operands()))
2346 return false;
2347 assert(vputils::getOpcodeOrIntrinsicID(L) &&
2348 vputils::getOpcodeOrIntrinsicID(R) &&
2349 "must have valid opcode info for both recipes");
2350 if (auto *LFlags = dyn_cast<VPRecipeWithIRFlags>(Val: L))
2351 if (LFlags->hasPredicate() &&
2352 LFlags->getPredicate() !=
2353 cast<VPRecipeWithIRFlags>(Val: R)->getPredicate())
2354 return false;
2355 if (auto *LSIV = dyn_cast<VPScalarIVStepsRecipe>(Val: L))
2356 if (LSIV->getInductionOpcode() !=
2357 cast<VPScalarIVStepsRecipe>(Val: R)->getInductionOpcode())
2358 return false;
2359 // Phi recipes can only be equal if they are in the same VPBB, as they
2360 // implicitly depend on their predecessors.
2361 if (isa<VPWidenPHIRecipe>(Val: L) && L->getParent() != R->getParent())
2362 return false;
2363 // Recipes in replicate regions implicitly depend on predicate. If either
2364 // recipe is in a replicate region, only consider them equal if both have
2365 // the same parent.
2366 const VPRegionBlock *RegionL = L->getRegion();
2367 const VPRegionBlock *RegionR = R->getRegion();
2368 if (((RegionL && RegionL->isReplicator()) ||
2369 (RegionR && RegionR->isReplicator())) &&
2370 L->getParent() != R->getParent())
2371 return false;
2372 return L->getScalarType() == R->getScalarType();
2373 }
2374};
2375} // end anonymous namespace
2376
2377/// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
2378/// Plan.
2379void VPlanTransforms::cse(VPlan &Plan) {
2380 VPDominatorTree VPDT(Plan);
2381 DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo> CSEMap;
2382
2383 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
2384 Plan.getEntry());
2385 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
2386 for (VPRecipeBase &R : *VPBB) {
2387 auto *Def = dyn_cast<VPSingleDefRecipe>(Val: &R);
2388 if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
2389 continue;
2390 if (VPSingleDefRecipe *V = CSEMap.lookup(Val: Def)) {
2391 // V must dominate Def for a valid replacement.
2392 if (!VPDT.dominates(A: V->getParent(), B: VPBB))
2393 continue;
2394 // Only keep flags present on both V and Def.
2395 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: V))
2396 RFlags->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Def));
2397 Def->replaceAllUsesWith(New: V);
2398 continue;
2399 }
2400 CSEMap[Def] = Def;
2401 }
2402 }
2403}
2404
2405/// Return true if we do not know how to (mechanically) hoist or sink a
2406/// non-memory or memory recipe \p R out of a loop region. When sinking, passing
2407/// \p Sinking = true ensures that assumes aren't sunk.
2408static bool cannotHoistOrSinkRecipe(VPRecipeBase &R, VPBasicBlock *FirstBB,
2409 VPBasicBlock *LastBB,
2410 bool Sinking = false) {
2411 if (!isa<VPReplicateRecipe>(Val: R) || !R.mayReadOrWriteMemory() ||
2412 match(V: &R, P: m_Intrinsic<Intrinsic::assume>()))
2413 return vputils::cannotHoistOrSinkRecipe(R, Sinking);
2414
2415 // Check that the memory operation doesn't alias between FirstBB and LastBB.
2416 auto MemLoc = vputils::getMemoryLocation(R);
2417
2418 // TODO: Could make use of SinkStoreInfo::isNoAliasViaDistance by collecting
2419 // stores upfront, and constructing a full SinkStoreInfo.
2420 auto SinkInfo =
2421 Sinking ? std::make_optional(t: SinkStoreInfo(cast<VPReplicateRecipe>(Val&: R)))
2422 : std::nullopt;
2423
2424 return !MemLoc ||
2425 !canHoistOrSinkWithNoAliasCheck(MemLoc: *MemLoc, FirstBB, LastBB, SinkInfo);
2426}
2427
2428/// Move loop-invariant recipes out of the vector loop region in \p Plan.
2429static void licm(VPlan &Plan) {
2430 VPBasicBlock *Preheader = Plan.getVectorPreheader();
2431
2432 // Hoist any loop invariant recipes from the vector loop region to the
2433 // preheader. Preform a shallow traversal of the vector loop region, to
2434 // exclude recipes in replicate regions. Since the top-level blocks in the
2435 // vector loop region are guaranteed to execute if the vector pre-header is,
2436 // we don't need to check speculation safety.
2437 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2438 assert(Preheader->getSingleSuccessor() == LoopRegion &&
2439 "Expected vector prehader's successor to be the vector loop region");
2440 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2441 Range: vp_depth_first_shallow(G: LoopRegion->getEntry()))) {
2442 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2443 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2444 LastBB: LoopRegion->getExitingBasicBlock()))
2445 continue;
2446 if (any_of(Range: R.operands(), P: [](VPValue *Op) {
2447 return !Op->isDefinedOutsideLoopRegions();
2448 }))
2449 continue;
2450 R.moveBefore(BB&: *Preheader, I: Preheader->end());
2451 }
2452 }
2453
2454#ifndef NDEBUG
2455 VPDominatorTree VPDT(Plan);
2456#endif
2457 // Sink recipes with no users inside the vector loop region if all users are
2458 // in the same exit block of the region.
2459 // TODO: Extend to sink recipes from inner loops.
2460 PostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> POT(
2461 LoopRegion->getEntry());
2462 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
2463 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
2464 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2465 LastBB: LoopRegion->getExitingBasicBlock(),
2466 /*Sinking=*/true))
2467 continue;
2468
2469 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
2470 assert(!RepR->isPredicated() &&
2471 "Expected prior transformation of predicated replicates to "
2472 "replicate regions");
2473 // narrowToSingleScalarRecipes should have already maximally narrowed
2474 // replicates to single-scalar replicates.
2475 // TODO: When unrolling, replicateByVF doesn't handle sunk
2476 // non-single-scalar replicates correctly.
2477 if (!RepR->isSingleScalar())
2478 continue;
2479
2480 // The pointer operand of stores must be loop-invariant.
2481 if (RepR->getOpcode() == Instruction::Store &&
2482 !RepR->getOperand(N: 1)->isDefinedOutsideLoopRegions())
2483 continue;
2484 }
2485
2486 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
2487 assert((!R.mayWriteToMemory() ||
2488 (RepR && RepR->getOpcode() == Instruction::Store &&
2489 RepR->getOperand(1)->isDefinedOutsideLoopRegions())) &&
2490 "The only recipes that may write to memory are expected to be "
2491 "stores with invariant pointer-operand");
2492
2493 // TODO: Use R.definedValues() instead of casting to VPSingleDefRecipe to
2494 // support recipes with multiple defined values (e.g., interleaved loads).
2495 auto *Def = cast<VPSingleDefRecipe>(Val: &R);
2496
2497 // Cannot sink the recipe if the user is defined in a loop region or a
2498 // non-successor of the vector loop region. Cannot sink if user is a phi
2499 // either.
2500 VPBasicBlock *SinkBB = nullptr;
2501 if (any_of(Range: Def->users(), P: [&SinkBB, &LoopRegion](VPUser *U) {
2502 auto *UserR = cast<VPRecipeBase>(Val: U);
2503 VPBasicBlock *Parent = UserR->getParent();
2504 // TODO: Support sinking when users are in multiple blocks.
2505 if (SinkBB && SinkBB != Parent)
2506 return true;
2507 SinkBB = Parent;
2508 // TODO: If the user is a PHI node, we should check the block of
2509 // incoming value. Support PHI node users if needed.
2510 return UserR->isPhi() || Parent->getEnclosingLoopRegion() ||
2511 Parent->getSinglePredecessor() != LoopRegion;
2512 }))
2513 continue;
2514
2515 if (!SinkBB)
2516 SinkBB = cast<VPBasicBlock>(Val: LoopRegion->getSingleSuccessor());
2517
2518 // TODO: This will need to be a check instead of a assert after
2519 // conditional branches in vectorized loops are supported.
2520 assert(VPDT.properlyDominates(VPBB, SinkBB) &&
2521 "Defining block must dominate sink block");
2522 // TODO: Clone the recipe if users are on multiple exit paths, instead of
2523 // just moving.
2524 Def->moveBefore(BB&: *SinkBB, I: SinkBB->getFirstNonPhi());
2525 }
2526 }
2527}
2528
2529void VPlanTransforms::truncateToMinimalBitwidths(
2530 VPlan &Plan, const MapVector<Instruction *, uint64_t> &MinBWs) {
2531 if (Plan.hasScalarVFOnly())
2532 return;
2533 // Keep track of created truncates, so they can be re-used. Note that we
2534 // cannot use RAUW after creating a new truncate, as this would could make
2535 // other uses have different types for their operands, making them invalidly
2536 // typed.
2537 DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
2538 VPBasicBlock *PH = Plan.getVectorPreheader();
2539 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2540 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
2541 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2542 if (!isa<VPWidenRecipe, VPWidenCastRecipe, VPReplicateRecipe,
2543 VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R))
2544 continue;
2545
2546 VPValue *ResultVPV = R.getVPSingleValue();
2547 auto *UI = cast_or_null<Instruction>(Val: ResultVPV->getUnderlyingValue());
2548 unsigned NewResSizeInBits = MinBWs.lookup(Key: UI);
2549 if (!NewResSizeInBits)
2550 continue;
2551
2552 // If the value wasn't vectorized, we must maintain the original scalar
2553 // type. Skip those here, after incrementing NumProcessedRecipes. Also
2554 // skip casts which do not need to be handled explicitly here, as
2555 // redundant casts will be removed during recipe simplification.
2556 if (isa<VPReplicateRecipe, VPWidenCastRecipe>(Val: &R))
2557 continue;
2558
2559 Type *OldResTy = ResultVPV->getScalarType();
2560 unsigned OldResSizeInBits = OldResTy->getScalarSizeInBits();
2561 assert(OldResTy->isIntegerTy() && "only integer types supported");
2562 (void)OldResSizeInBits;
2563
2564 auto *NewResTy = IntegerType::get(C&: Plan.getContext(), NumBits: NewResSizeInBits);
2565
2566 // Any wrapping introduced by shrinking this operation shouldn't be
2567 // considered undefined behavior. So, we can't unconditionally copy
2568 // arithmetic wrapping flags to VPW.
2569 if (auto *VPW = dyn_cast<VPRecipeWithIRFlags>(Val: &R))
2570 VPW->dropPoisonGeneratingFlags();
2571
2572 assert((OldResSizeInBits != NewResSizeInBits ||
2573 match(&R, m_ICmp(m_VPValue(), m_VPValue()))) &&
2574 "Only ICmps should not need extending the result.");
2575 assert(!isa<VPWidenStoreRecipe>(&R) && "stores cannot be narrowed");
2576
2577 // For loads/intrinsics we don't recreate the recipe; just wrap the
2578 // original wide result in a ZExt to OldResTy.
2579 if (isa<VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R)) {
2580 if (OldResSizeInBits != NewResSizeInBits) {
2581 auto *Ext = VPBuilder::getToInsertAfter(R: &R).createWidenCast(
2582 Opcode: Instruction::ZExt, Op: ResultVPV, ResultTy: OldResTy);
2583 ResultVPV->replaceAllUsesWith(New: Ext);
2584 Ext->setOperand(I: 0, New: ResultVPV);
2585 }
2586 continue;
2587 }
2588
2589 // Shrink operands by introducing truncates as needed.
2590 unsigned StartIdx =
2591 match(V: &R, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())) ? 1 : 0;
2592 SmallVector<VPValue *> NewOperands(R.operands());
2593 for (VPValue *&Op : drop_begin(RangeOrContainer&: NewOperands, N: StartIdx)) {
2594 unsigned OpSizeInBits = Op->getScalarType()->getScalarSizeInBits();
2595 if (OpSizeInBits == NewResSizeInBits)
2596 continue;
2597 assert(OpSizeInBits > NewResSizeInBits && "nothing to truncate");
2598 auto [ProcessedIter, Inserted] = ProcessedTruncs.try_emplace(Key: Op);
2599 if (Inserted) {
2600 VPBuilder Builder;
2601 if (isa<VPIRValue>(Val: Op))
2602 Builder.setInsertPoint(PH);
2603 else
2604 Builder.setInsertPoint(&R);
2605 ProcessedIter->second =
2606 Builder.createWidenCast(Opcode: Instruction::Trunc, Op, ResultTy: NewResTy);
2607 }
2608 Op = ProcessedIter->second;
2609 }
2610
2611 auto *NWR = cast<VPWidenRecipe>(Val: &R)->cloneWithOperands(NewOperands);
2612 NWR->insertBefore(InsertPos: &R);
2613
2614 // Wrap NWR in a ZExt to preserve the original wide type for downstream
2615 // users (unless this is an ICmp, which produces i1 regardless).
2616 VPValue *Replacement = NWR->getVPSingleValue();
2617 if (OldResSizeInBits != NewResSizeInBits)
2618 Replacement =
2619 VPBuilder::getToInsertAfter(R: NWR)
2620 .createWidenCast(Opcode: Instruction::ZExt, Op: Replacement, ResultTy: OldResTy)
2621 ->getVPSingleValue();
2622 ResultVPV->replaceAllUsesWith(New: Replacement);
2623 R.eraseFromParent();
2624 }
2625 }
2626}
2627
2628bool VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) {
2629 std::optional<VPDominatorTree> VPDT;
2630 if (OnlyLatches)
2631 VPDT.emplace(args&: Plan);
2632
2633 // Collect all blocks before modifying the CFG so we can identify unreachable
2634 // ones after constant branch removal.
2635 SmallVector<VPBlockBase *> AllBlocks(vp_depth_first_shallow(G: Plan.getEntry()));
2636
2637 bool SimplifiedPhi = false;
2638 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: AllBlocks)) {
2639 VPValue *Cond;
2640 // Skip blocks that are not terminated by BranchOnCond.
2641 if (VPBB->empty() || !match(V: &VPBB->back(), P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))))
2642 continue;
2643
2644 if (OnlyLatches && !VPBlockUtils::isLatch(VPB: VPBB, VPDT: *VPDT))
2645 continue;
2646
2647 assert(VPBB->getNumSuccessors() == 2 &&
2648 "Two successors expected for BranchOnCond");
2649 unsigned RemovedIdx;
2650 if (match(V: Cond, P: m_True()))
2651 RemovedIdx = 1;
2652 else if (match(V: Cond, P: m_False()))
2653 RemovedIdx = 0;
2654 else
2655 continue;
2656
2657 VPBasicBlock *RemovedSucc =
2658 cast<VPBasicBlock>(Val: VPBB->getSuccessors()[RemovedIdx]);
2659 assert(count(RemovedSucc->getPredecessors(), VPBB) == 1 &&
2660 "There must be a single edge between VPBB and its successor");
2661 // Values coming from VPBB into phi recipes of RemovedSucc are removed from
2662 // these recipes.
2663 auto Phis = RemovedSucc->phis();
2664 for (VPRecipeBase &R : Phis)
2665 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: VPBB);
2666 SimplifiedPhi |= !std::empty(cont: Phis);
2667
2668 // Disconnect blocks and remove the terminator.
2669 VPBlockUtils::disconnectBlocks(From: VPBB, To: RemovedSucc);
2670 VPBB->back().eraseFromParent();
2671 }
2672
2673 // Compute which blocks are still reachable from the entry after constant
2674 // branch removal.
2675 SmallPtrSet<VPBlockBase *, 16> Reachable(
2676 llvm::from_range, vp_depth_first_shallow(G: Plan.getEntry()));
2677
2678 // Detach all unreachable blocks from their successors, removing their recipes
2679 // and incoming values from phi recipes.
2680 VPSymbolicValue Tmp(nullptr);
2681 for (VPBlockBase *B : AllBlocks) {
2682 if (Reachable.contains(Ptr: B))
2683 continue;
2684 for (VPBlockBase *Succ : to_vector(Range: B->successors())) {
2685 if (auto *SuccBB = dyn_cast<VPBasicBlock>(Val: Succ))
2686 for (VPRecipeBase &R : SuccBB->phis())
2687 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: B);
2688 VPBlockUtils::disconnectBlocks(From: B, To: Succ);
2689 }
2690 for (VPBasicBlock *DeadBB :
2691 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_deep(G: B))) {
2692 for (VPRecipeBase &R : make_early_inc_range(Range&: *DeadBB)) {
2693 for (VPValue *Def : R.definedValues())
2694 Def->replaceAllUsesWith(New: &Tmp);
2695 R.eraseFromParent();
2696 }
2697 }
2698 }
2699 return SimplifiedPhi;
2700}
2701
2702void VPlanTransforms::optimize(VPlan &Plan) {
2703 RUN_VPLAN_PASS(removeRedundantInductionCasts, Plan);
2704
2705 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2706 RUN_VPLAN_PASS(simplifyRecipes, Plan);
2707 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2708 RUN_VPLAN_PASS(simplifyBlends, Plan);
2709 RUN_VPLAN_PASS(legalizeAndOptimizeInductions, Plan);
2710 RUN_VPLAN_PASS(narrowToSingleScalarRecipes, Plan);
2711 RUN_VPLAN_PASS(removeRedundantExpandSCEVRecipes, Plan);
2712 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2713 RUN_VPLAN_PASS(simplifyRecipes, Plan);
2714 RUN_VPLAN_PASS(removeBranchOnConst, Plan, /*OnlyLatches=*/false);
2715 RUN_VPLAN_PASS(simplifyReverses, Plan);
2716 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2717
2718 RUN_VPLAN_PASS(createAndOptimizeReplicateRegions, Plan);
2719 RUN_VPLAN_PASS(mergeBlocksIntoPredecessors, Plan);
2720 RUN_VPLAN_PASS(licm, Plan);
2721}
2722
2723// Add a VPActiveLaneMaskPHIRecipe and related recipes to \p Plan and replace
2724// the loop terminator with a branch-on-cond recipe with the negated
2725// active-lane-mask as operand. Note that this turns the loop into an
2726// uncountable one. Only the existing terminator is replaced, all other existing
2727// recipes/users remain unchanged, except for poison-generating flags being
2728// dropped from the canonical IV increment. Return the created
2729// VPActiveLaneMaskPHIRecipe.
2730//
2731// The function adds the following recipes:
2732//
2733// vector.ph:
2734// %EntryInc = canonical-iv-increment-for-part CanonicalIVStart
2735// %EntryALM = active-lane-mask %EntryInc, TC
2736//
2737// vector.body:
2738// ...
2739// %P = active-lane-mask-phi [ %EntryALM, %vector.ph ], [ %ALM, %vector.body ]
2740// ...
2741// %InLoopInc = canonical-iv-increment-for-part CanonicalIVIncrement
2742// %ALM = active-lane-mask %InLoopInc, TC
2743// %Negated = Not %ALM
2744// branch-on-cond %Negated
2745//
2746static VPActiveLaneMaskPHIRecipe *
2747addVPLaneMaskPhiAndUpdateExitBranch(VPlan &Plan) {
2748 VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
2749 VPBasicBlock *EB = TopRegion->getExitingBasicBlock();
2750 VPValue *StartV = Plan.getZero(Ty: TopRegion->getCanonicalIVType());
2751 auto *CanonicalIVIncrement = TopRegion->getOrCreateCanonicalIVIncrement();
2752 // TODO: Check if dropping the flags is needed.
2753 TopRegion->clearCanonicalIVNUW(Increment: CanonicalIVIncrement);
2754 DebugLoc DL = CanonicalIVIncrement->getDebugLoc();
2755 // We can't use StartV directly in the ActiveLaneMask VPInstruction, since
2756 // we have to take unrolling into account. Each part needs to start at
2757 // Part * VF
2758 auto *VecPreheader = Plan.getVectorPreheader();
2759 VPBuilder Builder(VecPreheader);
2760
2761 // Create the ActiveLaneMask instruction using the correct start values.
2762 VPValue *TC = Plan.getTripCount();
2763 VPValue *VF = &Plan.getVF();
2764
2765 auto *EntryIncrement =
2766 Builder.createOverflowingOp(Opcode: VPInstruction::CanonicalIVIncrementForPart,
2767 Operands: {StartV, VF}, WrapFlags: {}, DL, Name: "index.part.next");
2768
2769 // Create the active lane mask instruction in the VPlan preheader.
2770 VPValue *ALMMultiplier =
2771 Plan.getConstantInt(Ty: TopRegion->getCanonicalIVType(), Val: 1);
2772 auto *EntryALM = Builder.createNaryOp(Opcode: VPInstruction::ActiveLaneMask,
2773 Operands: {EntryIncrement, TC, ALMMultiplier}, DL,
2774 Name: "active.lane.mask.entry");
2775
2776 // Now create the ActiveLaneMaskPhi recipe in the main loop using the
2777 // preheader ActiveLaneMask instruction.
2778 auto *LaneMaskPhi =
2779 new VPActiveLaneMaskPHIRecipe(EntryALM, DebugLoc::getUnknown());
2780 auto *HeaderVPBB = TopRegion->getEntryBasicBlock();
2781 LaneMaskPhi->insertBefore(BB&: *HeaderVPBB, IP: HeaderVPBB->begin());
2782
2783 // Create the active lane mask for the next iteration of the loop before the
2784 // original terminator.
2785 VPRecipeBase *OriginalTerminator = EB->getTerminator();
2786 Builder.setInsertPoint(OriginalTerminator);
2787 auto *InLoopIncrement = Builder.createOverflowingOp(
2788 Opcode: VPInstruction::CanonicalIVIncrementForPart,
2789 Operands: {CanonicalIVIncrement, &Plan.getVF()}, WrapFlags: {}, DL);
2790 auto *ALM = Builder.createNaryOp(Opcode: VPInstruction::ActiveLaneMask,
2791 Operands: {InLoopIncrement, TC, ALMMultiplier}, DL,
2792 Name: "active.lane.mask.next");
2793 LaneMaskPhi->addBackedgeValue(V: ALM);
2794
2795 // Replace the original terminator with BranchOnCond. We have to invert the
2796 // mask here because a true condition means jumping to the exit block.
2797 auto *NotMask = Builder.createNot(Operand: ALM, DL);
2798 Builder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {NotMask}, DL);
2799 OriginalTerminator->eraseFromParent();
2800 return LaneMaskPhi;
2801}
2802
2803void VPlanTransforms::materializeHeaderMask(
2804 VPlan &Plan, bool UseActiveLaneMask, bool UseActiveLaneMaskForControlFlow) {
2805 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2806 VPValue *HeaderMask = LoopRegion->getUsedHeaderMask();
2807 if (!HeaderMask)
2808 return;
2809
2810 if (UseActiveLaneMaskForControlFlow) {
2811 HeaderMask->replaceAllUsesWith(New: addVPLaneMaskPhiAndUpdateExitBranch(Plan));
2812 return;
2813 }
2814
2815 VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
2816 VPBuilder Builder(Header, Header->getFirstNonPhi());
2817 auto *WideCanonicalIV = Builder.insert(R: new VPWidenCanonicalIVRecipe(
2818 LoopRegion->getCanonicalIV(),
2819 VPIRFlags::WrapFlagsTy(/*HasNUW=*/true, /*HasNSW=*/false)));
2820 VPValue *Mask;
2821 if (UseActiveLaneMask) {
2822 VPValue *ALMMultiplier =
2823 Plan.getConstantInt(Ty: LoopRegion->getCanonicalIVType(), Val: 1);
2824 Mask = Builder.createNaryOp(
2825 Opcode: VPInstruction::ActiveLaneMask,
2826 Operands: {WideCanonicalIV, Plan.getTripCount(), ALMMultiplier}, DL: nullptr,
2827 Name: "active.lane.mask");
2828 } else {
2829 Mask = Builder.createICmp(Pred: CmpInst::ICMP_ULE, A: WideCanonicalIV,
2830 B: Plan.getOrCreateBackedgeTakenCount());
2831 }
2832 HeaderMask->replaceAllUsesWith(New: Mask);
2833}
2834
2835void VPlanTransforms::replaceSymbolicStrides(
2836 VPlan &Plan, PredicatedScalarEvolution &PSE,
2837 const DenseMap<Value *, const SCEV *> &StridesMap,
2838 const VPDominatorTree &VPDT) {
2839 // Replace VPValues for known constant strides guaranteed by predicated scalar
2840 // evolution that are guaranteed to be guarded by the runtime checks; that is,
2841 // blocks dominated by the vector preheader.
2842 assert(!Plan.getVectorLoopRegion() &&
2843 "expected to run before loop regions are created");
2844 VPBlockBase *Preheader = Plan.getEntry()->getSuccessors()[1];
2845 auto CanUseVersionedStride = [&VPDT, Preheader](VPUser &U, unsigned) {
2846 auto *R = cast<VPRecipeBase>(Val: &U);
2847 VPBlockBase *Parent = R->getParent();
2848 return VPDT.dominates(A: Preheader, B: Parent);
2849 };
2850 ValueToSCEVMapTy RewriteMap;
2851 for (const SCEV *Stride : StridesMap.values()) {
2852 using namespace SCEVPatternMatch;
2853 auto *StrideV = cast<SCEVUnknown>(Val: Stride)->getValue();
2854 const APInt *StrideConst;
2855 if (!match(S: PSE.getSCEV(V: StrideV), P: m_scev_APInt(C&: StrideConst)))
2856 // Only handle constant strides for now.
2857 continue;
2858
2859 auto *CI = Plan.getConstantInt(Val: *StrideConst);
2860 if (VPValue *StrideVPV = Plan.getLiveIn(V: StrideV))
2861 StrideVPV->replaceUsesWithIf(New: CI, ShouldReplace: CanUseVersionedStride);
2862
2863 // The versioned value may not be used in the loop directly but through a
2864 // sext/zext. Add new live-ins in those cases.
2865 for (Value *U : StrideV->users()) {
2866 if (!isa<SExtInst, ZExtInst>(Val: U))
2867 continue;
2868 VPValue *StrideVPV = Plan.getLiveIn(V: U);
2869 if (!StrideVPV)
2870 continue;
2871 unsigned BW = U->getType()->getScalarSizeInBits();
2872 APInt C =
2873 isa<SExtInst>(Val: U) ? StrideConst->sext(width: BW) : StrideConst->zext(width: BW);
2874 VPValue *CI = Plan.getConstantInt(Val: C);
2875 StrideVPV->replaceUsesWithIf(New: CI, ShouldReplace: CanUseVersionedStride);
2876 }
2877 RewriteMap[StrideV] = PSE.getSCEV(V: StrideV);
2878 }
2879
2880 for (VPRecipeBase &R : *Plan.getEntry()) {
2881 auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(Val: &R);
2882 if (!ExpSCEV)
2883 continue;
2884 const SCEV *ScevExpr = ExpSCEV->getSCEV();
2885 auto *NewSCEV =
2886 SCEVParameterRewriter::rewrite(Scev: ScevExpr, SE&: *PSE.getSE(), Map&: RewriteMap);
2887 if (NewSCEV != ScevExpr) {
2888 VPValue *NewExp = vputils::getOrCreateVPValueForSCEVExpr(Plan, Expr: NewSCEV);
2889 ExpSCEV->replaceAllUsesWith(New: NewExp);
2890 if (Plan.getTripCount() == ExpSCEV)
2891 Plan.resetTripCount(NewTripCount: NewExp);
2892 }
2893 }
2894}
2895
2896void VPlanTransforms::dropPoisonGeneratingRecipes(VPlan &Plan) {
2897 // Collect recipes in the backward slice of `Root` that may generate a poison
2898 // value that is used after vectorization.
2899 SmallPtrSet<VPRecipeBase *, 16> Visited;
2900 auto CollectPoisonGeneratingInstrsInBackwardSlice([&](VPRecipeBase *Root) {
2901 SmallVector<VPRecipeBase *, 16> Worklist;
2902 Worklist.push_back(Elt: Root);
2903
2904 // Traverse the backward slice of Root through its use-def chain.
2905 while (!Worklist.empty()) {
2906 VPRecipeBase *CurRec = Worklist.pop_back_val();
2907
2908 if (!Visited.insert(Ptr: CurRec).second)
2909 continue;
2910
2911 // Prune search if we find another recipe generating a widen memory
2912 // instruction. Widen memory instructions involved in address computation
2913 // will lead to gather/scatter instructions, which don't need to be
2914 // handled.
2915 if (isa<VPWidenMemoryRecipe, VPInterleaveRecipe, VPScalarIVStepsRecipe,
2916 VPHeaderPHIRecipe>(Val: CurRec))
2917 continue;
2918
2919 // This recipe contributes to the address computation of a widen
2920 // load/store. If the underlying instruction has poison-generating flags,
2921 // drop them directly.
2922 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: CurRec)) {
2923 VPValue *A, *B;
2924 // Dropping disjoint from an OR may yield incorrect results, as some
2925 // analysis may have converted it to an Add implicitly (e.g. SCEV used
2926 // for dependence analysis). Instead, replace it with an equivalent Add.
2927 // This is possible as all users of the disjoint OR only access lanes
2928 // where the operands are disjoint or poison otherwise.
2929 if (match(V: RecWithFlags, P: m_BinaryOr(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))) &&
2930 RecWithFlags->isDisjoint()) {
2931 VPBuilder Builder(RecWithFlags);
2932 VPInstruction *New =
2933 Builder.createAdd(LHS: A, RHS: B, DL: RecWithFlags->getDebugLoc());
2934 New->setUnderlyingValue(RecWithFlags->getUnderlyingValue());
2935 RecWithFlags->replaceAllUsesWith(New);
2936 RecWithFlags->eraseFromParent();
2937 CurRec = New;
2938 } else
2939 RecWithFlags->dropPoisonGeneratingFlags();
2940 } else {
2941 Instruction *Instr = dyn_cast_or_null<Instruction>(
2942 Val: CurRec->getVPSingleValue()->getUnderlyingValue());
2943 (void)Instr;
2944 assert((!Instr || !Instr->hasPoisonGeneratingFlags()) &&
2945 "found instruction with poison generating flags not covered by "
2946 "VPRecipeWithIRFlags");
2947 }
2948
2949 // Add new definitions to the worklist.
2950 for (VPValue *Operand : CurRec->operands())
2951 if (VPRecipeBase *OpDef = Operand->getDefiningRecipe())
2952 Worklist.push_back(Elt: OpDef);
2953 }
2954 });
2955
2956 // We want to exclude the tail folding case, as we don't need to drop flags
2957 // for operations computing the first lane in this case: the first lane of the
2958 // header mask must always be true. For reverse memory accesses, the mask is
2959 // wrapped in a Reverse, which is just a permutation of the header mask, so
2960 // peel it off before checking. The header mask is still the abstract region
2961 // value at this point (materialization happens later).
2962 auto IsNotHeaderMask = [](VPValue *Mask) {
2963 return Mask &&
2964 !match(V: Mask, P: m_CombineOr(Ps: m_HeaderMask(), Ps: m_Reverse(Op0: m_HeaderMask())));
2965 };
2966
2967 // Traverse all the recipes in the VPlan and collect the poison-generating
2968 // recipes in the backward slice starting at the address of a VPWidenRecipe or
2969 // VPInterleaveRecipe.
2970 auto Iter =
2971 vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntryBasicBlock());
2972 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: Iter)) {
2973 for (VPRecipeBase &Recipe : *VPBB) {
2974 if (auto *WidenRec = dyn_cast<VPWidenMemoryRecipe>(Val: &Recipe)) {
2975 VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe();
2976 if (AddrDef && WidenRec->isConsecutive() &&
2977 IsNotHeaderMask(WidenRec->getMask()))
2978 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2979 } else if (auto *InterleaveRec = dyn_cast<VPInterleaveRecipe>(Val: &Recipe)) {
2980 VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe();
2981 if (AddrDef && IsNotHeaderMask(InterleaveRec->getMask()))
2982 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2983 }
2984 }
2985 }
2986}
2987
2988void VPlanTransforms::createInterleaveGroups(
2989 VPlan &Plan,
2990 const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
2991 &InterleaveGroups,
2992 const bool &EpilogueAllowed) {
2993 if (InterleaveGroups.empty())
2994 return;
2995
2996 DenseMap<Instruction *, VPWidenMemoryRecipe *> IRMemberToRecipe;
2997 for (VPBasicBlock *VPBB :
2998 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
2999 G: Plan.getVectorLoopRegion()->getEntryBasicBlock())))
3000 for (VPRecipeBase &R : make_filter_range(Range&: *VPBB, Pred: [](VPRecipeBase &R) {
3001 return isa<VPWidenMemoryRecipe>(Val: &R);
3002 })) {
3003 auto *MemR = cast<VPWidenMemoryRecipe>(Val: &R);
3004 IRMemberToRecipe[&MemR->getIngredient()] = MemR;
3005 }
3006
3007 // Interleave memory: for each Interleave Group we marked earlier as relevant
3008 // for this VPlan, replace the Recipes widening its memory instructions with a
3009 // single VPInterleaveRecipe at its insertion point.
3010 VPDominatorTree VPDT(Plan);
3011 for (const auto *IG : InterleaveGroups) {
3012 VPWidenMemoryRecipe *Start = nullptr;
3013 Instruction *StartMember = nullptr;
3014 for (auto *Member : IG->members())
3015 if (VPWidenMemoryRecipe *R = IRMemberToRecipe.lookup(Val: Member)) {
3016 StartMember = Member;
3017 Start = R;
3018 break;
3019 }
3020 if (!StartMember) // All member recipes are dead, so the group is dead.
3021 continue;
3022 VPIRMetadata InterleaveMD(*Start);
3023 SmallVector<VPValue *, 4> StoredValues;
3024 for (unsigned I = 0; I < IG->getFactor(); ++I) {
3025 Instruction *MemberI = IG->getMember(Index: I);
3026 if (!MemberI)
3027 continue;
3028 if (VPWidenMemoryRecipe *MemoryR = IRMemberToRecipe.lookup(Val: MemberI)) {
3029 if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Val: MemoryR->getAsRecipe()))
3030 StoredValues.push_back(Elt: StoreR->getStoredValue());
3031 InterleaveMD.intersect(MD: *MemoryR);
3032 } else {
3033 InterleaveMD.intersect(MD: VPIRMetadata(*MemberI));
3034 }
3035 }
3036
3037 bool NeedsMaskForGaps =
3038 (IG->requiresScalarEpilogue() && !EpilogueAllowed) ||
3039 (!StoredValues.empty() && !IG->isFull());
3040
3041 Instruction *IRInsertPos = IG->getInsertPos();
3042 auto *InsertPos = IRMemberToRecipe.lookup(Val: IRInsertPos);
3043 if (!InsertPos) {
3044 // InsertPos member is dead: find a new member that is alive.
3045 assert(isa<VPWidenLoadRecipe>(Start->getAsRecipe()) &&
3046 "Dead member in non-load group?");
3047 InsertPos = Start;
3048 for (Instruction *Member : IG->members())
3049 if (VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member))
3050 if (VPDT.properlyDominates(A: MemberR->getAsRecipe(),
3051 B: InsertPos->getAsRecipe()))
3052 InsertPos = MemberR;
3053 IRInsertPos = &InsertPos->getIngredient();
3054 }
3055 VPRecipeBase *InsertPosR = InsertPos->getAsRecipe();
3056
3057 GEPNoWrapFlags NW = GEPNoWrapFlags::none();
3058 if (auto *Gep = dyn_cast<GetElementPtrInst>(
3059 Val: getLoadStorePointerOperand(V: IRInsertPos)->stripPointerCasts()))
3060 NW = Gep->getNoWrapFlags().withoutNoUnsignedWrap();
3061
3062 // Get or create the start address for the interleave group.
3063 VPValue *Addr = Start->getAddr();
3064 VPRecipeBase *AddrDef = Addr->getDefiningRecipe();
3065 if (IG->getIndex(Instr: StartMember) != 0 ||
3066 (AddrDef && !VPDT.properlyDominates(A: AddrDef, B: InsertPosR))) {
3067 // Either member zero's recipe is dead, or we cannot re-use the address of
3068 // member zero because it does not dominate the insert position. Instead,
3069 // use the address of the insert position and create a PtrAdd adjusting it
3070 // to the address of member zero.
3071 // TODO: Hoist Addr's defining recipe (and any operands as needed) to
3072 // InsertPos or sink loads above zero members to join it.
3073 assert(IG->getIndex(IRInsertPos) != 0 &&
3074 "index of insert position shouldn't be zero");
3075 auto &DL = IRInsertPos->getDataLayout();
3076 APInt Offset(32,
3077 DL.getTypeAllocSize(Ty: getLoadStoreType(I: IRInsertPos)) *
3078 IG->getIndex(Instr: IRInsertPos),
3079 /*IsSigned=*/true);
3080 VPValue *OffsetVPV = Plan.getConstantInt(Val: -Offset);
3081 VPBuilder B(InsertPosR);
3082 Addr = B.createNoWrapPtrAdd(Ptr: InsertPos->getAddr(), Offset: OffsetVPV, GEPFlags: NW);
3083 }
3084 // If the group is reverse, adjust the index to refer to the last vector
3085 // lane instead of the first. We adjust the index from the first vector
3086 // lane, rather than directly getting the pointer for lane VF - 1, because
3087 // the pointer operand of the interleaved access is supposed to be uniform.
3088 if (IG->isReverse()) {
3089 auto *ReversePtr = new VPVectorEndPointerRecipe(
3090 Addr, &Plan.getVF(), getLoadStoreType(I: IRInsertPos),
3091 -(int64_t)IG->getFactor(), NW, InsertPosR->getDebugLoc());
3092 ReversePtr->insertBefore(InsertPos: InsertPosR);
3093 Addr = ReversePtr;
3094 }
3095 auto *VPIG = new VPInterleaveRecipe(
3096 IG, Addr, StoredValues, InsertPos->getMask(), NeedsMaskForGaps,
3097 InterleaveMD, InsertPosR->getDebugLoc());
3098 VPIG->insertBefore(InsertPos: InsertPosR);
3099
3100 unsigned J = 0;
3101 for (unsigned i = 0; i < IG->getFactor(); ++i)
3102 if (Instruction *Member = IG->getMember(Index: i)) {
3103 VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member);
3104 if (!Member->getType()->isVoidTy()) {
3105 if (MemberR) {
3106 VPValue *OriginalV = MemberR->getAsRecipe()->getVPSingleValue();
3107 OriginalV->replaceAllUsesWith(New: VPIG->getVPValue(I: J));
3108 }
3109 J++;
3110 }
3111 if (MemberR)
3112 MemberR->getAsRecipe()->eraseFromParent();
3113 }
3114 }
3115}
3116
3117/// Expand a VPWidenIntOrFpInduction into executable recipes, for the initial
3118/// value, phi and backedge value. In the following example:
3119///
3120/// vector.ph:
3121/// Successor(s): vector loop
3122///
3123/// <x1> vector loop: {
3124/// vector.body:
3125/// WIDEN-INDUCTION %i = phi %start, %step, %vf
3126/// ...
3127/// EMIT branch-on-count ...
3128/// No successors
3129/// }
3130///
3131/// WIDEN-INDUCTION will get expanded to:
3132///
3133/// vector.ph:
3134/// ...
3135/// vp<%induction.start> = ...
3136/// vp<%induction.increment> = ...
3137///
3138/// Successor(s): vector loop
3139///
3140/// <x1> vector loop: {
3141/// vector.body:
3142/// ir<%i> = WIDEN-PHI vp<%induction.start>, vp<%vec.ind.next>
3143/// ...
3144/// vp<%vec.ind.next> = add ir<%i>, vp<%induction.increment>
3145/// EMIT branch-on-count ...
3146/// No successors
3147/// }
3148static void
3149expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR) {
3150 VPlan *Plan = WidenIVR->getParent()->getPlan();
3151 VPValue *Start = WidenIVR->getStartValue();
3152 VPValue *Step = WidenIVR->getStepValue();
3153 VPValue *VF = WidenIVR->getVFValue();
3154 DebugLoc DL = WidenIVR->getDebugLoc();
3155
3156 // The value from the original loop to which we are mapping the new induction
3157 // variable.
3158 Type *Ty = WidenIVR->getScalarType();
3159
3160 const InductionDescriptor &ID = WidenIVR->getInductionDescriptor();
3161 Instruction::BinaryOps AddOp;
3162 Instruction::BinaryOps MulOp;
3163 VPIRFlags Flags = *WidenIVR;
3164 if (ID.getKind() == InductionDescriptor::IK_IntInduction) {
3165 AddOp = Instruction::Add;
3166 MulOp = Instruction::Mul;
3167 } else {
3168 AddOp = ID.getInductionOpcode();
3169 MulOp = Instruction::FMul;
3170 }
3171
3172 // If the phi is truncated, truncate the start and step values.
3173 VPBuilder Builder(Plan->getVectorPreheader());
3174 Type *StepTy = Step->getScalarType();
3175 if (Ty->getScalarSizeInBits() < StepTy->getScalarSizeInBits()) {
3176 assert(StepTy->isIntegerTy() && "Truncation requires an integer type");
3177 Step = Builder.createScalarCast(Opcode: Instruction::Trunc, Op: Step, ResultTy: Ty, DL);
3178 Start = Builder.createScalarCast(Opcode: Instruction::Trunc, Op: Start, ResultTy: Ty, DL);
3179 StepTy = Ty;
3180 }
3181
3182 // Construct the initial value of the vector IV in the vector loop preheader.
3183 Type *IVIntTy =
3184 IntegerType::get(C&: Plan->getContext(), NumBits: StepTy->getScalarSizeInBits());
3185 VPValue *Init = Builder.createNaryOp(Opcode: VPInstruction::StepVector, Operands: {}, ResultTy: IVIntTy);
3186 if (StepTy->isFloatingPointTy())
3187 Init = Builder.createWidenCast(Opcode: Instruction::UIToFP, Op: Init, ResultTy: StepTy);
3188
3189 VPValue *SplatStart = Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: Start);
3190 VPValue *SplatStep = Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: Step);
3191
3192 Init = Builder.createNaryOp(Opcode: MulOp, Operands: {Init, SplatStep}, Flags);
3193 Init = Builder.createNaryOp(Opcode: AddOp, Operands: {SplatStart, Init}, Flags,
3194 DL: DebugLoc::getUnknown(), Name: "induction");
3195
3196 // Create the widened phi of the vector IV.
3197 auto *WidePHI = VPBuilder(WidenIVR).createWidenPhi(
3198 IncomingValues: Init, DL: WidenIVR->getDebugLoc(), Name: "vec.ind");
3199
3200 // Create the backedge value for the vector IV.
3201 VPValue *Inc;
3202 VPValue *Prev;
3203 // If unrolled, use the increment and prev value from the operands.
3204 if (auto *SplatVF = WidenIVR->getSplatVFValue()) {
3205 Inc = SplatVF;
3206 Prev = WidenIVR->getLastUnrolledPartOperand();
3207 } else {
3208 // Move the insertion point after the VF definition when the VF is defined
3209 // inside a loop, such as for EVL tail-folding.
3210 if (VPRecipeBase *R = VF->getDefiningRecipe())
3211 if (R->getParent()->getEnclosingLoopRegion())
3212 Builder.setInsertPoint(TheBB: R->getParent(), IP: std::next(x: R->getIterator()));
3213
3214 // Multiply the vectorization factor by the step using integer or
3215 // floating-point arithmetic as appropriate.
3216 if (StepTy->isFloatingPointTy())
3217 VF = Builder.createScalarCast(Opcode: Instruction::CastOps::UIToFP, Op: VF, ResultTy: StepTy,
3218 DL);
3219 else
3220 VF = Builder.createScalarZExtOrTrunc(Op: VF, ResultTy: StepTy, DL);
3221
3222 Inc = Builder.createNaryOp(Opcode: MulOp, Operands: {Step, VF}, Flags);
3223 Inc = Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: Inc);
3224 Prev = WidePHI;
3225 }
3226
3227 VPBasicBlock *ExitingBB = Plan->getVectorLoopRegion()->getExitingBasicBlock();
3228 Builder.setInsertPoint(TheBB: ExitingBB, IP: ExitingBB->getTerminator()->getIterator());
3229 auto *Next = Builder.createNaryOp(Opcode: AddOp, Operands: {Prev, Inc}, Flags,
3230 DL: WidenIVR->getDebugLoc(), Name: "vec.ind.next");
3231
3232 WidePHI->addIncoming(IncomingV: Next);
3233
3234 WidenIVR->replaceAllUsesWith(New: WidePHI);
3235}
3236
3237/// Expand a VPWidenPointerInductionRecipe into executable recipes, for the
3238/// initial value, phi and backedge value. In the following example:
3239///
3240/// <x1> vector loop: {
3241/// vector.body:
3242/// EMIT ir<%ptr.iv> = WIDEN-POINTER-INDUCTION %start, %step, %vf
3243/// ...
3244/// EMIT branch-on-count ...
3245/// }
3246///
3247/// WIDEN-POINTER-INDUCTION will get expanded to:
3248///
3249/// <x1> vector loop: {
3250/// vector.body:
3251/// EMIT-SCALAR %pointer.phi = phi %start, %ptr.ind
3252/// EMIT %mul = mul %stepvector, %step
3253/// EMIT %vector.gep = wide-ptradd %pointer.phi, %mul
3254/// ...
3255/// EMIT %ptr.ind = ptradd %pointer.phi, %vf
3256/// EMIT branch-on-count ...
3257/// }
3258static void expandVPWidenPointerInduction(VPWidenPointerInductionRecipe *R) {
3259 VPlan *Plan = R->getParent()->getPlan();
3260 VPValue *Start = R->getStartValue();
3261 VPValue *Step = R->getStepValue();
3262 VPValue *VF = R->getVFValue();
3263
3264 assert(R->getInductionDescriptor().getKind() ==
3265 InductionDescriptor::IK_PtrInduction &&
3266 "Not a pointer induction according to InductionDescriptor!");
3267 assert(R->getScalarType()->isPointerTy() && "Unexpected type.");
3268 assert(!R->onlyScalarsGenerated(Plan->hasScalableVF()) &&
3269 "Recipe should have been replaced");
3270
3271 VPBuilder Builder(R);
3272 DebugLoc DL = R->getDebugLoc();
3273
3274 // Build a scalar pointer phi.
3275 VPPhi *ScalarPtrPhi = Builder.createScalarPhi(IncomingValues: Start, DL, Name: "pointer.phi");
3276
3277 // Create actual address geps that use the pointer phi as base and a
3278 // vectorized version of the step value (<step*0, ..., step*N>) as offset.
3279 Builder.setInsertPoint(TheBB: R->getParent(), IP: R->getParent()->getFirstNonPhi());
3280 Type *StepTy = Step->getScalarType();
3281 VPValue *Offset = Builder.createNaryOp(Opcode: VPInstruction::StepVector, Operands: {}, ResultTy: StepTy);
3282 Offset = Builder.createOverflowingOp(Opcode: Instruction::Mul, Operands: {Offset, Step});
3283 VPValue *PtrAdd =
3284 Builder.createWidePtrAdd(Ptr: ScalarPtrPhi, Offset, DL, Name: "vector.gep");
3285 R->replaceAllUsesWith(New: PtrAdd);
3286
3287 // Create the backedge value for the scalar pointer phi.
3288 VPBasicBlock *ExitingBB = Plan->getVectorLoopRegion()->getExitingBasicBlock();
3289 Builder.setInsertPoint(TheBB: ExitingBB, IP: ExitingBB->getTerminator()->getIterator());
3290 VF = Builder.createScalarZExtOrTrunc(Op: VF, ResultTy: StepTy, DL);
3291 VPValue *Inc = Builder.createOverflowingOp(Opcode: Instruction::Mul, Operands: {Step, VF});
3292
3293 VPValue *InductionGEP =
3294 Builder.createPtrAdd(Ptr: ScalarPtrPhi, Offset: Inc, DL, Name: "ptr.ind");
3295 ScalarPtrPhi->addIncoming(IncomingV: InductionGEP);
3296}
3297
3298/// Expand a VPDerivedIVRecipe into executable recipes.
3299static void expandVPDerivedIV(VPDerivedIVRecipe *R) {
3300 VPBuilder Builder(R);
3301 VPValue *Start = R->getStartValue();
3302 VPValue *Step = R->getStepValue();
3303 VPValue *Index = R->getIndex();
3304 Type *StepTy = Step->getScalarType();
3305 Index = StepTy->isIntegerTy()
3306 ? Builder.createScalarSExtOrTrunc(
3307 Op: Index, ResultTy: StepTy, DL: DebugLoc::getCompilerGenerated())
3308 : Builder.createScalarCast(Opcode: Instruction::SIToFP, Op: Index, ResultTy: StepTy,
3309 DL: DebugLoc::getCompilerGenerated());
3310 switch (R->getInductionKind()) {
3311 case InductionDescriptor::IK_IntInduction: {
3312 assert(Index->getScalarType() == Start->getScalarType() &&
3313 "Index type does not match StartValue type");
3314 return R->replaceAllUsesWith(New: Builder.createAdd(
3315 LHS: Start, RHS: Builder.createOverflowingOp(Opcode: Instruction::Mul, Operands: {Index, Step})));
3316 }
3317 case InductionDescriptor::IK_PtrInduction:
3318 return R->replaceAllUsesWith(New: Builder.createPtrAdd(
3319 Ptr: Start, Offset: Builder.createOverflowingOp(Opcode: Instruction::Mul, Operands: {Index, Step})));
3320 case InductionDescriptor::IK_FpInduction: {
3321 assert(StepTy->isFloatingPointTy() && "Expected FP Step value");
3322 const FPMathOperator *FPBinOp = R->getFPBinOp();
3323 assert(FPBinOp &&
3324 (FPBinOp->getOpcode() == Instruction::FAdd ||
3325 FPBinOp->getOpcode() == Instruction::FSub) &&
3326 "Original BinOp should be defined for FP induction");
3327 FastMathFlags FMF = FPBinOp->getFastMathFlags();
3328 VPValue *FMul = Builder.createNaryOp(Opcode: Instruction::FMul, Operands: {Step, Index}, Flags: FMF);
3329 return R->replaceAllUsesWith(
3330 New: Builder.createNaryOp(Opcode: FPBinOp->getOpcode(), Operands: {Start, FMul}, Flags: FMF));
3331 }
3332 case InductionDescriptor::IK_NoInduction:
3333 return;
3334 }
3335 llvm_unreachable("Unhandled induction kind");
3336}
3337
3338void VPlanTransforms::dissolveLoopRegions(VPlan &Plan) {
3339 // Replace loop regions with explicity CFG.
3340 SmallVector<VPRegionBlock *> LoopRegions;
3341 for (VPRegionBlock *R : VPBlockUtils::blocksOnly<VPRegionBlock>(
3342 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
3343 if (!R->isReplicator())
3344 LoopRegions.push_back(Elt: R);
3345 }
3346 for (VPRegionBlock *R : LoopRegions)
3347 R->dissolveToCFGLoop();
3348}
3349
3350void VPlanTransforms::expandBranchOnTwoConds(VPlan &Plan) {
3351 SmallVector<VPInstruction *> WorkList;
3352 // The transform runs after dissolving loop regions, so all VPBasicBlocks
3353 // terminated with BranchOnTwoConds are reached via a shallow traversal.
3354 for (VPBasicBlock *VPBB : VPBlockUtils::blocksAs<VPBasicBlock>(
3355 Range: vp_depth_first_shallow(G: Plan.getEntry()))) {
3356 if (!VPBB->empty() && match(V: &VPBB->back(), P: m_BranchOnTwoConds()))
3357 WorkList.push_back(Elt: cast<VPInstruction>(Val: &VPBB->back()));
3358 }
3359
3360 // Expand BranchOnTwoConds instructions into explicit CFG with two new
3361 // single-condition branches:
3362 // 1. A branch that replaces BranchOnTwoConds, jumps to the first successor if
3363 // the first condition is true, and otherwise jumps to a new interim block.
3364 // 2. A branch that ends the interim block, jumps to the second successor if
3365 // the second condition is true, and otherwise jumps to the third
3366 // successor.
3367 for (VPInstruction *Br : WorkList) {
3368 assert(Br->getNumOperands() == 2 &&
3369 "BranchOnTwoConds must have exactly 2 conditions");
3370 DebugLoc DL = Br->getDebugLoc();
3371 VPBasicBlock *BrOnTwoCondsBB = Br->getParent();
3372 const auto Successors = to_vector(Range&: BrOnTwoCondsBB->getSuccessors());
3373 assert(Successors.size() == 3 &&
3374 "BranchOnTwoConds must have exactly 3 successors");
3375
3376 for (VPBlockBase *Succ : Successors)
3377 VPBlockUtils::disconnectBlocks(From: BrOnTwoCondsBB, To: Succ);
3378
3379 VPValue *Cond0 = Br->getOperand(N: 0);
3380 VPValue *Cond1 = Br->getOperand(N: 1);
3381 VPBlockBase *Succ0 = Successors[0];
3382 VPBlockBase *Succ1 = Successors[1];
3383 VPBlockBase *Succ2 = Successors[2];
3384
3385 // If the successor block for both conditions is the same, then combine the
3386 // two conditions and plant a single conditional branch.
3387 if (Succ0 == Succ1) {
3388 VPBuilder Builder(Br);
3389 VPValue *Combined = Builder.createOr(LHS: Cond0, RHS: Cond1, DL);
3390 Builder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {Combined}, DL);
3391 VPBlockUtils::connectBlocks(From: BrOnTwoCondsBB, To: Succ0);
3392 VPBlockUtils::connectBlocks(From: BrOnTwoCondsBB, To: Succ2);
3393 Br->eraseFromParent();
3394 continue;
3395 }
3396
3397 assert(!Succ0->getParent() && !Succ1->getParent() && !Succ2->getParent() &&
3398 !BrOnTwoCondsBB->getParent() && "regions must already be dissolved");
3399
3400 VPBasicBlock *InterimBB =
3401 Plan.createVPBasicBlock(Name: BrOnTwoCondsBB->getName() + ".interim");
3402
3403 VPBuilder(BrOnTwoCondsBB)
3404 .createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {Cond0}, DL);
3405 VPBlockUtils::connectBlocks(From: BrOnTwoCondsBB, To: Succ0);
3406 VPBlockUtils::connectBlocks(From: BrOnTwoCondsBB, To: InterimBB);
3407
3408 VPBuilder(InterimBB).createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {Cond1}, DL);
3409 VPBlockUtils::connectBlocks(From: InterimBB, To: Succ1);
3410 VPBlockUtils::connectBlocks(From: InterimBB, To: Succ2);
3411 Br->eraseFromParent();
3412 }
3413}
3414
3415void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) {
3416 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3417 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
3418 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
3419 VPBuilder Builder(&R);
3420 if (auto *WidenIVR = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: &R)) {
3421 expandVPWidenIntOrFpInduction(WidenIVR);
3422 WidenIVR->eraseFromParent();
3423 continue;
3424 }
3425
3426 if (auto *WidenIVR = dyn_cast<VPWidenPointerInductionRecipe>(Val: &R)) {
3427 // If the recipe only generates scalars, scalarize it instead of
3428 // expanding it.
3429 if (WidenIVR->onlyScalarsGenerated(IsScalable: Plan.hasScalableVF())) {
3430 VPValue *PtrAdd =
3431 scalarizeVPWidenPointerInduction(PtrIV: WidenIVR, Plan, Builder);
3432 WidenIVR->replaceAllUsesWith(New: PtrAdd);
3433 WidenIVR->eraseFromParent();
3434 continue;
3435 }
3436 expandVPWidenPointerInduction(R: WidenIVR);
3437 WidenIVR->eraseFromParent();
3438 continue;
3439 }
3440
3441 if (auto *DerivedIVR = dyn_cast<VPDerivedIVRecipe>(Val: &R)) {
3442 expandVPDerivedIV(R: DerivedIVR);
3443 DerivedIVR->eraseFromParent();
3444 continue;
3445 }
3446
3447 if (auto *WideCanIV = dyn_cast<VPWidenCanonicalIVRecipe>(Val: &R)) {
3448 VPValue *CanIV = WideCanIV->getCanonicalIV();
3449 Type *CanIVTy = CanIV->getScalarType();
3450 VPValue *Step = WideCanIV->getStepValue();
3451 if (!Step) {
3452 assert(Plan.getConcreteUF() == 1 &&
3453 "Expected unroller to have materialized step for UF != 1");
3454 Step = Plan.getZero(Ty: CanIVTy);
3455 }
3456 CanIV = Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: CanIV);
3457 Step = Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: Step);
3458 Step = Builder.createAdd(
3459 LHS: Step, RHS: Builder.createNaryOp(Opcode: VPInstruction::StepVector, Operands: {}, ResultTy: CanIVTy));
3460 VPValue *CanVecIV =
3461 Builder.createAdd(LHS: CanIV, RHS: Step, DL: WideCanIV->getDebugLoc(), Name: "vec.iv",
3462 WrapFlags: WideCanIV->getNoWrapFlags());
3463 WideCanIV->replaceAllUsesWith(New: CanVecIV);
3464 WideCanIV->eraseFromParent();
3465 continue;
3466 }
3467
3468 // Expand VPBlendRecipe into VPInstruction::Select.
3469 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: &R)) {
3470 VPValue *Select = Blend->getIncomingValue(Idx: 0);
3471 for (unsigned I = 1; I != Blend->getNumIncomingValues(); ++I)
3472 Select = Builder.createSelect(Cond: Blend->getMask(Idx: I),
3473 TrueVal: Blend->getIncomingValue(Idx: I), FalseVal: Select,
3474 DL: R.getDebugLoc(), Name: "predphi", Flags: *Blend);
3475 Blend->replaceAllUsesWith(New: Select);
3476 Blend->eraseFromParent();
3477 continue;
3478 }
3479
3480 if (auto *VEPR = dyn_cast<VPVectorEndPointerRecipe>(Val: &R)) {
3481 if (!VEPR->getOffset()) {
3482 assert(Plan.getConcreteUF() == 1 &&
3483 "Expected unroller to have materialized offset for UF != 1");
3484 VEPR->materializeOffset();
3485 }
3486 continue;
3487 }
3488
3489 if (auto *Expr = dyn_cast<VPExpressionRecipe>(Val: &R)) {
3490 Expr->decompose();
3491 Expr->eraseFromParent();
3492 continue;
3493 }
3494
3495 // Expand LastActiveLane into Not + FirstActiveLane + Sub.
3496 auto *LastActiveL = dyn_cast<VPInstruction>(Val: &R);
3497 if (LastActiveL &&
3498 LastActiveL->getOpcode() == VPInstruction::LastActiveLane) {
3499 // Create Not(Mask) for all operands.
3500 SmallVector<VPValue *, 2> NotMasks;
3501 for (VPValue *Op : LastActiveL->operands()) {
3502 VPValue *NotMask = Builder.createNot(Operand: Op, DL: LastActiveL->getDebugLoc());
3503 NotMasks.push_back(Elt: NotMask);
3504 }
3505
3506 // Create FirstActiveLane on the inverted masks.
3507 VPValue *FirstInactiveLane = Builder.createFirstActiveLane(
3508 Masks: NotMasks, DL: LastActiveL->getDebugLoc(), Name: "first.inactive.lane");
3509
3510 // Subtract 1 to get the last active lane.
3511 VPValue *One =
3512 Plan.getConstantInt(Ty: FirstInactiveLane->getScalarType(), Val: 1);
3513 VPValue *LastLane =
3514 Builder.createSub(LHS: FirstInactiveLane, RHS: One,
3515 DL: LastActiveL->getDebugLoc(), Name: "last.active.lane");
3516
3517 LastActiveL->replaceAllUsesWith(New: LastLane);
3518 LastActiveL->eraseFromParent();
3519 continue;
3520 }
3521
3522 // Lower MaskedCond with block mask to LogicalAnd.
3523 if (match(V: &R, P: m_VPInstruction<VPInstruction::MaskedCond>())) {
3524 auto *VPI = cast<VPInstruction>(Val: &R);
3525 assert(VPI->isMasked() &&
3526 "Unmasked MaskedCond should be simplified earlier");
3527 VPI->replaceAllUsesWith(New: Builder.createNaryOp(
3528 Opcode: VPInstruction::LogicalAnd, Operands: {VPI->getMask(), VPI->getOperand(N: 0)}));
3529 VPI->eraseFromParent();
3530 continue;
3531 }
3532
3533 // Lower CanonicalIVIncrementForPart to plain Add.
3534 if (match(
3535 V: &R,
3536 P: m_VPInstruction<VPInstruction::CanonicalIVIncrementForPart>())) {
3537 auto *VPI = cast<VPInstruction>(Val: &R);
3538 VPValue *Add = Builder.createOverflowingOp(
3539 Opcode: Instruction::Add, Operands: VPI->operands(), WrapFlags: VPI->getNoWrapFlags(),
3540 DL: VPI->getDebugLoc());
3541 VPI->replaceAllUsesWith(New: Add);
3542 VPI->eraseFromParent();
3543 continue;
3544 }
3545
3546 // Lower BranchOnCount to ICmp + BranchOnCond.
3547 VPValue *IV, *TC;
3548 if (match(V: &R, P: m_BranchOnCount(Op0: m_VPValue(V&: IV), Op1: m_VPValue(V&: TC)))) {
3549 auto *BranchOnCountInst = cast<VPInstruction>(Val: &R);
3550 DebugLoc DL = BranchOnCountInst->getDebugLoc();
3551 VPValue *Cond = Builder.createICmp(Pred: CmpInst::ICMP_EQ, A: IV, B: TC, DL);
3552 Builder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: Cond, DL);
3553 BranchOnCountInst->eraseFromParent();
3554 continue;
3555 }
3556
3557 VPValue *VectorStep;
3558 VPValue *ScalarStep;
3559 if (!match(V: &R, P: m_VPInstruction<VPInstruction::WideIVStep>(
3560 Ops: m_VPValue(V&: VectorStep), Ops: m_VPValue(V&: ScalarStep))))
3561 continue;
3562
3563 // Expand WideIVStep.
3564 auto *VPI = cast<VPInstruction>(Val: &R);
3565 Type *IVTy = VPI->getScalarType();
3566 if (VectorStep->getScalarType() != IVTy) {
3567 Instruction::CastOps CastOp = IVTy->isFloatingPointTy()
3568 ? Instruction::UIToFP
3569 : Instruction::Trunc;
3570 VectorStep = Builder.createWidenCast(Opcode: CastOp, Op: VectorStep, ResultTy: IVTy);
3571 }
3572
3573 assert(!match(ScalarStep, m_One()) && "Expected non-unit scalar-step");
3574 if (ScalarStep->getScalarType() != IVTy) {
3575 ScalarStep =
3576 Builder.createWidenCast(Opcode: Instruction::Trunc, Op: ScalarStep, ResultTy: IVTy);
3577 }
3578
3579 VPIRFlags Flags;
3580 unsigned MulOpc;
3581 if (IVTy->isFloatingPointTy()) {
3582 MulOpc = Instruction::FMul;
3583 Flags = VPI->getFastMathFlagsOrNone();
3584 } else {
3585 MulOpc = Instruction::Mul;
3586 Flags = VPIRFlags::getDefaultFlags(Opcode: MulOpc);
3587 }
3588
3589 VPInstruction *Mul = Builder.createNaryOp(
3590 Opcode: MulOpc, Operands: {VectorStep, ScalarStep}, Flags, DL: R.getDebugLoc());
3591 VectorStep = Mul;
3592 VPI->replaceAllUsesWith(New: VectorStep);
3593 VPI->eraseFromParent();
3594 }
3595 }
3596}
3597
3598/// Returns the VPValue representing the uncountable exit comparison used by
3599/// AnyOf if the recipes it depends on can be traced back to live-ins and
3600/// the addresses (in GEP/PtrAdd form) of any (non-masked) load used in
3601/// generating the values for the comparison. The recipes are stored in
3602/// \p Recipes.
3603static std::optional<VPValue *>
3604getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
3605 VPBasicBlock *LatchVPBB) {
3606 // Given a plain CFG VPlan loop with countable latch exiting block
3607 // \p LatchVPBB, we're looking to match the recipes contributing to the
3608 // uncountable exit condition comparison (here, vp<%4>) back to either
3609 // live-ins or the address nodes for the load used as part of the uncountable
3610 // exit comparison so that we can either move them within the loop, or copy
3611 // them to the preheader depending on the chosen method for dealing with
3612 // stores in uncountable exit loops.
3613 //
3614 // Currently, the address of the load is restricted to a GEP with 2 operands
3615 // and a live-in base address. This constraint may be relaxed later.
3616 //
3617 // VPlan ' for UF>=1' {
3618 // Live-in vp<%0> = VF * UF
3619 // Live-in vp<%1> = vector-trip-count
3620 // Live-in ir<20> = original trip-count
3621 //
3622 // ir-bb<entry>:
3623 // Successor(s): scalar.ph, vector.ph
3624 //
3625 // vector.ph:
3626 // Successor(s): for.body
3627 //
3628 // for.body:
3629 // EMIT vp<%2> = phi ir<0>, vp<%index.next>
3630 // EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, for.inc ]
3631 // EMIT ir<%uncountable.addr> = getelementptr inbounds nuw ir<%pred>,ir<%iv>
3632 // EMIT ir<%uncountable.val> = load ir<%uncountable.addr>
3633 // EMIT ir<%uncountable.cond> = icmp sgt ir<%uncountable.val>, ir<500>
3634 // EMIT vp<%3> = masked-cond ir<%uncountable.cond>
3635 // Successor(s): for.inc
3636 //
3637 // for.inc:
3638 // EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
3639 // EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
3640 // EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
3641 // EMIT vp<%4> = any-of ir<%3>
3642 // EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
3643 // EMIT branch-on-two-conds vp<%4>, vp<%5>
3644 // Successor(s): middle.block, middle.block, for.body
3645 //
3646 // middle.block:
3647 // Successor(s): ir-bb<exit>, scalar.ph
3648 //
3649 // ir-bb<exit>:
3650 // No successors
3651 //
3652 // scalar.ph:
3653 // }
3654
3655 // Find the uncountable loop exit condition.
3656 VPValue *UncountableCondition = nullptr;
3657 if (!match(V: LatchVPBB->getTerminator(),
3658 P: m_BranchOnTwoConds(Op0: m_AnyOf(Op0: m_VPValue(V&: UncountableCondition)),
3659 Op1: m_VPValue())))
3660 return std::nullopt;
3661
3662 SmallVector<VPValue *, 4> Worklist;
3663 Worklist.push_back(Elt: UncountableCondition);
3664 while (!Worklist.empty()) {
3665 VPValue *V = Worklist.pop_back_val();
3666
3667 // Any value defined outside the loop does not need to be copied.
3668 if (V->isDefinedOutsideLoopRegions())
3669 continue;
3670
3671 // FIXME: Remove the single user restriction; it's here because we're
3672 // starting with the simplest set of loops we can, and multiple
3673 // users means needing to add PHI nodes in the transform.
3674 if (V->getNumUsers() > 1)
3675 return std::nullopt;
3676
3677 VPValue *Op1, *Op2;
3678 // Walk back through recipes until we find at least one load from memory.
3679 if (match(V, P: m_ICmp(Op0: m_VPValue(V&: Op1), Op1: m_VPValue(V&: Op2)))) {
3680 Worklist.push_back(Elt: Op1);
3681 Worklist.push_back(Elt: Op2);
3682 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3683 } else if (match(V, P: m_VPInstruction<Instruction::Load>(Ops: m_VPValue(V&: Op1)))) {
3684 VPRecipeBase *GepR = Op1->getDefiningRecipe();
3685 // Only matching base + single offset term for now.
3686 if (GepR->getNumOperands() != 2)
3687 return std::nullopt;
3688 // Matching a GEP with a loop-invariant base ptr.
3689 if (!match(V: GepR, P: m_VPInstruction<Instruction::GetElementPtr>(
3690 Ops: m_LiveIn(), Ops: m_VPValue())))
3691 return std::nullopt;
3692 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3693 Recipes.push_back(Elt: cast<VPInstruction>(Val: GepR));
3694 } else if (match(V, P: m_VPInstruction<VPInstruction::MaskedCond>(
3695 Ops: m_VPValue(V&: Op1)))) {
3696 Worklist.push_back(Elt: Op1);
3697 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3698 } else
3699 return std::nullopt;
3700 }
3701
3702 // If we couldn't match anything, don't return the condition. It may be
3703 // defined outside the loop.
3704 if (Recipes.empty() || none_of(Range&: Recipes, P: [](VPInstruction *I) {
3705 return match(V: I, P: m_VPInstruction<Instruction::GetElementPtr>());
3706 }))
3707 return std::nullopt;
3708
3709 return UncountableCondition;
3710}
3711
3712struct EarlyExitInfo {
3713 VPBasicBlock *EarlyExitingVPBB;
3714 VPIRBasicBlock *EarlyExitVPBB;
3715 VPValue *CondToExit;
3716};
3717
3718/// Update \p Plan to mask memory operations in the loop based on whether the
3719/// early exit is taken or not.
3720///
3721/// We're currently expecting to find a loop with properties similar to the
3722/// following:
3723///
3724/// for.body:
3725/// ir<%indvars.iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<%0>
3726/// EMIT ir<%arrayidx> = getelementptr inbounds nuw ir<@c>, ir<%indvars.iv>
3727/// EMIT-SCALAR ir<%0> = load ir<%arrayidx>
3728/// EMIT ir<%cmp1> = icmp sgt ir<%0>, ir<5>
3729/// EMIT vp<%1> = masked-cond ir<%cmp1>
3730/// Successor(s): if.end
3731///
3732/// if.end:
3733/// EMIT ir<%arrayidx3> = getelementptr inbounds nuw ir<@src>, ir<%indvars.iv>
3734/// EMIT-SCALAR ir<%2> = load ir<%arrayidx3>
3735/// EMIT ir<%add> = add nsw ir<%2>, ir<42>
3736/// EMIT ir<%arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<%indvars.iv>
3737/// EMIT store ir<%add>, ir<%arrayidx5>
3738/// EMIT ir<%indvars.iv.next> = add nuw nsw ir<%indvars.iv>, ir<1>
3739/// EMIT vp<%3> = any-of ir<%1>
3740/// EMIT ir<%exitcond.not> = icmp eq ir<%indvars.iv.next>, ir<10000>
3741/// EMIT branch-on-two-conds vp<%3>, ir<%exitcond.not>
3742/// Successor(s): middle.block, middle.block, for.body
3743///
3744/// We currently expect LoopVectorizationLegality to ensure that:
3745/// * There must also be a counted exit. We will need to support speculative
3746/// or first-faulting loads before we can remove this restriction.
3747/// * Any stores within the loop must not alias with the load used for the
3748/// uncountable exit. We can relax this a bit with runtime aliasing checks.
3749/// * Other memory operations in the loop can take place before or after the
3750/// uncountable exit, but must also be unconditional. We need to support
3751/// combining the conditions in VPlanPredicator.
3752/// * The loop must have a single unconditional load contributing to the
3753/// uncountable exit comparison, and the other term must be loop-invariant.
3754/// Improving upon this requires work in getRecipesForUncountableExit to
3755/// handle more complex recipe graphs.
3756static bool handleUncountableExitsWithSideEffects(
3757 VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
3758 VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB,
3759 Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT,
3760 AssumptionCache *AC) {
3761
3762 // Disconnect early exiting blocks from successors, remove branches. We
3763 // currently don't support multiple uses for recipes involved in creating
3764 // the uncountable exit condition.
3765 for (auto &Exit : Exits) {
3766 if (Exit.EarlyExitingVPBB == LatchVPBB)
3767 continue;
3768
3769 for (VPRecipeBase &R : Exit.EarlyExitVPBB->phis())
3770 cast<VPIRPhi>(Val: &R)->removeIncomingValueFor(IncomingBlock: Exit.EarlyExitingVPBB);
3771 Exit.EarlyExitingVPBB->getTerminator()->eraseFromParent();
3772 VPBlockUtils::disconnectBlocks(From: Exit.EarlyExitingVPBB, To: Exit.EarlyExitVPBB);
3773 }
3774
3775 VPDominatorTree VPDT(Plan);
3776
3777 // We can abandon a VPlan entirely if we return false here, so we shouldn't
3778 // crash if some earlier assumptions on scalar IR don't hold for the vplan
3779 // version of the loop.
3780 SmallVector<VPInstruction *, 8> ConditionRecipes;
3781
3782 std::optional<VPValue *> Cond =
3783 getRecipesForUncountableExit(Recipes&: ConditionRecipes, LatchVPBB);
3784 if (!Cond)
3785 return false;
3786
3787 // Find load contributing to condition.
3788 // At the moment LoopVectorizationLegality only supports a single
3789 // early-exit expression with a compare and a single load that must
3790 // be unconditional.
3791 // TODO: Support more than one load.
3792 auto *Load =
3793 find_singleton<VPInstruction>(Range&: ConditionRecipes, P: [](auto *I, bool _) {
3794 return match(I, m_VPInstruction<Instruction::Load>(Ops: m_VPValue()))
3795 ? I
3796 : nullptr;
3797 });
3798 assert(Load && "Couldn't find exactly one load");
3799 // TODO: Support conditional loads for uncountable exits.
3800 assert(VPDT.dominates(Load->getParent(), LatchVPBB) &&
3801 "Uncountable exit condition load is conditional.");
3802 VPInstruction *Ptr = cast<VPInstruction>(Val: Load->getOperand(N: 0));
3803
3804 // Ensure that we are guaranteed to be able to dereference the memory used
3805 // for determining the uncountable exit for the maximum possible number of
3806 // scalar iterations of the loop.
3807 //
3808 // TODO: Support first-faulting loads in cases where we don't know whether
3809 // all possible addresses are dereferenceable.
3810 {
3811 SmallVector<const SCEVPredicate *, 4> Predicates;
3812 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: TheLoop);
3813 const DataLayout &DL = Plan.getDataLayout();
3814 APInt EltSize(DL.getIndexTypeSizeInBits(Ty: Ptr->getScalarType()),
3815 DL.getTypeStoreSize(Ty: Load->getScalarType()).getFixedValue());
3816 if (!isDereferenceableAndAlignedInLoop(
3817 PtrSCEV, Alignment: cast<LoadInst>(Val: Load->getUnderlyingInstr())->getAlign(),
3818 EltSizeSCEV: PSE.getSE()->getConstant(Val: EltSize), L: TheLoop, SE&: *PSE.getSE(), DT, AC,
3819 Predicates: &Predicates))
3820 return false;
3821 }
3822
3823 // Check for a single GEP for the condition load to see if we can link it to
3824 // a widen IV recipe with a step of 1; we're only interested in contiguous
3825 // accesses for the condition load right now.
3826 auto *IV = cast<VPWidenInductionRecipe>(Val: &HeaderVPBB->front());
3827 if (!match(V: IV->getStartValue(), P: m_SpecificInt(V: 0)) ||
3828 !match(V: IV->getStepValue(), P: m_SpecificInt(V: 1)))
3829 return false;
3830 if (!match(V: Ptr, P: m_VPInstruction<Instruction::GetElementPtr>(Ops: m_LiveIn(),
3831 Ops: m_Specific(VPV: IV))))
3832 return false;
3833
3834 // We want to guarantee that the uncountable exit condition (and the mask
3835 // we will generate from it) are available for all operations in the loop
3836 // that need to be masked. If the condition recipes are not already the first
3837 // recipes in the header after the last phi, move them there.
3838 auto InsertIt = HeaderVPBB->getFirstNonPhi();
3839 while (InsertIt != HeaderVPBB->end() &&
3840 is_contained(Range&: ConditionRecipes, Element: &*InsertIt)) {
3841 erase(C&: ConditionRecipes, V: &*InsertIt);
3842 InsertIt++;
3843 }
3844 for (auto *Recipe : reverse(C&: ConditionRecipes))
3845 Recipe->moveBefore(BB&: *HeaderVPBB, I: InsertIt);
3846
3847 // Create a mask to represent all lanes that fully execute in the vector loop,
3848 // stopping short of any early exit.
3849 VPBuilder MaskBuilder(HeaderVPBB, InsertIt);
3850 VPValue *FirstActive = MaskBuilder.createFirstActiveLane(Masks: *Cond);
3851 Type *IVScalarTy = IV->getScalarType();
3852 VPValue *ALMMultiplier = Plan.getConstantInt(Ty: IVScalarTy, Val: 1);
3853 VPValue *Zero = Plan.getZero(Ty: IVScalarTy);
3854 FirstActive =
3855 MaskBuilder.createScalarZExtOrTrunc(Op: FirstActive, ResultTy: IVScalarTy, DL: DebugLoc());
3856 VPValue *Mask = MaskBuilder.createNaryOp(Opcode: VPInstruction::ActiveLaneMask,
3857 Operands: {Zero, FirstActive, ALMMultiplier},
3858 DL: DebugLoc(), Name: "uncountable.exit.mask");
3859
3860 // Convert all other memory operations to use the mask.
3861 for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(Header: HeaderVPBB))
3862 for (VPRecipeBase &R : *VPBB)
3863 if (R.mayReadOrWriteMemory() && &R != Load) {
3864 // TODO: Handle conditional memory operations in the loop.
3865 if (!VPDT.dominates(A: R.getParent(), B: LatchVPBB))
3866 return false;
3867 cast<VPInstruction>(Val: &R)->addMask(Mask);
3868 }
3869
3870 // Update middle block branch to compare (IV + however many lanes were active)
3871 // against the full trip count, since we may be exiting the vector loop early.
3872 // If we didn't take an early exit, we should get the equivalent of VF from
3873 // the FirstActiveLane.
3874 assert(match(MiddleVPBB->getTerminator(), m_BranchOnCond()) &&
3875 "Expected BranchOnCond terminator for MiddleVPBB");
3876 VPBuilder MiddleBuilder(MiddleVPBB->getTerminator());
3877 VPValue *ScalarIV = MiddleBuilder.createNaryOp(Opcode: VPInstruction::ExtractLane,
3878 Operands: {Zero, IV}, DL: DebugLoc());
3879 VPValue *ExitIV = MiddleBuilder.createAdd(LHS: ScalarIV, RHS: FirstActive);
3880 VPValue *FullTC =
3881 MiddleBuilder.createICmp(Pred: CmpInst::ICMP_EQ, A: ExitIV, B: Plan.getTripCount());
3882 MiddleVPBB->getTerminator()->setOperand(I: 0, New: FullTC);
3883
3884 // Update resume phi in scalar.ph.
3885 VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
3886 auto Phis = ScalarPH->phis();
3887 // TODO: Handle more than one Phi; re-derive from IV.
3888 // TODO: Handle reductions.
3889 if (range_size(Range&: Phis) != 1)
3890 return false;
3891 VPPhi *ContinueIV = cast<VPPhi>(Val: Phis.begin());
3892 // Make sure we're referring to the same IV.
3893 assert(
3894 match(ContinueIV->getOperand(0),
3895 m_VPInstruction<VPInstruction::ExitingIVValue>(m_Specific(IV))) &&
3896 "Continuing from different IV");
3897 ContinueIV->setOperand(I: 0, New: ExitIV);
3898 return true;
3899}
3900
3901bool VPlanTransforms::handleUncountableEarlyExits(
3902 VPlan &Plan, VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB,
3903 VPBasicBlock *MiddleVPBB, Loop *TheLoop, PredicatedScalarEvolution &PSE,
3904 DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style) {
3905#ifndef NDEBUG
3906 VPDominatorTree VPDT(Plan);
3907#endif
3908 VPBuilder LatchBuilder(LatchVPBB->getTerminator());
3909 SmallVector<EarlyExitInfo> Exits;
3910 for (VPIRBasicBlock *ExitBlock : Plan.getExitBlocks()) {
3911 for (VPBlockBase *Pred : to_vector(Range&: ExitBlock->getPredecessors())) {
3912 if (Pred == MiddleVPBB)
3913 continue;
3914 // Collect condition for this early exit.
3915 auto *EarlyExitingVPBB = cast<VPBasicBlock>(Val: Pred);
3916 VPBlockBase *TrueSucc = EarlyExitingVPBB->getSuccessors()[0];
3917 VPValue *CondOfEarlyExitingVPBB;
3918 [[maybe_unused]] bool Matched =
3919 match(V: EarlyExitingVPBB->getTerminator(),
3920 P: m_BranchOnCond(Op0: m_VPValue(V&: CondOfEarlyExitingVPBB)));
3921 assert(Matched && "Terminator must be BranchOnCond");
3922
3923 // Insert the MaskedCond in the EarlyExitingVPBB so the predicator adds
3924 // the correct block mask.
3925 VPBuilder EarlyExitingBuilder(EarlyExitingVPBB->getTerminator());
3926 auto *CondToEarlyExit = EarlyExitingBuilder.createNaryOp(
3927 Opcode: VPInstruction::MaskedCond,
3928 Operands: TrueSucc == ExitBlock
3929 ? CondOfEarlyExitingVPBB
3930 : EarlyExitingBuilder.createNot(Operand: CondOfEarlyExitingVPBB));
3931 assert((isa<VPIRValue>(CondOfEarlyExitingVPBB) ||
3932 !VPDT.properlyDominates(EarlyExitingVPBB, LatchVPBB) ||
3933 VPDT.properlyDominates(
3934 CondOfEarlyExitingVPBB->getDefiningRecipe()->getParent(),
3935 LatchVPBB)) &&
3936 "exit condition must dominate the latch");
3937 Exits.push_back(Elt: {
3938 .EarlyExitingVPBB: EarlyExitingVPBB,
3939 .EarlyExitVPBB: ExitBlock,
3940 .CondToExit: CondToEarlyExit,
3941 });
3942 }
3943 }
3944
3945 assert(!Exits.empty() && "must have at least one early exit");
3946 // Sort exits by RPO order to get correct program order. RPO gives a
3947 // topological ordering of the CFG, ensuring upstream exits are checked
3948 // before downstream exits in the dispatch chain.
3949 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
3950 HeaderVPBB);
3951 DenseMap<VPBlockBase *, unsigned> RPOIdx;
3952 for (const auto &[Num, VPB] : enumerate(First&: RPOT))
3953 RPOIdx[VPB] = Num;
3954 llvm::sort(C&: Exits, Comp: [&RPOIdx](const EarlyExitInfo &A, const EarlyExitInfo &B) {
3955 return RPOIdx[A.EarlyExitingVPBB] < RPOIdx[B.EarlyExitingVPBB];
3956 });
3957#ifndef NDEBUG
3958 // After RPO sorting, verify that for any pair where one exit dominates
3959 // another, the dominating exit comes first. This is guaranteed by RPO
3960 // (topological order) and is required for the dispatch chain correctness.
3961 for (unsigned I = 0; I + 1 < Exits.size(); ++I)
3962 for (unsigned J = I + 1; J < Exits.size(); ++J)
3963 assert(!VPDT.properlyDominates(Exits[J].EarlyExitingVPBB,
3964 Exits[I].EarlyExitingVPBB) &&
3965 "RPO sort must place dominating exits before dominated ones");
3966#endif
3967
3968 // Build the AnyOf condition for the latch terminator using logical OR
3969 // to avoid poison propagation from later exit conditions when an earlier
3970 // exit is taken.
3971 VPValue *Combined = Exits[0].CondToExit;
3972 for (const EarlyExitInfo &Info : drop_begin(RangeOrContainer&: Exits))
3973 Combined = LatchBuilder.createLogicalOr(LHS: Combined, RHS: Info.CondToExit);
3974
3975 VPValue *IsAnyExitTaken =
3976 LatchBuilder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: {Combined});
3977
3978 // Create a comparison for the latch exit condition and replace the
3979 // BranchOnCond with a BranchOnTwoConds. The original BranchOnCond's condition
3980 // is used as the latch-exit condition; canonical IV recipes have not been
3981 // introduced yet, so there is no BranchOnCount to derive the condition from.
3982 auto *LatchExitingBranch = cast<VPInstruction>(Val: LatchVPBB->getTerminator());
3983 assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCond &&
3984 "Unexpected terminator");
3985 VPValue *IsLatchExitTaken = LatchExitingBranch->getOperand(N: 0);
3986 DebugLoc LatchDL = LatchExitingBranch->getDebugLoc();
3987 LatchExitingBranch->eraseFromParent();
3988 LatchBuilder.setInsertPoint(LatchVPBB);
3989 LatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnTwoConds,
3990 Operands: {IsAnyExitTaken, IsLatchExitTaken}, DL: LatchDL);
3991 LatchVPBB->clearSuccessors();
3992
3993 if (Style == UncountableExitStyle::MaskedHandleExitInScalarLoop) {
3994 // If handling the exiting lane in the scalar loop, combine the exit
3995 // conditions into a single BranchOnCond.
3996 LatchVPBB->setSuccessors({MiddleVPBB, MiddleVPBB, HeaderVPBB});
3997 MiddleVPBB->clearPredecessors();
3998 MiddleVPBB->setPredecessors({LatchVPBB, LatchVPBB});
3999 return handleUncountableExitsWithSideEffects(
4000 Plan, Exits, HeaderVPBB, LatchVPBB, MiddleVPBB, TheLoop, PSE, DT, AC);
4001 }
4002
4003 // Create the vector.early.exit blocks.
4004 SmallVector<VPBasicBlock *> VectorEarlyExitVPBBs(Exits.size());
4005 for (unsigned Idx = 0; Idx != Exits.size(); ++Idx) {
4006 Twine BlockSuffix = Exits.size() == 1 ? "" : Twine(".") + Twine(Idx);
4007 VPBasicBlock *VectorEarlyExitVPBB =
4008 Plan.createVPBasicBlock(Name: "vector.early.exit" + BlockSuffix);
4009 VectorEarlyExitVPBBs[Idx] = VectorEarlyExitVPBB;
4010 }
4011
4012 // Create the dispatch block (or reuse the single exit block if only one
4013 // exit). The dispatch block computes the first active lane of the combined
4014 // condition and, for multiple exits, chains through conditions to determine
4015 // which exit to take.
4016 VPBasicBlock *DispatchVPBB =
4017 Exits.size() == 1 ? VectorEarlyExitVPBBs[0]
4018 : Plan.createVPBasicBlock(Name: "vector.early.exit.check");
4019 DispatchVPBB->setPredecessors({LatchVPBB});
4020 LatchVPBB->setSuccessors({DispatchVPBB, MiddleVPBB, HeaderVPBB});
4021 VPBuilder DispatchBuilder(DispatchVPBB, DispatchVPBB->begin());
4022 VPValue *FirstActiveLane = DispatchBuilder.createFirstActiveLane(
4023 Masks: {Combined}, DL: DebugLoc::getUnknown(), Name: "first.active.lane");
4024
4025 // For each early exit, disconnect the original exiting block
4026 // (early.exiting.I) from the exit block (ir-bb<exit.I>) and route through a
4027 // new vector.early.exit block. Update ir-bb<exit.I>'s phis to extract their
4028 // values at the first active lane:
4029 //
4030 // Input:
4031 // early.exiting.I:
4032 // ...
4033 // EMIT branch-on-cond vp<%cond.I>
4034 // Successor(s): in.loop.succ, ir-bb<exit.I>
4035 //
4036 // ir-bb<exit.I>:
4037 // IR %phi = phi [ vp<%incoming.I>, early.exiting.I ], ...
4038 //
4039 // Output:
4040 // early.exiting.I:
4041 // ...
4042 // Successor(s): in.loop.succ
4043 //
4044 // vector.early.exit.I:
4045 // EMIT vp<%exit.val> = extract-lane vp<%first.lane>, vp<%incoming.I>
4046 // Successor(s): ir-bb<exit.I>
4047 //
4048 // ir-bb<exit.I>:
4049 // IR %phi = phi ... (extra operand: vp<%exit.val> from
4050 // vector.early.exit.I)
4051 //
4052 for (auto [Exit, VectorEarlyExitVPBB] :
4053 zip_equal(t&: Exits, u&: VectorEarlyExitVPBBs)) {
4054 auto &[EarlyExitingVPBB, EarlyExitVPBB, _] = Exit;
4055 // Adjust the phi nodes in EarlyExitVPBB.
4056 // 1. remove incoming values from EarlyExitingVPBB,
4057 // 2. extract the incoming value at FirstActiveLane
4058 // 3. add back the extracts as last operands for the phis
4059 // Then adjust the CFG, removing the edge between EarlyExitingVPBB and
4060 // EarlyExitVPBB and adding a new edge between VectorEarlyExitVPBB and
4061 // EarlyExitVPBB. The extracts at FirstActiveLane are now the incoming
4062 // values from VectorEarlyExitVPBB.
4063 for (VPRecipeBase &R : EarlyExitVPBB->phis()) {
4064 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
4065 VPValue *IncomingVal =
4066 ExitIRI->getIncomingValueForBlock(VPBB: EarlyExitingVPBB);
4067 VPValue *NewIncoming = IncomingVal;
4068 if (!isa<VPIRValue>(Val: IncomingVal)) {
4069 VPBuilder EarlyExitBuilder(VectorEarlyExitVPBB);
4070 NewIncoming = EarlyExitBuilder.createNaryOp(
4071 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, IncomingVal},
4072 DL: DebugLoc::getUnknown(), Name: "early.exit.value");
4073 }
4074 ExitIRI->removeIncomingValueFor(IncomingBlock: EarlyExitingVPBB);
4075 ExitIRI->addIncoming(IncomingV: NewIncoming);
4076 }
4077
4078 EarlyExitingVPBB->getTerminator()->eraseFromParent();
4079 VPBlockUtils::disconnectBlocks(From: EarlyExitingVPBB, To: EarlyExitVPBB);
4080 VPBlockUtils::connectBlocks(From: VectorEarlyExitVPBB, To: EarlyExitVPBB);
4081 }
4082
4083 // Chain through exits: for each exit, check if its condition is true at
4084 // the first active lane. If so, take that exit; otherwise, try the next.
4085 // The last exit needs no check since it must be taken if all others fail.
4086 //
4087 // For 3 exits (cond.0, cond.1, cond.2), this creates:
4088 //
4089 // latch:
4090 // ...
4091 // EMIT vp<%combined> = logical-or vp<%cond.0>, vp<%cond.1>, vp<%cond.2>
4092 // ...
4093 //
4094 // vector.early.exit.check:
4095 // EMIT vp<%first.lane> = first-active-lane vp<%combined>
4096 // EMIT vp<%at.cond.0> = extract-lane vp<%first.lane>, vp<%cond.0>
4097 // EMIT branch-on-cond vp<%at.cond.0>
4098 // Successor(s): vector.early.exit.0, vector.early.exit.check.0
4099 //
4100 // vector.early.exit.check.0:
4101 // EMIT vp<%at.cond.1> = extract-lane vp<%first.lane>, vp<%cond.1>
4102 // EMIT branch-on-cond vp<%at.cond.1>
4103 // Successor(s): vector.early.exit.1, vector.early.exit.2
4104 VPBasicBlock *CurrentBB = DispatchVPBB;
4105 for (auto [I, Exit] : enumerate(First: ArrayRef(Exits).drop_back())) {
4106 VPValue *LaneVal = DispatchBuilder.createNaryOp(
4107 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, Exit.CondToExit},
4108 DL: DebugLoc::getUnknown(), Name: "exit.cond.at.lane");
4109
4110 // For the last dispatch, branch directly to the last exit on false;
4111 // otherwise, create a new check block.
4112 bool IsLastDispatch = (I + 2 == Exits.size());
4113 VPBasicBlock *FalseBB =
4114 IsLastDispatch ? VectorEarlyExitVPBBs.back()
4115 : Plan.createVPBasicBlock(
4116 Name: Twine("vector.early.exit.check.") + Twine(I));
4117
4118 DispatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {LaneVal});
4119 CurrentBB->setSuccessors({VectorEarlyExitVPBBs[I], FalseBB});
4120 VectorEarlyExitVPBBs[I]->setPredecessors({CurrentBB});
4121 FalseBB->setPredecessors({CurrentBB});
4122
4123 CurrentBB = FalseBB;
4124 DispatchBuilder.setInsertPoint(CurrentBB);
4125 }
4126
4127 return true;
4128}
4129
4130/// This function tries convert extended in-loop reductions to
4131/// VPExpressionRecipe and clamp the \p Range if it is beneficial and
4132/// valid. The created recipe must be decomposed to its constituent
4133/// recipes before execution.
4134static VPExpressionRecipe *
4135tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
4136 VFRange &Range) {
4137 Type *RedTy = Red->getScalarType();
4138 VPValue *VecOp = Red->getVecOp();
4139
4140 assert(!Red->isPartialReduction() &&
4141 "This path does not support partial reductions");
4142
4143 // Clamp the range if using extended-reduction is profitable.
4144 auto IsExtendedRedValidAndClampRange =
4145 [&](unsigned Opcode, Instruction::CastOps ExtOpc, Type *SrcTy) -> bool {
4146 return LoopVectorizationPlanner::getDecisionAndClampRange(
4147 Predicate: [&](ElementCount VF) {
4148 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
4149 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
4150
4151 InstructionCost ExtRedCost = InstructionCost::getInvalid();
4152 InstructionCost ExtCost =
4153 cast<VPWidenCastRecipe>(Val: VecOp)->computeCost(VF, Ctx);
4154 InstructionCost RedCost = Red->computeCost(VF, Ctx);
4155
4156 assert(!RedTy->isFloatingPointTy() &&
4157 "getExtendedReductionCost only supports integer types");
4158 ExtRedCost = Ctx.TTI.getExtendedReductionCost(
4159 Opcode, IsUnsigned: ExtOpc == Instruction::CastOps::ZExt, ResTy: RedTy, Ty: SrcVecTy,
4160 FMF: Red->getFastMathFlagsOrNone(), CostKind);
4161 return ExtRedCost.isValid() && ExtRedCost < ExtCost + RedCost;
4162 },
4163 Range);
4164 };
4165
4166 VPValue *A;
4167 // Match reduce(ext)).
4168 if (match(V: VecOp, P: m_Isa<VPWidenCastRecipe>(P: m_ZExtOrSExt(Op0: m_VPValue(V&: A)))) &&
4169 IsExtendedRedValidAndClampRange(
4170 RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind()),
4171 cast<VPWidenCastRecipe>(Val: VecOp)->getOpcode(), A->getScalarType()))
4172 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
4173
4174 return nullptr;
4175}
4176
4177/// This function tries convert extended in-loop reductions to
4178/// VPExpressionRecipe and clamp the \p Range if it is beneficial
4179/// and valid. The created VPExpressionRecipe must be decomposed to its
4180/// constituent recipes before execution. Patterns of the
4181/// VPExpressionRecipe:
4182/// reduce.add(mul(...)),
4183/// reduce.add(mul(ext(A), ext(B))),
4184/// reduce.add(ext(mul(ext(A), ext(B)))).
4185/// reduce.fadd(fmul(ext(A), ext(B)))
4186static VPExpressionRecipe *
4187tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
4188 VPCostContext &Ctx, VFRange &Range) {
4189 unsigned Opcode = RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind());
4190 if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
4191 Opcode != Instruction::FAdd)
4192 return nullptr;
4193
4194 assert(!Red->isPartialReduction() &&
4195 "This path does not support partial reductions");
4196 Type *RedTy = Red->getScalarType();
4197
4198 // Clamp the range if using multiply-accumulate-reduction is profitable.
4199 auto IsMulAccValidAndClampRange =
4200 [&](VPWidenRecipe *Mul, VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
4201 VPWidenCastRecipe *OuterExt) -> bool {
4202 return LoopVectorizationPlanner::getDecisionAndClampRange(
4203 Predicate: [&](ElementCount VF) {
4204 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
4205 Type *SrcTy = Ext0 ? Ext0->getOperand(N: 0)->getScalarType() : RedTy;
4206 InstructionCost MulAccCost;
4207
4208 // getMulAccReductionCost for in-loop reductions does not support
4209 // mixed or floating-point extends.
4210 if (Ext0 && Ext1 &&
4211 (Ext0->getOpcode() != Ext1->getOpcode() ||
4212 Ext0->getOpcode() == Instruction::CastOps::FPExt))
4213 return false;
4214
4215 bool IsZExt =
4216 !Ext0 || Ext0->getOpcode() == Instruction::CastOps::ZExt;
4217 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
4218 MulAccCost = Ctx.TTI.getMulAccReductionCost(IsUnsigned: IsZExt, RedOpcode: Opcode, ResTy: RedTy,
4219 Ty: SrcVecTy, CostKind);
4220
4221 InstructionCost MulCost = Mul->computeCost(VF, Ctx);
4222 InstructionCost RedCost = Red->computeCost(VF, Ctx);
4223 InstructionCost ExtCost = 0;
4224 if (Ext0)
4225 ExtCost += Ext0->computeCost(VF, Ctx);
4226 if (Ext1)
4227 ExtCost += Ext1->computeCost(VF, Ctx);
4228 if (OuterExt)
4229 ExtCost += OuterExt->computeCost(VF, Ctx);
4230
4231 return MulAccCost.isValid() &&
4232 MulAccCost < ExtCost + MulCost + RedCost;
4233 },
4234 Range);
4235 };
4236
4237 VPValue *VecOp = Red->getVecOp();
4238 VPRecipeBase *Sub = nullptr;
4239 VPValue *A, *B;
4240 VPValue *Tmp = nullptr;
4241
4242 if (RedTy->isFloatingPointTy())
4243 return nullptr;
4244
4245 // Sub reductions could have a sub between the add reduction and vec op.
4246 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Tmp)))) {
4247 Sub = VecOp->getDefiningRecipe();
4248 VecOp = Tmp;
4249 }
4250
4251 // If ValB is a constant and can be safely extended, truncate it to the same
4252 // type as ExtA's operand, then extend it to the same type as ExtA. This
4253 // creates two uniform extends that can more easily be matched by the rest of
4254 // the bundling code. The ExtB reference, ValB and operand 1 of Mul are all
4255 // replaced with the new extend of the constant.
4256 auto ExtendAndReplaceConstantOp = [](VPWidenCastRecipe *ExtA,
4257 VPWidenCastRecipe *&ExtB, VPValue *&ValB,
4258 VPWidenRecipe *Mul) {
4259 if (!ExtA || ExtB || !isa<VPIRValue>(Val: ValB))
4260 return;
4261 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
4262 Instruction::CastOps ExtOpc = ExtA->getOpcode();
4263 const APInt *Const;
4264 if (!match(V: ValB, P: m_APInt(C&: Const)) ||
4265 !llvm::canConstantBeExtended(
4266 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
4267 return;
4268 // The truncate ensures that the type of each extended operand is the
4269 // same, and it's been proven that the constant can be extended from
4270 // NarrowTy safely. Necessary since ExtA's extended operand would be
4271 // e.g. an i8, while the const will likely be an i32. This will be
4272 // elided by later optimisations.
4273 VPBuilder Builder(Mul);
4274 auto *Trunc =
4275 Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc, Op: ValB, ResultTy: NarrowTy);
4276 Type *WideTy = ExtA->getScalarType();
4277 ValB = ExtB = Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy);
4278 Mul->setOperand(I: 1, New: ExtB);
4279 };
4280
4281 // Try to match reduce.add(mul(...)).
4282 if (match(V: VecOp, P: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B)))) {
4283 auto *RecipeA = dyn_cast<VPWidenCastRecipe>(Val: A);
4284 auto *RecipeB = dyn_cast<VPWidenCastRecipe>(Val: B);
4285 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
4286
4287 // Convert reduce.add(mul(ext, const)) to reduce.add(mul(ext, ext(const)))
4288 ExtendAndReplaceConstantOp(RecipeA, RecipeB, B, Mul);
4289
4290 // Match reduce.add/sub(mul(ext, ext)).
4291 if (RecipeA && RecipeB && match(V: RecipeA, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
4292 match(V: RecipeB, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
4293 IsMulAccValidAndClampRange(Mul, RecipeA, RecipeB, nullptr)) {
4294 if (Sub)
4295 return new VPExpressionRecipe(RecipeA, RecipeB, Mul,
4296 cast<VPWidenRecipe>(Val: Sub), Red);
4297 return new VPExpressionRecipe(RecipeA, RecipeB, Mul, Red);
4298 }
4299 // TODO: Add an expression type for this variant with a negated mul
4300 if (!Sub && IsMulAccValidAndClampRange(Mul, nullptr, nullptr, nullptr))
4301 return new VPExpressionRecipe(Mul, Red);
4302 }
4303 // TODO: Add an expression type for negated versions of other expression
4304 // variants.
4305 if (Sub)
4306 return nullptr;
4307
4308 // Match reduce.add(ext(mul(A, B))).
4309 if (match(V: VecOp, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))))) {
4310 auto *Ext = cast<VPWidenCastRecipe>(Val: VecOp);
4311 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
4312 auto *Ext0 = dyn_cast<VPWidenCastRecipe>(Val: A);
4313 auto *Ext1 = dyn_cast<VPWidenCastRecipe>(Val: B);
4314
4315 // reduce.add(ext(mul(ext, const)))
4316 // -> reduce.add(ext(mul(ext, ext(const))))
4317 ExtendAndReplaceConstantOp(Ext0, Ext1, B, Mul);
4318
4319 // reduce.add(ext(mul(ext(A), ext(B))))
4320 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
4321 // The inner extends must either have the same opcode as the outer extend or
4322 // be the same, in which case the multiply can never result in a negative
4323 // value and the outer extend can be folded away by doing wider
4324 // extends for the operands of the mul.
4325 if (Ext0 && Ext1 &&
4326 (Ext->getOpcode() == Ext0->getOpcode() || Ext0 == Ext1) &&
4327 Ext0->getOpcode() == Ext1->getOpcode() &&
4328 IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext) && Mul->hasOneUse()) {
4329 auto *NewExt0 = new VPWidenCastRecipe(
4330 Ext0->getOpcode(), Ext0->getOperand(N: 0), Ext->getScalarType(), nullptr,
4331 *Ext0, *Ext0, Ext0->getDebugLoc());
4332 NewExt0->insertBefore(InsertPos: Ext0);
4333
4334 VPWidenCastRecipe *NewExt1 = NewExt0;
4335 if (Ext0 != Ext1) {
4336 NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(N: 0),
4337 Ext->getScalarType(), nullptr, *Ext1,
4338 *Ext1, Ext1->getDebugLoc());
4339 NewExt1->insertBefore(InsertPos: Ext1);
4340 }
4341 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewExt0, NewExt1});
4342 NewMul->insertBefore(InsertPos: Mul);
4343 Ext->replaceAllUsesWith(New: NewMul);
4344 Ext->eraseFromParent();
4345 Mul->eraseFromParent();
4346 return new VPExpressionRecipe(NewExt0, NewExt1, NewMul, Red);
4347 }
4348 }
4349 return nullptr;
4350}
4351
4352/// This function tries to create abstract recipes from the reduction recipe for
4353/// following optimizations and cost estimation.
4354static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
4355 VPCostContext &Ctx,
4356 VFRange &Range) {
4357 // Creation of VPExpressions for partial reductions is entirely handled in
4358 // transformToPartialReduction.
4359 assert(!Red->isPartialReduction() &&
4360 "This path does not support partial reductions");
4361
4362 VPExpressionRecipe *AbstractR = nullptr;
4363 auto IP = std::next(x: Red->getIterator());
4364 auto *VPBB = Red->getParent();
4365 if (auto *MulAcc = tryToMatchAndCreateMulAccumulateReduction(Red, Ctx, Range))
4366 AbstractR = MulAcc;
4367 else if (auto *ExtRed = tryToMatchAndCreateExtendedReduction(Red, Ctx, Range))
4368 AbstractR = ExtRed;
4369 // Cannot create abstract inloop reduction recipes.
4370 if (!AbstractR)
4371 return;
4372
4373 AbstractR->insertBefore(BB&: *VPBB, IP);
4374 Red->replaceAllUsesWith(New: AbstractR);
4375}
4376
4377void VPlanTransforms::convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
4378 VFRange &Range) {
4379 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
4380 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
4381 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
4382 if (auto *Red = dyn_cast<VPReductionRecipe>(Val: &R))
4383 tryToCreateAbstractReductionRecipe(Red, Ctx, Range);
4384 }
4385 }
4386}
4387
4388void VPlanTransforms::materializeBroadcasts(VPlan &Plan) {
4389 if (Plan.hasScalarVFOnly())
4390 return;
4391
4392#ifndef NDEBUG
4393 VPDominatorTree VPDT(Plan);
4394#endif
4395
4396 SmallVector<VPValue *> VPValues;
4397 if (VPValue *BTC = Plan.getBackedgeTakenCount())
4398 VPValues.push_back(Elt: BTC);
4399 append_range(C&: VPValues, R: Plan.getLiveIns());
4400 for (VPRecipeBase &R : *Plan.getEntry())
4401 append_range(C&: VPValues, R: R.definedValues());
4402
4403 auto *VectorPreheader = Plan.getVectorPreheader();
4404 for (VPValue *VPV : VPValues) {
4405 if (vputils::onlyScalarValuesUsed(Def: VPV) || isa<VPConstant>(Val: VPV))
4406 continue;
4407
4408 // Add explicit broadcast at the insert point that dominates all users.
4409 VPBasicBlock *HoistBlock = VectorPreheader;
4410 VPBasicBlock::iterator HoistPoint = VectorPreheader->end();
4411 for (VPUser *User : VPV->users()) {
4412 if (User->usesScalars(Op: VPV))
4413 continue;
4414 if (cast<VPRecipeBase>(Val: User)->getParent() == VectorPreheader)
4415 HoistPoint = HoistBlock->begin();
4416 else
4417 assert(VPDT.dominates(VectorPreheader,
4418 cast<VPRecipeBase>(User)->getParent()) &&
4419 "All users must be in the vector preheader or dominated by it");
4420 }
4421
4422 VPBuilder Builder(cast<VPBasicBlock>(Val: HoistBlock), HoistPoint);
4423 auto *Broadcast = Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: {VPV});
4424 VPV->replaceUsesWithIf(New: Broadcast,
4425 ShouldReplace: [VPV, Broadcast](VPUser &U, unsigned Idx) {
4426 return Broadcast != &U && !U.usesScalars(Op: VPV);
4427 });
4428 }
4429}
4430
4431// Collect common metadata from a group of replicate recipes by intersecting
4432// metadata from all recipes in the group.
4433static VPIRMetadata getCommonMetadata(ArrayRef<VPReplicateRecipe *> Recipes) {
4434 VPIRMetadata CommonMetadata = *Recipes.front();
4435 for (VPReplicateRecipe *Recipe : drop_begin(RangeOrContainer&: Recipes))
4436 CommonMetadata.intersect(MD: *Recipe);
4437 return CommonMetadata;
4438}
4439
4440template <unsigned Opcode>
4441static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
4442collectComplementaryPredicatedMemOps(VPlan &Plan,
4443 PredicatedScalarEvolution &PSE,
4444 const Loop *L) {
4445 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
4446 "Only Load and Store opcodes supported");
4447 [[maybe_unused]] constexpr bool IsLoad = (Opcode == Instruction::Load);
4448
4449 // For each address, collect operations with the same or complementary masks.
4450 SmallVector<SmallVector<VPReplicateRecipe *, 4>> AllGroups;
4451 auto Groups = collectGroupedReplicateMemOps<Opcode>(
4452 Plan, PSE, L,
4453 [](VPReplicateRecipe *RepR) { return RepR->isPredicated(); });
4454 for (auto Recipes : Groups) {
4455 if (Recipes.size() < 2)
4456 continue;
4457
4458 assert(all_equal(
4459 map_range(Recipes, bind_back<getLoadStoreValueType>(IsLoad))) &&
4460 "Expected all recipes in group to have the same load-store type");
4461
4462 // Collect groups with the same or complementary masks.
4463 for (VPReplicateRecipe *&RecipeI : Recipes) {
4464 if (!RecipeI)
4465 continue;
4466
4467 VPValue *MaskI = RecipeI->getMask();
4468 SmallVector<VPReplicateRecipe *, 4> Group;
4469 Group.push_back(Elt: RecipeI);
4470 RecipeI = nullptr;
4471
4472 // Find all operations with the same or complementary masks.
4473 bool HasComplementaryMask = false;
4474 for (VPReplicateRecipe *&RecipeJ : Recipes) {
4475 if (!RecipeJ)
4476 continue;
4477
4478 VPValue *MaskJ = RecipeJ->getMask();
4479 // Check if any operation in the group has a complementary mask with
4480 // another, that is M1 == NOT(M2) or M2 == NOT(M1).
4481 HasComplementaryMask |= match(V: MaskI, P: m_Not(Op0: m_Specific(VPV: MaskJ))) ||
4482 match(V: MaskJ, P: m_Not(Op0: m_Specific(VPV: MaskI)));
4483 Group.push_back(Elt: RecipeJ);
4484 RecipeJ = nullptr;
4485 }
4486
4487 if (HasComplementaryMask) {
4488 assert(Group.size() >= 2 && "must have at least 2 entries");
4489 AllGroups.push_back(Elt: std::move(Group));
4490 }
4491 }
4492 }
4493
4494 return AllGroups;
4495}
4496
4497// Find the recipe with minimum alignment in the group.
4498template <typename InstType>
4499static VPReplicateRecipe *
4500findRecipeWithMinAlign(ArrayRef<VPReplicateRecipe *> Group) {
4501 return *min_element(Group, [](VPReplicateRecipe *A, VPReplicateRecipe *B) {
4502 return cast<InstType>(A->getUnderlyingInstr())->getAlign() <
4503 cast<InstType>(B->getUnderlyingInstr())->getAlign();
4504 });
4505}
4506
4507void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
4508 PredicatedScalarEvolution &PSE,
4509 const Loop *L) {
4510 auto Groups =
4511 collectComplementaryPredicatedMemOps<Instruction::Load>(Plan, PSE, L);
4512 if (Groups.empty())
4513 return;
4514
4515 // Process each group of loads.
4516 for (auto &Group : Groups) {
4517 // Try to use the earliest (most dominating) load to replace all others.
4518 VPReplicateRecipe *EarliestLoad = Group[0];
4519 VPBasicBlock *FirstBB = EarliestLoad->getParent();
4520 VPBasicBlock *LastBB = Group.back()->getParent();
4521
4522 // Check that the load doesn't alias with stores between first and last.
4523 auto LoadLoc = vputils::getMemoryLocation(R: *EarliestLoad);
4524 if (!LoadLoc || !canHoistOrSinkWithNoAliasCheck(MemLoc: *LoadLoc, FirstBB, LastBB))
4525 continue;
4526
4527 // Collect common metadata from all loads in the group.
4528 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
4529
4530 // Find the load with minimum alignment to use.
4531 auto *LoadWithMinAlign = findRecipeWithMinAlign<LoadInst>(Group);
4532
4533 bool IsSingleScalar = EarliestLoad->isSingleScalar();
4534 assert(all_of(Group,
4535 [IsSingleScalar](VPReplicateRecipe *R) {
4536 return R->isSingleScalar() == IsSingleScalar;
4537 }) &&
4538 "all members in group must agree on IsSingleScalar");
4539
4540 // Create an unpredicated version of the earliest load with common
4541 // metadata.
4542 auto *UnpredicatedLoad = new VPReplicateRecipe(
4543 LoadWithMinAlign->getUnderlyingInstr(), {EarliestLoad->getOperand(N: 0)},
4544 IsSingleScalar, /*Mask=*/nullptr, *EarliestLoad, CommonMetadata);
4545
4546 UnpredicatedLoad->insertBefore(InsertPos: EarliestLoad);
4547
4548 // Replace all loads in the group with the unpredicated load.
4549 for (VPReplicateRecipe *Load : Group) {
4550 Load->replaceAllUsesWith(New: UnpredicatedLoad);
4551 Load->eraseFromParent();
4552 }
4553 }
4554}
4555
4556static bool
4557canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
4558 PredicatedScalarEvolution &PSE, const Loop &L) {
4559 auto StoreLoc = vputils::getMemoryLocation(R: *StoresToSink.front());
4560 if (!StoreLoc || !StoreLoc->AATags.Scope)
4561 return false;
4562
4563 // When sinking a group of stores, all members of the group alias each other.
4564 // Skip them during the alias checks.
4565 VPBasicBlock *FirstBB = StoresToSink.front()->getParent();
4566 VPBasicBlock *LastBB = StoresToSink.back()->getParent();
4567 SinkStoreInfo SinkInfo(StoresToSink, *StoresToSink[0], PSE, L);
4568 return canHoistOrSinkWithNoAliasCheck(MemLoc: *StoreLoc, FirstBB, LastBB, SinkInfo);
4569}
4570
4571void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
4572 PredicatedScalarEvolution &PSE,
4573 const Loop *L) {
4574 auto Groups =
4575 collectComplementaryPredicatedMemOps<Instruction::Store>(Plan, PSE, L);
4576 if (Groups.empty())
4577 return;
4578
4579 for (auto &Group : Groups) {
4580 if (!canSinkStoreWithNoAliasCheck(StoresToSink: Group, PSE, L: *L))
4581 continue;
4582
4583 // Use the last (most dominated) store's location for the unconditional
4584 // store.
4585 VPReplicateRecipe *LastStore = Group.back();
4586 VPBasicBlock *InsertBB = LastStore->getParent();
4587
4588 // Collect common alias metadata from all stores in the group.
4589 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
4590
4591 // Build select chain for stored values.
4592 VPValue *SelectedValue = Group[0]->getOperand(N: 0);
4593 VPBuilder Builder(InsertBB, LastStore->getIterator());
4594
4595 bool IsSingleScalar = Group[0]->isSingleScalar();
4596 for (unsigned I = 1; I < Group.size(); ++I) {
4597 assert(IsSingleScalar == Group[I]->isSingleScalar() &&
4598 "all members in group must agree on IsSingleScalar");
4599 VPValue *Mask = Group[I]->getMask();
4600 VPValue *Value = Group[I]->getOperand(N: 0);
4601 SelectedValue = Builder.createSelect(Cond: Mask, TrueVal: Value, FalseVal: SelectedValue,
4602 DL: Group[I]->getDebugLoc());
4603 }
4604
4605 // Find the store with minimum alignment to use.
4606 auto *StoreWithMinAlign = findRecipeWithMinAlign<StoreInst>(Group);
4607
4608 // Create unconditional store with selected value and common metadata.
4609 auto *UnpredicatedStore = new VPReplicateRecipe(
4610 StoreWithMinAlign->getUnderlyingInstr(),
4611 {SelectedValue, LastStore->getOperand(N: 1)}, IsSingleScalar,
4612 /*Mask=*/nullptr, *LastStore, CommonMetadata);
4613 UnpredicatedStore->insertBefore(BB&: *InsertBB, IP: LastStore->getIterator());
4614
4615 // Remove all predicated stores from the group.
4616 for (VPReplicateRecipe *Store : Group)
4617 Store->eraseFromParent();
4618 }
4619}
4620
4621void VPlanTransforms::materializeConstantVectorTripCount(
4622 VPlan &Plan, ElementCount BestVF, unsigned BestUF,
4623 PredicatedScalarEvolution &PSE) {
4624 assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
4625 assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
4626
4627 VPValue *TC = Plan.getTripCount();
4628 if (TC->user_empty())
4629 return;
4630
4631 // Skip cases for which the trip count may be non-trivial to materialize.
4632 // I.e., when a scalar tail is absent - due to tail folding, or when a scalar
4633 // tail is required.
4634 if (Plan.hasTailFolded() || !Plan.hasScalarTail() ||
4635 Plan.getMiddleBlock()->getSingleSuccessor() ==
4636 Plan.getScalarPreheader() ||
4637 !isa<VPIRValue>(Val: TC))
4638 return;
4639
4640 // Materialize vector trip counts for constants early if it can simply
4641 // be computed as (Original TC / VF * UF) * VF * UF.
4642 // TODO: Compute vector trip counts for loops requiring a scalar epilogue and
4643 // tail-folded loops.
4644 ScalarEvolution &SE = *PSE.getSE();
4645 auto *TCScev = SE.getSCEV(V: TC->getLiveInIRValue());
4646 if (!isa<SCEVConstant>(Val: TCScev))
4647 return;
4648 const SCEV *VFxUF = SE.getElementCount(Ty: TCScev->getType(), EC: BestVF * BestUF);
4649 auto VecTCScev = SE.getMulExpr(LHS: SE.getUDivExpr(LHS: TCScev, RHS: VFxUF), RHS: VFxUF);
4650 if (auto *ConstVecTC = dyn_cast<SCEVConstant>(Val: VecTCScev))
4651 Plan.getVectorTripCount().setUnderlyingValue(ConstVecTC->getValue());
4652}
4653
4654void VPlanTransforms::materializeBackedgeTakenCount(VPlan &Plan,
4655 VPBasicBlock *VectorPH) {
4656 VPValue *BTC = Plan.getOrCreateBackedgeTakenCount();
4657 if (BTC->user_empty())
4658 return;
4659
4660 VPBuilder Builder(VectorPH, VectorPH->begin());
4661 auto *TCTy = Plan.getTripCount()->getScalarType();
4662 auto *TCMO =
4663 Builder.createSub(LHS: Plan.getTripCount(), RHS: Plan.getConstantInt(Ty: TCTy, Val: 1),
4664 DL: DebugLoc::getCompilerGenerated(), Name: "trip.count.minus.1");
4665 BTC->replaceAllUsesWith(New: TCMO);
4666}
4667
4668void VPlanTransforms::materializePacksAndUnpacks(VPlan &Plan) {
4669 if (Plan.hasScalarVFOnly())
4670 return;
4671
4672 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
4673 auto VPBBsOutsideLoopRegion = VPBlockUtils::blocksOnly<VPBasicBlock>(
4674 Range: vp_depth_first_shallow(G: Plan.getEntry()));
4675 auto VPBBsInsideLoopRegion = VPBlockUtils::blocksOnly<VPBasicBlock>(
4676 Range: vp_depth_first_shallow(G: LoopRegion->getEntry()));
4677 // Materialize Build(Struct)Vector for all replicating VPReplicateRecipes,
4678 // VPScalarIVStepsRecipe and VPInstructions, excluding ones in replicate
4679 // regions. Those are not materialized explicitly yet.
4680 // TODO: materialize build vectors for replicating recipes in replicating
4681 // regions.
4682 for (VPBasicBlock *VPBB :
4683 concat<VPBasicBlock *>(Ranges&: VPBBsOutsideLoopRegion, Ranges&: VPBBsInsideLoopRegion)) {
4684 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
4685 if (!vputils::doesGeneratePerAllLanes(R: &R))
4686 continue;
4687 auto *DefR = cast<VPSingleDefRecipe>(Val: &R);
4688 auto UsesVectorOrInsideReplicateRegion = [DefR, LoopRegion](VPUser *U) {
4689 VPRegionBlock *ParentRegion = cast<VPRecipeBase>(Val: U)->getRegion();
4690 return !U->usesScalars(Op: DefR) || ParentRegion != LoopRegion;
4691 };
4692 if (none_of(Range: DefR->users(), P: UsesVectorOrInsideReplicateRegion))
4693 continue;
4694
4695 Type *ScalarTy = DefR->getScalarType();
4696 unsigned Opcode = ScalarTy->isStructTy()
4697 ? VPInstruction::BuildStructVector
4698 : VPInstruction::BuildVector;
4699 auto *BuildVector = new VPInstruction(Opcode, {DefR});
4700 BuildVector->insertAfter(InsertPos: DefR);
4701
4702 DefR->replaceUsesWithIf(
4703 New: BuildVector, ShouldReplace: [BuildVector, &UsesVectorOrInsideReplicateRegion](
4704 VPUser &U, unsigned) {
4705 return &U != BuildVector && UsesVectorOrInsideReplicateRegion(&U);
4706 });
4707 }
4708 }
4709
4710 // Create explicit VPInstructions to convert vectors to scalars. The current
4711 // implementation is conservative - it may miss some cases that may or may not
4712 // be vector values. TODO: introduce Unpacks speculatively - remove them later
4713 // if they are known to operate on scalar values.
4714 for (VPBasicBlock *VPBB : VPBBsInsideLoopRegion) {
4715 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
4716 if (isa<VPReplicateRecipe, VPInstruction, VPScalarIVStepsRecipe,
4717 VPDerivedIVRecipe>(Val: &R))
4718 continue;
4719 for (VPValue *Def : R.definedValues()) {
4720 // Skip recipes that are single-scalar.
4721 // TODO: The Defs skipped here may or may not be vector values.
4722 // Introduce Unpacks, and remove them later, if they are guaranteed to
4723 // produce scalar values.
4724 if (vputils::isSingleScalar(VPV: Def))
4725 continue;
4726
4727 // Only introduce an Unpack if some, but not all, users use the first
4728 // lane only.
4729 unsigned NumFirstLaneUsers = count_if(Range: Def->users(), P: [&Def](VPUser *U) {
4730 return U->usesFirstLaneOnly(Op: Def);
4731 });
4732 if (!NumFirstLaneUsers || NumFirstLaneUsers == Def->getNumUsers())
4733 continue;
4734
4735 auto *Unpack = new VPInstruction(VPInstruction::Unpack, {Def});
4736 if (R.isPhi())
4737 Unpack->insertBefore(BB&: *VPBB, IP: VPBB->getFirstNonPhi());
4738 else
4739 Unpack->insertAfter(InsertPos: &R);
4740 Def->replaceUsesWithIf(New: Unpack, ShouldReplace: [&Def](VPUser &U, unsigned) {
4741 return U.usesFirstLaneOnly(Op: Def);
4742 });
4743 }
4744 }
4745 }
4746}
4747
4748void VPlanTransforms::materializeVectorTripCount(
4749 VPlan &Plan, VPBasicBlock *VectorPHVPBB, bool TailByMasking,
4750 bool RequiresScalarEpilogue, VPValue *Step,
4751 std::optional<uint64_t> MaxRuntimeStep) {
4752 VPSymbolicValue &VectorTC = Plan.getVectorTripCount();
4753 // There's nothing to do if there are no users of the vector trip count or its
4754 // IR value has already been set.
4755 if (VectorTC.user_empty() || VectorTC.getUnderlyingValue())
4756 return;
4757
4758 VPValue *TC = Plan.getTripCount();
4759 Type *TCTy = TC->getScalarType();
4760 VPBasicBlock::iterator InsertPt = VectorPHVPBB->begin();
4761 if (auto *StepR = Step->getDefiningRecipe()) {
4762 assert(VPDominatorTree(Plan).dominates(StepR->getParent(), VectorPHVPBB) &&
4763 "Step VPBB must dominate VectorPHVPBB");
4764 // Insert after Step's definition to maintain valid def-use ordering.
4765 InsertPt = std::next(x: StepR->getIterator());
4766 }
4767 VPBuilder Builder(VectorPHVPBB, InsertPt);
4768
4769 // For scalable steps, if TC is a constant and is divisible by the maximum
4770 // possible runtime step, then TC % Step == 0 for all valid vscale values
4771 // and the vector trip count equals TC directly.
4772 const APInt *TCVal;
4773 if (!RequiresScalarEpilogue && match(V: TC, P: m_APInt(C&: TCVal)) && MaxRuntimeStep &&
4774 TCVal->urem(RHS: *MaxRuntimeStep) == 0) {
4775 VectorTC.replaceAllUsesWith(New: TC);
4776 return;
4777 }
4778
4779 // If the tail is to be folded by masking, round the number of iterations N
4780 // up to a multiple of Step instead of rounding down. This is done by first
4781 // adding Step-1 and then rounding down. Note that it's ok if this addition
4782 // overflows: the vector induction variable will eventually wrap to zero given
4783 // that it starts at zero and its Step is a power of two; the loop will then
4784 // exit, with the last early-exit vector comparison also producing all-true.
4785 if (TailByMasking) {
4786 TC = Builder.createAdd(
4787 LHS: TC, RHS: Builder.createSub(LHS: Step, RHS: Plan.getConstantInt(Ty: TCTy, Val: 1)),
4788 DL: DebugLoc::getCompilerGenerated(), Name: "n.rnd.up");
4789 }
4790
4791 // Now we need to generate the expression for the part of the loop that the
4792 // vectorized body will execute. This is equal to N - (N % Step) if scalar
4793 // iterations are not required for correctness, or N - Step, otherwise. Step
4794 // is equal to the vectorization factor (number of SIMD elements) times the
4795 // unroll factor (number of SIMD instructions).
4796 VPValue *R =
4797 Builder.createNaryOp(Opcode: Instruction::URem, Operands: {TC, Step},
4798 DL: DebugLoc::getCompilerGenerated(), Name: "n.mod.vf");
4799
4800 // There are cases where we *must* run at least one iteration in the remainder
4801 // loop. See the cost model for when this can happen. If the step evenly
4802 // divides the trip count, we set the remainder to be equal to the step. If
4803 // the step does not evenly divide the trip count, no adjustment is necessary
4804 // since there will already be scalar iterations. Note that the minimum
4805 // iterations check ensures that N >= Step.
4806 if (RequiresScalarEpilogue) {
4807 assert(!TailByMasking &&
4808 "requiring scalar epilogue is not supported with fail folding");
4809 VPValue *IsZero =
4810 Builder.createICmp(Pred: CmpInst::ICMP_EQ, A: R, B: Plan.getZero(Ty: TCTy));
4811 R = Builder.createSelect(Cond: IsZero, TrueVal: Step, FalseVal: R);
4812 }
4813
4814 VPValue *Res =
4815 Builder.createSub(LHS: TC, RHS: R, DL: DebugLoc::getCompilerGenerated(), Name: "n.vec");
4816 VectorTC.replaceAllUsesWith(New: Res);
4817}
4818
4819void VPlanTransforms::materializeFactors(VPlan &Plan, VPBasicBlock *VectorPH,
4820 ElementCount VFEC) {
4821 // If VF and VFxUF have already been materialized (no remaining users),
4822 // there's nothing more to do.
4823 if (Plan.getVF().isMaterialized()) {
4824 assert(Plan.getVFxUF().isMaterialized() &&
4825 "VF and VFxUF must be materialized together");
4826 return;
4827 }
4828
4829 VPBuilder Builder(VectorPH, VectorPH->begin());
4830 Type *TCTy = Plan.getTripCount()->getScalarType();
4831 VPValue &VF = Plan.getVF();
4832 VPValue &VFxUF = Plan.getVFxUF();
4833 // If there are no users of the runtime VF, compute VFxUF by constant folding
4834 // the multiplication of VF and UF.
4835 if (VF.user_empty()) {
4836 VPValue *RuntimeVFxUF =
4837 Builder.createElementCount(Ty: TCTy, EC: VFEC * Plan.getConcreteUF());
4838 VFxUF.replaceAllUsesWith(New: RuntimeVFxUF);
4839 return;
4840 }
4841
4842 // For users of the runtime VF, compute it as VF * vscale, and VFxUF as (VF *
4843 // vscale) * UF.
4844 VPValue *RuntimeVF = Builder.createElementCount(Ty: TCTy, EC: VFEC);
4845 if (!vputils::onlyScalarValuesUsed(Def: &VF)) {
4846 VPValue *BC = Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: RuntimeVF);
4847 VF.replaceUsesWithIf(
4848 New: BC, ShouldReplace: [&VF](VPUser &U, unsigned) { return !U.usesScalars(Op: &VF); });
4849 }
4850 VF.replaceAllUsesWith(New: RuntimeVF);
4851
4852 VPValue *MulByUF = Builder.createOverflowingOp(
4853 Opcode: Instruction::Mul,
4854 Operands: {RuntimeVF, Plan.getConstantInt(Ty: TCTy, Val: Plan.getConcreteUF())},
4855 WrapFlags: {true, false});
4856 VFxUF.replaceAllUsesWith(New: MulByUF);
4857}
4858
4859void VPlanTransforms::attachAliasMaskToHeaderMask(VPlan &Plan) {
4860 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
4861 VPValue *HeaderMask = LoopRegion->getHeaderMask();
4862 Type *I1Ty = IntegerType::getInt1Ty(C&: Plan.getContext());
4863
4864 VPBuilder Builder(Plan.getVectorPreheader());
4865 auto *AliasMask = Builder.createNaryOp(
4866 Opcode: VPInstruction::IncomingAliasMask, Operands: {}, Inst: nullptr, Flags: {}, MD: {},
4867 DL: DebugLoc::getUnknown(), Name: "incoming.alias.mask", ResultTy: I1Ty);
4868
4869 VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
4870 Builder = VPBuilder(Header, Header->getFirstNonPhi());
4871
4872 // Update all existing users of the header mask to "HeaderMask & AliasMask".
4873 auto *ClampedHeaderMask = Builder.createAnd(LHS: HeaderMask, RHS: AliasMask);
4874 HeaderMask->replaceUsesWithIf(New: ClampedHeaderMask, ShouldReplace: [&](VPUser &U, unsigned) {
4875 return &U != ClampedHeaderMask;
4876 });
4877}
4878
4879VPValue *
4880VPlanTransforms::materializeAliasMask(VPlan &Plan, VPBasicBlock *AliasCheckVPBB,
4881 ArrayRef<PointerDiffInfo> DiffChecks) {
4882 VPBuilder Builder(AliasCheckVPBB);
4883 Type *I1Ty = IntegerType::getInt1Ty(C&: Plan.getContext());
4884
4885 VPValue *IncomingAliasMask = vputils::findIncomingAliasMask(Plan);
4886 assert(IncomingAliasMask && "Expected an alias mask!");
4887
4888 VPValue *AliasMask = nullptr;
4889 for (const PointerDiffInfo &Check : DiffChecks) {
4890 VPValue *Src = vputils::getOrCreateVPValueForSCEVExpr(Plan, Expr: Check.SrcStart);
4891 VPValue *Sink =
4892 vputils::getOrCreateVPValueForSCEVExpr(Plan, Expr: Check.SinkStart);
4893 Type *AddrType = Src->getScalarType();
4894
4895 // TODO: Only freeze the required pointer (not both src and sink).
4896 if (Check.NeedsFreeze) {
4897 Src = Builder.createScalarFreeze(Op: Src, ResultTy: AddrType, DL: DebugLoc::getUnknown());
4898 Sink = Builder.createScalarFreeze(Op: Sink, ResultTy: AddrType, DL: DebugLoc::getUnknown());
4899 }
4900
4901 // TODO: Generate loop_dependence_raw_mask when there's a read-after-write
4902 // dependency between the source and the sink. This is not necessary for
4903 // correctness of the mask, but using the "raw" variant prevents loads
4904 // depending on the completion of stores.
4905 VPWidenIntrinsicRecipe *WARMask = Builder.insert(R: new VPWidenIntrinsicRecipe(
4906 Intrinsic::loop_dependence_war_mask,
4907 {Src, Sink, Plan.getConstantInt(Ty: AddrType, Val: Check.AccessSize)}, I1Ty));
4908
4909 if (AliasMask)
4910 AliasMask = Builder.createAnd(LHS: AliasMask, RHS: WARMask);
4911 else
4912 AliasMask = WARMask;
4913 }
4914
4915 Type *IVTy = Plan.getVectorLoopRegion()->getCanonicalIVType();
4916 Type *IndexTy = Plan.getDataLayout().getIndexType(C&: Plan.getContext(), AddressSpace: 0);
4917 VPValue *NumActive = Builder.createNaryOp(
4918 Opcode: VPInstruction::NumActiveLanes, Operands: {AliasMask}, Inst: nullptr, Flags: {}, MD: {},
4919 DL: DebugLoc::getUnknown(), Name: "num.active.lanes", ResultTy: IndexTy);
4920 VPValue *ClampedVF = Builder.createScalarZExtOrTrunc(
4921 Op: NumActive, ResultTy: IVTy, DL: DebugLoc::getCompilerGenerated());
4922
4923 IncomingAliasMask->replaceAllUsesWith(New: AliasMask);
4924
4925 return ClampedVF;
4926}
4927
4928void VPlanTransforms::materializeAliasMaskCheckBlock(
4929 VPlan &Plan, ArrayRef<PointerDiffInfo> DiffChecks, bool HasBranchWeights) {
4930 VPBasicBlock *ClampedVFCheck =
4931 Plan.createVPBasicBlock(Name: "vector.clamped.vf.check");
4932
4933 VPValue *ClampedVF = materializeAliasMask(Plan, AliasCheckVPBB: ClampedVFCheck, DiffChecks);
4934 VPBuilder Builder(ClampedVFCheck);
4935 DebugLoc DL = DebugLoc::getCompilerGenerated();
4936 Type *TCTy = Plan.getTripCount()->getScalarType();
4937
4938 // Check the "ClampedVF" from the alias mask is larger than one.
4939 VPValue *IsScalar =
4940 Builder.createICmp(Pred: CmpInst::ICMP_ULE, A: ClampedVF,
4941 B: Plan.getConstantInt(Ty: TCTy, Val: 1), DL, Name: "vf.is.scalar");
4942
4943 VPValue *TripCount = Plan.getTripCount();
4944 VPValue *MaxUIntTripCount =
4945 Plan.getConstantInt(Val: cast<IntegerType>(Val: TCTy)->getMask());
4946 VPValue *DistanceToMax = Builder.createSub(LHS: MaxUIntTripCount, RHS: TripCount);
4947
4948 // For tail-folding: Don't execute the vector loop if (UMax - n) < ClampedVF.
4949 // Note: The ClampedVF may not be a power-of-two. This means the loop exit
4950 // condition (index.next == n.vec) may not be correct in the case of an
4951 // overflow. The issue is `n.vec` could be zero due to an overflow, but
4952 // index.next is not guaranteed to overflow to zero as the ClampedVF is not a
4953 // power-of-two).
4954 VPValue *TripCountCheck = Builder.createICmp(
4955 Pred: ICmpInst::ICMP_ULT, A: DistanceToMax, B: ClampedVF, DL, Name: "vf.step.overflow");
4956
4957 VPValue *Cond = Builder.createOr(LHS: IsScalar, RHS: TripCountCheck, DL);
4958 attachVPCheckBlock(Plan, Cond, CheckBlock: ClampedVFCheck, AddBranchWeights: HasBranchWeights);
4959
4960 // Materialize the trip count early as this will add a use of (VFxUF) that
4961 // needs to be replaced with the ClampedVF.
4962 materializeVectorTripCount(Plan, VectorPHVPBB: Plan.getVectorPreheader(),
4963 /*TailByMasking=*/true,
4964 /*RequiresScalarEpilogue=*/false,
4965 Step: &Plan.getVFxUF());
4966
4967 assert(Plan.getConcreteUF() == 1 &&
4968 "Clamped VF not supported with interleaving");
4969 Plan.getVF().replaceAllUsesWith(New: ClampedVF);
4970 Plan.getVFxUF().replaceAllUsesWith(New: ClampedVF);
4971}
4972
4973void VPlanTransforms::expandSCEVsToVPInstructions(VPlan &Plan,
4974 ScalarEvolution &SE) {
4975 auto *Entry = Plan.getEntry();
4976 VPBuilder Builder(Entry, Entry->begin());
4977 DebugLoc DL = cast<VPIRBasicBlock>(Val: Entry)
4978 ->getIRBasicBlock()
4979 ->getTerminator()
4980 ->getDebugLoc();
4981 VPSCEVExpander Expander(Builder, SE, DL);
4982
4983 // Expand VPExpandSCEVRecipes to VPInstructions using VPSCEVExpander. During
4984 // the transition, unsupported VPExpandSCEVRecipes are skipped and left for
4985 // late expansion.
4986 for (VPRecipeBase &R : make_early_inc_range(Range&: *Entry)) {
4987 auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(Val: &R);
4988 if (!ExpSCEV || ExpSCEV->user_empty())
4989 continue;
4990 Builder.setInsertPoint(ExpSCEV);
4991 VPValue *Expanded = Expander.tryToExpand(S: ExpSCEV->getSCEV());
4992 if (!Expanded)
4993 continue;
4994 ExpSCEV->replaceAllUsesWith(New: Expanded);
4995 // TripCount should not be used after expansion to VPInstructions. Reset to
4996 // poison to avoid dangling references.
4997 if (Plan.getTripCount() == ExpSCEV)
4998 Plan.resetTripCount(NewTripCount: Plan.getPoison(Ty: ExpSCEV->getScalarType()));
4999 ExpSCEV->eraseFromParent();
5000 }
5001}
5002
5003DenseMap<const SCEV *, Value *>
5004VPlanTransforms::expandSCEVs(VPlan &Plan, ScalarEvolution &SE) {
5005 SCEVExpander Expander(SE, "induction", /*PreserveLCSSA=*/false);
5006
5007 auto *Entry = cast<VPIRBasicBlock>(Val: Plan.getEntry());
5008 BasicBlock *EntryBB = Entry->getIRBasicBlock();
5009 DenseMap<const SCEV *, Value *> ExpandedSCEVs;
5010 // Expand remaining VPExpandSCEVRecipes to IR instructions using SCEVExpander.
5011 for (VPRecipeBase &R : make_early_inc_range(Range&: *Entry)) {
5012 auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(Val: &R);
5013 if (!ExpSCEV)
5014 continue;
5015 const SCEV *Expr = ExpSCEV->getSCEV();
5016 Value *Res =
5017 Expander.expandCodeFor(SH: Expr, Ty: Expr->getType(), I: EntryBB->getTerminator());
5018 ExpandedSCEVs[Expr] = Res;
5019 VPValue *Exp = Plan.getOrAddLiveIn(V: Res);
5020 ExpSCEV->replaceAllUsesWith(New: Exp);
5021 if (Plan.getTripCount() == ExpSCEV)
5022 Plan.resetTripCount(NewTripCount: Exp);
5023 ExpSCEV->eraseFromParent();
5024 }
5025 assert(none_of(*Entry, IsaPred<VPExpandSCEVRecipe>) &&
5026 "all VPExpandSCEVRecipes must have been expanded");
5027 // Add IR instructions in the entry basic block but not in the VPIRBasicBlock
5028 // to the VPIRBasicBlock.
5029 auto EI = Entry->begin();
5030 for (Instruction &I : drop_end(RangeOrContainer&: *EntryBB)) {
5031 if (EI != Entry->end() && isa<VPIRInstruction>(Val: *EI) &&
5032 &cast<VPIRInstruction>(Val: &*EI)->getInstruction() == &I) {
5033 EI++;
5034 continue;
5035 }
5036 VPIRInstruction::create(I)->insertBefore(BB&: *Entry, IP: EI);
5037 }
5038
5039 return ExpandedSCEVs;
5040}
5041
5042/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
5043/// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
5044/// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
5045/// the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to
5046/// an index-independent load if it feeds all wide ops at all indices (\p OpV
5047/// must be the operand at index \p OpIdx for both the recipe at lane 0, \p
5048/// WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if \p V
5049/// is defined at \p Idx of a load interleave group.
5050/// A live-in or recipe defined outside the loop region can be converted, if it
5051/// is the same across all lanes, or we can create a BuildVector for it.
5052static bool canNarrowLoad(VPSingleDefRecipe *WideMember0, unsigned OpIdx,
5053 VPValue *OpV, unsigned Idx, bool IsScalable) {
5054 VPValue *Member0Op = WideMember0->getOperand(N: OpIdx);
5055 if (Member0Op->isDefinedOutsideLoopRegions()) {
5056 // Operand matches Member0, broadcast across all fields for both live-ins
5057 // and recipes.
5058 if (Member0Op == OpV)
5059 return true;
5060 // Otherwise distinct per-field VPValues are assembled into a BuildVector.
5061 return !IsScalable && OpV->isDefinedOutsideLoopRegions() &&
5062 OpV->getScalarType() == Member0Op->getScalarType();
5063 }
5064 VPRecipeBase *Member0OpR = Member0Op->getDefiningRecipe();
5065 if (auto *W = dyn_cast<VPWidenLoadRecipe>(Val: Member0OpR))
5066 // For scalable VFs, the narrowed plan processes vscale iterations at once,
5067 // so a shared wide load cannot be narrowed to a uniform scalar; bail out.
5068 return !IsScalable && !W->getMask() && W->isConsecutive() &&
5069 Member0Op == OpV;
5070 if (auto *IR = dyn_cast<VPInterleaveRecipe>(Val: Member0OpR))
5071 return IR->getInterleaveGroup()->isFull() && IR->getVPValue(I: Idx) == OpV;
5072 return false;
5073}
5074
5075static bool canNarrowOps(ArrayRef<VPValue *> Ops, bool IsScalable) {
5076 SmallVector<VPValue *> Ops0;
5077 auto *WideMember0 = dyn_cast<VPRecipeWithIRFlags>(Val: Ops[0]);
5078 if (!WideMember0)
5079 return false;
5080 for (VPValue *V : Ops) {
5081 if (!isa<VPWidenRecipe, VPWidenCastRecipe>(Val: V))
5082 return false;
5083 auto *R = cast<VPRecipeWithIRFlags>(Val: V);
5084 if (vputils::getOpcode(V: R) != vputils::getOpcode(V: WideMember0))
5085 return false;
5086 if (R->getScalarType() != WideMember0->getScalarType())
5087 return false;
5088 if (R->hasPredicate() && R->getPredicate() != WideMember0->getPredicate())
5089 return false;
5090 }
5091
5092 for (unsigned Idx = 0; Idx != WideMember0->getNumOperands(); ++Idx) {
5093 SmallVector<VPValue *> OpsI;
5094 for (VPValue *Op : Ops)
5095 OpsI.push_back(Elt: Op->getDefiningRecipe()->getOperand(N: Idx));
5096
5097 if (canNarrowOps(Ops: OpsI, IsScalable))
5098 continue;
5099
5100 if (any_of(Range: enumerate(First&: OpsI), P: [WideMember0, Idx, IsScalable](const auto &P) {
5101 const auto &[OpIdx, OpV] = P;
5102 return !canNarrowLoad(WideMember0, Idx, OpV, OpIdx, IsScalable);
5103 }))
5104 return false;
5105 }
5106
5107 return true;
5108}
5109
5110/// Returns VF from \p VFs if \p IR is a full interleave group with factor and
5111/// number of members both equal to VF. The interleave group must also access
5112/// the full vector width.
5113static std::optional<ElementCount>
5114isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR,
5115 ArrayRef<ElementCount> VFs,
5116 const TargetTransformInfo &TTI) {
5117 if (!InterleaveR || InterleaveR->getMask())
5118 return std::nullopt;
5119
5120 Type *GroupElementTy = nullptr;
5121 if (InterleaveR->getStoredValues().empty()) {
5122 GroupElementTy = InterleaveR->getVPValue(I: 0)->getScalarType();
5123 if (!all_of(Range: InterleaveR->definedValues(), P: [GroupElementTy](VPValue *Op) {
5124 return Op->getScalarType() == GroupElementTy;
5125 }))
5126 return std::nullopt;
5127 } else {
5128 GroupElementTy = InterleaveR->getStoredValues()[0]->getScalarType();
5129 if (!all_of(Range: InterleaveR->getStoredValues(), P: [GroupElementTy](VPValue *Op) {
5130 return Op->getScalarType() == GroupElementTy;
5131 }))
5132 return std::nullopt;
5133 }
5134
5135 auto IG = InterleaveR->getInterleaveGroup();
5136 if (IG->getFactor() != IG->getNumMembers())
5137 return std::nullopt;
5138
5139 auto GetVectorBitWidthForVF = [&TTI](ElementCount VF) {
5140 TypeSize Size = TTI.getRegisterBitWidth(
5141 K: VF.isFixed() ? TargetTransformInfo::RGK_FixedWidthVector
5142 : TargetTransformInfo::RGK_ScalableVector);
5143 assert(Size.isScalable() == VF.isScalable() &&
5144 "if Size is scalable, VF must be scalable and vice versa");
5145 return Size.getKnownMinValue();
5146 };
5147
5148 for (ElementCount VF : VFs) {
5149 unsigned MinVal = VF.getKnownMinValue();
5150 unsigned GroupSize = GroupElementTy->getScalarSizeInBits() * MinVal;
5151 if (IG->getFactor() == MinVal && GroupSize == GetVectorBitWidthForVF(VF))
5152 return {VF};
5153 }
5154 return std::nullopt;
5155}
5156
5157/// Returns true if \p VPValue is a narrow VPValue.
5158static bool isAlreadyNarrow(VPValue *VPV) {
5159 if (isa<VPIRValue>(Val: VPV))
5160 return true;
5161 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: VPV);
5162 return RepR && RepR->isSingleScalar();
5163}
5164
5165// Convert the wide recipes defining the VPValues in \p Members feeding an
5166// interleave group to a single narrow variant. The first member is reused as
5167// the narrowed recipe. BuildVectors for live-in operands are inserted into \p
5168// Preheader.
5169static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
5170 SmallPtrSetImpl<VPValue *> &NarrowedOps,
5171 VPBasicBlock *Preheader) {
5172 VPValue *V = Members.front();
5173 if (NarrowedOps.contains(Ptr: V))
5174 return V;
5175
5176 if (V->isDefinedOutsideLoopRegions()) {
5177 assert(all_of(Members,
5178 [V](VPValue *M) {
5179 return M->isDefinedOutsideLoopRegions() &&
5180 M->getScalarType() == V->getScalarType();
5181 }) &&
5182 "expected distinct loop-invariant values of matching scalar type");
5183 auto *BV = new VPInstruction(VPInstruction::BuildVector, Members);
5184 Preheader->appendRecipe(Recipe: BV);
5185 NarrowedOps.insert(Ptr: BV);
5186 return BV;
5187 }
5188
5189 if (isAlreadyNarrow(VPV: V))
5190 return V;
5191
5192 VPRecipeBase *R = V->getDefiningRecipe();
5193 if (isa<VPWidenRecipe, VPWidenCastRecipe>(Val: R)) {
5194 auto *WideMember0 = cast<VPRecipeWithIRFlags>(Val: R);
5195 for (VPValue *Member : Members.drop_front())
5196 WideMember0->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Member));
5197 for (unsigned Idx = 0, E = WideMember0->getNumOperands(); Idx != E; ++Idx) {
5198 SmallVector<VPValue *> OpsI;
5199 for (VPValue *Member : Members)
5200 OpsI.push_back(Elt: Member->getDefiningRecipe()->getOperand(N: Idx));
5201 WideMember0->setOperand(
5202 I: Idx, New: narrowInterleaveGroupOp(Members: OpsI, NarrowedOps, Preheader));
5203 }
5204 return V;
5205 }
5206
5207 if (auto *LoadGroup = dyn_cast<VPInterleaveRecipe>(Val: R)) {
5208 // Narrow interleave group to wide load, as transformed VPlan will only
5209 // process one original iteration.
5210 auto *LI = cast<LoadInst>(Val: LoadGroup->getInterleaveGroup()->getInsertPos());
5211 auto *L = new VPWidenLoadRecipe(*LI, LoadGroup->getAddr(),
5212 LoadGroup->getMask(), /*Consecutive=*/true,
5213 *LoadGroup, LoadGroup->getDebugLoc());
5214 L->insertBefore(InsertPos: LoadGroup);
5215 NarrowedOps.insert(Ptr: L);
5216 return L;
5217 }
5218
5219 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R)) {
5220 assert(RepR->isSingleScalar() && RepR->getOpcode() == Instruction::Load &&
5221 "must be a single scalar load");
5222 NarrowedOps.insert(Ptr: RepR);
5223 return RepR;
5224 }
5225
5226 auto *WideLoad = cast<VPWidenLoadRecipe>(Val: R);
5227 VPValue *PtrOp = WideLoad->getAddr();
5228 if (auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(Val: PtrOp))
5229 PtrOp = VecPtr->getOperand(N: 0);
5230 // Narrow wide load to uniform scalar load, as transformed VPlan will only
5231 // process one original iteration.
5232 auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(), {PtrOp},
5233 /*IsUniform*/ true,
5234 /*Mask*/ nullptr, {}, *WideLoad);
5235 N->insertBefore(InsertPos: WideLoad);
5236 NarrowedOps.insert(Ptr: N);
5237 return N;
5238}
5239
5240std::unique_ptr<VPlan>
5241VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
5242 const TargetTransformInfo &TTI) {
5243 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
5244
5245 if (!VectorLoop)
5246 return nullptr;
5247
5248 // Only handle single-block loops for now.
5249 if (VectorLoop->getEntryBasicBlock() != VectorLoop->getExitingBasicBlock())
5250 return nullptr;
5251
5252 // Skip plans when we may not be able to properly narrow.
5253 VPBasicBlock *Exiting = VectorLoop->getExitingBasicBlock();
5254 if (!match(V: &Exiting->back(), P: m_BranchOnCount()))
5255 return nullptr;
5256
5257 assert(match(&Exiting->back(),
5258 m_BranchOnCount(m_Add(m_VPValue(), m_Specific(&Plan.getVFxUF())),
5259 m_Specific(&Plan.getVectorTripCount()))) &&
5260 "unexpected branch-on-count");
5261
5262 SmallVector<VPInterleaveRecipe *> StoreGroups;
5263 std::optional<ElementCount> VFToOptimize;
5264 for (auto &R : *VectorLoop->getEntryBasicBlock()) {
5265 if (isa<VPDerivedIVRecipe, VPScalarIVStepsRecipe>(Val: &R) &&
5266 vputils::onlyFirstLaneUsed(Def: cast<VPSingleDefRecipe>(Val: &R)))
5267 continue;
5268
5269 // Bail out on recipes not supported at the moment:
5270 // * phi recipes other than the canonical induction
5271 // * recipes writing to memory except interleave groups
5272 // Only support plans with a canonical induction phi.
5273 if (R.isPhi())
5274 return nullptr;
5275
5276 auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(Val: &R);
5277 if (R.mayWriteToMemory() && !InterleaveR)
5278 return nullptr;
5279
5280 // Bail out if any recipe defines a vector value used outside the
5281 // vector loop region.
5282 if (any_of(Range: R.definedValues(), P: [&](VPValue *V) {
5283 return any_of(Range: V->users(), P: [&](VPUser *U) {
5284 auto *UR = cast<VPRecipeBase>(Val: U);
5285 return UR->getParent()->getParent() != VectorLoop;
5286 });
5287 }))
5288 return nullptr;
5289
5290 // All other ops are allowed, but we reject uses that cannot be converted
5291 // when checking all allowed consumers (store interleave groups) below.
5292 if (!InterleaveR)
5293 continue;
5294
5295 // Try to find a single VF, where all interleave groups are consecutive and
5296 // saturate the full vector width. If we already have a candidate VF, check
5297 // if it is applicable for the current InterleaveR, otherwise look for a
5298 // suitable VF across the Plan's VFs.
5299 SmallVector<ElementCount> VFs =
5300 VFToOptimize ? SmallVector<ElementCount>({*VFToOptimize})
5301 : to_vector(Range: Plan.vectorFactors());
5302 std::optional<ElementCount> NarrowedVF =
5303 isConsecutiveInterleaveGroup(InterleaveR, VFs, TTI);
5304 if (!NarrowedVF || (VFToOptimize && NarrowedVF != VFToOptimize))
5305 return nullptr;
5306 VFToOptimize = NarrowedVF;
5307
5308 // Skip read interleave groups.
5309 if (InterleaveR->getStoredValues().empty())
5310 continue;
5311
5312 // Narrow interleave groups, if all operands are already matching narrow
5313 // ops.
5314 auto *Member0 = InterleaveR->getStoredValues()[0];
5315 if (isAlreadyNarrow(VPV: Member0) &&
5316 all_of(Range: InterleaveR->getStoredValues(), P: equal_to(Arg&: Member0))) {
5317 StoreGroups.push_back(Elt: InterleaveR);
5318 continue;
5319 }
5320
5321 // For now, we only support full interleave groups storing load interleave
5322 // groups.
5323 if (all_of(Range: enumerate(First: InterleaveR->getStoredValues()), P: [](auto Op) {
5324 VPRecipeBase *DefR = Op.value()->getDefiningRecipe();
5325 if (!DefR)
5326 return false;
5327 auto *IR = dyn_cast<VPInterleaveRecipe>(Val: DefR);
5328 return IR && IR->getInterleaveGroup()->isFull() &&
5329 IR->getVPValue(Op.index()) == Op.value();
5330 })) {
5331 StoreGroups.push_back(Elt: InterleaveR);
5332 continue;
5333 }
5334
5335 // Check if all values feeding InterleaveR are matching wide recipes, which
5336 // operands that can be narrowed.
5337 if (!canNarrowOps(Ops: InterleaveR->getStoredValues(),
5338 IsScalable: VFToOptimize->isScalable()))
5339 return nullptr;
5340 StoreGroups.push_back(Elt: InterleaveR);
5341 }
5342
5343 if (StoreGroups.empty())
5344 return nullptr;
5345
5346 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
5347 bool RequiresScalarEpilogue =
5348 MiddleVPBB->getNumSuccessors() == 1 &&
5349 MiddleVPBB->getSingleSuccessor() == Plan.getScalarPreheader();
5350 // Bail out for tail-folding (middle block with a single successor to exit).
5351 if (MiddleVPBB->getNumSuccessors() != 2 && !RequiresScalarEpilogue)
5352 return nullptr;
5353
5354 // All interleave groups in Plan can be narrowed for VFToOptimize. Split the
5355 // original Plan into 2: a) a new clone which contains all VFs of Plan, except
5356 // VFToOptimize, and b) the original Plan with VFToOptimize as single VF.
5357 // TODO: Handle cases where only some interleave groups can be narrowed.
5358 std::unique_ptr<VPlan> NewPlan;
5359 if (size(Range: Plan.vectorFactors()) != 1) {
5360 NewPlan = std::unique_ptr<VPlan>(Plan.duplicate());
5361 Plan.setVF(*VFToOptimize);
5362 NewPlan->removeVF(VF: *VFToOptimize);
5363 }
5364
5365 // Convert InterleaveGroup \p R to a single VPWidenLoadRecipe.
5366 SmallPtrSet<VPValue *, 4> NarrowedOps;
5367 VPBasicBlock *Preheader = Plan.getVectorPreheader();
5368 // Narrow operation tree rooted at store groups.
5369 for (auto *StoreGroup : StoreGroups) {
5370 VPValue *Res = narrowInterleaveGroupOp(Members: StoreGroup->getStoredValues(),
5371 NarrowedOps, Preheader);
5372 auto *SI =
5373 cast<StoreInst>(Val: StoreGroup->getInterleaveGroup()->getInsertPos());
5374 auto *S = new VPWidenStoreRecipe(*SI, StoreGroup->getAddr(), Res, nullptr,
5375 /*Consecutive=*/true, *StoreGroup,
5376 StoreGroup->getDebugLoc());
5377 S->insertBefore(InsertPos: StoreGroup);
5378 StoreGroup->eraseFromParent();
5379 }
5380
5381 // Adjust induction to reflect that the transformed plan only processes one
5382 // original iteration.
5383 VPInstruction *CanIVInc = vputils::findCanonicalIVIncrement(Plan);
5384 Type *CanIVTy = VectorLoop->getCanonicalIVType();
5385 VPBasicBlock *VectorPH = Plan.getVectorPreheader();
5386 VPBuilder PHBuilder(VectorPH, VectorPH->begin());
5387
5388 VPValue *UF = &Plan.getUF();
5389 VPValue *Step;
5390 if (VFToOptimize->isScalable()) {
5391 VPValue *VScale =
5392 PHBuilder.createElementCount(Ty: CanIVTy, EC: ElementCount::getScalable(MinVal: 1));
5393 Step = PHBuilder.createOverflowingOp(Opcode: Instruction::Mul, Operands: {VScale, UF},
5394 WrapFlags: {true, false});
5395 Plan.getVF().replaceAllUsesWith(New: VScale);
5396 } else {
5397 Step = UF;
5398 Plan.getVF().replaceAllUsesWith(New: Plan.getConstantInt(Ty: CanIVTy, Val: 1));
5399 }
5400 // Materialize vector trip count with the narrowed step.
5401 materializeVectorTripCount(Plan, VectorPHVPBB: VectorPH, /*TailByMasking=*/false,
5402 RequiresScalarEpilogue, Step);
5403
5404 CanIVInc->setOperand(I: 1, New: Step);
5405 Plan.getVFxUF().replaceAllUsesWith(New: Step);
5406
5407 removeDeadRecipes(Plan);
5408 assert(none_of(*VectorLoop->getEntryBasicBlock(),
5409 IsaPred<VPVectorPointerRecipe>) &&
5410 "All VPVectorPointerRecipes should have been removed");
5411 return NewPlan;
5412}
5413
5414/// Add branch weight metadata, if the \p Plan's middle block is terminated by a
5415/// BranchOnCond recipe.
5416void VPlanTransforms::addBranchWeightToMiddleTerminator(
5417 VPlan &Plan, ElementCount VF, std::optional<unsigned> VScaleForTuning) {
5418 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
5419 auto *MiddleTerm =
5420 dyn_cast_or_null<VPInstruction>(Val: MiddleVPBB->getTerminator());
5421 // Only add branch metadata if there is a (conditional) terminator.
5422 if (!MiddleTerm)
5423 return;
5424
5425 assert(MiddleTerm->getOpcode() == VPInstruction::BranchOnCond &&
5426 "must have a BranchOnCond");
5427 // Assume that `TripCount % VectorStep ` is equally distributed.
5428 unsigned VectorStep = Plan.getConcreteUF() * VF.getKnownMinValue();
5429 if (VF.isScalable() && VScaleForTuning.has_value())
5430 VectorStep *= *VScaleForTuning;
5431 assert(VectorStep > 0 && "trip count should not be zero");
5432 MDBuilder MDB(Plan.getContext());
5433 MDNode *BranchWeights =
5434 MDB.createBranchWeights(Weights: {1, VectorStep - 1}, /*IsExpected=*/false);
5435 MiddleTerm->setMetadata(Kind: LLVMContext::MD_prof, Node: BranchWeights);
5436}
5437
5438void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
5439 VFRange &Range) {
5440 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
5441 auto *MiddleVPBB = Plan.getMiddleBlock();
5442 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
5443
5444 auto IsScalableOne = [](ElementCount VF) -> bool {
5445 return VF == ElementCount::getScalable(MinVal: 1);
5446 };
5447
5448 for (auto &HeaderPhi : VectorRegion->getEntryBasicBlock()->phis()) {
5449 auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Val: &HeaderPhi);
5450 if (!FOR)
5451 continue;
5452
5453 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
5454 "Cannot handle loops with uncountable early exits");
5455
5456 // Find the existing splice for this FOR, created in
5457 // createHeaderPhiRecipes. All uses of FOR have already been replaced with
5458 // RecurSplice there; only RecurSplice itself still references FOR.
5459 auto *RecurSplice =
5460 findUserOf<VPInstruction::FirstOrderRecurrenceSplice>(V: FOR);
5461 assert(RecurSplice && "expected FirstOrderRecurrenceSplice");
5462
5463 // For VF vscale x 1, if vscale = 1, we are unable to extract the
5464 // penultimate value of the recurrence. Instead we rely on the existing
5465 // extract of the last element from the result of
5466 // VPInstruction::FirstOrderRecurrenceSplice.
5467 // TODO: Consider vscale_range info and UF.
5468 if (any_of(Range: RecurSplice->users(),
5469 P: [](VPUser *U) { return !cast<VPRecipeBase>(Val: U)->getRegion(); }) &&
5470 LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsScalableOne,
5471 Range))
5472 return;
5473
5474 // This is the second phase of vectorizing first-order recurrences, creating
5475 // extracts for users outside the loop. An overview of the transformation is
5476 // described below. Suppose we have the following loop with some use after
5477 // the loop of the last a[i-1],
5478 //
5479 // for (int i = 0; i < n; ++i) {
5480 // t = a[i - 1];
5481 // b[i] = a[i] - t;
5482 // }
5483 // use t;
5484 //
5485 // There is a first-order recurrence on "a". For this loop, the shorthand
5486 // scalar IR looks like:
5487 //
5488 // scalar.ph:
5489 // s.init = a[-1]
5490 // br scalar.body
5491 //
5492 // scalar.body:
5493 // i = phi [0, scalar.ph], [i+1, scalar.body]
5494 // s1 = phi [s.init, scalar.ph], [s2, scalar.body]
5495 // s2 = a[i]
5496 // b[i] = s2 - s1
5497 // br cond, scalar.body, exit.block
5498 //
5499 // exit.block:
5500 // use = lcssa.phi [s1, scalar.body]
5501 //
5502 // In this example, s1 is a recurrence because it's value depends on the
5503 // previous iteration. In the first phase of vectorization, we created a
5504 // VPFirstOrderRecurrencePHIRecipe v1 for s1. Now we create the extracts
5505 // for users in the scalar preheader and exit block.
5506 //
5507 // vector.ph:
5508 // v_init = vector(..., ..., ..., a[-1])
5509 // br vector.body
5510 //
5511 // vector.body
5512 // i = phi [0, vector.ph], [i+4, vector.body]
5513 // v1 = phi [v_init, vector.ph], [v2, vector.body]
5514 // v2 = a[i, i+1, i+2, i+3]
5515 // v1' = splice(v1(3), v2(0, 1, 2))
5516 // b[i, i+1, i+2, i+3] = v2 - v1'
5517 // br cond, vector.body, middle.block
5518 //
5519 // middle.block:
5520 // vector.recur.extract.for.phi = v2(2)
5521 // vector.recur.extract = v2(3)
5522 // br cond, scalar.ph, exit.block
5523 //
5524 // scalar.ph:
5525 // scalar.recur.init = phi [vector.recur.extract, middle.block],
5526 // [s.init, otherwise]
5527 // br scalar.body
5528 //
5529 // scalar.body:
5530 // i = phi [0, scalar.ph], [i+1, scalar.body]
5531 // s1 = phi [scalar.recur.init, scalar.ph], [s2, scalar.body]
5532 // s2 = a[i]
5533 // b[i] = s2 - s1
5534 // br cond, scalar.body, exit.block
5535 //
5536 // exit.block:
5537 // lo = lcssa.phi [s1, scalar.body],
5538 // [vector.recur.extract.for.phi, middle.block]
5539 //
5540 // Update extracts of the splice in the middle block: they extract the
5541 // penultimate element of the recurrence.
5542 for (VPRecipeBase &R : make_early_inc_range(
5543 Range: make_range(x: MiddleVPBB->getFirstNonPhi(), y: MiddleVPBB->end()))) {
5544 if (!match(V: &R, P: m_ExtractLastLaneOfLastPart(Op0: m_Specific(VPV: RecurSplice))))
5545 continue;
5546
5547 auto *ExtractR = cast<VPInstruction>(Val: &R);
5548 VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
5549 Opcode: VPInstruction::ExtractPenultimateElement, Operands: RecurSplice->getOperand(N: 1),
5550 DL: {}, Name: "vector.recur.extract.for.phi");
5551 for (VPUser *ExitU : to_vector(Range: ExtractR->users())) {
5552 if (auto *ExitPhi = dyn_cast<VPIRPhi>(Val: ExitU))
5553 ExitPhi->replaceUsesOfWith(From: ExtractR, To: PenultimateElement);
5554 }
5555 }
5556 }
5557}
5558
5559/// Check if \p V is a binary expression of a widened IV and a loop-invariant
5560/// value. Returns the widened IV if found, nullptr otherwise.
5561static VPWidenIntOrFpInductionRecipe *getExpressionIV(VPValue *V) {
5562 auto *BinOp = dyn_cast<VPWidenRecipe>(Val: V);
5563 if (!BinOp || !Instruction::isBinaryOp(Opcode: BinOp->getOpcode()) ||
5564 Instruction::isIntDivRem(Opcode: BinOp->getOpcode()))
5565 return nullptr;
5566
5567 VPValue *WidenIVCandidate = BinOp->getOperand(N: 0);
5568 VPValue *InvariantCandidate = BinOp->getOperand(N: 1);
5569 if (!isa<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate))
5570 std::swap(a&: WidenIVCandidate, b&: InvariantCandidate);
5571
5572 if (!InvariantCandidate->isDefinedOutsideLoopRegions())
5573 return nullptr;
5574
5575 return dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate);
5576}
5577
5578/// Create a scalar version of \p BinOp, with its \p WidenIV operand replaced
5579/// by \p ScalarIV, and place it after \p ScalarIV's defining recipe.
5580static VPValue *cloneBinOpForScalarIV(VPWidenRecipe *BinOp, VPValue *ScalarIV,
5581 VPWidenIntOrFpInductionRecipe *WidenIV) {
5582 assert(Instruction::isBinaryOp(BinOp->getOpcode()) &&
5583 BinOp->getNumOperands() == 2 && "BinOp must have 2 operands");
5584 auto *ClonedOp = BinOp->clone();
5585 if (ClonedOp->getOperand(N: 0) == WidenIV) {
5586 ClonedOp->setOperand(I: 0, New: ScalarIV);
5587 } else {
5588 assert(ClonedOp->getOperand(1) == WidenIV && "one operand must be WideIV");
5589 ClonedOp->setOperand(I: 1, New: ScalarIV);
5590 }
5591 ClonedOp->insertAfter(InsertPos: ScalarIV->getDefiningRecipe());
5592 return ClonedOp;
5593}
5594
5595void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
5596 PredicatedScalarEvolution &PSE,
5597 Loop &L) {
5598 ScalarEvolution &SE = *PSE.getSE();
5599 VPRegionBlock *VectorLoopRegion = Plan.getVectorLoopRegion();
5600
5601 // Helper lambda to check if the IV range excludes the sentinel value. Try
5602 // signed first, then unsigned. Return an excluded sentinel if found,
5603 // otherwise return std::nullopt.
5604 auto CheckSentinel = [&SE](const SCEV *IVSCEV,
5605 bool UseMax) -> std::optional<APSInt> {
5606 unsigned BW = IVSCEV->getType()->getScalarSizeInBits();
5607 for (bool Signed : {true, false}) {
5608 APSInt Sentinel = UseMax ? APSInt::getMinValue(numBits: BW, /*Unsigned=*/!Signed)
5609 : APSInt::getMaxValue(numBits: BW, /*Unsigned=*/!Signed);
5610
5611 ConstantRange IVRange =
5612 Signed ? SE.getSignedRange(S: IVSCEV) : SE.getUnsignedRange(S: IVSCEV);
5613 if (!IVRange.contains(Val: Sentinel))
5614 return Sentinel;
5615 }
5616 return std::nullopt;
5617 };
5618
5619 VPValue *HeaderMask = VectorLoopRegion->getHeaderMask();
5620 for (VPRecipeBase &Phi :
5621 make_early_inc_range(Range: VectorLoopRegion->getEntryBasicBlock()->phis())) {
5622 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &Phi);
5623 if (!PhiR || !RecurrenceDescriptor::isFindLastRecurrenceKind(
5624 Kind: PhiR->getRecurrenceKind()))
5625 continue;
5626
5627 Type *PhiTy = PhiR->getScalarType();
5628 if (PhiTy->isPointerTy() || PhiTy->isFloatingPointTy())
5629 continue;
5630
5631 // If there's a header mask, the backedge select will not be the find-last
5632 // select.
5633 VPValue *BackedgeVal = PhiR->getBackedgeValue();
5634 auto *FindLastSelect = cast<VPSingleDefRecipe>(Val: BackedgeVal);
5635 if (HeaderMask &&
5636 !match(V: BackedgeVal,
5637 P: m_Select(Op0: m_Specific(VPV: HeaderMask),
5638 Op1: m_VPSingleDefRecipe(V&: FindLastSelect), Op2: m_Specific(VPV: PhiR))))
5639 continue;
5640
5641 // Get the find-last expression from the find-last select of the reduction
5642 // phi. The find-last select should be a select between the phi and the
5643 // find-last expression.
5644 VPValue *Cond, *FindLastExpression;
5645 if (!match(R: FindLastSelect, P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: PhiR),
5646 Op2: m_VPValue(V&: FindLastExpression))) &&
5647 !match(R: FindLastSelect,
5648 P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_VPValue(V&: FindLastExpression),
5649 Op2: m_Specific(VPV: PhiR))))
5650 continue;
5651
5652 // Check if FindLastExpression is a simple expression of a widened IV. If
5653 // so, we can track the underlying IV instead and sink the expression.
5654 auto *IVOfExpressionToSink = getExpressionIV(V: FindLastExpression);
5655 const SCEV *IVSCEV = vputils::getSCEVExprForVPValue(
5656 V: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression, PSE,
5657 L: &L);
5658 const SCEV *Step;
5659 if (!match(S: IVSCEV, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV(V&: Step)))) {
5660 assert(!match(vputils::getSCEVExprForVPValue(FindLastExpression, PSE, &L),
5661 m_scev_AffineAddRec(m_SCEV(), m_SCEV())) &&
5662 "IVOfExpressionToSink not being an AddRec must imply "
5663 "FindLastExpression not being an AddRec.");
5664 continue;
5665 }
5666
5667 // Determine direction from SCEV step.
5668 if (!SE.isKnownNonZero(S: Step))
5669 continue;
5670
5671 // Positive step means we need UMax/SMax to find the last IV value, and
5672 // UMin/SMin otherwise.
5673 bool UseMax = SE.isKnownPositive(S: Step);
5674 std::optional<APSInt> SentinelVal = CheckSentinel(IVSCEV, UseMax);
5675 bool UseSigned = SentinelVal && SentinelVal->isSigned();
5676
5677 // Sinking an expression will disable epilogue vectorization. Only use it,
5678 // if FindLastExpression cannot be vectorized via a sentinel. Sinking may
5679 // also prevent vectorizing using a sentinel (e.g., if the expression is a
5680 // multiply or divide by large constant, respectively), which also makes
5681 // sinking undesirable.
5682 if (IVOfExpressionToSink) {
5683 const SCEV *FindLastExpressionSCEV =
5684 vputils::getSCEVExprForVPValue(V: FindLastExpression, PSE, L: &L);
5685 if (match(S: FindLastExpressionSCEV,
5686 P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV(V&: Step)))) {
5687 bool NewUseMax = SE.isKnownPositive(S: Step);
5688 if (auto NewSentinel =
5689 CheckSentinel(FindLastExpressionSCEV, NewUseMax)) {
5690 // The original expression already has a sentinel, so prefer not
5691 // sinking to keep epilogue vectorization possible.
5692 SentinelVal = *NewSentinel;
5693 UseSigned = NewSentinel->isSigned();
5694 UseMax = NewUseMax;
5695 IVSCEV = FindLastExpressionSCEV;
5696 IVOfExpressionToSink = nullptr;
5697 }
5698 }
5699 }
5700
5701 // If no sentinel was found, fall back to a boolean AnyOf reduction to track
5702 // if the condition was ever true. Requires the IV to not wrap, otherwise we
5703 // cannot use min/max.
5704 if (!SentinelVal) {
5705 auto *AR = cast<SCEVAddRecExpr>(Val: IVSCEV);
5706 if (AR->hasNoSignedWrap())
5707 UseSigned = true;
5708 else if (AR->hasNoUnsignedWrap())
5709 UseSigned = false;
5710 else
5711 continue;
5712 }
5713
5714 VPInstruction *RdxResult = cast<VPInstruction>(Val: vputils::findRecipe(
5715 Start: BackedgeVal,
5716 Pred: match_fn(P: m_VPInstruction<VPInstruction::ComputeReductionResult>())));
5717
5718 VPValue *NewFindLastSelect = BackedgeVal;
5719 VPValue *SelectCond = Cond;
5720 if (!SentinelVal || IVOfExpressionToSink) {
5721 // When we need to create a new select, normalize the condition so that
5722 // PhiR is the last operand and include the header mask if needed.
5723 DebugLoc DL = FindLastSelect->getDefiningRecipe()->getDebugLoc();
5724 VPBuilder LoopBuilder(FindLastSelect->getDefiningRecipe());
5725 if (FindLastSelect->getDefiningRecipe()->getOperand(N: 1) == PhiR)
5726 SelectCond = LoopBuilder.createNot(Operand: SelectCond);
5727
5728 // When tail folding, mask the condition with the header mask to prevent
5729 // propagating poison from inactive lanes in the last vector iteration.
5730 if (HeaderMask)
5731 SelectCond = LoopBuilder.createLogicalAnd(LHS: HeaderMask, RHS: SelectCond);
5732
5733 if (SelectCond != Cond || IVOfExpressionToSink) {
5734 NewFindLastSelect = LoopBuilder.createSelect(
5735 Cond: SelectCond,
5736 TrueVal: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression,
5737 FalseVal: PhiR, DL);
5738 }
5739 }
5740
5741 // Create the reduction result in the middle block using sentinel directly.
5742 RecurKind MinMaxKind =
5743 UseMax ? (UseSigned ? RecurKind::SMax : RecurKind::UMax)
5744 : (UseSigned ? RecurKind::SMin : RecurKind::UMin);
5745 VPIRFlags Flags(MinMaxKind, /*IsOrdered=*/false, /*IsInLoop=*/false,
5746 FastMathFlags());
5747 DebugLoc ExitDL = RdxResult->getDebugLoc();
5748 VPBuilder MiddleBuilder(RdxResult);
5749 VPValue *ReducedIV =
5750 MiddleBuilder.createNaryOp(Opcode: VPInstruction::ComputeReductionResult,
5751 Operands: NewFindLastSelect, Flags, DL: ExitDL);
5752
5753 // If IVOfExpressionToSink is an expression to sink, sink it now.
5754 VPValue *VectorRegionExitingVal = ReducedIV;
5755 if (IVOfExpressionToSink)
5756 VectorRegionExitingVal =
5757 cloneBinOpForScalarIV(BinOp: cast<VPWidenRecipe>(Val: FindLastExpression),
5758 ScalarIV: ReducedIV, WidenIV: IVOfExpressionToSink);
5759
5760 VPValue *NewRdxResult;
5761 VPValue *StartVPV = PhiR->getStartValue();
5762 if (SentinelVal) {
5763 // Sentinel-based approach: reduce IVs with min/max, compare against
5764 // sentinel to detect if condition was ever true, select accordingly.
5765 VPValue *Sentinel = Plan.getConstantInt(Val: *SentinelVal);
5766 auto *Cmp = MiddleBuilder.createICmp(Pred: CmpInst::ICMP_NE, A: ReducedIV,
5767 B: Sentinel, DL: ExitDL);
5768 NewRdxResult = MiddleBuilder.createSelect(Cond: Cmp, TrueVal: VectorRegionExitingVal,
5769 FalseVal: StartVPV, DL: ExitDL);
5770 StartVPV = Sentinel;
5771 } else {
5772 // Introduce a boolean AnyOf reduction to track if the condition was ever
5773 // true in the loop. Use it to select the initial start value, if it was
5774 // never true.
5775 auto *AnyOfPhi = new VPReductionPHIRecipe(
5776 /*Phi=*/nullptr, RecurKind::Or, *Plan.getFalse(), *Plan.getFalse(),
5777 RdxUnordered{.VFScaleFactor: 1}, {}, /*HasUsesOutsideReductionChain=*/false);
5778 AnyOfPhi->insertAfter(InsertPos: PhiR);
5779
5780 VPBuilder LoopBuilder(BackedgeVal->getDefiningRecipe());
5781 VPValue *OrVal = LoopBuilder.createOr(LHS: AnyOfPhi, RHS: SelectCond);
5782 AnyOfPhi->setOperand(I: 1, New: OrVal);
5783
5784 NewRdxResult = MiddleBuilder.createAnyOfReduction(
5785 ChainOp: OrVal, TrueVal: VectorRegionExitingVal, FalseVal: StartVPV, DL: ExitDL);
5786
5787 // Initialize the IV reduction phi with the neutral element, not the
5788 // original start value, to ensure correct min/max reduction results.
5789 StartVPV = Plan.getOrAddLiveIn(
5790 V: getRecurrenceIdentity(K: MinMaxKind, Tp: IVSCEV->getType(), FMF: {}));
5791 }
5792 RdxResult->replaceAllUsesWith(New: NewRdxResult);
5793 RdxResult->eraseFromParent();
5794
5795 auto *NewPhiR = new VPReductionPHIRecipe(
5796 cast<PHINode>(Val: PhiR->getUnderlyingInstr()), RecurKind::FindIV, *StartVPV,
5797 *NewFindLastSelect, RdxUnordered{.VFScaleFactor: 1}, {},
5798 PhiR->hasUsesOutsideReductionChain());
5799 NewPhiR->insertBefore(InsertPos: PhiR);
5800 PhiR->replaceAllUsesWith(New: NewPhiR);
5801 PhiR->eraseFromParent();
5802 }
5803}
5804
5805namespace {
5806
5807using ExtendKind = TTI::PartialReductionExtendKind;
5808struct ReductionExtend {
5809 Type *SrcType = nullptr;
5810 ExtendKind Kind = ExtendKind::PR_None;
5811};
5812
5813/// Describes the extends used to compute the extended reduction operand.
5814/// ExtendB is optional. If ExtendB is present, ExtendsUser is a binary
5815/// operation.
5816struct ExtendedReductionOperand {
5817 /// The recipe that consumes the extends.
5818 VPWidenRecipe *ExtendsUser = nullptr;
5819 /// Extend descriptions (inputs to getPartialReductionCost).
5820 ReductionExtend ExtendA, ExtendB;
5821};
5822
5823/// A chain of recipes that form a partial reduction. Matches either
5824/// reduction_bin_op (extended op, accumulator), or
5825/// reduction_bin_op (accumulator, extended op).
5826/// The possible forms of the "extended op" are listed in
5827/// matchExtendedReductionOperand.
5828struct VPPartialReductionChain {
5829 /// The top-level binary operation that forms the reduction to a scalar
5830 /// after the loop body.
5831 VPWidenRecipe *ReductionBinOp = nullptr;
5832 /// The user of the extends that is then reduced.
5833 ExtendedReductionOperand ExtendedOp;
5834 /// The recurrence kind for the entire partial reduction chain.
5835 /// This allows distinguishing between Sub and AddWithSub recurrences,
5836 /// when the ReductionBinOp is a Instruction::Sub.
5837 RecurKind RK;
5838 /// The index of the accumulator operand of ReductionBinOp. The extended op
5839 /// is `1 - AccumulatorOpIdx`.
5840 unsigned AccumulatorOpIdx;
5841 unsigned ScaleFactor;
5842 /// Optional blend to represent predication for the block that updates the
5843 /// reduction.
5844 VPBlendRecipe *Blend = nullptr;
5845};
5846
5847// Return the incoming index of the single-use value in the blend, which is
5848// expected to be the predicated reduction update.
5849static std::optional<unsigned>
5850getBlendReductionUpdateValueIdx(VPBlendRecipe *Blend) {
5851 assert(Blend && !Blend->isNormalized() &&
5852 Blend->getNumIncomingValues() == 2 &&
5853 "Expected a non-normalized blend with two incoming values");
5854 bool FirstIncomingHasOneUse = Blend->getIncomingValue(Idx: 0)->hasOneUse();
5855
5856 // Only the update value should have one use (the blend). The previous
5857 // value should always have at least two uses, the blend and the reduction.
5858 if (FirstIncomingHasOneUse == Blend->getIncomingValue(Idx: 1)->hasOneUse())
5859 return std::nullopt;
5860 return FirstIncomingHasOneUse ? 0 : 1;
5861}
5862
5863static VPSingleDefRecipe *
5864optimizeExtendsForPartialReduction(VPSingleDefRecipe *Op) {
5865 // reduce.add(mul(ext(A), C))
5866 // -> reduce.add(mul(ext(A), ext(trunc(C))))
5867 const APInt *Const;
5868 if (match(R: Op, P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_APInt(C&: Const)))) {
5869 auto *ExtA = cast<VPWidenCastRecipe>(Val: Op->getOperand(N: 0));
5870 Instruction::CastOps ExtOpc = ExtA->getOpcode();
5871 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
5872 if (!Op->hasOneUse() ||
5873 !llvm::canConstantBeExtended(
5874 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
5875 return Op;
5876
5877 VPBuilder Builder(Op);
5878 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc,
5879 Op: Op->getOperand(N: 1), ResultTy: NarrowTy);
5880 Type *WideTy = ExtA->getScalarType();
5881 Op->setOperand(I: 1, New: Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy));
5882 return Op;
5883 }
5884
5885 // reduce.add(abs(sub(ext(A), ext(B))))
5886 // -> reduce.add(ext(absolute-difference(A, B)))
5887 VPValue *X, *Y;
5888 if (match(R: Op, P: m_WidenIntrinsic<Intrinsic::abs>(Ops: m_Sub(
5889 Op0: m_ZExtOrSExt(Op0: m_VPValue(V&: X)), Op1: m_ZExtOrSExt(Op0: m_VPValue(V&: Y)))))) {
5890 auto *Sub = Op->getOperand(N: 0)->getDefiningRecipe();
5891 auto *Ext = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
5892 assert(Ext->getOpcode() ==
5893 cast<VPWidenCastRecipe>(Sub->getOperand(1))->getOpcode() &&
5894 "Expected both the LHS and RHS extends to be the same");
5895 bool IsSigned = Ext->getOpcode() == Instruction::SExt;
5896 VPBuilder Builder(Op);
5897 Type *SrcTy = X->getScalarType();
5898 auto *FreezeX = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {X}));
5899 auto *FreezeY = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {Y}));
5900 auto *Max = Builder.insert(
5901 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smax : Intrinsic::umax,
5902 {FreezeX, FreezeY}, SrcTy));
5903 auto *Min = Builder.insert(
5904 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smin : Intrinsic::umin,
5905 {FreezeX, FreezeY}, SrcTy));
5906 auto *AbsDiff =
5907 Builder.insert(R: new VPWidenRecipe(Instruction::Sub, {Max, Min}));
5908 return Builder.createWidenCast(Opcode: Instruction::CastOps::ZExt, Op: AbsDiff,
5909 ResultTy: Op->getScalarType());
5910 }
5911
5912 // reduce.add(ext(mul(ext(A), ext(B))))
5913 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
5914 // TODO: Support this optimization for float types.
5915 if (match(R: Op, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
5916 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
5917 auto *Ext = cast<VPWidenCastRecipe>(Val: Op);
5918 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
5919 auto *MulLHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5920 auto *MulRHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5921 if (!Mul->hasOneUse() ||
5922 (Ext->getOpcode() != MulLHS->getOpcode() && MulLHS != MulRHS) ||
5923 MulLHS->getOpcode() != MulRHS->getOpcode())
5924 return Op;
5925 VPBuilder Builder(Mul);
5926 auto *NewLHS = Builder.createWidenCast(
5927 Opcode: MulLHS->getOpcode(), Op: MulLHS->getOperand(N: 0), ResultTy: Ext->getScalarType());
5928 auto *NewRHS = MulLHS == MulRHS
5929 ? NewLHS
5930 : Builder.createWidenCast(Opcode: MulRHS->getOpcode(),
5931 Op: MulRHS->getOperand(N: 0),
5932 ResultTy: Ext->getScalarType());
5933 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewLHS, NewRHS});
5934 Builder.insert(R: NewMul);
5935 Op->replaceAllUsesWith(New: NewMul);
5936 Op->eraseFromParent();
5937 Mul->eraseFromParent();
5938 return NewMul;
5939 }
5940
5941 return Op;
5942}
5943
5944static VPExpressionRecipe *
5945createPartialReductionExpression(VPReductionRecipe *Red) {
5946 VPValue *VecOp = Red->getVecOp();
5947
5948 // reduce.[f]add(ext(op))
5949 // -> VPExpressionRecipe(op, red)
5950 if (match(V: VecOp, P: m_WidenAnyExtend(Op0: m_VPValue())))
5951 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
5952
5953 // reduce.[f]add(neg(ext(op)))
5954 // -> VPExpressionRecipe(op, sub/neg, red)
5955 if (match(V: VecOp, P: m_AnyNeg(Op0: m_WidenAnyExtend(Op0: m_VPValue())))) {
5956 auto *Neg = cast<VPWidenRecipe>(Val: VecOp);
5957 auto *Ext =
5958 cast<VPWidenCastRecipe>(Val: Neg->getOperand(N: Neg->getNumOperands() - 1));
5959 return new VPExpressionRecipe(Ext, Neg, Red);
5960 }
5961
5962 // reduce.[f]add([f]mul(ext(a), ext(b)))
5963 // -> VPExpressionRecipe(a, b, mul, red)
5964 if (match(V: VecOp, P: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))) ||
5965 match(V: VecOp,
5966 P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_ZExtOrSExt(Op0: m_VPValue())))) {
5967 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
5968 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5969 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5970 return new VPExpressionRecipe(ExtA, ExtB, Mul, Red);
5971 }
5972
5973 // reduce.fadd(fneg(fmul(fpext(a), fpext(b))))
5974 // -> VPExpressionRecipe(a, b, fmul, fsub, red)
5975 if (match(V: VecOp,
5976 P: m_FNeg(Op0: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))))) {
5977 auto *FNeg = cast<VPWidenRecipe>(Val: VecOp);
5978 auto *FMul = cast<VPWidenRecipe>(Val: FNeg->getOperand(N: 0));
5979 auto *ExtA = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 0));
5980 auto *ExtB = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 1));
5981 return new VPExpressionRecipe(ExtA, ExtB, FMul, FNeg, Red);
5982 }
5983
5984 // reduce.add(neg(mul(ext(a), ext(b))))
5985 // -> VPExpressionRecipe(a, b, mul, sub, red)
5986 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
5987 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
5988 auto *Sub = cast<VPWidenRecipe>(Val: VecOp);
5989 auto *Mul = cast<VPWidenRecipe>(Val: Sub->getOperand(N: 1));
5990 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5991 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5992 return new VPExpressionRecipe(ExtA, ExtB, Mul, Sub, Red);
5993 }
5994
5995 llvm_unreachable("Unsupported expression");
5996}
5997
5998// Helper to transform a partial reduction chain into a partial reduction
5999// recipe. Assumes profitability has been checked.
6000static void transformToPartialReduction(const VPPartialReductionChain &Chain,
6001 VPlan &Plan,
6002 VPReductionPHIRecipe *RdxPhi) {
6003 VPWidenRecipe *WidenRecipe = Chain.ReductionBinOp;
6004 assert(WidenRecipe->getNumOperands() == 2 && "Expected binary operation");
6005
6006 VPValue *Accumulator = WidenRecipe->getOperand(N: Chain.AccumulatorOpIdx);
6007 auto *ExtendedOp = cast<VPSingleDefRecipe>(
6008 Val: WidenRecipe->getOperand(N: 1 - Chain.AccumulatorOpIdx));
6009
6010 // FIXME: Do these transforms before invoking the cost-model.
6011 ExtendedOp = optimizeExtendsForPartialReduction(Op: ExtendedOp);
6012
6013 // Sub-reductions can be implemented in two ways:
6014 // (1) negate the operand in the vector loop (the default way).
6015 // (2) subtract the reduced value from the init value in the middle block.
6016 // Both ways keep the reduction itself as an 'add' reduction.
6017 //
6018 // The ISD nodes for partial reductions don't support folding the
6019 // sub/negation into its operands because the following is not a valid
6020 // transformation:
6021 // sub(0, mul(ext(a), ext(b)))
6022 // -> mul(ext(a), ext(sub(0, b)))
6023 //
6024 // It's therefore better to choose option (2) such that the partial
6025 // reduction is always positive (starting at '0') and to do a final
6026 // subtract in the middle block.
6027 if ((WidenRecipe->getOpcode() == Instruction::Sub &&
6028 Chain.RK != RecurKind::Sub) ||
6029 (WidenRecipe->getOpcode() == Instruction::FSub &&
6030 Chain.RK != RecurKind::FSub)) {
6031 VPBuilder Builder(WidenRecipe);
6032 Type *ElemTy = ExtendedOp->getScalarType();
6033 VPWidenRecipe *NegRecipe;
6034 if (WidenRecipe->getOpcode() == Instruction::FSub) {
6035 NegRecipe =
6036 new VPWidenRecipe(Instruction::FNeg, {ExtendedOp}, VPIRFlags(),
6037 VPIRMetadata(), DebugLoc::getUnknown());
6038 } else {
6039 auto *Zero = Plan.getZero(Ty: ElemTy);
6040 NegRecipe =
6041 new VPWidenRecipe(Instruction::Sub, {Zero, ExtendedOp}, VPIRFlags(),
6042 VPIRMetadata(), DebugLoc::getUnknown());
6043 }
6044 Builder.insert(R: NegRecipe);
6045 ExtendedOp = NegRecipe;
6046 }
6047
6048 // Check if WidenRecipe is the final result of the reduction. If so, look
6049 // through the Select recipe introduced by tail-folding, otherwise look
6050 // through any Blend recipe introduced by predication for the block.
6051 VPValue *ExitSearch =
6052 Chain.Blend ? cast<VPValue>(Val: Chain.Blend) : cast<VPValue>(Val: WidenRecipe);
6053
6054 VPValue *Cond = nullptr;
6055 VPValue *ExitValue = cast_or_null<VPInstruction>(
6056 Val: findUserOf(V: ExitSearch, P: m_Select(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: ExitSearch),
6057 Op2: m_Specific(VPV: RdxPhi))));
6058
6059 if (Chain.Blend) {
6060 std::optional<unsigned> BlendReductionIdx =
6061 getBlendReductionUpdateValueIdx(Blend: Chain.Blend);
6062 assert(BlendReductionIdx &&
6063 Chain.Blend->getIncomingValue(*BlendReductionIdx) == WidenRecipe &&
6064 "Expected blend to contain the reduction update");
6065 VPValue *BlendCond = Chain.Blend->getMask(Idx: *BlendReductionIdx);
6066 Cond = ExitValue ? VPBuilder(WidenRecipe)
6067 .createLogicalAnd(LHS: Cond, RHS: BlendCond,
6068 DL: WidenRecipe->getDebugLoc())
6069 : BlendCond;
6070 }
6071
6072 bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
6073 RdxPhi->getBackedgeValue() == ExitValue ||
6074 RdxPhi->getBackedgeValue() == Chain.Blend;
6075 assert((!ExitValue || IsLastInChain) &&
6076 "if we found ExitValue, it must match RdxPhi's backedge value");
6077
6078 Type *PhiType = RdxPhi->getScalarType();
6079 RecurKind RdxKind =
6080 PhiType->isFloatingPointTy() ? RecurKind::FAdd : RecurKind::Add;
6081 auto *PartialRed = new VPReductionRecipe(
6082 RdxKind,
6083 RdxKind == RecurKind::FAdd ? WidenRecipe->getFastMathFlagsOrNone()
6084 : FastMathFlags(),
6085 WidenRecipe->getUnderlyingInstr(), Accumulator, ExtendedOp, Cond,
6086 RdxUnordered{/*VFScaleFactor=*/Chain.ScaleFactor});
6087 PartialRed->insertBefore(InsertPos: WidenRecipe);
6088
6089 if (ExitValue)
6090 ExitValue->replaceAllUsesWith(New: PartialRed);
6091 if (Chain.Blend)
6092 Chain.Blend->replaceAllUsesWith(New: PartialRed);
6093 WidenRecipe->replaceAllUsesWith(New: PartialRed);
6094
6095 // For cost-model purposes, fold this into a VPExpression.
6096 VPExpressionRecipe *E = createPartialReductionExpression(Red: PartialRed);
6097 E->insertBefore(InsertPos: WidenRecipe);
6098 PartialRed->replaceAllUsesWith(New: E);
6099
6100 // We only need to update the PHI node once, which is when we find the
6101 // last reduction in the chain.
6102 if (!IsLastInChain)
6103 return;
6104
6105 // Scale the PHI and ReductionStartVector by the VFScaleFactor
6106 assert(RdxPhi->getVFScaleFactor() == 1 && "scale factor must not be set");
6107 RdxPhi->setVFScaleFactor(Chain.ScaleFactor);
6108
6109 auto *StartInst = cast<VPInstruction>(Val: RdxPhi->getStartValue());
6110 assert(StartInst->getOpcode() == VPInstruction::ReductionStartVector);
6111 auto *NewScaleFactor = Plan.getConstantInt(BitWidth: 32, Val: Chain.ScaleFactor);
6112 StartInst->setOperand(I: 2, New: NewScaleFactor);
6113
6114 // If this is the last value in a sub-reduction chain, then update the PHI
6115 // node to start at `0` and update the reduction-result to subtract from
6116 // the PHI's start value.
6117 if (Chain.RK != RecurKind::Sub && Chain.RK != RecurKind::FSub)
6118 return;
6119
6120 VPValue *OldStartValue = StartInst->getOperand(N: 0);
6121 StartInst->setOperand(I: 0, New: StartInst->getOperand(N: 1));
6122
6123 // Replace reduction_result by 'sub (startval, reductionresult)'.
6124 VPInstruction *RdxResult = vputils::findComputeReductionResult(PhiR: RdxPhi);
6125 assert(RdxResult && "Could not find reduction result");
6126
6127 VPBuilder Builder = VPBuilder::getToInsertAfter(R: RdxResult);
6128 unsigned SubOpc = Chain.RK == RecurKind::FSub ? Instruction::BinaryOps::FSub
6129 : Instruction::BinaryOps::Sub;
6130 VPInstruction *NewResult = Builder.createNaryOp(
6131 Opcode: SubOpc, Operands: {OldStartValue, RdxResult}, Flags: VPIRFlags::getDefaultFlags(Opcode: SubOpc),
6132 DL: RdxPhi->getDebugLoc());
6133 RdxResult->replaceUsesWithIf(
6134 New: NewResult,
6135 ShouldReplace: [&NewResult](VPUser &U, unsigned Idx) { return &U != NewResult; });
6136}
6137
6138/// Returns the cost of a link in a partial-reduction chain for a given VF.
6139static InstructionCost
6140getPartialReductionLinkCost(VPCostContext &CostCtx,
6141 const VPPartialReductionChain &Link,
6142 ElementCount VF) {
6143 Type *RdxType = Link.ReductionBinOp->getScalarType();
6144 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
6145 std::optional<unsigned> BinOpc = std::nullopt;
6146 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
6147 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
6148 BinOpc = ExtendedOp.ExtendsUser->getOpcode();
6149
6150 std::optional<llvm::FastMathFlags> Flags;
6151 if (RdxType->isFloatingPointTy())
6152 Flags = Link.ReductionBinOp->getFastMathFlagsOrNone();
6153
6154 auto GetLinkOpcode = [&Link]() -> unsigned {
6155 switch (Link.RK) {
6156 case RecurKind::Sub:
6157 return Instruction::Add;
6158 case RecurKind::FSub:
6159 return Instruction::FAdd;
6160 default:
6161 return Link.ReductionBinOp->getOpcode();
6162 }
6163 };
6164
6165 return CostCtx.TTI.getPartialReductionCost(
6166 Opcode: GetLinkOpcode(), InputTypeA: ExtendedOp.ExtendA.SrcType, InputTypeB: ExtendedOp.ExtendB.SrcType,
6167 AccumType: RdxType, VF, OpAExtend: ExtendedOp.ExtendA.Kind, OpBExtend: ExtendedOp.ExtendB.Kind, BinOp: BinOpc,
6168 CostKind: CostCtx.CostKind, FMF: Flags);
6169}
6170
6171static ExtendKind getPartialReductionExtendKind(VPWidenCastRecipe *Cast) {
6172 return TTI::getPartialReductionExtendKind(CastOpc: Cast->getOpcode());
6173}
6174
6175/// Checks if \p Op (which is an operand of \p UpdateR) is an extended reduction
6176/// operand. This is an operand where the source of the value (e.g. a load) has
6177/// been extended (sext, zext, or fpext) before it is used in the reduction.
6178///
6179/// Possible forms matched by this function:
6180/// - UpdateR(PrevValue, ext(...))
6181/// - UpdateR(PrevValue, mul(ext(...), ext(...)))
6182/// - UpdateR(PrevValue, mul(ext(...), Constant))
6183/// - UpdateR(PrevValue, ext(mul(ext(...), ext(...))))
6184/// - UpdateR(PrevValue, ext(mul(ext(...), Constant)))
6185/// - UpdateR(PrevValue, abs(sub(ext(...), ext(...)))
6186///
6187/// Note: The second operand of UpdateR corresponds to \p Op in the examples.
6188static std::optional<ExtendedReductionOperand>
6189matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op) {
6190 assert(is_contained(UpdateR->operands(), Op) &&
6191 "Op should be operand of UpdateR");
6192
6193 // Try matching an absolute difference operand of the form
6194 // `abs(sub(ext(A), ext(B)))`. This will be later transformed into
6195 // `ext(absolute-difference(A, B))`. This allows us to perform the absolute
6196 // difference on a wider type and get the extend for "free" from the partial
6197 // reduction.
6198 VPValue *X, *Y;
6199 if (Op->hasOneUse() &&
6200 match(V: Op, P: m_WidenIntrinsic<Intrinsic::abs>(
6201 Ops: m_OneUse(SubPattern: m_Sub(Op0: m_WidenAnyExtend(Op0: m_VPValue(V&: X)),
6202 Op1: m_WidenAnyExtend(Op0: m_VPValue(V&: Y))))))) {
6203 auto *Abs = cast<VPWidenIntrinsicRecipe>(Val: Op);
6204 auto *Sub = cast<VPWidenRecipe>(Val: Abs->getOperand(N: 0));
6205 auto *LHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
6206 auto *RHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 1));
6207 Type *LHSInputType = X->getScalarType();
6208 Type *RHSInputType = Y->getScalarType();
6209 if (LHSInputType != RHSInputType ||
6210 LHSExt->getOpcode() != RHSExt->getOpcode())
6211 return std::nullopt;
6212 // Note: This is essentially the same as matching ext(...) as we will
6213 // rewrite this operand to ext(absolute-difference(A, B)).
6214 return ExtendedReductionOperand{
6215 .ExtendsUser: Sub,
6216 /*ExtendA=*/{.SrcType: LHSInputType, .Kind: getPartialReductionExtendKind(Cast: LHSExt)},
6217 /*ExtendB=*/{}};
6218 }
6219
6220 std::optional<TTI::PartialReductionExtendKind> OuterExtKind;
6221 if (match(V: Op, P: m_WidenAnyExtend(Op0: m_VPValue()))) {
6222 auto *CastRecipe = cast<VPWidenCastRecipe>(Val: Op);
6223 VPValue *CastSource = CastRecipe->getOperand(N: 0);
6224 OuterExtKind = getPartialReductionExtendKind(Cast: CastRecipe);
6225 if (match(V: CastSource, P: m_Mul(Op0: m_VPValue(), Op1: m_VPValue())) ||
6226 match(V: CastSource, P: m_FMul(Op0: m_VPValue(), Op1: m_VPValue()))) {
6227 // Match: ext(mul(...))
6228 // Record the outer extend kind and set `Op` to the mul. We can then match
6229 // this as a binary operation. Note: We can optimize out the outer extend
6230 // by widening the inner extends to match it. See
6231 // optimizeExtendsForPartialReduction.
6232 Op = CastSource;
6233 } else {
6234 return ExtendedReductionOperand{
6235 .ExtendsUser: UpdateR,
6236 /*ExtendA=*/{.SrcType: CastSource->getScalarType(), .Kind: *OuterExtKind},
6237 /*ExtendB=*/{}};
6238 }
6239 }
6240
6241 if (!Op->hasOneUse())
6242 return std::nullopt;
6243
6244 VPWidenRecipe *MulOp = dyn_cast<VPWidenRecipe>(Val: Op);
6245 if (!MulOp ||
6246 !is_contained(Set: {Instruction::Mul, Instruction::FMul}, Element: MulOp->getOpcode()))
6247 return std::nullopt;
6248
6249 // The rest of the matching assumes `Op` is a (possibly extended) mul
6250 // operation.
6251
6252 VPValue *LHS = MulOp->getOperand(N: 0);
6253 VPValue *RHS = MulOp->getOperand(N: 1);
6254
6255 // The LHS of the operation must always be an extend.
6256 if (!match(V: LHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
6257 return std::nullopt;
6258
6259 auto *LHSCast = cast<VPWidenCastRecipe>(Val: LHS);
6260 Type *LHSInputType = LHSCast->getOperand(N: 0)->getScalarType();
6261 ExtendKind LHSExtendKind = getPartialReductionExtendKind(Cast: LHSCast);
6262
6263 // The RHS of the operation can be an extend or a constant integer.
6264 const APInt *RHSConst = nullptr;
6265 VPWidenCastRecipe *RHSCast = nullptr;
6266 if (match(V: RHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
6267 RHSCast = cast<VPWidenCastRecipe>(Val: RHS);
6268 else if (!match(V: RHS, P: m_APInt(C&: RHSConst)) ||
6269 !canConstantBeExtended(C: RHSConst, NarrowType: LHSInputType, ExtKind: LHSExtendKind))
6270 return std::nullopt;
6271
6272 // The outer extend kind must match the inner extends for folding.
6273 for (VPWidenCastRecipe *Cast : {LHSCast, RHSCast})
6274 if (Cast && OuterExtKind &&
6275 getPartialReductionExtendKind(Cast) != OuterExtKind)
6276 return std::nullopt;
6277
6278 Type *RHSInputType = LHSInputType;
6279 ExtendKind RHSExtendKind = LHSExtendKind;
6280 if (RHSCast) {
6281 RHSInputType = RHSCast->getOperand(N: 0)->getScalarType();
6282 RHSExtendKind = getPartialReductionExtendKind(Cast: RHSCast);
6283 }
6284
6285 return ExtendedReductionOperand{
6286 .ExtendsUser: MulOp, .ExtendA: {.SrcType: LHSInputType, .Kind: LHSExtendKind}, .ExtendB: {.SrcType: RHSInputType, .Kind: RHSExtendKind}};
6287}
6288
6289/// Examines each operation in the reduction chain corresponding to \p RedPhiR,
6290/// and determines if the target can use a cheaper operation with a wider
6291/// per-iteration input VF and narrower PHI VF. If successful, returns the chain
6292/// of operations in the reduction.
6293static std::optional<SmallVector<VPPartialReductionChain>>
6294getScaledReductions(VPReductionPHIRecipe *RedPhiR) {
6295 // Get the backedge value from the reduction PHI and find the
6296 // ComputeReductionResult that uses it (directly or through a select for
6297 // predicated reductions).
6298 auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR);
6299 if (!RdxResult)
6300 return std::nullopt;
6301 VPValue *ExitValue = RdxResult->getOperand(N: 0);
6302 match(V: ExitValue, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: ExitValue), Op2: m_VPValue()));
6303
6304 SmallVector<VPPartialReductionChain> Chain;
6305 RecurKind RK = RedPhiR->getRecurrenceKind();
6306 Type *PhiType = RedPhiR->getScalarType();
6307 TypeSize PHISize = PhiType->getPrimitiveSizeInBits();
6308
6309 // Work backwards from the ExitValue examining each reduction operation.
6310 VPValue *CurrentValue = ExitValue;
6311 while (CurrentValue != RedPhiR) {
6312 VPBlendRecipe *Blend = dyn_cast<VPBlendRecipe>(Val: CurrentValue);
6313 std::optional<unsigned> BlendReductionIdx;
6314 if (Blend) {
6315 assert(!Blend->isNormalized() && "Expect Blend not to be normalized.");
6316 if (Blend->getNumIncomingValues() != 2)
6317 return std::nullopt;
6318
6319 BlendReductionIdx = getBlendReductionUpdateValueIdx(Blend);
6320 if (!BlendReductionIdx)
6321 return std::nullopt;
6322
6323 CurrentValue = Blend->getIncomingValue(Idx: *BlendReductionIdx);
6324 }
6325
6326 auto *UpdateR = dyn_cast<VPWidenRecipe>(Val: CurrentValue);
6327 if (!UpdateR || !Instruction::isBinaryOp(Opcode: UpdateR->getOpcode()))
6328 return std::nullopt;
6329
6330 VPValue *Op = UpdateR->getOperand(N: 1);
6331 VPValue *PrevValue = UpdateR->getOperand(N: 0);
6332
6333 // Find the extended operand. The other operand (PrevValue) is the next link
6334 // in the reduction chain.
6335 std::optional<ExtendedReductionOperand> ExtendedOp =
6336 matchExtendedReductionOperand(UpdateR, Op);
6337 if (!ExtendedOp) {
6338 ExtendedOp = matchExtendedReductionOperand(UpdateR, Op: PrevValue);
6339 if (!ExtendedOp)
6340 return std::nullopt;
6341 std::swap(a&: Op, b&: PrevValue);
6342 }
6343
6344 // Look for VPBlend(reduce(PrevValue, Op), PrevValue), where
6345 // reduce is equal to CurrentValue. This can be lowered as
6346 // a conditional reduction by hoisting the select to the inputs.
6347 if (Blend && Blend->getIncomingValue(Idx: 1 - *BlendReductionIdx) != PrevValue)
6348 return std::nullopt;
6349
6350 Type *ExtSrcType = ExtendedOp->ExtendA.SrcType;
6351 TypeSize ExtSrcSize = ExtSrcType->getPrimitiveSizeInBits();
6352 if (!PHISize.hasKnownScalarFactor(RHS: ExtSrcSize))
6353 return std::nullopt;
6354
6355 VPPartialReductionChain Link(
6356 {.ReductionBinOp: UpdateR, .ExtendedOp: *ExtendedOp, .RK: RK,
6357 .AccumulatorOpIdx: PrevValue == UpdateR->getOperand(N: 0) ? 0U : 1U,
6358 .ScaleFactor: static_cast<unsigned>(PHISize.getKnownScalarFactor(RHS: ExtSrcSize)),
6359 .Blend: Blend});
6360 Chain.push_back(Elt: Link);
6361 CurrentValue = PrevValue;
6362 }
6363
6364 // The chain links were collected by traversing backwards from the exit value.
6365 // Reverse the chains so they are in program order.
6366 std::reverse(first: Chain.begin(), last: Chain.end());
6367 return Chain;
6368}
6369} // namespace
6370
6371void VPlanTransforms::createPartialReductions(VPlan &Plan,
6372 VPCostContext &CostCtx,
6373 VFRange &Range) {
6374 // Find all possible valid partial reductions, grouping chains by their PHI.
6375 // This grouping allows invalidating the whole chain, if any link is not a
6376 // valid partial reduction.
6377 MapVector<VPReductionPHIRecipe *, SmallVector<VPPartialReductionChain>>
6378 ChainsByPhi;
6379 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
6380 for (VPRecipeBase &R : HeaderVPBB->phis()) {
6381 auto *RedPhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
6382 if (!RedPhiR)
6383 continue;
6384
6385 if (auto Chains = getScaledReductions(RedPhiR))
6386 ChainsByPhi.try_emplace(Key: RedPhiR, Args: std::move(*Chains));
6387 }
6388
6389 if (ChainsByPhi.empty())
6390 return;
6391
6392 // Build set of partial reduction operations and blends for user validation
6393 // and a map of reduction bin ops to their scale factors for scale validation.
6394 SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
6395 SmallPtrSet<VPBlendRecipe *, 4> PartialReductionBlends;
6396 DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
6397 for (const auto &[_, Chains] : ChainsByPhi)
6398 for (const VPPartialReductionChain &Chain : Chains) {
6399 PartialReductionOps.insert(Ptr: Chain.ExtendedOp.ExtendsUser);
6400 if (Chain.Blend)
6401 PartialReductionBlends.insert(Ptr: Chain.Blend);
6402 ScaledReductionMap[Chain.ReductionBinOp] = Chain.ScaleFactor;
6403 }
6404
6405 // A partial reduction is invalid if any of its extends are used by
6406 // something that isn't another partial reduction. This is because the
6407 // extends are intended to be lowered along with the reduction itself.
6408 auto ExtendUsersValid = [&](VPValue *Ext) {
6409 return !isa<VPWidenCastRecipe>(Val: Ext) || all_of(Range: Ext->users(), P: [&](VPUser *U) {
6410 return PartialReductionOps.contains(Ptr: cast<VPRecipeBase>(Val: U));
6411 });
6412 };
6413
6414 auto IsProfitablePartialReductionChainForVF =
6415 [&](ArrayRef<VPPartialReductionChain> Chain, ElementCount VF) -> bool {
6416 InstructionCost PartialCost = 0, RegularCost = 0;
6417
6418 // The chain is a profitable partial reduction chain if the cost of handling
6419 // the entire chain is cheaper when using partial reductions than when
6420 // handling the entire chain using regular reductions.
6421 for (const VPPartialReductionChain &Link : Chain) {
6422 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
6423 InstructionCost LinkCost = getPartialReductionLinkCost(CostCtx, Link, VF);
6424 if (!LinkCost.isValid())
6425 return false;
6426
6427 PartialCost += LinkCost;
6428 RegularCost += Link.ReductionBinOp->computeCost(VF, Ctx&: CostCtx);
6429 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
6430 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
6431 RegularCost += ExtendedOp.ExtendsUser->computeCost(VF, Ctx&: CostCtx);
6432 for (VPValue *Op : ExtendedOp.ExtendsUser->operands())
6433 if (auto *Extend = dyn_cast<VPWidenCastRecipe>(Val: Op))
6434 RegularCost += Extend->computeCost(VF, Ctx&: CostCtx);
6435 }
6436 return PartialCost.isValid() && PartialCost < RegularCost;
6437 };
6438
6439 // Validate chains: check that extends are only used by partial reductions,
6440 // and that reduction bin ops are only used by other partial reductions with
6441 // matching scale factors, are outside the loop region or the select
6442 // introduced by tail-folding. Otherwise we would create users of scaled
6443 // reductions where the types of the other operands don't match.
6444 for (auto &[RedPhiR, Chains] : ChainsByPhi) {
6445 for (const VPPartialReductionChain &Chain : Chains) {
6446 if (!all_of(Range: Chain.ExtendedOp.ExtendsUser->operands(), P: ExtendUsersValid)) {
6447 Chains.clear();
6448 break;
6449 }
6450 auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
6451 if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: U))
6452 return PhiR == RedPhiR;
6453 auto *R = cast<VPSingleDefRecipe>(Val: U);
6454
6455 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: R))
6456 return Blend == Chain.Blend || PartialReductionBlends.contains(Ptr: Blend);
6457
6458 return Chain.ScaleFactor == ScaledReductionMap.lookup_or(Val: R, Default: 0) ||
6459 match(R, P: m_ComputeReductionResult(
6460 Op0: m_Specific(VPV: Chain.ReductionBinOp))) ||
6461 match(R, P: m_Select(Op0: m_VPValue(), Op1: m_Specific(VPV: Chain.ReductionBinOp),
6462 Op2: m_Specific(VPV: RedPhiR)));
6463 };
6464 if (!all_of(Range: Chain.ReductionBinOp->users(), P: UseIsValid)) {
6465 Chains.clear();
6466 break;
6467 }
6468
6469 // Check if the compute-reduction-result is used by a sunk store.
6470 // TODO: Also form partial reductions in those cases.
6471 if (auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR)) {
6472 if (any_of(Range: RdxResult->users(), P: [](VPUser *U) {
6473 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
6474 return RepR && RepR->getOpcode() == Instruction::Store;
6475 })) {
6476 Chains.clear();
6477 break;
6478 }
6479 }
6480 }
6481
6482 // Clear the chain if it is not profitable.
6483 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
6484 Predicate: [&, &Chains = Chains](ElementCount VF) {
6485 return IsProfitablePartialReductionChainForVF(Chains, VF);
6486 },
6487 Range))
6488 Chains.clear();
6489 }
6490
6491 for (auto &[Phi, Chains] : ChainsByPhi)
6492 for (const VPPartialReductionChain &Chain : Chains)
6493 transformToPartialReduction(Chain, Plan, RdxPhi: Phi);
6494}
6495
6496/// If the pointer operand \p Addr of a memory access is an affine AddRec
6497/// w.r.t. \p L with a constant stride, return the stride in units of
6498/// \p AccessTy. Otherwise return std::nullopt.
6499static std::optional<int64_t> getConstantStride(VPValue *Addr, Type *AccessTy,
6500 PredicatedScalarEvolution &PSE,
6501 const Loop *L) {
6502 const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(V: Addr, PSE, L);
6503 auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: AddrSCEV);
6504 if (!AddRec)
6505 return {};
6506
6507 return getStrideFromAddRec(AR: AddRec, Lp: L, AccessTy, /*Ptr=*/nullptr, PSE);
6508}
6509
6510void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
6511 VPRecipeBuilder &RecipeBuilder,
6512 VPCostContext &CostCtx) {
6513 // Collect all loads/stores first. We will start with ones having simpler
6514 // decisions followed by more complex ones that are potentially
6515 // guided/dependent on the simpler ones.
6516 SmallVector<VPInstruction *> MemOps;
6517 for (VPBasicBlock *VPBB :
6518 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
6519 G: Plan.getVectorLoopRegion()->getEntryBasicBlock()))) {
6520 for (VPRecipeBase &R : *VPBB) {
6521 auto *VPI = dyn_cast<VPInstruction>(Val: &R);
6522 if (VPI && VPI->getUnderlyingValue() &&
6523 is_contained(Set: {Instruction::Load, Instruction::Store},
6524 Element: VPI->getOpcode()))
6525 MemOps.push_back(Elt: VPI);
6526 }
6527 }
6528
6529 // Few helpers to process different kinds of memory operations.
6530
6531 // To be used as argument to `VPlanTransforms::runPass` which explicitly
6532 // specified pass name, hence `VPlan &` parameter.
6533 auto ProcessSubset = [&](VPlan &, auto ProcessVPInst) {
6534 SmallVector<VPInstruction *> RemainingMemOps;
6535 for (VPInstruction *VPI : MemOps) {
6536 if (!ProcessVPInst(VPI))
6537 RemainingMemOps.push_back(Elt: VPI);
6538 }
6539
6540 MemOps.clear();
6541 std::swap(LHS&: MemOps, RHS&: RemainingMemOps);
6542 };
6543
6544 auto ReplaceWith = [&](VPInstruction *VPI, VPRecipeBase *New) {
6545 New->insertBefore(InsertPos: VPI);
6546 if (VPI->getOpcode() == Instruction::Load)
6547 VPI->replaceAllUsesWith(New: New->getVPSingleValue());
6548 VPI->eraseFromParent();
6549
6550 // VPI has been processed.
6551 return true;
6552 };
6553
6554 auto Scalarize = [&](VPInstruction *VPI) {
6555 return ReplaceWith(VPI, RecipeBuilder.handleReplication(VPI, Range));
6556 };
6557
6558 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
6559 VPBuilder FinalRedStoresBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
6560 VPlanTransforms::runPass(
6561 PassName: "lowerMemoryIdioms", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
6562 if (RecipeBuilder.replaceWithFinalIfReductionStore(
6563 VPI, FinalRedStoresBuilder))
6564 return true;
6565
6566 // Filter out scalar VPlan for the remaining idioms.
6567 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6568 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
6569 return false;
6570
6571 if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI))
6572 return ReplaceWith(VPI, Histogram);
6573
6574 return false;
6575 });
6576
6577 // Filter out scalar VPlan for the remaining memory operations.
6578 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6579 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
6580 return;
6581
6582 // If the instruction's allocated size doesn't equal it's type size, it
6583 // requires padding and will be scalarized.
6584 VPlanTransforms::runPass(
6585 PassName: "scalarizeMemOpsWithIrregularTypes", Pass&: ProcessSubset, Plan,
6586 Args: [&](VPInstruction *VPI) {
6587 Instruction *I = VPI->getUnderlyingInstr();
6588 if (hasIrregularType(Ty: getLoadStoreType(I), DL: I->getDataLayout()))
6589 return Scalarize(VPI);
6590
6591 return false;
6592 });
6593
6594 if (!RecipeBuilder.prefersVectorizedAddressing()) {
6595 VPlanTransforms::runPass(
6596 PassName: "makeVPlanMemOpDecision", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
6597 Instruction *I = VPI->getUnderlyingInstr();
6598 bool IsLoad = VPI->getOpcode() == Instruction::Load;
6599 if (RecipeBuilder.isPredicatedInst(I) || !IsLoad ||
6600 !vputils::isUsedByLoadStoreAddress(V: VPI))
6601 return false;
6602
6603 // Scalarize loads used as addresses, matching the legacy CM. The load
6604 // is single-scalar if the pointer is loop-invariant, otherwise it is
6605 // replicated per-lane. No mask is needed as the load is not
6606 // predicated.
6607 VPValue *Ptr = VPI->getOperand(N: 0);
6608 const SCEV *PtrSCEV =
6609 vputils::getSCEVExprForVPValue(V: Ptr, PSE&: CostCtx.PSE, L: CostCtx.L);
6610 bool IsSingleScalarLoad =
6611 !isa<SCEVCouldNotCompute>(Val: PtrSCEV) &&
6612 CostCtx.PSE.getSE()->isLoopInvariant(S: PtrSCEV, L: CostCtx.L);
6613
6614 ReplaceWith(VPI,
6615 new VPReplicateRecipe(
6616 I, Ptr, /*IsSingleScalar=*/IsSingleScalarLoad,
6617 /*Mask=*/nullptr, *VPI, *VPI, VPI->getDebugLoc()));
6618 return true;
6619 });
6620 }
6621
6622 // Widen unmasked unit-stride consecutive accesses, matching the legacy CM.
6623 VPlanTransforms::runPass(
6624 PassName: "widenConsecutiveMemOps", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
6625 Instruction *I = VPI->getUnderlyingInstr();
6626 if (RecipeBuilder.isPredicatedInst(I))
6627 return false;
6628
6629 bool IsLoad = VPI->getOpcode() == Instruction::Load;
6630 VPValue *Ptr = VPI->getOperand(N: !IsLoad);
6631 Type *ScalarTy =
6632 IsLoad ? VPI->getScalarType() : VPI->getOperand(N: 0)->getScalarType();
6633 if (getConstantStride(Addr: Ptr, AccessTy: ScalarTy, PSE&: CostCtx.PSE, L: CostCtx.L) != 1)
6634 return false;
6635
6636 Type *StrideTy =
6637 Plan.getDataLayout().getIndexType(PtrTy: Ptr->getScalarType());
6638 VPValue *StrideOne = Plan.getConstantInt(Ty: StrideTy, Val: 1);
6639 auto *VectorPtr = new VPVectorPointerRecipe(
6640 Ptr, ScalarTy, StrideOne, vputils::getGEPFlagsForPtr(Ptr),
6641 VPI->getDebugLoc());
6642 VectorPtr->insertBefore(InsertPos: VPI);
6643 VPRecipeBase *WidenedR;
6644 if (IsLoad)
6645 WidenedR = new VPWidenLoadRecipe(*cast<LoadInst>(Val: I), VectorPtr,
6646 /*Mask=*/nullptr,
6647 /*Consecutive=*/true, *VPI,
6648 VPI->getDebugLoc());
6649 else
6650 WidenedR = new VPWidenStoreRecipe(
6651 *cast<StoreInst>(Val: I), VectorPtr, VPI->getOperand(N: 0),
6652 /*Mask=*/nullptr, /*Consecutive=*/true, *VPI, VPI->getDebugLoc());
6653 return ReplaceWith(VPI, WidenedR);
6654 });
6655
6656 VPlanTransforms::runPass(PassName: "delegateMemOpWideningToLegacyCM", Pass&: ProcessSubset,
6657 Plan, Args: [&](VPInstruction *VPI) {
6658 if (VPRecipeBase *Recipe =
6659 RecipeBuilder.tryToWidenMemory(VPI, Range))
6660 return ReplaceWith(VPI, Recipe);
6661
6662 return Scalarize(VPI);
6663 });
6664}
6665
6666void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
6667 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6668 Predicate: [&](ElementCount VF) { return VF.isScalar(); }, Range))
6669 return;
6670
6671 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
6672 Plan.getEntry());
6673 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
6674 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
6675 auto *VPI = dyn_cast<VPInstruction>(Val: &R);
6676 if (!VPI)
6677 continue;
6678
6679 auto *I = cast_or_null<Instruction>(Val: VPI->getUnderlyingValue());
6680 // Wouldn't be able to create a `VPReplicateRecipe` anyway.
6681 if (!I)
6682 continue;
6683
6684 // If executing other lanes produces side-effects we can't avoid them.
6685 if (VPI->mayHaveSideEffects())
6686 continue;
6687
6688 // We want to drop the mask operand, verify we can safely do that.
6689 if (VPI->isMasked() && !VPI->isSafeToSpeculativelyExecute())
6690 continue;
6691
6692 // Avoid rewriting IV increment as that interferes with
6693 // `removeRedundantCanonicalIVs`.
6694 if (VPI->getOpcode() == Instruction::Add &&
6695 any_of(Range: VPI->operands(), P: IsaPred<VPWidenIntOrFpInductionRecipe>))
6696 continue;
6697
6698 // Other lanes are needed - can't drop them.
6699 if (!vputils::onlyFirstLaneUsed(Def: VPI))
6700 continue;
6701
6702 auto *Recipe = VPBuilder::createSingleScalarOp(
6703 Opcode: VPI->getOpcode(), Operands: VPI->operandsWithoutMask(), /*Mask=*/nullptr, Flags: *VPI,
6704 Metadata: *VPI, DL: VPI->getDebugLoc(), UV: I);
6705 Recipe->insertBefore(InsertPos: VPI);
6706 VPI->replaceAllUsesWith(New: Recipe);
6707 VPI->eraseFromParent();
6708 }
6709 }
6710}
6711
6712/// Returns true if \p Info's parameter kinds are compatible with \p Args.
6713static bool areVFParamsOk(const VFInfo &Info, ArrayRef<VPValue *> Args,
6714 PredicatedScalarEvolution &PSE, const Loop *L) {
6715 ScalarEvolution *SE = PSE.getSE();
6716 return all_of(Range: Info.Shape.Parameters, P: [&](VFParameter Param) {
6717 switch (Param.ParamKind) {
6718 case VFParamKind::Vector:
6719 case VFParamKind::GlobalPredicate:
6720 return true;
6721 case VFParamKind::OMP_Uniform:
6722 return SE->isSCEVable(Ty: Args[Param.ParamPos]->getScalarType()) &&
6723 SE->isLoopInvariant(
6724 S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
6725 L);
6726 case VFParamKind::OMP_Linear:
6727 return match(S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
6728 P: m_scev_AffineAddRec(
6729 Op0: m_SCEV(), Op1: m_scev_SpecificSInt(V: Param.LinearStepOrPos),
6730 L: m_SpecificLoop(L)));
6731 default:
6732 return false;
6733 }
6734 });
6735}
6736
6737/// Find a vector variant of \p CI for \p VF, respecting \p MaskRequired.
6738/// Returns the variant function, or nullptr. Masked variants are assumed to
6739/// take the mask as a trailing parameter.
6740static Function *findVectorVariant(CallInst *CI, ArrayRef<VPValue *> Args,
6741 ElementCount VF, bool MaskRequired,
6742 PredicatedScalarEvolution &PSE,
6743 const Loop *L) {
6744 if (CI->isNoBuiltin())
6745 return nullptr;
6746 auto Mappings = VFDatabase::getMappings(CI: *CI);
6747 const auto *It = find_if(Range&: Mappings, P: [&](const VFInfo &Info) {
6748 return Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()) &&
6749 areVFParamsOk(Info, Args, PSE, L);
6750 });
6751 if (It == Mappings.end())
6752 return nullptr;
6753 return CI->getModule()->getFunction(Name: It->VectorName);
6754}
6755
6756namespace {
6757/// The outcome of choosing how to widen a call at a given VF.
6758struct CallWideningDecision {
6759 enum class KindTy { Scalarize, Intrinsic, VectorVariant };
6760 CallWideningDecision(KindTy Kind, Function *Variant = nullptr)
6761 : Kind(Kind), Variant(Variant) {}
6762 KindTy Kind;
6763
6764 /// Set when Kind == VectorVariant.
6765 Function *Variant;
6766
6767 bool operator==(const CallWideningDecision &Other) const {
6768 return Kind == Other.Kind && Variant == Other.Variant;
6769 }
6770};
6771} // namespace
6772
6773/// Pick the cheapest widening for the call \p VPI at \p VF among scalarization,
6774/// vector intrinsic, and vector library variant.
6775static CallWideningDecision decideCallWidening(VPInstruction &VPI,
6776 ArrayRef<VPValue *> Ops,
6777 ElementCount VF,
6778 VPCostContext &CostCtx) {
6779 auto *CI = cast<CallInst>(Val: VPI.getUnderlyingInstr());
6780
6781 // Scalar VFs and calls forced or known to scalarize always replicate.
6782 if (VF.isScalar() || CostCtx.willBeScalarized(I: CI, VF))
6783 return CallWideningDecision::KindTy::Scalarize;
6784
6785 auto *CalledFn = cast<Function>(
6786 Val: VPI.getOperand(N: VPI.getNumOperandsWithoutMask() - 1)->getLiveInIRValue());
6787 Type *ResultTy = VPI.getScalarType();
6788 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
6789 bool MaskRequired = CostCtx.isMaskRequired(I: CI);
6790
6791 // Pseudo intrinsics (assume, lifetime, ...) are always scalarized.
6792 if (ID && VPCostContext::isFreeScalarIntrinsic(ID))
6793 return CallWideningDecision::KindTy::Scalarize;
6794
6795 InstructionCost ScalarCost =
6796 VPReplicateRecipe::computeCallCost(CalledFn, ResultTy, ArgOps: Ops,
6797 /*IsSingleScalar=*/false, VF, Ctx&: CostCtx);
6798
6799 Function *VecFunc =
6800 findVectorVariant(CI, Args: Ops, VF, MaskRequired, PSE&: CostCtx.PSE, L: CostCtx.L);
6801 InstructionCost VecCallCost = InstructionCost::getInvalid();
6802 if (VecFunc)
6803 VecCallCost = VPWidenCallRecipe::computeCallCost(Variant: VecFunc, Ctx&: CostCtx);
6804
6805 // Prefer the intrinsic if it is at least as cheap as scalarizing and any
6806 // available vector variant.
6807 if (ID) {
6808 InstructionCost IntrinsicCost =
6809 VPWidenIntrinsicRecipe::computeCallCost(ID, Operands: Ops, R: VPI, VF, Ctx&: CostCtx);
6810 if (IntrinsicCost.isValid() && ScalarCost >= IntrinsicCost &&
6811 (!VecFunc || VecCallCost >= IntrinsicCost))
6812 return CallWideningDecision::KindTy::Intrinsic;
6813 }
6814
6815 // Otherwise, use a vector library variant when it beats scalarizing.
6816 if (VecFunc && ScalarCost >= VecCallCost)
6817 return {CallWideningDecision::KindTy::VectorVariant, VecFunc};
6818
6819 return CallWideningDecision::KindTy::Scalarize;
6820}
6821
6822void VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
6823 VPRecipeBuilder &RecipeBuilder,
6824 VPCostContext &CostCtx) {
6825 for (VPBasicBlock *VPBB : VPBlockUtils::blocksAs<VPBasicBlock>(
6826 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
6827 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
6828 auto *VPI = dyn_cast<VPInstruction>(Val: &R);
6829 if (!VPI || !VPI->getUnderlyingValue() ||
6830 VPI->getOpcode() != Instruction::Call)
6831 continue;
6832
6833 auto *CI = cast<CallInst>(Val: VPI->getUnderlyingInstr());
6834 SmallVector<VPValue *, 4> Ops(VPI->op_begin(),
6835 VPI->op_begin() + CI->arg_size());
6836
6837 CallWideningDecision Decision =
6838 decideCallWidening(VPI&: *VPI, Ops, VF: Range.Start, CostCtx);
6839 LoopVectorizationPlanner::getDecisionAndClampRange(
6840 Predicate: [&](ElementCount VF) {
6841 return Decision == decideCallWidening(VPI&: *VPI, Ops, VF, CostCtx);
6842 },
6843 Range);
6844
6845 VPSingleDefRecipe *Replacement = nullptr;
6846 switch (Decision.Kind) {
6847 case CallWideningDecision::KindTy::Intrinsic: {
6848 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
6849 Type *ResultTy = VPI->getScalarType();
6850 Replacement = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, *VPI,
6851 *VPI, VPI->getDebugLoc());
6852 break;
6853 }
6854 case CallWideningDecision::KindTy::VectorVariant: {
6855 // Masked variants take the mask as a trailing parameter, so they have
6856 // one more parameter than the original call's arguments.
6857 if (Decision.Variant->arg_size() > Ops.size()) {
6858 VPValue *Mask = VPI->isMasked() ? VPI->getMask() : Plan.getTrue();
6859 Ops.push_back(Elt: Mask);
6860 }
6861 Ops.push_back(Elt: VPI->getOperand(N: VPI->getNumOperandsWithoutMask() - 1));
6862 Replacement = new VPWidenCallRecipe(CI, Decision.Variant, Ops, *VPI,
6863 *VPI, VPI->getDebugLoc());
6864 break;
6865 }
6866 case CallWideningDecision::KindTy::Scalarize:
6867 Replacement = RecipeBuilder.handleReplication(VPI, Range);
6868 break;
6869 }
6870
6871 Replacement->insertBefore(InsertPos: VPI);
6872 VPI->replaceAllUsesWith(New: Replacement);
6873 VPI->eraseFromParent();
6874 }
6875 }
6876}
6877
6878void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
6879 PredicatedScalarEvolution &PSE,
6880 Loop &L, VPCostContext &Ctx,
6881 VFRange &Range) {
6882 if (Plan.hasScalarVFOnly())
6883 return;
6884
6885 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
6886 VPValue *I32VF = nullptr;
6887 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6888 Range: vp_depth_first_shallow(G: VectorLoop->getEntry()))) {
6889 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
6890 auto *LoadR = dyn_cast<VPWidenLoadRecipe>(Val: &R);
6891 // TODO: Support strided store.
6892 // TODO: Transform reverse access into strided access with -1 stride.
6893 // TODO: Transform gather/scatter with uniform address into strided access
6894 // with 0 stride.
6895 // TODO: Transform interleave access into multiple strided accesses.
6896 if (!LoadR || LoadR->isConsecutive())
6897 continue;
6898
6899 VPValue *Ptr = LoadR->getAddr();
6900 // Check if this is a strided access by analyzing the address SCEV for an
6901 // affine addRec.
6902 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: &L);
6903 const SCEV *Start;
6904 const SCEVConstant *Step;
6905 // TODO: Support non-constant loop invariant stride.
6906 if (!match(S: PtrSCEV,
6907 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEVConstant(V&: Step),
6908 L: m_SpecificLoop(L: &L))))
6909 continue;
6910
6911 Type *LoadTy = LoadR->getScalarType();
6912 Align Alignment = LoadR->getAlign();
6913 auto IsProfitable = [&](ElementCount VF) {
6914 Type *DataTy = toVectorTy(Scalar: LoadTy, EC: VF);
6915 if (!Ctx.TTI.isLegalStridedLoadStore(DataType: DataTy, Alignment))
6916 return false;
6917 const InstructionCost CurrentCost = LoadR->computeCost(VF, Ctx);
6918 const InstructionCost StridedLoadStoreCost =
6919 VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost(
6920 IID: Intrinsic::experimental_vp_strided_load, Ty: DataTy,
6921 IsMasked: LoadR->isMasked(), Alignment, Ctx);
6922 return StridedLoadStoreCost < CurrentCost;
6923 };
6924
6925 if (!LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsProfitable,
6926 Range))
6927 continue;
6928
6929 // Invalidate the legacy widening decision so the cost of replaced load is
6930 // not counted during precomputeCosts.
6931 // TODO: Remove once the legacy exit cost computation is retired.
6932 for (ElementCount VF : Range)
6933 Ctx.invalidateWideningDecision(I: &LoadR->getIngredient(), VF);
6934
6935 // Get VF as i32 for the vector length operand.
6936 if (!I32VF) {
6937 VPBuilder Builder(Plan.getVectorPreheader());
6938 I32VF = Builder.createScalarZExtOrTrunc(
6939 Op: &Plan.getVF(), ResultTy: Type::getInt32Ty(C&: Plan.getContext()),
6940 DL: DebugLoc::getUnknown());
6941 }
6942
6943 VPBuilder Builder(LoadR);
6944 // Create the base pointer of strided access.
6945 // TODO: reuse VPDerivedIVRecipe for base pointer computation when it
6946 // supports a general VPValue as the start value.
6947 VPValue *StartVPV =
6948 VPSCEVExpander(Builder, *PSE.getSE(), LoadR->getDebugLoc())
6949 .tryToExpand(S: Start);
6950 if (!StartVPV)
6951 StartVPV = VPBuilder(Plan.getEntry()).createExpandSCEV(Expr: Start);
6952 VPValue *StrideInBytes = Plan.getOrAddLiveIn(V: Step->getValue());
6953 Type *IndexTy = Plan.getDataLayout().getIndexType(PtrTy: Ptr->getScalarType());
6954 assert(IndexTy == StrideInBytes->getScalarType() &&
6955 "Stride type from SCEV must match the index type");
6956 VPValue *CanIV = Builder.createScalarSExtOrTrunc(
6957 Op: VectorLoop->getCanonicalIV(), ResultTy: IndexTy, DL: DebugLoc::getUnknown());
6958 auto *AddRecPtr = cast<SCEVAddRecExpr>(Val: PtrSCEV);
6959 auto *Offset = Builder.createOverflowingOp(
6960 Opcode: Instruction::Mul, Operands: {CanIV, StrideInBytes},
6961 WrapFlags: {AddRecPtr->hasNoUnsignedWrap(), /*HasNSW=*/false});
6962 GEPNoWrapFlags NWFlags = AddRecPtr->hasNoUnsignedWrap()
6963 ? GEPNoWrapFlags::noUnsignedWrap()
6964 : GEPNoWrapFlags::none();
6965 VPValue *BasePtr = Builder.createNoWrapPtrAdd(Ptr: StartVPV, Offset, GEPFlags: NWFlags);
6966
6967 // Create a new vector pointer for strided access.
6968 VPValue *NewPtr = Builder.createVectorPointer(
6969 Ptr: BasePtr, SourceElementTy: Type::getInt8Ty(C&: Plan.getContext()), Stride: StrideInBytes, GEPFlags: NWFlags,
6970 DL: LoadR->getDebugLoc());
6971
6972 VPValue *Mask = LoadR->getMask();
6973 if (!Mask)
6974 Mask = Plan.getTrue();
6975 auto *StridedLoad = Builder.createWidenMemIntrinsic(
6976 VectorIntrinsicID: Intrinsic::experimental_vp_strided_load,
6977 CallArguments: {NewPtr, StrideInBytes, Mask, I32VF}, Ty: LoadTy, Alignment, MD: *LoadR,
6978 DL: LoadR->getDebugLoc());
6979 LoadR->replaceAllUsesWith(New: StridedLoad);
6980 }
6981 }
6982}
6983