1//===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements a set of utility VPlan to VPlan transformations.
11///
12//===----------------------------------------------------------------------===//
13
14#include "VPlanTransforms.h"
15#include "VPRecipeBuilder.h"
16#include "VPlan.h"
17#include "VPlanAnalysis.h"
18#include "VPlanCFG.h"
19#include "VPlanDominatorTree.h"
20#include "VPlanHelpers.h"
21#include "VPlanPatternMatch.h"
22#include "VPlanUtils.h"
23#include "llvm/ADT/APInt.h"
24#include "llvm/ADT/PostOrderIterator.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallPtrSet.h"
28#include "llvm/ADT/TypeSwitch.h"
29#include "llvm/Analysis/IVDescriptors.h"
30#include "llvm/Analysis/Loads.h"
31#include "llvm/Analysis/LoopInfo.h"
32#include "llvm/Analysis/MemoryLocation.h"
33#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
34#include "llvm/Analysis/ScopedNoAliasAA.h"
35#include "llvm/Analysis/VectorUtils.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/MDBuilder.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/IR/ProfDataUtils.h"
40#include "llvm/Support/Casting.h"
41#include "llvm/Support/CommandLine.h"
42#include "llvm/Support/TypeSize.h"
43#include "llvm/Transforms/Utils/LoopUtils.h"
44
45using namespace llvm;
46using namespace LoopVectorizationUtils;
47using namespace VPlanPatternMatch;
48using namespace SCEVPatternMatch;
49
50/// Returns the metadata attached to \p R, or an empty set for a recipe that
51/// does not carry any.
52static VPIRMetadata getMetadataOf(VPRecipeBase *R) {
53 if (auto *MD = dyn_cast<VPIRMetadata>(Val: R))
54 return *MD;
55 return {};
56}
57
58// TODO: Remove this once the partial reduction intrinsics are no worse than
59// normal vector operations.
60static cl::opt<bool> UsePartialReductionsByDefault(
61 "use-partial-reductions-by-default", cl::init(Val: false), cl::Hidden,
62 cl::desc("Use partial reduction intrinsics for "
63 "all supported unordered reductions."));
64
65bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
66 VPlan &Plan, const TargetLibraryInfo &TLI, PredicatedScalarEvolution &PSE,
67 Loop *OuterLoop) {
68
69 // Returns true if the access of \p AccessTy at \p Addr can be widened to a
70 // consecutive vector access.
71 auto IsConsecutiveAccess = [&](VPValue *Addr, Type *AccessTy) {
72 return !hasIrregularType(Ty: AccessTy, DL: Plan.getDataLayout()) &&
73 vputils::getConstantStride(Addr, AccessTy, PSE, L: OuterLoop) == 1;
74 };
75
76 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
77 Plan.getVectorLoopRegion());
78 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
79 // Skip blocks outside region
80 if (!VPBB->getParent())
81 break;
82 VPRecipeBase *Term = VPBB->getTerminator();
83 auto EndIter = Term ? Term->getIterator() : VPBB->end();
84 // Introduce each ingredient into VPlan.
85 for (VPRecipeBase &Ingredient :
86 make_early_inc_range(Range: make_range(x: VPBB->begin(), y: EndIter))) {
87
88 VPValue *VPV = Ingredient.getVPSingleValue();
89 if (!VPV->getUnderlyingValue())
90 continue;
91
92 Instruction *Inst = cast<Instruction>(Val: VPV->getUnderlyingValue());
93
94 VPRecipeBase *NewRecipe = nullptr;
95 if (auto *PhiR = dyn_cast<VPPhi>(Val: &Ingredient)) {
96 auto *Phi = cast<PHINode>(Val: PhiR->getUnderlyingValue());
97 NewRecipe = new VPWidenPHIRecipe(PhiR->operands(), PhiR->getDebugLoc(),
98 Phi->getName());
99 } else if (auto *VPI = dyn_cast<VPInstruction>(Val: &Ingredient)) {
100 assert(!isa<PHINode>(Inst) && "phis should be handled above");
101 // Create VPWidenMemoryRecipe for loads and stores.
102 if (LoadInst *Load = dyn_cast<LoadInst>(Val: Inst)) {
103 bool IsConsecutive =
104 IsConsecutiveAccess(VPI->getOperand(N: 0), VPI->getScalarType());
105 NewRecipe = new VPWidenLoadRecipe(*Load, Ingredient.getOperand(N: 0),
106 nullptr /*Mask*/, IsConsecutive,
107 *VPI, Ingredient.getDebugLoc());
108 } else if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst)) {
109 bool IsConsecutive = IsConsecutiveAccess(
110 VPI->getOperand(N: 1), VPI->getOperand(N: 0)->getScalarType());
111 NewRecipe = new VPWidenStoreRecipe(
112 *Store, Ingredient.getOperand(N: 1), Ingredient.getOperand(N: 0),
113 nullptr /*Mask*/, IsConsecutive, *VPI, Ingredient.getDebugLoc());
114 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Inst)) {
115 NewRecipe = new VPWidenGEPRecipe(GEP->getSourceElementType(),
116 Ingredient.operands(), *VPI,
117 Ingredient.getDebugLoc(), GEP);
118 } else if (CallInst *CI = dyn_cast<CallInst>(Val: Inst)) {
119 Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, TLI: &TLI);
120 if (VectorID == Intrinsic::not_intrinsic)
121 return false;
122
123 // The noalias.scope.decl intrinsic declares a noalias scope that
124 // is valid for a single iteration. Emitting it as a single-scalar
125 // replicate would incorrectly extend the scope across multiple
126 // original iterations packed into one vector iteration.
127 // FIXME: If we want to vectorize this loop, then we have to drop
128 // all the associated !alias.scope and !noalias.
129 if (VectorID == Intrinsic::experimental_noalias_scope_decl)
130 return false;
131
132 // These intrinsics are recognized by getVectorIntrinsicIDForCall
133 // but are not widenable. Emit them as replicate instead of widening.
134 if (VectorID == Intrinsic::assume ||
135 VectorID == Intrinsic::lifetime_end ||
136 VectorID == Intrinsic::lifetime_start ||
137 VectorID == Intrinsic::sideeffect ||
138 VectorID == Intrinsic::pseudoprobe) {
139 // If the operand of llvm.assume holds before vectorization, it will
140 // also hold per lane.
141 // llvm.pseudoprobe requires to be duplicated per lane for accurate
142 // sample count.
143 const bool IsSingleScalar = VectorID != Intrinsic::assume &&
144 VectorID != Intrinsic::pseudoprobe;
145 NewRecipe = new VPReplicateRecipe(CI, Ingredient.operands(),
146 /*IsSingleScalar=*/IsSingleScalar,
147 /*Mask=*/nullptr, *VPI, *VPI,
148 Ingredient.getDebugLoc());
149 } else {
150 NewRecipe = new VPWidenIntrinsicRecipe(
151 *CI, VectorID, drop_end(RangeOrContainer: Ingredient.operands()), CI->getType(),
152 VPIRFlags(*CI), *VPI, CI->getDebugLoc());
153 }
154 } else if (auto *CI = dyn_cast<CastInst>(Val: Inst)) {
155 NewRecipe = new VPWidenCastRecipe(
156 CI->getOpcode(), Ingredient.getOperand(N: 0), CI->getType(), CI,
157 VPIRFlags(*CI), VPIRMetadata(*CI));
158 } else {
159 NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI,
160 *VPI, Ingredient.getDebugLoc());
161 }
162 } else {
163 assert(isa<VPWidenIntOrFpInductionRecipe>(&Ingredient) &&
164 "inductions must be created earlier");
165 continue;
166 }
167
168 NewRecipe->insertBefore(InsertPos: &Ingredient);
169 if (NewRecipe->getNumDefinedValues() == 1)
170 VPV->replaceAllUsesWith(New: NewRecipe->getVPSingleValue());
171 else
172 assert(NewRecipe->getNumDefinedValues() == 0 &&
173 "Only recpies with zero or one defined values expected");
174 Ingredient.eraseFromParent();
175 }
176 }
177 return true;
178}
179
180/// Helper for extra no-alias checks via known-safe recipe and SCEV.
181class SinkStoreInfo {
182 SmallPtrSet<VPReplicateRecipe *, 4> ExcludeRecipes;
183 VPReplicateRecipe &GroupLeader;
184 PredicatedScalarEvolution *PSE = nullptr;
185 const Loop *L = nullptr;
186
187 // Return true if \p A and \p B are known to not alias for all VFs in the
188 // plan, checked via the distance between the accesses
189 bool isNoAliasViaDistance(VPReplicateRecipe *A, VPReplicateRecipe *B) const {
190 if (A->getOpcode() != Instruction::Store ||
191 B->getOpcode() != Instruction::Store)
192 return false;
193
194 if (!PSE || !L)
195 return A == B;
196
197 VPValue *AddrA = A->getOperand(N: 1);
198 const SCEV *SCEVA = vputils::getSCEVExprForVPValue(V: AddrA, PSE&: *PSE, L);
199 VPValue *AddrB = B->getOperand(N: 1);
200 const SCEV *SCEVB = vputils::getSCEVExprForVPValue(V: AddrB, PSE&: *PSE, L);
201 if (isa<SCEVCouldNotCompute>(Val: SCEVA) || isa<SCEVCouldNotCompute>(Val: SCEVB))
202 return false;
203
204 const APInt *Distance;
205 ScalarEvolution &SE = *PSE->getSE();
206 if (!match(S: SE.getMinusSCEV(LHS: SCEVA, RHS: SCEVB), P: m_scev_APInt(C&: Distance)))
207 return false;
208
209 const DataLayout &DL = SE.getDataLayout();
210 Type *TyA = A->getOperand(N: 0)->getScalarType();
211 uint64_t SizeA = DL.getTypeStoreSize(Ty: TyA);
212 Type *TyB = B->getOperand(N: 0)->getScalarType();
213 uint64_t SizeB = DL.getTypeStoreSize(Ty: TyB);
214
215 // Use the maximum store size to ensure no overlap from either direction.
216 // Currently only handles fixed sizes, as it is only used for
217 // replicating VPReplicateRecipes.
218 uint64_t MaxStoreSize = std::max(a: SizeA, b: SizeB);
219
220 auto VFs = B->getParent()->getPlan()->vectorFactors();
221 ElementCount MaxVF = *max_element(Range&: VFs, C: ElementCount::isKnownLT);
222 if (MaxVF.isScalable())
223 return false;
224 return Distance->abs().uge(RHS: MaxVF.getFixedValue() * MaxStoreSize);
225 }
226
227public:
228 SinkStoreInfo(ArrayRef<VPReplicateRecipe *> ExcludeRecipes,
229 VPReplicateRecipe &GroupLeader, PredicatedScalarEvolution &PSE,
230 const Loop &L)
231 : ExcludeRecipes(ExcludeRecipes.begin(), ExcludeRecipes.end()),
232 GroupLeader(GroupLeader), PSE(&PSE), L(&L) {}
233
234 SinkStoreInfo(VPReplicateRecipe &GroupLeader) : GroupLeader(GroupLeader) {}
235
236 /// Return true if \p R should be skipped during alias checking, either
237 /// because it's in the exclude set or because no-alias can be proven via
238 /// SCEV.
239 bool shouldSkip(VPRecipeBase &R) const {
240 auto *Store = dyn_cast<VPReplicateRecipe>(Val: &R);
241 return ExcludeRecipes.contains(Ptr: Store) ||
242 (Store && isNoAliasViaDistance(A: Store, B: &GroupLeader));
243 }
244};
245
246/// Check if a memory operation doesn't alias with memory operations using
247/// scoped noalias metadata, in blocks in the single-successor chain between \p
248/// FirstBB and \p LastBB. If \p SinkInfo is std::nullopt, only recipes that may
249/// write to memory are checked (for load hoisting). Otherwise recipes that both
250/// read and write memory are checked, and SCEV is used to prove no-alias
251/// between the group leader and other replicate recipes (for store sinking).
252static bool
253canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
254 VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
255 std::optional<SinkStoreInfo> SinkInfo = {}) {
256 bool CheckReads = SinkInfo.has_value();
257 for (VPBasicBlock *VPBB :
258 VPBlockUtils::blocksInSingleSuccessorChainBetween(FirstBB, LastBB)) {
259 for (VPRecipeBase &R : *VPBB) {
260 if (SinkInfo && SinkInfo->shouldSkip(R))
261 continue;
262
263 // Skip recipes that don't need checking.
264 if (!R.mayWriteToMemory() && !(CheckReads && R.mayReadFromMemory()))
265 continue;
266
267 auto Loc = vputils::getMemoryLocation(R);
268 if (!Loc)
269 // Conservatively assume aliasing for memory operations without
270 // location.
271 return false;
272
273 if (ScopedNoAliasAAResult::alias(LocA: *Loc, LocB: MemLoc) != AliasResult::NoAlias)
274 return false;
275 }
276 }
277 return true;
278}
279
280/// Get the value type of the replicate load or store. \p IsLoad indicates
281/// whether it is a load.
282static Type *getLoadStoreValueType(VPReplicateRecipe *R, bool IsLoad) {
283 return (IsLoad ? R : R->getOperand(N: 0))->getScalarType();
284}
285
286/// Collect either replicated Loads or Stores grouped by their address SCEV and
287/// their load-store type, in a deep-traversal of the vector loop region in \p
288/// Plan.
289template <unsigned Opcode>
290static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
291collectGroupedReplicateMemOps(
292 VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L,
293 function_ref<bool(VPReplicateRecipe *)> FilterFn) {
294 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
295 "Only Load and Store opcodes supported");
296 constexpr bool IsLoad = (Opcode == Instruction::Load);
297 SmallDenseMap<std::pair<const SCEV *, const Type *>,
298 SmallVector<VPReplicateRecipe *, 4>>
299 RecipesByAddressAndType;
300 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
301 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
302 for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(Range&: *VPBB)) {
303 if (RepR.getOpcode() != Opcode || !FilterFn(&RepR))
304 continue;
305
306 // For loads, operand 0 is address; for stores, operand 1 is address.
307 VPValue *Addr = RepR.getOperand(N: IsLoad ? 0 : 1);
308 const Type *LoadStoreTy = getLoadStoreValueType(R: &RepR, IsLoad);
309 const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(V: Addr, PSE, L);
310 if (!isa<SCEVCouldNotCompute>(Val: AddrSCEV))
311 RecipesByAddressAndType[{AddrSCEV, LoadStoreTy}].push_back(Elt: &RepR);
312 }
313 }
314 auto Groups = to_vector(Range: RecipesByAddressAndType.values());
315 VPDominatorTree VPDT(Plan);
316 for (auto &Group : Groups) {
317 // Sort mem ops by dominance order, with earliest (most dominating) first.
318 stable_sort(Group, [&VPDT](VPReplicateRecipe *A, VPReplicateRecipe *B) {
319 return VPDT.properlyDominates(A, B);
320 });
321 }
322 return Groups;
323}
324
325static bool sinkScalarOperands(VPlan &Plan) {
326 auto Iter = vp_depth_first_deep(G: Plan.getEntry());
327 bool ScalarVFOnly = Plan.hasScalarVFOnly();
328 bool Changed = false;
329
330 SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList;
331 auto InsertIfValidSinkCandidate = [ScalarVFOnly, &WorkList](
332 VPBasicBlock *SinkTo, VPValue *Op) {
333 auto *Candidate = dyn_cast<VPSingleDefRecipe>(Val: Op);
334 if (!isa_and_nonnull<VPReplicateRecipe, VPScalarIVStepsRecipe,
335 VPInstruction>(Val: Candidate))
336 return;
337
338 if (Candidate->getParent() == SinkTo ||
339 all_of(Range: Candidate->operands(),
340 P: [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); }) ||
341 vputils::cannotHoistOrSinkRecipe(R: *Candidate, /*Sinking=*/true))
342 return;
343
344 if (!ScalarVFOnly && !vputils::doesGeneratePerAllLanes(R: Candidate))
345 return;
346
347 // Only single-scalar VPInstructions can be sunk.
348 if (auto *VPI = dyn_cast<VPInstruction>(Val: Candidate))
349 if (!vputils::isSingleScalar(VPV: VPI))
350 return;
351
352 WorkList.insert(X: {SinkTo, Candidate});
353 };
354
355 // First, collect the operands of all recipes in replicate blocks as seeds for
356 // sinking.
357 for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Range&: Iter)) {
358 VPBasicBlock *EntryVPBB = VPR->getEntryBasicBlock();
359 if (!VPR->isReplicator() || EntryVPBB->getSuccessors().size() != 2)
360 continue;
361 VPBasicBlock *VPBB = cast<VPBasicBlock>(Val: EntryVPBB->getSuccessors().front());
362 if (VPBB->getSingleSuccessor() != VPR->getExitingBasicBlock())
363 continue;
364 for (auto &Recipe : *VPBB)
365 for (VPValue *Op : Recipe.operands())
366 InsertIfValidSinkCandidate(VPBB, Op);
367 }
368
369 // Try to sink each replicate or scalar IV steps recipe in the worklist.
370 for (unsigned I = 0; I != WorkList.size(); ++I) {
371 VPBasicBlock *SinkTo;
372 VPSingleDefRecipe *SinkCandidate;
373 std::tie(args&: SinkTo, args&: SinkCandidate) = WorkList[I];
374
375 // All recipe users of SinkCandidate must be in the same block SinkTo or all
376 // users outside of SinkTo must only use the first lane of SinkCandidate. In
377 // the latter case, we need to duplicate SinkCandidate.
378 auto UsersOutsideSinkTo =
379 make_filter_range(Range: SinkCandidate->users(), Pred: [SinkTo](VPUser *U) {
380 return cast<VPRecipeBase>(Val: U)->getParent() != SinkTo;
381 });
382 if (any_of(Range&: UsersOutsideSinkTo, P: [SinkCandidate](VPUser *U) {
383 return !U->usesFirstLaneOnly(Op: SinkCandidate);
384 }))
385 continue;
386 bool NeedsDuplicating = !UsersOutsideSinkTo.empty();
387
388 if (NeedsDuplicating) {
389 if (ScalarVFOnly)
390 continue;
391 VPSingleDefRecipe *Clone;
392 if (auto *SinkCandidateRepR =
393 dyn_cast<VPReplicateRecipe>(Val: SinkCandidate)) {
394 // TODO: Handle converting to uniform recipes as separate transform,
395 // then cloning should be sufficient here.
396 Clone = VPBuilder::createSingleScalarOp(
397 Opcode: SinkCandidateRepR->getOpcode(), Operands: SinkCandidate->operands(),
398 /*Mask=*/nullptr, Flags: *SinkCandidateRepR, Metadata: *SinkCandidateRepR,
399 DL: SinkCandidate->getDebugLoc(), ResultTy: SinkCandidate->getScalarType(),
400 UV: SinkCandidate->getUnderlyingInstr());
401 // TODO: add ".cloned" suffix to name of Clone's VPValue.
402 } else {
403 Clone = SinkCandidate->clone();
404 }
405
406 Clone->insertBefore(InsertPos: SinkCandidate);
407 SinkCandidate->replaceUsesWithIf(New: Clone, ShouldReplace: [SinkTo](VPUser &U) {
408 return cast<VPRecipeBase>(Val: &U)->getParent() != SinkTo;
409 });
410 }
411 SinkCandidate->moveBefore(BB&: *SinkTo, I: SinkTo->getFirstNonPhi());
412 for (VPValue *Op : SinkCandidate->operands())
413 InsertIfValidSinkCandidate(SinkTo, Op);
414 Changed = true;
415 }
416 return Changed;
417}
418
419/// If \p R is a triangle region, return the 'then' block of the triangle.
420static VPBasicBlock *getPredicatedThenBlock(VPRegionBlock *R) {
421 auto *EntryBB = cast<VPBasicBlock>(Val: R->getEntry());
422 if (EntryBB->getNumSuccessors() != 2)
423 return nullptr;
424
425 auto *Succ0 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[0]);
426 auto *Succ1 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[1]);
427 if (!Succ0 || !Succ1)
428 return nullptr;
429
430 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
431 return nullptr;
432 if (Succ0->getSingleSuccessor() == Succ1)
433 return Succ0;
434 if (Succ1->getSingleSuccessor() == Succ0)
435 return Succ1;
436 return nullptr;
437}
438
439// Merge replicate regions in their successor region, if a replicate region
440// is connected to a successor replicate region with the same predicate by a
441// single, empty VPBasicBlock.
442static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
443 SmallPtrSet<VPRegionBlock *, 4> TransformedRegions;
444
445 // Collect replicate regions followed by an empty block, followed by another
446 // replicate region with matching masks to process front. This is to avoid
447 // iterator invalidation issues while merging regions.
448 SmallVector<VPRegionBlock *, 8> WorkList;
449 for (VPRegionBlock *Region1 : VPBlockUtils::blocksOnly<VPRegionBlock>(
450 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
451 if (!Region1->isReplicator())
452 continue;
453 auto *MiddleBasicBlock =
454 dyn_cast_or_null<VPBasicBlock>(Val: Region1->getSingleSuccessor());
455 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
456 continue;
457
458 auto *Region2 =
459 dyn_cast_or_null<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
460 if (!Region2 || !Region2->isReplicator())
461 continue;
462
463 VPValue *Mask1 = Region1->getEntryBranchOnMask()->getOperand(N: 0);
464 VPValue *Mask2 = Region2->getEntryBranchOnMask()->getOperand(N: 0);
465 if (!Mask1 || Mask1 != Mask2)
466 continue;
467
468 assert(Mask1 && Mask2 && "both region must have conditions");
469 WorkList.push_back(Elt: Region1);
470 }
471
472 // Move recipes from Region1 to its successor region, if both are triangles.
473 for (VPRegionBlock *Region1 : WorkList) {
474 if (TransformedRegions.contains(Ptr: Region1))
475 continue;
476 auto *MiddleBasicBlock = cast<VPBasicBlock>(Val: Region1->getSingleSuccessor());
477 auto *Region2 = cast<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
478
479 VPBasicBlock *Then1 = getPredicatedThenBlock(R: Region1);
480 VPBasicBlock *Then2 = getPredicatedThenBlock(R: Region2);
481 if (!Then1 || !Then2)
482 continue;
483
484 // The merged region is entered whenever either of the original regions was,
485 // so use the higher, i.e. more conservative, of their entry frequencies.
486 // If only one of the two is known, the higher one is unknown, so the
487 // result must be unknown too.
488 VPBranchOnMaskRecipe *Guard2 = Region2->getEntryBranchOnMask();
489 std::optional<VPExecutionFrequency> Freq1 =
490 Region1->getEntryBranchOnMask()->getExecutionFrequency();
491 std::optional<VPExecutionFrequency> Freq2 = Guard2->getExecutionFrequency();
492 if (Freq1 && Freq2) {
493 if (Freq2->Freq < Freq1->Freq) {
494 // Freq1's frequency is taken, but it is only as trustworthy as the
495 // less trustworthy of the two.
496 Freq1.emplace(args: Freq1->Freq, args: Freq1->IsEstimated || Freq2->IsEstimated);
497 Guard2->setExecutionFrequency(Freq: Freq1, Ctx&: Plan.getContext());
498 }
499 } else if (Freq2) {
500 Guard2->clearExecutionFrequency();
501 }
502
503 // Note: No fusion-preventing memory dependencies are expected in either
504 // region. Such dependencies should be rejected during earlier dependence
505 // checks, which guarantee accesses can be re-ordered for vectorization.
506 //
507 // Move recipes to the successor region.
508 for (VPRecipeBase &ToMove : make_early_inc_range(Range: reverse(C&: *Then1)))
509 ToMove.moveBefore(BB&: *Then2, I: Then2->getFirstNonPhi());
510
511 auto *Merge1 = cast<VPBasicBlock>(Val: Then1->getSingleSuccessor());
512 auto *Merge2 = cast<VPBasicBlock>(Val: Then2->getSingleSuccessor());
513
514 // Move VPPredInstPHIRecipes from the merge block to the successor region's
515 // merge block. Update all users inside the successor region to use the
516 // original values.
517 for (VPRecipeBase &Phi1ToMove : make_early_inc_range(Range: reverse(C&: *Merge1))) {
518 VPValue *PredInst1 =
519 cast<VPPredInstPHIRecipe>(Val: &Phi1ToMove)->getOperand(N: 0);
520 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
521 Phi1ToMoveV->replaceUsesWithIf(New: PredInst1, ShouldReplace: [Then2](VPUser &U) {
522 return cast<VPRecipeBase>(Val: &U)->getParent() == Then2;
523 });
524
525 // Remove phi recipes that are unused after merging the regions.
526 if (Phi1ToMove.getVPSingleValue()->user_empty()) {
527 Phi1ToMove.eraseFromParent();
528 continue;
529 }
530 Phi1ToMove.moveBefore(BB&: *Merge2, I: Merge2->begin());
531 }
532
533 // Remove the dead recipes in Region1's entry block.
534 for (VPRecipeBase &R :
535 make_early_inc_range(Range: reverse(C&: *Region1->getEntryBasicBlock())))
536 R.eraseFromParent();
537
538 // Finally, remove the first region.
539 for (VPBlockBase *Pred : make_early_inc_range(Range&: Region1->getPredecessors())) {
540 VPBlockUtils::disconnectBlocks(From: Pred, To: Region1);
541 VPBlockUtils::connectBlocks(From: Pred, To: MiddleBasicBlock);
542 }
543 VPBlockUtils::disconnectBlocks(From: Region1, To: MiddleBasicBlock);
544 TransformedRegions.insert(Ptr: Region1);
545 }
546
547 return !TransformedRegions.empty();
548}
549
550static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
551 VPRegionBlock *ParentRegion,
552 VPlan &Plan) {
553 Instruction *Instr = PredRecipe->getUnderlyingInstr();
554 // Build the triangular if-then region.
555 std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str();
556 assert(Instr->getParent() && "Predicated instruction not in any basic block");
557 auto *BlockInMask = PredRecipe->getMask();
558 auto *MaskDef = BlockInMask->getDefiningRecipe();
559 auto *BOMRecipe = new VPBranchOnMaskRecipe(
560 BlockInMask, MaskDef ? MaskDef->getDebugLoc() : DebugLoc::getUnknown());
561 auto *Entry =
562 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".entry", Recipe: BOMRecipe);
563
564 // Replace predicated replicate recipe with a replicate recipe without a
565 // mask but in the replicate region.
566 auto *RecipeWithoutMask = new VPReplicateRecipe(
567 PredRecipe->getUnderlyingInstr(), PredRecipe->operandsWithoutMask(),
568 PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
569 PredRecipe->getDebugLoc());
570 // The predicated recipe executes exactly when the guarding branch-on-mask is
571 // taken, so move its execution frequency there.
572 BOMRecipe->setExecutionFrequency(Freq: RecipeWithoutMask->getExecutionFrequency(),
573 Ctx&: Plan.getContext());
574 RecipeWithoutMask->clearExecutionFrequency();
575 auto *Pred =
576 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".if", Recipe: RecipeWithoutMask);
577 auto *Exiting = Plan.createVPBasicBlock(Name: Twine(RegionName) + ".continue");
578 VPRegionBlock *Region =
579 Plan.createReplicateRegion(Entry, Exiting, Name: RegionName);
580
581 // Note: first set Entry as region entry and then connect successors starting
582 // from it in order, to propagate the "parent" of each VPBasicBlock.
583 Region->setParent(ParentRegion);
584 VPBlockUtils::insertTwoBlocksAfter(IfTrue: Pred, IfFalse: Exiting, BlockPtr: Entry);
585 VPBlockUtils::connectBlocks(From: Pred, To: Exiting);
586
587 if (!PredRecipe->user_empty()) {
588 auto *PHIRecipe = new VPPredInstPHIRecipe(RecipeWithoutMask,
589 RecipeWithoutMask->getDebugLoc());
590 Exiting->appendRecipe(Recipe: PHIRecipe);
591 PredRecipe->replaceAllUsesWith(New: PHIRecipe);
592 }
593 PredRecipe->eraseFromParent();
594 return Region;
595}
596
597static void addReplicateRegions(VPlan &Plan) {
598 SmallVector<VPReplicateRecipe *> WorkList;
599 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
600 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
601 for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(Range&: *VPBB))
602 if (RepR.isPredicated())
603 WorkList.push_back(Elt: &RepR);
604 }
605
606 unsigned BBNum = 0;
607 for (VPReplicateRecipe *RepR : WorkList) {
608 VPBasicBlock *CurrentBlock = RepR->getParent();
609 VPBasicBlock *SplitBlock = CurrentBlock->splitAt(SplitAt: RepR->getIterator());
610
611 BasicBlock *OrigBB = RepR->getUnderlyingInstr()->getParent();
612 SplitBlock->setName(
613 OrigBB->hasName() ? OrigBB->getName() + "." + Twine(BBNum++) : "");
614 // Record predicated instructions for above packing optimizations.
615 VPRegionBlock *Region =
616 createReplicateRegion(PredRecipe: RepR, ParentRegion: CurrentBlock->getParent(), Plan);
617 VPBlockUtils::insertOnEdge(From: CurrentBlock, To: SplitBlock, BlockPtr: Region);
618
619 VPRegionBlock *ParentRegion = Region->getParent();
620 if (ParentRegion && ParentRegion->getExiting() == CurrentBlock)
621 ParentRegion->setExiting(SplitBlock);
622 }
623}
624
625bool VPlanTransforms::mergeBlocksIntoPredecessors(VPlan &Plan) {
626 SmallVector<VPBasicBlock *> WorkList;
627 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
628 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
629 // Don't fold the blocks in the skeleton of the Plan into their single
630 // predecessors for now.
631 // TODO: Remove restriction once more of the skeleton is modeled in VPlan.
632 if (!VPBB->getParent())
633 continue;
634 auto *PredVPBB =
635 dyn_cast_or_null<VPBasicBlock>(Val: VPBB->getSinglePredecessor());
636 if (!PredVPBB || PredVPBB->getNumSuccessors() != 1 ||
637 isa<VPIRBasicBlock>(Val: PredVPBB))
638 continue;
639 WorkList.push_back(Elt: VPBB);
640 }
641
642 for (VPBasicBlock *VPBB : WorkList) {
643 VPBasicBlock *PredVPBB = cast<VPBasicBlock>(Val: VPBB->getSinglePredecessor());
644 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
645 R.moveBefore(BB&: *PredVPBB, I: PredVPBB->end());
646 VPBlockUtils::disconnectBlocks(From: PredVPBB, To: VPBB);
647 auto *ParentRegion = VPBB->getParent();
648 if (ParentRegion && ParentRegion->getExiting() == VPBB)
649 ParentRegion->setExiting(PredVPBB);
650 VPBlockUtils::transferSuccessors(Old: VPBB, New: PredVPBB);
651 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
652 }
653 return !WorkList.empty();
654}
655
656void VPlanTransforms::createAndOptimizeReplicateRegions(VPlan &Plan) {
657 // Convert masked VPReplicateRecipes to if-then region blocks.
658 addReplicateRegions(Plan);
659
660 bool ShouldSimplify = true;
661 while (ShouldSimplify) {
662 ShouldSimplify = sinkScalarOperands(Plan);
663 ShouldSimplify |= mergeReplicateRegionsIntoSuccessors(Plan);
664 ShouldSimplify |= mergeBlocksIntoPredecessors(Plan);
665 }
666}
667
668/// Remove redundant casts of inductions.
669///
670/// Such redundant casts are casts of induction variables that can be ignored,
671/// because we already proved that the casted phi is equal to the uncasted phi
672/// in the vectorized loop. There is no need to vectorize the cast - the same
673/// value can be used for both the phi and casts in the vector loop.
674static void removeRedundantInductionCasts(VPlan &Plan) {
675 for (VPWidenIntOrFpInductionRecipe &IV :
676 make_isa_range<VPWidenIntOrFpInductionRecipe>(
677 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
678 if (IV.getTruncInst())
679 continue;
680
681 // A sequence of IR Casts has potentially been recorded for IV, which
682 // *must be bypassed* when the IV is vectorized, because the vectorized IV
683 // will produce the desired casted value. This sequence forms a def-use
684 // chain and is provided in reverse order, ending with the cast that uses
685 // the IV phi. Search for the recipe of the last cast in the chain and
686 // replace it with the original IV. Note that only the final cast is
687 // expected to have users outside the cast-chain and the dead casts left
688 // over will be cleaned up later.
689 ArrayRef<Instruction *> Casts = IV.getInductionDescriptor().getCastInsts();
690 VPValue *FindMyCast = &IV;
691 for (Instruction *IRCast : reverse(C&: Casts)) {
692 VPSingleDefRecipe *FoundUserCast = nullptr;
693 for (auto *U : FindMyCast->users()) {
694 auto *UserCast = dyn_cast<VPSingleDefRecipe>(Val: U);
695 if (UserCast && UserCast->getUnderlyingValue() == IRCast) {
696 FoundUserCast = UserCast;
697 break;
698 }
699 }
700 // A cast recipe in the chain may have been removed by earlier DCE.
701 if (!FoundUserCast)
702 break;
703 FindMyCast = FoundUserCast;
704 }
705 if (FindMyCast != &IV)
706 FindMyCast->replaceAllUsesWith(New: &IV);
707 }
708}
709
710/// If R is a phi-like recipe starting a dead cycle of recipes, erase all
711/// reachable recipes of the dead cycle and return true. Otherwise leave the
712/// plan unchanged and return false.
713static bool tryToRemoveDeadCycle(VPRecipeBase *R) {
714 auto *PhiR = dyn_cast<VPSingleDefRecipe>(Val: R);
715 if (!PhiR || !isa<VPPhi, VPReductionPHIRecipe>(Val: R))
716 return false;
717
718 // The transitive users of PhiR are closed under users, so the cycle is dead
719 // if every one of them can be erased.
720 for (VPUser *U : vputils::collectUsersRecursively(V: PhiR)) {
721 auto *R = cast<VPRecipeBase>(Val: U);
722 // Bail out if a user must be retained, or if it is a phi-like recipe other
723 // than PhiR;
724 if (R->mayHaveSideEffects() || (R != PhiR && isa<VPPhiAccessors>(Val: R)))
725 return false;
726 }
727
728 // Break the cycle by replacing PhiR with its first incoming value, which is
729 // defined outside the cycle. That leaves the rest of the cycle dead.
730 PhiR->replaceAllUsesWith(New: PhiR->getOperand(N: 0));
731 SmallVector<VPValue *> Incoming(PhiR->operands());
732 PhiR->eraseFromParent();
733 for (VPValue *Op : Incoming)
734 vputils::recursivelyDeleteDeadRecipes(V: Op);
735 return true;
736}
737
738void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
739 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
740 Plan.getEntry());
741 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
742 // The recipes in the block are processed in reverse order, to catch chains
743 // of dead recipes.
744 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB)))
745 if (vputils::isDeadRecipe(R))
746 R.eraseFromParent();
747
748 // Erase dead cycles starting at one of VPBB's phi-like recipes. Erasing a
749 // cycle may also erase other phi-like recipes of VPBB, so restart the scan
750 // of the phi section after each removal. This terminates, as each removal
751 // erases the cycle's phi.
752 bool Changed = true;
753 while (Changed) {
754 Changed = false;
755 for (VPRecipeBase &R : VPBB->phis()) {
756 if (tryToRemoveDeadCycle(R: &R)) {
757 Changed = true;
758 break;
759 }
760 }
761 }
762 }
763}
764
765/// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd
766/// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as
767/// VPWidenPointerInductionRecipe will generate vectors only. If some users
768/// require vectors while other require scalars, the scalar uses need to extract
769/// the scalars from the generated vectors (Note that this is different to how
770/// int/fp inductions are handled). Legalize extract-from-ends using uniform
771/// VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so
772/// the correct end value is available. Also optimize
773/// VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by
774/// providing them scalar steps built on the canonical scalar IV and update the
775/// original IV's users. This is an optional optimization to reduce the needs of
776/// vector extracts.
777static void legalizeAndOptimizeInductions(VPlan &Plan) {
778 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
779 bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly();
780
781 SmallVector<VPWidenInductionRecipe *> WideIVs;
782 for (VPWidenInductionRecipe &PhiR :
783 make_isa_range<VPWidenInductionRecipe>(Range: HeaderVPBB->phis()))
784 WideIVs.push_back(Elt: &PhiR);
785
786 // Try to narrow wide and replicating recipes to uniform recipes, based on
787 // VPlan analysis.
788 // TODO: Apply to all recipes in the future, to replace legacy uniformity
789 // analysis.
790 for (VPWidenInductionRecipe *PhiR : WideIVs) {
791 auto Users = vputils::collectUsersRecursively(V: PhiR);
792 for (VPUser *U : reverse(C&: Users)) {
793 auto *Def = dyn_cast<VPRecipeWithIRFlags>(Val: U);
794 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
795 // Skip recipes that shouldn't be narrowed.
796 if (!Def ||
797 !isa<VPReplicateRecipe, VPWidenRecipe, VPWidenGEPRecipe>(Val: Def) ||
798 Def->user_empty() || !Def->getUnderlyingValue() ||
799 (RepR && (RepR->isSingleScalar() || RepR->isPredicated())))
800 continue;
801
802 // Skip recipes that may have other lanes than their first used.
803 if (!vputils::isSingleScalar(VPV: Def) && !vputils::onlyFirstLaneUsed(Def))
804 continue;
805
806 // TODO: Support scalarizing ExtractValue.
807 if (match(V: Def,
808 P: m_Binary<Instruction::ExtractValue>(Op0: m_VPValue(), Op1: m_VPValue())))
809 continue;
810
811 auto *Clone = VPBuilder::createSingleScalarOp(
812 Opcode: Def->getUnderlyingInstr()->getOpcode(), Operands: Def->operands(),
813 /*Mask=*/nullptr, Flags: *Def, Metadata: getMetadataOf(R: Def), DL: DebugLoc::getUnknown(),
814 ResultTy: Def->getScalarType(), UV: Def->getUnderlyingInstr());
815 Clone->insertAfter(InsertPos: Def);
816 Def->replaceAllUsesWith(New: Clone);
817 Def->eraseFromParent();
818 }
819 }
820
821 VPBuilder Builder(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
822 for (VPWidenInductionRecipe *PhiR : WideIVs) {
823 // Replace wide pointer inductions which have only their scalars used by
824 // PtrAdd(IndStart, ScalarIVSteps (0, Step)).
825 if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(Val: PhiR)) {
826 if (!Plan.hasScalarVFOnly() &&
827 !PtrIV->onlyScalarsGenerated(IsScalable: Plan.hasScalableVF()))
828 continue;
829
830 VPValue *PtrAdd =
831 vputils::scalarizeVPWidenPointerInduction(PtrIV, Plan, Builder);
832 PtrIV->replaceAllUsesWith(New: PtrAdd);
833 continue;
834 }
835
836 // Replace widened induction with scalar steps for users that only use
837 // scalars.
838 auto *WideIV = cast<VPWidenIntOrFpInductionRecipe>(Val: PhiR);
839 if (HasOnlyVectorVFs && none_of(Range: WideIV->users(), P: [WideIV](VPUser *U) {
840 return U->usesScalars(Op: WideIV);
841 }))
842 continue;
843
844 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
845 VPIRFlags::WrapFlagsTy WrapFlags;
846 // We can preserve nuw when the step is non-negative.
847 const APInt *Step;
848 if (match(V: WideIV->getStepValue(), P: m_APInt(C&: Step)) && Step->isNonNegative())
849 WrapFlags = WideIV->getNoWrapFlagsOrNone().withoutNoSignedWrap();
850 VPScalarIVStepsRecipe *Steps = vputils::createScalarIVSteps(
851 Plan, Kind: ID.getKind(), InductionOpcode: ID.getInductionOpcode(),
852 FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
853 TruncI: WideIV->getTruncInst(), StartV: WideIV->getStartValue(), Step: WideIV->getStepValue(),
854 DL: WideIV->getDebugLoc(), Builder, Flags: WrapFlags);
855
856 // Update scalar users of IV to use Step instead.
857 if (!HasOnlyVectorVFs) {
858 assert(!Plan.hasScalableVF() &&
859 "plans containing a scalar VF cannot also include scalable VFs");
860 WideIV->replaceAllUsesWith(New: Steps);
861 } else {
862 bool HasScalableVF = Plan.hasScalableVF();
863 WideIV->replaceUsesWithIf(New: Steps, ShouldReplace: [WideIV, HasScalableVF](VPUser &U) {
864 if (HasScalableVF)
865 return U.usesFirstLaneOnly(Op: WideIV);
866 return U.usesScalars(Op: WideIV);
867 });
868 }
869 }
870}
871
872/// Check if \p VPV is an untruncated wide induction, either before or after the
873/// increment. If so return the header IV (before the increment), otherwise
874/// return null.
875static VPWidenInductionRecipe *
876getOptimizableIVOf(VPValue *VPV, PredicatedScalarEvolution &PSE) {
877 auto *WideIV = dyn_cast<VPWidenInductionRecipe>(Val: VPV);
878 if (WideIV) {
879 // VPV itself is a wide induction, separately compute the end value for exit
880 // users if it is not a truncated IV.
881 auto *IntOrFpIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
882 return (IntOrFpIV && IntOrFpIV->getTruncInst()) ? nullptr : WideIV;
883 }
884
885 // Check if VPV is an optimizable induction increment.
886 VPRecipeBase *Def = VPV->getDefiningRecipe();
887 if (!Def || Def->getNumOperands() != 2)
888 return nullptr;
889 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 0));
890 if (!WideIV)
891 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 1));
892 if (!WideIV)
893 return nullptr;
894
895 auto IsWideIVInc = [&]() {
896 auto &ID = WideIV->getInductionDescriptor();
897
898 // Check if VPV increments the induction by the induction step.
899 VPValue *IVStep = WideIV->getStepValue();
900 switch (ID.getInductionOpcode()) {
901 case Instruction::Add:
902 return match(V: VPV, P: m_c_Add(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
903 case Instruction::FAdd:
904 return match(V: VPV, P: m_c_FAdd(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
905 case Instruction::FSub:
906 return match(V: VPV, P: m_Binary<Instruction::FSub>(Op0: m_Specific(VPV: WideIV),
907 Op1: m_Specific(VPV: IVStep)));
908 case Instruction::Sub: {
909 // IVStep will be the negated step of the subtraction. Check if Step == -1
910 // * IVStep.
911 VPValue *Step;
912 if (!match(V: VPV, P: m_Sub(Op0: m_VPValue(), Op1: m_VPValue(V&: Step))))
913 return false;
914 const SCEV *IVStepSCEV = vputils::getSCEVExprForVPValue(V: IVStep, PSE);
915 const SCEV *StepSCEV = vputils::getSCEVExprForVPValue(V: Step, PSE);
916 ScalarEvolution &SE = *PSE.getSE();
917 return !isa<SCEVCouldNotCompute>(Val: IVStepSCEV) &&
918 !isa<SCEVCouldNotCompute>(Val: StepSCEV) &&
919 IVStepSCEV == SE.getNegativeSCEV(V: StepSCEV);
920 }
921 default:
922 return ID.getKind() == InductionDescriptor::IK_PtrInduction &&
923 match(V: VPV, P: m_GetElementPtr(Op0: m_Specific(VPV: WideIV),
924 Op1: m_Specific(VPV: WideIV->getStepValue())));
925 }
926 llvm_unreachable("should have been covered by switch above");
927 };
928 return IsWideIVInc() ? WideIV : nullptr;
929}
930
931/// Attempts to optimize the induction variable exit values for users in the
932/// early exit block.
933static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan, VPValue *Op,
934 PredicatedScalarEvolution &PSE) {
935 VPValue *Incoming, *Mask;
936 if (!match(V: Op, P: m_ExtractLane(Op0: m_FirstActiveLane(Op0: m_VPValue(V&: Mask)),
937 Op1: m_VPValue(V&: Incoming))))
938 return nullptr;
939
940 auto *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
941 if (!WideIV)
942 return nullptr;
943
944 // Calculate the final index.
945 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
946 auto *CanonicalIV = LoopRegion->getCanonicalIV();
947 Type *CanonicalIVType = LoopRegion->getCanonicalIVType();
948 auto *ExtractR = cast<VPInstruction>(Val: Op);
949 VPBuilder B(ExtractR);
950
951 DebugLoc DL = ExtractR->getDebugLoc();
952 VPValue *FirstActiveLane = B.createFirstActiveLane(Masks: Mask, DL);
953 FirstActiveLane =
954 B.createScalarZExtOrTrunc(Op: FirstActiveLane, ResultTy: CanonicalIVType, DL);
955 VPValue *EndValue = B.createAdd(LHS: CanonicalIV, RHS: FirstActiveLane, DL);
956
957 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
958 // changed it means the exit is using the incremented value, so we need to
959 // add the step.
960 if (Incoming != WideIV) {
961 VPValue *One = Plan.getConstantInt(Ty: CanonicalIVType, Val: 1);
962 EndValue = B.createAdd(LHS: EndValue, RHS: One, DL);
963 }
964
965 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
966 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
967 VPValue *Start = WideIV->getStartValue();
968 VPValue *Step = WideIV->getStepValue();
969 EndValue = B.createDerivedIV(
970 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
971 Start, Current: EndValue, Step);
972 }
973
974 return EndValue;
975}
976
977/// Compute the end value for \p WideIV, unless it is truncated. Creates a
978/// VPDerivedIVRecipe for non-canonical inductions.
979static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV,
980 VPBuilder &VectorPHBuilder,
981 VPValue *VectorTC) {
982 auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
983 // Truncated wide inductions resume from the last lane of their vector value
984 // in the last vector iteration which is handled elsewhere.
985 if (WideIntOrFp && WideIntOrFp->getTruncInst())
986 return nullptr;
987
988 VPValue *Start = WideIV->getStartValue();
989 VPValue *Step = WideIV->getStepValue();
990 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
991 VPValue *EndValue = VectorTC;
992 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
993 EndValue = VectorPHBuilder.createDerivedIV(
994 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
995 Start, Current: VectorTC, Step);
996 }
997
998 // EndValue is derived from the vector trip count (which has the same type as
999 // the widest induction) and thus may be wider than the induction here.
1000 Type *ScalarTypeOfWideIV = WideIV->getScalarType();
1001 if (ScalarTypeOfWideIV != EndValue->getScalarType()) {
1002 EndValue = VectorPHBuilder.createScalarCast(Opcode: Instruction::Trunc, Op: EndValue,
1003 ResultTy: ScalarTypeOfWideIV,
1004 DL: WideIV->getDebugLoc());
1005 }
1006
1007 return EndValue;
1008}
1009
1010/// Attempts to optimize the induction variable exit values for users in the
1011/// exit block coming from the latch in the original scalar loop.
1012static VPValue *
1013optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
1014 DenseMap<VPValue *, VPValue *> &EndValues,
1015 PredicatedScalarEvolution &PSE) {
1016 VPValue *Incoming;
1017 if (!match(V: Op, P: m_CombineOr(Ps: m_ExtractLastLaneOfLastPart(Op0: m_VPValue(V&: Incoming)),
1018 Ps: m_ExtractLane(Op0: m_LastActiveLane(Op0: m_HeaderMask()),
1019 Op1: m_VPValue(V&: Incoming)))))
1020 return nullptr;
1021
1022 VPWidenInductionRecipe *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
1023 if (!WideIV)
1024 return nullptr;
1025
1026 VPValue *EndValue = EndValues.lookup(Val: WideIV);
1027 assert(EndValue && "Must have computed the end value up front");
1028
1029 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
1030 // changed it means the exit is using the incremented value, so we don't
1031 // need to subtract the step.
1032 if (Incoming != WideIV)
1033 return EndValue;
1034
1035 // Otherwise, subtract the step from the EndValue.
1036 auto *ExtractR = cast<VPInstruction>(Val: Op);
1037 VPBuilder B(ExtractR);
1038 VPValue *Step = WideIV->getStepValue();
1039 Type *ScalarTy = WideIV->getScalarType();
1040 if (ScalarTy->isIntegerTy())
1041 return B.createSub(LHS: EndValue, RHS: Step, DL: DebugLoc::getUnknown(), Name: "ind.escape");
1042 if (ScalarTy->isPointerTy()) {
1043 Type *StepTy = Step->getScalarType();
1044 auto *Zero = Plan.getZero(Ty: StepTy);
1045 return B.createPtrAdd(Ptr: EndValue, Offset: B.createSub(LHS: Zero, RHS: Step),
1046 DL: DebugLoc::getUnknown(), Name: "ind.escape");
1047 }
1048 if (ScalarTy->isFloatingPointTy()) {
1049 const auto &ID = WideIV->getInductionDescriptor();
1050 return B.createNaryOp(
1051 Opcode: ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
1052 ? Instruction::FSub
1053 : Instruction::FAdd,
1054 Operands: {EndValue, Step}, Flags: {ID.getInductionBinOp()->getFastMathFlags()});
1055 }
1056 llvm_unreachable("all possible induction types must be handled");
1057 return nullptr;
1058}
1059
1060static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
1061 PredicatedScalarEvolution &PSE,
1062 VPValue *ResumeTC,
1063 const Loop *L) {
1064 VPValue *Incoming;
1065 if (!match(V: Op, P: m_CombineOr(Ps: m_ExtractLastLaneOfLastPart(Op0: m_VPValue(V&: Incoming)),
1066 Ps: m_ExtractLane(Op0: m_LastActiveLane(Op0: m_HeaderMask()),
1067 Op1: m_VPValue(V&: Incoming)))))
1068 return nullptr;
1069
1070 const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(V: Incoming, PSE, L);
1071 const SCEV *Start, *Step;
1072 if (!match(S: IncomingSCEV, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step),
1073 L: m_SpecificLoop(L))))
1074 return nullptr;
1075
1076 auto *ExtractR = cast<VPInstruction>(Val: Op);
1077 DebugLoc DL = ExtractR->getDebugLoc();
1078 VPBuilder Builder(ExtractR);
1079 VPSCEVExpander Expander(Builder, *PSE.getSE(), DL);
1080 VPValue *StartVPV = Expander.expand(S: Start);
1081 VPValue *StepVPV = Expander.expand(S: Step);
1082
1083 Type *StartTy = StartVPV->getScalarType();
1084 assert(StartTy->isIntOrPtrTy() && "The type must be SCEVable");
1085 InductionDescriptor::InductionKind Kind =
1086 StartTy->isPointerTy() ? InductionDescriptor::IK_PtrInduction
1087 : InductionDescriptor::IK_IntInduction;
1088 Type *TCTy = ResumeTC->getScalarType();
1089 VPValue *ExitCount = Builder.createOverflowingOp(
1090 Opcode: Instruction::Sub, Operands: {ResumeTC, Plan.getConstantInt(Ty: TCTy, Val: 1)},
1091 WrapFlags: {/*HasNUW=*/true, /*HasNSW=*/false}, DL: DebugLoc::getUnknown());
1092 return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, Start: StartVPV, Current: ExitCount,
1093 Step: StepVPV);
1094}
1095
1096void VPlanTransforms::optimizeInductionLiveOutUsers(
1097 VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L) {
1098 // Compute end values for all inductions.
1099 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
1100 auto *VectorPH = cast<VPBasicBlock>(Val: VectorRegion->getSinglePredecessor());
1101 VPBuilder VectorPHBuilder(VectorPH, VectorPH->getFirstNonPhi());
1102 DenseMap<VPValue *, VPValue *> EndValues;
1103 VPValue *ResumeTC =
1104 Plan.hasTailFolded() ? Plan.getTripCount() : &Plan.getVectorTripCount();
1105 for (VPWidenInductionRecipe &WideIV : make_isa_range<VPWidenInductionRecipe>(
1106 Range: VectorRegion->getEntryBasicBlock()->phis())) {
1107 if (VPValue *EndValue = tryToComputeEndValueForInduction(
1108 WideIV: &WideIV, VectorPHBuilder, VectorTC: ResumeTC))
1109 EndValues[&WideIV] = EndValue;
1110 }
1111
1112 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
1113 for (VPRecipeBase &R : make_early_inc_range(Range&: *MiddleVPBB)) {
1114 VPValue *Op;
1115 if (!match(V: &R, P: m_ExitingIVValue(Op0: m_VPValue(V&: Op))))
1116 continue;
1117 auto *WideIV = cast<VPWidenInductionRecipe>(Val: Op);
1118 if (VPValue *EndValue = EndValues.lookup(Val: WideIV)) {
1119 R.getVPSingleValue()->replaceAllUsesWith(New: EndValue);
1120 R.eraseFromParent();
1121 }
1122 }
1123
1124 // Then, optimize exit block users.
1125 for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
1126 for (VPRecipeBase &R : ExitVPBB->phis()) {
1127 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
1128
1129 for (auto [Idx, PredVPBB] : enumerate(First&: ExitVPBB->getPredecessors())) {
1130 VPValue *Escape = nullptr;
1131 if (PredVPBB == MiddleVPBB) {
1132 Escape = optimizeLatchExitInductionUser(
1133 Plan, Op: ExitIRI->getOperand(N: Idx), EndValues, PSE);
1134 if (!Escape)
1135 Escape = optimizeLatchExitIVUserViaSCEV(
1136 Plan, Op: ExitIRI->getOperand(N: Idx), PSE, ResumeTC, L);
1137 } else {
1138 Escape = optimizeEarlyExitInductionUser(
1139 Plan, Op: ExitIRI->getOperand(N: Idx), PSE);
1140 }
1141 if (Escape)
1142 ExitIRI->setOperand(I: Idx, New: Escape);
1143 }
1144 }
1145 }
1146}
1147
1148/// Remove redundant ExpandSCEVRecipes in \p Plan's entry block by replacing
1149/// them with already existing recipes expanding the same SCEV expression.
1150static void removeRedundantExpandSCEVRecipes(VPlan &Plan) {
1151 DenseMap<const SCEV *, VPValue *> SCEV2VPV;
1152
1153 for (VPExpandSCEVRecipe &ExpR :
1154 make_early_inc_range(Range: make_isa_range<VPExpandSCEVRecipe>(
1155 Range&: *Plan.getEntry()->getEntryBasicBlock()))) {
1156 const auto &[V, Inserted] = SCEV2VPV.try_emplace(Key: ExpR.getSCEV(), Args: &ExpR);
1157 if (Inserted)
1158 continue;
1159
1160 ExpR.replaceAllUsesWith(New: V->second);
1161 if (&ExpR == Plan.getTripCount())
1162 Plan.resetTripCount(NewTripCount: V->second);
1163
1164 ExpR.eraseFromParent();
1165 }
1166}
1167
1168/// Try to simplify logical and bitwise recipes in \p Def.
1169static VPValue *simplifyLogicalRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
1170 VPValue *X;
1171
1172 // X | AllOnes -> AllOnes
1173 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_AllOnes())))
1174 return Plan.getAllOnesValue(Ty: Def->getScalarType());
1175
1176 // X | 0 -> X
1177 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_ZeroInt())))
1178 return X;
1179
1180 // X | !X -> AllOnes
1181 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X)))))
1182 return Plan.getAllOnesValue(Ty: Def->getScalarType());
1183
1184 // X & 0 -> 0
1185 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_ZeroInt())))
1186 return Plan.getZero(Ty: Def->getScalarType());
1187
1188 // X & AllOnes -> X
1189 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_AllOnes())))
1190 return X;
1191
1192 // X && false -> false
1193 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_False())))
1194 return Plan.getFalse();
1195
1196 // X && true -> X
1197 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_True())))
1198 return X;
1199
1200 // X && (X && Y) -> X && Y
1201 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1202 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue()))))
1203 return Def->getOperand(N: 1);
1204
1205 // X && !X -> 0
1206 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X)))))
1207 return Plan.getFalse();
1208
1209 if (match(R: Def, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: X), Op2: m_Deferred(V: X))))
1210 return X;
1211
1212 // X != false -> X
1213 if (match(R: Def, P: m_SpecificICmp(MatchPred: CmpInst::ICMP_NE, Op0: m_VPValue(V&: X), Op1: m_False()))) {
1214 assert(X->getScalarType()->isIntegerTy(1) && "must have boolean operands");
1215 return X;
1216 }
1217
1218 return nullptr;
1219}
1220
1221/// Swap the branch weights recorded for \p R, a select whose two selected
1222/// operands are being swapped,
1223static void swapSelectBranchWeights(VPRecipeBase &R, VPlan &Plan) {
1224 auto *MD = dyn_cast<VPIRMetadata>(Val: &R);
1225 if (!MD)
1226 return;
1227 SmallVector<uint32_t, 2> Weights;
1228 if (!extractBranchWeights(ProfileData: MD->getMetadata(Kind: LLVMContext::MD_prof), Weights))
1229 return;
1230 assert(Weights.size() == 2 && "unexpected branch weights");
1231 MD->setMetadata(
1232 Kind: LLVMContext::MD_prof,
1233 Node: MDBuilder(Plan.getContext()).createBranchWeights(TrueWeight: Weights[1], FalseWeight: Weights[0]));
1234}
1235
1236/// Return an existing value or a live in for VPSingleDefRecipe \p Def if
1237/// possible. This shouldn't create or modify recipes.
1238static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
1239 // Simplification of live-in IR values for SingleDef recipes using
1240 // InstSimplifyFolder.
1241 const DataLayout &DL = Plan.getDataLayout();
1242 if (VPValue *V = vputils::tryToFoldLiveIns(R&: *Def, Operands: Def->operands(), DL))
1243 return V;
1244
1245 // Fold PredPHI LiveIn -> LiveIn.
1246 if (auto *PredPHI = dyn_cast<VPPredInstPHIRecipe>(Val: Def)) {
1247 VPValue *Op = PredPHI->getOperand(N: 0);
1248 if (isa<VPIRValue>(Val: Op))
1249 return Op;
1250 }
1251
1252 if (VPValue *V = simplifyLogicalRecipe(Plan, Def))
1253 return V;
1254
1255 VPValue *A, *B;
1256
1257 if (match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))
1258 return A;
1259
1260 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_One())))
1261 return A;
1262
1263 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(), Op1: m_ZeroInt())))
1264 return Plan.getZero(Ty: Def->getScalarType());
1265
1266 // A bitcast to the same type is a no-op.
1267 if (match(R: Def, P: m_BitCast(Op0: m_VPValue(V&: A))) &&
1268 Def->getScalarType() == A->getScalarType())
1269 return A;
1270
1271 // Shifting by zero is a no-op.
1272 if (match(R: Def, P: m_CombineOr(Ps: m_Shl(Op0: m_VPValue(V&: A), Op1: m_ZeroInt()),
1273 Ps: m_CombineOr(Ps: m_LShr(Op0: m_VPValue(V&: A), Op1: m_ZeroInt()),
1274 Ps: m_AShr(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))))
1275 return A;
1276
1277 if (match(R: Def, P: m_Trunc(Op0: m_ZExtOrSExt(Op0: m_VPValue(V&: A)))))
1278 if (Def->getScalarType() == A->getScalarType())
1279 return A;
1280
1281 if (match(R: Def, P: m_Not(Op0: m_Not(Op0: m_VPValue(V&: A)))))
1282 return A;
1283
1284 // Remove redundant DerviedIVs, that is 0 + A * 1 -> A and 0 + 0 * x -> 0.
1285 if ((match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_VPValue(V&: A), Op2: m_One())) ||
1286 match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_VPValue(V&: A, Op: m_ZeroInt()),
1287 Op2: m_VPValue()))) &&
1288 A->getScalarType() == Def->getScalarType())
1289 return A;
1290
1291 // Simplify MaskedCond with no block mask to its single operand.
1292 if (match(R: Def, P: m_VPInstruction<VPInstruction::MaskedCond>()) &&
1293 !cast<VPInstruction>(Val: Def)->isMasked())
1294 return Def->getOperand(N: 0);
1295
1296 // Look through ExtractLastLane.
1297 if (match(R: Def, P: m_ExtractLastLane(Op0: m_VPValue(V&: A)))) {
1298 if (match(V: A, P: m_BuildVector())) {
1299 auto *BuildVector = cast<VPInstruction>(Val: A);
1300 return BuildVector->getLastOperand();
1301 }
1302
1303 if (match(V: A, P: m_Broadcast(Op0: m_VPValue(V&: B))))
1304 return B;
1305
1306 if (isa<VPInstruction, VPReplicateRecipe>(Val: A) && vputils::isSingleScalar(VPV: A))
1307 return A;
1308
1309 if (Plan.hasScalarVFOnly())
1310 return A;
1311 }
1312
1313 // Look through ExtractPenultimateElement (BuildVector ....).
1314 if (match(R: Def, P: m_ExtractPenultimateElement(Op0: m_BuildVector()))) {
1315 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1316 return BuildVector->getOperand(N: BuildVector->getNumOperands() - 2);
1317 }
1318
1319 uint64_t Idx;
1320 if (match(R: Def, P: m_ExtractElement(Op0: m_BuildVector(), Op1: m_ConstantInt(C&: Idx)))) {
1321 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1322 return BuildVector->getOperand(N: Idx);
1323 }
1324
1325 if (isa<VPPhi, VPWidenPHIRecipe, VPHeaderPHIRecipe>(Val: Def)) {
1326 if (Def->getNumOperands() == 1) {
1327 return Def->getOperand(N: 0);
1328 }
1329 if (auto *Phi = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Val: Def)) {
1330 if (all_equal(Range: Phi->incoming_values()))
1331 return Phi->getOperand(N: 0);
1332 }
1333 return nullptr;
1334 }
1335
1336 VPIRValue *IRV;
1337 if (Def->getNumOperands() == 1 &&
1338 match(R: Def, P: m_ComputeReductionResult(Op0: m_VPIRValue(V&: IRV))))
1339 return IRV;
1340
1341 if (match(R: Def, P: m_VPInstruction<VPInstruction::WideIVStep>(Ops: m_VPValue(V&: A),
1342 Ops: m_One())) &&
1343 A->getScalarType() == Def->getScalarType())
1344 return A;
1345
1346 // Some simplifications can only be applied after unrolling. Perform them
1347 // below.
1348 if (!Plan.isUnrolled())
1349 return nullptr;
1350
1351 // Simplify extracts of the same single-scalar.
1352 if (match(R: Def, P: m_VPInstruction<VPInstruction::ExtractLane>()) &&
1353 all_equal(Range: drop_begin(RangeOrContainer: Def->operands())) &&
1354 vputils::isSingleScalar(VPV: Def->getOperand(N: 1)))
1355 return Def->getOperand(N: 1);
1356
1357 // Replace extract-lane(0, canonical-WIDEN-INDUCTION) with the region's
1358 // scalar canonical IV.
1359 VPWidenIntOrFpInductionRecipe *WidenIV;
1360 if (match(R: Def, P: m_ExtractLane(Op0: m_ZeroInt(), Op1: m_CanonicalWidenIV(V&: WidenIV))))
1361 return WidenIV->getRegion()->getCanonicalIV();
1362
1363 // Simplify unrolled VectorPointer without offset, or with zero offset, to
1364 // just the pointer operand.
1365 if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(Val: Def))
1366 if (!VPR->getVFxPart() || match(V: VPR->getVFxPart(), P: m_ZeroInt()))
1367 return VPR->getOperand(N: 0);
1368
1369 // VPScalarIVSteps after unrolling can be replaced by their start value, if
1370 // the start index is zero and only the first lane 0 is demanded.
1371 if (auto *Steps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def))
1372 if (!Steps->getStartIndex() && vputils::onlyFirstLaneUsed(Def: Steps))
1373 return Steps->getOperand(N: 0);
1374
1375 if (Plan.getConcreteUF() == 1 && match(R: Def, P: m_ExtractLastPart(Op0: m_VPValue(V&: A))))
1376 return A;
1377
1378 return nullptr;
1379}
1380
1381/// Returns true if \p V is available at the end of \p VPBB, i.e. it either is a
1382/// live-in from the original IR or defined in \p VPBB.
1383static bool isAvailableAtEndOf(VPValue *V, const VPBasicBlock *VPBB) {
1384 VPRecipeBase *DefR = V->getDefiningRecipe();
1385 return DefR ? DefR->getParent() == VPBB : isa<VPIRValue>(Val: V);
1386}
1387
1388namespace {
1389/// Inserter for VPBuilderBase which appends all created VPSingleDefRecipes to a
1390/// worklist, so they get combined as well.
1391struct VPCombineInserter {
1392 SmallVectorImpl<VPSingleDefRecipe *> &Worklist;
1393
1394 void insertHelper(VPRecipeBase *R, VPBasicBlock *VPBB,
1395 VPBasicBlock::iterator It) {
1396 VPBB->insert(Recipe: R, InsertPt: It);
1397 if (auto *Def = dyn_cast<VPSingleDefRecipe>(Val: R))
1398 Worklist.push_back(Elt: Def);
1399 }
1400};
1401
1402using VPCombineBuilder = VPBuilderBase<VPCombineInserter>;
1403} // namespace
1404
1405/// Combine \p Def into a simpler recipe. May modify or create new recipes via
1406/// \p Builder.
1407static VPSingleDefRecipe *combineRecipe(VPlan &Plan, VPSingleDefRecipe *Def,
1408 VPCombineBuilder &Builder) {
1409 if (auto *V = simplifyRecipe(Plan, Def)) {
1410 Def->replaceAllUsesWith(New: V);
1411 return Def;
1412 }
1413
1414 // Drop the mask of a predicated store masked by the header mask (which is
1415 // guaranteed to be true at least for the first lane) and both the stored
1416 // value and the address are uniform across VF and UF. The header mask is
1417 // still the abstract region value here.
1418 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: Def);
1419 RepR && RepR->isPredicated() && RepR->getOpcode() == Instruction::Store &&
1420 all_of(Range: RepR->operandsWithoutMask(), P: vputils::isUniformAcrossVFsAndUFs) &&
1421 match(V: RepR->getMask(), P: m_HeaderMask())) {
1422 auto *Unmasked = new VPReplicateRecipe(
1423 RepR->getUnderlyingInstr(), RepR->operandsWithoutMask(),
1424 RepR->isSingleScalar(), /*Mask=*/nullptr, *RepR, *RepR,
1425 RepR->getDebugLoc());
1426 Builder.insert(R: Unmasked);
1427 return Unmasked;
1428 }
1429
1430 // Avoid replacing VPInstructions with underlying values with new
1431 // VPInstructions, as we would fail to create widen/replicate recpes from the
1432 // new VPInstructions without an underlying value, and miss out on some
1433 // transformations that only apply to widened/replicated recipes later, by
1434 // doing so.
1435 // TODO: We should also not replace non-VPInstructions like VPWidenRecipe with
1436 // VPInstructions without underlying values, as those will get skipped during
1437 // cost computation.
1438 bool CanCreateNewRecipe =
1439 !isa<VPInstruction>(Val: Def) || !Def->getUnderlyingValue();
1440
1441 VPValue *X, *Y, *Z;
1442
1443 // X && (Y && X) -> X && Y
1444 if (CanCreateNewRecipe &&
1445 match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1446 Op1: m_LogicalAnd(Op0: m_VPValue(V&: Y), Op1: m_Deferred(V: X)))))
1447 return Builder.createLogicalAnd(LHS: X, RHS: Y);
1448
1449 // (X && Y) | (X && Z) -> X && (Y | Z)
1450 if (CanCreateNewRecipe &&
1451 match(R: Def, P: m_c_BinaryOr(Op0: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y)),
1452 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue(V&: Z)))) &&
1453 // Simplify only if one of the operands has one use to avoid creating an
1454 // extra recipe.
1455 (!Def->getOperand(N: 0)->hasMoreThanOneUniqueUser() ||
1456 !Def->getOperand(N: 1)->hasMoreThanOneUniqueUser()))
1457 return Builder.createLogicalAnd(LHS: X, RHS: Builder.createOr(LHS: Y, RHS: Z));
1458
1459 // (X && Y) | !X -> !X || Y
1460 if (CanCreateNewRecipe &&
1461 match(R: Def,
1462 P: m_c_BinaryOr(Op0: m_OneUse(SubPattern: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y))),
1463 Op1: m_VPValue(V&: Z, Op: m_Not(Op0: m_Deferred(V: X))))))
1464 return Builder.createLogicalOr(LHS: Z, RHS: Y);
1465
1466 // select C, false, true -> not C
1467 VPValue *C;
1468 if (CanCreateNewRecipe &&
1469 match(R: Def, P: m_Select(Op0: m_VPValue(V&: C), Op1: m_False(), Op2: m_True())))
1470 return Builder.createNot(Operand: C);
1471
1472 // select !C, X, Y -> select C, Y, X
1473 if (match(R: Def, P: m_Select(Op0: m_Not(Op0: m_VPValue(V&: C)), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1474 Def->setOperand(I: 0, New: C);
1475 Def->setOperand(I: 1, New: Y);
1476 Def->setOperand(I: 2, New: X);
1477 swapSelectBranchWeights(R&: *Def, Plan);
1478 return Def;
1479 }
1480
1481 // select X, (i1 Y | Z), Y -> Y | (X && Z)
1482 if (CanCreateNewRecipe &&
1483 match(R: Def, P: m_Select(Op0: m_VPValue(V&: X),
1484 Op1: m_OneUse(SubPattern: m_c_BinaryOr(Op0: m_VPValue(V&: Y), Op1: m_VPValue(V&: Z))),
1485 Op2: m_Deferred(V: Y))) &&
1486 Y->getScalarType()->isIntegerTy(BitWidth: 1))
1487 return Builder.createOr(LHS: Y, RHS: Builder.createLogicalAnd(LHS: X, RHS: Z));
1488
1489 // select M0, (select M1, X, Y), Y -> select (M0 && M1), X, Y
1490 VPValue *Mask0, *Mask1;
1491 if (CanCreateNewRecipe &&
1492 match(R: Def,
1493 P: m_SelectLike(Op0: m_VPValue(V&: Mask0),
1494 Op1: m_OneUse(SubPattern: m_SelectLike(Op0: m_VPValue(V&: Mask1), Op1: m_VPValue(V&: X),
1495 Op2: m_VPValue(V&: Y))),
1496 Op2: m_Deferred(V: Y))))
1497 return Builder.createSelect(Cond: Builder.createLogicalAnd(LHS: Mask0, RHS: Mask1), TrueVal: X, FalseVal: Y,
1498 DL: Def->getDebugLoc());
1499
1500 if (match(R: Def, P: m_Trunc(Op0: m_VPValue(V&: Y, Op: m_ZExtOrSExt(Op0: m_VPValue(V&: X)))))) {
1501 // Don't replace a non-widened cast recipe with a widened cast.
1502 if (!isa<VPWidenCastRecipe>(Val: Def))
1503 return nullptr;
1504 Type *TruncTy = Def->getScalarType();
1505 Type *XTy = X->getScalarType();
1506 if (XTy->getScalarSizeInBits() < TruncTy->getScalarSizeInBits()) {
1507
1508 unsigned ExtOpcode =
1509 match(V: Y, P: m_SExt(Op0: m_VPValue())) ? Instruction::SExt : Instruction::ZExt;
1510 auto *Ext =
1511 Builder.createWidenCast(Opcode: Instruction::CastOps(ExtOpcode), Op: X, ResultTy: TruncTy);
1512 if (auto *UnderlyingExt = Y->getUnderlyingValue()) {
1513 // UnderlyingExt has distinct return type, used to retain legacy cost.
1514 Ext->setUnderlyingValue(UnderlyingExt);
1515 }
1516 return Ext;
1517 } else if (XTy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) {
1518 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::Trunc, Op: X, ResultTy: TruncTy);
1519 return Trunc;
1520 }
1521 }
1522
1523 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: X), Op1: m_AllOnes()))) {
1524 // Preserve nsw from the Mul on the new Sub.
1525 VPIRFlags::WrapFlagsTy NW = {
1526 false, cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap()};
1527 return Builder.createSub(LHS: Plan.getZero(Ty: X->getScalarType()), RHS: X,
1528 DL: Def->getDebugLoc(), Name: "", WrapFlags: NW);
1529 }
1530
1531 if (CanCreateNewRecipe &&
1532 match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: X),
1533 Op1: m_VPValue(V&: Z, Op: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Y)))))) {
1534 // Preserve nsw from the Add and the Sub, if it's present on both, on the
1535 // new Sub.
1536 VPIRFlags::WrapFlagsTy NW = {
1537 false, cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap() &&
1538 cast<VPRecipeWithIRFlags>(Val: Z)->hasNoSignedWrap()};
1539 return Builder.createSub(LHS: X, RHS: Y, DL: Def->getDebugLoc(), Name: "", WrapFlags: NW);
1540 }
1541
1542 const APInt *APC;
1543 if (CanCreateNewRecipe && match(R: Def, P: m_URem(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1544 APC->isPowerOf2())
1545 return Builder.createAnd(LHS: X, RHS: Plan.getConstantInt(Val: *APC - 1),
1546 DL: Def->getDebugLoc());
1547
1548 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1549 APC->isPowerOf2()) {
1550 auto *MulR = cast<VPRecipeWithIRFlags>(Val: Def);
1551 unsigned ShiftAmt = APC->exactLogBase2();
1552 VPIRFlags::WrapFlagsTy NW(MulR->hasNoUnsignedWrap(),
1553 MulR->hasNoSignedWrap() &&
1554 ShiftAmt != APC->getBitWidth() - 1);
1555 return Builder.createNaryOp(
1556 Opcode: Instruction::Shl,
1557 Operands: {X, Plan.getConstantInt(BitWidth: APC->getBitWidth(), Val: ShiftAmt)}, Flags: NW,
1558 DL: Def->getDebugLoc());
1559 }
1560
1561 if (CanCreateNewRecipe && match(R: Def, P: m_UDiv(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1562 APC->isPowerOf2())
1563 return Builder.createNaryOp(
1564 Opcode: Instruction::LShr,
1565 Operands: {X, Plan.getConstantInt(BitWidth: APC->getBitWidth(), Val: APC->exactLogBase2())},
1566 Flags: *cast<VPRecipeWithIRFlags>(Val: Def), DL: Def->getDebugLoc());
1567
1568 // (X >> C) << C -> X & (-1 << C).
1569 if (CanCreateNewRecipe &&
1570 match(R: Def, P: m_Shl(Op0: m_LShr(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y, Op: m_APInt(C&: APC))),
1571 Op1: m_Deferred(V: Y))))
1572 return Builder.createAnd(
1573 LHS: X, RHS: Plan.getConstantInt(Val: APInt::getAllOnes(numBits: APC->getBitWidth()) << *APC),
1574 DL: Def->getDebugLoc());
1575
1576 if (match(R: Def, P: m_Not(Op0: m_VPValue(V&: X)))) {
1577 // Try to fold Not into compares by adjusting the predicate in-place.
1578 CmpPredicate Pred;
1579 if (match(V: X, P: m_Cmp(Pred, Op0: m_VPValue(), Op1: m_VPValue()))) {
1580 auto *Cmp = cast<VPRecipeWithIRFlags>(Val: X);
1581 // Only fold if every user is a Not of the cmp, or a select using the cmp
1582 // solely as its condition.
1583 if (all_of(Range: Cmp->users(), P: [Cmp](VPUser *U) {
1584 return match(U, P: m_Not(Op0: m_Specific(VPV: Cmp))) ||
1585 (match(U, P: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(),
1586 Op2: m_VPValue())) &&
1587 U->getOperand(N: 1) != Cmp && U->getOperand(N: 2) != Cmp);
1588 })) {
1589 Cmp->setPredicate(CmpInst::getInversePredicate(pred: Pred));
1590 for (VPUser *U : to_vector(Range: Cmp->users())) {
1591 auto *R = cast<VPSingleDefRecipe>(Val: U);
1592 if (match(R, P: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1593 // select (cmp pred), X, Y -> select (cmp inv_pred), Y, X
1594 R->setOperand(I: 1, New: Y);
1595 R->setOperand(I: 2, New: X);
1596 swapSelectBranchWeights(R&: *R, Plan);
1597 } else {
1598 // not (cmp pred) -> cmp inv_pred
1599 assert(match(R, m_Not(m_Specific(Cmp))) && "Unexpected user");
1600 R->replaceAllUsesWith(New: Cmp);
1601 }
1602 }
1603 // If Cmp doesn't have a debug location, use the one from the negation,
1604 // to preserve the location.
1605 if (!Cmp->getDebugLoc() && Def->getDebugLoc())
1606 Cmp->setDebugLoc(Def->getDebugLoc());
1607 return Def;
1608 }
1609 }
1610 }
1611
1612 // Fold any-of (fcmp uno A, A), (fcmp uno B, B), ... ->
1613 // any-of (fcmp uno A, B), ...
1614 if (match(R: Def, P: m_AnyOf())) {
1615 SmallVector<VPValue *, 4> NewOps;
1616 VPRecipeBase *UnpairedCmp = nullptr;
1617 for (VPValue *Op : Def->operands()) {
1618 VPValue *X;
1619 if (Op->getNumUsers() > 1 ||
1620 !match(V: Op, P: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X),
1621 Op1: m_Deferred(V: X)))) {
1622 NewOps.push_back(Elt: Op);
1623 } else if (!UnpairedCmp) {
1624 UnpairedCmp = Op->getDefiningRecipe();
1625 } else {
1626 NewOps.push_back(Elt: Builder.createFCmp(Pred: CmpInst::FCMP_UNO,
1627 A: UnpairedCmp->getOperand(N: 0), B: X));
1628 UnpairedCmp = nullptr;
1629 }
1630 }
1631
1632 if (UnpairedCmp)
1633 NewOps.push_back(Elt: UnpairedCmp->getVPSingleValue());
1634
1635 if (NewOps.size() < Def->getNumOperands())
1636 return Builder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: NewOps);
1637 }
1638
1639 // Fold (fcmp uno X, X) | (fcmp uno Y, Y) -> fcmp uno X, Y
1640 // This is useful for fmax/fmin without fast-math flags, where we need to
1641 // check if any operand is NaN.
1642 if (CanCreateNewRecipe &&
1643 match(R: Def,
1644 P: m_BinaryOr(
1645 Op0: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X), Op1: m_Deferred(V: X)),
1646 Op1: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: Y), Op1: m_Deferred(V: Y)))))
1647 return Builder.createFCmp(Pred: CmpInst::FCMP_UNO, A: X, B: Y);
1648
1649 if (match(R: Def, P: m_VPInstruction<VPInstruction::WideIVStep>(Ops: m_VPValue(V&: X),
1650 Ops: m_One())) &&
1651 X->getScalarType() != Def->getScalarType())
1652 return Builder.createWidenCast(Opcode: Instruction::Trunc, Op: X, ResultTy: Def->getScalarType());
1653
1654 // For i1 vp.merges produced by AnyOf reductions:
1655 // vp.merge true, (or X, Y), X, evl -> vp.merge Y, true, X, evl
1656 if (match(R: Def, P: m_Intrinsic<Intrinsic::vp_merge>(Ops: m_True(), Ops: m_VPValue(V&: X),
1657 Ops: m_VPValue(V&: X), Ops: m_VPValue())) &&
1658 match(V: X, P: m_c_BinaryOr(Op0: m_Specific(VPV: X), Op1: m_VPValue(V&: Y))) &&
1659 Def->getScalarType()->isIntegerTy(BitWidth: 1)) {
1660 Def->setOperand(I: 1, New: Plan.getTrue());
1661 Def->setOperand(I: 0, New: Y);
1662 return Def;
1663 }
1664
1665 if (match(R: Def, P: m_BuildVector()) && all_equal(Range: Def->operands()))
1666 return Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: Def->getOperand(N: 0));
1667
1668 // Replace uses of a BuildVector by users that only use its first lane with
1669 // its first operand directly.
1670 if (match(R: Def, P: m_BuildVector())) {
1671 Def->replaceUsesWithIf(New: Def->getOperand(N: 0), ShouldReplace: [Def](VPUser &U) {
1672 return U.usesFirstLaneOnly(Op: Def);
1673 });
1674 return Def;
1675 }
1676
1677 // Look through broadcast of single-scalar when used as select conditions; in
1678 // that case the scalar condition can be used directly.
1679 if (match(R: Def,
1680 P: m_Select(Op0: m_Broadcast(Op0: m_VPValue(V&: Z)), Op1: m_VPValue(), Op2: m_VPValue()))) {
1681 assert(vputils::isSingleScalar(Z) &&
1682 "broadcast operand must be single-scalar");
1683 Def->setOperand(I: 0, New: Z);
1684 return Def;
1685 }
1686
1687 if (match(R: Def, P: m_Broadcast(Op0: m_VPValue(V&: X)))) {
1688 Def->replaceUsesWithIf(
1689 New: X, ShouldReplace: [Def](const VPUser &U) { return U.usesScalars(Op: Def); });
1690 return Def;
1691 }
1692
1693 // Some simplifications can only be applied after unrolling. Perform them
1694 // below.
1695 if (!Plan.isUnrolled())
1696 return nullptr;
1697
1698 // Simplify extract-lane with single source to extract-element.
1699 VPValue *LaneToExtract;
1700 if (match(R: Def, P: m_ExtractLane(Op0: m_VPValue(V&: LaneToExtract), Op1: m_VPValue(V&: X))))
1701 return Builder.createNaryOp(Opcode: Instruction::ExtractElement, Operands: {X, LaneToExtract},
1702 DL: Def->getDebugLoc());
1703
1704 // Look for cycles where Def is of the form:
1705 // X = phi(0, IVInc) ; used only by IVInc, or by IVInc and Inc = X + Y
1706 // IVInc = X + Step ; used by X and Def
1707 // Def = IVInc + Y
1708 // Fold the increment Y into the phi's start value, replace Def with IVInc,
1709 // and if Inc exists, replace it with X.
1710 VPValue *IVInc;
1711 if (match(R: Def, P: m_Add(Op0: m_VPValue(V&: IVInc, Op: m_Add(Op0: m_VPValue(V&: X), Op1: m_VPValue())),
1712 Op1: m_VPValue(V&: Y))) &&
1713 match(V: X, P: m_VPPhi(Op0: m_ZeroInt(), Op1: m_Specific(VPV: IVInc))) &&
1714 IVInc->getNumUsers() == 2) {
1715 auto *Phi = cast<VPPhi>(Val: X);
1716 // If Phi has a second user (besides IVInc's defining recipe), it must be
1717 // Inc = Phi + Y for the fold to apply.
1718 auto *Inc = dyn_cast_if_present<VPSingleDefRecipe>(
1719 Val: findUserOf(V: Phi, P: m_Add(Op0: m_Specific(VPV: Phi), Op1: m_Specific(VPV: Y))));
1720 if ((Phi->getNumUsers() == 1 || (Phi->getNumUsers() == 2 && Inc)) &&
1721 isAvailableAtEndOf(V: Y, VPBB: Phi->getIncomingBlock(Idx: 0))) {
1722 Def->replaceAllUsesWith(New: IVInc);
1723 if (Inc)
1724 Inc->replaceAllUsesWith(New: Phi);
1725 Phi->setOperand(I: 0, New: Y);
1726 return Def;
1727 }
1728 }
1729
1730 // Simplify redundant ReductionStartVector recipes after unrolling.
1731 VPValue *StartV;
1732 if (match(R: Def, P: m_VPInstruction<VPInstruction::ReductionStartVector>(
1733 Ops: m_VPValue(V&: StartV), Ops: m_VPValue(), Ops: m_VPValue()))) {
1734 Def->replaceUsesWithIf(New: StartV, ShouldReplace: [](const VPUser &U) {
1735 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &U);
1736 return PhiR && PhiR->isInLoop();
1737 });
1738 return Def;
1739 }
1740
1741 return nullptr;
1742}
1743
1744void VPlanTransforms::combineRecipes(VPlan &Plan) {
1745 SmallVector<VPSingleDefRecipe *, 256> Worklist;
1746 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
1747 Plan.getEntry());
1748 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT))
1749 for (VPSingleDefRecipe &Def :
1750 make_isa_range<VPSingleDefRecipe>(Range: reverse(C&: *VPBB)))
1751 Worklist.push_back(Elt: &Def);
1752
1753 [[maybe_unused]] unsigned InitWorklistSize = Worklist.size();
1754
1755 VPCombineBuilder Builder({.Worklist: Worklist});
1756 while (!Worklist.empty()) {
1757 assert(Worklist.size() < InitWorklistSize * 2 &&
1758 "Worklist is growing large, possible cycle?");
1759 VPSingleDefRecipe *Def = Worklist.pop_back_val();
1760 Builder.setInsertPoint(Def);
1761 VPSingleDefRecipe *New = combineRecipe(Plan, Def, Builder);
1762 if (!New)
1763 continue;
1764 if (New != Def) {
1765 // Replace the recipe with a new one.
1766 Def->replaceAllUsesWith(New);
1767 Def->eraseFromParent();
1768 // TODO: Append users to the worklist (might need a setvector)
1769 } else if (vputils::isDeadRecipe(R&: *Def)) {
1770 // Recipe was modified - it may be dead now.
1771 Def->eraseFromParent();
1772 }
1773 }
1774}
1775
1776void VPlanTransforms::simplifyReverses(VPlan &Plan) {
1777 // Pull out reverses from any elementwise op.
1778 // binop(reverse(x), reverse(y)) -> reverse(binop(x,y))
1779 vputils::pullOutPermutations(
1780 Plan, Perm: [](VPValue *&X) { return m_Reverse(Op0: m_VPValue(V&: X)); },
1781 Build: [](auto *X) { return new VPInstruction(VPInstruction::Reverse, X); });
1782
1783 // reverse(reverse(x)) -> x
1784 VPValue *X;
1785 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1786 Range: vp_depth_first_deep(G: Plan.getEntry())))
1787 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
1788 if (match(V: &R, P: m_Reverse(Op0: m_Reverse(Op0: m_VPValue(V&: X)))))
1789 R.getVPSingleValue()->replaceAllUsesWith(New: X);
1790}
1791
1792/// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the
1793/// header mask to be simplified further when tail folding, e.g. in
1794/// optimizeEVLMasks.
1795static void reassociateHeaderMask(VPlan &Plan) {
1796 VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
1797 if (!HeaderMask)
1798 return;
1799
1800 SmallVector<VPUser *> Worklist;
1801 for (VPUser *U : HeaderMask->users())
1802 if (match(U, P: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue())))
1803 append_range(C&: Worklist, R: cast<VPSingleDefRecipe>(Val: U)->users());
1804
1805 while (!Worklist.empty()) {
1806 auto *R = dyn_cast<VPSingleDefRecipe>(Val: Worklist.pop_back_val());
1807 VPValue *X, *Y;
1808 if (!R || !match(R, P: m_LogicalAnd(
1809 Op0: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue(V&: X)),
1810 Op1: m_VPValue(V&: Y))))
1811 continue;
1812 append_range(C&: Worklist, R: R->users());
1813 VPBuilder Builder(R);
1814 R->replaceAllUsesWith(
1815 New: Builder.createLogicalAnd(LHS: HeaderMask, RHS: Builder.createLogicalAnd(LHS: X, RHS: Y)));
1816 }
1817}
1818
1819static std::optional<Instruction::BinaryOps>
1820getUnmaskedDivRemOpcode(Intrinsic::ID ID) {
1821 switch (ID) {
1822 case Intrinsic::masked_udiv:
1823 return Instruction::UDiv;
1824 case Intrinsic::masked_sdiv:
1825 return Instruction::SDiv;
1826 case Intrinsic::masked_urem:
1827 return Instruction::URem;
1828 case Intrinsic::masked_srem:
1829 return Instruction::SRem;
1830 default:
1831 return {};
1832 }
1833}
1834
1835static void narrowToSingleScalarRecipes(VPlan &Plan) {
1836 if (Plan.hasScalarVFOnly())
1837 return;
1838
1839 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1840 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
1841 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
1842 if (!isa<VPWidenRecipe, VPWidenGEPRecipe, VPReplicateRecipe,
1843 VPWidenIntrinsicRecipe>(Val: &R))
1844 continue;
1845 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
1846 if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
1847 continue;
1848
1849 auto *RepOrWidenR = cast<VPRecipeWithIRFlags>(Val: &R);
1850 if (RepR && RepR->getOpcode() == Instruction::Store &&
1851 vputils::isSingleScalar(VPV: RepR->getOperand(N: 1))) {
1852 auto *Clone = new VPReplicateRecipe(
1853 RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
1854 true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
1855 *RepR /*Metadata*/, RepR->getDebugLoc());
1856 Clone->insertBefore(InsertPos: RepOrWidenR);
1857 VPBuilder Builder(Clone);
1858 VPValue *ExtractOp = Clone->getOperand(N: 0);
1859 if (vputils::isUniformAcrossVFsAndUFs(V: RepR->getOperand(N: 1)))
1860 ExtractOp =
1861 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastPart, Operands: ExtractOp);
1862 ExtractOp =
1863 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastLane, Operands: ExtractOp);
1864 Clone->setOperand(I: 0, New: ExtractOp);
1865 RepR->eraseFromParent();
1866 continue;
1867 }
1868
1869 // Narrow llvm.masked.{u,s}{div,rem} intrinsics with a safe divisor.
1870 if (auto *IntrR = dyn_cast<VPWidenIntrinsicRecipe>(Val: RepOrWidenR)) {
1871 if (!vputils::onlyFirstLaneUsed(Def: IntrR))
1872 continue;
1873 auto Opc = getUnmaskedDivRemOpcode(ID: IntrR->getVectorIntrinsicID());
1874 if (!Opc)
1875 continue;
1876 VPBuilder Builder(IntrR);
1877 VPValue *SafeDivisor = Builder.createSelect(
1878 Cond: IntrR->getOperand(N: 2), TrueVal: IntrR->getOperand(N: 1),
1879 FalseVal: Plan.getConstantInt(Ty: IntrR->getScalarType(), Val: 1));
1880 VPValue *Clone = Builder.createNaryOp(
1881 Opcode: *Opc, Operands: {IntrR->getOperand(N: 0), SafeDivisor},
1882 Flags: VPIRFlags::getDefaultFlags(Opcode: *Opc), DL: IntrR->getDebugLoc());
1883 IntrR->replaceAllUsesWith(New: Clone);
1884 IntrR->eraseFromParent();
1885 continue;
1886 }
1887
1888 // Skip recipes that aren't single scalars.
1889 if (!vputils::isSingleScalar(VPV: RepOrWidenR))
1890 continue;
1891
1892 // Predicate to check if a user of Op introduces extra broadcasts.
1893 auto IntroducesBCastOf = [](const VPValue *Op) {
1894 return [Op](const VPUser *U) {
1895 if (auto *VPI = dyn_cast<VPInstruction>(Val: U)) {
1896 if (is_contained(Set: {VPInstruction::ExtractLastLane,
1897 VPInstruction::ExtractLastPart,
1898 VPInstruction::ExtractPenultimateElement},
1899 Element: VPI->getOpcode()))
1900 return false;
1901 }
1902 return !U->usesScalars(Op);
1903 };
1904 };
1905
1906 if (any_of(Range: RepOrWidenR->users(), P: IntroducesBCastOf(RepOrWidenR)) &&
1907 none_of(Range: RepOrWidenR->operands(), P: [&](VPValue *Op) {
1908 if (any_of(
1909 Range: make_filter_range(Range: Op->users(), Pred: not_equal_to(Arg&: RepOrWidenR)),
1910 P: IntroducesBCastOf(Op)))
1911 return false;
1912 // Non-constant live-ins require broadcasts, while constants do not
1913 // need explicit broadcasts.
1914 bool LiveInNeedsBroadcast =
1915 isa<VPIRValue>(Val: Op) && !isa<VPConstant>(Val: Op);
1916 auto *OpR = dyn_cast<VPReplicateRecipe>(Val: Op);
1917 return LiveInNeedsBroadcast || (OpR && OpR->isSingleScalar());
1918 }))
1919 continue;
1920
1921 auto *Clone = VPBuilder::createSingleScalarOp(
1922 Opcode: vputils::getOpcode(V: RepOrWidenR), Operands: RepOrWidenR->operands(),
1923 /*Mask=*/nullptr, Flags: *RepOrWidenR, Metadata: getMetadataOf(R: RepOrWidenR),
1924 DL: DebugLoc::getUnknown(), ResultTy: RepOrWidenR->getScalarType(),
1925 UV: RepOrWidenR->getUnderlyingInstr());
1926 Clone->insertBefore(InsertPos: RepOrWidenR);
1927 RepOrWidenR->replaceAllUsesWith(New: Clone);
1928 if (vputils::isDeadRecipe(R&: *RepOrWidenR))
1929 RepOrWidenR->eraseFromParent();
1930 }
1931 }
1932}
1933
1934/// Try to see if all of \p Blend's masks share a common value logically and'ed
1935/// and remove it from the masks.
1936static void removeCommonBlendMask(VPBlendRecipe *Blend) {
1937 if (Blend->isNormalized())
1938 return;
1939 VPValue *CommonEdgeMask;
1940 if (!match(V: Blend->getMask(Idx: 0),
1941 P: m_LogicalAnd(Op0: m_VPValue(V&: CommonEdgeMask), Op1: m_VPValue())))
1942 return;
1943 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1944 if (!match(V: Blend->getMask(Idx: I),
1945 P: m_LogicalAnd(Op0: m_Specific(VPV: CommonEdgeMask), Op1: m_VPValue())))
1946 return;
1947 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1948 Blend->setMask(Idx: I, V: Blend->getMask(Idx: I)->getDefiningRecipe()->getOperand(N: 1));
1949}
1950
1951/// Normalize and simplify VPBlendRecipes. Should be run after combineRecipes
1952/// to make sure the masks are simplified.
1953static void simplifyBlends(VPlan &Plan) {
1954 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1955 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
1956 for (VPBlendRecipe &Blend :
1957 make_early_inc_range(Range: make_isa_range<VPBlendRecipe>(Range&: *VPBB))) {
1958 removeCommonBlendMask(Blend: &Blend);
1959
1960 // Try to remove redundant blend recipes.
1961 SmallPtrSet<VPValue *, 4> UniqueValues;
1962 if (Blend.isNormalized() || !match(V: Blend.getMask(Idx: 0), P: m_False()))
1963 UniqueValues.insert(Ptr: Blend.getIncomingValue(Idx: 0));
1964 for (unsigned I = 1; I != Blend.getNumIncomingValues(); ++I)
1965 if (!match(V: Blend.getMask(Idx: I), P: m_False()))
1966 UniqueValues.insert(Ptr: Blend.getIncomingValue(Idx: I));
1967
1968 if (UniqueValues.size() == 1) {
1969 Blend.replaceAllUsesWith(New: *UniqueValues.begin());
1970 Blend.eraseFromParent();
1971 continue;
1972 }
1973
1974 if (Blend.isNormalized())
1975 continue;
1976
1977 // Normalize the blend so its first incoming value is used as the initial
1978 // value with the others blended into it.
1979
1980 unsigned StartIndex = 0;
1981 for (unsigned I = 0; I != Blend.getNumIncomingValues(); ++I) {
1982 // If a value's mask is used only by the blend then is can be deadcoded.
1983 // TODO: Find the most expensive mask that can be deadcoded, or a mask
1984 // that's used by multiple blends where it can be removed from them all.
1985 VPValue *Mask = Blend.getMask(Idx: I);
1986 if (Mask->hasOneUse() && !match(V: Mask, P: m_False())) {
1987 StartIndex = I;
1988 break;
1989 }
1990 }
1991
1992 SmallVector<VPValue *, 4> OperandsWithMask;
1993 OperandsWithMask.push_back(Elt: Blend.getIncomingValue(Idx: StartIndex));
1994
1995 for (unsigned I = 0; I != Blend.getNumIncomingValues(); ++I) {
1996 if (I == StartIndex)
1997 continue;
1998 OperandsWithMask.push_back(Elt: Blend.getIncomingValue(Idx: I));
1999 OperandsWithMask.push_back(Elt: Blend.getMask(Idx: I));
2000 }
2001
2002 auto *NewBlend =
2003 new VPBlendRecipe(cast_or_null<PHINode>(Val: Blend.getUnderlyingValue()),
2004 OperandsWithMask, Blend, Blend.getDebugLoc());
2005 NewBlend->insertBefore(InsertPos: &Blend);
2006
2007 VPValue *DeadMask = Blend.getMask(Idx: StartIndex);
2008 Blend.replaceAllUsesWith(New: NewBlend);
2009 Blend.eraseFromParent();
2010 vputils::recursivelyDeleteDeadRecipes(V: DeadMask);
2011
2012 /// Simplify BLEND %a, %b, Not(%mask) -> BLEND %b, %a, %mask.
2013 VPValue *NewMask;
2014 if (NewBlend->getNumOperands() == 3 &&
2015 match(V: NewBlend->getMask(Idx: 1), P: m_Not(Op0: m_VPValue(V&: NewMask)))) {
2016 VPValue *Inc0 = NewBlend->getOperand(N: 0);
2017 VPValue *Inc1 = NewBlend->getOperand(N: 1);
2018 VPValue *OldMask = NewBlend->getOperand(N: 2);
2019 NewBlend->setOperand(I: 0, New: Inc1);
2020 NewBlend->setOperand(I: 1, New: Inc0);
2021 NewBlend->setOperand(I: 2, New: NewMask);
2022 if (OldMask->user_empty())
2023 cast<VPInstruction>(Val: OldMask)->eraseFromParent();
2024 }
2025 }
2026 }
2027}
2028
2029/// Optimize the width of vector induction variables in \p Plan based on a known
2030/// constant Trip Count, \p BestVF and \p BestUF.
2031static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan,
2032 ElementCount BestVF,
2033 unsigned BestUF) {
2034 // Only proceed if we have not completely removed the vector region.
2035 if (!Plan.getVectorLoopRegion())
2036 return false;
2037
2038 const APInt *TC;
2039 if (!BestVF.isFixed() || !match(V: Plan.getTripCount(), P: m_APInt(C&: TC)))
2040 return false;
2041
2042 // Calculate the minimum power-of-2 bit width that can fit the known TC, VF
2043 // and UF. Returns at least 8.
2044 auto ComputeBitWidth = [](APInt TC, uint64_t Align) {
2045 APInt AlignedTC =
2046 Align * APIntOps::RoundingUDiv(A: TC, B: APInt(TC.getBitWidth(), Align),
2047 RM: APInt::Rounding::UP);
2048 APInt MaxVal = AlignedTC - 1;
2049 return std::max<unsigned>(a: PowerOf2Ceil(A: MaxVal.getActiveBits()), b: 8);
2050 };
2051 unsigned NewBitWidth =
2052 ComputeBitWidth(*TC, BestVF.getKnownMinValue() * BestUF);
2053
2054 LLVMContext &Ctx = Plan.getContext();
2055 auto *NewIVTy = IntegerType::get(C&: Ctx, NumBits: NewBitWidth);
2056
2057 bool MadeChange = false;
2058
2059 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
2060 for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
2061 // Currently only handle canonical IVs as it is trivial to replace the start
2062 // and stop values, and we currently only perform the optimization when the
2063 // IV has a single use.
2064 VPWidenIntOrFpInductionRecipe *WideIV;
2065 if (!match(V: &Phi, P: m_CanonicalWidenIV(V&: WideIV)))
2066 continue;
2067 if (WideIV->hasMoreThanOneUniqueUser() ||
2068 NewIVTy == WideIV->getScalarType())
2069 continue;
2070
2071 // Currently only handle cases where the single user is a header-mask
2072 // comparison with the backedge-taken-count.
2073 VPUser *SingleUser = WideIV->getSingleUser();
2074 if (!SingleUser ||
2075 !match(U: SingleUser,
2076 P: m_ICmp(Op0: m_Specific(VPV: WideIV),
2077 Op1: m_Broadcast(Op0: m_Specific(VPV: Plan.getBackedgeTakenCount())))))
2078 continue;
2079
2080 // Update IV operands and comparison bound to use new narrower type.
2081 assert(!WideIV->getTruncInst() &&
2082 "canonical IV is not expected to have a truncation");
2083 auto *NewWideIV = new VPWidenIntOrFpInductionRecipe(
2084 WideIV->getPHINode(), Plan.getZero(Ty: NewIVTy),
2085 Plan.getConstantInt(Ty: NewIVTy, Val: 1), WideIV->getVFValue(),
2086 WideIV->getInductionDescriptor(), *WideIV, WideIV->getDebugLoc());
2087 NewWideIV->insertBefore(InsertPos: WideIV);
2088
2089 auto *NewBTC = new VPWidenCastRecipe(
2090 Instruction::Trunc, Plan.getOrCreateBackedgeTakenCount(), NewIVTy,
2091 nullptr, VPIRFlags::getDefaultFlags(Opcode: Instruction::Trunc));
2092 Plan.getVectorPreheader()->appendRecipe(Recipe: NewBTC);
2093 auto *Cmp = cast<VPInstruction>(Val: WideIV->getSingleUser());
2094 Cmp->replaceAllUsesWith(
2095 New: VPBuilder(Cmp).createICmp(Pred: Cmp->getPredicate(), A: NewWideIV, B: NewBTC));
2096
2097 MadeChange = true;
2098 }
2099
2100 return MadeChange;
2101}
2102
2103/// Return true if \p Cond is known to be true for given \p BestVF and \p
2104/// BestUF.
2105static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
2106 ElementCount BestVF, unsigned BestUF,
2107 PredicatedScalarEvolution &PSE) {
2108 if (match(V: Cond, P: m_BinaryOr(Op0: m_VPValue(), Op1: m_VPValue())))
2109 return any_of(Range: Cond->getDefiningRecipe()->operands(), P: [&Plan, BestVF, BestUF,
2110 &PSE](VPValue *C) {
2111 return isConditionTrueViaVFAndUF(Cond: C, Plan, BestVF, BestUF, PSE);
2112 });
2113
2114 auto *CanIV = Plan.getVectorLoopRegion()->getCanonicalIV();
2115 if (!match(V: Cond, P: m_SpecificICmp(
2116 MatchPred: CmpInst::ICMP_EQ,
2117 Op0: m_c_Add(Op0: m_Specific(VPV: CanIV), Op1: m_Specific(VPV: &Plan.getVFxUF())),
2118 Op1: m_Specific(VPV: &Plan.getVectorTripCount()))))
2119 return false;
2120
2121 // The compare checks CanIV + VFxUF == vector trip count. The vector trip
2122 // count is not conveniently available as SCEV so far, so we compare directly
2123 // against the original trip count. This is stricter than necessary, as we
2124 // will only return true if the trip count == vector trip count.
2125 const SCEV *VectorTripCount =
2126 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2127 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2128 VectorTripCount = vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2129 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2130 "Trip count SCEV must be computable");
2131 ScalarEvolution &SE = *PSE.getSE();
2132 ElementCount NumElements = BestVF * BestUF;
2133 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2134 return SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: VectorTripCount, RHS: C);
2135}
2136
2137// Replaces ExtractVectorForPart instructions with ICMP when the VF is scalar
2138// and the source is a WideActiveLaneMask. The unused mask is removed later
2139// when removing dead recipes.
2140static bool replaceMaskWithCompareForScalarPlan(VPlan &Plan,
2141 ElementCount BestVF) {
2142 if (!BestVF.isScalar())
2143 return false;
2144
2145 bool MadeChange = false;
2146 VPBuilder Builder;
2147 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2148 VPBasicBlock *PreheaderVPBB = Plan.getVectorPreheader();
2149 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2150
2151 VPValue *Start, *TC;
2152 uint64_t Idx;
2153 for (VPBasicBlock *VPBB : {PreheaderVPBB, ExitingVPBB}) {
2154 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2155 if (!match(V: &R, P: m_ExtractVectorForPart(
2156 Op0: m_WideActiveLaneMask(Op0: m_VPValue(V&: Start), Op1: m_VPValue(V&: TC),
2157 Op2: m_VPValue()),
2158 Op1: m_ConstantInt(C&: Idx))))
2159 continue;
2160
2161 auto *Extract = cast<VPInstruction>(Val: &R);
2162 Builder.setInsertPoint(Extract);
2163
2164 if (Idx > 0)
2165 Start = Builder.createAdd(
2166 LHS: Start, RHS: Plan.getConstantInt(Ty: Start->getScalarType(), Val: Idx));
2167
2168 VPValue *ICmp = Builder.createICmp(Pred: CmpInst::ICMP_ULT, A: Start, B: TC);
2169 Extract->replaceAllUsesWith(New: ICmp);
2170 Extract->eraseFromParent();
2171 MadeChange = true;
2172 }
2173 }
2174
2175 return MadeChange;
2176}
2177
2178/// Try to simplify the branch condition of \p Plan. This may restrict the
2179/// resulting plan to \p BestVF and \p BestUF.
2180static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
2181 unsigned BestUF,
2182 PredicatedScalarEvolution &PSE) {
2183 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2184 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2185 auto *Term = &ExitingVPBB->back();
2186 VPValue *Cond;
2187 VPValue *Offset = nullptr;
2188 auto m_CanIVInc = m_Add(Op0: m_VPValue(), Op1: m_Specific(VPV: &Plan.getVFxUF()));
2189 // Check if the branch condition compares the canonical IV increment (for main
2190 // loop), or the canonical IV increment plus an offset (for epilog loop).
2191 bool MatchedCanIVInc =
2192 match(V: Term,
2193 P: m_BranchOnCount(
2194 Op0: m_CombineOr(Ps: m_CanIVInc, Ps: m_c_Add(Op0: m_CanIVInc, Op1: m_VPValue(V&: Offset))),
2195 Op1: m_VPValue())) &&
2196 (!Offset || Offset->isDefinedOutsideLoopRegions());
2197 if (MatchedCanIVInc ||
2198 match(V: Term,
2199 P: m_BranchOnCond(Op0: m_Not(Op0: m_ExtractVectorForPart(
2200 Op0: m_WideActiveLaneMask(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue()),
2201 Op1: m_ZeroInt()))))) {
2202 // Try to simplify the branch condition if VectorTC <= VF * UF when the
2203 // latch terminator is BranchOnCount or
2204 // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask), 0))
2205 const SCEV *VectorTripCount =
2206 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2207 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2208 VectorTripCount =
2209 vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2210 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2211 "Trip count SCEV must be computable");
2212 ScalarEvolution &SE = *PSE.getSE();
2213 ElementCount NumElements = BestVF * BestUF;
2214 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2215 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_ULE, LHS: VectorTripCount, RHS: C))
2216 return false;
2217 } else if (match(V: Term, P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))) ||
2218 match(V: Term, P: m_BranchOnTwoConds(Op0: m_VPValue(), Op1: m_VPValue(V&: Cond)))) {
2219 // For BranchOnCond, check if we can prove the condition to be true using VF
2220 // and UF.
2221 if (!isConditionTrueViaVFAndUF(Cond, Plan, BestVF, BestUF, PSE))
2222 return false;
2223 } else {
2224 return false;
2225 }
2226
2227 // The vector loop region only executes once. Convert terminator of the
2228 // exiting block to exit in the first iteration.
2229 if (match(V: Term, P: m_BranchOnTwoConds())) {
2230 Term->setOperand(I: 1, New: Plan.getTrue());
2231 return true;
2232 }
2233
2234 auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, Plan.getTrue(), {},
2235 {}, Term->getDebugLoc());
2236 ExitingVPBB->appendRecipe(Recipe: BOC);
2237 Term->eraseFromParent();
2238
2239 return true;
2240}
2241
2242void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
2243 unsigned BestUF,
2244 PredicatedScalarEvolution &PSE) {
2245 assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
2246 assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
2247
2248 bool MadeChange =
2249 simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
2250 MadeChange |= replaceMaskWithCompareForScalarPlan(Plan, BestVF);
2251 MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
2252
2253 if (MadeChange) {
2254 Plan.setVF(BestVF);
2255 assert(Plan.getConcreteUF() == BestUF && "BestUF must match the Plan's UF");
2256 }
2257}
2258
2259void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
2260 for (VPReductionPHIRecipe &PhiR : make_isa_range<VPReductionPHIRecipe>(
2261 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
2262 RecurKind RK = PhiR.getRecurrenceKind();
2263 if (RK != RecurKind::Add && RK != RecurKind::Mul && RK != RecurKind::Sub &&
2264 RK != RecurKind::AddChainWithSubs)
2265 continue;
2266
2267 for (VPUser *U : vputils::collectUsersRecursively(V: &PhiR))
2268 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: U)) {
2269 RecWithFlags->dropPoisonGeneratingFlags();
2270 }
2271 }
2272}
2273
2274namespace {
2275struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
2276 /// If recipe \p R will lower to a GEP with a non-i8 source element type,
2277 /// return that source element type.
2278 static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) {
2279 // All VPInstructions that lower to GEPs must have the i8 source element
2280 // type (as they are PtrAdds), so we omit it.
2281 return TypeSwitch<const VPSingleDefRecipe *, Type *>(R)
2282 .Case(caseFn: [](const VPReplicateRecipe *I) -> Type * {
2283 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I->getUnderlyingValue()))
2284 return GEP->getSourceElementType();
2285 return nullptr;
2286 })
2287 .Case<VPVectorPointerRecipe, VPWidenGEPRecipe>(
2288 caseFn: [](auto *I) { return I->getSourceElementType(); })
2289 .Default(defaultFn: [](auto *) { return nullptr; });
2290 }
2291
2292 /// Returns true if recipe \p Def can be safely handed for CSE.
2293 static bool canHandle(const VPSingleDefRecipe *Def) {
2294 // We can extend the list of handled recipes in the future,
2295 // provided we account for the data embedded in them while checking for
2296 // equality or hashing.
2297 auto C = vputils::getOpcodeOrIntrinsicID(V: Def);
2298
2299 // The issue with (Insert|Extract)Value is that the index of the
2300 // insert/extract is not a proper operand in LLVM IR, and hence also not in
2301 // VPlan. Allocas must not be merged, as each creates a distinct allocation.
2302 if (!C || (!C->first && (C->second == Instruction::InsertValue ||
2303 C->second == Instruction::ExtractValue ||
2304 C->second == Instruction::Alloca)))
2305 return false;
2306
2307 // Widened loads (including the EVL variant) are handled, as cse() only
2308 // reuses them within a block with no intervening memory write. Any other
2309 // memory access is rejected.
2310 if (Def->mayWriteToMemory())
2311 return false;
2312 return !Def->mayReadFromMemory() ||
2313 isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Val: Def);
2314 }
2315
2316 /// Hash the underlying data of \p Def.
2317 static unsigned getHashValue(const VPSingleDefRecipe *Def) {
2318 hash_code Result = hash_combine(
2319 args: Def->getVPRecipeID(), args: vputils::getOpcodeOrIntrinsicID(V: Def),
2320 args: getGEPSourceElementType(R: Def), args: Def->getScalarType(),
2321 args: vputils::isSingleScalar(VPV: Def), args: hash_combine_range(R: Def->operands()));
2322 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: Def))
2323 if (RFlags->hasPredicate())
2324 return hash_combine(args: Result, args: RFlags->getPredicate());
2325 if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def))
2326 return hash_combine(args: Result, args: SIVSteps->getInductionOpcode());
2327 // Fold in the separately stored consecutive flag. Alignment is left out and
2328 // handled by cse.
2329 if (auto *Load = dyn_cast<VPWidenMemoryRecipe>(Val: Def))
2330 return hash_combine(args: Result, args: Load->isConsecutive());
2331 return Result;
2332 }
2333
2334 /// Check equality of underlying data of \p L and \p R.
2335 static bool isEqual(const VPSingleDefRecipe *L, const VPSingleDefRecipe *R) {
2336 if (L->getVPRecipeID() != R->getVPRecipeID() ||
2337 vputils::getOpcodeOrIntrinsicID(V: L) !=
2338 vputils::getOpcodeOrIntrinsicID(V: R) ||
2339 getGEPSourceElementType(R: L) != getGEPSourceElementType(R) ||
2340 vputils::isSingleScalar(VPV: L) != vputils::isSingleScalar(VPV: R) ||
2341 !equal(LRange: L->operands(), RRange: R->operands()))
2342 return false;
2343 assert(vputils::getOpcodeOrIntrinsicID(L) &&
2344 vputils::getOpcodeOrIntrinsicID(R) &&
2345 "must have valid opcode info for both recipes");
2346 if (auto *LFlags = dyn_cast<VPRecipeWithIRFlags>(Val: L))
2347 if (LFlags->hasPredicate() &&
2348 LFlags->getPredicate() !=
2349 cast<VPRecipeWithIRFlags>(Val: R)->getPredicate())
2350 return false;
2351 if (auto *LSIV = dyn_cast<VPScalarIVStepsRecipe>(Val: L))
2352 if (LSIV->getInductionOpcode() !=
2353 cast<VPScalarIVStepsRecipe>(Val: R)->getInductionOpcode())
2354 return false;
2355 // Compare the separately stored consecutive flag. Alignment is left out and
2356 // handled by cse.
2357 if (auto *LL = dyn_cast<VPWidenMemoryRecipe>(Val: L))
2358 if (LL->isConsecutive() != cast<VPWidenMemoryRecipe>(Val: R)->isConsecutive())
2359 return false;
2360 // Phi recipes can only be equal if they are in the same VPBB, as they
2361 // implicitly depend on their predecessors.
2362 if (isa<VPWidenPHIRecipe>(Val: L) && L->getParent() != R->getParent())
2363 return false;
2364 // Recipes in replicate regions implicitly depend on predicate. If either
2365 // recipe is in a replicate region, only consider them equal if both have
2366 // the same parent.
2367 const VPRegionBlock *RegionL = L->getRegion();
2368 const VPRegionBlock *RegionR = R->getRegion();
2369 if (((RegionL && RegionL->isReplicator()) ||
2370 (RegionR && RegionR->isReplicator())) &&
2371 L->getParent() != R->getParent())
2372 return false;
2373 return L->getScalarType() == R->getScalarType();
2374 }
2375};
2376} // end anonymous namespace
2377
2378/// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
2379/// Plan.
2380void VPlanTransforms::cse(VPlan &Plan) {
2381 VPDominatorTree VPDT(Plan);
2382 DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo> CSEMap;
2383 // CSE map for widened loads. Must be cleared on recipes that may write to
2384 // memory, and at the end of each VPBB.
2385 DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo>
2386 LoadCSEMap;
2387
2388 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
2389 Plan.getEntry());
2390 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
2391 for (VPRecipeBase &R : *VPBB) {
2392 if (R.mayWriteToMemory())
2393 LoadCSEMap.clear();
2394 auto *Def = dyn_cast<VPSingleDefRecipe>(Val: &R);
2395 if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
2396 continue;
2397 bool IsLoad = isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Val: Def);
2398 auto [It, Inserted] =
2399 (IsLoad ? LoadCSEMap : CSEMap).try_emplace(Key: Def, Args&: Def);
2400 if (Inserted)
2401 continue;
2402 VPSingleDefRecipe *V = It->second;
2403 // V must dominate Def for a valid replacement.
2404 if (!VPDT.dominates(A: V->getParent(), B: VPBB))
2405 continue;
2406 if (IsLoad) {
2407 auto *EarlierLoad = cast<VPWidenMemoryRecipe>(Val: V);
2408 auto *Load = cast<VPWidenMemoryRecipe>(Val: Def);
2409 if (EarlierLoad->getAlign() < Load->getAlign()) {
2410 // Record Load as the candidate for subsequent loads, as it may be
2411 // reusable where EarlierLoad is not.
2412 It->second = Def;
2413 continue;
2414 }
2415 // Keep only metadata common to both loads on the survivor.
2416 EarlierLoad->intersect(MD: *Load);
2417 }
2418 // Only keep flags present on both V and Def.
2419 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: V))
2420 RFlags->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Def));
2421 Def->replaceAllUsesWith(New: V);
2422 }
2423 LoadCSEMap.clear();
2424 }
2425}
2426
2427/// Return true if we do not know how to (mechanically) hoist or sink a
2428/// non-memory or memory recipe \p R out of a loop region. When sinking, passing
2429/// \p Sinking = true ensures that assumes aren't sunk.
2430static bool cannotHoistOrSinkRecipe(VPRecipeBase &R, VPBasicBlock *FirstBB,
2431 VPBasicBlock *LastBB,
2432 bool Sinking = false) {
2433 if (!isa<VPReplicateRecipe>(Val: R) || !R.mayReadOrWriteMemory() ||
2434 match(V: &R, P: m_Intrinsic<Intrinsic::assume>()))
2435 return vputils::cannotHoistOrSinkRecipe(R, Sinking);
2436
2437 // Check that the memory operation doesn't alias between FirstBB and LastBB.
2438 auto MemLoc = vputils::getMemoryLocation(R);
2439
2440 // TODO: Could make use of SinkStoreInfo::isNoAliasViaDistance by collecting
2441 // stores upfront, and constructing a full SinkStoreInfo.
2442 auto SinkInfo =
2443 Sinking ? std::make_optional(t: SinkStoreInfo(cast<VPReplicateRecipe>(Val&: R)))
2444 : std::nullopt;
2445
2446 return !MemLoc ||
2447 !canHoistOrSinkWithNoAliasCheck(MemLoc: *MemLoc, FirstBB, LastBB, SinkInfo);
2448}
2449
2450/// Move loop-invariant recipes out of the vector loop region in \p Plan.
2451static void licm(VPlan &Plan) {
2452 VPBasicBlock *Preheader = Plan.getVectorPreheader();
2453
2454 // Hoist any loop invariant recipes from the vector loop region to the
2455 // preheader. Preform a shallow traversal of the vector loop region, to
2456 // exclude recipes in replicate regions. Since the top-level blocks in the
2457 // vector loop region are guaranteed to execute if the vector pre-header is,
2458 // we don't need to check speculation safety.
2459 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2460 assert(Preheader->getSingleSuccessor() == LoopRegion &&
2461 "Expected vector prehader's successor to be the vector loop region");
2462 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2463 Range: vp_depth_first_shallow(G: LoopRegion->getEntry()))) {
2464 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2465 if (any_of(Range: R.operands(), P: [](VPValue *Op) {
2466 return !Op->isDefinedOutsideLoopRegions();
2467 }))
2468 continue;
2469 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2470 LastBB: LoopRegion->getExitingBasicBlock()))
2471 continue;
2472 R.moveBefore(BB&: *Preheader, I: Preheader->end());
2473 }
2474 }
2475
2476#ifndef NDEBUG
2477 VPDominatorTree VPDT(Plan);
2478#endif
2479 // Sink recipes with no users inside the vector loop region if all users are
2480 // in the same exit block of the region.
2481 // TODO: Extend to sink recipes from inner loops.
2482 PostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> POT(
2483 LoopRegion->getEntry());
2484 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
2485 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
2486 // TODO: Use R.definedValues() instead of casting to VPSingleDefRecipe to
2487 // support recipes with multiple defined values (e.g., interleaved loads).
2488 auto *Def = dyn_cast<VPSingleDefRecipe>(Val: &R);
2489 if (!Def)
2490 continue;
2491
2492 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
2493 assert(!RepR->isPredicated() &&
2494 "Expected prior transformation of predicated replicates to "
2495 "replicate regions");
2496 // narrowToSingleScalarRecipes should have already maximally narrowed
2497 // replicates to single-scalar replicates.
2498 // TODO: When unrolling, replicateByVF doesn't handle sunk
2499 // non-single-scalar replicates correctly.
2500 if (!RepR->isSingleScalar())
2501 continue;
2502
2503 // The pointer operand of stores must be loop-invariant.
2504 if (RepR->getOpcode() == Instruction::Store &&
2505 !RepR->getOperand(N: 1)->isDefinedOutsideLoopRegions())
2506 continue;
2507 }
2508
2509 // Cannot sink the recipe if the user is defined in a loop region or a
2510 // non-successor of the vector loop region. Cannot sink if user is a phi
2511 // either.
2512 VPBasicBlock *SinkBB = nullptr;
2513 if (any_of(Range: Def->users(), P: [&SinkBB, &LoopRegion](VPUser *U) {
2514 auto *UserR = cast<VPRecipeBase>(Val: U);
2515 VPBasicBlock *Parent = UserR->getParent();
2516 // TODO: Support sinking when users are in multiple blocks.
2517 if (SinkBB && SinkBB != Parent)
2518 return true;
2519 SinkBB = Parent;
2520 // TODO: If the user is a PHI node, we should check the block of
2521 // incoming value. Support PHI node users if needed.
2522 return UserR->isPhi() || Parent->getEnclosingLoopRegion() ||
2523 Parent->getSinglePredecessor() != LoopRegion;
2524 }))
2525 continue;
2526
2527 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2528 LastBB: LoopRegion->getExitingBasicBlock(),
2529 /*Sinking=*/true))
2530 continue;
2531
2532 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
2533 assert((!R.mayWriteToMemory() ||
2534 (RepR && RepR->getOpcode() == Instruction::Store &&
2535 RepR->getOperand(1)->isDefinedOutsideLoopRegions())) &&
2536 "The only recipes that may write to memory are expected to be "
2537 "stores with invariant pointer-operand");
2538
2539 if (!SinkBB)
2540 SinkBB = cast<VPBasicBlock>(Val: LoopRegion->getSingleSuccessor());
2541
2542 // TODO: This will need to be a check instead of a assert after
2543 // conditional branches in vectorized loops are supported.
2544 assert(VPDT.properlyDominates(VPBB, SinkBB) &&
2545 "Defining block must dominate sink block");
2546 // TODO: Clone the recipe if users are on multiple exit paths, instead of
2547 // just moving.
2548 Def->moveBefore(BB&: *SinkBB, I: SinkBB->getFirstNonPhi());
2549 }
2550 }
2551}
2552
2553void VPlanTransforms::truncateToMinimalBitwidths(
2554 VPlan &Plan, const MapVector<Instruction *, uint64_t> &MinBWs) {
2555 if (Plan.hasScalarVFOnly())
2556 return;
2557 // Keep track of created truncates, so they can be re-used. Note that we
2558 // cannot use RAUW after creating a new truncate, as this would could make
2559 // other uses have different types for their operands, making them invalidly
2560 // typed.
2561 DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
2562 VPBasicBlock *PH = Plan.getVectorPreheader();
2563 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2564 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
2565 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2566 if (!isa<VPWidenRecipe, VPWidenCastRecipe, VPReplicateRecipe,
2567 VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R))
2568 continue;
2569
2570 VPValue *ResultVPV = R.getVPSingleValue();
2571 auto *UI = cast_or_null<Instruction>(Val: ResultVPV->getUnderlyingValue());
2572 unsigned NewResSizeInBits = MinBWs.lookup(Key: UI);
2573 if (!NewResSizeInBits)
2574 continue;
2575
2576 // If the value wasn't vectorized, we must maintain the original scalar
2577 // type. Skip those here, after incrementing NumProcessedRecipes. Also
2578 // skip casts which do not need to be handled explicitly here, as
2579 // redundant casts will be removed during recipe simplification.
2580 if (isa<VPReplicateRecipe, VPWidenCastRecipe>(Val: &R))
2581 continue;
2582
2583 Type *OldResTy = ResultVPV->getScalarType();
2584 unsigned OldResSizeInBits = OldResTy->getScalarSizeInBits();
2585 assert(OldResTy->isIntegerTy() && "only integer types supported");
2586 (void)OldResSizeInBits;
2587
2588 auto *NewResTy = IntegerType::get(C&: Plan.getContext(), NumBits: NewResSizeInBits);
2589
2590 // Any wrapping introduced by shrinking this operation shouldn't be
2591 // considered undefined behavior. So, we can't unconditionally copy
2592 // arithmetic wrapping flags to VPW.
2593 if (auto *VPW = dyn_cast<VPRecipeWithIRFlags>(Val: &R))
2594 VPW->dropPoisonGeneratingFlags();
2595
2596 assert((OldResSizeInBits != NewResSizeInBits ||
2597 match(&R, m_ICmp(m_VPValue(), m_VPValue()))) &&
2598 "Only ICmps should not need extending the result.");
2599 assert(!isa<VPWidenStoreRecipe>(&R) && "stores cannot be narrowed");
2600
2601 // Loads/intrinsics are not recreated; they keep producing their original
2602 // wide result and narrowed users will truncate it as needed below.
2603 if (isa<VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R))
2604 continue;
2605
2606 // Shrink operands by introducing truncates as needed.
2607 unsigned StartIdx =
2608 match(V: &R, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())) ? 1 : 0;
2609 SmallVector<VPValue *> NewOperands(R.operands());
2610 for (VPValue *&Op : drop_begin(RangeOrContainer&: NewOperands, N: StartIdx)) {
2611 unsigned OpSizeInBits = Op->getScalarType()->getScalarSizeInBits();
2612 if (OpSizeInBits == NewResSizeInBits)
2613 continue;
2614 assert(OpSizeInBits > NewResSizeInBits && "nothing to truncate");
2615 auto [ProcessedIter, Inserted] = ProcessedTruncs.try_emplace(Key: Op);
2616 if (Inserted) {
2617 VPBuilder Builder;
2618 if (isa<VPIRValue>(Val: Op))
2619 Builder.setInsertPoint(PH);
2620 else
2621 Builder.setInsertPoint(&R);
2622 ProcessedIter->second =
2623 Builder.createWidenCast(Opcode: Instruction::Trunc, Op, ResultTy: NewResTy);
2624 }
2625 Op = ProcessedIter->second;
2626 }
2627
2628 auto *NWR = cast<VPWidenRecipe>(Val: &R)->cloneWithOperands(NewOperands);
2629 NWR->insertBefore(InsertPos: &R);
2630
2631 // Wrap NWR in a ZExt to preserve the original wide type for downstream
2632 // users. Not needed for ICmps, whose result type is i1 irrespective of
2633 // the narrowing of their operands.
2634 VPValue *Replacement = NWR->getVPSingleValue();
2635 if (Replacement->getScalarType() != OldResTy)
2636 Replacement =
2637 VPBuilder::getToInsertAfter(R: NWR)
2638 .createWidenCast(Opcode: Instruction::ZExt, Op: Replacement, ResultTy: OldResTy)
2639 ->getVPSingleValue();
2640 ResultVPV->replaceAllUsesWith(New: Replacement);
2641 R.eraseFromParent();
2642 }
2643 }
2644}
2645
2646bool VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) {
2647 std::optional<VPDominatorTree> VPDT;
2648 if (OnlyLatches)
2649 VPDT.emplace(args&: Plan);
2650
2651 // Collect all blocks before modifying the CFG so we can identify unreachable
2652 // ones after constant branch removal.
2653 SmallVector<VPBlockBase *> AllBlocks(vp_depth_first_shallow(G: Plan.getEntry()));
2654
2655 bool SimplifiedPhi = false;
2656 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: AllBlocks)) {
2657 VPValue *Cond;
2658 // Skip blocks that are not terminated by BranchOnCond.
2659 if (VPBB->empty() || !match(V: &VPBB->back(), P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))))
2660 continue;
2661
2662 if (OnlyLatches && !VPBlockUtils::isLatch(VPB: VPBB, VPDT: *VPDT))
2663 continue;
2664
2665 assert(VPBB->getNumSuccessors() == 2 &&
2666 "Two successors expected for BranchOnCond");
2667 unsigned RemovedIdx;
2668 if (match(V: Cond, P: m_True()))
2669 RemovedIdx = 1;
2670 else if (match(V: Cond, P: m_False()))
2671 RemovedIdx = 0;
2672 else
2673 continue;
2674
2675 VPBasicBlock *RemovedSucc =
2676 cast<VPBasicBlock>(Val: VPBB->getSuccessors()[RemovedIdx]);
2677 assert(count(RemovedSucc->getPredecessors(), VPBB) == 1 &&
2678 "There must be a single edge between VPBB and its successor");
2679 // Values coming from VPBB into phi recipes of RemovedSucc are removed from
2680 // these recipes and single-entry header phis are removed.
2681 for (VPRecipeBase &R : make_early_inc_range(Range: RemovedSucc->phis())) {
2682 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: VPBB);
2683 SimplifiedPhi = true;
2684 // Remove now invalid header phis that are left single-entry after
2685 // removing their backedges.
2686 auto *PhiR = dyn_cast<VPHeaderPHIRecipe>(Val: &R);
2687 if (!PhiR || PhiR->getNumIncoming() != 1)
2688 continue;
2689 PhiR->replaceAllUsesWith(New: PhiR->getOperand(N: 0));
2690 PhiR->eraseFromParent();
2691 }
2692
2693 // Disconnect blocks and remove the terminator.
2694 VPBlockUtils::disconnectBlocks(From: VPBB, To: RemovedSucc);
2695 VPBB->back().eraseFromParent();
2696 }
2697
2698 // Compute which blocks are still reachable from the entry after constant
2699 // branch removal.
2700 SmallPtrSet<VPBlockBase *, 16> Reachable(
2701 llvm::from_range, vp_depth_first_shallow(G: Plan.getEntry()));
2702
2703 // Detach all unreachable blocks from their successors, removing their recipes
2704 // and incoming values from phi recipes.
2705 VPSymbolicValue Tmp(nullptr);
2706 for (VPBlockBase *B : AllBlocks) {
2707 if (Reachable.contains(Ptr: B))
2708 continue;
2709 for (VPBlockBase *Succ : to_vector(Range: B->successors())) {
2710 if (auto *SuccBB = dyn_cast<VPBasicBlock>(Val: Succ))
2711 for (VPRecipeBase &R : SuccBB->phis())
2712 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: B);
2713 VPBlockUtils::disconnectBlocks(From: B, To: Succ);
2714 }
2715 for (VPBasicBlock *DeadBB :
2716 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_deep(G: B))) {
2717 for (VPRecipeBase &R : make_early_inc_range(Range&: *DeadBB)) {
2718 for (VPValue *Def : R.definedValues())
2719 Def->replaceAllUsesWith(New: &Tmp);
2720 R.eraseFromParent();
2721 }
2722 }
2723 }
2724 return SimplifiedPhi;
2725}
2726
2727void VPlanTransforms::optimize(VPlan &Plan) {
2728 RUN_VPLAN_PASS(removeRedundantInductionCasts, Plan);
2729
2730 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2731 RUN_VPLAN_PASS(combineRecipes, Plan);
2732 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2733 RUN_VPLAN_PASS(simplifyBlends, Plan);
2734 RUN_VPLAN_PASS(legalizeAndOptimizeInductions, Plan);
2735 RUN_VPLAN_PASS(narrowToSingleScalarRecipes, Plan);
2736 RUN_VPLAN_PASS(removeRedundantExpandSCEVRecipes, Plan);
2737 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2738 RUN_VPLAN_PASS(combineRecipes, Plan);
2739 RUN_VPLAN_PASS(removeBranchOnConst, Plan, /*OnlyLatches=*/false);
2740 RUN_VPLAN_PASS(simplifyReverses, Plan);
2741 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2742
2743 RUN_VPLAN_PASS(createAndOptimizeReplicateRegions, Plan);
2744 RUN_VPLAN_PASS(mergeBlocksIntoPredecessors, Plan);
2745 RUN_VPLAN_PASS(licm, Plan);
2746}
2747
2748void VPlanTransforms::simplifyLiveInsWithSCEV(VPlan &Plan,
2749 PredicatedScalarEvolution &PSE) {
2750 auto GetSimplifiedLiveInViaSCEV = [&](VPValue *VPV) -> VPValue * {
2751 const SCEV *Expr = vputils::getSCEVExprForVPValue(V: VPV, PSE);
2752 const APInt *C;
2753 if (match(S: Expr, P: m_scev_APInt(C)))
2754 return Plan.getConstantInt(Val: *C);
2755 return nullptr;
2756 };
2757
2758 for (VPValue *LiveIn : to_vector(Range: Plan.getLiveIns())) {
2759 if (VPValue *SimplifiedLiveIn = GetSimplifiedLiveInViaSCEV(LiveIn))
2760 LiveIn->replaceAllUsesWith(New: SimplifiedLiveIn);
2761 }
2762}
2763
2764void VPlanTransforms::replaceSymbolicStrides(
2765 VPlan &Plan, PredicatedScalarEvolution &PSE,
2766 const SymbolicStrideMap &StridesMap, const VPDominatorTree &VPDT) {
2767 // Replace VPValues for known constant strides guaranteed by predicated scalar
2768 // evolution that are guaranteed to be guarded by the runtime checks; that is,
2769 // blocks dominated by the vector header.
2770 assert(!Plan.getVectorLoopRegion() &&
2771 "expected to run before loop regions are created");
2772 const auto &[Header, _] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
2773 auto CanUseVersionedStride = [&VPDT, Header = Header, &Plan](VPUser &U) {
2774 auto *R = cast<VPRecipeBase>(Val: &U);
2775 // Skip phis if the loop if loop is not yet guarded.
2776 if (isa<VPPhiAccessors>(Val: R) &&
2777 Header == Plan.getEntry()->getSingleSuccessor())
2778 return false;
2779 return VPDT.dominates(A: Header, B: R->getParent());
2780 };
2781 ValueToSCEVMapTy RewriteMap;
2782 for (const SCEVUnknown *Stride : StridesMap.values()) {
2783 Value *StrideV = Stride->getValue();
2784 const APInt *StrideConst;
2785 const SCEV *StrideExpr = PSE.getSCEV(V: StrideV);
2786 if (!match(S: StrideExpr, P: m_scev_APInt(C&: StrideConst)))
2787 // Only handle constant strides for now.
2788 continue;
2789 if (VPValue *StrideVPV = Plan.getLiveIn(V: StrideV))
2790 StrideVPV->replaceUsesWithIf(New: Plan.getConstantInt(Val: *StrideConst),
2791 ShouldReplace: CanUseVersionedStride);
2792
2793 // The versioned value may not be used in the loop directly but through an
2794 // integral cast (sext/zext/trunc). Add new live-ins in those cases.
2795 for (Value *U : StrideV->users()) {
2796 if (!isa<SExtInst, ZExtInst, TruncInst>(Val: U))
2797 continue;
2798 VPValue *StrideVPV = Plan.getLiveIn(V: U);
2799 if (!StrideVPV)
2800 continue;
2801 unsigned BW = U->getType()->getScalarSizeInBits();
2802 APInt C = isa<SExtInst>(Val: U) ? StrideConst->sext(width: BW)
2803 : StrideConst->zextOrTrunc(width: BW);
2804 StrideVPV->replaceUsesWithIf(New: Plan.getConstantInt(Val: C),
2805 ShouldReplace: CanUseVersionedStride);
2806 }
2807 RewriteMap[StrideV] = StrideExpr;
2808 }
2809
2810 for (VPExpandSCEVRecipe &ExpSCEV :
2811 make_isa_range<VPExpandSCEVRecipe>(Range&: *Plan.getEntry())) {
2812 const SCEV *ScevExpr = ExpSCEV.getSCEV();
2813 auto *NewSCEV =
2814 SCEVParameterRewriter::rewrite(Scev: ScevExpr, SE&: *PSE.getSE(), Map&: RewriteMap);
2815 if (NewSCEV != ScevExpr) {
2816 VPValue *NewExp = vputils::getOrCreateVPValueForSCEVExpr(Plan, Expr: NewSCEV);
2817 ExpSCEV.replaceAllUsesWith(New: NewExp);
2818 if (Plan.getTripCount() == &ExpSCEV)
2819 Plan.resetTripCount(NewTripCount: NewExp);
2820 }
2821 }
2822}
2823
2824void VPlanTransforms::dropPoisonGeneratingRecipes(VPlan &Plan) {
2825 // Collect recipes in the backward slice of `Root` that may generate a poison
2826 // value that is used after vectorization.
2827 SmallPtrSet<VPRecipeBase *, 16> Visited;
2828 auto CollectPoisonGeneratingInstrsInBackwardSlice([&](VPRecipeBase *Root) {
2829 SmallVector<VPRecipeBase *, 16> Worklist;
2830 Worklist.push_back(Elt: Root);
2831
2832 // Traverse the backward slice of Root through its use-def chain.
2833 while (!Worklist.empty()) {
2834 VPRecipeBase *CurRec = Worklist.pop_back_val();
2835
2836 if (!Visited.insert(Ptr: CurRec).second)
2837 continue;
2838
2839 // Prune search if we find another recipe generating a widen memory
2840 // instruction. Widen memory instructions involved in address computation
2841 // will lead to gather/scatter instructions, which don't need to be
2842 // handled.
2843 if (isa<VPWidenMemoryRecipe, VPInterleaveRecipe, VPScalarIVStepsRecipe,
2844 VPHeaderPHIRecipe>(Val: CurRec))
2845 continue;
2846
2847 // This recipe contributes to the address computation of a widen
2848 // load/store. If the underlying instruction has poison-generating flags,
2849 // drop them directly.
2850 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: CurRec)) {
2851 VPValue *A, *B;
2852 // Dropping disjoint from an OR may yield incorrect results, as some
2853 // analysis may have converted it to an Add implicitly (e.g. SCEV used
2854 // for dependence analysis). Instead, replace it with an equivalent Add.
2855 // This is possible as all users of the disjoint OR only access lanes
2856 // where the operands are disjoint or poison otherwise.
2857 if (match(V: RecWithFlags, P: m_BinaryOr(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))) &&
2858 RecWithFlags->isDisjoint()) {
2859 VPBuilder Builder(RecWithFlags);
2860 VPInstruction *New =
2861 Builder.createAdd(LHS: A, RHS: B, DL: RecWithFlags->getDebugLoc());
2862 New->setUnderlyingValue(RecWithFlags->getUnderlyingValue());
2863 RecWithFlags->replaceAllUsesWith(New);
2864 RecWithFlags->eraseFromParent();
2865 CurRec = New;
2866 } else
2867 RecWithFlags->dropPoisonGeneratingFlags();
2868 } else {
2869 Instruction *Instr = dyn_cast_or_null<Instruction>(
2870 Val: CurRec->getVPSingleValue()->getUnderlyingValue());
2871 (void)Instr;
2872 assert((!Instr || !Instr->hasPoisonGeneratingFlags()) &&
2873 "found instruction with poison generating flags not covered by "
2874 "VPRecipeWithIRFlags");
2875 }
2876
2877 // Add new definitions to the worklist.
2878 for (VPValue *Operand : CurRec->operands())
2879 if (VPRecipeBase *OpDef = Operand->getDefiningRecipe())
2880 Worklist.push_back(Elt: OpDef);
2881 }
2882 });
2883
2884 // We want to exclude the tail folding case, as we don't need to drop flags
2885 // for operations computing the first lane in this case: the first lane of the
2886 // header mask must always be true. For reverse memory accesses, the mask is
2887 // wrapped in a Reverse, which is just a permutation of the header mask, so
2888 // peel it off before checking. The header mask is still the abstract region
2889 // value at this point (materialization happens later).
2890 auto m_UnlessHdrMask = m_Unless( // NOLINT
2891 P: m_CombineOr(Ps: m_HeaderMask(), Ps: m_Reverse(Op0: m_HeaderMask())));
2892
2893 // Traverse all the recipes in the VPlan and collect the poison-generating
2894 // recipes in the backward slice starting at the address of a VPWidenRecipe or
2895 // VPInterleaveRecipe.
2896 auto Iter =
2897 vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntryBasicBlock());
2898 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: Iter)) {
2899 for (VPRecipeBase &Recipe : *VPBB) {
2900 if (auto *WidenRec = dyn_cast<VPWidenMemoryRecipe>(Val: &Recipe)) {
2901 VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe();
2902 if (AddrDef && WidenRec->isConsecutive() && WidenRec->getMask() &&
2903 match(V: WidenRec->getMask(), P: m_UnlessHdrMask))
2904 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2905 } else if (auto *InterleaveRec = dyn_cast<VPInterleaveRecipe>(Val: &Recipe)) {
2906 VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe();
2907 if (AddrDef && InterleaveRec->getMask() &&
2908 match(V: InterleaveRec->getMask(), P: m_UnlessHdrMask))
2909 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2910 }
2911 }
2912 }
2913}
2914
2915void VPlanTransforms::createInterleaveGroups(
2916 VPlan &Plan,
2917 const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
2918 &InterleaveGroups,
2919 const bool &EpilogueAllowed) {
2920 if (InterleaveGroups.empty())
2921 return;
2922
2923 DenseMap<Instruction *, VPWidenMemoryRecipe *> IRMemberToRecipe;
2924 for (VPBasicBlock *VPBB :
2925 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
2926 G: Plan.getVectorLoopRegion()->getEntryBasicBlock())))
2927 for (VPRecipeBase &R : make_filter_range(Range&: *VPBB, Pred: [](VPRecipeBase &R) {
2928 return isa<VPWidenMemoryRecipe>(Val: &R);
2929 })) {
2930 auto *MemR = cast<VPWidenMemoryRecipe>(Val: &R);
2931 IRMemberToRecipe[&MemR->getIngredient()] = MemR;
2932 }
2933
2934 // Interleave memory: for each Interleave Group we marked earlier as relevant
2935 // for this VPlan, replace the Recipes widening its memory instructions with a
2936 // single VPInterleaveRecipe at its insertion point.
2937 VPDominatorTree VPDT(Plan);
2938 for (const auto *IG : InterleaveGroups) {
2939 VPWidenMemoryRecipe *Start = nullptr;
2940 Instruction *StartMember = nullptr;
2941 for (auto *Member : IG->members())
2942 if (VPWidenMemoryRecipe *R = IRMemberToRecipe.lookup(Val: Member)) {
2943 StartMember = Member;
2944 Start = R;
2945 break;
2946 }
2947 if (!StartMember) // All member recipes are dead, so the group is dead.
2948 continue;
2949 VPIRMetadata InterleaveMD(*Start);
2950 SmallVector<VPValue *, 4> StoredValues;
2951 for (unsigned I = 0; I < IG->getFactor(); ++I) {
2952 Instruction *MemberI = IG->getMember(Index: I);
2953 if (!MemberI)
2954 continue;
2955 if (VPWidenMemoryRecipe *MemoryR = IRMemberToRecipe.lookup(Val: MemberI)) {
2956 if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Val: MemoryR->getAsRecipe()))
2957 StoredValues.push_back(Elt: StoreR->getStoredValue());
2958 InterleaveMD.intersect(MD: *MemoryR);
2959 } else {
2960 InterleaveMD.intersect(MD: VPIRMetadata(*MemberI));
2961 }
2962 }
2963
2964 bool NeedsMaskForGaps =
2965 (IG->requiresScalarEpilogue() && !EpilogueAllowed) ||
2966 (!StoredValues.empty() && !IG->isFull());
2967
2968 Instruction *IRInsertPos = IG->getInsertPos();
2969 auto *InsertPos = IRMemberToRecipe.lookup(Val: IRInsertPos);
2970 if (!InsertPos) {
2971 // InsertPos member is dead: find a new member that is alive.
2972 assert(isa<VPWidenLoadRecipe>(Start->getAsRecipe()) &&
2973 "Dead member in non-load group?");
2974 InsertPos = Start;
2975 for (Instruction *Member : IG->members())
2976 if (VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member))
2977 if (VPDT.properlyDominates(A: MemberR->getAsRecipe(),
2978 B: InsertPos->getAsRecipe()))
2979 InsertPos = MemberR;
2980 IRInsertPos = &InsertPos->getIngredient();
2981 }
2982 VPRecipeBase *InsertPosR = InsertPos->getAsRecipe();
2983
2984 GEPNoWrapFlags NW = GEPNoWrapFlags::none();
2985 if (auto *Gep = dyn_cast<GetElementPtrInst>(
2986 Val: getLoadStorePointerOperand(V: IRInsertPos)->stripPointerCasts()))
2987 NW = Gep->getNoWrapFlags().withoutNoUnsignedWrap();
2988
2989 // Get or create the start address for the interleave group.
2990 VPValue *Addr = Start->getAddr();
2991 VPRecipeBase *AddrDef = Addr->getDefiningRecipe();
2992 if (IG->getIndex(Instr: StartMember) != 0 ||
2993 (AddrDef && !VPDT.properlyDominates(A: AddrDef, B: InsertPosR))) {
2994 // Either member zero's recipe is dead, or we cannot re-use the address of
2995 // member zero because it does not dominate the insert position. Instead,
2996 // use the address of the insert position and create a PtrAdd adjusting it
2997 // to the address of member zero.
2998 // TODO: Hoist Addr's defining recipe (and any operands as needed) to
2999 // InsertPos or sink loads above zero members to join it.
3000 assert(IG->getIndex(IRInsertPos) != 0 &&
3001 "index of insert position shouldn't be zero");
3002 auto &DL = IRInsertPos->getDataLayout();
3003 APInt Offset(32,
3004 DL.getTypeAllocSize(Ty: getLoadStoreType(I: IRInsertPos)) *
3005 IG->getIndex(Instr: IRInsertPos),
3006 /*IsSigned=*/true);
3007 VPValue *OffsetVPV = Plan.getConstantInt(Val: -Offset);
3008 VPBuilder B(InsertPosR);
3009 Addr = B.createNoWrapPtrAdd(Ptr: InsertPos->getAddr(), Offset: OffsetVPV, GEPFlags: NW);
3010 }
3011 // If the group is reverse, adjust the index to refer to the last vector
3012 // lane instead of the first. We adjust the index from the first vector
3013 // lane, rather than directly getting the pointer for lane VF - 1, because
3014 // the pointer operand of the interleaved access is supposed to be uniform.
3015 if (IG->isReverse()) {
3016 auto *ReversePtr = new VPVectorEndPointerRecipe(
3017 Addr, &Plan.getVF(), getLoadStoreType(I: IRInsertPos),
3018 -(int64_t)IG->getFactor(), NW, InsertPosR->getDebugLoc());
3019 ReversePtr->insertBefore(InsertPos: InsertPosR);
3020 Addr = ReversePtr;
3021 }
3022 auto *VPIG = new VPInterleaveRecipe(
3023 IG, Addr, StoredValues, InsertPos->getMask(), NeedsMaskForGaps,
3024 InterleaveMD, InsertPosR->getDebugLoc());
3025 VPIG->insertBefore(InsertPos: InsertPosR);
3026
3027 unsigned J = 0;
3028 for (unsigned i = 0; i < IG->getFactor(); ++i)
3029 if (Instruction *Member = IG->getMember(Index: i)) {
3030 VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member);
3031 if (!Member->getType()->isVoidTy()) {
3032 if (MemberR) {
3033 VPValue *OriginalV = MemberR->getAsRecipe()->getVPSingleValue();
3034 OriginalV->replaceAllUsesWith(New: VPIG->getVPValue(I: J));
3035 }
3036 J++;
3037 }
3038 if (MemberR)
3039 MemberR->getAsRecipe()->eraseFromParent();
3040 }
3041 }
3042}
3043
3044/// Matches an exit condition formed by comparing a value loaded from memory
3045/// with a loop-invariant term. Binds the comparison for the condition.
3046static auto m_Uncountable(VPValue *&Cond) {
3047 return m_VPValue(
3048 V&: Cond,
3049 Op: m_c_Cmp(Op0: m_VPInstruction<Instruction::Load>(Ops: m_VPValue()), Op1: m_LiveIn()));
3050}
3051
3052namespace {
3053struct CountableConditionMatch {
3054 VPValue *&Cmp;
3055 PredicatedScalarEvolution &PSE;
3056 Loop *L;
3057
3058 CountableConditionMatch(VPValue *&Cmp, PredicatedScalarEvolution &PSE,
3059 Loop *L)
3060 : Cmp(Cmp), PSE(PSE), L(L) {}
3061
3062 template <typename ITy> bool match(ITy *V) const {
3063 VPValue *Update;
3064 if (!VPlanPatternMatch::match(
3065 V, m_VPValue(V&: Cmp, Op: m_c_ICmp(Op0: m_VPValue(V&: Update, Op: m_Add(Op0: m_VPValue(),
3066 Op1: m_VPValue())),
3067 Op1: m_LiveIn()))))
3068 return false;
3069
3070 const SCEV *S = vputils::getSCEVExprForVPValue(V: Update, PSE, L);
3071 return SCEVPatternMatch::match(
3072 S, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_scev_One(), L: m_SpecificLoop(L)));
3073 }
3074};
3075} // end anonymous namespace
3076
3077/// Matches an exit condition formed by comparing the current value of a
3078/// affine add recurrence in the given loop with a stride of 1 against a
3079/// loop-invariant term. Binds the comparison for the condition.
3080static auto m_Countable(VPValue *&Cmp, PredicatedScalarEvolution &PSE,
3081 Loop *L) {
3082 return CountableConditionMatch(Cmp, PSE, L);
3083}
3084
3085bool VPlanTransforms::splitCombinedExits(VPlan &Plan,
3086 PredicatedScalarEvolution &PSE,
3087 Loop *L) {
3088 // Check for a single combined exit in the latch block.
3089 // TODO: Generalize to other blocks besides the latch.
3090 // If we don't find a combined condition in the latch, just return true
3091 // to proceed with vectorization.
3092 auto [_, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
3093
3094 // We're looking for a conditional branch...
3095 auto *Term = dyn_cast<VPInstruction>(Val: LatchVPBB->getTerminator());
3096 if (!Term || Term->getOpcode() != VPInstruction::BranchOnCond)
3097 return true;
3098
3099 // ...where the condition is a combination of both a countable and an
3100 // uncountable comparison.
3101 VPValue *Uncountable = nullptr;
3102 VPValue *Countable = nullptr;
3103 VPValue *Cond = Term->getOperand(N: 0);
3104 if (!match(V: Cond, P: m_OneUse(SubPattern: m_CombineOr(
3105 Ps: m_c_LogicalOr(Op0: m_Uncountable(Cond&: Uncountable),
3106 Op1: m_Countable(Cmp&: Countable, PSE, L)),
3107 Ps: m_c_BinaryOr(Op0: m_Uncountable(Cond&: Uncountable),
3108 Op1: m_Countable(Cmp&: Countable, PSE, L))))))
3109 return true;
3110
3111 // If the conditions are combined with a logical or (select), then we'll
3112 // need to freeze the individual terms when splitting.
3113 bool NeedsFreeze = match(V: Cond, P: m_LogicalOr(Op0: m_VPValue(), Op1: m_VPValue()));
3114
3115 // If we do have a combined exit condition, bail out if there's more than
3116 // one exit block.
3117 // TODO: Support additional exits.
3118 ArrayRef<VPIRBasicBlock *> ExitBlocks = Plan.getExitBlocks();
3119 if (ExitBlocks.size() != 1)
3120 return false;
3121
3122 // If there are any live-outs, bail out. The exit block is an existing IR
3123 // block, and if we split the exiting block then the incoming blocks and
3124 // values won't be correct.
3125 // TODO: Support live-outs with combined exits.
3126 if (!ExitBlocks.front()->phis().empty())
3127 return false;
3128
3129 // Split the latch block just before the terminator.
3130 VPBasicBlock *NewLatch = LatchVPBB->splitAt(SplitAt: Term->getIterator());
3131
3132 // Create new terminator for uncountable condition.
3133 VPBuilder EEBuilder(LatchVPBB);
3134 if (NeedsFreeze)
3135 Uncountable = EEBuilder.createFreeze(Op: Uncountable);
3136 EEBuilder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {Uncountable});
3137
3138 // We need to connect the uncountable exit to the sole exit block. The
3139 // latch is expected to connect to the middle block instead.
3140 // In canonical form, the backedge is the last successor for the latch. So
3141 // the first successor (true path) should be the exit for both conditions.
3142 LatchVPBB->clearSuccessors();
3143 NewLatch->clearPredecessors();
3144 VPBlockUtils::connectBlocks(From: LatchVPBB, To: ExitBlocks.front());
3145 VPBlockUtils::connectBlocks(From: LatchVPBB, To: NewLatch);
3146
3147 // Set condition for latch block to countable condition.
3148 if (NeedsFreeze) {
3149 VPBuilder NewLatchBuilder(Term);
3150 Countable = NewLatchBuilder.createFreeze(Op: Countable);
3151 }
3152 Term->setOperand(I: 0, New: Countable);
3153
3154 // Remove the combining or.
3155 cast<VPInstruction>(Val: Cond)->eraseFromParent();
3156
3157 return true;
3158}
3159
3160/// Returns the VPValue representing the uncountable exit comparison used by
3161/// AnyOf if the recipes it depends on can be traced back to live-ins and
3162/// the addresses (in GEP/PtrAdd form) of any (non-masked) load used in
3163/// generating the values for the comparison. The recipes are stored in
3164/// \p Recipes.
3165static VPValue *
3166getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
3167 VPBasicBlock *LatchVPBB) {
3168 // Given a plain CFG VPlan loop with countable latch exiting block
3169 // \p LatchVPBB, we're looking to match the recipes contributing to the
3170 // uncountable exit condition comparison (here, vp<%4>) back to either
3171 // live-ins or the address nodes for the load used as part of the uncountable
3172 // exit comparison so that we can either move them within the loop, or copy
3173 // them to the preheader depending on the chosen method for dealing with
3174 // stores in uncountable exit loops.
3175 //
3176 // Currently, the address of the load is restricted to a GEP with 2 operands
3177 // and a live-in base address. This constraint may be relaxed later.
3178 //
3179 // VPlan ' for UF>=1' {
3180 // Live-in vp<%0> = VF * UF
3181 // Live-in vp<%1> = vector-trip-count
3182 // Live-in ir<20> = original trip-count
3183 //
3184 // ir-bb<entry>:
3185 // Successor(s): scalar.ph, vector.ph
3186 //
3187 // vector.ph:
3188 // Successor(s): for.body
3189 //
3190 // for.body:
3191 // EMIT vp<%2> = phi ir<0>, vp<%index.next>
3192 // EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, for.inc ]
3193 // EMIT ir<%uncountable.addr> = getelementptr inbounds nuw ir<%pred>,ir<%iv>
3194 // EMIT ir<%uncountable.val> = load ir<%uncountable.addr>
3195 // EMIT ir<%uncountable.cond> = icmp sgt ir<%uncountable.val>, ir<500>
3196 // EMIT vp<%3> = masked-cond ir<%uncountable.cond>
3197 // Successor(s): for.inc
3198 //
3199 // for.inc:
3200 // EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
3201 // EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
3202 // EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
3203 // EMIT vp<%freeze> = freeze ir<%3>
3204 // EMIT vp<%4> = any-of ir<%freeze>
3205 // EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
3206 // EMIT branch-on-two-conds vp<%4>, vp<%5>
3207 // Successor(s): middle.block, middle.block, for.body
3208 //
3209 // middle.block:
3210 // Successor(s): ir-bb<exit>, scalar.ph
3211 //
3212 // ir-bb<exit>:
3213 // No successors
3214 //
3215 // scalar.ph:
3216 // }
3217
3218 // Find the uncountable loop exit condition.
3219 VPValue *UncountableCondition = nullptr;
3220 if (!match(V: LatchVPBB->getTerminator(),
3221 P: m_BranchOnTwoConds(Op0: m_AnyOf(Op0: m_VPValue(V&: UncountableCondition)),
3222 Op1: m_VPValue())))
3223 return nullptr;
3224
3225 SmallVector<VPValue *, 4> Worklist;
3226 Worklist.push_back(Elt: UncountableCondition);
3227 while (!Worklist.empty()) {
3228 VPValue *V = Worklist.pop_back_val();
3229
3230 // Any value defined outside the loop does not need to be copied.
3231 if (V->isDefinedOutsideLoopRegions())
3232 continue;
3233
3234 // FIXME: Remove the single user restriction; it's here because we're
3235 // starting with the simplest set of loops we can, and multiple
3236 // users means needing to add PHI nodes in the transform.
3237 if (V->getNumUsers() > 1)
3238 return nullptr;
3239
3240 VPValue *Op1, *Op2;
3241 // Walk back through recipes until we find at least one load from memory.
3242 if (match(V, P: m_Cmp(Op0: m_VPValue(V&: Op1), Op1: m_VPValue(V&: Op2)))) {
3243 Worklist.push_back(Elt: Op1);
3244 Worklist.push_back(Elt: Op2);
3245 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3246 } else if (match(V, P: m_VPInstruction<Instruction::Load>(Ops: m_VPValue(V&: Op1)))) {
3247 VPRecipeBase *GepR = Op1->getDefiningRecipe();
3248 // Only matching base + single offset term for now.
3249 if (GepR->getNumOperands() != 2)
3250 return nullptr;
3251 // Matching a GEP with a loop-invariant base ptr.
3252 if (!match(V: GepR, P: m_VPInstruction<Instruction::GetElementPtr>(
3253 Ops: m_LiveIn(), Ops: m_VPValue())))
3254 return nullptr;
3255 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3256 Recipes.push_back(Elt: cast<VPInstruction>(Val: GepR));
3257 } else if (match(V, P: m_Freeze(Op0: m_VPValue(V&: Op1)))) {
3258 Worklist.push_back(Elt: Op1);
3259 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3260 } else if (match(V, P: m_VPInstruction<VPInstruction::MaskedCond>(
3261 Ops: m_VPValue(V&: Op1)))) {
3262 Worklist.push_back(Elt: Op1);
3263 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3264 } else
3265 return nullptr;
3266 }
3267
3268 // If we couldn't match anything, don't return the condition. It may be
3269 // defined outside the loop.
3270 if (Recipes.empty() ||
3271 none_of(Range&: Recipes, P: match_fn(P: m_VPInstruction<Instruction::GetElementPtr>())))
3272 return nullptr;
3273
3274 return UncountableCondition;
3275}
3276
3277struct EarlyExitInfo {
3278 VPBasicBlock *EarlyExitingVPBB;
3279 VPIRBasicBlock *EarlyExitVPBB;
3280 VPValue *CondToExit;
3281};
3282
3283/// Update \p Plan to mask memory operations in the loop based on whether the
3284/// early exit is taken or not.
3285///
3286/// We're currently expecting to find a loop with properties similar to the
3287/// following:
3288///
3289/// for.body:
3290/// ir<%indvars.iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<%0>
3291/// EMIT ir<%arrayidx> = getelementptr inbounds nuw ir<@c>, ir<%indvars.iv>
3292/// EMIT-SCALAR ir<%0> = load ir<%arrayidx>
3293/// EMIT ir<%cmp1> = icmp sgt ir<%0>, ir<5>
3294/// EMIT vp<%1> = masked-cond ir<%cmp1>
3295/// Successor(s): if.end
3296///
3297/// if.end:
3298/// EMIT ir<%arrayidx3> = getelementptr inbounds nuw ir<@src>, ir<%indvars.iv>
3299/// EMIT-SCALAR ir<%2> = load ir<%arrayidx3>
3300/// EMIT ir<%add> = add nsw ir<%2>, ir<42>
3301/// EMIT ir<%arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<%indvars.iv>
3302/// EMIT store ir<%add>, ir<%arrayidx5>
3303/// EMIT ir<%indvars.iv.next> = add nuw nsw ir<%indvars.iv>, ir<1>
3304/// EMIT vp<%freeze> = freeze ir<%1>
3305/// EMIT vp<%3> = any-of ir<%freeze>
3306/// EMIT ir<%exitcond.not> = icmp eq ir<%indvars.iv.next>, ir<10000>
3307/// EMIT branch-on-two-conds vp<%3>, ir<%exitcond.not>
3308/// Successor(s): middle.block, middle.block, for.body
3309///
3310/// We currently expect LoopVectorizationLegality to ensure that:
3311/// * There must also be a counted exit. We will need to support speculative
3312/// or first-faulting loads before we can remove this restriction.
3313/// * Any stores within the loop must not alias with the load used for the
3314/// uncountable exit. We can relax this a bit with runtime aliasing checks.
3315/// * Other memory operations in the loop can take place before or after the
3316/// uncountable exit, but must also be unconditional. We need to support
3317/// combining the conditions in VPlanPredicator.
3318/// * The loop must have a single unconditional load contributing to the
3319/// uncountable exit comparison, and the other term must be loop-invariant.
3320/// Improving upon this requires work in getRecipesForUncountableExit to
3321/// handle more complex recipe graphs.
3322static bool handleUncountableExitsWithSideEffects(
3323 VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
3324 VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB,
3325 OptimizationRemarkEmitter *ORE, Loop *TheLoop,
3326 PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC) {
3327
3328 // Disconnect early exiting blocks from successors, remove branches. We
3329 // currently don't support multiple uses for recipes involved in creating
3330 // the uncountable exit condition.
3331 for (auto &Exit : Exits) {
3332 if (Exit.EarlyExitingVPBB == LatchVPBB)
3333 continue;
3334
3335 for (VPRecipeBase &R : Exit.EarlyExitVPBB->phis())
3336 cast<VPIRPhi>(Val: &R)->removeIncomingValueFor(IncomingBlock: Exit.EarlyExitingVPBB);
3337 Exit.EarlyExitingVPBB->getTerminator()->eraseFromParent();
3338 VPBlockUtils::disconnectBlocks(From: Exit.EarlyExitingVPBB, To: Exit.EarlyExitVPBB);
3339 }
3340
3341 VPDominatorTree VPDT(Plan);
3342
3343 // We can abandon a VPlan entirely if we return false here, so we shouldn't
3344 // crash if some earlier assumptions on scalar IR don't hold for the vplan
3345 // version of the loop.
3346 SmallVector<VPInstruction *, 8> ConditionRecipes;
3347
3348 VPValue *Cond = getRecipesForUncountableExit(Recipes&: ConditionRecipes, LatchVPBB);
3349 if (!Cond) {
3350 reportVectorizationFailure(DebugMsg: "Unable to determine early exit condition for "
3351 "loop with side effects",
3352 ORETag: "EarlyExitSideEffectsCond", ORE, TheLoop);
3353 return false;
3354 }
3355
3356 // Find load contributing to condition.
3357 // At the moment LoopVectorizationLegality only supports a single
3358 // early-exit expression with a compare and a single load that must
3359 // be unconditional.
3360 // TODO: Support more than one load.
3361 auto *Load =
3362 find_singleton<VPInstruction>(Range&: ConditionRecipes, P: [](auto *I, bool _) {
3363 return match(I, m_VPInstruction<Instruction::Load>(Ops: m_VPValue()))
3364 ? I
3365 : nullptr;
3366 });
3367 assert(Load && "Couldn't find exactly one load");
3368 // TODO: Support conditional loads for uncountable exits.
3369 assert(VPDT.dominates(Load->getParent(), LatchVPBB) &&
3370 "Uncountable exit condition load is conditional.");
3371 VPInstruction *Ptr = cast<VPInstruction>(Val: Load->getOperand(N: 0));
3372
3373 // Ensure that we are guaranteed to be able to dereference the memory used
3374 // for determining the uncountable exit for the maximum possible number of
3375 // scalar iterations of the loop.
3376 //
3377 // TODO: Support first-faulting loads in cases where we don't know whether
3378 // all possible addresses are dereferenceable.
3379 {
3380 SmallVector<const SCEVPredicate *, 4> Predicates;
3381 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: TheLoop);
3382 const DataLayout &DL = Plan.getDataLayout();
3383 APInt EltSize(DL.getIndexTypeSizeInBits(Ty: Ptr->getScalarType()),
3384 DL.getTypeStoreSize(Ty: Load->getScalarType()).getFixedValue());
3385 if (!isDereferenceableAndAlignedInLoop(
3386 PtrSCEV, Alignment: cast<LoadInst>(Val: Load->getUnderlyingInstr())->getAlign(),
3387 EltSizeSCEV: PSE.getSE()->getConstant(Val: EltSize), L: TheLoop, SE&: *PSE.getSE(), DT, AC,
3388 Predicates: &Predicates)) {
3389 reportVectorizationFailure(DebugMsg: "Early exit loop with side effects contains "
3390 "load used by the exit condition that may "
3391 "fault",
3392 ORETag: "EarlyExitSideEffectsFaultingLoad", ORE,
3393 TheLoop);
3394 return false;
3395 }
3396 }
3397
3398 // Check for a single GEP for the condition load to see if we can link it to
3399 // a widen IV recipe with a step of 1; we're only interested in contiguous
3400 // accesses for the condition load right now.
3401 auto *IV = cast<VPWidenInductionRecipe>(Val: &HeaderVPBB->front());
3402 if (!match(V: IV->getStartValue(), P: m_SpecificInt(V: 0)) ||
3403 !match(V: IV->getStepValue(), P: m_SpecificInt(V: 1))) {
3404 reportVectorizationFailure(DebugMsg: "Early exit loop with side effects contains "
3405 "load used by the exit condition with an "
3406 "unsupported memory access pattern",
3407 ORETag: "EarlyExitSideEffectsBadLoadAccessPattern", ORE,
3408 TheLoop);
3409 return false;
3410 }
3411
3412 if (!match(V: Ptr, P: m_VPInstruction<Instruction::GetElementPtr>(
3413 Ops: m_LiveIn(), Ops: m_Specific(VPV: IV)))) {
3414 reportVectorizationFailure(DebugMsg: "Early exit loop with side effects contains "
3415 "load used by the exit condition with an "
3416 "unsupported memory access pattern",
3417 ORETag: "EarlyExitSideEffectsBadLoadAccessPattern", ORE,
3418 TheLoop);
3419 return false;
3420 }
3421
3422 // We want to guarantee that the uncountable exit condition (and the mask
3423 // we will generate from it) are available for all operations in the loop
3424 // that need to be masked. If the condition recipes are not already the first
3425 // recipes in the header after the last phi, move them there.
3426 auto InsertIt = HeaderVPBB->getFirstNonPhi();
3427 while (InsertIt != HeaderVPBB->end() &&
3428 is_contained(Range&: ConditionRecipes, Element: &*InsertIt)) {
3429 erase(C&: ConditionRecipes, V: &*InsertIt);
3430 InsertIt++;
3431 }
3432 for (auto *Recipe : reverse(C&: ConditionRecipes))
3433 Recipe->moveBefore(BB&: *HeaderVPBB, I: InsertIt);
3434
3435 // Create a mask to represent all lanes that fully execute in the vector loop,
3436 // stopping short of any early exit.
3437 VPBuilder MaskBuilder(HeaderVPBB, InsertIt);
3438 VPValue *FirstActive = MaskBuilder.createFirstActiveLane(Masks: Cond);
3439 Type *IVScalarTy = IV->getScalarType();
3440 VPValue *Zero = Plan.getZero(Ty: IVScalarTy);
3441 FirstActive =
3442 MaskBuilder.createScalarZExtOrTrunc(Op: FirstActive, ResultTy: IVScalarTy, DL: DebugLoc());
3443 VPValue *Mask = MaskBuilder.createNaryOp(Opcode: VPInstruction::ActiveLaneMask,
3444 Operands: {Zero, FirstActive}, DL: DebugLoc(),
3445 Name: "uncountable.exit.mask");
3446
3447 // Convert all other memory operations to use the mask.
3448 for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(Header: HeaderVPBB))
3449 for (VPRecipeBase &R : *VPBB)
3450 if (R.mayReadOrWriteMemory() && &R != Load) {
3451 // TODO: Handle conditional memory operations in the loop.
3452 if (!VPDT.dominates(A: R.getParent(), B: LatchVPBB)) {
3453 reportVectorizationFailure(
3454 DebugMsg: "Early exit loop with side effects contains unsupported "
3455 "conditional memory operations",
3456 ORETag: "EarlyExitSideEffectsUnsupportedConditionalMemOps", ORE, TheLoop);
3457 return false;
3458 }
3459 cast<VPInstruction>(Val: &R)->addMask(Mask);
3460 }
3461
3462 // Update middle block branch to compare (IV + however many lanes were active)
3463 // against the full trip count, since we may be exiting the vector loop early.
3464 // If we didn't take an early exit, we should get the equivalent of VF from
3465 // the FirstActiveLane.
3466 assert(match(MiddleVPBB->getTerminator(), m_BranchOnCond()) &&
3467 "Expected BranchOnCond terminator for MiddleVPBB");
3468 VPBuilder MiddleBuilder(MiddleVPBB->getTerminator());
3469 VPValue *ScalarIV = MiddleBuilder.createNaryOp(Opcode: VPInstruction::ExtractLane,
3470 Operands: {Zero, IV}, DL: DebugLoc());
3471 VPValue *ExitIV = MiddleBuilder.createAdd(LHS: ScalarIV, RHS: FirstActive);
3472 VPValue *FullTC =
3473 MiddleBuilder.createICmp(Pred: CmpInst::ICMP_EQ, A: ExitIV, B: Plan.getTripCount());
3474 MiddleVPBB->getTerminator()->setOperand(I: 0, New: FullTC);
3475
3476 // Update resume phi in scalar.ph.
3477 VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
3478 auto Phis = ScalarPH->phis();
3479 // TODO: Handle more than one Phi; re-derive from IV.
3480 // TODO: Handle reductions.
3481 if (range_size(Range&: Phis) != 1) {
3482 reportVectorizationFailure(
3483 DebugMsg: "Early exit loop with side effects contains "
3484 "unsupported reductions, inductions or recurrences",
3485 ORETag: "EarlyExitSideEffectsReductions", ORE, TheLoop);
3486 return false;
3487 }
3488 VPPhi *ContinueIV = cast<VPPhi>(Val: Phis.begin());
3489 // Make sure we're referring to the same IV.
3490 assert(
3491 match(ContinueIV->getOperand(0),
3492 m_VPInstruction<VPInstruction::ExitingIVValue>(m_Specific(IV))) &&
3493 "Continuing from different IV");
3494 ContinueIV->setOperand(I: 0, New: ExitIV);
3495 return true;
3496}
3497
3498bool VPlanTransforms::handleUncountableEarlyExits(
3499 VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop,
3500 PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC,
3501 UncountableExitStyle Style) {
3502#ifndef NDEBUG
3503 VPDominatorTree VPDT(Plan);
3504#endif
3505
3506 auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
3507 auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
3508
3509 // Dereferenceability is checked separately for uncountable exit loops with
3510 // stores, as only the loads contributing to the exit condition need to
3511 // be checked.
3512 if (Style == UncountableExitStyle::ReadOnly &&
3513 !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC)) {
3514 reportVectorizationFailure(
3515 DebugMsg: "Auto-vectorization of early exit loops with potentially "
3516 "faulting loads is not supported",
3517 ORETag: "EarlyExitFaultingLoads", ORE, TheLoop);
3518 return false;
3519 }
3520
3521 VPBuilder LatchBuilder(LatchVPBB->getTerminator());
3522 SmallVector<EarlyExitInfo> Exits;
3523 for (auto [EarlyExitingVPBB, ExitBlock] :
3524 vputils::getEarlyExits(Plan, MiddleVPBB)) {
3525 // Collect condition for this early exit.
3526 VPBlockBase *TrueSucc = EarlyExitingVPBB->getSuccessors()[0];
3527 VPValue *CondOfEarlyExitingVPBB;
3528 [[maybe_unused]] bool Matched =
3529 match(V: EarlyExitingVPBB->getTerminator(),
3530 P: m_BranchOnCond(Op0: m_VPValue(V&: CondOfEarlyExitingVPBB)));
3531 assert(Matched && "Terminator must be BranchOnCond");
3532
3533 // Insert the MaskedCond in the EarlyExitingVPBB so the predicator adds
3534 // the correct block mask.
3535 VPBuilder EarlyExitingBuilder(EarlyExitingVPBB->getTerminator());
3536 auto *CondToEarlyExit = EarlyExitingBuilder.createNaryOp(
3537 Opcode: VPInstruction::MaskedCond,
3538 Operands: TrueSucc == ExitBlock
3539 ? CondOfEarlyExitingVPBB
3540 : EarlyExitingBuilder.createNot(Operand: CondOfEarlyExitingVPBB));
3541 assert((isa<VPIRValue>(CondOfEarlyExitingVPBB) ||
3542 !VPDT.properlyDominates(EarlyExitingVPBB, LatchVPBB) ||
3543 VPDT.properlyDominates(
3544 CondOfEarlyExitingVPBB->getDefiningRecipe()->getParent(),
3545 LatchVPBB)) &&
3546 "exit condition must dominate the latch");
3547 Exits.push_back(Elt: {
3548 .EarlyExitingVPBB: EarlyExitingVPBB,
3549 .EarlyExitVPBB: ExitBlock,
3550 .CondToExit: CondToEarlyExit,
3551 });
3552 }
3553
3554 assert(!Exits.empty() && "must have at least one early exit");
3555 // Sort exits by RPO order to get correct program order. RPO gives a
3556 // topological ordering of the CFG, ensuring upstream exits are checked
3557 // before downstream exits in the dispatch chain.
3558 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
3559 HeaderVPBB);
3560 DenseMap<VPBlockBase *, unsigned> RPOIdx;
3561 for (const auto &[Num, VPB] : enumerate(First&: RPOT))
3562 RPOIdx[VPB] = Num;
3563 llvm::sort(C&: Exits, Comp: [&RPOIdx](const EarlyExitInfo &A, const EarlyExitInfo &B) {
3564 return RPOIdx[A.EarlyExitingVPBB] < RPOIdx[B.EarlyExitingVPBB];
3565 });
3566#ifndef NDEBUG
3567 // After RPO sorting, verify that for any pair where one exit dominates
3568 // another, the dominating exit comes first. This is guaranteed by RPO
3569 // (topological order) and is required for the dispatch chain correctness.
3570 for (unsigned I = 0; I + 1 < Exits.size(); ++I)
3571 for (unsigned J = I + 1; J < Exits.size(); ++J)
3572 assert(!VPDT.properlyDominates(Exits[J].EarlyExitingVPBB,
3573 Exits[I].EarlyExitingVPBB) &&
3574 "RPO sort must place dominating exits before dominated ones");
3575#endif
3576
3577 // Build the AnyOf condition for the latch terminator using logical OR
3578 // to avoid poison propagation from later exit conditions when an earlier
3579 // exit is taken.
3580 VPValue *Combined = Exits[0].CondToExit;
3581 for (const EarlyExitInfo &Info : drop_begin(RangeOrContainer&: Exits))
3582 Combined = LatchBuilder.createLogicalOr(LHS: Combined, RHS: Info.CondToExit);
3583 Combined = LatchBuilder.createFreeze(Op: Combined);
3584
3585 // Even though the logical or prevents posion propagation, we need to freeze
3586 // Combined to prevent poisoning the entire AnyOf result:
3587 //
3588 // Exits[0].CondToExit = [0,1,0,0]
3589 // Exits[1].CondToExit = [0,0,p,p]
3590 // Combined = [0,1,p,p]
3591 // freeze(Combined) = [0,1,?,?]
3592 // AnyOf = 1
3593 VPValue *IsAnyExitTaken =
3594 LatchBuilder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: Combined);
3595
3596 // Create a comparison for the latch exit condition and replace the
3597 // BranchOnCond with a BranchOnTwoConds. The original BranchOnCond's condition
3598 // is used as the latch-exit condition; canonical IV recipes have not been
3599 // introduced yet, so there is no BranchOnCount to derive the condition from.
3600 auto *LatchExitingBranch = cast<VPInstruction>(Val: LatchVPBB->getTerminator());
3601 assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCond &&
3602 "Unexpected terminator");
3603 VPValue *IsLatchExitTaken = LatchExitingBranch->getOperand(N: 0);
3604 DebugLoc LatchDL = LatchExitingBranch->getDebugLoc();
3605 LatchExitingBranch->eraseFromParent();
3606 LatchBuilder.setInsertPoint(LatchVPBB);
3607 LatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnTwoConds,
3608 Operands: {IsAnyExitTaken, IsLatchExitTaken}, DL: LatchDL);
3609 LatchVPBB->clearSuccessors();
3610
3611 if (Style == UncountableExitStyle::MaskedHandleExitInScalarLoop) {
3612 // If handling the exiting lane in the scalar loop, combine the exit
3613 // conditions into a single BranchOnCond.
3614 LatchVPBB->setSuccessors({MiddleVPBB, MiddleVPBB, HeaderVPBB});
3615 MiddleVPBB->clearPredecessors();
3616 MiddleVPBB->setPredecessors({LatchVPBB, LatchVPBB});
3617 return handleUncountableExitsWithSideEffects(Plan, Exits, HeaderVPBB,
3618 LatchVPBB, MiddleVPBB, ORE,
3619 TheLoop, PSE, DT, AC);
3620 }
3621
3622 // Create the vector.early.exit blocks.
3623 SmallVector<VPBasicBlock *> VectorEarlyExitVPBBs(Exits.size());
3624 for (unsigned Idx = 0; Idx != Exits.size(); ++Idx) {
3625 Twine BlockSuffix = Exits.size() == 1 ? "" : Twine(".") + Twine(Idx);
3626 VPBasicBlock *VectorEarlyExitVPBB =
3627 Plan.createVPBasicBlock(Name: "vector.early.exit" + BlockSuffix);
3628 VectorEarlyExitVPBBs[Idx] = VectorEarlyExitVPBB;
3629 }
3630
3631 // Create the dispatch block (or reuse the single exit block if only one
3632 // exit). The dispatch block computes the first active lane of the combined
3633 // condition and, for multiple exits, chains through conditions to determine
3634 // which exit to take.
3635 VPBasicBlock *DispatchVPBB =
3636 Exits.size() == 1 ? VectorEarlyExitVPBBs[0]
3637 : Plan.createVPBasicBlock(Name: "vector.early.exit.check");
3638 DispatchVPBB->setPredecessors({LatchVPBB});
3639 LatchVPBB->setSuccessors({DispatchVPBB, MiddleVPBB, HeaderVPBB});
3640 VPBuilder DispatchBuilder(DispatchVPBB, DispatchVPBB->begin());
3641 VPValue *FirstActiveLane = DispatchBuilder.createFirstActiveLane(
3642 Masks: {Combined}, DL: DebugLoc::getUnknown(), Name: "first.active.lane");
3643
3644 // For each early exit, disconnect the original exiting block
3645 // (early.exiting.I) from the exit block (ir-bb<exit.I>) and route through a
3646 // new vector.early.exit block. Update ir-bb<exit.I>'s phis to extract their
3647 // values at the first active lane:
3648 //
3649 // Input:
3650 // early.exiting.I:
3651 // ...
3652 // EMIT branch-on-cond vp<%cond.I>
3653 // Successor(s): in.loop.succ, ir-bb<exit.I>
3654 //
3655 // ir-bb<exit.I>:
3656 // IR %phi = phi [ vp<%incoming.I>, early.exiting.I ], ...
3657 //
3658 // Output:
3659 // early.exiting.I:
3660 // ...
3661 // Successor(s): in.loop.succ
3662 //
3663 // vector.early.exit.I:
3664 // EMIT vp<%exit.val> = extract-lane vp<%first.lane>, vp<%incoming.I>
3665 // Successor(s): ir-bb<exit.I>
3666 //
3667 // ir-bb<exit.I>:
3668 // IR %phi = phi ... (extra operand: vp<%exit.val> from
3669 // vector.early.exit.I)
3670 //
3671 for (auto [Exit, VectorEarlyExitVPBB] :
3672 zip_equal(t&: Exits, u&: VectorEarlyExitVPBBs)) {
3673 auto &[EarlyExitingVPBB, EarlyExitVPBB, _] = Exit;
3674 // Adjust the phi nodes in EarlyExitVPBB.
3675 // 1. remove incoming values from EarlyExitingVPBB,
3676 // 2. extract the incoming value at FirstActiveLane
3677 // 3. add back the extracts as last operands for the phis
3678 // Then adjust the CFG, removing the edge between EarlyExitingVPBB and
3679 // EarlyExitVPBB and adding a new edge between VectorEarlyExitVPBB and
3680 // EarlyExitVPBB. The extracts at FirstActiveLane are now the incoming
3681 // values from VectorEarlyExitVPBB.
3682 for (VPRecipeBase &R : EarlyExitVPBB->phis()) {
3683 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
3684 VPValue *IncomingVal =
3685 ExitIRI->getIncomingValueForBlock(VPBB: EarlyExitingVPBB);
3686 VPValue *NewIncoming = IncomingVal;
3687 if (!isa<VPIRValue>(Val: IncomingVal)) {
3688 VPBuilder EarlyExitBuilder(VectorEarlyExitVPBB);
3689 NewIncoming = EarlyExitBuilder.createNaryOp(
3690 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, IncomingVal},
3691 DL: DebugLoc::getUnknown(), Name: "early.exit.value");
3692 }
3693 ExitIRI->removeIncomingValueFor(IncomingBlock: EarlyExitingVPBB);
3694 ExitIRI->addIncoming(IncomingV: NewIncoming);
3695 }
3696
3697 EarlyExitingVPBB->getTerminator()->eraseFromParent();
3698 VPBlockUtils::disconnectBlocks(From: EarlyExitingVPBB, To: EarlyExitVPBB);
3699 VPBlockUtils::connectBlocks(From: VectorEarlyExitVPBB, To: EarlyExitVPBB);
3700 }
3701
3702 // Chain through exits: for each exit, check if its condition is true at
3703 // the first active lane. If so, take that exit; otherwise, try the next.
3704 // The last exit needs no check since it must be taken if all others fail.
3705 //
3706 // For 3 exits (cond.0, cond.1, cond.2), this creates:
3707 //
3708 // latch:
3709 // ...
3710 // EMIT vp<%combined> = logical-or vp<%cond.0>, vp<%cond.1>, vp<%cond.2>
3711 // EMIT vp<%combined.freeze> = freeze vp<%combined>
3712 // ...
3713 //
3714 // vector.early.exit.check:
3715 // EMIT vp<%first.lane> = first-active-lane vp<%combined.freeze>
3716 // EMIT vp<%at.cond.0> = extract-lane vp<%first.lane>, vp<%cond.0>
3717 // EMIT branch-on-cond vp<%at.cond.0>
3718 // Successor(s): vector.early.exit.0, vector.early.exit.check.0
3719 //
3720 // vector.early.exit.check.0:
3721 // EMIT vp<%at.cond.1> = extract-lane vp<%first.lane>, vp<%cond.1>
3722 // EMIT branch-on-cond vp<%at.cond.1>
3723 // Successor(s): vector.early.exit.1, vector.early.exit.2
3724 VPBasicBlock *CurrentBB = DispatchVPBB;
3725 for (auto [I, Exit] : enumerate(First: ArrayRef(Exits).drop_back())) {
3726 VPValue *LaneVal = DispatchBuilder.createNaryOp(
3727 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, Exit.CondToExit},
3728 DL: DebugLoc::getUnknown(), Name: "exit.cond.at.lane");
3729
3730 // For the last dispatch, branch directly to the last exit on false;
3731 // otherwise, create a new check block.
3732 bool IsLastDispatch = (I + 2 == Exits.size());
3733 VPBasicBlock *FalseBB =
3734 IsLastDispatch ? VectorEarlyExitVPBBs.back()
3735 : Plan.createVPBasicBlock(
3736 Name: Twine("vector.early.exit.check.") + Twine(I));
3737
3738 DispatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {LaneVal});
3739 CurrentBB->setSuccessors({VectorEarlyExitVPBBs[I], FalseBB});
3740 VectorEarlyExitVPBBs[I]->setPredecessors({CurrentBB});
3741 FalseBB->setPredecessors({CurrentBB});
3742
3743 CurrentBB = FalseBB;
3744 DispatchBuilder.setInsertPoint(CurrentBB);
3745 }
3746
3747 return true;
3748}
3749
3750/// This function tries convert extended in-loop reductions to
3751/// VPExpressionRecipe and clamp the \p Range if it is beneficial and
3752/// valid. The created recipe must be decomposed to its constituent
3753/// recipes before execution.
3754static VPExpressionRecipe *
3755tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
3756 VFRange &Range) {
3757 Type *RedTy = Red->getScalarType();
3758 VPValue *VecOp = Red->getVecOp();
3759
3760 // We don't handle partial reductions here.
3761 if (Red->isPartialReduction())
3762 return nullptr;
3763
3764 // Clamp the range if using extended-reduction is profitable.
3765 auto IsExtendedRedValidAndClampRange =
3766 [&](unsigned Opcode, Instruction::CastOps ExtOpc, Type *SrcTy) -> bool {
3767 return LoopVectorizationPlanner::getDecisionAndClampRange(
3768 Predicate: [&](ElementCount VF) {
3769 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
3770 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
3771
3772 InstructionCost ExtRedCost = InstructionCost::getInvalid();
3773 InstructionCost ExtCost =
3774 cast<VPWidenCastRecipe>(Val: VecOp)->computeCost(VF, Ctx);
3775 InstructionCost RedCost = Red->computeCost(VF, Ctx);
3776
3777 assert(!RedTy->isFloatingPointTy() &&
3778 "getExtendedReductionCost only supports integer types");
3779 ExtRedCost = Ctx.TTI.getExtendedReductionCost(
3780 Opcode, IsUnsigned: ExtOpc == Instruction::CastOps::ZExt, ResTy: RedTy, Ty: SrcVecTy,
3781 FMF: Red->getFastMathFlagsOrNone(), CostKind);
3782 return ExtRedCost.isValid() && ExtRedCost < ExtCost + RedCost;
3783 },
3784 Range);
3785 };
3786
3787 VPValue *A;
3788 // Match reduce(ext)).
3789 if (match(V: VecOp, P: m_Isa<VPWidenCastRecipe>(P: m_ZExtOrSExt(Op0: m_VPValue(V&: A)))) &&
3790 IsExtendedRedValidAndClampRange(
3791 RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind()),
3792 cast<VPWidenCastRecipe>(Val: VecOp)->getOpcode(), A->getScalarType()))
3793 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
3794
3795 return nullptr;
3796}
3797
3798/// This function tries convert extended in-loop reductions to
3799/// VPExpressionRecipe and clamp the \p Range if it is beneficial
3800/// and valid. The created VPExpressionRecipe must be decomposed to its
3801/// constituent recipes before execution. Patterns of the
3802/// VPExpressionRecipe:
3803/// reduce.add(mul(...)),
3804/// reduce.add(mul(ext(A), ext(B))),
3805/// reduce.add(ext(mul(ext(A), ext(B)))).
3806/// reduce.fadd(fmul(ext(A), ext(B)))
3807static VPExpressionRecipe *
3808tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
3809 VPCostContext &Ctx, VFRange &Range) {
3810 unsigned Opcode = RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind());
3811 if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
3812 Opcode != Instruction::FAdd)
3813 return nullptr;
3814
3815 // We don't handle partial reductions here.
3816 if (Red->isPartialReduction())
3817 return nullptr;
3818
3819 Type *RedTy = Red->getScalarType();
3820
3821 // Clamp the range if using multiply-accumulate-reduction is profitable.
3822 auto IsMulAccValidAndClampRange =
3823 [&](VPWidenRecipe *Mul, VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
3824 VPWidenCastRecipe *OuterExt) -> bool {
3825 return LoopVectorizationPlanner::getDecisionAndClampRange(
3826 Predicate: [&](ElementCount VF) {
3827 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
3828 Type *SrcTy = Ext0 ? Ext0->getOperand(N: 0)->getScalarType() : RedTy;
3829 InstructionCost MulAccCost;
3830
3831 // getMulAccReductionCost for in-loop reductions does not support
3832 // mixed or floating-point extends.
3833 if (Ext0 && Ext1 &&
3834 (Ext0->getOpcode() != Ext1->getOpcode() ||
3835 Ext0->getOpcode() == Instruction::CastOps::FPExt))
3836 return false;
3837
3838 bool IsZExt =
3839 !Ext0 || Ext0->getOpcode() == Instruction::CastOps::ZExt;
3840 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
3841 MulAccCost = Ctx.TTI.getMulAccReductionCost(IsUnsigned: IsZExt, RedOpcode: Opcode, ResTy: RedTy,
3842 Ty: SrcVecTy, CostKind);
3843
3844 InstructionCost MulCost = Mul->computeCost(VF, Ctx);
3845 InstructionCost RedCost = Red->computeCost(VF, Ctx);
3846 InstructionCost ExtCost = 0;
3847 if (Ext0)
3848 ExtCost += Ext0->computeCost(VF, Ctx);
3849 if (Ext1)
3850 ExtCost += Ext1->computeCost(VF, Ctx);
3851 if (OuterExt)
3852 ExtCost += OuterExt->computeCost(VF, Ctx);
3853
3854 return MulAccCost.isValid() &&
3855 MulAccCost < ExtCost + MulCost + RedCost;
3856 },
3857 Range);
3858 };
3859
3860 VPValue *VecOp = Red->getVecOp();
3861 VPRecipeBase *Sub = nullptr;
3862 VPValue *A, *B;
3863 VPValue *Tmp = nullptr;
3864
3865 if (RedTy->isFloatingPointTy())
3866 return nullptr;
3867
3868 // Sub reductions could have a sub between the add reduction and vec op.
3869 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Tmp)))) {
3870 Sub = VecOp->getDefiningRecipe();
3871 VecOp = Tmp;
3872 }
3873
3874 // If ValB is a constant and can be safely extended, truncate it to the same
3875 // type as ExtA's operand, then extend it to the same type as ExtA. This
3876 // creates two uniform extends that can more easily be matched by the rest of
3877 // the bundling code. The ExtB reference, ValB and operand 1 of Mul are all
3878 // replaced with the new extend of the constant.
3879 auto ExtendAndReplaceConstantOp = [](VPWidenCastRecipe *ExtA,
3880 VPWidenCastRecipe *&ExtB, VPValue *&ValB,
3881 VPWidenRecipe *Mul) {
3882 if (!ExtA || ExtB || !isa<VPIRValue>(Val: ValB))
3883 return;
3884 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
3885 Instruction::CastOps ExtOpc = ExtA->getOpcode();
3886 const APInt *Const;
3887 if (!match(V: ValB, P: m_APInt(C&: Const)) ||
3888 !llvm::canConstantBeExtended(
3889 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
3890 return;
3891 // The truncate ensures that the type of each extended operand is the
3892 // same, and it's been proven that the constant can be extended from
3893 // NarrowTy safely. Necessary since ExtA's extended operand would be
3894 // e.g. an i8, while the const will likely be an i32. This will be
3895 // elided by later optimisations.
3896 VPBuilder Builder(Mul);
3897 auto *Trunc =
3898 Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc, Op: ValB, ResultTy: NarrowTy);
3899 Type *WideTy = ExtA->getScalarType();
3900 ValB = ExtB = Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy);
3901 Mul->setOperand(I: 1, New: ExtB);
3902 };
3903
3904 // Try to match reduce.add(mul(...)).
3905 if (match(V: VecOp, P: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B)))) {
3906 auto *RecipeA = dyn_cast<VPWidenCastRecipe>(Val: A);
3907 auto *RecipeB = dyn_cast<VPWidenCastRecipe>(Val: B);
3908 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
3909
3910 // Convert reduce.add(mul(ext, const)) to reduce.add(mul(ext, ext(const)))
3911 ExtendAndReplaceConstantOp(RecipeA, RecipeB, B, Mul);
3912
3913 // Match reduce.add/sub(mul(ext, ext)).
3914 if (RecipeA && RecipeB && match(V: RecipeA, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
3915 match(V: RecipeB, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
3916 IsMulAccValidAndClampRange(Mul, RecipeA, RecipeB, nullptr)) {
3917 if (Sub)
3918 return new VPExpressionRecipe(RecipeA, RecipeB, Mul,
3919 cast<VPWidenRecipe>(Val: Sub), Red);
3920 return new VPExpressionRecipe(RecipeA, RecipeB, Mul, Red);
3921 }
3922 // TODO: Add an expression type for this variant with a negated mul
3923 if (!Sub && IsMulAccValidAndClampRange(Mul, nullptr, nullptr, nullptr))
3924 return new VPExpressionRecipe(Mul, Red);
3925 }
3926 // TODO: Add an expression type for negated versions of other expression
3927 // variants.
3928 if (Sub)
3929 return nullptr;
3930
3931 // Match reduce.add(ext(mul(A, B))).
3932 if (match(V: VecOp, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))))) {
3933 auto *Ext = cast<VPWidenCastRecipe>(Val: VecOp);
3934 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
3935 auto *Ext0 = dyn_cast<VPWidenCastRecipe>(Val: A);
3936 auto *Ext1 = dyn_cast<VPWidenCastRecipe>(Val: B);
3937
3938 // reduce.add(ext(mul(ext, const)))
3939 // -> reduce.add(ext(mul(ext, ext(const))))
3940 ExtendAndReplaceConstantOp(Ext0, Ext1, B, Mul);
3941
3942 // reduce.add(ext(mul(ext(A), ext(B))))
3943 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
3944 // The inner extends must either have the same opcode as the outer extend or
3945 // be the same, in which case the multiply can never result in a negative
3946 // value and the outer extend can be folded away by doing wider
3947 // extends for the operands of the mul.
3948 if (Ext0 && Ext1 &&
3949 (Ext->getOpcode() == Ext0->getOpcode() || Ext0 == Ext1) &&
3950 Ext0->getOpcode() == Ext1->getOpcode() &&
3951 IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext) && Mul->hasOneUse()) {
3952 auto *NewExt0 = new VPWidenCastRecipe(
3953 Ext0->getOpcode(), Ext0->getOperand(N: 0), Ext->getScalarType(), nullptr,
3954 *Ext0, *Ext0, Ext0->getDebugLoc());
3955 NewExt0->insertBefore(InsertPos: Ext0);
3956
3957 VPWidenCastRecipe *NewExt1 = NewExt0;
3958 if (Ext0 != Ext1) {
3959 NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(N: 0),
3960 Ext->getScalarType(), nullptr, *Ext1,
3961 *Ext1, Ext1->getDebugLoc());
3962 NewExt1->insertBefore(InsertPos: Ext1);
3963 }
3964 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewExt0, NewExt1});
3965 NewMul->insertBefore(InsertPos: Mul);
3966 Ext->replaceAllUsesWith(New: NewMul);
3967 Ext->eraseFromParent();
3968 Mul->eraseFromParent();
3969 return new VPExpressionRecipe(NewExt0, NewExt1, NewMul, Red);
3970 }
3971 }
3972 return nullptr;
3973}
3974
3975/// This function tries to create abstract recipes from the reduction recipe for
3976/// following optimizations and cost estimation.
3977static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
3978 VPCostContext &Ctx,
3979 VFRange &Range) {
3980 // Creation of VPExpressions for partial reductions is entirely handled in
3981 // transformToPartialReduction.
3982 if (Red->isPartialReduction())
3983 return;
3984
3985 VPExpressionRecipe *AbstractR = nullptr;
3986 auto IP = std::next(x: Red->getIterator());
3987 auto *VPBB = Red->getParent();
3988 if (auto *MulAcc = tryToMatchAndCreateMulAccumulateReduction(Red, Ctx, Range))
3989 AbstractR = MulAcc;
3990 else if (auto *ExtRed = tryToMatchAndCreateExtendedReduction(Red, Ctx, Range))
3991 AbstractR = ExtRed;
3992 // Cannot create abstract inloop reduction recipes.
3993 if (!AbstractR)
3994 return;
3995
3996 AbstractR->insertBefore(BB&: *VPBB, IP);
3997 Red->replaceAllUsesWith(New: AbstractR);
3998}
3999
4000void VPlanTransforms::convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
4001 VFRange &Range) {
4002 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
4003 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
4004 for (VPReductionRecipe &Red :
4005 make_early_inc_range(Range: make_isa_range<VPReductionRecipe>(Range&: *VPBB)))
4006 tryToCreateAbstractReductionRecipe(Red: &Red, Ctx, Range);
4007 }
4008}
4009
4010// Collect common metadata from a group of replicate recipes by intersecting
4011// metadata from all recipes in the group.
4012static VPIRMetadata getCommonMetadata(ArrayRef<VPReplicateRecipe *> Recipes) {
4013 VPIRMetadata CommonMetadata = *Recipes.front();
4014 for (VPReplicateRecipe *Recipe : drop_begin(RangeOrContainer&: Recipes))
4015 CommonMetadata.intersect(MD: *Recipe);
4016 // The recipe using the common metadata is not predicated, so it does not
4017 // share the group's execution frequency.
4018 CommonMetadata.clearExecutionFrequency();
4019 return CommonMetadata;
4020}
4021
4022template <unsigned Opcode>
4023static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
4024collectComplementaryPredicatedMemOps(VPlan &Plan,
4025 PredicatedScalarEvolution &PSE,
4026 const Loop *L) {
4027 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
4028 "Only Load and Store opcodes supported");
4029 [[maybe_unused]] constexpr bool IsLoad = (Opcode == Instruction::Load);
4030
4031 // For each address, collect operations with the same or complementary masks.
4032 SmallVector<SmallVector<VPReplicateRecipe *, 4>> AllGroups;
4033 auto Groups = collectGroupedReplicateMemOps<Opcode>(
4034 Plan, PSE, L,
4035 [](VPReplicateRecipe *RepR) { return RepR->isPredicated(); });
4036 for (auto Recipes : Groups) {
4037 if (Recipes.size() < 2)
4038 continue;
4039
4040 assert(all_equal(
4041 map_range(Recipes, bind_back<getLoadStoreValueType>(IsLoad))) &&
4042 "Expected all recipes in group to have the same load-store type");
4043
4044 // Collect groups with the same or complementary masks.
4045 for (VPReplicateRecipe *&RecipeI : Recipes) {
4046 if (!RecipeI)
4047 continue;
4048
4049 VPValue *MaskI = RecipeI->getMask();
4050 SmallVector<VPReplicateRecipe *, 4> Group;
4051 Group.push_back(Elt: RecipeI);
4052 RecipeI = nullptr;
4053
4054 // Find all operations with the same or complementary masks.
4055 bool HasComplementaryMask = false;
4056 for (VPReplicateRecipe *&RecipeJ : Recipes) {
4057 if (!RecipeJ)
4058 continue;
4059
4060 VPValue *MaskJ = RecipeJ->getMask();
4061 // Check if any operation in the group has a complementary mask with
4062 // another, that is M1 == NOT(M2) or M2 == NOT(M1).
4063 HasComplementaryMask |= match(V: MaskI, P: m_Not(Op0: m_Specific(VPV: MaskJ))) ||
4064 match(V: MaskJ, P: m_Not(Op0: m_Specific(VPV: MaskI)));
4065 Group.push_back(Elt: RecipeJ);
4066 RecipeJ = nullptr;
4067 }
4068
4069 if (HasComplementaryMask) {
4070 assert(Group.size() >= 2 && "must have at least 2 entries");
4071 AllGroups.push_back(Elt: std::move(Group));
4072 }
4073 }
4074 }
4075
4076 return AllGroups;
4077}
4078
4079// Find the recipe with minimum alignment in the group.
4080template <typename InstType>
4081static VPReplicateRecipe *
4082findRecipeWithMinAlign(ArrayRef<VPReplicateRecipe *> Group) {
4083 return *min_element(Group, [](VPReplicateRecipe *A, VPReplicateRecipe *B) {
4084 return cast<InstType>(A->getUnderlyingInstr())->getAlign() <
4085 cast<InstType>(B->getUnderlyingInstr())->getAlign();
4086 });
4087}
4088
4089void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
4090 PredicatedScalarEvolution &PSE,
4091 const Loop *L) {
4092 auto Groups =
4093 collectComplementaryPredicatedMemOps<Instruction::Load>(Plan, PSE, L);
4094 if (Groups.empty())
4095 return;
4096
4097 // Process each group of loads.
4098 for (auto &Group : Groups) {
4099 // Try to use the earliest (most dominating) load to replace all others.
4100 VPReplicateRecipe *EarliestLoad = Group[0];
4101 VPBasicBlock *FirstBB = EarliestLoad->getParent();
4102 VPBasicBlock *LastBB = Group.back()->getParent();
4103
4104 // Check that the load doesn't alias with stores between first and last.
4105 auto LoadLoc = vputils::getMemoryLocation(R: *EarliestLoad);
4106 if (!LoadLoc || !canHoistOrSinkWithNoAliasCheck(MemLoc: *LoadLoc, FirstBB, LastBB))
4107 continue;
4108
4109 // Collect common metadata from all loads in the group.
4110 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
4111
4112 // Find the load with minimum alignment to use.
4113 auto *LoadWithMinAlign = findRecipeWithMinAlign<LoadInst>(Group);
4114
4115 bool IsSingleScalar = EarliestLoad->isSingleScalar();
4116 assert(all_of(Group,
4117 [IsSingleScalar](VPReplicateRecipe *R) {
4118 return R->isSingleScalar() == IsSingleScalar;
4119 }) &&
4120 "all members in group must agree on IsSingleScalar");
4121
4122 // Create an unpredicated version of the earliest load with common
4123 // metadata.
4124 auto *UnpredicatedLoad = new VPReplicateRecipe(
4125 LoadWithMinAlign->getUnderlyingInstr(), {EarliestLoad->getOperand(N: 0)},
4126 IsSingleScalar, /*Mask=*/nullptr, *EarliestLoad, CommonMetadata);
4127
4128 UnpredicatedLoad->insertBefore(InsertPos: EarliestLoad);
4129
4130 // Replace all loads in the group with the unpredicated load.
4131 for (VPReplicateRecipe *Load : Group) {
4132 Load->replaceAllUsesWith(New: UnpredicatedLoad);
4133 Load->eraseFromParent();
4134 }
4135 }
4136}
4137
4138static bool
4139canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
4140 PredicatedScalarEvolution &PSE, const Loop &L) {
4141 auto StoreLoc = vputils::getMemoryLocation(R: *StoresToSink.front());
4142 if (!StoreLoc || !StoreLoc->AATags.Scope)
4143 return false;
4144
4145 // When sinking a group of stores, all members of the group alias each other.
4146 // Skip them during the alias checks.
4147 VPBasicBlock *FirstBB = StoresToSink.front()->getParent();
4148 VPBasicBlock *LastBB = StoresToSink.back()->getParent();
4149 SinkStoreInfo SinkInfo(StoresToSink, *StoresToSink[0], PSE, L);
4150 return canHoistOrSinkWithNoAliasCheck(MemLoc: *StoreLoc, FirstBB, LastBB, SinkInfo);
4151}
4152
4153void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
4154 PredicatedScalarEvolution &PSE,
4155 const Loop *L) {
4156 auto Groups =
4157 collectComplementaryPredicatedMemOps<Instruction::Store>(Plan, PSE, L);
4158 if (Groups.empty())
4159 return;
4160
4161 for (auto &Group : Groups) {
4162 if (!canSinkStoreWithNoAliasCheck(StoresToSink: Group, PSE, L: *L))
4163 continue;
4164
4165 // Use the last (most dominated) store's location for the unconditional
4166 // store.
4167 VPReplicateRecipe *LastStore = Group.back();
4168 VPBasicBlock *InsertBB = LastStore->getParent();
4169
4170 // Collect common alias metadata from all stores in the group.
4171 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
4172
4173 // Build select chain for stored values.
4174 VPValue *SelectedValue = Group[0]->getOperand(N: 0);
4175 VPBuilder Builder(InsertBB, LastStore->getIterator());
4176
4177 bool IsSingleScalar = Group[0]->isSingleScalar();
4178 for (unsigned I = 1; I < Group.size(); ++I) {
4179 assert(IsSingleScalar == Group[I]->isSingleScalar() &&
4180 "all members in group must agree on IsSingleScalar");
4181 VPValue *Mask = Group[I]->getMask();
4182 VPValue *Value = Group[I]->getOperand(N: 0);
4183 SelectedValue = Builder.createSelect(
4184 Cond: Mask, TrueVal: Value, FalseVal: SelectedValue, DL: Group[I]->getDebugLoc(), Name: "",
4185 Flags: VPIRFlags::getDefaultFlags(Opcode: Instruction::Select,
4186 ResultTy: Value->getScalarType()));
4187 }
4188
4189 // Find the store with minimum alignment to use.
4190 auto *StoreWithMinAlign = findRecipeWithMinAlign<StoreInst>(Group);
4191
4192 // Create unconditional store with selected value and common metadata.
4193 auto *UnpredicatedStore = new VPReplicateRecipe(
4194 StoreWithMinAlign->getUnderlyingInstr(),
4195 {SelectedValue, LastStore->getOperand(N: 1)}, IsSingleScalar,
4196 /*Mask=*/nullptr, *LastStore, CommonMetadata);
4197 UnpredicatedStore->insertBefore(BB&: *InsertBB, IP: LastStore->getIterator());
4198
4199 // Remove all predicated stores from the group.
4200 for (VPReplicateRecipe *Store : Group)
4201 Store->eraseFromParent();
4202 }
4203}
4204
4205void VPlanTransforms::widenMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
4206 unsigned UF,
4207 const TargetTransformInfo &TTI) {
4208 assert(UF > 1 && "Expected plan to have an UF > 1");
4209
4210 auto m_ContiguousVecPtr = m_VecPtr(Op0: m_VPValue(), Op1: m_One());
4211 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
4212 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
4213 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
4214 VPValue *StoredValue = nullptr;
4215 if (!match(V: &R, P: m_WidenLoad(Op0: m_ContiguousVecPtr)) &&
4216 !match(V: &R, P: m_WidenStore(Op0: m_ContiguousVecPtr, Op1: m_VPValue(V&: StoredValue))))
4217 continue;
4218
4219 auto *MemOp = cast<VPWidenMemoryRecipe>(Val: &R);
4220 assert(MemOp->isConsecutive() && "Expected consecutive load/store");
4221
4222 // TODO: Support masked loads/stores. This requires widening the header
4223 // mask to the same factor as the memory operation.
4224 assert(!MemOp->isMasked() && "Masked accesses are not supported yet");
4225
4226 Type *AccessType = StoredValue ? StoredValue->getScalarType()
4227 : R.getVPSingleValue()->getScalarType();
4228 bool IsStore = isa<VPWidenStoreRecipe>(Val: MemOp->getAsRecipe());
4229 std::optional<Instruction::CastOps> CastHint;
4230 VPUser *MaybeCast = IsStore ? StoredValue->getDefiningRecipe()
4231 : R.getVPSingleValue()->getSingleUser();
4232 if (auto *Cast = dyn_cast_if_present<VPWidenCastRecipe>(Val: MaybeCast))
4233 CastHint = Cast->getOpcode();
4234
4235 VectorType *VectorAccessType = VectorType::get(ElementType: AccessType, EC: VF);
4236 if (!TTI.hasMultiVectorLoadStore(
4237 /*NumVectors=*/UF, Mask: TargetTransformInfo::MaskSource::None,
4238 VectorTy: VectorAccessType, IsStore, CastHint))
4239 continue;
4240
4241 DebugLoc DL = R.getDebugLoc();
4242 VPValue *Ptr = MemOp->getAddr();
4243 VPValue *Align = Plan.getConstantInt(BitWidth: 64, Val: MemOp->getAlign().value());
4244 VPValue *Multiplier = Plan.getConstantInt(BitWidth: 64, Val: 1);
4245
4246 VPBuilder Builder(VPBB, R.getIterator());
4247 if (IsStore) {
4248 VPValue *WideStoredValue = Builder.createNaryOp(
4249 Opcode: VPInstruction::ConcatVectors, Operands: {StoredValue}, DL);
4250 Builder.createNaryOp(Opcode: VPInstruction::WideVectorStore,
4251 Operands: {Multiplier, Ptr, Align, WideStoredValue}, Inst: nullptr,
4252 Flags: {}, MD: *MemOp, DL: R.getDebugLoc());
4253 } else {
4254 VPValue *OldLoad = R.getVPSingleValue();
4255 VPValue *Load = Builder.createNaryOp(
4256 Opcode: VPInstruction::WideVectorLoad, Operands: {Multiplier, Ptr, Align}, Inst: nullptr,
4257 Flags: {}, MD: *MemOp, DL: R.getDebugLoc(), Name: "", ResultTy: OldLoad->getScalarType());
4258 VPValue *Extract =
4259 Builder.createNaryOp(Opcode: VPInstruction::ExtractVectorForPart,
4260 Operands: {Load, Plan.getConstantInt(BitWidth: 64, Val: 0)}, DL);
4261 OldLoad->replaceAllUsesWith(New: Extract);
4262 }
4263
4264 R.eraseFromParent();
4265 }
4266 }
4267}
4268
4269/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
4270/// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
4271/// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
4272/// the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to
4273/// an index-independent load if it feeds all wide ops at all indices (\p OpV
4274/// must be the operand at index \p OpIdx for both the recipe at lane 0, \p
4275/// WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if \p V
4276/// is defined at \p Idx of a load interleave group.
4277/// A live-in or recipe defined outside the loop region can be converted, if it
4278/// is the same across all lanes, or we can create a BuildVector for it.
4279static bool canNarrowLoad(VPSingleDefRecipe *WideMember0, unsigned OpIdx,
4280 VPValue *OpV, unsigned Idx, bool IsScalable) {
4281 VPValue *Member0Op = WideMember0->getOperand(N: OpIdx);
4282 if (Member0Op->isDefinedOutsideLoopRegions()) {
4283 // Operand matches Member0, broadcast across all fields for both live-ins
4284 // and recipes.
4285 if (Member0Op == OpV)
4286 return true;
4287 // Otherwise distinct per-field VPValues are assembled into a BuildVector.
4288 return !IsScalable && OpV->isDefinedOutsideLoopRegions() &&
4289 OpV->getScalarType() == Member0Op->getScalarType();
4290 }
4291 VPRecipeBase *Member0OpR = Member0Op->getDefiningRecipe();
4292 if (auto *W = dyn_cast<VPWidenLoadRecipe>(Val: Member0OpR))
4293 // For scalable VFs, the narrowed plan processes vscale iterations at once,
4294 // so a shared wide load cannot be narrowed to a uniform scalar; bail out.
4295 return !IsScalable && !W->getMask() && W->isConsecutive() &&
4296 Member0Op == OpV;
4297 if (auto *IR = dyn_cast<VPInterleaveRecipe>(Val: Member0OpR))
4298 return IR->getInterleaveGroup()->isFull() && IR->getVPValue(I: Idx) == OpV;
4299 return false;
4300}
4301
4302/// Returns true if the wide recipes in \p Ops and their operand trees can be
4303/// narrowed. \p FirstMembersOf maps each in-loop value to the first member
4304/// list it was seen in, across all store groups.
4305static bool
4306canNarrowOps(ArrayRef<VPValue *> Ops, bool IsScalable,
4307 DenseMap<VPValue *, SmallVector<VPValue *>> &FirstMembersOf) {
4308 SmallVector<VPValue *> Ops0;
4309 auto *WideMember0 = dyn_cast<VPRecipeWithIRFlags>(Val: Ops[0]);
4310 if (!WideMember0)
4311 return false;
4312 for (VPValue *V : Ops) {
4313 if (!isa<VPWidenRecipe, VPWidenCastRecipe>(Val: V))
4314 return false;
4315 auto *R = cast<VPRecipeWithIRFlags>(Val: V);
4316 if (vputils::getOpcode(V: R) != vputils::getOpcode(V: WideMember0))
4317 return false;
4318 if (R->getScalarType() != WideMember0->getScalarType())
4319 return false;
4320 if (R->hasPredicate() && R->getPredicate() != WideMember0->getPredicate())
4321 return false;
4322 }
4323
4324 // The first member is narrowed in place using the other members' operands.
4325 // Bail out if a value is the first member of one list and also part of a
4326 // different list.
4327 for (VPValue *V : Ops) {
4328 if (V->isDefinedOutsideLoopRegions())
4329 continue;
4330 auto [It, Inserted] = FirstMembersOf.try_emplace(Key: V, Args&: Ops);
4331 if (!Inserted && (It->second.front() == V || V == WideMember0) &&
4332 !equal(LRange&: It->second, RRange&: Ops))
4333 return false;
4334 }
4335
4336 for (unsigned Idx = 0; Idx != WideMember0->getNumOperands(); ++Idx) {
4337 SmallVector<VPValue *> OpsI;
4338 for (VPValue *Op : Ops)
4339 OpsI.push_back(Elt: Op->getDefiningRecipe()->getOperand(N: Idx));
4340
4341 if (canNarrowOps(Ops: OpsI, IsScalable, FirstMembersOf))
4342 continue;
4343
4344 if (any_of(Range: enumerate(First&: OpsI), P: [WideMember0, Idx, IsScalable](const auto &P) {
4345 const auto &[OpIdx, OpV] = P;
4346 return !canNarrowLoad(WideMember0, Idx, OpV, OpIdx, IsScalable);
4347 }))
4348 return false;
4349 }
4350
4351 return true;
4352}
4353
4354/// Returns VF from \p VFs if \p IR is a full interleave group with factor and
4355/// number of members both equal to VF. The interleave group must also access
4356/// the full vector width.
4357static std::optional<ElementCount>
4358isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR,
4359 ArrayRef<ElementCount> VFs,
4360 const TargetTransformInfo &TTI) {
4361 if (!InterleaveR || InterleaveR->getMask())
4362 return std::nullopt;
4363
4364 Type *GroupElementTy = nullptr;
4365 if (InterleaveR->getStoredValues().empty()) {
4366 GroupElementTy = InterleaveR->getVPValue(I: 0)->getScalarType();
4367 if (!all_of(Range: InterleaveR->definedValues(), P: [GroupElementTy](VPValue *Op) {
4368 return Op->getScalarType() == GroupElementTy;
4369 }))
4370 return std::nullopt;
4371 } else {
4372 GroupElementTy = InterleaveR->getStoredValues()[0]->getScalarType();
4373 if (!all_of(Range: InterleaveR->getStoredValues(), P: [GroupElementTy](VPValue *Op) {
4374 return Op->getScalarType() == GroupElementTy;
4375 }))
4376 return std::nullopt;
4377 }
4378
4379 auto IG = InterleaveR->getInterleaveGroup();
4380 if (IG->getFactor() != IG->getNumMembers())
4381 return std::nullopt;
4382
4383 auto GetVectorBitWidthForVF = [&TTI](ElementCount VF) {
4384 TypeSize Size = TTI.getRegisterBitWidth(
4385 K: VF.isFixed() ? TargetTransformInfo::RGK_FixedWidthVector
4386 : TargetTransformInfo::RGK_ScalableVector);
4387 assert(Size.isScalable() == VF.isScalable() &&
4388 "if Size is scalable, VF must be scalable and vice versa");
4389 return Size.getKnownMinValue();
4390 };
4391
4392 for (ElementCount VF : VFs) {
4393 unsigned MinVal = VF.getKnownMinValue();
4394 unsigned GroupSize = GroupElementTy->getScalarSizeInBits() * MinVal;
4395 if (IG->getFactor() == MinVal && GroupSize == GetVectorBitWidthForVF(VF))
4396 return {VF};
4397 }
4398 return std::nullopt;
4399}
4400
4401/// Returns true if \p VPValue is a narrow VPValue.
4402static bool isAlreadyNarrow(VPValue *VPV) {
4403 if (isa<VPIRValue>(Val: VPV))
4404 return true;
4405 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: VPV);
4406 return RepR && RepR->isSingleScalar();
4407}
4408
4409// Convert the wide recipes defining the VPValues in \p Members feeding an
4410// interleave group to a single narrow variant. The first member is reused as
4411// the narrowed recipe. BuildVectors for live-in operands are inserted into \p
4412// Preheader.
4413static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
4414 SmallPtrSetImpl<VPValue *> &NarrowedOps,
4415 VPBasicBlock *Preheader) {
4416 VPValue *V = Members.front();
4417 if (NarrowedOps.contains(Ptr: V))
4418 return V;
4419
4420 if (V->isDefinedOutsideLoopRegions()) {
4421 assert(all_of(Members,
4422 [V](VPValue *M) {
4423 return M->isDefinedOutsideLoopRegions() &&
4424 M->getScalarType() == V->getScalarType();
4425 }) &&
4426 "expected distinct loop-invariant values of matching scalar type");
4427 auto *BV = new VPInstruction(VPInstruction::BuildVector, Members);
4428 Preheader->appendRecipe(Recipe: BV);
4429 NarrowedOps.insert(Ptr: BV);
4430 return BV;
4431 }
4432
4433 if (isAlreadyNarrow(VPV: V))
4434 return V;
4435
4436 VPRecipeBase *R = V->getDefiningRecipe();
4437 if (isa<VPWidenRecipe, VPWidenCastRecipe>(Val: R)) {
4438 auto *WideMember0 = cast<VPRecipeWithIRFlags>(Val: R);
4439 for (VPValue *Member : Members.drop_front())
4440 WideMember0->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Member));
4441 for (unsigned Idx = 0, E = WideMember0->getNumOperands(); Idx != E; ++Idx) {
4442 SmallVector<VPValue *> OpsI;
4443 for (VPValue *Member : Members)
4444 OpsI.push_back(Elt: Member->getDefiningRecipe()->getOperand(N: Idx));
4445 WideMember0->setOperand(
4446 I: Idx, New: narrowInterleaveGroupOp(Members: OpsI, NarrowedOps, Preheader));
4447 }
4448 return V;
4449 }
4450
4451 if (auto *LoadGroup = dyn_cast<VPInterleaveRecipe>(Val: R)) {
4452 // Narrow interleave group to wide load, as transformed VPlan will only
4453 // process one original iteration.
4454 auto *LI = cast<LoadInst>(Val: LoadGroup->getInterleaveGroup()->getInsertPos());
4455 auto *L = VPBuilder(LoadGroup).createWidenLoad(
4456 Load&: *LI, Addr: LoadGroup->getAddr(), Mask: LoadGroup->getMask(), /*Consecutive=*/true,
4457 Metadata: *LoadGroup, DL: LoadGroup->getDebugLoc());
4458 NarrowedOps.insert(Ptr: L);
4459 return L;
4460 }
4461
4462 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R)) {
4463 assert(RepR->isSingleScalar() && RepR->getOpcode() == Instruction::Load &&
4464 "must be a single scalar load");
4465 NarrowedOps.insert(Ptr: RepR);
4466 return RepR;
4467 }
4468
4469 auto *WideLoad = cast<VPWidenLoadRecipe>(Val: R);
4470 VPValue *PtrOp = WideLoad->getAddr();
4471 if (auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(Val: PtrOp))
4472 PtrOp = VecPtr->getOperand(N: 0);
4473 // Narrow wide load to uniform scalar load, as transformed VPlan will only
4474 // process one original iteration.
4475 auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(), {PtrOp},
4476 /*IsUniform*/ true,
4477 /*Mask*/ nullptr, {}, *WideLoad);
4478 N->insertBefore(InsertPos: WideLoad);
4479 NarrowedOps.insert(Ptr: N);
4480 return N;
4481}
4482
4483std::unique_ptr<VPlan>
4484VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
4485 const TargetTransformInfo &TTI) {
4486 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
4487
4488 if (!VectorLoop)
4489 return nullptr;
4490
4491 // Only handle single-block loops for now.
4492 if (VectorLoop->getEntryBasicBlock() != VectorLoop->getExitingBasicBlock())
4493 return nullptr;
4494
4495 // Skip plans when we may not be able to properly narrow.
4496 VPBasicBlock *Exiting = VectorLoop->getExitingBasicBlock();
4497 if (!match(V: &Exiting->back(), P: m_BranchOnCount()))
4498 return nullptr;
4499
4500 assert(match(&Exiting->back(),
4501 m_BranchOnCount(m_Add(m_VPValue(), m_Specific(&Plan.getVFxUF())),
4502 m_Specific(&Plan.getVectorTripCount()))) &&
4503 "unexpected branch-on-count");
4504
4505 SmallVector<VPInterleaveRecipe *> StoreGroups;
4506 DenseMap<VPValue *, SmallVector<VPValue *>> FirstMembersOf;
4507 std::optional<ElementCount> VFToOptimize;
4508 for (auto &R : *VectorLoop->getEntryBasicBlock()) {
4509 if (isa<VPDerivedIVRecipe, VPScalarIVStepsRecipe>(Val: &R) &&
4510 vputils::onlyFirstLaneUsed(Def: cast<VPSingleDefRecipe>(Val: &R)))
4511 continue;
4512
4513 // Bail out on recipes not supported at the moment:
4514 // * phi recipes other than the canonical induction
4515 // * recipes writing to memory except interleave groups
4516 // Only support plans with a canonical induction phi.
4517 if (R.isPhi())
4518 return nullptr;
4519
4520 auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(Val: &R);
4521 if (R.mayWriteToMemory() && !InterleaveR)
4522 return nullptr;
4523
4524 // Bail out if any recipe defines a vector value used outside the
4525 // vector loop region.
4526 if (any_of(Range: R.definedValues(), P: [&](VPValue *V) {
4527 return any_of(Range: V->users(), P: [&](VPUser *U) {
4528 auto *UR = cast<VPRecipeBase>(Val: U);
4529 return UR->getParent()->getParent() != VectorLoop;
4530 });
4531 }))
4532 return nullptr;
4533
4534 // All other ops are allowed, but we reject uses that cannot be converted
4535 // when checking all allowed consumers (store interleave groups) below.
4536 if (!InterleaveR)
4537 continue;
4538
4539 // Try to find a single VF, where all interleave groups are consecutive and
4540 // saturate the full vector width. If we already have a candidate VF, check
4541 // if it is applicable for the current InterleaveR, otherwise look for a
4542 // suitable VF across the Plan's VFs.
4543 SmallVector<ElementCount> VFs =
4544 VFToOptimize ? SmallVector<ElementCount>({*VFToOptimize})
4545 : to_vector(Range: Plan.vectorFactors());
4546 std::optional<ElementCount> NarrowedVF =
4547 isConsecutiveInterleaveGroup(InterleaveR, VFs, TTI);
4548 if (!NarrowedVF || (VFToOptimize && NarrowedVF != VFToOptimize))
4549 return nullptr;
4550 VFToOptimize = NarrowedVF;
4551
4552 // Skip read interleave groups.
4553 if (InterleaveR->getStoredValues().empty())
4554 continue;
4555
4556 // Narrow interleave groups, if all operands are already matching narrow
4557 // ops.
4558 auto *Member0 = InterleaveR->getStoredValues()[0];
4559 if (isAlreadyNarrow(VPV: Member0) &&
4560 all_of(Range: InterleaveR->getStoredValues(), P: equal_to(Arg&: Member0))) {
4561 StoreGroups.push_back(Elt: InterleaveR);
4562 continue;
4563 }
4564
4565 // For now, we only support full interleave groups storing load interleave
4566 // groups.
4567 if (all_of(Range: enumerate(First: InterleaveR->getStoredValues()), P: [](auto Op) {
4568 VPRecipeBase *DefR = Op.value()->getDefiningRecipe();
4569 if (!DefR)
4570 return false;
4571 auto *IR = dyn_cast<VPInterleaveRecipe>(Val: DefR);
4572 return IR && IR->getInterleaveGroup()->isFull() &&
4573 IR->getVPValue(Op.index()) == Op.value();
4574 })) {
4575 StoreGroups.push_back(Elt: InterleaveR);
4576 continue;
4577 }
4578
4579 // Check if all values feeding InterleaveR are matching wide recipes whose
4580 // operands can be narrowed, without conflicting uses across groups.
4581 if (!canNarrowOps(Ops: InterleaveR->getStoredValues(),
4582 IsScalable: VFToOptimize->isScalable(), FirstMembersOf))
4583 return nullptr;
4584 StoreGroups.push_back(Elt: InterleaveR);
4585 }
4586
4587 if (StoreGroups.empty())
4588 return nullptr;
4589
4590 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
4591 bool RequiresScalarEpilogue =
4592 MiddleVPBB->getNumSuccessors() == 1 &&
4593 MiddleVPBB->getSingleSuccessor() == Plan.getScalarPreheader();
4594 // Bail out for tail-folding (middle block with a single successor to exit).
4595 if (MiddleVPBB->getNumSuccessors() != 2 && !RequiresScalarEpilogue)
4596 return nullptr;
4597
4598 // All interleave groups in Plan can be narrowed for VFToOptimize. Split the
4599 // original Plan into 2: a) a new clone which contains all VFs of Plan, except
4600 // VFToOptimize, and b) the original Plan with VFToOptimize as single VF.
4601 // TODO: Handle cases where only some interleave groups can be narrowed.
4602 std::unique_ptr<VPlan> NewPlan;
4603 if (size(Range: Plan.vectorFactors()) != 1) {
4604 NewPlan = std::unique_ptr<VPlan>(Plan.duplicate());
4605 Plan.setVF(*VFToOptimize);
4606 NewPlan->removeVF(VF: *VFToOptimize);
4607 }
4608
4609 // Convert InterleaveGroup \p R to a single VPWidenLoadRecipe.
4610 SmallPtrSet<VPValue *, 4> NarrowedOps;
4611 VPBasicBlock *Preheader = Plan.getVectorPreheader();
4612 // Narrow operation tree rooted at store groups.
4613 for (auto *StoreGroup : StoreGroups) {
4614 VPValue *Res = narrowInterleaveGroupOp(Members: StoreGroup->getStoredValues(),
4615 NarrowedOps, Preheader);
4616 auto *SI =
4617 cast<StoreInst>(Val: StoreGroup->getInterleaveGroup()->getInsertPos());
4618 VPBuilder(StoreGroup)
4619 .createWidenStore(Store&: *SI, Addr: StoreGroup->getAddr(), StoredVal: Res, Mask: nullptr,
4620 /*Consecutive=*/true, Metadata: *StoreGroup,
4621 DL: StoreGroup->getDebugLoc());
4622 StoreGroup->eraseFromParent();
4623 }
4624
4625 // Adjust induction to reflect that the transformed plan only processes one
4626 // original iteration.
4627 VPInstruction *CanIVInc = vputils::findCanonicalIVIncrement(Plan);
4628 Type *CanIVTy = VectorLoop->getCanonicalIVType();
4629 VPBasicBlock *VectorPH = Plan.getVectorPreheader();
4630 VPBuilder PHBuilder(VectorPH, VectorPH->getFirstNonPhi());
4631
4632 VPValue *UF = &Plan.getUF();
4633 VPValue *Step;
4634 if (VFToOptimize->isScalable()) {
4635 VPValue *VScale =
4636 PHBuilder.createElementCount(Ty: CanIVTy, EC: ElementCount::getScalable(MinVal: 1));
4637 Step = PHBuilder.createOverflowingOp(Opcode: Instruction::Mul, Operands: {VScale, UF},
4638 WrapFlags: {true, false});
4639 Plan.getVF().replaceAllUsesWith(New: VScale);
4640 } else {
4641 Step = UF;
4642 Plan.getVF().replaceAllUsesWith(New: Plan.getConstantInt(Ty: CanIVTy, Val: 1));
4643 }
4644 // Materialize vector trip count with the narrowed step.
4645 materializeVectorTripCount(Plan, VectorPHVPBB: VectorPH, /*TailByMasking=*/false,
4646 RequiresScalarEpilogue, Step);
4647
4648 CanIVInc->setOperand(I: 1, New: Step);
4649 Plan.getVFxUF().replaceAllUsesWith(New: Step);
4650
4651 removeDeadRecipes(Plan);
4652 assert(none_of(*VectorLoop->getEntryBasicBlock(),
4653 IsaPred<VPVectorPointerRecipe>) &&
4654 "All VPVectorPointerRecipes should have been removed");
4655 return NewPlan;
4656}
4657
4658void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
4659 VFRange &Range) {
4660 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
4661 auto *MiddleVPBB = Plan.getMiddleBlock();
4662 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
4663
4664 auto IsScalableOne = [](ElementCount VF) -> bool {
4665 return VF == ElementCount::getScalable(MinVal: 1);
4666 };
4667
4668 for (VPFirstOrderRecurrencePHIRecipe &FOR :
4669 make_isa_range<VPFirstOrderRecurrencePHIRecipe>(
4670 Range: VectorRegion->getEntryBasicBlock()->phis())) {
4671 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
4672 "Cannot handle loops with uncountable early exits");
4673
4674 // Find the existing splice for this FOR, created in
4675 // createHeaderPhiRecipes. All uses of FOR have already been replaced with
4676 // RecurSplice there; only RecurSplice itself still references FOR.
4677 auto *RecurSplice =
4678 findUserOf<VPInstruction::FirstOrderRecurrenceSplice>(V: &FOR);
4679 assert(RecurSplice && "expected FirstOrderRecurrenceSplice");
4680
4681 // For VF vscale x 1, if vscale = 1, we are unable to extract the
4682 // penultimate value of the recurrence. Instead we rely on the existing
4683 // extract of the last element from the result of
4684 // VPInstruction::FirstOrderRecurrenceSplice.
4685 // TODO: Consider vscale_range info and UF.
4686 if (any_of(Range: RecurSplice->users(),
4687 P: [](VPUser *U) { return !cast<VPRecipeBase>(Val: U)->getRegion(); }) &&
4688 LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsScalableOne,
4689 Range))
4690 return;
4691
4692 // This is the second phase of vectorizing first-order recurrences, creating
4693 // extracts for users outside the loop. An overview of the transformation is
4694 // described below. Suppose we have the following loop with some use after
4695 // the loop of the last a[i-1],
4696 //
4697 // for (int i = 0; i < n; ++i) {
4698 // t = a[i - 1];
4699 // b[i] = a[i] - t;
4700 // }
4701 // use t;
4702 //
4703 // There is a first-order recurrence on "a". For this loop, the shorthand
4704 // scalar IR looks like:
4705 //
4706 // scalar.ph:
4707 // s.init = a[-1]
4708 // br scalar.body
4709 //
4710 // scalar.body:
4711 // i = phi [0, scalar.ph], [i+1, scalar.body]
4712 // s1 = phi [s.init, scalar.ph], [s2, scalar.body]
4713 // s2 = a[i]
4714 // b[i] = s2 - s1
4715 // br cond, scalar.body, exit.block
4716 //
4717 // exit.block:
4718 // use = lcssa.phi [s1, scalar.body]
4719 //
4720 // In this example, s1 is a recurrence because it's value depends on the
4721 // previous iteration. In the first phase of vectorization, we created a
4722 // VPFirstOrderRecurrencePHIRecipe v1 for s1. Now we create the extracts
4723 // for users in the scalar preheader and exit block.
4724 //
4725 // vector.ph:
4726 // v_init = vector(..., ..., ..., a[-1])
4727 // br vector.body
4728 //
4729 // vector.body
4730 // i = phi [0, vector.ph], [i+4, vector.body]
4731 // v1 = phi [v_init, vector.ph], [v2, vector.body]
4732 // v2 = a[i, i+1, i+2, i+3]
4733 // v1' = splice(v1(3), v2(0, 1, 2))
4734 // b[i, i+1, i+2, i+3] = v2 - v1'
4735 // br cond, vector.body, middle.block
4736 //
4737 // middle.block:
4738 // vector.recur.extract.for.phi = v2(2)
4739 // vector.recur.extract = v2(3)
4740 // br cond, scalar.ph, exit.block
4741 //
4742 // scalar.ph:
4743 // scalar.recur.init = phi [vector.recur.extract, middle.block],
4744 // [s.init, otherwise]
4745 // br scalar.body
4746 //
4747 // scalar.body:
4748 // i = phi [0, scalar.ph], [i+1, scalar.body]
4749 // s1 = phi [scalar.recur.init, scalar.ph], [s2, scalar.body]
4750 // s2 = a[i]
4751 // b[i] = s2 - s1
4752 // br cond, scalar.body, exit.block
4753 //
4754 // exit.block:
4755 // lo = lcssa.phi [s1, scalar.body],
4756 // [vector.recur.extract.for.phi, middle.block]
4757 //
4758 // Update extracts of the splice in the middle block: they extract the
4759 // penultimate element of the recurrence.
4760 for (VPRecipeBase &R : make_early_inc_range(
4761 Range: make_range(x: MiddleVPBB->getFirstNonPhi(), y: MiddleVPBB->end()))) {
4762 if (!match(V: &R, P: m_ExtractLastLaneOfLastPart(Op0: m_Specific(VPV: RecurSplice))))
4763 continue;
4764
4765 auto *ExtractR = cast<VPInstruction>(Val: &R);
4766 VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
4767 Opcode: VPInstruction::ExtractPenultimateElement, Operands: RecurSplice->getOperand(N: 1),
4768 DL: {}, Name: "vector.recur.extract.for.phi");
4769 for (VPUser *ExitU : to_vector(Range: ExtractR->users())) {
4770 if (auto *ExitPhi = dyn_cast<VPIRPhi>(Val: ExitU))
4771 ExitPhi->replaceUsesOfWith(From: ExtractR, To: PenultimateElement);
4772 }
4773 }
4774 }
4775}
4776
4777/// Check if \p V is a binary expression of a widened IV and a loop-invariant
4778/// value. Returns the widened IV if found, nullptr otherwise.
4779static VPWidenIntOrFpInductionRecipe *getExpressionIV(VPValue *V) {
4780 auto *BinOp = dyn_cast<VPWidenRecipe>(Val: V);
4781 if (!BinOp || !Instruction::isBinaryOp(Opcode: BinOp->getOpcode()) ||
4782 Instruction::isIntDivRem(Opcode: BinOp->getOpcode()))
4783 return nullptr;
4784
4785 VPValue *WidenIVCandidate = BinOp->getOperand(N: 0);
4786 VPValue *InvariantCandidate = BinOp->getOperand(N: 1);
4787 if (!isa<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate))
4788 std::swap(a&: WidenIVCandidate, b&: InvariantCandidate);
4789
4790 if (!InvariantCandidate->isDefinedOutsideLoopRegions())
4791 return nullptr;
4792
4793 return dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate);
4794}
4795
4796/// Create a scalar version of \p BinOp, with its \p WidenIV operand replaced
4797/// by \p ScalarIV, and place it after \p ScalarIV's defining recipe.
4798static VPValue *cloneBinOpForScalarIV(VPWidenRecipe *BinOp, VPValue *ScalarIV,
4799 VPWidenIntOrFpInductionRecipe *WidenIV) {
4800 assert(Instruction::isBinaryOp(BinOp->getOpcode()) &&
4801 BinOp->getNumOperands() == 2 && "BinOp must have 2 operands");
4802 auto *ClonedOp = BinOp->clone();
4803 if (ClonedOp->getOperand(N: 0) == WidenIV) {
4804 ClonedOp->setOperand(I: 0, New: ScalarIV);
4805 } else {
4806 assert(ClonedOp->getOperand(1) == WidenIV && "one operand must be WideIV");
4807 ClonedOp->setOperand(I: 1, New: ScalarIV);
4808 }
4809 ClonedOp->insertAfter(InsertPos: ScalarIV->getDefiningRecipe());
4810 return ClonedOp;
4811}
4812
4813/// If \p S is an affine AddRec, returns true if its step is known to be
4814/// positive and false if it is known to be negative. Returns std::nullopt if
4815/// \p S is not an affine AddRec, or if the sign of its step cannot be
4816/// determined.
4817static std::optional<bool> getStepDirection(const SCEV *S,
4818 ScalarEvolution &SE) {
4819 const SCEV *Step;
4820 if (!match(S, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV(V&: Step))))
4821 return std::nullopt;
4822 if (SE.isKnownPositive(S: Step))
4823 return true;
4824 if (SE.isKnownNegative(S: Step))
4825 return false;
4826 return std::nullopt;
4827}
4828
4829void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
4830 PredicatedScalarEvolution &PSE,
4831 Loop &L) {
4832 ScalarEvolution &SE = *PSE.getSE();
4833 VPRegionBlock *VectorLoopRegion = Plan.getVectorLoopRegion();
4834
4835 // Helper lambda to check if the IV range excludes the sentinel value. Try
4836 // signed first, then unsigned. Return an excluded sentinel if found,
4837 // otherwise return std::nullopt.
4838 auto CheckSentinel = [&SE](const SCEV *IVSCEV,
4839 bool UseMax) -> std::optional<APSInt> {
4840 unsigned BW = IVSCEV->getType()->getScalarSizeInBits();
4841 for (bool Signed : {true, false}) {
4842 APSInt Sentinel = UseMax ? APSInt::getMinValue(numBits: BW, /*Unsigned=*/!Signed)
4843 : APSInt::getMaxValue(numBits: BW, /*Unsigned=*/!Signed);
4844
4845 ConstantRange IVRange =
4846 Signed ? SE.getSignedRange(S: IVSCEV) : SE.getUnsignedRange(S: IVSCEV);
4847 if (!IVRange.contains(Val: Sentinel))
4848 return Sentinel;
4849 }
4850 return std::nullopt;
4851 };
4852
4853 VPValue *HeaderMask = VectorLoopRegion->getHeaderMask();
4854 for (VPRecipeBase &Phi :
4855 make_early_inc_range(Range: VectorLoopRegion->getEntryBasicBlock()->phis())) {
4856 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &Phi);
4857 if (!PhiR || !RecurrenceDescriptor::isFindLastRecurrenceKind(
4858 Kind: PhiR->getRecurrenceKind()))
4859 continue;
4860
4861 Type *PhiTy = PhiR->getScalarType();
4862 if (PhiTy->isPointerTy() || PhiTy->isFloatingPointTy())
4863 continue;
4864
4865 // If there's a header mask, the backedge select will not be the find-last
4866 // select.
4867 VPValue *BackedgeVal = PhiR->getBackedgeValue();
4868 auto *FindLastSelect = cast<VPSingleDefRecipe>(Val: BackedgeVal);
4869 if (HeaderMask &&
4870 !match(V: BackedgeVal,
4871 P: m_Select(Op0: m_Specific(VPV: HeaderMask),
4872 Op1: m_VPSingleDefRecipe(V&: FindLastSelect), Op2: m_Specific(VPV: PhiR))))
4873 continue;
4874
4875 // Get the find-last expression from the find-last select of the reduction
4876 // phi. The find-last select should be a select between the phi and the
4877 // find-last expression.
4878 VPValue *Cond, *FindLastExpression;
4879 if (!match(R: FindLastSelect, P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: PhiR),
4880 Op2: m_VPValue(V&: FindLastExpression))) &&
4881 !match(R: FindLastSelect,
4882 P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_VPValue(V&: FindLastExpression),
4883 Op2: m_Specific(VPV: PhiR))))
4884 continue;
4885
4886 // Check if FindLastExpression is a simple expression of a widened IV. If
4887 // so, we can track the underlying IV instead and sink the expression.
4888 auto *IVOfExpressionToSink = getExpressionIV(V: FindLastExpression);
4889 const SCEV *IVSCEV = vputils::getSCEVExprForVPValue(
4890 V: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression, PSE,
4891 L: &L);
4892 if (!match(S: IVSCEV, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV()))) {
4893 assert(!match(vputils::getSCEVExprForVPValue(FindLastExpression, PSE, &L),
4894 m_scev_AffineAddRec(m_SCEV(), m_SCEV())) &&
4895 "IVOfExpressionToSink not being an AddRec must imply "
4896 "FindLastExpression not being an AddRec.");
4897 continue;
4898 }
4899
4900 // Determine direction from the step of IVSCEV, if possible.
4901 std::optional<bool> StepDirection = getStepDirection(S: IVSCEV, SE);
4902 if (!StepDirection)
4903 continue;
4904
4905 bool UseMax = *StepDirection;
4906 std::optional<APSInt> SentinelVal = CheckSentinel(IVSCEV, UseMax);
4907 bool UseSigned = SentinelVal && SentinelVal->isSigned();
4908
4909 // Sinking an expression will disable epilogue vectorization. Only use it,
4910 // if FindLastExpression cannot be vectorized via a sentinel. Sinking may
4911 // also prevent vectorizing using a sentinel (e.g., if the expression is a
4912 // multiply or divide by large constant, respectively), which also makes
4913 // sinking undesirable.
4914 if (IVOfExpressionToSink) {
4915 const SCEV *FindLastExpressionSCEV =
4916 vputils::getSCEVExprForVPValue(V: FindLastExpression, PSE, L: &L);
4917 if (std::optional<bool> NewUseMax =
4918 getStepDirection(S: FindLastExpressionSCEV, SE)) {
4919 if (auto NewSentinel =
4920 CheckSentinel(FindLastExpressionSCEV, *NewUseMax)) {
4921 // The original expression already has a sentinel, so prefer not
4922 // sinking to keep epilogue vectorization possible.
4923 SentinelVal = *NewSentinel;
4924 UseSigned = NewSentinel->isSigned();
4925 UseMax = *NewUseMax;
4926 IVSCEV = FindLastExpressionSCEV;
4927 IVOfExpressionToSink = nullptr;
4928 }
4929 }
4930 }
4931
4932 // If no sentinel was found, fall back to a boolean AnyOf reduction to track
4933 // if the condition was ever true. Requires the IV to not wrap, otherwise we
4934 // cannot use min/max.
4935 if (!SentinelVal) {
4936 auto *AR = cast<SCEVAddRecExpr>(Val: IVSCEV);
4937 if (AR->hasNoSignedWrap())
4938 UseSigned = true;
4939 else if (AR->hasNoUnsignedWrap())
4940 UseSigned = false;
4941 else
4942 continue;
4943 }
4944
4945 VPInstruction *RdxResult = cast<VPInstruction>(Val: vputils::findRecipe(
4946 Start: BackedgeVal,
4947 Pred: match_fn(P: m_VPInstruction<VPInstruction::ComputeReductionResult>())));
4948
4949 VPValue *NewFindLastSelect = BackedgeVal;
4950 VPValue *SelectCond = Cond;
4951 if (!SentinelVal || IVOfExpressionToSink) {
4952 // When we need to create a new select, normalize the condition so that
4953 // PhiR is the last operand and include the header mask if needed.
4954 DebugLoc DL = FindLastSelect->getDefiningRecipe()->getDebugLoc();
4955 VPBuilder LoopBuilder(FindLastSelect->getDefiningRecipe());
4956 if (match(R: FindLastSelect,
4957 P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: PhiR), Op2: m_VPValue())))
4958 SelectCond = LoopBuilder.createNot(Operand: SelectCond);
4959
4960 // When tail folding, mask the condition with the header mask to prevent
4961 // propagating poison from inactive lanes in the last vector iteration.
4962 if (HeaderMask)
4963 SelectCond = LoopBuilder.createLogicalAnd(LHS: HeaderMask, RHS: SelectCond);
4964
4965 if (SelectCond != Cond || IVOfExpressionToSink) {
4966 NewFindLastSelect = LoopBuilder.createSelect(
4967 Cond: SelectCond,
4968 TrueVal: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression,
4969 FalseVal: PhiR, DL);
4970 }
4971 }
4972
4973 // Create the reduction result in the middle block using sentinel directly.
4974 RecurKind MinMaxKind =
4975 UseMax ? (UseSigned ? RecurKind::SMax : RecurKind::UMax)
4976 : (UseSigned ? RecurKind::SMin : RecurKind::UMin);
4977 VPIRFlags Flags(MinMaxKind, /*IsOrdered=*/false, /*IsInLoop=*/false,
4978 FastMathFlags());
4979 DebugLoc ExitDL = RdxResult->getDebugLoc();
4980 VPBuilder MiddleBuilder(RdxResult);
4981 VPValue *ReducedIV =
4982 MiddleBuilder.createNaryOp(Opcode: VPInstruction::ComputeReductionResult,
4983 Operands: NewFindLastSelect, Flags, DL: ExitDL);
4984
4985 // If IVOfExpressionToSink is an expression to sink, sink it now.
4986 VPValue *VectorRegionExitingVal = ReducedIV;
4987 bool SunkExpression = false;
4988 if (IVOfExpressionToSink) {
4989 VectorRegionExitingVal =
4990 cloneBinOpForScalarIV(BinOp: cast<VPWidenRecipe>(Val: FindLastExpression),
4991 ScalarIV: ReducedIV, WidenIV: IVOfExpressionToSink);
4992 SunkExpression = true;
4993 }
4994
4995 VPValue *NewRdxResult;
4996 VPValue *StartVPV = PhiR->getStartValue();
4997 if (SentinelVal) {
4998 // Sentinel-based approach: reduce IVs with min/max, compare against
4999 // sentinel to detect if condition was ever true, select accordingly.
5000 VPValue *Sentinel = Plan.getConstantInt(Val: *SentinelVal);
5001 auto *Cmp = MiddleBuilder.createICmp(Pred: CmpInst::ICMP_NE, A: ReducedIV,
5002 B: Sentinel, DL: ExitDL);
5003 NewRdxResult = MiddleBuilder.createSelect(Cond: Cmp, TrueVal: VectorRegionExitingVal,
5004 FalseVal: StartVPV, DL: ExitDL);
5005 StartVPV = Sentinel;
5006 } else {
5007 // Introduce a boolean AnyOf reduction to track if the condition was ever
5008 // true in the loop. Use it to select the initial start value, if it was
5009 // never true.
5010 auto *AnyOfPhi = new VPReductionPHIRecipe(
5011 /*Phi=*/nullptr, RecurKind::Or, *Plan.getFalse(), *Plan.getFalse(),
5012 RdxUnordered{.VFScaleFactor: 1}, {}, /*HasUsesOutsideReductionChain=*/false);
5013 AnyOfPhi->insertAfter(InsertPos: PhiR);
5014
5015 VPBuilder LoopBuilder(BackedgeVal->getDefiningRecipe());
5016 VPValue *OrVal = LoopBuilder.createOr(LHS: AnyOfPhi, RHS: SelectCond);
5017 AnyOfPhi->setOperand(I: 1, New: OrVal);
5018
5019 NewRdxResult = MiddleBuilder.createAnyOfReduction(
5020 ChainOp: OrVal, TrueVal: VectorRegionExitingVal, FalseVal: StartVPV, DL: ExitDL);
5021
5022 // Initialize the IV reduction phi with the neutral element, not the
5023 // original start value, to ensure correct min/max reduction results.
5024 StartVPV = Plan.getOrAddLiveIn(
5025 V: getRecurrenceIdentity(K: MinMaxKind, Tp: IVSCEV->getType(), FMF: {}));
5026 }
5027 RdxResult->replaceAllUsesWith(New: NewRdxResult);
5028 RdxResult->eraseFromParent();
5029
5030 auto *NewPhiR = new VPReductionPHIRecipe(
5031 cast<PHINode>(Val: PhiR->getUnderlyingInstr()), RecurKind::FindIV, *StartVPV,
5032 *NewFindLastSelect, RdxUnordered{.VFScaleFactor: 1}, {},
5033 PhiR->hasUsesOutsideReductionChain());
5034 if (SunkExpression)
5035 NewPhiR->setExpressionSunk();
5036 NewPhiR->insertBefore(InsertPos: PhiR);
5037 PhiR->replaceAllUsesWith(New: NewPhiR);
5038 PhiR->eraseFromParent();
5039 }
5040}
5041
5042namespace {
5043
5044using ExtendKind = TTI::PartialReductionExtendKind;
5045struct ReductionExtend {
5046 Type *SrcType = nullptr;
5047 ExtendKind Kind = ExtendKind::PR_None;
5048};
5049
5050/// Describes the extends used to compute the extended reduction operand.
5051/// ExtendB is optional. If ExtendB is present, ExtendsUser is a binary
5052/// operation.
5053struct ExtendedReductionOperand {
5054 /// The recipe that consumes the extends.
5055 VPWidenRecipe *ExtendsUser = nullptr;
5056 /// Extend descriptions (inputs to getPartialReductionCost).
5057 ReductionExtend ExtendA, ExtendB;
5058};
5059
5060/// A collection of recipes that describe a partial reduction. Matches either
5061/// reduction_bin_op (extended op, accumulator), or
5062/// reduction_bin_op (accumulator, extended op).
5063/// The possible forms of the "extended op" are listed in
5064/// matchExtendedReductionOperand.
5065struct PartialReductionDescriptor {
5066 /// The top-level binary operation that forms the reduction to a scalar
5067 /// after the loop body.
5068 VPWidenRecipe *ReductionBinOp = nullptr;
5069 /// The user of the extends that is then reduced.
5070 ExtendedReductionOperand ExtendedOp;
5071 /// The recurrence kind for the entire partial reduction chain.
5072 /// This allows distinguishing between Sub and AddWithSub recurrences,
5073 /// when the ReductionBinOp is a Instruction::Sub.
5074 RecurKind RK;
5075 /// The index of the accumulator operand of ReductionBinOp. The extended op
5076 /// is `1 - AccumulatorOpIdx`.
5077 unsigned AccumulatorOpIdx;
5078 unsigned ScaleFactor;
5079 /// Optional blend to represent predication for the block that updates the
5080 /// reduction.
5081 VPBlendRecipe *Blend = nullptr;
5082};
5083
5084// Return the incoming index of the single-use value in the blend, which is
5085// expected to be the predicated reduction update.
5086static std::optional<unsigned>
5087getBlendReductionUpdateValueIdx(VPBlendRecipe *Blend) {
5088 assert(Blend && !Blend->isNormalized() &&
5089 Blend->getNumIncomingValues() == 2 &&
5090 "Expected a non-normalized blend with two incoming values");
5091 bool FirstIncomingHasOneUse = Blend->getIncomingValue(Idx: 0)->hasOneUse();
5092
5093 // Only the update value should have one use (the blend). The previous
5094 // value should always have at least two uses, the blend and the reduction.
5095 if (FirstIncomingHasOneUse == Blend->getIncomingValue(Idx: 1)->hasOneUse())
5096 return std::nullopt;
5097 return FirstIncomingHasOneUse ? 0 : 1;
5098}
5099
5100static VPSingleDefRecipe *
5101optimizeExtendsForPartialReduction(VPSingleDefRecipe *Op) {
5102 // reduce.add(mul(ext(A), C))
5103 // -> reduce.add(mul(ext(A), ext(trunc(C))))
5104 const APInt *Const;
5105 if (match(R: Op, P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_APInt(C&: Const)))) {
5106 auto *ExtA = cast<VPWidenCastRecipe>(Val: Op->getOperand(N: 0));
5107 Instruction::CastOps ExtOpc = ExtA->getOpcode();
5108 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
5109 if (!Op->hasOneUse() ||
5110 !llvm::canConstantBeExtended(
5111 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
5112 return Op;
5113
5114 VPBuilder Builder(Op);
5115 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc,
5116 Op: Op->getOperand(N: 1), ResultTy: NarrowTy);
5117 Type *WideTy = ExtA->getScalarType();
5118 Op->setOperand(I: 1, New: Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy));
5119 return Op;
5120 }
5121
5122 // reduce.add(abs(sub(ext(A), ext(B))))
5123 // -> reduce.add(ext(absolute-difference(A, B)))
5124 VPValue *X, *Y;
5125 if (match(R: Op, P: m_WidenIntrinsic<Intrinsic::abs>(Ops: m_Sub(
5126 Op0: m_ZExtOrSExt(Op0: m_VPValue(V&: X)), Op1: m_ZExtOrSExt(Op0: m_VPValue(V&: Y)))))) {
5127 auto *Sub = Op->getOperand(N: 0)->getDefiningRecipe();
5128 auto *Ext = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
5129 assert(Ext->getOpcode() ==
5130 cast<VPWidenCastRecipe>(Sub->getOperand(1))->getOpcode() &&
5131 "Expected both the LHS and RHS extends to be the same");
5132 bool IsSigned = Ext->getOpcode() == Instruction::SExt;
5133 VPBuilder Builder(Op);
5134 Type *SrcTy = X->getScalarType();
5135 auto *FreezeX = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {X}));
5136 auto *FreezeY = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {Y}));
5137 auto *Max = Builder.insert(
5138 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smax : Intrinsic::umax,
5139 {FreezeX, FreezeY}, SrcTy));
5140 auto *Min = Builder.insert(
5141 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smin : Intrinsic::umin,
5142 {FreezeX, FreezeY}, SrcTy));
5143 auto *AbsDiff = Builder.insert(
5144 R: new VPWidenRecipe(Instruction::Sub, {Max, Min},
5145 VPIRFlags::getDefaultFlags(Opcode: Instruction::Sub)));
5146 return Builder.createWidenCast(Opcode: Instruction::CastOps::ZExt, Op: AbsDiff,
5147 ResultTy: Op->getScalarType());
5148 }
5149
5150 // reduce.add(ext(mul(ext(A), ext(B))))
5151 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
5152 // TODO: Support this optimization for float types.
5153 if (match(R: Op, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
5154 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
5155 auto *Ext = cast<VPWidenCastRecipe>(Val: Op);
5156 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
5157 auto *MulLHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5158 auto *MulRHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5159 if (!Mul->hasOneUse() ||
5160 (Ext->getOpcode() != MulLHS->getOpcode() && MulLHS != MulRHS) ||
5161 MulLHS->getOpcode() != MulRHS->getOpcode())
5162 return Op;
5163 VPBuilder Builder(Mul);
5164 auto *NewLHS = Builder.createWidenCast(
5165 Opcode: MulLHS->getOpcode(), Op: MulLHS->getOperand(N: 0), ResultTy: Ext->getScalarType());
5166 auto *NewRHS = MulLHS == MulRHS
5167 ? NewLHS
5168 : Builder.createWidenCast(Opcode: MulRHS->getOpcode(),
5169 Op: MulRHS->getOperand(N: 0),
5170 ResultTy: Ext->getScalarType());
5171 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewLHS, NewRHS});
5172 Builder.insert(R: NewMul);
5173 Op->replaceAllUsesWith(New: NewMul);
5174 Op->eraseFromParent();
5175 Mul->eraseFromParent();
5176 return NewMul;
5177 }
5178
5179 return Op;
5180}
5181
5182static VPExpressionRecipe *
5183createPartialReductionExpression(VPReductionRecipe *Red) {
5184 VPValue *VecOp = Red->getVecOp();
5185
5186 // reduce.[f]add(ext(op))
5187 // -> VPExpressionRecipe(op, red)
5188 if (match(V: VecOp, P: m_WidenAnyExtend(Op0: m_VPValue())))
5189 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
5190
5191 // reduce.[f]add(neg(ext(op)))
5192 // -> VPExpressionRecipe(op, sub/neg, red)
5193 if (match(V: VecOp, P: m_AnyNeg(Op0: m_WidenAnyExtend(Op0: m_VPValue())))) {
5194 auto *Neg = cast<VPWidenRecipe>(Val: VecOp);
5195 auto *Ext = cast<VPWidenCastRecipe>(Val: Neg->getLastOperand());
5196 return new VPExpressionRecipe(Ext, Neg, Red);
5197 }
5198
5199 // reduce.[f]add([f]mul(ext(a), ext(b)))
5200 // -> VPExpressionRecipe(a, b, mul, red)
5201 if (match(V: VecOp, P: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))) ||
5202 match(V: VecOp,
5203 P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_ZExtOrSExt(Op0: m_VPValue())))) {
5204 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
5205 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5206 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5207 return new VPExpressionRecipe(ExtA, ExtB, Mul, Red);
5208 }
5209
5210 // reduce.fadd(fneg(fmul(fpext(a), fpext(b))))
5211 // -> VPExpressionRecipe(a, b, fmul, fsub, red)
5212 if (match(V: VecOp,
5213 P: m_FNeg(Op0: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))))) {
5214 auto *FNeg = cast<VPWidenRecipe>(Val: VecOp);
5215 auto *FMul = cast<VPWidenRecipe>(Val: FNeg->getOperand(N: 0));
5216 auto *ExtA = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 0));
5217 auto *ExtB = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 1));
5218 return new VPExpressionRecipe(ExtA, ExtB, FMul, FNeg, Red);
5219 }
5220
5221 // reduce.add(neg(mul(ext(a), ext(b))))
5222 // -> VPExpressionRecipe(a, b, mul, sub, red)
5223 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
5224 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
5225 auto *Sub = cast<VPWidenRecipe>(Val: VecOp);
5226 auto *Mul = cast<VPWidenRecipe>(Val: Sub->getOperand(N: 1));
5227 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5228 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5229 return new VPExpressionRecipe(ExtA, ExtB, Mul, Sub, Red);
5230 }
5231
5232 llvm_unreachable("Unsupported expression");
5233}
5234
5235// Helper to transform a PartialReductionDescriptor into a partial reduction
5236// recipe. Assumes profitability has been checked.
5237static void transformToPartialReduction(const PartialReductionDescriptor &Link,
5238 VPlan &Plan,
5239 VPReductionPHIRecipe *RdxPhi) {
5240 VPWidenRecipe *WidenRecipe = Link.ReductionBinOp;
5241 assert(WidenRecipe->getNumOperands() == 2 && "Expected binary operation");
5242
5243 VPValue *Accumulator = WidenRecipe->getOperand(N: Link.AccumulatorOpIdx);
5244 auto *ExtendedOp = cast<VPSingleDefRecipe>(
5245 Val: WidenRecipe->getOperand(N: 1 - Link.AccumulatorOpIdx));
5246
5247 // FIXME: Do these transforms before invoking the cost-model.
5248 ExtendedOp = optimizeExtendsForPartialReduction(Op: ExtendedOp);
5249
5250 // Sub-reductions can be implemented in two ways:
5251 // (1) negate the operand in the vector loop (the default way).
5252 // (2) subtract the reduced value from the init value in the middle block.
5253 // Both ways keep the reduction itself as an 'add' reduction.
5254 //
5255 // The ISD nodes for partial reductions don't support folding the
5256 // sub/negation into its operands because the following is not a valid
5257 // transformation:
5258 // sub(0, mul(ext(a), ext(b)))
5259 // -> mul(ext(a), ext(sub(0, b)))
5260 //
5261 // It's therefore better to choose option (2) such that the partial
5262 // reduction is always positive (starting at '0') and to do a final
5263 // subtract in the middle block.
5264 if ((WidenRecipe->getOpcode() == Instruction::Sub &&
5265 Link.RK != RecurKind::Sub) ||
5266 (WidenRecipe->getOpcode() == Instruction::FSub &&
5267 Link.RK != RecurKind::FSub)) {
5268 VPBuilder Builder(WidenRecipe);
5269 Type *ElemTy = ExtendedOp->getScalarType();
5270 VPWidenRecipe *NegRecipe;
5271 if (WidenRecipe->getOpcode() == Instruction::FSub) {
5272 NegRecipe =
5273 new VPWidenRecipe(Instruction::FNeg, {ExtendedOp},
5274 VPIRFlags::getDefaultFlags(Opcode: Instruction::FNeg),
5275 VPIRMetadata(), DebugLoc::getUnknown());
5276 } else {
5277 auto *Zero = Plan.getZero(Ty: ElemTy);
5278 NegRecipe =
5279 new VPWidenRecipe(Instruction::Sub, {Zero, ExtendedOp},
5280 VPIRFlags::getDefaultFlags(Opcode: Instruction::Sub),
5281 VPIRMetadata(), DebugLoc::getUnknown());
5282 }
5283 Builder.insert(R: NegRecipe);
5284 ExtendedOp = NegRecipe;
5285 }
5286
5287 // Check if WidenRecipe is the final result of the reduction. If so, look
5288 // through the Select recipe introduced by tail-folding, otherwise look
5289 // through any Blend recipe introduced by predication for the block.
5290 VPValue *ExitSearch =
5291 Link.Blend ? cast<VPValue>(Val: Link.Blend) : cast<VPValue>(Val: WidenRecipe);
5292
5293 VPValue *Cond = nullptr;
5294 VPValue *ExitValue = cast_or_null<VPInstruction>(
5295 Val: findUserOf(V: ExitSearch, P: m_Select(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: ExitSearch),
5296 Op2: m_Specific(VPV: RdxPhi))));
5297
5298 if (Link.Blend) {
5299 std::optional<unsigned> BlendReductionIdx =
5300 getBlendReductionUpdateValueIdx(Blend: Link.Blend);
5301 assert(BlendReductionIdx &&
5302 Link.Blend->getIncomingValue(*BlendReductionIdx) == WidenRecipe &&
5303 "Expected blend to contain the reduction update");
5304 VPValue *BlendCond = Link.Blend->getMask(Idx: *BlendReductionIdx);
5305 Cond = ExitValue ? VPBuilder(WidenRecipe)
5306 .createLogicalAnd(LHS: Cond, RHS: BlendCond,
5307 DL: WidenRecipe->getDebugLoc())
5308 : BlendCond;
5309 }
5310
5311 // When folding the tail, the inactive lanes of the reduction update are
5312 // computed from values that do not correspond to any scalar iteration
5313 // and must not be accumulated.
5314 if (!Cond)
5315 Cond = Plan.getVectorLoopRegion()->getHeaderMask();
5316
5317 [[maybe_unused]] bool IsLastInChain =
5318 RdxPhi->getBackedgeValue() == WidenRecipe ||
5319 RdxPhi->getBackedgeValue() == ExitValue ||
5320 RdxPhi->getBackedgeValue() == Link.Blend;
5321 assert((!ExitValue || IsLastInChain) &&
5322 "if we found ExitValue, it must match RdxPhi's backedge value");
5323
5324 Type *PhiType = RdxPhi->getScalarType();
5325 RecurKind RdxKind =
5326 PhiType->isFloatingPointTy() ? RecurKind::FAdd : RecurKind::Add;
5327 auto *PartialRed = new VPReductionRecipe(
5328 RdxKind,
5329 RdxKind == RecurKind::FAdd ? WidenRecipe->getFastMathFlagsOrNone()
5330 : FastMathFlags(),
5331 WidenRecipe->getUnderlyingInstr(), Accumulator, ExtendedOp, Cond,
5332 RdxUnordered{/*VFScaleFactor=*/Link.ScaleFactor});
5333 PartialRed->insertBefore(InsertPos: WidenRecipe);
5334
5335 if (ExitValue)
5336 ExitValue->replaceAllUsesWith(New: PartialRed);
5337 if (Link.Blend)
5338 Link.Blend->replaceAllUsesWith(New: PartialRed);
5339 WidenRecipe->replaceAllUsesWith(New: PartialRed);
5340
5341 // For cost-model purposes, fold this into a VPExpression.
5342 VPExpressionRecipe *E = createPartialReductionExpression(Red: PartialRed);
5343 E->insertBefore(InsertPos: WidenRecipe);
5344 PartialRed->replaceAllUsesWith(New: E);
5345}
5346
5347/// Returns the cost of a link in a partial-reduction chain for a given VF.
5348static InstructionCost
5349getPartialReductionLinkCost(VPCostContext &CostCtx,
5350 const PartialReductionDescriptor &Link,
5351 ElementCount VF) {
5352 Type *RdxType = Link.ReductionBinOp->getScalarType();
5353 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
5354 std::optional<unsigned> BinOpc = std::nullopt;
5355 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
5356 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
5357 BinOpc = ExtendedOp.ExtendsUser->getOpcode();
5358
5359 std::optional<llvm::FastMathFlags> Flags;
5360 if (RdxType->isFloatingPointTy())
5361 Flags = Link.ReductionBinOp->getFastMathFlagsOrNone();
5362
5363 auto GetLinkOpcode = [&Link]() -> unsigned {
5364 switch (Link.RK) {
5365 case RecurKind::Sub:
5366 return Instruction::Add;
5367 case RecurKind::FSub:
5368 return Instruction::FAdd;
5369 default:
5370 return Link.ReductionBinOp->getOpcode();
5371 }
5372 };
5373
5374 return CostCtx.TTI.getPartialReductionCost(
5375 Opcode: GetLinkOpcode(), InputTypeA: ExtendedOp.ExtendA.SrcType, InputTypeB: ExtendedOp.ExtendB.SrcType,
5376 AccumType: RdxType, VF, OpAExtend: ExtendedOp.ExtendA.Kind, OpBExtend: ExtendedOp.ExtendB.Kind, BinOp: BinOpc,
5377 CostKind: CostCtx.CostKind, FMF: Flags);
5378}
5379
5380static ExtendKind getPartialReductionExtendKind(VPWidenCastRecipe *Cast) {
5381 return TTI::getPartialReductionExtendKind(CastOpc: Cast->getOpcode());
5382}
5383
5384/// Checks if \p Op (which is an operand of \p UpdateR) is an extended reduction
5385/// operand. This is an operand where the source of the value (e.g. a load) has
5386/// been extended (sext, zext, or fpext) before it is used in the reduction.
5387///
5388/// Possible forms matched by this function:
5389/// - UpdateR(PrevValue, ext(...))
5390/// - UpdateR(PrevValue, mul(ext(...), ext(...)))
5391/// - UpdateR(PrevValue, mul(ext(...), Constant))
5392/// - UpdateR(PrevValue, ext(mul(ext(...), ext(...))))
5393/// - UpdateR(PrevValue, ext(mul(ext(...), Constant)))
5394/// - UpdateR(PrevValue, abs(sub(ext(...), ext(...)))
5395///
5396/// Note: The second operand of UpdateR corresponds to \p Op in the examples.
5397static std::optional<ExtendedReductionOperand>
5398matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op) {
5399 assert(is_contained(UpdateR->operands(), Op) &&
5400 "Op should be operand of UpdateR");
5401
5402 // Try matching an absolute difference operand of the form
5403 // `abs(sub(ext(A), ext(B)))`. This will be later transformed into
5404 // `ext(absolute-difference(A, B))`. This allows us to perform the absolute
5405 // difference on a wider type and get the extend for "free" from the partial
5406 // reduction.
5407 VPValue *X, *Y;
5408 if (Op->hasOneUse() &&
5409 match(V: Op, P: m_WidenIntrinsic<Intrinsic::abs>(
5410 Ops: m_OneUse(SubPattern: m_Sub(Op0: m_WidenAnyExtend(Op0: m_VPValue(V&: X)),
5411 Op1: m_WidenAnyExtend(Op0: m_VPValue(V&: Y))))))) {
5412 auto *Abs = cast<VPWidenIntrinsicRecipe>(Val: Op);
5413 auto *Sub = cast<VPWidenRecipe>(Val: Abs->getOperand(N: 0));
5414 auto *LHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
5415 auto *RHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 1));
5416 Type *LHSInputType = X->getScalarType();
5417 Type *RHSInputType = Y->getScalarType();
5418 if (LHSInputType != RHSInputType ||
5419 LHSExt->getOpcode() != RHSExt->getOpcode())
5420 return std::nullopt;
5421 // Note: This is essentially the same as matching ext(...) as we will
5422 // rewrite this operand to ext(absolute-difference(A, B)).
5423 return ExtendedReductionOperand{
5424 .ExtendsUser: Sub,
5425 /*ExtendA=*/{.SrcType: LHSInputType, .Kind: getPartialReductionExtendKind(Cast: LHSExt)},
5426 /*ExtendB=*/{}};
5427 }
5428
5429 std::optional<TTI::PartialReductionExtendKind> OuterExtKind;
5430 if (match(V: Op, P: m_WidenAnyExtend(Op0: m_VPValue()))) {
5431 auto *CastRecipe = cast<VPWidenCastRecipe>(Val: Op);
5432 VPValue *CastSource = CastRecipe->getOperand(N: 0);
5433 OuterExtKind = getPartialReductionExtendKind(Cast: CastRecipe);
5434 if (match(V: CastSource, P: m_Mul(Op0: m_VPValue(), Op1: m_VPValue())) ||
5435 match(V: CastSource, P: m_FMul(Op0: m_VPValue(), Op1: m_VPValue()))) {
5436 // Match: ext(mul(...))
5437 // Record the outer extend kind and set `Op` to the mul. We can then match
5438 // this as a binary operation. Note: We can optimize out the outer extend
5439 // by widening the inner extends to match it. See
5440 // optimizeExtendsForPartialReduction.
5441 Op = CastSource;
5442 } else {
5443 return ExtendedReductionOperand{
5444 .ExtendsUser: UpdateR,
5445 /*ExtendA=*/{.SrcType: CastSource->getScalarType(), .Kind: *OuterExtKind},
5446 /*ExtendB=*/{}};
5447 }
5448 }
5449
5450 if (!Op->hasOneUse())
5451 return std::nullopt;
5452
5453 VPWidenRecipe *MulOp = dyn_cast<VPWidenRecipe>(Val: Op);
5454 if (!MulOp ||
5455 !is_contained(Set: {Instruction::Mul, Instruction::FMul}, Element: MulOp->getOpcode()))
5456 return std::nullopt;
5457
5458 // The rest of the matching assumes `Op` is a (possibly extended) mul
5459 // operation.
5460
5461 VPValue *LHS = MulOp->getOperand(N: 0);
5462 VPValue *RHS = MulOp->getOperand(N: 1);
5463
5464 // The LHS of the operation must always be an extend.
5465 if (!match(V: LHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
5466 return std::nullopt;
5467
5468 auto *LHSCast = cast<VPWidenCastRecipe>(Val: LHS);
5469 Type *LHSInputType = LHSCast->getOperand(N: 0)->getScalarType();
5470 ExtendKind LHSExtendKind = getPartialReductionExtendKind(Cast: LHSCast);
5471
5472 // The RHS of the operation can be an extend or a constant integer.
5473 const APInt *RHSConst = nullptr;
5474 VPWidenCastRecipe *RHSCast = nullptr;
5475 if (match(V: RHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
5476 RHSCast = cast<VPWidenCastRecipe>(Val: RHS);
5477 else if (!match(V: RHS, P: m_APInt(C&: RHSConst)) ||
5478 !canConstantBeExtended(C: RHSConst, NarrowType: LHSInputType, ExtKind: LHSExtendKind))
5479 return std::nullopt;
5480
5481 // The outer extend kind must match the inner extends for folding.
5482 for (VPWidenCastRecipe *Cast : {LHSCast, RHSCast})
5483 if (Cast && OuterExtKind &&
5484 getPartialReductionExtendKind(Cast) != OuterExtKind)
5485 return std::nullopt;
5486
5487 Type *RHSInputType = LHSInputType;
5488 ExtendKind RHSExtendKind = LHSExtendKind;
5489 if (RHSCast) {
5490 RHSInputType = RHSCast->getOperand(N: 0)->getScalarType();
5491 RHSExtendKind = getPartialReductionExtendKind(Cast: RHSCast);
5492 }
5493
5494 return ExtendedReductionOperand{
5495 .ExtendsUser: MulOp, .ExtendA: {.SrcType: LHSInputType, .Kind: LHSExtendKind}, .ExtendB: {.SrcType: RHSInputType, .Kind: RHSExtendKind}};
5496}
5497
5498/// Examines each operation in the reduction chain corresponding to \p RedPhiR,
5499/// and determines if the target can use a cheaper operation with a wider
5500/// per-iteration input VF and narrower PHI VF. If successful, returns the chain
5501/// of partial reduction descriptors (that are links in the reduction chain).
5502static std::optional<SmallVector<PartialReductionDescriptor>>
5503getScaledReductionChain(VPReductionPHIRecipe *RedPhiR) {
5504 // Get the backedge value from the reduction PHI and find the
5505 // ComputeReductionResult that uses it (directly or through a select for
5506 // predicated reductions).
5507 auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR);
5508 if (!RdxResult)
5509 return std::nullopt;
5510 VPValue *ExitValue = RdxResult->getOperand(N: 0);
5511 match(V: ExitValue, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: ExitValue), Op2: m_VPValue()));
5512
5513 SmallVector<PartialReductionDescriptor> Chain;
5514 RecurKind RK = RedPhiR->getRecurrenceKind();
5515 Type *PhiType = RedPhiR->getScalarType();
5516 TypeSize PHISize = PhiType->getPrimitiveSizeInBits();
5517
5518 // Work backwards from the ExitValue examining each reduction operation.
5519 VPValue *CurrentValue = ExitValue;
5520 while (CurrentValue != RedPhiR) {
5521 VPBlendRecipe *Blend = dyn_cast<VPBlendRecipe>(Val: CurrentValue);
5522 std::optional<unsigned> BlendReductionIdx;
5523 if (Blend) {
5524 assert(!Blend->isNormalized() && "Expect Blend not to be normalized.");
5525 if (Blend->getNumIncomingValues() != 2)
5526 return std::nullopt;
5527
5528 BlendReductionIdx = getBlendReductionUpdateValueIdx(Blend);
5529 if (!BlendReductionIdx)
5530 return std::nullopt;
5531
5532 CurrentValue = Blend->getIncomingValue(Idx: *BlendReductionIdx);
5533 }
5534
5535 auto *UpdateR = dyn_cast<VPWidenRecipe>(Val: CurrentValue);
5536 if (!UpdateR || !Instruction::isBinaryOp(Opcode: UpdateR->getOpcode()))
5537 return std::nullopt;
5538
5539 VPValue *Op = UpdateR->getOperand(N: 1);
5540 VPValue *PrevValue = UpdateR->getOperand(N: 0);
5541
5542 // Find the extended operand. The other operand (PrevValue) is the next link
5543 // in the reduction chain.
5544 std::optional<ExtendedReductionOperand> ExtendedOp =
5545 matchExtendedReductionOperand(UpdateR, Op);
5546 if (!ExtendedOp) {
5547 ExtendedOp = matchExtendedReductionOperand(UpdateR, Op: PrevValue);
5548 if (!ExtendedOp)
5549 return std::nullopt;
5550 std::swap(a&: Op, b&: PrevValue);
5551 }
5552
5553 // Look for VPBlend(reduce(PrevValue, Op), PrevValue), where
5554 // reduce is equal to CurrentValue. This can be lowered as
5555 // a conditional reduction by hoisting the select to the inputs.
5556 if (Blend && Blend->getIncomingValue(Idx: 1 - *BlendReductionIdx) != PrevValue)
5557 return std::nullopt;
5558
5559 Type *ExtSrcType = ExtendedOp->ExtendA.SrcType;
5560 TypeSize ExtSrcSize = ExtSrcType->getPrimitiveSizeInBits();
5561 if (!PHISize.hasKnownScalarFactor(RHS: ExtSrcSize))
5562 return std::nullopt;
5563
5564 PartialReductionDescriptor Link(
5565 {.ReductionBinOp: UpdateR, .ExtendedOp: *ExtendedOp, .RK: RK,
5566 .AccumulatorOpIdx: PrevValue == UpdateR->getOperand(N: 0) ? 0U : 1U,
5567 .ScaleFactor: static_cast<unsigned>(PHISize.getKnownScalarFactor(RHS: ExtSrcSize)),
5568 .Blend: Blend});
5569 Chain.push_back(Elt: Link);
5570 CurrentValue = PrevValue;
5571 }
5572
5573 // The chain links were collected by traversing backwards from the exit value.
5574 // Reverse the chain so the links follow program order.
5575 std::reverse(first: Chain.begin(), last: Chain.end());
5576 return Chain;
5577}
5578} // namespace
5579
5580// Scale the PHI and ReductionStartVector by \p Factor and if the recurrence is
5581// a sub-recurrence, negate the reduction result.
5582static void updatePartialReductionPhiAndResult(VPlan &Plan,
5583 VPReductionPHIRecipe *Phi,
5584 unsigned Factor, RecurKind RK) {
5585 assert(Phi->getVFScaleFactor() == 1 && "scale factor must not be set");
5586 Phi->setVFScaleFactor(Factor);
5587
5588 auto *StartInst = cast<VPInstruction>(Val: Phi->getStartValue());
5589 assert(StartInst->getOpcode() == VPInstruction::ReductionStartVector);
5590 auto *NewScaleFactor = Plan.getConstantInt(BitWidth: 32, Val: Factor);
5591 StartInst->setOperand(I: 2, New: NewScaleFactor);
5592
5593 if (RK != RecurKind::Sub && RK != RecurKind::FSub)
5594 return;
5595
5596 // Update the PHI node to start at `0` and update the reduction-result
5597 // to subtract from the PHI's start value.
5598 VPValue *OldStartValue = StartInst->getOperand(N: 0);
5599 StartInst->setOperand(I: 0, New: StartInst->getOperand(N: 1));
5600
5601 // Replace reduction_result by 'sub (startval, reductionresult)'.
5602 VPInstruction *RdxResult = vputils::findComputeReductionResult(PhiR: Phi);
5603 assert(RdxResult && "Could not find reduction result");
5604
5605 VPBuilder Builder = VPBuilder::getToInsertAfter(R: RdxResult);
5606 unsigned SubOpc = RK == RecurKind::FSub ? Instruction::BinaryOps::FSub
5607 : Instruction::BinaryOps::Sub;
5608 VPInstruction *NewResult = Builder.createNaryOp(
5609 Opcode: SubOpc, Operands: {OldStartValue, RdxResult}, Flags: VPIRFlags::getDefaultFlags(Opcode: SubOpc),
5610 DL: Phi->getDebugLoc());
5611 RdxResult->replaceUsesWithIf(
5612 New: NewResult, ShouldReplace: [&NewResult](VPUser &U) { return &U != NewResult; });
5613}
5614
5615void VPlanTransforms::createPartialReductions(VPlan &Plan,
5616 VPCostContext &CostCtx,
5617 VFRange &Range) {
5618 // Find all possible valid partial reductions, grouping chains by their PHI.
5619 // This grouping allows invalidating the whole chain, if any link is not a
5620 // valid partial reduction.
5621 MapVector<VPReductionPHIRecipe *, SmallVector<PartialReductionDescriptor>>
5622 PhiToChain;
5623 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
5624 SmallVector<VPReductionPHIRecipe *, 4> UnorderedReductions;
5625 for (VPReductionPHIRecipe &RedPhiR :
5626 make_isa_range<VPReductionPHIRecipe>(Range: HeaderVPBB->phis())) {
5627 if (auto Chain = getScaledReductionChain(RedPhiR: &RedPhiR))
5628 PhiToChain.try_emplace(Key: &RedPhiR, Args: std::move(*Chain));
5629 else if (UsePartialReductionsByDefault &&
5630 (RedPhiR.getRecurrenceKind() == RecurKind::Add ||
5631 (RedPhiR.getRecurrenceKind() == RecurKind::FAdd &&
5632 !RedPhiR.isOrdered() && !RedPhiR.isInLoop())))
5633 UnorderedReductions.push_back(Elt: &RedPhiR);
5634 }
5635
5636 // For general unordered reductions which aren't part of a candidate chain for
5637 // a scaled partial reduction, we can still use the intrinsic to allow for
5638 // more optimization later on.
5639 for (auto *Rdx : UnorderedReductions) {
5640 auto *Backedge = dyn_cast<VPWidenRecipe>(Val: Rdx->getBackedgeValue());
5641 VPValue *OtherOp;
5642 if (!Backedge ||
5643 !match(V: Backedge,
5644 P: m_CombineOr(Ps: m_c_FAdd(Op0: m_Specific(VPV: Rdx), Op1: m_VPValue(V&: OtherOp)),
5645 Ps: m_c_Add(Op0: m_Specific(VPV: Rdx), Op1: m_VPValue(V&: OtherOp)))))
5646 continue;
5647
5648 // If the target indicates that the intrinsic is as cheap as (or cheaper
5649 // than) the add, then prefer the intrinsic.
5650 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
5651 Predicate: [&CostCtx, Rdx, Backedge](ElementCount VF) {
5652 InstructionCost CurrentCost = Backedge->computeCost(VF, Ctx&: CostCtx);
5653 Type *ScalarTy = Backedge->getScalarType();
5654 auto FMF = ScalarTy->isFloatingPointTy()
5655 ? std::make_optional(t: Rdx->getFastMathFlagsOrNone())
5656 : std::nullopt;
5657
5658 InstructionCost PRCost = CostCtx.TTI.getPartialReductionCost(
5659 Opcode: Backedge->getOpcode(), InputTypeA: ScalarTy, /*InputTypeB=*/nullptr,
5660 AccumType: ScalarTy, VF, OpAExtend: TTI::PR_None, OpBExtend: TTI::PR_None,
5661 /*BinOp=*/std::nullopt, CostKind: CostCtx.CostKind, FMF);
5662 return PRCost <= CurrentCost;
5663 },
5664 Range))
5665 continue;
5666
5667 auto *Partial = new VPReductionRecipe(
5668 Rdx->getRecurrenceKind(), Rdx->getFastMathFlagsOrNone(),
5669 Backedge->getUnderlyingInstr(), Rdx, OtherOp, nullptr,
5670 getReductionStyle(/*InLoop=*/false, /*Ordered=*/false,
5671 /*ScaleFactor=*/1));
5672 Partial->insertBefore(InsertPos: Backedge);
5673 Backedge->replaceAllUsesWith(New: Partial);
5674 Backedge->eraseFromParent();
5675 }
5676
5677 if (PhiToChain.empty())
5678 return;
5679
5680 // Build set of partial reduction operations and blends for user validation
5681 // and a map of reduction bin ops to their scale factors for scale validation.
5682 SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
5683 SmallPtrSet<VPBlendRecipe *, 4> PartialReductionBlends;
5684 DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
5685 for (auto &[_, Chain] : PhiToChain)
5686 for (const PartialReductionDescriptor &Link : Chain) {
5687 PartialReductionOps.insert(Ptr: Link.ExtendedOp.ExtendsUser);
5688 if (Link.Blend)
5689 PartialReductionBlends.insert(Ptr: Link.Blend);
5690 ScaledReductionMap[Link.ReductionBinOp] = Link.ScaleFactor;
5691 }
5692
5693 // A partial reduction is invalid if any of its extends are used by
5694 // something that isn't another partial reduction. This is because the
5695 // extends are intended to be lowered along with the reduction itself.
5696 auto ExtendUsersValid = [&](VPValue *Ext) {
5697 return !isa<VPWidenCastRecipe>(Val: Ext) || all_of(Range: Ext->users(), P: [&](VPUser *U) {
5698 return PartialReductionOps.contains(Ptr: cast<VPRecipeBase>(Val: U));
5699 });
5700 };
5701
5702 auto IsProfitablePartialReductionChainForVF =
5703 [&](ArrayRef<PartialReductionDescriptor> Chain, ElementCount VF) -> bool {
5704 InstructionCost PartialCost = 0, RegularCost = 0;
5705
5706 // The chain is a profitable partial reduction chain if the cost of handling
5707 // the entire chain is cheaper when using partial reductions than when
5708 // handling the entire chain using regular reductions.
5709 for (const PartialReductionDescriptor &Link : Chain) {
5710 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
5711 InstructionCost LinkCost = getPartialReductionLinkCost(CostCtx, Link, VF);
5712 if (!LinkCost.isValid())
5713 return false;
5714
5715 PartialCost += LinkCost;
5716 RegularCost += Link.ReductionBinOp->computeCost(VF, Ctx&: CostCtx);
5717 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
5718 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
5719 RegularCost += ExtendedOp.ExtendsUser->computeCost(VF, Ctx&: CostCtx);
5720 for (VPValue *Op : ExtendedOp.ExtendsUser->operands())
5721 if (auto *Extend = dyn_cast<VPWidenCastRecipe>(Val: Op))
5722 RegularCost += Extend->computeCost(VF, Ctx&: CostCtx);
5723 }
5724 return PartialCost.isValid() && PartialCost < RegularCost;
5725 };
5726
5727 // Validate chains: check that extends are only used by partial reductions,
5728 // and that reduction bin ops are only used by other partial reductions with
5729 // matching scale factors, are outside the loop region or the select
5730 // introduced by tail-folding. Otherwise we would create users of scaled
5731 // reductions where the types of the other operands don't match.
5732 for (auto &[RedPhiR, Chain] : PhiToChain) {
5733 for (const PartialReductionDescriptor &Link : Chain) {
5734 if (!all_of(Range: Link.ExtendedOp.ExtendsUser->operands(), P: ExtendUsersValid)) {
5735 Chain.clear();
5736 break;
5737 }
5738 auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
5739 if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: U))
5740 return PhiR == RedPhiR;
5741 auto *R = cast<VPSingleDefRecipe>(Val: U);
5742
5743 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: R))
5744 return Blend == Link.Blend || PartialReductionBlends.contains(Ptr: Blend);
5745
5746 return Link.ScaleFactor == ScaledReductionMap.lookup_or(Val: R, Default: 0) ||
5747 match(R, P: m_ComputeReductionResult(
5748 Op0: m_Specific(VPV: Link.ReductionBinOp))) ||
5749 match(R, P: m_Select(Op0: m_VPValue(), Op1: m_Specific(VPV: Link.ReductionBinOp),
5750 Op2: m_Specific(VPV: RedPhiR)));
5751 };
5752 if (!all_of(Range: Link.ReductionBinOp->users(), P: UseIsValid)) {
5753 Chain.clear();
5754 break;
5755 }
5756
5757 // Check if the compute-reduction-result is used by a sunk store.
5758 // TODO: Also form partial reductions in those cases.
5759 if (auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR)) {
5760 if (any_of(Range: RdxResult->users(), P: [](VPUser *U) {
5761 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
5762 return RepR && RepR->getOpcode() == Instruction::Store;
5763 })) {
5764 Chain.clear();
5765 break;
5766 }
5767 }
5768 }
5769
5770 // Clear the chain if it is not profitable.
5771 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
5772 Predicate: [&, &Chain = Chain](ElementCount VF) {
5773 return IsProfitablePartialReductionChainForVF(Chain, VF);
5774 },
5775 Range))
5776 Chain.clear();
5777 }
5778
5779 for (auto &[Phi, Chain] : PhiToChain) {
5780 if (Chain.empty())
5781 continue;
5782
5783 for (const PartialReductionDescriptor &Link : Chain)
5784 transformToPartialReduction(Link, Plan, RdxPhi: Phi);
5785
5786 // After transforming all links in the chain, the PHI node and result need
5787 // updating. Note that we can pick any link in the chain for this, as the
5788 // ScaleFactor and RecurKind must match for all links in the chain.
5789 const PartialReductionDescriptor &Link = Chain[0];
5790 updatePartialReductionPhiAndResult(Plan, Phi, Factor: Link.ScaleFactor, RK: Link.RK);
5791 }
5792}
5793
5794void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
5795 VPRecipeBuilder &RecipeBuilder,
5796 VPCostContext &CostCtx) {
5797 // Collect all loads/stores first. We will start with ones having simpler
5798 // decisions followed by more complex ones that are potentially
5799 // guided/dependent on the simpler ones.
5800 SmallVector<VPInstruction *> MemOps;
5801 for (VPBasicBlock *VPBB :
5802 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
5803 G: Plan.getVectorLoopRegion()->getEntryBasicBlock()))) {
5804 for (VPInstruction &VPI : make_isa_range<VPInstruction>(Range&: *VPBB)) {
5805 if (VPI.getUnderlyingValue() &&
5806 is_contained(Set: {Instruction::Load, Instruction::Store},
5807 Element: VPI.getOpcode()))
5808 MemOps.push_back(Elt: &VPI);
5809 }
5810 }
5811
5812 // Few helpers to process different kinds of memory operations.
5813
5814 // To be used as argument to `VPlanTransforms::runPass` which explicitly
5815 // specified pass name, hence `VPlan &` parameter.
5816 auto ProcessSubset = [&](VPlan &, auto ProcessVPInst) {
5817 SmallVector<VPInstruction *> RemainingMemOps;
5818 for (VPInstruction *VPI : MemOps) {
5819 if (!ProcessVPInst(VPI))
5820 RemainingMemOps.push_back(Elt: VPI);
5821 }
5822
5823 MemOps.clear();
5824 std::swap(LHS&: MemOps, RHS&: RemainingMemOps);
5825 };
5826
5827 auto ReplaceWith = [&](VPInstruction *VPI, VPRecipeBase *New) {
5828 assert(New->getParent() && "New recipe must have been inserted");
5829 if (VPI->getOpcode() == Instruction::Load)
5830 VPI->replaceAllUsesWith(New: New->getVPSingleValue());
5831 VPI->eraseFromParent();
5832
5833 // VPI has been processed.
5834 return true;
5835 };
5836
5837 auto Scalarize = [&](VPInstruction *VPI) {
5838 return ReplaceWith(VPI, VPBuilder(VPI).insert(
5839 R: RecipeBuilder.handleReplication(VPI, Range)));
5840 };
5841
5842 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
5843 VPBuilder FinalRedStoresBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
5844 VPlanTransforms::runPass(
5845 PassName: "lowerMemoryIdioms", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5846 if (RecipeBuilder.replaceWithFinalIfReductionStore(
5847 VPI, FinalRedStoresBuilder))
5848 return true;
5849
5850 // Filter out scalar VPlan for the remaining idioms.
5851 if (LoopVectorizationPlanner::getDecisionAndClampRange(
5852 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
5853 return false;
5854
5855 if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI))
5856 return ReplaceWith(VPI, VPBuilder(VPI).insert(R: Histogram));
5857
5858 return false;
5859 });
5860
5861 // Filter out scalar VPlan for the remaining memory operations.
5862 if (LoopVectorizationPlanner::getDecisionAndClampRange(
5863 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
5864 return;
5865
5866 // If the instruction's allocated size doesn't equal it's type size, it
5867 // requires padding and will be scalarized.
5868 VPlanTransforms::runPass(
5869 PassName: "scalarizeMemOpsWithIrregularTypes", Pass&: ProcessSubset, Plan,
5870 Args: [&](VPInstruction *VPI) {
5871 Instruction *I = VPI->getUnderlyingInstr();
5872 if (hasIrregularType(Ty: getLoadStoreType(I), DL: I->getDataLayout()))
5873 return Scalarize(VPI);
5874
5875 return false;
5876 });
5877
5878 if (!RecipeBuilder.prefersVectorizedAddressing()) {
5879 VPlanTransforms::runPass(
5880 PassName: "makeVPlanMemOpDecision", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5881 Instruction *I = VPI->getUnderlyingInstr();
5882 bool IsLoad = VPI->getOpcode() == Instruction::Load;
5883 if (RecipeBuilder.isPredicatedInst(I) || !IsLoad ||
5884 !vputils::isUsedByLoadStoreAddress(V: VPI))
5885 return false;
5886
5887 // Scalarize loads used as addresses, matching the legacy CM. The load
5888 // is single-scalar if the pointer is loop-invariant, otherwise it is
5889 // replicated per-lane. No mask is needed as the load is not
5890 // predicated.
5891 VPValue *Ptr = VPI->getOperand(N: 0);
5892 const SCEV *PtrSCEV =
5893 vputils::getSCEVExprForVPValue(V: Ptr, PSE&: CostCtx.PSE, L: CostCtx.L);
5894 bool IsSingleScalarLoad =
5895 !isa<SCEVCouldNotCompute>(Val: PtrSCEV) &&
5896 CostCtx.PSE.getSE()->isLoopInvariant(S: PtrSCEV, L: CostCtx.L);
5897
5898 ReplaceWith(VPI,
5899 VPBuilder(VPI).insert(R: new VPReplicateRecipe(
5900 I, Ptr, /*IsSingleScalar=*/IsSingleScalarLoad,
5901 /*Mask=*/nullptr, *VPI, *VPI, VPI->getDebugLoc())));
5902 return true;
5903 });
5904 }
5905
5906 // Widen unit-stride consecutive accesses, matching the legacy CM. Both
5907 // forward (stride +1) and reverse (stride -1) accesses are handled.
5908 VPlanTransforms::runPass(
5909 PassName: "widenConsecutiveMemOps", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5910 Instruction *I = VPI->getUnderlyingInstr();
5911 bool IsLoad = VPI->getOpcode() == Instruction::Load;
5912 VPValue *Ptr = VPI->getOperand(N: !IsLoad);
5913 Type *ScalarTy =
5914 IsLoad ? VPI->getScalarType() : VPI->getOperand(N: 0)->getScalarType();
5915 std::optional<int64_t> Stride =
5916 vputils::getConstantStride(Addr: Ptr, AccessTy: ScalarTy, PSE&: CostCtx.PSE, L: CostCtx.L);
5917 if (Stride != 1 && Stride != -1)
5918 return false;
5919 bool Reverse = Stride == -1;
5920
5921 // A predicated access can only be widened (rather than scalarized) if
5922 // the target supports a masked load/store for it.
5923 // TODO: Determine if a load/store needs predication directly in VPlan.
5924 bool IsPredicated = RecipeBuilder.isPredicatedInst(I);
5925 if (IsPredicated && !CostCtx.Config.isLegalMaskedLoadOrStore(
5926 IsLoad, ScalarTy, Alignment: getLoadStoreAlignment(I),
5927 AddressSpace: getLoadStoreAddressSpace(I)))
5928 return false;
5929
5930 VPBuilder Builder(VPI);
5931 VPSingleDefRecipe *VectorPtr = Builder.createConsecutiveVectorPointer(
5932 Ptr, SourceElementTy: ScalarTy, Reverse, DL: VPI->getDebugLoc());
5933
5934 VPValue *Mask = IsPredicated ? VPI->getMask() : nullptr;
5935 // Reverse the mask so it matches the reversed access order.
5936 if (Reverse && Mask)
5937 Mask = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Mask,
5938 DL: VPI->getDebugLoc());
5939
5940 if (IsLoad) {
5941 VPSingleDefRecipe *Load = Builder.createWidenLoad(
5942 Load&: *cast<LoadInst>(Val: I), Addr: VectorPtr, Mask,
5943 /*Consecutive=*/true, Metadata: *VPI, DL: VPI->getDebugLoc());
5944 // Reverse the loaded values back into program order.
5945 if (Reverse)
5946 Load = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Load,
5947 DL: VPI->getDebugLoc());
5948 return ReplaceWith(VPI, Load);
5949 }
5950
5951 VPValue *StoredVal = VPI->getOperand(N: 0);
5952 if (Reverse)
5953 // Reverse the stored values so they are written in descending order.
5954 StoredVal = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: StoredVal,
5955 DL: VPI->getDebugLoc());
5956
5957 auto *StoreR = Builder.createWidenStore(
5958 Store&: *cast<StoreInst>(Val: I), Addr: VectorPtr, StoredVal, Mask,
5959 /*Consecutive=*/true, Metadata: *VPI, DL: VPI->getDebugLoc());
5960 return ReplaceWith(VPI, StoreR);
5961 });
5962
5963 VPlanTransforms::runPass(PassName: "delegateMemOpWideningToLegacyCM", Pass&: ProcessSubset,
5964 Plan, Args: [&](VPInstruction *VPI) {
5965 if (VPRecipeBase *Recipe =
5966 RecipeBuilder.tryToWidenMemory(VPI, Range))
5967 return ReplaceWith(VPI, Recipe);
5968
5969 return Scalarize(VPI);
5970 });
5971}
5972
5973void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
5974 if (LoopVectorizationPlanner::getDecisionAndClampRange(
5975 Predicate: [&](ElementCount VF) { return VF.isScalar(); }, Range))
5976 return;
5977
5978 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
5979 Plan.getEntry());
5980 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
5981 for (VPInstruction &VPI :
5982 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range: reverse(C&: *VPBB)))) {
5983 auto *I = cast_or_null<Instruction>(Val: VPI.getUnderlyingValue());
5984 // Wouldn't be able to create a `VPReplicateRecipe` anyway.
5985 if (!I)
5986 continue;
5987
5988 // If executing other lanes produces side-effects we can't avoid them.
5989 if (VPI.mayHaveSideEffects())
5990 continue;
5991
5992 // We want to drop the mask operand, verify we can safely do that.
5993 if (VPI.isMasked() && !VPI.isSafeToSpeculativelyExecute())
5994 continue;
5995
5996 // Avoid rewriting IV increment as that interferes with
5997 // `removeRedundantCanonicalIVs`.
5998 if (VPI.getOpcode() == Instruction::Add &&
5999 any_of(Range: VPI.operands(), P: IsaPred<VPWidenIntOrFpInductionRecipe>))
6000 continue;
6001
6002 // Other lanes are needed - can't drop them.
6003 if (!vputils::onlyFirstLaneUsed(Def: &VPI))
6004 continue;
6005
6006 auto *Recipe = VPBuilder::createSingleScalarOp(
6007 Opcode: VPI.getOpcode(), Operands: VPI.operandsWithoutMask(), /*Mask=*/nullptr, Flags: VPI,
6008 Metadata: VPI, DL: VPI.getDebugLoc(), ResultTy: VPI.getScalarType(), UV: I);
6009 Recipe->insertBefore(InsertPos: &VPI);
6010 VPI.replaceAllUsesWith(New: Recipe);
6011 VPI.eraseFromParent();
6012 }
6013 }
6014}
6015
6016/// Returns true if \p Info's parameter kinds are compatible with \p Args.
6017static bool areVFParamsOk(const VFInfo &Info, ArrayRef<VPValue *> Args,
6018 PredicatedScalarEvolution &PSE, const Loop *L) {
6019 ScalarEvolution *SE = PSE.getSE();
6020 return all_of(Range: Info.Shape.Parameters, P: [&](VFParameter Param) {
6021 switch (Param.ParamKind) {
6022 case VFParamKind::Vector:
6023 case VFParamKind::GlobalPredicate:
6024 return true;
6025 case VFParamKind::OMP_Uniform:
6026 return SE->isSCEVable(Ty: Args[Param.ParamPos]->getScalarType()) &&
6027 SE->isLoopInvariant(
6028 S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
6029 L);
6030 case VFParamKind::OMP_Linear:
6031 return match(S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
6032 P: m_scev_AffineAddRec(
6033 Op0: m_SCEV(), Op1: m_scev_SpecificSInt(V: Param.LinearStepOrPos),
6034 L: m_SpecificLoop(L)));
6035 default:
6036 return false;
6037 }
6038 });
6039}
6040
6041/// Find a vector variant of \p CI for \p VF, respecting \p MaskRequired.
6042/// Returns the variant function, or nullptr. Masked variants are assumed to
6043/// take the mask as a trailing parameter.
6044static Function *findVectorVariant(CallInst *CI, ArrayRef<VPValue *> Args,
6045 ElementCount VF, bool MaskRequired,
6046 PredicatedScalarEvolution &PSE,
6047 const Loop *L) {
6048 if (CI->isNoBuiltin())
6049 return nullptr;
6050 auto Mappings = VFDatabase::getMappings(CI: *CI);
6051 const auto *It = find_if(Range&: Mappings, P: [&](const VFInfo &Info) {
6052 return Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()) &&
6053 areVFParamsOk(Info, Args, PSE, L);
6054 });
6055 if (It == Mappings.end())
6056 return nullptr;
6057 return CI->getModule()->getFunction(Name: It->VectorName);
6058}
6059
6060namespace {
6061/// The outcome of choosing how to widen a call at a given VF.
6062struct CallWideningDecision {
6063 enum class KindTy { Scalarize, Intrinsic, VectorVariant };
6064 CallWideningDecision(KindTy Kind, Function *Variant = nullptr)
6065 : Kind(Kind), Variant(Variant) {}
6066 KindTy Kind;
6067
6068 /// Set when Kind == VectorVariant.
6069 Function *Variant;
6070
6071 bool operator==(const CallWideningDecision &Other) const {
6072 return Kind == Other.Kind && Variant == Other.Variant;
6073 }
6074};
6075} // namespace
6076
6077/// Pick the cheapest widening for the call \p VPI at \p VF among scalarization,
6078/// vector intrinsic, and vector library variant.
6079static CallWideningDecision decideCallWidening(VPInstruction &VPI,
6080 ArrayRef<VPValue *> Ops,
6081 ElementCount VF,
6082 VPCostContext &CostCtx) {
6083 auto *CI = cast<CallInst>(Val: VPI.getUnderlyingInstr());
6084
6085 // Scalar VFs and calls forced or known to scalarize always replicate.
6086 if (VF.isScalar() || CostCtx.willBeScalarized(I: CI, VF))
6087 return CallWideningDecision::KindTy::Scalarize;
6088
6089 auto *CalledFn = cast<Function>(
6090 Val: VPI.getOperand(N: VPI.getNumOperandsWithoutMask() - 1)->getLiveInIRValue());
6091 Type *ResultTy = VPI.getScalarType();
6092 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
6093 bool MaskRequired = CostCtx.isMaskRequired(I: CI);
6094
6095 // Pseudo intrinsics (assume, lifetime, ...) are always scalarized.
6096 if (ID && VPCostContext::isFreeScalarIntrinsic(ID))
6097 return CallWideningDecision::KindTy::Scalarize;
6098
6099 InstructionCost ScalarCost =
6100 VPReplicateRecipe::computeCallCost(CalledFn, ResultTy, ArgOps: Ops,
6101 /*IsSingleScalar=*/false, VF, Ctx&: CostCtx);
6102
6103 Function *VecFunc =
6104 findVectorVariant(CI, Args: Ops, VF, MaskRequired, PSE&: CostCtx.PSE, L: CostCtx.L);
6105 InstructionCost VecCallCost = InstructionCost::getInvalid();
6106 if (VecFunc)
6107 VecCallCost = VPWidenCallRecipe::computeCallCost(Variant: VecFunc, Ctx&: CostCtx);
6108
6109 // Prefer the intrinsic if it is at least as cheap as scalarizing and any
6110 // available vector variant.
6111 if (ID) {
6112 InstructionCost IntrinsicCost =
6113 VPWidenIntrinsicRecipe::computeCallCost(ID, Operands: Ops, R: VPI, VF, Ctx&: CostCtx);
6114 if (IntrinsicCost.isValid() && ScalarCost >= IntrinsicCost &&
6115 (!VecFunc || VecCallCost >= IntrinsicCost))
6116 return CallWideningDecision::KindTy::Intrinsic;
6117 }
6118
6119 // Otherwise, use a vector library variant when it beats scalarizing.
6120 if (VecFunc && ScalarCost >= VecCallCost)
6121 return {CallWideningDecision::KindTy::VectorVariant, VecFunc};
6122
6123 return CallWideningDecision::KindTy::Scalarize;
6124}
6125
6126bool VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
6127 VPRecipeBuilder &RecipeBuilder,
6128 VPCostContext &CostCtx) {
6129 bool Widened = false;
6130 for (VPBasicBlock *VPBB : VPBlockUtils::blocksAs<VPBasicBlock>(
6131 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
6132 for (VPInstruction &VPI :
6133 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range&: *VPBB))) {
6134 if (!VPI.getUnderlyingValue() || VPI.getOpcode() != Instruction::Call)
6135 continue;
6136
6137 auto *CI = cast<CallInst>(Val: VPI.getUnderlyingInstr());
6138 SmallVector<VPValue *, 4> Ops(VPI.op_begin(),
6139 VPI.op_begin() + CI->arg_size());
6140
6141 CallWideningDecision Decision =
6142 decideCallWidening(VPI, Ops, VF: Range.Start, CostCtx);
6143 LoopVectorizationPlanner::getDecisionAndClampRange(
6144 Predicate: [&](ElementCount VF) {
6145 return Decision == decideCallWidening(VPI, Ops, VF, CostCtx);
6146 },
6147 Range);
6148
6149 VPSingleDefRecipe *Replacement = nullptr;
6150 switch (Decision.Kind) {
6151 case CallWideningDecision::KindTy::Intrinsic: {
6152 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
6153 Type *ResultTy = VPI.getScalarType();
6154 Replacement = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, VPI,
6155 VPI, VPI.getDebugLoc());
6156 Widened = true;
6157 break;
6158 }
6159 case CallWideningDecision::KindTy::VectorVariant: {
6160 // Masked variants take the mask as a trailing parameter, so they have
6161 // one more parameter than the original call's arguments.
6162 if (Decision.Variant->arg_size() > Ops.size()) {
6163 VPValue *Mask = VPI.isMasked() ? VPI.getMask() : Plan.getTrue();
6164 Ops.push_back(Elt: Mask);
6165 }
6166 Ops.push_back(Elt: VPI.getOperand(N: VPI.getNumOperandsWithoutMask() - 1));
6167 Replacement = new VPWidenCallRecipe(CI, Decision.Variant, Ops, VPI, VPI,
6168 VPI.getDebugLoc());
6169 Widened = true;
6170 break;
6171 }
6172 case CallWideningDecision::KindTy::Scalarize:
6173 Replacement = RecipeBuilder.handleReplication(VPI: &VPI, Range);
6174 break;
6175 }
6176
6177 Replacement->insertBefore(InsertPos: &VPI);
6178 VPI.replaceAllUsesWith(New: Replacement);
6179 VPI.eraseFromParent();
6180 }
6181 }
6182 return Widened;
6183}
6184
6185void VPlanTransforms::narrowInductionTruncates(VPlan &Plan, VFRange &Range,
6186 const TargetTransformInfo &TTI,
6187 PredicatedScalarEvolution &PSE) {
6188 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
6189 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
6190 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6191 Range: vp_depth_first_shallow(G: LoopRegion->getEntry()))) {
6192 for (VPInstruction &VPI :
6193 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range&: *VPBB))) {
6194 // Only truncates are handled, as sext/zext may wrap, FP conversions lose
6195 // precision and other casts depend on the pointer size.
6196 if (VPI.getOpcode() != Instruction::Trunc)
6197 continue;
6198
6199 // Underlying Trunc is necessary to create VPWidenIntOrFpInductionRecipe.
6200 auto *Trunc = cast_or_null<TruncInst>(Val: VPI.getUnderlyingValue());
6201 if (!Trunc)
6202 continue;
6203
6204 // A truncate that is not widened is left to the scalarization decisions
6205 // made earlier.
6206 if (vputils::onlyFirstLaneUsed(Def: &VPI))
6207 continue;
6208
6209 VPValue *Op = VPI.getOperand(N: 0);
6210 auto *WideIV = getOptimizableIVOf(VPV: Op, PSE);
6211 if (!WideIV)
6212 continue;
6213
6214 // getOptimizableIVOf also matches an add of the IV and its step, which
6215 // is not handled here.
6216 // TODO: Also narrow truncates of the incremented IV.
6217 if (Op != WideIV)
6218 continue;
6219
6220 // Replacing a free truncate would add an induction update instruction to
6221 // each iteration of the loop. The canonical induction is exempt, as it
6222 // needs an update instruction regardless.
6223 auto IsNarrowingProfitable = [&](ElementCount VF) {
6224 return match(V: WideIV, P: m_CanonicalWidenIV()) ||
6225 !TTI.isTruncateFree(
6226 Ty1: toVectorTy(Scalar: VPI.getOperand(N: 0)->getScalarType(), EC: VF),
6227 Ty2: toVectorTy(Scalar: VPI.getScalarType(), EC: VF));
6228 };
6229 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
6230 Predicate: IsNarrowingProfitable, Range))
6231 continue;
6232
6233 // Wrap flags of the original induction do not hold in the truncated
6234 // type, so do not propagate them.
6235 auto *NarrowIV = new VPWidenIntOrFpInductionRecipe(
6236 WideIV->getPHINode(), WideIV->getStartValue(), WideIV->getStepValue(),
6237 WideIV->getVFValue(), WideIV->getInductionDescriptor(), Trunc,
6238 VPIRFlags::WrapFlagsTy(false, false), VPI.getDebugLoc());
6239 NarrowIV->insertBefore(BB&: *HeaderVPBB, IP: HeaderVPBB->getFirstNonPhi());
6240 VPI.replaceAllUsesWith(New: NarrowIV);
6241 VPI.eraseFromParent();
6242 }
6243 }
6244}
6245
6246void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
6247 PredicatedScalarEvolution &PSE,
6248 Loop &L, VPCostContext &Ctx,
6249 VFRange &Range) {
6250 if (Plan.hasScalarVFOnly())
6251 return;
6252
6253 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
6254 VPValue *I32VF = nullptr;
6255 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6256 Range: vp_depth_first_shallow(G: VectorLoop->getEntry()))) {
6257 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
6258 auto *MemR = dyn_cast<VPWidenMemoryRecipe>(Val: &R);
6259 // TODO: Transform reverse access into strided access with -1 stride.
6260 // TODO: Transform gather/scatter with uniform address into strided access
6261 // with 0 stride.
6262 // TODO: Transform interleave access into multiple strided accesses.
6263 if (!MemR || MemR->isConsecutive())
6264 continue;
6265
6266 VPValue *Ptr = MemR->getAddr();
6267 // Check if this is a strided access by analyzing the address SCEV for an
6268 // affine addRec.
6269 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: &L);
6270 const SCEV *Start;
6271 const SCEVConstant *Step;
6272 // TODO: Support non-constant loop invariant stride.
6273 if (!match(S: PtrSCEV,
6274 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEVConstant(V&: Step),
6275 L: m_SpecificLoop(L: &L))))
6276 continue;
6277
6278 VPValue *StoredValue = nullptr;
6279 Type *DataTy;
6280 Intrinsic::ID IntrinID;
6281 if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Val: &R)) {
6282 StoredValue = StoreR->getStoredValue();
6283 DataTy = StoredValue->getScalarType();
6284 IntrinID = Intrinsic::experimental_vp_strided_store;
6285 } else {
6286 auto *LoadR = cast<VPWidenLoadRecipe>(Val: &R);
6287 DataTy = LoadR->getScalarType();
6288 IntrinID = Intrinsic::experimental_vp_strided_load;
6289 }
6290
6291 Align Alignment = MemR->getAlign();
6292 auto IsProfitable = [&](ElementCount VF) {
6293 Type *VectorTy = toVectorTy(Scalar: DataTy, EC: VF);
6294 if (!Ctx.TTI.isLegalStridedLoadStore(DataType: VectorTy, Alignment))
6295 return false;
6296 const InstructionCost CurrentCost = MemR->computeCost(VF, Ctx);
6297 const InstructionCost StridedLoadStoreCost =
6298 VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost(
6299 IID: IntrinID, Ty: VectorTy, IsMasked: MemR->isMasked(), Alignment, Ctx);
6300 return StridedLoadStoreCost < CurrentCost;
6301 };
6302
6303 if (!LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsProfitable,
6304 Range))
6305 continue;
6306
6307 // Invalidate the legacy widening decision so the cost of replaced load is
6308 // not counted during precomputeCosts.
6309 // TODO: Remove once the legacy exit cost computation is retired.
6310 for (ElementCount VF : Range)
6311 Ctx.invalidateWideningDecision(I: &MemR->getIngredient(), VF);
6312
6313 // Get VF as i32 for the vector length operand.
6314 if (!I32VF) {
6315 VPBuilder Builder(Plan.getVectorPreheader());
6316 I32VF = Builder.createScalarZExtOrTrunc(
6317 Op: &Plan.getVF(), ResultTy: Type::getInt32Ty(C&: Plan.getContext()),
6318 DL: DebugLoc::getUnknown());
6319 }
6320
6321 VPBuilder Builder(&R);
6322 // Create the base pointer of strided access.
6323 // TODO: reuse VPDerivedIVRecipe for base pointer computation when it
6324 // supports a general VPValue as the start value.
6325 VPValue *StartVPV =
6326 VPSCEVExpander(Builder, *PSE.getSE(), R.getDebugLoc()).expand(S: Start);
6327 VPValue *StrideInBytes = Plan.getOrAddLiveIn(V: Step->getValue());
6328 Type *IndexTy = Plan.getDataLayout().getIndexType(PtrTy: Ptr->getScalarType());
6329 assert(IndexTy == StrideInBytes->getScalarType() &&
6330 "Stride type from SCEV must match the index type");
6331 VPValue *CanIV = Builder.createScalarZExtOrTrunc(
6332 Op: VectorLoop->getCanonicalIV(), ResultTy: IndexTy, DL: DebugLoc::getUnknown());
6333 auto *AddRecPtr = cast<SCEVAddRecExpr>(Val: PtrSCEV);
6334 auto *Offset = Builder.createOverflowingOp(
6335 Opcode: Instruction::Mul, Operands: {CanIV, StrideInBytes},
6336 WrapFlags: {AddRecPtr->hasNoUnsignedWrap(), /*HasNSW=*/false});
6337 GEPNoWrapFlags NWFlags = AddRecPtr->hasNoUnsignedWrap()
6338 ? GEPNoWrapFlags::noUnsignedWrap()
6339 : GEPNoWrapFlags::none();
6340 VPValue *BasePtr = Builder.createNoWrapPtrAdd(Ptr: StartVPV, Offset, GEPFlags: NWFlags);
6341
6342 // Create a new vector pointer for strided access.
6343 VPValue *NewPtr = Builder.createVectorPointer(
6344 Ptr: BasePtr, SourceElementTy: Type::getInt8Ty(C&: Plan.getContext()), Stride: StrideInBytes, GEPFlags: NWFlags,
6345 DL: R.getDebugLoc());
6346
6347 VPValue *Mask = MemR->getMask();
6348 if (!Mask)
6349 Mask = Plan.getTrue();
6350 SmallVector<VPValue *, 5> Ops;
6351 if (StoredValue)
6352 Ops.push_back(Elt: StoredValue);
6353 Ops.append(IL: {NewPtr, StrideInBytes, Mask, I32VF});
6354
6355 auto *StridedR = Builder.createWidenMemIntrinsic(
6356 VectorIntrinsicID: IntrinID, CallArguments: Ops,
6357 Ty: StoredValue ? Type::getVoidTy(C&: Plan.getContext()) : DataTy, Alignment,
6358 MD: *MemR, DL: R.getDebugLoc());
6359 if (!StoredValue)
6360 cast<VPWidenLoadRecipe>(Val: &R)->replaceAllUsesWith(New: StridedR);
6361 R.eraseFromParent();
6362 }
6363 }
6364}
6365