1//===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements a set of utility VPlan to VPlan transformations.
11///
12//===----------------------------------------------------------------------===//
13
14#include "VPlanTransforms.h"
15#include "VPRecipeBuilder.h"
16#include "VPlan.h"
17#include "VPlanAnalysis.h"
18#include "VPlanCFG.h"
19#include "VPlanDominatorTree.h"
20#include "VPlanHelpers.h"
21#include "VPlanPatternMatch.h"
22#include "VPlanUtils.h"
23#include "llvm/ADT/APInt.h"
24#include "llvm/ADT/PostOrderIterator.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallPtrSet.h"
28#include "llvm/ADT/TypeSwitch.h"
29#include "llvm/Analysis/IVDescriptors.h"
30#include "llvm/Analysis/Loads.h"
31#include "llvm/Analysis/LoopInfo.h"
32#include "llvm/Analysis/MemoryLocation.h"
33#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
34#include "llvm/Analysis/ScopedNoAliasAA.h"
35#include "llvm/Analysis/VectorUtils.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/Metadata.h"
38#include "llvm/Support/Casting.h"
39#include "llvm/Support/TypeSize.h"
40#include "llvm/Transforms/Utils/LoopUtils.h"
41
42using namespace llvm;
43using namespace VPlanPatternMatch;
44using namespace SCEVPatternMatch;
45
46/// If the pointer operand \p Addr of a memory access is an affine AddRec
47/// w.r.t. \p L with a constant stride, return the stride in units of
48/// \p AccessTy. Otherwise return std::nullopt.
49static std::optional<int64_t> getConstantStride(VPValue *Addr, Type *AccessTy,
50 PredicatedScalarEvolution &PSE,
51 const Loop *L) {
52 assert(!hasIrregularType(AccessTy, L->getHeader()->getDataLayout()) &&
53 "should not try to widen irregular types");
54 const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(V: Addr, PSE, L);
55 auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: AddrSCEV);
56 if (!AddRec)
57 return {};
58
59 return getStrideFromAddRec(AR: AddRec, Lp: L, AccessTy, /*Ptr=*/nullptr, PSE);
60}
61
62bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
63 VPlan &Plan, const TargetLibraryInfo &TLI, PredicatedScalarEvolution &PSE,
64 Loop *OuterLoop) {
65
66 // Returns true if the access of \p AccessTy at \p Addr can be widened to a
67 // consecutive vector access.
68 auto IsConsecutiveAccess = [&](VPValue *Addr, Type *AccessTy) {
69 return !hasIrregularType(Ty: AccessTy, DL: Plan.getDataLayout()) &&
70 getConstantStride(Addr, AccessTy, PSE, L: OuterLoop) == 1;
71 };
72
73 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
74 Plan.getVectorLoopRegion());
75 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
76 // Skip blocks outside region
77 if (!VPBB->getParent())
78 break;
79 VPRecipeBase *Term = VPBB->getTerminator();
80 auto EndIter = Term ? Term->getIterator() : VPBB->end();
81 // Introduce each ingredient into VPlan.
82 for (VPRecipeBase &Ingredient :
83 make_early_inc_range(Range: make_range(x: VPBB->begin(), y: EndIter))) {
84
85 VPValue *VPV = Ingredient.getVPSingleValue();
86 if (!VPV->getUnderlyingValue())
87 continue;
88
89 Instruction *Inst = cast<Instruction>(Val: VPV->getUnderlyingValue());
90
91 // Atomic accesses and fences have ordering/atomicity semantics that
92 // cannot be preserved by lane-wise widening.
93 if (isa<AtomicRMWInst, AtomicCmpXchgInst, FenceInst>(Val: Inst))
94 return false;
95
96 VPRecipeBase *NewRecipe = nullptr;
97 if (auto *PhiR = dyn_cast<VPPhi>(Val: &Ingredient)) {
98 auto *Phi = cast<PHINode>(Val: PhiR->getUnderlyingValue());
99 NewRecipe = new VPWidenPHIRecipe(PhiR->operands(), PhiR->getDebugLoc(),
100 Phi->getName());
101 } else if (auto *VPI = dyn_cast<VPInstruction>(Val: &Ingredient)) {
102 assert(!isa<PHINode>(Inst) && "phis should be handled above");
103 // Create VPWidenMemoryRecipe for loads and stores.
104 if (LoadInst *Load = dyn_cast<LoadInst>(Val: Inst)) {
105 bool IsConsecutive =
106 IsConsecutiveAccess(VPI->getOperand(N: 0), VPI->getScalarType());
107 NewRecipe = new VPWidenLoadRecipe(*Load, Ingredient.getOperand(N: 0),
108 nullptr /*Mask*/, IsConsecutive,
109 *VPI, Ingredient.getDebugLoc());
110 } else if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst)) {
111 bool IsConsecutive = IsConsecutiveAccess(
112 VPI->getOperand(N: 1), VPI->getOperand(N: 0)->getScalarType());
113 NewRecipe = new VPWidenStoreRecipe(
114 *Store, Ingredient.getOperand(N: 1), Ingredient.getOperand(N: 0),
115 nullptr /*Mask*/, IsConsecutive, *VPI, Ingredient.getDebugLoc());
116 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Inst)) {
117 NewRecipe = new VPWidenGEPRecipe(GEP->getSourceElementType(),
118 Ingredient.operands(), *VPI,
119 Ingredient.getDebugLoc(), GEP);
120 } else if (CallInst *CI = dyn_cast<CallInst>(Val: Inst)) {
121 Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, TLI: &TLI);
122 if (VectorID == Intrinsic::not_intrinsic)
123 return false;
124
125 // The noalias.scope.decl intrinsic declares a noalias scope that
126 // is valid for a single iteration. Emitting it as a single-scalar
127 // replicate would incorrectly extend the scope across multiple
128 // original iterations packed into one vector iteration.
129 // FIXME: If we want to vectorize this loop, then we have to drop
130 // all the associated !alias.scope and !noalias.
131 if (VectorID == Intrinsic::experimental_noalias_scope_decl)
132 return false;
133
134 // These intrinsics are recognized by getVectorIntrinsicIDForCall
135 // but are not widenable. Emit them as replicate instead of widening.
136 if (VectorID == Intrinsic::assume ||
137 VectorID == Intrinsic::lifetime_end ||
138 VectorID == Intrinsic::lifetime_start ||
139 VectorID == Intrinsic::sideeffect ||
140 VectorID == Intrinsic::pseudoprobe) {
141 // If the operand of llvm.assume holds before vectorization, it will
142 // also hold per lane.
143 // llvm.pseudoprobe requires to be duplicated per lane for accurate
144 // sample count.
145 const bool IsSingleScalar = VectorID != Intrinsic::assume &&
146 VectorID != Intrinsic::pseudoprobe;
147 NewRecipe = new VPReplicateRecipe(CI, Ingredient.operands(),
148 /*IsSingleScalar=*/IsSingleScalar,
149 /*Mask=*/nullptr, *VPI, *VPI,
150 Ingredient.getDebugLoc());
151 } else {
152 NewRecipe = new VPWidenIntrinsicRecipe(
153 *CI, VectorID, drop_end(RangeOrContainer: Ingredient.operands()), CI->getType(),
154 VPIRFlags(*CI), *VPI, CI->getDebugLoc());
155 }
156 } else if (auto *CI = dyn_cast<CastInst>(Val: Inst)) {
157 NewRecipe = new VPWidenCastRecipe(
158 CI->getOpcode(), Ingredient.getOperand(N: 0), CI->getType(), CI,
159 VPIRFlags(*CI), VPIRMetadata(*CI));
160 } else {
161 NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI,
162 *VPI, Ingredient.getDebugLoc());
163 }
164 } else {
165 assert(isa<VPWidenIntOrFpInductionRecipe>(&Ingredient) &&
166 "inductions must be created earlier");
167 continue;
168 }
169
170 NewRecipe->insertBefore(InsertPos: &Ingredient);
171 if (NewRecipe->getNumDefinedValues() == 1)
172 VPV->replaceAllUsesWith(New: NewRecipe->getVPSingleValue());
173 else
174 assert(NewRecipe->getNumDefinedValues() == 0 &&
175 "Only recpies with zero or one defined values expected");
176 Ingredient.eraseFromParent();
177 }
178 }
179 return true;
180}
181
182/// Helper for extra no-alias checks via known-safe recipe and SCEV.
183class SinkStoreInfo {
184 SmallPtrSet<VPReplicateRecipe *, 4> ExcludeRecipes;
185 VPReplicateRecipe &GroupLeader;
186 PredicatedScalarEvolution *PSE = nullptr;
187 const Loop *L = nullptr;
188
189 // Return true if \p A and \p B are known to not alias for all VFs in the
190 // plan, checked via the distance between the accesses
191 bool isNoAliasViaDistance(VPReplicateRecipe *A, VPReplicateRecipe *B) const {
192 if (A->getOpcode() != Instruction::Store ||
193 B->getOpcode() != Instruction::Store)
194 return false;
195
196 if (!PSE || !L)
197 return A == B;
198
199 VPValue *AddrA = A->getOperand(N: 1);
200 const SCEV *SCEVA = vputils::getSCEVExprForVPValue(V: AddrA, PSE&: *PSE, L);
201 VPValue *AddrB = B->getOperand(N: 1);
202 const SCEV *SCEVB = vputils::getSCEVExprForVPValue(V: AddrB, PSE&: *PSE, L);
203 if (isa<SCEVCouldNotCompute>(Val: SCEVA) || isa<SCEVCouldNotCompute>(Val: SCEVB))
204 return false;
205
206 const APInt *Distance;
207 ScalarEvolution &SE = *PSE->getSE();
208 if (!match(S: SE.getMinusSCEV(LHS: SCEVA, RHS: SCEVB), P: m_scev_APInt(C&: Distance)))
209 return false;
210
211 const DataLayout &DL = SE.getDataLayout();
212 Type *TyA = A->getOperand(N: 0)->getScalarType();
213 uint64_t SizeA = DL.getTypeStoreSize(Ty: TyA);
214 Type *TyB = B->getOperand(N: 0)->getScalarType();
215 uint64_t SizeB = DL.getTypeStoreSize(Ty: TyB);
216
217 // Use the maximum store size to ensure no overlap from either direction.
218 // Currently only handles fixed sizes, as it is only used for
219 // replicating VPReplicateRecipes.
220 uint64_t MaxStoreSize = std::max(a: SizeA, b: SizeB);
221
222 auto VFs = B->getParent()->getPlan()->vectorFactors();
223 ElementCount MaxVF = *max_element(Range&: VFs, C: ElementCount::isKnownLT);
224 if (MaxVF.isScalable())
225 return false;
226 return Distance->abs().uge(
227 RHS: MaxVF.multiplyCoefficientBy(RHS: MaxStoreSize).getFixedValue());
228 }
229
230public:
231 SinkStoreInfo(ArrayRef<VPReplicateRecipe *> ExcludeRecipes,
232 VPReplicateRecipe &GroupLeader, PredicatedScalarEvolution &PSE,
233 const Loop &L)
234 : ExcludeRecipes(ExcludeRecipes.begin(), ExcludeRecipes.end()),
235 GroupLeader(GroupLeader), PSE(&PSE), L(&L) {}
236
237 SinkStoreInfo(VPReplicateRecipe &GroupLeader) : GroupLeader(GroupLeader) {}
238
239 /// Return true if \p R should be skipped during alias checking, either
240 /// because it's in the exclude set or because no-alias can be proven via
241 /// SCEV.
242 bool shouldSkip(VPRecipeBase &R) const {
243 auto *Store = dyn_cast<VPReplicateRecipe>(Val: &R);
244 return ExcludeRecipes.contains(Ptr: Store) ||
245 (Store && isNoAliasViaDistance(A: Store, B: &GroupLeader));
246 }
247};
248
249/// Check if a memory operation doesn't alias with memory operations using
250/// scoped noalias metadata, in blocks in the single-successor chain between \p
251/// FirstBB and \p LastBB. If \p SinkInfo is std::nullopt, only recipes that may
252/// write to memory are checked (for load hoisting). Otherwise recipes that both
253/// read and write memory are checked, and SCEV is used to prove no-alias
254/// between the group leader and other replicate recipes (for store sinking).
255static bool
256canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
257 VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
258 std::optional<SinkStoreInfo> SinkInfo = {}) {
259 bool CheckReads = SinkInfo.has_value();
260 for (VPBasicBlock *VPBB :
261 VPBlockUtils::blocksInSingleSuccessorChainBetween(FirstBB, LastBB)) {
262 for (VPRecipeBase &R : *VPBB) {
263 if (SinkInfo && SinkInfo->shouldSkip(R))
264 continue;
265
266 // Skip recipes that don't need checking.
267 if (!R.mayWriteToMemory() && !(CheckReads && R.mayReadFromMemory()))
268 continue;
269
270 auto Loc = vputils::getMemoryLocation(R);
271 if (!Loc)
272 // Conservatively assume aliasing for memory operations without
273 // location.
274 return false;
275
276 if (ScopedNoAliasAAResult::alias(LocA: *Loc, LocB: MemLoc) != AliasResult::NoAlias)
277 return false;
278 }
279 }
280 return true;
281}
282
283/// Get the value type of the replicate load or store. \p IsLoad indicates
284/// whether it is a load.
285static Type *getLoadStoreValueType(VPReplicateRecipe *R, bool IsLoad) {
286 return (IsLoad ? R : R->getOperand(N: 0))->getScalarType();
287}
288
289/// Collect either replicated Loads or Stores grouped by their address SCEV and
290/// their load-store type, in a deep-traversal of the vector loop region in \p
291/// Plan.
292template <unsigned Opcode>
293static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
294collectGroupedReplicateMemOps(
295 VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L,
296 function_ref<bool(VPReplicateRecipe *)> FilterFn) {
297 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
298 "Only Load and Store opcodes supported");
299 constexpr bool IsLoad = (Opcode == Instruction::Load);
300 SmallDenseMap<std::pair<const SCEV *, const Type *>,
301 SmallVector<VPReplicateRecipe *, 4>>
302 RecipesByAddressAndType;
303 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
304 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
305 for (VPRecipeBase &R : *VPBB) {
306 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
307 if (!RepR || RepR->getOpcode() != Opcode || !FilterFn(RepR))
308 continue;
309
310 // For loads, operand 0 is address; for stores, operand 1 is address.
311 VPValue *Addr = RepR->getOperand(N: IsLoad ? 0 : 1);
312 const Type *LoadStoreTy = getLoadStoreValueType(R: RepR, IsLoad);
313 const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(V: Addr, PSE, L);
314 if (!isa<SCEVCouldNotCompute>(Val: AddrSCEV))
315 RecipesByAddressAndType[{AddrSCEV, LoadStoreTy}].push_back(Elt: RepR);
316 }
317 }
318 auto Groups = to_vector(Range: RecipesByAddressAndType.values());
319 VPDominatorTree VPDT(Plan);
320 for (auto &Group : Groups) {
321 // Sort mem ops by dominance order, with earliest (most dominating) first.
322 stable_sort(Group, [&VPDT](VPReplicateRecipe *A, VPReplicateRecipe *B) {
323 return VPDT.properlyDominates(A, B);
324 });
325 }
326 return Groups;
327}
328
329static bool sinkScalarOperands(VPlan &Plan) {
330 auto Iter = vp_depth_first_deep(G: Plan.getEntry());
331 bool ScalarVFOnly = Plan.hasScalarVFOnly();
332 bool Changed = false;
333
334 SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList;
335 auto InsertIfValidSinkCandidate = [ScalarVFOnly, &WorkList](
336 VPBasicBlock *SinkTo, VPValue *Op) {
337 auto *Candidate = dyn_cast<VPSingleDefRecipe>(Val: Op);
338 if (!isa_and_nonnull<VPReplicateRecipe, VPScalarIVStepsRecipe,
339 VPInstruction>(Val: Candidate))
340 return;
341
342 if (Candidate->getParent() == SinkTo ||
343 all_of(Range: Candidate->operands(),
344 P: [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); }) ||
345 vputils::cannotHoistOrSinkRecipe(R: *Candidate, /*Sinking=*/true))
346 return;
347
348 if (!ScalarVFOnly && !vputils::doesGeneratePerAllLanes(R: Candidate))
349 return;
350
351 // Only single-scalar VPInstructions can be sunk.
352 if (auto *VPI = dyn_cast<VPInstruction>(Val: Candidate))
353 if (!vputils::isSingleScalar(VPV: VPI))
354 return;
355
356 WorkList.insert(X: {SinkTo, Candidate});
357 };
358
359 // First, collect the operands of all recipes in replicate blocks as seeds for
360 // sinking.
361 for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Range&: Iter)) {
362 VPBasicBlock *EntryVPBB = VPR->getEntryBasicBlock();
363 if (!VPR->isReplicator() || EntryVPBB->getSuccessors().size() != 2)
364 continue;
365 VPBasicBlock *VPBB = cast<VPBasicBlock>(Val: EntryVPBB->getSuccessors().front());
366 if (VPBB->getSingleSuccessor() != VPR->getExitingBasicBlock())
367 continue;
368 for (auto &Recipe : *VPBB)
369 for (VPValue *Op : Recipe.operands())
370 InsertIfValidSinkCandidate(VPBB, Op);
371 }
372
373 // Try to sink each replicate or scalar IV steps recipe in the worklist.
374 for (unsigned I = 0; I != WorkList.size(); ++I) {
375 VPBasicBlock *SinkTo;
376 VPSingleDefRecipe *SinkCandidate;
377 std::tie(args&: SinkTo, args&: SinkCandidate) = WorkList[I];
378
379 // All recipe users of SinkCandidate must be in the same block SinkTo or all
380 // users outside of SinkTo must only use the first lane of SinkCandidate. In
381 // the latter case, we need to duplicate SinkCandidate.
382 auto UsersOutsideSinkTo =
383 make_filter_range(Range: SinkCandidate->users(), Pred: [SinkTo](VPUser *U) {
384 return cast<VPRecipeBase>(Val: U)->getParent() != SinkTo;
385 });
386 if (any_of(Range&: UsersOutsideSinkTo, P: [SinkCandidate](VPUser *U) {
387 return !U->usesFirstLaneOnly(Op: SinkCandidate);
388 }))
389 continue;
390 bool NeedsDuplicating = !UsersOutsideSinkTo.empty();
391
392 if (NeedsDuplicating) {
393 if (ScalarVFOnly)
394 continue;
395 VPSingleDefRecipe *Clone;
396 if (auto *SinkCandidateRepR =
397 dyn_cast<VPReplicateRecipe>(Val: SinkCandidate)) {
398 // TODO: Handle converting to uniform recipes as separate transform,
399 // then cloning should be sufficient here.
400 Clone = VPBuilder::createSingleScalarOp(
401 Opcode: SinkCandidateRepR->getOpcode(), Operands: SinkCandidate->operands(),
402 /*Mask=*/nullptr, Flags: *SinkCandidateRepR, Metadata: *SinkCandidateRepR,
403 DL: SinkCandidate->getDebugLoc(), UV: SinkCandidate->getUnderlyingInstr());
404 // TODO: add ".cloned" suffix to name of Clone's VPValue.
405 } else {
406 Clone = SinkCandidate->clone();
407 }
408
409 Clone->insertBefore(InsertPos: SinkCandidate);
410 SinkCandidate->replaceUsesWithIf(New: Clone, ShouldReplace: [SinkTo](VPUser &U, unsigned) {
411 return cast<VPRecipeBase>(Val: &U)->getParent() != SinkTo;
412 });
413 }
414 SinkCandidate->moveBefore(BB&: *SinkTo, I: SinkTo->getFirstNonPhi());
415 for (VPValue *Op : SinkCandidate->operands())
416 InsertIfValidSinkCandidate(SinkTo, Op);
417 Changed = true;
418 }
419 return Changed;
420}
421
422/// If \p R is a triangle region, return the 'then' block of the triangle.
423static VPBasicBlock *getPredicatedThenBlock(VPRegionBlock *R) {
424 auto *EntryBB = cast<VPBasicBlock>(Val: R->getEntry());
425 if (EntryBB->getNumSuccessors() != 2)
426 return nullptr;
427
428 auto *Succ0 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[0]);
429 auto *Succ1 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[1]);
430 if (!Succ0 || !Succ1)
431 return nullptr;
432
433 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
434 return nullptr;
435 if (Succ0->getSingleSuccessor() == Succ1)
436 return Succ0;
437 if (Succ1->getSingleSuccessor() == Succ0)
438 return Succ1;
439 return nullptr;
440}
441
442// Merge replicate regions in their successor region, if a replicate region
443// is connected to a successor replicate region with the same predicate by a
444// single, empty VPBasicBlock.
445static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
446 SmallPtrSet<VPRegionBlock *, 4> TransformedRegions;
447
448 // Collect replicate regions followed by an empty block, followed by another
449 // replicate region with matching masks to process front. This is to avoid
450 // iterator invalidation issues while merging regions.
451 SmallVector<VPRegionBlock *, 8> WorkList;
452 for (VPRegionBlock *Region1 : VPBlockUtils::blocksOnly<VPRegionBlock>(
453 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
454 if (!Region1->isReplicator())
455 continue;
456 auto *MiddleBasicBlock =
457 dyn_cast_or_null<VPBasicBlock>(Val: Region1->getSingleSuccessor());
458 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
459 continue;
460
461 auto *Region2 =
462 dyn_cast_or_null<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
463 if (!Region2 || !Region2->isReplicator())
464 continue;
465
466 VPValue *Mask1 = Region1->getEntryBranchOnMask()->getOperand(N: 0);
467 VPValue *Mask2 = Region2->getEntryBranchOnMask()->getOperand(N: 0);
468 if (!Mask1 || Mask1 != Mask2)
469 continue;
470
471 assert(Mask1 && Mask2 && "both region must have conditions");
472 WorkList.push_back(Elt: Region1);
473 }
474
475 // Move recipes from Region1 to its successor region, if both are triangles.
476 for (VPRegionBlock *Region1 : WorkList) {
477 if (TransformedRegions.contains(Ptr: Region1))
478 continue;
479 auto *MiddleBasicBlock = cast<VPBasicBlock>(Val: Region1->getSingleSuccessor());
480 auto *Region2 = cast<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
481
482 VPBasicBlock *Then1 = getPredicatedThenBlock(R: Region1);
483 VPBasicBlock *Then2 = getPredicatedThenBlock(R: Region2);
484 if (!Then1 || !Then2)
485 continue;
486
487 // Note: No fusion-preventing memory dependencies are expected in either
488 // region. Such dependencies should be rejected during earlier dependence
489 // checks, which guarantee accesses can be re-ordered for vectorization.
490 //
491 // Move recipes to the successor region.
492 for (VPRecipeBase &ToMove : make_early_inc_range(Range: reverse(C&: *Then1)))
493 ToMove.moveBefore(BB&: *Then2, I: Then2->getFirstNonPhi());
494
495 auto *Merge1 = cast<VPBasicBlock>(Val: Then1->getSingleSuccessor());
496 auto *Merge2 = cast<VPBasicBlock>(Val: Then2->getSingleSuccessor());
497
498 // Move VPPredInstPHIRecipes from the merge block to the successor region's
499 // merge block. Update all users inside the successor region to use the
500 // original values.
501 for (VPRecipeBase &Phi1ToMove : make_early_inc_range(Range: reverse(C&: *Merge1))) {
502 VPValue *PredInst1 =
503 cast<VPPredInstPHIRecipe>(Val: &Phi1ToMove)->getOperand(N: 0);
504 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
505 Phi1ToMoveV->replaceUsesWithIf(New: PredInst1, ShouldReplace: [Then2](VPUser &U, unsigned) {
506 return cast<VPRecipeBase>(Val: &U)->getParent() == Then2;
507 });
508
509 // Remove phi recipes that are unused after merging the regions.
510 if (Phi1ToMove.getVPSingleValue()->user_empty()) {
511 Phi1ToMove.eraseFromParent();
512 continue;
513 }
514 Phi1ToMove.moveBefore(BB&: *Merge2, I: Merge2->begin());
515 }
516
517 // Remove the dead recipes in Region1's entry block.
518 for (VPRecipeBase &R :
519 make_early_inc_range(Range: reverse(C&: *Region1->getEntryBasicBlock())))
520 R.eraseFromParent();
521
522 // Finally, remove the first region.
523 for (VPBlockBase *Pred : make_early_inc_range(Range&: Region1->getPredecessors())) {
524 VPBlockUtils::disconnectBlocks(From: Pred, To: Region1);
525 VPBlockUtils::connectBlocks(From: Pred, To: MiddleBasicBlock);
526 }
527 VPBlockUtils::disconnectBlocks(From: Region1, To: MiddleBasicBlock);
528 TransformedRegions.insert(Ptr: Region1);
529 }
530
531 return !TransformedRegions.empty();
532}
533
534static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
535 VPRegionBlock *ParentRegion,
536 VPlan &Plan) {
537 Instruction *Instr = PredRecipe->getUnderlyingInstr();
538 // Build the triangular if-then region.
539 std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str();
540 assert(Instr->getParent() && "Predicated instruction not in any basic block");
541 auto *BlockInMask = PredRecipe->getMask();
542 auto *MaskDef = BlockInMask->getDefiningRecipe();
543 auto *BOMRecipe = new VPBranchOnMaskRecipe(
544 BlockInMask, MaskDef ? MaskDef->getDebugLoc() : DebugLoc::getUnknown());
545 auto *Entry =
546 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".entry", Recipe: BOMRecipe);
547
548 // Replace predicated replicate recipe with a replicate recipe without a
549 // mask but in the replicate region.
550 auto *RecipeWithoutMask = new VPReplicateRecipe(
551 PredRecipe->getUnderlyingInstr(), PredRecipe->operandsWithoutMask(),
552 PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
553 PredRecipe->getDebugLoc());
554 auto *Pred =
555 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".if", Recipe: RecipeWithoutMask);
556 auto *Exiting = Plan.createVPBasicBlock(Name: Twine(RegionName) + ".continue");
557 VPRegionBlock *Region =
558 Plan.createReplicateRegion(Entry, Exiting, Name: RegionName);
559
560 // Note: first set Entry as region entry and then connect successors starting
561 // from it in order, to propagate the "parent" of each VPBasicBlock.
562 Region->setParent(ParentRegion);
563 VPBlockUtils::insertTwoBlocksAfter(IfTrue: Pred, IfFalse: Exiting, BlockPtr: Entry);
564 VPBlockUtils::connectBlocks(From: Pred, To: Exiting);
565
566 if (!PredRecipe->user_empty()) {
567 auto *PHIRecipe = new VPPredInstPHIRecipe(RecipeWithoutMask,
568 RecipeWithoutMask->getDebugLoc());
569 Exiting->appendRecipe(Recipe: PHIRecipe);
570 PredRecipe->replaceAllUsesWith(New: PHIRecipe);
571 }
572 PredRecipe->eraseFromParent();
573 return Region;
574}
575
576static void addReplicateRegions(VPlan &Plan) {
577 SmallVector<VPReplicateRecipe *> WorkList;
578 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
579 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
580 for (VPRecipeBase &R : *VPBB)
581 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
582 if (RepR->isPredicated())
583 WorkList.push_back(Elt: RepR);
584 }
585 }
586
587 unsigned BBNum = 0;
588 for (VPReplicateRecipe *RepR : WorkList) {
589 VPBasicBlock *CurrentBlock = RepR->getParent();
590 VPBasicBlock *SplitBlock = CurrentBlock->splitAt(SplitAt: RepR->getIterator());
591
592 BasicBlock *OrigBB = RepR->getUnderlyingInstr()->getParent();
593 SplitBlock->setName(
594 OrigBB->hasName() ? OrigBB->getName() + "." + Twine(BBNum++) : "");
595 // Record predicated instructions for above packing optimizations.
596 VPRegionBlock *Region =
597 createReplicateRegion(PredRecipe: RepR, ParentRegion: CurrentBlock->getParent(), Plan);
598 VPBlockUtils::insertOnEdge(From: CurrentBlock, To: SplitBlock, BlockPtr: Region);
599
600 VPRegionBlock *ParentRegion = Region->getParent();
601 if (ParentRegion && ParentRegion->getExiting() == CurrentBlock)
602 ParentRegion->setExiting(SplitBlock);
603 }
604}
605
606bool VPlanTransforms::mergeBlocksIntoPredecessors(VPlan &Plan) {
607 SmallVector<VPBasicBlock *> WorkList;
608 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
609 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
610 // Don't fold the blocks in the skeleton of the Plan into their single
611 // predecessors for now.
612 // TODO: Remove restriction once more of the skeleton is modeled in VPlan.
613 if (!VPBB->getParent())
614 continue;
615 auto *PredVPBB =
616 dyn_cast_or_null<VPBasicBlock>(Val: VPBB->getSinglePredecessor());
617 if (!PredVPBB || PredVPBB->getNumSuccessors() != 1 ||
618 isa<VPIRBasicBlock>(Val: PredVPBB))
619 continue;
620 WorkList.push_back(Elt: VPBB);
621 }
622
623 for (VPBasicBlock *VPBB : WorkList) {
624 VPBasicBlock *PredVPBB = cast<VPBasicBlock>(Val: VPBB->getSinglePredecessor());
625 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
626 R.moveBefore(BB&: *PredVPBB, I: PredVPBB->end());
627 VPBlockUtils::disconnectBlocks(From: PredVPBB, To: VPBB);
628 auto *ParentRegion = VPBB->getParent();
629 if (ParentRegion && ParentRegion->getExiting() == VPBB)
630 ParentRegion->setExiting(PredVPBB);
631 VPBlockUtils::transferSuccessors(Old: VPBB, New: PredVPBB);
632 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
633 }
634 return !WorkList.empty();
635}
636
637void VPlanTransforms::createAndOptimizeReplicateRegions(VPlan &Plan) {
638 // Convert masked VPReplicateRecipes to if-then region blocks.
639 addReplicateRegions(Plan);
640
641 bool ShouldSimplify = true;
642 while (ShouldSimplify) {
643 ShouldSimplify = sinkScalarOperands(Plan);
644 ShouldSimplify |= mergeReplicateRegionsIntoSuccessors(Plan);
645 ShouldSimplify |= mergeBlocksIntoPredecessors(Plan);
646 }
647}
648
649/// Remove redundant casts of inductions.
650///
651/// Such redundant casts are casts of induction variables that can be ignored,
652/// because we already proved that the casted phi is equal to the uncasted phi
653/// in the vectorized loop. There is no need to vectorize the cast - the same
654/// value can be used for both the phi and casts in the vector loop.
655static void removeRedundantInductionCasts(VPlan &Plan) {
656 for (auto &Phi : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
657 auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: &Phi);
658 if (!IV || IV->getTruncInst())
659 continue;
660
661 // A sequence of IR Casts has potentially been recorded for IV, which
662 // *must be bypassed* when the IV is vectorized, because the vectorized IV
663 // will produce the desired casted value. This sequence forms a def-use
664 // chain and is provided in reverse order, ending with the cast that uses
665 // the IV phi. Search for the recipe of the last cast in the chain and
666 // replace it with the original IV. Note that only the final cast is
667 // expected to have users outside the cast-chain and the dead casts left
668 // over will be cleaned up later.
669 ArrayRef<Instruction *> Casts = IV->getInductionDescriptor().getCastInsts();
670 VPValue *FindMyCast = IV;
671 for (Instruction *IRCast : reverse(C&: Casts)) {
672 VPSingleDefRecipe *FoundUserCast = nullptr;
673 for (auto *U : FindMyCast->users()) {
674 auto *UserCast = dyn_cast<VPSingleDefRecipe>(Val: U);
675 if (UserCast && UserCast->getUnderlyingValue() == IRCast) {
676 FoundUserCast = UserCast;
677 break;
678 }
679 }
680 // A cast recipe in the chain may have been removed by earlier DCE.
681 if (!FoundUserCast)
682 break;
683 FindMyCast = FoundUserCast;
684 }
685 if (FindMyCast != IV)
686 FindMyCast->replaceAllUsesWith(New: IV);
687 }
688}
689
690void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
691 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
692 Plan.getEntry());
693 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
694 // The recipes in the block are processed in reverse order, to catch chains
695 // of dead recipes.
696 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
697 if (vputils::isDeadRecipe(R)) {
698 R.eraseFromParent();
699 continue;
700 }
701
702 // Check if R is a dead VPPhi <-> update cycle and remove it.
703 VPValue *Start, *Incoming;
704 if (!match(V: &R, P: m_VPPhi(Op0: m_VPValue(V&: Start), Op1: m_VPValue(V&: Incoming))))
705 continue;
706 auto *PhiR = cast<VPPhi>(Val: &R);
707 VPUser *PhiUser = PhiR->getSingleUser();
708 if (!PhiUser)
709 continue;
710 if (PhiUser != Incoming->getDefiningRecipe() ||
711 Incoming->getNumUsers() != 1)
712 continue;
713 PhiR->replaceAllUsesWith(New: Start);
714 PhiR->eraseFromParent();
715 Incoming->getDefiningRecipe()->eraseFromParent();
716 }
717 }
718}
719
720/// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd
721/// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as
722/// VPWidenPointerInductionRecipe will generate vectors only. If some users
723/// require vectors while other require scalars, the scalar uses need to extract
724/// the scalars from the generated vectors (Note that this is different to how
725/// int/fp inductions are handled). Legalize extract-from-ends using uniform
726/// VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so
727/// the correct end value is available. Also optimize
728/// VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by
729/// providing them scalar steps built on the canonical scalar IV and update the
730/// original IV's users. This is an optional optimization to reduce the needs of
731/// vector extracts.
732static void legalizeAndOptimizeInductions(VPlan &Plan) {
733 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
734 bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly();
735
736 SmallVector<VPWidenInductionRecipe *> WideIVs;
737 for (VPRecipeBase &Phi : HeaderVPBB->phis())
738 if (auto *PhiR = dyn_cast<VPWidenInductionRecipe>(Val: &Phi))
739 WideIVs.push_back(Elt: PhiR);
740
741 // Try to narrow wide and replicating recipes to uniform recipes, based on
742 // VPlan analysis.
743 // TODO: Apply to all recipes in the future, to replace legacy uniformity
744 // analysis.
745 for (VPWidenInductionRecipe *PhiR : WideIVs) {
746 auto Users = vputils::collectUsersRecursively(V: PhiR);
747 for (VPUser *U : reverse(C&: Users)) {
748 auto *Def = dyn_cast<VPRecipeWithIRFlags>(Val: U);
749 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
750 // Skip recipes that shouldn't be narrowed.
751 if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Val: Def) ||
752 Def->user_empty() || !Def->getUnderlyingValue() ||
753 (RepR && (RepR->isSingleScalar() || RepR->isPredicated())))
754 continue;
755
756 // Skip recipes that may have other lanes than their first used.
757 if (!vputils::isSingleScalar(VPV: Def) && !vputils::onlyFirstLaneUsed(Def))
758 continue;
759
760 // TODO: Support scalarizing ExtractValue.
761 if (match(V: Def,
762 P: m_Binary<Instruction::ExtractValue>(Op0: m_VPValue(), Op1: m_VPValue())))
763 continue;
764
765 auto *Clone = VPBuilder::createSingleScalarOp(
766 Opcode: Def->getUnderlyingInstr()->getOpcode(), Operands: Def->operands(),
767 /*Mask=*/nullptr, Flags: *Def, Metadata: {}, DL: DebugLoc::getUnknown(),
768 UV: Def->getUnderlyingInstr());
769 Clone->insertAfter(InsertPos: Def);
770 Def->replaceAllUsesWith(New: Clone);
771 Def->eraseFromParent();
772 }
773 }
774
775 VPBuilder Builder(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
776 for (VPWidenInductionRecipe *PhiR : WideIVs) {
777 // Replace wide pointer inductions which have only their scalars used by
778 // PtrAdd(IndStart, ScalarIVSteps (0, Step)).
779 if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(Val: PhiR)) {
780 if (!Plan.hasScalarVFOnly() &&
781 !PtrIV->onlyScalarsGenerated(IsScalable: Plan.hasScalableVF()))
782 continue;
783
784 VPValue *PtrAdd =
785 vputils::scalarizeVPWidenPointerInduction(PtrIV, Plan, Builder);
786 PtrIV->replaceAllUsesWith(New: PtrAdd);
787 continue;
788 }
789
790 // Replace widened induction with scalar steps for users that only use
791 // scalars.
792 auto *WideIV = cast<VPWidenIntOrFpInductionRecipe>(Val: PhiR);
793 if (HasOnlyVectorVFs && none_of(Range: WideIV->users(), P: [WideIV](VPUser *U) {
794 return U->usesScalars(Op: WideIV);
795 }))
796 continue;
797
798 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
799 VPIRFlags::WrapFlagsTy WrapFlags;
800 // We can preserve nuw when the step is non-negative.
801 const APInt *Step;
802 if (match(V: WideIV->getStepValue(), P: m_APInt(C&: Step)) && Step->isNonNegative())
803 WrapFlags = {static_cast<bool>(WideIV->getNoWrapFlagsOrNone().HasNUW),
804 false};
805 VPScalarIVStepsRecipe *Steps = vputils::createScalarIVSteps(
806 Plan, Kind: ID.getKind(), InductionOpcode: ID.getInductionOpcode(),
807 FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
808 TruncI: WideIV->getTruncInst(), StartV: WideIV->getStartValue(), Step: WideIV->getStepValue(),
809 DL: WideIV->getDebugLoc(), Builder, Flags: WrapFlags);
810
811 // Update scalar users of IV to use Step instead.
812 if (!HasOnlyVectorVFs) {
813 assert(!Plan.hasScalableVF() &&
814 "plans containing a scalar VF cannot also include scalable VFs");
815 WideIV->replaceAllUsesWith(New: Steps);
816 } else {
817 bool HasScalableVF = Plan.hasScalableVF();
818 WideIV->replaceUsesWithIf(New: Steps,
819 ShouldReplace: [WideIV, HasScalableVF](VPUser &U, unsigned) {
820 if (HasScalableVF)
821 return U.usesFirstLaneOnly(Op: WideIV);
822 return U.usesScalars(Op: WideIV);
823 });
824 }
825 }
826}
827
828/// Check if \p VPV is an untruncated wide induction, either before or after the
829/// increment. If so return the header IV (before the increment), otherwise
830/// return null.
831static VPWidenInductionRecipe *
832getOptimizableIVOf(VPValue *VPV, PredicatedScalarEvolution &PSE) {
833 auto *WideIV = dyn_cast<VPWidenInductionRecipe>(Val: VPV);
834 if (WideIV) {
835 // VPV itself is a wide induction, separately compute the end value for exit
836 // users if it is not a truncated IV.
837 auto *IntOrFpIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
838 return (IntOrFpIV && IntOrFpIV->getTruncInst()) ? nullptr : WideIV;
839 }
840
841 // Check if VPV is an optimizable induction increment.
842 VPRecipeBase *Def = VPV->getDefiningRecipe();
843 if (!Def || Def->getNumOperands() != 2)
844 return nullptr;
845 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 0));
846 if (!WideIV)
847 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 1));
848 if (!WideIV)
849 return nullptr;
850
851 auto IsWideIVInc = [&]() {
852 auto &ID = WideIV->getInductionDescriptor();
853
854 // Check if VPV increments the induction by the induction step.
855 VPValue *IVStep = WideIV->getStepValue();
856 switch (ID.getInductionOpcode()) {
857 case Instruction::Add:
858 return match(V: VPV, P: m_c_Add(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
859 case Instruction::FAdd:
860 return match(V: VPV, P: m_c_FAdd(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
861 case Instruction::FSub:
862 return match(V: VPV, P: m_Binary<Instruction::FSub>(Op0: m_Specific(VPV: WideIV),
863 Op1: m_Specific(VPV: IVStep)));
864 case Instruction::Sub: {
865 // IVStep will be the negated step of the subtraction. Check if Step == -1
866 // * IVStep.
867 VPValue *Step;
868 if (!match(V: VPV, P: m_Sub(Op0: m_VPValue(), Op1: m_VPValue(V&: Step))))
869 return false;
870 const SCEV *IVStepSCEV = vputils::getSCEVExprForVPValue(V: IVStep, PSE);
871 const SCEV *StepSCEV = vputils::getSCEVExprForVPValue(V: Step, PSE);
872 ScalarEvolution &SE = *PSE.getSE();
873 return !isa<SCEVCouldNotCompute>(Val: IVStepSCEV) &&
874 !isa<SCEVCouldNotCompute>(Val: StepSCEV) &&
875 IVStepSCEV == SE.getNegativeSCEV(V: StepSCEV);
876 }
877 default:
878 return ID.getKind() == InductionDescriptor::IK_PtrInduction &&
879 match(V: VPV, P: m_GetElementPtr(Op0: m_Specific(VPV: WideIV),
880 Op1: m_Specific(VPV: WideIV->getStepValue())));
881 }
882 llvm_unreachable("should have been covered by switch above");
883 };
884 return IsWideIVInc() ? WideIV : nullptr;
885}
886
887/// Attempts to optimize the induction variable exit values for users in the
888/// early exit block.
889static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan, VPValue *Op,
890 PredicatedScalarEvolution &PSE) {
891 VPValue *Incoming, *Mask;
892 if (!match(V: Op, P: m_ExtractLane(Op0: m_FirstActiveLane(Op0: m_VPValue(V&: Mask)),
893 Op1: m_VPValue(V&: Incoming))))
894 return nullptr;
895
896 auto *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
897 if (!WideIV)
898 return nullptr;
899
900 // Calculate the final index.
901 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
902 auto *CanonicalIV = LoopRegion->getCanonicalIV();
903 Type *CanonicalIVType = LoopRegion->getCanonicalIVType();
904 auto *ExtractR = cast<VPInstruction>(Val: Op);
905 VPBuilder B(ExtractR);
906
907 DebugLoc DL = ExtractR->getDebugLoc();
908 VPValue *FirstActiveLane = B.createFirstActiveLane(Masks: Mask, DL);
909 FirstActiveLane =
910 B.createScalarZExtOrTrunc(Op: FirstActiveLane, ResultTy: CanonicalIVType, DL);
911 VPValue *EndValue = B.createAdd(LHS: CanonicalIV, RHS: FirstActiveLane, DL);
912
913 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
914 // changed it means the exit is using the incremented value, so we need to
915 // add the step.
916 if (Incoming != WideIV) {
917 VPValue *One = Plan.getConstantInt(Ty: CanonicalIVType, Val: 1);
918 EndValue = B.createAdd(LHS: EndValue, RHS: One, DL);
919 }
920
921 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
922 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
923 VPIRValue *Start = WideIV->getStartValue();
924 VPValue *Step = WideIV->getStepValue();
925 EndValue = B.createDerivedIV(
926 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
927 Start, Current: EndValue, Step);
928 }
929
930 return EndValue;
931}
932
933/// Compute the end value for \p WideIV, unless it is truncated. Creates a
934/// VPDerivedIVRecipe for non-canonical inductions.
935static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV,
936 VPBuilder &VectorPHBuilder,
937 VPValue *VectorTC) {
938 auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
939 // Truncated wide inductions resume from the last lane of their vector value
940 // in the last vector iteration which is handled elsewhere.
941 if (WideIntOrFp && WideIntOrFp->getTruncInst())
942 return nullptr;
943
944 VPIRValue *Start = WideIV->getStartValue();
945 VPValue *Step = WideIV->getStepValue();
946 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
947 VPValue *EndValue = VectorTC;
948 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
949 EndValue = VectorPHBuilder.createDerivedIV(
950 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
951 Start, Current: VectorTC, Step);
952 }
953
954 // EndValue is derived from the vector trip count (which has the same type as
955 // the widest induction) and thus may be wider than the induction here.
956 Type *ScalarTypeOfWideIV = WideIV->getScalarType();
957 if (ScalarTypeOfWideIV != EndValue->getScalarType()) {
958 EndValue = VectorPHBuilder.createScalarCast(Opcode: Instruction::Trunc, Op: EndValue,
959 ResultTy: ScalarTypeOfWideIV,
960 DL: WideIV->getDebugLoc());
961 }
962
963 return EndValue;
964}
965
966/// Attempts to optimize the induction variable exit values for users in the
967/// exit block coming from the latch in the original scalar loop.
968static VPValue *
969optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
970 DenseMap<VPValue *, VPValue *> &EndValues,
971 PredicatedScalarEvolution &PSE) {
972 VPValue *Incoming;
973 if (!match(V: Op, P: m_CombineOr(Ps: m_ExtractLastLaneOfLastPart(Op0: m_VPValue(V&: Incoming)),
974 Ps: m_ExtractLane(Op0: m_LastActiveLane(Op0: m_HeaderMask()),
975 Op1: m_VPValue(V&: Incoming)))))
976 return nullptr;
977
978 VPWidenInductionRecipe *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
979 if (!WideIV)
980 return nullptr;
981
982 VPValue *EndValue = EndValues.lookup(Val: WideIV);
983 assert(EndValue && "Must have computed the end value up front");
984
985 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
986 // changed it means the exit is using the incremented value, so we don't
987 // need to subtract the step.
988 if (Incoming != WideIV)
989 return EndValue;
990
991 // Otherwise, subtract the step from the EndValue.
992 auto *ExtractR = cast<VPInstruction>(Val: Op);
993 VPBuilder B(ExtractR);
994 VPValue *Step = WideIV->getStepValue();
995 Type *ScalarTy = WideIV->getScalarType();
996 if (ScalarTy->isIntegerTy())
997 return B.createSub(LHS: EndValue, RHS: Step, DL: DebugLoc::getUnknown(), Name: "ind.escape");
998 if (ScalarTy->isPointerTy()) {
999 Type *StepTy = Step->getScalarType();
1000 auto *Zero = Plan.getZero(Ty: StepTy);
1001 return B.createPtrAdd(Ptr: EndValue, Offset: B.createSub(LHS: Zero, RHS: Step),
1002 DL: DebugLoc::getUnknown(), Name: "ind.escape");
1003 }
1004 if (ScalarTy->isFloatingPointTy()) {
1005 const auto &ID = WideIV->getInductionDescriptor();
1006 return B.createNaryOp(
1007 Opcode: ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
1008 ? Instruction::FSub
1009 : Instruction::FAdd,
1010 Operands: {EndValue, Step}, Flags: {ID.getInductionBinOp()->getFastMathFlags()});
1011 }
1012 llvm_unreachable("all possible induction types must be handled");
1013 return nullptr;
1014}
1015
1016static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
1017 PredicatedScalarEvolution &PSE,
1018 VPValue *ResumeTC,
1019 const Loop *L) {
1020 VPValue *Incoming;
1021 if (!match(V: Op, P: m_ExtractLastLaneOfLastPart(Op0: m_VPValue(V&: Incoming))))
1022 return nullptr;
1023
1024 const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(V: Incoming, PSE, L);
1025 const SCEV *Start, *Step;
1026 if (!match(S: IncomingSCEV, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step),
1027 L: m_SpecificLoop(L))))
1028 return nullptr;
1029
1030 auto *ExtractR = cast<VPInstruction>(Val: Op);
1031 DebugLoc DL = ExtractR->getDebugLoc();
1032 VPBuilder Builder(ExtractR);
1033 VPSCEVExpander Expander(Builder, *PSE.getSE(), DL);
1034 VPValue *StartVPV = Expander.expand(S: Start);
1035 VPValue *StepVPV = Expander.expand(S: Step);
1036
1037 Type *StartTy = StartVPV->getScalarType();
1038 assert(StartTy->isIntOrPtrTy() && "The type must be SCEVable");
1039 InductionDescriptor::InductionKind Kind =
1040 StartTy->isPointerTy() ? InductionDescriptor::IK_PtrInduction
1041 : InductionDescriptor::IK_IntInduction;
1042 Type *TCTy = ResumeTC->getScalarType();
1043 VPValue *ExitCount = Builder.createOverflowingOp(
1044 Opcode: Instruction::Sub, Operands: {ResumeTC, Plan.getConstantInt(Ty: TCTy, Val: 1)},
1045 WrapFlags: {/*HasNUW=*/true, /*HasNSW=*/false}, DL: DebugLoc::getUnknown());
1046 return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, Start: StartVPV, Current: ExitCount,
1047 Step: StepVPV);
1048}
1049
1050void VPlanTransforms::optimizeInductionLiveOutUsers(
1051 VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L) {
1052 // Compute end values for all inductions.
1053 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
1054 auto *VectorPH = cast<VPBasicBlock>(Val: VectorRegion->getSinglePredecessor());
1055 VPBuilder VectorPHBuilder(VectorPH, VectorPH->begin());
1056 DenseMap<VPValue *, VPValue *> EndValues;
1057 VPValue *ResumeTC =
1058 Plan.hasTailFolded() ? Plan.getTripCount() : &Plan.getVectorTripCount();
1059 for (auto &Phi : VectorRegion->getEntryBasicBlock()->phis()) {
1060 auto *WideIV = dyn_cast<VPWidenInductionRecipe>(Val: &Phi);
1061 if (!WideIV)
1062 continue;
1063 if (VPValue *EndValue =
1064 tryToComputeEndValueForInduction(WideIV, VectorPHBuilder, VectorTC: ResumeTC))
1065 EndValues[WideIV] = EndValue;
1066 }
1067
1068 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
1069 for (VPRecipeBase &R : make_early_inc_range(Range&: *MiddleVPBB)) {
1070 VPValue *Op;
1071 if (!match(V: &R, P: m_ExitingIVValue(Op0: m_VPValue(V&: Op))))
1072 continue;
1073 auto *WideIV = cast<VPWidenInductionRecipe>(Val: Op);
1074 if (VPValue *EndValue = EndValues.lookup(Val: WideIV)) {
1075 R.getVPSingleValue()->replaceAllUsesWith(New: EndValue);
1076 R.eraseFromParent();
1077 }
1078 }
1079
1080 // Then, optimize exit block users.
1081 for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
1082 for (VPRecipeBase &R : ExitVPBB->phis()) {
1083 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
1084
1085 for (auto [Idx, PredVPBB] : enumerate(First&: ExitVPBB->getPredecessors())) {
1086 VPValue *Escape = nullptr;
1087 if (PredVPBB == MiddleVPBB) {
1088 Escape = optimizeLatchExitInductionUser(
1089 Plan, Op: ExitIRI->getOperand(N: Idx), EndValues, PSE);
1090 if (!Escape)
1091 Escape = optimizeLatchExitIVUserViaSCEV(
1092 Plan, Op: ExitIRI->getOperand(N: Idx), PSE, ResumeTC, L);
1093 } else {
1094 Escape = optimizeEarlyExitInductionUser(
1095 Plan, Op: ExitIRI->getOperand(N: Idx), PSE);
1096 }
1097 if (Escape)
1098 ExitIRI->setOperand(I: Idx, New: Escape);
1099 }
1100 }
1101 }
1102}
1103
1104/// Remove redundant ExpandSCEVRecipes in \p Plan's entry block by replacing
1105/// them with already existing recipes expanding the same SCEV expression.
1106static void removeRedundantExpandSCEVRecipes(VPlan &Plan) {
1107 DenseMap<const SCEV *, VPValue *> SCEV2VPV;
1108
1109 for (VPRecipeBase &R :
1110 make_early_inc_range(Range&: *Plan.getEntry()->getEntryBasicBlock())) {
1111 auto *ExpR = dyn_cast<VPExpandSCEVRecipe>(Val: &R);
1112 if (!ExpR)
1113 continue;
1114
1115 const auto &[V, Inserted] = SCEV2VPV.try_emplace(Key: ExpR->getSCEV(), Args&: ExpR);
1116 if (Inserted)
1117 continue;
1118
1119 ExpR->replaceAllUsesWith(New: V->second);
1120 if (ExpR == Plan.getTripCount())
1121 Plan.resetTripCount(NewTripCount: V->second);
1122
1123 ExpR->eraseFromParent();
1124 }
1125}
1126
1127/// Try to simplify logical and bitwise recipes in \p Def.
1128static VPValue *simplifyLogicalRecipe(VPSingleDefRecipe *Def,
1129 VPBuilder &Builder,
1130 bool CanCreateNewRecipe) {
1131 VPlan *Plan = Def->getParent()->getPlan();
1132
1133 // Simplify (X && Y) | (X && !Y) -> X.
1134 // TODO: Split up into simpler, modular combines: (X && Y) | (X && Z) into X
1135 // && (Y | Z) and (X | !X) into true. This requires queuing newly created
1136 // recipes to be visited during simplification.
1137 VPValue *X, *Y, *Z;
1138 if (match(R: Def,
1139 P: m_c_BinaryOr(Op0: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y)),
1140 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_Not(Op0: m_Deferred(V: Y))))))
1141 return X;
1142
1143 // x | AllOnes -> AllOnes
1144 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_AllOnes())))
1145 return Plan->getAllOnesValue(Ty: Def->getScalarType());
1146
1147 // x | 0 -> x
1148 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_ZeroInt())))
1149 return X;
1150
1151 // x | !x -> AllOnes
1152 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X)))))
1153 return Plan->getAllOnesValue(Ty: Def->getScalarType());
1154
1155 // x & 0 -> 0
1156 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_ZeroInt())))
1157 return Plan->getZero(Ty: Def->getScalarType());
1158
1159 // x & AllOnes -> x
1160 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_AllOnes())))
1161 return X;
1162
1163 // x && false -> false
1164 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_False())))
1165 return Plan->getFalse();
1166
1167 // x && true -> x
1168 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_True())))
1169 return X;
1170
1171 // (x && y) | (x && z) -> x && (y | z)
1172 if (CanCreateNewRecipe &&
1173 match(R: Def, P: m_c_BinaryOr(Op0: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y)),
1174 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue(V&: Z)))) &&
1175 // Simplify only if one of the operands has one use to avoid creating an
1176 // extra recipe.
1177 (!Def->getOperand(N: 0)->hasMoreThanOneUniqueUser() ||
1178 !Def->getOperand(N: 1)->hasMoreThanOneUniqueUser()))
1179 return Builder.createLogicalAnd(LHS: X, RHS: Builder.createOr(LHS: Y, RHS: Z));
1180
1181 // x && (x && y) -> x && y
1182 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1183 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue()))))
1184 return Def->getOperand(N: 1);
1185
1186 // x && (y && x) -> x && y
1187 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1188 Op1: m_LogicalAnd(Op0: m_VPValue(V&: Y), Op1: m_Deferred(V: X)))))
1189 return Builder.createLogicalAnd(LHS: X, RHS: Y);
1190
1191 // x && !x -> 0
1192 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X)))))
1193 return Plan->getFalse();
1194
1195 if (match(R: Def, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: X), Op2: m_Deferred(V: X))))
1196 return X;
1197
1198 // select c, false, true -> not c
1199 VPValue *C;
1200 if (CanCreateNewRecipe &&
1201 match(R: Def, P: m_Select(Op0: m_VPValue(V&: C), Op1: m_False(), Op2: m_True())))
1202 return Builder.createNot(Operand: C);
1203
1204 // select !c, x, y -> select c, y, x
1205 if (match(R: Def, P: m_Select(Op0: m_Not(Op0: m_VPValue(V&: C)), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1206 Def->setOperand(I: 0, New: C);
1207 Def->setOperand(I: 1, New: Y);
1208 Def->setOperand(I: 2, New: X);
1209 return Def;
1210 }
1211
1212 // select x, (i1 y | z), y -> y | (x && z)
1213 if (CanCreateNewRecipe &&
1214 match(R: Def, P: m_Select(Op0: m_VPValue(V&: X),
1215 Op1: m_OneUse(SubPattern: m_c_BinaryOr(Op0: m_VPValue(V&: Y), Op1: m_VPValue(V&: Z))),
1216 Op2: m_Deferred(V: Y))) &&
1217 Y->getScalarType()->isIntegerTy(BitWidth: 1))
1218 return Builder.createOr(LHS: Y, RHS: Builder.createLogicalAnd(LHS: X, RHS: Z));
1219
1220 // select %M0, (select %M1, %X, %Y), %Y -> select (%M0 && %M1), %X, %Y
1221 VPValue *Mask0, *Mask1;
1222 if (CanCreateNewRecipe &&
1223 match(R: Def,
1224 P: m_SelectLike(Op0: m_VPValue(V&: Mask0),
1225 Op1: m_OneUse(SubPattern: m_SelectLike(Op0: m_VPValue(V&: Mask1), Op1: m_VPValue(V&: X),
1226 Op2: m_VPValue(V&: Y))),
1227 Op2: m_Deferred(V: Y))))
1228 return Builder.createSelect(Cond: Builder.createLogicalAnd(LHS: Mask0, RHS: Mask1), TrueVal: X, FalseVal: Y,
1229 DL: Def->getDebugLoc());
1230
1231 return nullptr;
1232}
1233
1234/// Try to simplify VPSingleDefRecipe \p Def. Returns a new recipe if it should
1235/// be replaced, or the existing recipe if it was modified. Returns nullptr if
1236/// nothing was simplified.
1237static VPValue *simplifyRecipe(VPSingleDefRecipe *Def) {
1238 VPlan *Plan = Def->getParent()->getPlan();
1239
1240 // Simplification of live-in IR values for SingleDef recipes using
1241 // InstSimplifyFolder.
1242 const DataLayout &DL = Plan->getDataLayout();
1243 if (VPValue *V = vputils::tryToFoldLiveIns(R&: *Def, Operands: Def->operands(), DL))
1244 return V;
1245
1246 // Fold PredPHI LiveIn -> LiveIn.
1247 if (auto *PredPHI = dyn_cast<VPPredInstPHIRecipe>(Val: Def)) {
1248 VPValue *Op = PredPHI->getOperand(N: 0);
1249 if (isa<VPIRValue>(Val: Op))
1250 return Op;
1251 }
1252
1253 // Drop the mask of a predicated store masked by the header mask (which is
1254 // guaranteed to be true at least for the first lane) and both the stored
1255 // value and the address are uniform across VF and UF. The header mask is
1256 // still the abstract region value here.
1257 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: Def);
1258 RepR && RepR->isPredicated() && RepR->getOpcode() == Instruction::Store &&
1259 all_of(Range: RepR->operandsWithoutMask(), P: vputils::isUniformAcrossVFsAndUFs) &&
1260 match(V: RepR->getMask(), P: m_HeaderMask())) {
1261 auto *Unmasked = new VPReplicateRecipe(
1262 RepR->getUnderlyingInstr(), RepR->operandsWithoutMask(),
1263 RepR->isSingleScalar(), /*Mask=*/nullptr, *RepR, *RepR,
1264 RepR->getDebugLoc());
1265 Unmasked->insertBefore(InsertPos: RepR);
1266 return Unmasked;
1267 }
1268
1269 VPBuilder Builder(Def);
1270
1271 // Avoid replacing VPInstructions with underlying values with new
1272 // VPInstructions, as we would fail to create widen/replicate recpes from the
1273 // new VPInstructions without an underlying value, and miss out on some
1274 // transformations that only apply to widened/replicated recipes later, by
1275 // doing so.
1276 // TODO: We should also not replace non-VPInstructions like VPWidenRecipe with
1277 // VPInstructions without underlying values, as those will get skipped during
1278 // cost computation.
1279 bool CanCreateNewRecipe =
1280 !isa<VPInstruction>(Val: Def) || !Def->getUnderlyingValue();
1281
1282 VPValue *A, *Z;
1283 if (match(R: Def, P: m_Trunc(Op0: m_VPValue(V&: Z, Op: m_ZExtOrSExt(Op0: m_VPValue(V&: A)))))) {
1284 Type *TruncTy = Def->getScalarType();
1285 Type *ATy = A->getScalarType();
1286 if (TruncTy == ATy) {
1287 return A;
1288 } else {
1289 // Don't replace a non-widened cast recipe with a widened cast.
1290 if (!isa<VPWidenCastRecipe>(Val: Def))
1291 return nullptr;
1292 if (ATy->getScalarSizeInBits() < TruncTy->getScalarSizeInBits()) {
1293
1294 unsigned ExtOpcode = match(V: Z, P: m_SExt(Op0: m_VPValue())) ? Instruction::SExt
1295 : Instruction::ZExt;
1296 auto *Ext = Builder.createWidenCast(Opcode: Instruction::CastOps(ExtOpcode), Op: A,
1297 ResultTy: TruncTy);
1298 if (auto *UnderlyingExt = Z->getUnderlyingValue()) {
1299 // UnderlyingExt has distinct return type, used to retain legacy cost.
1300 Ext->setUnderlyingValue(UnderlyingExt);
1301 }
1302 return Ext;
1303 } else if (ATy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) {
1304 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::Trunc, Op: A, ResultTy: TruncTy);
1305 return Trunc;
1306 }
1307 }
1308 }
1309
1310 if (VPValue *V = simplifyLogicalRecipe(Def, Builder, CanCreateNewRecipe))
1311 return V;
1312
1313 VPValue *X, *Y;
1314 if (match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))
1315 return A;
1316
1317 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_One())))
1318 return A;
1319
1320 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))
1321 return Plan->getZero(Ty: Def->getScalarType());
1322
1323 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_AllOnes()))) {
1324 // Preserve nsw from the Mul on the new Sub.
1325 VPIRFlags::WrapFlagsTy NW = {
1326 false, cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap()};
1327 return Builder.createSub(LHS: Plan->getZero(Ty: A->getScalarType()), RHS: A,
1328 DL: Def->getDebugLoc(), Name: "", WrapFlags: NW);
1329 }
1330
1331 if (CanCreateNewRecipe &&
1332 match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: X),
1333 Op1: m_VPValue(V&: Z, Op: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Y)))))) {
1334 // Preserve nsw from the Add and the Sub, if it's present on both, on the
1335 // new Sub.
1336 VPIRFlags::WrapFlagsTy NW = {
1337 false, cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap() &&
1338 cast<VPRecipeWithIRFlags>(Val: Z)->hasNoSignedWrap()};
1339 return Builder.createSub(LHS: X, RHS: Y, DL: Def->getDebugLoc(), Name: "", WrapFlags: NW);
1340 }
1341
1342 const APInt *APC;
1343 if (CanCreateNewRecipe && match(R: Def, P: m_URem(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1344 APC->isPowerOf2())
1345 return Builder.createAnd(LHS: X, RHS: Plan->getConstantInt(Val: *APC - 1),
1346 DL: Def->getDebugLoc());
1347
1348 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_APInt(C&: APC))) &&
1349 APC->isPowerOf2()) {
1350 auto *MulR = cast<VPRecipeWithIRFlags>(Val: Def);
1351 unsigned ShiftAmt = APC->exactLogBase2();
1352 VPIRFlags::WrapFlagsTy NW(MulR->hasNoUnsignedWrap(),
1353 MulR->hasNoSignedWrap() &&
1354 ShiftAmt != APC->getBitWidth() - 1);
1355 return Builder.createNaryOp(
1356 Opcode: Instruction::Shl,
1357 Operands: {A, Plan->getConstantInt(BitWidth: APC->getBitWidth(), Val: ShiftAmt)}, Flags: NW,
1358 DL: Def->getDebugLoc());
1359 }
1360
1361 if (CanCreateNewRecipe && match(R: Def, P: m_UDiv(Op0: m_VPValue(V&: A), Op1: m_APInt(C&: APC))) &&
1362 APC->isPowerOf2())
1363 return Builder.createNaryOp(
1364 Opcode: Instruction::LShr,
1365 Operands: {A, Plan->getConstantInt(BitWidth: APC->getBitWidth(), Val: APC->exactLogBase2())},
1366 Flags: *cast<VPRecipeWithIRFlags>(Val: Def), DL: Def->getDebugLoc());
1367
1368 if (match(R: Def, P: m_Not(Op0: m_VPValue(V&: A)))) {
1369 if (match(V: A, P: m_Not(Op0: m_VPValue(V&: A))))
1370 return A;
1371
1372 // Try to fold Not into compares by adjusting the predicate in-place.
1373 CmpPredicate Pred;
1374 if (match(V: A, P: m_Cmp(Pred, Op0: m_VPValue(), Op1: m_VPValue()))) {
1375 auto *Cmp = cast<VPRecipeWithIRFlags>(Val: A);
1376 // Only fold if every user is a Not of the cmp, or a select using the cmp
1377 // solely as its condition.
1378 if (all_of(Range: Cmp->users(), P: [Cmp](VPUser *U) {
1379 return match(U, P: m_Not(Op0: m_Specific(VPV: Cmp))) ||
1380 (match(U, P: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(),
1381 Op2: m_VPValue())) &&
1382 U->getOperand(N: 1) != Cmp && U->getOperand(N: 2) != Cmp);
1383 })) {
1384 Cmp->setPredicate(CmpInst::getInversePredicate(pred: Pred));
1385 for (VPUser *U : to_vector(Range: Cmp->users())) {
1386 auto *R = cast<VPSingleDefRecipe>(Val: U);
1387 if (match(R, P: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1388 // select (cmp pred), x, y -> select (cmp inv_pred), y, x
1389 R->setOperand(I: 1, New: Y);
1390 R->setOperand(I: 2, New: X);
1391 } else {
1392 // not (cmp pred) -> cmp inv_pred
1393 assert(match(R, m_Not(m_Specific(Cmp))) && "Unexpected user");
1394 R->replaceAllUsesWith(New: Cmp);
1395 }
1396 }
1397 // If Cmp doesn't have a debug location, use the one from the negation,
1398 // to preserve the location.
1399 if (!Cmp->getDebugLoc() && Def->getDebugLoc())
1400 Cmp->setDebugLoc(Def->getDebugLoc());
1401 return Def;
1402 }
1403 }
1404 }
1405
1406 // Fold any-of (fcmp uno %A, %A), (fcmp uno %B, %B), ... ->
1407 // any-of (fcmp uno %A, %B), ...
1408 if (match(R: Def, P: m_AnyOf())) {
1409 SmallVector<VPValue *, 4> NewOps;
1410 VPRecipeBase *UnpairedCmp = nullptr;
1411 for (VPValue *Op : Def->operands()) {
1412 VPValue *X;
1413 if (Op->getNumUsers() > 1 ||
1414 !match(V: Op, P: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X),
1415 Op1: m_Deferred(V: X)))) {
1416 NewOps.push_back(Elt: Op);
1417 } else if (!UnpairedCmp) {
1418 UnpairedCmp = Op->getDefiningRecipe();
1419 } else {
1420 NewOps.push_back(Elt: Builder.createFCmp(Pred: CmpInst::FCMP_UNO,
1421 A: UnpairedCmp->getOperand(N: 0), B: X));
1422 UnpairedCmp = nullptr;
1423 }
1424 }
1425
1426 if (UnpairedCmp)
1427 NewOps.push_back(Elt: UnpairedCmp->getVPSingleValue());
1428
1429 if (NewOps.size() < Def->getNumOperands()) {
1430 VPValue *NewAnyOf = Builder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: NewOps);
1431 return NewAnyOf;
1432 }
1433 }
1434
1435 // Fold (fcmp uno %X, %X) or (fcmp uno %Y, %Y) -> fcmp uno %X, %Y
1436 // This is useful for fmax/fmin without fast-math flags, where we need to
1437 // check if any operand is NaN.
1438 if (CanCreateNewRecipe &&
1439 match(R: Def,
1440 P: m_BinaryOr(
1441 Op0: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X), Op1: m_Deferred(V: X)),
1442 Op1: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: Y), Op1: m_Deferred(V: Y)))))
1443 return Builder.createFCmp(Pred: CmpInst::FCMP_UNO, A: X, B: Y);
1444
1445 // Remove redundant DerviedIVs, that is 0 + A * 1 -> A and 0 + 0 * x -> 0.
1446 if ((match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_VPValue(V&: A), Op2: m_One())) ||
1447 match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_VPValue(V&: A, Op: m_ZeroInt()),
1448 Op2: m_VPValue()))) &&
1449 A->getScalarType() == Def->getScalarType())
1450 return A;
1451
1452 if (match(R: Def, P: m_VPInstruction<VPInstruction::WideIVStep>(Ops: m_VPValue(V&: X),
1453 Ops: m_One()))) {
1454 Type *WideStepTy = Def->getScalarType();
1455 if (X->getScalarType() != WideStepTy)
1456 X = Builder.createWidenCast(Opcode: Instruction::Trunc, Op: X, ResultTy: WideStepTy);
1457 return X;
1458 }
1459
1460 // For i1 vp.merges produced by AnyOf reductions:
1461 // vp.merge true, (or x, y), x, evl -> vp.merge y, true, x, evl
1462 if (match(R: Def, P: m_Intrinsic<Intrinsic::vp_merge>(Ops: m_True(), Ops: m_VPValue(V&: A),
1463 Ops: m_VPValue(V&: X), Ops: m_VPValue())) &&
1464 match(V: A, P: m_c_BinaryOr(Op0: m_Specific(VPV: X), Op1: m_VPValue(V&: Y))) &&
1465 Def->getScalarType()->isIntegerTy(BitWidth: 1)) {
1466 Def->setOperand(I: 1, New: Plan->getTrue());
1467 Def->setOperand(I: 0, New: Y);
1468 return Def;
1469 }
1470
1471 // Simplify MaskedCond with no block mask to its single operand.
1472 if (match(R: Def, P: m_VPInstruction<VPInstruction::MaskedCond>()) &&
1473 !cast<VPInstruction>(Val: Def)->isMasked())
1474 return Def->getOperand(N: 0);
1475
1476 // Look through ExtractLastLane.
1477 if (match(R: Def, P: m_ExtractLastLane(Op0: m_VPValue(V&: A)))) {
1478 if (match(V: A, P: m_BuildVector())) {
1479 auto *BuildVector = cast<VPInstruction>(Val: A);
1480 return BuildVector->getOperand(N: BuildVector->getNumOperands() - 1);
1481 }
1482
1483 if (match(V: A, P: m_Broadcast(Op0: m_VPValue(V&: X))))
1484 return X;
1485
1486 if (isa<VPInstruction, VPReplicateRecipe>(Val: A) && vputils::isSingleScalar(VPV: A))
1487 return A;
1488
1489 if (Plan->hasScalarVFOnly())
1490 return A;
1491 }
1492
1493 // Look through ExtractPenultimateElement (BuildVector ....).
1494 if (match(R: Def, P: m_ExtractPenultimateElement(Op0: m_BuildVector()))) {
1495 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1496 return BuildVector->getOperand(N: BuildVector->getNumOperands() - 2);
1497 }
1498
1499 uint64_t Idx;
1500 if (match(R: Def, P: m_ExtractElement(Op0: m_BuildVector(), Op1: m_ConstantInt(C&: Idx)))) {
1501 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1502 return BuildVector->getOperand(N: Idx);
1503 }
1504
1505 if (match(R: Def, P: m_BuildVector()) && all_equal(Range: Def->operands()))
1506 return Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: Def->getOperand(N: 0));
1507
1508 // Replace uses of a BuildVector by users that only use its first lane with
1509 // its first operand directly.
1510 if (match(R: Def, P: m_BuildVector())) {
1511 Def->replaceUsesWithIf(New: Def->getOperand(N: 0), ShouldReplace: [Def](VPUser &U, unsigned) {
1512 return U.usesFirstLaneOnly(Op: Def);
1513 });
1514 return Def;
1515 }
1516
1517 // Look through broadcast of single-scalar when used as select conditions; in
1518 // that case the scalar condition can be used directly.
1519 if (match(R: Def,
1520 P: m_Select(Op0: m_Broadcast(Op0: m_VPValue(V&: Z)), Op1: m_VPValue(), Op2: m_VPValue()))) {
1521 assert(vputils::isSingleScalar(Z) &&
1522 "broadcast operand must be single-scalar");
1523 Def->setOperand(I: 0, New: Z);
1524 return Def;
1525 }
1526
1527 if (match(R: Def, P: m_Broadcast(Op0: m_VPValue(V&: X)))) {
1528 Def->replaceUsesWithIf(
1529 New: X, ShouldReplace: [Def](const VPUser &U, unsigned) { return U.usesScalars(Op: Def); });
1530 return Def;
1531 }
1532
1533 if (isa<VPPhi, VPWidenPHIRecipe, VPHeaderPHIRecipe>(Val: Def)) {
1534 if (Def->getNumOperands() == 1) {
1535 return Def->getOperand(N: 0);
1536 }
1537 if (auto *Phi = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Val: Def)) {
1538 if (all_equal(Range: Phi->incoming_values()))
1539 return Phi->getOperand(N: 0);
1540 }
1541 return nullptr;
1542 }
1543
1544 VPIRValue *IRV;
1545 if (Def->getNumOperands() == 1 &&
1546 match(R: Def, P: m_ComputeReductionResult(Op0: m_VPIRValue(V&: IRV))))
1547 return IRV;
1548
1549 // Some simplifications can only be applied after unrolling. Perform them
1550 // below.
1551 if (!Plan->isUnrolled())
1552 return nullptr;
1553
1554 // After unrolling, extract-lane may be used to extract values from multiple
1555 // scalar sources. Only simplify when extracting from a single scalar source.
1556 VPValue *LaneToExtract;
1557 if (match(R: Def, P: m_ExtractLane(Op0: m_VPValue(V&: LaneToExtract), Op1: m_VPValue(V&: A)))) {
1558 // Simplify extract-lane(%lane_num, %scalar_val) -> %scalar_val.
1559 if (vputils::isSingleScalar(VPV: A))
1560 return A;
1561
1562 // Replace extract-lane(0, canonical-WIDEN-INDUCTION) with the region's
1563 // scalar canonical IV.
1564 VPWidenIntOrFpInductionRecipe *WidenIV;
1565 if (match(V: LaneToExtract, P: m_ZeroInt()) &&
1566 match(V: A, P: m_CanonicalWidenIV(V&: WidenIV)))
1567 return WidenIV->getRegion()->getCanonicalIV();
1568
1569 // Simplify extract-lane with single source to extract-element.
1570 return Builder.createNaryOp(Opcode: Instruction::ExtractElement, Operands: {A, LaneToExtract},
1571 DL: Def->getDebugLoc());
1572 }
1573
1574 // Look for cycles where Def is of the form:
1575 // X = phi(0, IVInc) ; used only by IVInc, or by IVInc and Inc = X + Y
1576 // IVInc = X + Step ; used by X and Def
1577 // Def = IVInc + Y
1578 // Fold the increment Y into the phi's start value, replace Def with IVInc,
1579 // and if Inc exists, replace it with X.
1580 VPValue *IVInc;
1581 if (match(R: Def, P: m_Add(Op0: m_VPValue(V&: IVInc, Op: m_Add(Op0: m_VPValue(V&: X), Op1: m_VPValue())),
1582 Op1: m_VPValue(V&: Y))) &&
1583 isa<VPIRValue>(Val: Y) && match(V: X, P: m_VPPhi(Op0: m_ZeroInt(), Op1: m_Specific(VPV: IVInc)))) {
1584 auto *Phi = cast<VPPhi>(Val: X);
1585 if (IVInc->getNumUsers() == 2) {
1586 // If Phi has a second user (besides IVInc's defining recipe), it must
1587 // be Inc = Phi + Y for the fold to apply.
1588 auto *Inc = dyn_cast_or_null<VPSingleDefRecipe>(
1589 Val: findUserOf(V: Phi, P: m_Add(Op0: m_Specific(VPV: Phi), Op1: m_Specific(VPV: Y))));
1590 if (Phi->getNumUsers() == 1 || (Phi->getNumUsers() == 2 && Inc)) {
1591 Def->replaceAllUsesWith(New: IVInc);
1592 if (Inc)
1593 Inc->replaceAllUsesWith(New: Phi);
1594 Phi->setOperand(I: 0, New: Y);
1595 return Def;
1596 }
1597 }
1598 }
1599
1600 // Simplify unrolled VectorPointer without offset, or with zero offset, to
1601 // just the pointer operand.
1602 if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(Val: Def))
1603 if (!VPR->getVFxPart() || match(V: VPR->getVFxPart(), P: m_ZeroInt()))
1604 return VPR->getOperand(N: 0);
1605
1606 // VPScalarIVSteps after unrolling can be replaced by their start value, if
1607 // the start index is zero and only the first lane 0 is demanded.
1608 if (auto *Steps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def))
1609 if (!Steps->getStartIndex() && vputils::onlyFirstLaneUsed(Def: Steps))
1610 return Steps->getOperand(N: 0);
1611
1612 // Simplify redundant ReductionStartVector recipes after unrolling.
1613 VPValue *StartV;
1614 if (match(R: Def, P: m_VPInstruction<VPInstruction::ReductionStartVector>(
1615 Ops: m_VPValue(V&: StartV), Ops: m_VPValue(), Ops: m_VPValue()))) {
1616 Def->replaceUsesWithIf(New: StartV, ShouldReplace: [](const VPUser &U, unsigned Idx) {
1617 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &U);
1618 return PhiR && PhiR->isInLoop();
1619 });
1620 return Def;
1621 }
1622
1623 if (Plan->getConcreteUF() == 1 && match(R: Def, P: m_ExtractLastPart(Op0: m_VPValue(V&: A))))
1624 return A;
1625
1626 return nullptr;
1627}
1628
1629void VPlanTransforms::simplifyRecipes(VPlan &Plan) {
1630 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
1631 Plan.getEntry());
1632 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
1633 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
1634 if (auto *Def = dyn_cast<VPSingleDefRecipe>(Val: &R))
1635 if (VPValue *New = simplifyRecipe(Def)) {
1636 if (New != Def) {
1637 // Replace the recipe with a new one.
1638 Def->replaceAllUsesWith(New);
1639 Def->eraseFromParent();
1640 } else if (vputils::isDeadRecipe(R)) {
1641 // Recipe was modified - it may be dead now.
1642 Def->eraseFromParent();
1643 }
1644 }
1645 }
1646}
1647
1648void VPlanTransforms::simplifyReverses(VPlan &Plan) {
1649 // Pull out reverses from any elementwise op.
1650 // binop(reverse(x), reverse(y)) -> reverse(binop(x,y))
1651 vputils::pullOutPermutations(
1652 Plan, Perm: [](VPValue *&X) { return m_Reverse(Op0: m_VPValue(V&: X)); },
1653 Build: [](auto *X) { return new VPInstruction(VPInstruction::Reverse, X); });
1654
1655 // reverse(reverse(x)) -> x
1656 VPValue *X;
1657 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1658 Range: vp_depth_first_deep(G: Plan.getEntry())))
1659 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
1660 if (match(V: &R, P: m_Reverse(Op0: m_Reverse(Op0: m_VPValue(V&: X)))))
1661 R.getVPSingleValue()->replaceAllUsesWith(New: X);
1662}
1663
1664/// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the
1665/// header mask to be simplified further when tail folding, e.g. in
1666/// optimizeEVLMasks.
1667static void reassociateHeaderMask(VPlan &Plan) {
1668 VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
1669 if (!HeaderMask)
1670 return;
1671
1672 SmallVector<VPUser *> Worklist;
1673 for (VPUser *U : HeaderMask->users())
1674 if (match(U, P: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue())))
1675 append_range(C&: Worklist, R: cast<VPSingleDefRecipe>(Val: U)->users());
1676
1677 while (!Worklist.empty()) {
1678 auto *R = dyn_cast<VPSingleDefRecipe>(Val: Worklist.pop_back_val());
1679 VPValue *X, *Y;
1680 if (!R || !match(R, P: m_LogicalAnd(
1681 Op0: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue(V&: X)),
1682 Op1: m_VPValue(V&: Y))))
1683 continue;
1684 append_range(C&: Worklist, R: R->users());
1685 VPBuilder Builder(R);
1686 R->replaceAllUsesWith(
1687 New: Builder.createLogicalAnd(LHS: HeaderMask, RHS: Builder.createLogicalAnd(LHS: X, RHS: Y)));
1688 }
1689}
1690
1691static std::optional<Instruction::BinaryOps>
1692getUnmaskedDivRemOpcode(Intrinsic::ID ID) {
1693 switch (ID) {
1694 case Intrinsic::masked_udiv:
1695 return Instruction::UDiv;
1696 case Intrinsic::masked_sdiv:
1697 return Instruction::SDiv;
1698 case Intrinsic::masked_urem:
1699 return Instruction::URem;
1700 case Intrinsic::masked_srem:
1701 return Instruction::SRem;
1702 default:
1703 return {};
1704 }
1705}
1706
1707static void narrowToSingleScalarRecipes(VPlan &Plan) {
1708 if (Plan.hasScalarVFOnly())
1709 return;
1710
1711 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1712 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
1713 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
1714 if (!isa<VPWidenRecipe, VPWidenGEPRecipe, VPReplicateRecipe,
1715 VPWidenIntrinsicRecipe>(Val: &R))
1716 continue;
1717 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
1718 if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
1719 continue;
1720
1721 auto *RepOrWidenR = cast<VPRecipeWithIRFlags>(Val: &R);
1722 if (RepR && RepR->getOpcode() == Instruction::Store &&
1723 vputils::isSingleScalar(VPV: RepR->getOperand(N: 1))) {
1724 auto *Clone = new VPReplicateRecipe(
1725 RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
1726 true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
1727 *RepR /*Metadata*/, RepR->getDebugLoc());
1728 Clone->insertBefore(InsertPos: RepOrWidenR);
1729 VPBuilder Builder(Clone);
1730 VPValue *ExtractOp = Clone->getOperand(N: 0);
1731 if (vputils::isUniformAcrossVFsAndUFs(V: RepR->getOperand(N: 1)))
1732 ExtractOp =
1733 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastPart, Operands: ExtractOp);
1734 ExtractOp =
1735 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastLane, Operands: ExtractOp);
1736 Clone->setOperand(I: 0, New: ExtractOp);
1737 RepR->eraseFromParent();
1738 continue;
1739 }
1740
1741 // Narrow llvm.masked.{u,s}{div,rem} intrinsics with a safe divisor.
1742 if (auto *IntrR = dyn_cast<VPWidenIntrinsicRecipe>(Val: RepOrWidenR)) {
1743 if (!vputils::onlyFirstLaneUsed(Def: IntrR))
1744 continue;
1745 auto Opc = getUnmaskedDivRemOpcode(ID: IntrR->getVectorIntrinsicID());
1746 if (!Opc)
1747 continue;
1748 VPBuilder Builder(IntrR);
1749 VPValue *SafeDivisor = Builder.createSelect(
1750 Cond: IntrR->getOperand(N: 2), TrueVal: IntrR->getOperand(N: 1),
1751 FalseVal: Plan.getConstantInt(Ty: IntrR->getScalarType(), Val: 1));
1752 VPValue *Clone = Builder.createNaryOp(
1753 Opcode: *Opc, Operands: {IntrR->getOperand(N: 0), SafeDivisor},
1754 Flags: VPIRFlags::getDefaultFlags(Opcode: *Opc), DL: IntrR->getDebugLoc());
1755 IntrR->replaceAllUsesWith(New: Clone);
1756 IntrR->eraseFromParent();
1757 continue;
1758 }
1759
1760 // Skip recipes that aren't single scalars.
1761 if (!vputils::isSingleScalar(VPV: RepOrWidenR))
1762 continue;
1763
1764 // Predicate to check if a user of Op introduces extra broadcasts.
1765 auto IntroducesBCastOf = [](const VPValue *Op) {
1766 return [Op](const VPUser *U) {
1767 if (auto *VPI = dyn_cast<VPInstruction>(Val: U)) {
1768 if (is_contained(Set: {VPInstruction::ExtractLastLane,
1769 VPInstruction::ExtractLastPart,
1770 VPInstruction::ExtractPenultimateElement},
1771 Element: VPI->getOpcode()))
1772 return false;
1773 }
1774 return !U->usesScalars(Op);
1775 };
1776 };
1777
1778 if (any_of(Range: RepOrWidenR->users(), P: IntroducesBCastOf(RepOrWidenR)) &&
1779 none_of(Range: RepOrWidenR->operands(), P: [&](VPValue *Op) {
1780 if (any_of(
1781 Range: make_filter_range(Range: Op->users(), Pred: not_equal_to(Arg&: RepOrWidenR)),
1782 P: IntroducesBCastOf(Op)))
1783 return false;
1784 // Non-constant live-ins require broadcasts, while constants do not
1785 // need explicit broadcasts.
1786 bool LiveInNeedsBroadcast =
1787 isa<VPIRValue>(Val: Op) && !isa<VPConstant>(Val: Op);
1788 auto *OpR = dyn_cast<VPReplicateRecipe>(Val: Op);
1789 return LiveInNeedsBroadcast || (OpR && OpR->isSingleScalar());
1790 }))
1791 continue;
1792
1793 auto *Clone = VPBuilder::createSingleScalarOp(
1794 Opcode: vputils::getOpcode(V: RepOrWidenR), Operands: RepOrWidenR->operands(),
1795 /*Mask=*/nullptr, Flags: *RepOrWidenR, Metadata: {}, DL: DebugLoc::getUnknown(),
1796 UV: RepOrWidenR->getUnderlyingInstr());
1797 Clone->insertBefore(InsertPos: RepOrWidenR);
1798 RepOrWidenR->replaceAllUsesWith(New: Clone);
1799 if (vputils::isDeadRecipe(R&: *RepOrWidenR))
1800 RepOrWidenR->eraseFromParent();
1801 }
1802 }
1803}
1804
1805/// Try to see if all of \p Blend's masks share a common value logically and'ed
1806/// and remove it from the masks.
1807static void removeCommonBlendMask(VPBlendRecipe *Blend) {
1808 if (Blend->isNormalized())
1809 return;
1810 VPValue *CommonEdgeMask;
1811 if (!match(V: Blend->getMask(Idx: 0),
1812 P: m_LogicalAnd(Op0: m_VPValue(V&: CommonEdgeMask), Op1: m_VPValue())))
1813 return;
1814 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1815 if (!match(V: Blend->getMask(Idx: I),
1816 P: m_LogicalAnd(Op0: m_Specific(VPV: CommonEdgeMask), Op1: m_VPValue())))
1817 return;
1818 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1819 Blend->setMask(Idx: I, V: Blend->getMask(Idx: I)->getDefiningRecipe()->getOperand(N: 1));
1820}
1821
1822/// Normalize and simplify VPBlendRecipes. Should be run after simplifyRecipes
1823/// to make sure the masks are simplified.
1824static void simplifyBlends(VPlan &Plan) {
1825 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1826 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
1827 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
1828 auto *Blend = dyn_cast<VPBlendRecipe>(Val: &R);
1829 if (!Blend)
1830 continue;
1831
1832 removeCommonBlendMask(Blend);
1833
1834 // Try to remove redundant blend recipes.
1835 SmallPtrSet<VPValue *, 4> UniqueValues;
1836 if (Blend->isNormalized() || !match(V: Blend->getMask(Idx: 0), P: m_False()))
1837 UniqueValues.insert(Ptr: Blend->getIncomingValue(Idx: 0));
1838 for (unsigned I = 1; I != Blend->getNumIncomingValues(); ++I)
1839 if (!match(V: Blend->getMask(Idx: I), P: m_False()))
1840 UniqueValues.insert(Ptr: Blend->getIncomingValue(Idx: I));
1841
1842 if (UniqueValues.size() == 1) {
1843 Blend->replaceAllUsesWith(New: *UniqueValues.begin());
1844 Blend->eraseFromParent();
1845 continue;
1846 }
1847
1848 if (Blend->isNormalized())
1849 continue;
1850
1851 // Normalize the blend so its first incoming value is used as the initial
1852 // value with the others blended into it.
1853
1854 unsigned StartIndex = 0;
1855 for (unsigned I = 0; I != Blend->getNumIncomingValues(); ++I) {
1856 // If a value's mask is used only by the blend then is can be deadcoded.
1857 // TODO: Find the most expensive mask that can be deadcoded, or a mask
1858 // that's used by multiple blends where it can be removed from them all.
1859 VPValue *Mask = Blend->getMask(Idx: I);
1860 if (Mask->hasOneUse() && !match(V: Mask, P: m_False())) {
1861 StartIndex = I;
1862 break;
1863 }
1864 }
1865
1866 SmallVector<VPValue *, 4> OperandsWithMask;
1867 OperandsWithMask.push_back(Elt: Blend->getIncomingValue(Idx: StartIndex));
1868
1869 for (unsigned I = 0; I != Blend->getNumIncomingValues(); ++I) {
1870 if (I == StartIndex)
1871 continue;
1872 OperandsWithMask.push_back(Elt: Blend->getIncomingValue(Idx: I));
1873 OperandsWithMask.push_back(Elt: Blend->getMask(Idx: I));
1874 }
1875
1876 auto *NewBlend =
1877 new VPBlendRecipe(cast_or_null<PHINode>(Val: Blend->getUnderlyingValue()),
1878 OperandsWithMask, *Blend, Blend->getDebugLoc());
1879 NewBlend->insertBefore(InsertPos: &R);
1880
1881 VPValue *DeadMask = Blend->getMask(Idx: StartIndex);
1882 Blend->replaceAllUsesWith(New: NewBlend);
1883 Blend->eraseFromParent();
1884 vputils::recursivelyDeleteDeadRecipes(V: DeadMask);
1885
1886 /// Simplify BLEND %a, %b, Not(%mask) -> BLEND %b, %a, %mask.
1887 VPValue *NewMask;
1888 if (NewBlend->getNumOperands() == 3 &&
1889 match(V: NewBlend->getMask(Idx: 1), P: m_Not(Op0: m_VPValue(V&: NewMask)))) {
1890 VPValue *Inc0 = NewBlend->getOperand(N: 0);
1891 VPValue *Inc1 = NewBlend->getOperand(N: 1);
1892 VPValue *OldMask = NewBlend->getOperand(N: 2);
1893 NewBlend->setOperand(I: 0, New: Inc1);
1894 NewBlend->setOperand(I: 1, New: Inc0);
1895 NewBlend->setOperand(I: 2, New: NewMask);
1896 if (OldMask->user_empty())
1897 cast<VPInstruction>(Val: OldMask)->eraseFromParent();
1898 }
1899 }
1900 }
1901}
1902
1903/// Optimize the width of vector induction variables in \p Plan based on a known
1904/// constant Trip Count, \p BestVF and \p BestUF.
1905static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan,
1906 ElementCount BestVF,
1907 unsigned BestUF) {
1908 // Only proceed if we have not completely removed the vector region.
1909 if (!Plan.getVectorLoopRegion())
1910 return false;
1911
1912 const APInt *TC;
1913 if (!BestVF.isFixed() || !match(V: Plan.getTripCount(), P: m_APInt(C&: TC)))
1914 return false;
1915
1916 // Calculate the minimum power-of-2 bit width that can fit the known TC, VF
1917 // and UF. Returns at least 8.
1918 auto ComputeBitWidth = [](APInt TC, uint64_t Align) {
1919 APInt AlignedTC =
1920 Align * APIntOps::RoundingUDiv(A: TC, B: APInt(TC.getBitWidth(), Align),
1921 RM: APInt::Rounding::UP);
1922 APInt MaxVal = AlignedTC - 1;
1923 return std::max<unsigned>(a: PowerOf2Ceil(A: MaxVal.getActiveBits()), b: 8);
1924 };
1925 unsigned NewBitWidth =
1926 ComputeBitWidth(*TC, BestVF.getKnownMinValue() * BestUF);
1927
1928 LLVMContext &Ctx = Plan.getContext();
1929 auto *NewIVTy = IntegerType::get(C&: Ctx, NumBits: NewBitWidth);
1930
1931 bool MadeChange = false;
1932
1933 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
1934 for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
1935 // Currently only handle canonical IVs as it is trivial to replace the start
1936 // and stop values, and we currently only perform the optimization when the
1937 // IV has a single use.
1938 VPWidenIntOrFpInductionRecipe *WideIV;
1939 if (!match(V: &Phi, P: m_CanonicalWidenIV(V&: WideIV)))
1940 continue;
1941 if (WideIV->hasMoreThanOneUniqueUser() ||
1942 NewIVTy == WideIV->getScalarType())
1943 continue;
1944
1945 // Currently only handle cases where the single user is a header-mask
1946 // comparison with the backedge-taken-count.
1947 VPUser *SingleUser = WideIV->getSingleUser();
1948 if (!SingleUser ||
1949 !match(U: SingleUser,
1950 P: m_ICmp(Op0: m_Specific(VPV: WideIV),
1951 Op1: m_Broadcast(Op0: m_Specific(VPV: Plan.getBackedgeTakenCount())))))
1952 continue;
1953
1954 // Update IV operands and comparison bound to use new narrower type.
1955 assert(!WideIV->getTruncInst() &&
1956 "canonical IV is not expected to have a truncation");
1957 auto *NewWideIV = new VPWidenIntOrFpInductionRecipe(
1958 WideIV->getPHINode(), Plan.getZero(Ty: NewIVTy),
1959 Plan.getConstantInt(Ty: NewIVTy, Val: 1), WideIV->getVFValue(),
1960 WideIV->getInductionDescriptor(), *WideIV, WideIV->getDebugLoc());
1961 NewWideIV->insertBefore(InsertPos: WideIV);
1962
1963 auto *NewBTC = new VPWidenCastRecipe(
1964 Instruction::Trunc, Plan.getOrCreateBackedgeTakenCount(), NewIVTy,
1965 nullptr, VPIRFlags::getDefaultFlags(Opcode: Instruction::Trunc));
1966 Plan.getVectorPreheader()->appendRecipe(Recipe: NewBTC);
1967 auto *Cmp = cast<VPInstruction>(Val: WideIV->getSingleUser());
1968 Cmp->replaceAllUsesWith(
1969 New: VPBuilder(Cmp).createICmp(Pred: Cmp->getPredicate(), A: NewWideIV, B: NewBTC));
1970
1971 MadeChange = true;
1972 }
1973
1974 return MadeChange;
1975}
1976
1977/// Return true if \p Cond is known to be true for given \p BestVF and \p
1978/// BestUF.
1979static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
1980 ElementCount BestVF, unsigned BestUF,
1981 PredicatedScalarEvolution &PSE) {
1982 if (match(V: Cond, P: m_BinaryOr(Op0: m_VPValue(), Op1: m_VPValue())))
1983 return any_of(Range: Cond->getDefiningRecipe()->operands(), P: [&Plan, BestVF, BestUF,
1984 &PSE](VPValue *C) {
1985 return isConditionTrueViaVFAndUF(Cond: C, Plan, BestVF, BestUF, PSE);
1986 });
1987
1988 auto *CanIV = Plan.getVectorLoopRegion()->getCanonicalIV();
1989 if (!match(V: Cond, P: m_SpecificICmp(
1990 MatchPred: CmpInst::ICMP_EQ,
1991 Op0: m_c_Add(Op0: m_Specific(VPV: CanIV), Op1: m_Specific(VPV: &Plan.getVFxUF())),
1992 Op1: m_Specific(VPV: &Plan.getVectorTripCount()))))
1993 return false;
1994
1995 // The compare checks CanIV + VFxUF == vector trip count. The vector trip
1996 // count is not conveniently available as SCEV so far, so we compare directly
1997 // against the original trip count. This is stricter than necessary, as we
1998 // will only return true if the trip count == vector trip count.
1999 const SCEV *VectorTripCount =
2000 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2001 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2002 VectorTripCount = vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2003 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2004 "Trip count SCEV must be computable");
2005 ScalarEvolution &SE = *PSE.getSE();
2006 ElementCount NumElements = BestVF.multiplyCoefficientBy(RHS: BestUF);
2007 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2008 return SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: VectorTripCount, RHS: C);
2009}
2010
2011// Replaces ExtractVectorForPart instructions with ICMP when the VF is scalar
2012// and the source is a WideActiveLaneMask. The unused mask is removed later
2013// when removing dead recipes.
2014static bool replaceMaskWithCompareForScalarPlan(VPlan &Plan,
2015 ElementCount BestVF) {
2016 if (!BestVF.isScalar())
2017 return false;
2018
2019 bool MadeChange = false;
2020 VPBuilder Builder;
2021 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2022 VPBasicBlock *PreheaderVPBB = Plan.getVectorPreheader();
2023 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2024
2025 VPValue *Start, *TC;
2026 uint64_t Idx;
2027 for (VPBasicBlock *VPBB : {PreheaderVPBB, ExitingVPBB}) {
2028 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2029 if (!match(V: &R, P: m_ExtractVectorForPart(
2030 Op0: m_WideActiveLaneMask(Op0: m_VPValue(V&: Start), Op1: m_VPValue(V&: TC),
2031 Op2: m_VPValue()),
2032 Op1: m_ConstantInt(C&: Idx))))
2033 continue;
2034
2035 auto *Extract = cast<VPInstruction>(Val: &R);
2036 Builder.setInsertPoint(Extract);
2037
2038 if (Idx > 0)
2039 Start = Builder.createAdd(
2040 LHS: Start, RHS: Plan.getConstantInt(Ty: Start->getScalarType(), Val: Idx));
2041
2042 VPValue *ICmp = Builder.createICmp(Pred: CmpInst::ICMP_ULT, A: Start, B: TC);
2043 Extract->replaceAllUsesWith(New: ICmp);
2044 Extract->eraseFromParent();
2045 MadeChange = true;
2046 }
2047 }
2048
2049 return MadeChange;
2050}
2051
2052/// Try to simplify the branch condition of \p Plan. This may restrict the
2053/// resulting plan to \p BestVF and \p BestUF.
2054static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
2055 unsigned BestUF,
2056 PredicatedScalarEvolution &PSE) {
2057 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2058 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2059 auto *Term = &ExitingVPBB->back();
2060 VPValue *Cond;
2061 auto m_CanIVInc = m_Add(Op0: m_VPValue(), Op1: m_Specific(VPV: &Plan.getVFxUF()));
2062 // Check if the branch condition compares the canonical IV increment (for main
2063 // loop), or the canonical IV increment plus an offset (for epilog loop).
2064 if (match(V: Term, P: m_BranchOnCount(
2065 Op0: m_CombineOr(Ps: m_CanIVInc, Ps: m_c_Add(Op0: m_CanIVInc, Op1: m_LiveIn())),
2066 Op1: m_VPValue())) ||
2067 match(V: Term,
2068 P: m_BranchOnCond(Op0: m_Not(Op0: m_ExtractVectorForPart(
2069 Op0: m_WideActiveLaneMask(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue()),
2070 Op1: m_ZeroInt()))))) {
2071 // Try to simplify the branch condition if VectorTC <= VF * UF when the
2072 // latch terminator is BranchOnCount or
2073 // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask), 0))
2074 const SCEV *VectorTripCount =
2075 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2076 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2077 VectorTripCount =
2078 vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2079 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2080 "Trip count SCEV must be computable");
2081 ScalarEvolution &SE = *PSE.getSE();
2082 ElementCount NumElements = BestVF.multiplyCoefficientBy(RHS: BestUF);
2083 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2084 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_ULE, LHS: VectorTripCount, RHS: C))
2085 return false;
2086 } else if (match(V: Term, P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))) ||
2087 match(V: Term, P: m_BranchOnTwoConds(Op0: m_VPValue(), Op1: m_VPValue(V&: Cond)))) {
2088 // For BranchOnCond, check if we can prove the condition to be true using VF
2089 // and UF.
2090 if (!isConditionTrueViaVFAndUF(Cond, Plan, BestVF, BestUF, PSE))
2091 return false;
2092 } else {
2093 return false;
2094 }
2095
2096 // The vector loop region only executes once. Convert terminator of the
2097 // exiting block to exit in the first iteration.
2098 if (match(V: Term, P: m_BranchOnTwoConds())) {
2099 Term->setOperand(I: 1, New: Plan.getTrue());
2100 return true;
2101 }
2102
2103 auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, Plan.getTrue(), {},
2104 {}, Term->getDebugLoc());
2105 ExitingVPBB->appendRecipe(Recipe: BOC);
2106 Term->eraseFromParent();
2107
2108 return true;
2109}
2110
2111void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
2112 unsigned BestUF,
2113 PredicatedScalarEvolution &PSE) {
2114 assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
2115 assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
2116
2117 bool MadeChange =
2118 simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
2119 MadeChange |= replaceMaskWithCompareForScalarPlan(Plan, BestVF);
2120 MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
2121
2122 if (MadeChange) {
2123 Plan.setVF(BestVF);
2124 assert(Plan.getConcreteUF() == BestUF && "BestUF must match the Plan's UF");
2125 }
2126}
2127
2128void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
2129 for (VPRecipeBase &R :
2130 Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
2131 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
2132 if (!PhiR)
2133 continue;
2134 RecurKind RK = PhiR->getRecurrenceKind();
2135 if (RK != RecurKind::Add && RK != RecurKind::Mul && RK != RecurKind::Sub &&
2136 RK != RecurKind::AddChainWithSubs)
2137 continue;
2138
2139 for (VPUser *U : vputils::collectUsersRecursively(V: PhiR))
2140 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: U)) {
2141 RecWithFlags->dropPoisonGeneratingFlags();
2142 }
2143 }
2144}
2145
2146namespace {
2147struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
2148 /// If recipe \p R will lower to a GEP with a non-i8 source element type,
2149 /// return that source element type.
2150 static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) {
2151 // All VPInstructions that lower to GEPs must have the i8 source element
2152 // type (as they are PtrAdds), so we omit it.
2153 return TypeSwitch<const VPSingleDefRecipe *, Type *>(R)
2154 .Case(caseFn: [](const VPReplicateRecipe *I) -> Type * {
2155 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I->getUnderlyingValue()))
2156 return GEP->getSourceElementType();
2157 return nullptr;
2158 })
2159 .Case<VPVectorPointerRecipe, VPWidenGEPRecipe>(
2160 caseFn: [](auto *I) { return I->getSourceElementType(); })
2161 .Default(defaultFn: [](auto *) { return nullptr; });
2162 }
2163
2164 /// Returns true if recipe \p Def can be safely handed for CSE.
2165 static bool canHandle(const VPSingleDefRecipe *Def) {
2166 // We can extend the list of handled recipes in the future,
2167 // provided we account for the data embedded in them while checking for
2168 // equality or hashing.
2169 auto C = vputils::getOpcodeOrIntrinsicID(V: Def);
2170
2171 // The issue with (Insert|Extract)Value is that the index of the
2172 // insert/extract is not a proper operand in LLVM IR, and hence also not in
2173 // VPlan.
2174 if (!C || (!C->first && (C->second == Instruction::InsertValue ||
2175 C->second == Instruction::ExtractValue)))
2176 return false;
2177
2178 // During CSE, we can only handle non-memory recipes, as memory can alias.
2179 return !Def->mayReadOrWriteMemory();
2180 }
2181
2182 /// Hash the underlying data of \p Def.
2183 static unsigned getHashValue(const VPSingleDefRecipe *Def) {
2184 hash_code Result = hash_combine(
2185 args: Def->getVPRecipeID(), args: vputils::getOpcodeOrIntrinsicID(V: Def),
2186 args: getGEPSourceElementType(R: Def), args: Def->getScalarType(),
2187 args: vputils::isSingleScalar(VPV: Def), args: hash_combine_range(R: Def->operands()));
2188 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: Def))
2189 if (RFlags->hasPredicate())
2190 return hash_combine(args: Result, args: RFlags->getPredicate());
2191 if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def))
2192 return hash_combine(args: Result, args: SIVSteps->getInductionOpcode());
2193 return Result;
2194 }
2195
2196 /// Check equality of underlying data of \p L and \p R.
2197 static bool isEqual(const VPSingleDefRecipe *L, const VPSingleDefRecipe *R) {
2198 if (L->getVPRecipeID() != R->getVPRecipeID() ||
2199 vputils::getOpcodeOrIntrinsicID(V: L) !=
2200 vputils::getOpcodeOrIntrinsicID(V: R) ||
2201 getGEPSourceElementType(R: L) != getGEPSourceElementType(R) ||
2202 vputils::isSingleScalar(VPV: L) != vputils::isSingleScalar(VPV: R) ||
2203 !equal(LRange: L->operands(), RRange: R->operands()))
2204 return false;
2205 assert(vputils::getOpcodeOrIntrinsicID(L) &&
2206 vputils::getOpcodeOrIntrinsicID(R) &&
2207 "must have valid opcode info for both recipes");
2208 if (auto *LFlags = dyn_cast<VPRecipeWithIRFlags>(Val: L))
2209 if (LFlags->hasPredicate() &&
2210 LFlags->getPredicate() !=
2211 cast<VPRecipeWithIRFlags>(Val: R)->getPredicate())
2212 return false;
2213 if (auto *LSIV = dyn_cast<VPScalarIVStepsRecipe>(Val: L))
2214 if (LSIV->getInductionOpcode() !=
2215 cast<VPScalarIVStepsRecipe>(Val: R)->getInductionOpcode())
2216 return false;
2217 // Phi recipes can only be equal if they are in the same VPBB, as they
2218 // implicitly depend on their predecessors.
2219 if (isa<VPWidenPHIRecipe>(Val: L) && L->getParent() != R->getParent())
2220 return false;
2221 // Recipes in replicate regions implicitly depend on predicate. If either
2222 // recipe is in a replicate region, only consider them equal if both have
2223 // the same parent.
2224 const VPRegionBlock *RegionL = L->getRegion();
2225 const VPRegionBlock *RegionR = R->getRegion();
2226 if (((RegionL && RegionL->isReplicator()) ||
2227 (RegionR && RegionR->isReplicator())) &&
2228 L->getParent() != R->getParent())
2229 return false;
2230 return L->getScalarType() == R->getScalarType();
2231 }
2232};
2233} // end anonymous namespace
2234
2235/// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
2236/// Plan.
2237void VPlanTransforms::cse(VPlan &Plan) {
2238 VPDominatorTree VPDT(Plan);
2239 DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo> CSEMap;
2240
2241 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
2242 Plan.getEntry());
2243 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
2244 for (VPRecipeBase &R : *VPBB) {
2245 auto *Def = dyn_cast<VPSingleDefRecipe>(Val: &R);
2246 if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
2247 continue;
2248 if (VPSingleDefRecipe *V = CSEMap.lookup(Val: Def)) {
2249 // V must dominate Def for a valid replacement.
2250 if (!VPDT.dominates(A: V->getParent(), B: VPBB))
2251 continue;
2252 // Only keep flags present on both V and Def.
2253 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: V))
2254 RFlags->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Def));
2255 Def->replaceAllUsesWith(New: V);
2256 continue;
2257 }
2258 CSEMap[Def] = Def;
2259 }
2260 }
2261}
2262
2263/// Return true if we do not know how to (mechanically) hoist or sink a
2264/// non-memory or memory recipe \p R out of a loop region. When sinking, passing
2265/// \p Sinking = true ensures that assumes aren't sunk.
2266static bool cannotHoistOrSinkRecipe(VPRecipeBase &R, VPBasicBlock *FirstBB,
2267 VPBasicBlock *LastBB,
2268 bool Sinking = false) {
2269 if (!isa<VPReplicateRecipe>(Val: R) || !R.mayReadOrWriteMemory() ||
2270 match(V: &R, P: m_Intrinsic<Intrinsic::assume>()))
2271 return vputils::cannotHoistOrSinkRecipe(R, Sinking);
2272
2273 // Check that the memory operation doesn't alias between FirstBB and LastBB.
2274 auto MemLoc = vputils::getMemoryLocation(R);
2275
2276 // TODO: Could make use of SinkStoreInfo::isNoAliasViaDistance by collecting
2277 // stores upfront, and constructing a full SinkStoreInfo.
2278 auto SinkInfo =
2279 Sinking ? std::make_optional(t: SinkStoreInfo(cast<VPReplicateRecipe>(Val&: R)))
2280 : std::nullopt;
2281
2282 return !MemLoc ||
2283 !canHoistOrSinkWithNoAliasCheck(MemLoc: *MemLoc, FirstBB, LastBB, SinkInfo);
2284}
2285
2286/// Move loop-invariant recipes out of the vector loop region in \p Plan.
2287static void licm(VPlan &Plan) {
2288 VPBasicBlock *Preheader = Plan.getVectorPreheader();
2289
2290 // Hoist any loop invariant recipes from the vector loop region to the
2291 // preheader. Preform a shallow traversal of the vector loop region, to
2292 // exclude recipes in replicate regions. Since the top-level blocks in the
2293 // vector loop region are guaranteed to execute if the vector pre-header is,
2294 // we don't need to check speculation safety.
2295 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2296 assert(Preheader->getSingleSuccessor() == LoopRegion &&
2297 "Expected vector prehader's successor to be the vector loop region");
2298 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2299 Range: vp_depth_first_shallow(G: LoopRegion->getEntry()))) {
2300 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2301 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2302 LastBB: LoopRegion->getExitingBasicBlock()))
2303 continue;
2304 if (any_of(Range: R.operands(), P: [](VPValue *Op) {
2305 return !Op->isDefinedOutsideLoopRegions();
2306 }))
2307 continue;
2308 R.moveBefore(BB&: *Preheader, I: Preheader->end());
2309 }
2310 }
2311
2312#ifndef NDEBUG
2313 VPDominatorTree VPDT(Plan);
2314#endif
2315 // Sink recipes with no users inside the vector loop region if all users are
2316 // in the same exit block of the region.
2317 // TODO: Extend to sink recipes from inner loops.
2318 PostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> POT(
2319 LoopRegion->getEntry());
2320 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
2321 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
2322 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2323 LastBB: LoopRegion->getExitingBasicBlock(),
2324 /*Sinking=*/true))
2325 continue;
2326
2327 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
2328 assert(!RepR->isPredicated() &&
2329 "Expected prior transformation of predicated replicates to "
2330 "replicate regions");
2331 // narrowToSingleScalarRecipes should have already maximally narrowed
2332 // replicates to single-scalar replicates.
2333 // TODO: When unrolling, replicateByVF doesn't handle sunk
2334 // non-single-scalar replicates correctly.
2335 if (!RepR->isSingleScalar())
2336 continue;
2337
2338 // The pointer operand of stores must be loop-invariant.
2339 if (RepR->getOpcode() == Instruction::Store &&
2340 !RepR->getOperand(N: 1)->isDefinedOutsideLoopRegions())
2341 continue;
2342 }
2343
2344 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
2345 assert((!R.mayWriteToMemory() ||
2346 (RepR && RepR->getOpcode() == Instruction::Store &&
2347 RepR->getOperand(1)->isDefinedOutsideLoopRegions())) &&
2348 "The only recipes that may write to memory are expected to be "
2349 "stores with invariant pointer-operand");
2350
2351 // TODO: Use R.definedValues() instead of casting to VPSingleDefRecipe to
2352 // support recipes with multiple defined values (e.g., interleaved loads).
2353 auto *Def = cast<VPSingleDefRecipe>(Val: &R);
2354
2355 // Cannot sink the recipe if the user is defined in a loop region or a
2356 // non-successor of the vector loop region. Cannot sink if user is a phi
2357 // either.
2358 VPBasicBlock *SinkBB = nullptr;
2359 if (any_of(Range: Def->users(), P: [&SinkBB, &LoopRegion](VPUser *U) {
2360 auto *UserR = cast<VPRecipeBase>(Val: U);
2361 VPBasicBlock *Parent = UserR->getParent();
2362 // TODO: Support sinking when users are in multiple blocks.
2363 if (SinkBB && SinkBB != Parent)
2364 return true;
2365 SinkBB = Parent;
2366 // TODO: If the user is a PHI node, we should check the block of
2367 // incoming value. Support PHI node users if needed.
2368 return UserR->isPhi() || Parent->getEnclosingLoopRegion() ||
2369 Parent->getSinglePredecessor() != LoopRegion;
2370 }))
2371 continue;
2372
2373 if (!SinkBB)
2374 SinkBB = cast<VPBasicBlock>(Val: LoopRegion->getSingleSuccessor());
2375
2376 // TODO: This will need to be a check instead of a assert after
2377 // conditional branches in vectorized loops are supported.
2378 assert(VPDT.properlyDominates(VPBB, SinkBB) &&
2379 "Defining block must dominate sink block");
2380 // TODO: Clone the recipe if users are on multiple exit paths, instead of
2381 // just moving.
2382 Def->moveBefore(BB&: *SinkBB, I: SinkBB->getFirstNonPhi());
2383 }
2384 }
2385}
2386
2387void VPlanTransforms::truncateToMinimalBitwidths(
2388 VPlan &Plan, const MapVector<Instruction *, uint64_t> &MinBWs) {
2389 if (Plan.hasScalarVFOnly())
2390 return;
2391 // Keep track of created truncates, so they can be re-used. Note that we
2392 // cannot use RAUW after creating a new truncate, as this would could make
2393 // other uses have different types for their operands, making them invalidly
2394 // typed.
2395 DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
2396 VPBasicBlock *PH = Plan.getVectorPreheader();
2397 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2398 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
2399 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2400 if (!isa<VPWidenRecipe, VPWidenCastRecipe, VPReplicateRecipe,
2401 VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R))
2402 continue;
2403
2404 VPValue *ResultVPV = R.getVPSingleValue();
2405 auto *UI = cast_or_null<Instruction>(Val: ResultVPV->getUnderlyingValue());
2406 unsigned NewResSizeInBits = MinBWs.lookup(Key: UI);
2407 if (!NewResSizeInBits)
2408 continue;
2409
2410 // If the value wasn't vectorized, we must maintain the original scalar
2411 // type. Skip those here, after incrementing NumProcessedRecipes. Also
2412 // skip casts which do not need to be handled explicitly here, as
2413 // redundant casts will be removed during recipe simplification.
2414 if (isa<VPReplicateRecipe, VPWidenCastRecipe>(Val: &R))
2415 continue;
2416
2417 Type *OldResTy = ResultVPV->getScalarType();
2418 unsigned OldResSizeInBits = OldResTy->getScalarSizeInBits();
2419 assert(OldResTy->isIntegerTy() && "only integer types supported");
2420 (void)OldResSizeInBits;
2421
2422 auto *NewResTy = IntegerType::get(C&: Plan.getContext(), NumBits: NewResSizeInBits);
2423
2424 // Any wrapping introduced by shrinking this operation shouldn't be
2425 // considered undefined behavior. So, we can't unconditionally copy
2426 // arithmetic wrapping flags to VPW.
2427 if (auto *VPW = dyn_cast<VPRecipeWithIRFlags>(Val: &R))
2428 VPW->dropPoisonGeneratingFlags();
2429
2430 assert((OldResSizeInBits != NewResSizeInBits ||
2431 match(&R, m_ICmp(m_VPValue(), m_VPValue()))) &&
2432 "Only ICmps should not need extending the result.");
2433 assert(!isa<VPWidenStoreRecipe>(&R) && "stores cannot be narrowed");
2434
2435 // For loads/intrinsics we don't recreate the recipe; just wrap the
2436 // original wide result in a ZExt to OldResTy.
2437 if (isa<VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R)) {
2438 if (OldResSizeInBits != NewResSizeInBits) {
2439 auto *Ext = VPBuilder::getToInsertAfter(R: &R).createWidenCast(
2440 Opcode: Instruction::ZExt, Op: ResultVPV, ResultTy: OldResTy);
2441 ResultVPV->replaceAllUsesWith(New: Ext);
2442 Ext->setOperand(I: 0, New: ResultVPV);
2443 }
2444 continue;
2445 }
2446
2447 // Shrink operands by introducing truncates as needed.
2448 unsigned StartIdx =
2449 match(V: &R, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())) ? 1 : 0;
2450 SmallVector<VPValue *> NewOperands(R.operands());
2451 for (VPValue *&Op : drop_begin(RangeOrContainer&: NewOperands, N: StartIdx)) {
2452 unsigned OpSizeInBits = Op->getScalarType()->getScalarSizeInBits();
2453 if (OpSizeInBits == NewResSizeInBits)
2454 continue;
2455 assert(OpSizeInBits > NewResSizeInBits && "nothing to truncate");
2456 auto [ProcessedIter, Inserted] = ProcessedTruncs.try_emplace(Key: Op);
2457 if (Inserted) {
2458 VPBuilder Builder;
2459 if (isa<VPIRValue>(Val: Op))
2460 Builder.setInsertPoint(PH);
2461 else
2462 Builder.setInsertPoint(&R);
2463 ProcessedIter->second =
2464 Builder.createWidenCast(Opcode: Instruction::Trunc, Op, ResultTy: NewResTy);
2465 }
2466 Op = ProcessedIter->second;
2467 }
2468
2469 auto *NWR = cast<VPWidenRecipe>(Val: &R)->cloneWithOperands(NewOperands);
2470 NWR->insertBefore(InsertPos: &R);
2471
2472 // Wrap NWR in a ZExt to preserve the original wide type for downstream
2473 // users (unless this is an ICmp, which produces i1 regardless).
2474 VPValue *Replacement = NWR->getVPSingleValue();
2475 if (OldResSizeInBits != NewResSizeInBits)
2476 Replacement =
2477 VPBuilder::getToInsertAfter(R: NWR)
2478 .createWidenCast(Opcode: Instruction::ZExt, Op: Replacement, ResultTy: OldResTy)
2479 ->getVPSingleValue();
2480 ResultVPV->replaceAllUsesWith(New: Replacement);
2481 R.eraseFromParent();
2482 }
2483 }
2484}
2485
2486bool VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) {
2487 std::optional<VPDominatorTree> VPDT;
2488 if (OnlyLatches)
2489 VPDT.emplace(args&: Plan);
2490
2491 // Collect all blocks before modifying the CFG so we can identify unreachable
2492 // ones after constant branch removal.
2493 SmallVector<VPBlockBase *> AllBlocks(vp_depth_first_shallow(G: Plan.getEntry()));
2494
2495 bool SimplifiedPhi = false;
2496 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: AllBlocks)) {
2497 VPValue *Cond;
2498 // Skip blocks that are not terminated by BranchOnCond.
2499 if (VPBB->empty() || !match(V: &VPBB->back(), P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))))
2500 continue;
2501
2502 if (OnlyLatches && !VPBlockUtils::isLatch(VPB: VPBB, VPDT: *VPDT))
2503 continue;
2504
2505 assert(VPBB->getNumSuccessors() == 2 &&
2506 "Two successors expected for BranchOnCond");
2507 unsigned RemovedIdx;
2508 if (match(V: Cond, P: m_True()))
2509 RemovedIdx = 1;
2510 else if (match(V: Cond, P: m_False()))
2511 RemovedIdx = 0;
2512 else
2513 continue;
2514
2515 VPBasicBlock *RemovedSucc =
2516 cast<VPBasicBlock>(Val: VPBB->getSuccessors()[RemovedIdx]);
2517 assert(count(RemovedSucc->getPredecessors(), VPBB) == 1 &&
2518 "There must be a single edge between VPBB and its successor");
2519 // Values coming from VPBB into phi recipes of RemovedSucc are removed from
2520 // these recipes and single-entry header phis are removed.
2521 for (VPRecipeBase &R : make_early_inc_range(Range: RemovedSucc->phis())) {
2522 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: VPBB);
2523 SimplifiedPhi = true;
2524 // Remove now invalid header phis that are left single-entry after
2525 // removing their backedges.
2526 auto *PhiR = dyn_cast<VPHeaderPHIRecipe>(Val: &R);
2527 if (!PhiR || PhiR->getNumIncoming() != 1)
2528 continue;
2529 PhiR->replaceAllUsesWith(New: PhiR->getOperand(N: 0));
2530 PhiR->eraseFromParent();
2531 }
2532
2533 // Disconnect blocks and remove the terminator.
2534 VPBlockUtils::disconnectBlocks(From: VPBB, To: RemovedSucc);
2535 VPBB->back().eraseFromParent();
2536 }
2537
2538 // Compute which blocks are still reachable from the entry after constant
2539 // branch removal.
2540 SmallPtrSet<VPBlockBase *, 16> Reachable(
2541 llvm::from_range, vp_depth_first_shallow(G: Plan.getEntry()));
2542
2543 // Detach all unreachable blocks from their successors, removing their recipes
2544 // and incoming values from phi recipes.
2545 VPSymbolicValue Tmp(nullptr);
2546 for (VPBlockBase *B : AllBlocks) {
2547 if (Reachable.contains(Ptr: B))
2548 continue;
2549 for (VPBlockBase *Succ : to_vector(Range: B->successors())) {
2550 if (auto *SuccBB = dyn_cast<VPBasicBlock>(Val: Succ))
2551 for (VPRecipeBase &R : SuccBB->phis())
2552 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: B);
2553 VPBlockUtils::disconnectBlocks(From: B, To: Succ);
2554 }
2555 for (VPBasicBlock *DeadBB :
2556 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_deep(G: B))) {
2557 for (VPRecipeBase &R : make_early_inc_range(Range&: *DeadBB)) {
2558 for (VPValue *Def : R.definedValues())
2559 Def->replaceAllUsesWith(New: &Tmp);
2560 R.eraseFromParent();
2561 }
2562 }
2563 }
2564 return SimplifiedPhi;
2565}
2566
2567void VPlanTransforms::optimize(VPlan &Plan) {
2568 RUN_VPLAN_PASS(removeRedundantInductionCasts, Plan);
2569
2570 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2571 RUN_VPLAN_PASS(simplifyRecipes, Plan);
2572 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2573 RUN_VPLAN_PASS(simplifyBlends, Plan);
2574 RUN_VPLAN_PASS(legalizeAndOptimizeInductions, Plan);
2575 RUN_VPLAN_PASS(narrowToSingleScalarRecipes, Plan);
2576 RUN_VPLAN_PASS(removeRedundantExpandSCEVRecipes, Plan);
2577 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2578 RUN_VPLAN_PASS(simplifyRecipes, Plan);
2579 RUN_VPLAN_PASS(removeBranchOnConst, Plan, /*OnlyLatches=*/false);
2580 RUN_VPLAN_PASS(simplifyReverses, Plan);
2581 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2582
2583 RUN_VPLAN_PASS(createAndOptimizeReplicateRegions, Plan);
2584 RUN_VPLAN_PASS(mergeBlocksIntoPredecessors, Plan);
2585 RUN_VPLAN_PASS(licm, Plan);
2586}
2587
2588void VPlanTransforms::simplifyLiveInsWithSCEV(VPlan &Plan,
2589 PredicatedScalarEvolution &PSE) {
2590 auto GetSimplifiedLiveInViaSCEV = [&](VPValue *VPV) -> VPValue * {
2591 const SCEV *Expr = vputils::getSCEVExprForVPValue(V: VPV, PSE);
2592 const APInt *C;
2593 if (match(S: Expr, P: m_scev_APInt(C)))
2594 return Plan.getConstantInt(Val: *C);
2595 return nullptr;
2596 };
2597
2598 for (VPValue *LiveIn : to_vector(Range: Plan.getLiveIns())) {
2599 if (VPValue *SimplifiedLiveIn = GetSimplifiedLiveInViaSCEV(LiveIn))
2600 LiveIn->replaceAllUsesWith(New: SimplifiedLiveIn);
2601 }
2602}
2603
2604void VPlanTransforms::replaceSymbolicStrides(
2605 VPlan &Plan, PredicatedScalarEvolution &PSE,
2606 const DenseMap<Value *, const SCEV *> &StridesMap,
2607 const VPDominatorTree &VPDT) {
2608 // Replace VPValues for known constant strides guaranteed by predicated scalar
2609 // evolution that are guaranteed to be guarded by the runtime checks; that is,
2610 // blocks dominated by the vector header.
2611 assert(!Plan.getVectorLoopRegion() &&
2612 "expected to run before loop regions are created");
2613 const auto &[Header, _] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
2614 auto CanUseVersionedStride = [&VPDT, Header = Header, &Plan](VPUser &U,
2615 unsigned Idx) {
2616 auto *R = cast<VPRecipeBase>(Val: &U);
2617 // Skip phis if the loop if loop is not yet guarded.
2618 if (isa<VPPhiAccessors>(Val: R) &&
2619 Header == Plan.getEntry()->getSingleSuccessor())
2620 return false;
2621 return VPDT.dominates(A: Header, B: R->getParent());
2622 };
2623 ValueToSCEVMapTy RewriteMap;
2624 for (const SCEV *Stride : StridesMap.values()) {
2625 Value *StrideV = cast<SCEVUnknown>(Val: Stride)->getValue();
2626 const APInt *StrideConst;
2627 const SCEV *StrideExpr = PSE.getSCEV(V: StrideV);
2628 if (!match(S: StrideExpr, P: m_scev_APInt(C&: StrideConst)))
2629 // Only handle constant strides for now.
2630 continue;
2631 if (VPValue *StrideVPV = Plan.getLiveIn(V: StrideV))
2632 StrideVPV->replaceUsesWithIf(New: Plan.getConstantInt(Val: *StrideConst),
2633 ShouldReplace: CanUseVersionedStride);
2634
2635 // The versioned value may not be used in the loop directly but through an
2636 // integral cast (sext/zext/trunc). Add new live-ins in those cases.
2637 for (Value *U : StrideV->users()) {
2638 if (!isa<SExtInst, ZExtInst, TruncInst>(Val: U))
2639 continue;
2640 VPValue *StrideVPV = Plan.getLiveIn(V: U);
2641 if (!StrideVPV)
2642 continue;
2643 unsigned BW = U->getType()->getScalarSizeInBits();
2644 APInt C = isa<SExtInst>(Val: U) ? StrideConst->sext(width: BW)
2645 : StrideConst->zextOrTrunc(width: BW);
2646 StrideVPV->replaceUsesWithIf(New: Plan.getConstantInt(Val: C),
2647 ShouldReplace: CanUseVersionedStride);
2648 }
2649 RewriteMap[StrideV] = StrideExpr;
2650 }
2651
2652 for (VPRecipeBase &R : *Plan.getEntry()) {
2653 auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(Val: &R);
2654 if (!ExpSCEV)
2655 continue;
2656 const SCEV *ScevExpr = ExpSCEV->getSCEV();
2657 auto *NewSCEV =
2658 SCEVParameterRewriter::rewrite(Scev: ScevExpr, SE&: *PSE.getSE(), Map&: RewriteMap);
2659 if (NewSCEV != ScevExpr) {
2660 VPValue *NewExp = vputils::getOrCreateVPValueForSCEVExpr(Plan, Expr: NewSCEV);
2661 ExpSCEV->replaceAllUsesWith(New: NewExp);
2662 if (Plan.getTripCount() == ExpSCEV)
2663 Plan.resetTripCount(NewTripCount: NewExp);
2664 }
2665 }
2666}
2667
2668void VPlanTransforms::dropPoisonGeneratingRecipes(VPlan &Plan) {
2669 // Collect recipes in the backward slice of `Root` that may generate a poison
2670 // value that is used after vectorization.
2671 SmallPtrSet<VPRecipeBase *, 16> Visited;
2672 auto CollectPoisonGeneratingInstrsInBackwardSlice([&](VPRecipeBase *Root) {
2673 SmallVector<VPRecipeBase *, 16> Worklist;
2674 Worklist.push_back(Elt: Root);
2675
2676 // Traverse the backward slice of Root through its use-def chain.
2677 while (!Worklist.empty()) {
2678 VPRecipeBase *CurRec = Worklist.pop_back_val();
2679
2680 if (!Visited.insert(Ptr: CurRec).second)
2681 continue;
2682
2683 // Prune search if we find another recipe generating a widen memory
2684 // instruction. Widen memory instructions involved in address computation
2685 // will lead to gather/scatter instructions, which don't need to be
2686 // handled.
2687 if (isa<VPWidenMemoryRecipe, VPInterleaveRecipe, VPScalarIVStepsRecipe,
2688 VPHeaderPHIRecipe>(Val: CurRec))
2689 continue;
2690
2691 // This recipe contributes to the address computation of a widen
2692 // load/store. If the underlying instruction has poison-generating flags,
2693 // drop them directly.
2694 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: CurRec)) {
2695 VPValue *A, *B;
2696 // Dropping disjoint from an OR may yield incorrect results, as some
2697 // analysis may have converted it to an Add implicitly (e.g. SCEV used
2698 // for dependence analysis). Instead, replace it with an equivalent Add.
2699 // This is possible as all users of the disjoint OR only access lanes
2700 // where the operands are disjoint or poison otherwise.
2701 if (match(V: RecWithFlags, P: m_BinaryOr(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))) &&
2702 RecWithFlags->isDisjoint()) {
2703 VPBuilder Builder(RecWithFlags);
2704 VPInstruction *New =
2705 Builder.createAdd(LHS: A, RHS: B, DL: RecWithFlags->getDebugLoc());
2706 New->setUnderlyingValue(RecWithFlags->getUnderlyingValue());
2707 RecWithFlags->replaceAllUsesWith(New);
2708 RecWithFlags->eraseFromParent();
2709 CurRec = New;
2710 } else
2711 RecWithFlags->dropPoisonGeneratingFlags();
2712 } else {
2713 Instruction *Instr = dyn_cast_or_null<Instruction>(
2714 Val: CurRec->getVPSingleValue()->getUnderlyingValue());
2715 (void)Instr;
2716 assert((!Instr || !Instr->hasPoisonGeneratingFlags()) &&
2717 "found instruction with poison generating flags not covered by "
2718 "VPRecipeWithIRFlags");
2719 }
2720
2721 // Add new definitions to the worklist.
2722 for (VPValue *Operand : CurRec->operands())
2723 if (VPRecipeBase *OpDef = Operand->getDefiningRecipe())
2724 Worklist.push_back(Elt: OpDef);
2725 }
2726 });
2727
2728 // We want to exclude the tail folding case, as we don't need to drop flags
2729 // for operations computing the first lane in this case: the first lane of the
2730 // header mask must always be true. For reverse memory accesses, the mask is
2731 // wrapped in a Reverse, which is just a permutation of the header mask, so
2732 // peel it off before checking. The header mask is still the abstract region
2733 // value at this point (materialization happens later).
2734 auto m_UnlessHdrMask = m_Unless( // NOLINT
2735 P: m_CombineOr(Ps: m_HeaderMask(), Ps: m_Reverse(Op0: m_HeaderMask())));
2736
2737 // Traverse all the recipes in the VPlan and collect the poison-generating
2738 // recipes in the backward slice starting at the address of a VPWidenRecipe or
2739 // VPInterleaveRecipe.
2740 auto Iter =
2741 vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntryBasicBlock());
2742 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: Iter)) {
2743 for (VPRecipeBase &Recipe : *VPBB) {
2744 if (auto *WidenRec = dyn_cast<VPWidenMemoryRecipe>(Val: &Recipe)) {
2745 VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe();
2746 if (AddrDef && WidenRec->isConsecutive() && WidenRec->getMask() &&
2747 match(V: WidenRec->getMask(), P: m_UnlessHdrMask))
2748 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2749 } else if (auto *InterleaveRec = dyn_cast<VPInterleaveRecipe>(Val: &Recipe)) {
2750 VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe();
2751 if (AddrDef && InterleaveRec->getMask() &&
2752 match(V: InterleaveRec->getMask(), P: m_UnlessHdrMask))
2753 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2754 }
2755 }
2756 }
2757}
2758
2759void VPlanTransforms::createInterleaveGroups(
2760 VPlan &Plan,
2761 const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
2762 &InterleaveGroups,
2763 const bool &EpilogueAllowed) {
2764 if (InterleaveGroups.empty())
2765 return;
2766
2767 DenseMap<Instruction *, VPWidenMemoryRecipe *> IRMemberToRecipe;
2768 for (VPBasicBlock *VPBB :
2769 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
2770 G: Plan.getVectorLoopRegion()->getEntryBasicBlock())))
2771 for (VPRecipeBase &R : make_filter_range(Range&: *VPBB, Pred: [](VPRecipeBase &R) {
2772 return isa<VPWidenMemoryRecipe>(Val: &R);
2773 })) {
2774 auto *MemR = cast<VPWidenMemoryRecipe>(Val: &R);
2775 IRMemberToRecipe[&MemR->getIngredient()] = MemR;
2776 }
2777
2778 // Interleave memory: for each Interleave Group we marked earlier as relevant
2779 // for this VPlan, replace the Recipes widening its memory instructions with a
2780 // single VPInterleaveRecipe at its insertion point.
2781 VPDominatorTree VPDT(Plan);
2782 for (const auto *IG : InterleaveGroups) {
2783 VPWidenMemoryRecipe *Start = nullptr;
2784 Instruction *StartMember = nullptr;
2785 for (auto *Member : IG->members())
2786 if (VPWidenMemoryRecipe *R = IRMemberToRecipe.lookup(Val: Member)) {
2787 StartMember = Member;
2788 Start = R;
2789 break;
2790 }
2791 if (!StartMember) // All member recipes are dead, so the group is dead.
2792 continue;
2793 VPIRMetadata InterleaveMD(*Start);
2794 SmallVector<VPValue *, 4> StoredValues;
2795 for (unsigned I = 0; I < IG->getFactor(); ++I) {
2796 Instruction *MemberI = IG->getMember(Index: I);
2797 if (!MemberI)
2798 continue;
2799 if (VPWidenMemoryRecipe *MemoryR = IRMemberToRecipe.lookup(Val: MemberI)) {
2800 if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Val: MemoryR->getAsRecipe()))
2801 StoredValues.push_back(Elt: StoreR->getStoredValue());
2802 InterleaveMD.intersect(MD: *MemoryR);
2803 } else {
2804 InterleaveMD.intersect(MD: VPIRMetadata(*MemberI));
2805 }
2806 }
2807
2808 bool NeedsMaskForGaps =
2809 (IG->requiresScalarEpilogue() && !EpilogueAllowed) ||
2810 (!StoredValues.empty() && !IG->isFull());
2811
2812 Instruction *IRInsertPos = IG->getInsertPos();
2813 auto *InsertPos = IRMemberToRecipe.lookup(Val: IRInsertPos);
2814 if (!InsertPos) {
2815 // InsertPos member is dead: find a new member that is alive.
2816 assert(isa<VPWidenLoadRecipe>(Start->getAsRecipe()) &&
2817 "Dead member in non-load group?");
2818 InsertPos = Start;
2819 for (Instruction *Member : IG->members())
2820 if (VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member))
2821 if (VPDT.properlyDominates(A: MemberR->getAsRecipe(),
2822 B: InsertPos->getAsRecipe()))
2823 InsertPos = MemberR;
2824 IRInsertPos = &InsertPos->getIngredient();
2825 }
2826 VPRecipeBase *InsertPosR = InsertPos->getAsRecipe();
2827
2828 GEPNoWrapFlags NW = GEPNoWrapFlags::none();
2829 if (auto *Gep = dyn_cast<GetElementPtrInst>(
2830 Val: getLoadStorePointerOperand(V: IRInsertPos)->stripPointerCasts()))
2831 NW = Gep->getNoWrapFlags().withoutNoUnsignedWrap();
2832
2833 // Get or create the start address for the interleave group.
2834 VPValue *Addr = Start->getAddr();
2835 VPRecipeBase *AddrDef = Addr->getDefiningRecipe();
2836 if (IG->getIndex(Instr: StartMember) != 0 ||
2837 (AddrDef && !VPDT.properlyDominates(A: AddrDef, B: InsertPosR))) {
2838 // Either member zero's recipe is dead, or we cannot re-use the address of
2839 // member zero because it does not dominate the insert position. Instead,
2840 // use the address of the insert position and create a PtrAdd adjusting it
2841 // to the address of member zero.
2842 // TODO: Hoist Addr's defining recipe (and any operands as needed) to
2843 // InsertPos or sink loads above zero members to join it.
2844 assert(IG->getIndex(IRInsertPos) != 0 &&
2845 "index of insert position shouldn't be zero");
2846 auto &DL = IRInsertPos->getDataLayout();
2847 APInt Offset(32,
2848 DL.getTypeAllocSize(Ty: getLoadStoreType(I: IRInsertPos)) *
2849 IG->getIndex(Instr: IRInsertPos),
2850 /*IsSigned=*/true);
2851 VPValue *OffsetVPV = Plan.getConstantInt(Val: -Offset);
2852 VPBuilder B(InsertPosR);
2853 Addr = B.createNoWrapPtrAdd(Ptr: InsertPos->getAddr(), Offset: OffsetVPV, GEPFlags: NW);
2854 }
2855 // If the group is reverse, adjust the index to refer to the last vector
2856 // lane instead of the first. We adjust the index from the first vector
2857 // lane, rather than directly getting the pointer for lane VF - 1, because
2858 // the pointer operand of the interleaved access is supposed to be uniform.
2859 if (IG->isReverse()) {
2860 auto *ReversePtr = new VPVectorEndPointerRecipe(
2861 Addr, &Plan.getVF(), getLoadStoreType(I: IRInsertPos),
2862 -(int64_t)IG->getFactor(), NW, InsertPosR->getDebugLoc());
2863 ReversePtr->insertBefore(InsertPos: InsertPosR);
2864 Addr = ReversePtr;
2865 }
2866 auto *VPIG = new VPInterleaveRecipe(
2867 IG, Addr, StoredValues, InsertPos->getMask(), NeedsMaskForGaps,
2868 InterleaveMD, InsertPosR->getDebugLoc());
2869 VPIG->insertBefore(InsertPos: InsertPosR);
2870
2871 unsigned J = 0;
2872 for (unsigned i = 0; i < IG->getFactor(); ++i)
2873 if (Instruction *Member = IG->getMember(Index: i)) {
2874 VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member);
2875 if (!Member->getType()->isVoidTy()) {
2876 if (MemberR) {
2877 VPValue *OriginalV = MemberR->getAsRecipe()->getVPSingleValue();
2878 OriginalV->replaceAllUsesWith(New: VPIG->getVPValue(I: J));
2879 }
2880 J++;
2881 }
2882 if (MemberR)
2883 MemberR->getAsRecipe()->eraseFromParent();
2884 }
2885 }
2886}
2887
2888/// Returns the VPValue representing the uncountable exit comparison used by
2889/// AnyOf if the recipes it depends on can be traced back to live-ins and
2890/// the addresses (in GEP/PtrAdd form) of any (non-masked) load used in
2891/// generating the values for the comparison. The recipes are stored in
2892/// \p Recipes.
2893static std::optional<VPValue *>
2894getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
2895 VPBasicBlock *LatchVPBB) {
2896 // Given a plain CFG VPlan loop with countable latch exiting block
2897 // \p LatchVPBB, we're looking to match the recipes contributing to the
2898 // uncountable exit condition comparison (here, vp<%4>) back to either
2899 // live-ins or the address nodes for the load used as part of the uncountable
2900 // exit comparison so that we can either move them within the loop, or copy
2901 // them to the preheader depending on the chosen method for dealing with
2902 // stores in uncountable exit loops.
2903 //
2904 // Currently, the address of the load is restricted to a GEP with 2 operands
2905 // and a live-in base address. This constraint may be relaxed later.
2906 //
2907 // VPlan ' for UF>=1' {
2908 // Live-in vp<%0> = VF * UF
2909 // Live-in vp<%1> = vector-trip-count
2910 // Live-in ir<20> = original trip-count
2911 //
2912 // ir-bb<entry>:
2913 // Successor(s): scalar.ph, vector.ph
2914 //
2915 // vector.ph:
2916 // Successor(s): for.body
2917 //
2918 // for.body:
2919 // EMIT vp<%2> = phi ir<0>, vp<%index.next>
2920 // EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, for.inc ]
2921 // EMIT ir<%uncountable.addr> = getelementptr inbounds nuw ir<%pred>,ir<%iv>
2922 // EMIT ir<%uncountable.val> = load ir<%uncountable.addr>
2923 // EMIT ir<%uncountable.cond> = icmp sgt ir<%uncountable.val>, ir<500>
2924 // EMIT vp<%3> = masked-cond ir<%uncountable.cond>
2925 // Successor(s): for.inc
2926 //
2927 // for.inc:
2928 // EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
2929 // EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
2930 // EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
2931 // EMIT vp<%4> = any-of ir<%3>
2932 // EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
2933 // EMIT branch-on-two-conds vp<%4>, vp<%5>
2934 // Successor(s): middle.block, middle.block, for.body
2935 //
2936 // middle.block:
2937 // Successor(s): ir-bb<exit>, scalar.ph
2938 //
2939 // ir-bb<exit>:
2940 // No successors
2941 //
2942 // scalar.ph:
2943 // }
2944
2945 // Find the uncountable loop exit condition.
2946 VPValue *UncountableCondition = nullptr;
2947 if (!match(V: LatchVPBB->getTerminator(),
2948 P: m_BranchOnTwoConds(Op0: m_AnyOf(Op0: m_VPValue(V&: UncountableCondition)),
2949 Op1: m_VPValue())))
2950 return std::nullopt;
2951
2952 SmallVector<VPValue *, 4> Worklist;
2953 Worklist.push_back(Elt: UncountableCondition);
2954 while (!Worklist.empty()) {
2955 VPValue *V = Worklist.pop_back_val();
2956
2957 // Any value defined outside the loop does not need to be copied.
2958 if (V->isDefinedOutsideLoopRegions())
2959 continue;
2960
2961 // FIXME: Remove the single user restriction; it's here because we're
2962 // starting with the simplest set of loops we can, and multiple
2963 // users means needing to add PHI nodes in the transform.
2964 if (V->getNumUsers() > 1)
2965 return std::nullopt;
2966
2967 VPValue *Op1, *Op2;
2968 // Walk back through recipes until we find at least one load from memory.
2969 if (match(V, P: m_ICmp(Op0: m_VPValue(V&: Op1), Op1: m_VPValue(V&: Op2)))) {
2970 Worklist.push_back(Elt: Op1);
2971 Worklist.push_back(Elt: Op2);
2972 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
2973 } else if (match(V, P: m_VPInstruction<Instruction::Load>(Ops: m_VPValue(V&: Op1)))) {
2974 VPRecipeBase *GepR = Op1->getDefiningRecipe();
2975 // Only matching base + single offset term for now.
2976 if (GepR->getNumOperands() != 2)
2977 return std::nullopt;
2978 // Matching a GEP with a loop-invariant base ptr.
2979 if (!match(V: GepR, P: m_VPInstruction<Instruction::GetElementPtr>(
2980 Ops: m_LiveIn(), Ops: m_VPValue())))
2981 return std::nullopt;
2982 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
2983 Recipes.push_back(Elt: cast<VPInstruction>(Val: GepR));
2984 } else if (match(V, P: m_VPInstruction<VPInstruction::MaskedCond>(
2985 Ops: m_VPValue(V&: Op1)))) {
2986 Worklist.push_back(Elt: Op1);
2987 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
2988 } else
2989 return std::nullopt;
2990 }
2991
2992 // If we couldn't match anything, don't return the condition. It may be
2993 // defined outside the loop.
2994 if (Recipes.empty() ||
2995 none_of(Range&: Recipes, P: match_fn(P: m_VPInstruction<Instruction::GetElementPtr>())))
2996 return std::nullopt;
2997
2998 return UncountableCondition;
2999}
3000
3001struct EarlyExitInfo {
3002 VPBasicBlock *EarlyExitingVPBB;
3003 VPIRBasicBlock *EarlyExitVPBB;
3004 VPValue *CondToExit;
3005};
3006
3007/// Update \p Plan to mask memory operations in the loop based on whether the
3008/// early exit is taken or not.
3009///
3010/// We're currently expecting to find a loop with properties similar to the
3011/// following:
3012///
3013/// for.body:
3014/// ir<%indvars.iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<%0>
3015/// EMIT ir<%arrayidx> = getelementptr inbounds nuw ir<@c>, ir<%indvars.iv>
3016/// EMIT-SCALAR ir<%0> = load ir<%arrayidx>
3017/// EMIT ir<%cmp1> = icmp sgt ir<%0>, ir<5>
3018/// EMIT vp<%1> = masked-cond ir<%cmp1>
3019/// Successor(s): if.end
3020///
3021/// if.end:
3022/// EMIT ir<%arrayidx3> = getelementptr inbounds nuw ir<@src>, ir<%indvars.iv>
3023/// EMIT-SCALAR ir<%2> = load ir<%arrayidx3>
3024/// EMIT ir<%add> = add nsw ir<%2>, ir<42>
3025/// EMIT ir<%arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<%indvars.iv>
3026/// EMIT store ir<%add>, ir<%arrayidx5>
3027/// EMIT ir<%indvars.iv.next> = add nuw nsw ir<%indvars.iv>, ir<1>
3028/// EMIT vp<%3> = any-of ir<%1>
3029/// EMIT ir<%exitcond.not> = icmp eq ir<%indvars.iv.next>, ir<10000>
3030/// EMIT branch-on-two-conds vp<%3>, ir<%exitcond.not>
3031/// Successor(s): middle.block, middle.block, for.body
3032///
3033/// We currently expect LoopVectorizationLegality to ensure that:
3034/// * There must also be a counted exit. We will need to support speculative
3035/// or first-faulting loads before we can remove this restriction.
3036/// * Any stores within the loop must not alias with the load used for the
3037/// uncountable exit. We can relax this a bit with runtime aliasing checks.
3038/// * Other memory operations in the loop can take place before or after the
3039/// uncountable exit, but must also be unconditional. We need to support
3040/// combining the conditions in VPlanPredicator.
3041/// * The loop must have a single unconditional load contributing to the
3042/// uncountable exit comparison, and the other term must be loop-invariant.
3043/// Improving upon this requires work in getRecipesForUncountableExit to
3044/// handle more complex recipe graphs.
3045static bool handleUncountableExitsWithSideEffects(
3046 VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
3047 VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB,
3048 Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT,
3049 AssumptionCache *AC) {
3050
3051 // Disconnect early exiting blocks from successors, remove branches. We
3052 // currently don't support multiple uses for recipes involved in creating
3053 // the uncountable exit condition.
3054 for (auto &Exit : Exits) {
3055 if (Exit.EarlyExitingVPBB == LatchVPBB)
3056 continue;
3057
3058 for (VPRecipeBase &R : Exit.EarlyExitVPBB->phis())
3059 cast<VPIRPhi>(Val: &R)->removeIncomingValueFor(IncomingBlock: Exit.EarlyExitingVPBB);
3060 Exit.EarlyExitingVPBB->getTerminator()->eraseFromParent();
3061 VPBlockUtils::disconnectBlocks(From: Exit.EarlyExitingVPBB, To: Exit.EarlyExitVPBB);
3062 }
3063
3064 VPDominatorTree VPDT(Plan);
3065
3066 // We can abandon a VPlan entirely if we return false here, so we shouldn't
3067 // crash if some earlier assumptions on scalar IR don't hold for the vplan
3068 // version of the loop.
3069 SmallVector<VPInstruction *, 8> ConditionRecipes;
3070
3071 std::optional<VPValue *> Cond =
3072 getRecipesForUncountableExit(Recipes&: ConditionRecipes, LatchVPBB);
3073 if (!Cond)
3074 return false;
3075
3076 // Find load contributing to condition.
3077 // At the moment LoopVectorizationLegality only supports a single
3078 // early-exit expression with a compare and a single load that must
3079 // be unconditional.
3080 // TODO: Support more than one load.
3081 auto *Load =
3082 find_singleton<VPInstruction>(Range&: ConditionRecipes, P: [](auto *I, bool _) {
3083 return match(I, m_VPInstruction<Instruction::Load>(Ops: m_VPValue()))
3084 ? I
3085 : nullptr;
3086 });
3087 assert(Load && "Couldn't find exactly one load");
3088 // TODO: Support conditional loads for uncountable exits.
3089 assert(VPDT.dominates(Load->getParent(), LatchVPBB) &&
3090 "Uncountable exit condition load is conditional.");
3091 VPInstruction *Ptr = cast<VPInstruction>(Val: Load->getOperand(N: 0));
3092
3093 // Ensure that we are guaranteed to be able to dereference the memory used
3094 // for determining the uncountable exit for the maximum possible number of
3095 // scalar iterations of the loop.
3096 //
3097 // TODO: Support first-faulting loads in cases where we don't know whether
3098 // all possible addresses are dereferenceable.
3099 {
3100 SmallVector<const SCEVPredicate *, 4> Predicates;
3101 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: TheLoop);
3102 const DataLayout &DL = Plan.getDataLayout();
3103 APInt EltSize(DL.getIndexTypeSizeInBits(Ty: Ptr->getScalarType()),
3104 DL.getTypeStoreSize(Ty: Load->getScalarType()).getFixedValue());
3105 if (!isDereferenceableAndAlignedInLoop(
3106 PtrSCEV, Alignment: cast<LoadInst>(Val: Load->getUnderlyingInstr())->getAlign(),
3107 EltSizeSCEV: PSE.getSE()->getConstant(Val: EltSize), L: TheLoop, SE&: *PSE.getSE(), DT, AC,
3108 Predicates: &Predicates))
3109 return false;
3110 }
3111
3112 // Check for a single GEP for the condition load to see if we can link it to
3113 // a widen IV recipe with a step of 1; we're only interested in contiguous
3114 // accesses for the condition load right now.
3115 auto *IV = cast<VPWidenInductionRecipe>(Val: &HeaderVPBB->front());
3116 if (!match(V: IV->getStartValue(), P: m_SpecificInt(V: 0)) ||
3117 !match(V: IV->getStepValue(), P: m_SpecificInt(V: 1)))
3118 return false;
3119 if (!match(V: Ptr, P: m_VPInstruction<Instruction::GetElementPtr>(Ops: m_LiveIn(),
3120 Ops: m_Specific(VPV: IV))))
3121 return false;
3122
3123 // We want to guarantee that the uncountable exit condition (and the mask
3124 // we will generate from it) are available for all operations in the loop
3125 // that need to be masked. If the condition recipes are not already the first
3126 // recipes in the header after the last phi, move them there.
3127 auto InsertIt = HeaderVPBB->getFirstNonPhi();
3128 while (InsertIt != HeaderVPBB->end() &&
3129 is_contained(Range&: ConditionRecipes, Element: &*InsertIt)) {
3130 erase(C&: ConditionRecipes, V: &*InsertIt);
3131 InsertIt++;
3132 }
3133 for (auto *Recipe : reverse(C&: ConditionRecipes))
3134 Recipe->moveBefore(BB&: *HeaderVPBB, I: InsertIt);
3135
3136 // Create a mask to represent all lanes that fully execute in the vector loop,
3137 // stopping short of any early exit.
3138 VPBuilder MaskBuilder(HeaderVPBB, InsertIt);
3139 VPValue *FirstActive = MaskBuilder.createFirstActiveLane(Masks: *Cond);
3140 Type *IVScalarTy = IV->getScalarType();
3141 VPValue *Zero = Plan.getZero(Ty: IVScalarTy);
3142 FirstActive =
3143 MaskBuilder.createScalarZExtOrTrunc(Op: FirstActive, ResultTy: IVScalarTy, DL: DebugLoc());
3144 VPValue *Mask = MaskBuilder.createNaryOp(Opcode: VPInstruction::ActiveLaneMask,
3145 Operands: {Zero, FirstActive}, DL: DebugLoc(),
3146 Name: "uncountable.exit.mask");
3147
3148 // Convert all other memory operations to use the mask.
3149 for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(Header: HeaderVPBB))
3150 for (VPRecipeBase &R : *VPBB)
3151 if (R.mayReadOrWriteMemory() && &R != Load) {
3152 // TODO: Handle conditional memory operations in the loop.
3153 if (!VPDT.dominates(A: R.getParent(), B: LatchVPBB))
3154 return false;
3155 cast<VPInstruction>(Val: &R)->addMask(Mask);
3156 }
3157
3158 // Update middle block branch to compare (IV + however many lanes were active)
3159 // against the full trip count, since we may be exiting the vector loop early.
3160 // If we didn't take an early exit, we should get the equivalent of VF from
3161 // the FirstActiveLane.
3162 assert(match(MiddleVPBB->getTerminator(), m_BranchOnCond()) &&
3163 "Expected BranchOnCond terminator for MiddleVPBB");
3164 VPBuilder MiddleBuilder(MiddleVPBB->getTerminator());
3165 VPValue *ScalarIV = MiddleBuilder.createNaryOp(Opcode: VPInstruction::ExtractLane,
3166 Operands: {Zero, IV}, DL: DebugLoc());
3167 VPValue *ExitIV = MiddleBuilder.createAdd(LHS: ScalarIV, RHS: FirstActive);
3168 VPValue *FullTC =
3169 MiddleBuilder.createICmp(Pred: CmpInst::ICMP_EQ, A: ExitIV, B: Plan.getTripCount());
3170 MiddleVPBB->getTerminator()->setOperand(I: 0, New: FullTC);
3171
3172 // Update resume phi in scalar.ph.
3173 VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
3174 auto Phis = ScalarPH->phis();
3175 // TODO: Handle more than one Phi; re-derive from IV.
3176 // TODO: Handle reductions.
3177 if (range_size(Range&: Phis) != 1)
3178 return false;
3179 VPPhi *ContinueIV = cast<VPPhi>(Val: Phis.begin());
3180 // Make sure we're referring to the same IV.
3181 assert(
3182 match(ContinueIV->getOperand(0),
3183 m_VPInstruction<VPInstruction::ExitingIVValue>(m_Specific(IV))) &&
3184 "Continuing from different IV");
3185 ContinueIV->setOperand(I: 0, New: ExitIV);
3186 return true;
3187}
3188
3189bool VPlanTransforms::handleUncountableEarlyExits(
3190 VPlan &Plan, Loop *TheLoop, PredicatedScalarEvolution &PSE,
3191 DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style) {
3192#ifndef NDEBUG
3193 VPDominatorTree VPDT(Plan);
3194#endif
3195
3196 auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
3197 auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
3198
3199 // Dereferenceability is checked separately for uncountable exit loops with
3200 // stores, as only the loads contributing to the exit condition need to
3201 // be checked.
3202 if (Style == UncountableExitStyle::ReadOnly &&
3203 !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC))
3204 return false;
3205
3206 VPBuilder LatchBuilder(LatchVPBB->getTerminator());
3207 SmallVector<EarlyExitInfo> Exits;
3208 for (auto [EarlyExitingVPBB, ExitBlock] :
3209 vputils::getEarlyExits(Plan, MiddleVPBB)) {
3210 // Collect condition for this early exit.
3211 VPBlockBase *TrueSucc = EarlyExitingVPBB->getSuccessors()[0];
3212 VPValue *CondOfEarlyExitingVPBB;
3213 [[maybe_unused]] bool Matched =
3214 match(V: EarlyExitingVPBB->getTerminator(),
3215 P: m_BranchOnCond(Op0: m_VPValue(V&: CondOfEarlyExitingVPBB)));
3216 assert(Matched && "Terminator must be BranchOnCond");
3217
3218 // Insert the MaskedCond in the EarlyExitingVPBB so the predicator adds
3219 // the correct block mask.
3220 VPBuilder EarlyExitingBuilder(EarlyExitingVPBB->getTerminator());
3221 auto *CondToEarlyExit = EarlyExitingBuilder.createNaryOp(
3222 Opcode: VPInstruction::MaskedCond,
3223 Operands: TrueSucc == ExitBlock
3224 ? CondOfEarlyExitingVPBB
3225 : EarlyExitingBuilder.createNot(Operand: CondOfEarlyExitingVPBB));
3226 assert((isa<VPIRValue>(CondOfEarlyExitingVPBB) ||
3227 !VPDT.properlyDominates(EarlyExitingVPBB, LatchVPBB) ||
3228 VPDT.properlyDominates(
3229 CondOfEarlyExitingVPBB->getDefiningRecipe()->getParent(),
3230 LatchVPBB)) &&
3231 "exit condition must dominate the latch");
3232 Exits.push_back(Elt: {
3233 .EarlyExitingVPBB: EarlyExitingVPBB,
3234 .EarlyExitVPBB: ExitBlock,
3235 .CondToExit: CondToEarlyExit,
3236 });
3237 }
3238
3239 assert(!Exits.empty() && "must have at least one early exit");
3240 // Sort exits by RPO order to get correct program order. RPO gives a
3241 // topological ordering of the CFG, ensuring upstream exits are checked
3242 // before downstream exits in the dispatch chain.
3243 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
3244 HeaderVPBB);
3245 DenseMap<VPBlockBase *, unsigned> RPOIdx;
3246 for (const auto &[Num, VPB] : enumerate(First&: RPOT))
3247 RPOIdx[VPB] = Num;
3248 llvm::sort(C&: Exits, Comp: [&RPOIdx](const EarlyExitInfo &A, const EarlyExitInfo &B) {
3249 return RPOIdx[A.EarlyExitingVPBB] < RPOIdx[B.EarlyExitingVPBB];
3250 });
3251#ifndef NDEBUG
3252 // After RPO sorting, verify that for any pair where one exit dominates
3253 // another, the dominating exit comes first. This is guaranteed by RPO
3254 // (topological order) and is required for the dispatch chain correctness.
3255 for (unsigned I = 0; I + 1 < Exits.size(); ++I)
3256 for (unsigned J = I + 1; J < Exits.size(); ++J)
3257 assert(!VPDT.properlyDominates(Exits[J].EarlyExitingVPBB,
3258 Exits[I].EarlyExitingVPBB) &&
3259 "RPO sort must place dominating exits before dominated ones");
3260#endif
3261
3262 // Build the AnyOf condition for the latch terminator using logical OR
3263 // to avoid poison propagation from later exit conditions when an earlier
3264 // exit is taken.
3265 VPValue *Combined = Exits[0].CondToExit;
3266 for (const EarlyExitInfo &Info : drop_begin(RangeOrContainer&: Exits))
3267 Combined = LatchBuilder.createLogicalOr(LHS: Combined, RHS: Info.CondToExit);
3268
3269 VPValue *IsAnyExitTaken =
3270 LatchBuilder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: {Combined});
3271
3272 // Create a comparison for the latch exit condition and replace the
3273 // BranchOnCond with a BranchOnTwoConds. The original BranchOnCond's condition
3274 // is used as the latch-exit condition; canonical IV recipes have not been
3275 // introduced yet, so there is no BranchOnCount to derive the condition from.
3276 auto *LatchExitingBranch = cast<VPInstruction>(Val: LatchVPBB->getTerminator());
3277 assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCond &&
3278 "Unexpected terminator");
3279 VPValue *IsLatchExitTaken = LatchExitingBranch->getOperand(N: 0);
3280 DebugLoc LatchDL = LatchExitingBranch->getDebugLoc();
3281 LatchExitingBranch->eraseFromParent();
3282 LatchBuilder.setInsertPoint(LatchVPBB);
3283 LatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnTwoConds,
3284 Operands: {IsAnyExitTaken, IsLatchExitTaken}, DL: LatchDL);
3285 LatchVPBB->clearSuccessors();
3286
3287 if (Style == UncountableExitStyle::MaskedHandleExitInScalarLoop) {
3288 // If handling the exiting lane in the scalar loop, combine the exit
3289 // conditions into a single BranchOnCond.
3290 LatchVPBB->setSuccessors({MiddleVPBB, MiddleVPBB, HeaderVPBB});
3291 MiddleVPBB->clearPredecessors();
3292 MiddleVPBB->setPredecessors({LatchVPBB, LatchVPBB});
3293 return handleUncountableExitsWithSideEffects(
3294 Plan, Exits, HeaderVPBB, LatchVPBB, MiddleVPBB, TheLoop, PSE, DT, AC);
3295 }
3296
3297 // Create the vector.early.exit blocks.
3298 SmallVector<VPBasicBlock *> VectorEarlyExitVPBBs(Exits.size());
3299 for (unsigned Idx = 0; Idx != Exits.size(); ++Idx) {
3300 Twine BlockSuffix = Exits.size() == 1 ? "" : Twine(".") + Twine(Idx);
3301 VPBasicBlock *VectorEarlyExitVPBB =
3302 Plan.createVPBasicBlock(Name: "vector.early.exit" + BlockSuffix);
3303 VectorEarlyExitVPBBs[Idx] = VectorEarlyExitVPBB;
3304 }
3305
3306 // Create the dispatch block (or reuse the single exit block if only one
3307 // exit). The dispatch block computes the first active lane of the combined
3308 // condition and, for multiple exits, chains through conditions to determine
3309 // which exit to take.
3310 VPBasicBlock *DispatchVPBB =
3311 Exits.size() == 1 ? VectorEarlyExitVPBBs[0]
3312 : Plan.createVPBasicBlock(Name: "vector.early.exit.check");
3313 DispatchVPBB->setPredecessors({LatchVPBB});
3314 LatchVPBB->setSuccessors({DispatchVPBB, MiddleVPBB, HeaderVPBB});
3315 VPBuilder DispatchBuilder(DispatchVPBB, DispatchVPBB->begin());
3316 VPValue *FirstActiveLane = DispatchBuilder.createFirstActiveLane(
3317 Masks: {Combined}, DL: DebugLoc::getUnknown(), Name: "first.active.lane");
3318
3319 // For each early exit, disconnect the original exiting block
3320 // (early.exiting.I) from the exit block (ir-bb<exit.I>) and route through a
3321 // new vector.early.exit block. Update ir-bb<exit.I>'s phis to extract their
3322 // values at the first active lane:
3323 //
3324 // Input:
3325 // early.exiting.I:
3326 // ...
3327 // EMIT branch-on-cond vp<%cond.I>
3328 // Successor(s): in.loop.succ, ir-bb<exit.I>
3329 //
3330 // ir-bb<exit.I>:
3331 // IR %phi = phi [ vp<%incoming.I>, early.exiting.I ], ...
3332 //
3333 // Output:
3334 // early.exiting.I:
3335 // ...
3336 // Successor(s): in.loop.succ
3337 //
3338 // vector.early.exit.I:
3339 // EMIT vp<%exit.val> = extract-lane vp<%first.lane>, vp<%incoming.I>
3340 // Successor(s): ir-bb<exit.I>
3341 //
3342 // ir-bb<exit.I>:
3343 // IR %phi = phi ... (extra operand: vp<%exit.val> from
3344 // vector.early.exit.I)
3345 //
3346 for (auto [Exit, VectorEarlyExitVPBB] :
3347 zip_equal(t&: Exits, u&: VectorEarlyExitVPBBs)) {
3348 auto &[EarlyExitingVPBB, EarlyExitVPBB, _] = Exit;
3349 // Adjust the phi nodes in EarlyExitVPBB.
3350 // 1. remove incoming values from EarlyExitingVPBB,
3351 // 2. extract the incoming value at FirstActiveLane
3352 // 3. add back the extracts as last operands for the phis
3353 // Then adjust the CFG, removing the edge between EarlyExitingVPBB and
3354 // EarlyExitVPBB and adding a new edge between VectorEarlyExitVPBB and
3355 // EarlyExitVPBB. The extracts at FirstActiveLane are now the incoming
3356 // values from VectorEarlyExitVPBB.
3357 for (VPRecipeBase &R : EarlyExitVPBB->phis()) {
3358 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
3359 VPValue *IncomingVal =
3360 ExitIRI->getIncomingValueForBlock(VPBB: EarlyExitingVPBB);
3361 VPValue *NewIncoming = IncomingVal;
3362 if (!isa<VPIRValue>(Val: IncomingVal)) {
3363 VPBuilder EarlyExitBuilder(VectorEarlyExitVPBB);
3364 NewIncoming = EarlyExitBuilder.createNaryOp(
3365 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, IncomingVal},
3366 DL: DebugLoc::getUnknown(), Name: "early.exit.value");
3367 }
3368 ExitIRI->removeIncomingValueFor(IncomingBlock: EarlyExitingVPBB);
3369 ExitIRI->addIncoming(IncomingV: NewIncoming);
3370 }
3371
3372 EarlyExitingVPBB->getTerminator()->eraseFromParent();
3373 VPBlockUtils::disconnectBlocks(From: EarlyExitingVPBB, To: EarlyExitVPBB);
3374 VPBlockUtils::connectBlocks(From: VectorEarlyExitVPBB, To: EarlyExitVPBB);
3375 }
3376
3377 // Chain through exits: for each exit, check if its condition is true at
3378 // the first active lane. If so, take that exit; otherwise, try the next.
3379 // The last exit needs no check since it must be taken if all others fail.
3380 //
3381 // For 3 exits (cond.0, cond.1, cond.2), this creates:
3382 //
3383 // latch:
3384 // ...
3385 // EMIT vp<%combined> = logical-or vp<%cond.0>, vp<%cond.1>, vp<%cond.2>
3386 // ...
3387 //
3388 // vector.early.exit.check:
3389 // EMIT vp<%first.lane> = first-active-lane vp<%combined>
3390 // EMIT vp<%at.cond.0> = extract-lane vp<%first.lane>, vp<%cond.0>
3391 // EMIT branch-on-cond vp<%at.cond.0>
3392 // Successor(s): vector.early.exit.0, vector.early.exit.check.0
3393 //
3394 // vector.early.exit.check.0:
3395 // EMIT vp<%at.cond.1> = extract-lane vp<%first.lane>, vp<%cond.1>
3396 // EMIT branch-on-cond vp<%at.cond.1>
3397 // Successor(s): vector.early.exit.1, vector.early.exit.2
3398 VPBasicBlock *CurrentBB = DispatchVPBB;
3399 for (auto [I, Exit] : enumerate(First: ArrayRef(Exits).drop_back())) {
3400 VPValue *LaneVal = DispatchBuilder.createNaryOp(
3401 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, Exit.CondToExit},
3402 DL: DebugLoc::getUnknown(), Name: "exit.cond.at.lane");
3403
3404 // For the last dispatch, branch directly to the last exit on false;
3405 // otherwise, create a new check block.
3406 bool IsLastDispatch = (I + 2 == Exits.size());
3407 VPBasicBlock *FalseBB =
3408 IsLastDispatch ? VectorEarlyExitVPBBs.back()
3409 : Plan.createVPBasicBlock(
3410 Name: Twine("vector.early.exit.check.") + Twine(I));
3411
3412 DispatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {LaneVal});
3413 CurrentBB->setSuccessors({VectorEarlyExitVPBBs[I], FalseBB});
3414 VectorEarlyExitVPBBs[I]->setPredecessors({CurrentBB});
3415 FalseBB->setPredecessors({CurrentBB});
3416
3417 CurrentBB = FalseBB;
3418 DispatchBuilder.setInsertPoint(CurrentBB);
3419 }
3420
3421 return true;
3422}
3423
3424/// This function tries convert extended in-loop reductions to
3425/// VPExpressionRecipe and clamp the \p Range if it is beneficial and
3426/// valid. The created recipe must be decomposed to its constituent
3427/// recipes before execution.
3428static VPExpressionRecipe *
3429tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
3430 VFRange &Range) {
3431 Type *RedTy = Red->getScalarType();
3432 VPValue *VecOp = Red->getVecOp();
3433
3434 assert(!Red->isPartialReduction() &&
3435 "This path does not support partial reductions");
3436
3437 // Clamp the range if using extended-reduction is profitable.
3438 auto IsExtendedRedValidAndClampRange =
3439 [&](unsigned Opcode, Instruction::CastOps ExtOpc, Type *SrcTy) -> bool {
3440 return LoopVectorizationPlanner::getDecisionAndClampRange(
3441 Predicate: [&](ElementCount VF) {
3442 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
3443 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
3444
3445 InstructionCost ExtRedCost = InstructionCost::getInvalid();
3446 InstructionCost ExtCost =
3447 cast<VPWidenCastRecipe>(Val: VecOp)->computeCost(VF, Ctx);
3448 InstructionCost RedCost = Red->computeCost(VF, Ctx);
3449
3450 assert(!RedTy->isFloatingPointTy() &&
3451 "getExtendedReductionCost only supports integer types");
3452 ExtRedCost = Ctx.TTI.getExtendedReductionCost(
3453 Opcode, IsUnsigned: ExtOpc == Instruction::CastOps::ZExt, ResTy: RedTy, Ty: SrcVecTy,
3454 FMF: Red->getFastMathFlagsOrNone(), CostKind);
3455 return ExtRedCost.isValid() && ExtRedCost < ExtCost + RedCost;
3456 },
3457 Range);
3458 };
3459
3460 VPValue *A;
3461 // Match reduce(ext)).
3462 if (match(V: VecOp, P: m_Isa<VPWidenCastRecipe>(P: m_ZExtOrSExt(Op0: m_VPValue(V&: A)))) &&
3463 IsExtendedRedValidAndClampRange(
3464 RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind()),
3465 cast<VPWidenCastRecipe>(Val: VecOp)->getOpcode(), A->getScalarType()))
3466 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
3467
3468 return nullptr;
3469}
3470
3471/// This function tries convert extended in-loop reductions to
3472/// VPExpressionRecipe and clamp the \p Range if it is beneficial
3473/// and valid. The created VPExpressionRecipe must be decomposed to its
3474/// constituent recipes before execution. Patterns of the
3475/// VPExpressionRecipe:
3476/// reduce.add(mul(...)),
3477/// reduce.add(mul(ext(A), ext(B))),
3478/// reduce.add(ext(mul(ext(A), ext(B)))).
3479/// reduce.fadd(fmul(ext(A), ext(B)))
3480static VPExpressionRecipe *
3481tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
3482 VPCostContext &Ctx, VFRange &Range) {
3483 unsigned Opcode = RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind());
3484 if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
3485 Opcode != Instruction::FAdd)
3486 return nullptr;
3487
3488 assert(!Red->isPartialReduction() &&
3489 "This path does not support partial reductions");
3490 Type *RedTy = Red->getScalarType();
3491
3492 // Clamp the range if using multiply-accumulate-reduction is profitable.
3493 auto IsMulAccValidAndClampRange =
3494 [&](VPWidenRecipe *Mul, VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
3495 VPWidenCastRecipe *OuterExt) -> bool {
3496 return LoopVectorizationPlanner::getDecisionAndClampRange(
3497 Predicate: [&](ElementCount VF) {
3498 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
3499 Type *SrcTy = Ext0 ? Ext0->getOperand(N: 0)->getScalarType() : RedTy;
3500 InstructionCost MulAccCost;
3501
3502 // getMulAccReductionCost for in-loop reductions does not support
3503 // mixed or floating-point extends.
3504 if (Ext0 && Ext1 &&
3505 (Ext0->getOpcode() != Ext1->getOpcode() ||
3506 Ext0->getOpcode() == Instruction::CastOps::FPExt))
3507 return false;
3508
3509 bool IsZExt =
3510 !Ext0 || Ext0->getOpcode() == Instruction::CastOps::ZExt;
3511 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
3512 MulAccCost = Ctx.TTI.getMulAccReductionCost(IsUnsigned: IsZExt, RedOpcode: Opcode, ResTy: RedTy,
3513 Ty: SrcVecTy, CostKind);
3514
3515 InstructionCost MulCost = Mul->computeCost(VF, Ctx);
3516 InstructionCost RedCost = Red->computeCost(VF, Ctx);
3517 InstructionCost ExtCost = 0;
3518 if (Ext0)
3519 ExtCost += Ext0->computeCost(VF, Ctx);
3520 if (Ext1)
3521 ExtCost += Ext1->computeCost(VF, Ctx);
3522 if (OuterExt)
3523 ExtCost += OuterExt->computeCost(VF, Ctx);
3524
3525 return MulAccCost.isValid() &&
3526 MulAccCost < ExtCost + MulCost + RedCost;
3527 },
3528 Range);
3529 };
3530
3531 VPValue *VecOp = Red->getVecOp();
3532 VPRecipeBase *Sub = nullptr;
3533 VPValue *A, *B;
3534 VPValue *Tmp = nullptr;
3535
3536 if (RedTy->isFloatingPointTy())
3537 return nullptr;
3538
3539 // Sub reductions could have a sub between the add reduction and vec op.
3540 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Tmp)))) {
3541 Sub = VecOp->getDefiningRecipe();
3542 VecOp = Tmp;
3543 }
3544
3545 // If ValB is a constant and can be safely extended, truncate it to the same
3546 // type as ExtA's operand, then extend it to the same type as ExtA. This
3547 // creates two uniform extends that can more easily be matched by the rest of
3548 // the bundling code. The ExtB reference, ValB and operand 1 of Mul are all
3549 // replaced with the new extend of the constant.
3550 auto ExtendAndReplaceConstantOp = [](VPWidenCastRecipe *ExtA,
3551 VPWidenCastRecipe *&ExtB, VPValue *&ValB,
3552 VPWidenRecipe *Mul) {
3553 if (!ExtA || ExtB || !isa<VPIRValue>(Val: ValB))
3554 return;
3555 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
3556 Instruction::CastOps ExtOpc = ExtA->getOpcode();
3557 const APInt *Const;
3558 if (!match(V: ValB, P: m_APInt(C&: Const)) ||
3559 !llvm::canConstantBeExtended(
3560 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
3561 return;
3562 // The truncate ensures that the type of each extended operand is the
3563 // same, and it's been proven that the constant can be extended from
3564 // NarrowTy safely. Necessary since ExtA's extended operand would be
3565 // e.g. an i8, while the const will likely be an i32. This will be
3566 // elided by later optimisations.
3567 VPBuilder Builder(Mul);
3568 auto *Trunc =
3569 Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc, Op: ValB, ResultTy: NarrowTy);
3570 Type *WideTy = ExtA->getScalarType();
3571 ValB = ExtB = Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy);
3572 Mul->setOperand(I: 1, New: ExtB);
3573 };
3574
3575 // Try to match reduce.add(mul(...)).
3576 if (match(V: VecOp, P: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B)))) {
3577 auto *RecipeA = dyn_cast<VPWidenCastRecipe>(Val: A);
3578 auto *RecipeB = dyn_cast<VPWidenCastRecipe>(Val: B);
3579 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
3580
3581 // Convert reduce.add(mul(ext, const)) to reduce.add(mul(ext, ext(const)))
3582 ExtendAndReplaceConstantOp(RecipeA, RecipeB, B, Mul);
3583
3584 // Match reduce.add/sub(mul(ext, ext)).
3585 if (RecipeA && RecipeB && match(V: RecipeA, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
3586 match(V: RecipeB, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
3587 IsMulAccValidAndClampRange(Mul, RecipeA, RecipeB, nullptr)) {
3588 if (Sub)
3589 return new VPExpressionRecipe(RecipeA, RecipeB, Mul,
3590 cast<VPWidenRecipe>(Val: Sub), Red);
3591 return new VPExpressionRecipe(RecipeA, RecipeB, Mul, Red);
3592 }
3593 // TODO: Add an expression type for this variant with a negated mul
3594 if (!Sub && IsMulAccValidAndClampRange(Mul, nullptr, nullptr, nullptr))
3595 return new VPExpressionRecipe(Mul, Red);
3596 }
3597 // TODO: Add an expression type for negated versions of other expression
3598 // variants.
3599 if (Sub)
3600 return nullptr;
3601
3602 // Match reduce.add(ext(mul(A, B))).
3603 if (match(V: VecOp, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))))) {
3604 auto *Ext = cast<VPWidenCastRecipe>(Val: VecOp);
3605 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
3606 auto *Ext0 = dyn_cast<VPWidenCastRecipe>(Val: A);
3607 auto *Ext1 = dyn_cast<VPWidenCastRecipe>(Val: B);
3608
3609 // reduce.add(ext(mul(ext, const)))
3610 // -> reduce.add(ext(mul(ext, ext(const))))
3611 ExtendAndReplaceConstantOp(Ext0, Ext1, B, Mul);
3612
3613 // reduce.add(ext(mul(ext(A), ext(B))))
3614 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
3615 // The inner extends must either have the same opcode as the outer extend or
3616 // be the same, in which case the multiply can never result in a negative
3617 // value and the outer extend can be folded away by doing wider
3618 // extends for the operands of the mul.
3619 if (Ext0 && Ext1 &&
3620 (Ext->getOpcode() == Ext0->getOpcode() || Ext0 == Ext1) &&
3621 Ext0->getOpcode() == Ext1->getOpcode() &&
3622 IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext) && Mul->hasOneUse()) {
3623 auto *NewExt0 = new VPWidenCastRecipe(
3624 Ext0->getOpcode(), Ext0->getOperand(N: 0), Ext->getScalarType(), nullptr,
3625 *Ext0, *Ext0, Ext0->getDebugLoc());
3626 NewExt0->insertBefore(InsertPos: Ext0);
3627
3628 VPWidenCastRecipe *NewExt1 = NewExt0;
3629 if (Ext0 != Ext1) {
3630 NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(N: 0),
3631 Ext->getScalarType(), nullptr, *Ext1,
3632 *Ext1, Ext1->getDebugLoc());
3633 NewExt1->insertBefore(InsertPos: Ext1);
3634 }
3635 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewExt0, NewExt1});
3636 NewMul->insertBefore(InsertPos: Mul);
3637 Ext->replaceAllUsesWith(New: NewMul);
3638 Ext->eraseFromParent();
3639 Mul->eraseFromParent();
3640 return new VPExpressionRecipe(NewExt0, NewExt1, NewMul, Red);
3641 }
3642 }
3643 return nullptr;
3644}
3645
3646/// This function tries to create abstract recipes from the reduction recipe for
3647/// following optimizations and cost estimation.
3648static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
3649 VPCostContext &Ctx,
3650 VFRange &Range) {
3651 // Creation of VPExpressions for partial reductions is entirely handled in
3652 // transformToPartialReduction.
3653 assert(!Red->isPartialReduction() &&
3654 "This path does not support partial reductions");
3655
3656 VPExpressionRecipe *AbstractR = nullptr;
3657 auto IP = std::next(x: Red->getIterator());
3658 auto *VPBB = Red->getParent();
3659 if (auto *MulAcc = tryToMatchAndCreateMulAccumulateReduction(Red, Ctx, Range))
3660 AbstractR = MulAcc;
3661 else if (auto *ExtRed = tryToMatchAndCreateExtendedReduction(Red, Ctx, Range))
3662 AbstractR = ExtRed;
3663 // Cannot create abstract inloop reduction recipes.
3664 if (!AbstractR)
3665 return;
3666
3667 AbstractR->insertBefore(BB&: *VPBB, IP);
3668 Red->replaceAllUsesWith(New: AbstractR);
3669}
3670
3671void VPlanTransforms::convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
3672 VFRange &Range) {
3673 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3674 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
3675 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
3676 if (auto *Red = dyn_cast<VPReductionRecipe>(Val: &R))
3677 tryToCreateAbstractReductionRecipe(Red, Ctx, Range);
3678 }
3679 }
3680}
3681
3682// Collect common metadata from a group of replicate recipes by intersecting
3683// metadata from all recipes in the group.
3684static VPIRMetadata getCommonMetadata(ArrayRef<VPReplicateRecipe *> Recipes) {
3685 VPIRMetadata CommonMetadata = *Recipes.front();
3686 for (VPReplicateRecipe *Recipe : drop_begin(RangeOrContainer&: Recipes))
3687 CommonMetadata.intersect(MD: *Recipe);
3688 return CommonMetadata;
3689}
3690
3691template <unsigned Opcode>
3692static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
3693collectComplementaryPredicatedMemOps(VPlan &Plan,
3694 PredicatedScalarEvolution &PSE,
3695 const Loop *L) {
3696 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
3697 "Only Load and Store opcodes supported");
3698 [[maybe_unused]] constexpr bool IsLoad = (Opcode == Instruction::Load);
3699
3700 // For each address, collect operations with the same or complementary masks.
3701 SmallVector<SmallVector<VPReplicateRecipe *, 4>> AllGroups;
3702 auto Groups = collectGroupedReplicateMemOps<Opcode>(
3703 Plan, PSE, L,
3704 [](VPReplicateRecipe *RepR) { return RepR->isPredicated(); });
3705 for (auto Recipes : Groups) {
3706 if (Recipes.size() < 2)
3707 continue;
3708
3709 assert(all_equal(
3710 map_range(Recipes, bind_back<getLoadStoreValueType>(IsLoad))) &&
3711 "Expected all recipes in group to have the same load-store type");
3712
3713 // Collect groups with the same or complementary masks.
3714 for (VPReplicateRecipe *&RecipeI : Recipes) {
3715 if (!RecipeI)
3716 continue;
3717
3718 VPValue *MaskI = RecipeI->getMask();
3719 SmallVector<VPReplicateRecipe *, 4> Group;
3720 Group.push_back(Elt: RecipeI);
3721 RecipeI = nullptr;
3722
3723 // Find all operations with the same or complementary masks.
3724 bool HasComplementaryMask = false;
3725 for (VPReplicateRecipe *&RecipeJ : Recipes) {
3726 if (!RecipeJ)
3727 continue;
3728
3729 VPValue *MaskJ = RecipeJ->getMask();
3730 // Check if any operation in the group has a complementary mask with
3731 // another, that is M1 == NOT(M2) or M2 == NOT(M1).
3732 HasComplementaryMask |= match(V: MaskI, P: m_Not(Op0: m_Specific(VPV: MaskJ))) ||
3733 match(V: MaskJ, P: m_Not(Op0: m_Specific(VPV: MaskI)));
3734 Group.push_back(Elt: RecipeJ);
3735 RecipeJ = nullptr;
3736 }
3737
3738 if (HasComplementaryMask) {
3739 assert(Group.size() >= 2 && "must have at least 2 entries");
3740 AllGroups.push_back(Elt: std::move(Group));
3741 }
3742 }
3743 }
3744
3745 return AllGroups;
3746}
3747
3748// Find the recipe with minimum alignment in the group.
3749template <typename InstType>
3750static VPReplicateRecipe *
3751findRecipeWithMinAlign(ArrayRef<VPReplicateRecipe *> Group) {
3752 return *min_element(Group, [](VPReplicateRecipe *A, VPReplicateRecipe *B) {
3753 return cast<InstType>(A->getUnderlyingInstr())->getAlign() <
3754 cast<InstType>(B->getUnderlyingInstr())->getAlign();
3755 });
3756}
3757
3758void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
3759 PredicatedScalarEvolution &PSE,
3760 const Loop *L) {
3761 auto Groups =
3762 collectComplementaryPredicatedMemOps<Instruction::Load>(Plan, PSE, L);
3763 if (Groups.empty())
3764 return;
3765
3766 // Process each group of loads.
3767 for (auto &Group : Groups) {
3768 // Try to use the earliest (most dominating) load to replace all others.
3769 VPReplicateRecipe *EarliestLoad = Group[0];
3770 VPBasicBlock *FirstBB = EarliestLoad->getParent();
3771 VPBasicBlock *LastBB = Group.back()->getParent();
3772
3773 // Check that the load doesn't alias with stores between first and last.
3774 auto LoadLoc = vputils::getMemoryLocation(R: *EarliestLoad);
3775 if (!LoadLoc || !canHoistOrSinkWithNoAliasCheck(MemLoc: *LoadLoc, FirstBB, LastBB))
3776 continue;
3777
3778 // Collect common metadata from all loads in the group.
3779 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
3780
3781 // Find the load with minimum alignment to use.
3782 auto *LoadWithMinAlign = findRecipeWithMinAlign<LoadInst>(Group);
3783
3784 bool IsSingleScalar = EarliestLoad->isSingleScalar();
3785 assert(all_of(Group,
3786 [IsSingleScalar](VPReplicateRecipe *R) {
3787 return R->isSingleScalar() == IsSingleScalar;
3788 }) &&
3789 "all members in group must agree on IsSingleScalar");
3790
3791 // Create an unpredicated version of the earliest load with common
3792 // metadata.
3793 auto *UnpredicatedLoad = new VPReplicateRecipe(
3794 LoadWithMinAlign->getUnderlyingInstr(), {EarliestLoad->getOperand(N: 0)},
3795 IsSingleScalar, /*Mask=*/nullptr, *EarliestLoad, CommonMetadata);
3796
3797 UnpredicatedLoad->insertBefore(InsertPos: EarliestLoad);
3798
3799 // Replace all loads in the group with the unpredicated load.
3800 for (VPReplicateRecipe *Load : Group) {
3801 Load->replaceAllUsesWith(New: UnpredicatedLoad);
3802 Load->eraseFromParent();
3803 }
3804 }
3805}
3806
3807static bool
3808canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
3809 PredicatedScalarEvolution &PSE, const Loop &L) {
3810 auto StoreLoc = vputils::getMemoryLocation(R: *StoresToSink.front());
3811 if (!StoreLoc || !StoreLoc->AATags.Scope)
3812 return false;
3813
3814 // When sinking a group of stores, all members of the group alias each other.
3815 // Skip them during the alias checks.
3816 VPBasicBlock *FirstBB = StoresToSink.front()->getParent();
3817 VPBasicBlock *LastBB = StoresToSink.back()->getParent();
3818 SinkStoreInfo SinkInfo(StoresToSink, *StoresToSink[0], PSE, L);
3819 return canHoistOrSinkWithNoAliasCheck(MemLoc: *StoreLoc, FirstBB, LastBB, SinkInfo);
3820}
3821
3822void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
3823 PredicatedScalarEvolution &PSE,
3824 const Loop *L) {
3825 auto Groups =
3826 collectComplementaryPredicatedMemOps<Instruction::Store>(Plan, PSE, L);
3827 if (Groups.empty())
3828 return;
3829
3830 for (auto &Group : Groups) {
3831 if (!canSinkStoreWithNoAliasCheck(StoresToSink: Group, PSE, L: *L))
3832 continue;
3833
3834 // Use the last (most dominated) store's location for the unconditional
3835 // store.
3836 VPReplicateRecipe *LastStore = Group.back();
3837 VPBasicBlock *InsertBB = LastStore->getParent();
3838
3839 // Collect common alias metadata from all stores in the group.
3840 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
3841
3842 // Build select chain for stored values.
3843 VPValue *SelectedValue = Group[0]->getOperand(N: 0);
3844 VPBuilder Builder(InsertBB, LastStore->getIterator());
3845
3846 bool IsSingleScalar = Group[0]->isSingleScalar();
3847 for (unsigned I = 1; I < Group.size(); ++I) {
3848 assert(IsSingleScalar == Group[I]->isSingleScalar() &&
3849 "all members in group must agree on IsSingleScalar");
3850 VPValue *Mask = Group[I]->getMask();
3851 VPValue *Value = Group[I]->getOperand(N: 0);
3852 SelectedValue = Builder.createSelect(
3853 Cond: Mask, TrueVal: Value, FalseVal: SelectedValue, DL: Group[I]->getDebugLoc(), Name: "",
3854 Flags: VPIRFlags::getDefaultFlags(Opcode: Instruction::Select,
3855 ResultTy: Value->getScalarType()));
3856 }
3857
3858 // Find the store with minimum alignment to use.
3859 auto *StoreWithMinAlign = findRecipeWithMinAlign<StoreInst>(Group);
3860
3861 // Create unconditional store with selected value and common metadata.
3862 auto *UnpredicatedStore = new VPReplicateRecipe(
3863 StoreWithMinAlign->getUnderlyingInstr(),
3864 {SelectedValue, LastStore->getOperand(N: 1)}, IsSingleScalar,
3865 /*Mask=*/nullptr, *LastStore, CommonMetadata);
3866 UnpredicatedStore->insertBefore(BB&: *InsertBB, IP: LastStore->getIterator());
3867
3868 // Remove all predicated stores from the group.
3869 for (VPReplicateRecipe *Store : Group)
3870 Store->eraseFromParent();
3871 }
3872}
3873
3874/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
3875/// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
3876/// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
3877/// the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to
3878/// an index-independent load if it feeds all wide ops at all indices (\p OpV
3879/// must be the operand at index \p OpIdx for both the recipe at lane 0, \p
3880/// WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if \p V
3881/// is defined at \p Idx of a load interleave group.
3882/// A live-in or recipe defined outside the loop region can be converted, if it
3883/// is the same across all lanes, or we can create a BuildVector for it.
3884static bool canNarrowLoad(VPSingleDefRecipe *WideMember0, unsigned OpIdx,
3885 VPValue *OpV, unsigned Idx, bool IsScalable) {
3886 VPValue *Member0Op = WideMember0->getOperand(N: OpIdx);
3887 if (Member0Op->isDefinedOutsideLoopRegions()) {
3888 // Operand matches Member0, broadcast across all fields for both live-ins
3889 // and recipes.
3890 if (Member0Op == OpV)
3891 return true;
3892 // Otherwise distinct per-field VPValues are assembled into a BuildVector.
3893 return !IsScalable && OpV->isDefinedOutsideLoopRegions() &&
3894 OpV->getScalarType() == Member0Op->getScalarType();
3895 }
3896 VPRecipeBase *Member0OpR = Member0Op->getDefiningRecipe();
3897 if (auto *W = dyn_cast<VPWidenLoadRecipe>(Val: Member0OpR))
3898 // For scalable VFs, the narrowed plan processes vscale iterations at once,
3899 // so a shared wide load cannot be narrowed to a uniform scalar; bail out.
3900 return !IsScalable && !W->getMask() && W->isConsecutive() &&
3901 Member0Op == OpV;
3902 if (auto *IR = dyn_cast<VPInterleaveRecipe>(Val: Member0OpR))
3903 return IR->getInterleaveGroup()->isFull() && IR->getVPValue(I: Idx) == OpV;
3904 return false;
3905}
3906
3907static bool canNarrowOps(ArrayRef<VPValue *> Ops, bool IsScalable) {
3908 SmallVector<VPValue *> Ops0;
3909 auto *WideMember0 = dyn_cast<VPRecipeWithIRFlags>(Val: Ops[0]);
3910 if (!WideMember0)
3911 return false;
3912 for (VPValue *V : Ops) {
3913 if (!isa<VPWidenRecipe, VPWidenCastRecipe>(Val: V))
3914 return false;
3915 auto *R = cast<VPRecipeWithIRFlags>(Val: V);
3916 if (vputils::getOpcode(V: R) != vputils::getOpcode(V: WideMember0))
3917 return false;
3918 if (R->getScalarType() != WideMember0->getScalarType())
3919 return false;
3920 if (R->hasPredicate() && R->getPredicate() != WideMember0->getPredicate())
3921 return false;
3922 }
3923
3924 for (unsigned Idx = 0; Idx != WideMember0->getNumOperands(); ++Idx) {
3925 SmallVector<VPValue *> OpsI;
3926 for (VPValue *Op : Ops)
3927 OpsI.push_back(Elt: Op->getDefiningRecipe()->getOperand(N: Idx));
3928
3929 if (canNarrowOps(Ops: OpsI, IsScalable))
3930 continue;
3931
3932 if (any_of(Range: enumerate(First&: OpsI), P: [WideMember0, Idx, IsScalable](const auto &P) {
3933 const auto &[OpIdx, OpV] = P;
3934 return !canNarrowLoad(WideMember0, Idx, OpV, OpIdx, IsScalable);
3935 }))
3936 return false;
3937 }
3938
3939 return true;
3940}
3941
3942/// Returns VF from \p VFs if \p IR is a full interleave group with factor and
3943/// number of members both equal to VF. The interleave group must also access
3944/// the full vector width.
3945static std::optional<ElementCount>
3946isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR,
3947 ArrayRef<ElementCount> VFs,
3948 const TargetTransformInfo &TTI) {
3949 if (!InterleaveR || InterleaveR->getMask())
3950 return std::nullopt;
3951
3952 Type *GroupElementTy = nullptr;
3953 if (InterleaveR->getStoredValues().empty()) {
3954 GroupElementTy = InterleaveR->getVPValue(I: 0)->getScalarType();
3955 if (!all_of(Range: InterleaveR->definedValues(), P: [GroupElementTy](VPValue *Op) {
3956 return Op->getScalarType() == GroupElementTy;
3957 }))
3958 return std::nullopt;
3959 } else {
3960 GroupElementTy = InterleaveR->getStoredValues()[0]->getScalarType();
3961 if (!all_of(Range: InterleaveR->getStoredValues(), P: [GroupElementTy](VPValue *Op) {
3962 return Op->getScalarType() == GroupElementTy;
3963 }))
3964 return std::nullopt;
3965 }
3966
3967 auto IG = InterleaveR->getInterleaveGroup();
3968 if (IG->getFactor() != IG->getNumMembers())
3969 return std::nullopt;
3970
3971 auto GetVectorBitWidthForVF = [&TTI](ElementCount VF) {
3972 TypeSize Size = TTI.getRegisterBitWidth(
3973 K: VF.isFixed() ? TargetTransformInfo::RGK_FixedWidthVector
3974 : TargetTransformInfo::RGK_ScalableVector);
3975 assert(Size.isScalable() == VF.isScalable() &&
3976 "if Size is scalable, VF must be scalable and vice versa");
3977 return Size.getKnownMinValue();
3978 };
3979
3980 for (ElementCount VF : VFs) {
3981 unsigned MinVal = VF.getKnownMinValue();
3982 unsigned GroupSize = GroupElementTy->getScalarSizeInBits() * MinVal;
3983 if (IG->getFactor() == MinVal && GroupSize == GetVectorBitWidthForVF(VF))
3984 return {VF};
3985 }
3986 return std::nullopt;
3987}
3988
3989/// Returns true if \p VPValue is a narrow VPValue.
3990static bool isAlreadyNarrow(VPValue *VPV) {
3991 if (isa<VPIRValue>(Val: VPV))
3992 return true;
3993 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: VPV);
3994 return RepR && RepR->isSingleScalar();
3995}
3996
3997// Convert the wide recipes defining the VPValues in \p Members feeding an
3998// interleave group to a single narrow variant. The first member is reused as
3999// the narrowed recipe. BuildVectors for live-in operands are inserted into \p
4000// Preheader.
4001static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
4002 SmallPtrSetImpl<VPValue *> &NarrowedOps,
4003 VPBasicBlock *Preheader) {
4004 VPValue *V = Members.front();
4005 if (NarrowedOps.contains(Ptr: V))
4006 return V;
4007
4008 if (V->isDefinedOutsideLoopRegions()) {
4009 assert(all_of(Members,
4010 [V](VPValue *M) {
4011 return M->isDefinedOutsideLoopRegions() &&
4012 M->getScalarType() == V->getScalarType();
4013 }) &&
4014 "expected distinct loop-invariant values of matching scalar type");
4015 auto *BV = new VPInstruction(VPInstruction::BuildVector, Members);
4016 Preheader->appendRecipe(Recipe: BV);
4017 NarrowedOps.insert(Ptr: BV);
4018 return BV;
4019 }
4020
4021 if (isAlreadyNarrow(VPV: V))
4022 return V;
4023
4024 VPRecipeBase *R = V->getDefiningRecipe();
4025 if (isa<VPWidenRecipe, VPWidenCastRecipe>(Val: R)) {
4026 auto *WideMember0 = cast<VPRecipeWithIRFlags>(Val: R);
4027 for (VPValue *Member : Members.drop_front())
4028 WideMember0->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Member));
4029 for (unsigned Idx = 0, E = WideMember0->getNumOperands(); Idx != E; ++Idx) {
4030 SmallVector<VPValue *> OpsI;
4031 for (VPValue *Member : Members)
4032 OpsI.push_back(Elt: Member->getDefiningRecipe()->getOperand(N: Idx));
4033 WideMember0->setOperand(
4034 I: Idx, New: narrowInterleaveGroupOp(Members: OpsI, NarrowedOps, Preheader));
4035 }
4036 return V;
4037 }
4038
4039 if (auto *LoadGroup = dyn_cast<VPInterleaveRecipe>(Val: R)) {
4040 // Narrow interleave group to wide load, as transformed VPlan will only
4041 // process one original iteration.
4042 auto *LI = cast<LoadInst>(Val: LoadGroup->getInterleaveGroup()->getInsertPos());
4043 auto *L = VPBuilder(LoadGroup).createWidenLoad(
4044 Load&: *LI, Addr: LoadGroup->getAddr(), Mask: LoadGroup->getMask(), /*Consecutive=*/true,
4045 Metadata: *LoadGroup, DL: LoadGroup->getDebugLoc());
4046 NarrowedOps.insert(Ptr: L);
4047 return L;
4048 }
4049
4050 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R)) {
4051 assert(RepR->isSingleScalar() && RepR->getOpcode() == Instruction::Load &&
4052 "must be a single scalar load");
4053 NarrowedOps.insert(Ptr: RepR);
4054 return RepR;
4055 }
4056
4057 auto *WideLoad = cast<VPWidenLoadRecipe>(Val: R);
4058 VPValue *PtrOp = WideLoad->getAddr();
4059 if (auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(Val: PtrOp))
4060 PtrOp = VecPtr->getOperand(N: 0);
4061 // Narrow wide load to uniform scalar load, as transformed VPlan will only
4062 // process one original iteration.
4063 auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(), {PtrOp},
4064 /*IsUniform*/ true,
4065 /*Mask*/ nullptr, {}, *WideLoad);
4066 N->insertBefore(InsertPos: WideLoad);
4067 NarrowedOps.insert(Ptr: N);
4068 return N;
4069}
4070
4071std::unique_ptr<VPlan>
4072VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
4073 const TargetTransformInfo &TTI) {
4074 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
4075
4076 if (!VectorLoop)
4077 return nullptr;
4078
4079 // Only handle single-block loops for now.
4080 if (VectorLoop->getEntryBasicBlock() != VectorLoop->getExitingBasicBlock())
4081 return nullptr;
4082
4083 // Skip plans when we may not be able to properly narrow.
4084 VPBasicBlock *Exiting = VectorLoop->getExitingBasicBlock();
4085 if (!match(V: &Exiting->back(), P: m_BranchOnCount()))
4086 return nullptr;
4087
4088 assert(match(&Exiting->back(),
4089 m_BranchOnCount(m_Add(m_VPValue(), m_Specific(&Plan.getVFxUF())),
4090 m_Specific(&Plan.getVectorTripCount()))) &&
4091 "unexpected branch-on-count");
4092
4093 SmallVector<VPInterleaveRecipe *> StoreGroups;
4094 std::optional<ElementCount> VFToOptimize;
4095 for (auto &R : *VectorLoop->getEntryBasicBlock()) {
4096 if (isa<VPDerivedIVRecipe, VPScalarIVStepsRecipe>(Val: &R) &&
4097 vputils::onlyFirstLaneUsed(Def: cast<VPSingleDefRecipe>(Val: &R)))
4098 continue;
4099
4100 // Bail out on recipes not supported at the moment:
4101 // * phi recipes other than the canonical induction
4102 // * recipes writing to memory except interleave groups
4103 // Only support plans with a canonical induction phi.
4104 if (R.isPhi())
4105 return nullptr;
4106
4107 auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(Val: &R);
4108 if (R.mayWriteToMemory() && !InterleaveR)
4109 return nullptr;
4110
4111 // Bail out if any recipe defines a vector value used outside the
4112 // vector loop region.
4113 if (any_of(Range: R.definedValues(), P: [&](VPValue *V) {
4114 return any_of(Range: V->users(), P: [&](VPUser *U) {
4115 auto *UR = cast<VPRecipeBase>(Val: U);
4116 return UR->getParent()->getParent() != VectorLoop;
4117 });
4118 }))
4119 return nullptr;
4120
4121 // All other ops are allowed, but we reject uses that cannot be converted
4122 // when checking all allowed consumers (store interleave groups) below.
4123 if (!InterleaveR)
4124 continue;
4125
4126 // Try to find a single VF, where all interleave groups are consecutive and
4127 // saturate the full vector width. If we already have a candidate VF, check
4128 // if it is applicable for the current InterleaveR, otherwise look for a
4129 // suitable VF across the Plan's VFs.
4130 SmallVector<ElementCount> VFs =
4131 VFToOptimize ? SmallVector<ElementCount>({*VFToOptimize})
4132 : to_vector(Range: Plan.vectorFactors());
4133 std::optional<ElementCount> NarrowedVF =
4134 isConsecutiveInterleaveGroup(InterleaveR, VFs, TTI);
4135 if (!NarrowedVF || (VFToOptimize && NarrowedVF != VFToOptimize))
4136 return nullptr;
4137 VFToOptimize = NarrowedVF;
4138
4139 // Skip read interleave groups.
4140 if (InterleaveR->getStoredValues().empty())
4141 continue;
4142
4143 // Narrow interleave groups, if all operands are already matching narrow
4144 // ops.
4145 auto *Member0 = InterleaveR->getStoredValues()[0];
4146 if (isAlreadyNarrow(VPV: Member0) &&
4147 all_of(Range: InterleaveR->getStoredValues(), P: equal_to(Arg&: Member0))) {
4148 StoreGroups.push_back(Elt: InterleaveR);
4149 continue;
4150 }
4151
4152 // For now, we only support full interleave groups storing load interleave
4153 // groups.
4154 if (all_of(Range: enumerate(First: InterleaveR->getStoredValues()), P: [](auto Op) {
4155 VPRecipeBase *DefR = Op.value()->getDefiningRecipe();
4156 if (!DefR)
4157 return false;
4158 auto *IR = dyn_cast<VPInterleaveRecipe>(Val: DefR);
4159 return IR && IR->getInterleaveGroup()->isFull() &&
4160 IR->getVPValue(Op.index()) == Op.value();
4161 })) {
4162 StoreGroups.push_back(Elt: InterleaveR);
4163 continue;
4164 }
4165
4166 // Check if all values feeding InterleaveR are matching wide recipes, which
4167 // operands that can be narrowed.
4168 if (!canNarrowOps(Ops: InterleaveR->getStoredValues(),
4169 IsScalable: VFToOptimize->isScalable()))
4170 return nullptr;
4171 StoreGroups.push_back(Elt: InterleaveR);
4172 }
4173
4174 if (StoreGroups.empty())
4175 return nullptr;
4176
4177 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
4178 bool RequiresScalarEpilogue =
4179 MiddleVPBB->getNumSuccessors() == 1 &&
4180 MiddleVPBB->getSingleSuccessor() == Plan.getScalarPreheader();
4181 // Bail out for tail-folding (middle block with a single successor to exit).
4182 if (MiddleVPBB->getNumSuccessors() != 2 && !RequiresScalarEpilogue)
4183 return nullptr;
4184
4185 // All interleave groups in Plan can be narrowed for VFToOptimize. Split the
4186 // original Plan into 2: a) a new clone which contains all VFs of Plan, except
4187 // VFToOptimize, and b) the original Plan with VFToOptimize as single VF.
4188 // TODO: Handle cases where only some interleave groups can be narrowed.
4189 std::unique_ptr<VPlan> NewPlan;
4190 if (size(Range: Plan.vectorFactors()) != 1) {
4191 NewPlan = std::unique_ptr<VPlan>(Plan.duplicate());
4192 Plan.setVF(*VFToOptimize);
4193 NewPlan->removeVF(VF: *VFToOptimize);
4194 }
4195
4196 // Convert InterleaveGroup \p R to a single VPWidenLoadRecipe.
4197 SmallPtrSet<VPValue *, 4> NarrowedOps;
4198 VPBasicBlock *Preheader = Plan.getVectorPreheader();
4199 // Narrow operation tree rooted at store groups.
4200 for (auto *StoreGroup : StoreGroups) {
4201 VPValue *Res = narrowInterleaveGroupOp(Members: StoreGroup->getStoredValues(),
4202 NarrowedOps, Preheader);
4203 auto *SI =
4204 cast<StoreInst>(Val: StoreGroup->getInterleaveGroup()->getInsertPos());
4205 VPBuilder(StoreGroup)
4206 .createWidenStore(Store&: *SI, Addr: StoreGroup->getAddr(), StoredVal: Res, Mask: nullptr,
4207 /*Consecutive=*/true, Metadata: *StoreGroup,
4208 DL: StoreGroup->getDebugLoc());
4209 StoreGroup->eraseFromParent();
4210 }
4211
4212 // Adjust induction to reflect that the transformed plan only processes one
4213 // original iteration.
4214 VPInstruction *CanIVInc = vputils::findCanonicalIVIncrement(Plan);
4215 Type *CanIVTy = VectorLoop->getCanonicalIVType();
4216 VPBasicBlock *VectorPH = Plan.getVectorPreheader();
4217 VPBuilder PHBuilder(VectorPH, VectorPH->begin());
4218
4219 VPValue *UF = &Plan.getUF();
4220 VPValue *Step;
4221 if (VFToOptimize->isScalable()) {
4222 VPValue *VScale =
4223 PHBuilder.createElementCount(Ty: CanIVTy, EC: ElementCount::getScalable(MinVal: 1));
4224 Step = PHBuilder.createOverflowingOp(Opcode: Instruction::Mul, Operands: {VScale, UF},
4225 WrapFlags: {true, false});
4226 Plan.getVF().replaceAllUsesWith(New: VScale);
4227 } else {
4228 Step = UF;
4229 Plan.getVF().replaceAllUsesWith(New: Plan.getConstantInt(Ty: CanIVTy, Val: 1));
4230 }
4231 // Materialize vector trip count with the narrowed step.
4232 materializeVectorTripCount(Plan, VectorPHVPBB: VectorPH, /*TailByMasking=*/false,
4233 RequiresScalarEpilogue, Step);
4234
4235 CanIVInc->setOperand(I: 1, New: Step);
4236 Plan.getVFxUF().replaceAllUsesWith(New: Step);
4237
4238 removeDeadRecipes(Plan);
4239 assert(none_of(*VectorLoop->getEntryBasicBlock(),
4240 IsaPred<VPVectorPointerRecipe>) &&
4241 "All VPVectorPointerRecipes should have been removed");
4242 return NewPlan;
4243}
4244
4245void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
4246 VFRange &Range) {
4247 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
4248 auto *MiddleVPBB = Plan.getMiddleBlock();
4249 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
4250
4251 auto IsScalableOne = [](ElementCount VF) -> bool {
4252 return VF == ElementCount::getScalable(MinVal: 1);
4253 };
4254
4255 for (auto &HeaderPhi : VectorRegion->getEntryBasicBlock()->phis()) {
4256 auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Val: &HeaderPhi);
4257 if (!FOR)
4258 continue;
4259
4260 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
4261 "Cannot handle loops with uncountable early exits");
4262
4263 // Find the existing splice for this FOR, created in
4264 // createHeaderPhiRecipes. All uses of FOR have already been replaced with
4265 // RecurSplice there; only RecurSplice itself still references FOR.
4266 auto *RecurSplice =
4267 findUserOf<VPInstruction::FirstOrderRecurrenceSplice>(V: FOR);
4268 assert(RecurSplice && "expected FirstOrderRecurrenceSplice");
4269
4270 // For VF vscale x 1, if vscale = 1, we are unable to extract the
4271 // penultimate value of the recurrence. Instead we rely on the existing
4272 // extract of the last element from the result of
4273 // VPInstruction::FirstOrderRecurrenceSplice.
4274 // TODO: Consider vscale_range info and UF.
4275 if (any_of(Range: RecurSplice->users(),
4276 P: [](VPUser *U) { return !cast<VPRecipeBase>(Val: U)->getRegion(); }) &&
4277 LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsScalableOne,
4278 Range))
4279 return;
4280
4281 // This is the second phase of vectorizing first-order recurrences, creating
4282 // extracts for users outside the loop. An overview of the transformation is
4283 // described below. Suppose we have the following loop with some use after
4284 // the loop of the last a[i-1],
4285 //
4286 // for (int i = 0; i < n; ++i) {
4287 // t = a[i - 1];
4288 // b[i] = a[i] - t;
4289 // }
4290 // use t;
4291 //
4292 // There is a first-order recurrence on "a". For this loop, the shorthand
4293 // scalar IR looks like:
4294 //
4295 // scalar.ph:
4296 // s.init = a[-1]
4297 // br scalar.body
4298 //
4299 // scalar.body:
4300 // i = phi [0, scalar.ph], [i+1, scalar.body]
4301 // s1 = phi [s.init, scalar.ph], [s2, scalar.body]
4302 // s2 = a[i]
4303 // b[i] = s2 - s1
4304 // br cond, scalar.body, exit.block
4305 //
4306 // exit.block:
4307 // use = lcssa.phi [s1, scalar.body]
4308 //
4309 // In this example, s1 is a recurrence because it's value depends on the
4310 // previous iteration. In the first phase of vectorization, we created a
4311 // VPFirstOrderRecurrencePHIRecipe v1 for s1. Now we create the extracts
4312 // for users in the scalar preheader and exit block.
4313 //
4314 // vector.ph:
4315 // v_init = vector(..., ..., ..., a[-1])
4316 // br vector.body
4317 //
4318 // vector.body
4319 // i = phi [0, vector.ph], [i+4, vector.body]
4320 // v1 = phi [v_init, vector.ph], [v2, vector.body]
4321 // v2 = a[i, i+1, i+2, i+3]
4322 // v1' = splice(v1(3), v2(0, 1, 2))
4323 // b[i, i+1, i+2, i+3] = v2 - v1'
4324 // br cond, vector.body, middle.block
4325 //
4326 // middle.block:
4327 // vector.recur.extract.for.phi = v2(2)
4328 // vector.recur.extract = v2(3)
4329 // br cond, scalar.ph, exit.block
4330 //
4331 // scalar.ph:
4332 // scalar.recur.init = phi [vector.recur.extract, middle.block],
4333 // [s.init, otherwise]
4334 // br scalar.body
4335 //
4336 // scalar.body:
4337 // i = phi [0, scalar.ph], [i+1, scalar.body]
4338 // s1 = phi [scalar.recur.init, scalar.ph], [s2, scalar.body]
4339 // s2 = a[i]
4340 // b[i] = s2 - s1
4341 // br cond, scalar.body, exit.block
4342 //
4343 // exit.block:
4344 // lo = lcssa.phi [s1, scalar.body],
4345 // [vector.recur.extract.for.phi, middle.block]
4346 //
4347 // Update extracts of the splice in the middle block: they extract the
4348 // penultimate element of the recurrence.
4349 for (VPRecipeBase &R : make_early_inc_range(
4350 Range: make_range(x: MiddleVPBB->getFirstNonPhi(), y: MiddleVPBB->end()))) {
4351 if (!match(V: &R, P: m_ExtractLastLaneOfLastPart(Op0: m_Specific(VPV: RecurSplice))))
4352 continue;
4353
4354 auto *ExtractR = cast<VPInstruction>(Val: &R);
4355 VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
4356 Opcode: VPInstruction::ExtractPenultimateElement, Operands: RecurSplice->getOperand(N: 1),
4357 DL: {}, Name: "vector.recur.extract.for.phi");
4358 for (VPUser *ExitU : to_vector(Range: ExtractR->users())) {
4359 if (auto *ExitPhi = dyn_cast<VPIRPhi>(Val: ExitU))
4360 ExitPhi->replaceUsesOfWith(From: ExtractR, To: PenultimateElement);
4361 }
4362 }
4363 }
4364}
4365
4366/// Check if \p V is a binary expression of a widened IV and a loop-invariant
4367/// value. Returns the widened IV if found, nullptr otherwise.
4368static VPWidenIntOrFpInductionRecipe *getExpressionIV(VPValue *V) {
4369 auto *BinOp = dyn_cast<VPWidenRecipe>(Val: V);
4370 if (!BinOp || !Instruction::isBinaryOp(Opcode: BinOp->getOpcode()) ||
4371 Instruction::isIntDivRem(Opcode: BinOp->getOpcode()))
4372 return nullptr;
4373
4374 VPValue *WidenIVCandidate = BinOp->getOperand(N: 0);
4375 VPValue *InvariantCandidate = BinOp->getOperand(N: 1);
4376 if (!isa<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate))
4377 std::swap(a&: WidenIVCandidate, b&: InvariantCandidate);
4378
4379 if (!InvariantCandidate->isDefinedOutsideLoopRegions())
4380 return nullptr;
4381
4382 return dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate);
4383}
4384
4385/// Create a scalar version of \p BinOp, with its \p WidenIV operand replaced
4386/// by \p ScalarIV, and place it after \p ScalarIV's defining recipe.
4387static VPValue *cloneBinOpForScalarIV(VPWidenRecipe *BinOp, VPValue *ScalarIV,
4388 VPWidenIntOrFpInductionRecipe *WidenIV) {
4389 assert(Instruction::isBinaryOp(BinOp->getOpcode()) &&
4390 BinOp->getNumOperands() == 2 && "BinOp must have 2 operands");
4391 auto *ClonedOp = BinOp->clone();
4392 if (ClonedOp->getOperand(N: 0) == WidenIV) {
4393 ClonedOp->setOperand(I: 0, New: ScalarIV);
4394 } else {
4395 assert(ClonedOp->getOperand(1) == WidenIV && "one operand must be WideIV");
4396 ClonedOp->setOperand(I: 1, New: ScalarIV);
4397 }
4398 ClonedOp->insertAfter(InsertPos: ScalarIV->getDefiningRecipe());
4399 return ClonedOp;
4400}
4401
4402/// If \p S is an affine AddRec, returns true if its step is known to be
4403/// positive and false if it is known to be negative. Returns std::nullopt if
4404/// \p S is not an affine AddRec, or if the sign of its step cannot be
4405/// determined.
4406static std::optional<bool> getStepDirection(const SCEV *S,
4407 ScalarEvolution &SE) {
4408 const SCEV *Step;
4409 if (!match(S, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV(V&: Step))))
4410 return std::nullopt;
4411 if (SE.isKnownPositive(S: Step))
4412 return true;
4413 if (SE.isKnownNegative(S: Step))
4414 return false;
4415 return std::nullopt;
4416}
4417
4418void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
4419 PredicatedScalarEvolution &PSE,
4420 Loop &L) {
4421 ScalarEvolution &SE = *PSE.getSE();
4422 VPRegionBlock *VectorLoopRegion = Plan.getVectorLoopRegion();
4423
4424 // Helper lambda to check if the IV range excludes the sentinel value. Try
4425 // signed first, then unsigned. Return an excluded sentinel if found,
4426 // otherwise return std::nullopt.
4427 auto CheckSentinel = [&SE](const SCEV *IVSCEV,
4428 bool UseMax) -> std::optional<APSInt> {
4429 unsigned BW = IVSCEV->getType()->getScalarSizeInBits();
4430 for (bool Signed : {true, false}) {
4431 APSInt Sentinel = UseMax ? APSInt::getMinValue(numBits: BW, /*Unsigned=*/!Signed)
4432 : APSInt::getMaxValue(numBits: BW, /*Unsigned=*/!Signed);
4433
4434 ConstantRange IVRange =
4435 Signed ? SE.getSignedRange(S: IVSCEV) : SE.getUnsignedRange(S: IVSCEV);
4436 if (!IVRange.contains(Val: Sentinel))
4437 return Sentinel;
4438 }
4439 return std::nullopt;
4440 };
4441
4442 VPValue *HeaderMask = VectorLoopRegion->getHeaderMask();
4443 for (VPRecipeBase &Phi :
4444 make_early_inc_range(Range: VectorLoopRegion->getEntryBasicBlock()->phis())) {
4445 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &Phi);
4446 if (!PhiR || !RecurrenceDescriptor::isFindLastRecurrenceKind(
4447 Kind: PhiR->getRecurrenceKind()))
4448 continue;
4449
4450 Type *PhiTy = PhiR->getScalarType();
4451 if (PhiTy->isPointerTy() || PhiTy->isFloatingPointTy())
4452 continue;
4453
4454 // If there's a header mask, the backedge select will not be the find-last
4455 // select.
4456 VPValue *BackedgeVal = PhiR->getBackedgeValue();
4457 auto *FindLastSelect = cast<VPSingleDefRecipe>(Val: BackedgeVal);
4458 if (HeaderMask &&
4459 !match(V: BackedgeVal,
4460 P: m_Select(Op0: m_Specific(VPV: HeaderMask),
4461 Op1: m_VPSingleDefRecipe(V&: FindLastSelect), Op2: m_Specific(VPV: PhiR))))
4462 continue;
4463
4464 // Get the find-last expression from the find-last select of the reduction
4465 // phi. The find-last select should be a select between the phi and the
4466 // find-last expression.
4467 VPValue *Cond, *FindLastExpression;
4468 if (!match(R: FindLastSelect, P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: PhiR),
4469 Op2: m_VPValue(V&: FindLastExpression))) &&
4470 !match(R: FindLastSelect,
4471 P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_VPValue(V&: FindLastExpression),
4472 Op2: m_Specific(VPV: PhiR))))
4473 continue;
4474
4475 // Check if FindLastExpression is a simple expression of a widened IV. If
4476 // so, we can track the underlying IV instead and sink the expression.
4477 auto *IVOfExpressionToSink = getExpressionIV(V: FindLastExpression);
4478 const SCEV *IVSCEV = vputils::getSCEVExprForVPValue(
4479 V: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression, PSE,
4480 L: &L);
4481 if (!match(S: IVSCEV, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV()))) {
4482 assert(!match(vputils::getSCEVExprForVPValue(FindLastExpression, PSE, &L),
4483 m_scev_AffineAddRec(m_SCEV(), m_SCEV())) &&
4484 "IVOfExpressionToSink not being an AddRec must imply "
4485 "FindLastExpression not being an AddRec.");
4486 continue;
4487 }
4488
4489 // Determine direction from the step of IVSCEV, if possible.
4490 std::optional<bool> StepDirection = getStepDirection(S: IVSCEV, SE);
4491 if (!StepDirection)
4492 continue;
4493
4494 bool UseMax = *StepDirection;
4495 std::optional<APSInt> SentinelVal = CheckSentinel(IVSCEV, UseMax);
4496 bool UseSigned = SentinelVal && SentinelVal->isSigned();
4497
4498 // Sinking an expression will disable epilogue vectorization. Only use it,
4499 // if FindLastExpression cannot be vectorized via a sentinel. Sinking may
4500 // also prevent vectorizing using a sentinel (e.g., if the expression is a
4501 // multiply or divide by large constant, respectively), which also makes
4502 // sinking undesirable.
4503 if (IVOfExpressionToSink) {
4504 const SCEV *FindLastExpressionSCEV =
4505 vputils::getSCEVExprForVPValue(V: FindLastExpression, PSE, L: &L);
4506 if (std::optional<bool> NewUseMax =
4507 getStepDirection(S: FindLastExpressionSCEV, SE)) {
4508 if (auto NewSentinel =
4509 CheckSentinel(FindLastExpressionSCEV, *NewUseMax)) {
4510 // The original expression already has a sentinel, so prefer not
4511 // sinking to keep epilogue vectorization possible.
4512 SentinelVal = *NewSentinel;
4513 UseSigned = NewSentinel->isSigned();
4514 UseMax = *NewUseMax;
4515 IVSCEV = FindLastExpressionSCEV;
4516 IVOfExpressionToSink = nullptr;
4517 }
4518 }
4519 }
4520
4521 // If no sentinel was found, fall back to a boolean AnyOf reduction to track
4522 // if the condition was ever true. Requires the IV to not wrap, otherwise we
4523 // cannot use min/max.
4524 if (!SentinelVal) {
4525 auto *AR = cast<SCEVAddRecExpr>(Val: IVSCEV);
4526 if (AR->hasNoSignedWrap())
4527 UseSigned = true;
4528 else if (AR->hasNoUnsignedWrap())
4529 UseSigned = false;
4530 else
4531 continue;
4532 }
4533
4534 VPInstruction *RdxResult = cast<VPInstruction>(Val: vputils::findRecipe(
4535 Start: BackedgeVal,
4536 Pred: match_fn(P: m_VPInstruction<VPInstruction::ComputeReductionResult>())));
4537
4538 VPValue *NewFindLastSelect = BackedgeVal;
4539 VPValue *SelectCond = Cond;
4540 if (!SentinelVal || IVOfExpressionToSink) {
4541 // When we need to create a new select, normalize the condition so that
4542 // PhiR is the last operand and include the header mask if needed.
4543 DebugLoc DL = FindLastSelect->getDefiningRecipe()->getDebugLoc();
4544 VPBuilder LoopBuilder(FindLastSelect->getDefiningRecipe());
4545 if (match(R: FindLastSelect,
4546 P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: PhiR), Op2: m_VPValue())))
4547 SelectCond = LoopBuilder.createNot(Operand: SelectCond);
4548
4549 // When tail folding, mask the condition with the header mask to prevent
4550 // propagating poison from inactive lanes in the last vector iteration.
4551 if (HeaderMask)
4552 SelectCond = LoopBuilder.createLogicalAnd(LHS: HeaderMask, RHS: SelectCond);
4553
4554 if (SelectCond != Cond || IVOfExpressionToSink) {
4555 NewFindLastSelect = LoopBuilder.createSelect(
4556 Cond: SelectCond,
4557 TrueVal: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression,
4558 FalseVal: PhiR, DL);
4559 }
4560 }
4561
4562 // Create the reduction result in the middle block using sentinel directly.
4563 RecurKind MinMaxKind =
4564 UseMax ? (UseSigned ? RecurKind::SMax : RecurKind::UMax)
4565 : (UseSigned ? RecurKind::SMin : RecurKind::UMin);
4566 VPIRFlags Flags(MinMaxKind, /*IsOrdered=*/false, /*IsInLoop=*/false,
4567 FastMathFlags());
4568 DebugLoc ExitDL = RdxResult->getDebugLoc();
4569 VPBuilder MiddleBuilder(RdxResult);
4570 VPValue *ReducedIV =
4571 MiddleBuilder.createNaryOp(Opcode: VPInstruction::ComputeReductionResult,
4572 Operands: NewFindLastSelect, Flags, DL: ExitDL);
4573
4574 // If IVOfExpressionToSink is an expression to sink, sink it now.
4575 VPValue *VectorRegionExitingVal = ReducedIV;
4576 if (IVOfExpressionToSink)
4577 VectorRegionExitingVal =
4578 cloneBinOpForScalarIV(BinOp: cast<VPWidenRecipe>(Val: FindLastExpression),
4579 ScalarIV: ReducedIV, WidenIV: IVOfExpressionToSink);
4580
4581 VPValue *NewRdxResult;
4582 VPValue *StartVPV = PhiR->getStartValue();
4583 if (SentinelVal) {
4584 // Sentinel-based approach: reduce IVs with min/max, compare against
4585 // sentinel to detect if condition was ever true, select accordingly.
4586 VPValue *Sentinel = Plan.getConstantInt(Val: *SentinelVal);
4587 auto *Cmp = MiddleBuilder.createICmp(Pred: CmpInst::ICMP_NE, A: ReducedIV,
4588 B: Sentinel, DL: ExitDL);
4589 NewRdxResult = MiddleBuilder.createSelect(Cond: Cmp, TrueVal: VectorRegionExitingVal,
4590 FalseVal: StartVPV, DL: ExitDL);
4591 StartVPV = Sentinel;
4592 } else {
4593 // Introduce a boolean AnyOf reduction to track if the condition was ever
4594 // true in the loop. Use it to select the initial start value, if it was
4595 // never true.
4596 auto *AnyOfPhi = new VPReductionPHIRecipe(
4597 /*Phi=*/nullptr, RecurKind::Or, *Plan.getFalse(), *Plan.getFalse(),
4598 RdxUnordered{.VFScaleFactor: 1}, {}, /*HasUsesOutsideReductionChain=*/false);
4599 AnyOfPhi->insertAfter(InsertPos: PhiR);
4600
4601 VPBuilder LoopBuilder(BackedgeVal->getDefiningRecipe());
4602 VPValue *OrVal = LoopBuilder.createOr(LHS: AnyOfPhi, RHS: SelectCond);
4603 AnyOfPhi->setOperand(I: 1, New: OrVal);
4604
4605 NewRdxResult = MiddleBuilder.createAnyOfReduction(
4606 ChainOp: OrVal, TrueVal: VectorRegionExitingVal, FalseVal: StartVPV, DL: ExitDL);
4607
4608 // Initialize the IV reduction phi with the neutral element, not the
4609 // original start value, to ensure correct min/max reduction results.
4610 StartVPV = Plan.getOrAddLiveIn(
4611 V: getRecurrenceIdentity(K: MinMaxKind, Tp: IVSCEV->getType(), FMF: {}));
4612 }
4613 RdxResult->replaceAllUsesWith(New: NewRdxResult);
4614 RdxResult->eraseFromParent();
4615
4616 auto *NewPhiR = new VPReductionPHIRecipe(
4617 cast<PHINode>(Val: PhiR->getUnderlyingInstr()), RecurKind::FindIV, *StartVPV,
4618 *NewFindLastSelect, RdxUnordered{.VFScaleFactor: 1}, {},
4619 PhiR->hasUsesOutsideReductionChain());
4620 NewPhiR->insertBefore(InsertPos: PhiR);
4621 PhiR->replaceAllUsesWith(New: NewPhiR);
4622 PhiR->eraseFromParent();
4623 }
4624}
4625
4626namespace {
4627
4628using ExtendKind = TTI::PartialReductionExtendKind;
4629struct ReductionExtend {
4630 Type *SrcType = nullptr;
4631 ExtendKind Kind = ExtendKind::PR_None;
4632};
4633
4634/// Describes the extends used to compute the extended reduction operand.
4635/// ExtendB is optional. If ExtendB is present, ExtendsUser is a binary
4636/// operation.
4637struct ExtendedReductionOperand {
4638 /// The recipe that consumes the extends.
4639 VPWidenRecipe *ExtendsUser = nullptr;
4640 /// Extend descriptions (inputs to getPartialReductionCost).
4641 ReductionExtend ExtendA, ExtendB;
4642};
4643
4644/// A chain of recipes that form a partial reduction. Matches either
4645/// reduction_bin_op (extended op, accumulator), or
4646/// reduction_bin_op (accumulator, extended op).
4647/// The possible forms of the "extended op" are listed in
4648/// matchExtendedReductionOperand.
4649struct VPPartialReductionChain {
4650 /// The top-level binary operation that forms the reduction to a scalar
4651 /// after the loop body.
4652 VPWidenRecipe *ReductionBinOp = nullptr;
4653 /// The user of the extends that is then reduced.
4654 ExtendedReductionOperand ExtendedOp;
4655 /// The recurrence kind for the entire partial reduction chain.
4656 /// This allows distinguishing between Sub and AddWithSub recurrences,
4657 /// when the ReductionBinOp is a Instruction::Sub.
4658 RecurKind RK;
4659 /// The index of the accumulator operand of ReductionBinOp. The extended op
4660 /// is `1 - AccumulatorOpIdx`.
4661 unsigned AccumulatorOpIdx;
4662 unsigned ScaleFactor;
4663 /// Optional blend to represent predication for the block that updates the
4664 /// reduction.
4665 VPBlendRecipe *Blend = nullptr;
4666};
4667
4668// Return the incoming index of the single-use value in the blend, which is
4669// expected to be the predicated reduction update.
4670static std::optional<unsigned>
4671getBlendReductionUpdateValueIdx(VPBlendRecipe *Blend) {
4672 assert(Blend && !Blend->isNormalized() &&
4673 Blend->getNumIncomingValues() == 2 &&
4674 "Expected a non-normalized blend with two incoming values");
4675 bool FirstIncomingHasOneUse = Blend->getIncomingValue(Idx: 0)->hasOneUse();
4676
4677 // Only the update value should have one use (the blend). The previous
4678 // value should always have at least two uses, the blend and the reduction.
4679 if (FirstIncomingHasOneUse == Blend->getIncomingValue(Idx: 1)->hasOneUse())
4680 return std::nullopt;
4681 return FirstIncomingHasOneUse ? 0 : 1;
4682}
4683
4684static VPSingleDefRecipe *
4685optimizeExtendsForPartialReduction(VPSingleDefRecipe *Op) {
4686 // reduce.add(mul(ext(A), C))
4687 // -> reduce.add(mul(ext(A), ext(trunc(C))))
4688 const APInt *Const;
4689 if (match(R: Op, P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_APInt(C&: Const)))) {
4690 auto *ExtA = cast<VPWidenCastRecipe>(Val: Op->getOperand(N: 0));
4691 Instruction::CastOps ExtOpc = ExtA->getOpcode();
4692 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
4693 if (!Op->hasOneUse() ||
4694 !llvm::canConstantBeExtended(
4695 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
4696 return Op;
4697
4698 VPBuilder Builder(Op);
4699 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc,
4700 Op: Op->getOperand(N: 1), ResultTy: NarrowTy);
4701 Type *WideTy = ExtA->getScalarType();
4702 Op->setOperand(I: 1, New: Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy));
4703 return Op;
4704 }
4705
4706 // reduce.add(abs(sub(ext(A), ext(B))))
4707 // -> reduce.add(ext(absolute-difference(A, B)))
4708 VPValue *X, *Y;
4709 if (match(R: Op, P: m_WidenIntrinsic<Intrinsic::abs>(Ops: m_Sub(
4710 Op0: m_ZExtOrSExt(Op0: m_VPValue(V&: X)), Op1: m_ZExtOrSExt(Op0: m_VPValue(V&: Y)))))) {
4711 auto *Sub = Op->getOperand(N: 0)->getDefiningRecipe();
4712 auto *Ext = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
4713 assert(Ext->getOpcode() ==
4714 cast<VPWidenCastRecipe>(Sub->getOperand(1))->getOpcode() &&
4715 "Expected both the LHS and RHS extends to be the same");
4716 bool IsSigned = Ext->getOpcode() == Instruction::SExt;
4717 VPBuilder Builder(Op);
4718 Type *SrcTy = X->getScalarType();
4719 auto *FreezeX = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {X}));
4720 auto *FreezeY = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {Y}));
4721 auto *Max = Builder.insert(
4722 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smax : Intrinsic::umax,
4723 {FreezeX, FreezeY}, SrcTy));
4724 auto *Min = Builder.insert(
4725 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smin : Intrinsic::umin,
4726 {FreezeX, FreezeY}, SrcTy));
4727 auto *AbsDiff = Builder.insert(
4728 R: new VPWidenRecipe(Instruction::Sub, {Max, Min},
4729 VPIRFlags::getDefaultFlags(Opcode: Instruction::Sub)));
4730 return Builder.createWidenCast(Opcode: Instruction::CastOps::ZExt, Op: AbsDiff,
4731 ResultTy: Op->getScalarType());
4732 }
4733
4734 // reduce.add(ext(mul(ext(A), ext(B))))
4735 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
4736 // TODO: Support this optimization for float types.
4737 if (match(R: Op, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
4738 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
4739 auto *Ext = cast<VPWidenCastRecipe>(Val: Op);
4740 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
4741 auto *MulLHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
4742 auto *MulRHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
4743 if (!Mul->hasOneUse() ||
4744 (Ext->getOpcode() != MulLHS->getOpcode() && MulLHS != MulRHS) ||
4745 MulLHS->getOpcode() != MulRHS->getOpcode())
4746 return Op;
4747 VPBuilder Builder(Mul);
4748 auto *NewLHS = Builder.createWidenCast(
4749 Opcode: MulLHS->getOpcode(), Op: MulLHS->getOperand(N: 0), ResultTy: Ext->getScalarType());
4750 auto *NewRHS = MulLHS == MulRHS
4751 ? NewLHS
4752 : Builder.createWidenCast(Opcode: MulRHS->getOpcode(),
4753 Op: MulRHS->getOperand(N: 0),
4754 ResultTy: Ext->getScalarType());
4755 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewLHS, NewRHS});
4756 Builder.insert(R: NewMul);
4757 Op->replaceAllUsesWith(New: NewMul);
4758 Op->eraseFromParent();
4759 Mul->eraseFromParent();
4760 return NewMul;
4761 }
4762
4763 return Op;
4764}
4765
4766static VPExpressionRecipe *
4767createPartialReductionExpression(VPReductionRecipe *Red) {
4768 VPValue *VecOp = Red->getVecOp();
4769
4770 // reduce.[f]add(ext(op))
4771 // -> VPExpressionRecipe(op, red)
4772 if (match(V: VecOp, P: m_WidenAnyExtend(Op0: m_VPValue())))
4773 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
4774
4775 // reduce.[f]add(neg(ext(op)))
4776 // -> VPExpressionRecipe(op, sub/neg, red)
4777 if (match(V: VecOp, P: m_AnyNeg(Op0: m_WidenAnyExtend(Op0: m_VPValue())))) {
4778 auto *Neg = cast<VPWidenRecipe>(Val: VecOp);
4779 auto *Ext =
4780 cast<VPWidenCastRecipe>(Val: Neg->getOperand(N: Neg->getNumOperands() - 1));
4781 return new VPExpressionRecipe(Ext, Neg, Red);
4782 }
4783
4784 // reduce.[f]add([f]mul(ext(a), ext(b)))
4785 // -> VPExpressionRecipe(a, b, mul, red)
4786 if (match(V: VecOp, P: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))) ||
4787 match(V: VecOp,
4788 P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_ZExtOrSExt(Op0: m_VPValue())))) {
4789 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
4790 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
4791 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
4792 return new VPExpressionRecipe(ExtA, ExtB, Mul, Red);
4793 }
4794
4795 // reduce.fadd(fneg(fmul(fpext(a), fpext(b))))
4796 // -> VPExpressionRecipe(a, b, fmul, fsub, red)
4797 if (match(V: VecOp,
4798 P: m_FNeg(Op0: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))))) {
4799 auto *FNeg = cast<VPWidenRecipe>(Val: VecOp);
4800 auto *FMul = cast<VPWidenRecipe>(Val: FNeg->getOperand(N: 0));
4801 auto *ExtA = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 0));
4802 auto *ExtB = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 1));
4803 return new VPExpressionRecipe(ExtA, ExtB, FMul, FNeg, Red);
4804 }
4805
4806 // reduce.add(neg(mul(ext(a), ext(b))))
4807 // -> VPExpressionRecipe(a, b, mul, sub, red)
4808 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
4809 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
4810 auto *Sub = cast<VPWidenRecipe>(Val: VecOp);
4811 auto *Mul = cast<VPWidenRecipe>(Val: Sub->getOperand(N: 1));
4812 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
4813 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
4814 return new VPExpressionRecipe(ExtA, ExtB, Mul, Sub, Red);
4815 }
4816
4817 llvm_unreachable("Unsupported expression");
4818}
4819
4820// Helper to transform a partial reduction chain into a partial reduction
4821// recipe. Assumes profitability has been checked.
4822static void transformToPartialReduction(const VPPartialReductionChain &Chain,
4823 VPlan &Plan,
4824 VPReductionPHIRecipe *RdxPhi) {
4825 VPWidenRecipe *WidenRecipe = Chain.ReductionBinOp;
4826 assert(WidenRecipe->getNumOperands() == 2 && "Expected binary operation");
4827
4828 VPValue *Accumulator = WidenRecipe->getOperand(N: Chain.AccumulatorOpIdx);
4829 auto *ExtendedOp = cast<VPSingleDefRecipe>(
4830 Val: WidenRecipe->getOperand(N: 1 - Chain.AccumulatorOpIdx));
4831
4832 // FIXME: Do these transforms before invoking the cost-model.
4833 ExtendedOp = optimizeExtendsForPartialReduction(Op: ExtendedOp);
4834
4835 // Sub-reductions can be implemented in two ways:
4836 // (1) negate the operand in the vector loop (the default way).
4837 // (2) subtract the reduced value from the init value in the middle block.
4838 // Both ways keep the reduction itself as an 'add' reduction.
4839 //
4840 // The ISD nodes for partial reductions don't support folding the
4841 // sub/negation into its operands because the following is not a valid
4842 // transformation:
4843 // sub(0, mul(ext(a), ext(b)))
4844 // -> mul(ext(a), ext(sub(0, b)))
4845 //
4846 // It's therefore better to choose option (2) such that the partial
4847 // reduction is always positive (starting at '0') and to do a final
4848 // subtract in the middle block.
4849 if ((WidenRecipe->getOpcode() == Instruction::Sub &&
4850 Chain.RK != RecurKind::Sub) ||
4851 (WidenRecipe->getOpcode() == Instruction::FSub &&
4852 Chain.RK != RecurKind::FSub)) {
4853 VPBuilder Builder(WidenRecipe);
4854 Type *ElemTy = ExtendedOp->getScalarType();
4855 VPWidenRecipe *NegRecipe;
4856 if (WidenRecipe->getOpcode() == Instruction::FSub) {
4857 NegRecipe =
4858 new VPWidenRecipe(Instruction::FNeg, {ExtendedOp},
4859 VPIRFlags::getDefaultFlags(Opcode: Instruction::FNeg),
4860 VPIRMetadata(), DebugLoc::getUnknown());
4861 } else {
4862 auto *Zero = Plan.getZero(Ty: ElemTy);
4863 NegRecipe =
4864 new VPWidenRecipe(Instruction::Sub, {Zero, ExtendedOp},
4865 VPIRFlags::getDefaultFlags(Opcode: Instruction::Sub),
4866 VPIRMetadata(), DebugLoc::getUnknown());
4867 }
4868 Builder.insert(R: NegRecipe);
4869 ExtendedOp = NegRecipe;
4870 }
4871
4872 // Check if WidenRecipe is the final result of the reduction. If so, look
4873 // through the Select recipe introduced by tail-folding, otherwise look
4874 // through any Blend recipe introduced by predication for the block.
4875 VPValue *ExitSearch =
4876 Chain.Blend ? cast<VPValue>(Val: Chain.Blend) : cast<VPValue>(Val: WidenRecipe);
4877
4878 VPValue *Cond = nullptr;
4879 VPValue *ExitValue = cast_or_null<VPInstruction>(
4880 Val: findUserOf(V: ExitSearch, P: m_Select(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: ExitSearch),
4881 Op2: m_Specific(VPV: RdxPhi))));
4882
4883 if (Chain.Blend) {
4884 std::optional<unsigned> BlendReductionIdx =
4885 getBlendReductionUpdateValueIdx(Blend: Chain.Blend);
4886 assert(BlendReductionIdx &&
4887 Chain.Blend->getIncomingValue(*BlendReductionIdx) == WidenRecipe &&
4888 "Expected blend to contain the reduction update");
4889 VPValue *BlendCond = Chain.Blend->getMask(Idx: *BlendReductionIdx);
4890 Cond = ExitValue ? VPBuilder(WidenRecipe)
4891 .createLogicalAnd(LHS: Cond, RHS: BlendCond,
4892 DL: WidenRecipe->getDebugLoc())
4893 : BlendCond;
4894 }
4895
4896 // When folding the tail, the inactive lanes of the reduction update are
4897 // computed from values that do not correspond to any scalar iteration
4898 // and must not be accumulated.
4899 if (!Cond)
4900 Cond = Plan.getVectorLoopRegion()->getHeaderMask();
4901
4902 bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
4903 RdxPhi->getBackedgeValue() == ExitValue ||
4904 RdxPhi->getBackedgeValue() == Chain.Blend;
4905 assert((!ExitValue || IsLastInChain) &&
4906 "if we found ExitValue, it must match RdxPhi's backedge value");
4907
4908 Type *PhiType = RdxPhi->getScalarType();
4909 RecurKind RdxKind =
4910 PhiType->isFloatingPointTy() ? RecurKind::FAdd : RecurKind::Add;
4911 auto *PartialRed = new VPReductionRecipe(
4912 RdxKind,
4913 RdxKind == RecurKind::FAdd ? WidenRecipe->getFastMathFlagsOrNone()
4914 : FastMathFlags(),
4915 WidenRecipe->getUnderlyingInstr(), Accumulator, ExtendedOp, Cond,
4916 RdxUnordered{/*VFScaleFactor=*/Chain.ScaleFactor});
4917 PartialRed->insertBefore(InsertPos: WidenRecipe);
4918
4919 if (ExitValue)
4920 ExitValue->replaceAllUsesWith(New: PartialRed);
4921 if (Chain.Blend)
4922 Chain.Blend->replaceAllUsesWith(New: PartialRed);
4923 WidenRecipe->replaceAllUsesWith(New: PartialRed);
4924
4925 // For cost-model purposes, fold this into a VPExpression.
4926 VPExpressionRecipe *E = createPartialReductionExpression(Red: PartialRed);
4927 E->insertBefore(InsertPos: WidenRecipe);
4928 PartialRed->replaceAllUsesWith(New: E);
4929
4930 // We only need to update the PHI node once, which is when we find the
4931 // last reduction in the chain.
4932 if (!IsLastInChain)
4933 return;
4934
4935 // Scale the PHI and ReductionStartVector by the VFScaleFactor
4936 assert(RdxPhi->getVFScaleFactor() == 1 && "scale factor must not be set");
4937 RdxPhi->setVFScaleFactor(Chain.ScaleFactor);
4938
4939 auto *StartInst = cast<VPInstruction>(Val: RdxPhi->getStartValue());
4940 assert(StartInst->getOpcode() == VPInstruction::ReductionStartVector);
4941 auto *NewScaleFactor = Plan.getConstantInt(BitWidth: 32, Val: Chain.ScaleFactor);
4942 StartInst->setOperand(I: 2, New: NewScaleFactor);
4943
4944 // If this is the last value in a sub-reduction chain, then update the PHI
4945 // node to start at `0` and update the reduction-result to subtract from
4946 // the PHI's start value.
4947 if (Chain.RK != RecurKind::Sub && Chain.RK != RecurKind::FSub)
4948 return;
4949
4950 VPValue *OldStartValue = StartInst->getOperand(N: 0);
4951 StartInst->setOperand(I: 0, New: StartInst->getOperand(N: 1));
4952
4953 // Replace reduction_result by 'sub (startval, reductionresult)'.
4954 VPInstruction *RdxResult = vputils::findComputeReductionResult(PhiR: RdxPhi);
4955 assert(RdxResult && "Could not find reduction result");
4956
4957 VPBuilder Builder = VPBuilder::getToInsertAfter(R: RdxResult);
4958 unsigned SubOpc = Chain.RK == RecurKind::FSub ? Instruction::BinaryOps::FSub
4959 : Instruction::BinaryOps::Sub;
4960 VPInstruction *NewResult = Builder.createNaryOp(
4961 Opcode: SubOpc, Operands: {OldStartValue, RdxResult}, Flags: VPIRFlags::getDefaultFlags(Opcode: SubOpc),
4962 DL: RdxPhi->getDebugLoc());
4963 RdxResult->replaceUsesWithIf(
4964 New: NewResult,
4965 ShouldReplace: [&NewResult](VPUser &U, unsigned Idx) { return &U != NewResult; });
4966}
4967
4968/// Returns the cost of a link in a partial-reduction chain for a given VF.
4969static InstructionCost
4970getPartialReductionLinkCost(VPCostContext &CostCtx,
4971 const VPPartialReductionChain &Link,
4972 ElementCount VF) {
4973 Type *RdxType = Link.ReductionBinOp->getScalarType();
4974 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
4975 std::optional<unsigned> BinOpc = std::nullopt;
4976 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
4977 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
4978 BinOpc = ExtendedOp.ExtendsUser->getOpcode();
4979
4980 std::optional<llvm::FastMathFlags> Flags;
4981 if (RdxType->isFloatingPointTy())
4982 Flags = Link.ReductionBinOp->getFastMathFlagsOrNone();
4983
4984 auto GetLinkOpcode = [&Link]() -> unsigned {
4985 switch (Link.RK) {
4986 case RecurKind::Sub:
4987 return Instruction::Add;
4988 case RecurKind::FSub:
4989 return Instruction::FAdd;
4990 default:
4991 return Link.ReductionBinOp->getOpcode();
4992 }
4993 };
4994
4995 return CostCtx.TTI.getPartialReductionCost(
4996 Opcode: GetLinkOpcode(), InputTypeA: ExtendedOp.ExtendA.SrcType, InputTypeB: ExtendedOp.ExtendB.SrcType,
4997 AccumType: RdxType, VF, OpAExtend: ExtendedOp.ExtendA.Kind, OpBExtend: ExtendedOp.ExtendB.Kind, BinOp: BinOpc,
4998 CostKind: CostCtx.CostKind, FMF: Flags);
4999}
5000
5001static ExtendKind getPartialReductionExtendKind(VPWidenCastRecipe *Cast) {
5002 return TTI::getPartialReductionExtendKind(CastOpc: Cast->getOpcode());
5003}
5004
5005/// Checks if \p Op (which is an operand of \p UpdateR) is an extended reduction
5006/// operand. This is an operand where the source of the value (e.g. a load) has
5007/// been extended (sext, zext, or fpext) before it is used in the reduction.
5008///
5009/// Possible forms matched by this function:
5010/// - UpdateR(PrevValue, ext(...))
5011/// - UpdateR(PrevValue, mul(ext(...), ext(...)))
5012/// - UpdateR(PrevValue, mul(ext(...), Constant))
5013/// - UpdateR(PrevValue, ext(mul(ext(...), ext(...))))
5014/// - UpdateR(PrevValue, ext(mul(ext(...), Constant)))
5015/// - UpdateR(PrevValue, abs(sub(ext(...), ext(...)))
5016///
5017/// Note: The second operand of UpdateR corresponds to \p Op in the examples.
5018static std::optional<ExtendedReductionOperand>
5019matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op) {
5020 assert(is_contained(UpdateR->operands(), Op) &&
5021 "Op should be operand of UpdateR");
5022
5023 // Try matching an absolute difference operand of the form
5024 // `abs(sub(ext(A), ext(B)))`. This will be later transformed into
5025 // `ext(absolute-difference(A, B))`. This allows us to perform the absolute
5026 // difference on a wider type and get the extend for "free" from the partial
5027 // reduction.
5028 VPValue *X, *Y;
5029 if (Op->hasOneUse() &&
5030 match(V: Op, P: m_WidenIntrinsic<Intrinsic::abs>(
5031 Ops: m_OneUse(SubPattern: m_Sub(Op0: m_WidenAnyExtend(Op0: m_VPValue(V&: X)),
5032 Op1: m_WidenAnyExtend(Op0: m_VPValue(V&: Y))))))) {
5033 auto *Abs = cast<VPWidenIntrinsicRecipe>(Val: Op);
5034 auto *Sub = cast<VPWidenRecipe>(Val: Abs->getOperand(N: 0));
5035 auto *LHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
5036 auto *RHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 1));
5037 Type *LHSInputType = X->getScalarType();
5038 Type *RHSInputType = Y->getScalarType();
5039 if (LHSInputType != RHSInputType ||
5040 LHSExt->getOpcode() != RHSExt->getOpcode())
5041 return std::nullopt;
5042 // Note: This is essentially the same as matching ext(...) as we will
5043 // rewrite this operand to ext(absolute-difference(A, B)).
5044 return ExtendedReductionOperand{
5045 .ExtendsUser: Sub,
5046 /*ExtendA=*/{.SrcType: LHSInputType, .Kind: getPartialReductionExtendKind(Cast: LHSExt)},
5047 /*ExtendB=*/{}};
5048 }
5049
5050 std::optional<TTI::PartialReductionExtendKind> OuterExtKind;
5051 if (match(V: Op, P: m_WidenAnyExtend(Op0: m_VPValue()))) {
5052 auto *CastRecipe = cast<VPWidenCastRecipe>(Val: Op);
5053 VPValue *CastSource = CastRecipe->getOperand(N: 0);
5054 OuterExtKind = getPartialReductionExtendKind(Cast: CastRecipe);
5055 if (match(V: CastSource, P: m_Mul(Op0: m_VPValue(), Op1: m_VPValue())) ||
5056 match(V: CastSource, P: m_FMul(Op0: m_VPValue(), Op1: m_VPValue()))) {
5057 // Match: ext(mul(...))
5058 // Record the outer extend kind and set `Op` to the mul. We can then match
5059 // this as a binary operation. Note: We can optimize out the outer extend
5060 // by widening the inner extends to match it. See
5061 // optimizeExtendsForPartialReduction.
5062 Op = CastSource;
5063 } else {
5064 return ExtendedReductionOperand{
5065 .ExtendsUser: UpdateR,
5066 /*ExtendA=*/{.SrcType: CastSource->getScalarType(), .Kind: *OuterExtKind},
5067 /*ExtendB=*/{}};
5068 }
5069 }
5070
5071 if (!Op->hasOneUse())
5072 return std::nullopt;
5073
5074 VPWidenRecipe *MulOp = dyn_cast<VPWidenRecipe>(Val: Op);
5075 if (!MulOp ||
5076 !is_contained(Set: {Instruction::Mul, Instruction::FMul}, Element: MulOp->getOpcode()))
5077 return std::nullopt;
5078
5079 // The rest of the matching assumes `Op` is a (possibly extended) mul
5080 // operation.
5081
5082 VPValue *LHS = MulOp->getOperand(N: 0);
5083 VPValue *RHS = MulOp->getOperand(N: 1);
5084
5085 // The LHS of the operation must always be an extend.
5086 if (!match(V: LHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
5087 return std::nullopt;
5088
5089 auto *LHSCast = cast<VPWidenCastRecipe>(Val: LHS);
5090 Type *LHSInputType = LHSCast->getOperand(N: 0)->getScalarType();
5091 ExtendKind LHSExtendKind = getPartialReductionExtendKind(Cast: LHSCast);
5092
5093 // The RHS of the operation can be an extend or a constant integer.
5094 const APInt *RHSConst = nullptr;
5095 VPWidenCastRecipe *RHSCast = nullptr;
5096 if (match(V: RHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
5097 RHSCast = cast<VPWidenCastRecipe>(Val: RHS);
5098 else if (!match(V: RHS, P: m_APInt(C&: RHSConst)) ||
5099 !canConstantBeExtended(C: RHSConst, NarrowType: LHSInputType, ExtKind: LHSExtendKind))
5100 return std::nullopt;
5101
5102 // The outer extend kind must match the inner extends for folding.
5103 for (VPWidenCastRecipe *Cast : {LHSCast, RHSCast})
5104 if (Cast && OuterExtKind &&
5105 getPartialReductionExtendKind(Cast) != OuterExtKind)
5106 return std::nullopt;
5107
5108 Type *RHSInputType = LHSInputType;
5109 ExtendKind RHSExtendKind = LHSExtendKind;
5110 if (RHSCast) {
5111 RHSInputType = RHSCast->getOperand(N: 0)->getScalarType();
5112 RHSExtendKind = getPartialReductionExtendKind(Cast: RHSCast);
5113 }
5114
5115 return ExtendedReductionOperand{
5116 .ExtendsUser: MulOp, .ExtendA: {.SrcType: LHSInputType, .Kind: LHSExtendKind}, .ExtendB: {.SrcType: RHSInputType, .Kind: RHSExtendKind}};
5117}
5118
5119/// Examines each operation in the reduction chain corresponding to \p RedPhiR,
5120/// and determines if the target can use a cheaper operation with a wider
5121/// per-iteration input VF and narrower PHI VF. If successful, returns the chain
5122/// of operations in the reduction.
5123static std::optional<SmallVector<VPPartialReductionChain>>
5124getScaledReductions(VPReductionPHIRecipe *RedPhiR) {
5125 // Get the backedge value from the reduction PHI and find the
5126 // ComputeReductionResult that uses it (directly or through a select for
5127 // predicated reductions).
5128 auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR);
5129 if (!RdxResult)
5130 return std::nullopt;
5131 VPValue *ExitValue = RdxResult->getOperand(N: 0);
5132 match(V: ExitValue, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: ExitValue), Op2: m_VPValue()));
5133
5134 SmallVector<VPPartialReductionChain> Chain;
5135 RecurKind RK = RedPhiR->getRecurrenceKind();
5136 Type *PhiType = RedPhiR->getScalarType();
5137 TypeSize PHISize = PhiType->getPrimitiveSizeInBits();
5138
5139 // Work backwards from the ExitValue examining each reduction operation.
5140 VPValue *CurrentValue = ExitValue;
5141 while (CurrentValue != RedPhiR) {
5142 VPBlendRecipe *Blend = dyn_cast<VPBlendRecipe>(Val: CurrentValue);
5143 std::optional<unsigned> BlendReductionIdx;
5144 if (Blend) {
5145 assert(!Blend->isNormalized() && "Expect Blend not to be normalized.");
5146 if (Blend->getNumIncomingValues() != 2)
5147 return std::nullopt;
5148
5149 BlendReductionIdx = getBlendReductionUpdateValueIdx(Blend);
5150 if (!BlendReductionIdx)
5151 return std::nullopt;
5152
5153 CurrentValue = Blend->getIncomingValue(Idx: *BlendReductionIdx);
5154 }
5155
5156 auto *UpdateR = dyn_cast<VPWidenRecipe>(Val: CurrentValue);
5157 if (!UpdateR || !Instruction::isBinaryOp(Opcode: UpdateR->getOpcode()))
5158 return std::nullopt;
5159
5160 VPValue *Op = UpdateR->getOperand(N: 1);
5161 VPValue *PrevValue = UpdateR->getOperand(N: 0);
5162
5163 // Find the extended operand. The other operand (PrevValue) is the next link
5164 // in the reduction chain.
5165 std::optional<ExtendedReductionOperand> ExtendedOp =
5166 matchExtendedReductionOperand(UpdateR, Op);
5167 if (!ExtendedOp) {
5168 ExtendedOp = matchExtendedReductionOperand(UpdateR, Op: PrevValue);
5169 if (!ExtendedOp)
5170 return std::nullopt;
5171 std::swap(a&: Op, b&: PrevValue);
5172 }
5173
5174 // Look for VPBlend(reduce(PrevValue, Op), PrevValue), where
5175 // reduce is equal to CurrentValue. This can be lowered as
5176 // a conditional reduction by hoisting the select to the inputs.
5177 if (Blend && Blend->getIncomingValue(Idx: 1 - *BlendReductionIdx) != PrevValue)
5178 return std::nullopt;
5179
5180 Type *ExtSrcType = ExtendedOp->ExtendA.SrcType;
5181 TypeSize ExtSrcSize = ExtSrcType->getPrimitiveSizeInBits();
5182 if (!PHISize.hasKnownScalarFactor(RHS: ExtSrcSize))
5183 return std::nullopt;
5184
5185 VPPartialReductionChain Link(
5186 {.ReductionBinOp: UpdateR, .ExtendedOp: *ExtendedOp, .RK: RK,
5187 .AccumulatorOpIdx: PrevValue == UpdateR->getOperand(N: 0) ? 0U : 1U,
5188 .ScaleFactor: static_cast<unsigned>(PHISize.getKnownScalarFactor(RHS: ExtSrcSize)),
5189 .Blend: Blend});
5190 Chain.push_back(Elt: Link);
5191 CurrentValue = PrevValue;
5192 }
5193
5194 // The chain links were collected by traversing backwards from the exit value.
5195 // Reverse the chains so they are in program order.
5196 std::reverse(first: Chain.begin(), last: Chain.end());
5197 return Chain;
5198}
5199} // namespace
5200
5201void VPlanTransforms::createPartialReductions(VPlan &Plan,
5202 VPCostContext &CostCtx,
5203 VFRange &Range) {
5204 // Find all possible valid partial reductions, grouping chains by their PHI.
5205 // This grouping allows invalidating the whole chain, if any link is not a
5206 // valid partial reduction.
5207 MapVector<VPReductionPHIRecipe *, SmallVector<VPPartialReductionChain>>
5208 ChainsByPhi;
5209 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
5210 for (VPRecipeBase &R : HeaderVPBB->phis()) {
5211 auto *RedPhiR = dyn_cast<VPReductionPHIRecipe>(Val: &R);
5212 if (!RedPhiR)
5213 continue;
5214
5215 if (auto Chains = getScaledReductions(RedPhiR))
5216 ChainsByPhi.try_emplace(Key: RedPhiR, Args: std::move(*Chains));
5217 }
5218
5219 if (ChainsByPhi.empty())
5220 return;
5221
5222 // Build set of partial reduction operations and blends for user validation
5223 // and a map of reduction bin ops to their scale factors for scale validation.
5224 SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
5225 SmallPtrSet<VPBlendRecipe *, 4> PartialReductionBlends;
5226 DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
5227 for (const auto &[_, Chains] : ChainsByPhi)
5228 for (const VPPartialReductionChain &Chain : Chains) {
5229 PartialReductionOps.insert(Ptr: Chain.ExtendedOp.ExtendsUser);
5230 if (Chain.Blend)
5231 PartialReductionBlends.insert(Ptr: Chain.Blend);
5232 ScaledReductionMap[Chain.ReductionBinOp] = Chain.ScaleFactor;
5233 }
5234
5235 // A partial reduction is invalid if any of its extends are used by
5236 // something that isn't another partial reduction. This is because the
5237 // extends are intended to be lowered along with the reduction itself.
5238 auto ExtendUsersValid = [&](VPValue *Ext) {
5239 return !isa<VPWidenCastRecipe>(Val: Ext) || all_of(Range: Ext->users(), P: [&](VPUser *U) {
5240 return PartialReductionOps.contains(Ptr: cast<VPRecipeBase>(Val: U));
5241 });
5242 };
5243
5244 auto IsProfitablePartialReductionChainForVF =
5245 [&](ArrayRef<VPPartialReductionChain> Chain, ElementCount VF) -> bool {
5246 InstructionCost PartialCost = 0, RegularCost = 0;
5247
5248 // The chain is a profitable partial reduction chain if the cost of handling
5249 // the entire chain is cheaper when using partial reductions than when
5250 // handling the entire chain using regular reductions.
5251 for (const VPPartialReductionChain &Link : Chain) {
5252 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
5253 InstructionCost LinkCost = getPartialReductionLinkCost(CostCtx, Link, VF);
5254 if (!LinkCost.isValid())
5255 return false;
5256
5257 PartialCost += LinkCost;
5258 RegularCost += Link.ReductionBinOp->computeCost(VF, Ctx&: CostCtx);
5259 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
5260 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
5261 RegularCost += ExtendedOp.ExtendsUser->computeCost(VF, Ctx&: CostCtx);
5262 for (VPValue *Op : ExtendedOp.ExtendsUser->operands())
5263 if (auto *Extend = dyn_cast<VPWidenCastRecipe>(Val: Op))
5264 RegularCost += Extend->computeCost(VF, Ctx&: CostCtx);
5265 }
5266 return PartialCost.isValid() && PartialCost < RegularCost;
5267 };
5268
5269 // Validate chains: check that extends are only used by partial reductions,
5270 // and that reduction bin ops are only used by other partial reductions with
5271 // matching scale factors, are outside the loop region or the select
5272 // introduced by tail-folding. Otherwise we would create users of scaled
5273 // reductions where the types of the other operands don't match.
5274 for (auto &[RedPhiR, Chains] : ChainsByPhi) {
5275 for (const VPPartialReductionChain &Chain : Chains) {
5276 if (!all_of(Range: Chain.ExtendedOp.ExtendsUser->operands(), P: ExtendUsersValid)) {
5277 Chains.clear();
5278 break;
5279 }
5280 auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
5281 if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: U))
5282 return PhiR == RedPhiR;
5283 auto *R = cast<VPSingleDefRecipe>(Val: U);
5284
5285 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: R))
5286 return Blend == Chain.Blend || PartialReductionBlends.contains(Ptr: Blend);
5287
5288 return Chain.ScaleFactor == ScaledReductionMap.lookup_or(Val: R, Default: 0) ||
5289 match(R, P: m_ComputeReductionResult(
5290 Op0: m_Specific(VPV: Chain.ReductionBinOp))) ||
5291 match(R, P: m_Select(Op0: m_VPValue(), Op1: m_Specific(VPV: Chain.ReductionBinOp),
5292 Op2: m_Specific(VPV: RedPhiR)));
5293 };
5294 if (!all_of(Range: Chain.ReductionBinOp->users(), P: UseIsValid)) {
5295 Chains.clear();
5296 break;
5297 }
5298
5299 // Check if the compute-reduction-result is used by a sunk store.
5300 // TODO: Also form partial reductions in those cases.
5301 if (auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR)) {
5302 if (any_of(Range: RdxResult->users(), P: [](VPUser *U) {
5303 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
5304 return RepR && RepR->getOpcode() == Instruction::Store;
5305 })) {
5306 Chains.clear();
5307 break;
5308 }
5309 }
5310 }
5311
5312 // Clear the chain if it is not profitable.
5313 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
5314 Predicate: [&, &Chains = Chains](ElementCount VF) {
5315 return IsProfitablePartialReductionChainForVF(Chains, VF);
5316 },
5317 Range))
5318 Chains.clear();
5319 }
5320
5321 for (auto &[Phi, Chains] : ChainsByPhi)
5322 for (const VPPartialReductionChain &Chain : Chains)
5323 transformToPartialReduction(Chain, Plan, RdxPhi: Phi);
5324}
5325
5326void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
5327 VPRecipeBuilder &RecipeBuilder,
5328 VPCostContext &CostCtx) {
5329 // Collect all loads/stores first. We will start with ones having simpler
5330 // decisions followed by more complex ones that are potentially
5331 // guided/dependent on the simpler ones.
5332 SmallVector<VPInstruction *> MemOps;
5333 for (VPBasicBlock *VPBB :
5334 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
5335 G: Plan.getVectorLoopRegion()->getEntryBasicBlock()))) {
5336 for (VPRecipeBase &R : *VPBB) {
5337 auto *VPI = dyn_cast<VPInstruction>(Val: &R);
5338 if (VPI && VPI->getUnderlyingValue() &&
5339 is_contained(Set: {Instruction::Load, Instruction::Store},
5340 Element: VPI->getOpcode()))
5341 MemOps.push_back(Elt: VPI);
5342 }
5343 }
5344
5345 // Few helpers to process different kinds of memory operations.
5346
5347 // To be used as argument to `VPlanTransforms::runPass` which explicitly
5348 // specified pass name, hence `VPlan &` parameter.
5349 auto ProcessSubset = [&](VPlan &, auto ProcessVPInst) {
5350 SmallVector<VPInstruction *> RemainingMemOps;
5351 for (VPInstruction *VPI : MemOps) {
5352 if (!ProcessVPInst(VPI))
5353 RemainingMemOps.push_back(Elt: VPI);
5354 }
5355
5356 MemOps.clear();
5357 std::swap(LHS&: MemOps, RHS&: RemainingMemOps);
5358 };
5359
5360 auto ReplaceWith = [&](VPInstruction *VPI, VPRecipeBase *New) {
5361 assert(New->getParent() && "New recipe must have been inserted");
5362 if (VPI->getOpcode() == Instruction::Load)
5363 VPI->replaceAllUsesWith(New: New->getVPSingleValue());
5364 VPI->eraseFromParent();
5365
5366 // VPI has been processed.
5367 return true;
5368 };
5369
5370 auto Scalarize = [&](VPInstruction *VPI) {
5371 return ReplaceWith(VPI, VPBuilder(VPI).insert(
5372 R: RecipeBuilder.handleReplication(VPI, Range)));
5373 };
5374
5375 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
5376 VPBuilder FinalRedStoresBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
5377 VPlanTransforms::runPass(
5378 PassName: "lowerMemoryIdioms", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5379 if (RecipeBuilder.replaceWithFinalIfReductionStore(
5380 VPI, FinalRedStoresBuilder))
5381 return true;
5382
5383 // Filter out scalar VPlan for the remaining idioms.
5384 if (LoopVectorizationPlanner::getDecisionAndClampRange(
5385 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
5386 return false;
5387
5388 if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI))
5389 return ReplaceWith(VPI, VPBuilder(VPI).insert(R: Histogram));
5390
5391 return false;
5392 });
5393
5394 // Filter out scalar VPlan for the remaining memory operations.
5395 if (LoopVectorizationPlanner::getDecisionAndClampRange(
5396 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
5397 return;
5398
5399 // If the instruction's allocated size doesn't equal it's type size, it
5400 // requires padding and will be scalarized.
5401 VPlanTransforms::runPass(
5402 PassName: "scalarizeMemOpsWithIrregularTypes", Pass&: ProcessSubset, Plan,
5403 Args: [&](VPInstruction *VPI) {
5404 Instruction *I = VPI->getUnderlyingInstr();
5405 if (hasIrregularType(Ty: getLoadStoreType(I), DL: I->getDataLayout()))
5406 return Scalarize(VPI);
5407
5408 return false;
5409 });
5410
5411 if (!RecipeBuilder.prefersVectorizedAddressing()) {
5412 VPlanTransforms::runPass(
5413 PassName: "makeVPlanMemOpDecision", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5414 Instruction *I = VPI->getUnderlyingInstr();
5415 bool IsLoad = VPI->getOpcode() == Instruction::Load;
5416 if (RecipeBuilder.isPredicatedInst(I) || !IsLoad ||
5417 !vputils::isUsedByLoadStoreAddress(V: VPI))
5418 return false;
5419
5420 // Scalarize loads used as addresses, matching the legacy CM. The load
5421 // is single-scalar if the pointer is loop-invariant, otherwise it is
5422 // replicated per-lane. No mask is needed as the load is not
5423 // predicated.
5424 VPValue *Ptr = VPI->getOperand(N: 0);
5425 const SCEV *PtrSCEV =
5426 vputils::getSCEVExprForVPValue(V: Ptr, PSE&: CostCtx.PSE, L: CostCtx.L);
5427 bool IsSingleScalarLoad =
5428 !isa<SCEVCouldNotCompute>(Val: PtrSCEV) &&
5429 CostCtx.PSE.getSE()->isLoopInvariant(S: PtrSCEV, L: CostCtx.L);
5430
5431 ReplaceWith(VPI,
5432 VPBuilder(VPI).insert(R: new VPReplicateRecipe(
5433 I, Ptr, /*IsSingleScalar=*/IsSingleScalarLoad,
5434 /*Mask=*/nullptr, *VPI, *VPI, VPI->getDebugLoc())));
5435 return true;
5436 });
5437 }
5438
5439 // Widen unit-stride consecutive accesses, matching the legacy CM. Both
5440 // forward (stride +1) and reverse (stride -1) accesses are handled.
5441 VPlanTransforms::runPass(
5442 PassName: "widenConsecutiveMemOps", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5443 Instruction *I = VPI->getUnderlyingInstr();
5444 bool IsLoad = VPI->getOpcode() == Instruction::Load;
5445 VPValue *Ptr = VPI->getOperand(N: !IsLoad);
5446 Type *ScalarTy =
5447 IsLoad ? VPI->getScalarType() : VPI->getOperand(N: 0)->getScalarType();
5448 std::optional<int64_t> Stride =
5449 getConstantStride(Addr: Ptr, AccessTy: ScalarTy, PSE&: CostCtx.PSE, L: CostCtx.L);
5450 if (Stride != 1 && Stride != -1)
5451 return false;
5452 bool Reverse = Stride == -1;
5453
5454 // A predicated access can only be widened (rather than scalarized) if
5455 // the target supports a masked load/store for it.
5456 // TODO: Determine if a load/store needs predication directly in VPlan.
5457 bool IsPredicated = RecipeBuilder.isPredicatedInst(I);
5458 if (IsPredicated && !CostCtx.Config.isLegalMaskedLoadOrStore(
5459 IsLoad, ScalarTy, Alignment: getLoadStoreAlignment(I),
5460 AddressSpace: getLoadStoreAddressSpace(I)))
5461 return false;
5462
5463 VPBuilder Builder(VPI);
5464 VPSingleDefRecipe *VectorPtr = Builder.createConsecutiveVectorPointer(
5465 Ptr, SourceElementTy: ScalarTy, Reverse, DL: VPI->getDebugLoc());
5466
5467 VPValue *Mask = IsPredicated ? VPI->getMask() : nullptr;
5468 // Reverse the mask so it matches the reversed access order.
5469 if (Reverse && Mask)
5470 Mask = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Mask,
5471 DL: VPI->getDebugLoc());
5472
5473 if (IsLoad) {
5474 VPSingleDefRecipe *Load = Builder.createWidenLoad(
5475 Load&: *cast<LoadInst>(Val: I), Addr: VectorPtr, Mask,
5476 /*Consecutive=*/true, Metadata: *VPI, DL: VPI->getDebugLoc());
5477 // Reverse the loaded values back into program order.
5478 if (Reverse)
5479 Load = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Load,
5480 DL: VPI->getDebugLoc());
5481 return ReplaceWith(VPI, Load);
5482 }
5483
5484 VPValue *StoredVal = VPI->getOperand(N: 0);
5485 if (Reverse)
5486 // Reverse the stored values so they are written in descending order.
5487 StoredVal = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: StoredVal,
5488 DL: VPI->getDebugLoc());
5489
5490 auto *StoreR = Builder.createWidenStore(
5491 Store&: *cast<StoreInst>(Val: I), Addr: VectorPtr, StoredVal, Mask,
5492 /*Consecutive=*/true, Metadata: *VPI, DL: VPI->getDebugLoc());
5493 return ReplaceWith(VPI, StoreR);
5494 });
5495
5496 VPlanTransforms::runPass(PassName: "delegateMemOpWideningToLegacyCM", Pass&: ProcessSubset,
5497 Plan, Args: [&](VPInstruction *VPI) {
5498 if (VPRecipeBase *Recipe =
5499 RecipeBuilder.tryToWidenMemory(VPI, Range))
5500 return ReplaceWith(VPI, Recipe);
5501
5502 return Scalarize(VPI);
5503 });
5504}
5505
5506void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
5507 if (LoopVectorizationPlanner::getDecisionAndClampRange(
5508 Predicate: [&](ElementCount VF) { return VF.isScalar(); }, Range))
5509 return;
5510
5511 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
5512 Plan.getEntry());
5513 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
5514 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
5515 auto *VPI = dyn_cast<VPInstruction>(Val: &R);
5516 if (!VPI)
5517 continue;
5518
5519 auto *I = cast_or_null<Instruction>(Val: VPI->getUnderlyingValue());
5520 // Wouldn't be able to create a `VPReplicateRecipe` anyway.
5521 if (!I)
5522 continue;
5523
5524 // If executing other lanes produces side-effects we can't avoid them.
5525 if (VPI->mayHaveSideEffects())
5526 continue;
5527
5528 // We want to drop the mask operand, verify we can safely do that.
5529 if (VPI->isMasked() && !VPI->isSafeToSpeculativelyExecute())
5530 continue;
5531
5532 // Avoid rewriting IV increment as that interferes with
5533 // `removeRedundantCanonicalIVs`.
5534 if (VPI->getOpcode() == Instruction::Add &&
5535 any_of(Range: VPI->operands(), P: IsaPred<VPWidenIntOrFpInductionRecipe>))
5536 continue;
5537
5538 // Other lanes are needed - can't drop them.
5539 if (!vputils::onlyFirstLaneUsed(Def: VPI))
5540 continue;
5541
5542 auto *Recipe = VPBuilder::createSingleScalarOp(
5543 Opcode: VPI->getOpcode(), Operands: VPI->operandsWithoutMask(), /*Mask=*/nullptr, Flags: *VPI,
5544 Metadata: *VPI, DL: VPI->getDebugLoc(), UV: I);
5545 Recipe->insertBefore(InsertPos: VPI);
5546 VPI->replaceAllUsesWith(New: Recipe);
5547 VPI->eraseFromParent();
5548 }
5549 }
5550}
5551
5552/// Returns true if \p Info's parameter kinds are compatible with \p Args.
5553static bool areVFParamsOk(const VFInfo &Info, ArrayRef<VPValue *> Args,
5554 PredicatedScalarEvolution &PSE, const Loop *L) {
5555 ScalarEvolution *SE = PSE.getSE();
5556 return all_of(Range: Info.Shape.Parameters, P: [&](VFParameter Param) {
5557 switch (Param.ParamKind) {
5558 case VFParamKind::Vector:
5559 case VFParamKind::GlobalPredicate:
5560 return true;
5561 case VFParamKind::OMP_Uniform:
5562 return SE->isSCEVable(Ty: Args[Param.ParamPos]->getScalarType()) &&
5563 SE->isLoopInvariant(
5564 S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
5565 L);
5566 case VFParamKind::OMP_Linear:
5567 return match(S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
5568 P: m_scev_AffineAddRec(
5569 Op0: m_SCEV(), Op1: m_scev_SpecificSInt(V: Param.LinearStepOrPos),
5570 L: m_SpecificLoop(L)));
5571 default:
5572 return false;
5573 }
5574 });
5575}
5576
5577/// Find a vector variant of \p CI for \p VF, respecting \p MaskRequired.
5578/// Returns the variant function, or nullptr. Masked variants are assumed to
5579/// take the mask as a trailing parameter.
5580static Function *findVectorVariant(CallInst *CI, ArrayRef<VPValue *> Args,
5581 ElementCount VF, bool MaskRequired,
5582 PredicatedScalarEvolution &PSE,
5583 const Loop *L) {
5584 if (CI->isNoBuiltin())
5585 return nullptr;
5586 auto Mappings = VFDatabase::getMappings(CI: *CI);
5587 const auto *It = find_if(Range&: Mappings, P: [&](const VFInfo &Info) {
5588 return Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()) &&
5589 areVFParamsOk(Info, Args, PSE, L);
5590 });
5591 if (It == Mappings.end())
5592 return nullptr;
5593 return CI->getModule()->getFunction(Name: It->VectorName);
5594}
5595
5596namespace {
5597/// The outcome of choosing how to widen a call at a given VF.
5598struct CallWideningDecision {
5599 enum class KindTy { Scalarize, Intrinsic, VectorVariant };
5600 CallWideningDecision(KindTy Kind, Function *Variant = nullptr)
5601 : Kind(Kind), Variant(Variant) {}
5602 KindTy Kind;
5603
5604 /// Set when Kind == VectorVariant.
5605 Function *Variant;
5606
5607 bool operator==(const CallWideningDecision &Other) const {
5608 return Kind == Other.Kind && Variant == Other.Variant;
5609 }
5610};
5611} // namespace
5612
5613/// Pick the cheapest widening for the call \p VPI at \p VF among scalarization,
5614/// vector intrinsic, and vector library variant.
5615static CallWideningDecision decideCallWidening(VPInstruction &VPI,
5616 ArrayRef<VPValue *> Ops,
5617 ElementCount VF,
5618 VPCostContext &CostCtx) {
5619 auto *CI = cast<CallInst>(Val: VPI.getUnderlyingInstr());
5620
5621 // Scalar VFs and calls forced or known to scalarize always replicate.
5622 if (VF.isScalar() || CostCtx.willBeScalarized(I: CI, VF))
5623 return CallWideningDecision::KindTy::Scalarize;
5624
5625 auto *CalledFn = cast<Function>(
5626 Val: VPI.getOperand(N: VPI.getNumOperandsWithoutMask() - 1)->getLiveInIRValue());
5627 Type *ResultTy = VPI.getScalarType();
5628 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
5629 bool MaskRequired = CostCtx.isMaskRequired(I: CI);
5630
5631 // Pseudo intrinsics (assume, lifetime, ...) are always scalarized.
5632 if (ID && VPCostContext::isFreeScalarIntrinsic(ID))
5633 return CallWideningDecision::KindTy::Scalarize;
5634
5635 InstructionCost ScalarCost =
5636 VPReplicateRecipe::computeCallCost(CalledFn, ResultTy, ArgOps: Ops,
5637 /*IsSingleScalar=*/false, VF, Ctx&: CostCtx);
5638
5639 Function *VecFunc =
5640 findVectorVariant(CI, Args: Ops, VF, MaskRequired, PSE&: CostCtx.PSE, L: CostCtx.L);
5641 InstructionCost VecCallCost = InstructionCost::getInvalid();
5642 if (VecFunc)
5643 VecCallCost = VPWidenCallRecipe::computeCallCost(Variant: VecFunc, Ctx&: CostCtx);
5644
5645 // Prefer the intrinsic if it is at least as cheap as scalarizing and any
5646 // available vector variant.
5647 if (ID) {
5648 InstructionCost IntrinsicCost =
5649 VPWidenIntrinsicRecipe::computeCallCost(ID, Operands: Ops, R: VPI, VF, Ctx&: CostCtx);
5650 if (IntrinsicCost.isValid() && ScalarCost >= IntrinsicCost &&
5651 (!VecFunc || VecCallCost >= IntrinsicCost))
5652 return CallWideningDecision::KindTy::Intrinsic;
5653 }
5654
5655 // Otherwise, use a vector library variant when it beats scalarizing.
5656 if (VecFunc && ScalarCost >= VecCallCost)
5657 return {CallWideningDecision::KindTy::VectorVariant, VecFunc};
5658
5659 return CallWideningDecision::KindTy::Scalarize;
5660}
5661
5662void VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
5663 VPRecipeBuilder &RecipeBuilder,
5664 VPCostContext &CostCtx) {
5665 for (VPBasicBlock *VPBB : VPBlockUtils::blocksAs<VPBasicBlock>(
5666 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
5667 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
5668 auto *VPI = dyn_cast<VPInstruction>(Val: &R);
5669 if (!VPI || !VPI->getUnderlyingValue() ||
5670 VPI->getOpcode() != Instruction::Call)
5671 continue;
5672
5673 auto *CI = cast<CallInst>(Val: VPI->getUnderlyingInstr());
5674 SmallVector<VPValue *, 4> Ops(VPI->op_begin(),
5675 VPI->op_begin() + CI->arg_size());
5676
5677 CallWideningDecision Decision =
5678 decideCallWidening(VPI&: *VPI, Ops, VF: Range.Start, CostCtx);
5679 LoopVectorizationPlanner::getDecisionAndClampRange(
5680 Predicate: [&](ElementCount VF) {
5681 return Decision == decideCallWidening(VPI&: *VPI, Ops, VF, CostCtx);
5682 },
5683 Range);
5684
5685 VPSingleDefRecipe *Replacement = nullptr;
5686 switch (Decision.Kind) {
5687 case CallWideningDecision::KindTy::Intrinsic: {
5688 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
5689 Type *ResultTy = VPI->getScalarType();
5690 Replacement = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, *VPI,
5691 *VPI, VPI->getDebugLoc());
5692 break;
5693 }
5694 case CallWideningDecision::KindTy::VectorVariant: {
5695 // Masked variants take the mask as a trailing parameter, so they have
5696 // one more parameter than the original call's arguments.
5697 if (Decision.Variant->arg_size() > Ops.size()) {
5698 VPValue *Mask = VPI->isMasked() ? VPI->getMask() : Plan.getTrue();
5699 Ops.push_back(Elt: Mask);
5700 }
5701 Ops.push_back(Elt: VPI->getOperand(N: VPI->getNumOperandsWithoutMask() - 1));
5702 Replacement = new VPWidenCallRecipe(CI, Decision.Variant, Ops, *VPI,
5703 *VPI, VPI->getDebugLoc());
5704 break;
5705 }
5706 case CallWideningDecision::KindTy::Scalarize:
5707 Replacement = RecipeBuilder.handleReplication(VPI, Range);
5708 break;
5709 }
5710
5711 Replacement->insertBefore(InsertPos: VPI);
5712 VPI->replaceAllUsesWith(New: Replacement);
5713 VPI->eraseFromParent();
5714 }
5715 }
5716}
5717
5718void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
5719 PredicatedScalarEvolution &PSE,
5720 Loop &L, VPCostContext &Ctx,
5721 VFRange &Range) {
5722 if (Plan.hasScalarVFOnly())
5723 return;
5724
5725 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
5726 VPValue *I32VF = nullptr;
5727 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
5728 Range: vp_depth_first_shallow(G: VectorLoop->getEntry()))) {
5729 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
5730 auto *MemR = dyn_cast<VPWidenMemoryRecipe>(Val: &R);
5731 // TODO: Transform reverse access into strided access with -1 stride.
5732 // TODO: Transform gather/scatter with uniform address into strided access
5733 // with 0 stride.
5734 // TODO: Transform interleave access into multiple strided accesses.
5735 if (!MemR || MemR->isConsecutive())
5736 continue;
5737
5738 VPValue *Ptr = MemR->getAddr();
5739 // Check if this is a strided access by analyzing the address SCEV for an
5740 // affine addRec.
5741 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: &L);
5742 const SCEV *Start;
5743 const SCEVConstant *Step;
5744 // TODO: Support non-constant loop invariant stride.
5745 if (!match(S: PtrSCEV,
5746 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEVConstant(V&: Step),
5747 L: m_SpecificLoop(L: &L))))
5748 continue;
5749
5750 VPValue *StoredValue = nullptr;
5751 Type *DataTy;
5752 Intrinsic::ID IntrinID;
5753 if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Val: &R)) {
5754 StoredValue = StoreR->getStoredValue();
5755 DataTy = StoredValue->getScalarType();
5756 IntrinID = Intrinsic::experimental_vp_strided_store;
5757 } else {
5758 auto *LoadR = cast<VPWidenLoadRecipe>(Val: &R);
5759 DataTy = LoadR->getScalarType();
5760 IntrinID = Intrinsic::experimental_vp_strided_load;
5761 }
5762
5763 Align Alignment = MemR->getAlign();
5764 auto IsProfitable = [&](ElementCount VF) {
5765 Type *VectorTy = toVectorTy(Scalar: DataTy, EC: VF);
5766 if (!Ctx.TTI.isLegalStridedLoadStore(DataType: VectorTy, Alignment))
5767 return false;
5768 const InstructionCost CurrentCost = MemR->computeCost(VF, Ctx);
5769 const InstructionCost StridedLoadStoreCost =
5770 VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost(
5771 IID: IntrinID, Ty: VectorTy, IsMasked: MemR->isMasked(), Alignment, Ctx);
5772 return StridedLoadStoreCost < CurrentCost;
5773 };
5774
5775 if (!LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsProfitable,
5776 Range))
5777 continue;
5778
5779 // Invalidate the legacy widening decision so the cost of replaced load is
5780 // not counted during precomputeCosts.
5781 // TODO: Remove once the legacy exit cost computation is retired.
5782 for (ElementCount VF : Range)
5783 Ctx.invalidateWideningDecision(I: &MemR->getIngredient(), VF);
5784
5785 // Get VF as i32 for the vector length operand.
5786 if (!I32VF) {
5787 VPBuilder Builder(Plan.getVectorPreheader());
5788 I32VF = Builder.createScalarZExtOrTrunc(
5789 Op: &Plan.getVF(), ResultTy: Type::getInt32Ty(C&: Plan.getContext()),
5790 DL: DebugLoc::getUnknown());
5791 }
5792
5793 VPBuilder Builder(&R);
5794 // Create the base pointer of strided access.
5795 // TODO: reuse VPDerivedIVRecipe for base pointer computation when it
5796 // supports a general VPValue as the start value.
5797 VPValue *StartVPV =
5798 VPSCEVExpander(Builder, *PSE.getSE(), R.getDebugLoc()).expand(S: Start);
5799 VPValue *StrideInBytes = Plan.getOrAddLiveIn(V: Step->getValue());
5800 Type *IndexTy = Plan.getDataLayout().getIndexType(PtrTy: Ptr->getScalarType());
5801 assert(IndexTy == StrideInBytes->getScalarType() &&
5802 "Stride type from SCEV must match the index type");
5803 VPValue *CanIV = Builder.createScalarZExtOrTrunc(
5804 Op: VectorLoop->getCanonicalIV(), ResultTy: IndexTy, DL: DebugLoc::getUnknown());
5805 auto *AddRecPtr = cast<SCEVAddRecExpr>(Val: PtrSCEV);
5806 auto *Offset = Builder.createOverflowingOp(
5807 Opcode: Instruction::Mul, Operands: {CanIV, StrideInBytes},
5808 WrapFlags: {AddRecPtr->hasNoUnsignedWrap(), /*HasNSW=*/false});
5809 GEPNoWrapFlags NWFlags = AddRecPtr->hasNoUnsignedWrap()
5810 ? GEPNoWrapFlags::noUnsignedWrap()
5811 : GEPNoWrapFlags::none();
5812 VPValue *BasePtr = Builder.createNoWrapPtrAdd(Ptr: StartVPV, Offset, GEPFlags: NWFlags);
5813
5814 // Create a new vector pointer for strided access.
5815 VPValue *NewPtr = Builder.createVectorPointer(
5816 Ptr: BasePtr, SourceElementTy: Type::getInt8Ty(C&: Plan.getContext()), Stride: StrideInBytes, GEPFlags: NWFlags,
5817 DL: R.getDebugLoc());
5818
5819 VPValue *Mask = MemR->getMask();
5820 if (!Mask)
5821 Mask = Plan.getTrue();
5822 SmallVector<VPValue *, 5> Ops;
5823 if (StoredValue)
5824 Ops.push_back(Elt: StoredValue);
5825 Ops.append(IL: {NewPtr, StrideInBytes, Mask, I32VF});
5826
5827 auto *StridedR = Builder.createWidenMemIntrinsic(
5828 VectorIntrinsicID: IntrinID, CallArguments: Ops,
5829 Ty: StoredValue ? Type::getVoidTy(C&: Plan.getContext()) : DataTy, Alignment,
5830 MD: *MemR, DL: R.getDebugLoc());
5831 if (!StoredValue)
5832 cast<VPWidenLoadRecipe>(Val: &R)->replaceAllUsesWith(New: StridedR);
5833 R.eraseFromParent();
5834 }
5835 }
5836}
5837