1//===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements a set of utility VPlan to VPlan transformations.
11///
12//===----------------------------------------------------------------------===//
13
14#include "VPlanTransforms.h"
15#include "VPRecipeBuilder.h"
16#include "VPlan.h"
17#include "VPlanAnalysis.h"
18#include "VPlanCFG.h"
19#include "VPlanDominatorTree.h"
20#include "VPlanHelpers.h"
21#include "VPlanPatternMatch.h"
22#include "VPlanUtils.h"
23#include "llvm/ADT/APInt.h"
24#include "llvm/ADT/PostOrderIterator.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SetVector.h"
27#include "llvm/ADT/SmallPtrSet.h"
28#include "llvm/ADT/TypeSwitch.h"
29#include "llvm/Analysis/IVDescriptors.h"
30#include "llvm/Analysis/Loads.h"
31#include "llvm/Analysis/LoopInfo.h"
32#include "llvm/Analysis/MemoryLocation.h"
33#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
34#include "llvm/Analysis/ScopedNoAliasAA.h"
35#include "llvm/Analysis/VectorUtils.h"
36#include "llvm/IR/Intrinsics.h"
37#include "llvm/IR/Metadata.h"
38#include "llvm/Support/Casting.h"
39#include "llvm/Support/CommandLine.h"
40#include "llvm/Support/TypeSize.h"
41#include "llvm/Transforms/Utils/LoopUtils.h"
42
43using namespace llvm;
44using namespace VPlanPatternMatch;
45using namespace SCEVPatternMatch;
46
47/// Returns the metadata attached to \p R, or an empty set for a recipe that
48/// does not carry any.
49static VPIRMetadata getMetadataOf(VPRecipeBase *R) {
50 if (auto *MD = dyn_cast<VPIRMetadata>(Val: R))
51 return *MD;
52 return {};
53}
54
55// TODO: Remove this once the partial reduction intrinsics are no worse than
56// normal vector operations.
57static cl::opt<bool> UsePartialReductionsByDefault(
58 "use-partial-reductions-by-default", cl::init(Val: false), cl::Hidden,
59 cl::desc("Use partial reduction intrinsics for "
60 "all supported unordered reductions."));
61
62/// If the pointer operand \p Addr of a memory access is an affine AddRec
63/// w.r.t. \p L with a constant stride, return the stride in units of
64/// \p AccessTy. Otherwise return std::nullopt.
65static std::optional<int64_t> getConstantStride(VPValue *Addr, Type *AccessTy,
66 PredicatedScalarEvolution &PSE,
67 const Loop *L) {
68 assert(!hasIrregularType(AccessTy, L->getHeader()->getDataLayout()) &&
69 "should not try to widen irregular types");
70 const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(V: Addr, PSE, L);
71 auto *AddRec = dyn_cast<SCEVAddRecExpr>(Val: AddrSCEV);
72 if (!AddRec)
73 return {};
74
75 return getStrideFromAddRec(AR: AddRec, Lp: L, AccessTy, /*Ptr=*/nullptr, PSE);
76}
77
78bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
79 VPlan &Plan, const TargetLibraryInfo &TLI, PredicatedScalarEvolution &PSE,
80 Loop *OuterLoop) {
81
82 // Returns true if the access of \p AccessTy at \p Addr can be widened to a
83 // consecutive vector access.
84 auto IsConsecutiveAccess = [&](VPValue *Addr, Type *AccessTy) {
85 return !hasIrregularType(Ty: AccessTy, DL: Plan.getDataLayout()) &&
86 getConstantStride(Addr, AccessTy, PSE, L: OuterLoop) == 1;
87 };
88
89 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
90 Plan.getVectorLoopRegion());
91 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
92 // Skip blocks outside region
93 if (!VPBB->getParent())
94 break;
95 VPRecipeBase *Term = VPBB->getTerminator();
96 auto EndIter = Term ? Term->getIterator() : VPBB->end();
97 // Introduce each ingredient into VPlan.
98 for (VPRecipeBase &Ingredient :
99 make_early_inc_range(Range: make_range(x: VPBB->begin(), y: EndIter))) {
100
101 VPValue *VPV = Ingredient.getVPSingleValue();
102 if (!VPV->getUnderlyingValue())
103 continue;
104
105 Instruction *Inst = cast<Instruction>(Val: VPV->getUnderlyingValue());
106
107 // Atomic accesses and fences have ordering/atomicity semantics that
108 // cannot be preserved by lane-wise widening.
109 if (isa<AtomicRMWInst, AtomicCmpXchgInst, FenceInst>(Val: Inst))
110 return false;
111
112 VPRecipeBase *NewRecipe = nullptr;
113 if (auto *PhiR = dyn_cast<VPPhi>(Val: &Ingredient)) {
114 auto *Phi = cast<PHINode>(Val: PhiR->getUnderlyingValue());
115 NewRecipe = new VPWidenPHIRecipe(PhiR->operands(), PhiR->getDebugLoc(),
116 Phi->getName());
117 } else if (auto *VPI = dyn_cast<VPInstruction>(Val: &Ingredient)) {
118 assert(!isa<PHINode>(Inst) && "phis should be handled above");
119 // Create VPWidenMemoryRecipe for loads and stores.
120 if (LoadInst *Load = dyn_cast<LoadInst>(Val: Inst)) {
121 bool IsConsecutive =
122 IsConsecutiveAccess(VPI->getOperand(N: 0), VPI->getScalarType());
123 NewRecipe = new VPWidenLoadRecipe(*Load, Ingredient.getOperand(N: 0),
124 nullptr /*Mask*/, IsConsecutive,
125 *VPI, Ingredient.getDebugLoc());
126 } else if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst)) {
127 bool IsConsecutive = IsConsecutiveAccess(
128 VPI->getOperand(N: 1), VPI->getOperand(N: 0)->getScalarType());
129 NewRecipe = new VPWidenStoreRecipe(
130 *Store, Ingredient.getOperand(N: 1), Ingredient.getOperand(N: 0),
131 nullptr /*Mask*/, IsConsecutive, *VPI, Ingredient.getDebugLoc());
132 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Inst)) {
133 NewRecipe = new VPWidenGEPRecipe(GEP->getSourceElementType(),
134 Ingredient.operands(), *VPI,
135 Ingredient.getDebugLoc(), GEP);
136 } else if (CallInst *CI = dyn_cast<CallInst>(Val: Inst)) {
137 Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, TLI: &TLI);
138 if (VectorID == Intrinsic::not_intrinsic)
139 return false;
140
141 // The noalias.scope.decl intrinsic declares a noalias scope that
142 // is valid for a single iteration. Emitting it as a single-scalar
143 // replicate would incorrectly extend the scope across multiple
144 // original iterations packed into one vector iteration.
145 // FIXME: If we want to vectorize this loop, then we have to drop
146 // all the associated !alias.scope and !noalias.
147 if (VectorID == Intrinsic::experimental_noalias_scope_decl)
148 return false;
149
150 // These intrinsics are recognized by getVectorIntrinsicIDForCall
151 // but are not widenable. Emit them as replicate instead of widening.
152 if (VectorID == Intrinsic::assume ||
153 VectorID == Intrinsic::lifetime_end ||
154 VectorID == Intrinsic::lifetime_start ||
155 VectorID == Intrinsic::sideeffect ||
156 VectorID == Intrinsic::pseudoprobe) {
157 // If the operand of llvm.assume holds before vectorization, it will
158 // also hold per lane.
159 // llvm.pseudoprobe requires to be duplicated per lane for accurate
160 // sample count.
161 const bool IsSingleScalar = VectorID != Intrinsic::assume &&
162 VectorID != Intrinsic::pseudoprobe;
163 NewRecipe = new VPReplicateRecipe(CI, Ingredient.operands(),
164 /*IsSingleScalar=*/IsSingleScalar,
165 /*Mask=*/nullptr, *VPI, *VPI,
166 Ingredient.getDebugLoc());
167 } else {
168 NewRecipe = new VPWidenIntrinsicRecipe(
169 *CI, VectorID, drop_end(RangeOrContainer: Ingredient.operands()), CI->getType(),
170 VPIRFlags(*CI), *VPI, CI->getDebugLoc());
171 }
172 } else if (auto *CI = dyn_cast<CastInst>(Val: Inst)) {
173 NewRecipe = new VPWidenCastRecipe(
174 CI->getOpcode(), Ingredient.getOperand(N: 0), CI->getType(), CI,
175 VPIRFlags(*CI), VPIRMetadata(*CI));
176 } else {
177 NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI,
178 *VPI, Ingredient.getDebugLoc());
179 }
180 } else {
181 assert(isa<VPWidenIntOrFpInductionRecipe>(&Ingredient) &&
182 "inductions must be created earlier");
183 continue;
184 }
185
186 NewRecipe->insertBefore(InsertPos: &Ingredient);
187 if (NewRecipe->getNumDefinedValues() == 1)
188 VPV->replaceAllUsesWith(New: NewRecipe->getVPSingleValue());
189 else
190 assert(NewRecipe->getNumDefinedValues() == 0 &&
191 "Only recpies with zero or one defined values expected");
192 Ingredient.eraseFromParent();
193 }
194 }
195 return true;
196}
197
198/// Helper for extra no-alias checks via known-safe recipe and SCEV.
199class SinkStoreInfo {
200 SmallPtrSet<VPReplicateRecipe *, 4> ExcludeRecipes;
201 VPReplicateRecipe &GroupLeader;
202 PredicatedScalarEvolution *PSE = nullptr;
203 const Loop *L = nullptr;
204
205 // Return true if \p A and \p B are known to not alias for all VFs in the
206 // plan, checked via the distance between the accesses
207 bool isNoAliasViaDistance(VPReplicateRecipe *A, VPReplicateRecipe *B) const {
208 if (A->getOpcode() != Instruction::Store ||
209 B->getOpcode() != Instruction::Store)
210 return false;
211
212 if (!PSE || !L)
213 return A == B;
214
215 VPValue *AddrA = A->getOperand(N: 1);
216 const SCEV *SCEVA = vputils::getSCEVExprForVPValue(V: AddrA, PSE&: *PSE, L);
217 VPValue *AddrB = B->getOperand(N: 1);
218 const SCEV *SCEVB = vputils::getSCEVExprForVPValue(V: AddrB, PSE&: *PSE, L);
219 if (isa<SCEVCouldNotCompute>(Val: SCEVA) || isa<SCEVCouldNotCompute>(Val: SCEVB))
220 return false;
221
222 const APInt *Distance;
223 ScalarEvolution &SE = *PSE->getSE();
224 if (!match(S: SE.getMinusSCEV(LHS: SCEVA, RHS: SCEVB), P: m_scev_APInt(C&: Distance)))
225 return false;
226
227 const DataLayout &DL = SE.getDataLayout();
228 Type *TyA = A->getOperand(N: 0)->getScalarType();
229 uint64_t SizeA = DL.getTypeStoreSize(Ty: TyA);
230 Type *TyB = B->getOperand(N: 0)->getScalarType();
231 uint64_t SizeB = DL.getTypeStoreSize(Ty: TyB);
232
233 // Use the maximum store size to ensure no overlap from either direction.
234 // Currently only handles fixed sizes, as it is only used for
235 // replicating VPReplicateRecipes.
236 uint64_t MaxStoreSize = std::max(a: SizeA, b: SizeB);
237
238 auto VFs = B->getParent()->getPlan()->vectorFactors();
239 ElementCount MaxVF = *max_element(Range&: VFs, C: ElementCount::isKnownLT);
240 if (MaxVF.isScalable())
241 return false;
242 return Distance->abs().uge(RHS: MaxVF.getFixedValue() * MaxStoreSize);
243 }
244
245public:
246 SinkStoreInfo(ArrayRef<VPReplicateRecipe *> ExcludeRecipes,
247 VPReplicateRecipe &GroupLeader, PredicatedScalarEvolution &PSE,
248 const Loop &L)
249 : ExcludeRecipes(ExcludeRecipes.begin(), ExcludeRecipes.end()),
250 GroupLeader(GroupLeader), PSE(&PSE), L(&L) {}
251
252 SinkStoreInfo(VPReplicateRecipe &GroupLeader) : GroupLeader(GroupLeader) {}
253
254 /// Return true if \p R should be skipped during alias checking, either
255 /// because it's in the exclude set or because no-alias can be proven via
256 /// SCEV.
257 bool shouldSkip(VPRecipeBase &R) const {
258 auto *Store = dyn_cast<VPReplicateRecipe>(Val: &R);
259 return ExcludeRecipes.contains(Ptr: Store) ||
260 (Store && isNoAliasViaDistance(A: Store, B: &GroupLeader));
261 }
262};
263
264/// Check if a memory operation doesn't alias with memory operations using
265/// scoped noalias metadata, in blocks in the single-successor chain between \p
266/// FirstBB and \p LastBB. If \p SinkInfo is std::nullopt, only recipes that may
267/// write to memory are checked (for load hoisting). Otherwise recipes that both
268/// read and write memory are checked, and SCEV is used to prove no-alias
269/// between the group leader and other replicate recipes (for store sinking).
270static bool
271canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
272 VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
273 std::optional<SinkStoreInfo> SinkInfo = {}) {
274 bool CheckReads = SinkInfo.has_value();
275 for (VPBasicBlock *VPBB :
276 VPBlockUtils::blocksInSingleSuccessorChainBetween(FirstBB, LastBB)) {
277 for (VPRecipeBase &R : *VPBB) {
278 if (SinkInfo && SinkInfo->shouldSkip(R))
279 continue;
280
281 // Skip recipes that don't need checking.
282 if (!R.mayWriteToMemory() && !(CheckReads && R.mayReadFromMemory()))
283 continue;
284
285 auto Loc = vputils::getMemoryLocation(R);
286 if (!Loc)
287 // Conservatively assume aliasing for memory operations without
288 // location.
289 return false;
290
291 if (ScopedNoAliasAAResult::alias(LocA: *Loc, LocB: MemLoc) != AliasResult::NoAlias)
292 return false;
293 }
294 }
295 return true;
296}
297
298/// Get the value type of the replicate load or store. \p IsLoad indicates
299/// whether it is a load.
300static Type *getLoadStoreValueType(VPReplicateRecipe *R, bool IsLoad) {
301 return (IsLoad ? R : R->getOperand(N: 0))->getScalarType();
302}
303
304/// Collect either replicated Loads or Stores grouped by their address SCEV and
305/// their load-store type, in a deep-traversal of the vector loop region in \p
306/// Plan.
307template <unsigned Opcode>
308static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
309collectGroupedReplicateMemOps(
310 VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L,
311 function_ref<bool(VPReplicateRecipe *)> FilterFn) {
312 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
313 "Only Load and Store opcodes supported");
314 constexpr bool IsLoad = (Opcode == Instruction::Load);
315 SmallDenseMap<std::pair<const SCEV *, const Type *>,
316 SmallVector<VPReplicateRecipe *, 4>>
317 RecipesByAddressAndType;
318 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
319 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
320 for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(Range&: *VPBB)) {
321 if (RepR.getOpcode() != Opcode || !FilterFn(&RepR))
322 continue;
323
324 // For loads, operand 0 is address; for stores, operand 1 is address.
325 VPValue *Addr = RepR.getOperand(N: IsLoad ? 0 : 1);
326 const Type *LoadStoreTy = getLoadStoreValueType(R: &RepR, IsLoad);
327 const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(V: Addr, PSE, L);
328 if (!isa<SCEVCouldNotCompute>(Val: AddrSCEV))
329 RecipesByAddressAndType[{AddrSCEV, LoadStoreTy}].push_back(Elt: &RepR);
330 }
331 }
332 auto Groups = to_vector(Range: RecipesByAddressAndType.values());
333 VPDominatorTree VPDT(Plan);
334 for (auto &Group : Groups) {
335 // Sort mem ops by dominance order, with earliest (most dominating) first.
336 stable_sort(Group, [&VPDT](VPReplicateRecipe *A, VPReplicateRecipe *B) {
337 return VPDT.properlyDominates(A, B);
338 });
339 }
340 return Groups;
341}
342
343static bool sinkScalarOperands(VPlan &Plan) {
344 auto Iter = vp_depth_first_deep(G: Plan.getEntry());
345 bool ScalarVFOnly = Plan.hasScalarVFOnly();
346 bool Changed = false;
347
348 SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList;
349 auto InsertIfValidSinkCandidate = [ScalarVFOnly, &WorkList](
350 VPBasicBlock *SinkTo, VPValue *Op) {
351 auto *Candidate = dyn_cast<VPSingleDefRecipe>(Val: Op);
352 if (!isa_and_nonnull<VPReplicateRecipe, VPScalarIVStepsRecipe,
353 VPInstruction>(Val: Candidate))
354 return;
355
356 if (Candidate->getParent() == SinkTo ||
357 all_of(Range: Candidate->operands(),
358 P: [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); }) ||
359 vputils::cannotHoistOrSinkRecipe(R: *Candidate, /*Sinking=*/true))
360 return;
361
362 if (!ScalarVFOnly && !vputils::doesGeneratePerAllLanes(R: Candidate))
363 return;
364
365 // Only single-scalar VPInstructions can be sunk.
366 if (auto *VPI = dyn_cast<VPInstruction>(Val: Candidate))
367 if (!vputils::isSingleScalar(VPV: VPI))
368 return;
369
370 WorkList.insert(X: {SinkTo, Candidate});
371 };
372
373 // First, collect the operands of all recipes in replicate blocks as seeds for
374 // sinking.
375 for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Range&: Iter)) {
376 VPBasicBlock *EntryVPBB = VPR->getEntryBasicBlock();
377 if (!VPR->isReplicator() || EntryVPBB->getSuccessors().size() != 2)
378 continue;
379 VPBasicBlock *VPBB = cast<VPBasicBlock>(Val: EntryVPBB->getSuccessors().front());
380 if (VPBB->getSingleSuccessor() != VPR->getExitingBasicBlock())
381 continue;
382 for (auto &Recipe : *VPBB)
383 for (VPValue *Op : Recipe.operands())
384 InsertIfValidSinkCandidate(VPBB, Op);
385 }
386
387 // Try to sink each replicate or scalar IV steps recipe in the worklist.
388 for (unsigned I = 0; I != WorkList.size(); ++I) {
389 VPBasicBlock *SinkTo;
390 VPSingleDefRecipe *SinkCandidate;
391 std::tie(args&: SinkTo, args&: SinkCandidate) = WorkList[I];
392
393 // All recipe users of SinkCandidate must be in the same block SinkTo or all
394 // users outside of SinkTo must only use the first lane of SinkCandidate. In
395 // the latter case, we need to duplicate SinkCandidate.
396 auto UsersOutsideSinkTo =
397 make_filter_range(Range: SinkCandidate->users(), Pred: [SinkTo](VPUser *U) {
398 return cast<VPRecipeBase>(Val: U)->getParent() != SinkTo;
399 });
400 if (any_of(Range&: UsersOutsideSinkTo, P: [SinkCandidate](VPUser *U) {
401 return !U->usesFirstLaneOnly(Op: SinkCandidate);
402 }))
403 continue;
404 bool NeedsDuplicating = !UsersOutsideSinkTo.empty();
405
406 if (NeedsDuplicating) {
407 if (ScalarVFOnly)
408 continue;
409 VPSingleDefRecipe *Clone;
410 if (auto *SinkCandidateRepR =
411 dyn_cast<VPReplicateRecipe>(Val: SinkCandidate)) {
412 // TODO: Handle converting to uniform recipes as separate transform,
413 // then cloning should be sufficient here.
414 Clone = VPBuilder::createSingleScalarOp(
415 Opcode: SinkCandidateRepR->getOpcode(), Operands: SinkCandidate->operands(),
416 /*Mask=*/nullptr, Flags: *SinkCandidateRepR, Metadata: *SinkCandidateRepR,
417 DL: SinkCandidate->getDebugLoc(), UV: SinkCandidate->getUnderlyingInstr());
418 // TODO: add ".cloned" suffix to name of Clone's VPValue.
419 } else {
420 Clone = SinkCandidate->clone();
421 }
422
423 Clone->insertBefore(InsertPos: SinkCandidate);
424 SinkCandidate->replaceUsesWithIf(New: Clone, ShouldReplace: [SinkTo](VPUser &U, unsigned) {
425 return cast<VPRecipeBase>(Val: &U)->getParent() != SinkTo;
426 });
427 }
428 SinkCandidate->moveBefore(BB&: *SinkTo, I: SinkTo->getFirstNonPhi());
429 for (VPValue *Op : SinkCandidate->operands())
430 InsertIfValidSinkCandidate(SinkTo, Op);
431 Changed = true;
432 }
433 return Changed;
434}
435
436/// If \p R is a triangle region, return the 'then' block of the triangle.
437static VPBasicBlock *getPredicatedThenBlock(VPRegionBlock *R) {
438 auto *EntryBB = cast<VPBasicBlock>(Val: R->getEntry());
439 if (EntryBB->getNumSuccessors() != 2)
440 return nullptr;
441
442 auto *Succ0 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[0]);
443 auto *Succ1 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[1]);
444 if (!Succ0 || !Succ1)
445 return nullptr;
446
447 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
448 return nullptr;
449 if (Succ0->getSingleSuccessor() == Succ1)
450 return Succ0;
451 if (Succ1->getSingleSuccessor() == Succ0)
452 return Succ1;
453 return nullptr;
454}
455
456// Merge replicate regions in their successor region, if a replicate region
457// is connected to a successor replicate region with the same predicate by a
458// single, empty VPBasicBlock.
459static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
460 SmallPtrSet<VPRegionBlock *, 4> TransformedRegions;
461
462 // Collect replicate regions followed by an empty block, followed by another
463 // replicate region with matching masks to process front. This is to avoid
464 // iterator invalidation issues while merging regions.
465 SmallVector<VPRegionBlock *, 8> WorkList;
466 for (VPRegionBlock *Region1 : VPBlockUtils::blocksOnly<VPRegionBlock>(
467 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
468 if (!Region1->isReplicator())
469 continue;
470 auto *MiddleBasicBlock =
471 dyn_cast_or_null<VPBasicBlock>(Val: Region1->getSingleSuccessor());
472 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
473 continue;
474
475 auto *Region2 =
476 dyn_cast_or_null<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
477 if (!Region2 || !Region2->isReplicator())
478 continue;
479
480 VPValue *Mask1 = Region1->getEntryBranchOnMask()->getOperand(N: 0);
481 VPValue *Mask2 = Region2->getEntryBranchOnMask()->getOperand(N: 0);
482 if (!Mask1 || Mask1 != Mask2)
483 continue;
484
485 assert(Mask1 && Mask2 && "both region must have conditions");
486 WorkList.push_back(Elt: Region1);
487 }
488
489 // Move recipes from Region1 to its successor region, if both are triangles.
490 for (VPRegionBlock *Region1 : WorkList) {
491 if (TransformedRegions.contains(Ptr: Region1))
492 continue;
493 auto *MiddleBasicBlock = cast<VPBasicBlock>(Val: Region1->getSingleSuccessor());
494 auto *Region2 = cast<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
495
496 VPBasicBlock *Then1 = getPredicatedThenBlock(R: Region1);
497 VPBasicBlock *Then2 = getPredicatedThenBlock(R: Region2);
498 if (!Then1 || !Then2)
499 continue;
500
501 // The merged region is entered whenever either of the original regions was,
502 // so use the higher, i.e. more conservative, of their entry frequencies.
503 // If only one of the two is known, the higher one is unknown, so the
504 // result must be unknown too.
505 VPBranchOnMaskRecipe *Guard2 = Region2->getEntryBranchOnMask();
506 std::optional<VPExecutionFrequency> Freq1 =
507 Region1->getEntryBranchOnMask()->getExecutionFrequency();
508 std::optional<VPExecutionFrequency> Freq2 = Guard2->getExecutionFrequency();
509 if (Freq1 && Freq2) {
510 if (Freq2->Freq < Freq1->Freq) {
511 // Freq1's frequency is taken, but it is only as trustworthy as the
512 // less trustworthy of the two.
513 Freq1.emplace(args: Freq1->Freq, args: Freq1->IsEstimated || Freq2->IsEstimated);
514 Guard2->setExecutionFrequency(Freq: Freq1, Ctx&: Plan.getContext());
515 }
516 } else if (Freq2) {
517 Guard2->clearExecutionFrequency();
518 }
519
520 // Note: No fusion-preventing memory dependencies are expected in either
521 // region. Such dependencies should be rejected during earlier dependence
522 // checks, which guarantee accesses can be re-ordered for vectorization.
523 //
524 // Move recipes to the successor region.
525 for (VPRecipeBase &ToMove : make_early_inc_range(Range: reverse(C&: *Then1)))
526 ToMove.moveBefore(BB&: *Then2, I: Then2->getFirstNonPhi());
527
528 auto *Merge1 = cast<VPBasicBlock>(Val: Then1->getSingleSuccessor());
529 auto *Merge2 = cast<VPBasicBlock>(Val: Then2->getSingleSuccessor());
530
531 // Move VPPredInstPHIRecipes from the merge block to the successor region's
532 // merge block. Update all users inside the successor region to use the
533 // original values.
534 for (VPRecipeBase &Phi1ToMove : make_early_inc_range(Range: reverse(C&: *Merge1))) {
535 VPValue *PredInst1 =
536 cast<VPPredInstPHIRecipe>(Val: &Phi1ToMove)->getOperand(N: 0);
537 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
538 Phi1ToMoveV->replaceUsesWithIf(New: PredInst1, ShouldReplace: [Then2](VPUser &U, unsigned) {
539 return cast<VPRecipeBase>(Val: &U)->getParent() == Then2;
540 });
541
542 // Remove phi recipes that are unused after merging the regions.
543 if (Phi1ToMove.getVPSingleValue()->user_empty()) {
544 Phi1ToMove.eraseFromParent();
545 continue;
546 }
547 Phi1ToMove.moveBefore(BB&: *Merge2, I: Merge2->begin());
548 }
549
550 // Remove the dead recipes in Region1's entry block.
551 for (VPRecipeBase &R :
552 make_early_inc_range(Range: reverse(C&: *Region1->getEntryBasicBlock())))
553 R.eraseFromParent();
554
555 // Finally, remove the first region.
556 for (VPBlockBase *Pred : make_early_inc_range(Range&: Region1->getPredecessors())) {
557 VPBlockUtils::disconnectBlocks(From: Pred, To: Region1);
558 VPBlockUtils::connectBlocks(From: Pred, To: MiddleBasicBlock);
559 }
560 VPBlockUtils::disconnectBlocks(From: Region1, To: MiddleBasicBlock);
561 TransformedRegions.insert(Ptr: Region1);
562 }
563
564 return !TransformedRegions.empty();
565}
566
567static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
568 VPRegionBlock *ParentRegion,
569 VPlan &Plan) {
570 Instruction *Instr = PredRecipe->getUnderlyingInstr();
571 // Build the triangular if-then region.
572 std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str();
573 assert(Instr->getParent() && "Predicated instruction not in any basic block");
574 auto *BlockInMask = PredRecipe->getMask();
575 auto *MaskDef = BlockInMask->getDefiningRecipe();
576 auto *BOMRecipe = new VPBranchOnMaskRecipe(
577 BlockInMask, MaskDef ? MaskDef->getDebugLoc() : DebugLoc::getUnknown());
578 auto *Entry =
579 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".entry", Recipe: BOMRecipe);
580
581 // Replace predicated replicate recipe with a replicate recipe without a
582 // mask but in the replicate region.
583 auto *RecipeWithoutMask = new VPReplicateRecipe(
584 PredRecipe->getUnderlyingInstr(), PredRecipe->operandsWithoutMask(),
585 PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
586 PredRecipe->getDebugLoc());
587 // The predicated recipe executes exactly when the guarding branch-on-mask is
588 // taken, so move its execution frequency there.
589 BOMRecipe->setExecutionFrequency(Freq: RecipeWithoutMask->getExecutionFrequency(),
590 Ctx&: Plan.getContext());
591 RecipeWithoutMask->clearExecutionFrequency();
592 auto *Pred =
593 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".if", Recipe: RecipeWithoutMask);
594 auto *Exiting = Plan.createVPBasicBlock(Name: Twine(RegionName) + ".continue");
595 VPRegionBlock *Region =
596 Plan.createReplicateRegion(Entry, Exiting, Name: RegionName);
597
598 // Note: first set Entry as region entry and then connect successors starting
599 // from it in order, to propagate the "parent" of each VPBasicBlock.
600 Region->setParent(ParentRegion);
601 VPBlockUtils::insertTwoBlocksAfter(IfTrue: Pred, IfFalse: Exiting, BlockPtr: Entry);
602 VPBlockUtils::connectBlocks(From: Pred, To: Exiting);
603
604 if (!PredRecipe->user_empty()) {
605 auto *PHIRecipe = new VPPredInstPHIRecipe(RecipeWithoutMask,
606 RecipeWithoutMask->getDebugLoc());
607 Exiting->appendRecipe(Recipe: PHIRecipe);
608 PredRecipe->replaceAllUsesWith(New: PHIRecipe);
609 }
610 PredRecipe->eraseFromParent();
611 return Region;
612}
613
614static void addReplicateRegions(VPlan &Plan) {
615 SmallVector<VPReplicateRecipe *> WorkList;
616 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
617 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
618 for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(Range&: *VPBB))
619 if (RepR.isPredicated())
620 WorkList.push_back(Elt: &RepR);
621 }
622
623 unsigned BBNum = 0;
624 for (VPReplicateRecipe *RepR : WorkList) {
625 VPBasicBlock *CurrentBlock = RepR->getParent();
626 VPBasicBlock *SplitBlock = CurrentBlock->splitAt(SplitAt: RepR->getIterator());
627
628 BasicBlock *OrigBB = RepR->getUnderlyingInstr()->getParent();
629 SplitBlock->setName(
630 OrigBB->hasName() ? OrigBB->getName() + "." + Twine(BBNum++) : "");
631 // Record predicated instructions for above packing optimizations.
632 VPRegionBlock *Region =
633 createReplicateRegion(PredRecipe: RepR, ParentRegion: CurrentBlock->getParent(), Plan);
634 VPBlockUtils::insertOnEdge(From: CurrentBlock, To: SplitBlock, BlockPtr: Region);
635
636 VPRegionBlock *ParentRegion = Region->getParent();
637 if (ParentRegion && ParentRegion->getExiting() == CurrentBlock)
638 ParentRegion->setExiting(SplitBlock);
639 }
640}
641
642bool VPlanTransforms::mergeBlocksIntoPredecessors(VPlan &Plan) {
643 SmallVector<VPBasicBlock *> WorkList;
644 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
645 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
646 // Don't fold the blocks in the skeleton of the Plan into their single
647 // predecessors for now.
648 // TODO: Remove restriction once more of the skeleton is modeled in VPlan.
649 if (!VPBB->getParent())
650 continue;
651 auto *PredVPBB =
652 dyn_cast_or_null<VPBasicBlock>(Val: VPBB->getSinglePredecessor());
653 if (!PredVPBB || PredVPBB->getNumSuccessors() != 1 ||
654 isa<VPIRBasicBlock>(Val: PredVPBB))
655 continue;
656 WorkList.push_back(Elt: VPBB);
657 }
658
659 for (VPBasicBlock *VPBB : WorkList) {
660 VPBasicBlock *PredVPBB = cast<VPBasicBlock>(Val: VPBB->getSinglePredecessor());
661 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
662 R.moveBefore(BB&: *PredVPBB, I: PredVPBB->end());
663 VPBlockUtils::disconnectBlocks(From: PredVPBB, To: VPBB);
664 auto *ParentRegion = VPBB->getParent();
665 if (ParentRegion && ParentRegion->getExiting() == VPBB)
666 ParentRegion->setExiting(PredVPBB);
667 VPBlockUtils::transferSuccessors(Old: VPBB, New: PredVPBB);
668 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
669 }
670 return !WorkList.empty();
671}
672
673void VPlanTransforms::createAndOptimizeReplicateRegions(VPlan &Plan) {
674 // Convert masked VPReplicateRecipes to if-then region blocks.
675 addReplicateRegions(Plan);
676
677 bool ShouldSimplify = true;
678 while (ShouldSimplify) {
679 ShouldSimplify = sinkScalarOperands(Plan);
680 ShouldSimplify |= mergeReplicateRegionsIntoSuccessors(Plan);
681 ShouldSimplify |= mergeBlocksIntoPredecessors(Plan);
682 }
683}
684
685/// Remove redundant casts of inductions.
686///
687/// Such redundant casts are casts of induction variables that can be ignored,
688/// because we already proved that the casted phi is equal to the uncasted phi
689/// in the vectorized loop. There is no need to vectorize the cast - the same
690/// value can be used for both the phi and casts in the vector loop.
691static void removeRedundantInductionCasts(VPlan &Plan) {
692 for (VPWidenIntOrFpInductionRecipe &IV :
693 make_isa_range<VPWidenIntOrFpInductionRecipe>(
694 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
695 if (IV.getTruncInst())
696 continue;
697
698 // A sequence of IR Casts has potentially been recorded for IV, which
699 // *must be bypassed* when the IV is vectorized, because the vectorized IV
700 // will produce the desired casted value. This sequence forms a def-use
701 // chain and is provided in reverse order, ending with the cast that uses
702 // the IV phi. Search for the recipe of the last cast in the chain and
703 // replace it with the original IV. Note that only the final cast is
704 // expected to have users outside the cast-chain and the dead casts left
705 // over will be cleaned up later.
706 ArrayRef<Instruction *> Casts = IV.getInductionDescriptor().getCastInsts();
707 VPValue *FindMyCast = &IV;
708 for (Instruction *IRCast : reverse(C&: Casts)) {
709 VPSingleDefRecipe *FoundUserCast = nullptr;
710 for (auto *U : FindMyCast->users()) {
711 auto *UserCast = dyn_cast<VPSingleDefRecipe>(Val: U);
712 if (UserCast && UserCast->getUnderlyingValue() == IRCast) {
713 FoundUserCast = UserCast;
714 break;
715 }
716 }
717 // A cast recipe in the chain may have been removed by earlier DCE.
718 if (!FoundUserCast)
719 break;
720 FindMyCast = FoundUserCast;
721 }
722 if (FindMyCast != &IV)
723 FindMyCast->replaceAllUsesWith(New: &IV);
724 }
725}
726
727/// If R is a phi-like recipe starting a dead cycle of recipes, erase all
728/// reachable recipes of the dead cycle and return true. Otherwise leave the
729/// plan unchanged and return false.
730static bool tryToRemoveDeadCycle(VPRecipeBase *R) {
731 auto *PhiR = dyn_cast<VPSingleDefRecipe>(Val: R);
732 if (!PhiR || !isa<VPPhi, VPReductionPHIRecipe>(Val: R))
733 return false;
734
735 // The transitive users of PhiR are closed under users, so the cycle is dead
736 // if every one of them can be erased.
737 for (VPUser *U : vputils::collectUsersRecursively(V: PhiR)) {
738 auto *R = cast<VPRecipeBase>(Val: U);
739 // Bail out if a user must be retained, or if it is a phi-like recipe other
740 // than PhiR;
741 if (R->mayHaveSideEffects() || (R != PhiR && isa<VPPhiAccessors>(Val: R)))
742 return false;
743 }
744
745 // Break the cycle by replacing PhiR with its first incoming value, which is
746 // defined outside the cycle. That leaves the rest of the cycle dead.
747 PhiR->replaceAllUsesWith(New: PhiR->getOperand(N: 0));
748 SmallVector<VPValue *> Incoming(PhiR->operands());
749 PhiR->eraseFromParent();
750 for (VPValue *Op : Incoming)
751 vputils::recursivelyDeleteDeadRecipes(V: Op);
752 return true;
753}
754
755void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
756 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
757 Plan.getEntry());
758 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
759 // The recipes in the block are processed in reverse order, to catch chains
760 // of dead recipes.
761 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB)))
762 if (vputils::isDeadRecipe(R))
763 R.eraseFromParent();
764
765 // Erase dead cycles starting at one of VPBB's phi-like recipes. Erasing a
766 // cycle may also erase other phi-like recipes of VPBB, so restart the scan
767 // of the phi section after each removal. This terminates, as each removal
768 // erases the cycle's phi.
769 bool Changed = true;
770 while (Changed) {
771 Changed = false;
772 for (VPRecipeBase &R : VPBB->phis()) {
773 if (tryToRemoveDeadCycle(R: &R)) {
774 Changed = true;
775 break;
776 }
777 }
778 }
779 }
780}
781
782/// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd
783/// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as
784/// VPWidenPointerInductionRecipe will generate vectors only. If some users
785/// require vectors while other require scalars, the scalar uses need to extract
786/// the scalars from the generated vectors (Note that this is different to how
787/// int/fp inductions are handled). Legalize extract-from-ends using uniform
788/// VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so
789/// the correct end value is available. Also optimize
790/// VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by
791/// providing them scalar steps built on the canonical scalar IV and update the
792/// original IV's users. This is an optional optimization to reduce the needs of
793/// vector extracts.
794static void legalizeAndOptimizeInductions(VPlan &Plan) {
795 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
796 bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly();
797
798 SmallVector<VPWidenInductionRecipe *> WideIVs;
799 for (VPWidenInductionRecipe &PhiR :
800 make_isa_range<VPWidenInductionRecipe>(Range: HeaderVPBB->phis()))
801 WideIVs.push_back(Elt: &PhiR);
802
803 // Try to narrow wide and replicating recipes to uniform recipes, based on
804 // VPlan analysis.
805 // TODO: Apply to all recipes in the future, to replace legacy uniformity
806 // analysis.
807 for (VPWidenInductionRecipe *PhiR : WideIVs) {
808 auto Users = vputils::collectUsersRecursively(V: PhiR);
809 for (VPUser *U : reverse(C&: Users)) {
810 auto *Def = dyn_cast<VPRecipeWithIRFlags>(Val: U);
811 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
812 // Skip recipes that shouldn't be narrowed.
813 if (!Def ||
814 !isa<VPReplicateRecipe, VPWidenRecipe, VPWidenGEPRecipe>(Val: Def) ||
815 Def->user_empty() || !Def->getUnderlyingValue() ||
816 (RepR && (RepR->isSingleScalar() || RepR->isPredicated())))
817 continue;
818
819 // Skip recipes that may have other lanes than their first used.
820 if (!vputils::isSingleScalar(VPV: Def) && !vputils::onlyFirstLaneUsed(Def))
821 continue;
822
823 // TODO: Support scalarizing ExtractValue.
824 if (match(V: Def,
825 P: m_Binary<Instruction::ExtractValue>(Op0: m_VPValue(), Op1: m_VPValue())))
826 continue;
827
828 auto *Clone = VPBuilder::createSingleScalarOp(
829 Opcode: Def->getUnderlyingInstr()->getOpcode(), Operands: Def->operands(),
830 /*Mask=*/nullptr, Flags: *Def, Metadata: getMetadataOf(R: Def), DL: DebugLoc::getUnknown(),
831 UV: Def->getUnderlyingInstr());
832 Clone->insertAfter(InsertPos: Def);
833 Def->replaceAllUsesWith(New: Clone);
834 Def->eraseFromParent();
835 }
836 }
837
838 VPBuilder Builder(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
839 for (VPWidenInductionRecipe *PhiR : WideIVs) {
840 // Replace wide pointer inductions which have only their scalars used by
841 // PtrAdd(IndStart, ScalarIVSteps (0, Step)).
842 if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(Val: PhiR)) {
843 if (!Plan.hasScalarVFOnly() &&
844 !PtrIV->onlyScalarsGenerated(IsScalable: Plan.hasScalableVF()))
845 continue;
846
847 VPValue *PtrAdd =
848 vputils::scalarizeVPWidenPointerInduction(PtrIV, Plan, Builder);
849 PtrIV->replaceAllUsesWith(New: PtrAdd);
850 continue;
851 }
852
853 // Replace widened induction with scalar steps for users that only use
854 // scalars.
855 auto *WideIV = cast<VPWidenIntOrFpInductionRecipe>(Val: PhiR);
856 if (HasOnlyVectorVFs && none_of(Range: WideIV->users(), P: [WideIV](VPUser *U) {
857 return U->usesScalars(Op: WideIV);
858 }))
859 continue;
860
861 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
862 VPIRFlags::WrapFlagsTy WrapFlags;
863 // We can preserve nuw when the step is non-negative.
864 const APInt *Step;
865 if (match(V: WideIV->getStepValue(), P: m_APInt(C&: Step)) && Step->isNonNegative())
866 WrapFlags = {static_cast<bool>(WideIV->getNoWrapFlagsOrNone().HasNUW),
867 false};
868 VPScalarIVStepsRecipe *Steps = vputils::createScalarIVSteps(
869 Plan, Kind: ID.getKind(), InductionOpcode: ID.getInductionOpcode(),
870 FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
871 TruncI: WideIV->getTruncInst(), StartV: WideIV->getStartValue(), Step: WideIV->getStepValue(),
872 DL: WideIV->getDebugLoc(), Builder, Flags: WrapFlags);
873
874 // Update scalar users of IV to use Step instead.
875 if (!HasOnlyVectorVFs) {
876 assert(!Plan.hasScalableVF() &&
877 "plans containing a scalar VF cannot also include scalable VFs");
878 WideIV->replaceAllUsesWith(New: Steps);
879 } else {
880 bool HasScalableVF = Plan.hasScalableVF();
881 WideIV->replaceUsesWithIf(New: Steps,
882 ShouldReplace: [WideIV, HasScalableVF](VPUser &U, unsigned) {
883 if (HasScalableVF)
884 return U.usesFirstLaneOnly(Op: WideIV);
885 return U.usesScalars(Op: WideIV);
886 });
887 }
888 }
889}
890
891/// Check if \p VPV is an untruncated wide induction, either before or after the
892/// increment. If so return the header IV (before the increment), otherwise
893/// return null.
894static VPWidenInductionRecipe *
895getOptimizableIVOf(VPValue *VPV, PredicatedScalarEvolution &PSE) {
896 auto *WideIV = dyn_cast<VPWidenInductionRecipe>(Val: VPV);
897 if (WideIV) {
898 // VPV itself is a wide induction, separately compute the end value for exit
899 // users if it is not a truncated IV.
900 auto *IntOrFpIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
901 return (IntOrFpIV && IntOrFpIV->getTruncInst()) ? nullptr : WideIV;
902 }
903
904 // Check if VPV is an optimizable induction increment.
905 VPRecipeBase *Def = VPV->getDefiningRecipe();
906 if (!Def || Def->getNumOperands() != 2)
907 return nullptr;
908 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 0));
909 if (!WideIV)
910 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 1));
911 if (!WideIV)
912 return nullptr;
913
914 auto IsWideIVInc = [&]() {
915 auto &ID = WideIV->getInductionDescriptor();
916
917 // Check if VPV increments the induction by the induction step.
918 VPValue *IVStep = WideIV->getStepValue();
919 switch (ID.getInductionOpcode()) {
920 case Instruction::Add:
921 return match(V: VPV, P: m_c_Add(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
922 case Instruction::FAdd:
923 return match(V: VPV, P: m_c_FAdd(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
924 case Instruction::FSub:
925 return match(V: VPV, P: m_Binary<Instruction::FSub>(Op0: m_Specific(VPV: WideIV),
926 Op1: m_Specific(VPV: IVStep)));
927 case Instruction::Sub: {
928 // IVStep will be the negated step of the subtraction. Check if Step == -1
929 // * IVStep.
930 VPValue *Step;
931 if (!match(V: VPV, P: m_Sub(Op0: m_VPValue(), Op1: m_VPValue(V&: Step))))
932 return false;
933 const SCEV *IVStepSCEV = vputils::getSCEVExprForVPValue(V: IVStep, PSE);
934 const SCEV *StepSCEV = vputils::getSCEVExprForVPValue(V: Step, PSE);
935 ScalarEvolution &SE = *PSE.getSE();
936 return !isa<SCEVCouldNotCompute>(Val: IVStepSCEV) &&
937 !isa<SCEVCouldNotCompute>(Val: StepSCEV) &&
938 IVStepSCEV == SE.getNegativeSCEV(V: StepSCEV);
939 }
940 default:
941 return ID.getKind() == InductionDescriptor::IK_PtrInduction &&
942 match(V: VPV, P: m_GetElementPtr(Op0: m_Specific(VPV: WideIV),
943 Op1: m_Specific(VPV: WideIV->getStepValue())));
944 }
945 llvm_unreachable("should have been covered by switch above");
946 };
947 return IsWideIVInc() ? WideIV : nullptr;
948}
949
950/// Attempts to optimize the induction variable exit values for users in the
951/// early exit block.
952static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan, VPValue *Op,
953 PredicatedScalarEvolution &PSE) {
954 VPValue *Incoming, *Mask;
955 if (!match(V: Op, P: m_ExtractLane(Op0: m_FirstActiveLane(Op0: m_VPValue(V&: Mask)),
956 Op1: m_VPValue(V&: Incoming))))
957 return nullptr;
958
959 auto *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
960 if (!WideIV)
961 return nullptr;
962
963 // Calculate the final index.
964 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
965 auto *CanonicalIV = LoopRegion->getCanonicalIV();
966 Type *CanonicalIVType = LoopRegion->getCanonicalIVType();
967 auto *ExtractR = cast<VPInstruction>(Val: Op);
968 VPBuilder B(ExtractR);
969
970 DebugLoc DL = ExtractR->getDebugLoc();
971 VPValue *FirstActiveLane = B.createFirstActiveLane(Masks: Mask, DL);
972 FirstActiveLane =
973 B.createScalarZExtOrTrunc(Op: FirstActiveLane, ResultTy: CanonicalIVType, DL);
974 VPValue *EndValue = B.createAdd(LHS: CanonicalIV, RHS: FirstActiveLane, DL);
975
976 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
977 // changed it means the exit is using the incremented value, so we need to
978 // add the step.
979 if (Incoming != WideIV) {
980 VPValue *One = Plan.getConstantInt(Ty: CanonicalIVType, Val: 1);
981 EndValue = B.createAdd(LHS: EndValue, RHS: One, DL);
982 }
983
984 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
985 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
986 VPValue *Start = WideIV->getStartValue();
987 VPValue *Step = WideIV->getStepValue();
988 EndValue = B.createDerivedIV(
989 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
990 Start, Current: EndValue, Step);
991 }
992
993 return EndValue;
994}
995
996/// Compute the end value for \p WideIV, unless it is truncated. Creates a
997/// VPDerivedIVRecipe for non-canonical inductions.
998static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV,
999 VPBuilder &VectorPHBuilder,
1000 VPValue *VectorTC) {
1001 auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
1002 // Truncated wide inductions resume from the last lane of their vector value
1003 // in the last vector iteration which is handled elsewhere.
1004 if (WideIntOrFp && WideIntOrFp->getTruncInst())
1005 return nullptr;
1006
1007 VPValue *Start = WideIV->getStartValue();
1008 VPValue *Step = WideIV->getStepValue();
1009 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
1010 VPValue *EndValue = VectorTC;
1011 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
1012 EndValue = VectorPHBuilder.createDerivedIV(
1013 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
1014 Start, Current: VectorTC, Step);
1015 }
1016
1017 // EndValue is derived from the vector trip count (which has the same type as
1018 // the widest induction) and thus may be wider than the induction here.
1019 Type *ScalarTypeOfWideIV = WideIV->getScalarType();
1020 if (ScalarTypeOfWideIV != EndValue->getScalarType()) {
1021 EndValue = VectorPHBuilder.createScalarCast(Opcode: Instruction::Trunc, Op: EndValue,
1022 ResultTy: ScalarTypeOfWideIV,
1023 DL: WideIV->getDebugLoc());
1024 }
1025
1026 return EndValue;
1027}
1028
1029/// Attempts to optimize the induction variable exit values for users in the
1030/// exit block coming from the latch in the original scalar loop.
1031static VPValue *
1032optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
1033 DenseMap<VPValue *, VPValue *> &EndValues,
1034 PredicatedScalarEvolution &PSE) {
1035 VPValue *Incoming;
1036 if (!match(V: Op, P: m_CombineOr(Ps: m_ExtractLastLaneOfLastPart(Op0: m_VPValue(V&: Incoming)),
1037 Ps: m_ExtractLane(Op0: m_LastActiveLane(Op0: m_HeaderMask()),
1038 Op1: m_VPValue(V&: Incoming)))))
1039 return nullptr;
1040
1041 VPWidenInductionRecipe *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
1042 if (!WideIV)
1043 return nullptr;
1044
1045 VPValue *EndValue = EndValues.lookup(Val: WideIV);
1046 assert(EndValue && "Must have computed the end value up front");
1047
1048 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
1049 // changed it means the exit is using the incremented value, so we don't
1050 // need to subtract the step.
1051 if (Incoming != WideIV)
1052 return EndValue;
1053
1054 // Otherwise, subtract the step from the EndValue.
1055 auto *ExtractR = cast<VPInstruction>(Val: Op);
1056 VPBuilder B(ExtractR);
1057 VPValue *Step = WideIV->getStepValue();
1058 Type *ScalarTy = WideIV->getScalarType();
1059 if (ScalarTy->isIntegerTy())
1060 return B.createSub(LHS: EndValue, RHS: Step, DL: DebugLoc::getUnknown(), Name: "ind.escape");
1061 if (ScalarTy->isPointerTy()) {
1062 Type *StepTy = Step->getScalarType();
1063 auto *Zero = Plan.getZero(Ty: StepTy);
1064 return B.createPtrAdd(Ptr: EndValue, Offset: B.createSub(LHS: Zero, RHS: Step),
1065 DL: DebugLoc::getUnknown(), Name: "ind.escape");
1066 }
1067 if (ScalarTy->isFloatingPointTy()) {
1068 const auto &ID = WideIV->getInductionDescriptor();
1069 return B.createNaryOp(
1070 Opcode: ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
1071 ? Instruction::FSub
1072 : Instruction::FAdd,
1073 Operands: {EndValue, Step}, Flags: {ID.getInductionBinOp()->getFastMathFlags()});
1074 }
1075 llvm_unreachable("all possible induction types must be handled");
1076 return nullptr;
1077}
1078
1079static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
1080 PredicatedScalarEvolution &PSE,
1081 VPValue *ResumeTC,
1082 const Loop *L) {
1083 VPValue *Incoming;
1084 if (!match(V: Op, P: m_CombineOr(Ps: m_ExtractLastLaneOfLastPart(Op0: m_VPValue(V&: Incoming)),
1085 Ps: m_ExtractLane(Op0: m_LastActiveLane(Op0: m_HeaderMask()),
1086 Op1: m_VPValue(V&: Incoming)))))
1087 return nullptr;
1088
1089 const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(V: Incoming, PSE, L);
1090 const SCEV *Start, *Step;
1091 if (!match(S: IncomingSCEV, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step),
1092 L: m_SpecificLoop(L))))
1093 return nullptr;
1094
1095 auto *ExtractR = cast<VPInstruction>(Val: Op);
1096 DebugLoc DL = ExtractR->getDebugLoc();
1097 VPBuilder Builder(ExtractR);
1098 VPSCEVExpander Expander(Builder, *PSE.getSE(), DL);
1099 VPValue *StartVPV = Expander.expand(S: Start);
1100 VPValue *StepVPV = Expander.expand(S: Step);
1101
1102 Type *StartTy = StartVPV->getScalarType();
1103 assert(StartTy->isIntOrPtrTy() && "The type must be SCEVable");
1104 InductionDescriptor::InductionKind Kind =
1105 StartTy->isPointerTy() ? InductionDescriptor::IK_PtrInduction
1106 : InductionDescriptor::IK_IntInduction;
1107 Type *TCTy = ResumeTC->getScalarType();
1108 VPValue *ExitCount = Builder.createOverflowingOp(
1109 Opcode: Instruction::Sub, Operands: {ResumeTC, Plan.getConstantInt(Ty: TCTy, Val: 1)},
1110 WrapFlags: {/*HasNUW=*/true, /*HasNSW=*/false}, DL: DebugLoc::getUnknown());
1111 return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, Start: StartVPV, Current: ExitCount,
1112 Step: StepVPV);
1113}
1114
1115void VPlanTransforms::optimizeInductionLiveOutUsers(
1116 VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L) {
1117 // Compute end values for all inductions.
1118 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
1119 auto *VectorPH = cast<VPBasicBlock>(Val: VectorRegion->getSinglePredecessor());
1120 VPBuilder VectorPHBuilder(VectorPH, VectorPH->getFirstNonPhi());
1121 DenseMap<VPValue *, VPValue *> EndValues;
1122 VPValue *ResumeTC =
1123 Plan.hasTailFolded() ? Plan.getTripCount() : &Plan.getVectorTripCount();
1124 for (VPWidenInductionRecipe &WideIV : make_isa_range<VPWidenInductionRecipe>(
1125 Range: VectorRegion->getEntryBasicBlock()->phis())) {
1126 if (VPValue *EndValue = tryToComputeEndValueForInduction(
1127 WideIV: &WideIV, VectorPHBuilder, VectorTC: ResumeTC))
1128 EndValues[&WideIV] = EndValue;
1129 }
1130
1131 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
1132 for (VPRecipeBase &R : make_early_inc_range(Range&: *MiddleVPBB)) {
1133 VPValue *Op;
1134 if (!match(V: &R, P: m_ExitingIVValue(Op0: m_VPValue(V&: Op))))
1135 continue;
1136 auto *WideIV = cast<VPWidenInductionRecipe>(Val: Op);
1137 if (VPValue *EndValue = EndValues.lookup(Val: WideIV)) {
1138 R.getVPSingleValue()->replaceAllUsesWith(New: EndValue);
1139 R.eraseFromParent();
1140 }
1141 }
1142
1143 // Then, optimize exit block users.
1144 for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
1145 for (VPRecipeBase &R : ExitVPBB->phis()) {
1146 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
1147
1148 for (auto [Idx, PredVPBB] : enumerate(First&: ExitVPBB->getPredecessors())) {
1149 VPValue *Escape = nullptr;
1150 if (PredVPBB == MiddleVPBB) {
1151 Escape = optimizeLatchExitInductionUser(
1152 Plan, Op: ExitIRI->getOperand(N: Idx), EndValues, PSE);
1153 if (!Escape)
1154 Escape = optimizeLatchExitIVUserViaSCEV(
1155 Plan, Op: ExitIRI->getOperand(N: Idx), PSE, ResumeTC, L);
1156 } else {
1157 Escape = optimizeEarlyExitInductionUser(
1158 Plan, Op: ExitIRI->getOperand(N: Idx), PSE);
1159 }
1160 if (Escape)
1161 ExitIRI->setOperand(I: Idx, New: Escape);
1162 }
1163 }
1164 }
1165}
1166
1167/// Remove redundant ExpandSCEVRecipes in \p Plan's entry block by replacing
1168/// them with already existing recipes expanding the same SCEV expression.
1169static void removeRedundantExpandSCEVRecipes(VPlan &Plan) {
1170 DenseMap<const SCEV *, VPValue *> SCEV2VPV;
1171
1172 for (VPExpandSCEVRecipe &ExpR :
1173 make_early_inc_range(Range: make_isa_range<VPExpandSCEVRecipe>(
1174 Range&: *Plan.getEntry()->getEntryBasicBlock()))) {
1175 const auto &[V, Inserted] = SCEV2VPV.try_emplace(Key: ExpR.getSCEV(), Args: &ExpR);
1176 if (Inserted)
1177 continue;
1178
1179 ExpR.replaceAllUsesWith(New: V->second);
1180 if (&ExpR == Plan.getTripCount())
1181 Plan.resetTripCount(NewTripCount: V->second);
1182
1183 ExpR.eraseFromParent();
1184 }
1185}
1186
1187/// Try to simplify logical and bitwise recipes in \p Def.
1188static VPValue *simplifyLogicalRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
1189 // Simplify (X && Y) | (X && !Y) -> X.
1190 // TODO: Split up into simpler, modular combines: (X && Y) | (X && Z) into X
1191 // && (Y | Z) and (X | !X) into true. This requires queuing newly created
1192 // recipes to be visited during simplification.
1193 VPValue *X, *Y;
1194 if (match(R: Def,
1195 P: m_c_BinaryOr(Op0: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y)),
1196 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_Not(Op0: m_Deferred(V: Y))))))
1197 return X;
1198
1199 // X | AllOnes -> AllOnes
1200 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_AllOnes())))
1201 return Plan.getAllOnesValue(Ty: Def->getScalarType());
1202
1203 // X | 0 -> X
1204 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_ZeroInt())))
1205 return X;
1206
1207 // X | !X -> AllOnes
1208 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X)))))
1209 return Plan.getAllOnesValue(Ty: Def->getScalarType());
1210
1211 // X & 0 -> 0
1212 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_ZeroInt())))
1213 return Plan.getZero(Ty: Def->getScalarType());
1214
1215 // X & AllOnes -> X
1216 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_AllOnes())))
1217 return X;
1218
1219 // X && false -> false
1220 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_False())))
1221 return Plan.getFalse();
1222
1223 // X && true -> X
1224 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_True())))
1225 return X;
1226
1227 // X && (X && Y) -> X && Y
1228 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1229 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue()))))
1230 return Def->getOperand(N: 1);
1231
1232 // X && !X -> 0
1233 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X)))))
1234 return Plan.getFalse();
1235
1236 if (match(R: Def, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: X), Op2: m_Deferred(V: X))))
1237 return X;
1238
1239 return nullptr;
1240}
1241
1242/// Return an existing value or a live in for VPSingleDefRecipe \p Def if
1243/// possible. This shouldn't create or modify recipes.
1244static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
1245 // Simplification of live-in IR values for SingleDef recipes using
1246 // InstSimplifyFolder.
1247 const DataLayout &DL = Plan.getDataLayout();
1248 if (VPValue *V = vputils::tryToFoldLiveIns(R&: *Def, Operands: Def->operands(), DL))
1249 return V;
1250
1251 // Fold PredPHI LiveIn -> LiveIn.
1252 if (auto *PredPHI = dyn_cast<VPPredInstPHIRecipe>(Val: Def)) {
1253 VPValue *Op = PredPHI->getOperand(N: 0);
1254 if (isa<VPIRValue>(Val: Op))
1255 return Op;
1256 }
1257
1258 if (VPValue *V = simplifyLogicalRecipe(Plan, Def))
1259 return V;
1260
1261 VPValue *A, *B;
1262
1263 if (match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))
1264 return A;
1265
1266 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_One())))
1267 return A;
1268
1269 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(), Op1: m_ZeroInt())))
1270 return Plan.getZero(Ty: Def->getScalarType());
1271
1272 // A bitcast to the same type is a no-op.
1273 if (match(R: Def, P: m_BitCast(Op0: m_VPValue(V&: A))) &&
1274 Def->getScalarType() == A->getScalarType())
1275 return A;
1276
1277 // Shifting by zero is a no-op.
1278 if (match(R: Def, P: m_CombineOr(Ps: m_Shl(Op0: m_VPValue(V&: A), Op1: m_ZeroInt()),
1279 Ps: m_CombineOr(Ps: m_LShr(Op0: m_VPValue(V&: A), Op1: m_ZeroInt()),
1280 Ps: m_AShr(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))))
1281 return A;
1282
1283 if (match(R: Def, P: m_Trunc(Op0: m_ZExtOrSExt(Op0: m_VPValue(V&: A)))))
1284 if (Def->getScalarType() == A->getScalarType())
1285 return A;
1286
1287 if (match(R: Def, P: m_Not(Op0: m_Not(Op0: m_VPValue(V&: A)))))
1288 return A;
1289
1290 // Remove redundant DerviedIVs, that is 0 + A * 1 -> A and 0 + 0 * x -> 0.
1291 if ((match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_VPValue(V&: A), Op2: m_One())) ||
1292 match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_VPValue(V&: A, Op: m_ZeroInt()),
1293 Op2: m_VPValue()))) &&
1294 A->getScalarType() == Def->getScalarType())
1295 return A;
1296
1297 // Simplify MaskedCond with no block mask to its single operand.
1298 if (match(R: Def, P: m_VPInstruction<VPInstruction::MaskedCond>()) &&
1299 !cast<VPInstruction>(Val: Def)->isMasked())
1300 return Def->getOperand(N: 0);
1301
1302 // Look through ExtractLastLane.
1303 if (match(R: Def, P: m_ExtractLastLane(Op0: m_VPValue(V&: A)))) {
1304 if (match(V: A, P: m_BuildVector())) {
1305 auto *BuildVector = cast<VPInstruction>(Val: A);
1306 return BuildVector->getOperand(N: BuildVector->getNumOperands() - 1);
1307 }
1308
1309 if (match(V: A, P: m_Broadcast(Op0: m_VPValue(V&: B))))
1310 return B;
1311
1312 if (isa<VPInstruction, VPReplicateRecipe>(Val: A) && vputils::isSingleScalar(VPV: A))
1313 return A;
1314
1315 if (Plan.hasScalarVFOnly())
1316 return A;
1317 }
1318
1319 // Look through ExtractPenultimateElement (BuildVector ....).
1320 if (match(R: Def, P: m_ExtractPenultimateElement(Op0: m_BuildVector()))) {
1321 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1322 return BuildVector->getOperand(N: BuildVector->getNumOperands() - 2);
1323 }
1324
1325 uint64_t Idx;
1326 if (match(R: Def, P: m_ExtractElement(Op0: m_BuildVector(), Op1: m_ConstantInt(C&: Idx)))) {
1327 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1328 return BuildVector->getOperand(N: Idx);
1329 }
1330
1331 if (isa<VPPhi, VPWidenPHIRecipe, VPHeaderPHIRecipe>(Val: Def)) {
1332 if (Def->getNumOperands() == 1) {
1333 return Def->getOperand(N: 0);
1334 }
1335 if (auto *Phi = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Val: Def)) {
1336 if (all_equal(Range: Phi->incoming_values()))
1337 return Phi->getOperand(N: 0);
1338 }
1339 return nullptr;
1340 }
1341
1342 VPIRValue *IRV;
1343 if (Def->getNumOperands() == 1 &&
1344 match(R: Def, P: m_ComputeReductionResult(Op0: m_VPIRValue(V&: IRV))))
1345 return IRV;
1346
1347 if (match(R: Def, P: m_VPInstruction<VPInstruction::WideIVStep>(Ops: m_VPValue(V&: A),
1348 Ops: m_One())) &&
1349 A->getScalarType() == Def->getScalarType())
1350 return A;
1351
1352 // Some simplifications can only be applied after unrolling. Perform them
1353 // below.
1354 if (!Plan.isUnrolled())
1355 return nullptr;
1356
1357 // After unrolling, extract-lane may be used to extract values from multiple
1358 // scalar sources. Only simplify when extracting from a single scalar source.
1359 VPValue *LaneToExtract;
1360 if (match(R: Def, P: m_ExtractLane(Op0: m_VPValue(V&: LaneToExtract), Op1: m_VPValue(V&: A)))) {
1361 // Simplify extract-lane(%lane_num, %scalar_val) -> %scalar_val.
1362 if (vputils::isSingleScalar(VPV: A))
1363 return A;
1364
1365 // Replace extract-lane(0, canonical-WIDEN-INDUCTION) with the region's
1366 // scalar canonical IV.
1367 VPWidenIntOrFpInductionRecipe *WidenIV;
1368 if (match(V: LaneToExtract, P: m_ZeroInt()) &&
1369 match(V: A, P: m_CanonicalWidenIV(V&: WidenIV)))
1370 return WidenIV->getRegion()->getCanonicalIV();
1371 }
1372
1373 // Simplify unrolled VectorPointer without offset, or with zero offset, to
1374 // just the pointer operand.
1375 if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(Val: Def))
1376 if (!VPR->getVFxPart() || match(V: VPR->getVFxPart(), P: m_ZeroInt()))
1377 return VPR->getOperand(N: 0);
1378
1379 // VPScalarIVSteps after unrolling can be replaced by their start value, if
1380 // the start index is zero and only the first lane 0 is demanded.
1381 if (auto *Steps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def))
1382 if (!Steps->getStartIndex() && vputils::onlyFirstLaneUsed(Def: Steps))
1383 return Steps->getOperand(N: 0);
1384
1385 if (Plan.getConcreteUF() == 1 && match(R: Def, P: m_ExtractLastPart(Op0: m_VPValue(V&: A))))
1386 return A;
1387
1388 return nullptr;
1389}
1390
1391/// Returns true if \p V is available at the end of \p VPBB, i.e. it either is a
1392/// live-in from the original IR or defined in \p VPBB.
1393static bool isAvailableAtEndOf(VPValue *V, const VPBasicBlock *VPBB) {
1394 VPRecipeBase *DefR = V->getDefiningRecipe();
1395 return DefR ? DefR->getParent() == VPBB : isa<VPIRValue>(Val: V);
1396}
1397
1398/// Combine \p Def into a simpler recipe. May modify or create new recipes.
1399static VPSingleDefRecipe *combineRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
1400 if (auto *V = simplifyRecipe(Plan, Def)) {
1401 Def->replaceAllUsesWith(New: V);
1402 return Def;
1403 }
1404
1405 // Drop the mask of a predicated store masked by the header mask (which is
1406 // guaranteed to be true at least for the first lane) and both the stored
1407 // value and the address are uniform across VF and UF. The header mask is
1408 // still the abstract region value here.
1409 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: Def);
1410 RepR && RepR->isPredicated() && RepR->getOpcode() == Instruction::Store &&
1411 all_of(Range: RepR->operandsWithoutMask(), P: vputils::isUniformAcrossVFsAndUFs) &&
1412 match(V: RepR->getMask(), P: m_HeaderMask())) {
1413 auto *Unmasked = new VPReplicateRecipe(
1414 RepR->getUnderlyingInstr(), RepR->operandsWithoutMask(),
1415 RepR->isSingleScalar(), /*Mask=*/nullptr, *RepR, *RepR,
1416 RepR->getDebugLoc());
1417 Unmasked->insertBefore(InsertPos: RepR);
1418 return Unmasked;
1419 }
1420
1421 VPBuilder Builder(Def);
1422
1423 // Avoid replacing VPInstructions with underlying values with new
1424 // VPInstructions, as we would fail to create widen/replicate recpes from the
1425 // new VPInstructions without an underlying value, and miss out on some
1426 // transformations that only apply to widened/replicated recipes later, by
1427 // doing so.
1428 // TODO: We should also not replace non-VPInstructions like VPWidenRecipe with
1429 // VPInstructions without underlying values, as those will get skipped during
1430 // cost computation.
1431 bool CanCreateNewRecipe =
1432 !isa<VPInstruction>(Val: Def) || !Def->getUnderlyingValue();
1433
1434 VPValue *X, *Y, *Z;
1435
1436 // X && (Y && X) -> X && Y
1437 if (CanCreateNewRecipe &&
1438 match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1439 Op1: m_LogicalAnd(Op0: m_VPValue(V&: Y), Op1: m_Deferred(V: X)))))
1440 return Builder.createLogicalAnd(LHS: X, RHS: Y);
1441
1442 // (X && Y) | (X && Z) -> X && (Y | Z)
1443 if (CanCreateNewRecipe &&
1444 match(R: Def, P: m_c_BinaryOr(Op0: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y)),
1445 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue(V&: Z)))) &&
1446 // Simplify only if one of the operands has one use to avoid creating an
1447 // extra recipe.
1448 (!Def->getOperand(N: 0)->hasMoreThanOneUniqueUser() ||
1449 !Def->getOperand(N: 1)->hasMoreThanOneUniqueUser()))
1450 return Builder.createLogicalAnd(LHS: X, RHS: Builder.createOr(LHS: Y, RHS: Z));
1451
1452 // (X && Y) | !X -> !X || Y
1453 if (CanCreateNewRecipe &&
1454 match(R: Def,
1455 P: m_c_BinaryOr(Op0: m_OneUse(SubPattern: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y))),
1456 Op1: m_VPValue(V&: Z, Op: m_Not(Op0: m_Deferred(V: X))))))
1457 return Builder.createLogicalOr(LHS: Z, RHS: Y);
1458
1459 // select C, false, true -> not C
1460 VPValue *C;
1461 if (CanCreateNewRecipe &&
1462 match(R: Def, P: m_Select(Op0: m_VPValue(V&: C), Op1: m_False(), Op2: m_True())))
1463 return Builder.createNot(Operand: C);
1464
1465 // select !C, X, Y -> select C, Y, X
1466 if (match(R: Def, P: m_Select(Op0: m_Not(Op0: m_VPValue(V&: C)), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1467 Def->setOperand(I: 0, New: C);
1468 Def->setOperand(I: 1, New: Y);
1469 Def->setOperand(I: 2, New: X);
1470 return Def;
1471 }
1472
1473 // select X, (i1 Y | Z), Y -> Y | (X && Z)
1474 if (CanCreateNewRecipe &&
1475 match(R: Def, P: m_Select(Op0: m_VPValue(V&: X),
1476 Op1: m_OneUse(SubPattern: m_c_BinaryOr(Op0: m_VPValue(V&: Y), Op1: m_VPValue(V&: Z))),
1477 Op2: m_Deferred(V: Y))) &&
1478 Y->getScalarType()->isIntegerTy(BitWidth: 1))
1479 return Builder.createOr(LHS: Y, RHS: Builder.createLogicalAnd(LHS: X, RHS: Z));
1480
1481 // select M0, (select M1, X, Y), Y -> select (M0 && M1), X, Y
1482 VPValue *Mask0, *Mask1;
1483 if (CanCreateNewRecipe &&
1484 match(R: Def,
1485 P: m_SelectLike(Op0: m_VPValue(V&: Mask0),
1486 Op1: m_OneUse(SubPattern: m_SelectLike(Op0: m_VPValue(V&: Mask1), Op1: m_VPValue(V&: X),
1487 Op2: m_VPValue(V&: Y))),
1488 Op2: m_Deferred(V: Y))))
1489 return Builder.createSelect(Cond: Builder.createLogicalAnd(LHS: Mask0, RHS: Mask1), TrueVal: X, FalseVal: Y,
1490 DL: Def->getDebugLoc());
1491
1492 if (match(R: Def, P: m_Trunc(Op0: m_VPValue(V&: Y, Op: m_ZExtOrSExt(Op0: m_VPValue(V&: X)))))) {
1493 // Don't replace a non-widened cast recipe with a widened cast.
1494 if (!isa<VPWidenCastRecipe>(Val: Def))
1495 return nullptr;
1496 Type *TruncTy = Def->getScalarType();
1497 Type *XTy = X->getScalarType();
1498 if (XTy->getScalarSizeInBits() < TruncTy->getScalarSizeInBits()) {
1499
1500 unsigned ExtOpcode =
1501 match(V: Y, P: m_SExt(Op0: m_VPValue())) ? Instruction::SExt : Instruction::ZExt;
1502 auto *Ext =
1503 Builder.createWidenCast(Opcode: Instruction::CastOps(ExtOpcode), Op: X, ResultTy: TruncTy);
1504 if (auto *UnderlyingExt = Y->getUnderlyingValue()) {
1505 // UnderlyingExt has distinct return type, used to retain legacy cost.
1506 Ext->setUnderlyingValue(UnderlyingExt);
1507 }
1508 return Ext;
1509 } else if (XTy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) {
1510 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::Trunc, Op: X, ResultTy: TruncTy);
1511 return Trunc;
1512 }
1513 }
1514
1515 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: X), Op1: m_AllOnes()))) {
1516 // Preserve nsw from the Mul on the new Sub.
1517 VPIRFlags::WrapFlagsTy NW = {
1518 false, cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap()};
1519 return Builder.createSub(LHS: Plan.getZero(Ty: X->getScalarType()), RHS: X,
1520 DL: Def->getDebugLoc(), Name: "", WrapFlags: NW);
1521 }
1522
1523 if (CanCreateNewRecipe &&
1524 match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: X),
1525 Op1: m_VPValue(V&: Z, Op: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Y)))))) {
1526 // Preserve nsw from the Add and the Sub, if it's present on both, on the
1527 // new Sub.
1528 VPIRFlags::WrapFlagsTy NW = {
1529 false, cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap() &&
1530 cast<VPRecipeWithIRFlags>(Val: Z)->hasNoSignedWrap()};
1531 return Builder.createSub(LHS: X, RHS: Y, DL: Def->getDebugLoc(), Name: "", WrapFlags: NW);
1532 }
1533
1534 const APInt *APC;
1535 if (CanCreateNewRecipe && match(R: Def, P: m_URem(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1536 APC->isPowerOf2())
1537 return Builder.createAnd(LHS: X, RHS: Plan.getConstantInt(Val: *APC - 1),
1538 DL: Def->getDebugLoc());
1539
1540 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1541 APC->isPowerOf2()) {
1542 auto *MulR = cast<VPRecipeWithIRFlags>(Val: Def);
1543 unsigned ShiftAmt = APC->exactLogBase2();
1544 VPIRFlags::WrapFlagsTy NW(MulR->hasNoUnsignedWrap(),
1545 MulR->hasNoSignedWrap() &&
1546 ShiftAmt != APC->getBitWidth() - 1);
1547 return Builder.createNaryOp(
1548 Opcode: Instruction::Shl,
1549 Operands: {X, Plan.getConstantInt(BitWidth: APC->getBitWidth(), Val: ShiftAmt)}, Flags: NW,
1550 DL: Def->getDebugLoc());
1551 }
1552
1553 if (CanCreateNewRecipe && match(R: Def, P: m_UDiv(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1554 APC->isPowerOf2())
1555 return Builder.createNaryOp(
1556 Opcode: Instruction::LShr,
1557 Operands: {X, Plan.getConstantInt(BitWidth: APC->getBitWidth(), Val: APC->exactLogBase2())},
1558 Flags: *cast<VPRecipeWithIRFlags>(Val: Def), DL: Def->getDebugLoc());
1559
1560 if (match(R: Def, P: m_Not(Op0: m_VPValue(V&: X)))) {
1561 // Try to fold Not into compares by adjusting the predicate in-place.
1562 CmpPredicate Pred;
1563 if (match(V: X, P: m_Cmp(Pred, Op0: m_VPValue(), Op1: m_VPValue()))) {
1564 auto *Cmp = cast<VPRecipeWithIRFlags>(Val: X);
1565 // Only fold if every user is a Not of the cmp, or a select using the cmp
1566 // solely as its condition.
1567 if (all_of(Range: Cmp->users(), P: [Cmp](VPUser *U) {
1568 return match(U, P: m_Not(Op0: m_Specific(VPV: Cmp))) ||
1569 (match(U, P: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(),
1570 Op2: m_VPValue())) &&
1571 U->getOperand(N: 1) != Cmp && U->getOperand(N: 2) != Cmp);
1572 })) {
1573 Cmp->setPredicate(CmpInst::getInversePredicate(pred: Pred));
1574 for (VPUser *U : to_vector(Range: Cmp->users())) {
1575 auto *R = cast<VPSingleDefRecipe>(Val: U);
1576 if (match(R, P: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1577 // select (cmp pred), X, Y -> select (cmp inv_pred), Y, X
1578 R->setOperand(I: 1, New: Y);
1579 R->setOperand(I: 2, New: X);
1580 } else {
1581 // not (cmp pred) -> cmp inv_pred
1582 assert(match(R, m_Not(m_Specific(Cmp))) && "Unexpected user");
1583 R->replaceAllUsesWith(New: Cmp);
1584 }
1585 }
1586 // If Cmp doesn't have a debug location, use the one from the negation,
1587 // to preserve the location.
1588 if (!Cmp->getDebugLoc() && Def->getDebugLoc())
1589 Cmp->setDebugLoc(Def->getDebugLoc());
1590 return Def;
1591 }
1592 }
1593 }
1594
1595 // Fold any-of (fcmp uno A, A), (fcmp uno B, B), ... ->
1596 // any-of (fcmp uno A, B), ...
1597 if (match(R: Def, P: m_AnyOf())) {
1598 SmallVector<VPValue *, 4> NewOps;
1599 VPRecipeBase *UnpairedCmp = nullptr;
1600 for (VPValue *Op : Def->operands()) {
1601 VPValue *X;
1602 if (Op->getNumUsers() > 1 ||
1603 !match(V: Op, P: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X),
1604 Op1: m_Deferred(V: X)))) {
1605 NewOps.push_back(Elt: Op);
1606 } else if (!UnpairedCmp) {
1607 UnpairedCmp = Op->getDefiningRecipe();
1608 } else {
1609 NewOps.push_back(Elt: Builder.createFCmp(Pred: CmpInst::FCMP_UNO,
1610 A: UnpairedCmp->getOperand(N: 0), B: X));
1611 UnpairedCmp = nullptr;
1612 }
1613 }
1614
1615 if (UnpairedCmp)
1616 NewOps.push_back(Elt: UnpairedCmp->getVPSingleValue());
1617
1618 if (NewOps.size() < Def->getNumOperands())
1619 return Builder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: NewOps);
1620 }
1621
1622 // Fold (fcmp uno X, X) | (fcmp uno Y, Y) -> fcmp uno X, Y
1623 // This is useful for fmax/fmin without fast-math flags, where we need to
1624 // check if any operand is NaN.
1625 if (CanCreateNewRecipe &&
1626 match(R: Def,
1627 P: m_BinaryOr(
1628 Op0: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X), Op1: m_Deferred(V: X)),
1629 Op1: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: Y), Op1: m_Deferred(V: Y)))))
1630 return Builder.createFCmp(Pred: CmpInst::FCMP_UNO, A: X, B: Y);
1631
1632 if (match(R: Def, P: m_VPInstruction<VPInstruction::WideIVStep>(Ops: m_VPValue(V&: X),
1633 Ops: m_One())) &&
1634 X->getScalarType() != Def->getScalarType())
1635 return Builder.createWidenCast(Opcode: Instruction::Trunc, Op: X, ResultTy: Def->getScalarType());
1636
1637 // For i1 vp.merges produced by AnyOf reductions:
1638 // vp.merge true, (or X, Y), X, evl -> vp.merge Y, true, X, evl
1639 if (match(R: Def, P: m_Intrinsic<Intrinsic::vp_merge>(Ops: m_True(), Ops: m_VPValue(V&: X),
1640 Ops: m_VPValue(V&: X), Ops: m_VPValue())) &&
1641 match(V: X, P: m_c_BinaryOr(Op0: m_Specific(VPV: X), Op1: m_VPValue(V&: Y))) &&
1642 Def->getScalarType()->isIntegerTy(BitWidth: 1)) {
1643 Def->setOperand(I: 1, New: Plan.getTrue());
1644 Def->setOperand(I: 0, New: Y);
1645 return Def;
1646 }
1647
1648 if (match(R: Def, P: m_BuildVector()) && all_equal(Range: Def->operands()))
1649 return Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: Def->getOperand(N: 0));
1650
1651 // Replace uses of a BuildVector by users that only use its first lane with
1652 // its first operand directly.
1653 if (match(R: Def, P: m_BuildVector())) {
1654 Def->replaceUsesWithIf(New: Def->getOperand(N: 0), ShouldReplace: [Def](VPUser &U, unsigned) {
1655 return U.usesFirstLaneOnly(Op: Def);
1656 });
1657 return Def;
1658 }
1659
1660 // Look through broadcast of single-scalar when used as select conditions; in
1661 // that case the scalar condition can be used directly.
1662 if (match(R: Def,
1663 P: m_Select(Op0: m_Broadcast(Op0: m_VPValue(V&: Z)), Op1: m_VPValue(), Op2: m_VPValue()))) {
1664 assert(vputils::isSingleScalar(Z) &&
1665 "broadcast operand must be single-scalar");
1666 Def->setOperand(I: 0, New: Z);
1667 return Def;
1668 }
1669
1670 if (match(R: Def, P: m_Broadcast(Op0: m_VPValue(V&: X)))) {
1671 Def->replaceUsesWithIf(
1672 New: X, ShouldReplace: [Def](const VPUser &U, unsigned) { return U.usesScalars(Op: Def); });
1673 return Def;
1674 }
1675
1676 // Some simplifications can only be applied after unrolling. Perform them
1677 // below.
1678 if (!Plan.isUnrolled())
1679 return nullptr;
1680
1681 // Simplify extract-lane with single source to extract-element.
1682 VPValue *LaneToExtract;
1683 if (match(R: Def, P: m_ExtractLane(Op0: m_VPValue(V&: LaneToExtract), Op1: m_VPValue(V&: X))))
1684 return Builder.createNaryOp(Opcode: Instruction::ExtractElement, Operands: {X, LaneToExtract},
1685 DL: Def->getDebugLoc());
1686
1687 // Look for cycles where Def is of the form:
1688 // X = phi(0, IVInc) ; used only by IVInc, or by IVInc and Inc = X + Y
1689 // IVInc = X + Step ; used by X and Def
1690 // Def = IVInc + Y
1691 // Fold the increment Y into the phi's start value, replace Def with IVInc,
1692 // and if Inc exists, replace it with X.
1693 VPValue *IVInc;
1694 if (match(R: Def, P: m_Add(Op0: m_VPValue(V&: IVInc, Op: m_Add(Op0: m_VPValue(V&: X), Op1: m_VPValue())),
1695 Op1: m_VPValue(V&: Y))) &&
1696 match(V: X, P: m_VPPhi(Op0: m_ZeroInt(), Op1: m_Specific(VPV: IVInc))) &&
1697 IVInc->getNumUsers() == 2) {
1698 auto *Phi = cast<VPPhi>(Val: X);
1699 // If Phi has a second user (besides IVInc's defining recipe), it must be
1700 // Inc = Phi + Y for the fold to apply.
1701 auto *Inc = dyn_cast_if_present<VPSingleDefRecipe>(
1702 Val: findUserOf(V: Phi, P: m_Add(Op0: m_Specific(VPV: Phi), Op1: m_Specific(VPV: Y))));
1703 if ((Phi->getNumUsers() == 1 || (Phi->getNumUsers() == 2 && Inc)) &&
1704 isAvailableAtEndOf(V: Y, VPBB: Phi->getIncomingBlock(Idx: 0))) {
1705 Def->replaceAllUsesWith(New: IVInc);
1706 if (Inc)
1707 Inc->replaceAllUsesWith(New: Phi);
1708 Phi->setOperand(I: 0, New: Y);
1709 return Def;
1710 }
1711 }
1712
1713 // Simplify redundant ReductionStartVector recipes after unrolling.
1714 VPValue *StartV;
1715 if (match(R: Def, P: m_VPInstruction<VPInstruction::ReductionStartVector>(
1716 Ops: m_VPValue(V&: StartV), Ops: m_VPValue(), Ops: m_VPValue()))) {
1717 Def->replaceUsesWithIf(New: StartV, ShouldReplace: [](const VPUser &U, unsigned Idx) {
1718 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &U);
1719 return PhiR && PhiR->isInLoop();
1720 });
1721 return Def;
1722 }
1723
1724 return nullptr;
1725}
1726
1727void VPlanTransforms::combineRecipes(VPlan &Plan) {
1728 SmallVector<VPSingleDefRecipe *, 256> Worklist;
1729 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
1730 Plan.getEntry());
1731 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT))
1732 for (VPSingleDefRecipe &Def :
1733 make_isa_range<VPSingleDefRecipe>(Range: reverse(C&: *VPBB)))
1734 Worklist.push_back(Elt: &Def);
1735
1736 [[maybe_unused]] unsigned InitWorklistSize = Worklist.size();
1737
1738 while (!Worklist.empty()) {
1739 assert(Worklist.size() < InitWorklistSize * 2 &&
1740 "Worklist is growing large, possible cycle?");
1741 VPSingleDefRecipe *Def = Worklist.pop_back_val();
1742 VPSingleDefRecipe *New = combineRecipe(Plan, Def);
1743 if (!New)
1744 continue;
1745 if (New != Def) {
1746 // Replace the recipe with a new one.
1747 Def->replaceAllUsesWith(New);
1748 Def->eraseFromParent();
1749 Worklist.push_back(Elt: New);
1750 // TODO: Append users to the worklist (might need a setvector)
1751 } else if (vputils::isDeadRecipe(R&: *Def)) {
1752 // Recipe was modified - it may be dead now.
1753 Def->eraseFromParent();
1754 }
1755 }
1756}
1757
1758void VPlanTransforms::simplifyReverses(VPlan &Plan) {
1759 // Pull out reverses from any elementwise op.
1760 // binop(reverse(x), reverse(y)) -> reverse(binop(x,y))
1761 vputils::pullOutPermutations(
1762 Plan, Perm: [](VPValue *&X) { return m_Reverse(Op0: m_VPValue(V&: X)); },
1763 Build: [](auto *X) { return new VPInstruction(VPInstruction::Reverse, X); });
1764
1765 // reverse(reverse(x)) -> x
1766 VPValue *X;
1767 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1768 Range: vp_depth_first_deep(G: Plan.getEntry())))
1769 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
1770 if (match(V: &R, P: m_Reverse(Op0: m_Reverse(Op0: m_VPValue(V&: X)))))
1771 R.getVPSingleValue()->replaceAllUsesWith(New: X);
1772}
1773
1774/// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the
1775/// header mask to be simplified further when tail folding, e.g. in
1776/// optimizeEVLMasks.
1777static void reassociateHeaderMask(VPlan &Plan) {
1778 VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
1779 if (!HeaderMask)
1780 return;
1781
1782 SmallVector<VPUser *> Worklist;
1783 for (VPUser *U : HeaderMask->users())
1784 if (match(U, P: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue())))
1785 append_range(C&: Worklist, R: cast<VPSingleDefRecipe>(Val: U)->users());
1786
1787 while (!Worklist.empty()) {
1788 auto *R = dyn_cast<VPSingleDefRecipe>(Val: Worklist.pop_back_val());
1789 VPValue *X, *Y;
1790 if (!R || !match(R, P: m_LogicalAnd(
1791 Op0: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue(V&: X)),
1792 Op1: m_VPValue(V&: Y))))
1793 continue;
1794 append_range(C&: Worklist, R: R->users());
1795 VPBuilder Builder(R);
1796 R->replaceAllUsesWith(
1797 New: Builder.createLogicalAnd(LHS: HeaderMask, RHS: Builder.createLogicalAnd(LHS: X, RHS: Y)));
1798 }
1799}
1800
1801static std::optional<Instruction::BinaryOps>
1802getUnmaskedDivRemOpcode(Intrinsic::ID ID) {
1803 switch (ID) {
1804 case Intrinsic::masked_udiv:
1805 return Instruction::UDiv;
1806 case Intrinsic::masked_sdiv:
1807 return Instruction::SDiv;
1808 case Intrinsic::masked_urem:
1809 return Instruction::URem;
1810 case Intrinsic::masked_srem:
1811 return Instruction::SRem;
1812 default:
1813 return {};
1814 }
1815}
1816
1817static void narrowToSingleScalarRecipes(VPlan &Plan) {
1818 if (Plan.hasScalarVFOnly())
1819 return;
1820
1821 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1822 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
1823 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
1824 if (!isa<VPWidenRecipe, VPWidenGEPRecipe, VPReplicateRecipe,
1825 VPWidenIntrinsicRecipe>(Val: &R))
1826 continue;
1827 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
1828 if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
1829 continue;
1830
1831 auto *RepOrWidenR = cast<VPRecipeWithIRFlags>(Val: &R);
1832 if (RepR && RepR->getOpcode() == Instruction::Store &&
1833 vputils::isSingleScalar(VPV: RepR->getOperand(N: 1))) {
1834 auto *Clone = new VPReplicateRecipe(
1835 RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
1836 true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
1837 *RepR /*Metadata*/, RepR->getDebugLoc());
1838 Clone->insertBefore(InsertPos: RepOrWidenR);
1839 VPBuilder Builder(Clone);
1840 VPValue *ExtractOp = Clone->getOperand(N: 0);
1841 if (vputils::isUniformAcrossVFsAndUFs(V: RepR->getOperand(N: 1)))
1842 ExtractOp =
1843 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastPart, Operands: ExtractOp);
1844 ExtractOp =
1845 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastLane, Operands: ExtractOp);
1846 Clone->setOperand(I: 0, New: ExtractOp);
1847 RepR->eraseFromParent();
1848 continue;
1849 }
1850
1851 // Narrow llvm.masked.{u,s}{div,rem} intrinsics with a safe divisor.
1852 if (auto *IntrR = dyn_cast<VPWidenIntrinsicRecipe>(Val: RepOrWidenR)) {
1853 if (!vputils::onlyFirstLaneUsed(Def: IntrR))
1854 continue;
1855 auto Opc = getUnmaskedDivRemOpcode(ID: IntrR->getVectorIntrinsicID());
1856 if (!Opc)
1857 continue;
1858 VPBuilder Builder(IntrR);
1859 VPValue *SafeDivisor = Builder.createSelect(
1860 Cond: IntrR->getOperand(N: 2), TrueVal: IntrR->getOperand(N: 1),
1861 FalseVal: Plan.getConstantInt(Ty: IntrR->getScalarType(), Val: 1));
1862 VPValue *Clone = Builder.createNaryOp(
1863 Opcode: *Opc, Operands: {IntrR->getOperand(N: 0), SafeDivisor},
1864 Flags: VPIRFlags::getDefaultFlags(Opcode: *Opc), DL: IntrR->getDebugLoc());
1865 IntrR->replaceAllUsesWith(New: Clone);
1866 IntrR->eraseFromParent();
1867 continue;
1868 }
1869
1870 // Skip recipes that aren't single scalars.
1871 if (!vputils::isSingleScalar(VPV: RepOrWidenR))
1872 continue;
1873
1874 // Predicate to check if a user of Op introduces extra broadcasts.
1875 auto IntroducesBCastOf = [](const VPValue *Op) {
1876 return [Op](const VPUser *U) {
1877 if (auto *VPI = dyn_cast<VPInstruction>(Val: U)) {
1878 if (is_contained(Set: {VPInstruction::ExtractLastLane,
1879 VPInstruction::ExtractLastPart,
1880 VPInstruction::ExtractPenultimateElement},
1881 Element: VPI->getOpcode()))
1882 return false;
1883 }
1884 return !U->usesScalars(Op);
1885 };
1886 };
1887
1888 if (any_of(Range: RepOrWidenR->users(), P: IntroducesBCastOf(RepOrWidenR)) &&
1889 none_of(Range: RepOrWidenR->operands(), P: [&](VPValue *Op) {
1890 if (any_of(
1891 Range: make_filter_range(Range: Op->users(), Pred: not_equal_to(Arg&: RepOrWidenR)),
1892 P: IntroducesBCastOf(Op)))
1893 return false;
1894 // Non-constant live-ins require broadcasts, while constants do not
1895 // need explicit broadcasts.
1896 bool LiveInNeedsBroadcast =
1897 isa<VPIRValue>(Val: Op) && !isa<VPConstant>(Val: Op);
1898 auto *OpR = dyn_cast<VPReplicateRecipe>(Val: Op);
1899 return LiveInNeedsBroadcast || (OpR && OpR->isSingleScalar());
1900 }))
1901 continue;
1902
1903 auto *Clone = VPBuilder::createSingleScalarOp(
1904 Opcode: vputils::getOpcode(V: RepOrWidenR), Operands: RepOrWidenR->operands(),
1905 /*Mask=*/nullptr, Flags: *RepOrWidenR, Metadata: getMetadataOf(R: RepOrWidenR),
1906 DL: DebugLoc::getUnknown(), UV: RepOrWidenR->getUnderlyingInstr());
1907 Clone->insertBefore(InsertPos: RepOrWidenR);
1908 RepOrWidenR->replaceAllUsesWith(New: Clone);
1909 if (vputils::isDeadRecipe(R&: *RepOrWidenR))
1910 RepOrWidenR->eraseFromParent();
1911 }
1912 }
1913}
1914
1915/// Try to see if all of \p Blend's masks share a common value logically and'ed
1916/// and remove it from the masks.
1917static void removeCommonBlendMask(VPBlendRecipe *Blend) {
1918 if (Blend->isNormalized())
1919 return;
1920 VPValue *CommonEdgeMask;
1921 if (!match(V: Blend->getMask(Idx: 0),
1922 P: m_LogicalAnd(Op0: m_VPValue(V&: CommonEdgeMask), Op1: m_VPValue())))
1923 return;
1924 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1925 if (!match(V: Blend->getMask(Idx: I),
1926 P: m_LogicalAnd(Op0: m_Specific(VPV: CommonEdgeMask), Op1: m_VPValue())))
1927 return;
1928 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1929 Blend->setMask(Idx: I, V: Blend->getMask(Idx: I)->getDefiningRecipe()->getOperand(N: 1));
1930}
1931
1932/// Normalize and simplify VPBlendRecipes. Should be run after combineRecipes
1933/// to make sure the masks are simplified.
1934static void simplifyBlends(VPlan &Plan) {
1935 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1936 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
1937 for (VPBlendRecipe &Blend :
1938 make_early_inc_range(Range: make_isa_range<VPBlendRecipe>(Range&: *VPBB))) {
1939 removeCommonBlendMask(Blend: &Blend);
1940
1941 // Try to remove redundant blend recipes.
1942 SmallPtrSet<VPValue *, 4> UniqueValues;
1943 if (Blend.isNormalized() || !match(V: Blend.getMask(Idx: 0), P: m_False()))
1944 UniqueValues.insert(Ptr: Blend.getIncomingValue(Idx: 0));
1945 for (unsigned I = 1; I != Blend.getNumIncomingValues(); ++I)
1946 if (!match(V: Blend.getMask(Idx: I), P: m_False()))
1947 UniqueValues.insert(Ptr: Blend.getIncomingValue(Idx: I));
1948
1949 if (UniqueValues.size() == 1) {
1950 Blend.replaceAllUsesWith(New: *UniqueValues.begin());
1951 Blend.eraseFromParent();
1952 continue;
1953 }
1954
1955 if (Blend.isNormalized())
1956 continue;
1957
1958 // Normalize the blend so its first incoming value is used as the initial
1959 // value with the others blended into it.
1960
1961 unsigned StartIndex = 0;
1962 for (unsigned I = 0; I != Blend.getNumIncomingValues(); ++I) {
1963 // If a value's mask is used only by the blend then is can be deadcoded.
1964 // TODO: Find the most expensive mask that can be deadcoded, or a mask
1965 // that's used by multiple blends where it can be removed from them all.
1966 VPValue *Mask = Blend.getMask(Idx: I);
1967 if (Mask->hasOneUse() && !match(V: Mask, P: m_False())) {
1968 StartIndex = I;
1969 break;
1970 }
1971 }
1972
1973 SmallVector<VPValue *, 4> OperandsWithMask;
1974 OperandsWithMask.push_back(Elt: Blend.getIncomingValue(Idx: StartIndex));
1975
1976 for (unsigned I = 0; I != Blend.getNumIncomingValues(); ++I) {
1977 if (I == StartIndex)
1978 continue;
1979 OperandsWithMask.push_back(Elt: Blend.getIncomingValue(Idx: I));
1980 OperandsWithMask.push_back(Elt: Blend.getMask(Idx: I));
1981 }
1982
1983 auto *NewBlend =
1984 new VPBlendRecipe(cast_or_null<PHINode>(Val: Blend.getUnderlyingValue()),
1985 OperandsWithMask, Blend, Blend.getDebugLoc());
1986 NewBlend->insertBefore(InsertPos: &Blend);
1987
1988 VPValue *DeadMask = Blend.getMask(Idx: StartIndex);
1989 Blend.replaceAllUsesWith(New: NewBlend);
1990 Blend.eraseFromParent();
1991 vputils::recursivelyDeleteDeadRecipes(V: DeadMask);
1992
1993 /// Simplify BLEND %a, %b, Not(%mask) -> BLEND %b, %a, %mask.
1994 VPValue *NewMask;
1995 if (NewBlend->getNumOperands() == 3 &&
1996 match(V: NewBlend->getMask(Idx: 1), P: m_Not(Op0: m_VPValue(V&: NewMask)))) {
1997 VPValue *Inc0 = NewBlend->getOperand(N: 0);
1998 VPValue *Inc1 = NewBlend->getOperand(N: 1);
1999 VPValue *OldMask = NewBlend->getOperand(N: 2);
2000 NewBlend->setOperand(I: 0, New: Inc1);
2001 NewBlend->setOperand(I: 1, New: Inc0);
2002 NewBlend->setOperand(I: 2, New: NewMask);
2003 if (OldMask->user_empty())
2004 cast<VPInstruction>(Val: OldMask)->eraseFromParent();
2005 }
2006 }
2007 }
2008}
2009
2010/// Optimize the width of vector induction variables in \p Plan based on a known
2011/// constant Trip Count, \p BestVF and \p BestUF.
2012static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan,
2013 ElementCount BestVF,
2014 unsigned BestUF) {
2015 // Only proceed if we have not completely removed the vector region.
2016 if (!Plan.getVectorLoopRegion())
2017 return false;
2018
2019 const APInt *TC;
2020 if (!BestVF.isFixed() || !match(V: Plan.getTripCount(), P: m_APInt(C&: TC)))
2021 return false;
2022
2023 // Calculate the minimum power-of-2 bit width that can fit the known TC, VF
2024 // and UF. Returns at least 8.
2025 auto ComputeBitWidth = [](APInt TC, uint64_t Align) {
2026 APInt AlignedTC =
2027 Align * APIntOps::RoundingUDiv(A: TC, B: APInt(TC.getBitWidth(), Align),
2028 RM: APInt::Rounding::UP);
2029 APInt MaxVal = AlignedTC - 1;
2030 return std::max<unsigned>(a: PowerOf2Ceil(A: MaxVal.getActiveBits()), b: 8);
2031 };
2032 unsigned NewBitWidth =
2033 ComputeBitWidth(*TC, BestVF.getKnownMinValue() * BestUF);
2034
2035 LLVMContext &Ctx = Plan.getContext();
2036 auto *NewIVTy = IntegerType::get(C&: Ctx, NumBits: NewBitWidth);
2037
2038 bool MadeChange = false;
2039
2040 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
2041 for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
2042 // Currently only handle canonical IVs as it is trivial to replace the start
2043 // and stop values, and we currently only perform the optimization when the
2044 // IV has a single use.
2045 VPWidenIntOrFpInductionRecipe *WideIV;
2046 if (!match(V: &Phi, P: m_CanonicalWidenIV(V&: WideIV)))
2047 continue;
2048 if (WideIV->hasMoreThanOneUniqueUser() ||
2049 NewIVTy == WideIV->getScalarType())
2050 continue;
2051
2052 // Currently only handle cases where the single user is a header-mask
2053 // comparison with the backedge-taken-count.
2054 VPUser *SingleUser = WideIV->getSingleUser();
2055 if (!SingleUser ||
2056 !match(U: SingleUser,
2057 P: m_ICmp(Op0: m_Specific(VPV: WideIV),
2058 Op1: m_Broadcast(Op0: m_Specific(VPV: Plan.getBackedgeTakenCount())))))
2059 continue;
2060
2061 // Update IV operands and comparison bound to use new narrower type.
2062 assert(!WideIV->getTruncInst() &&
2063 "canonical IV is not expected to have a truncation");
2064 auto *NewWideIV = new VPWidenIntOrFpInductionRecipe(
2065 WideIV->getPHINode(), Plan.getZero(Ty: NewIVTy),
2066 Plan.getConstantInt(Ty: NewIVTy, Val: 1), WideIV->getVFValue(),
2067 WideIV->getInductionDescriptor(), *WideIV, WideIV->getDebugLoc());
2068 NewWideIV->insertBefore(InsertPos: WideIV);
2069
2070 auto *NewBTC = new VPWidenCastRecipe(
2071 Instruction::Trunc, Plan.getOrCreateBackedgeTakenCount(), NewIVTy,
2072 nullptr, VPIRFlags::getDefaultFlags(Opcode: Instruction::Trunc));
2073 Plan.getVectorPreheader()->appendRecipe(Recipe: NewBTC);
2074 auto *Cmp = cast<VPInstruction>(Val: WideIV->getSingleUser());
2075 Cmp->replaceAllUsesWith(
2076 New: VPBuilder(Cmp).createICmp(Pred: Cmp->getPredicate(), A: NewWideIV, B: NewBTC));
2077
2078 MadeChange = true;
2079 }
2080
2081 return MadeChange;
2082}
2083
2084/// Return true if \p Cond is known to be true for given \p BestVF and \p
2085/// BestUF.
2086static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
2087 ElementCount BestVF, unsigned BestUF,
2088 PredicatedScalarEvolution &PSE) {
2089 if (match(V: Cond, P: m_BinaryOr(Op0: m_VPValue(), Op1: m_VPValue())))
2090 return any_of(Range: Cond->getDefiningRecipe()->operands(), P: [&Plan, BestVF, BestUF,
2091 &PSE](VPValue *C) {
2092 return isConditionTrueViaVFAndUF(Cond: C, Plan, BestVF, BestUF, PSE);
2093 });
2094
2095 auto *CanIV = Plan.getVectorLoopRegion()->getCanonicalIV();
2096 if (!match(V: Cond, P: m_SpecificICmp(
2097 MatchPred: CmpInst::ICMP_EQ,
2098 Op0: m_c_Add(Op0: m_Specific(VPV: CanIV), Op1: m_Specific(VPV: &Plan.getVFxUF())),
2099 Op1: m_Specific(VPV: &Plan.getVectorTripCount()))))
2100 return false;
2101
2102 // The compare checks CanIV + VFxUF == vector trip count. The vector trip
2103 // count is not conveniently available as SCEV so far, so we compare directly
2104 // against the original trip count. This is stricter than necessary, as we
2105 // will only return true if the trip count == vector trip count.
2106 const SCEV *VectorTripCount =
2107 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2108 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2109 VectorTripCount = vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2110 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2111 "Trip count SCEV must be computable");
2112 ScalarEvolution &SE = *PSE.getSE();
2113 ElementCount NumElements = BestVF * BestUF;
2114 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2115 return SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: VectorTripCount, RHS: C);
2116}
2117
2118// Replaces ExtractVectorForPart instructions with ICMP when the VF is scalar
2119// and the source is a WideActiveLaneMask. The unused mask is removed later
2120// when removing dead recipes.
2121static bool replaceMaskWithCompareForScalarPlan(VPlan &Plan,
2122 ElementCount BestVF) {
2123 if (!BestVF.isScalar())
2124 return false;
2125
2126 bool MadeChange = false;
2127 VPBuilder Builder;
2128 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2129 VPBasicBlock *PreheaderVPBB = Plan.getVectorPreheader();
2130 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2131
2132 VPValue *Start, *TC;
2133 uint64_t Idx;
2134 for (VPBasicBlock *VPBB : {PreheaderVPBB, ExitingVPBB}) {
2135 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2136 if (!match(V: &R, P: m_ExtractVectorForPart(
2137 Op0: m_WideActiveLaneMask(Op0: m_VPValue(V&: Start), Op1: m_VPValue(V&: TC),
2138 Op2: m_VPValue()),
2139 Op1: m_ConstantInt(C&: Idx))))
2140 continue;
2141
2142 auto *Extract = cast<VPInstruction>(Val: &R);
2143 Builder.setInsertPoint(Extract);
2144
2145 if (Idx > 0)
2146 Start = Builder.createAdd(
2147 LHS: Start, RHS: Plan.getConstantInt(Ty: Start->getScalarType(), Val: Idx));
2148
2149 VPValue *ICmp = Builder.createICmp(Pred: CmpInst::ICMP_ULT, A: Start, B: TC);
2150 Extract->replaceAllUsesWith(New: ICmp);
2151 Extract->eraseFromParent();
2152 MadeChange = true;
2153 }
2154 }
2155
2156 return MadeChange;
2157}
2158
2159/// Try to simplify the branch condition of \p Plan. This may restrict the
2160/// resulting plan to \p BestVF and \p BestUF.
2161static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
2162 unsigned BestUF,
2163 PredicatedScalarEvolution &PSE) {
2164 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2165 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2166 auto *Term = &ExitingVPBB->back();
2167 VPValue *Cond;
2168 VPValue *Offset = nullptr;
2169 auto m_CanIVInc = m_Add(Op0: m_VPValue(), Op1: m_Specific(VPV: &Plan.getVFxUF()));
2170 // Check if the branch condition compares the canonical IV increment (for main
2171 // loop), or the canonical IV increment plus an offset (for epilog loop).
2172 bool MatchedCanIVInc =
2173 match(V: Term,
2174 P: m_BranchOnCount(
2175 Op0: m_CombineOr(Ps: m_CanIVInc, Ps: m_c_Add(Op0: m_CanIVInc, Op1: m_VPValue(V&: Offset))),
2176 Op1: m_VPValue())) &&
2177 (!Offset || Offset->isDefinedOutsideLoopRegions());
2178 if (MatchedCanIVInc ||
2179 match(V: Term,
2180 P: m_BranchOnCond(Op0: m_Not(Op0: m_ExtractVectorForPart(
2181 Op0: m_WideActiveLaneMask(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue()),
2182 Op1: m_ZeroInt()))))) {
2183 // Try to simplify the branch condition if VectorTC <= VF * UF when the
2184 // latch terminator is BranchOnCount or
2185 // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask), 0))
2186 const SCEV *VectorTripCount =
2187 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2188 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2189 VectorTripCount =
2190 vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2191 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2192 "Trip count SCEV must be computable");
2193 ScalarEvolution &SE = *PSE.getSE();
2194 ElementCount NumElements = BestVF * BestUF;
2195 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2196 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_ULE, LHS: VectorTripCount, RHS: C))
2197 return false;
2198 } else if (match(V: Term, P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))) ||
2199 match(V: Term, P: m_BranchOnTwoConds(Op0: m_VPValue(), Op1: m_VPValue(V&: Cond)))) {
2200 // For BranchOnCond, check if we can prove the condition to be true using VF
2201 // and UF.
2202 if (!isConditionTrueViaVFAndUF(Cond, Plan, BestVF, BestUF, PSE))
2203 return false;
2204 } else {
2205 return false;
2206 }
2207
2208 // The vector loop region only executes once. Convert terminator of the
2209 // exiting block to exit in the first iteration.
2210 if (match(V: Term, P: m_BranchOnTwoConds())) {
2211 Term->setOperand(I: 1, New: Plan.getTrue());
2212 return true;
2213 }
2214
2215 auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, Plan.getTrue(), {},
2216 {}, Term->getDebugLoc());
2217 ExitingVPBB->appendRecipe(Recipe: BOC);
2218 Term->eraseFromParent();
2219
2220 return true;
2221}
2222
2223void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
2224 unsigned BestUF,
2225 PredicatedScalarEvolution &PSE) {
2226 assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
2227 assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
2228
2229 bool MadeChange =
2230 simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
2231 MadeChange |= replaceMaskWithCompareForScalarPlan(Plan, BestVF);
2232 MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
2233
2234 if (MadeChange) {
2235 Plan.setVF(BestVF);
2236 assert(Plan.getConcreteUF() == BestUF && "BestUF must match the Plan's UF");
2237 }
2238}
2239
2240void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
2241 for (VPReductionPHIRecipe &PhiR : make_isa_range<VPReductionPHIRecipe>(
2242 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
2243 RecurKind RK = PhiR.getRecurrenceKind();
2244 if (RK != RecurKind::Add && RK != RecurKind::Mul && RK != RecurKind::Sub &&
2245 RK != RecurKind::AddChainWithSubs)
2246 continue;
2247
2248 for (VPUser *U : vputils::collectUsersRecursively(V: &PhiR))
2249 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: U)) {
2250 RecWithFlags->dropPoisonGeneratingFlags();
2251 }
2252 }
2253}
2254
2255namespace {
2256struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
2257 /// If recipe \p R will lower to a GEP with a non-i8 source element type,
2258 /// return that source element type.
2259 static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) {
2260 // All VPInstructions that lower to GEPs must have the i8 source element
2261 // type (as they are PtrAdds), so we omit it.
2262 return TypeSwitch<const VPSingleDefRecipe *, Type *>(R)
2263 .Case(caseFn: [](const VPReplicateRecipe *I) -> Type * {
2264 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I->getUnderlyingValue()))
2265 return GEP->getSourceElementType();
2266 return nullptr;
2267 })
2268 .Case<VPVectorPointerRecipe, VPWidenGEPRecipe>(
2269 caseFn: [](auto *I) { return I->getSourceElementType(); })
2270 .Default(defaultFn: [](auto *) { return nullptr; });
2271 }
2272
2273 /// Returns true if recipe \p Def can be safely handed for CSE.
2274 static bool canHandle(const VPSingleDefRecipe *Def) {
2275 // We can extend the list of handled recipes in the future,
2276 // provided we account for the data embedded in them while checking for
2277 // equality or hashing.
2278 auto C = vputils::getOpcodeOrIntrinsicID(V: Def);
2279
2280 // The issue with (Insert|Extract)Value is that the index of the
2281 // insert/extract is not a proper operand in LLVM IR, and hence also not in
2282 // VPlan.
2283 if (!C || (!C->first && (C->second == Instruction::InsertValue ||
2284 C->second == Instruction::ExtractValue)))
2285 return false;
2286
2287 // Widened loads (including the EVL variant) are handled, as cse() only
2288 // reuses them within a block with no intervening memory write. Any other
2289 // memory access is rejected.
2290 if (Def->mayWriteToMemory())
2291 return false;
2292 return !Def->mayReadFromMemory() ||
2293 isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Val: Def);
2294 }
2295
2296 /// Hash the underlying data of \p Def.
2297 static unsigned getHashValue(const VPSingleDefRecipe *Def) {
2298 hash_code Result = hash_combine(
2299 args: Def->getVPRecipeID(), args: vputils::getOpcodeOrIntrinsicID(V: Def),
2300 args: getGEPSourceElementType(R: Def), args: Def->getScalarType(),
2301 args: vputils::isSingleScalar(VPV: Def), args: hash_combine_range(R: Def->operands()));
2302 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: Def))
2303 if (RFlags->hasPredicate())
2304 return hash_combine(args: Result, args: RFlags->getPredicate());
2305 if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def))
2306 return hash_combine(args: Result, args: SIVSteps->getInductionOpcode());
2307 // Fold in the separately stored consecutive flag. Alignment is left out and
2308 // handled by cse.
2309 if (auto *Load = dyn_cast<VPWidenMemoryRecipe>(Val: Def))
2310 return hash_combine(args: Result, args: Load->isConsecutive());
2311 return Result;
2312 }
2313
2314 /// Check equality of underlying data of \p L and \p R.
2315 static bool isEqual(const VPSingleDefRecipe *L, const VPSingleDefRecipe *R) {
2316 if (L->getVPRecipeID() != R->getVPRecipeID() ||
2317 vputils::getOpcodeOrIntrinsicID(V: L) !=
2318 vputils::getOpcodeOrIntrinsicID(V: R) ||
2319 getGEPSourceElementType(R: L) != getGEPSourceElementType(R) ||
2320 vputils::isSingleScalar(VPV: L) != vputils::isSingleScalar(VPV: R) ||
2321 !equal(LRange: L->operands(), RRange: R->operands()))
2322 return false;
2323 assert(vputils::getOpcodeOrIntrinsicID(L) &&
2324 vputils::getOpcodeOrIntrinsicID(R) &&
2325 "must have valid opcode info for both recipes");
2326 if (auto *LFlags = dyn_cast<VPRecipeWithIRFlags>(Val: L))
2327 if (LFlags->hasPredicate() &&
2328 LFlags->getPredicate() !=
2329 cast<VPRecipeWithIRFlags>(Val: R)->getPredicate())
2330 return false;
2331 if (auto *LSIV = dyn_cast<VPScalarIVStepsRecipe>(Val: L))
2332 if (LSIV->getInductionOpcode() !=
2333 cast<VPScalarIVStepsRecipe>(Val: R)->getInductionOpcode())
2334 return false;
2335 // Compare the separately stored consecutive flag. Alignment is left out and
2336 // handled by cse.
2337 if (auto *LL = dyn_cast<VPWidenMemoryRecipe>(Val: L))
2338 if (LL->isConsecutive() != cast<VPWidenMemoryRecipe>(Val: R)->isConsecutive())
2339 return false;
2340 // Phi recipes can only be equal if they are in the same VPBB, as they
2341 // implicitly depend on their predecessors.
2342 if (isa<VPWidenPHIRecipe>(Val: L) && L->getParent() != R->getParent())
2343 return false;
2344 // Recipes in replicate regions implicitly depend on predicate. If either
2345 // recipe is in a replicate region, only consider them equal if both have
2346 // the same parent.
2347 const VPRegionBlock *RegionL = L->getRegion();
2348 const VPRegionBlock *RegionR = R->getRegion();
2349 if (((RegionL && RegionL->isReplicator()) ||
2350 (RegionR && RegionR->isReplicator())) &&
2351 L->getParent() != R->getParent())
2352 return false;
2353 return L->getScalarType() == R->getScalarType();
2354 }
2355};
2356} // end anonymous namespace
2357
2358/// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
2359/// Plan.
2360void VPlanTransforms::cse(VPlan &Plan) {
2361 VPDominatorTree VPDT(Plan);
2362 DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo> CSEMap;
2363 // CSE map for widened loads. Must be cleared on recipes that may write to
2364 // memory, and at the end of each VPBB.
2365 DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo>
2366 LoadCSEMap;
2367
2368 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
2369 Plan.getEntry());
2370 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
2371 for (VPRecipeBase &R : *VPBB) {
2372 if (R.mayWriteToMemory())
2373 LoadCSEMap.clear();
2374 auto *Def = dyn_cast<VPSingleDefRecipe>(Val: &R);
2375 if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
2376 continue;
2377 bool IsLoad = isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Val: Def);
2378 auto [It, Inserted] =
2379 (IsLoad ? LoadCSEMap : CSEMap).try_emplace(Key: Def, Args&: Def);
2380 if (Inserted)
2381 continue;
2382 VPSingleDefRecipe *V = It->second;
2383 // V must dominate Def for a valid replacement.
2384 if (!VPDT.dominates(A: V->getParent(), B: VPBB))
2385 continue;
2386 if (IsLoad) {
2387 auto *EarlierLoad = cast<VPWidenMemoryRecipe>(Val: V);
2388 auto *Load = cast<VPWidenMemoryRecipe>(Val: Def);
2389 if (EarlierLoad->getAlign() < Load->getAlign()) {
2390 // Record Load as the candidate for subsequent loads, as it may be
2391 // reusable where EarlierLoad is not.
2392 It->second = Def;
2393 continue;
2394 }
2395 // Keep only metadata common to both loads on the survivor.
2396 EarlierLoad->intersect(MD: *Load);
2397 }
2398 // Only keep flags present on both V and Def.
2399 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: V))
2400 RFlags->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Def));
2401 Def->replaceAllUsesWith(New: V);
2402 }
2403 LoadCSEMap.clear();
2404 }
2405}
2406
2407/// Return true if we do not know how to (mechanically) hoist or sink a
2408/// non-memory or memory recipe \p R out of a loop region. When sinking, passing
2409/// \p Sinking = true ensures that assumes aren't sunk.
2410static bool cannotHoistOrSinkRecipe(VPRecipeBase &R, VPBasicBlock *FirstBB,
2411 VPBasicBlock *LastBB,
2412 bool Sinking = false) {
2413 if (!isa<VPReplicateRecipe>(Val: R) || !R.mayReadOrWriteMemory() ||
2414 match(V: &R, P: m_Intrinsic<Intrinsic::assume>()))
2415 return vputils::cannotHoistOrSinkRecipe(R, Sinking);
2416
2417 // Check that the memory operation doesn't alias between FirstBB and LastBB.
2418 auto MemLoc = vputils::getMemoryLocation(R);
2419
2420 // TODO: Could make use of SinkStoreInfo::isNoAliasViaDistance by collecting
2421 // stores upfront, and constructing a full SinkStoreInfo.
2422 auto SinkInfo =
2423 Sinking ? std::make_optional(t: SinkStoreInfo(cast<VPReplicateRecipe>(Val&: R)))
2424 : std::nullopt;
2425
2426 return !MemLoc ||
2427 !canHoistOrSinkWithNoAliasCheck(MemLoc: *MemLoc, FirstBB, LastBB, SinkInfo);
2428}
2429
2430/// Move loop-invariant recipes out of the vector loop region in \p Plan.
2431static void licm(VPlan &Plan) {
2432 VPBasicBlock *Preheader = Plan.getVectorPreheader();
2433
2434 // Hoist any loop invariant recipes from the vector loop region to the
2435 // preheader. Preform a shallow traversal of the vector loop region, to
2436 // exclude recipes in replicate regions. Since the top-level blocks in the
2437 // vector loop region are guaranteed to execute if the vector pre-header is,
2438 // we don't need to check speculation safety.
2439 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2440 assert(Preheader->getSingleSuccessor() == LoopRegion &&
2441 "Expected vector prehader's successor to be the vector loop region");
2442 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2443 Range: vp_depth_first_shallow(G: LoopRegion->getEntry()))) {
2444 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2445 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2446 LastBB: LoopRegion->getExitingBasicBlock()))
2447 continue;
2448 if (any_of(Range: R.operands(), P: [](VPValue *Op) {
2449 return !Op->isDefinedOutsideLoopRegions();
2450 }))
2451 continue;
2452 R.moveBefore(BB&: *Preheader, I: Preheader->end());
2453 }
2454 }
2455
2456#ifndef NDEBUG
2457 VPDominatorTree VPDT(Plan);
2458#endif
2459 // Sink recipes with no users inside the vector loop region if all users are
2460 // in the same exit block of the region.
2461 // TODO: Extend to sink recipes from inner loops.
2462 PostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> POT(
2463 LoopRegion->getEntry());
2464 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
2465 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
2466 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2467 LastBB: LoopRegion->getExitingBasicBlock(),
2468 /*Sinking=*/true))
2469 continue;
2470
2471 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
2472 assert(!RepR->isPredicated() &&
2473 "Expected prior transformation of predicated replicates to "
2474 "replicate regions");
2475 // narrowToSingleScalarRecipes should have already maximally narrowed
2476 // replicates to single-scalar replicates.
2477 // TODO: When unrolling, replicateByVF doesn't handle sunk
2478 // non-single-scalar replicates correctly.
2479 if (!RepR->isSingleScalar())
2480 continue;
2481
2482 // The pointer operand of stores must be loop-invariant.
2483 if (RepR->getOpcode() == Instruction::Store &&
2484 !RepR->getOperand(N: 1)->isDefinedOutsideLoopRegions())
2485 continue;
2486 }
2487
2488 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
2489 assert((!R.mayWriteToMemory() ||
2490 (RepR && RepR->getOpcode() == Instruction::Store &&
2491 RepR->getOperand(1)->isDefinedOutsideLoopRegions())) &&
2492 "The only recipes that may write to memory are expected to be "
2493 "stores with invariant pointer-operand");
2494
2495 // TODO: Use R.definedValues() instead of casting to VPSingleDefRecipe to
2496 // support recipes with multiple defined values (e.g., interleaved loads).
2497 auto *Def = cast<VPSingleDefRecipe>(Val: &R);
2498
2499 // Cannot sink the recipe if the user is defined in a loop region or a
2500 // non-successor of the vector loop region. Cannot sink if user is a phi
2501 // either.
2502 VPBasicBlock *SinkBB = nullptr;
2503 if (any_of(Range: Def->users(), P: [&SinkBB, &LoopRegion](VPUser *U) {
2504 auto *UserR = cast<VPRecipeBase>(Val: U);
2505 VPBasicBlock *Parent = UserR->getParent();
2506 // TODO: Support sinking when users are in multiple blocks.
2507 if (SinkBB && SinkBB != Parent)
2508 return true;
2509 SinkBB = Parent;
2510 // TODO: If the user is a PHI node, we should check the block of
2511 // incoming value. Support PHI node users if needed.
2512 return UserR->isPhi() || Parent->getEnclosingLoopRegion() ||
2513 Parent->getSinglePredecessor() != LoopRegion;
2514 }))
2515 continue;
2516
2517 if (!SinkBB)
2518 SinkBB = cast<VPBasicBlock>(Val: LoopRegion->getSingleSuccessor());
2519
2520 // TODO: This will need to be a check instead of a assert after
2521 // conditional branches in vectorized loops are supported.
2522 assert(VPDT.properlyDominates(VPBB, SinkBB) &&
2523 "Defining block must dominate sink block");
2524 // TODO: Clone the recipe if users are on multiple exit paths, instead of
2525 // just moving.
2526 Def->moveBefore(BB&: *SinkBB, I: SinkBB->getFirstNonPhi());
2527 }
2528 }
2529}
2530
2531void VPlanTransforms::truncateToMinimalBitwidths(
2532 VPlan &Plan, const MapVector<Instruction *, uint64_t> &MinBWs) {
2533 if (Plan.hasScalarVFOnly())
2534 return;
2535 // Keep track of created truncates, so they can be re-used. Note that we
2536 // cannot use RAUW after creating a new truncate, as this would could make
2537 // other uses have different types for their operands, making them invalidly
2538 // typed.
2539 DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
2540 VPBasicBlock *PH = Plan.getVectorPreheader();
2541 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2542 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
2543 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2544 if (!isa<VPWidenRecipe, VPWidenCastRecipe, VPReplicateRecipe,
2545 VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R))
2546 continue;
2547
2548 VPValue *ResultVPV = R.getVPSingleValue();
2549 auto *UI = cast_or_null<Instruction>(Val: ResultVPV->getUnderlyingValue());
2550 unsigned NewResSizeInBits = MinBWs.lookup(Key: UI);
2551 if (!NewResSizeInBits)
2552 continue;
2553
2554 // If the value wasn't vectorized, we must maintain the original scalar
2555 // type. Skip those here, after incrementing NumProcessedRecipes. Also
2556 // skip casts which do not need to be handled explicitly here, as
2557 // redundant casts will be removed during recipe simplification.
2558 if (isa<VPReplicateRecipe, VPWidenCastRecipe>(Val: &R))
2559 continue;
2560
2561 Type *OldResTy = ResultVPV->getScalarType();
2562 unsigned OldResSizeInBits = OldResTy->getScalarSizeInBits();
2563 assert(OldResTy->isIntegerTy() && "only integer types supported");
2564 (void)OldResSizeInBits;
2565
2566 auto *NewResTy = IntegerType::get(C&: Plan.getContext(), NumBits: NewResSizeInBits);
2567
2568 // Any wrapping introduced by shrinking this operation shouldn't be
2569 // considered undefined behavior. So, we can't unconditionally copy
2570 // arithmetic wrapping flags to VPW.
2571 if (auto *VPW = dyn_cast<VPRecipeWithIRFlags>(Val: &R))
2572 VPW->dropPoisonGeneratingFlags();
2573
2574 assert((OldResSizeInBits != NewResSizeInBits ||
2575 match(&R, m_ICmp(m_VPValue(), m_VPValue()))) &&
2576 "Only ICmps should not need extending the result.");
2577 assert(!isa<VPWidenStoreRecipe>(&R) && "stores cannot be narrowed");
2578
2579 // Loads/intrinsics are not recreated; they keep producing their original
2580 // wide result and narrowed users will truncate it as needed below.
2581 if (isa<VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R))
2582 continue;
2583
2584 // Shrink operands by introducing truncates as needed.
2585 unsigned StartIdx =
2586 match(V: &R, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())) ? 1 : 0;
2587 SmallVector<VPValue *> NewOperands(R.operands());
2588 for (VPValue *&Op : drop_begin(RangeOrContainer&: NewOperands, N: StartIdx)) {
2589 unsigned OpSizeInBits = Op->getScalarType()->getScalarSizeInBits();
2590 if (OpSizeInBits == NewResSizeInBits)
2591 continue;
2592 assert(OpSizeInBits > NewResSizeInBits && "nothing to truncate");
2593 auto [ProcessedIter, Inserted] = ProcessedTruncs.try_emplace(Key: Op);
2594 if (Inserted) {
2595 VPBuilder Builder;
2596 if (isa<VPIRValue>(Val: Op))
2597 Builder.setInsertPoint(PH);
2598 else
2599 Builder.setInsertPoint(&R);
2600 ProcessedIter->second =
2601 Builder.createWidenCast(Opcode: Instruction::Trunc, Op, ResultTy: NewResTy);
2602 }
2603 Op = ProcessedIter->second;
2604 }
2605
2606 auto *NWR = cast<VPWidenRecipe>(Val: &R)->cloneWithOperands(NewOperands);
2607 NWR->insertBefore(InsertPos: &R);
2608
2609 // Wrap NWR in a ZExt to preserve the original wide type for downstream
2610 // users. Not needed for ICmps, whose result type is i1 irrespective of
2611 // the narrowing of their operands.
2612 VPValue *Replacement = NWR->getVPSingleValue();
2613 if (Replacement->getScalarType() != OldResTy)
2614 Replacement =
2615 VPBuilder::getToInsertAfter(R: NWR)
2616 .createWidenCast(Opcode: Instruction::ZExt, Op: Replacement, ResultTy: OldResTy)
2617 ->getVPSingleValue();
2618 ResultVPV->replaceAllUsesWith(New: Replacement);
2619 R.eraseFromParent();
2620 }
2621 }
2622}
2623
2624bool VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) {
2625 std::optional<VPDominatorTree> VPDT;
2626 if (OnlyLatches)
2627 VPDT.emplace(args&: Plan);
2628
2629 // Collect all blocks before modifying the CFG so we can identify unreachable
2630 // ones after constant branch removal.
2631 SmallVector<VPBlockBase *> AllBlocks(vp_depth_first_shallow(G: Plan.getEntry()));
2632
2633 bool SimplifiedPhi = false;
2634 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: AllBlocks)) {
2635 VPValue *Cond;
2636 // Skip blocks that are not terminated by BranchOnCond.
2637 if (VPBB->empty() || !match(V: &VPBB->back(), P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))))
2638 continue;
2639
2640 if (OnlyLatches && !VPBlockUtils::isLatch(VPB: VPBB, VPDT: *VPDT))
2641 continue;
2642
2643 assert(VPBB->getNumSuccessors() == 2 &&
2644 "Two successors expected for BranchOnCond");
2645 unsigned RemovedIdx;
2646 if (match(V: Cond, P: m_True()))
2647 RemovedIdx = 1;
2648 else if (match(V: Cond, P: m_False()))
2649 RemovedIdx = 0;
2650 else
2651 continue;
2652
2653 VPBasicBlock *RemovedSucc =
2654 cast<VPBasicBlock>(Val: VPBB->getSuccessors()[RemovedIdx]);
2655 assert(count(RemovedSucc->getPredecessors(), VPBB) == 1 &&
2656 "There must be a single edge between VPBB and its successor");
2657 // Values coming from VPBB into phi recipes of RemovedSucc are removed from
2658 // these recipes and single-entry header phis are removed.
2659 for (VPRecipeBase &R : make_early_inc_range(Range: RemovedSucc->phis())) {
2660 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: VPBB);
2661 SimplifiedPhi = true;
2662 // Remove now invalid header phis that are left single-entry after
2663 // removing their backedges.
2664 auto *PhiR = dyn_cast<VPHeaderPHIRecipe>(Val: &R);
2665 if (!PhiR || PhiR->getNumIncoming() != 1)
2666 continue;
2667 PhiR->replaceAllUsesWith(New: PhiR->getOperand(N: 0));
2668 PhiR->eraseFromParent();
2669 }
2670
2671 // Disconnect blocks and remove the terminator.
2672 VPBlockUtils::disconnectBlocks(From: VPBB, To: RemovedSucc);
2673 VPBB->back().eraseFromParent();
2674 }
2675
2676 // Compute which blocks are still reachable from the entry after constant
2677 // branch removal.
2678 SmallPtrSet<VPBlockBase *, 16> Reachable(
2679 llvm::from_range, vp_depth_first_shallow(G: Plan.getEntry()));
2680
2681 // Detach all unreachable blocks from their successors, removing their recipes
2682 // and incoming values from phi recipes.
2683 VPSymbolicValue Tmp(nullptr);
2684 for (VPBlockBase *B : AllBlocks) {
2685 if (Reachable.contains(Ptr: B))
2686 continue;
2687 for (VPBlockBase *Succ : to_vector(Range: B->successors())) {
2688 if (auto *SuccBB = dyn_cast<VPBasicBlock>(Val: Succ))
2689 for (VPRecipeBase &R : SuccBB->phis())
2690 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: B);
2691 VPBlockUtils::disconnectBlocks(From: B, To: Succ);
2692 }
2693 for (VPBasicBlock *DeadBB :
2694 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_deep(G: B))) {
2695 for (VPRecipeBase &R : make_early_inc_range(Range&: *DeadBB)) {
2696 for (VPValue *Def : R.definedValues())
2697 Def->replaceAllUsesWith(New: &Tmp);
2698 R.eraseFromParent();
2699 }
2700 }
2701 }
2702 return SimplifiedPhi;
2703}
2704
2705void VPlanTransforms::optimize(VPlan &Plan) {
2706 RUN_VPLAN_PASS(removeRedundantInductionCasts, Plan);
2707
2708 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2709 RUN_VPLAN_PASS(combineRecipes, Plan);
2710 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2711 RUN_VPLAN_PASS(simplifyBlends, Plan);
2712 RUN_VPLAN_PASS(legalizeAndOptimizeInductions, Plan);
2713 RUN_VPLAN_PASS(narrowToSingleScalarRecipes, Plan);
2714 RUN_VPLAN_PASS(removeRedundantExpandSCEVRecipes, Plan);
2715 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2716 RUN_VPLAN_PASS(combineRecipes, Plan);
2717 RUN_VPLAN_PASS(removeBranchOnConst, Plan, /*OnlyLatches=*/false);
2718 RUN_VPLAN_PASS(simplifyReverses, Plan);
2719 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2720
2721 RUN_VPLAN_PASS(createAndOptimizeReplicateRegions, Plan);
2722 RUN_VPLAN_PASS(mergeBlocksIntoPredecessors, Plan);
2723 RUN_VPLAN_PASS(licm, Plan);
2724}
2725
2726void VPlanTransforms::simplifyLiveInsWithSCEV(VPlan &Plan,
2727 PredicatedScalarEvolution &PSE) {
2728 auto GetSimplifiedLiveInViaSCEV = [&](VPValue *VPV) -> VPValue * {
2729 const SCEV *Expr = vputils::getSCEVExprForVPValue(V: VPV, PSE);
2730 const APInt *C;
2731 if (match(S: Expr, P: m_scev_APInt(C)))
2732 return Plan.getConstantInt(Val: *C);
2733 return nullptr;
2734 };
2735
2736 for (VPValue *LiveIn : to_vector(Range: Plan.getLiveIns())) {
2737 if (VPValue *SimplifiedLiveIn = GetSimplifiedLiveInViaSCEV(LiveIn))
2738 LiveIn->replaceAllUsesWith(New: SimplifiedLiveIn);
2739 }
2740}
2741
2742void VPlanTransforms::replaceSymbolicStrides(
2743 VPlan &Plan, PredicatedScalarEvolution &PSE,
2744 const SymbolicStrideMap &StridesMap, const VPDominatorTree &VPDT) {
2745 // Replace VPValues for known constant strides guaranteed by predicated scalar
2746 // evolution that are guaranteed to be guarded by the runtime checks; that is,
2747 // blocks dominated by the vector header.
2748 assert(!Plan.getVectorLoopRegion() &&
2749 "expected to run before loop regions are created");
2750 const auto &[Header, _] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
2751 auto CanUseVersionedStride = [&VPDT, Header = Header, &Plan](VPUser &U,
2752 unsigned Idx) {
2753 auto *R = cast<VPRecipeBase>(Val: &U);
2754 // Skip phis if the loop if loop is not yet guarded.
2755 if (isa<VPPhiAccessors>(Val: R) &&
2756 Header == Plan.getEntry()->getSingleSuccessor())
2757 return false;
2758 return VPDT.dominates(A: Header, B: R->getParent());
2759 };
2760 ValueToSCEVMapTy RewriteMap;
2761 for (const SCEVUnknown *Stride : StridesMap.values()) {
2762 Value *StrideV = Stride->getValue();
2763 const APInt *StrideConst;
2764 const SCEV *StrideExpr = PSE.getSCEV(V: StrideV);
2765 if (!match(S: StrideExpr, P: m_scev_APInt(C&: StrideConst)))
2766 // Only handle constant strides for now.
2767 continue;
2768 if (VPValue *StrideVPV = Plan.getLiveIn(V: StrideV))
2769 StrideVPV->replaceUsesWithIf(New: Plan.getConstantInt(Val: *StrideConst),
2770 ShouldReplace: CanUseVersionedStride);
2771
2772 // The versioned value may not be used in the loop directly but through an
2773 // integral cast (sext/zext/trunc). Add new live-ins in those cases.
2774 for (Value *U : StrideV->users()) {
2775 if (!isa<SExtInst, ZExtInst, TruncInst>(Val: U))
2776 continue;
2777 VPValue *StrideVPV = Plan.getLiveIn(V: U);
2778 if (!StrideVPV)
2779 continue;
2780 unsigned BW = U->getType()->getScalarSizeInBits();
2781 APInt C = isa<SExtInst>(Val: U) ? StrideConst->sext(width: BW)
2782 : StrideConst->zextOrTrunc(width: BW);
2783 StrideVPV->replaceUsesWithIf(New: Plan.getConstantInt(Val: C),
2784 ShouldReplace: CanUseVersionedStride);
2785 }
2786 RewriteMap[StrideV] = StrideExpr;
2787 }
2788
2789 for (VPExpandSCEVRecipe &ExpSCEV :
2790 make_isa_range<VPExpandSCEVRecipe>(Range&: *Plan.getEntry())) {
2791 const SCEV *ScevExpr = ExpSCEV.getSCEV();
2792 auto *NewSCEV =
2793 SCEVParameterRewriter::rewrite(Scev: ScevExpr, SE&: *PSE.getSE(), Map&: RewriteMap);
2794 if (NewSCEV != ScevExpr) {
2795 VPValue *NewExp = vputils::getOrCreateVPValueForSCEVExpr(Plan, Expr: NewSCEV);
2796 ExpSCEV.replaceAllUsesWith(New: NewExp);
2797 if (Plan.getTripCount() == &ExpSCEV)
2798 Plan.resetTripCount(NewTripCount: NewExp);
2799 }
2800 }
2801}
2802
2803void VPlanTransforms::dropPoisonGeneratingRecipes(VPlan &Plan) {
2804 // Collect recipes in the backward slice of `Root` that may generate a poison
2805 // value that is used after vectorization.
2806 SmallPtrSet<VPRecipeBase *, 16> Visited;
2807 auto CollectPoisonGeneratingInstrsInBackwardSlice([&](VPRecipeBase *Root) {
2808 SmallVector<VPRecipeBase *, 16> Worklist;
2809 Worklist.push_back(Elt: Root);
2810
2811 // Traverse the backward slice of Root through its use-def chain.
2812 while (!Worklist.empty()) {
2813 VPRecipeBase *CurRec = Worklist.pop_back_val();
2814
2815 if (!Visited.insert(Ptr: CurRec).second)
2816 continue;
2817
2818 // Prune search if we find another recipe generating a widen memory
2819 // instruction. Widen memory instructions involved in address computation
2820 // will lead to gather/scatter instructions, which don't need to be
2821 // handled.
2822 if (isa<VPWidenMemoryRecipe, VPInterleaveRecipe, VPScalarIVStepsRecipe,
2823 VPHeaderPHIRecipe>(Val: CurRec))
2824 continue;
2825
2826 // This recipe contributes to the address computation of a widen
2827 // load/store. If the underlying instruction has poison-generating flags,
2828 // drop them directly.
2829 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: CurRec)) {
2830 VPValue *A, *B;
2831 // Dropping disjoint from an OR may yield incorrect results, as some
2832 // analysis may have converted it to an Add implicitly (e.g. SCEV used
2833 // for dependence analysis). Instead, replace it with an equivalent Add.
2834 // This is possible as all users of the disjoint OR only access lanes
2835 // where the operands are disjoint or poison otherwise.
2836 if (match(V: RecWithFlags, P: m_BinaryOr(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))) &&
2837 RecWithFlags->isDisjoint()) {
2838 VPBuilder Builder(RecWithFlags);
2839 VPInstruction *New =
2840 Builder.createAdd(LHS: A, RHS: B, DL: RecWithFlags->getDebugLoc());
2841 New->setUnderlyingValue(RecWithFlags->getUnderlyingValue());
2842 RecWithFlags->replaceAllUsesWith(New);
2843 RecWithFlags->eraseFromParent();
2844 CurRec = New;
2845 } else
2846 RecWithFlags->dropPoisonGeneratingFlags();
2847 } else {
2848 Instruction *Instr = dyn_cast_or_null<Instruction>(
2849 Val: CurRec->getVPSingleValue()->getUnderlyingValue());
2850 (void)Instr;
2851 assert((!Instr || !Instr->hasPoisonGeneratingFlags()) &&
2852 "found instruction with poison generating flags not covered by "
2853 "VPRecipeWithIRFlags");
2854 }
2855
2856 // Add new definitions to the worklist.
2857 for (VPValue *Operand : CurRec->operands())
2858 if (VPRecipeBase *OpDef = Operand->getDefiningRecipe())
2859 Worklist.push_back(Elt: OpDef);
2860 }
2861 });
2862
2863 // We want to exclude the tail folding case, as we don't need to drop flags
2864 // for operations computing the first lane in this case: the first lane of the
2865 // header mask must always be true. For reverse memory accesses, the mask is
2866 // wrapped in a Reverse, which is just a permutation of the header mask, so
2867 // peel it off before checking. The header mask is still the abstract region
2868 // value at this point (materialization happens later).
2869 auto m_UnlessHdrMask = m_Unless( // NOLINT
2870 P: m_CombineOr(Ps: m_HeaderMask(), Ps: m_Reverse(Op0: m_HeaderMask())));
2871
2872 // Traverse all the recipes in the VPlan and collect the poison-generating
2873 // recipes in the backward slice starting at the address of a VPWidenRecipe or
2874 // VPInterleaveRecipe.
2875 auto Iter =
2876 vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntryBasicBlock());
2877 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: Iter)) {
2878 for (VPRecipeBase &Recipe : *VPBB) {
2879 if (auto *WidenRec = dyn_cast<VPWidenMemoryRecipe>(Val: &Recipe)) {
2880 VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe();
2881 if (AddrDef && WidenRec->isConsecutive() && WidenRec->getMask() &&
2882 match(V: WidenRec->getMask(), P: m_UnlessHdrMask))
2883 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2884 } else if (auto *InterleaveRec = dyn_cast<VPInterleaveRecipe>(Val: &Recipe)) {
2885 VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe();
2886 if (AddrDef && InterleaveRec->getMask() &&
2887 match(V: InterleaveRec->getMask(), P: m_UnlessHdrMask))
2888 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2889 }
2890 }
2891 }
2892}
2893
2894void VPlanTransforms::createInterleaveGroups(
2895 VPlan &Plan,
2896 const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
2897 &InterleaveGroups,
2898 const bool &EpilogueAllowed) {
2899 if (InterleaveGroups.empty())
2900 return;
2901
2902 DenseMap<Instruction *, VPWidenMemoryRecipe *> IRMemberToRecipe;
2903 for (VPBasicBlock *VPBB :
2904 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
2905 G: Plan.getVectorLoopRegion()->getEntryBasicBlock())))
2906 for (VPRecipeBase &R : make_filter_range(Range&: *VPBB, Pred: [](VPRecipeBase &R) {
2907 return isa<VPWidenMemoryRecipe>(Val: &R);
2908 })) {
2909 auto *MemR = cast<VPWidenMemoryRecipe>(Val: &R);
2910 IRMemberToRecipe[&MemR->getIngredient()] = MemR;
2911 }
2912
2913 // Interleave memory: for each Interleave Group we marked earlier as relevant
2914 // for this VPlan, replace the Recipes widening its memory instructions with a
2915 // single VPInterleaveRecipe at its insertion point.
2916 VPDominatorTree VPDT(Plan);
2917 for (const auto *IG : InterleaveGroups) {
2918 VPWidenMemoryRecipe *Start = nullptr;
2919 Instruction *StartMember = nullptr;
2920 for (auto *Member : IG->members())
2921 if (VPWidenMemoryRecipe *R = IRMemberToRecipe.lookup(Val: Member)) {
2922 StartMember = Member;
2923 Start = R;
2924 break;
2925 }
2926 if (!StartMember) // All member recipes are dead, so the group is dead.
2927 continue;
2928 VPIRMetadata InterleaveMD(*Start);
2929 SmallVector<VPValue *, 4> StoredValues;
2930 for (unsigned I = 0; I < IG->getFactor(); ++I) {
2931 Instruction *MemberI = IG->getMember(Index: I);
2932 if (!MemberI)
2933 continue;
2934 if (VPWidenMemoryRecipe *MemoryR = IRMemberToRecipe.lookup(Val: MemberI)) {
2935 if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Val: MemoryR->getAsRecipe()))
2936 StoredValues.push_back(Elt: StoreR->getStoredValue());
2937 InterleaveMD.intersect(MD: *MemoryR);
2938 } else {
2939 InterleaveMD.intersect(MD: VPIRMetadata(*MemberI));
2940 }
2941 }
2942
2943 bool NeedsMaskForGaps =
2944 (IG->requiresScalarEpilogue() && !EpilogueAllowed) ||
2945 (!StoredValues.empty() && !IG->isFull());
2946
2947 Instruction *IRInsertPos = IG->getInsertPos();
2948 auto *InsertPos = IRMemberToRecipe.lookup(Val: IRInsertPos);
2949 if (!InsertPos) {
2950 // InsertPos member is dead: find a new member that is alive.
2951 assert(isa<VPWidenLoadRecipe>(Start->getAsRecipe()) &&
2952 "Dead member in non-load group?");
2953 InsertPos = Start;
2954 for (Instruction *Member : IG->members())
2955 if (VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member))
2956 if (VPDT.properlyDominates(A: MemberR->getAsRecipe(),
2957 B: InsertPos->getAsRecipe()))
2958 InsertPos = MemberR;
2959 IRInsertPos = &InsertPos->getIngredient();
2960 }
2961 VPRecipeBase *InsertPosR = InsertPos->getAsRecipe();
2962
2963 GEPNoWrapFlags NW = GEPNoWrapFlags::none();
2964 if (auto *Gep = dyn_cast<GetElementPtrInst>(
2965 Val: getLoadStorePointerOperand(V: IRInsertPos)->stripPointerCasts()))
2966 NW = Gep->getNoWrapFlags().withoutNoUnsignedWrap();
2967
2968 // Get or create the start address for the interleave group.
2969 VPValue *Addr = Start->getAddr();
2970 VPRecipeBase *AddrDef = Addr->getDefiningRecipe();
2971 if (IG->getIndex(Instr: StartMember) != 0 ||
2972 (AddrDef && !VPDT.properlyDominates(A: AddrDef, B: InsertPosR))) {
2973 // Either member zero's recipe is dead, or we cannot re-use the address of
2974 // member zero because it does not dominate the insert position. Instead,
2975 // use the address of the insert position and create a PtrAdd adjusting it
2976 // to the address of member zero.
2977 // TODO: Hoist Addr's defining recipe (and any operands as needed) to
2978 // InsertPos or sink loads above zero members to join it.
2979 assert(IG->getIndex(IRInsertPos) != 0 &&
2980 "index of insert position shouldn't be zero");
2981 auto &DL = IRInsertPos->getDataLayout();
2982 APInt Offset(32,
2983 DL.getTypeAllocSize(Ty: getLoadStoreType(I: IRInsertPos)) *
2984 IG->getIndex(Instr: IRInsertPos),
2985 /*IsSigned=*/true);
2986 VPValue *OffsetVPV = Plan.getConstantInt(Val: -Offset);
2987 VPBuilder B(InsertPosR);
2988 Addr = B.createNoWrapPtrAdd(Ptr: InsertPos->getAddr(), Offset: OffsetVPV, GEPFlags: NW);
2989 }
2990 // If the group is reverse, adjust the index to refer to the last vector
2991 // lane instead of the first. We adjust the index from the first vector
2992 // lane, rather than directly getting the pointer for lane VF - 1, because
2993 // the pointer operand of the interleaved access is supposed to be uniform.
2994 if (IG->isReverse()) {
2995 auto *ReversePtr = new VPVectorEndPointerRecipe(
2996 Addr, &Plan.getVF(), getLoadStoreType(I: IRInsertPos),
2997 -(int64_t)IG->getFactor(), NW, InsertPosR->getDebugLoc());
2998 ReversePtr->insertBefore(InsertPos: InsertPosR);
2999 Addr = ReversePtr;
3000 }
3001 auto *VPIG = new VPInterleaveRecipe(
3002 IG, Addr, StoredValues, InsertPos->getMask(), NeedsMaskForGaps,
3003 InterleaveMD, InsertPosR->getDebugLoc());
3004 VPIG->insertBefore(InsertPos: InsertPosR);
3005
3006 unsigned J = 0;
3007 for (unsigned i = 0; i < IG->getFactor(); ++i)
3008 if (Instruction *Member = IG->getMember(Index: i)) {
3009 VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member);
3010 if (!Member->getType()->isVoidTy()) {
3011 if (MemberR) {
3012 VPValue *OriginalV = MemberR->getAsRecipe()->getVPSingleValue();
3013 OriginalV->replaceAllUsesWith(New: VPIG->getVPValue(I: J));
3014 }
3015 J++;
3016 }
3017 if (MemberR)
3018 MemberR->getAsRecipe()->eraseFromParent();
3019 }
3020 }
3021}
3022
3023/// Returns the VPValue representing the uncountable exit comparison used by
3024/// AnyOf if the recipes it depends on can be traced back to live-ins and
3025/// the addresses (in GEP/PtrAdd form) of any (non-masked) load used in
3026/// generating the values for the comparison. The recipes are stored in
3027/// \p Recipes.
3028static VPValue *
3029getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
3030 VPBasicBlock *LatchVPBB) {
3031 // Given a plain CFG VPlan loop with countable latch exiting block
3032 // \p LatchVPBB, we're looking to match the recipes contributing to the
3033 // uncountable exit condition comparison (here, vp<%4>) back to either
3034 // live-ins or the address nodes for the load used as part of the uncountable
3035 // exit comparison so that we can either move them within the loop, or copy
3036 // them to the preheader depending on the chosen method for dealing with
3037 // stores in uncountable exit loops.
3038 //
3039 // Currently, the address of the load is restricted to a GEP with 2 operands
3040 // and a live-in base address. This constraint may be relaxed later.
3041 //
3042 // VPlan ' for UF>=1' {
3043 // Live-in vp<%0> = VF * UF
3044 // Live-in vp<%1> = vector-trip-count
3045 // Live-in ir<20> = original trip-count
3046 //
3047 // ir-bb<entry>:
3048 // Successor(s): scalar.ph, vector.ph
3049 //
3050 // vector.ph:
3051 // Successor(s): for.body
3052 //
3053 // for.body:
3054 // EMIT vp<%2> = phi ir<0>, vp<%index.next>
3055 // EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, for.inc ]
3056 // EMIT ir<%uncountable.addr> = getelementptr inbounds nuw ir<%pred>,ir<%iv>
3057 // EMIT ir<%uncountable.val> = load ir<%uncountable.addr>
3058 // EMIT ir<%uncountable.cond> = icmp sgt ir<%uncountable.val>, ir<500>
3059 // EMIT vp<%3> = masked-cond ir<%uncountable.cond>
3060 // Successor(s): for.inc
3061 //
3062 // for.inc:
3063 // EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
3064 // EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
3065 // EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
3066 // EMIT vp<%4> = any-of ir<%3>
3067 // EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
3068 // EMIT branch-on-two-conds vp<%4>, vp<%5>
3069 // Successor(s): middle.block, middle.block, for.body
3070 //
3071 // middle.block:
3072 // Successor(s): ir-bb<exit>, scalar.ph
3073 //
3074 // ir-bb<exit>:
3075 // No successors
3076 //
3077 // scalar.ph:
3078 // }
3079
3080 // Find the uncountable loop exit condition.
3081 VPValue *UncountableCondition = nullptr;
3082 if (!match(V: LatchVPBB->getTerminator(),
3083 P: m_BranchOnTwoConds(Op0: m_AnyOf(Op0: m_VPValue(V&: UncountableCondition)),
3084 Op1: m_VPValue())))
3085 return nullptr;
3086
3087 SmallVector<VPValue *, 4> Worklist;
3088 Worklist.push_back(Elt: UncountableCondition);
3089 while (!Worklist.empty()) {
3090 VPValue *V = Worklist.pop_back_val();
3091
3092 // Any value defined outside the loop does not need to be copied.
3093 if (V->isDefinedOutsideLoopRegions())
3094 continue;
3095
3096 // FIXME: Remove the single user restriction; it's here because we're
3097 // starting with the simplest set of loops we can, and multiple
3098 // users means needing to add PHI nodes in the transform.
3099 if (V->getNumUsers() > 1)
3100 return nullptr;
3101
3102 VPValue *Op1, *Op2;
3103 // Walk back through recipes until we find at least one load from memory.
3104 if (match(V, P: m_ICmp(Op0: m_VPValue(V&: Op1), Op1: m_VPValue(V&: Op2)))) {
3105 Worklist.push_back(Elt: Op1);
3106 Worklist.push_back(Elt: Op2);
3107 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3108 } else if (match(V, P: m_VPInstruction<Instruction::Load>(Ops: m_VPValue(V&: Op1)))) {
3109 VPRecipeBase *GepR = Op1->getDefiningRecipe();
3110 // Only matching base + single offset term for now.
3111 if (GepR->getNumOperands() != 2)
3112 return nullptr;
3113 // Matching a GEP with a loop-invariant base ptr.
3114 if (!match(V: GepR, P: m_VPInstruction<Instruction::GetElementPtr>(
3115 Ops: m_LiveIn(), Ops: m_VPValue())))
3116 return nullptr;
3117 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3118 Recipes.push_back(Elt: cast<VPInstruction>(Val: GepR));
3119 } else if (match(V, P: m_VPInstruction<VPInstruction::MaskedCond>(
3120 Ops: m_VPValue(V&: Op1)))) {
3121 Worklist.push_back(Elt: Op1);
3122 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3123 } else
3124 return nullptr;
3125 }
3126
3127 // If we couldn't match anything, don't return the condition. It may be
3128 // defined outside the loop.
3129 if (Recipes.empty() ||
3130 none_of(Range&: Recipes, P: match_fn(P: m_VPInstruction<Instruction::GetElementPtr>())))
3131 return nullptr;
3132
3133 return UncountableCondition;
3134}
3135
3136struct EarlyExitInfo {
3137 VPBasicBlock *EarlyExitingVPBB;
3138 VPIRBasicBlock *EarlyExitVPBB;
3139 VPValue *CondToExit;
3140};
3141
3142/// Update \p Plan to mask memory operations in the loop based on whether the
3143/// early exit is taken or not.
3144///
3145/// We're currently expecting to find a loop with properties similar to the
3146/// following:
3147///
3148/// for.body:
3149/// ir<%indvars.iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<%0>
3150/// EMIT ir<%arrayidx> = getelementptr inbounds nuw ir<@c>, ir<%indvars.iv>
3151/// EMIT-SCALAR ir<%0> = load ir<%arrayidx>
3152/// EMIT ir<%cmp1> = icmp sgt ir<%0>, ir<5>
3153/// EMIT vp<%1> = masked-cond ir<%cmp1>
3154/// Successor(s): if.end
3155///
3156/// if.end:
3157/// EMIT ir<%arrayidx3> = getelementptr inbounds nuw ir<@src>, ir<%indvars.iv>
3158/// EMIT-SCALAR ir<%2> = load ir<%arrayidx3>
3159/// EMIT ir<%add> = add nsw ir<%2>, ir<42>
3160/// EMIT ir<%arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<%indvars.iv>
3161/// EMIT store ir<%add>, ir<%arrayidx5>
3162/// EMIT ir<%indvars.iv.next> = add nuw nsw ir<%indvars.iv>, ir<1>
3163/// EMIT vp<%3> = any-of ir<%1>
3164/// EMIT ir<%exitcond.not> = icmp eq ir<%indvars.iv.next>, ir<10000>
3165/// EMIT branch-on-two-conds vp<%3>, ir<%exitcond.not>
3166/// Successor(s): middle.block, middle.block, for.body
3167///
3168/// We currently expect LoopVectorizationLegality to ensure that:
3169/// * There must also be a counted exit. We will need to support speculative
3170/// or first-faulting loads before we can remove this restriction.
3171/// * Any stores within the loop must not alias with the load used for the
3172/// uncountable exit. We can relax this a bit with runtime aliasing checks.
3173/// * Other memory operations in the loop can take place before or after the
3174/// uncountable exit, but must also be unconditional. We need to support
3175/// combining the conditions in VPlanPredicator.
3176/// * The loop must have a single unconditional load contributing to the
3177/// uncountable exit comparison, and the other term must be loop-invariant.
3178/// Improving upon this requires work in getRecipesForUncountableExit to
3179/// handle more complex recipe graphs.
3180static bool handleUncountableExitsWithSideEffects(
3181 VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
3182 VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB,
3183 Loop *TheLoop, PredicatedScalarEvolution &PSE, DominatorTree &DT,
3184 AssumptionCache *AC) {
3185
3186 // Disconnect early exiting blocks from successors, remove branches. We
3187 // currently don't support multiple uses for recipes involved in creating
3188 // the uncountable exit condition.
3189 for (auto &Exit : Exits) {
3190 if (Exit.EarlyExitingVPBB == LatchVPBB)
3191 continue;
3192
3193 for (VPRecipeBase &R : Exit.EarlyExitVPBB->phis())
3194 cast<VPIRPhi>(Val: &R)->removeIncomingValueFor(IncomingBlock: Exit.EarlyExitingVPBB);
3195 Exit.EarlyExitingVPBB->getTerminator()->eraseFromParent();
3196 VPBlockUtils::disconnectBlocks(From: Exit.EarlyExitingVPBB, To: Exit.EarlyExitVPBB);
3197 }
3198
3199 VPDominatorTree VPDT(Plan);
3200
3201 // We can abandon a VPlan entirely if we return false here, so we shouldn't
3202 // crash if some earlier assumptions on scalar IR don't hold for the vplan
3203 // version of the loop.
3204 SmallVector<VPInstruction *, 8> ConditionRecipes;
3205
3206 VPValue *Cond = getRecipesForUncountableExit(Recipes&: ConditionRecipes, LatchVPBB);
3207 if (!Cond)
3208 return false;
3209
3210 // Find load contributing to condition.
3211 // At the moment LoopVectorizationLegality only supports a single
3212 // early-exit expression with a compare and a single load that must
3213 // be unconditional.
3214 // TODO: Support more than one load.
3215 auto *Load =
3216 find_singleton<VPInstruction>(Range&: ConditionRecipes, P: [](auto *I, bool _) {
3217 return match(I, m_VPInstruction<Instruction::Load>(Ops: m_VPValue()))
3218 ? I
3219 : nullptr;
3220 });
3221 assert(Load && "Couldn't find exactly one load");
3222 // TODO: Support conditional loads for uncountable exits.
3223 assert(VPDT.dominates(Load->getParent(), LatchVPBB) &&
3224 "Uncountable exit condition load is conditional.");
3225 VPInstruction *Ptr = cast<VPInstruction>(Val: Load->getOperand(N: 0));
3226
3227 // Ensure that we are guaranteed to be able to dereference the memory used
3228 // for determining the uncountable exit for the maximum possible number of
3229 // scalar iterations of the loop.
3230 //
3231 // TODO: Support first-faulting loads in cases where we don't know whether
3232 // all possible addresses are dereferenceable.
3233 {
3234 SmallVector<const SCEVPredicate *, 4> Predicates;
3235 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: TheLoop);
3236 const DataLayout &DL = Plan.getDataLayout();
3237 APInt EltSize(DL.getIndexTypeSizeInBits(Ty: Ptr->getScalarType()),
3238 DL.getTypeStoreSize(Ty: Load->getScalarType()).getFixedValue());
3239 if (!isDereferenceableAndAlignedInLoop(
3240 PtrSCEV, Alignment: cast<LoadInst>(Val: Load->getUnderlyingInstr())->getAlign(),
3241 EltSizeSCEV: PSE.getSE()->getConstant(Val: EltSize), L: TheLoop, SE&: *PSE.getSE(), DT, AC,
3242 Predicates: &Predicates))
3243 return false;
3244 }
3245
3246 // Check for a single GEP for the condition load to see if we can link it to
3247 // a widen IV recipe with a step of 1; we're only interested in contiguous
3248 // accesses for the condition load right now.
3249 auto *IV = cast<VPWidenInductionRecipe>(Val: &HeaderVPBB->front());
3250 if (!match(V: IV->getStartValue(), P: m_SpecificInt(V: 0)) ||
3251 !match(V: IV->getStepValue(), P: m_SpecificInt(V: 1)))
3252 return false;
3253 if (!match(V: Ptr, P: m_VPInstruction<Instruction::GetElementPtr>(Ops: m_LiveIn(),
3254 Ops: m_Specific(VPV: IV))))
3255 return false;
3256
3257 // We want to guarantee that the uncountable exit condition (and the mask
3258 // we will generate from it) are available for all operations in the loop
3259 // that need to be masked. If the condition recipes are not already the first
3260 // recipes in the header after the last phi, move them there.
3261 auto InsertIt = HeaderVPBB->getFirstNonPhi();
3262 while (InsertIt != HeaderVPBB->end() &&
3263 is_contained(Range&: ConditionRecipes, Element: &*InsertIt)) {
3264 erase(C&: ConditionRecipes, V: &*InsertIt);
3265 InsertIt++;
3266 }
3267 for (auto *Recipe : reverse(C&: ConditionRecipes))
3268 Recipe->moveBefore(BB&: *HeaderVPBB, I: InsertIt);
3269
3270 // Create a mask to represent all lanes that fully execute in the vector loop,
3271 // stopping short of any early exit.
3272 VPBuilder MaskBuilder(HeaderVPBB, InsertIt);
3273 VPValue *FirstActive = MaskBuilder.createFirstActiveLane(Masks: Cond);
3274 Type *IVScalarTy = IV->getScalarType();
3275 VPValue *Zero = Plan.getZero(Ty: IVScalarTy);
3276 FirstActive =
3277 MaskBuilder.createScalarZExtOrTrunc(Op: FirstActive, ResultTy: IVScalarTy, DL: DebugLoc());
3278 VPValue *Mask = MaskBuilder.createNaryOp(Opcode: VPInstruction::ActiveLaneMask,
3279 Operands: {Zero, FirstActive}, DL: DebugLoc(),
3280 Name: "uncountable.exit.mask");
3281
3282 // Convert all other memory operations to use the mask.
3283 for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(Header: HeaderVPBB))
3284 for (VPRecipeBase &R : *VPBB)
3285 if (R.mayReadOrWriteMemory() && &R != Load) {
3286 // TODO: Handle conditional memory operations in the loop.
3287 if (!VPDT.dominates(A: R.getParent(), B: LatchVPBB))
3288 return false;
3289 cast<VPInstruction>(Val: &R)->addMask(Mask);
3290 }
3291
3292 // Update middle block branch to compare (IV + however many lanes were active)
3293 // against the full trip count, since we may be exiting the vector loop early.
3294 // If we didn't take an early exit, we should get the equivalent of VF from
3295 // the FirstActiveLane.
3296 assert(match(MiddleVPBB->getTerminator(), m_BranchOnCond()) &&
3297 "Expected BranchOnCond terminator for MiddleVPBB");
3298 VPBuilder MiddleBuilder(MiddleVPBB->getTerminator());
3299 VPValue *ScalarIV = MiddleBuilder.createNaryOp(Opcode: VPInstruction::ExtractLane,
3300 Operands: {Zero, IV}, DL: DebugLoc());
3301 VPValue *ExitIV = MiddleBuilder.createAdd(LHS: ScalarIV, RHS: FirstActive);
3302 VPValue *FullTC =
3303 MiddleBuilder.createICmp(Pred: CmpInst::ICMP_EQ, A: ExitIV, B: Plan.getTripCount());
3304 MiddleVPBB->getTerminator()->setOperand(I: 0, New: FullTC);
3305
3306 // Update resume phi in scalar.ph.
3307 VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
3308 auto Phis = ScalarPH->phis();
3309 // TODO: Handle more than one Phi; re-derive from IV.
3310 // TODO: Handle reductions.
3311 if (range_size(Range&: Phis) != 1)
3312 return false;
3313 VPPhi *ContinueIV = cast<VPPhi>(Val: Phis.begin());
3314 // Make sure we're referring to the same IV.
3315 assert(
3316 match(ContinueIV->getOperand(0),
3317 m_VPInstruction<VPInstruction::ExitingIVValue>(m_Specific(IV))) &&
3318 "Continuing from different IV");
3319 ContinueIV->setOperand(I: 0, New: ExitIV);
3320 return true;
3321}
3322
3323bool VPlanTransforms::handleUncountableEarlyExits(
3324 VPlan &Plan, Loop *TheLoop, PredicatedScalarEvolution &PSE,
3325 DominatorTree &DT, AssumptionCache *AC, UncountableExitStyle Style) {
3326#ifndef NDEBUG
3327 VPDominatorTree VPDT(Plan);
3328#endif
3329
3330 auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
3331 auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
3332
3333 // Dereferenceability is checked separately for uncountable exit loops with
3334 // stores, as only the loads contributing to the exit condition need to
3335 // be checked.
3336 if (Style == UncountableExitStyle::ReadOnly &&
3337 !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC))
3338 return false;
3339
3340 VPBuilder LatchBuilder(LatchVPBB->getTerminator());
3341 SmallVector<EarlyExitInfo> Exits;
3342 for (auto [EarlyExitingVPBB, ExitBlock] :
3343 vputils::getEarlyExits(Plan, MiddleVPBB)) {
3344 // Collect condition for this early exit.
3345 VPBlockBase *TrueSucc = EarlyExitingVPBB->getSuccessors()[0];
3346 VPValue *CondOfEarlyExitingVPBB;
3347 [[maybe_unused]] bool Matched =
3348 match(V: EarlyExitingVPBB->getTerminator(),
3349 P: m_BranchOnCond(Op0: m_VPValue(V&: CondOfEarlyExitingVPBB)));
3350 assert(Matched && "Terminator must be BranchOnCond");
3351
3352 // Insert the MaskedCond in the EarlyExitingVPBB so the predicator adds
3353 // the correct block mask.
3354 VPBuilder EarlyExitingBuilder(EarlyExitingVPBB->getTerminator());
3355 auto *CondToEarlyExit = EarlyExitingBuilder.createNaryOp(
3356 Opcode: VPInstruction::MaskedCond,
3357 Operands: TrueSucc == ExitBlock
3358 ? CondOfEarlyExitingVPBB
3359 : EarlyExitingBuilder.createNot(Operand: CondOfEarlyExitingVPBB));
3360 assert((isa<VPIRValue>(CondOfEarlyExitingVPBB) ||
3361 !VPDT.properlyDominates(EarlyExitingVPBB, LatchVPBB) ||
3362 VPDT.properlyDominates(
3363 CondOfEarlyExitingVPBB->getDefiningRecipe()->getParent(),
3364 LatchVPBB)) &&
3365 "exit condition must dominate the latch");
3366 Exits.push_back(Elt: {
3367 .EarlyExitingVPBB: EarlyExitingVPBB,
3368 .EarlyExitVPBB: ExitBlock,
3369 .CondToExit: CondToEarlyExit,
3370 });
3371 }
3372
3373 assert(!Exits.empty() && "must have at least one early exit");
3374 // Sort exits by RPO order to get correct program order. RPO gives a
3375 // topological ordering of the CFG, ensuring upstream exits are checked
3376 // before downstream exits in the dispatch chain.
3377 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
3378 HeaderVPBB);
3379 DenseMap<VPBlockBase *, unsigned> RPOIdx;
3380 for (const auto &[Num, VPB] : enumerate(First&: RPOT))
3381 RPOIdx[VPB] = Num;
3382 llvm::sort(C&: Exits, Comp: [&RPOIdx](const EarlyExitInfo &A, const EarlyExitInfo &B) {
3383 return RPOIdx[A.EarlyExitingVPBB] < RPOIdx[B.EarlyExitingVPBB];
3384 });
3385#ifndef NDEBUG
3386 // After RPO sorting, verify that for any pair where one exit dominates
3387 // another, the dominating exit comes first. This is guaranteed by RPO
3388 // (topological order) and is required for the dispatch chain correctness.
3389 for (unsigned I = 0; I + 1 < Exits.size(); ++I)
3390 for (unsigned J = I + 1; J < Exits.size(); ++J)
3391 assert(!VPDT.properlyDominates(Exits[J].EarlyExitingVPBB,
3392 Exits[I].EarlyExitingVPBB) &&
3393 "RPO sort must place dominating exits before dominated ones");
3394#endif
3395
3396 // Build the AnyOf condition for the latch terminator using logical OR
3397 // to avoid poison propagation from later exit conditions when an earlier
3398 // exit is taken.
3399 VPValue *Combined = Exits[0].CondToExit;
3400 for (const EarlyExitInfo &Info : drop_begin(RangeOrContainer&: Exits))
3401 Combined = LatchBuilder.createLogicalOr(LHS: Combined, RHS: Info.CondToExit);
3402
3403 VPValue *IsAnyExitTaken =
3404 LatchBuilder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: {Combined});
3405
3406 // Create a comparison for the latch exit condition and replace the
3407 // BranchOnCond with a BranchOnTwoConds. The original BranchOnCond's condition
3408 // is used as the latch-exit condition; canonical IV recipes have not been
3409 // introduced yet, so there is no BranchOnCount to derive the condition from.
3410 auto *LatchExitingBranch = cast<VPInstruction>(Val: LatchVPBB->getTerminator());
3411 assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCond &&
3412 "Unexpected terminator");
3413 VPValue *IsLatchExitTaken = LatchExitingBranch->getOperand(N: 0);
3414 DebugLoc LatchDL = LatchExitingBranch->getDebugLoc();
3415 LatchExitingBranch->eraseFromParent();
3416 LatchBuilder.setInsertPoint(LatchVPBB);
3417 LatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnTwoConds,
3418 Operands: {IsAnyExitTaken, IsLatchExitTaken}, DL: LatchDL);
3419 LatchVPBB->clearSuccessors();
3420
3421 if (Style == UncountableExitStyle::MaskedHandleExitInScalarLoop) {
3422 // If handling the exiting lane in the scalar loop, combine the exit
3423 // conditions into a single BranchOnCond.
3424 LatchVPBB->setSuccessors({MiddleVPBB, MiddleVPBB, HeaderVPBB});
3425 MiddleVPBB->clearPredecessors();
3426 MiddleVPBB->setPredecessors({LatchVPBB, LatchVPBB});
3427 return handleUncountableExitsWithSideEffects(
3428 Plan, Exits, HeaderVPBB, LatchVPBB, MiddleVPBB, TheLoop, PSE, DT, AC);
3429 }
3430
3431 // Create the vector.early.exit blocks.
3432 SmallVector<VPBasicBlock *> VectorEarlyExitVPBBs(Exits.size());
3433 for (unsigned Idx = 0; Idx != Exits.size(); ++Idx) {
3434 Twine BlockSuffix = Exits.size() == 1 ? "" : Twine(".") + Twine(Idx);
3435 VPBasicBlock *VectorEarlyExitVPBB =
3436 Plan.createVPBasicBlock(Name: "vector.early.exit" + BlockSuffix);
3437 VectorEarlyExitVPBBs[Idx] = VectorEarlyExitVPBB;
3438 }
3439
3440 // Create the dispatch block (or reuse the single exit block if only one
3441 // exit). The dispatch block computes the first active lane of the combined
3442 // condition and, for multiple exits, chains through conditions to determine
3443 // which exit to take.
3444 VPBasicBlock *DispatchVPBB =
3445 Exits.size() == 1 ? VectorEarlyExitVPBBs[0]
3446 : Plan.createVPBasicBlock(Name: "vector.early.exit.check");
3447 DispatchVPBB->setPredecessors({LatchVPBB});
3448 LatchVPBB->setSuccessors({DispatchVPBB, MiddleVPBB, HeaderVPBB});
3449 VPBuilder DispatchBuilder(DispatchVPBB, DispatchVPBB->begin());
3450 VPValue *FirstActiveLane = DispatchBuilder.createFirstActiveLane(
3451 Masks: {Combined}, DL: DebugLoc::getUnknown(), Name: "first.active.lane");
3452
3453 // For each early exit, disconnect the original exiting block
3454 // (early.exiting.I) from the exit block (ir-bb<exit.I>) and route through a
3455 // new vector.early.exit block. Update ir-bb<exit.I>'s phis to extract their
3456 // values at the first active lane:
3457 //
3458 // Input:
3459 // early.exiting.I:
3460 // ...
3461 // EMIT branch-on-cond vp<%cond.I>
3462 // Successor(s): in.loop.succ, ir-bb<exit.I>
3463 //
3464 // ir-bb<exit.I>:
3465 // IR %phi = phi [ vp<%incoming.I>, early.exiting.I ], ...
3466 //
3467 // Output:
3468 // early.exiting.I:
3469 // ...
3470 // Successor(s): in.loop.succ
3471 //
3472 // vector.early.exit.I:
3473 // EMIT vp<%exit.val> = extract-lane vp<%first.lane>, vp<%incoming.I>
3474 // Successor(s): ir-bb<exit.I>
3475 //
3476 // ir-bb<exit.I>:
3477 // IR %phi = phi ... (extra operand: vp<%exit.val> from
3478 // vector.early.exit.I)
3479 //
3480 for (auto [Exit, VectorEarlyExitVPBB] :
3481 zip_equal(t&: Exits, u&: VectorEarlyExitVPBBs)) {
3482 auto &[EarlyExitingVPBB, EarlyExitVPBB, _] = Exit;
3483 // Adjust the phi nodes in EarlyExitVPBB.
3484 // 1. remove incoming values from EarlyExitingVPBB,
3485 // 2. extract the incoming value at FirstActiveLane
3486 // 3. add back the extracts as last operands for the phis
3487 // Then adjust the CFG, removing the edge between EarlyExitingVPBB and
3488 // EarlyExitVPBB and adding a new edge between VectorEarlyExitVPBB and
3489 // EarlyExitVPBB. The extracts at FirstActiveLane are now the incoming
3490 // values from VectorEarlyExitVPBB.
3491 for (VPRecipeBase &R : EarlyExitVPBB->phis()) {
3492 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
3493 VPValue *IncomingVal =
3494 ExitIRI->getIncomingValueForBlock(VPBB: EarlyExitingVPBB);
3495 VPValue *NewIncoming = IncomingVal;
3496 if (!isa<VPIRValue>(Val: IncomingVal)) {
3497 VPBuilder EarlyExitBuilder(VectorEarlyExitVPBB);
3498 NewIncoming = EarlyExitBuilder.createNaryOp(
3499 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, IncomingVal},
3500 DL: DebugLoc::getUnknown(), Name: "early.exit.value");
3501 }
3502 ExitIRI->removeIncomingValueFor(IncomingBlock: EarlyExitingVPBB);
3503 ExitIRI->addIncoming(IncomingV: NewIncoming);
3504 }
3505
3506 EarlyExitingVPBB->getTerminator()->eraseFromParent();
3507 VPBlockUtils::disconnectBlocks(From: EarlyExitingVPBB, To: EarlyExitVPBB);
3508 VPBlockUtils::connectBlocks(From: VectorEarlyExitVPBB, To: EarlyExitVPBB);
3509 }
3510
3511 // Chain through exits: for each exit, check if its condition is true at
3512 // the first active lane. If so, take that exit; otherwise, try the next.
3513 // The last exit needs no check since it must be taken if all others fail.
3514 //
3515 // For 3 exits (cond.0, cond.1, cond.2), this creates:
3516 //
3517 // latch:
3518 // ...
3519 // EMIT vp<%combined> = logical-or vp<%cond.0>, vp<%cond.1>, vp<%cond.2>
3520 // ...
3521 //
3522 // vector.early.exit.check:
3523 // EMIT vp<%first.lane> = first-active-lane vp<%combined>
3524 // EMIT vp<%at.cond.0> = extract-lane vp<%first.lane>, vp<%cond.0>
3525 // EMIT branch-on-cond vp<%at.cond.0>
3526 // Successor(s): vector.early.exit.0, vector.early.exit.check.0
3527 //
3528 // vector.early.exit.check.0:
3529 // EMIT vp<%at.cond.1> = extract-lane vp<%first.lane>, vp<%cond.1>
3530 // EMIT branch-on-cond vp<%at.cond.1>
3531 // Successor(s): vector.early.exit.1, vector.early.exit.2
3532 VPBasicBlock *CurrentBB = DispatchVPBB;
3533 for (auto [I, Exit] : enumerate(First: ArrayRef(Exits).drop_back())) {
3534 VPValue *LaneVal = DispatchBuilder.createNaryOp(
3535 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, Exit.CondToExit},
3536 DL: DebugLoc::getUnknown(), Name: "exit.cond.at.lane");
3537
3538 // For the last dispatch, branch directly to the last exit on false;
3539 // otherwise, create a new check block.
3540 bool IsLastDispatch = (I + 2 == Exits.size());
3541 VPBasicBlock *FalseBB =
3542 IsLastDispatch ? VectorEarlyExitVPBBs.back()
3543 : Plan.createVPBasicBlock(
3544 Name: Twine("vector.early.exit.check.") + Twine(I));
3545
3546 DispatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {LaneVal});
3547 CurrentBB->setSuccessors({VectorEarlyExitVPBBs[I], FalseBB});
3548 VectorEarlyExitVPBBs[I]->setPredecessors({CurrentBB});
3549 FalseBB->setPredecessors({CurrentBB});
3550
3551 CurrentBB = FalseBB;
3552 DispatchBuilder.setInsertPoint(CurrentBB);
3553 }
3554
3555 return true;
3556}
3557
3558/// This function tries convert extended in-loop reductions to
3559/// VPExpressionRecipe and clamp the \p Range if it is beneficial and
3560/// valid. The created recipe must be decomposed to its constituent
3561/// recipes before execution.
3562static VPExpressionRecipe *
3563tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
3564 VFRange &Range) {
3565 Type *RedTy = Red->getScalarType();
3566 VPValue *VecOp = Red->getVecOp();
3567
3568 // We don't handle partial reductions here.
3569 if (Red->isPartialReduction())
3570 return nullptr;
3571
3572 // Clamp the range if using extended-reduction is profitable.
3573 auto IsExtendedRedValidAndClampRange =
3574 [&](unsigned Opcode, Instruction::CastOps ExtOpc, Type *SrcTy) -> bool {
3575 return LoopVectorizationPlanner::getDecisionAndClampRange(
3576 Predicate: [&](ElementCount VF) {
3577 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
3578 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
3579
3580 InstructionCost ExtRedCost = InstructionCost::getInvalid();
3581 InstructionCost ExtCost =
3582 cast<VPWidenCastRecipe>(Val: VecOp)->computeCost(VF, Ctx);
3583 InstructionCost RedCost = Red->computeCost(VF, Ctx);
3584
3585 assert(!RedTy->isFloatingPointTy() &&
3586 "getExtendedReductionCost only supports integer types");
3587 ExtRedCost = Ctx.TTI.getExtendedReductionCost(
3588 Opcode, IsUnsigned: ExtOpc == Instruction::CastOps::ZExt, ResTy: RedTy, Ty: SrcVecTy,
3589 FMF: Red->getFastMathFlagsOrNone(), CostKind);
3590 return ExtRedCost.isValid() && ExtRedCost < ExtCost + RedCost;
3591 },
3592 Range);
3593 };
3594
3595 VPValue *A;
3596 // Match reduce(ext)).
3597 if (match(V: VecOp, P: m_Isa<VPWidenCastRecipe>(P: m_ZExtOrSExt(Op0: m_VPValue(V&: A)))) &&
3598 IsExtendedRedValidAndClampRange(
3599 RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind()),
3600 cast<VPWidenCastRecipe>(Val: VecOp)->getOpcode(), A->getScalarType()))
3601 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
3602
3603 return nullptr;
3604}
3605
3606/// This function tries convert extended in-loop reductions to
3607/// VPExpressionRecipe and clamp the \p Range if it is beneficial
3608/// and valid. The created VPExpressionRecipe must be decomposed to its
3609/// constituent recipes before execution. Patterns of the
3610/// VPExpressionRecipe:
3611/// reduce.add(mul(...)),
3612/// reduce.add(mul(ext(A), ext(B))),
3613/// reduce.add(ext(mul(ext(A), ext(B)))).
3614/// reduce.fadd(fmul(ext(A), ext(B)))
3615static VPExpressionRecipe *
3616tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
3617 VPCostContext &Ctx, VFRange &Range) {
3618 unsigned Opcode = RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind());
3619 if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
3620 Opcode != Instruction::FAdd)
3621 return nullptr;
3622
3623 // We don't handle partial reductions here.
3624 if (Red->isPartialReduction())
3625 return nullptr;
3626
3627 Type *RedTy = Red->getScalarType();
3628
3629 // Clamp the range if using multiply-accumulate-reduction is profitable.
3630 auto IsMulAccValidAndClampRange =
3631 [&](VPWidenRecipe *Mul, VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
3632 VPWidenCastRecipe *OuterExt) -> bool {
3633 return LoopVectorizationPlanner::getDecisionAndClampRange(
3634 Predicate: [&](ElementCount VF) {
3635 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
3636 Type *SrcTy = Ext0 ? Ext0->getOperand(N: 0)->getScalarType() : RedTy;
3637 InstructionCost MulAccCost;
3638
3639 // getMulAccReductionCost for in-loop reductions does not support
3640 // mixed or floating-point extends.
3641 if (Ext0 && Ext1 &&
3642 (Ext0->getOpcode() != Ext1->getOpcode() ||
3643 Ext0->getOpcode() == Instruction::CastOps::FPExt))
3644 return false;
3645
3646 bool IsZExt =
3647 !Ext0 || Ext0->getOpcode() == Instruction::CastOps::ZExt;
3648 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
3649 MulAccCost = Ctx.TTI.getMulAccReductionCost(IsUnsigned: IsZExt, RedOpcode: Opcode, ResTy: RedTy,
3650 Ty: SrcVecTy, CostKind);
3651
3652 InstructionCost MulCost = Mul->computeCost(VF, Ctx);
3653 InstructionCost RedCost = Red->computeCost(VF, Ctx);
3654 InstructionCost ExtCost = 0;
3655 if (Ext0)
3656 ExtCost += Ext0->computeCost(VF, Ctx);
3657 if (Ext1)
3658 ExtCost += Ext1->computeCost(VF, Ctx);
3659 if (OuterExt)
3660 ExtCost += OuterExt->computeCost(VF, Ctx);
3661
3662 return MulAccCost.isValid() &&
3663 MulAccCost < ExtCost + MulCost + RedCost;
3664 },
3665 Range);
3666 };
3667
3668 VPValue *VecOp = Red->getVecOp();
3669 VPRecipeBase *Sub = nullptr;
3670 VPValue *A, *B;
3671 VPValue *Tmp = nullptr;
3672
3673 if (RedTy->isFloatingPointTy())
3674 return nullptr;
3675
3676 // Sub reductions could have a sub between the add reduction and vec op.
3677 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Tmp)))) {
3678 Sub = VecOp->getDefiningRecipe();
3679 VecOp = Tmp;
3680 }
3681
3682 // If ValB is a constant and can be safely extended, truncate it to the same
3683 // type as ExtA's operand, then extend it to the same type as ExtA. This
3684 // creates two uniform extends that can more easily be matched by the rest of
3685 // the bundling code. The ExtB reference, ValB and operand 1 of Mul are all
3686 // replaced with the new extend of the constant.
3687 auto ExtendAndReplaceConstantOp = [](VPWidenCastRecipe *ExtA,
3688 VPWidenCastRecipe *&ExtB, VPValue *&ValB,
3689 VPWidenRecipe *Mul) {
3690 if (!ExtA || ExtB || !isa<VPIRValue>(Val: ValB))
3691 return;
3692 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
3693 Instruction::CastOps ExtOpc = ExtA->getOpcode();
3694 const APInt *Const;
3695 if (!match(V: ValB, P: m_APInt(C&: Const)) ||
3696 !llvm::canConstantBeExtended(
3697 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
3698 return;
3699 // The truncate ensures that the type of each extended operand is the
3700 // same, and it's been proven that the constant can be extended from
3701 // NarrowTy safely. Necessary since ExtA's extended operand would be
3702 // e.g. an i8, while the const will likely be an i32. This will be
3703 // elided by later optimisations.
3704 VPBuilder Builder(Mul);
3705 auto *Trunc =
3706 Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc, Op: ValB, ResultTy: NarrowTy);
3707 Type *WideTy = ExtA->getScalarType();
3708 ValB = ExtB = Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy);
3709 Mul->setOperand(I: 1, New: ExtB);
3710 };
3711
3712 // Try to match reduce.add(mul(...)).
3713 if (match(V: VecOp, P: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B)))) {
3714 auto *RecipeA = dyn_cast<VPWidenCastRecipe>(Val: A);
3715 auto *RecipeB = dyn_cast<VPWidenCastRecipe>(Val: B);
3716 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
3717
3718 // Convert reduce.add(mul(ext, const)) to reduce.add(mul(ext, ext(const)))
3719 ExtendAndReplaceConstantOp(RecipeA, RecipeB, B, Mul);
3720
3721 // Match reduce.add/sub(mul(ext, ext)).
3722 if (RecipeA && RecipeB && match(V: RecipeA, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
3723 match(V: RecipeB, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
3724 IsMulAccValidAndClampRange(Mul, RecipeA, RecipeB, nullptr)) {
3725 if (Sub)
3726 return new VPExpressionRecipe(RecipeA, RecipeB, Mul,
3727 cast<VPWidenRecipe>(Val: Sub), Red);
3728 return new VPExpressionRecipe(RecipeA, RecipeB, Mul, Red);
3729 }
3730 // TODO: Add an expression type for this variant with a negated mul
3731 if (!Sub && IsMulAccValidAndClampRange(Mul, nullptr, nullptr, nullptr))
3732 return new VPExpressionRecipe(Mul, Red);
3733 }
3734 // TODO: Add an expression type for negated versions of other expression
3735 // variants.
3736 if (Sub)
3737 return nullptr;
3738
3739 // Match reduce.add(ext(mul(A, B))).
3740 if (match(V: VecOp, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))))) {
3741 auto *Ext = cast<VPWidenCastRecipe>(Val: VecOp);
3742 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
3743 auto *Ext0 = dyn_cast<VPWidenCastRecipe>(Val: A);
3744 auto *Ext1 = dyn_cast<VPWidenCastRecipe>(Val: B);
3745
3746 // reduce.add(ext(mul(ext, const)))
3747 // -> reduce.add(ext(mul(ext, ext(const))))
3748 ExtendAndReplaceConstantOp(Ext0, Ext1, B, Mul);
3749
3750 // reduce.add(ext(mul(ext(A), ext(B))))
3751 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
3752 // The inner extends must either have the same opcode as the outer extend or
3753 // be the same, in which case the multiply can never result in a negative
3754 // value and the outer extend can be folded away by doing wider
3755 // extends for the operands of the mul.
3756 if (Ext0 && Ext1 &&
3757 (Ext->getOpcode() == Ext0->getOpcode() || Ext0 == Ext1) &&
3758 Ext0->getOpcode() == Ext1->getOpcode() &&
3759 IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext) && Mul->hasOneUse()) {
3760 auto *NewExt0 = new VPWidenCastRecipe(
3761 Ext0->getOpcode(), Ext0->getOperand(N: 0), Ext->getScalarType(), nullptr,
3762 *Ext0, *Ext0, Ext0->getDebugLoc());
3763 NewExt0->insertBefore(InsertPos: Ext0);
3764
3765 VPWidenCastRecipe *NewExt1 = NewExt0;
3766 if (Ext0 != Ext1) {
3767 NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(N: 0),
3768 Ext->getScalarType(), nullptr, *Ext1,
3769 *Ext1, Ext1->getDebugLoc());
3770 NewExt1->insertBefore(InsertPos: Ext1);
3771 }
3772 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewExt0, NewExt1});
3773 NewMul->insertBefore(InsertPos: Mul);
3774 Ext->replaceAllUsesWith(New: NewMul);
3775 Ext->eraseFromParent();
3776 Mul->eraseFromParent();
3777 return new VPExpressionRecipe(NewExt0, NewExt1, NewMul, Red);
3778 }
3779 }
3780 return nullptr;
3781}
3782
3783/// This function tries to create abstract recipes from the reduction recipe for
3784/// following optimizations and cost estimation.
3785static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
3786 VPCostContext &Ctx,
3787 VFRange &Range) {
3788 // Creation of VPExpressions for partial reductions is entirely handled in
3789 // transformToPartialReduction.
3790 if (Red->isPartialReduction())
3791 return;
3792
3793 VPExpressionRecipe *AbstractR = nullptr;
3794 auto IP = std::next(x: Red->getIterator());
3795 auto *VPBB = Red->getParent();
3796 if (auto *MulAcc = tryToMatchAndCreateMulAccumulateReduction(Red, Ctx, Range))
3797 AbstractR = MulAcc;
3798 else if (auto *ExtRed = tryToMatchAndCreateExtendedReduction(Red, Ctx, Range))
3799 AbstractR = ExtRed;
3800 // Cannot create abstract inloop reduction recipes.
3801 if (!AbstractR)
3802 return;
3803
3804 AbstractR->insertBefore(BB&: *VPBB, IP);
3805 Red->replaceAllUsesWith(New: AbstractR);
3806}
3807
3808void VPlanTransforms::convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
3809 VFRange &Range) {
3810 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
3811 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
3812 for (VPReductionRecipe &Red :
3813 make_early_inc_range(Range: make_isa_range<VPReductionRecipe>(Range&: *VPBB)))
3814 tryToCreateAbstractReductionRecipe(Red: &Red, Ctx, Range);
3815 }
3816}
3817
3818// Collect common metadata from a group of replicate recipes by intersecting
3819// metadata from all recipes in the group.
3820static VPIRMetadata getCommonMetadata(ArrayRef<VPReplicateRecipe *> Recipes) {
3821 VPIRMetadata CommonMetadata = *Recipes.front();
3822 for (VPReplicateRecipe *Recipe : drop_begin(RangeOrContainer&: Recipes))
3823 CommonMetadata.intersect(MD: *Recipe);
3824 // The recipe using the common metadata is not predicated, so it does not
3825 // share the group's execution frequency.
3826 CommonMetadata.clearExecutionFrequency();
3827 return CommonMetadata;
3828}
3829
3830template <unsigned Opcode>
3831static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
3832collectComplementaryPredicatedMemOps(VPlan &Plan,
3833 PredicatedScalarEvolution &PSE,
3834 const Loop *L) {
3835 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
3836 "Only Load and Store opcodes supported");
3837 [[maybe_unused]] constexpr bool IsLoad = (Opcode == Instruction::Load);
3838
3839 // For each address, collect operations with the same or complementary masks.
3840 SmallVector<SmallVector<VPReplicateRecipe *, 4>> AllGroups;
3841 auto Groups = collectGroupedReplicateMemOps<Opcode>(
3842 Plan, PSE, L,
3843 [](VPReplicateRecipe *RepR) { return RepR->isPredicated(); });
3844 for (auto Recipes : Groups) {
3845 if (Recipes.size() < 2)
3846 continue;
3847
3848 assert(all_equal(
3849 map_range(Recipes, bind_back<getLoadStoreValueType>(IsLoad))) &&
3850 "Expected all recipes in group to have the same load-store type");
3851
3852 // Collect groups with the same or complementary masks.
3853 for (VPReplicateRecipe *&RecipeI : Recipes) {
3854 if (!RecipeI)
3855 continue;
3856
3857 VPValue *MaskI = RecipeI->getMask();
3858 SmallVector<VPReplicateRecipe *, 4> Group;
3859 Group.push_back(Elt: RecipeI);
3860 RecipeI = nullptr;
3861
3862 // Find all operations with the same or complementary masks.
3863 bool HasComplementaryMask = false;
3864 for (VPReplicateRecipe *&RecipeJ : Recipes) {
3865 if (!RecipeJ)
3866 continue;
3867
3868 VPValue *MaskJ = RecipeJ->getMask();
3869 // Check if any operation in the group has a complementary mask with
3870 // another, that is M1 == NOT(M2) or M2 == NOT(M1).
3871 HasComplementaryMask |= match(V: MaskI, P: m_Not(Op0: m_Specific(VPV: MaskJ))) ||
3872 match(V: MaskJ, P: m_Not(Op0: m_Specific(VPV: MaskI)));
3873 Group.push_back(Elt: RecipeJ);
3874 RecipeJ = nullptr;
3875 }
3876
3877 if (HasComplementaryMask) {
3878 assert(Group.size() >= 2 && "must have at least 2 entries");
3879 AllGroups.push_back(Elt: std::move(Group));
3880 }
3881 }
3882 }
3883
3884 return AllGroups;
3885}
3886
3887// Find the recipe with minimum alignment in the group.
3888template <typename InstType>
3889static VPReplicateRecipe *
3890findRecipeWithMinAlign(ArrayRef<VPReplicateRecipe *> Group) {
3891 return *min_element(Group, [](VPReplicateRecipe *A, VPReplicateRecipe *B) {
3892 return cast<InstType>(A->getUnderlyingInstr())->getAlign() <
3893 cast<InstType>(B->getUnderlyingInstr())->getAlign();
3894 });
3895}
3896
3897void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
3898 PredicatedScalarEvolution &PSE,
3899 const Loop *L) {
3900 auto Groups =
3901 collectComplementaryPredicatedMemOps<Instruction::Load>(Plan, PSE, L);
3902 if (Groups.empty())
3903 return;
3904
3905 // Process each group of loads.
3906 for (auto &Group : Groups) {
3907 // Try to use the earliest (most dominating) load to replace all others.
3908 VPReplicateRecipe *EarliestLoad = Group[0];
3909 VPBasicBlock *FirstBB = EarliestLoad->getParent();
3910 VPBasicBlock *LastBB = Group.back()->getParent();
3911
3912 // Check that the load doesn't alias with stores between first and last.
3913 auto LoadLoc = vputils::getMemoryLocation(R: *EarliestLoad);
3914 if (!LoadLoc || !canHoistOrSinkWithNoAliasCheck(MemLoc: *LoadLoc, FirstBB, LastBB))
3915 continue;
3916
3917 // Collect common metadata from all loads in the group.
3918 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
3919
3920 // Find the load with minimum alignment to use.
3921 auto *LoadWithMinAlign = findRecipeWithMinAlign<LoadInst>(Group);
3922
3923 bool IsSingleScalar = EarliestLoad->isSingleScalar();
3924 assert(all_of(Group,
3925 [IsSingleScalar](VPReplicateRecipe *R) {
3926 return R->isSingleScalar() == IsSingleScalar;
3927 }) &&
3928 "all members in group must agree on IsSingleScalar");
3929
3930 // Create an unpredicated version of the earliest load with common
3931 // metadata.
3932 auto *UnpredicatedLoad = new VPReplicateRecipe(
3933 LoadWithMinAlign->getUnderlyingInstr(), {EarliestLoad->getOperand(N: 0)},
3934 IsSingleScalar, /*Mask=*/nullptr, *EarliestLoad, CommonMetadata);
3935
3936 UnpredicatedLoad->insertBefore(InsertPos: EarliestLoad);
3937
3938 // Replace all loads in the group with the unpredicated load.
3939 for (VPReplicateRecipe *Load : Group) {
3940 Load->replaceAllUsesWith(New: UnpredicatedLoad);
3941 Load->eraseFromParent();
3942 }
3943 }
3944}
3945
3946static bool
3947canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
3948 PredicatedScalarEvolution &PSE, const Loop &L) {
3949 auto StoreLoc = vputils::getMemoryLocation(R: *StoresToSink.front());
3950 if (!StoreLoc || !StoreLoc->AATags.Scope)
3951 return false;
3952
3953 // When sinking a group of stores, all members of the group alias each other.
3954 // Skip them during the alias checks.
3955 VPBasicBlock *FirstBB = StoresToSink.front()->getParent();
3956 VPBasicBlock *LastBB = StoresToSink.back()->getParent();
3957 SinkStoreInfo SinkInfo(StoresToSink, *StoresToSink[0], PSE, L);
3958 return canHoistOrSinkWithNoAliasCheck(MemLoc: *StoreLoc, FirstBB, LastBB, SinkInfo);
3959}
3960
3961void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
3962 PredicatedScalarEvolution &PSE,
3963 const Loop *L) {
3964 auto Groups =
3965 collectComplementaryPredicatedMemOps<Instruction::Store>(Plan, PSE, L);
3966 if (Groups.empty())
3967 return;
3968
3969 for (auto &Group : Groups) {
3970 if (!canSinkStoreWithNoAliasCheck(StoresToSink: Group, PSE, L: *L))
3971 continue;
3972
3973 // Use the last (most dominated) store's location for the unconditional
3974 // store.
3975 VPReplicateRecipe *LastStore = Group.back();
3976 VPBasicBlock *InsertBB = LastStore->getParent();
3977
3978 // Collect common alias metadata from all stores in the group.
3979 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
3980
3981 // Build select chain for stored values.
3982 VPValue *SelectedValue = Group[0]->getOperand(N: 0);
3983 VPBuilder Builder(InsertBB, LastStore->getIterator());
3984
3985 bool IsSingleScalar = Group[0]->isSingleScalar();
3986 for (unsigned I = 1; I < Group.size(); ++I) {
3987 assert(IsSingleScalar == Group[I]->isSingleScalar() &&
3988 "all members in group must agree on IsSingleScalar");
3989 VPValue *Mask = Group[I]->getMask();
3990 VPValue *Value = Group[I]->getOperand(N: 0);
3991 SelectedValue = Builder.createSelect(
3992 Cond: Mask, TrueVal: Value, FalseVal: SelectedValue, DL: Group[I]->getDebugLoc(), Name: "",
3993 Flags: VPIRFlags::getDefaultFlags(Opcode: Instruction::Select,
3994 ResultTy: Value->getScalarType()));
3995 }
3996
3997 // Find the store with minimum alignment to use.
3998 auto *StoreWithMinAlign = findRecipeWithMinAlign<StoreInst>(Group);
3999
4000 // Create unconditional store with selected value and common metadata.
4001 auto *UnpredicatedStore = new VPReplicateRecipe(
4002 StoreWithMinAlign->getUnderlyingInstr(),
4003 {SelectedValue, LastStore->getOperand(N: 1)}, IsSingleScalar,
4004 /*Mask=*/nullptr, *LastStore, CommonMetadata);
4005 UnpredicatedStore->insertBefore(BB&: *InsertBB, IP: LastStore->getIterator());
4006
4007 // Remove all predicated stores from the group.
4008 for (VPReplicateRecipe *Store : Group)
4009 Store->eraseFromParent();
4010 }
4011}
4012
4013/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
4014/// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
4015/// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
4016/// the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to
4017/// an index-independent load if it feeds all wide ops at all indices (\p OpV
4018/// must be the operand at index \p OpIdx for both the recipe at lane 0, \p
4019/// WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if \p V
4020/// is defined at \p Idx of a load interleave group.
4021/// A live-in or recipe defined outside the loop region can be converted, if it
4022/// is the same across all lanes, or we can create a BuildVector for it.
4023static bool canNarrowLoad(VPSingleDefRecipe *WideMember0, unsigned OpIdx,
4024 VPValue *OpV, unsigned Idx, bool IsScalable) {
4025 VPValue *Member0Op = WideMember0->getOperand(N: OpIdx);
4026 if (Member0Op->isDefinedOutsideLoopRegions()) {
4027 // Operand matches Member0, broadcast across all fields for both live-ins
4028 // and recipes.
4029 if (Member0Op == OpV)
4030 return true;
4031 // Otherwise distinct per-field VPValues are assembled into a BuildVector.
4032 return !IsScalable && OpV->isDefinedOutsideLoopRegions() &&
4033 OpV->getScalarType() == Member0Op->getScalarType();
4034 }
4035 VPRecipeBase *Member0OpR = Member0Op->getDefiningRecipe();
4036 if (auto *W = dyn_cast<VPWidenLoadRecipe>(Val: Member0OpR))
4037 // For scalable VFs, the narrowed plan processes vscale iterations at once,
4038 // so a shared wide load cannot be narrowed to a uniform scalar; bail out.
4039 return !IsScalable && !W->getMask() && W->isConsecutive() &&
4040 Member0Op == OpV;
4041 if (auto *IR = dyn_cast<VPInterleaveRecipe>(Val: Member0OpR))
4042 return IR->getInterleaveGroup()->isFull() && IR->getVPValue(I: Idx) == OpV;
4043 return false;
4044}
4045
4046static bool canNarrowOps(ArrayRef<VPValue *> Ops, bool IsScalable) {
4047 SmallVector<VPValue *> Ops0;
4048 auto *WideMember0 = dyn_cast<VPRecipeWithIRFlags>(Val: Ops[0]);
4049 if (!WideMember0)
4050 return false;
4051 for (VPValue *V : Ops) {
4052 if (!isa<VPWidenRecipe, VPWidenCastRecipe>(Val: V))
4053 return false;
4054 auto *R = cast<VPRecipeWithIRFlags>(Val: V);
4055 if (vputils::getOpcode(V: R) != vputils::getOpcode(V: WideMember0))
4056 return false;
4057 if (R->getScalarType() != WideMember0->getScalarType())
4058 return false;
4059 if (R->hasPredicate() && R->getPredicate() != WideMember0->getPredicate())
4060 return false;
4061 }
4062
4063 for (unsigned Idx = 0; Idx != WideMember0->getNumOperands(); ++Idx) {
4064 SmallVector<VPValue *> OpsI;
4065 for (VPValue *Op : Ops)
4066 OpsI.push_back(Elt: Op->getDefiningRecipe()->getOperand(N: Idx));
4067
4068 if (canNarrowOps(Ops: OpsI, IsScalable))
4069 continue;
4070
4071 if (any_of(Range: enumerate(First&: OpsI), P: [WideMember0, Idx, IsScalable](const auto &P) {
4072 const auto &[OpIdx, OpV] = P;
4073 return !canNarrowLoad(WideMember0, Idx, OpV, OpIdx, IsScalable);
4074 }))
4075 return false;
4076 }
4077
4078 return true;
4079}
4080
4081/// Returns VF from \p VFs if \p IR is a full interleave group with factor and
4082/// number of members both equal to VF. The interleave group must also access
4083/// the full vector width.
4084static std::optional<ElementCount>
4085isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR,
4086 ArrayRef<ElementCount> VFs,
4087 const TargetTransformInfo &TTI) {
4088 if (!InterleaveR || InterleaveR->getMask())
4089 return std::nullopt;
4090
4091 Type *GroupElementTy = nullptr;
4092 if (InterleaveR->getStoredValues().empty()) {
4093 GroupElementTy = InterleaveR->getVPValue(I: 0)->getScalarType();
4094 if (!all_of(Range: InterleaveR->definedValues(), P: [GroupElementTy](VPValue *Op) {
4095 return Op->getScalarType() == GroupElementTy;
4096 }))
4097 return std::nullopt;
4098 } else {
4099 GroupElementTy = InterleaveR->getStoredValues()[0]->getScalarType();
4100 if (!all_of(Range: InterleaveR->getStoredValues(), P: [GroupElementTy](VPValue *Op) {
4101 return Op->getScalarType() == GroupElementTy;
4102 }))
4103 return std::nullopt;
4104 }
4105
4106 auto IG = InterleaveR->getInterleaveGroup();
4107 if (IG->getFactor() != IG->getNumMembers())
4108 return std::nullopt;
4109
4110 auto GetVectorBitWidthForVF = [&TTI](ElementCount VF) {
4111 TypeSize Size = TTI.getRegisterBitWidth(
4112 K: VF.isFixed() ? TargetTransformInfo::RGK_FixedWidthVector
4113 : TargetTransformInfo::RGK_ScalableVector);
4114 assert(Size.isScalable() == VF.isScalable() &&
4115 "if Size is scalable, VF must be scalable and vice versa");
4116 return Size.getKnownMinValue();
4117 };
4118
4119 for (ElementCount VF : VFs) {
4120 unsigned MinVal = VF.getKnownMinValue();
4121 unsigned GroupSize = GroupElementTy->getScalarSizeInBits() * MinVal;
4122 if (IG->getFactor() == MinVal && GroupSize == GetVectorBitWidthForVF(VF))
4123 return {VF};
4124 }
4125 return std::nullopt;
4126}
4127
4128/// Returns true if \p VPValue is a narrow VPValue.
4129static bool isAlreadyNarrow(VPValue *VPV) {
4130 if (isa<VPIRValue>(Val: VPV))
4131 return true;
4132 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: VPV);
4133 return RepR && RepR->isSingleScalar();
4134}
4135
4136// Convert the wide recipes defining the VPValues in \p Members feeding an
4137// interleave group to a single narrow variant. The first member is reused as
4138// the narrowed recipe. BuildVectors for live-in operands are inserted into \p
4139// Preheader.
4140static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
4141 SmallPtrSetImpl<VPValue *> &NarrowedOps,
4142 VPBasicBlock *Preheader) {
4143 VPValue *V = Members.front();
4144 if (NarrowedOps.contains(Ptr: V))
4145 return V;
4146
4147 if (V->isDefinedOutsideLoopRegions()) {
4148 assert(all_of(Members,
4149 [V](VPValue *M) {
4150 return M->isDefinedOutsideLoopRegions() &&
4151 M->getScalarType() == V->getScalarType();
4152 }) &&
4153 "expected distinct loop-invariant values of matching scalar type");
4154 auto *BV = new VPInstruction(VPInstruction::BuildVector, Members);
4155 Preheader->appendRecipe(Recipe: BV);
4156 NarrowedOps.insert(Ptr: BV);
4157 return BV;
4158 }
4159
4160 if (isAlreadyNarrow(VPV: V))
4161 return V;
4162
4163 VPRecipeBase *R = V->getDefiningRecipe();
4164 if (isa<VPWidenRecipe, VPWidenCastRecipe>(Val: R)) {
4165 auto *WideMember0 = cast<VPRecipeWithIRFlags>(Val: R);
4166 for (VPValue *Member : Members.drop_front())
4167 WideMember0->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Member));
4168 for (unsigned Idx = 0, E = WideMember0->getNumOperands(); Idx != E; ++Idx) {
4169 SmallVector<VPValue *> OpsI;
4170 for (VPValue *Member : Members)
4171 OpsI.push_back(Elt: Member->getDefiningRecipe()->getOperand(N: Idx));
4172 WideMember0->setOperand(
4173 I: Idx, New: narrowInterleaveGroupOp(Members: OpsI, NarrowedOps, Preheader));
4174 }
4175 return V;
4176 }
4177
4178 if (auto *LoadGroup = dyn_cast<VPInterleaveRecipe>(Val: R)) {
4179 // Narrow interleave group to wide load, as transformed VPlan will only
4180 // process one original iteration.
4181 auto *LI = cast<LoadInst>(Val: LoadGroup->getInterleaveGroup()->getInsertPos());
4182 auto *L = VPBuilder(LoadGroup).createWidenLoad(
4183 Load&: *LI, Addr: LoadGroup->getAddr(), Mask: LoadGroup->getMask(), /*Consecutive=*/true,
4184 Metadata: *LoadGroup, DL: LoadGroup->getDebugLoc());
4185 NarrowedOps.insert(Ptr: L);
4186 return L;
4187 }
4188
4189 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R)) {
4190 assert(RepR->isSingleScalar() && RepR->getOpcode() == Instruction::Load &&
4191 "must be a single scalar load");
4192 NarrowedOps.insert(Ptr: RepR);
4193 return RepR;
4194 }
4195
4196 auto *WideLoad = cast<VPWidenLoadRecipe>(Val: R);
4197 VPValue *PtrOp = WideLoad->getAddr();
4198 if (auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(Val: PtrOp))
4199 PtrOp = VecPtr->getOperand(N: 0);
4200 // Narrow wide load to uniform scalar load, as transformed VPlan will only
4201 // process one original iteration.
4202 auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(), {PtrOp},
4203 /*IsUniform*/ true,
4204 /*Mask*/ nullptr, {}, *WideLoad);
4205 N->insertBefore(InsertPos: WideLoad);
4206 NarrowedOps.insert(Ptr: N);
4207 return N;
4208}
4209
4210std::unique_ptr<VPlan>
4211VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
4212 const TargetTransformInfo &TTI) {
4213 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
4214
4215 if (!VectorLoop)
4216 return nullptr;
4217
4218 // Only handle single-block loops for now.
4219 if (VectorLoop->getEntryBasicBlock() != VectorLoop->getExitingBasicBlock())
4220 return nullptr;
4221
4222 // Skip plans when we may not be able to properly narrow.
4223 VPBasicBlock *Exiting = VectorLoop->getExitingBasicBlock();
4224 if (!match(V: &Exiting->back(), P: m_BranchOnCount()))
4225 return nullptr;
4226
4227 assert(match(&Exiting->back(),
4228 m_BranchOnCount(m_Add(m_VPValue(), m_Specific(&Plan.getVFxUF())),
4229 m_Specific(&Plan.getVectorTripCount()))) &&
4230 "unexpected branch-on-count");
4231
4232 SmallVector<VPInterleaveRecipe *> StoreGroups;
4233 std::optional<ElementCount> VFToOptimize;
4234 for (auto &R : *VectorLoop->getEntryBasicBlock()) {
4235 if (isa<VPDerivedIVRecipe, VPScalarIVStepsRecipe>(Val: &R) &&
4236 vputils::onlyFirstLaneUsed(Def: cast<VPSingleDefRecipe>(Val: &R)))
4237 continue;
4238
4239 // Bail out on recipes not supported at the moment:
4240 // * phi recipes other than the canonical induction
4241 // * recipes writing to memory except interleave groups
4242 // Only support plans with a canonical induction phi.
4243 if (R.isPhi())
4244 return nullptr;
4245
4246 auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(Val: &R);
4247 if (R.mayWriteToMemory() && !InterleaveR)
4248 return nullptr;
4249
4250 // Bail out if any recipe defines a vector value used outside the
4251 // vector loop region.
4252 if (any_of(Range: R.definedValues(), P: [&](VPValue *V) {
4253 return any_of(Range: V->users(), P: [&](VPUser *U) {
4254 auto *UR = cast<VPRecipeBase>(Val: U);
4255 return UR->getParent()->getParent() != VectorLoop;
4256 });
4257 }))
4258 return nullptr;
4259
4260 // All other ops are allowed, but we reject uses that cannot be converted
4261 // when checking all allowed consumers (store interleave groups) below.
4262 if (!InterleaveR)
4263 continue;
4264
4265 // Try to find a single VF, where all interleave groups are consecutive and
4266 // saturate the full vector width. If we already have a candidate VF, check
4267 // if it is applicable for the current InterleaveR, otherwise look for a
4268 // suitable VF across the Plan's VFs.
4269 SmallVector<ElementCount> VFs =
4270 VFToOptimize ? SmallVector<ElementCount>({*VFToOptimize})
4271 : to_vector(Range: Plan.vectorFactors());
4272 std::optional<ElementCount> NarrowedVF =
4273 isConsecutiveInterleaveGroup(InterleaveR, VFs, TTI);
4274 if (!NarrowedVF || (VFToOptimize && NarrowedVF != VFToOptimize))
4275 return nullptr;
4276 VFToOptimize = NarrowedVF;
4277
4278 // Skip read interleave groups.
4279 if (InterleaveR->getStoredValues().empty())
4280 continue;
4281
4282 // Narrow interleave groups, if all operands are already matching narrow
4283 // ops.
4284 auto *Member0 = InterleaveR->getStoredValues()[0];
4285 if (isAlreadyNarrow(VPV: Member0) &&
4286 all_of(Range: InterleaveR->getStoredValues(), P: equal_to(Arg&: Member0))) {
4287 StoreGroups.push_back(Elt: InterleaveR);
4288 continue;
4289 }
4290
4291 // For now, we only support full interleave groups storing load interleave
4292 // groups.
4293 if (all_of(Range: enumerate(First: InterleaveR->getStoredValues()), P: [](auto Op) {
4294 VPRecipeBase *DefR = Op.value()->getDefiningRecipe();
4295 if (!DefR)
4296 return false;
4297 auto *IR = dyn_cast<VPInterleaveRecipe>(Val: DefR);
4298 return IR && IR->getInterleaveGroup()->isFull() &&
4299 IR->getVPValue(Op.index()) == Op.value();
4300 })) {
4301 StoreGroups.push_back(Elt: InterleaveR);
4302 continue;
4303 }
4304
4305 // Check if all values feeding InterleaveR are matching wide recipes, which
4306 // operands that can be narrowed.
4307 if (!canNarrowOps(Ops: InterleaveR->getStoredValues(),
4308 IsScalable: VFToOptimize->isScalable()))
4309 return nullptr;
4310 StoreGroups.push_back(Elt: InterleaveR);
4311 }
4312
4313 if (StoreGroups.empty())
4314 return nullptr;
4315
4316 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
4317 bool RequiresScalarEpilogue =
4318 MiddleVPBB->getNumSuccessors() == 1 &&
4319 MiddleVPBB->getSingleSuccessor() == Plan.getScalarPreheader();
4320 // Bail out for tail-folding (middle block with a single successor to exit).
4321 if (MiddleVPBB->getNumSuccessors() != 2 && !RequiresScalarEpilogue)
4322 return nullptr;
4323
4324 // All interleave groups in Plan can be narrowed for VFToOptimize. Split the
4325 // original Plan into 2: a) a new clone which contains all VFs of Plan, except
4326 // VFToOptimize, and b) the original Plan with VFToOptimize as single VF.
4327 // TODO: Handle cases where only some interleave groups can be narrowed.
4328 std::unique_ptr<VPlan> NewPlan;
4329 if (size(Range: Plan.vectorFactors()) != 1) {
4330 NewPlan = std::unique_ptr<VPlan>(Plan.duplicate());
4331 Plan.setVF(*VFToOptimize);
4332 NewPlan->removeVF(VF: *VFToOptimize);
4333 }
4334
4335 // Convert InterleaveGroup \p R to a single VPWidenLoadRecipe.
4336 SmallPtrSet<VPValue *, 4> NarrowedOps;
4337 VPBasicBlock *Preheader = Plan.getVectorPreheader();
4338 // Narrow operation tree rooted at store groups.
4339 for (auto *StoreGroup : StoreGroups) {
4340 VPValue *Res = narrowInterleaveGroupOp(Members: StoreGroup->getStoredValues(),
4341 NarrowedOps, Preheader);
4342 auto *SI =
4343 cast<StoreInst>(Val: StoreGroup->getInterleaveGroup()->getInsertPos());
4344 VPBuilder(StoreGroup)
4345 .createWidenStore(Store&: *SI, Addr: StoreGroup->getAddr(), StoredVal: Res, Mask: nullptr,
4346 /*Consecutive=*/true, Metadata: *StoreGroup,
4347 DL: StoreGroup->getDebugLoc());
4348 StoreGroup->eraseFromParent();
4349 }
4350
4351 // Adjust induction to reflect that the transformed plan only processes one
4352 // original iteration.
4353 VPInstruction *CanIVInc = vputils::findCanonicalIVIncrement(Plan);
4354 Type *CanIVTy = VectorLoop->getCanonicalIVType();
4355 VPBasicBlock *VectorPH = Plan.getVectorPreheader();
4356 VPBuilder PHBuilder(VectorPH, VectorPH->getFirstNonPhi());
4357
4358 VPValue *UF = &Plan.getUF();
4359 VPValue *Step;
4360 if (VFToOptimize->isScalable()) {
4361 VPValue *VScale =
4362 PHBuilder.createElementCount(Ty: CanIVTy, EC: ElementCount::getScalable(MinVal: 1));
4363 Step = PHBuilder.createOverflowingOp(Opcode: Instruction::Mul, Operands: {VScale, UF},
4364 WrapFlags: {true, false});
4365 Plan.getVF().replaceAllUsesWith(New: VScale);
4366 } else {
4367 Step = UF;
4368 Plan.getVF().replaceAllUsesWith(New: Plan.getConstantInt(Ty: CanIVTy, Val: 1));
4369 }
4370 // Materialize vector trip count with the narrowed step.
4371 materializeVectorTripCount(Plan, VectorPHVPBB: VectorPH, /*TailByMasking=*/false,
4372 RequiresScalarEpilogue, Step);
4373
4374 CanIVInc->setOperand(I: 1, New: Step);
4375 Plan.getVFxUF().replaceAllUsesWith(New: Step);
4376
4377 removeDeadRecipes(Plan);
4378 assert(none_of(*VectorLoop->getEntryBasicBlock(),
4379 IsaPred<VPVectorPointerRecipe>) &&
4380 "All VPVectorPointerRecipes should have been removed");
4381 return NewPlan;
4382}
4383
4384void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
4385 VFRange &Range) {
4386 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
4387 auto *MiddleVPBB = Plan.getMiddleBlock();
4388 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
4389
4390 auto IsScalableOne = [](ElementCount VF) -> bool {
4391 return VF == ElementCount::getScalable(MinVal: 1);
4392 };
4393
4394 for (VPFirstOrderRecurrencePHIRecipe &FOR :
4395 make_isa_range<VPFirstOrderRecurrencePHIRecipe>(
4396 Range: VectorRegion->getEntryBasicBlock()->phis())) {
4397 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
4398 "Cannot handle loops with uncountable early exits");
4399
4400 // Find the existing splice for this FOR, created in
4401 // createHeaderPhiRecipes. All uses of FOR have already been replaced with
4402 // RecurSplice there; only RecurSplice itself still references FOR.
4403 auto *RecurSplice =
4404 findUserOf<VPInstruction::FirstOrderRecurrenceSplice>(V: &FOR);
4405 assert(RecurSplice && "expected FirstOrderRecurrenceSplice");
4406
4407 // For VF vscale x 1, if vscale = 1, we are unable to extract the
4408 // penultimate value of the recurrence. Instead we rely on the existing
4409 // extract of the last element from the result of
4410 // VPInstruction::FirstOrderRecurrenceSplice.
4411 // TODO: Consider vscale_range info and UF.
4412 if (any_of(Range: RecurSplice->users(),
4413 P: [](VPUser *U) { return !cast<VPRecipeBase>(Val: U)->getRegion(); }) &&
4414 LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsScalableOne,
4415 Range))
4416 return;
4417
4418 // This is the second phase of vectorizing first-order recurrences, creating
4419 // extracts for users outside the loop. An overview of the transformation is
4420 // described below. Suppose we have the following loop with some use after
4421 // the loop of the last a[i-1],
4422 //
4423 // for (int i = 0; i < n; ++i) {
4424 // t = a[i - 1];
4425 // b[i] = a[i] - t;
4426 // }
4427 // use t;
4428 //
4429 // There is a first-order recurrence on "a". For this loop, the shorthand
4430 // scalar IR looks like:
4431 //
4432 // scalar.ph:
4433 // s.init = a[-1]
4434 // br scalar.body
4435 //
4436 // scalar.body:
4437 // i = phi [0, scalar.ph], [i+1, scalar.body]
4438 // s1 = phi [s.init, scalar.ph], [s2, scalar.body]
4439 // s2 = a[i]
4440 // b[i] = s2 - s1
4441 // br cond, scalar.body, exit.block
4442 //
4443 // exit.block:
4444 // use = lcssa.phi [s1, scalar.body]
4445 //
4446 // In this example, s1 is a recurrence because it's value depends on the
4447 // previous iteration. In the first phase of vectorization, we created a
4448 // VPFirstOrderRecurrencePHIRecipe v1 for s1. Now we create the extracts
4449 // for users in the scalar preheader and exit block.
4450 //
4451 // vector.ph:
4452 // v_init = vector(..., ..., ..., a[-1])
4453 // br vector.body
4454 //
4455 // vector.body
4456 // i = phi [0, vector.ph], [i+4, vector.body]
4457 // v1 = phi [v_init, vector.ph], [v2, vector.body]
4458 // v2 = a[i, i+1, i+2, i+3]
4459 // v1' = splice(v1(3), v2(0, 1, 2))
4460 // b[i, i+1, i+2, i+3] = v2 - v1'
4461 // br cond, vector.body, middle.block
4462 //
4463 // middle.block:
4464 // vector.recur.extract.for.phi = v2(2)
4465 // vector.recur.extract = v2(3)
4466 // br cond, scalar.ph, exit.block
4467 //
4468 // scalar.ph:
4469 // scalar.recur.init = phi [vector.recur.extract, middle.block],
4470 // [s.init, otherwise]
4471 // br scalar.body
4472 //
4473 // scalar.body:
4474 // i = phi [0, scalar.ph], [i+1, scalar.body]
4475 // s1 = phi [scalar.recur.init, scalar.ph], [s2, scalar.body]
4476 // s2 = a[i]
4477 // b[i] = s2 - s1
4478 // br cond, scalar.body, exit.block
4479 //
4480 // exit.block:
4481 // lo = lcssa.phi [s1, scalar.body],
4482 // [vector.recur.extract.for.phi, middle.block]
4483 //
4484 // Update extracts of the splice in the middle block: they extract the
4485 // penultimate element of the recurrence.
4486 for (VPRecipeBase &R : make_early_inc_range(
4487 Range: make_range(x: MiddleVPBB->getFirstNonPhi(), y: MiddleVPBB->end()))) {
4488 if (!match(V: &R, P: m_ExtractLastLaneOfLastPart(Op0: m_Specific(VPV: RecurSplice))))
4489 continue;
4490
4491 auto *ExtractR = cast<VPInstruction>(Val: &R);
4492 VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
4493 Opcode: VPInstruction::ExtractPenultimateElement, Operands: RecurSplice->getOperand(N: 1),
4494 DL: {}, Name: "vector.recur.extract.for.phi");
4495 for (VPUser *ExitU : to_vector(Range: ExtractR->users())) {
4496 if (auto *ExitPhi = dyn_cast<VPIRPhi>(Val: ExitU))
4497 ExitPhi->replaceUsesOfWith(From: ExtractR, To: PenultimateElement);
4498 }
4499 }
4500 }
4501}
4502
4503/// Check if \p V is a binary expression of a widened IV and a loop-invariant
4504/// value. Returns the widened IV if found, nullptr otherwise.
4505static VPWidenIntOrFpInductionRecipe *getExpressionIV(VPValue *V) {
4506 auto *BinOp = dyn_cast<VPWidenRecipe>(Val: V);
4507 if (!BinOp || !Instruction::isBinaryOp(Opcode: BinOp->getOpcode()) ||
4508 Instruction::isIntDivRem(Opcode: BinOp->getOpcode()))
4509 return nullptr;
4510
4511 VPValue *WidenIVCandidate = BinOp->getOperand(N: 0);
4512 VPValue *InvariantCandidate = BinOp->getOperand(N: 1);
4513 if (!isa<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate))
4514 std::swap(a&: WidenIVCandidate, b&: InvariantCandidate);
4515
4516 if (!InvariantCandidate->isDefinedOutsideLoopRegions())
4517 return nullptr;
4518
4519 return dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate);
4520}
4521
4522/// Create a scalar version of \p BinOp, with its \p WidenIV operand replaced
4523/// by \p ScalarIV, and place it after \p ScalarIV's defining recipe.
4524static VPValue *cloneBinOpForScalarIV(VPWidenRecipe *BinOp, VPValue *ScalarIV,
4525 VPWidenIntOrFpInductionRecipe *WidenIV) {
4526 assert(Instruction::isBinaryOp(BinOp->getOpcode()) &&
4527 BinOp->getNumOperands() == 2 && "BinOp must have 2 operands");
4528 auto *ClonedOp = BinOp->clone();
4529 if (ClonedOp->getOperand(N: 0) == WidenIV) {
4530 ClonedOp->setOperand(I: 0, New: ScalarIV);
4531 } else {
4532 assert(ClonedOp->getOperand(1) == WidenIV && "one operand must be WideIV");
4533 ClonedOp->setOperand(I: 1, New: ScalarIV);
4534 }
4535 ClonedOp->insertAfter(InsertPos: ScalarIV->getDefiningRecipe());
4536 return ClonedOp;
4537}
4538
4539/// If \p S is an affine AddRec, returns true if its step is known to be
4540/// positive and false if it is known to be negative. Returns std::nullopt if
4541/// \p S is not an affine AddRec, or if the sign of its step cannot be
4542/// determined.
4543static std::optional<bool> getStepDirection(const SCEV *S,
4544 ScalarEvolution &SE) {
4545 const SCEV *Step;
4546 if (!match(S, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV(V&: Step))))
4547 return std::nullopt;
4548 if (SE.isKnownPositive(S: Step))
4549 return true;
4550 if (SE.isKnownNegative(S: Step))
4551 return false;
4552 return std::nullopt;
4553}
4554
4555void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
4556 PredicatedScalarEvolution &PSE,
4557 Loop &L) {
4558 ScalarEvolution &SE = *PSE.getSE();
4559 VPRegionBlock *VectorLoopRegion = Plan.getVectorLoopRegion();
4560
4561 // Helper lambda to check if the IV range excludes the sentinel value. Try
4562 // signed first, then unsigned. Return an excluded sentinel if found,
4563 // otherwise return std::nullopt.
4564 auto CheckSentinel = [&SE](const SCEV *IVSCEV,
4565 bool UseMax) -> std::optional<APSInt> {
4566 unsigned BW = IVSCEV->getType()->getScalarSizeInBits();
4567 for (bool Signed : {true, false}) {
4568 APSInt Sentinel = UseMax ? APSInt::getMinValue(numBits: BW, /*Unsigned=*/!Signed)
4569 : APSInt::getMaxValue(numBits: BW, /*Unsigned=*/!Signed);
4570
4571 ConstantRange IVRange =
4572 Signed ? SE.getSignedRange(S: IVSCEV) : SE.getUnsignedRange(S: IVSCEV);
4573 if (!IVRange.contains(Val: Sentinel))
4574 return Sentinel;
4575 }
4576 return std::nullopt;
4577 };
4578
4579 VPValue *HeaderMask = VectorLoopRegion->getHeaderMask();
4580 for (VPRecipeBase &Phi :
4581 make_early_inc_range(Range: VectorLoopRegion->getEntryBasicBlock()->phis())) {
4582 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &Phi);
4583 if (!PhiR || !RecurrenceDescriptor::isFindLastRecurrenceKind(
4584 Kind: PhiR->getRecurrenceKind()))
4585 continue;
4586
4587 Type *PhiTy = PhiR->getScalarType();
4588 if (PhiTy->isPointerTy() || PhiTy->isFloatingPointTy())
4589 continue;
4590
4591 // If there's a header mask, the backedge select will not be the find-last
4592 // select.
4593 VPValue *BackedgeVal = PhiR->getBackedgeValue();
4594 auto *FindLastSelect = cast<VPSingleDefRecipe>(Val: BackedgeVal);
4595 if (HeaderMask &&
4596 !match(V: BackedgeVal,
4597 P: m_Select(Op0: m_Specific(VPV: HeaderMask),
4598 Op1: m_VPSingleDefRecipe(V&: FindLastSelect), Op2: m_Specific(VPV: PhiR))))
4599 continue;
4600
4601 // Get the find-last expression from the find-last select of the reduction
4602 // phi. The find-last select should be a select between the phi and the
4603 // find-last expression.
4604 VPValue *Cond, *FindLastExpression;
4605 if (!match(R: FindLastSelect, P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: PhiR),
4606 Op2: m_VPValue(V&: FindLastExpression))) &&
4607 !match(R: FindLastSelect,
4608 P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_VPValue(V&: FindLastExpression),
4609 Op2: m_Specific(VPV: PhiR))))
4610 continue;
4611
4612 // Check if FindLastExpression is a simple expression of a widened IV. If
4613 // so, we can track the underlying IV instead and sink the expression.
4614 auto *IVOfExpressionToSink = getExpressionIV(V: FindLastExpression);
4615 const SCEV *IVSCEV = vputils::getSCEVExprForVPValue(
4616 V: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression, PSE,
4617 L: &L);
4618 if (!match(S: IVSCEV, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV()))) {
4619 assert(!match(vputils::getSCEVExprForVPValue(FindLastExpression, PSE, &L),
4620 m_scev_AffineAddRec(m_SCEV(), m_SCEV())) &&
4621 "IVOfExpressionToSink not being an AddRec must imply "
4622 "FindLastExpression not being an AddRec.");
4623 continue;
4624 }
4625
4626 // Determine direction from the step of IVSCEV, if possible.
4627 std::optional<bool> StepDirection = getStepDirection(S: IVSCEV, SE);
4628 if (!StepDirection)
4629 continue;
4630
4631 bool UseMax = *StepDirection;
4632 std::optional<APSInt> SentinelVal = CheckSentinel(IVSCEV, UseMax);
4633 bool UseSigned = SentinelVal && SentinelVal->isSigned();
4634
4635 // Sinking an expression will disable epilogue vectorization. Only use it,
4636 // if FindLastExpression cannot be vectorized via a sentinel. Sinking may
4637 // also prevent vectorizing using a sentinel (e.g., if the expression is a
4638 // multiply or divide by large constant, respectively), which also makes
4639 // sinking undesirable.
4640 if (IVOfExpressionToSink) {
4641 const SCEV *FindLastExpressionSCEV =
4642 vputils::getSCEVExprForVPValue(V: FindLastExpression, PSE, L: &L);
4643 if (std::optional<bool> NewUseMax =
4644 getStepDirection(S: FindLastExpressionSCEV, SE)) {
4645 if (auto NewSentinel =
4646 CheckSentinel(FindLastExpressionSCEV, *NewUseMax)) {
4647 // The original expression already has a sentinel, so prefer not
4648 // sinking to keep epilogue vectorization possible.
4649 SentinelVal = *NewSentinel;
4650 UseSigned = NewSentinel->isSigned();
4651 UseMax = *NewUseMax;
4652 IVSCEV = FindLastExpressionSCEV;
4653 IVOfExpressionToSink = nullptr;
4654 }
4655 }
4656 }
4657
4658 // If no sentinel was found, fall back to a boolean AnyOf reduction to track
4659 // if the condition was ever true. Requires the IV to not wrap, otherwise we
4660 // cannot use min/max.
4661 if (!SentinelVal) {
4662 auto *AR = cast<SCEVAddRecExpr>(Val: IVSCEV);
4663 if (AR->hasNoSignedWrap())
4664 UseSigned = true;
4665 else if (AR->hasNoUnsignedWrap())
4666 UseSigned = false;
4667 else
4668 continue;
4669 }
4670
4671 VPInstruction *RdxResult = cast<VPInstruction>(Val: vputils::findRecipe(
4672 Start: BackedgeVal,
4673 Pred: match_fn(P: m_VPInstruction<VPInstruction::ComputeReductionResult>())));
4674
4675 VPValue *NewFindLastSelect = BackedgeVal;
4676 VPValue *SelectCond = Cond;
4677 if (!SentinelVal || IVOfExpressionToSink) {
4678 // When we need to create a new select, normalize the condition so that
4679 // PhiR is the last operand and include the header mask if needed.
4680 DebugLoc DL = FindLastSelect->getDefiningRecipe()->getDebugLoc();
4681 VPBuilder LoopBuilder(FindLastSelect->getDefiningRecipe());
4682 if (match(R: FindLastSelect,
4683 P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: PhiR), Op2: m_VPValue())))
4684 SelectCond = LoopBuilder.createNot(Operand: SelectCond);
4685
4686 // When tail folding, mask the condition with the header mask to prevent
4687 // propagating poison from inactive lanes in the last vector iteration.
4688 if (HeaderMask)
4689 SelectCond = LoopBuilder.createLogicalAnd(LHS: HeaderMask, RHS: SelectCond);
4690
4691 if (SelectCond != Cond || IVOfExpressionToSink) {
4692 NewFindLastSelect = LoopBuilder.createSelect(
4693 Cond: SelectCond,
4694 TrueVal: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression,
4695 FalseVal: PhiR, DL);
4696 }
4697 }
4698
4699 // Create the reduction result in the middle block using sentinel directly.
4700 RecurKind MinMaxKind =
4701 UseMax ? (UseSigned ? RecurKind::SMax : RecurKind::UMax)
4702 : (UseSigned ? RecurKind::SMin : RecurKind::UMin);
4703 VPIRFlags Flags(MinMaxKind, /*IsOrdered=*/false, /*IsInLoop=*/false,
4704 FastMathFlags());
4705 DebugLoc ExitDL = RdxResult->getDebugLoc();
4706 VPBuilder MiddleBuilder(RdxResult);
4707 VPValue *ReducedIV =
4708 MiddleBuilder.createNaryOp(Opcode: VPInstruction::ComputeReductionResult,
4709 Operands: NewFindLastSelect, Flags, DL: ExitDL);
4710
4711 // If IVOfExpressionToSink is an expression to sink, sink it now.
4712 VPValue *VectorRegionExitingVal = ReducedIV;
4713 if (IVOfExpressionToSink)
4714 VectorRegionExitingVal =
4715 cloneBinOpForScalarIV(BinOp: cast<VPWidenRecipe>(Val: FindLastExpression),
4716 ScalarIV: ReducedIV, WidenIV: IVOfExpressionToSink);
4717
4718 VPValue *NewRdxResult;
4719 VPValue *StartVPV = PhiR->getStartValue();
4720 if (SentinelVal) {
4721 // Sentinel-based approach: reduce IVs with min/max, compare against
4722 // sentinel to detect if condition was ever true, select accordingly.
4723 VPValue *Sentinel = Plan.getConstantInt(Val: *SentinelVal);
4724 auto *Cmp = MiddleBuilder.createICmp(Pred: CmpInst::ICMP_NE, A: ReducedIV,
4725 B: Sentinel, DL: ExitDL);
4726 NewRdxResult = MiddleBuilder.createSelect(Cond: Cmp, TrueVal: VectorRegionExitingVal,
4727 FalseVal: StartVPV, DL: ExitDL);
4728 StartVPV = Sentinel;
4729 } else {
4730 // Introduce a boolean AnyOf reduction to track if the condition was ever
4731 // true in the loop. Use it to select the initial start value, if it was
4732 // never true.
4733 auto *AnyOfPhi = new VPReductionPHIRecipe(
4734 /*Phi=*/nullptr, RecurKind::Or, *Plan.getFalse(), *Plan.getFalse(),
4735 RdxUnordered{.VFScaleFactor: 1}, {}, /*HasUsesOutsideReductionChain=*/false);
4736 AnyOfPhi->insertAfter(InsertPos: PhiR);
4737
4738 VPBuilder LoopBuilder(BackedgeVal->getDefiningRecipe());
4739 VPValue *OrVal = LoopBuilder.createOr(LHS: AnyOfPhi, RHS: SelectCond);
4740 AnyOfPhi->setOperand(I: 1, New: OrVal);
4741
4742 NewRdxResult = MiddleBuilder.createAnyOfReduction(
4743 ChainOp: OrVal, TrueVal: VectorRegionExitingVal, FalseVal: StartVPV, DL: ExitDL);
4744
4745 // Initialize the IV reduction phi with the neutral element, not the
4746 // original start value, to ensure correct min/max reduction results.
4747 StartVPV = Plan.getOrAddLiveIn(
4748 V: getRecurrenceIdentity(K: MinMaxKind, Tp: IVSCEV->getType(), FMF: {}));
4749 }
4750 RdxResult->replaceAllUsesWith(New: NewRdxResult);
4751 RdxResult->eraseFromParent();
4752
4753 auto *NewPhiR = new VPReductionPHIRecipe(
4754 cast<PHINode>(Val: PhiR->getUnderlyingInstr()), RecurKind::FindIV, *StartVPV,
4755 *NewFindLastSelect, RdxUnordered{.VFScaleFactor: 1}, {},
4756 PhiR->hasUsesOutsideReductionChain());
4757 NewPhiR->insertBefore(InsertPos: PhiR);
4758 PhiR->replaceAllUsesWith(New: NewPhiR);
4759 PhiR->eraseFromParent();
4760 }
4761}
4762
4763namespace {
4764
4765using ExtendKind = TTI::PartialReductionExtendKind;
4766struct ReductionExtend {
4767 Type *SrcType = nullptr;
4768 ExtendKind Kind = ExtendKind::PR_None;
4769};
4770
4771/// Describes the extends used to compute the extended reduction operand.
4772/// ExtendB is optional. If ExtendB is present, ExtendsUser is a binary
4773/// operation.
4774struct ExtendedReductionOperand {
4775 /// The recipe that consumes the extends.
4776 VPWidenRecipe *ExtendsUser = nullptr;
4777 /// Extend descriptions (inputs to getPartialReductionCost).
4778 ReductionExtend ExtendA, ExtendB;
4779};
4780
4781/// A chain of recipes that form a partial reduction. Matches either
4782/// reduction_bin_op (extended op, accumulator), or
4783/// reduction_bin_op (accumulator, extended op).
4784/// The possible forms of the "extended op" are listed in
4785/// matchExtendedReductionOperand.
4786struct VPPartialReductionChain {
4787 /// The top-level binary operation that forms the reduction to a scalar
4788 /// after the loop body.
4789 VPWidenRecipe *ReductionBinOp = nullptr;
4790 /// The user of the extends that is then reduced.
4791 ExtendedReductionOperand ExtendedOp;
4792 /// The recurrence kind for the entire partial reduction chain.
4793 /// This allows distinguishing between Sub and AddWithSub recurrences,
4794 /// when the ReductionBinOp is a Instruction::Sub.
4795 RecurKind RK;
4796 /// The index of the accumulator operand of ReductionBinOp. The extended op
4797 /// is `1 - AccumulatorOpIdx`.
4798 unsigned AccumulatorOpIdx;
4799 unsigned ScaleFactor;
4800 /// Optional blend to represent predication for the block that updates the
4801 /// reduction.
4802 VPBlendRecipe *Blend = nullptr;
4803};
4804
4805// Return the incoming index of the single-use value in the blend, which is
4806// expected to be the predicated reduction update.
4807static std::optional<unsigned>
4808getBlendReductionUpdateValueIdx(VPBlendRecipe *Blend) {
4809 assert(Blend && !Blend->isNormalized() &&
4810 Blend->getNumIncomingValues() == 2 &&
4811 "Expected a non-normalized blend with two incoming values");
4812 bool FirstIncomingHasOneUse = Blend->getIncomingValue(Idx: 0)->hasOneUse();
4813
4814 // Only the update value should have one use (the blend). The previous
4815 // value should always have at least two uses, the blend and the reduction.
4816 if (FirstIncomingHasOneUse == Blend->getIncomingValue(Idx: 1)->hasOneUse())
4817 return std::nullopt;
4818 return FirstIncomingHasOneUse ? 0 : 1;
4819}
4820
4821static VPSingleDefRecipe *
4822optimizeExtendsForPartialReduction(VPSingleDefRecipe *Op) {
4823 // reduce.add(mul(ext(A), C))
4824 // -> reduce.add(mul(ext(A), ext(trunc(C))))
4825 const APInt *Const;
4826 if (match(R: Op, P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_APInt(C&: Const)))) {
4827 auto *ExtA = cast<VPWidenCastRecipe>(Val: Op->getOperand(N: 0));
4828 Instruction::CastOps ExtOpc = ExtA->getOpcode();
4829 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
4830 if (!Op->hasOneUse() ||
4831 !llvm::canConstantBeExtended(
4832 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
4833 return Op;
4834
4835 VPBuilder Builder(Op);
4836 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc,
4837 Op: Op->getOperand(N: 1), ResultTy: NarrowTy);
4838 Type *WideTy = ExtA->getScalarType();
4839 Op->setOperand(I: 1, New: Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy));
4840 return Op;
4841 }
4842
4843 // reduce.add(abs(sub(ext(A), ext(B))))
4844 // -> reduce.add(ext(absolute-difference(A, B)))
4845 VPValue *X, *Y;
4846 if (match(R: Op, P: m_WidenIntrinsic<Intrinsic::abs>(Ops: m_Sub(
4847 Op0: m_ZExtOrSExt(Op0: m_VPValue(V&: X)), Op1: m_ZExtOrSExt(Op0: m_VPValue(V&: Y)))))) {
4848 auto *Sub = Op->getOperand(N: 0)->getDefiningRecipe();
4849 auto *Ext = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
4850 assert(Ext->getOpcode() ==
4851 cast<VPWidenCastRecipe>(Sub->getOperand(1))->getOpcode() &&
4852 "Expected both the LHS and RHS extends to be the same");
4853 bool IsSigned = Ext->getOpcode() == Instruction::SExt;
4854 VPBuilder Builder(Op);
4855 Type *SrcTy = X->getScalarType();
4856 auto *FreezeX = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {X}));
4857 auto *FreezeY = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {Y}));
4858 auto *Max = Builder.insert(
4859 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smax : Intrinsic::umax,
4860 {FreezeX, FreezeY}, SrcTy));
4861 auto *Min = Builder.insert(
4862 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smin : Intrinsic::umin,
4863 {FreezeX, FreezeY}, SrcTy));
4864 auto *AbsDiff = Builder.insert(
4865 R: new VPWidenRecipe(Instruction::Sub, {Max, Min},
4866 VPIRFlags::getDefaultFlags(Opcode: Instruction::Sub)));
4867 return Builder.createWidenCast(Opcode: Instruction::CastOps::ZExt, Op: AbsDiff,
4868 ResultTy: Op->getScalarType());
4869 }
4870
4871 // reduce.add(ext(mul(ext(A), ext(B))))
4872 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
4873 // TODO: Support this optimization for float types.
4874 if (match(R: Op, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
4875 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
4876 auto *Ext = cast<VPWidenCastRecipe>(Val: Op);
4877 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
4878 auto *MulLHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
4879 auto *MulRHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
4880 if (!Mul->hasOneUse() ||
4881 (Ext->getOpcode() != MulLHS->getOpcode() && MulLHS != MulRHS) ||
4882 MulLHS->getOpcode() != MulRHS->getOpcode())
4883 return Op;
4884 VPBuilder Builder(Mul);
4885 auto *NewLHS = Builder.createWidenCast(
4886 Opcode: MulLHS->getOpcode(), Op: MulLHS->getOperand(N: 0), ResultTy: Ext->getScalarType());
4887 auto *NewRHS = MulLHS == MulRHS
4888 ? NewLHS
4889 : Builder.createWidenCast(Opcode: MulRHS->getOpcode(),
4890 Op: MulRHS->getOperand(N: 0),
4891 ResultTy: Ext->getScalarType());
4892 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewLHS, NewRHS});
4893 Builder.insert(R: NewMul);
4894 Op->replaceAllUsesWith(New: NewMul);
4895 Op->eraseFromParent();
4896 Mul->eraseFromParent();
4897 return NewMul;
4898 }
4899
4900 return Op;
4901}
4902
4903static VPExpressionRecipe *
4904createPartialReductionExpression(VPReductionRecipe *Red) {
4905 VPValue *VecOp = Red->getVecOp();
4906
4907 // reduce.[f]add(ext(op))
4908 // -> VPExpressionRecipe(op, red)
4909 if (match(V: VecOp, P: m_WidenAnyExtend(Op0: m_VPValue())))
4910 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
4911
4912 // reduce.[f]add(neg(ext(op)))
4913 // -> VPExpressionRecipe(op, sub/neg, red)
4914 if (match(V: VecOp, P: m_AnyNeg(Op0: m_WidenAnyExtend(Op0: m_VPValue())))) {
4915 auto *Neg = cast<VPWidenRecipe>(Val: VecOp);
4916 auto *Ext =
4917 cast<VPWidenCastRecipe>(Val: Neg->getOperand(N: Neg->getNumOperands() - 1));
4918 return new VPExpressionRecipe(Ext, Neg, Red);
4919 }
4920
4921 // reduce.[f]add([f]mul(ext(a), ext(b)))
4922 // -> VPExpressionRecipe(a, b, mul, red)
4923 if (match(V: VecOp, P: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))) ||
4924 match(V: VecOp,
4925 P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_ZExtOrSExt(Op0: m_VPValue())))) {
4926 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
4927 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
4928 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
4929 return new VPExpressionRecipe(ExtA, ExtB, Mul, Red);
4930 }
4931
4932 // reduce.fadd(fneg(fmul(fpext(a), fpext(b))))
4933 // -> VPExpressionRecipe(a, b, fmul, fsub, red)
4934 if (match(V: VecOp,
4935 P: m_FNeg(Op0: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))))) {
4936 auto *FNeg = cast<VPWidenRecipe>(Val: VecOp);
4937 auto *FMul = cast<VPWidenRecipe>(Val: FNeg->getOperand(N: 0));
4938 auto *ExtA = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 0));
4939 auto *ExtB = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 1));
4940 return new VPExpressionRecipe(ExtA, ExtB, FMul, FNeg, Red);
4941 }
4942
4943 // reduce.add(neg(mul(ext(a), ext(b))))
4944 // -> VPExpressionRecipe(a, b, mul, sub, red)
4945 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
4946 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
4947 auto *Sub = cast<VPWidenRecipe>(Val: VecOp);
4948 auto *Mul = cast<VPWidenRecipe>(Val: Sub->getOperand(N: 1));
4949 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
4950 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
4951 return new VPExpressionRecipe(ExtA, ExtB, Mul, Sub, Red);
4952 }
4953
4954 llvm_unreachable("Unsupported expression");
4955}
4956
4957// Helper to transform a partial reduction chain into a partial reduction
4958// recipe. Assumes profitability has been checked.
4959static void transformToPartialReduction(const VPPartialReductionChain &Chain,
4960 VPlan &Plan,
4961 VPReductionPHIRecipe *RdxPhi) {
4962 VPWidenRecipe *WidenRecipe = Chain.ReductionBinOp;
4963 assert(WidenRecipe->getNumOperands() == 2 && "Expected binary operation");
4964
4965 VPValue *Accumulator = WidenRecipe->getOperand(N: Chain.AccumulatorOpIdx);
4966 auto *ExtendedOp = cast<VPSingleDefRecipe>(
4967 Val: WidenRecipe->getOperand(N: 1 - Chain.AccumulatorOpIdx));
4968
4969 // FIXME: Do these transforms before invoking the cost-model.
4970 ExtendedOp = optimizeExtendsForPartialReduction(Op: ExtendedOp);
4971
4972 // Sub-reductions can be implemented in two ways:
4973 // (1) negate the operand in the vector loop (the default way).
4974 // (2) subtract the reduced value from the init value in the middle block.
4975 // Both ways keep the reduction itself as an 'add' reduction.
4976 //
4977 // The ISD nodes for partial reductions don't support folding the
4978 // sub/negation into its operands because the following is not a valid
4979 // transformation:
4980 // sub(0, mul(ext(a), ext(b)))
4981 // -> mul(ext(a), ext(sub(0, b)))
4982 //
4983 // It's therefore better to choose option (2) such that the partial
4984 // reduction is always positive (starting at '0') and to do a final
4985 // subtract in the middle block.
4986 if ((WidenRecipe->getOpcode() == Instruction::Sub &&
4987 Chain.RK != RecurKind::Sub) ||
4988 (WidenRecipe->getOpcode() == Instruction::FSub &&
4989 Chain.RK != RecurKind::FSub)) {
4990 VPBuilder Builder(WidenRecipe);
4991 Type *ElemTy = ExtendedOp->getScalarType();
4992 VPWidenRecipe *NegRecipe;
4993 if (WidenRecipe->getOpcode() == Instruction::FSub) {
4994 NegRecipe =
4995 new VPWidenRecipe(Instruction::FNeg, {ExtendedOp},
4996 VPIRFlags::getDefaultFlags(Opcode: Instruction::FNeg),
4997 VPIRMetadata(), DebugLoc::getUnknown());
4998 } else {
4999 auto *Zero = Plan.getZero(Ty: ElemTy);
5000 NegRecipe =
5001 new VPWidenRecipe(Instruction::Sub, {Zero, ExtendedOp},
5002 VPIRFlags::getDefaultFlags(Opcode: Instruction::Sub),
5003 VPIRMetadata(), DebugLoc::getUnknown());
5004 }
5005 Builder.insert(R: NegRecipe);
5006 ExtendedOp = NegRecipe;
5007 }
5008
5009 // Check if WidenRecipe is the final result of the reduction. If so, look
5010 // through the Select recipe introduced by tail-folding, otherwise look
5011 // through any Blend recipe introduced by predication for the block.
5012 VPValue *ExitSearch =
5013 Chain.Blend ? cast<VPValue>(Val: Chain.Blend) : cast<VPValue>(Val: WidenRecipe);
5014
5015 VPValue *Cond = nullptr;
5016 VPValue *ExitValue = cast_or_null<VPInstruction>(
5017 Val: findUserOf(V: ExitSearch, P: m_Select(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: ExitSearch),
5018 Op2: m_Specific(VPV: RdxPhi))));
5019
5020 if (Chain.Blend) {
5021 std::optional<unsigned> BlendReductionIdx =
5022 getBlendReductionUpdateValueIdx(Blend: Chain.Blend);
5023 assert(BlendReductionIdx &&
5024 Chain.Blend->getIncomingValue(*BlendReductionIdx) == WidenRecipe &&
5025 "Expected blend to contain the reduction update");
5026 VPValue *BlendCond = Chain.Blend->getMask(Idx: *BlendReductionIdx);
5027 Cond = ExitValue ? VPBuilder(WidenRecipe)
5028 .createLogicalAnd(LHS: Cond, RHS: BlendCond,
5029 DL: WidenRecipe->getDebugLoc())
5030 : BlendCond;
5031 }
5032
5033 // When folding the tail, the inactive lanes of the reduction update are
5034 // computed from values that do not correspond to any scalar iteration
5035 // and must not be accumulated.
5036 if (!Cond)
5037 Cond = Plan.getVectorLoopRegion()->getHeaderMask();
5038
5039 bool IsLastInChain = RdxPhi->getBackedgeValue() == WidenRecipe ||
5040 RdxPhi->getBackedgeValue() == ExitValue ||
5041 RdxPhi->getBackedgeValue() == Chain.Blend;
5042 assert((!ExitValue || IsLastInChain) &&
5043 "if we found ExitValue, it must match RdxPhi's backedge value");
5044
5045 Type *PhiType = RdxPhi->getScalarType();
5046 RecurKind RdxKind =
5047 PhiType->isFloatingPointTy() ? RecurKind::FAdd : RecurKind::Add;
5048 auto *PartialRed = new VPReductionRecipe(
5049 RdxKind,
5050 RdxKind == RecurKind::FAdd ? WidenRecipe->getFastMathFlagsOrNone()
5051 : FastMathFlags(),
5052 WidenRecipe->getUnderlyingInstr(), Accumulator, ExtendedOp, Cond,
5053 RdxUnordered{/*VFScaleFactor=*/Chain.ScaleFactor});
5054 PartialRed->insertBefore(InsertPos: WidenRecipe);
5055
5056 if (ExitValue)
5057 ExitValue->replaceAllUsesWith(New: PartialRed);
5058 if (Chain.Blend)
5059 Chain.Blend->replaceAllUsesWith(New: PartialRed);
5060 WidenRecipe->replaceAllUsesWith(New: PartialRed);
5061
5062 // For cost-model purposes, fold this into a VPExpression.
5063 VPExpressionRecipe *E = createPartialReductionExpression(Red: PartialRed);
5064 E->insertBefore(InsertPos: WidenRecipe);
5065 PartialRed->replaceAllUsesWith(New: E);
5066
5067 // We only need to update the PHI node once, which is when we find the
5068 // last reduction in the chain.
5069 if (!IsLastInChain)
5070 return;
5071
5072 // Scale the PHI and ReductionStartVector by the VFScaleFactor
5073 assert(RdxPhi->getVFScaleFactor() == 1 && "scale factor must not be set");
5074 RdxPhi->setVFScaleFactor(Chain.ScaleFactor);
5075
5076 auto *StartInst = cast<VPInstruction>(Val: RdxPhi->getStartValue());
5077 assert(StartInst->getOpcode() == VPInstruction::ReductionStartVector);
5078 auto *NewScaleFactor = Plan.getConstantInt(BitWidth: 32, Val: Chain.ScaleFactor);
5079 StartInst->setOperand(I: 2, New: NewScaleFactor);
5080
5081 // If this is the last value in a sub-reduction chain, then update the PHI
5082 // node to start at `0` and update the reduction-result to subtract from
5083 // the PHI's start value.
5084 if (Chain.RK != RecurKind::Sub && Chain.RK != RecurKind::FSub)
5085 return;
5086
5087 VPValue *OldStartValue = StartInst->getOperand(N: 0);
5088 StartInst->setOperand(I: 0, New: StartInst->getOperand(N: 1));
5089
5090 // Replace reduction_result by 'sub (startval, reductionresult)'.
5091 VPInstruction *RdxResult = vputils::findComputeReductionResult(PhiR: RdxPhi);
5092 assert(RdxResult && "Could not find reduction result");
5093
5094 VPBuilder Builder = VPBuilder::getToInsertAfter(R: RdxResult);
5095 unsigned SubOpc = Chain.RK == RecurKind::FSub ? Instruction::BinaryOps::FSub
5096 : Instruction::BinaryOps::Sub;
5097 VPInstruction *NewResult = Builder.createNaryOp(
5098 Opcode: SubOpc, Operands: {OldStartValue, RdxResult}, Flags: VPIRFlags::getDefaultFlags(Opcode: SubOpc),
5099 DL: RdxPhi->getDebugLoc());
5100 RdxResult->replaceUsesWithIf(
5101 New: NewResult,
5102 ShouldReplace: [&NewResult](VPUser &U, unsigned Idx) { return &U != NewResult; });
5103}
5104
5105/// Returns the cost of a link in a partial-reduction chain for a given VF.
5106static InstructionCost
5107getPartialReductionLinkCost(VPCostContext &CostCtx,
5108 const VPPartialReductionChain &Link,
5109 ElementCount VF) {
5110 Type *RdxType = Link.ReductionBinOp->getScalarType();
5111 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
5112 std::optional<unsigned> BinOpc = std::nullopt;
5113 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
5114 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
5115 BinOpc = ExtendedOp.ExtendsUser->getOpcode();
5116
5117 std::optional<llvm::FastMathFlags> Flags;
5118 if (RdxType->isFloatingPointTy())
5119 Flags = Link.ReductionBinOp->getFastMathFlagsOrNone();
5120
5121 auto GetLinkOpcode = [&Link]() -> unsigned {
5122 switch (Link.RK) {
5123 case RecurKind::Sub:
5124 return Instruction::Add;
5125 case RecurKind::FSub:
5126 return Instruction::FAdd;
5127 default:
5128 return Link.ReductionBinOp->getOpcode();
5129 }
5130 };
5131
5132 return CostCtx.TTI.getPartialReductionCost(
5133 Opcode: GetLinkOpcode(), InputTypeA: ExtendedOp.ExtendA.SrcType, InputTypeB: ExtendedOp.ExtendB.SrcType,
5134 AccumType: RdxType, VF, OpAExtend: ExtendedOp.ExtendA.Kind, OpBExtend: ExtendedOp.ExtendB.Kind, BinOp: BinOpc,
5135 CostKind: CostCtx.CostKind, FMF: Flags);
5136}
5137
5138static ExtendKind getPartialReductionExtendKind(VPWidenCastRecipe *Cast) {
5139 return TTI::getPartialReductionExtendKind(CastOpc: Cast->getOpcode());
5140}
5141
5142/// Checks if \p Op (which is an operand of \p UpdateR) is an extended reduction
5143/// operand. This is an operand where the source of the value (e.g. a load) has
5144/// been extended (sext, zext, or fpext) before it is used in the reduction.
5145///
5146/// Possible forms matched by this function:
5147/// - UpdateR(PrevValue, ext(...))
5148/// - UpdateR(PrevValue, mul(ext(...), ext(...)))
5149/// - UpdateR(PrevValue, mul(ext(...), Constant))
5150/// - UpdateR(PrevValue, ext(mul(ext(...), ext(...))))
5151/// - UpdateR(PrevValue, ext(mul(ext(...), Constant)))
5152/// - UpdateR(PrevValue, abs(sub(ext(...), ext(...)))
5153///
5154/// Note: The second operand of UpdateR corresponds to \p Op in the examples.
5155static std::optional<ExtendedReductionOperand>
5156matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op) {
5157 assert(is_contained(UpdateR->operands(), Op) &&
5158 "Op should be operand of UpdateR");
5159
5160 // Try matching an absolute difference operand of the form
5161 // `abs(sub(ext(A), ext(B)))`. This will be later transformed into
5162 // `ext(absolute-difference(A, B))`. This allows us to perform the absolute
5163 // difference on a wider type and get the extend for "free" from the partial
5164 // reduction.
5165 VPValue *X, *Y;
5166 if (Op->hasOneUse() &&
5167 match(V: Op, P: m_WidenIntrinsic<Intrinsic::abs>(
5168 Ops: m_OneUse(SubPattern: m_Sub(Op0: m_WidenAnyExtend(Op0: m_VPValue(V&: X)),
5169 Op1: m_WidenAnyExtend(Op0: m_VPValue(V&: Y))))))) {
5170 auto *Abs = cast<VPWidenIntrinsicRecipe>(Val: Op);
5171 auto *Sub = cast<VPWidenRecipe>(Val: Abs->getOperand(N: 0));
5172 auto *LHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
5173 auto *RHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 1));
5174 Type *LHSInputType = X->getScalarType();
5175 Type *RHSInputType = Y->getScalarType();
5176 if (LHSInputType != RHSInputType ||
5177 LHSExt->getOpcode() != RHSExt->getOpcode())
5178 return std::nullopt;
5179 // Note: This is essentially the same as matching ext(...) as we will
5180 // rewrite this operand to ext(absolute-difference(A, B)).
5181 return ExtendedReductionOperand{
5182 .ExtendsUser: Sub,
5183 /*ExtendA=*/{.SrcType: LHSInputType, .Kind: getPartialReductionExtendKind(Cast: LHSExt)},
5184 /*ExtendB=*/{}};
5185 }
5186
5187 std::optional<TTI::PartialReductionExtendKind> OuterExtKind;
5188 if (match(V: Op, P: m_WidenAnyExtend(Op0: m_VPValue()))) {
5189 auto *CastRecipe = cast<VPWidenCastRecipe>(Val: Op);
5190 VPValue *CastSource = CastRecipe->getOperand(N: 0);
5191 OuterExtKind = getPartialReductionExtendKind(Cast: CastRecipe);
5192 if (match(V: CastSource, P: m_Mul(Op0: m_VPValue(), Op1: m_VPValue())) ||
5193 match(V: CastSource, P: m_FMul(Op0: m_VPValue(), Op1: m_VPValue()))) {
5194 // Match: ext(mul(...))
5195 // Record the outer extend kind and set `Op` to the mul. We can then match
5196 // this as a binary operation. Note: We can optimize out the outer extend
5197 // by widening the inner extends to match it. See
5198 // optimizeExtendsForPartialReduction.
5199 Op = CastSource;
5200 } else {
5201 return ExtendedReductionOperand{
5202 .ExtendsUser: UpdateR,
5203 /*ExtendA=*/{.SrcType: CastSource->getScalarType(), .Kind: *OuterExtKind},
5204 /*ExtendB=*/{}};
5205 }
5206 }
5207
5208 if (!Op->hasOneUse())
5209 return std::nullopt;
5210
5211 VPWidenRecipe *MulOp = dyn_cast<VPWidenRecipe>(Val: Op);
5212 if (!MulOp ||
5213 !is_contained(Set: {Instruction::Mul, Instruction::FMul}, Element: MulOp->getOpcode()))
5214 return std::nullopt;
5215
5216 // The rest of the matching assumes `Op` is a (possibly extended) mul
5217 // operation.
5218
5219 VPValue *LHS = MulOp->getOperand(N: 0);
5220 VPValue *RHS = MulOp->getOperand(N: 1);
5221
5222 // The LHS of the operation must always be an extend.
5223 if (!match(V: LHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
5224 return std::nullopt;
5225
5226 auto *LHSCast = cast<VPWidenCastRecipe>(Val: LHS);
5227 Type *LHSInputType = LHSCast->getOperand(N: 0)->getScalarType();
5228 ExtendKind LHSExtendKind = getPartialReductionExtendKind(Cast: LHSCast);
5229
5230 // The RHS of the operation can be an extend or a constant integer.
5231 const APInt *RHSConst = nullptr;
5232 VPWidenCastRecipe *RHSCast = nullptr;
5233 if (match(V: RHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
5234 RHSCast = cast<VPWidenCastRecipe>(Val: RHS);
5235 else if (!match(V: RHS, P: m_APInt(C&: RHSConst)) ||
5236 !canConstantBeExtended(C: RHSConst, NarrowType: LHSInputType, ExtKind: LHSExtendKind))
5237 return std::nullopt;
5238
5239 // The outer extend kind must match the inner extends for folding.
5240 for (VPWidenCastRecipe *Cast : {LHSCast, RHSCast})
5241 if (Cast && OuterExtKind &&
5242 getPartialReductionExtendKind(Cast) != OuterExtKind)
5243 return std::nullopt;
5244
5245 Type *RHSInputType = LHSInputType;
5246 ExtendKind RHSExtendKind = LHSExtendKind;
5247 if (RHSCast) {
5248 RHSInputType = RHSCast->getOperand(N: 0)->getScalarType();
5249 RHSExtendKind = getPartialReductionExtendKind(Cast: RHSCast);
5250 }
5251
5252 return ExtendedReductionOperand{
5253 .ExtendsUser: MulOp, .ExtendA: {.SrcType: LHSInputType, .Kind: LHSExtendKind}, .ExtendB: {.SrcType: RHSInputType, .Kind: RHSExtendKind}};
5254}
5255
5256/// Examines each operation in the reduction chain corresponding to \p RedPhiR,
5257/// and determines if the target can use a cheaper operation with a wider
5258/// per-iteration input VF and narrower PHI VF. If successful, returns the chain
5259/// of operations in the reduction.
5260static std::optional<SmallVector<VPPartialReductionChain>>
5261getScaledReductions(VPReductionPHIRecipe *RedPhiR) {
5262 // Get the backedge value from the reduction PHI and find the
5263 // ComputeReductionResult that uses it (directly or through a select for
5264 // predicated reductions).
5265 auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR);
5266 if (!RdxResult)
5267 return std::nullopt;
5268 VPValue *ExitValue = RdxResult->getOperand(N: 0);
5269 match(V: ExitValue, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: ExitValue), Op2: m_VPValue()));
5270
5271 SmallVector<VPPartialReductionChain> Chain;
5272 RecurKind RK = RedPhiR->getRecurrenceKind();
5273 Type *PhiType = RedPhiR->getScalarType();
5274 TypeSize PHISize = PhiType->getPrimitiveSizeInBits();
5275
5276 // Work backwards from the ExitValue examining each reduction operation.
5277 VPValue *CurrentValue = ExitValue;
5278 while (CurrentValue != RedPhiR) {
5279 VPBlendRecipe *Blend = dyn_cast<VPBlendRecipe>(Val: CurrentValue);
5280 std::optional<unsigned> BlendReductionIdx;
5281 if (Blend) {
5282 assert(!Blend->isNormalized() && "Expect Blend not to be normalized.");
5283 if (Blend->getNumIncomingValues() != 2)
5284 return std::nullopt;
5285
5286 BlendReductionIdx = getBlendReductionUpdateValueIdx(Blend);
5287 if (!BlendReductionIdx)
5288 return std::nullopt;
5289
5290 CurrentValue = Blend->getIncomingValue(Idx: *BlendReductionIdx);
5291 }
5292
5293 auto *UpdateR = dyn_cast<VPWidenRecipe>(Val: CurrentValue);
5294 if (!UpdateR || !Instruction::isBinaryOp(Opcode: UpdateR->getOpcode()))
5295 return std::nullopt;
5296
5297 VPValue *Op = UpdateR->getOperand(N: 1);
5298 VPValue *PrevValue = UpdateR->getOperand(N: 0);
5299
5300 // Find the extended operand. The other operand (PrevValue) is the next link
5301 // in the reduction chain.
5302 std::optional<ExtendedReductionOperand> ExtendedOp =
5303 matchExtendedReductionOperand(UpdateR, Op);
5304 if (!ExtendedOp) {
5305 ExtendedOp = matchExtendedReductionOperand(UpdateR, Op: PrevValue);
5306 if (!ExtendedOp)
5307 return std::nullopt;
5308 std::swap(a&: Op, b&: PrevValue);
5309 }
5310
5311 // Look for VPBlend(reduce(PrevValue, Op), PrevValue), where
5312 // reduce is equal to CurrentValue. This can be lowered as
5313 // a conditional reduction by hoisting the select to the inputs.
5314 if (Blend && Blend->getIncomingValue(Idx: 1 - *BlendReductionIdx) != PrevValue)
5315 return std::nullopt;
5316
5317 Type *ExtSrcType = ExtendedOp->ExtendA.SrcType;
5318 TypeSize ExtSrcSize = ExtSrcType->getPrimitiveSizeInBits();
5319 if (!PHISize.hasKnownScalarFactor(RHS: ExtSrcSize))
5320 return std::nullopt;
5321
5322 VPPartialReductionChain Link(
5323 {.ReductionBinOp: UpdateR, .ExtendedOp: *ExtendedOp, .RK: RK,
5324 .AccumulatorOpIdx: PrevValue == UpdateR->getOperand(N: 0) ? 0U : 1U,
5325 .ScaleFactor: static_cast<unsigned>(PHISize.getKnownScalarFactor(RHS: ExtSrcSize)),
5326 .Blend: Blend});
5327 Chain.push_back(Elt: Link);
5328 CurrentValue = PrevValue;
5329 }
5330
5331 // The chain links were collected by traversing backwards from the exit value.
5332 // Reverse the chains so they are in program order.
5333 std::reverse(first: Chain.begin(), last: Chain.end());
5334 return Chain;
5335}
5336} // namespace
5337
5338void VPlanTransforms::createPartialReductions(VPlan &Plan,
5339 VPCostContext &CostCtx,
5340 VFRange &Range) {
5341 // Find all possible valid partial reductions, grouping chains by their PHI.
5342 // This grouping allows invalidating the whole chain, if any link is not a
5343 // valid partial reduction.
5344 MapVector<VPReductionPHIRecipe *, SmallVector<VPPartialReductionChain>>
5345 ChainsByPhi;
5346 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
5347 SmallVector<VPReductionPHIRecipe *, 4> UnorderedReductions;
5348 for (VPReductionPHIRecipe &RedPhiR :
5349 make_isa_range<VPReductionPHIRecipe>(Range: HeaderVPBB->phis())) {
5350 if (auto Chains = getScaledReductions(RedPhiR: &RedPhiR))
5351 ChainsByPhi.try_emplace(Key: &RedPhiR, Args: std::move(*Chains));
5352 else if (UsePartialReductionsByDefault &&
5353 (RedPhiR.getRecurrenceKind() == RecurKind::Add ||
5354 (RedPhiR.getRecurrenceKind() == RecurKind::FAdd &&
5355 !RedPhiR.isOrdered() && !RedPhiR.isInLoop())))
5356 UnorderedReductions.push_back(Elt: &RedPhiR);
5357 }
5358
5359 // For general unordered reductions which aren't part of a candidate chain for
5360 // a scaled partial reduction, we can still use the intrinsic to allow for
5361 // more optimization later on.
5362 for (auto *Rdx : UnorderedReductions) {
5363 auto *Backedge = dyn_cast<VPWidenRecipe>(Val: Rdx->getBackedgeValue());
5364 VPValue *OtherOp;
5365 if (!Backedge ||
5366 !match(V: Backedge,
5367 P: m_CombineOr(Ps: m_c_FAdd(Op0: m_Specific(VPV: Rdx), Op1: m_VPValue(V&: OtherOp)),
5368 Ps: m_c_Add(Op0: m_Specific(VPV: Rdx), Op1: m_VPValue(V&: OtherOp)))))
5369 continue;
5370
5371 // If the target indicates that the intrinsic is as cheap as (or cheaper
5372 // than) the add, then prefer the intrinsic.
5373 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
5374 Predicate: [&CostCtx, Rdx, Backedge](ElementCount VF) {
5375 InstructionCost CurrentCost = Backedge->computeCost(VF, Ctx&: CostCtx);
5376 Type *ScalarTy = Backedge->getScalarType();
5377 auto FMF = ScalarTy->isFloatingPointTy()
5378 ? std::make_optional(t: Rdx->getFastMathFlagsOrNone())
5379 : std::nullopt;
5380
5381 InstructionCost PRCost = CostCtx.TTI.getPartialReductionCost(
5382 Opcode: Backedge->getOpcode(), InputTypeA: ScalarTy, /*InputTypeB=*/nullptr,
5383 AccumType: ScalarTy, VF, OpAExtend: TTI::PR_None, OpBExtend: TTI::PR_None,
5384 /*BinOp=*/std::nullopt, CostKind: CostCtx.CostKind, FMF);
5385 return PRCost <= CurrentCost;
5386 },
5387 Range))
5388 continue;
5389
5390 auto *Partial = new VPReductionRecipe(
5391 Rdx->getRecurrenceKind(), Rdx->getFastMathFlagsOrNone(),
5392 Backedge->getUnderlyingInstr(), Rdx, OtherOp, nullptr,
5393 getReductionStyle(/*InLoop=*/false, /*Ordered=*/false,
5394 /*ScaleFactor=*/1));
5395 Partial->insertBefore(InsertPos: Backedge);
5396 Backedge->replaceAllUsesWith(New: Partial);
5397 Backedge->eraseFromParent();
5398 }
5399
5400 if (ChainsByPhi.empty())
5401 return;
5402
5403 // Build set of partial reduction operations and blends for user validation
5404 // and a map of reduction bin ops to their scale factors for scale validation.
5405 SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
5406 SmallPtrSet<VPBlendRecipe *, 4> PartialReductionBlends;
5407 DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
5408 for (const auto &[_, Chains] : ChainsByPhi)
5409 for (const VPPartialReductionChain &Chain : Chains) {
5410 PartialReductionOps.insert(Ptr: Chain.ExtendedOp.ExtendsUser);
5411 if (Chain.Blend)
5412 PartialReductionBlends.insert(Ptr: Chain.Blend);
5413 ScaledReductionMap[Chain.ReductionBinOp] = Chain.ScaleFactor;
5414 }
5415
5416 // A partial reduction is invalid if any of its extends are used by
5417 // something that isn't another partial reduction. This is because the
5418 // extends are intended to be lowered along with the reduction itself.
5419 auto ExtendUsersValid = [&](VPValue *Ext) {
5420 return !isa<VPWidenCastRecipe>(Val: Ext) || all_of(Range: Ext->users(), P: [&](VPUser *U) {
5421 return PartialReductionOps.contains(Ptr: cast<VPRecipeBase>(Val: U));
5422 });
5423 };
5424
5425 auto IsProfitablePartialReductionChainForVF =
5426 [&](ArrayRef<VPPartialReductionChain> Chain, ElementCount VF) -> bool {
5427 InstructionCost PartialCost = 0, RegularCost = 0;
5428
5429 // The chain is a profitable partial reduction chain if the cost of handling
5430 // the entire chain is cheaper when using partial reductions than when
5431 // handling the entire chain using regular reductions.
5432 for (const VPPartialReductionChain &Link : Chain) {
5433 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
5434 InstructionCost LinkCost = getPartialReductionLinkCost(CostCtx, Link, VF);
5435 if (!LinkCost.isValid())
5436 return false;
5437
5438 PartialCost += LinkCost;
5439 RegularCost += Link.ReductionBinOp->computeCost(VF, Ctx&: CostCtx);
5440 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
5441 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
5442 RegularCost += ExtendedOp.ExtendsUser->computeCost(VF, Ctx&: CostCtx);
5443 for (VPValue *Op : ExtendedOp.ExtendsUser->operands())
5444 if (auto *Extend = dyn_cast<VPWidenCastRecipe>(Val: Op))
5445 RegularCost += Extend->computeCost(VF, Ctx&: CostCtx);
5446 }
5447 return PartialCost.isValid() && PartialCost < RegularCost;
5448 };
5449
5450 // Validate chains: check that extends are only used by partial reductions,
5451 // and that reduction bin ops are only used by other partial reductions with
5452 // matching scale factors, are outside the loop region or the select
5453 // introduced by tail-folding. Otherwise we would create users of scaled
5454 // reductions where the types of the other operands don't match.
5455 for (auto &[RedPhiR, Chains] : ChainsByPhi) {
5456 for (const VPPartialReductionChain &Chain : Chains) {
5457 if (!all_of(Range: Chain.ExtendedOp.ExtendsUser->operands(), P: ExtendUsersValid)) {
5458 Chains.clear();
5459 break;
5460 }
5461 auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
5462 if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: U))
5463 return PhiR == RedPhiR;
5464 auto *R = cast<VPSingleDefRecipe>(Val: U);
5465
5466 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: R))
5467 return Blend == Chain.Blend || PartialReductionBlends.contains(Ptr: Blend);
5468
5469 return Chain.ScaleFactor == ScaledReductionMap.lookup_or(Val: R, Default: 0) ||
5470 match(R, P: m_ComputeReductionResult(
5471 Op0: m_Specific(VPV: Chain.ReductionBinOp))) ||
5472 match(R, P: m_Select(Op0: m_VPValue(), Op1: m_Specific(VPV: Chain.ReductionBinOp),
5473 Op2: m_Specific(VPV: RedPhiR)));
5474 };
5475 if (!all_of(Range: Chain.ReductionBinOp->users(), P: UseIsValid)) {
5476 Chains.clear();
5477 break;
5478 }
5479
5480 // Check if the compute-reduction-result is used by a sunk store.
5481 // TODO: Also form partial reductions in those cases.
5482 if (auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR)) {
5483 if (any_of(Range: RdxResult->users(), P: [](VPUser *U) {
5484 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
5485 return RepR && RepR->getOpcode() == Instruction::Store;
5486 })) {
5487 Chains.clear();
5488 break;
5489 }
5490 }
5491 }
5492
5493 // Clear the chain if it is not profitable.
5494 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
5495 Predicate: [&, &Chains = Chains](ElementCount VF) {
5496 return IsProfitablePartialReductionChainForVF(Chains, VF);
5497 },
5498 Range))
5499 Chains.clear();
5500 }
5501
5502 for (auto &[Phi, Chains] : ChainsByPhi)
5503 for (const VPPartialReductionChain &Chain : Chains)
5504 transformToPartialReduction(Chain, Plan, RdxPhi: Phi);
5505}
5506
5507void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
5508 VPRecipeBuilder &RecipeBuilder,
5509 VPCostContext &CostCtx) {
5510 // Collect all loads/stores first. We will start with ones having simpler
5511 // decisions followed by more complex ones that are potentially
5512 // guided/dependent on the simpler ones.
5513 SmallVector<VPInstruction *> MemOps;
5514 for (VPBasicBlock *VPBB :
5515 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
5516 G: Plan.getVectorLoopRegion()->getEntryBasicBlock()))) {
5517 for (VPInstruction &VPI : make_isa_range<VPInstruction>(Range&: *VPBB)) {
5518 if (VPI.getUnderlyingValue() &&
5519 is_contained(Set: {Instruction::Load, Instruction::Store},
5520 Element: VPI.getOpcode()))
5521 MemOps.push_back(Elt: &VPI);
5522 }
5523 }
5524
5525 // Few helpers to process different kinds of memory operations.
5526
5527 // To be used as argument to `VPlanTransforms::runPass` which explicitly
5528 // specified pass name, hence `VPlan &` parameter.
5529 auto ProcessSubset = [&](VPlan &, auto ProcessVPInst) {
5530 SmallVector<VPInstruction *> RemainingMemOps;
5531 for (VPInstruction *VPI : MemOps) {
5532 if (!ProcessVPInst(VPI))
5533 RemainingMemOps.push_back(Elt: VPI);
5534 }
5535
5536 MemOps.clear();
5537 std::swap(LHS&: MemOps, RHS&: RemainingMemOps);
5538 };
5539
5540 auto ReplaceWith = [&](VPInstruction *VPI, VPRecipeBase *New) {
5541 assert(New->getParent() && "New recipe must have been inserted");
5542 if (VPI->getOpcode() == Instruction::Load)
5543 VPI->replaceAllUsesWith(New: New->getVPSingleValue());
5544 VPI->eraseFromParent();
5545
5546 // VPI has been processed.
5547 return true;
5548 };
5549
5550 auto Scalarize = [&](VPInstruction *VPI) {
5551 return ReplaceWith(VPI, VPBuilder(VPI).insert(
5552 R: RecipeBuilder.handleReplication(VPI, Range)));
5553 };
5554
5555 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
5556 VPBuilder FinalRedStoresBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
5557 VPlanTransforms::runPass(
5558 PassName: "lowerMemoryIdioms", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5559 if (RecipeBuilder.replaceWithFinalIfReductionStore(
5560 VPI, FinalRedStoresBuilder))
5561 return true;
5562
5563 // Filter out scalar VPlan for the remaining idioms.
5564 if (LoopVectorizationPlanner::getDecisionAndClampRange(
5565 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
5566 return false;
5567
5568 if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI))
5569 return ReplaceWith(VPI, VPBuilder(VPI).insert(R: Histogram));
5570
5571 return false;
5572 });
5573
5574 // Filter out scalar VPlan for the remaining memory operations.
5575 if (LoopVectorizationPlanner::getDecisionAndClampRange(
5576 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
5577 return;
5578
5579 // If the instruction's allocated size doesn't equal it's type size, it
5580 // requires padding and will be scalarized.
5581 VPlanTransforms::runPass(
5582 PassName: "scalarizeMemOpsWithIrregularTypes", Pass&: ProcessSubset, Plan,
5583 Args: [&](VPInstruction *VPI) {
5584 Instruction *I = VPI->getUnderlyingInstr();
5585 if (hasIrregularType(Ty: getLoadStoreType(I), DL: I->getDataLayout()))
5586 return Scalarize(VPI);
5587
5588 return false;
5589 });
5590
5591 if (!RecipeBuilder.prefersVectorizedAddressing()) {
5592 VPlanTransforms::runPass(
5593 PassName: "makeVPlanMemOpDecision", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5594 Instruction *I = VPI->getUnderlyingInstr();
5595 bool IsLoad = VPI->getOpcode() == Instruction::Load;
5596 if (RecipeBuilder.isPredicatedInst(I) || !IsLoad ||
5597 !vputils::isUsedByLoadStoreAddress(V: VPI))
5598 return false;
5599
5600 // Scalarize loads used as addresses, matching the legacy CM. The load
5601 // is single-scalar if the pointer is loop-invariant, otherwise it is
5602 // replicated per-lane. No mask is needed as the load is not
5603 // predicated.
5604 VPValue *Ptr = VPI->getOperand(N: 0);
5605 const SCEV *PtrSCEV =
5606 vputils::getSCEVExprForVPValue(V: Ptr, PSE&: CostCtx.PSE, L: CostCtx.L);
5607 bool IsSingleScalarLoad =
5608 !isa<SCEVCouldNotCompute>(Val: PtrSCEV) &&
5609 CostCtx.PSE.getSE()->isLoopInvariant(S: PtrSCEV, L: CostCtx.L);
5610
5611 ReplaceWith(VPI,
5612 VPBuilder(VPI).insert(R: new VPReplicateRecipe(
5613 I, Ptr, /*IsSingleScalar=*/IsSingleScalarLoad,
5614 /*Mask=*/nullptr, *VPI, *VPI, VPI->getDebugLoc())));
5615 return true;
5616 });
5617 }
5618
5619 // Widen unit-stride consecutive accesses, matching the legacy CM. Both
5620 // forward (stride +1) and reverse (stride -1) accesses are handled.
5621 VPlanTransforms::runPass(
5622 PassName: "widenConsecutiveMemOps", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5623 Instruction *I = VPI->getUnderlyingInstr();
5624 bool IsLoad = VPI->getOpcode() == Instruction::Load;
5625 VPValue *Ptr = VPI->getOperand(N: !IsLoad);
5626 Type *ScalarTy =
5627 IsLoad ? VPI->getScalarType() : VPI->getOperand(N: 0)->getScalarType();
5628 std::optional<int64_t> Stride =
5629 getConstantStride(Addr: Ptr, AccessTy: ScalarTy, PSE&: CostCtx.PSE, L: CostCtx.L);
5630 if (Stride != 1 && Stride != -1)
5631 return false;
5632 bool Reverse = Stride == -1;
5633
5634 // A predicated access can only be widened (rather than scalarized) if
5635 // the target supports a masked load/store for it.
5636 // TODO: Determine if a load/store needs predication directly in VPlan.
5637 bool IsPredicated = RecipeBuilder.isPredicatedInst(I);
5638 if (IsPredicated && !CostCtx.Config.isLegalMaskedLoadOrStore(
5639 IsLoad, ScalarTy, Alignment: getLoadStoreAlignment(I),
5640 AddressSpace: getLoadStoreAddressSpace(I)))
5641 return false;
5642
5643 VPBuilder Builder(VPI);
5644 VPSingleDefRecipe *VectorPtr = Builder.createConsecutiveVectorPointer(
5645 Ptr, SourceElementTy: ScalarTy, Reverse, DL: VPI->getDebugLoc());
5646
5647 VPValue *Mask = IsPredicated ? VPI->getMask() : nullptr;
5648 // Reverse the mask so it matches the reversed access order.
5649 if (Reverse && Mask)
5650 Mask = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Mask,
5651 DL: VPI->getDebugLoc());
5652
5653 if (IsLoad) {
5654 VPSingleDefRecipe *Load = Builder.createWidenLoad(
5655 Load&: *cast<LoadInst>(Val: I), Addr: VectorPtr, Mask,
5656 /*Consecutive=*/true, Metadata: *VPI, DL: VPI->getDebugLoc());
5657 // Reverse the loaded values back into program order.
5658 if (Reverse)
5659 Load = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Load,
5660 DL: VPI->getDebugLoc());
5661 return ReplaceWith(VPI, Load);
5662 }
5663
5664 VPValue *StoredVal = VPI->getOperand(N: 0);
5665 if (Reverse)
5666 // Reverse the stored values so they are written in descending order.
5667 StoredVal = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: StoredVal,
5668 DL: VPI->getDebugLoc());
5669
5670 auto *StoreR = Builder.createWidenStore(
5671 Store&: *cast<StoreInst>(Val: I), Addr: VectorPtr, StoredVal, Mask,
5672 /*Consecutive=*/true, Metadata: *VPI, DL: VPI->getDebugLoc());
5673 return ReplaceWith(VPI, StoreR);
5674 });
5675
5676 VPlanTransforms::runPass(PassName: "delegateMemOpWideningToLegacyCM", Pass&: ProcessSubset,
5677 Plan, Args: [&](VPInstruction *VPI) {
5678 if (VPRecipeBase *Recipe =
5679 RecipeBuilder.tryToWidenMemory(VPI, Range))
5680 return ReplaceWith(VPI, Recipe);
5681
5682 return Scalarize(VPI);
5683 });
5684}
5685
5686void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
5687 if (LoopVectorizationPlanner::getDecisionAndClampRange(
5688 Predicate: [&](ElementCount VF) { return VF.isScalar(); }, Range))
5689 return;
5690
5691 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
5692 Plan.getEntry());
5693 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
5694 for (VPInstruction &VPI :
5695 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range: reverse(C&: *VPBB)))) {
5696 auto *I = cast_or_null<Instruction>(Val: VPI.getUnderlyingValue());
5697 // Wouldn't be able to create a `VPReplicateRecipe` anyway.
5698 if (!I)
5699 continue;
5700
5701 // If executing other lanes produces side-effects we can't avoid them.
5702 if (VPI.mayHaveSideEffects())
5703 continue;
5704
5705 // We want to drop the mask operand, verify we can safely do that.
5706 if (VPI.isMasked() && !VPI.isSafeToSpeculativelyExecute())
5707 continue;
5708
5709 // Avoid rewriting IV increment as that interferes with
5710 // `removeRedundantCanonicalIVs`.
5711 if (VPI.getOpcode() == Instruction::Add &&
5712 any_of(Range: VPI.operands(), P: IsaPred<VPWidenIntOrFpInductionRecipe>))
5713 continue;
5714
5715 // Other lanes are needed - can't drop them.
5716 if (!vputils::onlyFirstLaneUsed(Def: &VPI))
5717 continue;
5718
5719 auto *Recipe = VPBuilder::createSingleScalarOp(
5720 Opcode: VPI.getOpcode(), Operands: VPI.operandsWithoutMask(), /*Mask=*/nullptr, Flags: VPI,
5721 Metadata: VPI, DL: VPI.getDebugLoc(), UV: I);
5722 Recipe->insertBefore(InsertPos: &VPI);
5723 VPI.replaceAllUsesWith(New: Recipe);
5724 VPI.eraseFromParent();
5725 }
5726 }
5727}
5728
5729/// Returns true if \p Info's parameter kinds are compatible with \p Args.
5730static bool areVFParamsOk(const VFInfo &Info, ArrayRef<VPValue *> Args,
5731 PredicatedScalarEvolution &PSE, const Loop *L) {
5732 ScalarEvolution *SE = PSE.getSE();
5733 return all_of(Range: Info.Shape.Parameters, P: [&](VFParameter Param) {
5734 switch (Param.ParamKind) {
5735 case VFParamKind::Vector:
5736 case VFParamKind::GlobalPredicate:
5737 return true;
5738 case VFParamKind::OMP_Uniform:
5739 return SE->isSCEVable(Ty: Args[Param.ParamPos]->getScalarType()) &&
5740 SE->isLoopInvariant(
5741 S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
5742 L);
5743 case VFParamKind::OMP_Linear:
5744 return match(S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
5745 P: m_scev_AffineAddRec(
5746 Op0: m_SCEV(), Op1: m_scev_SpecificSInt(V: Param.LinearStepOrPos),
5747 L: m_SpecificLoop(L)));
5748 default:
5749 return false;
5750 }
5751 });
5752}
5753
5754/// Find a vector variant of \p CI for \p VF, respecting \p MaskRequired.
5755/// Returns the variant function, or nullptr. Masked variants are assumed to
5756/// take the mask as a trailing parameter.
5757static Function *findVectorVariant(CallInst *CI, ArrayRef<VPValue *> Args,
5758 ElementCount VF, bool MaskRequired,
5759 PredicatedScalarEvolution &PSE,
5760 const Loop *L) {
5761 if (CI->isNoBuiltin())
5762 return nullptr;
5763 auto Mappings = VFDatabase::getMappings(CI: *CI);
5764 const auto *It = find_if(Range&: Mappings, P: [&](const VFInfo &Info) {
5765 return Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()) &&
5766 areVFParamsOk(Info, Args, PSE, L);
5767 });
5768 if (It == Mappings.end())
5769 return nullptr;
5770 return CI->getModule()->getFunction(Name: It->VectorName);
5771}
5772
5773namespace {
5774/// The outcome of choosing how to widen a call at a given VF.
5775struct CallWideningDecision {
5776 enum class KindTy { Scalarize, Intrinsic, VectorVariant };
5777 CallWideningDecision(KindTy Kind, Function *Variant = nullptr)
5778 : Kind(Kind), Variant(Variant) {}
5779 KindTy Kind;
5780
5781 /// Set when Kind == VectorVariant.
5782 Function *Variant;
5783
5784 bool operator==(const CallWideningDecision &Other) const {
5785 return Kind == Other.Kind && Variant == Other.Variant;
5786 }
5787};
5788} // namespace
5789
5790/// Pick the cheapest widening for the call \p VPI at \p VF among scalarization,
5791/// vector intrinsic, and vector library variant.
5792static CallWideningDecision decideCallWidening(VPInstruction &VPI,
5793 ArrayRef<VPValue *> Ops,
5794 ElementCount VF,
5795 VPCostContext &CostCtx) {
5796 auto *CI = cast<CallInst>(Val: VPI.getUnderlyingInstr());
5797
5798 // Scalar VFs and calls forced or known to scalarize always replicate.
5799 if (VF.isScalar() || CostCtx.willBeScalarized(I: CI, VF))
5800 return CallWideningDecision::KindTy::Scalarize;
5801
5802 auto *CalledFn = cast<Function>(
5803 Val: VPI.getOperand(N: VPI.getNumOperandsWithoutMask() - 1)->getLiveInIRValue());
5804 Type *ResultTy = VPI.getScalarType();
5805 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
5806 bool MaskRequired = CostCtx.isMaskRequired(I: CI);
5807
5808 // Pseudo intrinsics (assume, lifetime, ...) are always scalarized.
5809 if (ID && VPCostContext::isFreeScalarIntrinsic(ID))
5810 return CallWideningDecision::KindTy::Scalarize;
5811
5812 InstructionCost ScalarCost =
5813 VPReplicateRecipe::computeCallCost(CalledFn, ResultTy, ArgOps: Ops,
5814 /*IsSingleScalar=*/false, VF, Ctx&: CostCtx);
5815
5816 Function *VecFunc =
5817 findVectorVariant(CI, Args: Ops, VF, MaskRequired, PSE&: CostCtx.PSE, L: CostCtx.L);
5818 InstructionCost VecCallCost = InstructionCost::getInvalid();
5819 if (VecFunc)
5820 VecCallCost = VPWidenCallRecipe::computeCallCost(Variant: VecFunc, Ctx&: CostCtx);
5821
5822 // Prefer the intrinsic if it is at least as cheap as scalarizing and any
5823 // available vector variant.
5824 if (ID) {
5825 InstructionCost IntrinsicCost =
5826 VPWidenIntrinsicRecipe::computeCallCost(ID, Operands: Ops, R: VPI, VF, Ctx&: CostCtx);
5827 if (IntrinsicCost.isValid() && ScalarCost >= IntrinsicCost &&
5828 (!VecFunc || VecCallCost >= IntrinsicCost))
5829 return CallWideningDecision::KindTy::Intrinsic;
5830 }
5831
5832 // Otherwise, use a vector library variant when it beats scalarizing.
5833 if (VecFunc && ScalarCost >= VecCallCost)
5834 return {CallWideningDecision::KindTy::VectorVariant, VecFunc};
5835
5836 return CallWideningDecision::KindTy::Scalarize;
5837}
5838
5839void VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
5840 VPRecipeBuilder &RecipeBuilder,
5841 VPCostContext &CostCtx) {
5842 for (VPBasicBlock *VPBB : VPBlockUtils::blocksAs<VPBasicBlock>(
5843 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
5844 for (VPInstruction &VPI :
5845 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range&: *VPBB))) {
5846 if (!VPI.getUnderlyingValue() || VPI.getOpcode() != Instruction::Call)
5847 continue;
5848
5849 auto *CI = cast<CallInst>(Val: VPI.getUnderlyingInstr());
5850 SmallVector<VPValue *, 4> Ops(VPI.op_begin(),
5851 VPI.op_begin() + CI->arg_size());
5852
5853 CallWideningDecision Decision =
5854 decideCallWidening(VPI, Ops, VF: Range.Start, CostCtx);
5855 LoopVectorizationPlanner::getDecisionAndClampRange(
5856 Predicate: [&](ElementCount VF) {
5857 return Decision == decideCallWidening(VPI, Ops, VF, CostCtx);
5858 },
5859 Range);
5860
5861 VPSingleDefRecipe *Replacement = nullptr;
5862 switch (Decision.Kind) {
5863 case CallWideningDecision::KindTy::Intrinsic: {
5864 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
5865 Type *ResultTy = VPI.getScalarType();
5866 Replacement = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, VPI,
5867 VPI, VPI.getDebugLoc());
5868 break;
5869 }
5870 case CallWideningDecision::KindTy::VectorVariant: {
5871 // Masked variants take the mask as a trailing parameter, so they have
5872 // one more parameter than the original call's arguments.
5873 if (Decision.Variant->arg_size() > Ops.size()) {
5874 VPValue *Mask = VPI.isMasked() ? VPI.getMask() : Plan.getTrue();
5875 Ops.push_back(Elt: Mask);
5876 }
5877 Ops.push_back(Elt: VPI.getOperand(N: VPI.getNumOperandsWithoutMask() - 1));
5878 Replacement = new VPWidenCallRecipe(CI, Decision.Variant, Ops, VPI, VPI,
5879 VPI.getDebugLoc());
5880 break;
5881 }
5882 case CallWideningDecision::KindTy::Scalarize:
5883 Replacement = RecipeBuilder.handleReplication(VPI: &VPI, Range);
5884 break;
5885 }
5886
5887 Replacement->insertBefore(InsertPos: &VPI);
5888 VPI.replaceAllUsesWith(New: Replacement);
5889 VPI.eraseFromParent();
5890 }
5891 }
5892}
5893
5894void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
5895 PredicatedScalarEvolution &PSE,
5896 Loop &L, VPCostContext &Ctx,
5897 VFRange &Range) {
5898 if (Plan.hasScalarVFOnly())
5899 return;
5900
5901 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
5902 VPValue *I32VF = nullptr;
5903 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
5904 Range: vp_depth_first_shallow(G: VectorLoop->getEntry()))) {
5905 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
5906 auto *MemR = dyn_cast<VPWidenMemoryRecipe>(Val: &R);
5907 // TODO: Transform reverse access into strided access with -1 stride.
5908 // TODO: Transform gather/scatter with uniform address into strided access
5909 // with 0 stride.
5910 // TODO: Transform interleave access into multiple strided accesses.
5911 if (!MemR || MemR->isConsecutive())
5912 continue;
5913
5914 VPValue *Ptr = MemR->getAddr();
5915 // Check if this is a strided access by analyzing the address SCEV for an
5916 // affine addRec.
5917 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: &L);
5918 const SCEV *Start;
5919 const SCEVConstant *Step;
5920 // TODO: Support non-constant loop invariant stride.
5921 if (!match(S: PtrSCEV,
5922 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEVConstant(V&: Step),
5923 L: m_SpecificLoop(L: &L))))
5924 continue;
5925
5926 VPValue *StoredValue = nullptr;
5927 Type *DataTy;
5928 Intrinsic::ID IntrinID;
5929 if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Val: &R)) {
5930 StoredValue = StoreR->getStoredValue();
5931 DataTy = StoredValue->getScalarType();
5932 IntrinID = Intrinsic::experimental_vp_strided_store;
5933 } else {
5934 auto *LoadR = cast<VPWidenLoadRecipe>(Val: &R);
5935 DataTy = LoadR->getScalarType();
5936 IntrinID = Intrinsic::experimental_vp_strided_load;
5937 }
5938
5939 Align Alignment = MemR->getAlign();
5940 auto IsProfitable = [&](ElementCount VF) {
5941 Type *VectorTy = toVectorTy(Scalar: DataTy, EC: VF);
5942 if (!Ctx.TTI.isLegalStridedLoadStore(DataType: VectorTy, Alignment))
5943 return false;
5944 const InstructionCost CurrentCost = MemR->computeCost(VF, Ctx);
5945 const InstructionCost StridedLoadStoreCost =
5946 VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost(
5947 IID: IntrinID, Ty: VectorTy, IsMasked: MemR->isMasked(), Alignment, Ctx);
5948 return StridedLoadStoreCost < CurrentCost;
5949 };
5950
5951 if (!LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsProfitable,
5952 Range))
5953 continue;
5954
5955 // Invalidate the legacy widening decision so the cost of replaced load is
5956 // not counted during precomputeCosts.
5957 // TODO: Remove once the legacy exit cost computation is retired.
5958 for (ElementCount VF : Range)
5959 Ctx.invalidateWideningDecision(I: &MemR->getIngredient(), VF);
5960
5961 // Get VF as i32 for the vector length operand.
5962 if (!I32VF) {
5963 VPBuilder Builder(Plan.getVectorPreheader());
5964 I32VF = Builder.createScalarZExtOrTrunc(
5965 Op: &Plan.getVF(), ResultTy: Type::getInt32Ty(C&: Plan.getContext()),
5966 DL: DebugLoc::getUnknown());
5967 }
5968
5969 VPBuilder Builder(&R);
5970 // Create the base pointer of strided access.
5971 // TODO: reuse VPDerivedIVRecipe for base pointer computation when it
5972 // supports a general VPValue as the start value.
5973 VPValue *StartVPV =
5974 VPSCEVExpander(Builder, *PSE.getSE(), R.getDebugLoc()).expand(S: Start);
5975 VPValue *StrideInBytes = Plan.getOrAddLiveIn(V: Step->getValue());
5976 Type *IndexTy = Plan.getDataLayout().getIndexType(PtrTy: Ptr->getScalarType());
5977 assert(IndexTy == StrideInBytes->getScalarType() &&
5978 "Stride type from SCEV must match the index type");
5979 VPValue *CanIV = Builder.createScalarZExtOrTrunc(
5980 Op: VectorLoop->getCanonicalIV(), ResultTy: IndexTy, DL: DebugLoc::getUnknown());
5981 auto *AddRecPtr = cast<SCEVAddRecExpr>(Val: PtrSCEV);
5982 auto *Offset = Builder.createOverflowingOp(
5983 Opcode: Instruction::Mul, Operands: {CanIV, StrideInBytes},
5984 WrapFlags: {AddRecPtr->hasNoUnsignedWrap(), /*HasNSW=*/false});
5985 GEPNoWrapFlags NWFlags = AddRecPtr->hasNoUnsignedWrap()
5986 ? GEPNoWrapFlags::noUnsignedWrap()
5987 : GEPNoWrapFlags::none();
5988 VPValue *BasePtr = Builder.createNoWrapPtrAdd(Ptr: StartVPV, Offset, GEPFlags: NWFlags);
5989
5990 // Create a new vector pointer for strided access.
5991 VPValue *NewPtr = Builder.createVectorPointer(
5992 Ptr: BasePtr, SourceElementTy: Type::getInt8Ty(C&: Plan.getContext()), Stride: StrideInBytes, GEPFlags: NWFlags,
5993 DL: R.getDebugLoc());
5994
5995 VPValue *Mask = MemR->getMask();
5996 if (!Mask)
5997 Mask = Plan.getTrue();
5998 SmallVector<VPValue *, 5> Ops;
5999 if (StoredValue)
6000 Ops.push_back(Elt: StoredValue);
6001 Ops.append(IL: {NewPtr, StrideInBytes, Mask, I32VF});
6002
6003 auto *StridedR = Builder.createWidenMemIntrinsic(
6004 VectorIntrinsicID: IntrinID, CallArguments: Ops,
6005 Ty: StoredValue ? Type::getVoidTy(C&: Plan.getContext()) : DataTy, Alignment,
6006 MD: *MemR, DL: R.getDebugLoc());
6007 if (!StoredValue)
6008 cast<VPWidenLoadRecipe>(Val: &R)->replaceAllUsesWith(New: StridedR);
6009 R.eraseFromParent();
6010 }
6011 }
6012}
6013