1//===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements a set of utility VPlan to VPlan transformations.
11///
12//===----------------------------------------------------------------------===//
13
14#include "VPlanTransforms.h"
15#include "VPRecipeBuilder.h"
16#include "VPlan.h"
17#include "VPlanAnalysis.h"
18#include "VPlanCFG.h"
19#include "VPlanDominatorTree.h"
20#include "VPlanHelpers.h"
21#include "VPlanPatternMatch.h"
22#include "VPlanUtils.h"
23#include "VectorizeOptions.h"
24#include "llvm/ADT/APInt.h"
25#include "llvm/ADT/PostOrderIterator.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SetVector.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/TypeSwitch.h"
30#include "llvm/Analysis/IVDescriptors.h"
31#include "llvm/Analysis/Loads.h"
32#include "llvm/Analysis/LoopInfo.h"
33#include "llvm/Analysis/MemoryLocation.h"
34#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
35#include "llvm/Analysis/ScopedNoAliasAA.h"
36#include "llvm/Analysis/VectorUtils.h"
37#include "llvm/IR/Intrinsics.h"
38#include "llvm/IR/MDBuilder.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/ProfDataUtils.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/Regex.h"
43#include "llvm/Support/TypeSize.h"
44#include "llvm/Transforms/Utils/LoopUtils.h"
45
46using namespace llvm;
47using namespace LoopVectorizationUtils;
48using namespace VPlanPatternMatch;
49using namespace SCEVPatternMatch;
50
51void VPlanTransforms::verifyIfRequested(VPlan &Plan) {
52 const VectorizeOptions &Opts =
53 Plan.getContext().getLibraryOptions().get<VectorizeOptions>();
54 if (Opts.vplan_verify_each && !verifyVPlanIsValid(Plan))
55 report_fatal_error(reason: "Broken VPlan found, compilation aborted!");
56}
57
58#ifndef NDEBUG
59bool VPlanTransforms::isPrintingRequested(const VPlan &Plan) {
60 const VectorizeOptions &Opts =
61 Plan.getContext().getLibraryOptions().get<VectorizeOptions>();
62 return Opts.vplan_print_before_all || Opts.vplan_print_after_all ||
63 !Opts.vplan_print_before.empty() || !Opts.vplan_print_after.empty();
64}
65
66void VPlanTransforms::printIfRequested(VPlan &Plan, Function *Fn,
67 StringRef NumberedPassName, bool After) {
68 const VectorizeOptions &Opts =
69 Plan.getContext().getLibraryOptions().get<VectorizeOptions>();
70 bool All = After ? Opts.vplan_print_after_all : Opts.vplan_print_before_all;
71 ArrayRef<StringRef> Passes =
72 After ? Opts.vplan_print_after : Opts.vplan_print_before;
73 if (!All && none_of(Passes, [&](StringRef Entry) {
74 return Regex(Entry).match(NumberedPassName);
75 }))
76 return;
77 dbgs() << "VPlan for loop in '" << Fn->getName() << "' "
78 << (After ? "after" : "before") << " " << NumberedPassName << '\n';
79 if (Opts.vplan_print_vector_region_scope && Plan.getVectorLoopRegion())
80 Plan.getVectorLoopRegion()->print(dbgs());
81 else
82 dbgs() << Plan << '\n';
83}
84#endif
85
86/// Returns the metadata attached to \p R, or an empty set for a recipe that
87/// does not carry any.
88static VPIRMetadata getMetadataOf(VPRecipeBase *R) {
89 if (auto *MD = dyn_cast<VPIRMetadata>(Val: R))
90 return *MD;
91 return {};
92}
93
94bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
95 VPlan &Plan, const TargetLibraryInfo &TLI, PredicatedScalarEvolution &PSE,
96 Loop *OuterLoop) {
97
98 // Returns true if the access of \p AccessTy at \p Addr can be widened to a
99 // consecutive vector access.
100 auto IsConsecutiveAccess = [&](VPValue *Addr, Type *AccessTy) {
101 return !hasIrregularType(Ty: AccessTy, DL: Plan.getDataLayout()) &&
102 vputils::getConstantStride(Addr, AccessTy, PSE, L: OuterLoop) == 1;
103 };
104
105 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
106 Plan.getVectorLoopRegion());
107 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
108 // Skip blocks outside region
109 if (!VPBB->getParent())
110 break;
111 VPRecipeBase *Term = VPBB->getTerminator();
112 auto EndIter = Term ? Term->getIterator() : VPBB->end();
113 // Introduce each ingredient into VPlan.
114 for (VPRecipeBase &Ingredient :
115 make_early_inc_range(Range: make_range(x: VPBB->begin(), y: EndIter))) {
116
117 VPValue *VPV = Ingredient.getVPSingleValue();
118 if (!VPV->getUnderlyingValue())
119 continue;
120
121 Instruction *Inst = cast<Instruction>(Val: VPV->getUnderlyingValue());
122
123 VPRecipeBase *NewRecipe = nullptr;
124 if (auto *PhiR = dyn_cast<VPPhi>(Val: &Ingredient)) {
125 auto *Phi = cast<PHINode>(Val: PhiR->getUnderlyingValue());
126 NewRecipe = new VPWidenPHIRecipe(PhiR->operands(), PhiR->getDebugLoc(),
127 Phi->getName());
128 } else if (auto *VPI = dyn_cast<VPInstruction>(Val: &Ingredient)) {
129 assert(!isa<PHINode>(Inst) && "phis should be handled above");
130 // Create VPWidenMemoryRecipe for loads and stores.
131 if (LoadInst *Load = dyn_cast<LoadInst>(Val: Inst)) {
132 bool IsConsecutive =
133 IsConsecutiveAccess(VPI->getOperand(N: 0), VPI->getScalarType());
134 NewRecipe = new VPWidenLoadRecipe(*Load, Ingredient.getOperand(N: 0),
135 nullptr /*Mask*/, IsConsecutive,
136 *VPI, Ingredient.getDebugLoc());
137 } else if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst)) {
138 bool IsConsecutive = IsConsecutiveAccess(
139 VPI->getOperand(N: 1), VPI->getOperand(N: 0)->getScalarType());
140 NewRecipe = new VPWidenStoreRecipe(
141 *Store, Ingredient.getOperand(N: 1), Ingredient.getOperand(N: 0),
142 nullptr /*Mask*/, IsConsecutive, *VPI, Ingredient.getDebugLoc());
143 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Inst)) {
144 NewRecipe = new VPWidenGEPRecipe(GEP->getSourceElementType(),
145 Ingredient.operands(), *VPI,
146 Ingredient.getDebugLoc(), GEP);
147 } else if (CallInst *CI = dyn_cast<CallInst>(Val: Inst)) {
148 Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, TLI: &TLI);
149 if (VectorID == Intrinsic::not_intrinsic)
150 return false;
151
152 // The noalias.scope.decl intrinsic declares a noalias scope that
153 // is valid for a single iteration. Emitting it as a single-scalar
154 // replicate would incorrectly extend the scope across multiple
155 // original iterations packed into one vector iteration.
156 // FIXME: If we want to vectorize this loop, then we have to drop
157 // all the associated !alias.scope and !noalias.
158 if (VectorID == Intrinsic::experimental_noalias_scope_decl)
159 return false;
160
161 // These intrinsics are recognized by getVectorIntrinsicIDForCall
162 // but are not widenable. Emit them as replicate instead of widening.
163 if (VectorID == Intrinsic::assume ||
164 VectorID == Intrinsic::lifetime_end ||
165 VectorID == Intrinsic::lifetime_start ||
166 VectorID == Intrinsic::sideeffect ||
167 VectorID == Intrinsic::pseudoprobe) {
168 // If the operand of llvm.assume holds before vectorization, it will
169 // also hold per lane.
170 // llvm.pseudoprobe requires to be duplicated per lane for accurate
171 // sample count.
172 const bool IsSingleScalar = VectorID != Intrinsic::assume &&
173 VectorID != Intrinsic::pseudoprobe;
174 NewRecipe = new VPReplicateRecipe(CI, Ingredient.operands(),
175 /*IsSingleScalar=*/IsSingleScalar,
176 /*Mask=*/nullptr, *VPI, *VPI,
177 Ingredient.getDebugLoc());
178 } else {
179 assert(!CI->hasOperandBundles() &&
180 "calls with operand bundles must not be widened");
181 NewRecipe = new VPWidenIntrinsicRecipe(
182 *CI, VectorID, drop_end(RangeOrContainer: Ingredient.operands()), CI->getType(),
183 VPIRFlags(*CI), *VPI, CI->getDebugLoc());
184 }
185 } else if (auto *CI = dyn_cast<CastInst>(Val: Inst)) {
186 NewRecipe = new VPWidenCastRecipe(
187 CI->getOpcode(), Ingredient.getOperand(N: 0), CI->getType(), CI,
188 VPIRFlags(*CI), VPIRMetadata(*CI));
189 } else {
190 NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI,
191 *VPI, Ingredient.getDebugLoc());
192 }
193 } else {
194 assert(isa<VPWidenIntOrFpInductionRecipe>(&Ingredient) &&
195 "inductions must be created earlier");
196 continue;
197 }
198
199 NewRecipe->insertBefore(InsertPos: &Ingredient);
200 if (NewRecipe->getNumDefinedValues() == 1)
201 VPV->replaceAllUsesWith(New: NewRecipe->getVPSingleValue());
202 else
203 assert(NewRecipe->getNumDefinedValues() == 0 &&
204 "Only recpies with zero or one defined values expected");
205 Ingredient.eraseFromParent();
206 }
207 }
208 return true;
209}
210
211/// Helper for extra no-alias checks via known-safe recipe and SCEV.
212class SinkStoreInfo {
213 SmallPtrSet<VPReplicateRecipe *, 4> ExcludeRecipes;
214 VPReplicateRecipe &GroupLeader;
215 PredicatedScalarEvolution *PSE = nullptr;
216 const Loop *L = nullptr;
217
218 // Return true if \p A and \p B are known to not alias for all VFs in the
219 // plan, checked via the distance between the accesses
220 bool isNoAliasViaDistance(VPReplicateRecipe *A, VPReplicateRecipe *B) const {
221 if (A->getOpcode() != Instruction::Store ||
222 B->getOpcode() != Instruction::Store)
223 return false;
224
225 if (!PSE || !L)
226 return A == B;
227
228 VPValue *AddrA = A->getOperand(N: 1);
229 const SCEV *SCEVA = vputils::getSCEVExprForVPValue(V: AddrA, PSE&: *PSE, L);
230 VPValue *AddrB = B->getOperand(N: 1);
231 const SCEV *SCEVB = vputils::getSCEVExprForVPValue(V: AddrB, PSE&: *PSE, L);
232 if (isa<SCEVCouldNotCompute>(Val: SCEVA) || isa<SCEVCouldNotCompute>(Val: SCEVB))
233 return false;
234
235 const APInt *Distance;
236 ScalarEvolution &SE = *PSE->getSE();
237 if (!match(S: SE.getMinusSCEV(LHS: SCEVA, RHS: SCEVB), P: m_scev_APInt(C&: Distance)))
238 return false;
239
240 const DataLayout &DL = SE.getDataLayout();
241 Type *TyA = A->getOperand(N: 0)->getScalarType();
242 uint64_t SizeA = DL.getTypeStoreSize(Ty: TyA);
243 Type *TyB = B->getOperand(N: 0)->getScalarType();
244 uint64_t SizeB = DL.getTypeStoreSize(Ty: TyB);
245
246 // Use the maximum store size to ensure no overlap from either direction.
247 // Currently only handles fixed sizes, as it is only used for
248 // replicating VPReplicateRecipes.
249 uint64_t MaxStoreSize = std::max(a: SizeA, b: SizeB);
250
251 auto VFs = B->getParent()->getPlan()->vectorFactors();
252 ElementCount MaxVF = *max_element(Range&: VFs, C: ElementCount::isKnownLT);
253 if (MaxVF.isScalable())
254 return false;
255 return Distance->abs().uge(RHS: MaxVF.getFixedValue() * MaxStoreSize);
256 }
257
258public:
259 SinkStoreInfo(ArrayRef<VPReplicateRecipe *> ExcludeRecipes,
260 VPReplicateRecipe &GroupLeader, PredicatedScalarEvolution &PSE,
261 const Loop &L)
262 : ExcludeRecipes(ExcludeRecipes.begin(), ExcludeRecipes.end()),
263 GroupLeader(GroupLeader), PSE(&PSE), L(&L) {}
264
265 SinkStoreInfo(VPReplicateRecipe &GroupLeader) : GroupLeader(GroupLeader) {}
266
267 /// Return true if \p R should be skipped during alias checking, either
268 /// because it's in the exclude set or because no-alias can be proven via
269 /// SCEV.
270 bool shouldSkip(VPRecipeBase &R) const {
271 auto *Store = dyn_cast<VPReplicateRecipe>(Val: &R);
272 return ExcludeRecipes.contains(Ptr: Store) ||
273 (Store && isNoAliasViaDistance(A: Store, B: &GroupLeader));
274 }
275};
276
277/// Check if a memory operation doesn't alias with memory operations using
278/// scoped noalias metadata, in blocks in the single-successor chain between \p
279/// FirstBB and \p LastBB. If \p SinkInfo is std::nullopt, only recipes that may
280/// write to memory are checked (for load hoisting). Otherwise recipes that both
281/// read and write memory are checked, and SCEV is used to prove no-alias
282/// between the group leader and other replicate recipes (for store sinking).
283static bool
284canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
285 VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
286 std::optional<SinkStoreInfo> SinkInfo = {}) {
287 bool CheckReads = SinkInfo.has_value();
288 for (VPBasicBlock *VPBB :
289 VPBlockUtils::blocksInSingleSuccessorChainBetween(FirstBB, LastBB)) {
290 for (VPRecipeBase &R : *VPBB) {
291 if (SinkInfo && SinkInfo->shouldSkip(R))
292 continue;
293
294 // Skip recipes that don't need checking.
295 if (!R.mayWriteToMemory() && !(CheckReads && R.mayReadFromMemory()))
296 continue;
297
298 auto Loc = vputils::getMemoryLocation(R);
299 if (!Loc)
300 // Conservatively assume aliasing for memory operations without
301 // location.
302 return false;
303
304 if (ScopedNoAliasAAResult::alias(LocA: *Loc, LocB: MemLoc) != AliasResult::NoAlias)
305 return false;
306 }
307 }
308 return true;
309}
310
311/// Get the value type of a VPInstruction or VPReplicateRecipe load or store.
312static Type *getLoadStoreValueType(VPSingleDefRecipe *R) {
313 bool IsLoad = vputils::getOpcode(V: R) == Instruction::Load;
314 return (IsLoad ? R : R->getOperand(N: 0))->getScalarType();
315}
316
317/// Collect either replicated Loads or Stores grouped by their address SCEV and
318/// their load-store type, in a deep-traversal of the vector loop region in \p
319/// Plan.
320template <unsigned Opcode>
321static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
322collectGroupedReplicateMemOps(
323 VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L,
324 function_ref<bool(VPReplicateRecipe *)> FilterFn) {
325 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
326 "Only Load and Store opcodes supported");
327 constexpr bool IsLoad = (Opcode == Instruction::Load);
328 SmallDenseMap<std::pair<const SCEV *, const Type *>,
329 SmallVector<VPReplicateRecipe *, 4>>
330 RecipesByAddressAndType;
331 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
332 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
333 for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(Range&: *VPBB)) {
334 if (RepR.getOpcode() != Opcode || !FilterFn(&RepR))
335 continue;
336
337 // For loads, operand 0 is address; for stores, operand 1 is address.
338 VPValue *Addr = RepR.getOperand(N: IsLoad ? 0 : 1);
339 const Type *LoadStoreTy = getLoadStoreValueType(R: &RepR);
340 const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(V: Addr, PSE, L);
341 if (!isa<SCEVCouldNotCompute>(Val: AddrSCEV))
342 RecipesByAddressAndType[{AddrSCEV, LoadStoreTy}].push_back(Elt: &RepR);
343 }
344 }
345 auto Groups = to_vector(Range: RecipesByAddressAndType.values());
346 VPDominatorTree VPDT(Plan);
347 for (auto &Group : Groups) {
348 // Sort mem ops by dominance order, with earliest (most dominating) first.
349 stable_sort(Group, [&VPDT](VPReplicateRecipe *A, VPReplicateRecipe *B) {
350 return VPDT.properlyDominates(A, B);
351 });
352 }
353 return Groups;
354}
355
356static bool sinkScalarOperands(VPlan &Plan) {
357 auto Iter = vp_depth_first_deep(G: Plan.getEntry());
358 bool ScalarVFOnly = Plan.hasScalarVFOnly();
359 bool Changed = false;
360
361 SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList;
362 auto InsertIfValidSinkCandidate = [ScalarVFOnly, &WorkList](
363 VPBasicBlock *SinkTo, VPValue *Op) {
364 auto *Candidate = dyn_cast<VPSingleDefRecipe>(Val: Op);
365 if (!isa_and_nonnull<VPReplicateRecipe, VPScalarIVStepsRecipe,
366 VPInstruction>(Val: Candidate))
367 return;
368
369 if (Candidate->getParent() == SinkTo ||
370 all_of(Range: Candidate->operands(),
371 P: [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); }) ||
372 vputils::cannotHoistOrSinkRecipe(R: *Candidate, /*Sinking=*/true))
373 return;
374
375 if (!ScalarVFOnly && !vputils::doesGeneratePerAllLanes(R: Candidate))
376 return;
377
378 // Only single-scalar VPInstructions can be sunk.
379 if (auto *VPI = dyn_cast<VPInstruction>(Val: Candidate))
380 if (!vputils::isSingleScalar(VPV: VPI))
381 return;
382
383 WorkList.insert(X: {SinkTo, Candidate});
384 };
385
386 // First, collect the operands of all recipes in replicate blocks as seeds for
387 // sinking.
388 for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Range&: Iter)) {
389 VPBasicBlock *EntryVPBB = VPR->getEntryBasicBlock();
390 if (!VPR->isReplicator() || EntryVPBB->getSuccessors().size() != 2)
391 continue;
392 VPBasicBlock *VPBB = cast<VPBasicBlock>(Val: EntryVPBB->getSuccessors().front());
393 if (VPBB->getSingleSuccessor() != VPR->getExitingBasicBlock())
394 continue;
395 for (auto &Recipe : *VPBB)
396 for (VPValue *Op : Recipe.operands())
397 InsertIfValidSinkCandidate(VPBB, Op);
398 }
399
400 // Try to sink each replicate or scalar IV steps recipe in the worklist.
401 for (unsigned I = 0; I != WorkList.size(); ++I) {
402 VPBasicBlock *SinkTo;
403 VPSingleDefRecipe *SinkCandidate;
404 std::tie(args&: SinkTo, args&: SinkCandidate) = WorkList[I];
405
406 // All recipe users of SinkCandidate must be in the same block SinkTo or all
407 // users outside of SinkTo must only use the first lane of SinkCandidate. In
408 // the latter case, we need to duplicate SinkCandidate.
409 auto UsersOutsideSinkTo =
410 make_filter_range(Range: SinkCandidate->users(), Pred: [SinkTo](VPUser *U) {
411 return cast<VPRecipeBase>(Val: U)->getParent() != SinkTo;
412 });
413 if (any_of(Range&: UsersOutsideSinkTo, P: [SinkCandidate](VPUser *U) {
414 return !U->usesFirstLaneOnly(Op: SinkCandidate);
415 }))
416 continue;
417 bool NeedsDuplicating = !UsersOutsideSinkTo.empty();
418
419 if (NeedsDuplicating) {
420 if (ScalarVFOnly)
421 continue;
422 VPSingleDefRecipe *Clone;
423 if (auto *SinkCandidateRepR =
424 dyn_cast<VPReplicateRecipe>(Val: SinkCandidate)) {
425 // TODO: Handle converting to uniform recipes as separate transform,
426 // then cloning should be sufficient here.
427 Clone = VPBuilder::createSingleScalarOp(
428 Opcode: SinkCandidateRepR->getOpcode(), Operands: SinkCandidate->operands(),
429 /*Mask=*/nullptr, Flags: *SinkCandidateRepR, Metadata: *SinkCandidateRepR,
430 DL: SinkCandidate->getDebugLoc(), ResultTy: SinkCandidate->getScalarType(),
431 UV: SinkCandidate->getUnderlyingInstr());
432 // TODO: add ".cloned" suffix to name of Clone's VPValue.
433 } else {
434 Clone = SinkCandidate->clone();
435 }
436
437 Clone->insertBefore(InsertPos: SinkCandidate);
438 SinkCandidate->replaceUsesWithIf(New: Clone, ShouldReplace: [SinkTo](VPUser *U) {
439 return cast<VPRecipeBase>(Val: U)->getParent() != SinkTo;
440 });
441 }
442 SinkCandidate->moveBefore(BB&: *SinkTo, I: SinkTo->getFirstNonPhi());
443 for (VPValue *Op : SinkCandidate->operands())
444 InsertIfValidSinkCandidate(SinkTo, Op);
445 Changed = true;
446 }
447 return Changed;
448}
449
450/// If \p R is a triangle region, return the 'then' block of the triangle.
451static VPBasicBlock *getPredicatedThenBlock(VPRegionBlock *R) {
452 auto *EntryBB = cast<VPBasicBlock>(Val: R->getEntry());
453 if (EntryBB->getNumSuccessors() != 2)
454 return nullptr;
455
456 auto *Succ0 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[0]);
457 auto *Succ1 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[1]);
458 if (!Succ0 || !Succ1)
459 return nullptr;
460
461 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
462 return nullptr;
463 if (Succ0->getSingleSuccessor() == Succ1)
464 return Succ0;
465 if (Succ1->getSingleSuccessor() == Succ0)
466 return Succ1;
467 return nullptr;
468}
469
470// Merge replicate regions in their successor region, if a replicate region
471// is connected to a successor replicate region with the same predicate by a
472// single, empty VPBasicBlock.
473static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
474 SmallPtrSet<VPRegionBlock *, 4> TransformedRegions;
475
476 // Collect replicate regions followed by an empty block, followed by another
477 // replicate region with matching masks to process front. This is to avoid
478 // iterator invalidation issues while merging regions.
479 SmallVector<VPRegionBlock *, 8> WorkList;
480 for (VPRegionBlock *Region1 : VPBlockUtils::blocksOnly<VPRegionBlock>(
481 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
482 if (!Region1->isReplicator())
483 continue;
484 auto *MiddleBasicBlock =
485 dyn_cast_or_null<VPBasicBlock>(Val: Region1->getSingleSuccessor());
486 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
487 continue;
488
489 auto *Region2 =
490 dyn_cast_or_null<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
491 if (!Region2 || !Region2->isReplicator())
492 continue;
493
494 VPValue *Mask1 = Region1->getEntryBranchOnMask()->getOperand(N: 0);
495 VPValue *Mask2 = Region2->getEntryBranchOnMask()->getOperand(N: 0);
496 if (!Mask1 || Mask1 != Mask2)
497 continue;
498
499 assert(Mask1 && Mask2 && "both region must have conditions");
500 WorkList.push_back(Elt: Region1);
501 }
502
503 // Move recipes from Region1 to its successor region, if both are triangles.
504 for (VPRegionBlock *Region1 : WorkList) {
505 if (TransformedRegions.contains(Ptr: Region1))
506 continue;
507 auto *MiddleBasicBlock = cast<VPBasicBlock>(Val: Region1->getSingleSuccessor());
508 auto *Region2 = cast<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
509
510 VPBasicBlock *Then1 = getPredicatedThenBlock(R: Region1);
511 VPBasicBlock *Then2 = getPredicatedThenBlock(R: Region2);
512 if (!Then1 || !Then2)
513 continue;
514
515 // The merged region is entered whenever either of the original regions was,
516 // so use the higher, i.e. more conservative, of their entry frequencies.
517 // If only one of the two is known, the higher one is unknown, so the
518 // result must be unknown too.
519 VPBranchOnMaskRecipe *Guard2 = Region2->getEntryBranchOnMask();
520 std::optional<VPExecutionFrequency> Freq1 =
521 Region1->getEntryBranchOnMask()->getExecutionFrequency();
522 std::optional<VPExecutionFrequency> Freq2 = Guard2->getExecutionFrequency();
523 if (Freq1 && Freq2) {
524 if (Freq2->Freq < Freq1->Freq) {
525 // Freq1's frequency is taken, but it is only as trustworthy as the
526 // less trustworthy of the two.
527 Freq1.emplace(args: Freq1->Freq, args: Freq1->IsEstimated || Freq2->IsEstimated);
528 Guard2->setExecutionFrequency(Freq: Freq1, Ctx&: Plan.getContext());
529 }
530 } else if (Freq2) {
531 Guard2->clearExecutionFrequency();
532 }
533
534 // Note: No fusion-preventing memory dependencies are expected in either
535 // region. Such dependencies should be rejected during earlier dependence
536 // checks, which guarantee accesses can be re-ordered for vectorization.
537 //
538 // Move recipes to the successor region.
539 for (VPRecipeBase &ToMove : make_early_inc_range(Range: reverse(C&: *Then1)))
540 ToMove.moveBefore(BB&: *Then2, I: Then2->getFirstNonPhi());
541
542 auto *Merge1 = cast<VPBasicBlock>(Val: Then1->getSingleSuccessor());
543 auto *Merge2 = cast<VPBasicBlock>(Val: Then2->getSingleSuccessor());
544
545 // Move VPPredInstPHIRecipes from the merge block to the successor region's
546 // merge block. Update all users inside the successor region to use the
547 // original values.
548 for (VPRecipeBase &Phi1ToMove : make_early_inc_range(Range: reverse(C&: *Merge1))) {
549 VPValue *PredInst1 =
550 cast<VPPredInstPHIRecipe>(Val: &Phi1ToMove)->getOperand(N: 0);
551 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
552 Phi1ToMoveV->replaceUsesWithIf(New: PredInst1, ShouldReplace: [Then2](VPUser *U) {
553 return cast<VPRecipeBase>(Val: U)->getParent() == Then2;
554 });
555
556 // Remove phi recipes that are unused after merging the regions.
557 if (Phi1ToMove.getVPSingleValue()->user_empty()) {
558 Phi1ToMove.eraseFromParent();
559 continue;
560 }
561 Phi1ToMove.moveBefore(BB&: *Merge2, I: Merge2->begin());
562 }
563
564 // Remove the dead recipes in Region1's entry block.
565 for (VPRecipeBase &R :
566 make_early_inc_range(Range: reverse(C&: *Region1->getEntryBasicBlock())))
567 R.eraseFromParent();
568
569 // Finally, remove the first region.
570 for (VPBlockBase *Pred : make_early_inc_range(Range&: Region1->getPredecessors())) {
571 VPBlockUtils::disconnectBlocks(From: Pred, To: Region1);
572 VPBlockUtils::connectBlocks(From: Pred, To: MiddleBasicBlock);
573 }
574 VPBlockUtils::disconnectBlocks(From: Region1, To: MiddleBasicBlock);
575 TransformedRegions.insert(Ptr: Region1);
576 }
577
578 return !TransformedRegions.empty();
579}
580
581static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
582 VPRegionBlock *ParentRegion,
583 VPlan &Plan) {
584 Instruction *Instr = PredRecipe->getUnderlyingInstr();
585 // Build the triangular if-then region.
586 std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str();
587 assert(Instr->getParent() && "Predicated instruction not in any basic block");
588 auto *BlockInMask = PredRecipe->getMask();
589 auto *MaskDef = BlockInMask->getDefiningRecipe();
590 auto *BOMRecipe = new VPBranchOnMaskRecipe(
591 BlockInMask, MaskDef ? MaskDef->getDebugLoc() : DebugLoc::getUnknown());
592 auto *Entry =
593 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".entry", Recipe: BOMRecipe);
594
595 // Replace predicated replicate recipe with a replicate recipe without a
596 // mask but in the replicate region.
597 auto *RecipeWithoutMask = new VPReplicateRecipe(
598 PredRecipe->getUnderlyingInstr(), PredRecipe->operandsWithoutMask(),
599 PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
600 PredRecipe->getDebugLoc());
601 // The predicated recipe executes exactly when the guarding branch-on-mask is
602 // taken, so move its execution frequency there.
603 BOMRecipe->setExecutionFrequency(Freq: RecipeWithoutMask->getExecutionFrequency(),
604 Ctx&: Plan.getContext());
605 RecipeWithoutMask->clearExecutionFrequency();
606 auto *Pred =
607 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".if", Recipe: RecipeWithoutMask);
608 auto *Exiting = Plan.createVPBasicBlock(Name: Twine(RegionName) + ".continue");
609 VPRegionBlock *Region =
610 Plan.createReplicateRegion(Entry, Exiting, Name: RegionName);
611
612 // Note: first set Entry as region entry and then connect successors starting
613 // from it in order, to propagate the "parent" of each VPBasicBlock.
614 Region->setParent(ParentRegion);
615 VPBlockUtils::insertTwoBlocksAfter(IfTrue: Pred, IfFalse: Exiting, BlockPtr: Entry);
616 VPBlockUtils::connectBlocks(From: Pred, To: Exiting);
617
618 PredRecipe->replaceAllUsesWith(New: RecipeWithoutMask);
619 // Move the extracts of a struct result into the region, so only scalars
620 // leave it.
621 SmallVector<VPUser *> Defs = {RecipeWithoutMask};
622 if (RecipeWithoutMask->getScalarType()->isStructTy())
623 Defs = to_vector(Range: RecipeWithoutMask->users());
624 for (VPUser *U : Defs) {
625 auto *Def = cast<VPReplicateRecipe>(Val: U);
626 assert((Def == RecipeWithoutMask ||
627 Def->getOpcode() == Instruction::ExtractValue) &&
628 "only extracts can use a predicated struct");
629 Def->moveBefore(BB&: *Pred, I: Pred->end());
630 if (Def->user_empty())
631 continue;
632 auto *PHIRecipe = new VPPredInstPHIRecipe(Def, Def->getDebugLoc());
633 Exiting->appendRecipe(Recipe: PHIRecipe);
634 Def->replaceAllUsesWith(New: PHIRecipe);
635 PHIRecipe->setOperand(I: 0, New: Def);
636 }
637 PredRecipe->eraseFromParent();
638 return Region;
639}
640
641static void addReplicateRegions(VPlan &Plan) {
642 SmallVector<VPReplicateRecipe *> WorkList;
643 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
644 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
645 for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(Range&: *VPBB))
646 if (RepR.isPredicated())
647 WorkList.push_back(Elt: &RepR);
648 }
649
650 unsigned BBNum = 0;
651 for (VPReplicateRecipe *RepR : WorkList) {
652 VPBasicBlock *CurrentBlock = RepR->getParent();
653 VPBasicBlock *SplitBlock = CurrentBlock->splitAt(SplitAt: RepR->getIterator());
654
655 BasicBlock *OrigBB = RepR->getUnderlyingInstr()->getParent();
656 SplitBlock->setName(
657 OrigBB->hasName() ? OrigBB->getName() + "." + Twine(BBNum++) : "");
658 // Record predicated instructions for above packing optimizations.
659 VPRegionBlock *Region =
660 createReplicateRegion(PredRecipe: RepR, ParentRegion: CurrentBlock->getParent(), Plan);
661 VPBlockUtils::insertOnEdge(From: CurrentBlock, To: SplitBlock, BlockPtr: Region);
662
663 VPRegionBlock *ParentRegion = Region->getParent();
664 if (ParentRegion && ParentRegion->getExiting() == CurrentBlock)
665 ParentRegion->setExiting(SplitBlock);
666 }
667}
668
669bool VPlanTransforms::mergeBlocksIntoPredecessors(VPlan &Plan) {
670 SmallVector<VPBasicBlock *> WorkList;
671
672 // Collect single-successor chains in the loop region.
673 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
674 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
675 Range: vp_depth_first_deep(G: LoopRegion->getEntry()))) {
676 if (!VPBB->getParent())
677 continue;
678 VPBlockBase *PredBlock = VPBB->getSinglePredecessor();
679 if (!isa_and_nonnull<VPBasicBlock>(Val: PredBlock) ||
680 !PredBlock->getSingleSuccessor())
681 continue;
682 WorkList.push_back(Elt: VPBB);
683 }
684
685 for (VPBasicBlock *VPBB : WorkList) {
686 VPBasicBlock *PredVPBB = cast<VPBasicBlock>(Val: VPBB->getSinglePredecessor());
687 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
688 R.moveBefore(BB&: *PredVPBB, I: PredVPBB->end());
689 VPBlockUtils::disconnectBlocks(From: PredVPBB, To: VPBB);
690 auto *ParentRegion = VPBB->getParent();
691 if (ParentRegion && ParentRegion->getExiting() == VPBB)
692 ParentRegion->setExiting(PredVPBB);
693 VPBlockUtils::transferSuccessors(Old: VPBB, New: PredVPBB);
694 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
695 }
696 return !WorkList.empty();
697}
698
699void VPlanTransforms::createAndOptimizeReplicateRegions(VPlan &Plan) {
700 // Convert masked VPReplicateRecipes to if-then region blocks.
701 addReplicateRegions(Plan);
702
703 bool ShouldSimplify = true;
704 while (ShouldSimplify) {
705 ShouldSimplify = sinkScalarOperands(Plan);
706 ShouldSimplify |= mergeReplicateRegionsIntoSuccessors(Plan);
707 ShouldSimplify |= mergeBlocksIntoPredecessors(Plan);
708 }
709}
710
711/// Remove redundant casts of inductions.
712///
713/// Such redundant casts are casts of induction variables that can be ignored,
714/// because we already proved that the casted phi is equal to the uncasted phi
715/// in the vectorized loop. There is no need to vectorize the cast - the same
716/// value can be used for both the phi and casts in the vector loop.
717static void removeRedundantInductionCasts(VPlan &Plan) {
718 for (VPWidenIntOrFpInductionRecipe &IV :
719 make_isa_range<VPWidenIntOrFpInductionRecipe>(
720 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
721 if (IV.isTruncated())
722 continue;
723
724 // A sequence of IR Casts has potentially been recorded for IV, which
725 // *must be bypassed* when the IV is vectorized, because the vectorized IV
726 // will produce the desired casted value. This sequence forms a def-use
727 // chain and is provided in reverse order, ending with the cast that uses
728 // the IV phi. Search for the recipe of the last cast in the chain and
729 // replace it with the original IV. Note that only the final cast is
730 // expected to have users outside the cast-chain and the dead casts left
731 // over will be cleaned up later.
732 ArrayRef<Instruction *> Casts = IV.getInductionDescriptor().getCastInsts();
733 VPValue *FindMyCast = &IV;
734 for (Instruction *IRCast : reverse(C&: Casts)) {
735 VPSingleDefRecipe *FoundUserCast = nullptr;
736 for (auto *U : FindMyCast->users()) {
737 auto *UserCast = dyn_cast<VPSingleDefRecipe>(Val: U);
738 if (UserCast && UserCast->getUnderlyingValue() == IRCast) {
739 FoundUserCast = UserCast;
740 break;
741 }
742 }
743 // A cast recipe in the chain may have been removed by earlier DCE.
744 if (!FoundUserCast)
745 break;
746 FindMyCast = FoundUserCast;
747 }
748 if (FindMyCast != &IV)
749 FindMyCast->replaceAllUsesWith(New: &IV);
750 }
751}
752
753/// If R is a phi-like recipe starting a dead cycle of recipes, erase all
754/// reachable recipes of the dead cycle and return true. Otherwise leave the
755/// plan unchanged and return false.
756static bool tryToRemoveDeadCycle(VPRecipeBase *R) {
757 auto *PhiR = dyn_cast<VPSingleDefRecipe>(Val: R);
758 if (!PhiR || !isa<VPPhi, VPReductionPHIRecipe>(Val: R))
759 return false;
760
761 // The transitive users of PhiR are closed under users, so the cycle is dead
762 // if every one of them can be erased.
763 for (VPUser *U : vputils::collectUsersRecursively(V: PhiR)) {
764 auto *R = cast<VPRecipeBase>(Val: U);
765 // Bail out if a user must be retained, or if it is a phi-like recipe other
766 // than PhiR;
767 if (R->mayHaveSideEffects() || (R != PhiR && isa<VPPhiAccessors>(Val: R)))
768 return false;
769 }
770
771 // Break the cycle by replacing PhiR with its first incoming value, which is
772 // defined outside the cycle. That leaves the rest of the cycle dead.
773 PhiR->replaceAllUsesWith(New: PhiR->getOperand(N: 0));
774 SmallVector<VPValue *> Incoming(PhiR->operands());
775 PhiR->eraseFromParent();
776 for (VPValue *Op : Incoming)
777 vputils::recursivelyDeleteDeadRecipes(V: Op);
778 return true;
779}
780
781void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
782 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
783 Plan.getEntry());
784 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
785 // The recipes in the block are processed in reverse order, to catch chains
786 // of dead recipes.
787 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB)))
788 if (vputils::isDeadRecipe(R))
789 R.eraseFromParent();
790
791 // Erase dead cycles starting at one of VPBB's phi-like recipes. Erasing a
792 // cycle may also erase other phi-like recipes of VPBB, so restart the scan
793 // of the phi section after each removal. This terminates, as each removal
794 // erases the cycle's phi.
795 bool Changed = true;
796 while (Changed) {
797 Changed = false;
798 for (VPRecipeBase &R : VPBB->phis()) {
799 if (tryToRemoveDeadCycle(R: &R)) {
800 Changed = true;
801 break;
802 }
803 }
804 }
805 }
806}
807
808/// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd
809/// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as
810/// VPWidenPointerInductionRecipe will generate vectors only. If some users
811/// require vectors while other require scalars, the scalar uses need to extract
812/// the scalars from the generated vectors (Note that this is different to how
813/// int/fp inductions are handled). Legalize extract-from-ends using uniform
814/// VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so
815/// the correct end value is available. Also optimize
816/// VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by
817/// providing them scalar steps built on the canonical scalar IV and update the
818/// original IV's users. This is an optional optimization to reduce the needs of
819/// vector extracts.
820static void legalizeAndOptimizeInductions(VPlan &Plan) {
821 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
822 bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly();
823
824 SmallVector<VPWidenInductionRecipe *> WideIVs;
825 for (VPWidenInductionRecipe &PhiR :
826 make_isa_range<VPWidenInductionRecipe>(Range: HeaderVPBB->phis()))
827 WideIVs.push_back(Elt: &PhiR);
828
829 // Try to narrow wide and replicating recipes to uniform recipes, based on
830 // VPlan analysis.
831 // TODO: Apply to all recipes in the future, to replace legacy uniformity
832 // analysis.
833 for (VPWidenInductionRecipe *PhiR : WideIVs) {
834 auto Users = vputils::collectUsersRecursively(V: PhiR);
835 for (VPUser *U : reverse(C&: Users)) {
836 auto *Def = dyn_cast<VPRecipeWithIRFlags>(Val: U);
837 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
838 // Skip recipes that shouldn't be narrowed.
839 if (!Def ||
840 !isa<VPReplicateRecipe, VPWidenRecipe, VPWidenGEPRecipe>(Val: Def) ||
841 Def->user_empty() || !Def->getUnderlyingValue() ||
842 (RepR && (RepR->isSingleScalar() || RepR->isPredicated())))
843 continue;
844
845 // Skip recipes that may have other lanes than their first used.
846 if (!vputils::isSingleScalar(VPV: Def) && !vputils::onlyFirstLaneUsed(Def))
847 continue;
848
849 // TODO: Support scalarizing ExtractValue.
850 if (match(V: Def,
851 P: m_Binary<Instruction::ExtractValue>(Op0: m_VPValue(), Op1: m_VPValue())))
852 continue;
853
854 auto *Clone = VPBuilder::createSingleScalarOp(
855 Opcode: Def->getUnderlyingInstr()->getOpcode(), Operands: Def->operands(),
856 /*Mask=*/nullptr, Flags: *Def, Metadata: getMetadataOf(R: Def), DL: DebugLoc::getUnknown(),
857 ResultTy: Def->getScalarType(), UV: Def->getUnderlyingInstr());
858 Clone->insertAfter(InsertPos: Def);
859 Def->replaceAllUsesWith(New: Clone);
860 Def->eraseFromParent();
861 }
862 }
863
864 VPBuilder Builder(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
865 for (VPWidenInductionRecipe *PhiR : WideIVs) {
866 // Replace wide pointer inductions which have only their scalars used by
867 // PtrAdd(IndStart, ScalarIVSteps (0, Step)).
868 if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(Val: PhiR)) {
869 if (!Plan.hasScalarVFOnly() &&
870 !PtrIV->onlyScalarsGenerated(IsScalable: Plan.hasScalableVF()))
871 continue;
872
873 VPValue *PtrAdd =
874 vputils::scalarizeVPWidenPointerInduction(PtrIV, Plan, Builder);
875 PtrIV->replaceAllUsesWith(New: PtrAdd);
876 continue;
877 }
878
879 // Replace widened induction with scalar steps for users that only use
880 // scalars.
881 auto *WideIV = cast<VPWidenIntOrFpInductionRecipe>(Val: PhiR);
882 if (HasOnlyVectorVFs && none_of(Range: WideIV->users(), P: [WideIV](VPUser *U) {
883 return U->usesScalars(Op: WideIV);
884 }))
885 continue;
886
887 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
888 VPIRFlags::WrapFlagsTy WrapFlags;
889 // We can preserve nuw when the step is non-negative.
890 const APInt *Step;
891 if (match(V: WideIV->getStepValue(), P: m_APInt(C&: Step)) && Step->isNonNegative())
892 WrapFlags = WideIV->getNoWrapFlagsOrNone().withoutNoSignedWrap();
893 VPScalarIVStepsRecipe *Steps = vputils::createScalarIVSteps(
894 Plan, Kind: ID.getKind(), InductionOpcode: ID.getInductionOpcode(),
895 FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
896 ResultTy: WideIV->getScalarType(), StartV: WideIV->getStartValue(),
897 Step: WideIV->getStepValue(), DL: WideIV->getDebugLoc(), Builder, Flags: WrapFlags);
898
899 // Update scalar users of IV to use Step instead.
900 if (!HasOnlyVectorVFs) {
901 assert(!Plan.hasScalableVF() &&
902 "plans containing a scalar VF cannot also include scalable VFs");
903 WideIV->replaceAllUsesWith(New: Steps);
904 } else {
905 bool HasScalableVF = Plan.hasScalableVF();
906 WideIV->replaceUsesWithIf(New: Steps, ShouldReplace: [WideIV, HasScalableVF](VPUser *U) {
907 if (HasScalableVF)
908 return U->usesFirstLaneOnly(Op: WideIV);
909 return U->usesScalars(Op: WideIV);
910 });
911 }
912 }
913}
914
915/// Check if \p VPV is an untruncated wide induction, either before or after the
916/// increment. If so return the header IV (before the increment), otherwise
917/// return null.
918static VPWidenInductionRecipe *
919getOptimizableIVOf(VPValue *VPV, PredicatedScalarEvolution &PSE) {
920 auto *WideIV = dyn_cast<VPWidenInductionRecipe>(Val: VPV);
921 if (WideIV) {
922 // VPV itself is a wide induction, separately compute the end value for exit
923 // users if it is not a truncated IV.
924 auto *IntOrFpIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
925 return (IntOrFpIV && IntOrFpIV->isTruncated()) ? nullptr : WideIV;
926 }
927
928 // Check if VPV is an optimizable induction increment.
929 VPRecipeBase *Def = VPV->getDefiningRecipe();
930 if (!Def || Def->getNumOperands() != 2)
931 return nullptr;
932 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 0));
933 if (!WideIV)
934 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 1));
935 if (!WideIV)
936 return nullptr;
937
938 auto IsWideIVInc = [&]() {
939 auto &ID = WideIV->getInductionDescriptor();
940
941 // Check if VPV increments the induction by the induction step.
942 VPValue *IVStep = WideIV->getStepValue();
943 switch (ID.getInductionOpcode()) {
944 case Instruction::Add:
945 return match(V: VPV, P: m_c_Add(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
946 case Instruction::FAdd:
947 return match(V: VPV, P: m_c_FAdd(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
948 case Instruction::FSub:
949 return match(V: VPV, P: m_Binary<Instruction::FSub>(Op0: m_Specific(VPV: WideIV),
950 Op1: m_Specific(VPV: IVStep)));
951 case Instruction::Sub: {
952 // IVStep will be the negated step of the subtraction. Check if Step == -1
953 // * IVStep.
954 VPValue *Step;
955 if (!match(V: VPV, P: m_Sub(Op0: m_VPValue(), Op1: m_VPValue(V&: Step))))
956 return false;
957 const SCEV *IVStepSCEV = vputils::getSCEVExprForVPValue(V: IVStep, PSE);
958 const SCEV *StepSCEV = vputils::getSCEVExprForVPValue(V: Step, PSE);
959 ScalarEvolution &SE = *PSE.getSE();
960 return !isa<SCEVCouldNotCompute>(Val: IVStepSCEV) &&
961 !isa<SCEVCouldNotCompute>(Val: StepSCEV) &&
962 IVStepSCEV == SE.getNegativeSCEV(V: StepSCEV);
963 }
964 default:
965 return ID.getKind() == InductionDescriptor::IK_PtrInduction &&
966 match(V: VPV, P: m_GetElementPtr(Op0: m_Specific(VPV: WideIV),
967 Op1: m_Specific(VPV: WideIV->getStepValue())));
968 }
969 llvm_unreachable("should have been covered by switch above");
970 };
971 return IsWideIVInc() ? WideIV : nullptr;
972}
973
974/// Attempts to optimize the induction variable exit values for users in the
975/// early exit block.
976static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan, VPValue *Op,
977 PredicatedScalarEvolution &PSE) {
978 VPValue *Incoming, *Mask;
979 if (!match(V: Op, P: m_ExtractLane(Op0: m_FirstActiveLane(Op0: m_VPValue(V&: Mask)),
980 Op1: m_VPValue(V&: Incoming))))
981 return nullptr;
982
983 auto *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
984 if (!WideIV)
985 return nullptr;
986
987 // Calculate the final index.
988 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
989 auto *CanonicalIV = LoopRegion->getCanonicalIV();
990 Type *CanonicalIVType = LoopRegion->getCanonicalIVType();
991 auto *ExtractR = cast<VPInstruction>(Val: Op);
992 VPBuilder B(ExtractR);
993
994 DebugLoc DL = ExtractR->getDebugLoc();
995 VPValue *FirstActiveLane = B.createFirstActiveLane(Masks: Mask, DL);
996 FirstActiveLane =
997 B.createScalarZExtOrTrunc(Op: FirstActiveLane, ResultTy: CanonicalIVType, DL);
998 VPValue *EndValue = B.createAdd(LHS: CanonicalIV, RHS: FirstActiveLane, DL);
999
1000 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
1001 // changed it means the exit is using the incremented value, so we need to
1002 // add the step.
1003 if (Incoming != WideIV) {
1004 VPValue *One = Plan.getConstantInt(Ty: CanonicalIVType, Val: 1);
1005 EndValue = B.createAdd(LHS: EndValue, RHS: One, DL);
1006 }
1007
1008 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
1009 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
1010 VPValue *Start = WideIV->getStartValue();
1011 VPValue *Step = WideIV->getStepValue();
1012 EndValue = B.createDerivedIV(
1013 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
1014 Start, Current: EndValue, Step);
1015 }
1016
1017 return EndValue;
1018}
1019
1020/// Compute the end value for \p WideIV, unless it is truncated. Creates a
1021/// VPDerivedIVRecipe for non-canonical inductions.
1022static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV,
1023 VPBuilder &VectorPHBuilder,
1024 VPValue *VectorTC) {
1025 auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
1026 // Truncated wide inductions resume from the last lane of their vector value
1027 // in the last vector iteration which is handled elsewhere.
1028 if (WideIntOrFp && WideIntOrFp->isTruncated())
1029 return nullptr;
1030
1031 VPValue *Start = WideIV->getStartValue();
1032 VPValue *Step = WideIV->getStepValue();
1033 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
1034 VPValue *EndValue = VectorTC;
1035 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
1036 EndValue = VectorPHBuilder.createDerivedIV(
1037 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
1038 Start, Current: VectorTC, Step);
1039 }
1040
1041 // EndValue is derived from the vector trip count (which has the same type as
1042 // the widest induction) and thus may be wider than the induction here.
1043 Type *ScalarTypeOfWideIV = WideIV->getScalarType();
1044 if (ScalarTypeOfWideIV != EndValue->getScalarType()) {
1045 EndValue = VectorPHBuilder.createScalarCast(Opcode: Instruction::Trunc, Op: EndValue,
1046 ResultTy: ScalarTypeOfWideIV,
1047 DL: WideIV->getDebugLoc());
1048 }
1049
1050 return EndValue;
1051}
1052
1053/// Attempts to optimize the induction variable exit values for users in the
1054/// exit block coming from the latch in the original scalar loop.
1055static VPValue *
1056optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
1057 DenseMap<VPValue *, VPValue *> &EndValues,
1058 PredicatedScalarEvolution &PSE) {
1059 VPValue *Incoming;
1060 if (!match(V: Op, P: m_CombineOr(Ps: m_ExtractLastLaneOfLastPart(Op0: m_VPValue(V&: Incoming)),
1061 Ps: m_ExtractLane(Op0: m_LastActiveLane(Op0: m_HeaderMask()),
1062 Op1: m_VPValue(V&: Incoming)))))
1063 return nullptr;
1064
1065 VPWidenInductionRecipe *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
1066 if (!WideIV)
1067 return nullptr;
1068
1069 VPValue *EndValue = EndValues.lookup(Val: WideIV);
1070 assert(EndValue && "Must have computed the end value up front");
1071
1072 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
1073 // changed it means the exit is using the incremented value, so we don't
1074 // need to subtract the step.
1075 if (Incoming != WideIV)
1076 return EndValue;
1077
1078 // Otherwise, subtract the step from the EndValue.
1079 auto *ExtractR = cast<VPInstruction>(Val: Op);
1080 VPBuilder B(ExtractR);
1081 VPValue *Step = WideIV->getStepValue();
1082 Type *ScalarTy = WideIV->getScalarType();
1083 if (ScalarTy->isIntegerTy())
1084 return B.createSub(LHS: EndValue, RHS: Step, DL: DebugLoc::getUnknown(), Name: "ind.escape");
1085 if (ScalarTy->isPointerTy()) {
1086 Type *StepTy = Step->getScalarType();
1087 auto *Zero = Plan.getZero(Ty: StepTy);
1088 return B.createPtrAdd(Ptr: EndValue, Offset: B.createSub(LHS: Zero, RHS: Step),
1089 DL: DebugLoc::getUnknown(), Name: "ind.escape");
1090 }
1091 if (ScalarTy->isFloatingPointTy()) {
1092 const auto &ID = WideIV->getInductionDescriptor();
1093 return B.createNaryOp(
1094 Opcode: ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
1095 ? Instruction::FSub
1096 : Instruction::FAdd,
1097 Operands: {EndValue, Step}, Flags: {ID.getInductionBinOp()->getFastMathFlags()});
1098 }
1099 llvm_unreachable("all possible induction types must be handled");
1100 return nullptr;
1101}
1102
1103static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
1104 PredicatedScalarEvolution &PSE,
1105 VPValue *ResumeTC,
1106 const Loop *L) {
1107 VPValue *Incoming;
1108 if (!match(V: Op, P: m_CombineOr(Ps: m_ExtractLastLaneOfLastPart(Op0: m_VPValue(V&: Incoming)),
1109 Ps: m_ExtractLane(Op0: m_LastActiveLane(Op0: m_HeaderMask()),
1110 Op1: m_VPValue(V&: Incoming)))))
1111 return nullptr;
1112
1113 const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(V: Incoming, PSE, L);
1114 const SCEV *Start, *Step;
1115 if (!match(S: IncomingSCEV, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step),
1116 L: m_SpecificLoop(L))))
1117 return nullptr;
1118
1119 auto *ExtractR = cast<VPInstruction>(Val: Op);
1120 DebugLoc DL = ExtractR->getDebugLoc();
1121 VPBuilder Builder(ExtractR);
1122 VPSCEVExpander Expander(Builder, *PSE.getSE(), DL);
1123 VPValue *StartVPV = Expander.expand(S: Start);
1124 VPValue *StepVPV = Expander.expand(S: Step);
1125
1126 Type *StartTy = StartVPV->getScalarType();
1127 assert(StartTy->isIntOrPtrTy() && "The type must be SCEVable");
1128 InductionDescriptor::InductionKind Kind =
1129 StartTy->isPointerTy() ? InductionDescriptor::IK_PtrInduction
1130 : InductionDescriptor::IK_IntInduction;
1131 Type *TCTy = ResumeTC->getScalarType();
1132 VPValue *ExitCount = Builder.createOverflowingOp(
1133 Opcode: Instruction::Sub, Operands: {ResumeTC, Plan.getConstantInt(Ty: TCTy, Val: 1)},
1134 WrapFlags: {/*HasNUW=*/true, /*HasNSW=*/false}, DL: DebugLoc::getUnknown());
1135 return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, Start: StartVPV, Current: ExitCount,
1136 Step: StepVPV);
1137}
1138
1139void VPlanTransforms::optimizeInductionLiveOutUsers(
1140 VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L) {
1141 // Compute end values for all inductions.
1142 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
1143 auto *VectorPH = cast<VPBasicBlock>(Val: VectorRegion->getSinglePredecessor());
1144 VPBuilder VectorPHBuilder(VectorPH, VectorPH->getFirstNonPhi());
1145 DenseMap<VPValue *, VPValue *> EndValues;
1146 VPValue *ResumeTC =
1147 Plan.hasTailFolded() ? Plan.getTripCount() : &Plan.getVectorTripCount();
1148 for (VPWidenInductionRecipe &WideIV : make_isa_range<VPWidenInductionRecipe>(
1149 Range: VectorRegion->getEntryBasicBlock()->phis())) {
1150 if (VPValue *EndValue = tryToComputeEndValueForInduction(
1151 WideIV: &WideIV, VectorPHBuilder, VectorTC: ResumeTC))
1152 EndValues[&WideIV] = EndValue;
1153 }
1154
1155 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
1156 for (VPRecipeBase &R : make_early_inc_range(Range&: *MiddleVPBB)) {
1157 VPValue *Op;
1158 if (!match(V: &R, P: m_ExitingIVValue(Op0: m_VPValue(V&: Op))))
1159 continue;
1160 auto *WideIV = cast<VPWidenInductionRecipe>(Val: Op);
1161 if (VPValue *EndValue = EndValues.lookup(Val: WideIV)) {
1162 R.getVPSingleValue()->replaceAllUsesWith(New: EndValue);
1163 R.eraseFromParent();
1164 }
1165 }
1166
1167 // Then, optimize exit block users.
1168 for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
1169 for (VPRecipeBase &R : ExitVPBB->phis()) {
1170 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
1171
1172 for (auto [Idx, PredVPBB] : enumerate(First&: ExitVPBB->getPredecessors())) {
1173 VPValue *Escape = nullptr;
1174 if (PredVPBB == MiddleVPBB) {
1175 Escape = optimizeLatchExitInductionUser(
1176 Plan, Op: ExitIRI->getOperand(N: Idx), EndValues, PSE);
1177 if (!Escape)
1178 Escape = optimizeLatchExitIVUserViaSCEV(
1179 Plan, Op: ExitIRI->getOperand(N: Idx), PSE, ResumeTC, L);
1180 } else {
1181 Escape = optimizeEarlyExitInductionUser(
1182 Plan, Op: ExitIRI->getOperand(N: Idx), PSE);
1183 }
1184 if (Escape)
1185 ExitIRI->setOperand(I: Idx, New: Escape);
1186 }
1187 }
1188 }
1189}
1190
1191void VPlanTransforms::removeRedundantExpandSCEVRecipes(VPlan &Plan) {
1192 DenseMap<const SCEV *, VPValue *> SCEV2VPV;
1193
1194 for (VPExpandSCEVRecipe &ExpR :
1195 make_early_inc_range(Range: make_isa_range<VPExpandSCEVRecipe>(
1196 Range&: *Plan.getEntry()->getEntryBasicBlock()))) {
1197 const auto &[V, Inserted] = SCEV2VPV.try_emplace(Key: ExpR.getSCEV(), Args: &ExpR);
1198 if (Inserted)
1199 continue;
1200
1201 ExpR.replaceAllUsesWith(New: V->second);
1202 if (&ExpR == Plan.getTripCount())
1203 Plan.resetTripCount(NewTripCount: V->second);
1204
1205 ExpR.eraseFromParent();
1206 }
1207}
1208
1209/// Try to simplify logical and bitwise recipes in \p Def.
1210static VPValue *simplifyLogicalRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
1211 VPValue *X;
1212
1213 // X | AllOnes -> AllOnes
1214 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_AllOnes())))
1215 return Plan.getAllOnesValue(Ty: Def->getScalarType());
1216
1217 // X | 0 -> X
1218 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_ZeroInt())))
1219 return X;
1220
1221 // X | !X -> AllOnes
1222 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X)))))
1223 return Plan.getAllOnesValue(Ty: Def->getScalarType());
1224
1225 // X & 0 -> 0
1226 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_ZeroInt())))
1227 return Plan.getZero(Ty: Def->getScalarType());
1228
1229 // X & AllOnes -> X
1230 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_AllOnes())))
1231 return X;
1232
1233 // X && false -> false
1234 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_False())))
1235 return Plan.getFalse();
1236
1237 // X && true -> X
1238 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_True())))
1239 return X;
1240
1241 // X && (X && Y) -> X && Y
1242 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1243 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue()))))
1244 return Def->getOperand(N: 1);
1245
1246 // X && !X -> 0
1247 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X)))))
1248 return Plan.getFalse();
1249
1250 if (match(R: Def, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: X), Op2: m_Deferred(V: X))))
1251 return X;
1252
1253 // X != false -> X
1254 if (match(R: Def, P: m_SpecificICmp(MatchPred: CmpInst::ICMP_NE, Op0: m_VPValue(V&: X), Op1: m_False()))) {
1255 assert(X->getScalarType()->isIntegerTy(1) && "must have boolean operands");
1256 return X;
1257 }
1258
1259 return nullptr;
1260}
1261
1262/// Swap the branch weights recorded for \p R, a select whose two selected
1263/// operands are being swapped,
1264static void swapSelectBranchWeights(VPRecipeBase &R, VPlan &Plan) {
1265 auto *MD = dyn_cast<VPIRMetadata>(Val: &R);
1266 if (!MD)
1267 return;
1268 SmallVector<uint32_t, 2> Weights;
1269 if (!extractBranchWeights(ProfileData: MD->getMetadata(Kind: LLVMContext::MD_prof), Weights))
1270 return;
1271 assert(Weights.size() == 2 && "unexpected branch weights");
1272 MD->setMetadata(
1273 Kind: LLVMContext::MD_prof,
1274 Node: MDBuilder(Plan.getContext()).createBranchWeights(TrueWeight: Weights[1], FalseWeight: Weights[0]));
1275}
1276
1277/// Return an existing value or a live in for VPSingleDefRecipe \p Def if
1278/// possible. This shouldn't create or modify recipes.
1279static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
1280 // Simplification of live-in IR values for SingleDef recipes using
1281 // InstSimplifyFolder.
1282 const DataLayout &DL = Plan.getDataLayout();
1283 if (VPValue *V = vputils::tryToFoldLiveIns(R&: *Def, Operands: Def->operands(), DL))
1284 return V;
1285
1286 // Fold PredPHI LiveIn -> LiveIn.
1287 if (auto *PredPHI = dyn_cast<VPPredInstPHIRecipe>(Val: Def)) {
1288 VPValue *Op = PredPHI->getOperand(N: 0);
1289 if (isa<VPIRValue>(Val: Op))
1290 return Op;
1291 }
1292
1293 if (VPValue *V = simplifyLogicalRecipe(Plan, Def))
1294 return V;
1295
1296 VPValue *A, *B;
1297
1298 if (match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))
1299 return A;
1300
1301 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_One())))
1302 return A;
1303
1304 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(), Op1: m_ZeroInt())))
1305 return Plan.getZero(Ty: Def->getScalarType());
1306
1307 // A bitcast to the same type is a no-op.
1308 if (match(R: Def, P: m_BitCast(Op0: m_VPValue(V&: A))) &&
1309 Def->getScalarType() == A->getScalarType())
1310 return A;
1311
1312 // Shifting by zero is a no-op.
1313 if (match(R: Def, P: m_CombineOr(Ps: m_Shl(Op0: m_VPValue(V&: A), Op1: m_ZeroInt()),
1314 Ps: m_CombineOr(Ps: m_LShr(Op0: m_VPValue(V&: A), Op1: m_ZeroInt()),
1315 Ps: m_AShr(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))))
1316 return A;
1317
1318 if (match(R: Def, P: m_Trunc(Op0: m_ZExtOrSExt(Op0: m_VPValue(V&: A)))))
1319 if (Def->getScalarType() == A->getScalarType())
1320 return A;
1321
1322 if (match(R: Def, P: m_Not(Op0: m_Not(Op0: m_VPValue(V&: A)))))
1323 return A;
1324
1325 // Remove redundant DerviedIVs, that is 0 + A * 1 -> A and 0 + 0 * x -> 0.
1326 if ((match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_VPValue(V&: A), Op2: m_One())) ||
1327 match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_VPValue(V&: A, Op: m_ZeroInt()),
1328 Op2: m_VPValue()))) &&
1329 A->getScalarType() == Def->getScalarType())
1330 return A;
1331
1332 // Simplify MaskedCond with no block mask to its single operand.
1333 if (match(R: Def, P: m_VPInstruction<VPInstruction::MaskedCond>()) &&
1334 !cast<VPInstruction>(Val: Def)->isMasked())
1335 return Def->getOperand(N: 0);
1336
1337 // Look through ExtractLastLane.
1338 if (match(R: Def, P: m_ExtractLastLane(Op0: m_VPValue(V&: A)))) {
1339 if (match(V: A, P: m_BuildVector())) {
1340 auto *BuildVector = cast<VPInstruction>(Val: A);
1341 return BuildVector->getLastOperand();
1342 }
1343
1344 if (match(V: A, P: m_Broadcast(Op0: m_VPValue(V&: B))))
1345 return B;
1346
1347 if (isa<VPIRValue, VPInstruction, VPReplicateRecipe>(Val: A) &&
1348 vputils::isSingleScalar(VPV: A))
1349 return A;
1350
1351 if (Plan.hasScalarVFOnly())
1352 return A;
1353 }
1354
1355 // Look through ExtractPenultimateElement (BuildVector ....).
1356 if (match(R: Def, P: m_ExtractPenultimateElement(Op0: m_BuildVector()))) {
1357 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1358 return BuildVector->getOperand(N: BuildVector->getNumOperands() - 2);
1359 }
1360
1361 uint64_t Idx;
1362 if (match(R: Def, P: m_ExtractElement(Op0: m_BuildVector(), Op1: m_ConstantInt(C&: Idx)))) {
1363 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1364 return BuildVector->getOperand(N: Idx);
1365 }
1366
1367 if (isa<VPPhi, VPWidenPHIRecipe, VPHeaderPHIRecipe>(Val: Def)) {
1368 if (Def->getNumOperands() == 1) {
1369 return Def->getOperand(N: 0);
1370 }
1371 if (auto *Phi = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Val: Def)) {
1372 if (all_equal(Range: Phi->incoming_values()))
1373 return Phi->getOperand(N: 0);
1374 }
1375 return nullptr;
1376 }
1377
1378 VPIRValue *IRV;
1379 if (Def->getNumOperands() == 1 &&
1380 match(R: Def, P: m_ComputeReductionResult(Op0: m_VPIRValue(V&: IRV))))
1381 return IRV;
1382
1383 if (match(R: Def, P: m_VPInstruction<VPInstruction::WideIVStep>(Ops: m_VPValue(V&: A),
1384 Ops: m_One())) &&
1385 A->getScalarType() == Def->getScalarType())
1386 return A;
1387
1388 // Some simplifications can only be applied after unrolling. Perform them
1389 // below.
1390 if (!Plan.isUnrolled())
1391 return nullptr;
1392
1393 // Simplify extracts of the same single-scalar.
1394 if (match(R: Def, P: m_VPInstruction<VPInstruction::ExtractLane>()) &&
1395 all_equal(Range: drop_begin(RangeOrContainer: Def->operands())) &&
1396 vputils::isSingleScalar(VPV: Def->getOperand(N: 1)))
1397 return Def->getOperand(N: 1);
1398
1399 // Replace extract-lane(0, canonical-WIDEN-INDUCTION) with the region's
1400 // scalar canonical IV.
1401 VPWidenIntOrFpInductionRecipe *WidenIV;
1402 if (match(R: Def, P: m_ExtractLane(Op0: m_ZeroInt(), Op1: m_CanonicalWidenIV(V&: WidenIV))))
1403 return WidenIV->getRegion()->getCanonicalIV();
1404
1405 // Simplify unrolled VectorPointer without offset, or with zero offset, to
1406 // just the pointer operand.
1407 if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(Val: Def))
1408 if (!VPR->getVFxPart() || match(V: VPR->getVFxPart(), P: m_ZeroInt()))
1409 return VPR->getOperand(N: 0);
1410
1411 // VPScalarIVSteps after unrolling can be replaced by their start value, if
1412 // the start index is zero and only the first lane 0 is demanded.
1413 if (auto *Steps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def))
1414 if (!Steps->getStartIndex() && vputils::onlyFirstLaneUsed(Def: Steps))
1415 return Steps->getOperand(N: 0);
1416
1417 if (Plan.getConcreteUF() == 1 && match(R: Def, P: m_ExtractLastPart(Op0: m_VPValue(V&: A))))
1418 return A;
1419
1420 return nullptr;
1421}
1422
1423/// Returns true if \p V is available at the end of \p VPBB, i.e. it either is a
1424/// live-in from the original IR or defined in \p VPBB.
1425static bool isAvailableAtEndOf(VPValue *V, const VPBasicBlock *VPBB) {
1426 VPRecipeBase *DefR = V->getDefiningRecipe();
1427 return DefR ? DefR->getParent() == VPBB : isa<VPIRValue>(Val: V);
1428}
1429
1430namespace {
1431/// Inserter for VPBuilderBase which appends all created VPSingleDefRecipes to a
1432/// worklist, so they get combined as well.
1433struct VPCombineInserter {
1434 SmallVectorImpl<VPSingleDefRecipe *> &Worklist;
1435
1436 void insertHelper(VPRecipeBase *R, VPBasicBlock *VPBB,
1437 VPBasicBlock::iterator It) {
1438 VPBB->insert(Recipe: R, InsertPt: It);
1439 if (auto *Def = dyn_cast<VPSingleDefRecipe>(Val: R))
1440 Worklist.push_back(Elt: Def);
1441 }
1442};
1443
1444using VPCombineBuilder = VPBuilderBase<VPCombineInserter>;
1445} // namespace
1446
1447/// Combine \p Def into a simpler recipe. May modify or create new recipes via
1448/// \p Builder.
1449static VPSingleDefRecipe *combineRecipe(VPlan &Plan, VPSingleDefRecipe *Def,
1450 VPCombineBuilder &Builder) {
1451 if (auto *V = simplifyRecipe(Plan, Def)) {
1452 Def->replaceAllUsesWith(New: V);
1453 return Def;
1454 }
1455
1456 // Drop the mask of a predicated store masked by the header mask (which is
1457 // guaranteed to be true at least for the first lane) and both the stored
1458 // value and the address are uniform across VF and UF. The header mask is
1459 // still the abstract region value here.
1460 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: Def);
1461 RepR && RepR->isPredicated() && RepR->getOpcode() == Instruction::Store &&
1462 all_of(Range: RepR->operandsWithoutMask(), P: vputils::isUniformAcrossVFsAndUFs) &&
1463 match(V: RepR->getMask(), P: m_HeaderMask())) {
1464 auto *Unmasked = new VPReplicateRecipe(
1465 RepR->getUnderlyingInstr(), RepR->operandsWithoutMask(),
1466 RepR->isSingleScalar(), /*Mask=*/nullptr, *RepR, *RepR,
1467 RepR->getDebugLoc());
1468 Builder.insert(R: Unmasked);
1469 return Unmasked;
1470 }
1471
1472 // Avoid replacing VPInstructions with underlying values with new
1473 // VPInstructions, as we would fail to create widen/replicate recpes from the
1474 // new VPInstructions without an underlying value, and miss out on some
1475 // transformations that only apply to widened/replicated recipes later, by
1476 // doing so.
1477 // TODO: We should also not replace non-VPInstructions like VPWidenRecipe with
1478 // VPInstructions without underlying values, as those will get skipped during
1479 // cost computation.
1480 bool CanCreateNewRecipe =
1481 !isa<VPInstruction>(Val: Def) || !Def->getUnderlyingValue();
1482
1483 VPValue *X, *Y, *Z;
1484
1485 // X && (Y && X) -> X && Y
1486 if (CanCreateNewRecipe &&
1487 match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1488 Op1: m_LogicalAnd(Op0: m_VPValue(V&: Y), Op1: m_Deferred(V: X)))))
1489 return Builder.createLogicalAnd(LHS: X, RHS: Y);
1490
1491 // (X && Y) | (X && Z) -> X && (Y | Z)
1492 if (CanCreateNewRecipe &&
1493 match(R: Def, P: m_c_BinaryOr(Op0: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y)),
1494 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue(V&: Z)))) &&
1495 // Simplify only if one of the operands has one use to avoid creating an
1496 // extra recipe.
1497 (!Def->getOperand(N: 0)->hasMoreThanOneUniqueUser() ||
1498 !Def->getOperand(N: 1)->hasMoreThanOneUniqueUser()))
1499 return Builder.createLogicalAnd(LHS: X, RHS: Builder.createOr(LHS: Y, RHS: Z));
1500
1501 // (X && Y) | !X -> !X || Y
1502 if (CanCreateNewRecipe &&
1503 match(R: Def,
1504 P: m_c_BinaryOr(Op0: m_OneUse(SubPattern: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y))),
1505 Op1: m_VPValue(V&: Z, Op: m_Not(Op0: m_Deferred(V: X))))))
1506 return Builder.createLogicalOr(LHS: Z, RHS: Y);
1507
1508 // select C, false, true -> not C
1509 VPValue *C;
1510 if (CanCreateNewRecipe &&
1511 match(R: Def, P: m_Select(Op0: m_VPValue(V&: C), Op1: m_False(), Op2: m_True())))
1512 return Builder.createNot(Operand: C);
1513
1514 // select !C, X, Y -> select C, Y, X
1515 if (match(R: Def, P: m_Select(Op0: m_Not(Op0: m_VPValue(V&: C)), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1516 Def->setOperand(I: 0, New: C);
1517 Def->setOperand(I: 1, New: Y);
1518 Def->setOperand(I: 2, New: X);
1519 swapSelectBranchWeights(R&: *Def, Plan);
1520 return Def;
1521 }
1522
1523 // select X, (i1 Y | Z), Y -> Y | (X && Z)
1524 if (CanCreateNewRecipe &&
1525 match(R: Def, P: m_Select(Op0: m_VPValue(V&: X),
1526 Op1: m_OneUse(SubPattern: m_c_BinaryOr(Op0: m_VPValue(V&: Y), Op1: m_VPValue(V&: Z))),
1527 Op2: m_Deferred(V: Y))) &&
1528 Y->getScalarType()->isIntegerTy(BitWidth: 1))
1529 return Builder.createOr(LHS: Y, RHS: Builder.createLogicalAnd(LHS: X, RHS: Z));
1530
1531 // select M0, (select M1, X, Y), Y -> select (M0 && M1), X, Y
1532 VPValue *Mask0, *Mask1;
1533 if (CanCreateNewRecipe &&
1534 match(R: Def,
1535 P: m_SelectLike(Op0: m_VPValue(V&: Mask0),
1536 Op1: m_OneUse(SubPattern: m_SelectLike(Op0: m_VPValue(V&: Mask1), Op1: m_VPValue(V&: X),
1537 Op2: m_VPValue(V&: Y))),
1538 Op2: m_Deferred(V: Y))))
1539 return Builder.createSelect(Cond: Builder.createLogicalAnd(LHS: Mask0, RHS: Mask1), TrueVal: X, FalseVal: Y,
1540 DL: Def->getDebugLoc());
1541
1542 if (match(R: Def, P: m_Trunc(Op0: m_VPValue(V&: Y, Op: m_ZExtOrSExt(Op0: m_VPValue(V&: X)))))) {
1543 // Don't replace a non-widened cast recipe with a widened cast.
1544 if (!isa<VPWidenCastRecipe>(Val: Def))
1545 return nullptr;
1546 Type *TruncTy = Def->getScalarType();
1547 Type *XTy = X->getScalarType();
1548 if (XTy->getScalarSizeInBits() < TruncTy->getScalarSizeInBits()) {
1549
1550 unsigned ExtOpcode =
1551 match(V: Y, P: m_SExt(Op0: m_VPValue())) ? Instruction::SExt : Instruction::ZExt;
1552 auto *Ext =
1553 Builder.createWidenCast(Opcode: Instruction::CastOps(ExtOpcode), Op: X, ResultTy: TruncTy);
1554 if (auto *UnderlyingExt = Y->getUnderlyingValue()) {
1555 // UnderlyingExt has distinct return type, used to retain legacy cost.
1556 Ext->setUnderlyingValue(UnderlyingExt);
1557 }
1558 return Ext;
1559 } else if (XTy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) {
1560 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::Trunc, Op: X, ResultTy: TruncTy);
1561 return Trunc;
1562 }
1563 }
1564
1565 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: X), Op1: m_AllOnes()))) {
1566 // Preserve nsw from the Mul on the new Sub.
1567 VPIRFlags::WrapFlagsTy NW = {
1568 false, cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap()};
1569 return Builder.createSub(LHS: Plan.getZero(Ty: X->getScalarType()), RHS: X,
1570 DL: Def->getDebugLoc(), Name: "", WrapFlags: NW);
1571 }
1572
1573 if (CanCreateNewRecipe &&
1574 match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: X),
1575 Op1: m_VPValue(V&: Z, Op: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Y)))))) {
1576 // Preserve nsw from the Add and the Sub, if it's present on both, on the
1577 // new Sub.
1578 VPIRFlags::WrapFlagsTy NW = {
1579 false, cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap() &&
1580 cast<VPRecipeWithIRFlags>(Val: Z)->hasNoSignedWrap()};
1581 return Builder.createSub(LHS: X, RHS: Y, DL: Def->getDebugLoc(), Name: "", WrapFlags: NW);
1582 }
1583
1584 const APInt *APC;
1585 if (CanCreateNewRecipe && match(R: Def, P: m_URem(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1586 APC->isPowerOf2())
1587 return Builder.createAnd(LHS: X, RHS: Plan.getConstantInt(Val: *APC - 1),
1588 DL: Def->getDebugLoc());
1589
1590 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1591 APC->isPowerOf2()) {
1592 auto *MulR = cast<VPRecipeWithIRFlags>(Val: Def);
1593 unsigned ShiftAmt = APC->exactLogBase2();
1594 VPIRFlags::WrapFlagsTy NW(MulR->hasNoUnsignedWrap(),
1595 MulR->hasNoSignedWrap() &&
1596 ShiftAmt != APC->getBitWidth() - 1);
1597 return Builder.createNaryOp(
1598 Opcode: Instruction::Shl,
1599 Operands: {X, Plan.getConstantInt(BitWidth: APC->getBitWidth(), Val: ShiftAmt)}, Flags: NW,
1600 DL: Def->getDebugLoc());
1601 }
1602
1603 if (CanCreateNewRecipe && match(R: Def, P: m_UDiv(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1604 APC->isPowerOf2())
1605 return Builder.createNaryOp(
1606 Opcode: Instruction::LShr,
1607 Operands: {X, Plan.getConstantInt(BitWidth: APC->getBitWidth(), Val: APC->exactLogBase2())},
1608 Flags: *cast<VPRecipeWithIRFlags>(Val: Def), DL: Def->getDebugLoc());
1609
1610 // (X >> C) << C -> X & (-1 << C).
1611 if (CanCreateNewRecipe &&
1612 match(R: Def, P: m_Shl(Op0: m_LShr(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y, Op: m_APInt(C&: APC))),
1613 Op1: m_Deferred(V: Y))))
1614 return Builder.createAnd(
1615 LHS: X, RHS: Plan.getConstantInt(Val: APInt::getAllOnes(numBits: APC->getBitWidth()) << *APC),
1616 DL: Def->getDebugLoc());
1617
1618 if (match(R: Def, P: m_Not(Op0: m_VPValue(V&: X)))) {
1619 // Try to fold Not into compares by adjusting the predicate in-place.
1620 CmpPredicate Pred;
1621 if (match(V: X, P: m_Cmp(Pred, Op0: m_VPValue(), Op1: m_VPValue()))) {
1622 auto *Cmp = cast<VPRecipeWithIRFlags>(Val: X);
1623 // Only fold if every user is a Not of the cmp, or a select using the cmp
1624 // solely as its condition.
1625 if (all_of(Range: Cmp->users(), P: [Cmp](VPUser *U) {
1626 return match(U, P: m_Not(Op0: m_Specific(VPV: Cmp))) ||
1627 (match(U, P: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(),
1628 Op2: m_VPValue())) &&
1629 U->getOperand(N: 1) != Cmp && U->getOperand(N: 2) != Cmp);
1630 })) {
1631 Cmp->setPredicate(CmpInst::getInversePredicate(pred: Pred));
1632 for (VPUser *U : to_vector(Range: Cmp->users())) {
1633 auto *R = cast<VPSingleDefRecipe>(Val: U);
1634 if (match(R, P: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1635 // select (cmp pred), X, Y -> select (cmp inv_pred), Y, X
1636 R->setOperand(I: 1, New: Y);
1637 R->setOperand(I: 2, New: X);
1638 swapSelectBranchWeights(R&: *R, Plan);
1639 } else {
1640 // not (cmp pred) -> cmp inv_pred
1641 assert(match(R, m_Not(m_Specific(Cmp))) && "Unexpected user");
1642 R->replaceAllUsesWith(New: Cmp);
1643 }
1644 }
1645 // If Cmp doesn't have a debug location, use the one from the negation,
1646 // to preserve the location.
1647 if (!Cmp->getDebugLoc() && Def->getDebugLoc())
1648 Cmp->setDebugLoc(Def->getDebugLoc());
1649 return Def;
1650 }
1651 }
1652 }
1653
1654 // Fold any-of (fcmp uno A, A), (fcmp uno B, B), ... ->
1655 // any-of (fcmp uno A, B), ...
1656 if (match(R: Def, P: m_AnyOf())) {
1657 SmallVector<VPValue *, 4> NewOps;
1658 VPRecipeBase *UnpairedCmp = nullptr;
1659 for (VPValue *Op : Def->operands()) {
1660 VPValue *X;
1661 if (Op->getNumUsers() > 1 ||
1662 !match(V: Op, P: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X),
1663 Op1: m_Deferred(V: X)))) {
1664 NewOps.push_back(Elt: Op);
1665 } else if (!UnpairedCmp) {
1666 UnpairedCmp = Op->getDefiningRecipe();
1667 } else {
1668 NewOps.push_back(Elt: Builder.createFCmp(Pred: CmpInst::FCMP_UNO,
1669 A: UnpairedCmp->getOperand(N: 0), B: X));
1670 UnpairedCmp = nullptr;
1671 }
1672 }
1673
1674 if (UnpairedCmp)
1675 NewOps.push_back(Elt: UnpairedCmp->getVPSingleValue());
1676
1677 if (NewOps.size() < Def->getNumOperands())
1678 return Builder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: NewOps);
1679 }
1680
1681 // Fold (fcmp uno X, X) | (fcmp uno Y, Y) -> fcmp uno X, Y
1682 // This is useful for fmax/fmin without fast-math flags, where we need to
1683 // check if any operand is NaN.
1684 if (CanCreateNewRecipe &&
1685 match(R: Def, P: m_BinaryOr(Op0: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X),
1686 Op1: m_Deferred(V: X)),
1687 Op1: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: Y),
1688 Op1: m_Deferred(V: Y)))) &&
1689 X->getScalarType() == Y->getScalarType())
1690 return Builder.createFCmp(Pred: CmpInst::FCMP_UNO, A: X, B: Y);
1691
1692 if (match(R: Def, P: m_VPInstruction<VPInstruction::WideIVStep>(Ops: m_VPValue(V&: X),
1693 Ops: m_One())) &&
1694 X->getScalarType() != Def->getScalarType())
1695 return Builder.createWidenCast(Opcode: Instruction::Trunc, Op: X, ResultTy: Def->getScalarType());
1696
1697 // For i1 vp.merges produced by AnyOf reductions:
1698 // vp.merge true, (or X, Y), X, evl -> vp.merge Y, true, X, evl
1699 if (match(R: Def, P: m_Intrinsic<Intrinsic::vp_merge>(Ops: m_True(), Ops: m_VPValue(V&: X),
1700 Ops: m_VPValue(V&: X), Ops: m_VPValue())) &&
1701 match(V: X, P: m_c_BinaryOr(Op0: m_Specific(VPV: X), Op1: m_VPValue(V&: Y))) &&
1702 Def->getScalarType()->isIntegerTy(BitWidth: 1)) {
1703 Def->setOperand(I: 1, New: Plan.getTrue());
1704 Def->setOperand(I: 0, New: Y);
1705 return Def;
1706 }
1707
1708 if (match(R: Def, P: m_BuildVector()) && all_equal(Range: Def->operands()))
1709 return Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: Def->getOperand(N: 0));
1710
1711 // Replace uses of a BuildVector by users that only use its first lane with
1712 // its first operand directly.
1713 if (match(R: Def, P: m_BuildVector())) {
1714 Def->replaceUsesWithIf(New: Def->getOperand(N: 0), ShouldReplace: [Def](VPUser *U) {
1715 return U->usesFirstLaneOnly(Op: Def);
1716 });
1717 return Def;
1718 }
1719
1720 // Look through broadcast of single-scalar when used as select conditions; in
1721 // that case the scalar condition can be used directly.
1722 if (match(R: Def,
1723 P: m_Select(Op0: m_Broadcast(Op0: m_VPValue(V&: Z)), Op1: m_VPValue(), Op2: m_VPValue()))) {
1724 assert(vputils::isSingleScalar(Z) &&
1725 "broadcast operand must be single-scalar");
1726 Def->setOperand(I: 0, New: Z);
1727 return Def;
1728 }
1729
1730 if (match(R: Def, P: m_Broadcast(Op0: m_VPValue(V&: X)))) {
1731 Def->replaceUsesWithIf(
1732 New: X, ShouldReplace: [Def](const VPUser *U) { return U->usesScalars(Op: Def); });
1733 return Def;
1734 }
1735
1736 // Some simplifications can only be applied after unrolling. Perform them
1737 // below.
1738 if (!Plan.isUnrolled())
1739 return nullptr;
1740
1741 // Simplify extract-lane with single source to extract-element.
1742 VPValue *LaneToExtract;
1743 if (match(R: Def, P: m_ExtractLane(Op0: m_VPValue(V&: LaneToExtract), Op1: m_VPValue(V&: X))))
1744 return Builder.createNaryOp(Opcode: Instruction::ExtractElement, Operands: {X, LaneToExtract},
1745 DL: Def->getDebugLoc());
1746
1747 // Look for cycles where Def is of the form:
1748 // X = phi(0, IVInc) ; used only by IVInc, or by IVInc and Inc = X + Y
1749 // IVInc = X + Step ; used by X and Def
1750 // Def = IVInc + Y
1751 // Fold the increment Y into the phi's start value, replace Def with IVInc,
1752 // and if Inc exists, replace it with X.
1753 VPValue *IVInc;
1754 if (match(R: Def, P: m_Add(Op0: m_VPValue(V&: IVInc, Op: m_Add(Op0: m_VPValue(V&: X), Op1: m_VPValue())),
1755 Op1: m_VPValue(V&: Y))) &&
1756 match(V: X, P: m_VPPhi(Op0: m_ZeroInt(), Op1: m_Specific(VPV: IVInc))) &&
1757 IVInc->getNumUsers() == 2) {
1758 auto *Phi = cast<VPPhi>(Val: X);
1759 // If Phi has a second user (besides IVInc's defining recipe), it must be
1760 // Inc = Phi + Y for the fold to apply.
1761 auto *Inc = dyn_cast_if_present<VPSingleDefRecipe>(
1762 Val: findUserOf(V: Phi, P: m_Add(Op0: m_Specific(VPV: Phi), Op1: m_Specific(VPV: Y))));
1763 if ((Phi->getNumUsers() == 1 || (Phi->getNumUsers() == 2 && Inc)) &&
1764 isAvailableAtEndOf(V: Y, VPBB: Phi->getIncomingBlock(Idx: 0))) {
1765 Def->replaceAllUsesWith(New: IVInc);
1766 if (Inc)
1767 Inc->replaceAllUsesWith(New: Phi);
1768 Phi->setOperand(I: 0, New: Y);
1769 return Def;
1770 }
1771 }
1772
1773 // Simplify redundant ReductionStartVector recipes after unrolling.
1774 VPValue *StartV;
1775 if (match(R: Def, P: m_VPInstruction<VPInstruction::ReductionStartVector>(
1776 Ops: m_VPValue(V&: StartV), Ops: m_VPValue(), Ops: m_VPValue()))) {
1777 Def->replaceUsesWithIf(New: StartV, ShouldReplace: [](const VPUser *U) {
1778 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: U);
1779 return PhiR && PhiR->isInLoop();
1780 });
1781 return Def;
1782 }
1783
1784 return nullptr;
1785}
1786
1787void VPlanTransforms::combineRecipes(VPlan &Plan) {
1788 SmallVector<VPSingleDefRecipe *, 256> Worklist;
1789 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
1790 Plan.getEntry());
1791 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT))
1792 for (VPSingleDefRecipe &Def :
1793 make_isa_range<VPSingleDefRecipe>(Range: reverse(C&: *VPBB)))
1794 Worklist.push_back(Elt: &Def);
1795
1796 [[maybe_unused]] unsigned InitWorklistSize = Worklist.size();
1797
1798 VPCombineBuilder Builder({.Worklist: Worklist});
1799 while (!Worklist.empty()) {
1800 assert(Worklist.size() < InitWorklistSize * 2 &&
1801 "Worklist is growing large, possible cycle?");
1802 VPSingleDefRecipe *Def = Worklist.pop_back_val();
1803 Builder.setInsertPoint(Def);
1804 VPSingleDefRecipe *New = combineRecipe(Plan, Def, Builder);
1805 if (!New)
1806 continue;
1807 if (New != Def) {
1808 // Replace the recipe with a new one.
1809 Def->replaceAllUsesWith(New);
1810 Def->eraseFromParent();
1811 // TODO: Append users to the worklist (might need a setvector)
1812 } else if (vputils::isDeadRecipe(R&: *Def)) {
1813 // Recipe was modified - it may be dead now.
1814 Def->eraseFromParent();
1815 }
1816 }
1817}
1818
1819void VPlanTransforms::simplifyReverses(VPlan &Plan) {
1820 // Pull out reverses from any elementwise op.
1821 // binop(reverse(x), reverse(y)) -> reverse(binop(x,y))
1822 vputils::pullOutPermutations(
1823 Plan, Perm: [](VPValue *&X) { return m_Reverse(Op0: m_VPValue(V&: X)); },
1824 Build: [](auto *X) { return new VPInstruction(VPInstruction::Reverse, X); });
1825
1826 // reverse(reverse(x)) -> x
1827 VPValue *X;
1828 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1829 Range: vp_depth_first_deep(G: Plan.getEntry())))
1830 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
1831 if (match(V: &R, P: m_Reverse(Op0: m_Reverse(Op0: m_VPValue(V&: X)))))
1832 R.getVPSingleValue()->replaceAllUsesWith(New: X);
1833}
1834
1835/// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the
1836/// header mask to be simplified further when tail folding, e.g. in
1837/// optimizeEVLMasks.
1838static void reassociateHeaderMask(VPlan &Plan) {
1839 VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
1840 if (!HeaderMask)
1841 return;
1842
1843 SmallVector<VPUser *> Worklist;
1844 for (VPUser *U : HeaderMask->users())
1845 if (match(U, P: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue())))
1846 append_range(C&: Worklist, R: cast<VPSingleDefRecipe>(Val: U)->users());
1847
1848 while (!Worklist.empty()) {
1849 auto *R = dyn_cast<VPSingleDefRecipe>(Val: Worklist.pop_back_val());
1850 VPValue *X, *Y;
1851 if (!R || !match(R, P: m_LogicalAnd(
1852 Op0: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue(V&: X)),
1853 Op1: m_VPValue(V&: Y))))
1854 continue;
1855 append_range(C&: Worklist, R: R->users());
1856 VPBuilder Builder(R);
1857 R->replaceAllUsesWith(
1858 New: Builder.createLogicalAnd(LHS: HeaderMask, RHS: Builder.createLogicalAnd(LHS: X, RHS: Y)));
1859 }
1860}
1861
1862static std::optional<Instruction::BinaryOps>
1863getUnmaskedDivRemOpcode(Intrinsic::ID ID) {
1864 switch (ID) {
1865 case Intrinsic::masked_udiv:
1866 return Instruction::UDiv;
1867 case Intrinsic::masked_sdiv:
1868 return Instruction::SDiv;
1869 case Intrinsic::masked_urem:
1870 return Instruction::URem;
1871 case Intrinsic::masked_srem:
1872 return Instruction::SRem;
1873 default:
1874 return {};
1875 }
1876}
1877
1878static void narrowToSingleScalarRecipes(VPlan &Plan) {
1879 if (Plan.hasScalarVFOnly())
1880 return;
1881
1882 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1883 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
1884 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
1885 if (!isa<VPWidenRecipe, VPWidenGEPRecipe, VPReplicateRecipe,
1886 VPWidenIntrinsicRecipe>(Val: &R))
1887 continue;
1888 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
1889 if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
1890 continue;
1891
1892 auto *RepOrWidenR = cast<VPRecipeWithIRFlags>(Val: &R);
1893 if (RepR && RepR->getOpcode() == Instruction::Store &&
1894 vputils::isSingleScalar(VPV: RepR->getOperand(N: 1))) {
1895 auto *Clone = new VPReplicateRecipe(
1896 RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
1897 true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
1898 *RepR /*Metadata*/, RepR->getDebugLoc());
1899 Clone->insertBefore(InsertPos: RepOrWidenR);
1900 VPBuilder Builder(Clone);
1901 VPValue *ExtractOp = Clone->getOperand(N: 0);
1902 if (vputils::isUniformAcrossVFsAndUFs(V: RepR->getOperand(N: 1)))
1903 ExtractOp =
1904 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastPart, Operands: ExtractOp);
1905 ExtractOp =
1906 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastLane, Operands: ExtractOp);
1907 Clone->setOperand(I: 0, New: ExtractOp);
1908 RepR->eraseFromParent();
1909 continue;
1910 }
1911
1912 // Narrow llvm.masked.{u,s}{div,rem} intrinsics with a safe divisor.
1913 if (auto *IntrR = dyn_cast<VPWidenIntrinsicRecipe>(Val: RepOrWidenR)) {
1914 if (!vputils::onlyFirstLaneUsed(Def: IntrR))
1915 continue;
1916 auto Opc = getUnmaskedDivRemOpcode(ID: IntrR->getVectorIntrinsicID());
1917 if (!Opc)
1918 continue;
1919 VPBuilder Builder(IntrR);
1920 VPValue *SafeDivisor = Builder.createSelect(
1921 Cond: IntrR->getOperand(N: 2), TrueVal: IntrR->getOperand(N: 1),
1922 FalseVal: Plan.getConstantInt(Ty: IntrR->getScalarType(), Val: 1));
1923 VPValue *Clone = Builder.createNaryOp(
1924 Opcode: *Opc, Operands: {IntrR->getOperand(N: 0), SafeDivisor},
1925 Flags: VPIRFlags::getDefaultFlags(Opcode: *Opc), DL: IntrR->getDebugLoc());
1926 IntrR->replaceAllUsesWith(New: Clone);
1927 IntrR->eraseFromParent();
1928 continue;
1929 }
1930
1931 // Skip recipes that aren't single scalars.
1932 if (!vputils::isSingleScalar(VPV: RepOrWidenR))
1933 continue;
1934
1935 // Predicate to check if a user of Op introduces extra broadcasts.
1936 auto IntroducesBCastOf = [](const VPValue *Op) {
1937 return [Op](const VPUser *U) {
1938 if (auto *VPI = dyn_cast<VPInstruction>(Val: U)) {
1939 if (is_contained(Set: {VPInstruction::ExtractLastLane,
1940 VPInstruction::ExtractLastPart,
1941 VPInstruction::ExtractPenultimateElement},
1942 Element: VPI->getOpcode()))
1943 return false;
1944 }
1945 return !U->usesScalars(Op);
1946 };
1947 };
1948
1949 if (any_of(Range: RepOrWidenR->users(), P: IntroducesBCastOf(RepOrWidenR)) &&
1950 none_of(Range: RepOrWidenR->operands(), P: [&](VPValue *Op) {
1951 if (any_of(
1952 Range: make_filter_range(Range: Op->users(), Pred: not_equal_to(Arg&: RepOrWidenR)),
1953 P: IntroducesBCastOf(Op)))
1954 return false;
1955 // Non-constant live-ins require broadcasts, while constants do not
1956 // need explicit broadcasts.
1957 bool LiveInNeedsBroadcast =
1958 isa<VPIRValue>(Val: Op) && !isa<VPConstant>(Val: Op);
1959 auto *OpR = dyn_cast<VPReplicateRecipe>(Val: Op);
1960 return LiveInNeedsBroadcast || (OpR && OpR->isSingleScalar());
1961 }))
1962 continue;
1963
1964 auto *Clone = VPBuilder::createSingleScalarOp(
1965 Opcode: vputils::getOpcode(V: RepOrWidenR), Operands: RepOrWidenR->operands(),
1966 /*Mask=*/nullptr, Flags: *RepOrWidenR, Metadata: getMetadataOf(R: RepOrWidenR),
1967 DL: DebugLoc::getUnknown(), ResultTy: RepOrWidenR->getScalarType(),
1968 UV: RepOrWidenR->getUnderlyingInstr());
1969 Clone->insertBefore(InsertPos: RepOrWidenR);
1970 RepOrWidenR->replaceAllUsesWith(New: Clone);
1971 if (vputils::isDeadRecipe(R&: *RepOrWidenR))
1972 RepOrWidenR->eraseFromParent();
1973 }
1974 }
1975}
1976
1977/// Try to see if all of \p Blend's masks share a common value logically and'ed
1978/// and remove it from the masks.
1979static void removeCommonBlendMask(VPBlendRecipe *Blend) {
1980 if (Blend->isNormalized())
1981 return;
1982 VPValue *CommonEdgeMask;
1983 if (!match(V: Blend->getMask(Idx: 0),
1984 P: m_LogicalAnd(Op0: m_VPValue(V&: CommonEdgeMask), Op1: m_VPValue())))
1985 return;
1986 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1987 if (!match(V: Blend->getMask(Idx: I),
1988 P: m_LogicalAnd(Op0: m_Specific(VPV: CommonEdgeMask), Op1: m_VPValue())))
1989 return;
1990 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1991 Blend->setMask(Idx: I, V: Blend->getMask(Idx: I)->getDefiningRecipe()->getOperand(N: 1));
1992}
1993
1994/// Normalize and simplify VPBlendRecipes. Should be run after combineRecipes
1995/// to make sure the masks are simplified.
1996static void simplifyBlends(VPlan &Plan) {
1997 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1998 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
1999 for (VPBlendRecipe &Blend :
2000 make_early_inc_range(Range: make_isa_range<VPBlendRecipe>(Range&: *VPBB))) {
2001 removeCommonBlendMask(Blend: &Blend);
2002
2003 // Try to remove redundant blend recipes.
2004 SmallPtrSet<VPValue *, 4> UniqueValues;
2005 if (Blend.isNormalized() || !match(V: Blend.getMask(Idx: 0), P: m_False()))
2006 UniqueValues.insert(Ptr: Blend.getIncomingValue(Idx: 0));
2007 for (unsigned I = 1; I != Blend.getNumIncomingValues(); ++I)
2008 if (!match(V: Blend.getMask(Idx: I), P: m_False()))
2009 UniqueValues.insert(Ptr: Blend.getIncomingValue(Idx: I));
2010
2011 if (UniqueValues.size() == 1) {
2012 Blend.replaceAllUsesWith(New: *UniqueValues.begin());
2013 Blend.eraseFromParent();
2014 continue;
2015 }
2016
2017 if (Blend.isNormalized())
2018 continue;
2019
2020 // Normalize the blend so its first incoming value is used as the initial
2021 // value with the others blended into it.
2022
2023 unsigned StartIndex = 0;
2024 for (unsigned I = 0; I != Blend.getNumIncomingValues(); ++I) {
2025 // If a value's mask is used only by the blend then is can be deadcoded.
2026 // TODO: Find the most expensive mask that can be deadcoded, or a mask
2027 // that's used by multiple blends where it can be removed from them all.
2028 VPValue *Mask = Blend.getMask(Idx: I);
2029 if (Mask->hasOneUse() && !match(V: Mask, P: m_False())) {
2030 StartIndex = I;
2031 break;
2032 }
2033 }
2034
2035 SmallVector<VPValue *, 4> OperandsWithMask;
2036 OperandsWithMask.push_back(Elt: Blend.getIncomingValue(Idx: StartIndex));
2037
2038 for (unsigned I = 0; I != Blend.getNumIncomingValues(); ++I) {
2039 if (I == StartIndex)
2040 continue;
2041 OperandsWithMask.push_back(Elt: Blend.getIncomingValue(Idx: I));
2042 OperandsWithMask.push_back(Elt: Blend.getMask(Idx: I));
2043 }
2044
2045 auto *NewBlend =
2046 new VPBlendRecipe(cast_or_null<PHINode>(Val: Blend.getUnderlyingValue()),
2047 OperandsWithMask, Blend, Blend.getDebugLoc());
2048 NewBlend->insertBefore(InsertPos: &Blend);
2049
2050 VPValue *DeadMask = Blend.getMask(Idx: StartIndex);
2051 Blend.replaceAllUsesWith(New: NewBlend);
2052 Blend.eraseFromParent();
2053 vputils::recursivelyDeleteDeadRecipes(V: DeadMask);
2054
2055 /// Simplify BLEND %a, %b, Not(%mask) -> BLEND %b, %a, %mask.
2056 VPValue *NewMask;
2057 if (NewBlend->getNumOperands() == 3 &&
2058 match(V: NewBlend->getMask(Idx: 1), P: m_Not(Op0: m_VPValue(V&: NewMask)))) {
2059 VPValue *Inc0 = NewBlend->getOperand(N: 0);
2060 VPValue *Inc1 = NewBlend->getOperand(N: 1);
2061 VPValue *OldMask = NewBlend->getOperand(N: 2);
2062 NewBlend->setOperand(I: 0, New: Inc1);
2063 NewBlend->setOperand(I: 1, New: Inc0);
2064 NewBlend->setOperand(I: 2, New: NewMask);
2065 if (OldMask->user_empty())
2066 cast<VPInstruction>(Val: OldMask)->eraseFromParent();
2067 }
2068 }
2069 }
2070}
2071
2072/// Optimize the width of vector induction variables in \p Plan based on a known
2073/// constant Trip Count, \p BestVF and \p BestUF.
2074static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan,
2075 ElementCount BestVF,
2076 unsigned BestUF) {
2077 // Only proceed if we have not completely removed the vector region.
2078 if (!Plan.getVectorLoopRegion())
2079 return false;
2080
2081 const APInt *TC;
2082 if (!BestVF.isFixed() || !match(V: Plan.getTripCount(), P: m_APInt(C&: TC)))
2083 return false;
2084
2085 // Calculate the minimum power-of-2 bit width that can fit the known TC, VF
2086 // and UF. Returns at least 8.
2087 auto ComputeBitWidth = [](APInt TC, uint64_t Align) {
2088 APInt AlignedTC =
2089 Align * APIntOps::RoundingUDiv(A: TC, B: APInt(TC.getBitWidth(), Align),
2090 RM: APInt::Rounding::UP);
2091 APInt MaxVal = AlignedTC - 1;
2092 return std::max<unsigned>(a: PowerOf2Ceil(A: MaxVal.getActiveBits()), b: 8);
2093 };
2094 unsigned NewBitWidth =
2095 ComputeBitWidth(*TC, BestVF.getKnownMinValue() * BestUF);
2096
2097 LLVMContext &Ctx = Plan.getContext();
2098 auto *NewIVTy = IntegerType::get(C&: Ctx, NumBits: NewBitWidth);
2099
2100 bool MadeChange = false;
2101
2102 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
2103 for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
2104 // Currently only handle canonical IVs as it is trivial to replace the start
2105 // and stop values, and we currently only perform the optimization when the
2106 // IV has a single use.
2107 VPWidenIntOrFpInductionRecipe *WideIV;
2108 if (!match(V: &Phi, P: m_CanonicalWidenIV(V&: WideIV)))
2109 continue;
2110 if (WideIV->hasMoreThanOneUniqueUser() ||
2111 NewIVTy == WideIV->getScalarType())
2112 continue;
2113
2114 // Currently only handle cases where the single user is a header-mask
2115 // comparison with the backedge-taken-count.
2116 VPUser *SingleUser = WideIV->getSingleUser();
2117 if (!SingleUser ||
2118 !match(U: SingleUser,
2119 P: m_ICmp(Op0: m_Specific(VPV: WideIV),
2120 Op1: m_Broadcast(Op0: m_Specific(VPV: Plan.getBackedgeTakenCount())))))
2121 continue;
2122
2123 // Update IV operands and comparison bound to use new narrower type.
2124 assert(!WideIV->isTruncated() &&
2125 "canonical IV is not expected to have a truncation");
2126 auto *NewWideIV = new VPWidenIntOrFpInductionRecipe(
2127 WideIV->getPHINode(), Plan.getZero(Ty: NewIVTy),
2128 Plan.getConstantInt(Ty: NewIVTy, Val: 1), WideIV->getVFValue(),
2129 WideIV->getInductionDescriptor(), *WideIV, WideIV->getDebugLoc());
2130 NewWideIV->insertBefore(InsertPos: WideIV);
2131
2132 auto *NewBTC = new VPWidenCastRecipe(
2133 Instruction::Trunc, Plan.getOrCreateBackedgeTakenCount(), NewIVTy,
2134 nullptr, VPIRFlags::getDefaultFlags(Opcode: Instruction::Trunc));
2135 Plan.getVectorPreheader()->appendRecipe(Recipe: NewBTC);
2136 auto *Cmp = cast<VPInstruction>(Val: WideIV->getSingleUser());
2137 Cmp->replaceAllUsesWith(
2138 New: VPBuilder(Cmp).createICmp(Pred: Cmp->getPredicate(), A: NewWideIV, B: NewBTC));
2139
2140 MadeChange = true;
2141 }
2142
2143 return MadeChange;
2144}
2145
2146/// Return true if \p Cond is known to be true for given \p BestVF and \p
2147/// BestUF.
2148static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
2149 ElementCount BestVF, unsigned BestUF,
2150 PredicatedScalarEvolution &PSE) {
2151 if (match(V: Cond, P: m_BinaryOr(Op0: m_VPValue(), Op1: m_VPValue())))
2152 return any_of(Range: Cond->getDefiningRecipe()->operands(), P: [&Plan, BestVF, BestUF,
2153 &PSE](VPValue *C) {
2154 return isConditionTrueViaVFAndUF(Cond: C, Plan, BestVF, BestUF, PSE);
2155 });
2156
2157 auto *CanIV = Plan.getVectorLoopRegion()->getCanonicalIV();
2158 if (!match(V: Cond, P: m_SpecificICmp(
2159 MatchPred: CmpInst::ICMP_EQ,
2160 Op0: m_c_Add(Op0: m_Specific(VPV: CanIV), Op1: m_Specific(VPV: &Plan.getVFxUF())),
2161 Op1: m_Specific(VPV: &Plan.getVectorTripCount()))))
2162 return false;
2163
2164 // The compare checks CanIV + VFxUF == vector trip count. The vector trip
2165 // count is not conveniently available as SCEV so far, so we compare directly
2166 // against the original trip count. This is stricter than necessary, as we
2167 // will only return true if the trip count == vector trip count.
2168 const SCEV *VectorTripCount =
2169 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2170 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2171 VectorTripCount = vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2172 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2173 "Trip count SCEV must be computable");
2174 ScalarEvolution &SE = *PSE.getSE();
2175 ElementCount NumElements = BestVF * BestUF;
2176 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2177 return SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: VectorTripCount, RHS: C);
2178}
2179
2180// Replaces ExtractVectorForPart instructions with ICMP when the VF is scalar
2181// and the source is a WideActiveLaneMask. The unused mask is removed later
2182// when removing dead recipes.
2183static bool replaceMaskWithCompareForScalarPlan(VPlan &Plan,
2184 ElementCount BestVF) {
2185 if (!BestVF.isScalar())
2186 return false;
2187
2188 bool MadeChange = false;
2189 VPBuilder Builder;
2190 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2191 VPBasicBlock *PreheaderVPBB = Plan.getVectorPreheader();
2192 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2193
2194 VPValue *Start, *TC;
2195 uint64_t Idx;
2196 for (VPBasicBlock *VPBB : {PreheaderVPBB, ExitingVPBB}) {
2197 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2198 if (!match(V: &R, P: m_ExtractVectorForPart(
2199 Op0: m_WideActiveLaneMask(Op0: m_VPValue(V&: Start), Op1: m_VPValue(V&: TC),
2200 Op2: m_VPValue()),
2201 Op1: m_ConstantInt(C&: Idx))))
2202 continue;
2203
2204 auto *Extract = cast<VPInstruction>(Val: &R);
2205 Builder.setInsertPoint(Extract);
2206
2207 if (Idx > 0)
2208 Start = Builder.createAdd(
2209 LHS: Start, RHS: Plan.getConstantInt(Ty: Start->getScalarType(), Val: Idx));
2210
2211 VPValue *ICmp = Builder.createICmp(Pred: CmpInst::ICMP_ULT, A: Start, B: TC);
2212 Extract->replaceAllUsesWith(New: ICmp);
2213 Extract->eraseFromParent();
2214 MadeChange = true;
2215 }
2216 }
2217
2218 return MadeChange;
2219}
2220
2221/// Try to simplify the branch condition of \p Plan. This may restrict the
2222/// resulting plan to \p BestVF and \p BestUF.
2223static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
2224 unsigned BestUF,
2225 PredicatedScalarEvolution &PSE) {
2226 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2227 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2228 auto *Term = &ExitingVPBB->back();
2229 VPValue *Cond;
2230 VPValue *Offset = nullptr;
2231 auto m_CanIVInc = m_Add(Op0: m_VPValue(), Op1: m_Specific(VPV: &Plan.getVFxUF()));
2232 // Check if the branch condition compares the canonical IV increment (for main
2233 // loop), or the canonical IV increment plus an offset (for epilog loop).
2234 bool MatchedCanIVInc =
2235 match(V: Term,
2236 P: m_BranchOnCount(
2237 Op0: m_CombineOr(Ps: m_CanIVInc, Ps: m_c_Add(Op0: m_CanIVInc, Op1: m_VPValue(V&: Offset))),
2238 Op1: m_VPValue())) &&
2239 (!Offset || Offset->isDefinedOutsideLoopRegions());
2240 if (MatchedCanIVInc ||
2241 match(V: Term,
2242 P: m_BranchOnCond(Op0: m_Not(Op0: m_ExtractVectorForPart(
2243 Op0: m_WideActiveLaneMask(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue()),
2244 Op1: m_ZeroInt()))))) {
2245 // Try to simplify the branch condition if VectorTC <= VF * UF when the
2246 // latch terminator is BranchOnCount or
2247 // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask), 0))
2248 const SCEV *VectorTripCount =
2249 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2250 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2251 VectorTripCount =
2252 vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2253 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2254 "Trip count SCEV must be computable");
2255 ScalarEvolution &SE = *PSE.getSE();
2256 ElementCount NumElements = BestVF * BestUF;
2257 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2258 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_ULE, LHS: VectorTripCount, RHS: C))
2259 return false;
2260 } else if (match(V: Term, P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))) ||
2261 match(V: Term, P: m_BranchOnTwoConds(Op0: m_VPValue(), Op1: m_VPValue(V&: Cond)))) {
2262 // For BranchOnCond, check if we can prove the condition to be true using VF
2263 // and UF.
2264 if (!isConditionTrueViaVFAndUF(Cond, Plan, BestVF, BestUF, PSE))
2265 return false;
2266 } else {
2267 return false;
2268 }
2269
2270 // The vector loop region only executes once. Convert terminator of the
2271 // exiting block to exit in the first iteration.
2272 if (match(V: Term, P: m_BranchOnTwoConds())) {
2273 Term->setOperand(I: 1, New: Plan.getTrue());
2274 return true;
2275 }
2276
2277 auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, Plan.getTrue(), {},
2278 {}, Term->getDebugLoc());
2279 ExitingVPBB->appendRecipe(Recipe: BOC);
2280 Term->eraseFromParent();
2281
2282 return true;
2283}
2284
2285void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
2286 unsigned BestUF,
2287 PredicatedScalarEvolution &PSE) {
2288 assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
2289 assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
2290
2291 bool MadeChange =
2292 simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
2293 MadeChange |= replaceMaskWithCompareForScalarPlan(Plan, BestVF);
2294 MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
2295
2296 if (MadeChange) {
2297 Plan.setVF(BestVF);
2298 assert(Plan.getConcreteUF() == BestUF && "BestUF must match the Plan's UF");
2299 }
2300}
2301
2302void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
2303 for (VPReductionPHIRecipe &PhiR : make_isa_range<VPReductionPHIRecipe>(
2304 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
2305 RecurKind RK = PhiR.getRecurrenceKind();
2306 if (RK != RecurKind::Add && RK != RecurKind::Mul && RK != RecurKind::Sub &&
2307 RK != RecurKind::AddChainWithSubs)
2308 continue;
2309
2310 for (VPUser *U : vputils::collectUsersRecursively(V: &PhiR))
2311 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: U)) {
2312 RecWithFlags->dropPoisonGeneratingFlags();
2313 }
2314 }
2315}
2316
2317namespace {
2318struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
2319 /// If recipe \p R will lower to a GEP with a non-i8 source element type,
2320 /// return that source element type.
2321 static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) {
2322 // All VPInstructions that lower to GEPs must have the i8 source element
2323 // type (as they are PtrAdds), so we omit it.
2324 return TypeSwitch<const VPSingleDefRecipe *, Type *>(R)
2325 .Case(caseFn: [](const VPReplicateRecipe *I) -> Type * {
2326 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I->getUnderlyingValue()))
2327 return GEP->getSourceElementType();
2328 return nullptr;
2329 })
2330 .Case<VPVectorPointerRecipe, VPWidenGEPRecipe>(
2331 caseFn: [](auto *I) { return I->getSourceElementType(); })
2332 .Default(defaultFn: [](auto *) { return nullptr; });
2333 }
2334
2335 /// Returns true if recipe \p Def can be safely handed for CSE.
2336 static bool canHandle(const VPSingleDefRecipe *Def) {
2337 // We can extend the list of handled recipes in the future,
2338 // provided we account for the data embedded in them while checking for
2339 // equality or hashing.
2340 auto C = vputils::getOpcodeOrIntrinsicID(V: Def);
2341
2342 // Allocas must not be merged, as each creates a distinct allocation.
2343 if (!C || (!C->first && C->second == Instruction::Alloca))
2344 return false;
2345
2346 // Widened loads (including the EVL variant) are handled, as cse() only
2347 // reuses them within a block with no intervening memory write. Any other
2348 // memory access is rejected.
2349 if (Def->mayWriteToMemory())
2350 return false;
2351 return !Def->mayReadFromMemory() ||
2352 isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Val: Def);
2353 }
2354
2355 /// Hash the underlying data of \p Def.
2356 static unsigned getHashValue(const VPSingleDefRecipe *Def) {
2357 hash_code Result = hash_combine(
2358 args: Def->getVPRecipeID(), args: vputils::getOpcodeOrIntrinsicID(V: Def),
2359 args: getGEPSourceElementType(R: Def), args: Def->getScalarType(),
2360 args: vputils::isSingleScalar(VPV: Def), args: hash_combine_range(R: Def->operands()));
2361 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: Def))
2362 if (RFlags->hasPredicate())
2363 return hash_combine(args: Result, args: RFlags->getPredicate());
2364 if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def))
2365 return hash_combine(args: Result, args: SIVSteps->getInductionOpcode());
2366 // Fold in the separately stored consecutive flag. Alignment is left out and
2367 // handled by cse.
2368 if (auto *Load = dyn_cast<VPWidenMemoryRecipe>(Val: Def))
2369 return hash_combine(args: Result, args: Load->isConsecutive());
2370 return Result;
2371 }
2372
2373 /// Check equality of underlying data of \p L and \p R.
2374 static bool isEqual(const VPSingleDefRecipe *L, const VPSingleDefRecipe *R) {
2375 if (L->getVPRecipeID() != R->getVPRecipeID() ||
2376 vputils::getOpcodeOrIntrinsicID(V: L) !=
2377 vputils::getOpcodeOrIntrinsicID(V: R) ||
2378 getGEPSourceElementType(R: L) != getGEPSourceElementType(R) ||
2379 vputils::isSingleScalar(VPV: L) != vputils::isSingleScalar(VPV: R) ||
2380 !equal(LRange: L->operands(), RRange: R->operands()))
2381 return false;
2382 assert(vputils::getOpcodeOrIntrinsicID(L) &&
2383 vputils::getOpcodeOrIntrinsicID(R) &&
2384 "must have valid opcode info for both recipes");
2385 if (auto *LFlags = dyn_cast<VPRecipeWithIRFlags>(Val: L))
2386 if (LFlags->hasPredicate() &&
2387 LFlags->getPredicate() !=
2388 cast<VPRecipeWithIRFlags>(Val: R)->getPredicate())
2389 return false;
2390 if (auto *LSIV = dyn_cast<VPScalarIVStepsRecipe>(Val: L))
2391 if (LSIV->getInductionOpcode() !=
2392 cast<VPScalarIVStepsRecipe>(Val: R)->getInductionOpcode())
2393 return false;
2394 // Compare the separately stored consecutive flag. Alignment is left out and
2395 // handled by cse.
2396 if (auto *LL = dyn_cast<VPWidenMemoryRecipe>(Val: L))
2397 if (LL->isConsecutive() != cast<VPWidenMemoryRecipe>(Val: R)->isConsecutive())
2398 return false;
2399 // Phi recipes can only be equal if they are in the same VPBB, as they
2400 // implicitly depend on their predecessors.
2401 if (isa<VPWidenPHIRecipe>(Val: L) && L->getParent() != R->getParent())
2402 return false;
2403 // Recipes in replicate regions implicitly depend on predicate. If either
2404 // recipe is in a replicate region, only consider them equal if both have
2405 // the same parent.
2406 const VPRegionBlock *RegionL = L->getRegion();
2407 const VPRegionBlock *RegionR = R->getRegion();
2408 if (((RegionL && RegionL->isReplicator()) ||
2409 (RegionR && RegionR->isReplicator())) &&
2410 L->getParent() != R->getParent())
2411 return false;
2412 return L->getScalarType() == R->getScalarType();
2413 }
2414};
2415} // end anonymous namespace
2416
2417/// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
2418/// Plan.
2419void VPlanTransforms::cse(VPlan &Plan, const VPDominatorTree &VPDT) {
2420 DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo> CSEMap;
2421 // CSE map for widened loads. Must be cleared on recipes that may write to
2422 // memory, and at the end of each VPBB.
2423 DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo>
2424 LoadCSEMap;
2425
2426 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
2427 Plan.getEntry());
2428 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
2429 for (VPRecipeBase &R : *VPBB) {
2430 if (R.mayWriteToMemory())
2431 LoadCSEMap.clear();
2432 auto *Def = dyn_cast<VPSingleDefRecipe>(Val: &R);
2433 if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
2434 continue;
2435 bool IsLoad = isa<VPWidenLoadRecipe, VPWidenLoadEVLRecipe>(Val: Def);
2436 auto [It, Inserted] =
2437 (IsLoad ? LoadCSEMap : CSEMap).try_emplace(Key: Def, Args&: Def);
2438 if (Inserted)
2439 continue;
2440 VPSingleDefRecipe *V = It->second;
2441 // V must dominate Def for a valid replacement.
2442 if (!VPDT.dominates(A: V->getParent(), B: VPBB))
2443 continue;
2444 if (IsLoad) {
2445 auto *EarlierLoad = cast<VPWidenMemoryRecipe>(Val: V);
2446 auto *Load = cast<VPWidenMemoryRecipe>(Val: Def);
2447 if (EarlierLoad->getAlign() < Load->getAlign()) {
2448 // Record Load as the candidate for subsequent loads, as it may be
2449 // reusable where EarlierLoad is not.
2450 It->second = Def;
2451 continue;
2452 }
2453 // Keep only metadata common to both loads on the survivor.
2454 EarlierLoad->intersect(MD: *Load);
2455 }
2456 // Only keep flags present on both V and Def.
2457 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: V))
2458 RFlags->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Def));
2459 // Only keep call attributes valid for both V and Def.
2460 if (auto *Call = dyn_cast<VPWidenCallRecipe>(Val: V))
2461 Call->intersectAttrs(Other: *cast<VPWidenCallRecipe>(Val: Def), Ctx&: Plan.getContext());
2462 Def->replaceAllUsesWith(New: V);
2463 }
2464 LoadCSEMap.clear();
2465 }
2466}
2467
2468/// Return true if we do not know how to (mechanically) hoist or sink a
2469/// non-memory or memory recipe \p R out of a loop region. When sinking, passing
2470/// \p Sinking = true ensures that assumes aren't sunk.
2471static bool cannotHoistOrSinkRecipe(VPRecipeBase &R, VPBasicBlock *FirstBB,
2472 VPBasicBlock *LastBB,
2473 bool Sinking = false) {
2474 if (!isa<VPReplicateRecipe>(Val: R) || !R.mayReadOrWriteMemory() ||
2475 match(V: &R, P: m_Intrinsic<Intrinsic::assume>()))
2476 return vputils::cannotHoistOrSinkRecipe(R, Sinking);
2477
2478 // Check that the memory operation doesn't alias between FirstBB and LastBB.
2479 auto MemLoc = vputils::getMemoryLocation(R);
2480
2481 // TODO: Could make use of SinkStoreInfo::isNoAliasViaDistance by collecting
2482 // stores upfront, and constructing a full SinkStoreInfo.
2483 auto SinkInfo =
2484 Sinking ? std::make_optional(t: SinkStoreInfo(cast<VPReplicateRecipe>(Val&: R)))
2485 : std::nullopt;
2486
2487 return !MemLoc ||
2488 !canHoistOrSinkWithNoAliasCheck(MemLoc: *MemLoc, FirstBB, LastBB, SinkInfo);
2489}
2490
2491/// Move loop-invariant recipes out of the vector loop region in \p Plan.
2492static void licm(VPlan &Plan, const VPDominatorTree &VPDT) {
2493 VPBasicBlock *Preheader = Plan.getVectorPreheader();
2494
2495 // Hoist any loop invariant recipes from the vector loop region to the
2496 // preheader. Perform a shallow traversal of the vector loop region, to
2497 // exclude recipes in replicate regions. Only hoist from blocks dominating the
2498 // exiting block, which are guaranteed to execute if the vector pre-header is,
2499 // so we don't need to check speculation safety.
2500 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2501 assert(Preheader->getSingleSuccessor() == LoopRegion &&
2502 "Expected vector prehader's successor to be the vector loop region");
2503 VPBasicBlock *ExitingVPBB = LoopRegion->getExitingBasicBlock();
2504 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2505 Range: vp_depth_first_shallow(G: LoopRegion->getEntry()))) {
2506 if (!VPDT.dominates(A: VPBB, B: ExitingVPBB))
2507 continue;
2508 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2509 if (any_of(Range: R.operands(), P: [](VPValue *Op) {
2510 return !Op->isDefinedOutsideLoopRegions();
2511 }))
2512 continue;
2513 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2514 LastBB: LoopRegion->getExitingBasicBlock()))
2515 continue;
2516 R.moveBefore(BB&: *Preheader, I: Preheader->end());
2517 }
2518 }
2519
2520 // Sink recipes with no users inside the vector loop region if all users are
2521 // in the same exit block of the region.
2522 // TODO: Extend to sink recipes from inner loops.
2523 PostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> POT(
2524 LoopRegion->getEntry());
2525 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
2526 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
2527 // TODO: Use R.definedValues() instead of casting to VPSingleDefRecipe to
2528 // support recipes with multiple defined values (e.g., interleaved loads).
2529 auto *Def = dyn_cast<VPSingleDefRecipe>(Val: &R);
2530 if (!Def)
2531 continue;
2532
2533 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
2534 assert(!RepR->isPredicated() &&
2535 "Expected prior transformation of predicated replicates to "
2536 "replicate regions");
2537 // narrowToSingleScalarRecipes should have already maximally narrowed
2538 // replicates to single-scalar replicates.
2539 // TODO: When unrolling, replicateByVF doesn't handle sunk
2540 // non-single-scalar replicates correctly.
2541 if (!RepR->isSingleScalar())
2542 continue;
2543
2544 // The pointer operand of stores must be loop-invariant.
2545 if (RepR->getOpcode() == Instruction::Store &&
2546 !RepR->getOperand(N: 1)->isDefinedOutsideLoopRegions())
2547 continue;
2548 }
2549
2550 // Cannot sink the recipe if the user is defined in a loop region or a
2551 // non-successor of the vector loop region. Cannot sink if user is a phi
2552 // either.
2553 VPBasicBlock *SinkBB = nullptr;
2554 if (any_of(Range: Def->users(), P: [&SinkBB, &LoopRegion](VPUser *U) {
2555 auto *UserR = cast<VPRecipeBase>(Val: U);
2556 VPBasicBlock *Parent = UserR->getParent();
2557 // TODO: Support sinking when users are in multiple blocks.
2558 if (SinkBB && SinkBB != Parent)
2559 return true;
2560 SinkBB = Parent;
2561 // TODO: If the user is a PHI node, we should check the block of
2562 // incoming value. Support PHI node users if needed.
2563 return UserR->isPhi() || Parent->getEnclosingLoopRegion() ||
2564 Parent->getSinglePredecessor() != LoopRegion;
2565 }))
2566 continue;
2567
2568 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2569 LastBB: LoopRegion->getExitingBasicBlock(),
2570 /*Sinking=*/true))
2571 continue;
2572
2573 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
2574 assert((!R.mayWriteToMemory() ||
2575 (RepR && RepR->getOpcode() == Instruction::Store &&
2576 RepR->getOperand(1)->isDefinedOutsideLoopRegions())) &&
2577 "The only recipes that may write to memory are expected to be "
2578 "stores with invariant pointer-operand");
2579
2580 if (!SinkBB)
2581 SinkBB = cast<VPBasicBlock>(Val: LoopRegion->getSingleSuccessor());
2582
2583 // TODO: This will need to be a check instead of a assert after
2584 // conditional branches in vectorized loops are supported.
2585 assert(VPDT.properlyDominates(VPBB, SinkBB) &&
2586 "Defining block must dominate sink block");
2587 // TODO: Clone the recipe if users are on multiple exit paths, instead of
2588 // just moving.
2589 Def->moveBefore(BB&: *SinkBB, I: SinkBB->getFirstNonPhi());
2590 }
2591 }
2592}
2593
2594void VPlanTransforms::truncateToMinimalBitwidths(
2595 VPlan &Plan, const MapVector<Instruction *, uint64_t> &MinBWs) {
2596 if (Plan.hasScalarVFOnly())
2597 return;
2598 // Keep track of created truncates, so they can be re-used. Note that we
2599 // cannot use RAUW after creating a new truncate, as this would could make
2600 // other uses have different types for their operands, making them invalidly
2601 // typed.
2602 DenseMap<VPValue *, VPWidenCastRecipe *> ProcessedTruncs;
2603 VPBasicBlock *PH = Plan.getVectorPreheader();
2604 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2605 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
2606 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2607 if (!isa<VPWidenRecipe, VPWidenCastRecipe, VPReplicateRecipe,
2608 VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R))
2609 continue;
2610
2611 VPValue *ResultVPV = R.getVPSingleValue();
2612 auto *UI = cast_or_null<Instruction>(Val: ResultVPV->getUnderlyingValue());
2613 unsigned NewResSizeInBits = MinBWs.lookup(Key: UI);
2614 if (!NewResSizeInBits)
2615 continue;
2616
2617 // If the value wasn't vectorized, we must maintain the original scalar
2618 // type. Skip those here, after incrementing NumProcessedRecipes. Also
2619 // skip casts which do not need to be handled explicitly here, as
2620 // redundant casts will be removed during recipe simplification.
2621 if (isa<VPReplicateRecipe, VPWidenCastRecipe>(Val: &R))
2622 continue;
2623
2624 Type *OldResTy = ResultVPV->getScalarType();
2625 unsigned OldResSizeInBits = OldResTy->getScalarSizeInBits();
2626 assert(OldResTy->isIntegerTy() && "only integer types supported");
2627 (void)OldResSizeInBits;
2628
2629 auto *NewResTy = IntegerType::get(C&: Plan.getContext(), NumBits: NewResSizeInBits);
2630
2631 // Any wrapping introduced by shrinking this operation shouldn't be
2632 // considered undefined behavior. So, we can't unconditionally copy
2633 // arithmetic wrapping flags to VPW.
2634 if (auto *VPW = dyn_cast<VPRecipeWithIRFlags>(Val: &R))
2635 VPW->dropPoisonGeneratingFlags();
2636
2637 assert((OldResSizeInBits != NewResSizeInBits ||
2638 match(&R, m_ICmp(m_VPValue(), m_VPValue()))) &&
2639 "Only ICmps should not need extending the result.");
2640 assert(!isa<VPWidenStoreRecipe>(&R) && "stores cannot be narrowed");
2641
2642 // Loads/intrinsics are not recreated; they keep producing their original
2643 // wide result and narrowed users will truncate it as needed below.
2644 if (isa<VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R))
2645 continue;
2646
2647 // Shrink operands by introducing truncates as needed.
2648 unsigned StartIdx =
2649 match(V: &R, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())) ? 1 : 0;
2650 SmallVector<VPValue *> NewOperands(R.operands());
2651 for (VPValue *&Op : drop_begin(RangeOrContainer&: NewOperands, N: StartIdx)) {
2652 unsigned OpSizeInBits = Op->getScalarType()->getScalarSizeInBits();
2653 if (OpSizeInBits == NewResSizeInBits)
2654 continue;
2655 assert(OpSizeInBits > NewResSizeInBits && "nothing to truncate");
2656 auto [ProcessedIter, Inserted] = ProcessedTruncs.try_emplace(Key: Op);
2657 if (Inserted) {
2658 VPBuilder Builder;
2659 if (isa<VPIRValue>(Val: Op))
2660 Builder.setInsertPoint(PH);
2661 else
2662 Builder.setInsertPoint(&R);
2663 ProcessedIter->second =
2664 Builder.createWidenCast(Opcode: Instruction::Trunc, Op, ResultTy: NewResTy);
2665 }
2666 Op = ProcessedIter->second;
2667 }
2668
2669 auto *NWR = cast<VPWidenRecipe>(Val: &R)->cloneWithOperands(NewOperands);
2670 NWR->insertBefore(InsertPos: &R);
2671
2672 // Wrap NWR in a ZExt to preserve the original wide type for downstream
2673 // users. Not needed for ICmps, whose result type is i1 irrespective of
2674 // the narrowing of their operands.
2675 VPValue *Replacement = NWR->getVPSingleValue();
2676 if (Replacement->getScalarType() != OldResTy)
2677 Replacement =
2678 VPBuilder::getToInsertAfter(R: NWR)
2679 .createWidenCast(Opcode: Instruction::ZExt, Op: Replacement, ResultTy: OldResTy)
2680 ->getVPSingleValue();
2681 ResultVPV->replaceAllUsesWith(New: Replacement);
2682 R.eraseFromParent();
2683 }
2684 }
2685}
2686
2687bool VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) {
2688 std::optional<VPDominatorTree> VPDT;
2689 if (OnlyLatches)
2690 VPDT.emplace(args&: Plan);
2691
2692 // Collect all blocks before modifying the CFG so we can identify unreachable
2693 // ones after constant branch removal.
2694 SmallVector<VPBlockBase *> AllBlocks(vp_depth_first_shallow(G: Plan.getEntry()));
2695
2696 bool SimplifiedPhi = false;
2697 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: AllBlocks)) {
2698 VPValue *Cond;
2699 // Skip blocks that are not terminated by BranchOnCond.
2700 if (VPBB->empty() || !match(V: &VPBB->back(), P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))))
2701 continue;
2702
2703 if (OnlyLatches && !VPBlockUtils::isLatch(VPB: VPBB, VPDT: *VPDT))
2704 continue;
2705
2706 assert(VPBB->getNumSuccessors() == 2 &&
2707 "Two successors expected for BranchOnCond");
2708 unsigned RemovedIdx;
2709 if (match(V: Cond, P: m_True()))
2710 RemovedIdx = 1;
2711 else if (match(V: Cond, P: m_False()))
2712 RemovedIdx = 0;
2713 else
2714 continue;
2715
2716 VPBasicBlock *RemovedSucc =
2717 cast<VPBasicBlock>(Val: VPBB->getSuccessors()[RemovedIdx]);
2718 assert(count(RemovedSucc->getPredecessors(), VPBB) == 1 &&
2719 "There must be a single edge between VPBB and its successor");
2720 // Values coming from VPBB into phi recipes of RemovedSucc are removed from
2721 // these recipes and single-entry header phis are removed.
2722 for (VPRecipeBase &R : make_early_inc_range(Range: RemovedSucc->phis())) {
2723 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: VPBB);
2724 SimplifiedPhi = true;
2725 // Remove now invalid header phis that are left single-entry after
2726 // removing their backedges.
2727 auto *PhiR = dyn_cast<VPHeaderPHIRecipe>(Val: &R);
2728 if (!PhiR || PhiR->getNumIncoming() != 1)
2729 continue;
2730 PhiR->replaceAllUsesWith(New: PhiR->getOperand(N: 0));
2731 PhiR->eraseFromParent();
2732 }
2733
2734 // Disconnect blocks and remove the terminator.
2735 VPBlockUtils::disconnectBlocks(From: VPBB, To: RemovedSucc);
2736 VPBB->back().eraseFromParent();
2737 }
2738
2739 // Compute which blocks are still reachable from the entry after constant
2740 // branch removal.
2741 SmallPtrSet<VPBlockBase *, 16> Reachable(
2742 llvm::from_range, vp_depth_first_shallow(G: Plan.getEntry()));
2743
2744 // Detach all unreachable blocks from their successors, removing their recipes
2745 // and incoming values from phi recipes.
2746 VPSymbolicValue Tmp(nullptr);
2747 for (VPBlockBase *B : AllBlocks) {
2748 if (Reachable.contains(Ptr: B))
2749 continue;
2750 for (VPBlockBase *Succ : to_vector(Range: B->successors())) {
2751 if (auto *SuccBB = dyn_cast<VPBasicBlock>(Val: Succ))
2752 for (VPRecipeBase &R : SuccBB->phis())
2753 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: B);
2754 VPBlockUtils::disconnectBlocks(From: B, To: Succ);
2755 }
2756 for (VPBasicBlock *DeadBB :
2757 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_deep(G: B))) {
2758 for (VPRecipeBase &R : make_early_inc_range(Range&: *DeadBB)) {
2759 for (VPValue *Def : R.definedValues())
2760 Def->replaceAllUsesWith(New: &Tmp);
2761 R.eraseFromParent();
2762 }
2763 }
2764 }
2765 return SimplifiedPhi;
2766}
2767
2768void VPlanTransforms::optimize(VPlan &Plan) {
2769 RUN_VPLAN_PASS(removeRedundantInductionCasts, Plan);
2770
2771 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2772 RUN_VPLAN_PASS(combineRecipes, Plan);
2773 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2774 RUN_VPLAN_PASS(simplifyBlends, Plan);
2775 RUN_VPLAN_PASS(legalizeAndOptimizeInductions, Plan);
2776 RUN_VPLAN_PASS(narrowToSingleScalarRecipes, Plan);
2777 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2778 RUN_VPLAN_PASS(combineRecipes, Plan);
2779 RUN_VPLAN_PASS(removeBranchOnConst, Plan, /*OnlyLatches=*/false);
2780 RUN_VPLAN_PASS(simplifyReverses, Plan);
2781 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2782
2783 RUN_VPLAN_PASS(createAndOptimizeReplicateRegions, Plan);
2784 RUN_VPLAN_PASS(mergeBlocksIntoPredecessors, Plan);
2785 VPDominatorTree VPDT(Plan);
2786 RUN_VPLAN_PASS(licm, Plan, VPDT);
2787 RUN_VPLAN_PASS(cse, Plan, VPDT);
2788 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2789}
2790
2791void VPlanTransforms::simplifyLiveInsWithSCEV(VPlan &Plan,
2792 PredicatedScalarEvolution &PSE) {
2793 auto GetSimplifiedLiveInViaSCEV = [&](VPValue *VPV) -> VPValue * {
2794 const SCEV *Expr = vputils::getSCEVExprForVPValue(V: VPV, PSE);
2795 const APInt *C;
2796 if (match(S: Expr, P: m_scev_APInt(C)))
2797 return Plan.getConstantInt(Val: *C);
2798 return nullptr;
2799 };
2800
2801 for (VPValue *LiveIn : to_vector(Range: Plan.getLiveIns())) {
2802 if (VPValue *SimplifiedLiveIn = GetSimplifiedLiveInViaSCEV(LiveIn))
2803 LiveIn->replaceAllUsesWith(New: SimplifiedLiveIn);
2804 }
2805}
2806
2807void VPlanTransforms::replaceSymbolicStrides(
2808 VPlan &Plan, PredicatedScalarEvolution &PSE,
2809 const SymbolicStrideMap &StridesMap, const VPDominatorTree &VPDT) {
2810 // Replace VPValues for known constant strides guaranteed by predicated scalar
2811 // evolution that are guaranteed to be guarded by the runtime checks; that is,
2812 // blocks dominated by the vector header.
2813 assert(!Plan.getVectorLoopRegion() &&
2814 "expected to run before loop regions are created");
2815 const auto &[Header, _] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
2816 auto CanUseVersionedStride = [&VPDT, Header = Header, &Plan](VPUser *U) {
2817 auto *R = cast<VPRecipeBase>(Val: U);
2818 // Skip phis if the loop if loop is not yet guarded.
2819 if (isa<VPPhiAccessors>(Val: R) &&
2820 Header == Plan.getEntry()->getSingleSuccessor())
2821 return false;
2822 return VPDT.dominates(A: Header, B: R->getParent());
2823 };
2824 ValueToSCEVMapTy RewriteMap;
2825 for (const SCEVUnknown *Stride : StridesMap.values()) {
2826 Value *StrideV = Stride->getValue();
2827 const APInt *StrideConst;
2828 const SCEV *StrideExpr = PSE.getSCEV(V: StrideV);
2829 if (!match(S: StrideExpr, P: m_scev_APInt(C&: StrideConst)))
2830 // Only handle constant strides for now.
2831 continue;
2832 if (VPValue *StrideVPV = Plan.getLiveIn(V: StrideV))
2833 StrideVPV->replaceUsesWithIf(New: Plan.getConstantInt(Val: *StrideConst),
2834 ShouldReplace: CanUseVersionedStride);
2835
2836 // The versioned value may not be used in the loop directly but through an
2837 // integral cast (sext/zext/trunc). Add new live-ins in those cases.
2838 for (Value *U : StrideV->users()) {
2839 if (!isa<SExtInst, ZExtInst, TruncInst>(Val: U))
2840 continue;
2841 VPValue *StrideVPV = Plan.getLiveIn(V: U);
2842 if (!StrideVPV)
2843 continue;
2844 unsigned BW = U->getType()->getScalarSizeInBits();
2845 APInt C = isa<SExtInst>(Val: U) ? StrideConst->sext(width: BW)
2846 : StrideConst->zextOrTrunc(width: BW);
2847 StrideVPV->replaceUsesWithIf(New: Plan.getConstantInt(Val: C),
2848 ShouldReplace: CanUseVersionedStride);
2849 }
2850 RewriteMap[StrideV] = StrideExpr;
2851 }
2852
2853 for (VPExpandSCEVRecipe &ExpSCEV :
2854 make_isa_range<VPExpandSCEVRecipe>(Range&: *Plan.getEntry())) {
2855 const SCEV *ScevExpr = ExpSCEV.getSCEV();
2856 auto *NewSCEV =
2857 SCEVParameterRewriter::rewrite(Scev: ScevExpr, SE&: *PSE.getSE(), Map&: RewriteMap);
2858 if (NewSCEV != ScevExpr) {
2859 VPValue *NewExp = vputils::getOrCreateVPValueForSCEVExpr(Plan, Expr: NewSCEV);
2860 ExpSCEV.replaceAllUsesWith(New: NewExp);
2861 if (Plan.getTripCount() == &ExpSCEV)
2862 Plan.resetTripCount(NewTripCount: NewExp);
2863 }
2864 }
2865}
2866
2867void VPlanTransforms::dropPoisonGeneratingRecipes(VPlan &Plan) {
2868 // Collect recipes in the backward slice of `Root` that may generate a poison
2869 // value that is used after vectorization.
2870 SmallPtrSet<VPRecipeBase *, 16> Visited;
2871 auto CollectPoisonGeneratingInstrsInBackwardSlice([&](VPRecipeBase *Root) {
2872 SmallVector<VPRecipeBase *, 16> Worklist;
2873 Worklist.push_back(Elt: Root);
2874
2875 // Traverse the backward slice of Root through its use-def chain.
2876 while (!Worklist.empty()) {
2877 VPRecipeBase *CurRec = Worklist.pop_back_val();
2878
2879 if (!Visited.insert(Ptr: CurRec).second)
2880 continue;
2881
2882 // Prune search if we find another recipe generating a widen memory
2883 // instruction. Widen memory instructions involved in address computation
2884 // will lead to gather/scatter instructions, which don't need to be
2885 // handled.
2886 if (isa<VPWidenMemoryRecipe, VPInterleaveRecipe, VPScalarIVStepsRecipe,
2887 VPHeaderPHIRecipe>(Val: CurRec))
2888 continue;
2889
2890 // This recipe contributes to the address computation of a widen
2891 // load/store. If the underlying instruction has poison-generating flags,
2892 // drop them directly.
2893 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: CurRec)) {
2894 VPValue *A, *B;
2895 // Dropping disjoint from an OR may yield incorrect results, as some
2896 // analysis may have converted it to an Add implicitly (e.g. SCEV used
2897 // for dependence analysis). Instead, replace it with an equivalent Add.
2898 // This is possible as all users of the disjoint OR only access lanes
2899 // where the operands are disjoint or poison otherwise.
2900 if (match(V: RecWithFlags, P: m_BinaryOr(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))) &&
2901 RecWithFlags->isDisjoint()) {
2902 VPBuilder Builder(RecWithFlags);
2903 VPInstruction *New =
2904 Builder.createAdd(LHS: A, RHS: B, DL: RecWithFlags->getDebugLoc());
2905 New->setUnderlyingValue(RecWithFlags->getUnderlyingValue());
2906 RecWithFlags->replaceAllUsesWith(New);
2907 RecWithFlags->eraseFromParent();
2908 CurRec = New;
2909 } else
2910 RecWithFlags->dropPoisonGeneratingFlags();
2911 } else {
2912 Instruction *Instr = dyn_cast_or_null<Instruction>(
2913 Val: CurRec->getVPSingleValue()->getUnderlyingValue());
2914 (void)Instr;
2915 assert((!Instr || !Instr->hasPoisonGeneratingFlags()) &&
2916 "found instruction with poison generating flags not covered by "
2917 "VPRecipeWithIRFlags");
2918 }
2919
2920 // Add new definitions to the worklist.
2921 for (VPValue *Operand : CurRec->operands())
2922 if (VPRecipeBase *OpDef = Operand->getDefiningRecipe())
2923 Worklist.push_back(Elt: OpDef);
2924 }
2925 });
2926
2927 // We want to exclude the tail folding case, as we don't need to drop flags
2928 // for operations computing the first lane in this case: the first lane of the
2929 // header mask must always be true. For reverse memory accesses, the mask is
2930 // wrapped in a Reverse, which is just a permutation of the header mask, so
2931 // peel it off before checking. The header mask is still the abstract region
2932 // value at this point (materialization happens later).
2933 auto m_UnlessHdrMask = m_Unless( // NOLINT
2934 P: m_CombineOr(Ps: m_HeaderMask(), Ps: m_Reverse(Op0: m_HeaderMask())));
2935
2936 // Traverse all the recipes in the VPlan and collect the poison-generating
2937 // recipes in the backward slice starting at the address of a VPWidenRecipe or
2938 // VPInterleaveRecipe.
2939 auto Iter =
2940 vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntryBasicBlock());
2941 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: Iter)) {
2942 for (VPRecipeBase &Recipe : *VPBB) {
2943 if (auto *WidenRec = dyn_cast<VPWidenMemoryRecipe>(Val: &Recipe)) {
2944 VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe();
2945 if (AddrDef && WidenRec->isConsecutive() && WidenRec->getMask() &&
2946 match(V: WidenRec->getMask(), P: m_UnlessHdrMask))
2947 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2948 } else if (auto *InterleaveRec = dyn_cast<VPInterleaveRecipe>(Val: &Recipe)) {
2949 VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe();
2950 if (AddrDef && InterleaveRec->getMask() &&
2951 match(V: InterleaveRec->getMask(), P: m_UnlessHdrMask))
2952 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2953 }
2954 }
2955 }
2956}
2957
2958void VPlanTransforms::createInterleaveGroups(
2959 VPlan &Plan,
2960 const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
2961 &InterleaveGroups,
2962 const bool &EpilogueAllowed) {
2963 if (InterleaveGroups.empty())
2964 return;
2965
2966 DenseMap<Instruction *, VPWidenMemoryRecipe *> IRMemberToRecipe;
2967 for (VPBasicBlock *VPBB :
2968 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
2969 G: Plan.getVectorLoopRegion()->getEntryBasicBlock())))
2970 for (VPRecipeBase &R : make_filter_range(Range&: *VPBB, Pred: [](VPRecipeBase &R) {
2971 return isa<VPWidenMemoryRecipe>(Val: &R);
2972 })) {
2973 auto *MemR = cast<VPWidenMemoryRecipe>(Val: &R);
2974 IRMemberToRecipe[&MemR->getIngredient()] = MemR;
2975 }
2976
2977 // Interleave memory: for each Interleave Group we marked earlier as relevant
2978 // for this VPlan, replace the Recipes widening its memory instructions with a
2979 // single VPInterleaveRecipe at its insertion point.
2980 VPDominatorTree VPDT(Plan);
2981 for (const auto *IG : InterleaveGroups) {
2982 VPWidenMemoryRecipe *Start = nullptr;
2983 Instruction *StartMember = nullptr;
2984 for (auto *Member : IG->members())
2985 if (VPWidenMemoryRecipe *R = IRMemberToRecipe.lookup(Val: Member)) {
2986 StartMember = Member;
2987 Start = R;
2988 break;
2989 }
2990 if (!StartMember) // All member recipes are dead, so the group is dead.
2991 continue;
2992 VPIRMetadata InterleaveMD(*Start);
2993 SmallVector<VPValue *, 4> StoredValues;
2994 for (unsigned I = 0; I < IG->getFactor(); ++I) {
2995 Instruction *MemberI = IG->getMember(Index: I);
2996 if (!MemberI)
2997 continue;
2998 if (VPWidenMemoryRecipe *MemoryR = IRMemberToRecipe.lookup(Val: MemberI)) {
2999 if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Val: MemoryR->getAsRecipe()))
3000 StoredValues.push_back(Elt: StoreR->getStoredValue());
3001 InterleaveMD.intersect(MD: *MemoryR);
3002 } else {
3003 InterleaveMD.intersect(MD: VPIRMetadata(*MemberI));
3004 }
3005 }
3006
3007 bool NeedsMaskForGaps =
3008 (IG->requiresScalarEpilogue() && !EpilogueAllowed) ||
3009 (!StoredValues.empty() && !IG->isFull());
3010
3011 Instruction *IRInsertPos = IG->getInsertPos();
3012 auto *InsertPos = IRMemberToRecipe.lookup(Val: IRInsertPos);
3013 if (!InsertPos) {
3014 // InsertPos member is dead: find a new member that is alive.
3015 assert(isa<VPWidenLoadRecipe>(Start->getAsRecipe()) &&
3016 "Dead member in non-load group?");
3017 InsertPos = Start;
3018 for (Instruction *Member : IG->members())
3019 if (VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member))
3020 if (VPDT.properlyDominates(A: MemberR->getAsRecipe(),
3021 B: InsertPos->getAsRecipe()))
3022 InsertPos = MemberR;
3023 IRInsertPos = &InsertPos->getIngredient();
3024 }
3025 VPRecipeBase *InsertPosR = InsertPos->getAsRecipe();
3026
3027 GEPNoWrapFlags NW = GEPNoWrapFlags::none();
3028 if (auto *Gep = dyn_cast<GetElementPtrInst>(
3029 Val: getLoadStorePointerOperand(V: IRInsertPos)->stripPointerCasts()))
3030 NW = Gep->getNoWrapFlags().withoutNoUnsignedWrap();
3031
3032 // Get or create the start address for the interleave group.
3033 VPValue *Addr = Start->getAddr();
3034 VPRecipeBase *AddrDef = Addr->getDefiningRecipe();
3035 if (IG->getIndex(Instr: StartMember) != 0 ||
3036 (AddrDef && !VPDT.properlyDominates(A: AddrDef, B: InsertPosR))) {
3037 // Either member zero's recipe is dead, or we cannot re-use the address of
3038 // member zero because it does not dominate the insert position. Instead,
3039 // use the address of the insert position and create a PtrAdd adjusting it
3040 // to the address of member zero.
3041 // TODO: Hoist Addr's defining recipe (and any operands as needed) to
3042 // InsertPos or sink loads above zero members to join it.
3043 assert(IG->getIndex(IRInsertPos) != 0 &&
3044 "index of insert position shouldn't be zero");
3045 auto &DL = IRInsertPos->getDataLayout();
3046 APInt Offset(32,
3047 DL.getTypeAllocSize(Ty: getLoadStoreType(I: IRInsertPos)) *
3048 IG->getIndex(Instr: IRInsertPos),
3049 /*IsSigned=*/true);
3050 VPValue *OffsetVPV = Plan.getConstantInt(Val: -Offset);
3051 VPBuilder B(InsertPosR);
3052 Addr = B.createNoWrapPtrAdd(Ptr: InsertPos->getAddr(), Offset: OffsetVPV, GEPFlags: NW);
3053 }
3054 // If the group is reverse, adjust the index to refer to the last vector
3055 // lane instead of the first. We adjust the index from the first vector
3056 // lane, rather than directly getting the pointer for lane VF - 1, because
3057 // the pointer operand of the interleaved access is supposed to be uniform.
3058 if (IG->isReverse()) {
3059 auto *ReversePtr = new VPVectorEndPointerRecipe(
3060 Addr, &Plan.getVF(), getLoadStoreType(I: IRInsertPos),
3061 -(int64_t)IG->getFactor(), NW, InsertPosR->getDebugLoc());
3062 ReversePtr->insertBefore(InsertPos: InsertPosR);
3063 Addr = ReversePtr;
3064 }
3065 auto *VPIG = new VPInterleaveRecipe(
3066 IG, Addr, StoredValues, InsertPos->getMask(), NeedsMaskForGaps,
3067 InterleaveMD, InsertPosR->getDebugLoc());
3068 VPIG->insertBefore(InsertPos: InsertPosR);
3069
3070 unsigned J = 0;
3071 for (unsigned i = 0; i < IG->getFactor(); ++i)
3072 if (Instruction *Member = IG->getMember(Index: i)) {
3073 VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member);
3074 if (!Member->getType()->isVoidTy()) {
3075 if (MemberR) {
3076 VPValue *OriginalV = MemberR->getAsRecipe()->getVPSingleValue();
3077 OriginalV->replaceAllUsesWith(New: VPIG->getVPValue(I: J));
3078 }
3079 J++;
3080 }
3081 if (MemberR)
3082 MemberR->getAsRecipe()->eraseFromParent();
3083 }
3084 }
3085}
3086
3087/// Matches an exit condition formed by comparing a value loaded from memory
3088/// with a loop-invariant term. Binds the comparison for the condition.
3089static auto m_Uncountable(VPValue *&Cond) {
3090 return m_VPValue(
3091 V&: Cond,
3092 Op: m_c_Cmp(Op0: m_VPInstruction<Instruction::Load>(Ops: m_VPValue()), Op1: m_LiveIn()));
3093}
3094
3095namespace {
3096struct CountableConditionMatch {
3097 VPValue *&Cmp;
3098 PredicatedScalarEvolution &PSE;
3099 Loop *L;
3100
3101 CountableConditionMatch(VPValue *&Cmp, PredicatedScalarEvolution &PSE,
3102 Loop *L)
3103 : Cmp(Cmp), PSE(PSE), L(L) {}
3104
3105 template <typename ITy> bool match(ITy *V) const {
3106 VPValue *Update;
3107 if (!VPlanPatternMatch::match(
3108 V, m_VPValue(V&: Cmp, Op: m_c_ICmp(Op0: m_VPValue(V&: Update, Op: m_Add(Op0: m_VPValue(),
3109 Op1: m_VPValue())),
3110 Op1: m_LiveIn()))))
3111 return false;
3112
3113 const SCEV *S = vputils::getSCEVExprForVPValue(V: Update, PSE, L);
3114 return SCEVPatternMatch::match(
3115 S, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_scev_One(), L: m_SpecificLoop(L)));
3116 }
3117};
3118} // end anonymous namespace
3119
3120/// Matches an exit condition formed by comparing the current value of a
3121/// affine add recurrence in the given loop with a stride of 1 against a
3122/// loop-invariant term. Binds the comparison for the condition.
3123static auto m_Countable(VPValue *&Cmp, PredicatedScalarEvolution &PSE,
3124 Loop *L) {
3125 return CountableConditionMatch(Cmp, PSE, L);
3126}
3127
3128bool VPlanTransforms::splitCombinedExits(VPlan &Plan,
3129 PredicatedScalarEvolution &PSE,
3130 Loop *L) {
3131 // Check for a single combined exit in the latch block.
3132 // TODO: Generalize to other blocks besides the latch.
3133 // If we don't find a combined condition in the latch, just return true
3134 // to proceed with vectorization.
3135 auto [_, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
3136
3137 // We're looking for a conditional branch...
3138 auto *Term = dyn_cast<VPInstruction>(Val: LatchVPBB->getTerminator());
3139 if (!Term || Term->getOpcode() != VPInstruction::BranchOnCond)
3140 return true;
3141
3142 // ...where the condition is a combination of both a countable and an
3143 // uncountable comparison.
3144 VPValue *Uncountable = nullptr;
3145 VPValue *Countable = nullptr;
3146 VPValue *Cond = Term->getOperand(N: 0);
3147 if (!match(V: Cond, P: m_CombineOr(Ps: m_c_LogicalOr(Op0: m_Uncountable(Cond&: Uncountable),
3148 Op1: m_Countable(Cmp&: Countable, PSE, L)),
3149 Ps: m_c_BinaryOr(Op0: m_Uncountable(Cond&: Uncountable),
3150 Op1: m_Countable(Cmp&: Countable, PSE, L)))))
3151 return true;
3152
3153 // If the conditions are combined with a logical or (select), then we'll
3154 // need to freeze the individual terms when splitting.
3155 bool NeedsFreeze = match(V: Cond, P: m_LogicalOr(Op0: m_VPValue(), Op1: m_VPValue()));
3156
3157 // If we do have a combined exit condition, bail out if there's more than
3158 // one exit block.
3159 // TODO: Support additional exits.
3160 ArrayRef<VPIRBasicBlock *> ExitBlocks = Plan.getExitBlocks();
3161 if (ExitBlocks.size() != 1)
3162 return false;
3163
3164 // If there are any live-outs, bail out. The exit block is an existing IR
3165 // block, and if we split the exiting block then the incoming blocks and
3166 // values won't be correct.
3167 // TODO: Support live-outs with combined exits.
3168 if (!ExitBlocks.front()->phis().empty())
3169 return false;
3170
3171 // Split the latch block just before the terminator.
3172 VPBasicBlock *NewLatch = LatchVPBB->splitAt(SplitAt: Term->getIterator());
3173
3174 // Create new terminator for uncountable condition.
3175 VPBuilder EEBuilder(LatchVPBB);
3176 if (NeedsFreeze)
3177 Uncountable = EEBuilder.createFreeze(Op: Uncountable);
3178 EEBuilder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {Uncountable});
3179
3180 // We need to connect the uncountable exit to the sole exit block. The
3181 // latch is expected to connect to the middle block instead.
3182 // In canonical form, the backedge is the last successor for the latch. So
3183 // the first successor (true path) should be the exit for both conditions.
3184 LatchVPBB->clearSuccessors();
3185 NewLatch->clearPredecessors();
3186 VPBlockUtils::connectBlocks(From: LatchVPBB, To: ExitBlocks.front());
3187 VPBlockUtils::connectBlocks(From: LatchVPBB, To: NewLatch);
3188
3189 // Set condition for latch block to countable condition.
3190 if (NeedsFreeze) {
3191 VPBuilder NewLatchBuilder(Term);
3192 Countable = NewLatchBuilder.createFreeze(Op: Countable);
3193 }
3194 Term->setOperand(I: 0, New: Countable);
3195
3196 return true;
3197}
3198
3199/// Returns the VPValue representing the uncountable exit comparison used by
3200/// AnyOf if the recipes it depends on can be traced back to live-ins and
3201/// the addresses (in GEP/PtrAdd form) of any (non-masked) load used in
3202/// generating the values for the comparison. The recipes are stored in
3203/// \p Recipes.
3204static VPValue *
3205getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
3206 VPBasicBlock *LatchVPBB) {
3207 // Given a plain CFG VPlan loop with countable latch exiting block
3208 // \p LatchVPBB, we're looking to match the recipes contributing to the
3209 // uncountable exit condition comparison (here, vp<%4>) back to either
3210 // live-ins or the address nodes for the load used as part of the uncountable
3211 // exit comparison so that we can either move them within the loop, or copy
3212 // them to the preheader depending on the chosen method for dealing with
3213 // stores in uncountable exit loops.
3214 //
3215 // Currently, the address of the load is restricted to a GEP with 2 operands
3216 // and a live-in base address. This constraint may be relaxed later.
3217 //
3218 // VPlan ' for UF>=1' {
3219 // Live-in vp<%0> = VF * UF
3220 // Live-in vp<%1> = vector-trip-count
3221 // Live-in ir<20> = original trip-count
3222 //
3223 // ir-bb<entry>:
3224 // Successor(s): scalar.ph, vector.ph
3225 //
3226 // vector.ph:
3227 // Successor(s): for.body
3228 //
3229 // for.body:
3230 // EMIT vp<%2> = phi ir<0>, vp<%index.next>
3231 // EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, for.inc ]
3232 // EMIT ir<%uncountable.addr> = getelementptr inbounds nuw ir<%pred>,ir<%iv>
3233 // EMIT ir<%uncountable.val> = load ir<%uncountable.addr>
3234 // EMIT ir<%uncountable.cond> = icmp sgt ir<%uncountable.val>, ir<500>
3235 // EMIT vp<%3> = masked-cond ir<%uncountable.cond>
3236 // Successor(s): for.inc
3237 //
3238 // for.inc:
3239 // EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
3240 // EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
3241 // EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
3242 // EMIT vp<%freeze> = freeze ir<%3>
3243 // EMIT vp<%4> = any-of ir<%freeze>
3244 // EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
3245 // EMIT branch-on-two-conds vp<%4>, vp<%5>
3246 // Successor(s): middle.block, middle.block, for.body
3247 //
3248 // middle.block:
3249 // Successor(s): ir-bb<exit>, scalar.ph
3250 //
3251 // ir-bb<exit>:
3252 // No successors
3253 //
3254 // scalar.ph:
3255 // }
3256
3257 // Find the uncountable loop exit condition.
3258 VPValue *UncountableCondition = nullptr;
3259 if (!match(V: LatchVPBB->getTerminator(),
3260 P: m_BranchOnTwoConds(Op0: m_AnyOf(Op0: m_VPValue(V&: UncountableCondition)),
3261 Op1: m_VPValue())))
3262 return nullptr;
3263
3264 SmallVector<VPValue *, 4> Worklist;
3265 Worklist.push_back(Elt: UncountableCondition);
3266 while (!Worklist.empty()) {
3267 VPValue *V = Worklist.pop_back_val();
3268
3269 // Any value defined outside the loop does not need to be copied.
3270 if (V->isDefinedOutsideLoopRegions())
3271 continue;
3272
3273 VPValue *Op1, *Op2;
3274 // Walk back through recipes until we find at least one load from memory.
3275 if (match(V, P: m_Cmp(Op0: m_VPValue(V&: Op1), Op1: m_VPValue(V&: Op2)))) {
3276 Worklist.push_back(Elt: Op1);
3277 Worklist.push_back(Elt: Op2);
3278 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3279 } else if (match(V, P: m_VPInstruction<Instruction::Load>(Ops: m_VPValue(V&: Op1)))) {
3280 VPRecipeBase *GepR = Op1->getDefiningRecipe();
3281 // Only matching base + single offset term for now.
3282 if (GepR->getNumOperands() != 2)
3283 return nullptr;
3284 // Matching a GEP with a loop-invariant base ptr.
3285 if (!match(V: GepR, P: m_VPInstruction<Instruction::GetElementPtr>(
3286 Ops: m_LiveIn(), Ops: m_VPValue())))
3287 return nullptr;
3288 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3289 Recipes.push_back(Elt: cast<VPInstruction>(Val: GepR));
3290 } else if (match(V, P: m_Freeze(Op0: m_VPValue(V&: Op1)))) {
3291 Worklist.push_back(Elt: Op1);
3292 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3293 } else if (match(V, P: m_VPInstruction<VPInstruction::MaskedCond>(
3294 Ops: m_VPValue(V&: Op1)))) {
3295 Worklist.push_back(Elt: Op1);
3296 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3297 } else
3298 return nullptr;
3299 }
3300
3301 // If we couldn't match anything, don't return the condition. It may be
3302 // defined outside the loop.
3303 if (Recipes.empty() ||
3304 none_of(Range&: Recipes, P: match_fn(P: m_VPInstruction<Instruction::GetElementPtr>())))
3305 return nullptr;
3306
3307 return UncountableCondition;
3308}
3309
3310struct EarlyExitInfo {
3311 VPBasicBlock *EarlyExitingVPBB;
3312 VPIRBasicBlock *EarlyExitVPBB;
3313 VPValue *CondToExit;
3314};
3315
3316/// Update \p Plan to mask memory operations in the loop based on whether the
3317/// early exit is taken or not.
3318///
3319/// We're currently expecting to find a loop with properties similar to the
3320/// following:
3321///
3322/// for.body:
3323/// ir<%indvars.iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<%0>
3324/// EMIT ir<%arrayidx> = getelementptr inbounds nuw ir<@c>, ir<%indvars.iv>
3325/// EMIT-SCALAR ir<%0> = load ir<%arrayidx>
3326/// EMIT ir<%cmp1> = icmp sgt ir<%0>, ir<5>
3327/// EMIT vp<%1> = masked-cond ir<%cmp1>
3328/// Successor(s): if.end
3329///
3330/// if.end:
3331/// EMIT ir<%arrayidx3> = getelementptr inbounds nuw ir<@src>, ir<%indvars.iv>
3332/// EMIT-SCALAR ir<%2> = load ir<%arrayidx3>
3333/// EMIT ir<%add> = add nsw ir<%2>, ir<42>
3334/// EMIT ir<%arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<%indvars.iv>
3335/// EMIT store ir<%add>, ir<%arrayidx5>
3336/// EMIT ir<%indvars.iv.next> = add nuw nsw ir<%indvars.iv>, ir<1>
3337/// EMIT vp<%freeze> = freeze ir<%1>
3338/// EMIT vp<%3> = any-of ir<%freeze>
3339/// EMIT ir<%exitcond.not> = icmp eq ir<%indvars.iv.next>, ir<10000>
3340/// EMIT branch-on-two-conds vp<%3>, ir<%exitcond.not>
3341/// Successor(s): middle.block, middle.block, for.body
3342///
3343/// We currently expect LoopVectorizationLegality to ensure that:
3344/// * There must also be a counted exit. We will need to support speculative
3345/// or first-faulting loads before we can remove this restriction.
3346/// * Any stores within the loop must not alias with the load used for the
3347/// uncountable exit. We can relax this a bit with runtime aliasing checks.
3348/// * Other memory operations in the loop can take place before or after the
3349/// uncountable exit, but must also be unconditional. We need to support
3350/// combining the conditions in VPlanPredicator.
3351/// * The loop must have a single unconditional load contributing to the
3352/// uncountable exit comparison, and the other term must be loop-invariant.
3353/// Improving upon this requires work in getRecipesForUncountableExit to
3354/// handle more complex recipe graphs.
3355static bool handleUncountableExitsWithSideEffects(
3356 VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
3357 VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB,
3358 OptimizationRemarkEmitter *ORE, Loop *TheLoop,
3359 PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC) {
3360
3361 // Disconnect early exiting blocks from successors, remove branches.
3362 for (auto &Exit : Exits) {
3363 if (Exit.EarlyExitingVPBB == LatchVPBB)
3364 continue;
3365
3366 for (VPRecipeBase &R : Exit.EarlyExitVPBB->phis())
3367 cast<VPIRPhi>(Val: &R)->removeIncomingValueFor(IncomingBlock: Exit.EarlyExitingVPBB);
3368 Exit.EarlyExitingVPBB->getTerminator()->eraseFromParent();
3369 VPBlockUtils::disconnectBlocks(From: Exit.EarlyExitingVPBB, To: Exit.EarlyExitVPBB);
3370 }
3371
3372 VPDominatorTree VPDT(Plan);
3373
3374 // We can abandon a VPlan entirely if we return false here, so we shouldn't
3375 // crash if some earlier assumptions on scalar IR don't hold for the vplan
3376 // version of the loop.
3377 SmallVector<VPInstruction *, 8> ConditionRecipes;
3378
3379 VPValue *Cond = getRecipesForUncountableExit(Recipes&: ConditionRecipes, LatchVPBB);
3380 if (!Cond) {
3381 reportVectorizationFailure(DebugMsg: "Unable to determine early exit condition for "
3382 "loop with side effects",
3383 ORETag: "EarlyExitSideEffectsCond", ORE, TheLoop);
3384 return false;
3385 }
3386
3387 // Find load contributing to condition.
3388 // At the moment LoopVectorizationLegality only supports a single
3389 // early-exit expression with a compare and a single load that must
3390 // be unconditional.
3391 // TODO: Support more than one load.
3392 auto *Load =
3393 find_singleton<VPInstruction>(Range&: ConditionRecipes, P: [](auto *I, bool _) {
3394 return match(I, m_VPInstruction<Instruction::Load>(Ops: m_VPValue()))
3395 ? I
3396 : nullptr;
3397 });
3398 assert(Load && "Couldn't find exactly one load");
3399 // TODO: Support conditional loads for uncountable exits.
3400 assert(VPDT.dominates(Load->getParent(), LatchVPBB) &&
3401 "Uncountable exit condition load is conditional.");
3402 VPInstruction *Ptr = cast<VPInstruction>(Val: Load->getOperand(N: 0));
3403
3404 // Ensure that we are guaranteed to be able to dereference the memory used
3405 // for determining the uncountable exit for the maximum possible number of
3406 // scalar iterations of the loop.
3407 //
3408 // TODO: Support first-faulting loads in cases where we don't know whether
3409 // all possible addresses are dereferenceable.
3410 {
3411 SmallVector<const SCEVPredicate *, 4> Predicates;
3412 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: TheLoop);
3413 const DataLayout &DL = Plan.getDataLayout();
3414 APInt EltSize(DL.getIndexTypeSizeInBits(Ty: Ptr->getScalarType()),
3415 DL.getTypeStoreSize(Ty: Load->getScalarType()).getFixedValue());
3416 if (!isDereferenceableAndAlignedInLoop(
3417 PtrSCEV, Alignment: cast<LoadInst>(Val: Load->getUnderlyingInstr())->getAlign(),
3418 EltSizeSCEV: PSE.getSE()->getConstant(Val: EltSize), L: TheLoop, SE&: *PSE.getSE(), DT, AC,
3419 Predicates: &Predicates)) {
3420 reportVectorizationFailure(DebugMsg: "Early exit loop with side effects contains "
3421 "load used by the exit condition that may "
3422 "fault",
3423 ORETag: "EarlyExitSideEffectsFaultingLoad", ORE,
3424 TheLoop);
3425 return false;
3426 }
3427 }
3428
3429 // Check for a single GEP for the condition load to see if we can link it to
3430 // a widen IV recipe with a step of 1; we're only interested in contiguous
3431 // accesses for the condition load right now.
3432 auto *IV = cast<VPWidenInductionRecipe>(Val: &HeaderVPBB->front());
3433 if (!match(V: IV->getStartValue(), P: m_SpecificInt(V: 0)) ||
3434 !match(V: IV->getStepValue(), P: m_SpecificInt(V: 1))) {
3435 reportVectorizationFailure(DebugMsg: "Early exit loop with side effects contains "
3436 "load used by the exit condition with an "
3437 "unsupported memory access pattern",
3438 ORETag: "EarlyExitSideEffectsBadLoadAccessPattern", ORE,
3439 TheLoop);
3440 return false;
3441 }
3442
3443 if (!match(V: Ptr, P: m_VPInstruction<Instruction::GetElementPtr>(
3444 Ops: m_LiveIn(), Ops: m_Specific(VPV: IV)))) {
3445 reportVectorizationFailure(DebugMsg: "Early exit loop with side effects contains "
3446 "load used by the exit condition with an "
3447 "unsupported memory access pattern",
3448 ORETag: "EarlyExitSideEffectsBadLoadAccessPattern", ORE,
3449 TheLoop);
3450 return false;
3451 }
3452
3453 // We want to guarantee that the uncountable exit condition (and the mask
3454 // we will generate from it) are available for all operations in the loop
3455 // that need to be masked. If the condition recipes are not already the first
3456 // recipes in the header after the last phi, move them there.
3457 auto InsertIt = HeaderVPBB->getFirstNonPhi();
3458 while (InsertIt != HeaderVPBB->end() &&
3459 is_contained(Range&: ConditionRecipes, Element: &*InsertIt)) {
3460 erase(C&: ConditionRecipes, V: &*InsertIt);
3461 InsertIt++;
3462 }
3463 for (auto *Recipe : reverse(C&: ConditionRecipes))
3464 Recipe->moveBefore(BB&: *HeaderVPBB, I: InsertIt);
3465
3466 // Create a mask to represent all lanes that fully execute in the vector loop,
3467 // stopping short of any early exit.
3468 VPBuilder MaskBuilder(HeaderVPBB, InsertIt);
3469 VPValue *FirstActive = MaskBuilder.createFirstActiveLane(Masks: Cond);
3470 Type *IVScalarTy = IV->getScalarType();
3471 VPValue *Zero = Plan.getZero(Ty: IVScalarTy);
3472 FirstActive =
3473 MaskBuilder.createScalarZExtOrTrunc(Op: FirstActive, ResultTy: IVScalarTy, DL: DebugLoc());
3474 VPValue *Mask = MaskBuilder.createNaryOp(Opcode: VPInstruction::ActiveLaneMask,
3475 Operands: {Zero, FirstActive}, DL: DebugLoc(),
3476 Name: "uncountable.exit.mask");
3477
3478 // Convert all other memory operations to use the mask.
3479 for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(Header: HeaderVPBB))
3480 for (VPRecipeBase &R : *VPBB)
3481 if (R.mayReadOrWriteMemory() && &R != Load) {
3482 // TODO: Handle conditional memory operations in the loop.
3483 if (!VPDT.dominates(A: R.getParent(), B: LatchVPBB)) {
3484 reportVectorizationFailure(
3485 DebugMsg: "Early exit loop with side effects contains unsupported "
3486 "conditional memory operations",
3487 ORETag: "EarlyExitSideEffectsUnsupportedConditionalMemOps", ORE, TheLoop);
3488 return false;
3489 }
3490 cast<VPInstruction>(Val: &R)->addMask(Mask);
3491 }
3492
3493 // Update middle block branch to compare (IV + however many lanes were active)
3494 // against the full trip count, since we may be exiting the vector loop early.
3495 // If we didn't take an early exit, we should get the equivalent of VF from
3496 // the FirstActiveLane.
3497 assert(match(MiddleVPBB->getTerminator(), m_BranchOnCond()) &&
3498 "Expected BranchOnCond terminator for MiddleVPBB");
3499 VPBuilder MiddleBuilder(MiddleVPBB->getTerminator());
3500 VPValue *ScalarIV = MiddleBuilder.createNaryOp(Opcode: VPInstruction::ExtractLane,
3501 Operands: {Zero, IV}, DL: DebugLoc());
3502 VPValue *ExitIV = MiddleBuilder.createAdd(LHS: ScalarIV, RHS: FirstActive);
3503 VPValue *FullTC =
3504 MiddleBuilder.createICmp(Pred: CmpInst::ICMP_EQ, A: ExitIV, B: Plan.getTripCount());
3505 MiddleVPBB->getTerminator()->setOperand(I: 0, New: FullTC);
3506
3507 // Update resume phi in scalar.ph.
3508 VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
3509 auto Phis = ScalarPH->phis();
3510 // TODO: Handle more than one Phi; re-derive from IV.
3511 // TODO: Handle reductions.
3512 if (range_size(Range&: Phis) != 1) {
3513 reportVectorizationFailure(
3514 DebugMsg: "Early exit loop with side effects contains "
3515 "unsupported reductions, inductions or recurrences",
3516 ORETag: "EarlyExitSideEffectsReductions", ORE, TheLoop);
3517 return false;
3518 }
3519 VPPhi *ContinueIV = cast<VPPhi>(Val: Phis.begin());
3520 // Make sure we're referring to the same IV.
3521 assert(
3522 match(ContinueIV->getOperand(0),
3523 m_VPInstruction<VPInstruction::ExitingIVValue>(m_Specific(IV))) &&
3524 "Continuing from different IV");
3525 ContinueIV->setOperand(I: 0, New: ExitIV);
3526 return true;
3527}
3528
3529// Returns true if any non-branch recipe in the loop may have side effects.
3530// For now this relies on LoopVectorizationLegality::isVectorizableEarlyExitLoop
3531// rejecting loops with side effects other than stores.
3532static bool loopMayHaveSideEffects(VPBasicBlock *HeaderVPBB) {
3533 return any_of(Range: vp_rpo_plain_cfg_loop_body(Header: HeaderVPBB), P: [](VPBasicBlock *VPBB) {
3534 return any_of(Range&: *VPBB, P: [](VPRecipeBase &R) {
3535 return match(V: &R, P: m_VPInstruction<Instruction::Store>());
3536 });
3537 });
3538}
3539
3540bool VPlanTransforms::handleUncountableEarlyExits(
3541 VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop,
3542 PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC,
3543 UncountableExitStyle Style) {
3544#ifndef NDEBUG
3545 VPDominatorTree VPDT(Plan);
3546#endif
3547
3548 auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
3549 auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
3550
3551 if (loopMayHaveSideEffects(HeaderVPBB))
3552 Style = UncountableExitStyle::MaskedHandleExitInScalarLoop;
3553
3554 // Dereferenceability is checked separately for uncountable exit loops with
3555 // stores, as only the loads contributing to the exit condition need to
3556 // be checked.
3557 if (Style == UncountableExitStyle::ReadOnly &&
3558 !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC)) {
3559 reportVectorizationFailure(
3560 DebugMsg: "Auto-vectorization of early exit loops with potentially "
3561 "faulting loads is not supported",
3562 ORETag: "EarlyExitFaultingLoads", ORE, TheLoop);
3563 return false;
3564 }
3565
3566 VPBuilder LatchBuilder(LatchVPBB->getTerminator());
3567 SmallVector<EarlyExitInfo> Exits;
3568 for (auto [EarlyExitingVPBB, ExitBlock] :
3569 vputils::getEarlyExits(Plan, MiddleVPBB)) {
3570 // Collect condition for this early exit.
3571 VPBlockBase *TrueSucc = EarlyExitingVPBB->getSuccessors()[0];
3572 VPValue *CondOfEarlyExitingVPBB;
3573 [[maybe_unused]] bool Matched =
3574 match(V: EarlyExitingVPBB->getTerminator(),
3575 P: m_BranchOnCond(Op0: m_VPValue(V&: CondOfEarlyExitingVPBB)));
3576 assert(Matched && "Terminator must be BranchOnCond");
3577
3578 // Insert the MaskedCond in the EarlyExitingVPBB so the predicator adds
3579 // the correct block mask.
3580 VPBuilder EarlyExitingBuilder(EarlyExitingVPBB->getTerminator());
3581 auto *CondToEarlyExit = EarlyExitingBuilder.createNaryOp(
3582 Opcode: VPInstruction::MaskedCond,
3583 Operands: TrueSucc == ExitBlock
3584 ? CondOfEarlyExitingVPBB
3585 : EarlyExitingBuilder.createNot(Operand: CondOfEarlyExitingVPBB));
3586 assert((isa<VPIRValue>(CondOfEarlyExitingVPBB) ||
3587 !VPDT.properlyDominates(EarlyExitingVPBB, LatchVPBB) ||
3588 VPDT.properlyDominates(
3589 CondOfEarlyExitingVPBB->getDefiningRecipe()->getParent(),
3590 LatchVPBB)) &&
3591 "exit condition must dominate the latch");
3592 Exits.push_back(Elt: {
3593 .EarlyExitingVPBB: EarlyExitingVPBB,
3594 .EarlyExitVPBB: ExitBlock,
3595 .CondToExit: CondToEarlyExit,
3596 });
3597 }
3598
3599 assert(!Exits.empty() && "must have at least one early exit");
3600 // Sort exits by RPO order to get correct program order. RPO gives a
3601 // topological ordering of the CFG, ensuring upstream exits are checked
3602 // before downstream exits in the dispatch chain.
3603 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
3604 HeaderVPBB);
3605 DenseMap<VPBlockBase *, unsigned> RPOIdx;
3606 for (const auto &[Num, VPB] : enumerate(First&: RPOT))
3607 RPOIdx[VPB] = Num;
3608 llvm::sort(C&: Exits, Comp: [&RPOIdx](const EarlyExitInfo &A, const EarlyExitInfo &B) {
3609 return RPOIdx[A.EarlyExitingVPBB] < RPOIdx[B.EarlyExitingVPBB];
3610 });
3611#ifndef NDEBUG
3612 // After RPO sorting, verify that for any pair where one exit dominates
3613 // another, the dominating exit comes first. This is guaranteed by RPO
3614 // (topological order) and is required for the dispatch chain correctness.
3615 for (unsigned I = 0; I + 1 < Exits.size(); ++I)
3616 for (unsigned J = I + 1; J < Exits.size(); ++J)
3617 assert(!VPDT.properlyDominates(Exits[J].EarlyExitingVPBB,
3618 Exits[I].EarlyExitingVPBB) &&
3619 "RPO sort must place dominating exits before dominated ones");
3620#endif
3621
3622 // Build the AnyOf condition for the latch terminator using logical OR
3623 // to avoid poison propagation from later exit conditions when an earlier
3624 // exit is taken.
3625 VPValue *Combined = Exits[0].CondToExit;
3626 for (const EarlyExitInfo &Info : drop_begin(RangeOrContainer&: Exits))
3627 Combined = LatchBuilder.createLogicalOr(LHS: Combined, RHS: Info.CondToExit);
3628 Combined = LatchBuilder.createFreeze(Op: Combined);
3629
3630 // Even though the logical or prevents posion propagation, we need to freeze
3631 // Combined to prevent poisoning the entire AnyOf result:
3632 //
3633 // Exits[0].CondToExit = [0,1,0,0]
3634 // Exits[1].CondToExit = [0,0,p,p]
3635 // Combined = [0,1,p,p]
3636 // freeze(Combined) = [0,1,?,?]
3637 // AnyOf = 1
3638 VPValue *IsAnyExitTaken =
3639 LatchBuilder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: Combined);
3640
3641 // Create a comparison for the latch exit condition and replace the
3642 // BranchOnCond with a BranchOnTwoConds. The original BranchOnCond's condition
3643 // is used as the latch-exit condition; canonical IV recipes have not been
3644 // introduced yet, so there is no BranchOnCount to derive the condition from.
3645 auto *LatchExitingBranch = cast<VPInstruction>(Val: LatchVPBB->getTerminator());
3646 assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCond &&
3647 "Unexpected terminator");
3648 VPValue *IsLatchExitTaken = LatchExitingBranch->getOperand(N: 0);
3649 DebugLoc LatchDL = LatchExitingBranch->getDebugLoc();
3650 LatchExitingBranch->eraseFromParent();
3651 LatchBuilder.setInsertPoint(LatchVPBB);
3652 LatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnTwoConds,
3653 Operands: {IsAnyExitTaken, IsLatchExitTaken}, DL: LatchDL);
3654 LatchVPBB->clearSuccessors();
3655
3656 if (Style == UncountableExitStyle::MaskedHandleExitInScalarLoop) {
3657 // If handling the exiting lane in the scalar loop, combine the exit
3658 // conditions into a single BranchOnCond.
3659 LatchVPBB->setSuccessors({MiddleVPBB, MiddleVPBB, HeaderVPBB});
3660 MiddleVPBB->clearPredecessors();
3661 MiddleVPBB->setPredecessors({LatchVPBB, LatchVPBB});
3662 return handleUncountableExitsWithSideEffects(Plan, Exits, HeaderVPBB,
3663 LatchVPBB, MiddleVPBB, ORE,
3664 TheLoop, PSE, DT, AC);
3665 }
3666
3667 // Create the vector.early.exit blocks.
3668 SmallVector<VPBasicBlock *> VectorEarlyExitVPBBs(Exits.size());
3669 for (unsigned Idx = 0; Idx != Exits.size(); ++Idx) {
3670 Twine BlockSuffix = Exits.size() == 1 ? "" : Twine(".") + Twine(Idx);
3671 VPBasicBlock *VectorEarlyExitVPBB =
3672 Plan.createVPBasicBlock(Name: "vector.early.exit" + BlockSuffix);
3673 VectorEarlyExitVPBBs[Idx] = VectorEarlyExitVPBB;
3674 }
3675
3676 // Create the dispatch block (or reuse the single exit block if only one
3677 // exit). The dispatch block computes the first active lane of the combined
3678 // condition and, for multiple exits, chains through conditions to determine
3679 // which exit to take.
3680 VPBasicBlock *DispatchVPBB =
3681 Exits.size() == 1 ? VectorEarlyExitVPBBs[0]
3682 : Plan.createVPBasicBlock(Name: "vector.early.exit.check");
3683 DispatchVPBB->setPredecessors({LatchVPBB});
3684 LatchVPBB->setSuccessors({DispatchVPBB, MiddleVPBB, HeaderVPBB});
3685 VPBuilder DispatchBuilder(DispatchVPBB, DispatchVPBB->begin());
3686 VPValue *FirstActiveLane = DispatchBuilder.createFirstActiveLane(
3687 Masks: {Combined}, DL: DebugLoc::getUnknown(), Name: "first.active.lane");
3688
3689 // For each early exit, disconnect the original exiting block
3690 // (early.exiting.I) from the exit block (ir-bb<exit.I>) and route through a
3691 // new vector.early.exit block. Update ir-bb<exit.I>'s phis to extract their
3692 // values at the first active lane:
3693 //
3694 // Input:
3695 // early.exiting.I:
3696 // ...
3697 // EMIT branch-on-cond vp<%cond.I>
3698 // Successor(s): in.loop.succ, ir-bb<exit.I>
3699 //
3700 // ir-bb<exit.I>:
3701 // IR %phi = phi [ vp<%incoming.I>, early.exiting.I ], ...
3702 //
3703 // Output:
3704 // early.exiting.I:
3705 // ...
3706 // Successor(s): in.loop.succ
3707 //
3708 // vector.early.exit.I:
3709 // EMIT vp<%exit.val> = extract-lane vp<%first.lane>, vp<%incoming.I>
3710 // Successor(s): ir-bb<exit.I>
3711 //
3712 // ir-bb<exit.I>:
3713 // IR %phi = phi ... (extra operand: vp<%exit.val> from
3714 // vector.early.exit.I)
3715 //
3716 for (auto [Exit, VectorEarlyExitVPBB] :
3717 zip_equal(t&: Exits, u&: VectorEarlyExitVPBBs)) {
3718 auto &[EarlyExitingVPBB, EarlyExitVPBB, _] = Exit;
3719 // Adjust the phi nodes in EarlyExitVPBB.
3720 // 1. remove incoming values from EarlyExitingVPBB,
3721 // 2. extract the incoming value at FirstActiveLane
3722 // 3. add back the extracts as last operands for the phis
3723 // Then adjust the CFG, removing the edge between EarlyExitingVPBB and
3724 // EarlyExitVPBB and adding a new edge between VectorEarlyExitVPBB and
3725 // EarlyExitVPBB. The extracts at FirstActiveLane are now the incoming
3726 // values from VectorEarlyExitVPBB.
3727 for (VPRecipeBase &R : EarlyExitVPBB->phis()) {
3728 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
3729 VPValue *IncomingVal =
3730 ExitIRI->getIncomingValueForBlock(VPBB: EarlyExitingVPBB);
3731 VPValue *NewIncoming = IncomingVal;
3732 if (!isa<VPIRValue>(Val: IncomingVal)) {
3733 VPBuilder EarlyExitBuilder(VectorEarlyExitVPBB);
3734 NewIncoming = EarlyExitBuilder.createNaryOp(
3735 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, IncomingVal},
3736 DL: DebugLoc::getUnknown(), Name: "early.exit.value");
3737 }
3738 ExitIRI->removeIncomingValueFor(IncomingBlock: EarlyExitingVPBB);
3739 ExitIRI->addIncoming(IncomingV: NewIncoming);
3740 }
3741
3742 EarlyExitingVPBB->getTerminator()->eraseFromParent();
3743 VPBlockUtils::disconnectBlocks(From: EarlyExitingVPBB, To: EarlyExitVPBB);
3744 VPBlockUtils::connectBlocks(From: VectorEarlyExitVPBB, To: EarlyExitVPBB);
3745 }
3746
3747 // Chain through exits: for each exit, check if its condition is true at
3748 // the first active lane. If so, take that exit; otherwise, try the next.
3749 // The last exit needs no check since it must be taken if all others fail.
3750 //
3751 // For 3 exits (cond.0, cond.1, cond.2), this creates:
3752 //
3753 // latch:
3754 // ...
3755 // EMIT vp<%combined> = logical-or vp<%cond.0>, vp<%cond.1>, vp<%cond.2>
3756 // EMIT vp<%combined.freeze> = freeze vp<%combined>
3757 // ...
3758 //
3759 // vector.early.exit.check:
3760 // EMIT vp<%first.lane> = first-active-lane vp<%combined.freeze>
3761 // EMIT vp<%at.cond.0> = extract-lane vp<%first.lane>, vp<%cond.0>
3762 // EMIT branch-on-cond vp<%at.cond.0>
3763 // Successor(s): vector.early.exit.0, vector.early.exit.check.0
3764 //
3765 // vector.early.exit.check.0:
3766 // EMIT vp<%at.cond.1> = extract-lane vp<%first.lane>, vp<%cond.1>
3767 // EMIT branch-on-cond vp<%at.cond.1>
3768 // Successor(s): vector.early.exit.1, vector.early.exit.2
3769 VPBasicBlock *CurrentBB = DispatchVPBB;
3770 for (auto [I, Exit] : enumerate(First: ArrayRef(Exits).drop_back())) {
3771 VPValue *LaneVal = DispatchBuilder.createNaryOp(
3772 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, Exit.CondToExit},
3773 DL: DebugLoc::getUnknown(), Name: "exit.cond.at.lane");
3774
3775 // For the last dispatch, branch directly to the last exit on false;
3776 // otherwise, create a new check block.
3777 bool IsLastDispatch = (I + 2 == Exits.size());
3778 VPBasicBlock *FalseBB =
3779 IsLastDispatch ? VectorEarlyExitVPBBs.back()
3780 : Plan.createVPBasicBlock(
3781 Name: Twine("vector.early.exit.check.") + Twine(I));
3782
3783 DispatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {LaneVal});
3784 CurrentBB->setSuccessors({VectorEarlyExitVPBBs[I], FalseBB});
3785 VectorEarlyExitVPBBs[I]->setPredecessors({CurrentBB});
3786 FalseBB->setPredecessors({CurrentBB});
3787
3788 CurrentBB = FalseBB;
3789 DispatchBuilder.setInsertPoint(CurrentBB);
3790 }
3791
3792 return true;
3793}
3794
3795/// This function tries convert extended in-loop reductions to
3796/// VPExpressionRecipe and clamp the \p Range if it is beneficial and
3797/// valid. The created recipe must be decomposed to its constituent
3798/// recipes before execution.
3799static VPExpressionRecipe *
3800tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
3801 VFRange &Range) {
3802 Type *RedTy = Red->getScalarType();
3803 VPValue *VecOp = Red->getVecOp();
3804
3805 // We don't handle partial reductions here.
3806 if (Red->isPartialReduction())
3807 return nullptr;
3808
3809 // Clamp the range if using extended-reduction is profitable.
3810 auto IsExtendedRedValidAndClampRange =
3811 [&](unsigned Opcode, Instruction::CastOps ExtOpc, Type *SrcTy) -> bool {
3812 return LoopVectorizationPlanner::getDecisionAndClampRange(
3813 Predicate: [&](ElementCount VF) {
3814 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
3815 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
3816
3817 InstructionCost ExtRedCost = InstructionCost::getInvalid();
3818 InstructionCost ExtCost =
3819 cast<VPWidenCastRecipe>(Val: VecOp)->computeCost(VF, Ctx);
3820 InstructionCost RedCost = Red->computeCost(VF, Ctx);
3821
3822 assert(!RedTy->isFloatingPointTy() &&
3823 "getExtendedReductionCost only supports integer types");
3824 ExtRedCost = Ctx.TTI.getExtendedReductionCost(
3825 Opcode, IsUnsigned: ExtOpc == Instruction::CastOps::ZExt, ResTy: RedTy, Ty: SrcVecTy,
3826 FMF: Red->getFastMathFlagsOrNone(), CostKind);
3827 return ExtRedCost.isValid() && ExtRedCost < ExtCost + RedCost;
3828 },
3829 Range);
3830 };
3831
3832 VPValue *A;
3833 // Match reduce(ext)).
3834 if (match(V: VecOp, P: m_Isa<VPWidenCastRecipe>(P: m_ZExtOrSExt(Op0: m_VPValue(V&: A)))) &&
3835 IsExtendedRedValidAndClampRange(
3836 RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind()),
3837 cast<VPWidenCastRecipe>(Val: VecOp)->getOpcode(), A->getScalarType()))
3838 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
3839
3840 return nullptr;
3841}
3842
3843using ExtendKind = TTI::PartialReductionExtendKind;
3844
3845static ExtendKind getPartialReductionExtendKind(VPWidenCastRecipe *Cast) {
3846 return TTI::getPartialReductionExtendKind(CastOpc: Cast->getOpcode());
3847}
3848
3849/// Returns true if the narrow \p Mul in ext(mul(ext(A), ext(B))) or
3850/// ext(mul(ext(A), C)) cannot wrap. The kind of the inner extends selects
3851/// whether nuw (zext) or nsw (sext) is required on \p Mul.
3852static bool canFoldOuterExtendIntoMul(VPWidenRecipe *Mul) {
3853 auto *LHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
3854 auto *RHS = dyn_cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
3855 Type *LHSSrcTy = LHS->getOperand(N: 0)->getScalarType();
3856 // A constant RHS must fit in the source type of the LHS.
3857 Type *RHSSrcTy = RHS ? RHS->getOperand(N: 0)->getScalarType() : LHSSrcTy;
3858 ExtendKind ExtKind = getPartialReductionExtendKind(Cast: LHS);
3859 if (Mul->getScalarType()->getScalarSizeInBits() >=
3860 LHSSrcTy->getScalarSizeInBits() + RHSSrcTy->getScalarSizeInBits())
3861 return true;
3862 if (ExtKind == ExtendKind::PR_ZeroExtend)
3863 return Mul->hasNoUnsignedWrap();
3864 return ExtKind == ExtendKind::PR_SignExtend && Mul->hasNoSignedWrap();
3865}
3866
3867/// This function tries convert extended in-loop reductions to
3868/// VPExpressionRecipe and clamp the \p Range if it is beneficial
3869/// and valid. The created VPExpressionRecipe must be decomposed to its
3870/// constituent recipes before execution. Patterns of the
3871/// VPExpressionRecipe:
3872/// reduce.add(mul(...)),
3873/// reduce.add(mul(ext(A), ext(B))),
3874/// reduce.add(ext(mul(ext(A), ext(B)))).
3875/// reduce.fadd(fmul(ext(A), ext(B)))
3876static VPExpressionRecipe *
3877tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
3878 VPCostContext &Ctx, VFRange &Range) {
3879 unsigned Opcode = RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind());
3880 if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
3881 Opcode != Instruction::FAdd)
3882 return nullptr;
3883
3884 // We don't handle partial reductions here.
3885 if (Red->isPartialReduction())
3886 return nullptr;
3887
3888 Type *RedTy = Red->getScalarType();
3889
3890 // Clamp the range if using multiply-accumulate-reduction is profitable.
3891 auto IsMulAccValidAndClampRange =
3892 [&](VPWidenRecipe *Mul, VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
3893 VPWidenCastRecipe *OuterExt) -> bool {
3894 return LoopVectorizationPlanner::getDecisionAndClampRange(
3895 Predicate: [&](ElementCount VF) {
3896 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
3897 Type *SrcTy = Ext0 ? Ext0->getOperand(N: 0)->getScalarType() : RedTy;
3898 InstructionCost MulAccCost;
3899
3900 // getMulAccReductionCost for in-loop reductions does not support
3901 // mixed or floating-point extends.
3902 if (Ext0 && Ext1 &&
3903 (Ext0->getOpcode() != Ext1->getOpcode() ||
3904 Ext0->getOpcode() == Instruction::CastOps::FPExt))
3905 return false;
3906
3907 bool IsZExt =
3908 !Ext0 || Ext0->getOpcode() == Instruction::CastOps::ZExt;
3909 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
3910 MulAccCost = Ctx.TTI.getMulAccReductionCost(IsUnsigned: IsZExt, RedOpcode: Opcode, ResTy: RedTy,
3911 Ty: SrcVecTy, CostKind);
3912
3913 InstructionCost MulCost = Mul->computeCost(VF, Ctx);
3914 InstructionCost RedCost = Red->computeCost(VF, Ctx);
3915 InstructionCost ExtCost = 0;
3916 if (Ext0)
3917 ExtCost += Ext0->computeCost(VF, Ctx);
3918 if (Ext1)
3919 ExtCost += Ext1->computeCost(VF, Ctx);
3920 if (OuterExt)
3921 ExtCost += OuterExt->computeCost(VF, Ctx);
3922
3923 return MulAccCost.isValid() &&
3924 MulAccCost < ExtCost + MulCost + RedCost;
3925 },
3926 Range);
3927 };
3928
3929 VPValue *VecOp = Red->getVecOp();
3930 VPRecipeBase *Sub = nullptr;
3931 VPValue *A, *B;
3932 VPValue *Tmp = nullptr;
3933
3934 if (RedTy->isFloatingPointTy())
3935 return nullptr;
3936
3937 // Sub reductions could have a sub between the add reduction and vec op.
3938 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Tmp)))) {
3939 Sub = VecOp->getDefiningRecipe();
3940 VecOp = Tmp;
3941 }
3942
3943 // If ValB is a constant and can be safely extended, truncate it to the same
3944 // type as ExtA's operand, then extend it to the same type as ExtA. This
3945 // creates two uniform extends that can more easily be matched by the rest of
3946 // the bundling code. The ExtB reference, ValB and operand 1 of Mul are all
3947 // replaced with the new extend of the constant.
3948 auto ExtendAndReplaceConstantOp = [](VPWidenCastRecipe *ExtA,
3949 VPWidenCastRecipe *&ExtB, VPValue *&ValB,
3950 VPWidenRecipe *Mul) {
3951 if (!ExtA || ExtB || !isa<VPIRValue>(Val: ValB))
3952 return;
3953 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
3954 Instruction::CastOps ExtOpc = ExtA->getOpcode();
3955 const APInt *Const;
3956 if (!match(V: ValB, P: m_APInt(C&: Const)) ||
3957 !llvm::canConstantBeExtended(
3958 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
3959 return;
3960 // The truncate ensures that the type of each extended operand is the
3961 // same, and it's been proven that the constant can be extended from
3962 // NarrowTy safely. Necessary since ExtA's extended operand would be
3963 // e.g. an i8, while the const will likely be an i32. This will be
3964 // elided by later optimisations.
3965 VPBuilder Builder(Mul);
3966 auto *Trunc =
3967 Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc, Op: ValB, ResultTy: NarrowTy);
3968 Type *WideTy = ExtA->getScalarType();
3969 ValB = ExtB = Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy);
3970 Mul->setOperand(I: 1, New: ExtB);
3971 };
3972
3973 // Try to match reduce.add(mul(...)).
3974 if (match(V: VecOp, P: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B)))) {
3975 auto *RecipeA = dyn_cast<VPWidenCastRecipe>(Val: A);
3976 auto *RecipeB = dyn_cast<VPWidenCastRecipe>(Val: B);
3977 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
3978
3979 // Convert reduce.add(mul(ext, const)) to reduce.add(mul(ext, ext(const)))
3980 ExtendAndReplaceConstantOp(RecipeA, RecipeB, B, Mul);
3981
3982 // Match reduce.add/sub(mul(ext, ext)).
3983 if (RecipeA && RecipeB && match(V: RecipeA, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
3984 match(V: RecipeB, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
3985 IsMulAccValidAndClampRange(Mul, RecipeA, RecipeB, nullptr)) {
3986 if (Sub)
3987 return new VPExpressionRecipe(RecipeA, RecipeB, Mul,
3988 cast<VPWidenRecipe>(Val: Sub), Red);
3989 return new VPExpressionRecipe(RecipeA, RecipeB, Mul, Red);
3990 }
3991 // TODO: Add an expression type for this variant with a negated mul
3992 if (!Sub && IsMulAccValidAndClampRange(Mul, nullptr, nullptr, nullptr))
3993 return new VPExpressionRecipe(Mul, Red);
3994 }
3995 // TODO: Add an expression type for negated versions of other expression
3996 // variants.
3997 if (Sub)
3998 return nullptr;
3999
4000 // Match reduce.add(ext(mul(A, B))).
4001 if (match(V: VecOp, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))))) {
4002 auto *Ext = cast<VPWidenCastRecipe>(Val: VecOp);
4003 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
4004 auto *Ext0 = dyn_cast<VPWidenCastRecipe>(Val: A);
4005 auto *Ext1 = dyn_cast<VPWidenCastRecipe>(Val: B);
4006
4007 // reduce.add(ext(mul(ext, const)))
4008 // -> reduce.add(ext(mul(ext, ext(const))))
4009 ExtendAndReplaceConstantOp(Ext0, Ext1, B, Mul);
4010
4011 // reduce.add(ext(mul(ext(A), ext(B))))
4012 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
4013 // The inner extends must either have the same opcode as the outer extend or
4014 // be the same sext, in which case the multiply cannot result in a negative
4015 // value without wrapping. If the narrow mul cannot wrap, the outer extend
4016 // can be folded away by doing wider extends for the operands of the mul.
4017 if (Ext0 && Ext1 &&
4018 (Ext->getOpcode() == Ext0->getOpcode() ||
4019 (Ext0 == Ext1 && Ext0->getOpcode() == Instruction::SExt)) &&
4020 Ext0->getOpcode() == Ext1->getOpcode() && Mul->hasOneUse() &&
4021 canFoldOuterExtendIntoMul(Mul) &&
4022 IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext)) {
4023 auto *NewExt0 = new VPWidenCastRecipe(
4024 Ext0->getOpcode(), Ext0->getOperand(N: 0), Ext->getScalarType(), nullptr,
4025 *Ext0, *Ext0, Ext0->getDebugLoc());
4026 NewExt0->insertBefore(InsertPos: Ext0);
4027
4028 VPWidenCastRecipe *NewExt1 = NewExt0;
4029 if (Ext0 != Ext1) {
4030 NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(N: 0),
4031 Ext->getScalarType(), nullptr, *Ext1,
4032 *Ext1, Ext1->getDebugLoc());
4033 NewExt1->insertBefore(InsertPos: Ext1);
4034 }
4035 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewExt0, NewExt1});
4036 NewMul->insertBefore(InsertPos: Mul);
4037 Ext->replaceAllUsesWith(New: NewMul);
4038 Ext->eraseFromParent();
4039 Mul->eraseFromParent();
4040 return new VPExpressionRecipe(NewExt0, NewExt1, NewMul, Red);
4041 }
4042 }
4043 return nullptr;
4044}
4045
4046/// This function tries to create abstract recipes from the reduction recipe for
4047/// following optimizations and cost estimation.
4048static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
4049 VPCostContext &Ctx,
4050 VFRange &Range) {
4051 // Creation of VPExpressions for partial reductions is entirely handled in
4052 // transformToPartialReduction.
4053 if (Red->isPartialReduction())
4054 return;
4055
4056 VPExpressionRecipe *AbstractR = nullptr;
4057 auto IP = std::next(x: Red->getIterator());
4058 auto *VPBB = Red->getParent();
4059 if (auto *MulAcc = tryToMatchAndCreateMulAccumulateReduction(Red, Ctx, Range))
4060 AbstractR = MulAcc;
4061 else if (auto *ExtRed = tryToMatchAndCreateExtendedReduction(Red, Ctx, Range))
4062 AbstractR = ExtRed;
4063 // Cannot create abstract inloop reduction recipes.
4064 if (!AbstractR)
4065 return;
4066
4067 AbstractR->insertBefore(BB&: *VPBB, IP);
4068 Red->replaceAllUsesWith(New: AbstractR);
4069}
4070
4071void VPlanTransforms::convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
4072 VFRange &Range) {
4073 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
4074 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
4075 for (VPReductionRecipe &Red :
4076 make_early_inc_range(Range: make_isa_range<VPReductionRecipe>(Range&: *VPBB)))
4077 tryToCreateAbstractReductionRecipe(Red: &Red, Ctx, Range);
4078 }
4079}
4080
4081// Collect common metadata from a group of replicate recipes by intersecting
4082// metadata from all recipes in the group.
4083static VPIRMetadata getCommonMetadata(ArrayRef<VPReplicateRecipe *> Recipes) {
4084 VPIRMetadata CommonMetadata = *Recipes.front();
4085 for (VPReplicateRecipe *Recipe : drop_begin(RangeOrContainer&: Recipes))
4086 CommonMetadata.intersect(MD: *Recipe);
4087 // The recipe using the common metadata is not predicated, so it does not
4088 // share the group's execution frequency.
4089 CommonMetadata.clearExecutionFrequency();
4090 return CommonMetadata;
4091}
4092
4093template <unsigned Opcode>
4094static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
4095collectComplementaryPredicatedMemOps(VPlan &Plan,
4096 PredicatedScalarEvolution &PSE,
4097 const Loop *L) {
4098 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
4099 "Only Load and Store opcodes supported");
4100
4101 // For each address, collect operations with the same or complementary masks.
4102 SmallVector<SmallVector<VPReplicateRecipe *, 4>> AllGroups;
4103 auto Groups = collectGroupedReplicateMemOps<Opcode>(
4104 Plan, PSE, L,
4105 [](VPReplicateRecipe *RepR) { return RepR->isPredicated(); });
4106 for (auto Recipes : Groups) {
4107 if (Recipes.size() < 2)
4108 continue;
4109
4110 assert(all_equal(map_range(Recipes, getLoadStoreValueType)) &&
4111 "Expected all recipes in group to have the same load-store type");
4112
4113 // Collect groups with the same or complementary masks.
4114 for (VPReplicateRecipe *&RecipeI : Recipes) {
4115 if (!RecipeI)
4116 continue;
4117
4118 VPValue *MaskI = RecipeI->getMask();
4119 SmallVector<VPReplicateRecipe *, 4> Group;
4120 Group.push_back(Elt: RecipeI);
4121 RecipeI = nullptr;
4122
4123 // Find all operations with the same or complementary masks.
4124 bool HasComplementaryMask = false;
4125 for (VPReplicateRecipe *&RecipeJ : Recipes) {
4126 if (!RecipeJ)
4127 continue;
4128
4129 VPValue *MaskJ = RecipeJ->getMask();
4130 // Check if any operation in the group has a complementary mask with
4131 // another, that is M1 == NOT(M2) or M2 == NOT(M1).
4132 HasComplementaryMask |= match(V: MaskI, P: m_Not(Op0: m_Specific(VPV: MaskJ))) ||
4133 match(V: MaskJ, P: m_Not(Op0: m_Specific(VPV: MaskI)));
4134 Group.push_back(Elt: RecipeJ);
4135 RecipeJ = nullptr;
4136 }
4137
4138 if (HasComplementaryMask) {
4139 assert(Group.size() >= 2 && "must have at least 2 entries");
4140 AllGroups.push_back(Elt: std::move(Group));
4141 }
4142 }
4143 }
4144
4145 return AllGroups;
4146}
4147
4148// Find the recipe with minimum alignment in the group.
4149template <typename InstType>
4150static VPReplicateRecipe *
4151findRecipeWithMinAlign(ArrayRef<VPReplicateRecipe *> Group) {
4152 return *min_element(Group, [](VPReplicateRecipe *A, VPReplicateRecipe *B) {
4153 return cast<InstType>(A->getUnderlyingInstr())->getAlign() <
4154 cast<InstType>(B->getUnderlyingInstr())->getAlign();
4155 });
4156}
4157
4158void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
4159 PredicatedScalarEvolution &PSE,
4160 const Loop *L) {
4161 auto Groups =
4162 collectComplementaryPredicatedMemOps<Instruction::Load>(Plan, PSE, L);
4163 if (Groups.empty())
4164 return;
4165
4166 // Process each group of loads.
4167 for (auto &Group : Groups) {
4168 // Try to use the earliest (most dominating) load to replace all others.
4169 VPReplicateRecipe *EarliestLoad = Group[0];
4170 VPBasicBlock *FirstBB = EarliestLoad->getParent();
4171 VPBasicBlock *LastBB = Group.back()->getParent();
4172
4173 // Check that the load doesn't alias with stores between first and last.
4174 auto LoadLoc = vputils::getMemoryLocation(R: *EarliestLoad);
4175 if (!LoadLoc || !canHoistOrSinkWithNoAliasCheck(MemLoc: *LoadLoc, FirstBB, LastBB))
4176 continue;
4177
4178 // Collect common metadata from all loads in the group.
4179 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
4180
4181 // Find the load with minimum alignment to use.
4182 auto *LoadWithMinAlign = findRecipeWithMinAlign<LoadInst>(Group);
4183
4184 bool IsSingleScalar = EarliestLoad->isSingleScalar();
4185 assert(all_of(Group,
4186 [IsSingleScalar](VPReplicateRecipe *R) {
4187 return R->isSingleScalar() == IsSingleScalar;
4188 }) &&
4189 "all members in group must agree on IsSingleScalar");
4190
4191 // Create an unpredicated version of the earliest load with common
4192 // metadata.
4193 auto *UnpredicatedLoad = new VPReplicateRecipe(
4194 LoadWithMinAlign->getUnderlyingInstr(), {EarliestLoad->getOperand(N: 0)},
4195 IsSingleScalar, /*Mask=*/nullptr, *EarliestLoad, CommonMetadata);
4196
4197 UnpredicatedLoad->insertBefore(InsertPos: EarliestLoad);
4198
4199 // Replace all loads in the group with the unpredicated load.
4200 for (VPReplicateRecipe *Load : Group) {
4201 Load->replaceAllUsesWith(New: UnpredicatedLoad);
4202 Load->eraseFromParent();
4203 }
4204 }
4205}
4206
4207static bool
4208canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
4209 PredicatedScalarEvolution &PSE, const Loop &L) {
4210 auto StoreLoc = vputils::getMemoryLocation(R: *StoresToSink.front());
4211 if (!StoreLoc || !StoreLoc->AATags.Scope)
4212 return false;
4213
4214 // When sinking a group of stores, all members of the group alias each other.
4215 // Skip them during the alias checks.
4216 VPBasicBlock *FirstBB = StoresToSink.front()->getParent();
4217 VPBasicBlock *LastBB = StoresToSink.back()->getParent();
4218 SinkStoreInfo SinkInfo(StoresToSink, *StoresToSink[0], PSE, L);
4219 return canHoistOrSinkWithNoAliasCheck(MemLoc: *StoreLoc, FirstBB, LastBB, SinkInfo);
4220}
4221
4222void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
4223 PredicatedScalarEvolution &PSE,
4224 const Loop *L) {
4225 auto Groups =
4226 collectComplementaryPredicatedMemOps<Instruction::Store>(Plan, PSE, L);
4227 if (Groups.empty())
4228 return;
4229
4230 for (auto &Group : Groups) {
4231 if (!canSinkStoreWithNoAliasCheck(StoresToSink: Group, PSE, L: *L))
4232 continue;
4233
4234 // Use the last (most dominated) store's location for the unconditional
4235 // store.
4236 VPReplicateRecipe *LastStore = Group.back();
4237 VPBasicBlock *InsertBB = LastStore->getParent();
4238
4239 // Collect common alias metadata from all stores in the group.
4240 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
4241
4242 // Build select chain for stored values.
4243 VPValue *SelectedValue = Group[0]->getOperand(N: 0);
4244 VPBuilder Builder(InsertBB, LastStore->getIterator());
4245
4246 bool IsSingleScalar = Group[0]->isSingleScalar();
4247 for (unsigned I = 1; I < Group.size(); ++I) {
4248 assert(IsSingleScalar == Group[I]->isSingleScalar() &&
4249 "all members in group must agree on IsSingleScalar");
4250 VPValue *Mask = Group[I]->getMask();
4251 VPValue *Value = Group[I]->getOperand(N: 0);
4252 SelectedValue = Builder.createSelect(
4253 Cond: Mask, TrueVal: Value, FalseVal: SelectedValue, DL: Group[I]->getDebugLoc(), Name: "",
4254 Flags: VPIRFlags::getDefaultFlags(Opcode: Instruction::Select,
4255 ResultTy: Value->getScalarType()));
4256 }
4257
4258 // Find the store with minimum alignment to use.
4259 auto *StoreWithMinAlign = findRecipeWithMinAlign<StoreInst>(Group);
4260
4261 // Create unconditional store with selected value and common metadata.
4262 auto *UnpredicatedStore = new VPReplicateRecipe(
4263 StoreWithMinAlign->getUnderlyingInstr(),
4264 {SelectedValue, LastStore->getOperand(N: 1)}, IsSingleScalar,
4265 /*Mask=*/nullptr, *LastStore, CommonMetadata);
4266 UnpredicatedStore->insertBefore(BB&: *InsertBB, IP: LastStore->getIterator());
4267
4268 // Remove all predicated stores from the group.
4269 for (VPReplicateRecipe *Store : Group)
4270 Store->eraseFromParent();
4271 }
4272}
4273
4274void VPlanTransforms::widenMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
4275 unsigned UF,
4276 const TargetTransformInfo &TTI) {
4277 assert(UF > 1 && "Expected plan to have an UF > 1");
4278
4279 auto m_ContiguousVecPtr = m_VecPtr(Op0: m_VPValue(), Op1: m_One());
4280 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
4281 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
4282 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
4283 VPValue *StoredValue = nullptr;
4284 if (!match(V: &R, P: m_WidenLoad(Op0: m_ContiguousVecPtr)) &&
4285 !match(V: &R, P: m_WidenStore(Op0: m_ContiguousVecPtr, Op1: m_VPValue(V&: StoredValue))))
4286 continue;
4287
4288 auto *MemOp = cast<VPWidenMemoryRecipe>(Val: &R);
4289 assert(MemOp->isConsecutive() && "Expected consecutive load/store");
4290
4291 // TODO: Support masked loads/stores. This requires widening the header
4292 // mask to the same factor as the memory operation.
4293 assert(!MemOp->isMasked() && "Masked accesses are not supported yet");
4294
4295 Type *AccessType = StoredValue ? StoredValue->getScalarType()
4296 : R.getVPSingleValue()->getScalarType();
4297 bool IsStore = isa<VPWidenStoreRecipe>(Val: MemOp->getAsRecipe());
4298 std::optional<Instruction::CastOps> CastHint;
4299 VPUser *MaybeCast = IsStore ? StoredValue->getDefiningRecipe()
4300 : R.getVPSingleValue()->getSingleUser();
4301 if (auto *Cast = dyn_cast_if_present<VPWidenCastRecipe>(Val: MaybeCast))
4302 CastHint = Cast->getOpcode();
4303
4304 VectorType *VectorAccessType = VectorType::get(ElementType: AccessType, EC: VF);
4305 if (!TTI.hasMultiVectorLoadStore(
4306 /*NumVectors=*/UF, Mask: TargetTransformInfo::MaskSource::None,
4307 VectorTy: VectorAccessType, IsStore, CastHint))
4308 continue;
4309
4310 DebugLoc DL = R.getDebugLoc();
4311 VPValue *Ptr = MemOp->getAddr();
4312 VPValue *Align = Plan.getConstantInt(BitWidth: 64, Val: MemOp->getAlign().value());
4313 VPValue *Multiplier = Plan.getConstantInt(BitWidth: 64, Val: 1);
4314
4315 VPBuilder Builder(VPBB, R.getIterator());
4316 if (IsStore) {
4317 VPValue *WideStoredValue = Builder.createNaryOp(
4318 Opcode: VPInstruction::ConcatVectors, Operands: {StoredValue}, DL);
4319 Builder.createNaryOp(Opcode: VPInstruction::WideVectorStore,
4320 Operands: {Multiplier, Ptr, Align, WideStoredValue}, Inst: nullptr,
4321 Flags: {}, MD: *MemOp, DL: R.getDebugLoc());
4322 } else {
4323 VPValue *OldLoad = R.getVPSingleValue();
4324 VPValue *Load = Builder.createNaryOp(
4325 Opcode: VPInstruction::WideVectorLoad, Operands: {Multiplier, Ptr, Align}, Inst: nullptr,
4326 Flags: {}, MD: *MemOp, DL: R.getDebugLoc(), Name: "", ResultTy: OldLoad->getScalarType());
4327 VPValue *Extract =
4328 Builder.createNaryOp(Opcode: VPInstruction::ExtractVectorForPart,
4329 Operands: {Load, Plan.getConstantInt(BitWidth: 64, Val: 0)}, DL);
4330 OldLoad->replaceAllUsesWith(New: Extract);
4331 }
4332
4333 R.eraseFromParent();
4334 }
4335 }
4336}
4337
4338/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
4339/// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
4340/// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
4341/// the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to
4342/// an index-independent load if it feeds all wide ops at all indices (\p OpV
4343/// must be the operand at index \p OpIdx for both the recipe at lane 0, \p
4344/// WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if \p V
4345/// is defined at \p Idx of a load interleave group.
4346/// A live-in or recipe defined outside the loop region can be converted, if it
4347/// is the same across all lanes, or we can create a BuildVector for it.
4348static bool canNarrowLoad(VPSingleDefRecipe *WideMember0, unsigned OpIdx,
4349 VPValue *OpV, unsigned Idx, bool IsScalable) {
4350 VPValue *Member0Op = WideMember0->getOperand(N: OpIdx);
4351 if (Member0Op->isDefinedOutsideLoopRegions()) {
4352 // Operand matches Member0, broadcast across all fields for both live-ins
4353 // and recipes.
4354 if (Member0Op == OpV)
4355 return true;
4356 // Otherwise distinct per-field VPValues are assembled into a BuildVector.
4357 return !IsScalable && OpV->isDefinedOutsideLoopRegions() &&
4358 OpV->getScalarType() == Member0Op->getScalarType();
4359 }
4360 VPRecipeBase *Member0OpR = Member0Op->getDefiningRecipe();
4361 if (auto *W = dyn_cast<VPWidenLoadRecipe>(Val: Member0OpR))
4362 // For scalable VFs, the narrowed plan processes vscale iterations at once,
4363 // so a shared wide load cannot be narrowed to a uniform scalar; bail out.
4364 return !IsScalable && !W->getMask() && W->isConsecutive() &&
4365 Member0Op == OpV;
4366 if (auto *IR = dyn_cast<VPInterleaveRecipe>(Val: Member0OpR))
4367 return IR->getInterleaveGroup()->isFull() && IR->getVPValue(I: Idx) == OpV;
4368 return false;
4369}
4370
4371/// Returns true if the wide recipes in \p Ops and their operand trees can be
4372/// narrowed. \p FirstMembersOf maps each in-loop value to the first member
4373/// list it was seen in, across all store groups.
4374static bool
4375canNarrowOps(ArrayRef<VPValue *> Ops, bool IsScalable,
4376 DenseMap<VPValue *, SmallVector<VPValue *>> &FirstMembersOf) {
4377 SmallVector<VPValue *> Ops0;
4378 auto *WideMember0 = dyn_cast<VPRecipeWithIRFlags>(Val: Ops[0]);
4379 if (!WideMember0)
4380 return false;
4381 for (VPValue *V : Ops) {
4382 if (!isa<VPWidenRecipe, VPWidenCastRecipe>(Val: V))
4383 return false;
4384 auto *R = cast<VPRecipeWithIRFlags>(Val: V);
4385 if (vputils::getOpcode(V: R) != vputils::getOpcode(V: WideMember0))
4386 return false;
4387 if (R->getScalarType() != WideMember0->getScalarType())
4388 return false;
4389 if (R->hasPredicate() && R->getPredicate() != WideMember0->getPredicate())
4390 return false;
4391 }
4392
4393 // The first member is narrowed in place using the other members' operands.
4394 // Bail out if a value is the first member of one list and also part of a
4395 // different list.
4396 for (VPValue *V : Ops) {
4397 if (V->isDefinedOutsideLoopRegions())
4398 continue;
4399 auto [It, Inserted] = FirstMembersOf.try_emplace(Key: V, Args&: Ops);
4400 if (!Inserted && (It->second.front() == V || V == WideMember0) &&
4401 !equal(LRange&: It->second, RRange&: Ops))
4402 return false;
4403 }
4404
4405 for (unsigned Idx = 0; Idx != WideMember0->getNumOperands(); ++Idx) {
4406 SmallVector<VPValue *> OpsI;
4407 for (VPValue *Op : Ops)
4408 OpsI.push_back(Elt: Op->getDefiningRecipe()->getOperand(N: Idx));
4409
4410 if (canNarrowOps(Ops: OpsI, IsScalable, FirstMembersOf))
4411 continue;
4412
4413 if (any_of(Range: enumerate(First&: OpsI), P: [WideMember0, Idx, IsScalable](const auto &P) {
4414 const auto &[OpIdx, OpV] = P;
4415 return !canNarrowLoad(WideMember0, Idx, OpV, OpIdx, IsScalable);
4416 }))
4417 return false;
4418 }
4419
4420 return true;
4421}
4422
4423/// Returns VF from \p VFs if \p IR is a full interleave group with factor and
4424/// number of members both equal to VF. The interleave group must also access
4425/// the full vector width.
4426static std::optional<ElementCount>
4427isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR,
4428 ArrayRef<ElementCount> VFs,
4429 const TargetTransformInfo &TTI) {
4430 if (!InterleaveR || InterleaveR->getMask())
4431 return std::nullopt;
4432
4433 Type *GroupElementTy = nullptr;
4434 if (InterleaveR->getStoredValues().empty()) {
4435 GroupElementTy = InterleaveR->getVPValue(I: 0)->getScalarType();
4436 if (!all_of(Range: InterleaveR->definedValues(), P: [GroupElementTy](VPValue *Op) {
4437 return Op->getScalarType() == GroupElementTy;
4438 }))
4439 return std::nullopt;
4440 } else {
4441 GroupElementTy = InterleaveR->getStoredValues()[0]->getScalarType();
4442 if (!all_of(Range: InterleaveR->getStoredValues(), P: [GroupElementTy](VPValue *Op) {
4443 return Op->getScalarType() == GroupElementTy;
4444 }))
4445 return std::nullopt;
4446 }
4447
4448 auto IG = InterleaveR->getInterleaveGroup();
4449 if (IG->getFactor() != IG->getNumMembers() || IG->isReverse())
4450 return std::nullopt;
4451
4452 auto GetVectorBitWidthForVF = [&TTI](ElementCount VF) {
4453 TypeSize Size = TTI.getRegisterBitWidth(
4454 K: VF.isFixed() ? TargetTransformInfo::RGK_FixedWidthVector
4455 : TargetTransformInfo::RGK_ScalableVector);
4456 assert(Size.isScalable() == VF.isScalable() &&
4457 "if Size is scalable, VF must be scalable and vice versa");
4458 return Size.getKnownMinValue();
4459 };
4460
4461 for (ElementCount VF : VFs) {
4462 unsigned MinVal = VF.getKnownMinValue();
4463 unsigned GroupSize = GroupElementTy->getScalarSizeInBits() * MinVal;
4464 if (IG->getFactor() == MinVal && GroupSize == GetVectorBitWidthForVF(VF))
4465 return {VF};
4466 }
4467 return std::nullopt;
4468}
4469
4470/// Returns true if \p VPValue is a narrow VPValue.
4471static bool isAlreadyNarrow(VPValue *VPV) {
4472 if (isa<VPIRValue>(Val: VPV))
4473 return true;
4474 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: VPV);
4475 return RepR && RepR->isSingleScalar();
4476}
4477
4478// Convert the wide recipes defining the VPValues in \p Members feeding an
4479// interleave group to a single narrow variant. The first member is reused as
4480// the narrowed recipe. BuildVectors for live-in operands are inserted into \p
4481// Preheader.
4482static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
4483 SmallPtrSetImpl<VPValue *> &NarrowedOps,
4484 VPBasicBlock *Preheader) {
4485 VPValue *V = Members.front();
4486 if (NarrowedOps.contains(Ptr: V))
4487 return V;
4488
4489 if (V->isDefinedOutsideLoopRegions()) {
4490 assert(all_of(Members,
4491 [V](VPValue *M) {
4492 return M->isDefinedOutsideLoopRegions() &&
4493 M->getScalarType() == V->getScalarType();
4494 }) &&
4495 "expected distinct loop-invariant values of matching scalar type");
4496 auto *BV = new VPInstruction(VPInstruction::BuildVector, Members);
4497 Preheader->appendRecipe(Recipe: BV);
4498 NarrowedOps.insert(Ptr: BV);
4499 return BV;
4500 }
4501
4502 if (isAlreadyNarrow(VPV: V))
4503 return V;
4504
4505 VPRecipeBase *R = V->getDefiningRecipe();
4506 if (isa<VPWidenRecipe, VPWidenCastRecipe>(Val: R)) {
4507 auto *WideMember0 = cast<VPRecipeWithIRFlags>(Val: R);
4508 for (VPValue *Member : Members.drop_front())
4509 WideMember0->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Member));
4510 for (unsigned Idx = 0, E = WideMember0->getNumOperands(); Idx != E; ++Idx) {
4511 SmallVector<VPValue *> OpsI;
4512 for (VPValue *Member : Members)
4513 OpsI.push_back(Elt: Member->getDefiningRecipe()->getOperand(N: Idx));
4514 WideMember0->setOperand(
4515 I: Idx, New: narrowInterleaveGroupOp(Members: OpsI, NarrowedOps, Preheader));
4516 }
4517 return V;
4518 }
4519
4520 if (auto *LoadGroup = dyn_cast<VPInterleaveRecipe>(Val: R)) {
4521 // Narrow interleave group to wide load, as transformed VPlan will only
4522 // process one original iteration.
4523 auto *LI = cast<LoadInst>(Val: LoadGroup->getInterleaveGroup()->getInsertPos());
4524 auto *L = VPBuilder(LoadGroup).createWidenLoad(
4525 Load&: *LI, Addr: LoadGroup->getAddr(), Mask: LoadGroup->getMask(), /*Consecutive=*/true,
4526 Metadata: *LoadGroup, DL: LoadGroup->getDebugLoc());
4527 NarrowedOps.insert(Ptr: L);
4528 return L;
4529 }
4530
4531 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R)) {
4532 assert(RepR->isSingleScalar() && RepR->getOpcode() == Instruction::Load &&
4533 "must be a single scalar load");
4534 NarrowedOps.insert(Ptr: RepR);
4535 return RepR;
4536 }
4537
4538 auto *WideLoad = cast<VPWidenLoadRecipe>(Val: R);
4539 VPValue *PtrOp = WideLoad->getAddr();
4540 if (auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(Val: PtrOp))
4541 PtrOp = VecPtr->getOperand(N: 0);
4542 // Narrow wide load to uniform scalar load, as transformed VPlan will only
4543 // process one original iteration.
4544 auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(), {PtrOp},
4545 /*IsUniform*/ true,
4546 /*Mask*/ nullptr, {}, *WideLoad);
4547 N->insertBefore(InsertPos: WideLoad);
4548 NarrowedOps.insert(Ptr: N);
4549 return N;
4550}
4551
4552std::unique_ptr<VPlan>
4553VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
4554 const TargetTransformInfo &TTI) {
4555 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
4556
4557 if (!VectorLoop)
4558 return nullptr;
4559
4560 // Only handle single-block loops for now.
4561 if (VectorLoop->getEntryBasicBlock() != VectorLoop->getExitingBasicBlock())
4562 return nullptr;
4563
4564 // Skip plans when we may not be able to properly narrow.
4565 VPBasicBlock *Exiting = VectorLoop->getExitingBasicBlock();
4566 if (!match(V: &Exiting->back(), P: m_BranchOnCount()))
4567 return nullptr;
4568
4569 assert(match(&Exiting->back(),
4570 m_BranchOnCount(m_Add(m_VPValue(), m_Specific(&Plan.getVFxUF())),
4571 m_Specific(&Plan.getVectorTripCount()))) &&
4572 "unexpected branch-on-count");
4573
4574 SmallVector<VPInterleaveRecipe *> StoreGroups;
4575 DenseMap<VPValue *, SmallVector<VPValue *>> FirstMembersOf;
4576 std::optional<ElementCount> VFToOptimize;
4577 for (auto &R : *VectorLoop->getEntryBasicBlock()) {
4578 if (isa<VPDerivedIVRecipe, VPScalarIVStepsRecipe>(Val: &R) &&
4579 vputils::onlyFirstLaneUsed(Def: cast<VPSingleDefRecipe>(Val: &R)))
4580 continue;
4581
4582 // Bail out on recipes not supported at the moment:
4583 // * phi recipes other than the canonical induction
4584 // * recipes writing to memory except interleave groups
4585 // Only support plans with a canonical induction phi.
4586 if (R.isPhi())
4587 return nullptr;
4588
4589 auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(Val: &R);
4590 if (R.mayWriteToMemory() && !InterleaveR)
4591 return nullptr;
4592
4593 // Bail out if any recipe defines a vector value used outside the
4594 // vector loop region.
4595 if (any_of(Range: R.definedValues(), P: [&](VPValue *V) {
4596 return any_of(Range: V->users(), P: [&](VPUser *U) {
4597 auto *UR = cast<VPRecipeBase>(Val: U);
4598 return UR->getParent()->getParent() != VectorLoop;
4599 });
4600 }))
4601 return nullptr;
4602
4603 // All other ops are allowed, but we reject uses that cannot be converted
4604 // when checking all allowed consumers (store interleave groups) below.
4605 if (!InterleaveR)
4606 continue;
4607
4608 // Try to find a single VF, where all interleave groups are consecutive and
4609 // saturate the full vector width. If we already have a candidate VF, check
4610 // if it is applicable for the current InterleaveR, otherwise look for a
4611 // suitable VF across the Plan's VFs.
4612 SmallVector<ElementCount> VFs =
4613 VFToOptimize ? SmallVector<ElementCount>({*VFToOptimize})
4614 : to_vector(Range: Plan.vectorFactors());
4615 std::optional<ElementCount> NarrowedVF =
4616 isConsecutiveInterleaveGroup(InterleaveR, VFs, TTI);
4617 if (!NarrowedVF || (VFToOptimize && NarrowedVF != VFToOptimize))
4618 return nullptr;
4619 VFToOptimize = NarrowedVF;
4620
4621 // Skip read interleave groups.
4622 if (InterleaveR->getStoredValues().empty())
4623 continue;
4624
4625 // Narrow interleave groups, if all operands are already matching narrow
4626 // ops.
4627 auto *Member0 = InterleaveR->getStoredValues()[0];
4628 if (isAlreadyNarrow(VPV: Member0) &&
4629 all_of(Range: InterleaveR->getStoredValues(), P: equal_to(Arg&: Member0))) {
4630 StoreGroups.push_back(Elt: InterleaveR);
4631 continue;
4632 }
4633
4634 // For now, we only support full interleave groups storing load interleave
4635 // groups.
4636 if (all_of(Range: enumerate(First: InterleaveR->getStoredValues()), P: [](auto Op) {
4637 VPRecipeBase *DefR = Op.value()->getDefiningRecipe();
4638 if (!DefR)
4639 return false;
4640 auto *IR = dyn_cast<VPInterleaveRecipe>(Val: DefR);
4641 return IR && IR->getInterleaveGroup()->isFull() &&
4642 IR->getVPValue(Op.index()) == Op.value();
4643 })) {
4644 StoreGroups.push_back(Elt: InterleaveR);
4645 continue;
4646 }
4647
4648 // Check if all values feeding InterleaveR are matching wide recipes whose
4649 // operands can be narrowed, without conflicting uses across groups.
4650 if (!canNarrowOps(Ops: InterleaveR->getStoredValues(),
4651 IsScalable: VFToOptimize->isScalable(), FirstMembersOf))
4652 return nullptr;
4653 StoreGroups.push_back(Elt: InterleaveR);
4654 }
4655
4656 if (StoreGroups.empty())
4657 return nullptr;
4658
4659 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
4660 bool RequiresScalarEpilogue =
4661 MiddleVPBB->getNumSuccessors() == 1 &&
4662 MiddleVPBB->getSingleSuccessor() == Plan.getScalarPreheader();
4663 // Bail out for tail-folding (middle block with a single successor to exit).
4664 if (MiddleVPBB->getNumSuccessors() != 2 && !RequiresScalarEpilogue)
4665 return nullptr;
4666
4667 // All interleave groups in Plan can be narrowed for VFToOptimize. Split the
4668 // original Plan into 2: a) a new clone which contains all VFs of Plan, except
4669 // VFToOptimize, and b) the original Plan with VFToOptimize as single VF.
4670 // TODO: Handle cases where only some interleave groups can be narrowed.
4671 std::unique_ptr<VPlan> NewPlan;
4672 if (size(Range: Plan.vectorFactors()) != 1) {
4673 NewPlan = std::unique_ptr<VPlan>(Plan.duplicate());
4674 Plan.setVF(*VFToOptimize);
4675 NewPlan->removeVF(VF: *VFToOptimize);
4676 }
4677
4678 // Convert InterleaveGroup \p R to a single VPWidenLoadRecipe.
4679 SmallPtrSet<VPValue *, 4> NarrowedOps;
4680 VPBasicBlock *Preheader = Plan.getVectorPreheader();
4681 // Narrow operation tree rooted at store groups.
4682 for (auto *StoreGroup : StoreGroups) {
4683 VPValue *Res = narrowInterleaveGroupOp(Members: StoreGroup->getStoredValues(),
4684 NarrowedOps, Preheader);
4685 auto *SI =
4686 cast<StoreInst>(Val: StoreGroup->getInterleaveGroup()->getInsertPos());
4687 VPBuilder(StoreGroup)
4688 .createWidenStore(Store&: *SI, Addr: StoreGroup->getAddr(), StoredVal: Res, Mask: nullptr,
4689 /*Consecutive=*/true, Metadata: *StoreGroup,
4690 DL: StoreGroup->getDebugLoc());
4691 StoreGroup->eraseFromParent();
4692 }
4693
4694 // Adjust induction to reflect that the transformed plan only processes one
4695 // original iteration.
4696 VPInstruction *CanIVInc = vputils::findCanonicalIVIncrement(Plan);
4697 Type *CanIVTy = VectorLoop->getCanonicalIVType();
4698 VPBasicBlock *VectorPH = Plan.getVectorPreheader();
4699 VPBuilder PHBuilder(VectorPH, VectorPH->getFirstNonPhi());
4700
4701 VPValue *UF = &Plan.getUF();
4702 VPValue *Step;
4703 if (VFToOptimize->isScalable()) {
4704 VPValue *VScale =
4705 PHBuilder.createElementCount(Ty: CanIVTy, EC: ElementCount::getScalable(MinVal: 1));
4706 Step = PHBuilder.createOverflowingOp(Opcode: Instruction::Mul, Operands: {VScale, UF},
4707 WrapFlags: {true, false});
4708 Plan.getVF().replaceAllUsesWith(New: VScale);
4709 } else {
4710 Step = UF;
4711 Plan.getVF().replaceAllUsesWith(New: Plan.getConstantInt(Ty: CanIVTy, Val: 1));
4712 }
4713 // Materialize vector trip count with the narrowed step.
4714 materializeVectorTripCount(Plan, VectorPHVPBB: VectorPH, /*TailByMasking=*/false,
4715 RequiresScalarEpilogue, Step);
4716
4717 CanIVInc->setOperand(I: 1, New: Step);
4718 Plan.getVFxUF().replaceAllUsesWith(New: Step);
4719
4720 removeDeadRecipes(Plan);
4721 assert(none_of(*VectorLoop->getEntryBasicBlock(),
4722 IsaPred<VPVectorPointerRecipe>) &&
4723 "All VPVectorPointerRecipes should have been removed");
4724 return NewPlan;
4725}
4726
4727void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
4728 VFRange &Range) {
4729 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
4730 auto *MiddleVPBB = Plan.getMiddleBlock();
4731 VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
4732
4733 auto IsScalableOne = [](ElementCount VF) -> bool {
4734 return VF == ElementCount::getScalable(MinVal: 1);
4735 };
4736
4737 for (VPFirstOrderRecurrencePHIRecipe &FOR :
4738 make_isa_range<VPFirstOrderRecurrencePHIRecipe>(
4739 Range: VectorRegion->getEntryBasicBlock()->phis())) {
4740 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
4741 "Cannot handle loops with uncountable early exits");
4742
4743 // Find the existing splice for this FOR, created in
4744 // createHeaderPhiRecipes. All uses of FOR have already been replaced with
4745 // RecurSplice there; only RecurSplice itself still references FOR.
4746 auto *RecurSplice =
4747 findUserOf<VPInstruction::FirstOrderRecurrenceSplice>(V: &FOR);
4748 assert(RecurSplice && "expected FirstOrderRecurrenceSplice");
4749
4750 // For VF vscale x 1, if vscale = 1, we are unable to extract the
4751 // penultimate value of the recurrence. Instead we rely on the existing
4752 // extract of the last element from the result of
4753 // VPInstruction::FirstOrderRecurrenceSplice.
4754 // TODO: Consider vscale_range info and UF.
4755 if (any_of(Range: RecurSplice->users(),
4756 P: [](VPUser *U) { return !cast<VPRecipeBase>(Val: U)->getRegion(); }) &&
4757 LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsScalableOne,
4758 Range))
4759 return;
4760
4761 // This is the second phase of vectorizing first-order recurrences, creating
4762 // extracts for users outside the loop. An overview of the transformation is
4763 // described below. Suppose we have the following loop with some use after
4764 // the loop of the last a[i-1],
4765 //
4766 // for (int i = 0; i < n; ++i) {
4767 // t = a[i - 1];
4768 // b[i] = a[i] - t;
4769 // }
4770 // use t;
4771 //
4772 // There is a first-order recurrence on "a". For this loop, the shorthand
4773 // scalar IR looks like:
4774 //
4775 // scalar.ph:
4776 // s.init = a[-1]
4777 // br scalar.body
4778 //
4779 // scalar.body:
4780 // i = phi [0, scalar.ph], [i+1, scalar.body]
4781 // s1 = phi [s.init, scalar.ph], [s2, scalar.body]
4782 // s2 = a[i]
4783 // b[i] = s2 - s1
4784 // br cond, scalar.body, exit.block
4785 //
4786 // exit.block:
4787 // use = lcssa.phi [s1, scalar.body]
4788 //
4789 // In this example, s1 is a recurrence because it's value depends on the
4790 // previous iteration. In the first phase of vectorization, we created a
4791 // VPFirstOrderRecurrencePHIRecipe v1 for s1. Now we create the extracts
4792 // for users in the scalar preheader and exit block.
4793 //
4794 // vector.ph:
4795 // v_init = vector(..., ..., ..., a[-1])
4796 // br vector.body
4797 //
4798 // vector.body
4799 // i = phi [0, vector.ph], [i+4, vector.body]
4800 // v1 = phi [v_init, vector.ph], [v2, vector.body]
4801 // v2 = a[i, i+1, i+2, i+3]
4802 // v1' = splice(v1(3), v2(0, 1, 2))
4803 // b[i, i+1, i+2, i+3] = v2 - v1'
4804 // br cond, vector.body, middle.block
4805 //
4806 // middle.block:
4807 // vector.recur.extract.for.phi = v2(2)
4808 // vector.recur.extract = v2(3)
4809 // br cond, scalar.ph, exit.block
4810 //
4811 // scalar.ph:
4812 // scalar.recur.init = phi [vector.recur.extract, middle.block],
4813 // [s.init, otherwise]
4814 // br scalar.body
4815 //
4816 // scalar.body:
4817 // i = phi [0, scalar.ph], [i+1, scalar.body]
4818 // s1 = phi [scalar.recur.init, scalar.ph], [s2, scalar.body]
4819 // s2 = a[i]
4820 // b[i] = s2 - s1
4821 // br cond, scalar.body, exit.block
4822 //
4823 // exit.block:
4824 // lo = lcssa.phi [s1, scalar.body],
4825 // [vector.recur.extract.for.phi, middle.block]
4826 //
4827 // Update extracts of the splice in the middle block: they extract the
4828 // penultimate element of the recurrence.
4829 for (VPRecipeBase &R : make_early_inc_range(
4830 Range: make_range(x: MiddleVPBB->getFirstNonPhi(), y: MiddleVPBB->end()))) {
4831 if (!match(V: &R, P: m_ExtractLastLaneOfLastPart(Op0: m_Specific(VPV: RecurSplice))))
4832 continue;
4833
4834 auto *ExtractR = cast<VPInstruction>(Val: &R);
4835 VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
4836 Opcode: VPInstruction::ExtractPenultimateElement, Operands: RecurSplice->getOperand(N: 1),
4837 DL: {}, Name: "vector.recur.extract.for.phi");
4838 for (VPUser *ExitU : to_vector(Range: ExtractR->users())) {
4839 if (auto *ExitPhi = dyn_cast<VPIRPhi>(Val: ExitU))
4840 ExitPhi->replaceUsesOfWith(From: ExtractR, To: PenultimateElement);
4841 }
4842 }
4843 }
4844}
4845
4846/// Check if \p V is a binary expression of a widened IV and a loop-invariant
4847/// value. Returns the widened IV if found, nullptr otherwise.
4848static VPWidenIntOrFpInductionRecipe *getExpressionIV(VPValue *V) {
4849 auto *BinOp = dyn_cast<VPWidenRecipe>(Val: V);
4850 if (!BinOp || !Instruction::isBinaryOp(Opcode: BinOp->getOpcode()) ||
4851 Instruction::isIntDivRem(Opcode: BinOp->getOpcode()))
4852 return nullptr;
4853
4854 VPValue *WidenIVCandidate = BinOp->getOperand(N: 0);
4855 VPValue *InvariantCandidate = BinOp->getOperand(N: 1);
4856 if (!isa<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate))
4857 std::swap(a&: WidenIVCandidate, b&: InvariantCandidate);
4858
4859 if (!InvariantCandidate->isDefinedOutsideLoopRegions())
4860 return nullptr;
4861
4862 return dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate);
4863}
4864
4865/// Create a scalar version of \p BinOp, with its \p WidenIV operand replaced
4866/// by \p ScalarIV, and place it after \p ScalarIV's defining recipe.
4867static VPValue *cloneBinOpForScalarIV(VPWidenRecipe *BinOp, VPValue *ScalarIV,
4868 VPWidenIntOrFpInductionRecipe *WidenIV) {
4869 assert(Instruction::isBinaryOp(BinOp->getOpcode()) &&
4870 BinOp->getNumOperands() == 2 && "BinOp must have 2 operands");
4871 auto *ClonedOp = BinOp->clone();
4872 if (ClonedOp->getOperand(N: 0) == WidenIV) {
4873 ClonedOp->setOperand(I: 0, New: ScalarIV);
4874 } else {
4875 assert(ClonedOp->getOperand(1) == WidenIV && "one operand must be WideIV");
4876 ClonedOp->setOperand(I: 1, New: ScalarIV);
4877 }
4878 ClonedOp->insertAfter(InsertPos: ScalarIV->getDefiningRecipe());
4879 return ClonedOp;
4880}
4881
4882/// If \p S is an affine AddRec, returns true if its step is known to be
4883/// positive and false if it is known to be negative. Returns std::nullopt if
4884/// \p S is not an affine AddRec, or if the sign of its step cannot be
4885/// determined.
4886static std::optional<bool> getStepDirection(const SCEV *S,
4887 ScalarEvolution &SE) {
4888 const SCEV *Step;
4889 if (!match(S, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV(V&: Step))))
4890 return std::nullopt;
4891 if (SE.isKnownPositive(S: Step))
4892 return true;
4893 if (SE.isKnownNegative(S: Step))
4894 return false;
4895 return std::nullopt;
4896}
4897
4898void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
4899 PredicatedScalarEvolution &PSE,
4900 Loop &L) {
4901 ScalarEvolution &SE = *PSE.getSE();
4902 VPRegionBlock *VectorLoopRegion = Plan.getVectorLoopRegion();
4903
4904 // Helper lambda to check if the IV range excludes the sentinel value. Try
4905 // signed first, then unsigned. Return an excluded sentinel if found,
4906 // otherwise return std::nullopt.
4907 auto CheckSentinel = [&SE](const SCEV *IVSCEV,
4908 bool UseMax) -> std::optional<APSInt> {
4909 unsigned BW = IVSCEV->getType()->getScalarSizeInBits();
4910 for (bool Signed : {true, false}) {
4911 APSInt Sentinel = UseMax ? APSInt::getMinValue(numBits: BW, /*Unsigned=*/!Signed)
4912 : APSInt::getMaxValue(numBits: BW, /*Unsigned=*/!Signed);
4913
4914 ConstantRange IVRange =
4915 Signed ? SE.getSignedRange(S: IVSCEV) : SE.getUnsignedRange(S: IVSCEV);
4916 if (!IVRange.contains(Val: Sentinel))
4917 return Sentinel;
4918 }
4919 return std::nullopt;
4920 };
4921
4922 VPValue *HeaderMask = VectorLoopRegion->getHeaderMask();
4923 for (VPRecipeBase &Phi :
4924 make_early_inc_range(Range: VectorLoopRegion->getEntryBasicBlock()->phis())) {
4925 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &Phi);
4926 if (!PhiR || !RecurrenceDescriptor::isFindLastRecurrenceKind(
4927 Kind: PhiR->getRecurrenceKind()))
4928 continue;
4929
4930 Type *PhiTy = PhiR->getScalarType();
4931 if (PhiTy->isPointerTy() || PhiTy->isFloatingPointTy())
4932 continue;
4933
4934 // If there's a header mask, the backedge select will not be the find-last
4935 // select.
4936 VPValue *BackedgeVal = PhiR->getBackedgeValue();
4937 auto *FindLastSelect = cast<VPSingleDefRecipe>(Val: BackedgeVal);
4938 if (HeaderMask &&
4939 !match(V: BackedgeVal,
4940 P: m_Select(Op0: m_Specific(VPV: HeaderMask),
4941 Op1: m_VPSingleDefRecipe(V&: FindLastSelect), Op2: m_Specific(VPV: PhiR))))
4942 continue;
4943
4944 // Get the find-last expression from the find-last select of the reduction
4945 // phi. The find-last select should be a select between the phi and the
4946 // find-last expression.
4947 VPValue *Cond, *FindLastExpression;
4948 if (!match(R: FindLastSelect, P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: PhiR),
4949 Op2: m_VPValue(V&: FindLastExpression))) &&
4950 !match(R: FindLastSelect,
4951 P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_VPValue(V&: FindLastExpression),
4952 Op2: m_Specific(VPV: PhiR))))
4953 continue;
4954
4955 // Check if FindLastExpression is a simple expression of a widened IV. If
4956 // so, we can track the underlying IV instead and sink the expression.
4957 auto *IVOfExpressionToSink = getExpressionIV(V: FindLastExpression);
4958 const SCEV *IVSCEV = vputils::getSCEVExprForVPValue(
4959 V: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression, PSE,
4960 L: &L);
4961 if (!match(S: IVSCEV, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV()))) {
4962 assert(!match(vputils::getSCEVExprForVPValue(FindLastExpression, PSE, &L),
4963 m_scev_AffineAddRec(m_SCEV(), m_SCEV())) &&
4964 "IVOfExpressionToSink not being an AddRec must imply "
4965 "FindLastExpression not being an AddRec.");
4966 continue;
4967 }
4968
4969 // Determine direction from the step of IVSCEV, if possible.
4970 std::optional<bool> StepDirection = getStepDirection(S: IVSCEV, SE);
4971 if (!StepDirection)
4972 continue;
4973
4974 bool UseMax = *StepDirection;
4975 std::optional<APSInt> SentinelVal = CheckSentinel(IVSCEV, UseMax);
4976 bool UseSigned = SentinelVal && SentinelVal->isSigned();
4977
4978 // Sinking an expression will disable epilogue vectorization. Only use it,
4979 // if FindLastExpression cannot be vectorized via a sentinel. Sinking may
4980 // also prevent vectorizing using a sentinel (e.g., if the expression is a
4981 // multiply or divide by large constant, respectively), which also makes
4982 // sinking undesirable.
4983 if (IVOfExpressionToSink) {
4984 const SCEV *FindLastExpressionSCEV =
4985 vputils::getSCEVExprForVPValue(V: FindLastExpression, PSE, L: &L);
4986 if (std::optional<bool> NewUseMax =
4987 getStepDirection(S: FindLastExpressionSCEV, SE)) {
4988 if (auto NewSentinel =
4989 CheckSentinel(FindLastExpressionSCEV, *NewUseMax)) {
4990 // The original expression already has a sentinel, so prefer not
4991 // sinking to keep epilogue vectorization possible.
4992 SentinelVal = *NewSentinel;
4993 UseSigned = NewSentinel->isSigned();
4994 UseMax = *NewUseMax;
4995 IVSCEV = FindLastExpressionSCEV;
4996 IVOfExpressionToSink = nullptr;
4997 }
4998 }
4999 }
5000
5001 // If no sentinel was found, fall back to a boolean AnyOf reduction to track
5002 // if the condition was ever true. Requires the IV to not wrap, otherwise we
5003 // cannot use min/max.
5004 if (!SentinelVal) {
5005 auto *AR = cast<SCEVAddRecExpr>(Val: IVSCEV);
5006 if (AR->hasNoSignedWrap())
5007 UseSigned = true;
5008 else if (AR->hasNoUnsignedWrap())
5009 UseSigned = false;
5010 else
5011 continue;
5012 }
5013
5014 VPInstruction *RdxResult = cast<VPInstruction>(Val: vputils::findRecipe(
5015 Start: BackedgeVal,
5016 Pred: match_fn(P: m_VPInstruction<VPInstruction::ComputeReductionResult>())));
5017
5018 VPValue *NewFindLastSelect = BackedgeVal;
5019 VPValue *SelectCond = Cond;
5020 if (!SentinelVal || IVOfExpressionToSink) {
5021 // When we need to create a new select, normalize the condition so that
5022 // PhiR is the last operand and include the header mask if needed.
5023 DebugLoc DL = FindLastSelect->getDefiningRecipe()->getDebugLoc();
5024 VPBuilder LoopBuilder(FindLastSelect->getDefiningRecipe());
5025 if (match(R: FindLastSelect,
5026 P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: PhiR), Op2: m_VPValue())))
5027 SelectCond = LoopBuilder.createNot(Operand: SelectCond);
5028
5029 // When tail folding, mask the condition with the header mask to prevent
5030 // propagating poison from inactive lanes in the last vector iteration.
5031 if (HeaderMask)
5032 SelectCond = LoopBuilder.createLogicalAnd(LHS: HeaderMask, RHS: SelectCond);
5033
5034 if (SelectCond != Cond || IVOfExpressionToSink) {
5035 NewFindLastSelect = LoopBuilder.createSelect(
5036 Cond: SelectCond,
5037 TrueVal: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression,
5038 FalseVal: PhiR, DL);
5039 }
5040 }
5041
5042 // Create the reduction result in the middle block using sentinel directly.
5043 RecurKind MinMaxKind =
5044 UseMax ? (UseSigned ? RecurKind::SMax : RecurKind::UMax)
5045 : (UseSigned ? RecurKind::SMin : RecurKind::UMin);
5046 VPIRFlags Flags(MinMaxKind, /*IsOrdered=*/false, /*IsInLoop=*/false,
5047 FastMathFlags());
5048 DebugLoc ExitDL = RdxResult->getDebugLoc();
5049 VPBuilder MiddleBuilder(RdxResult);
5050 VPValue *ReducedIV =
5051 MiddleBuilder.createNaryOp(Opcode: VPInstruction::ComputeReductionResult,
5052 Operands: NewFindLastSelect, Flags, DL: ExitDL);
5053
5054 // If IVOfExpressionToSink is an expression to sink, sink it now.
5055 VPValue *VectorRegionExitingVal = ReducedIV;
5056 bool SunkExpression = false;
5057 if (IVOfExpressionToSink) {
5058 VectorRegionExitingVal =
5059 cloneBinOpForScalarIV(BinOp: cast<VPWidenRecipe>(Val: FindLastExpression),
5060 ScalarIV: ReducedIV, WidenIV: IVOfExpressionToSink);
5061 SunkExpression = true;
5062 }
5063
5064 VPValue *NewRdxResult;
5065 VPValue *StartVPV = PhiR->getStartValue();
5066 if (SentinelVal) {
5067 // Sentinel-based approach: reduce IVs with min/max, compare against
5068 // sentinel to detect if condition was ever true, select accordingly.
5069 VPValue *Sentinel = Plan.getConstantInt(Val: *SentinelVal);
5070 auto *Cmp = MiddleBuilder.createICmp(Pred: CmpInst::ICMP_NE, A: ReducedIV,
5071 B: Sentinel, DL: ExitDL);
5072 NewRdxResult = MiddleBuilder.createSelect(Cond: Cmp, TrueVal: VectorRegionExitingVal,
5073 FalseVal: StartVPV, DL: ExitDL);
5074 StartVPV = Sentinel;
5075 } else {
5076 // Introduce a boolean AnyOf reduction to track if the condition was ever
5077 // true in the loop. Use it to select the initial start value, if it was
5078 // never true.
5079 auto *AnyOfPhi = new VPReductionPHIRecipe(
5080 /*Phi=*/nullptr, RecurKind::Or, *Plan.getFalse(), *Plan.getFalse(),
5081 RdxUnordered{.VFScaleFactor: 1}, {}, /*HasUsesOutsideReductionChain=*/false);
5082 AnyOfPhi->insertAfter(InsertPos: PhiR);
5083
5084 VPBuilder LoopBuilder(BackedgeVal->getDefiningRecipe());
5085 VPValue *OrVal = LoopBuilder.createOr(LHS: AnyOfPhi, RHS: SelectCond);
5086 AnyOfPhi->setOperand(I: 1, New: OrVal);
5087
5088 NewRdxResult = MiddleBuilder.createAnyOfReduction(
5089 ChainOp: OrVal, TrueVal: VectorRegionExitingVal, FalseVal: StartVPV, DL: ExitDL);
5090
5091 // Initialize the IV reduction phi with the neutral element, not the
5092 // original start value, to ensure correct min/max reduction results.
5093 StartVPV = Plan.getOrAddLiveIn(
5094 V: getRecurrenceIdentity(K: MinMaxKind, Tp: IVSCEV->getType(), FMF: {}));
5095 }
5096 RdxResult->replaceAllUsesWith(New: NewRdxResult);
5097 RdxResult->eraseFromParent();
5098
5099 auto *NewPhiR = new VPReductionPHIRecipe(
5100 cast<PHINode>(Val: PhiR->getUnderlyingInstr()), RecurKind::FindIV, *StartVPV,
5101 *NewFindLastSelect, RdxUnordered{.VFScaleFactor: 1}, {},
5102 PhiR->hasUsesOutsideReductionChain());
5103 if (SunkExpression)
5104 NewPhiR->setExpressionSunk();
5105 NewPhiR->insertBefore(InsertPos: PhiR);
5106 PhiR->replaceAllUsesWith(New: NewPhiR);
5107 PhiR->eraseFromParent();
5108 }
5109}
5110
5111namespace {
5112
5113struct ReductionExtend {
5114 Type *SrcType = nullptr;
5115 ExtendKind Kind = ExtendKind::PR_None;
5116};
5117
5118/// Describes the extends used to compute the extended reduction operand.
5119/// ExtendB is optional. If ExtendB is present, ExtendsUser is a binary
5120/// operation.
5121struct ExtendedReductionOperand {
5122 /// The recipe that consumes the extends.
5123 VPWidenRecipe *ExtendsUser = nullptr;
5124 /// Extend descriptions (inputs to getPartialReductionCost).
5125 ReductionExtend ExtendA, ExtendB;
5126};
5127
5128/// A collection of recipes that describe a partial reduction. Matches either
5129/// reduction_bin_op (extended op, accumulator), or
5130/// reduction_bin_op (accumulator, extended op).
5131/// The possible forms of the "extended op" are listed in
5132/// matchExtendedReductionOperand.
5133struct PartialReductionDescriptor {
5134 /// The top-level binary operation that forms the reduction to a scalar
5135 /// after the loop body.
5136 VPWidenRecipe *ReductionBinOp = nullptr;
5137 /// The user of the extends that is then reduced.
5138 ExtendedReductionOperand ExtendedOp;
5139 /// The recurrence kind for the entire partial reduction chain.
5140 /// This allows distinguishing between Sub and AddWithSub recurrences,
5141 /// when the ReductionBinOp is a Instruction::Sub.
5142 RecurKind RK;
5143 /// The index of the accumulator operand of ReductionBinOp. The extended op
5144 /// is `1 - AccumulatorOpIdx`.
5145 unsigned AccumulatorOpIdx;
5146 unsigned ScaleFactor;
5147 /// Optional blend to represent predication for the block that updates the
5148 /// reduction.
5149 VPBlendRecipe *Blend = nullptr;
5150};
5151
5152// Return the incoming index of the single-use value in the blend, which is
5153// expected to be the predicated reduction update.
5154static std::optional<unsigned>
5155getBlendReductionUpdateValueIdx(VPBlendRecipe *Blend) {
5156 assert(Blend && !Blend->isNormalized() &&
5157 Blend->getNumIncomingValues() == 2 &&
5158 "Expected a non-normalized blend with two incoming values");
5159 bool FirstIncomingHasOneUse = Blend->getIncomingValue(Idx: 0)->hasOneUse();
5160
5161 // Only the update value should have one use (the blend). The previous
5162 // value should always have at least two uses, the blend and the reduction.
5163 if (FirstIncomingHasOneUse == Blend->getIncomingValue(Idx: 1)->hasOneUse())
5164 return std::nullopt;
5165 return FirstIncomingHasOneUse ? 0 : 1;
5166}
5167
5168static VPSingleDefRecipe *
5169optimizeExtendsForPartialReduction(VPSingleDefRecipe *Op) {
5170 // reduce.add(mul(ext(A), C))
5171 // -> reduce.add(mul(ext(A), ext(trunc(C))))
5172 const APInt *Const;
5173 if (match(R: Op, P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_APInt(C&: Const)))) {
5174 auto *ExtA = cast<VPWidenCastRecipe>(Val: Op->getOperand(N: 0));
5175 Instruction::CastOps ExtOpc = ExtA->getOpcode();
5176 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
5177 if (!Op->hasOneUse() ||
5178 !llvm::canConstantBeExtended(
5179 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
5180 return Op;
5181
5182 VPBuilder Builder(Op);
5183 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc,
5184 Op: Op->getOperand(N: 1), ResultTy: NarrowTy);
5185 Type *WideTy = ExtA->getScalarType();
5186 Op->setOperand(I: 1, New: Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy));
5187 return Op;
5188 }
5189
5190 // reduce.add(abs(sub(ext(A), ext(B))))
5191 // -> reduce.add(ext(absolute-difference(A, B)))
5192 VPValue *X, *Y;
5193 if (match(R: Op, P: m_WidenIntrinsic<Intrinsic::abs>(Ops: m_Sub(
5194 Op0: m_ZExtOrSExt(Op0: m_VPValue(V&: X)), Op1: m_ZExtOrSExt(Op0: m_VPValue(V&: Y)))))) {
5195 auto *Sub = Op->getOperand(N: 0)->getDefiningRecipe();
5196 auto *Ext = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
5197 assert(Ext->getOpcode() ==
5198 cast<VPWidenCastRecipe>(Sub->getOperand(1))->getOpcode() &&
5199 "Expected both the LHS and RHS extends to be the same");
5200 bool IsSigned = Ext->getOpcode() == Instruction::SExt;
5201 VPBuilder Builder(Op);
5202 Type *SrcTy = X->getScalarType();
5203 auto *FreezeX = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {X}));
5204 auto *FreezeY = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {Y}));
5205 auto *Max = Builder.insert(
5206 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smax : Intrinsic::umax,
5207 {FreezeX, FreezeY}, SrcTy));
5208 auto *Min = Builder.insert(
5209 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smin : Intrinsic::umin,
5210 {FreezeX, FreezeY}, SrcTy));
5211 auto *AbsDiff = Builder.insert(
5212 R: new VPWidenRecipe(Instruction::Sub, {Max, Min},
5213 VPIRFlags::getDefaultFlags(Opcode: Instruction::Sub)));
5214 return Builder.createWidenCast(Opcode: Instruction::CastOps::ZExt, Op: AbsDiff,
5215 ResultTy: Op->getScalarType());
5216 }
5217
5218 // reduce.add(ext(mul(ext(A), ext(B))))
5219 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
5220 // TODO: Support this optimization for float types.
5221 if (match(R: Op, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
5222 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
5223 auto *Ext = cast<VPWidenCastRecipe>(Val: Op);
5224 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
5225 auto *MulLHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5226 auto *MulRHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5227 assert(Mul->hasOneUse() && Ext->getOpcode() == MulLHS->getOpcode() &&
5228 MulLHS->getOpcode() == MulRHS->getOpcode() &&
5229 canFoldOuterExtendIntoMul(Mul) &&
5230 "Expected the outer extend to be foldable");
5231 VPBuilder Builder(Mul);
5232 auto *NewLHS = Builder.createWidenCast(
5233 Opcode: MulLHS->getOpcode(), Op: MulLHS->getOperand(N: 0), ResultTy: Ext->getScalarType());
5234 auto *NewRHS = MulLHS == MulRHS
5235 ? NewLHS
5236 : Builder.createWidenCast(Opcode: MulRHS->getOpcode(),
5237 Op: MulRHS->getOperand(N: 0),
5238 ResultTy: Ext->getScalarType());
5239 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewLHS, NewRHS});
5240 Builder.insert(R: NewMul);
5241 Op->replaceAllUsesWith(New: NewMul);
5242 Op->eraseFromParent();
5243 Mul->eraseFromParent();
5244 return NewMul;
5245 }
5246
5247 return Op;
5248}
5249
5250static VPExpressionRecipe *
5251createPartialReductionExpression(VPReductionRecipe *Red) {
5252 VPValue *VecOp = Red->getVecOp();
5253
5254 // reduce.[f]add(ext(op))
5255 // -> VPExpressionRecipe(op, red)
5256 if (match(V: VecOp, P: m_WidenAnyExtend(Op0: m_VPValue())))
5257 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
5258
5259 // reduce.[f]add(neg(ext(op)))
5260 // -> VPExpressionRecipe(op, sub/neg, red)
5261 if (match(V: VecOp, P: m_AnyNeg(Op0: m_WidenAnyExtend(Op0: m_VPValue())))) {
5262 auto *Neg = cast<VPWidenRecipe>(Val: VecOp);
5263 auto *Ext = cast<VPWidenCastRecipe>(Val: Neg->getLastOperand());
5264 return new VPExpressionRecipe(Ext, Neg, Red);
5265 }
5266
5267 // reduce.[f]add([f]mul(ext(a), ext(b)))
5268 // -> VPExpressionRecipe(a, b, mul, red)
5269 if (match(V: VecOp, P: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))) ||
5270 match(V: VecOp,
5271 P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_ZExtOrSExt(Op0: m_VPValue())))) {
5272 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
5273 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5274 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5275 return new VPExpressionRecipe(ExtA, ExtB, Mul, Red);
5276 }
5277
5278 // reduce.fadd(fneg(fmul(fpext(a), fpext(b))))
5279 // -> VPExpressionRecipe(a, b, fmul, fsub, red)
5280 if (match(V: VecOp,
5281 P: m_FNeg(Op0: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))))) {
5282 auto *FNeg = cast<VPWidenRecipe>(Val: VecOp);
5283 auto *FMul = cast<VPWidenRecipe>(Val: FNeg->getOperand(N: 0));
5284 auto *ExtA = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 0));
5285 auto *ExtB = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 1));
5286 return new VPExpressionRecipe(ExtA, ExtB, FMul, FNeg, Red);
5287 }
5288
5289 // reduce.add(neg(mul(ext(a), ext(b))))
5290 // -> VPExpressionRecipe(a, b, mul, sub, red)
5291 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
5292 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
5293 auto *Sub = cast<VPWidenRecipe>(Val: VecOp);
5294 auto *Mul = cast<VPWidenRecipe>(Val: Sub->getOperand(N: 1));
5295 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5296 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5297 return new VPExpressionRecipe(ExtA, ExtB, Mul, Sub, Red);
5298 }
5299
5300 llvm_unreachable("Unsupported expression");
5301}
5302
5303// Helper to transform a PartialReductionDescriptor into a partial reduction
5304// recipe. Assumes profitability has been checked.
5305static void transformToPartialReduction(const PartialReductionDescriptor &Link,
5306 VPlan &Plan,
5307 VPReductionPHIRecipe *RdxPhi) {
5308 VPWidenRecipe *WidenRecipe = Link.ReductionBinOp;
5309 assert(WidenRecipe->getNumOperands() == 2 && "Expected binary operation");
5310
5311 VPValue *Accumulator = WidenRecipe->getOperand(N: Link.AccumulatorOpIdx);
5312 auto *ExtendedOp = cast<VPSingleDefRecipe>(
5313 Val: WidenRecipe->getOperand(N: 1 - Link.AccumulatorOpIdx));
5314
5315 // FIXME: Do these transforms before invoking the cost-model.
5316 ExtendedOp = optimizeExtendsForPartialReduction(Op: ExtendedOp);
5317
5318 // Sub-reductions can be implemented in two ways:
5319 // (1) negate the operand in the vector loop (the default way).
5320 // (2) subtract the reduced value from the init value in the middle block.
5321 // Both ways keep the reduction itself as an 'add' reduction.
5322 //
5323 // The ISD nodes for partial reductions don't support folding the
5324 // sub/negation into its operands because the following is not a valid
5325 // transformation:
5326 // sub(0, mul(ext(a), ext(b)))
5327 // -> mul(ext(a), ext(sub(0, b)))
5328 //
5329 // It's therefore better to choose option (2) such that the partial
5330 // reduction is always positive (starting at '0') and to do a final
5331 // subtract in the middle block.
5332 if ((WidenRecipe->getOpcode() == Instruction::Sub &&
5333 Link.RK != RecurKind::Sub) ||
5334 (WidenRecipe->getOpcode() == Instruction::FSub &&
5335 Link.RK != RecurKind::FSub)) {
5336 VPBuilder Builder(WidenRecipe);
5337 Type *ElemTy = ExtendedOp->getScalarType();
5338 VPWidenRecipe *NegRecipe;
5339 if (WidenRecipe->getOpcode() == Instruction::FSub) {
5340 NegRecipe =
5341 new VPWidenRecipe(Instruction::FNeg, {ExtendedOp},
5342 VPIRFlags::getDefaultFlags(Opcode: Instruction::FNeg),
5343 VPIRMetadata(), DebugLoc::getUnknown());
5344 } else {
5345 auto *Zero = Plan.getZero(Ty: ElemTy);
5346 NegRecipe =
5347 new VPWidenRecipe(Instruction::Sub, {Zero, ExtendedOp},
5348 VPIRFlags::getDefaultFlags(Opcode: Instruction::Sub),
5349 VPIRMetadata(), DebugLoc::getUnknown());
5350 }
5351 Builder.insert(R: NegRecipe);
5352 ExtendedOp = NegRecipe;
5353 }
5354
5355 // Check if WidenRecipe is the final result of the reduction. If so, look
5356 // through the Select recipe introduced by tail-folding, otherwise look
5357 // through any Blend recipe introduced by predication for the block.
5358 VPValue *ExitSearch =
5359 Link.Blend ? cast<VPValue>(Val: Link.Blend) : cast<VPValue>(Val: WidenRecipe);
5360
5361 VPValue *Cond = nullptr;
5362 VPValue *ExitValue = cast_or_null<VPInstruction>(
5363 Val: findUserOf(V: ExitSearch, P: m_Select(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: ExitSearch),
5364 Op2: m_Specific(VPV: RdxPhi))));
5365
5366 if (Link.Blend) {
5367 std::optional<unsigned> BlendReductionIdx =
5368 getBlendReductionUpdateValueIdx(Blend: Link.Blend);
5369 assert(BlendReductionIdx &&
5370 Link.Blend->getIncomingValue(*BlendReductionIdx) == WidenRecipe &&
5371 "Expected blend to contain the reduction update");
5372 VPValue *BlendCond = Link.Blend->getMask(Idx: *BlendReductionIdx);
5373 Cond = ExitValue ? VPBuilder(WidenRecipe)
5374 .createLogicalAnd(LHS: Cond, RHS: BlendCond,
5375 DL: WidenRecipe->getDebugLoc())
5376 : BlendCond;
5377 }
5378
5379 // When folding the tail, the inactive lanes of the reduction update are
5380 // computed from values that do not correspond to any scalar iteration
5381 // and must not be accumulated.
5382 if (!Cond)
5383 Cond = Plan.getVectorLoopRegion()->getHeaderMask();
5384
5385 [[maybe_unused]] bool IsLastInChain =
5386 RdxPhi->getBackedgeValue() == WidenRecipe ||
5387 RdxPhi->getBackedgeValue() == ExitValue ||
5388 RdxPhi->getBackedgeValue() == Link.Blend;
5389 assert((!ExitValue || IsLastInChain) &&
5390 "if we found ExitValue, it must match RdxPhi's backedge value");
5391
5392 Type *PhiType = RdxPhi->getScalarType();
5393 RecurKind RdxKind =
5394 PhiType->isFloatingPointTy() ? RecurKind::FAdd : RecurKind::Add;
5395 auto *PartialRed = new VPReductionRecipe(
5396 RdxKind,
5397 RdxKind == RecurKind::FAdd ? WidenRecipe->getFastMathFlagsOrNone()
5398 : FastMathFlags(),
5399 WidenRecipe->getUnderlyingInstr(), Accumulator, ExtendedOp, Cond,
5400 RdxUnordered{/*VFScaleFactor=*/Link.ScaleFactor});
5401 PartialRed->insertBefore(InsertPos: WidenRecipe);
5402
5403 if (ExitValue)
5404 ExitValue->replaceAllUsesWith(New: PartialRed);
5405 if (Link.Blend)
5406 Link.Blend->replaceAllUsesWith(New: PartialRed);
5407 WidenRecipe->replaceAllUsesWith(New: PartialRed);
5408
5409 // For cost-model purposes, fold this into a VPExpression.
5410 VPExpressionRecipe *E = createPartialReductionExpression(Red: PartialRed);
5411 E->insertBefore(InsertPos: WidenRecipe);
5412 PartialRed->replaceAllUsesWith(New: E);
5413}
5414
5415/// Returns the cost of a link in a partial-reduction chain for a given VF.
5416static InstructionCost
5417getPartialReductionLinkCost(VPCostContext &CostCtx,
5418 const PartialReductionDescriptor &Link,
5419 ElementCount VF) {
5420 Type *RdxType = Link.ReductionBinOp->getScalarType();
5421 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
5422 std::optional<unsigned> BinOpc = std::nullopt;
5423 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
5424 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
5425 BinOpc = ExtendedOp.ExtendsUser->getOpcode();
5426
5427 std::optional<llvm::FastMathFlags> Flags;
5428 if (RdxType->isFloatingPointTy())
5429 Flags = Link.ReductionBinOp->getFastMathFlagsOrNone();
5430
5431 auto GetLinkOpcode = [&Link]() -> unsigned {
5432 switch (Link.RK) {
5433 case RecurKind::Sub:
5434 return Instruction::Add;
5435 case RecurKind::FSub:
5436 return Instruction::FAdd;
5437 default:
5438 return Link.ReductionBinOp->getOpcode();
5439 }
5440 };
5441
5442 return CostCtx.TTI.getPartialReductionCost(
5443 Opcode: GetLinkOpcode(), InputTypeA: ExtendedOp.ExtendA.SrcType, InputTypeB: ExtendedOp.ExtendB.SrcType,
5444 AccumType: RdxType, VF, OpAExtend: ExtendedOp.ExtendA.Kind, OpBExtend: ExtendedOp.ExtendB.Kind, BinOp: BinOpc,
5445 CostKind: CostCtx.CostKind, FMF: Flags);
5446}
5447
5448/// Checks if \p Op (which is an operand of \p UpdateR) is an extended reduction
5449/// operand. This is an operand where the source of the value (e.g. a load) has
5450/// been extended (sext, zext, or fpext) before it is used in the reduction.
5451///
5452/// Possible forms matched by this function:
5453/// - UpdateR(PrevValue, ext(...))
5454/// - UpdateR(PrevValue, mul(ext(...), ext(...)))
5455/// - UpdateR(PrevValue, mul(ext(...), Constant))
5456/// - UpdateR(PrevValue, ext(mul(ext(...), ext(...))))
5457/// - UpdateR(PrevValue, ext(mul(ext(...), Constant)))
5458/// - UpdateR(PrevValue, abs(sub(ext(...), ext(...)))
5459///
5460/// Note: The second operand of UpdateR corresponds to \p Op in the examples.
5461static std::optional<ExtendedReductionOperand>
5462matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op) {
5463 assert(is_contained(UpdateR->operands(), Op) &&
5464 "Op should be operand of UpdateR");
5465
5466 // Try matching an absolute difference operand of the form
5467 // `abs(sub(ext(A), ext(B)))`. This will be later transformed into
5468 // `ext(absolute-difference(A, B))`. This allows us to perform the absolute
5469 // difference on a wider type and get the extend for "free" from the partial
5470 // reduction.
5471 VPValue *X, *Y;
5472 if (Op->hasOneUse() &&
5473 match(V: Op, P: m_WidenIntrinsic<Intrinsic::abs>(
5474 Ops: m_OneUse(SubPattern: m_Sub(Op0: m_WidenAnyExtend(Op0: m_VPValue(V&: X)),
5475 Op1: m_WidenAnyExtend(Op0: m_VPValue(V&: Y))))))) {
5476 auto *Abs = cast<VPWidenIntrinsicRecipe>(Val: Op);
5477 auto *Sub = cast<VPWidenRecipe>(Val: Abs->getOperand(N: 0));
5478 auto *LHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
5479 auto *RHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 1));
5480 Type *LHSInputType = X->getScalarType();
5481 Type *RHSInputType = Y->getScalarType();
5482 if (LHSInputType != RHSInputType ||
5483 LHSExt->getOpcode() != RHSExt->getOpcode())
5484 return std::nullopt;
5485 // Note: This is essentially the same as matching ext(...) as we will
5486 // rewrite this operand to ext(absolute-difference(A, B)).
5487 return ExtendedReductionOperand{
5488 .ExtendsUser: Sub,
5489 /*ExtendA=*/{.SrcType: LHSInputType, .Kind: getPartialReductionExtendKind(Cast: LHSExt)},
5490 /*ExtendB=*/{}};
5491 }
5492
5493 std::optional<TTI::PartialReductionExtendKind> OuterExtKind;
5494 if (match(V: Op, P: m_WidenAnyExtend(Op0: m_VPValue()))) {
5495 auto *CastRecipe = cast<VPWidenCastRecipe>(Val: Op);
5496 VPValue *CastSource = CastRecipe->getOperand(N: 0);
5497 OuterExtKind = getPartialReductionExtendKind(Cast: CastRecipe);
5498 if (match(V: CastSource, P: m_Mul(Op0: m_VPValue(), Op1: m_VPValue())) ||
5499 match(V: CastSource, P: m_FMul(Op0: m_VPValue(), Op1: m_VPValue()))) {
5500 // Match: ext(mul(...))
5501 // Record the outer extend kind and set `Op` to the mul. We can then match
5502 // this as a binary operation. Note: We can optimize out the outer extend
5503 // by widening the inner extends to match it. See
5504 // optimizeExtendsForPartialReduction.
5505 Op = CastSource;
5506 } else {
5507 return ExtendedReductionOperand{
5508 .ExtendsUser: UpdateR,
5509 /*ExtendA=*/{.SrcType: CastSource->getScalarType(), .Kind: *OuterExtKind},
5510 /*ExtendB=*/{}};
5511 }
5512 }
5513
5514 if (!Op->hasOneUse())
5515 return std::nullopt;
5516
5517 VPWidenRecipe *MulOp = dyn_cast<VPWidenRecipe>(Val: Op);
5518 if (!MulOp ||
5519 !is_contained(Set: {Instruction::Mul, Instruction::FMul}, Element: MulOp->getOpcode()))
5520 return std::nullopt;
5521
5522 // The rest of the matching assumes `Op` is a (possibly extended) mul
5523 // operation.
5524
5525 VPValue *LHS = MulOp->getOperand(N: 0);
5526 VPValue *RHS = MulOp->getOperand(N: 1);
5527
5528 // The LHS of the operation must always be an extend.
5529 if (!match(V: LHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
5530 return std::nullopt;
5531
5532 auto *LHSCast = cast<VPWidenCastRecipe>(Val: LHS);
5533 Type *LHSInputType = LHSCast->getOperand(N: 0)->getScalarType();
5534 ExtendKind LHSExtendKind = getPartialReductionExtendKind(Cast: LHSCast);
5535
5536 // The RHS of the operation can be an extend or a constant integer.
5537 const APInt *RHSConst = nullptr;
5538 VPWidenCastRecipe *RHSCast = nullptr;
5539 if (match(V: RHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
5540 RHSCast = cast<VPWidenCastRecipe>(Val: RHS);
5541 else if (!match(V: RHS, P: m_APInt(C&: RHSConst)) ||
5542 !canConstantBeExtended(C: RHSConst, NarrowType: LHSInputType, ExtKind: LHSExtendKind))
5543 return std::nullopt;
5544
5545 // The outer extend kind must match the inner extends for folding.
5546 for (VPWidenCastRecipe *Cast : {LHSCast, RHSCast})
5547 if (Cast && OuterExtKind &&
5548 getPartialReductionExtendKind(Cast) != OuterExtKind)
5549 return std::nullopt;
5550
5551 Type *RHSInputType = LHSInputType;
5552 ExtendKind RHSExtendKind = LHSExtendKind;
5553 if (RHSCast) {
5554 RHSInputType = RHSCast->getOperand(N: 0)->getScalarType();
5555 RHSExtendKind = getPartialReductionExtendKind(Cast: RHSCast);
5556 }
5557
5558 // The outer extend can only be treated as extending the operands of the mul
5559 // if the narrow mul cannot wrap.
5560 if (OuterExtKind && MulOp->getOpcode() == Instruction::Mul &&
5561 !canFoldOuterExtendIntoMul(Mul: MulOp))
5562 return std::nullopt;
5563
5564 return ExtendedReductionOperand{
5565 .ExtendsUser: MulOp, .ExtendA: {.SrcType: LHSInputType, .Kind: LHSExtendKind}, .ExtendB: {.SrcType: RHSInputType, .Kind: RHSExtendKind}};
5566}
5567
5568/// Examines each operation in the reduction chain corresponding to \p RedPhiR,
5569/// and determines if the target can use a cheaper operation with a wider
5570/// per-iteration input VF and narrower PHI VF. If successful, returns the chain
5571/// of partial reduction descriptors (that are links in the reduction chain).
5572static std::optional<SmallVector<PartialReductionDescriptor>>
5573getScaledReductionChain(VPReductionPHIRecipe *RedPhiR) {
5574 // Get the backedge value from the reduction PHI and find the
5575 // ComputeReductionResult that uses it (directly or through a select for
5576 // predicated reductions).
5577 auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR);
5578 if (!RdxResult)
5579 return std::nullopt;
5580 VPValue *ExitValue = RdxResult->getOperand(N: 0);
5581 match(V: ExitValue, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: ExitValue), Op2: m_VPValue()));
5582
5583 SmallVector<PartialReductionDescriptor> Chain;
5584 RecurKind RK = RedPhiR->getRecurrenceKind();
5585 Type *PhiType = RedPhiR->getScalarType();
5586 TypeSize PHISize = PhiType->getPrimitiveSizeInBits();
5587
5588 // Work backwards from the ExitValue examining each reduction operation.
5589 VPValue *CurrentValue = ExitValue;
5590 while (CurrentValue != RedPhiR) {
5591 VPBlendRecipe *Blend = dyn_cast<VPBlendRecipe>(Val: CurrentValue);
5592 std::optional<unsigned> BlendReductionIdx;
5593 if (Blend) {
5594 assert(!Blend->isNormalized() && "Expect Blend not to be normalized.");
5595 if (Blend->getNumIncomingValues() != 2)
5596 return std::nullopt;
5597
5598 BlendReductionIdx = getBlendReductionUpdateValueIdx(Blend);
5599 if (!BlendReductionIdx)
5600 return std::nullopt;
5601
5602 CurrentValue = Blend->getIncomingValue(Idx: *BlendReductionIdx);
5603 }
5604
5605 auto *UpdateR = dyn_cast<VPWidenRecipe>(Val: CurrentValue);
5606 if (!UpdateR || !Instruction::isBinaryOp(Opcode: UpdateR->getOpcode()))
5607 return std::nullopt;
5608
5609 VPValue *Op = UpdateR->getOperand(N: 1);
5610 VPValue *PrevValue = UpdateR->getOperand(N: 0);
5611
5612 // Find the extended operand. The other operand (PrevValue) is the next link
5613 // in the reduction chain.
5614 std::optional<ExtendedReductionOperand> ExtendedOp =
5615 matchExtendedReductionOperand(UpdateR, Op);
5616 if (!ExtendedOp) {
5617 ExtendedOp = matchExtendedReductionOperand(UpdateR, Op: PrevValue);
5618 if (!ExtendedOp)
5619 return std::nullopt;
5620 std::swap(a&: Op, b&: PrevValue);
5621 }
5622
5623 // Look for VPBlend(reduce(PrevValue, Op), PrevValue), where
5624 // reduce is equal to CurrentValue. This can be lowered as
5625 // a conditional reduction by hoisting the select to the inputs.
5626 if (Blend && Blend->getIncomingValue(Idx: 1 - *BlendReductionIdx) != PrevValue)
5627 return std::nullopt;
5628
5629 Type *ExtSrcType = ExtendedOp->ExtendA.SrcType;
5630 TypeSize ExtSrcSize = ExtSrcType->getPrimitiveSizeInBits();
5631 if (!PHISize.hasKnownScalarFactor(RHS: ExtSrcSize))
5632 return std::nullopt;
5633
5634 PartialReductionDescriptor Link(
5635 {.ReductionBinOp: UpdateR, .ExtendedOp: *ExtendedOp, .RK: RK,
5636 .AccumulatorOpIdx: PrevValue == UpdateR->getOperand(N: 0) ? 0U : 1U,
5637 .ScaleFactor: static_cast<unsigned>(PHISize.getKnownScalarFactor(RHS: ExtSrcSize)),
5638 .Blend: Blend});
5639 Chain.push_back(Elt: Link);
5640 CurrentValue = PrevValue;
5641 }
5642
5643 // The chain links were collected by traversing backwards from the exit value.
5644 // Reverse the chain so the links follow program order.
5645 std::reverse(first: Chain.begin(), last: Chain.end());
5646 return Chain;
5647}
5648} // namespace
5649
5650// Scale the PHI and ReductionStartVector by \p Factor and if the recurrence is
5651// a sub-recurrence, negate the reduction result.
5652static void updatePartialReductionPhiAndResult(VPlan &Plan,
5653 VPReductionPHIRecipe *Phi,
5654 unsigned Factor, RecurKind RK) {
5655 assert(Phi->getVFScaleFactor() == 1 && "scale factor must not be set");
5656 Phi->setVFScaleFactor(Factor);
5657
5658 auto *StartInst = cast<VPInstruction>(Val: Phi->getStartValue());
5659 assert(StartInst->getOpcode() == VPInstruction::ReductionStartVector);
5660 auto *NewScaleFactor = Plan.getConstantInt(BitWidth: 32, Val: Factor);
5661 StartInst->setOperand(I: 2, New: NewScaleFactor);
5662
5663 if (RK != RecurKind::Sub && RK != RecurKind::FSub)
5664 return;
5665
5666 // Update the PHI node to start at `0` and update the reduction-result
5667 // to subtract from the PHI's start value.
5668 VPValue *OldStartValue = StartInst->getOperand(N: 0);
5669 StartInst->setOperand(I: 0, New: StartInst->getOperand(N: 1));
5670
5671 // Replace reduction_result by 'sub (startval, reductionresult)'.
5672 VPInstruction *RdxResult = vputils::findComputeReductionResult(PhiR: Phi);
5673 assert(RdxResult && "Could not find reduction result");
5674
5675 VPBuilder Builder = VPBuilder::getToInsertAfter(R: RdxResult);
5676 unsigned SubOpc = RK == RecurKind::FSub ? Instruction::BinaryOps::FSub
5677 : Instruction::BinaryOps::Sub;
5678 VPInstruction *NewResult = Builder.createNaryOp(
5679 Opcode: SubOpc, Operands: {OldStartValue, RdxResult}, Flags: VPIRFlags::getDefaultFlags(Opcode: SubOpc),
5680 DL: Phi->getDebugLoc());
5681 RdxResult->replaceUsesWithIf(New: NewResult, ShouldReplace: not_equal_to(Arg&: NewResult));
5682}
5683
5684void VPlanTransforms::createPartialReductions(VPlan &Plan,
5685 VPCostContext &CostCtx,
5686 VFRange &Range) {
5687 // Find all possible valid partial reductions, grouping chains by their PHI.
5688 // This grouping allows invalidating the whole chain, if any link is not a
5689 // valid partial reduction.
5690 MapVector<VPReductionPHIRecipe *, SmallVector<PartialReductionDescriptor>>
5691 PhiToChain;
5692 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
5693 SmallVector<VPReductionPHIRecipe *, 4> UnorderedReductions;
5694 for (VPReductionPHIRecipe &RedPhiR :
5695 make_isa_range<VPReductionPHIRecipe>(Range: HeaderVPBB->phis())) {
5696 if (auto Chain = getScaledReductionChain(RedPhiR: &RedPhiR))
5697 PhiToChain.try_emplace(Key: &RedPhiR, Args: std::move(*Chain));
5698 else if (CostCtx.Opts.use_partial_reductions_by_default &&
5699 (RedPhiR.getRecurrenceKind() == RecurKind::Add ||
5700 (RedPhiR.getRecurrenceKind() == RecurKind::FAdd &&
5701 !RedPhiR.isOrdered() && !RedPhiR.isInLoop())))
5702 UnorderedReductions.push_back(Elt: &RedPhiR);
5703 }
5704
5705 // For general unordered reductions which aren't part of a candidate chain for
5706 // a scaled partial reduction, we can still use the intrinsic to allow for
5707 // more optimization later on.
5708 for (auto *Rdx : UnorderedReductions) {
5709 auto *Backedge = dyn_cast<VPWidenRecipe>(Val: Rdx->getBackedgeValue());
5710 VPValue *OtherOp;
5711 if (!Backedge ||
5712 !match(V: Backedge,
5713 P: m_CombineOr(Ps: m_c_FAdd(Op0: m_Specific(VPV: Rdx), Op1: m_VPValue(V&: OtherOp)),
5714 Ps: m_c_Add(Op0: m_Specific(VPV: Rdx), Op1: m_VPValue(V&: OtherOp)))))
5715 continue;
5716
5717 // If the target indicates that the intrinsic is as cheap as (or cheaper
5718 // than) the add, then prefer the intrinsic.
5719 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
5720 Predicate: [&CostCtx, Rdx, Backedge](ElementCount VF) {
5721 InstructionCost CurrentCost = Backedge->computeCost(VF, Ctx&: CostCtx);
5722 Type *ScalarTy = Backedge->getScalarType();
5723 auto FMF = ScalarTy->isFloatingPointTy()
5724 ? std::make_optional(t: Rdx->getFastMathFlagsOrNone())
5725 : std::nullopt;
5726
5727 InstructionCost PRCost = CostCtx.TTI.getPartialReductionCost(
5728 Opcode: Backedge->getOpcode(), InputTypeA: ScalarTy, /*InputTypeB=*/nullptr,
5729 AccumType: ScalarTy, VF, OpAExtend: TTI::PR_None, OpBExtend: TTI::PR_None,
5730 /*BinOp=*/std::nullopt, CostKind: CostCtx.CostKind, FMF);
5731 return PRCost <= CurrentCost;
5732 },
5733 Range))
5734 continue;
5735
5736 auto *Partial = new VPReductionRecipe(
5737 Rdx->getRecurrenceKind(), Rdx->getFastMathFlagsOrNone(),
5738 Backedge->getUnderlyingInstr(), Rdx, OtherOp, nullptr,
5739 getReductionStyle(/*InLoop=*/false, /*Ordered=*/false,
5740 /*ScaleFactor=*/1));
5741 Partial->insertBefore(InsertPos: Backedge);
5742 Backedge->replaceAllUsesWith(New: Partial);
5743 Backedge->eraseFromParent();
5744 }
5745
5746 if (PhiToChain.empty())
5747 return;
5748
5749 // Build set of partial reduction operations and blends for user validation
5750 // and a map of reduction bin ops to their scale factors for scale validation.
5751 SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
5752 SmallPtrSet<VPBlendRecipe *, 4> PartialReductionBlends;
5753 DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
5754 for (auto &[_, Chain] : PhiToChain)
5755 for (const PartialReductionDescriptor &Link : Chain) {
5756 PartialReductionOps.insert(Ptr: Link.ExtendedOp.ExtendsUser);
5757 if (Link.Blend)
5758 PartialReductionBlends.insert(Ptr: Link.Blend);
5759 ScaledReductionMap[Link.ReductionBinOp] = Link.ScaleFactor;
5760 }
5761
5762 // A partial reduction is invalid if any of its extends are used by
5763 // something that isn't another partial reduction. This is because the
5764 // extends are intended to be lowered along with the reduction itself.
5765 auto ExtendUsersValid = [&](VPValue *Ext) {
5766 return !isa<VPWidenCastRecipe>(Val: Ext) || all_of(Range: Ext->users(), P: [&](VPUser *U) {
5767 return PartialReductionOps.contains(Ptr: cast<VPRecipeBase>(Val: U));
5768 });
5769 };
5770
5771 auto IsProfitablePartialReductionChainForVF =
5772 [&](ArrayRef<PartialReductionDescriptor> Chain, ElementCount VF) -> bool {
5773 InstructionCost PartialCost = 0, RegularCost = 0;
5774
5775 // The chain is a profitable partial reduction chain if the cost of handling
5776 // the entire chain is cheaper when using partial reductions than when
5777 // handling the entire chain using regular reductions.
5778 for (const PartialReductionDescriptor &Link : Chain) {
5779 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
5780 InstructionCost LinkCost = getPartialReductionLinkCost(CostCtx, Link, VF);
5781 if (!LinkCost.isValid())
5782 return false;
5783
5784 PartialCost += LinkCost;
5785 RegularCost += Link.ReductionBinOp->computeCost(VF, Ctx&: CostCtx);
5786 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
5787 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
5788 RegularCost += ExtendedOp.ExtendsUser->computeCost(VF, Ctx&: CostCtx);
5789 for (VPValue *Op : ExtendedOp.ExtendsUser->operands())
5790 if (auto *Extend = dyn_cast<VPWidenCastRecipe>(Val: Op))
5791 RegularCost += Extend->computeCost(VF, Ctx&: CostCtx);
5792 }
5793 return PartialCost.isValid() && PartialCost < RegularCost;
5794 };
5795
5796 // Validate chains: check that extends are only used by partial reductions,
5797 // and that reduction bin ops are only used by other partial reductions with
5798 // matching scale factors, are outside the loop region or the select
5799 // introduced by tail-folding. Otherwise we would create users of scaled
5800 // reductions where the types of the other operands don't match.
5801 for (auto &[RedPhiR, Chain] : PhiToChain) {
5802 for (const PartialReductionDescriptor &Link : Chain) {
5803 if (!all_of(Range: Link.ExtendedOp.ExtendsUser->operands(), P: ExtendUsersValid)) {
5804 Chain.clear();
5805 break;
5806 }
5807 auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
5808 if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: U))
5809 return PhiR == RedPhiR;
5810 auto *R = cast<VPSingleDefRecipe>(Val: U);
5811
5812 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: R))
5813 return Blend == Link.Blend || PartialReductionBlends.contains(Ptr: Blend);
5814
5815 return Link.ScaleFactor == ScaledReductionMap.lookup_or(Val: R, Default: 0) ||
5816 match(R, P: m_ComputeReductionResult(
5817 Op0: m_Specific(VPV: Link.ReductionBinOp))) ||
5818 match(R, P: m_Select(Op0: m_VPValue(), Op1: m_Specific(VPV: Link.ReductionBinOp),
5819 Op2: m_Specific(VPV: RedPhiR)));
5820 };
5821 if (!all_of(Range: Link.ReductionBinOp->users(), P: UseIsValid)) {
5822 Chain.clear();
5823 break;
5824 }
5825
5826 // Check if the compute-reduction-result is used by a sunk store.
5827 // TODO: Also form partial reductions in those cases.
5828 if (auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR)) {
5829 if (any_of(Range: RdxResult->users(), P: [](VPUser *U) {
5830 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
5831 return RepR && RepR->getOpcode() == Instruction::Store;
5832 })) {
5833 Chain.clear();
5834 break;
5835 }
5836 }
5837 }
5838
5839 // Clear the chain if it is not profitable.
5840 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
5841 Predicate: [&, &Chain = Chain](ElementCount VF) {
5842 return IsProfitablePartialReductionChainForVF(Chain, VF);
5843 },
5844 Range))
5845 Chain.clear();
5846 }
5847
5848 for (auto &[Phi, Chain] : PhiToChain) {
5849 if (Chain.empty())
5850 continue;
5851
5852 for (const PartialReductionDescriptor &Link : Chain)
5853 transformToPartialReduction(Link, Plan, RdxPhi: Phi);
5854
5855 // After transforming all links in the chain, the PHI node and result need
5856 // updating. Note that we can pick any link in the chain for this, as the
5857 // ScaleFactor and RecurKind must match for all links in the chain.
5858 const PartialReductionDescriptor &Link = Chain[0];
5859 updatePartialReductionPhiAndResult(Plan, Phi, Factor: Link.ScaleFactor, RK: Link.RK);
5860 }
5861}
5862
5863void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
5864 VPRecipeBuilder &RecipeBuilder,
5865 VPCostContext &CostCtx) {
5866 // Collect all loads/stores first. We will start with ones having simpler
5867 // decisions followed by more complex ones that are potentially
5868 // guided/dependent on the simpler ones.
5869 SmallVector<VPInstruction *> MemOps;
5870 for (VPBasicBlock *VPBB :
5871 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
5872 G: Plan.getVectorLoopRegion()->getEntryBasicBlock()))) {
5873 for (VPInstruction &VPI : make_isa_range<VPInstruction>(Range&: *VPBB)) {
5874 if (VPI.getUnderlyingValue() &&
5875 is_contained(Set: {Instruction::Load, Instruction::Store},
5876 Element: VPI.getOpcode()))
5877 MemOps.push_back(Elt: &VPI);
5878 }
5879 }
5880
5881 // Few helpers to process different kinds of memory operations.
5882
5883 // To be used as argument to `VPlanTransforms::runPass` which explicitly
5884 // specified pass name, hence `VPlan &` parameter.
5885 auto ProcessSubset = [&](VPlan &, auto ProcessVPInst) {
5886 SmallVector<VPInstruction *> RemainingMemOps;
5887 for (VPInstruction *VPI : MemOps) {
5888 if (!ProcessVPInst(VPI))
5889 RemainingMemOps.push_back(Elt: VPI);
5890 }
5891
5892 MemOps.clear();
5893 std::swap(LHS&: MemOps, RHS&: RemainingMemOps);
5894 };
5895
5896 auto ReplaceWith = [&](VPInstruction *VPI, VPRecipeBase *New) {
5897 assert(New->getParent() && "New recipe must have been inserted");
5898 if (VPI->getOpcode() == Instruction::Load)
5899 VPI->replaceAllUsesWith(New: New->getVPSingleValue());
5900 VPI->eraseFromParent();
5901
5902 // VPI has been processed.
5903 return true;
5904 };
5905
5906 auto Scalarize = [&](VPInstruction *VPI) {
5907 return ReplaceWith(VPI, VPBuilder(VPI).insert(
5908 R: RecipeBuilder.handleReplication(VPI, Range)));
5909 };
5910
5911 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
5912 VPBuilder FinalRedStoresBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
5913 VPlanTransforms::runPass(
5914 PassName: "lowerMemoryIdioms", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5915 if (RecipeBuilder.replaceWithFinalIfReductionStore(
5916 VPI, FinalRedStoresBuilder))
5917 return true;
5918
5919 // Filter out scalar VPlan for the remaining idioms.
5920 if (LoopVectorizationPlanner::getDecisionAndClampRange(
5921 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
5922 return false;
5923
5924 if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI))
5925 return ReplaceWith(VPI, VPBuilder(VPI).insert(R: Histogram));
5926
5927 return false;
5928 });
5929
5930 // Filter out scalar VPlan for the remaining memory operations.
5931 if (LoopVectorizationPlanner::getDecisionAndClampRange(
5932 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
5933 return;
5934
5935 // If the instruction's allocated size doesn't equal it's type size, it
5936 // requires padding and will be scalarized.
5937 VPlanTransforms::runPass(
5938 PassName: "scalarizeMemOpsWithIrregularTypes", Pass&: ProcessSubset, Plan,
5939 Args: [&](VPInstruction *VPI) {
5940 Instruction *I = VPI->getUnderlyingInstr();
5941 if (hasIrregularType(Ty: getLoadStoreType(I), DL: I->getDataLayout()))
5942 return Scalarize(VPI);
5943
5944 return false;
5945 });
5946
5947 if (!RecipeBuilder.prefersVectorizedAddressing()) {
5948 VPlanTransforms::runPass(
5949 PassName: "makeVPlanMemOpDecision", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5950 Instruction *I = VPI->getUnderlyingInstr();
5951 bool IsLoad = VPI->getOpcode() == Instruction::Load;
5952 if (RecipeBuilder.isPredicatedInst(I) || !IsLoad ||
5953 !vputils::isUsedByLoadStoreAddress(V: VPI))
5954 return false;
5955
5956 // Scalarize loads used as addresses, matching the legacy CM. The load
5957 // is single-scalar if the pointer is loop-invariant, otherwise it is
5958 // replicated per-lane. No mask is needed as the load is not
5959 // predicated.
5960 VPValue *Ptr = VPI->getOperand(N: 0);
5961 const SCEV *PtrSCEV =
5962 vputils::getSCEVExprForVPValue(V: Ptr, PSE&: CostCtx.PSE, L: CostCtx.L);
5963 bool IsSingleScalarLoad =
5964 !isa<SCEVCouldNotCompute>(Val: PtrSCEV) &&
5965 CostCtx.PSE.getSE()->isLoopInvariant(S: PtrSCEV, L: CostCtx.L);
5966
5967 ReplaceWith(VPI,
5968 VPBuilder(VPI).insert(R: new VPReplicateRecipe(
5969 I, Ptr, /*IsSingleScalar=*/IsSingleScalarLoad,
5970 /*Mask=*/nullptr, *VPI, *VPI, VPI->getDebugLoc())));
5971 return true;
5972 });
5973 }
5974
5975 // Widen unit-stride consecutive accesses, matching the legacy CM. Both
5976 // forward (stride +1) and reverse (stride -1) accesses are handled.
5977 VPlanTransforms::runPass(
5978 PassName: "widenConsecutiveMemOps", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
5979 Instruction *I = VPI->getUnderlyingInstr();
5980 bool IsLoad = VPI->getOpcode() == Instruction::Load;
5981 VPValue *Ptr = VPI->getOperand(N: !IsLoad);
5982 Type *ScalarTy =
5983 IsLoad ? VPI->getScalarType() : VPI->getOperand(N: 0)->getScalarType();
5984 std::optional<int64_t> Stride =
5985 vputils::getConstantStride(Addr: Ptr, AccessTy: ScalarTy, PSE&: CostCtx.PSE, L: CostCtx.L);
5986 if (Stride != 1 && Stride != -1)
5987 return false;
5988 bool Reverse = Stride == -1;
5989
5990 // A predicated access can only be widened (rather than scalarized) if
5991 // the target supports a masked load/store for it.
5992 // TODO: Determine if a load/store needs predication directly in VPlan.
5993 bool IsPredicated = RecipeBuilder.isPredicatedInst(I);
5994 if (IsPredicated && !CostCtx.Config.isLegalMaskedLoadOrStore(
5995 IsLoad, ScalarTy, Alignment: getLoadStoreAlignment(I),
5996 AddressSpace: getLoadStoreAddressSpace(I)))
5997 return false;
5998
5999 VPBuilder Builder(VPI);
6000 VPSingleDefRecipe *VectorPtr = Builder.createConsecutiveVectorPointer(
6001 Ptr, SourceElementTy: ScalarTy, Reverse, DL: VPI->getDebugLoc());
6002
6003 VPValue *Mask = IsPredicated ? VPI->getMask() : nullptr;
6004 // Reverse the mask so it matches the reversed access order.
6005 if (Reverse && Mask)
6006 Mask = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Mask,
6007 DL: VPI->getDebugLoc());
6008
6009 if (IsLoad) {
6010 VPSingleDefRecipe *Load = Builder.createWidenLoad(
6011 Load&: *cast<LoadInst>(Val: I), Addr: VectorPtr, Mask,
6012 /*Consecutive=*/true, Metadata: *VPI, DL: VPI->getDebugLoc());
6013 // Reverse the loaded values back into program order.
6014 if (Reverse)
6015 Load = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Load,
6016 DL: VPI->getDebugLoc());
6017 return ReplaceWith(VPI, Load);
6018 }
6019
6020 VPValue *StoredVal = VPI->getOperand(N: 0);
6021 if (Reverse)
6022 // Reverse the stored values so they are written in descending order.
6023 StoredVal = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: StoredVal,
6024 DL: VPI->getDebugLoc());
6025
6026 auto *StoreR = Builder.createWidenStore(
6027 Store&: *cast<StoreInst>(Val: I), Addr: VectorPtr, StoredVal, Mask,
6028 /*Consecutive=*/true, Metadata: *VPI, DL: VPI->getDebugLoc());
6029 return ReplaceWith(VPI, StoreR);
6030 });
6031
6032 VPlanTransforms::runPass(PassName: "delegateMemOpWideningToLegacyCM", Pass&: ProcessSubset,
6033 Plan, Args: [&](VPInstruction *VPI) {
6034 if (VPRecipeBase *Recipe =
6035 RecipeBuilder.tryToWidenMemory(VPI, Range))
6036 return ReplaceWith(VPI, Recipe);
6037
6038 return Scalarize(VPI);
6039 });
6040}
6041
6042void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
6043 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6044 Predicate: [&](ElementCount VF) { return VF.isScalar(); }, Range))
6045 return;
6046
6047 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
6048 Plan.getEntry());
6049 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
6050 for (VPInstruction &VPI :
6051 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range: reverse(C&: *VPBB)))) {
6052 auto *I = cast_or_null<Instruction>(Val: VPI.getUnderlyingValue());
6053 // Wouldn't be able to create a `VPReplicateRecipe` anyway.
6054 if (!I)
6055 continue;
6056
6057 // If executing other lanes produces side-effects we can't avoid them.
6058 if (VPI.mayHaveSideEffects())
6059 continue;
6060
6061 // ExtractValue takes the shape of its aggregate, decided in
6062 // tryToCreateWidenNonPhiRecipe.
6063 if (VPI.getOpcode() == Instruction::ExtractValue)
6064 continue;
6065
6066 // We want to drop the mask operand, verify we can safely do that.
6067 if (VPI.isMasked() && !VPI.isSafeToSpeculativelyExecute())
6068 continue;
6069
6070 // Avoid rewriting IV increment as that interferes with
6071 // `removeRedundantCanonicalIVs`.
6072 if (VPI.getOpcode() == Instruction::Add &&
6073 any_of(Range: VPI.operands(), P: IsaPred<VPWidenIntOrFpInductionRecipe>))
6074 continue;
6075
6076 // Other lanes are needed - can't drop them.
6077 if (!vputils::onlyFirstLaneUsed(Def: &VPI))
6078 continue;
6079
6080 auto *Recipe = VPBuilder::createSingleScalarOp(
6081 Opcode: VPI.getOpcode(), Operands: VPI.operandsWithoutMask(), /*Mask=*/nullptr, Flags: VPI,
6082 Metadata: VPI, DL: VPI.getDebugLoc(), ResultTy: VPI.getScalarType(), UV: I);
6083 Recipe->insertBefore(InsertPos: &VPI);
6084 VPI.replaceAllUsesWith(New: Recipe);
6085 VPI.eraseFromParent();
6086 }
6087 }
6088}
6089
6090/// Returns true if \p Info's parameter kinds are compatible with \p Args.
6091static bool areVFParamsOk(const VFInfo &Info, ArrayRef<VPValue *> Args,
6092 PredicatedScalarEvolution &PSE, const Loop *L) {
6093 ScalarEvolution *SE = PSE.getSE();
6094 return all_of(Range: Info.Shape.Parameters, P: [&](VFParameter Param) {
6095 switch (Param.ParamKind) {
6096 case VFParamKind::Vector:
6097 case VFParamKind::GlobalPredicate:
6098 return true;
6099 case VFParamKind::OMP_Uniform:
6100 return SE->isSCEVable(Ty: Args[Param.ParamPos]->getScalarType()) &&
6101 SE->isLoopInvariant(
6102 S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
6103 L);
6104 case VFParamKind::OMP_Linear:
6105 return match(S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
6106 P: m_scev_AffineAddRec(
6107 Op0: m_SCEV(), Op1: m_scev_SpecificSInt(V: Param.LinearStepOrPos),
6108 L: m_SpecificLoop(L)));
6109 default:
6110 return false;
6111 }
6112 });
6113}
6114
6115/// Find a vector variant of \p CI for \p VF, respecting \p MaskRequired.
6116/// Returns the variant function, or nullptr. Masked variants are assumed to
6117/// take the mask as a trailing parameter.
6118static Function *findVectorVariant(CallInst *CI, ArrayRef<VPValue *> Args,
6119 ElementCount VF, bool MaskRequired,
6120 PredicatedScalarEvolution &PSE,
6121 const Loop *L) {
6122 if (CI->isNoBuiltin())
6123 return nullptr;
6124 auto Mappings = VFDatabase::getMappings(CI: *CI);
6125 const auto *It = find_if(Range&: Mappings, P: [&](const VFInfo &Info) {
6126 return Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()) &&
6127 areVFParamsOk(Info, Args, PSE, L);
6128 });
6129 if (It == Mappings.end())
6130 return nullptr;
6131 return CI->getModule()->getFunction(Name: It->VectorName);
6132}
6133
6134namespace {
6135/// The outcome of choosing how to widen a call at a given VF.
6136struct CallWideningDecision {
6137 enum class KindTy { Scalarize, Intrinsic, VectorVariant };
6138 CallWideningDecision(KindTy Kind, Function *Variant = nullptr)
6139 : Kind(Kind), Variant(Variant) {}
6140 KindTy Kind;
6141
6142 /// Set when Kind == VectorVariant.
6143 Function *Variant;
6144
6145 bool operator==(const CallWideningDecision &Other) const {
6146 return Kind == Other.Kind && Variant == Other.Variant;
6147 }
6148};
6149} // namespace
6150
6151/// Pick the cheapest widening for the call \p VPI at \p VF among scalarization,
6152/// vector intrinsic, and vector library variant.
6153static CallWideningDecision decideCallWidening(VPInstruction &VPI,
6154 ArrayRef<VPValue *> Ops,
6155 ElementCount VF,
6156 VPCostContext &CostCtx) {
6157 auto *CI = cast<CallInst>(Val: VPI.getUnderlyingInstr());
6158
6159 // Scalar VFs and calls forced or known to scalarize always replicate.
6160 if (VF.isScalar() || CostCtx.willBeScalarized(I: CI, VF))
6161 return CallWideningDecision::KindTy::Scalarize;
6162
6163 auto *CalledFn = cast<Function>(
6164 Val: VPI.getOperand(N: VPI.getNumOperandsWithoutMask() - 1)->getLiveInIRValue());
6165 Type *ResultTy = VPI.getScalarType();
6166 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
6167 bool MaskRequired = CostCtx.isMaskRequired(I: CI);
6168
6169 // Pseudo intrinsics (assume, lifetime, ...) are always scalarized.
6170 if (ID && VPCostContext::isFreeScalarIntrinsic(ID))
6171 return CallWideningDecision::KindTy::Scalarize;
6172
6173 InstructionCost ScalarCost =
6174 VPReplicateRecipe::computeCallCost(CalledFn, ResultTy, ArgOps: Ops,
6175 /*IsSingleScalar=*/false, VF, Ctx&: CostCtx);
6176
6177 Function *VecFunc =
6178 findVectorVariant(CI, Args: Ops, VF, MaskRequired, PSE&: CostCtx.PSE, L: CostCtx.L);
6179 InstructionCost VecCallCost = InstructionCost::getInvalid();
6180 if (VecFunc)
6181 VecCallCost = VPWidenCallRecipe::computeCallCost(Variant: VecFunc, Ctx&: CostCtx);
6182
6183 // Prefer the intrinsic if it is at least as cheap as scalarizing and any
6184 // available vector variant.
6185 if (ID) {
6186 InstructionCost IntrinsicCost =
6187 VPWidenIntrinsicRecipe::computeCallCost(ID, Operands: Ops, R: VPI, VF, Ctx&: CostCtx);
6188 if (IntrinsicCost.isValid() && ScalarCost >= IntrinsicCost &&
6189 (!VecFunc || VecCallCost >= IntrinsicCost))
6190 return CallWideningDecision::KindTy::Intrinsic;
6191 }
6192
6193 // Otherwise, use a vector library variant when it beats scalarizing.
6194 if (VecFunc && ScalarCost >= VecCallCost)
6195 return {CallWideningDecision::KindTy::VectorVariant, VecFunc};
6196
6197 return CallWideningDecision::KindTy::Scalarize;
6198}
6199
6200bool VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
6201 VPRecipeBuilder &RecipeBuilder,
6202 VPCostContext &CostCtx) {
6203 bool Widened = false;
6204 for (VPBasicBlock *VPBB : VPBlockUtils::blocksAs<VPBasicBlock>(
6205 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
6206 for (VPInstruction &VPI :
6207 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range&: *VPBB))) {
6208 if (!VPI.getUnderlyingValue() || VPI.getOpcode() != Instruction::Call)
6209 continue;
6210
6211 auto *CI = cast<CallInst>(Val: VPI.getUnderlyingInstr());
6212 SmallVector<VPValue *, 4> Ops(VPI.op_begin(),
6213 VPI.op_begin() + CI->arg_size());
6214
6215 CallWideningDecision Decision =
6216 decideCallWidening(VPI, Ops, VF: Range.Start, CostCtx);
6217 LoopVectorizationPlanner::getDecisionAndClampRange(
6218 Predicate: [&](ElementCount VF) {
6219 return Decision == decideCallWidening(VPI, Ops, VF, CostCtx);
6220 },
6221 Range);
6222
6223 assert((Decision.Kind == CallWideningDecision::KindTy::Scalarize ||
6224 !CI->hasOperandBundles()) &&
6225 "calls with operand bundles must not be widened");
6226 VPSingleDefRecipe *Replacement = nullptr;
6227 switch (Decision.Kind) {
6228 case CallWideningDecision::KindTy::Intrinsic: {
6229 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
6230 Type *ResultTy = VPI.getScalarType();
6231 Replacement = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, VPI,
6232 VPI, VPI.getDebugLoc());
6233 Widened = true;
6234 break;
6235 }
6236 case CallWideningDecision::KindTy::VectorVariant: {
6237 // Masked variants take the mask as a trailing parameter, so they have
6238 // one more parameter than the original call's arguments.
6239 if (Decision.Variant->arg_size() > Ops.size()) {
6240 VPValue *Mask = VPI.isMasked() ? VPI.getMask() : Plan.getTrue();
6241 Ops.push_back(Elt: Mask);
6242 }
6243 Ops.push_back(Elt: VPI.getOperand(N: VPI.getNumOperandsWithoutMask() - 1));
6244 VPIRAttributes Attrs(*CI, Decision.Variant->getFunctionType(),
6245 VPI.isMasked());
6246 Replacement = new VPWidenCallRecipe(CI, Decision.Variant, Ops, VPI, VPI,
6247 Attrs, VPI.getDebugLoc());
6248 Widened = true;
6249 break;
6250 }
6251 case CallWideningDecision::KindTy::Scalarize:
6252 Replacement = RecipeBuilder.handleReplication(VPI: &VPI, Range);
6253 break;
6254 }
6255
6256 Replacement->insertBefore(InsertPos: &VPI);
6257 VPI.replaceAllUsesWith(New: Replacement);
6258 VPI.eraseFromParent();
6259 }
6260 }
6261 return Widened;
6262}
6263
6264void VPlanTransforms::narrowInductionTruncates(VPlan &Plan, VFRange &Range,
6265 const TargetTransformInfo &TTI,
6266 PredicatedScalarEvolution &PSE) {
6267 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
6268 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
6269 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6270 Range: vp_depth_first_shallow(G: LoopRegion->getEntry()))) {
6271 for (VPInstruction &VPI :
6272 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range&: *VPBB))) {
6273 // Only truncates are handled, as sext/zext may wrap, FP conversions lose
6274 // precision and other casts depend on the pointer size.
6275 if (VPI.getOpcode() != Instruction::Trunc)
6276 continue;
6277
6278 VPValue *Op = VPI.getOperand(N: 0);
6279 auto *WideIV = getOptimizableIVOf(VPV: Op, PSE);
6280 if (!WideIV)
6281 continue;
6282
6283 // getOptimizableIVOf also matches an add of the IV and its step, which
6284 // is not handled here.
6285 // TODO: Also narrow truncates of the incremented IV.
6286 if (Op != WideIV)
6287 continue;
6288
6289 // A truncate that is not widened is left to the scalarization decisions
6290 // made earlier.
6291 if (vputils::onlyFirstLaneUsed(Def: &VPI))
6292 continue;
6293
6294 // Replacing a free truncate would add an induction update instruction to
6295 // each iteration of the loop. The canonical induction is exempt, as it
6296 // needs an update instruction regardless.
6297 auto IsNarrowingProfitable = [&](ElementCount VF) {
6298 return match(V: WideIV, P: m_CanonicalWidenIV()) ||
6299 !TTI.isTruncateFree(
6300 Ty1: toVectorTy(Scalar: VPI.getOperand(N: 0)->getScalarType(), EC: VF),
6301 Ty2: toVectorTy(Scalar: VPI.getScalarType(), EC: VF));
6302 };
6303 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
6304 Predicate: IsNarrowingProfitable, Range))
6305 continue;
6306
6307 // Wrap flags of the original induction do not hold in the truncated
6308 // type, so do not propagate them.
6309 auto *NarrowIV = new VPWidenIntOrFpInductionRecipe(
6310 WideIV->getPHINode(), WideIV->getStartValue(), WideIV->getStepValue(),
6311 WideIV->getVFValue(), WideIV->getInductionDescriptor(),
6312 VPI.getScalarType(), VPIRFlags::WrapFlagsTy(false, false),
6313 VPI.getDebugLoc());
6314 NarrowIV->insertBefore(BB&: *HeaderVPBB, IP: HeaderVPBB->getFirstNonPhi());
6315 VPI.replaceAllUsesWith(New: NarrowIV);
6316 VPI.eraseFromParent();
6317 }
6318 }
6319}
6320
6321void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
6322 PredicatedScalarEvolution &PSE,
6323 Loop &L, VPCostContext &Ctx,
6324 VFRange &Range) {
6325 if (Plan.hasScalarVFOnly())
6326 return;
6327
6328 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
6329 VPValue *I32VF = nullptr;
6330 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6331 Range: vp_depth_first_shallow(G: VectorLoop->getEntry()))) {
6332 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
6333 auto *MemR = dyn_cast<VPWidenMemoryRecipe>(Val: &R);
6334 // TODO: Transform reverse access into strided access with -1 stride.
6335 // TODO: Transform gather/scatter with uniform address into strided access
6336 // with 0 stride.
6337 // TODO: Transform interleave access into multiple strided accesses.
6338 if (!MemR || MemR->isConsecutive())
6339 continue;
6340
6341 VPValue *Ptr = MemR->getAddr();
6342 // Check if this is a strided access by analyzing the address SCEV for an
6343 // affine addRec.
6344 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: &L);
6345 const SCEV *Start;
6346 const SCEVConstant *Step;
6347 // TODO: Support non-constant loop invariant stride.
6348 if (!match(S: PtrSCEV,
6349 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEVConstant(V&: Step),
6350 L: m_SpecificLoop(L: &L))))
6351 continue;
6352
6353 VPValue *StoredValue = nullptr;
6354 Type *DataTy;
6355 Intrinsic::ID IntrinID;
6356 if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Val: &R)) {
6357 StoredValue = StoreR->getStoredValue();
6358 DataTy = StoredValue->getScalarType();
6359 IntrinID = Intrinsic::experimental_vp_strided_store;
6360 } else {
6361 auto *LoadR = cast<VPWidenLoadRecipe>(Val: &R);
6362 DataTy = LoadR->getScalarType();
6363 IntrinID = Intrinsic::experimental_vp_strided_load;
6364 }
6365
6366 Align Alignment = MemR->getAlign();
6367 auto IsProfitable = [&](ElementCount VF) {
6368 Type *VectorTy = toVectorTy(Scalar: DataTy, EC: VF);
6369 if (!Ctx.TTI.isLegalStridedLoadStore(DataType: VectorTy, Alignment))
6370 return false;
6371 const InstructionCost CurrentCost = MemR->computeCost(VF, Ctx);
6372 const InstructionCost StridedLoadStoreCost =
6373 VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost(
6374 IID: IntrinID, Ty: VectorTy, IsMasked: MemR->isMasked(), Alignment, Ctx);
6375 return StridedLoadStoreCost < CurrentCost;
6376 };
6377
6378 if (!LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsProfitable,
6379 Range))
6380 continue;
6381
6382 // Invalidate the legacy widening decision so the cost of replaced load is
6383 // not counted during precomputeCosts.
6384 // TODO: Remove once the legacy exit cost computation is retired.
6385 for (ElementCount VF : Range)
6386 Ctx.invalidateWideningDecision(I: &MemR->getIngredient(), VF);
6387
6388 // Get VF as i32 for the vector length operand.
6389 if (!I32VF) {
6390 VPBuilder Builder(Plan.getVectorPreheader());
6391 I32VF = Builder.createScalarZExtOrTrunc(
6392 Op: &Plan.getVF(), ResultTy: Type::getInt32Ty(C&: Plan.getContext()),
6393 DL: DebugLoc::getUnknown());
6394 }
6395
6396 VPBuilder Builder(&R);
6397 // Create the base pointer of strided access.
6398 // TODO: reuse VPDerivedIVRecipe for base pointer computation when it
6399 // supports a general VPValue as the start value.
6400 VPValue *StartVPV =
6401 VPSCEVExpander(Builder, *PSE.getSE(), R.getDebugLoc()).expand(S: Start);
6402 VPValue *StrideInBytes = Plan.getOrAddLiveIn(V: Step->getValue());
6403 Type *IndexTy = Plan.getDataLayout().getIndexType(PtrTy: Ptr->getScalarType());
6404 assert(IndexTy == StrideInBytes->getScalarType() &&
6405 "Stride type from SCEV must match the index type");
6406 VPValue *CanIV = Builder.createScalarZExtOrTrunc(
6407 Op: VectorLoop->getCanonicalIV(), ResultTy: IndexTy, DL: DebugLoc::getUnknown());
6408 auto *AddRecPtr = cast<SCEVAddRecExpr>(Val: PtrSCEV);
6409 auto *Offset = Builder.createOverflowingOp(
6410 Opcode: Instruction::Mul, Operands: {CanIV, StrideInBytes},
6411 WrapFlags: {AddRecPtr->hasNoUnsignedWrap(), /*HasNSW=*/false});
6412 GEPNoWrapFlags NWFlags = AddRecPtr->hasNoUnsignedWrap()
6413 ? GEPNoWrapFlags::noUnsignedWrap()
6414 : GEPNoWrapFlags::none();
6415 VPValue *BasePtr = Builder.createNoWrapPtrAdd(Ptr: StartVPV, Offset, GEPFlags: NWFlags);
6416
6417 // Create a new vector pointer for strided access.
6418 VPValue *NewPtr = Builder.createVectorPointer(
6419 Ptr: BasePtr, SourceElementTy: Type::getInt8Ty(C&: Plan.getContext()), Stride: StrideInBytes, GEPFlags: NWFlags,
6420 DL: R.getDebugLoc());
6421
6422 VPValue *Mask = MemR->getMask();
6423 if (!Mask)
6424 Mask = Plan.getTrue();
6425 SmallVector<VPValue *, 5> Ops;
6426 if (StoredValue)
6427 Ops.push_back(Elt: StoredValue);
6428 Ops.append(IL: {NewPtr, StrideInBytes, Mask, I32VF});
6429
6430 auto *StridedR = Builder.createWidenMemIntrinsic(
6431 VectorIntrinsicID: IntrinID, CallArguments: Ops,
6432 Ty: StoredValue ? Type::getVoidTy(C&: Plan.getContext()) : DataTy, Alignment,
6433 MD: *MemR, DL: R.getDebugLoc());
6434 if (!StoredValue)
6435 cast<VPWidenLoadRecipe>(Val: &R)->replaceAllUsesWith(New: StridedR);
6436 R.eraseFromParent();
6437 }
6438 }
6439}
6440