1//===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements a set of utility VPlan to VPlan transformations.
11///
12//===----------------------------------------------------------------------===//
13
14#include "VPlanTransforms.h"
15#include "VPRecipeBuilder.h"
16#include "VPlan.h"
17#include "VPlanAnalysis.h"
18#include "VPlanCFG.h"
19#include "VPlanDominatorTree.h"
20#include "VPlanHelpers.h"
21#include "VPlanPatternMatch.h"
22#include "VPlanUtils.h"
23#include "VectorizeOptions.h"
24#include "llvm/ADT/APInt.h"
25#include "llvm/ADT/PostOrderIterator.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/SetVector.h"
28#include "llvm/ADT/SmallPtrSet.h"
29#include "llvm/ADT/TypeSwitch.h"
30#include "llvm/Analysis/IVDescriptors.h"
31#include "llvm/Analysis/Loads.h"
32#include "llvm/Analysis/LoopInfo.h"
33#include "llvm/Analysis/MemoryLocation.h"
34#include "llvm/Analysis/ScalarEvolutionPatternMatch.h"
35#include "llvm/Analysis/ScopedNoAliasAA.h"
36#include "llvm/Analysis/VectorUtils.h"
37#include "llvm/IR/Intrinsics.h"
38#include "llvm/IR/MDBuilder.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/ProfDataUtils.h"
41#include "llvm/Support/Casting.h"
42#include "llvm/Support/Regex.h"
43#include "llvm/Support/TypeSize.h"
44#include "llvm/Transforms/Utils/LoopUtils.h"
45
46using namespace llvm;
47using namespace LoopVectorizationUtils;
48using namespace VPlanPatternMatch;
49using namespace SCEVPatternMatch;
50
51void VPlanTransforms::verifyIfRequested(VPlan &Plan) {
52 const VectorizeOptions &Opts = getVectorizeOptions(C: Plan.getContext());
53 if (Opts.vplan_verify_each && !verifyVPlanIsValid(Plan))
54 report_fatal_error(reason: "Broken VPlan found, compilation aborted!");
55}
56
57#ifndef NDEBUG
58bool VPlanTransforms::isPrintingRequested(const VPlan &Plan) {
59 const VectorizeOptions &Opts = getVectorizeOptions(Plan.getContext());
60 return Opts.vplan_print_before_all || Opts.vplan_print_after_all ||
61 !Opts.vplan_print_before.empty() || !Opts.vplan_print_after.empty();
62}
63
64void VPlanTransforms::printIfRequested(VPlan &Plan, Function *Fn,
65 StringRef NumberedPassName, bool After) {
66 const VectorizeOptions &Opts = getVectorizeOptions(Plan.getContext());
67 bool All = After ? Opts.vplan_print_after_all : Opts.vplan_print_before_all;
68 ArrayRef<StringRef> Passes =
69 After ? Opts.vplan_print_after : Opts.vplan_print_before;
70 if (!All && none_of(Passes, [&](StringRef Entry) {
71 return Regex(Entry).match(NumberedPassName);
72 }))
73 return;
74 dbgs() << "VPlan for loop in '" << Fn->getName() << "' "
75 << (After ? "after" : "before") << " " << NumberedPassName << '\n';
76 if (Opts.vplan_print_vector_region_scope && Plan.getVectorLoopRegion())
77 Plan.getVectorLoopRegion()->print(dbgs());
78 else
79 dbgs() << Plan << '\n';
80}
81#endif
82
83/// Returns the metadata attached to \p R, or an empty set for a recipe that
84/// does not carry any.
85static VPIRMetadata getMetadataOf(VPRecipeBase *R) {
86 if (auto *MD = dyn_cast<VPIRMetadata>(Val: R))
87 return *MD;
88 return {};
89}
90
91bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes(
92 VPlan &Plan, const TargetLibraryInfo &TLI, PredicatedScalarEvolution &PSE,
93 Loop *OuterLoop) {
94
95 // Returns true if the access of \p AccessTy at \p Addr can be widened to a
96 // consecutive vector access.
97 auto IsConsecutiveAccess = [&](VPValue *Addr, Type *AccessTy) {
98 return !hasIrregularType(Ty: AccessTy, DL: Plan.getDataLayout()) &&
99 vputils::getConstantStride(Addr, AccessTy, PSE, L: OuterLoop) == 1;
100 };
101
102 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
103 Plan.getVectorLoopRegion());
104 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
105 // Skip blocks outside region
106 if (!VPBB->getParent())
107 break;
108 VPRecipeBase *Term = VPBB->getTerminator();
109 auto EndIter = Term ? Term->getIterator() : VPBB->end();
110 // Introduce each ingredient into VPlan.
111 for (VPRecipeBase &Ingredient :
112 make_early_inc_range(Range: make_range(x: VPBB->begin(), y: EndIter))) {
113
114 VPValue *VPV = Ingredient.getVPSingleValue();
115 if (!VPV->getUnderlyingValue())
116 continue;
117
118 Instruction *Inst = cast<Instruction>(Val: VPV->getUnderlyingValue());
119
120 VPRecipeBase *NewRecipe = nullptr;
121 if (auto *PhiR = dyn_cast<VPPhi>(Val: &Ingredient)) {
122 auto *Phi = cast<PHINode>(Val: PhiR->getUnderlyingValue());
123 NewRecipe = new VPWidenPHIRecipe(PhiR->operands(), PhiR->getDebugLoc(),
124 Phi->getName());
125 } else if (auto *VPI = dyn_cast<VPInstruction>(Val: &Ingredient)) {
126 assert(!isa<PHINode>(Inst) && "phis should be handled above");
127 // Create VPWidenMemoryRecipe for loads and stores.
128 if (LoadInst *Load = dyn_cast<LoadInst>(Val: Inst)) {
129 bool IsConsecutive =
130 IsConsecutiveAccess(VPI->getOperand(N: 0), VPI->getScalarType());
131 NewRecipe = new VPWidenLoadRecipe(
132 *Load, Ingredient.getOperand(N: 0), nullptr /*Mask*/, IsConsecutive,
133 Load->getAlign(), *VPI, Ingredient.getDebugLoc());
134 } else if (StoreInst *Store = dyn_cast<StoreInst>(Val: Inst)) {
135 bool IsConsecutive = IsConsecutiveAccess(
136 VPI->getOperand(N: 1), VPI->getOperand(N: 0)->getScalarType());
137 NewRecipe = new VPWidenStoreRecipe(
138 *Store, Ingredient.getOperand(N: 1), Ingredient.getOperand(N: 0),
139 nullptr /*Mask*/, IsConsecutive, Store->getAlign(), *VPI,
140 Ingredient.getDebugLoc());
141 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Val: Inst)) {
142 NewRecipe = new VPWidenGEPRecipe(GEP->getSourceElementType(),
143 Ingredient.operands(), *VPI,
144 Ingredient.getDebugLoc(), GEP);
145 } else if (CallInst *CI = dyn_cast<CallInst>(Val: Inst)) {
146 Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, TLI: &TLI);
147 if (VectorID == Intrinsic::not_intrinsic)
148 return false;
149
150 // The noalias.scope.decl intrinsic declares a noalias scope that
151 // is valid for a single iteration. Emitting it as a single-scalar
152 // replicate would incorrectly extend the scope across multiple
153 // original iterations packed into one vector iteration.
154 // FIXME: If we want to vectorize this loop, then we have to drop
155 // all the associated !alias.scope and !noalias.
156 if (VectorID == Intrinsic::experimental_noalias_scope_decl)
157 return false;
158
159 // These intrinsics are recognized by getVectorIntrinsicIDForCall
160 // but are not widenable. Emit them as replicate instead of widening.
161 if (VectorID == Intrinsic::assume ||
162 VectorID == Intrinsic::lifetime_end ||
163 VectorID == Intrinsic::lifetime_start ||
164 VectorID == Intrinsic::sideeffect ||
165 VectorID == Intrinsic::pseudoprobe) {
166 // If the operand of llvm.assume holds before vectorization, it will
167 // also hold per lane.
168 // llvm.pseudoprobe requires to be duplicated per lane for accurate
169 // sample count.
170 const bool IsSingleScalar = VectorID != Intrinsic::assume &&
171 VectorID != Intrinsic::pseudoprobe;
172 NewRecipe = new VPReplicateRecipe(CI, Ingredient.operands(),
173 /*IsSingleScalar=*/IsSingleScalar,
174 /*Mask=*/nullptr, *VPI, *VPI,
175 Ingredient.getDebugLoc());
176 } else {
177 assert(!CI->hasOperandBundles() &&
178 "calls with operand bundles must not be widened");
179 NewRecipe = new VPWidenIntrinsicRecipe(
180 *CI, VectorID, drop_end(RangeOrContainer: Ingredient.operands()), CI->getType(),
181 VPIRFlags(*CI), *VPI, CI->getDebugLoc());
182 }
183 } else if (auto *CI = dyn_cast<CastInst>(Val: Inst)) {
184 NewRecipe = new VPWidenCastRecipe(
185 CI->getOpcode(), Ingredient.getOperand(N: 0), CI->getType(), CI,
186 VPIRFlags(*CI), VPIRMetadata(*CI));
187 } else {
188 NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI,
189 *VPI, Ingredient.getDebugLoc());
190 }
191 } else {
192 assert(isa<VPWidenInductionRecipe>(&Ingredient) &&
193 "inductions must be created earlier");
194 continue;
195 }
196
197 NewRecipe->insertBefore(InsertPos: &Ingredient);
198 if (NewRecipe->getNumDefinedValues() == 1)
199 VPV->replaceAllUsesWith(New: NewRecipe->getVPSingleValue());
200 else
201 assert(NewRecipe->getNumDefinedValues() == 0 &&
202 "Only recpies with zero or one defined values expected");
203 Ingredient.eraseFromParent();
204 }
205 }
206 return true;
207}
208
209/// Helper for extra no-alias checks via known-safe recipe and SCEV.
210class SinkStoreInfo {
211 SmallPtrSet<VPReplicateRecipe *, 4> ExcludeRecipes;
212 VPReplicateRecipe &GroupLeader;
213 PredicatedScalarEvolution *PSE = nullptr;
214 const Loop *L = nullptr;
215
216 // Return true if \p A and \p B are known to not alias for all VFs in the
217 // plan, checked via the distance between the accesses
218 bool isNoAliasViaDistance(VPReplicateRecipe *A, VPReplicateRecipe *B) const {
219 if (A->getOpcode() != Instruction::Store ||
220 B->getOpcode() != Instruction::Store)
221 return false;
222
223 if (!PSE || !L)
224 return A == B;
225
226 VPValue *AddrA = A->getOperand(N: 1);
227 const SCEV *SCEVA = vputils::getSCEVExprForVPValue(V: AddrA, PSE&: *PSE, L);
228 VPValue *AddrB = B->getOperand(N: 1);
229 const SCEV *SCEVB = vputils::getSCEVExprForVPValue(V: AddrB, PSE&: *PSE, L);
230 if (isa<SCEVCouldNotCompute>(Val: SCEVA) || isa<SCEVCouldNotCompute>(Val: SCEVB))
231 return false;
232
233 const APInt *Distance;
234 ScalarEvolution &SE = *PSE->getSE();
235 if (!match(S: SE.getMinusSCEV(LHS: SCEVA, RHS: SCEVB), P: m_scev_APInt(C&: Distance)))
236 return false;
237
238 const DataLayout &DL = SE.getDataLayout();
239 Type *TyA = A->getOperand(N: 0)->getScalarType();
240 uint64_t SizeA = DL.getTypeStoreSize(Ty: TyA);
241 Type *TyB = B->getOperand(N: 0)->getScalarType();
242 uint64_t SizeB = DL.getTypeStoreSize(Ty: TyB);
243
244 // Use the maximum store size to ensure no overlap from either direction.
245 // Currently only handles fixed sizes, as it is only used for
246 // replicating VPReplicateRecipes.
247 uint64_t MaxStoreSize = std::max(a: SizeA, b: SizeB);
248
249 auto VFs = B->getParent()->getPlan()->vectorFactors();
250 ElementCount MaxVF = *max_element(Range&: VFs, C: ElementCount::isKnownLT);
251 if (MaxVF.isScalable())
252 return false;
253 return Distance->abs().uge(RHS: MaxVF.getFixedValue() * MaxStoreSize);
254 }
255
256public:
257 SinkStoreInfo(ArrayRef<VPReplicateRecipe *> ExcludeRecipes,
258 VPReplicateRecipe &GroupLeader, PredicatedScalarEvolution &PSE,
259 const Loop &L)
260 : ExcludeRecipes(ExcludeRecipes.begin(), ExcludeRecipes.end()),
261 GroupLeader(GroupLeader), PSE(&PSE), L(&L) {}
262
263 SinkStoreInfo(VPReplicateRecipe &GroupLeader) : GroupLeader(GroupLeader) {}
264
265 /// Return true if \p R should be skipped during alias checking, either
266 /// because it's in the exclude set or because no-alias can be proven via
267 /// SCEV.
268 bool shouldSkip(VPRecipeBase &R) const {
269 auto *Store = dyn_cast<VPReplicateRecipe>(Val: &R);
270 return ExcludeRecipes.contains(Ptr: Store) ||
271 (Store && isNoAliasViaDistance(A: Store, B: &GroupLeader));
272 }
273};
274
275/// Check if a memory operation doesn't alias with memory operations using
276/// scoped noalias metadata, in blocks in the single-successor chain between \p
277/// FirstBB and \p LastBB. If \p SinkInfo is std::nullopt, only recipes that may
278/// write to memory are checked (for load hoisting). Otherwise recipes that both
279/// read and write memory are checked, and SCEV is used to prove no-alias
280/// between the group leader and other replicate recipes (for store sinking).
281static bool
282canHoistOrSinkWithNoAliasCheck(const MemoryLocation &MemLoc,
283 VPBasicBlock *FirstBB, VPBasicBlock *LastBB,
284 std::optional<SinkStoreInfo> SinkInfo = {}) {
285 bool CheckReads = SinkInfo.has_value();
286 ScopedNoAliasAAResult ScopedNoAlias(FirstBB->getPlan()->getContext());
287 for (VPBasicBlock *VPBB :
288 VPBlockUtils::blocksInSingleSuccessorChainBetween(FirstBB, LastBB)) {
289 for (VPRecipeBase &R : *VPBB) {
290 if (SinkInfo && SinkInfo->shouldSkip(R))
291 continue;
292
293 // Skip recipes that don't need checking.
294 if (!R.mayWriteToMemory() && !(CheckReads && R.mayReadFromMemory()))
295 continue;
296
297 auto Loc = vputils::getMemoryLocation(R);
298 if (!Loc)
299 // Conservatively assume aliasing for memory operations without
300 // location.
301 return false;
302
303 if (ScopedNoAlias.alias(LocA: *Loc, LocB: MemLoc) != AliasResult::NoAlias)
304 return false;
305 }
306 }
307 return true;
308}
309
310/// Collect either replicated Loads or Stores grouped by their address SCEV and
311/// their load-store type, in a deep-traversal of the vector loop region in \p
312/// Plan.
313template <unsigned Opcode>
314static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
315collectGroupedReplicateMemOps(
316 VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L,
317 function_ref<bool(VPReplicateRecipe *)> FilterFn) {
318 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
319 "Only Load and Store opcodes supported");
320 constexpr bool IsLoad = (Opcode == Instruction::Load);
321 SmallDenseMap<std::pair<const SCEV *, const Type *>,
322 SmallVector<VPReplicateRecipe *, 4>>
323 RecipesByAddressAndType;
324 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
325 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
326 for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(Range&: *VPBB)) {
327 if (RepR.getOpcode() != Opcode || !FilterFn(&RepR))
328 continue;
329
330 // For loads, operand 0 is address; for stores, operand 1 is address.
331 VPValue *Addr = RepR.getOperand(N: IsLoad ? 0 : 1);
332 const Type *LoadStoreTy = vputils::getLoadStoreValueType(R: &RepR);
333 const SCEV *AddrSCEV = vputils::getSCEVExprForVPValue(V: Addr, PSE, L);
334 if (!isa<SCEVCouldNotCompute>(Val: AddrSCEV))
335 RecipesByAddressAndType[{AddrSCEV, LoadStoreTy}].push_back(Elt: &RepR);
336 }
337 }
338 auto Groups = to_vector(Range: RecipesByAddressAndType.values());
339 VPDominatorTree VPDT(Plan);
340 for (auto &Group : Groups) {
341 // Sort mem ops by dominance order, with earliest (most dominating) first.
342 stable_sort(Group, [&VPDT](VPReplicateRecipe *A, VPReplicateRecipe *B) {
343 return VPDT.properlyDominates(A, B);
344 });
345 }
346 return Groups;
347}
348
349static bool sinkScalarOperands(VPlan &Plan) {
350 auto Iter = vp_depth_first_deep(G: Plan.getEntry());
351 bool ScalarVFOnly = Plan.hasScalarVFOnly();
352 bool Changed = false;
353
354 SetVector<std::pair<VPBasicBlock *, VPSingleDefRecipe *>> WorkList;
355 auto InsertIfValidSinkCandidate = [ScalarVFOnly, &WorkList](
356 VPBasicBlock *SinkTo, VPValue *Op) {
357 auto *Candidate = dyn_cast<VPSingleDefRecipe>(Val: Op);
358 if (!isa_and_nonnull<VPReplicateRecipe, VPScalarIVStepsRecipe,
359 VPInstruction>(Val: Candidate))
360 return;
361
362 if (Candidate->getParent() == SinkTo ||
363 all_of(Range: Candidate->operands(),
364 P: [](VPValue *Op) { return Op->isDefinedOutsideLoopRegions(); }) ||
365 vputils::cannotHoistOrSinkRecipe(R: *Candidate, /*Sinking=*/true))
366 return;
367
368 if (!ScalarVFOnly && !vputils::doesGeneratePerAllLanes(R: Candidate))
369 return;
370
371 // Only single-scalar VPInstructions can be sunk.
372 if (auto *VPI = dyn_cast<VPInstruction>(Val: Candidate))
373 if (!vputils::isSingleScalar(VPV: VPI))
374 return;
375
376 WorkList.insert(X: {SinkTo, Candidate});
377 };
378
379 // First, collect the operands of all recipes in replicate blocks as seeds for
380 // sinking.
381 for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Range&: Iter)) {
382 VPBasicBlock *EntryVPBB = VPR->getEntryBasicBlock();
383 if (!VPR->isReplicator() || EntryVPBB->getSuccessors().size() != 2)
384 continue;
385 VPBasicBlock *VPBB = cast<VPBasicBlock>(Val: EntryVPBB->getSuccessors().front());
386 if (VPBB->getSingleSuccessor() != VPR->getExitingBasicBlock())
387 continue;
388 for (auto &Recipe : *VPBB)
389 for (VPValue *Op : Recipe.operands())
390 InsertIfValidSinkCandidate(VPBB, Op);
391 }
392
393 // Try to sink each replicate or scalar IV steps recipe in the worklist.
394 for (unsigned I = 0; I != WorkList.size(); ++I) {
395 VPBasicBlock *SinkTo;
396 VPSingleDefRecipe *SinkCandidate;
397 std::tie(args&: SinkTo, args&: SinkCandidate) = WorkList[I];
398
399 // All recipe users of SinkCandidate must be in the same block SinkTo or all
400 // users outside of SinkTo must only use the first lane of SinkCandidate. In
401 // the latter case, we need to duplicate SinkCandidate.
402 auto UsersOutsideSinkTo =
403 make_filter_range(Range: SinkCandidate->users(), Pred: [SinkTo](VPUser *U) {
404 return cast<VPRecipeBase>(Val: U)->getParent() != SinkTo;
405 });
406 if (any_of(Range&: UsersOutsideSinkTo, P: [SinkCandidate](VPUser *U) {
407 return !U->usesFirstLaneOnly(Op: SinkCandidate);
408 }))
409 continue;
410 bool NeedsDuplicating = !UsersOutsideSinkTo.empty();
411
412 if (NeedsDuplicating) {
413 if (ScalarVFOnly)
414 continue;
415 VPSingleDefRecipe *Clone;
416 if (auto *SinkCandidateRepR =
417 dyn_cast<VPReplicateRecipe>(Val: SinkCandidate)) {
418 // TODO: Handle converting to uniform recipes as separate transform,
419 // then cloning should be sufficient here.
420 Clone = VPBuilder::createSingleScalarOp(
421 Opcode: SinkCandidateRepR->getOpcode(), Operands: SinkCandidate->operands(),
422 /*Mask=*/nullptr, Flags: *SinkCandidateRepR, Metadata: *SinkCandidateRepR,
423 DL: SinkCandidate->getDebugLoc(), ResultTy: SinkCandidate->getScalarType(),
424 UV: SinkCandidate->getUnderlyingInstr());
425 // TODO: add ".cloned" suffix to name of Clone's VPValue.
426 } else {
427 Clone = SinkCandidate->clone();
428 }
429
430 Clone->insertBefore(InsertPos: SinkCandidate);
431 SinkCandidate->replaceUsesWithIf(New: Clone, ShouldReplace: [SinkTo](VPUser *U) {
432 return cast<VPRecipeBase>(Val: U)->getParent() != SinkTo;
433 });
434 }
435 SinkCandidate->moveBefore(I: SinkTo->getFirstNonPhi());
436 for (VPValue *Op : SinkCandidate->operands())
437 InsertIfValidSinkCandidate(SinkTo, Op);
438 Changed = true;
439 }
440 return Changed;
441}
442
443/// If \p R is a triangle region, return the 'then' block of the triangle.
444static VPBasicBlock *getPredicatedThenBlock(VPRegionBlock *R) {
445 auto *EntryBB = cast<VPBasicBlock>(Val: R->getEntry());
446 if (EntryBB->getNumSuccessors() != 2)
447 return nullptr;
448
449 auto *Succ0 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[0]);
450 auto *Succ1 = dyn_cast<VPBasicBlock>(Val: EntryBB->getSuccessors()[1]);
451 if (!Succ0 || !Succ1)
452 return nullptr;
453
454 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
455 return nullptr;
456 if (Succ0->getSingleSuccessor() == Succ1)
457 return Succ0;
458 if (Succ1->getSingleSuccessor() == Succ0)
459 return Succ1;
460 return nullptr;
461}
462
463// Merge replicate regions in their successor region, if a replicate region
464// is connected to a successor replicate region with the same predicate by a
465// single, empty VPBasicBlock.
466static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) {
467 SmallPtrSet<VPRegionBlock *, 4> TransformedRegions;
468
469 // Collect replicate regions followed by an empty block, followed by another
470 // replicate region with matching masks to process front. This is to avoid
471 // iterator invalidation issues while merging regions.
472 SmallVector<VPRegionBlock *, 8> WorkList;
473 for (VPRegionBlock *Region1 : VPBlockUtils::blocksOnly<VPRegionBlock>(
474 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
475 if (!Region1->isReplicator())
476 continue;
477 auto *MiddleBasicBlock =
478 dyn_cast_or_null<VPBasicBlock>(Val: Region1->getSingleSuccessor());
479 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
480 continue;
481
482 auto *Region2 =
483 dyn_cast_or_null<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
484 if (!Region2 || !Region2->isReplicator())
485 continue;
486
487 VPValue *Mask1 = Region1->getEntryBranchOnMask()->getOperand(N: 0);
488 VPValue *Mask2 = Region2->getEntryBranchOnMask()->getOperand(N: 0);
489 if (!Mask1 || Mask1 != Mask2)
490 continue;
491
492 assert(Mask1 && Mask2 && "both region must have conditions");
493 WorkList.push_back(Elt: Region1);
494 }
495
496 // Move recipes from Region1 to its successor region, if both are triangles.
497 for (VPRegionBlock *Region1 : WorkList) {
498 if (TransformedRegions.contains(Ptr: Region1))
499 continue;
500 auto *MiddleBasicBlock = cast<VPBasicBlock>(Val: Region1->getSingleSuccessor());
501 auto *Region2 = cast<VPRegionBlock>(Val: MiddleBasicBlock->getSingleSuccessor());
502
503 VPBasicBlock *Then1 = getPredicatedThenBlock(R: Region1);
504 VPBasicBlock *Then2 = getPredicatedThenBlock(R: Region2);
505 if (!Then1 || !Then2)
506 continue;
507
508 // The merged region is entered whenever either of the original regions was,
509 // so use the higher, i.e. more conservative, of their entry frequencies.
510 // If only one of the two is known, the higher one is unknown, so the
511 // result must be unknown too.
512 VPBranchOnMaskRecipe *Guard2 = Region2->getEntryBranchOnMask();
513 std::optional<VPExecutionFrequency> Freq1 =
514 Region1->getEntryBranchOnMask()->getExecutionFrequency();
515 std::optional<VPExecutionFrequency> Freq2 = Guard2->getExecutionFrequency();
516 if (Freq1 && Freq2) {
517 if (Freq2->Freq < Freq1->Freq) {
518 // Freq1's frequency is taken, but it is only as trustworthy as the
519 // less trustworthy of the two.
520 Freq1.emplace(args: Freq1->Freq, args: Freq1->IsEstimated || Freq2->IsEstimated);
521 Guard2->setExecutionFrequency(Freq: Freq1, Ctx&: Plan.getContext());
522 }
523 } else if (Freq2) {
524 Guard2->clearExecutionFrequency();
525 }
526
527 // Note: No fusion-preventing memory dependencies are expected in either
528 // region. Such dependencies should be rejected during earlier dependence
529 // checks, which guarantee accesses can be re-ordered for vectorization.
530 //
531 // Move recipes to the successor region.
532 for (VPRecipeBase &ToMove : make_early_inc_range(Range: reverse(C&: *Then1)))
533 ToMove.moveBefore(I: Then2->getFirstNonPhi());
534
535 auto *Merge1 = cast<VPBasicBlock>(Val: Then1->getSingleSuccessor());
536 auto *Merge2 = cast<VPBasicBlock>(Val: Then2->getSingleSuccessor());
537
538 // Move VPPredInstPHIRecipes from the merge block to the successor region's
539 // merge block. Update all users inside the successor region to use the
540 // original values.
541 for (VPRecipeBase &Phi1ToMove : make_early_inc_range(Range: reverse(C&: *Merge1))) {
542 VPValue *PredInst1 =
543 cast<VPPredInstPHIRecipe>(Val: &Phi1ToMove)->getOperand(N: 0);
544 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
545 Phi1ToMoveV->replaceUsesWithIf(New: PredInst1, ShouldReplace: [Then2](VPUser *U) {
546 return cast<VPRecipeBase>(Val: U)->getParent() == Then2;
547 });
548
549 // Remove phi recipes that are unused after merging the regions.
550 if (Phi1ToMove.getVPSingleValue()->user_empty()) {
551 Phi1ToMove.eraseFromParent();
552 continue;
553 }
554 Phi1ToMove.moveBefore(I: Merge2->begin());
555 }
556
557 // Remove the dead recipes in Region1's entry block.
558 for (VPRecipeBase &R :
559 make_early_inc_range(Range: reverse(C&: *Region1->getEntryBasicBlock())))
560 R.eraseFromParent();
561
562 // Finally, remove the first region.
563 for (VPBlockBase *Pred : make_early_inc_range(Range&: Region1->getPredecessors())) {
564 VPBlockUtils::disconnectBlocks(From: Pred, To: Region1);
565 VPBlockUtils::connectBlocks(From: Pred, To: MiddleBasicBlock);
566 }
567 VPBlockUtils::disconnectBlocks(From: Region1, To: MiddleBasicBlock);
568 TransformedRegions.insert(Ptr: Region1);
569 }
570
571 return !TransformedRegions.empty();
572}
573
574static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
575 VPRegionBlock *ParentRegion,
576 VPlan &Plan) {
577 Instruction *Instr = PredRecipe->getUnderlyingInstr();
578 // Build the triangular if-then region.
579 std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str();
580 assert(Instr->getParent() && "Predicated instruction not in any basic block");
581 auto *BlockInMask = PredRecipe->getMask();
582 auto *MaskDef = BlockInMask->getDefiningRecipe();
583 auto *BOMRecipe = new VPBranchOnMaskRecipe(
584 BlockInMask, MaskDef ? MaskDef->getDebugLoc() : DebugLoc::getUnknown());
585 auto *Entry =
586 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".entry", Recipe: BOMRecipe);
587
588 // Replace predicated replicate recipe with a replicate recipe without a
589 // mask but in the replicate region.
590 auto *RecipeWithoutMask = new VPReplicateRecipe(
591 PredRecipe->getUnderlyingInstr(), PredRecipe->operandsWithoutMask(),
592 PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe,
593 PredRecipe->getDebugLoc());
594 // The predicated recipe executes exactly when the guarding branch-on-mask is
595 // taken, so move its execution frequency there.
596 BOMRecipe->setExecutionFrequency(Freq: RecipeWithoutMask->getExecutionFrequency(),
597 Ctx&: Plan.getContext());
598 RecipeWithoutMask->clearExecutionFrequency();
599 auto *Pred =
600 Plan.createVPBasicBlock(Name: Twine(RegionName) + ".if", Recipe: RecipeWithoutMask);
601 auto *Exiting = Plan.createVPBasicBlock(Name: Twine(RegionName) + ".continue");
602 VPRegionBlock *Region =
603 Plan.createReplicateRegion(Entry, Exiting, Name: RegionName);
604
605 // Note: first set Entry as region entry and then connect successors starting
606 // from it in order, to propagate the "parent" of each VPBasicBlock.
607 Region->setParent(ParentRegion);
608 VPBlockUtils::insertTwoBlocksAfter(IfTrue: Pred, IfFalse: Exiting, BlockPtr: Entry);
609 VPBlockUtils::connectBlocks(From: Pred, To: Exiting);
610
611 PredRecipe->replaceAllUsesWith(New: RecipeWithoutMask);
612 // Move the extracts of a struct result into the region, so only scalars
613 // leave it.
614 SmallVector<VPUser *> Defs = {RecipeWithoutMask};
615 if (RecipeWithoutMask->getScalarType()->isStructTy())
616 Defs = to_vector(Range: RecipeWithoutMask->users());
617 for (VPUser *U : Defs) {
618 auto *Def = cast<VPReplicateRecipe>(Val: U);
619 assert((Def == RecipeWithoutMask ||
620 Def->getOpcode() == Instruction::ExtractValue) &&
621 "only extracts can use a predicated struct");
622 Def->moveBefore(I: Pred->end());
623 if (Def->user_empty())
624 continue;
625 auto *PHIRecipe = new VPPredInstPHIRecipe(Def, Def->getDebugLoc());
626 Exiting->appendRecipe(Recipe: PHIRecipe);
627 Def->replaceAllUsesWith(New: PHIRecipe);
628 PHIRecipe->setOperand(I: 0, New: Def);
629 }
630 PredRecipe->eraseFromParent();
631 return Region;
632}
633
634static void addReplicateRegions(VPlan &Plan) {
635 SmallVector<VPReplicateRecipe *> WorkList;
636 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
637 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
638 for (VPReplicateRecipe &RepR : make_isa_range<VPReplicateRecipe>(Range&: *VPBB))
639 if (RepR.isPredicated())
640 WorkList.push_back(Elt: &RepR);
641 }
642
643 unsigned BBNum = 0;
644 for (VPReplicateRecipe *RepR : WorkList) {
645 VPBasicBlock *CurrentBlock = RepR->getParent();
646 VPBasicBlock *SplitBlock = CurrentBlock->splitAt(SplitAt: RepR->getIterator());
647
648 BasicBlock *OrigBB = RepR->getUnderlyingInstr()->getParent();
649 SplitBlock->setName(
650 OrigBB->hasName() ? OrigBB->getName() + "." + Twine(BBNum++) : "");
651 // Record predicated instructions for above packing optimizations.
652 VPRegionBlock *Region =
653 createReplicateRegion(PredRecipe: RepR, ParentRegion: CurrentBlock->getParent(), Plan);
654 VPBlockUtils::insertOnEdge(From: CurrentBlock, To: SplitBlock, BlockPtr: Region);
655
656 VPRegionBlock *ParentRegion = Region->getParent();
657 if (ParentRegion && ParentRegion->getExiting() == CurrentBlock)
658 ParentRegion->setExiting(SplitBlock);
659 }
660}
661
662bool VPlanTransforms::mergeBlocksIntoPredecessors(VPlan &Plan) {
663 SmallVector<VPBasicBlock *> WorkList;
664
665 // Collect single-successor chains in the loop region.
666 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
667 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
668 Range: vp_depth_first_deep(G: LoopRegion->getEntry()))) {
669 if (!VPBB->getParent())
670 continue;
671 VPBlockBase *PredBlock = VPBB->getSinglePredecessor();
672 if (!isa_and_nonnull<VPBasicBlock>(Val: PredBlock) ||
673 !PredBlock->getSingleSuccessor())
674 continue;
675 WorkList.push_back(Elt: VPBB);
676 }
677
678 for (VPBasicBlock *VPBB : WorkList) {
679 VPBasicBlock *PredVPBB = cast<VPBasicBlock>(Val: VPBB->getSinglePredecessor());
680 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
681 R.moveBefore(I: PredVPBB->end());
682 VPBlockUtils::disconnectBlocks(From: PredVPBB, To: VPBB);
683 auto *ParentRegion = VPBB->getParent();
684 if (ParentRegion && ParentRegion->getExiting() == VPBB)
685 ParentRegion->setExiting(PredVPBB);
686 VPBlockUtils::transferSuccessors(Old: VPBB, New: PredVPBB);
687 // VPBB is now dead and will be cleaned up when the plan gets destroyed.
688 }
689 return !WorkList.empty();
690}
691
692void VPlanTransforms::createAndOptimizeReplicateRegions(VPlan &Plan) {
693 // Convert masked VPReplicateRecipes to if-then region blocks.
694 addReplicateRegions(Plan);
695
696 bool ShouldSimplify = true;
697 while (ShouldSimplify) {
698 ShouldSimplify = sinkScalarOperands(Plan);
699 ShouldSimplify |= mergeReplicateRegionsIntoSuccessors(Plan);
700 ShouldSimplify |= mergeBlocksIntoPredecessors(Plan);
701 }
702}
703
704/// Remove redundant casts of inductions.
705///
706/// Such redundant casts are casts of induction variables that can be ignored,
707/// because we already proved that the casted phi is equal to the uncasted phi
708/// in the vectorized loop. There is no need to vectorize the cast - the same
709/// value can be used for both the phi and casts in the vector loop.
710static void removeRedundantInductionCasts(VPlan &Plan) {
711 for (VPWidenIntOrFpInductionRecipe &IV :
712 make_isa_range<VPWidenIntOrFpInductionRecipe>(
713 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
714 if (IV.isTruncated())
715 continue;
716
717 // A sequence of IR Casts has potentially been recorded for IV, which
718 // *must be bypassed* when the IV is vectorized, because the vectorized IV
719 // will produce the desired casted value. This sequence forms a def-use
720 // chain and is provided in reverse order, ending with the cast that uses
721 // the IV phi. Search for the recipe of the last cast in the chain and
722 // replace it with the original IV. Note that only the final cast is
723 // expected to have users outside the cast-chain and the dead casts left
724 // over will be cleaned up later.
725 ArrayRef<Instruction *> Casts = IV.getInductionDescriptor().getCastInsts();
726 VPValue *FindMyCast = &IV;
727 for (Instruction *IRCast : reverse(C&: Casts)) {
728 VPSingleDefRecipe *FoundUserCast = nullptr;
729 for (auto *U : FindMyCast->users()) {
730 auto *UserCast = dyn_cast<VPSingleDefRecipe>(Val: U);
731 if (UserCast && UserCast->getUnderlyingValue() == IRCast) {
732 FoundUserCast = UserCast;
733 break;
734 }
735 }
736 // A cast recipe in the chain may have been removed by earlier DCE.
737 if (!FoundUserCast)
738 break;
739 FindMyCast = FoundUserCast;
740 }
741 if (FindMyCast != &IV)
742 FindMyCast->replaceAllUsesWith(New: &IV);
743 }
744}
745
746/// If R is a phi-like recipe starting a dead cycle of recipes, erase all
747/// reachable recipes of the dead cycle and return true. Otherwise leave the
748/// plan unchanged and return false.
749static bool tryToRemoveDeadCycle(VPRecipeBase *R) {
750 auto *PhiR = dyn_cast<VPSingleDefRecipe>(Val: R);
751 if (!PhiR || !isa<VPPhi, VPReductionPHIRecipe>(Val: R))
752 return false;
753
754 // The transitive users of PhiR are closed under users, so the cycle is dead
755 // if every one of them can be erased.
756 for (VPUser *U : vputils::collectUsersRecursively(V: PhiR)) {
757 auto *R = cast<VPRecipeBase>(Val: U);
758 // Bail out if a user must be retained, or if it is a phi-like recipe other
759 // than PhiR;
760 if (R->mayHaveSideEffects() || (R != PhiR && isa<VPPhiAccessors>(Val: R)))
761 return false;
762 }
763
764 // Break the cycle by replacing PhiR with its first incoming value, which is
765 // defined outside the cycle. That leaves the rest of the cycle dead.
766 PhiR->replaceAllUsesWith(New: PhiR->getOperand(N: 0));
767 SmallVector<VPValue *> Incoming(PhiR->operands());
768 PhiR->eraseFromParent();
769 for (VPValue *Op : Incoming)
770 vputils::recursivelyDeleteDeadRecipes(V: Op);
771 return true;
772}
773
774void VPlanTransforms::removeDeadRecipes(VPlan &Plan) {
775 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
776 Plan.getEntry());
777 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
778 // The recipes in the block are processed in reverse order, to catch chains
779 // of dead recipes.
780 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB)))
781 if (vputils::isDeadRecipe(R))
782 R.eraseFromParent();
783
784 // Erase dead cycles starting at one of VPBB's phi-like recipes. Erasing a
785 // cycle may also erase other phi-like recipes of VPBB, so restart the scan
786 // of the phi section after each removal. This terminates, as each removal
787 // erases the cycle's phi.
788 bool Changed = true;
789 while (Changed) {
790 Changed = false;
791 for (VPRecipeBase &R : VPBB->phis()) {
792 if (tryToRemoveDeadCycle(R: &R)) {
793 Changed = true;
794 break;
795 }
796 }
797 }
798 }
799}
800
801/// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd
802/// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as
803/// VPWidenPointerInductionRecipe will generate vectors only. If some users
804/// require vectors while other require scalars, the scalar uses need to extract
805/// the scalars from the generated vectors (Note that this is different to how
806/// int/fp inductions are handled). Legalize extract-from-ends using uniform
807/// VPReplicateRecipe of wide inductions to use regular VPReplicateRecipe, so
808/// the correct end value is available. Also optimize
809/// VPWidenIntOrFpInductionRecipe, if any of its users needs scalar values, by
810/// providing them scalar steps built on the canonical scalar IV and update the
811/// original IV's users. This is an optional optimization to reduce the needs of
812/// vector extracts.
813static void legalizeAndOptimizeInductions(VPlan &Plan) {
814 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
815 bool HasOnlyVectorVFs = !Plan.hasScalarVFOnly();
816
817 SmallVector<VPWidenInductionRecipe *> WideIVs;
818 for (VPWidenInductionRecipe &PhiR :
819 make_isa_range<VPWidenInductionRecipe>(Range: HeaderVPBB->phis()))
820 WideIVs.push_back(Elt: &PhiR);
821
822 // Try to narrow wide and replicating recipes to uniform recipes, based on
823 // VPlan analysis.
824 // TODO: Apply to all recipes in the future, to replace legacy uniformity
825 // analysis.
826 for (VPWidenInductionRecipe *PhiR : WideIVs) {
827 auto Users = vputils::collectUsersRecursively(V: PhiR);
828 for (VPUser *U : reverse(C&: Users)) {
829 auto *Def = dyn_cast<VPRecipeWithIRFlags>(Val: U);
830 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
831 // Skip recipes that shouldn't be narrowed.
832 if (!Def ||
833 !isa<VPReplicateRecipe, VPWidenRecipe, VPWidenGEPRecipe>(Val: Def) ||
834 Def->user_empty() || !Def->getUnderlyingValue() ||
835 (RepR && (RepR->isSingleScalar() || RepR->isPredicated())))
836 continue;
837
838 // Skip recipes that may have other lanes than their first used.
839 if (!vputils::isSingleScalar(VPV: Def) && !vputils::onlyFirstLaneUsed(Def))
840 continue;
841
842 // TODO: Support scalarizing ExtractValue.
843 if (match(V: Def,
844 P: m_Binary<Instruction::ExtractValue>(Op0: m_VPValue(), Op1: m_VPValue())))
845 continue;
846
847 auto *Clone = VPBuilder::createSingleScalarOp(
848 Opcode: Def->getUnderlyingInstr()->getOpcode(), Operands: Def->operands(),
849 /*Mask=*/nullptr, Flags: *Def, Metadata: getMetadataOf(R: Def), DL: DebugLoc::getUnknown(),
850 ResultTy: Def->getScalarType(), UV: Def->getUnderlyingInstr());
851 Clone->insertAfter(InsertPos: Def);
852 Def->replaceAllUsesWith(New: Clone);
853 Def->eraseFromParent();
854 }
855 }
856
857 VPBuilder Builder(HeaderVPBB->getFirstNonPhi());
858 for (VPWidenInductionRecipe *PhiR : WideIVs) {
859 // Replace wide pointer inductions which have only their scalars used by
860 // PtrAdd(IndStart, ScalarIVSteps (0, Step)).
861 if (auto *PtrIV = dyn_cast<VPWidenPointerInductionRecipe>(Val: PhiR)) {
862 if (!Plan.hasScalarVFOnly() &&
863 !PtrIV->onlyScalarsGenerated(IsScalable: Plan.hasScalableVF()))
864 continue;
865
866 VPValue *PtrAdd =
867 vputils::scalarizeVPWidenPointerInduction(PtrIV, Plan, Builder);
868 PtrIV->replaceAllUsesWith(New: PtrAdd);
869 continue;
870 }
871
872 // Replace widened induction with scalar steps for users that only use
873 // scalars.
874 auto *WideIV = cast<VPWidenIntOrFpInductionRecipe>(Val: PhiR);
875 if (HasOnlyVectorVFs && none_of(Range: WideIV->users(), P: [WideIV](VPUser *U) {
876 return U->usesScalars(Op: WideIV);
877 }))
878 continue;
879
880 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
881 VPIRFlags::WrapFlagsTy WrapFlags;
882 // We can preserve nuw when the step is non-negative.
883 const APInt *Step;
884 if (match(V: WideIV->getStepValue(), P: m_APInt(C&: Step)) && Step->isNonNegative())
885 WrapFlags = WideIV->getNoWrapFlagsOrNone().withoutNoSignedWrap();
886 VPScalarIVStepsRecipe *Steps = vputils::createScalarIVSteps(
887 Plan, Kind: ID.getKind(), InductionOpcode: ID.getInductionOpcode(),
888 FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
889 ResultTy: WideIV->getScalarType(), StartV: WideIV->getStartValue(),
890 Step: WideIV->getStepValue(), DL: WideIV->getDebugLoc(), Builder, Flags: WrapFlags);
891
892 // Update scalar users of IV to use Step instead.
893 if (!HasOnlyVectorVFs) {
894 assert(!Plan.hasScalableVF() &&
895 "plans containing a scalar VF cannot also include scalable VFs");
896 WideIV->replaceAllUsesWith(New: Steps);
897 } else {
898 bool HasScalableVF = Plan.hasScalableVF();
899 WideIV->replaceUsesWithIf(New: Steps, ShouldReplace: [WideIV, HasScalableVF](VPUser *U) {
900 if (HasScalableVF)
901 return U->usesFirstLaneOnly(Op: WideIV);
902 return U->usesScalars(Op: WideIV);
903 });
904 }
905 }
906}
907
908/// Check if \p VPV is an untruncated wide induction, either before or after the
909/// increment. If so return the header IV (before the increment), otherwise
910/// return null.
911static VPWidenInductionRecipe *
912getOptimizableIVOf(VPValue *VPV, PredicatedScalarEvolution &PSE) {
913 auto *WideIV = dyn_cast<VPWidenInductionRecipe>(Val: VPV);
914 if (WideIV) {
915 // VPV itself is a wide induction, separately compute the end value for exit
916 // users if it is not a truncated IV.
917 auto *IntOrFpIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
918 return (IntOrFpIV && IntOrFpIV->isTruncated()) ? nullptr : WideIV;
919 }
920
921 // Check if VPV is an optimizable induction increment.
922 VPRecipeBase *Def = VPV->getDefiningRecipe();
923 if (!Def || Def->getNumOperands() != 2)
924 return nullptr;
925 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 0));
926 if (!WideIV)
927 WideIV = dyn_cast<VPWidenInductionRecipe>(Val: Def->getOperand(N: 1));
928 if (!WideIV)
929 return nullptr;
930
931 auto IsWideIVInc = [&]() {
932 auto &ID = WideIV->getInductionDescriptor();
933
934 // Check if VPV increments the induction by the induction step.
935 VPValue *IVStep = WideIV->getStepValue();
936 switch (ID.getInductionOpcode()) {
937 case Instruction::Add:
938 return match(V: VPV, P: m_c_Add(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
939 case Instruction::FAdd:
940 return match(V: VPV, P: m_c_FAdd(Op0: m_Specific(VPV: WideIV), Op1: m_Specific(VPV: IVStep)));
941 case Instruction::FSub:
942 return match(V: VPV, P: m_Binary<Instruction::FSub>(Op0: m_Specific(VPV: WideIV),
943 Op1: m_Specific(VPV: IVStep)));
944 case Instruction::Sub: {
945 // IVStep will be the negated step of the subtraction. Check if Step == -1
946 // * IVStep.
947 VPValue *Step;
948 if (!match(V: VPV, P: m_Sub(Op0: m_VPValue(), Op1: m_VPValue(V&: Step))))
949 return false;
950 const SCEV *IVStepSCEV = vputils::getSCEVExprForVPValue(V: IVStep, PSE);
951 const SCEV *StepSCEV = vputils::getSCEVExprForVPValue(V: Step, PSE);
952 ScalarEvolution &SE = *PSE.getSE();
953 return !isa<SCEVCouldNotCompute>(Val: IVStepSCEV) &&
954 !isa<SCEVCouldNotCompute>(Val: StepSCEV) &&
955 IVStepSCEV == SE.getNegativeSCEV(V: StepSCEV);
956 }
957 default:
958 return ID.getKind() == InductionDescriptor::IK_PtrInduction &&
959 match(V: VPV, P: m_GetElementPtr(Op0: m_Specific(VPV: WideIV),
960 Op1: m_Specific(VPV: WideIV->getStepValue())));
961 }
962 llvm_unreachable("should have been covered by switch above");
963 };
964 return IsWideIVInc() ? WideIV : nullptr;
965}
966
967/// Attempts to optimize the induction variable exit values for users in the
968/// early exit block.
969static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan, VPValue *Op,
970 PredicatedScalarEvolution &PSE) {
971 VPValue *Incoming, *Mask;
972 if (!match(V: Op, P: m_ExtractLane(Op0: m_FirstActiveLane(Op0: m_VPValue(V&: Mask)),
973 Op1: m_VPValue(V&: Incoming))))
974 return nullptr;
975
976 auto *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
977 if (!WideIV)
978 return nullptr;
979
980 // Calculate the final index.
981 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
982 auto *CanonicalIV = LoopRegion->getCanonicalIV();
983 Type *CanonicalIVType = LoopRegion->getCanonicalIVType();
984 auto *ExtractR = cast<VPInstruction>(Val: Op);
985 VPBuilder B(ExtractR);
986
987 DebugLoc DL = ExtractR->getDebugLoc();
988 VPValue *FirstActiveLane = B.createFirstActiveLane(Masks: Mask, DL);
989 FirstActiveLane =
990 B.createScalarZExtOrTrunc(Op: FirstActiveLane, ResultTy: CanonicalIVType, DL);
991 VPValue *EndValue = B.createAdd(LHS: CanonicalIV, RHS: FirstActiveLane, DL);
992
993 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
994 // changed it means the exit is using the incremented value, so we need to
995 // add the step.
996 if (Incoming != WideIV) {
997 VPValue *One = Plan.getConstantInt(Ty: CanonicalIVType, Val: 1);
998 EndValue = B.createAdd(LHS: EndValue, RHS: One, DL);
999 }
1000
1001 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
1002 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
1003 VPValue *Start = WideIV->getStartValue();
1004 VPValue *Step = WideIV->getStepValue();
1005 EndValue = B.createDerivedIV(
1006 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
1007 Start, Current: EndValue, Step);
1008 }
1009
1010 return EndValue;
1011}
1012
1013/// Compute the end value for \p WideIV, unless it is truncated. Creates a
1014/// VPDerivedIVRecipe for non-canonical inductions.
1015static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV,
1016 VPBuilder &VectorPHBuilder,
1017 VPValue *VectorTC) {
1018 auto *WideIntOrFp = dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WideIV);
1019 // Truncated wide inductions resume from the last lane of their vector value
1020 // in the last vector iteration which is handled elsewhere.
1021 if (WideIntOrFp && WideIntOrFp->isTruncated())
1022 return nullptr;
1023
1024 VPValue *Start = WideIV->getStartValue();
1025 VPValue *Step = WideIV->getStepValue();
1026 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
1027 VPValue *EndValue = VectorTC;
1028 if (!match(V: WideIV, P: m_CanonicalWidenIV())) {
1029 EndValue = VectorPHBuilder.createDerivedIV(
1030 Kind: ID.getKind(), FPBinOp: dyn_cast_or_null<FPMathOperator>(Val: ID.getInductionBinOp()),
1031 Start, Current: VectorTC, Step);
1032 }
1033
1034 // EndValue is derived from the vector trip count (which has the same type as
1035 // the widest induction) and thus may be wider than the induction here.
1036 Type *ScalarTypeOfWideIV = WideIV->getScalarType();
1037 if (ScalarTypeOfWideIV != EndValue->getScalarType()) {
1038 EndValue = VectorPHBuilder.createScalarCast(Opcode: Instruction::Trunc, Op: EndValue,
1039 ResultTy: ScalarTypeOfWideIV,
1040 DL: WideIV->getDebugLoc());
1041 }
1042
1043 return EndValue;
1044}
1045
1046/// Attempts to optimize the induction variable exit values for users in the
1047/// exit block coming from the latch in the original scalar loop.
1048static VPValue *
1049optimizeLatchExitInductionUser(VPlan &Plan, VPValue *Op,
1050 DenseMap<VPValue *, VPValue *> &EndValues,
1051 PredicatedScalarEvolution &PSE) {
1052 VPValue *Incoming;
1053 if (!match(V: Op, P: m_CombineOr(Ps: m_ExtractLastLaneOfLastPart(Op0: m_VPValue(V&: Incoming)),
1054 Ps: m_ExtractLane(Op0: m_LastActiveLane(Op0: m_HeaderMask()),
1055 Op1: m_VPValue(V&: Incoming)))))
1056 return nullptr;
1057
1058 VPWidenInductionRecipe *WideIV = getOptimizableIVOf(VPV: Incoming, PSE);
1059 if (!WideIV)
1060 return nullptr;
1061
1062 VPValue *EndValue = EndValues.lookup(Val: WideIV);
1063 assert(EndValue && "Must have computed the end value up front");
1064
1065 // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
1066 // changed it means the exit is using the incremented value, so we don't
1067 // need to subtract the step.
1068 if (Incoming != WideIV)
1069 return EndValue;
1070
1071 // Otherwise, subtract the step from the EndValue.
1072 auto *ExtractR = cast<VPInstruction>(Val: Op);
1073 VPBuilder B(ExtractR);
1074 VPValue *Step = WideIV->getStepValue();
1075 Type *ScalarTy = WideIV->getScalarType();
1076 if (ScalarTy->isIntegerTy())
1077 return B.createSub(LHS: EndValue, RHS: Step, DL: DebugLoc::getUnknown(), Name: "ind.escape");
1078 if (ScalarTy->isPointerTy()) {
1079 Type *StepTy = Step->getScalarType();
1080 auto *Zero = Plan.getZero(Ty: StepTy);
1081 return B.createPtrAdd(Ptr: EndValue, Offset: B.createSub(LHS: Zero, RHS: Step),
1082 DL: DebugLoc::getUnknown(), Name: "ind.escape");
1083 }
1084 if (ScalarTy->isFloatingPointTy()) {
1085 const auto &ID = WideIV->getInductionDescriptor();
1086 return B.createNaryOp(
1087 Opcode: ID.getInductionBinOp()->getOpcode() == Instruction::FAdd
1088 ? Instruction::FSub
1089 : Instruction::FAdd,
1090 Operands: {EndValue, Step}, Flags: {ID.getInductionBinOp()->getFastMathFlags()});
1091 }
1092 llvm_unreachable("all possible induction types must be handled");
1093 return nullptr;
1094}
1095
1096static VPValue *optimizeLatchExitIVUserViaSCEV(VPlan &Plan, VPValue *Op,
1097 PredicatedScalarEvolution &PSE,
1098 VPValue *ResumeTC,
1099 const Loop *L) {
1100 VPValue *Incoming;
1101 if (!match(V: Op, P: m_CombineOr(Ps: m_ExtractLastLaneOfLastPart(Op0: m_VPValue(V&: Incoming)),
1102 Ps: m_ExtractLane(Op0: m_LastActiveLane(Op0: m_HeaderMask()),
1103 Op1: m_VPValue(V&: Incoming)))))
1104 return nullptr;
1105
1106 const SCEV *IncomingSCEV = vputils::getSCEVExprForVPValue(V: Incoming, PSE, L);
1107 const SCEV *Start, *Step;
1108 if (!match(S: IncomingSCEV, P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEV(V&: Step),
1109 L: m_SpecificLoop(L))))
1110 return nullptr;
1111
1112 auto *ExtractR = cast<VPInstruction>(Val: Op);
1113 DebugLoc DL = ExtractR->getDebugLoc();
1114 VPBuilder Builder(ExtractR);
1115 VPSCEVExpander Expander(Builder, *PSE.getSE(), DL);
1116 VPValue *StartVPV = Expander.expand(S: Start);
1117 VPValue *StepVPV = Expander.expand(S: Step);
1118
1119 Type *StartTy = StartVPV->getScalarType();
1120 assert(StartTy->isIntOrPtrTy() && "The type must be SCEVable");
1121 InductionDescriptor::InductionKind Kind =
1122 StartTy->isPointerTy() ? InductionDescriptor::IK_PtrInduction
1123 : InductionDescriptor::IK_IntInduction;
1124 Type *TCTy = ResumeTC->getScalarType();
1125 VPValue *ExitCount = Builder.createOverflowingOp(
1126 Opcode: Instruction::Sub, Operands: {ResumeTC, Plan.getConstantInt(Ty: TCTy, Val: 1)},
1127 WrapFlags: {/*HasNUW=*/true, /*HasNSW=*/false}, DL: DebugLoc::getUnknown());
1128 return Builder.createDerivedIV(Kind, /*FPBinOp=*/nullptr, Start: StartVPV, Current: ExitCount,
1129 Step: StepVPV);
1130}
1131
1132void VPlanTransforms::optimizeInductionLiveOutUsers(
1133 VPlan &Plan, PredicatedScalarEvolution &PSE, const Loop *L) {
1134 // Compute end values for all inductions.
1135 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
1136 auto *VectorPH = cast<VPBasicBlock>(Val: VectorRegion->getSinglePredecessor());
1137 VPBuilder VectorPHBuilder(VectorPH->getFirstNonPhi());
1138 DenseMap<VPValue *, VPValue *> EndValues;
1139 VPValue *ResumeTC =
1140 Plan.hasTailFolded() ? Plan.getTripCount() : &Plan.getVectorTripCount();
1141 for (VPWidenInductionRecipe &WideIV : make_isa_range<VPWidenInductionRecipe>(
1142 Range: VectorRegion->getEntryBasicBlock()->phis())) {
1143 if (VPValue *EndValue = tryToComputeEndValueForInduction(
1144 WideIV: &WideIV, VectorPHBuilder, VectorTC: ResumeTC))
1145 EndValues[&WideIV] = EndValue;
1146 }
1147
1148 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
1149 for (VPRecipeBase &R : make_early_inc_range(Range&: *MiddleVPBB)) {
1150 VPValue *Op;
1151 if (!match(V: &R, P: m_ExitingIVValue(Op0: m_VPValue(V&: Op))))
1152 continue;
1153 auto *WideIV = cast<VPWidenInductionRecipe>(Val: Op);
1154 if (VPValue *EndValue = EndValues.lookup(Val: WideIV)) {
1155 R.getVPSingleValue()->replaceAllUsesWith(New: EndValue);
1156 R.eraseFromParent();
1157 }
1158 }
1159
1160 // Then, optimize exit block users.
1161 for (VPIRBasicBlock *ExitVPBB : Plan.getExitBlocks()) {
1162 for (VPRecipeBase &R : ExitVPBB->phis()) {
1163 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
1164
1165 for (auto [Idx, PredVPBB] : enumerate(First&: ExitVPBB->getPredecessors())) {
1166 VPValue *Escape = nullptr;
1167 if (PredVPBB == MiddleVPBB) {
1168 Escape = optimizeLatchExitInductionUser(
1169 Plan, Op: ExitIRI->getOperand(N: Idx), EndValues, PSE);
1170 if (!Escape)
1171 Escape = optimizeLatchExitIVUserViaSCEV(
1172 Plan, Op: ExitIRI->getOperand(N: Idx), PSE, ResumeTC, L);
1173 } else {
1174 Escape = optimizeEarlyExitInductionUser(
1175 Plan, Op: ExitIRI->getOperand(N: Idx), PSE);
1176 }
1177 if (Escape)
1178 ExitIRI->setOperand(I: Idx, New: Escape);
1179 }
1180 }
1181 }
1182}
1183
1184/// Try to simplify logical and bitwise recipes in \p Def.
1185static VPValue *simplifyLogicalRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
1186 VPValue *X;
1187
1188 // X | AllOnes -> AllOnes
1189 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_AllOnes())))
1190 return Plan.getAllOnesValue(Ty: Def->getScalarType());
1191
1192 // X | 0 -> X
1193 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_ZeroInt())))
1194 return X;
1195
1196 // X | !X -> AllOnes
1197 if (match(R: Def, P: m_c_BinaryOr(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X)))))
1198 return Plan.getAllOnesValue(Ty: Def->getScalarType());
1199
1200 // X & 0 -> 0
1201 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_ZeroInt())))
1202 return Plan.getZero(Ty: Def->getScalarType());
1203
1204 // X & AllOnes -> X
1205 if (match(R: Def, P: m_c_BinaryAnd(Op0: m_VPValue(V&: X), Op1: m_AllOnes())))
1206 return X;
1207
1208 // X && false -> false
1209 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_False())))
1210 return Plan.getFalse();
1211
1212 // X && true -> X
1213 if (match(R: Def, P: m_c_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_True())))
1214 return X;
1215
1216 // X && (X && Y) -> X && Y
1217 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1218 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue()))))
1219 return Def->getOperand(N: 1);
1220
1221 // X && !X -> 0
1222 if (match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_Not(Op0: m_Deferred(V: X)))))
1223 return Plan.getFalse();
1224
1225 if (match(R: Def, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: X), Op2: m_Deferred(V: X))))
1226 return X;
1227
1228 // X != false -> X
1229 if (match(R: Def, P: m_SpecificICmp(MatchPred: CmpInst::ICMP_NE, Op0: m_VPValue(V&: X), Op1: m_False()))) {
1230 assert(X->getScalarType()->isIntegerTy(1) && "must have boolean operands");
1231 return X;
1232 }
1233
1234 return nullptr;
1235}
1236
1237/// Swap the branch weights recorded for \p R, a select whose two selected
1238/// operands are being swapped,
1239static void swapSelectBranchWeights(VPRecipeBase &R, VPlan &Plan) {
1240 auto *MD = dyn_cast<VPIRMetadata>(Val: &R);
1241 if (!MD)
1242 return;
1243 SmallVector<uint32_t, 2> Weights;
1244 if (!extractBranchWeights(ProfileData: MD->getMetadata(Kind: LLVMContext::MD_prof), Weights))
1245 return;
1246 assert(Weights.size() == 2 && "unexpected branch weights");
1247 MD->setMetadata(
1248 Kind: LLVMContext::MD_prof,
1249 Node: MDBuilder(Plan.getContext()).createBranchWeights(TrueWeight: Weights[1], FalseWeight: Weights[0]));
1250}
1251
1252/// Return an existing value or a live in for VPSingleDefRecipe \p Def if
1253/// possible. This shouldn't create or modify recipes.
1254static VPValue *simplifyRecipe(VPlan &Plan, VPSingleDefRecipe *Def) {
1255 // Simplification of live-in IR values for SingleDef recipes using
1256 // InstSimplifyFolder.
1257 const DataLayout &DL = Plan.getDataLayout();
1258 if (VPValue *V = vputils::tryToFoldLiveIns(R&: *Def, Operands: Def->operands(), DL))
1259 return V;
1260
1261 // Fold PredPHI LiveIn -> LiveIn.
1262 if (auto *PredPHI = dyn_cast<VPPredInstPHIRecipe>(Val: Def)) {
1263 VPValue *Op = PredPHI->getOperand(N: 0);
1264 if (isa<VPIRValue>(Val: Op))
1265 return Op;
1266 }
1267
1268 if (VPValue *V = simplifyLogicalRecipe(Plan, Def))
1269 return V;
1270
1271 VPValue *A, *B;
1272
1273 // icmp Pred X, X -> true if Pred is true when equal, false otherwise.
1274 CmpPredicate Pred;
1275 if (match(R: Def, P: m_ICmp(Pred, Op0: m_VPValue(V&: A), Op1: m_Deferred(V: A))))
1276 return ICmpInst::isTrueWhenEqual(predicate: Pred) ? Plan.getTrue() : Plan.getFalse();
1277
1278 if (match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))
1279 return A;
1280
1281 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: A), Op1: m_One())))
1282 return A;
1283
1284 if (match(R: Def, P: m_c_Mul(Op0: m_VPValue(), Op1: m_ZeroInt())))
1285 return Plan.getZero(Ty: Def->getScalarType());
1286
1287 // A bitcast to the same type is a no-op.
1288 if (match(R: Def, P: m_BitCast(Op0: m_VPValue(V&: A))) &&
1289 Def->getScalarType() == A->getScalarType())
1290 return A;
1291
1292 // Shifting by zero is a no-op.
1293 if (match(R: Def, P: m_CombineOr(Ps: m_Shl(Op0: m_VPValue(V&: A), Op1: m_ZeroInt()),
1294 Ps: m_CombineOr(Ps: m_LShr(Op0: m_VPValue(V&: A), Op1: m_ZeroInt()),
1295 Ps: m_AShr(Op0: m_VPValue(V&: A), Op1: m_ZeroInt())))))
1296 return A;
1297
1298 if (match(R: Def, P: m_Trunc(Op0: m_ZExtOrSExt(Op0: m_VPValue(V&: A)))))
1299 if (Def->getScalarType() == A->getScalarType())
1300 return A;
1301
1302 if (match(R: Def, P: m_Not(Op0: m_Not(Op0: m_VPValue(V&: A)))))
1303 return A;
1304
1305 // Remove redundant DerviedIVs, that is 0 + A * 1 -> A and 0 + 0 * x -> 0.
1306 if ((match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_VPValue(V&: A), Op2: m_One())) ||
1307 match(R: Def, P: m_DerivedIV(Op0: m_ZeroInt(), Op1: m_VPValue(V&: A, Op: m_ZeroInt()),
1308 Op2: m_VPValue()))) &&
1309 A->getScalarType() == Def->getScalarType())
1310 return A;
1311
1312 // Look through ExtractLastLane.
1313 if (match(R: Def, P: m_ExtractLastLane(Op0: m_VPValue(V&: A)))) {
1314 if (match(V: A, P: m_BuildVector())) {
1315 auto *BuildVector = cast<VPInstruction>(Val: A);
1316 return BuildVector->getLastOperand();
1317 }
1318
1319 if (match(V: A, P: m_Broadcast(Op0: m_VPValue(V&: B))))
1320 return B;
1321
1322 if (isa<VPIRValue, VPInstruction, VPReplicateRecipe>(Val: A) &&
1323 vputils::isSingleScalar(VPV: A))
1324 return A;
1325
1326 if (Plan.hasScalarVFOnly())
1327 return A;
1328 }
1329
1330 // Look through ExtractPenultimateElement (BuildVector ....).
1331 if (match(R: Def, P: m_ExtractPenultimateElement(Op0: m_BuildVector()))) {
1332 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1333 return BuildVector->getOperand(N: BuildVector->getNumOperands() - 2);
1334 }
1335
1336 uint64_t Idx;
1337 if (match(R: Def, P: m_ExtractElement(Op0: m_BuildVector(), Op1: m_ConstantInt(C&: Idx)))) {
1338 auto *BuildVector = cast<VPInstruction>(Val: Def->getOperand(N: 0));
1339 return BuildVector->getOperand(N: Idx);
1340 }
1341
1342 if (isa<VPPhi, VPWidenPHIRecipe, VPHeaderPHIRecipe>(Val: Def)) {
1343 if (Def->getNumOperands() == 1) {
1344 return Def->getOperand(N: 0);
1345 }
1346 if (auto *Phi = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(Val: Def)) {
1347 if (all_equal(Range: Phi->incoming_values()))
1348 return Phi->getOperand(N: 0);
1349 }
1350 return nullptr;
1351 }
1352
1353 VPIRValue *IRV;
1354 if (Def->getNumOperands() == 1 &&
1355 match(R: Def, P: m_ComputeReductionResult(Op0: m_VPIRValue(V&: IRV))))
1356 return IRV;
1357
1358 if (match(R: Def, P: m_VPInstruction<VPInstruction::WideIVStep>(Ops: m_VPValue(V&: A),
1359 Ops: m_One())) &&
1360 A->getScalarType() == Def->getScalarType())
1361 return A;
1362
1363 // Some simplifications can only be applied after unrolling. Perform them
1364 // below.
1365 if (!Plan.isUnrolled())
1366 return nullptr;
1367
1368 // Simplify extracts of the same single-scalar.
1369 if (match(R: Def, P: m_VPInstruction<VPInstruction::ExtractLane>()) &&
1370 all_equal(Range: drop_begin(RangeOrContainer: Def->operands())) &&
1371 vputils::isSingleScalar(VPV: Def->getOperand(N: 1)))
1372 return Def->getOperand(N: 1);
1373
1374 // Replace extract-lane(0, canonical-WIDEN-INDUCTION) with the region's
1375 // scalar canonical IV.
1376 VPWidenIntOrFpInductionRecipe *WidenIV;
1377 if (match(R: Def, P: m_ExtractLane(Op0: m_ZeroInt(), Op1: m_CanonicalWidenIV(V&: WidenIV))))
1378 return WidenIV->getRegion()->getCanonicalIV();
1379
1380 // Simplify unrolled VectorPointer without offset, or with zero offset, to
1381 // just the pointer operand.
1382 if (auto *VPR = dyn_cast<VPVectorPointerRecipe>(Val: Def))
1383 if (!VPR->getVFxPart() || match(V: VPR->getVFxPart(), P: m_ZeroInt()))
1384 return VPR->getOperand(N: 0);
1385
1386 // VPScalarIVSteps after unrolling can be replaced by their start value, if
1387 // the start index is zero and only the first lane 0 is demanded.
1388 if (auto *Steps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def))
1389 if (!Steps->getStartIndex() && vputils::onlyFirstLaneUsed(Def: Steps))
1390 return Steps->getOperand(N: 0);
1391
1392 return nullptr;
1393}
1394
1395/// Returns true if \p V is available at the end of \p VPBB, i.e. it either is a
1396/// live-in from the original IR or defined in \p VPBB.
1397static bool isAvailableAtEndOf(VPValue *V, const VPBasicBlock *VPBB) {
1398 VPRecipeBase *DefR = V->getDefiningRecipe();
1399 return DefR ? DefR->getParent() == VPBB : isa<VPIRValue>(Val: V);
1400}
1401
1402namespace {
1403/// Inserter for VPBuilderBase which appends all created VPSingleDefRecipes to a
1404/// worklist, so they get combined as well.
1405struct VPCombineInserter {
1406 SmallVectorImpl<VPSingleDefRecipe *> &Worklist;
1407
1408 void insertHelper(VPRecipeBase *R, VPBasicBlock::iterator It) {
1409 R->insertBefore(IP: It);
1410 if (auto *Def = dyn_cast<VPSingleDefRecipe>(Val: R))
1411 Worklist.push_back(Elt: Def);
1412 }
1413};
1414
1415using VPCombineBuilder = VPBuilderBase<VPCombineInserter>;
1416} // namespace
1417
1418/// Combine \p Def into a simpler recipe. May modify or create new recipes via
1419/// \p Builder.
1420static VPSingleDefRecipe *combineRecipe(VPlan &Plan, VPSingleDefRecipe *Def,
1421 VPCombineBuilder &Builder) {
1422 if (auto *V = simplifyRecipe(Plan, Def)) {
1423 Def->replaceAllUsesWith(New: V);
1424 return Def;
1425 }
1426
1427 // Drop the mask of a predicated store masked by the header mask (which is
1428 // guaranteed to be true at least for the first lane) and both the stored
1429 // value and the address are uniform across VF and UF. The header mask is
1430 // still the abstract region value here.
1431 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: Def);
1432 RepR && RepR->isPredicated() && RepR->getOpcode() == Instruction::Store &&
1433 all_of(Range: RepR->operandsWithoutMask(), P: vputils::isUniformAcrossVFsAndUFs) &&
1434 match(V: RepR->getMask(), P: m_HeaderMask())) {
1435 auto *Unmasked = new VPReplicateRecipe(
1436 RepR->getUnderlyingInstr(), RepR->operandsWithoutMask(),
1437 RepR->isSingleScalar(), /*Mask=*/nullptr, *RepR, *RepR,
1438 RepR->getDebugLoc());
1439 Builder.insert(R: Unmasked);
1440 return Unmasked;
1441 }
1442
1443 // Avoid replacing VPInstructions with underlying values with new
1444 // VPInstructions, as we would fail to create widen/replicate recpes from the
1445 // new VPInstructions without an underlying value, and miss out on some
1446 // transformations that only apply to widened/replicated recipes later, by
1447 // doing so.
1448 // TODO: We should also not replace non-VPInstructions like VPWidenRecipe with
1449 // VPInstructions without underlying values, as those will get skipped during
1450 // cost computation.
1451 bool CanCreateNewRecipe =
1452 !isa<VPInstruction>(Val: Def) || !Def->getUnderlyingValue();
1453
1454 VPValue *X, *Y, *Z;
1455
1456 // X && (Y && X) -> X && Y
1457 if (CanCreateNewRecipe &&
1458 match(R: Def, P: m_LogicalAnd(Op0: m_VPValue(V&: X),
1459 Op1: m_LogicalAnd(Op0: m_VPValue(V&: Y), Op1: m_Deferred(V: X)))))
1460 return Builder.createLogicalAnd(LHS: X, RHS: Y);
1461
1462 // (X && Y) | (X && Z) -> X && (Y | Z)
1463 if (CanCreateNewRecipe &&
1464 match(R: Def, P: m_c_BinaryOr(Op0: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y)),
1465 Op1: m_LogicalAnd(Op0: m_Deferred(V: X), Op1: m_VPValue(V&: Z)))) &&
1466 // Simplify only if one of the operands has one use to avoid creating an
1467 // extra recipe.
1468 (!Def->getOperand(N: 0)->hasMoreThanOneUniqueUser() ||
1469 !Def->getOperand(N: 1)->hasMoreThanOneUniqueUser()))
1470 return Builder.createLogicalAnd(LHS: X, RHS: Builder.createOr(LHS: Y, RHS: Z));
1471
1472 // (X && Y) | !X -> !X || Y
1473 if (CanCreateNewRecipe &&
1474 match(R: Def,
1475 P: m_c_BinaryOr(Op0: m_OneUse(SubPattern: m_LogicalAnd(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y))),
1476 Op1: m_VPValue(V&: Z, Op: m_Not(Op0: m_Deferred(V: X))))))
1477 return Builder.createLogicalOr(LHS: Z, RHS: Y);
1478
1479 // select C, false, true -> not C
1480 VPValue *C;
1481 if (CanCreateNewRecipe &&
1482 match(R: Def, P: m_Select(Op0: m_VPValue(V&: C), Op1: m_False(), Op2: m_True())))
1483 return Builder.createNot(Operand: C);
1484
1485 // select !C, X, Y -> select C, Y, X
1486 if (match(R: Def, P: m_Select(Op0: m_Not(Op0: m_VPValue(V&: C)), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1487 Def->setOperand(I: 0, New: C);
1488 Def->setOperand(I: 1, New: Y);
1489 Def->setOperand(I: 2, New: X);
1490 swapSelectBranchWeights(R&: *Def, Plan);
1491 return Def;
1492 }
1493
1494 // select X, (i1 Y | Z), Y -> Y | (X && Z)
1495 if (CanCreateNewRecipe &&
1496 match(R: Def, P: m_Select(Op0: m_VPValue(V&: X),
1497 Op1: m_OneUse(SubPattern: m_c_BinaryOr(Op0: m_VPValue(V&: Y), Op1: m_VPValue(V&: Z))),
1498 Op2: m_Deferred(V: Y))) &&
1499 Y->getScalarType()->isIntegerTy(BitWidth: 1))
1500 return Builder.createOr(LHS: Y, RHS: Builder.createLogicalAnd(LHS: X, RHS: Z));
1501
1502 // select M0, (select M1, X, Y), Y -> select (M0 && M1), X, Y
1503 VPValue *Mask0, *Mask1;
1504 if (CanCreateNewRecipe &&
1505 match(R: Def,
1506 P: m_SelectLike(Op0: m_VPValue(V&: Mask0),
1507 Op1: m_OneUse(SubPattern: m_SelectLike(Op0: m_VPValue(V&: Mask1), Op1: m_VPValue(V&: X),
1508 Op2: m_VPValue(V&: Y))),
1509 Op2: m_Deferred(V: Y))))
1510 return Builder.createSelect(Cond: Builder.createLogicalAnd(LHS: Mask0, RHS: Mask1), TrueVal: X, FalseVal: Y,
1511 DL: Def->getDebugLoc());
1512
1513 if (match(R: Def, P: m_Trunc(Op0: m_VPValue(V&: Y, Op: m_ZExtOrSExt(Op0: m_VPValue(V&: X)))))) {
1514 // Don't replace a non-widened cast recipe with a widened cast.
1515 if (!isa<VPWidenCastRecipe>(Val: Def))
1516 return nullptr;
1517 Type *TruncTy = Def->getScalarType();
1518 Type *XTy = X->getScalarType();
1519 if (XTy->getScalarSizeInBits() < TruncTy->getScalarSizeInBits()) {
1520
1521 unsigned ExtOpcode =
1522 match(V: Y, P: m_SExt(Op0: m_VPValue())) ? Instruction::SExt : Instruction::ZExt;
1523 auto *Ext =
1524 Builder.createWidenCast(Opcode: Instruction::CastOps(ExtOpcode), Op: X, ResultTy: TruncTy);
1525 if (auto *UnderlyingExt = Y->getUnderlyingValue()) {
1526 // UnderlyingExt has distinct return type, used to retain legacy cost.
1527 Ext->setUnderlyingValue(UnderlyingExt);
1528 }
1529 return Ext;
1530 } else if (XTy->getScalarSizeInBits() > TruncTy->getScalarSizeInBits()) {
1531 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::Trunc, Op: X, ResultTy: TruncTy);
1532 return Trunc;
1533 }
1534 }
1535
1536 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: X), Op1: m_AllOnes()))) {
1537 // Preserve nsw from the Mul on the new Sub.
1538 VPIRFlags::WrapFlagsTy NW = {
1539 false, cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap()};
1540 return Builder.createSub(LHS: Plan.getZero(Ty: X->getScalarType()), RHS: X,
1541 DL: Def->getDebugLoc(), Name: "", WrapFlags: NW);
1542 }
1543
1544 if (CanCreateNewRecipe &&
1545 match(R: Def, P: m_c_Add(Op0: m_VPValue(V&: X),
1546 Op1: m_VPValue(V&: Z, Op: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Y)))))) {
1547 // Preserve nsw from the Add and the Sub, if it's present on both, on the
1548 // new Sub.
1549 VPIRFlags::WrapFlagsTy NW = {
1550 false, cast<VPRecipeWithIRFlags>(Val: Def)->hasNoSignedWrap() &&
1551 cast<VPRecipeWithIRFlags>(Val: Z)->hasNoSignedWrap()};
1552 return Builder.createSub(LHS: X, RHS: Y, DL: Def->getDebugLoc(), Name: "", WrapFlags: NW);
1553 }
1554
1555 const APInt *APC;
1556 if (CanCreateNewRecipe && match(R: Def, P: m_URem(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1557 APC->isPowerOf2())
1558 return Builder.createAnd(LHS: X, RHS: Plan.getConstantInt(Val: *APC - 1),
1559 DL: Def->getDebugLoc());
1560
1561 if (CanCreateNewRecipe && match(R: Def, P: m_c_Mul(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1562 APC->isPowerOf2()) {
1563 auto *MulR = cast<VPRecipeWithIRFlags>(Val: Def);
1564 unsigned ShiftAmt = APC->exactLogBase2();
1565 VPIRFlags::WrapFlagsTy NW(MulR->hasNoUnsignedWrap(),
1566 MulR->hasNoSignedWrap() &&
1567 ShiftAmt != APC->getBitWidth() - 1);
1568 return Builder.createNaryOp(
1569 Opcode: Instruction::Shl,
1570 Operands: {X, Plan.getConstantInt(BitWidth: APC->getBitWidth(), Val: ShiftAmt)}, Flags: NW,
1571 DL: Def->getDebugLoc());
1572 }
1573
1574 if (CanCreateNewRecipe && match(R: Def, P: m_UDiv(Op0: m_VPValue(V&: X), Op1: m_APInt(C&: APC))) &&
1575 APC->isPowerOf2())
1576 return Builder.createNaryOp(
1577 Opcode: Instruction::LShr,
1578 Operands: {X, Plan.getConstantInt(BitWidth: APC->getBitWidth(), Val: APC->exactLogBase2())},
1579 Flags: *cast<VPRecipeWithIRFlags>(Val: Def), DL: Def->getDebugLoc());
1580
1581 // (X >> C) << C -> X & (-1 << C).
1582 if (CanCreateNewRecipe &&
1583 match(R: Def, P: m_Shl(Op0: m_LShr(Op0: m_VPValue(V&: X), Op1: m_VPValue(V&: Y, Op: m_APInt(C&: APC))),
1584 Op1: m_Deferred(V: Y))))
1585 return Builder.createAnd(
1586 LHS: X, RHS: Plan.getConstantInt(Val: APInt::getAllOnes(numBits: APC->getBitWidth()) << *APC),
1587 DL: Def->getDebugLoc());
1588
1589 if (match(R: Def, P: m_Not(Op0: m_VPValue(V&: X)))) {
1590 // Try to fold Not into compares by adjusting the predicate in-place.
1591 CmpPredicate Pred;
1592 if (match(V: X, P: m_Cmp(Pred, Op0: m_VPValue(), Op1: m_VPValue()))) {
1593 auto *Cmp = cast<VPRecipeWithIRFlags>(Val: X);
1594 // Only fold if every user is a Not of the cmp, or a select using the cmp
1595 // solely as its condition.
1596 if (all_of(Range: Cmp->users(), P: [Cmp](VPUser *U) {
1597 return match(U, P: m_Not(Op0: m_Specific(VPV: Cmp))) ||
1598 (match(U, P: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(),
1599 Op2: m_VPValue())) &&
1600 U->getOperand(N: 1) != Cmp && U->getOperand(N: 2) != Cmp);
1601 })) {
1602 Cmp->setPredicate(CmpInst::getInversePredicate(pred: Pred));
1603 for (VPUser *U : to_vector(Range: Cmp->users())) {
1604 auto *R = cast<VPSingleDefRecipe>(Val: U);
1605 if (match(R, P: m_Select(Op0: m_Specific(VPV: Cmp), Op1: m_VPValue(V&: X), Op2: m_VPValue(V&: Y)))) {
1606 // select (cmp pred), X, Y -> select (cmp inv_pred), Y, X
1607 R->setOperand(I: 1, New: Y);
1608 R->setOperand(I: 2, New: X);
1609 swapSelectBranchWeights(R&: *R, Plan);
1610 } else {
1611 // not (cmp pred) -> cmp inv_pred
1612 assert(match(R, m_Not(m_Specific(Cmp))) && "Unexpected user");
1613 R->replaceAllUsesWith(New: Cmp);
1614 }
1615 }
1616 // If Cmp doesn't have a debug location, use the one from the negation,
1617 // to preserve the location.
1618 if (!Cmp->getDebugLoc() && Def->getDebugLoc())
1619 Cmp->setDebugLoc(Def->getDebugLoc());
1620 return Def;
1621 }
1622 }
1623 }
1624
1625 // Fold any-of (fcmp uno A, A), (fcmp uno B, B), ... ->
1626 // any-of (fcmp uno A, B), ...
1627 if (match(R: Def, P: m_AnyOf())) {
1628 SmallVector<VPValue *, 4> NewOps;
1629 VPRecipeBase *UnpairedCmp = nullptr;
1630 for (VPValue *Op : Def->operands()) {
1631 VPValue *X;
1632 if (Op->getNumUsers() > 1 ||
1633 !match(V: Op, P: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X),
1634 Op1: m_Deferred(V: X)))) {
1635 NewOps.push_back(Elt: Op);
1636 } else if (!UnpairedCmp) {
1637 UnpairedCmp = Op->getDefiningRecipe();
1638 } else {
1639 NewOps.push_back(Elt: Builder.createFCmp(Pred: CmpInst::FCMP_UNO,
1640 A: UnpairedCmp->getOperand(N: 0), B: X));
1641 UnpairedCmp = nullptr;
1642 }
1643 }
1644
1645 if (UnpairedCmp)
1646 NewOps.push_back(Elt: UnpairedCmp->getVPSingleValue());
1647
1648 if (NewOps.size() < Def->getNumOperands())
1649 return Builder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: NewOps);
1650 }
1651
1652 // Fold (fcmp uno X, X) | (fcmp uno Y, Y) -> fcmp uno X, Y
1653 // This is useful for fmax/fmin without fast-math flags, where we need to
1654 // check if any operand is NaN.
1655 if (CanCreateNewRecipe &&
1656 match(R: Def, P: m_BinaryOr(Op0: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: X),
1657 Op1: m_Deferred(V: X)),
1658 Op1: m_SpecificCmp(MatchPred: CmpInst::FCMP_UNO, Op0: m_VPValue(V&: Y),
1659 Op1: m_Deferred(V: Y)))) &&
1660 X->getScalarType() == Y->getScalarType())
1661 return Builder.createFCmp(Pred: CmpInst::FCMP_UNO, A: X, B: Y);
1662
1663 if (match(R: Def, P: m_VPInstruction<VPInstruction::WideIVStep>(Ops: m_VPValue(V&: X),
1664 Ops: m_One())) &&
1665 X->getScalarType() != Def->getScalarType())
1666 return Builder.createWidenCast(Opcode: Instruction::Trunc, Op: X, ResultTy: Def->getScalarType());
1667
1668 // For i1 vp.merges produced by AnyOf reductions:
1669 // vp.merge true, (or X, Y), X, evl -> vp.merge Y, true, X, evl
1670 if (match(R: Def, P: m_Intrinsic<Intrinsic::vp_merge>(Ops: m_True(), Ops: m_VPValue(V&: X),
1671 Ops: m_VPValue(V&: X), Ops: m_VPValue())) &&
1672 match(V: X, P: m_c_BinaryOr(Op0: m_Specific(VPV: X), Op1: m_VPValue(V&: Y))) &&
1673 Def->getScalarType()->isIntegerTy(BitWidth: 1)) {
1674 Def->setOperand(I: 1, New: Plan.getTrue());
1675 Def->setOperand(I: 0, New: Y);
1676 return Def;
1677 }
1678
1679 if (match(R: Def, P: m_BuildVector()) && all_equal(Range: Def->operands()))
1680 return Builder.createNaryOp(Opcode: VPInstruction::Broadcast, Operands: Def->getOperand(N: 0));
1681
1682 // Replace uses of a BuildVector by users that only use its first lane with
1683 // its first operand directly.
1684 if (match(R: Def, P: m_BuildVector())) {
1685 Def->replaceUsesWithIf(New: Def->getOperand(N: 0), ShouldReplace: [Def](VPUser *U) {
1686 return U->usesFirstLaneOnly(Op: Def);
1687 });
1688 return Def;
1689 }
1690
1691 // Look through broadcast of single-scalar when used as select conditions; in
1692 // that case the scalar condition can be used directly.
1693 if (match(R: Def,
1694 P: m_Select(Op0: m_Broadcast(Op0: m_VPValue(V&: Z)), Op1: m_VPValue(), Op2: m_VPValue()))) {
1695 assert(vputils::isSingleScalar(Z) &&
1696 "broadcast operand must be single-scalar");
1697 Def->setOperand(I: 0, New: Z);
1698 return Def;
1699 }
1700
1701 if (match(R: Def, P: m_Broadcast(Op0: m_VPValue(V&: X)))) {
1702 Def->replaceUsesWithIf(
1703 New: X, ShouldReplace: [Def](const VPUser *U) { return U->usesScalars(Op: Def); });
1704 return Def;
1705 }
1706
1707 // Some simplifications can only be applied after unrolling. Perform them
1708 // below.
1709 if (!Plan.isUnrolled())
1710 return nullptr;
1711
1712 // Simplify extract-lane with single source to extract-element.
1713 VPValue *LaneToExtract;
1714 if (match(R: Def, P: m_ExtractLane(Op0: m_VPValue(V&: LaneToExtract), Op1: m_VPValue(V&: X))))
1715 return Builder.createNaryOp(Opcode: Instruction::ExtractElement, Operands: {X, LaneToExtract},
1716 DL: Def->getDebugLoc());
1717
1718 // Look for cycles where Def is of the form:
1719 // X = phi(0, IVInc) ; used only by IVInc, or by IVInc and Inc = X + Y
1720 // IVInc = X + Step ; used by X and Def
1721 // Def = IVInc + Y
1722 // Fold the increment Y into the phi's start value, replace Def with IVInc,
1723 // and if Inc exists, replace it with X.
1724 VPValue *IVInc;
1725 if (match(R: Def, P: m_Add(Op0: m_VPValue(V&: IVInc, Op: m_Add(Op0: m_VPValue(V&: X), Op1: m_VPValue())),
1726 Op1: m_VPValue(V&: Y))) &&
1727 match(V: X, P: m_VPPhi(Op0: m_ZeroInt(), Op1: m_Specific(VPV: IVInc))) &&
1728 IVInc->getNumUsers() == 2) {
1729 auto *Phi = cast<VPPhi>(Val: X);
1730 // If Phi has a second user (besides IVInc's defining recipe), it must be
1731 // Inc = Phi + Y for the fold to apply.
1732 auto *Inc = findSingleUser<VPSingleDefRecipe>(
1733 V: Phi, P: m_Add(Op0: m_Specific(VPV: Phi), Op1: m_Specific(VPV: Y)));
1734 if ((Phi->getNumUsers() == 1 || (Phi->getNumUsers() == 2 && Inc)) &&
1735 isAvailableAtEndOf(V: Y, VPBB: Phi->getIncomingBlock(Idx: 0))) {
1736 Def->replaceAllUsesWith(New: IVInc);
1737 if (Inc)
1738 Inc->replaceAllUsesWith(New: Phi);
1739 Phi->setOperand(I: 0, New: Y);
1740 return Def;
1741 }
1742 }
1743
1744 // Simplify redundant ReductionStartVector recipes after unrolling.
1745 VPValue *StartV;
1746 if (match(R: Def, P: m_VPInstruction<VPInstruction::ReductionStartVector>(
1747 Ops: m_VPValue(V&: StartV), Ops: m_VPValue(), Ops: m_VPValue()))) {
1748 Def->replaceUsesWithIf(New: StartV, ShouldReplace: [](const VPUser *U) {
1749 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: U);
1750 return PhiR && PhiR->isInLoop();
1751 });
1752 return Def;
1753 }
1754
1755 return nullptr;
1756}
1757
1758void VPlanTransforms::combineRecipes(VPlan &Plan) {
1759 SmallVector<VPSingleDefRecipe *, 256> Worklist;
1760 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
1761 Plan.getEntry());
1762 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT))
1763 for (VPSingleDefRecipe &Def :
1764 make_isa_range<VPSingleDefRecipe>(Range: reverse(C&: *VPBB)))
1765 Worklist.push_back(Elt: &Def);
1766
1767 [[maybe_unused]] unsigned InitWorklistSize = Worklist.size();
1768
1769 VPCombineBuilder Builder({.Worklist: Worklist});
1770 while (!Worklist.empty()) {
1771 assert(Worklist.size() < InitWorklistSize * 2 &&
1772 "Worklist is growing large, possible cycle?");
1773 VPSingleDefRecipe *Def = Worklist.pop_back_val();
1774 Builder.setInsertPoint(Def);
1775 VPSingleDefRecipe *New = combineRecipe(Plan, Def, Builder);
1776 if (!New)
1777 continue;
1778 if (New != Def) {
1779 // Replace the recipe with a new one.
1780 Def->replaceAllUsesWith(New);
1781 Def->eraseFromParent();
1782 // TODO: Append users to the worklist (might need a setvector)
1783 } else if (vputils::isDeadRecipe(R&: *Def)) {
1784 // Recipe was modified - it may be dead now.
1785 Def->eraseFromParent();
1786 }
1787 }
1788}
1789
1790void VPlanTransforms::simplifyReverses(VPlan &Plan) {
1791 // Pull out reverses from any elementwise op.
1792 // binop(reverse(x), reverse(y)) -> reverse(binop(x,y))
1793 vputils::pullOutPermutations(
1794 Plan, Perm: [](VPValue *&X) { return m_Reverse(Op0: m_VPValue(V&: X)); },
1795 Build: [](auto *X) { return new VPInstruction(VPInstruction::Reverse, X); });
1796
1797 // reverse(reverse(x)) -> x
1798 VPValue *X;
1799 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1800 Range: vp_depth_first_deep(G: Plan.getEntry())))
1801 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB))
1802 if (match(V: &R, P: m_Reverse(Op0: m_Reverse(Op0: m_VPValue(V&: X)))))
1803 R.getVPSingleValue()->replaceAllUsesWith(New: X);
1804}
1805
1806/// Reassociate (headermask && x) && y -> headermask && (x && y) to allow the
1807/// header mask to be simplified further when tail folding, e.g. in
1808/// optimizeEVLMasks.
1809static void reassociateHeaderMask(VPlan &Plan) {
1810 VPValue *HeaderMask = Plan.getVectorLoopRegion()->getHeaderMask();
1811 if (!HeaderMask)
1812 return;
1813
1814 SmallVector<VPUser *> Worklist;
1815 for (VPUser *U : HeaderMask->users())
1816 if (match(U, P: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue())))
1817 append_range(C&: Worklist, R: cast<VPSingleDefRecipe>(Val: U)->users());
1818
1819 while (!Worklist.empty()) {
1820 auto *R = dyn_cast<VPSingleDefRecipe>(Val: Worklist.pop_back_val());
1821 VPValue *X, *Y;
1822 if (!R || !match(R, P: m_LogicalAnd(
1823 Op0: m_LogicalAnd(Op0: m_Specific(VPV: HeaderMask), Op1: m_VPValue(V&: X)),
1824 Op1: m_VPValue(V&: Y))))
1825 continue;
1826 append_range(C&: Worklist, R: R->users());
1827 VPBuilder Builder(R);
1828 R->replaceAllUsesWith(
1829 New: Builder.createLogicalAnd(LHS: HeaderMask, RHS: Builder.createLogicalAnd(LHS: X, RHS: Y)));
1830 }
1831}
1832
1833static std::optional<Instruction::BinaryOps>
1834getUnmaskedDivRemOpcode(Intrinsic::ID ID) {
1835 switch (ID) {
1836 case Intrinsic::masked_udiv:
1837 return Instruction::UDiv;
1838 case Intrinsic::masked_sdiv:
1839 return Instruction::SDiv;
1840 case Intrinsic::masked_urem:
1841 return Instruction::URem;
1842 case Intrinsic::masked_srem:
1843 return Instruction::SRem;
1844 default:
1845 return {};
1846 }
1847}
1848
1849static void narrowToSingleScalarRecipes(VPlan &Plan) {
1850 if (Plan.hasScalarVFOnly())
1851 return;
1852
1853 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1854 Range: vp_depth_first_deep(G: Plan.getEntry()))) {
1855 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
1856 if (!isa<VPWidenRecipe, VPWidenGEPRecipe, VPReplicateRecipe,
1857 VPWidenIntrinsicRecipe>(Val: &R))
1858 continue;
1859 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
1860 if (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))
1861 continue;
1862
1863 auto *RepOrWidenR = cast<VPRecipeWithIRFlags>(Val: &R);
1864 if (RepR && RepR->getOpcode() == Instruction::Store &&
1865 vputils::isSingleScalar(VPV: RepR->getOperand(N: 1))) {
1866 auto *Clone = new VPReplicateRecipe(
1867 RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(),
1868 true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/,
1869 *RepR /*Metadata*/, RepR->getDebugLoc());
1870 Clone->insertBefore(InsertPos: RepOrWidenR);
1871 VPBuilder Builder(Clone);
1872 VPValue *ExtractOp = Clone->getOperand(N: 0);
1873 if (vputils::isUniformAcrossVFsAndUFs(V: RepR->getOperand(N: 1)))
1874 ExtractOp =
1875 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastPart, Operands: ExtractOp);
1876 ExtractOp =
1877 Builder.createNaryOp(Opcode: VPInstruction::ExtractLastLane, Operands: ExtractOp);
1878 Clone->setOperand(I: 0, New: ExtractOp);
1879 RepR->eraseFromParent();
1880 continue;
1881 }
1882
1883 // Narrow llvm.masked.{u,s}{div,rem} intrinsics with a safe divisor.
1884 if (auto *IntrR = dyn_cast<VPWidenIntrinsicRecipe>(Val: RepOrWidenR)) {
1885 if (!vputils::onlyFirstLaneUsed(Def: IntrR))
1886 continue;
1887 auto Opc = getUnmaskedDivRemOpcode(ID: IntrR->getVectorIntrinsicID());
1888 if (!Opc)
1889 continue;
1890 VPBuilder Builder(IntrR);
1891 VPValue *SafeDivisor = Builder.createSelect(
1892 Cond: IntrR->getOperand(N: 2), TrueVal: IntrR->getOperand(N: 1),
1893 FalseVal: Plan.getConstantInt(Ty: IntrR->getScalarType(), Val: 1));
1894 VPValue *Clone = Builder.createNaryOp(
1895 Opcode: *Opc, Operands: {IntrR->getOperand(N: 0), SafeDivisor},
1896 Flags: VPIRFlags::getDefaultFlags(Opcode: *Opc), DL: IntrR->getDebugLoc());
1897 IntrR->replaceAllUsesWith(New: Clone);
1898 IntrR->eraseFromParent();
1899 continue;
1900 }
1901
1902 // Skip recipes that aren't single scalars or have no underlying
1903 // instruction to create a single-scalar recipe from.
1904 if (!RepOrWidenR->getUnderlyingValue() ||
1905 !vputils::isSingleScalar(VPV: RepOrWidenR))
1906 continue;
1907
1908 // Predicate to check if a user of Op introduces extra broadcasts.
1909 auto IntroducesBCastOf = [](const VPValue *Op) {
1910 return [Op](const VPUser *U) {
1911 if (auto *VPI = dyn_cast<VPInstruction>(Val: U)) {
1912 if (is_contained(Set: {VPInstruction::ExtractLastLane,
1913 VPInstruction::ExtractLastPart,
1914 VPInstruction::ExtractPenultimateElement},
1915 Element: VPI->getOpcode()))
1916 return false;
1917 }
1918 return !U->usesScalars(Op);
1919 };
1920 };
1921
1922 if (any_of(Range: RepOrWidenR->users(), P: IntroducesBCastOf(RepOrWidenR)) &&
1923 none_of(Range: RepOrWidenR->operands(), P: [&](VPValue *Op) {
1924 if (any_of(
1925 Range: make_filter_range(Range: Op->users(), Pred: not_equal_to(Arg&: RepOrWidenR)),
1926 P: IntroducesBCastOf(Op)))
1927 return false;
1928 // Non-constant live-ins require broadcasts, while constants do not
1929 // need explicit broadcasts.
1930 bool LiveInNeedsBroadcast =
1931 isa<VPIRValue>(Val: Op) && !isa<VPConstant>(Val: Op);
1932 auto *OpR = dyn_cast<VPReplicateRecipe>(Val: Op);
1933 return LiveInNeedsBroadcast || (OpR && OpR->isSingleScalar());
1934 }))
1935 continue;
1936
1937 auto *Clone = VPBuilder::createSingleScalarOp(
1938 Opcode: vputils::getOpcode(V: RepOrWidenR), Operands: RepOrWidenR->operands(),
1939 /*Mask=*/nullptr, Flags: *RepOrWidenR, Metadata: getMetadataOf(R: RepOrWidenR),
1940 DL: DebugLoc::getUnknown(), ResultTy: RepOrWidenR->getScalarType(),
1941 UV: RepOrWidenR->getUnderlyingInstr());
1942 Clone->insertBefore(InsertPos: RepOrWidenR);
1943 RepOrWidenR->replaceAllUsesWith(New: Clone);
1944 if (vputils::isDeadRecipe(R&: *RepOrWidenR))
1945 RepOrWidenR->eraseFromParent();
1946 }
1947 }
1948}
1949
1950/// Try to see if all of \p Blend's masks share a common value logically and'ed
1951/// and remove it from the masks.
1952static void removeCommonBlendMask(VPBlendRecipe *Blend) {
1953 if (Blend->isNormalized())
1954 return;
1955 VPValue *CommonEdgeMask;
1956 if (!match(V: Blend->getMask(Idx: 0),
1957 P: m_LogicalAnd(Op0: m_VPValue(V&: CommonEdgeMask), Op1: m_VPValue())))
1958 return;
1959 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1960 if (!match(V: Blend->getMask(Idx: I),
1961 P: m_LogicalAnd(Op0: m_Specific(VPV: CommonEdgeMask), Op1: m_VPValue())))
1962 return;
1963 for (unsigned I = 0; I < Blend->getNumIncomingValues(); I++)
1964 Blend->setMask(Idx: I, V: Blend->getMask(Idx: I)->getDefiningRecipe()->getOperand(N: 1));
1965}
1966
1967/// Normalize and simplify VPBlendRecipes. Should be run after combineRecipes
1968/// to make sure the masks are simplified.
1969static void simplifyBlends(VPlan &Plan) {
1970 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1971 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
1972 for (VPBlendRecipe &Blend :
1973 make_early_inc_range(Range: make_isa_range<VPBlendRecipe>(Range&: *VPBB))) {
1974 removeCommonBlendMask(Blend: &Blend);
1975
1976 // Try to remove redundant blend recipes.
1977 SmallPtrSet<VPValue *, 4> UniqueValues;
1978 if (Blend.isNormalized() || !match(V: Blend.getMask(Idx: 0), P: m_False()))
1979 UniqueValues.insert(Ptr: Blend.getIncomingValue(Idx: 0));
1980 for (unsigned I = 1; I != Blend.getNumIncomingValues(); ++I)
1981 if (!match(V: Blend.getMask(Idx: I), P: m_False()))
1982 UniqueValues.insert(Ptr: Blend.getIncomingValue(Idx: I));
1983
1984 if (UniqueValues.size() == 1) {
1985 Blend.replaceAllUsesWith(New: *UniqueValues.begin());
1986 Blend.eraseFromParent();
1987 continue;
1988 }
1989
1990 if (Blend.isNormalized())
1991 continue;
1992
1993 // Normalize the blend so its first incoming value is used as the initial
1994 // value with the others blended into it.
1995
1996 unsigned StartIndex = 0;
1997 for (unsigned I = 0; I != Blend.getNumIncomingValues(); ++I) {
1998 // If a value's mask is used only by the blend then is can be deadcoded.
1999 // TODO: Find the most expensive mask that can be deadcoded, or a mask
2000 // that's used by multiple blends where it can be removed from them all.
2001 VPValue *Mask = Blend.getMask(Idx: I);
2002 if (Mask->hasOneUse() && !match(V: Mask, P: m_False())) {
2003 StartIndex = I;
2004 break;
2005 }
2006 }
2007
2008 SmallVector<VPValue *, 4> OperandsWithMask;
2009 OperandsWithMask.push_back(Elt: Blend.getIncomingValue(Idx: StartIndex));
2010
2011 for (unsigned I = 0; I != Blend.getNumIncomingValues(); ++I) {
2012 if (I == StartIndex)
2013 continue;
2014 OperandsWithMask.push_back(Elt: Blend.getIncomingValue(Idx: I));
2015 OperandsWithMask.push_back(Elt: Blend.getMask(Idx: I));
2016 }
2017
2018 auto *NewBlend =
2019 new VPBlendRecipe(cast_or_null<PHINode>(Val: Blend.getUnderlyingValue()),
2020 OperandsWithMask, Blend, Blend.getDebugLoc());
2021 NewBlend->insertBefore(InsertPos: &Blend);
2022
2023 VPValue *DeadMask = Blend.getMask(Idx: StartIndex);
2024 Blend.replaceAllUsesWith(New: NewBlend);
2025 Blend.eraseFromParent();
2026 vputils::recursivelyDeleteDeadRecipes(V: DeadMask);
2027
2028 /// Simplify BLEND %a, %b, Not(%mask) -> BLEND %b, %a, %mask.
2029 VPValue *NewMask;
2030 if (NewBlend->getNumOperands() == 3 &&
2031 match(V: NewBlend->getMask(Idx: 1), P: m_Not(Op0: m_VPValue(V&: NewMask)))) {
2032 VPValue *Inc0 = NewBlend->getOperand(N: 0);
2033 VPValue *Inc1 = NewBlend->getOperand(N: 1);
2034 VPValue *OldMask = NewBlend->getOperand(N: 2);
2035 NewBlend->setOperand(I: 0, New: Inc1);
2036 NewBlend->setOperand(I: 1, New: Inc0);
2037 NewBlend->setOperand(I: 2, New: NewMask);
2038 if (OldMask->user_empty())
2039 cast<VPInstruction>(Val: OldMask)->eraseFromParent();
2040 }
2041 }
2042 }
2043}
2044
2045/// Optimize the width of vector induction variables in \p Plan based on a known
2046/// constant Trip Count, \p BestVF and \p BestUF.
2047static bool optimizeVectorInductionWidthForTCAndVFUF(VPlan &Plan,
2048 ElementCount BestVF,
2049 unsigned BestUF) {
2050 // Only proceed if we have not completely removed the vector region.
2051 if (!Plan.getVectorLoopRegion())
2052 return false;
2053
2054 const APInt *TC;
2055 if (!BestVF.isFixed() || !match(V: Plan.getTripCount(), P: m_APInt(C&: TC)))
2056 return false;
2057
2058 // Calculate the minimum power-of-2 bit width that can fit the known TC, VF
2059 // and UF. Returns at least 8.
2060 auto ComputeBitWidth = [](APInt TC, uint64_t Align) {
2061 APInt AlignedTC =
2062 Align * APIntOps::RoundingUDiv(A: TC, B: APInt(TC.getBitWidth(), Align),
2063 RM: APInt::Rounding::UP);
2064 APInt MaxVal = AlignedTC - 1;
2065 return std::max<unsigned>(a: PowerOf2Ceil(A: MaxVal.getActiveBits()), b: 8);
2066 };
2067 unsigned NewBitWidth =
2068 ComputeBitWidth(*TC, BestVF.getKnownMinValue() * BestUF);
2069
2070 LLVMContext &Ctx = Plan.getContext();
2071 auto *NewIVTy = IntegerType::get(C&: Ctx, NumBits: NewBitWidth);
2072
2073 bool MadeChange = false;
2074
2075 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
2076 for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
2077 // Currently only handle canonical IVs as it is trivial to replace the start
2078 // and stop values, and we currently only perform the optimization when the
2079 // IV has a single use.
2080 VPWidenIntOrFpInductionRecipe *WideIV;
2081 if (!match(V: &Phi, P: m_CanonicalWidenIV(V&: WideIV)))
2082 continue;
2083 if (WideIV->hasMoreThanOneUniqueUser() ||
2084 NewIVTy == WideIV->getScalarType())
2085 continue;
2086
2087 // Currently only handle cases where the single user is a header-mask
2088 // comparison with the backedge-taken-count, TC - 1.
2089 VPUser *SingleUser = WideIV->getSingleUser();
2090 VPValue *BTC;
2091 if (!SingleUser ||
2092 !match(
2093 U: SingleUser,
2094 P: m_ICmp(Op0: m_Specific(VPV: WideIV),
2095 Op1: m_Broadcast(Op0: m_VPValue(
2096 V&: BTC, Op: m_Sub(Op0: m_Specific(VPV: Plan.getTripCount()), Op1: m_One()))))))
2097 continue;
2098
2099 // Update IV operands and comparison bound to use new narrower type.
2100 assert(!WideIV->isTruncated() &&
2101 "canonical IV is not expected to have a truncation");
2102 auto *NewWideIV = new VPWidenIntOrFpInductionRecipe(
2103 WideIV->getPHINode(), Plan.getZero(Ty: NewIVTy),
2104 Plan.getConstantInt(Ty: NewIVTy, Val: 1), WideIV->getVFValue(),
2105 WideIV->getInductionDescriptor(), *WideIV, WideIV->getDebugLoc());
2106 NewWideIV->insertBefore(InsertPos: WideIV);
2107
2108 auto *NewBTC =
2109 new VPWidenCastRecipe(Instruction::Trunc, BTC, NewIVTy, nullptr,
2110 VPIRFlags::getDefaultFlags(Opcode: Instruction::Trunc));
2111 Plan.getVectorPreheader()->appendRecipe(Recipe: NewBTC);
2112 auto *Cmp = cast<VPInstruction>(Val: WideIV->getSingleUser());
2113 Cmp->replaceAllUsesWith(
2114 New: VPBuilder(Cmp).createICmp(Pred: Cmp->getPredicate(), A: NewWideIV, B: NewBTC));
2115
2116 MadeChange = true;
2117 }
2118
2119 return MadeChange;
2120}
2121
2122/// Return true if \p Cond is known to be true for given \p BestVF and \p
2123/// BestUF.
2124static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
2125 ElementCount BestVF, unsigned BestUF,
2126 PredicatedScalarEvolution &PSE) {
2127 if (match(V: Cond, P: m_BinaryOr(Op0: m_VPValue(), Op1: m_VPValue())))
2128 return any_of(Range: Cond->getDefiningRecipe()->operands(), P: [&Plan, BestVF, BestUF,
2129 &PSE](VPValue *C) {
2130 return isConditionTrueViaVFAndUF(Cond: C, Plan, BestVF, BestUF, PSE);
2131 });
2132
2133 auto *CanIV = Plan.getVectorLoopRegion()->getCanonicalIV();
2134 if (!match(V: Cond, P: m_SpecificICmp(
2135 MatchPred: CmpInst::ICMP_EQ,
2136 Op0: m_c_Add(Op0: m_Specific(VPV: CanIV), Op1: m_Specific(VPV: &Plan.getVFxUF())),
2137 Op1: m_Specific(VPV: &Plan.getVectorTripCount()))))
2138 return false;
2139
2140 // The compare checks CanIV + VFxUF == vector trip count. The vector trip
2141 // count is not conveniently available as SCEV so far, so we compare directly
2142 // against the original trip count. This is stricter than necessary, as we
2143 // will only return true if the trip count == vector trip count.
2144 const SCEV *VectorTripCount =
2145 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2146 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2147 VectorTripCount = vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2148 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2149 "Trip count SCEV must be computable");
2150 ScalarEvolution &SE = *PSE.getSE();
2151 ElementCount NumElements = BestVF * BestUF;
2152 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2153 return SE.isKnownPredicate(Pred: CmpInst::ICMP_EQ, LHS: VectorTripCount, RHS: C);
2154}
2155
2156// Replaces ExtractVectorForPart instructions with ICMP when the VF is scalar
2157// and the source is a WideActiveLaneMask. The unused mask is removed later
2158// when removing dead recipes.
2159static bool replaceMaskWithCompareForScalarPlan(VPlan &Plan,
2160 ElementCount BestVF) {
2161 if (!BestVF.isScalar())
2162 return false;
2163
2164 bool MadeChange = false;
2165 VPBuilder Builder;
2166 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2167 VPBasicBlock *PreheaderVPBB = Plan.getVectorPreheader();
2168 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2169
2170 VPValue *Start, *TC;
2171 uint64_t Idx;
2172 for (VPBasicBlock *VPBB : {PreheaderVPBB, ExitingVPBB}) {
2173 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2174 if (!match(V: &R, P: m_ExtractVectorForPart(
2175 Op: m_WideActiveLaneMask(Op: m_VPValue(V&: Start), Op: m_VPValue(V&: TC),
2176 Op: m_VPValue()),
2177 Op: m_ConstantInt(C&: Idx))))
2178 continue;
2179
2180 auto *Extract = cast<VPInstruction>(Val: &R);
2181 Builder.setInsertPoint(Extract);
2182
2183 if (Idx > 0)
2184 Start = Builder.createAdd(
2185 LHS: Start, RHS: Plan.getConstantInt(Ty: Start->getScalarType(), Val: Idx));
2186
2187 VPValue *ICmp = Builder.createICmp(Pred: CmpInst::ICMP_ULT, A: Start, B: TC);
2188 Extract->replaceAllUsesWith(New: ICmp);
2189 Extract->eraseFromParent();
2190 MadeChange = true;
2191 }
2192 }
2193
2194 return MadeChange;
2195}
2196
2197/// Return true if the branch \p Term with condition \p Cond (null if \p Term
2198/// has no condition operand) is the vector loop's latch check.
2199static bool isVectorLatchCheck(VPRecipeBase *Term, VPValue *Cond, VPlan &Plan) {
2200 // BranchOnCount compares the canonical IV increment against the vector trip
2201 // count, plus an offset for epilogue loops.
2202 VPValue *Offset = nullptr;
2203 auto m_CanIVInc = m_Add(Op0: m_VPValue(), Op1: m_Specific(VPV: &Plan.getVFxUF()));
2204 if (match(V: Term,
2205 P: m_BranchOnCount(
2206 Op0: m_CombineOr(Ps: m_CanIVInc, Ps: m_c_Add(Op0: m_CanIVInc, Op1: m_VPValue(V&: Offset))),
2207 Op1: m_VPValue())))
2208 return !Offset || Offset->isDefinedOutsideLoopRegions();
2209
2210 if (!Cond)
2211 return false;
2212
2213 // The active-lane-mask form used when the tail is folded by masking.
2214 return match(V: Cond,
2215 P: m_Not(Op0: m_CombineOr(Ps: m_WideActiveLaneMask(),
2216 Ps: m_ExtractVectorForPart(Op: m_WideActiveLaneMask(),
2217 Op: m_ZeroInt()))));
2218}
2219
2220/// Try to simplify the branch condition of \p Plan. This may restrict the
2221/// resulting plan to \p BestVF and \p BestUF.
2222static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF,
2223 unsigned BestUF,
2224 PredicatedScalarEvolution &PSE) {
2225 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
2226 VPBasicBlock *ExitingVPBB = VectorRegion->getExitingBasicBlock();
2227 auto *Term = &ExitingVPBB->back();
2228
2229 // Find the branch condition, if there is one.
2230 VPValue *Cond = nullptr;
2231 if (!match(V: Term, P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))))
2232 match(V: Term, P: m_BranchOnTwoConds(Op0: m_VPValue(), Op1: m_VPValue(V&: Cond)));
2233
2234 if (isVectorLatchCheck(Term, Cond, Plan)) {
2235 // Try to simplify the branch condition if VectorTC <= VF * UF when the
2236 // latch terminator is BranchOnCount or
2237 // BranchOnCond(Not(ExtractVectorForPart(WideActiveLaneMask), 0))
2238 const SCEV *VectorTripCount =
2239 vputils::getSCEVExprForVPValue(V: &Plan.getVectorTripCount(), PSE);
2240 if (isa<SCEVCouldNotCompute>(Val: VectorTripCount))
2241 VectorTripCount =
2242 vputils::getSCEVExprForVPValue(V: Plan.getTripCount(), PSE);
2243 assert(!isa<SCEVCouldNotCompute>(VectorTripCount) &&
2244 "Trip count SCEV must be computable");
2245 ScalarEvolution &SE = *PSE.getSE();
2246 ElementCount NumElements = BestVF * BestUF;
2247 const SCEV *C = SE.getElementCount(Ty: VectorTripCount->getType(), EC: NumElements);
2248 if (!SE.isKnownPredicate(Pred: CmpInst::ICMP_ULE, LHS: VectorTripCount, RHS: C))
2249 return false;
2250 } else if (!Cond ||
2251 !isConditionTrueViaVFAndUF(Cond, Plan, BestVF, BestUF, PSE)) {
2252 return false;
2253 }
2254
2255 // The vector loop region only executes once. Convert terminator of the
2256 // exiting block to exit in the first iteration.
2257 if (match(V: Term, P: m_BranchOnTwoConds())) {
2258 Term->setOperand(I: 1, New: Plan.getTrue());
2259 return true;
2260 }
2261
2262 auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, Plan.getTrue(), {},
2263 {}, Term->getDebugLoc());
2264 ExitingVPBB->appendRecipe(Recipe: BOC);
2265 Term->eraseFromParent();
2266
2267 return true;
2268}
2269
2270void VPlanTransforms::optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
2271 unsigned BestUF,
2272 PredicatedScalarEvolution &PSE) {
2273 assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
2274 assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
2275
2276 bool MadeChange =
2277 simplifyBranchConditionForVFAndUF(Plan, BestVF, BestUF, PSE);
2278 MadeChange |= replaceMaskWithCompareForScalarPlan(Plan, BestVF);
2279 MadeChange |= optimizeVectorInductionWidthForTCAndVFUF(Plan, BestVF, BestUF);
2280
2281 if (MadeChange) {
2282 Plan.setVF(BestVF);
2283 assert(Plan.getConcreteUF() == BestUF && "BestUF must match the Plan's UF");
2284 }
2285}
2286
2287void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
2288 for (VPReductionPHIRecipe &PhiR : make_isa_range<VPReductionPHIRecipe>(
2289 Range: Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis())) {
2290 RecurKind RK = PhiR.getRecurrenceKind();
2291 if (RK != RecurKind::Add && RK != RecurKind::Mul && RK != RecurKind::Sub &&
2292 RK != RecurKind::AddChainWithSubs)
2293 continue;
2294
2295 for (VPUser *U : vputils::collectUsersRecursively(V: &PhiR))
2296 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: U)) {
2297 RecWithFlags->dropPoisonGeneratingFlags();
2298 }
2299 }
2300}
2301
2302static Align getLoadAlign(const VPSingleDefRecipe *R) {
2303 if (auto *MemR = dyn_cast<VPWidenMemoryRecipe>(Val: R))
2304 return MemR->getAlign();
2305 return cast<LoadInst>(Val: R->getUnderlyingInstr())->getAlign();
2306}
2307
2308namespace {
2309struct VPCSEDenseMapInfo : public DenseMapInfo<VPSingleDefRecipe *> {
2310 /// If recipe \p R will lower to a GEP with a non-i8 source element type,
2311 /// return that source element type.
2312 static Type *getGEPSourceElementType(const VPSingleDefRecipe *R) {
2313 // All VPInstructions that lower to GEPs must have the i8 source element
2314 // type (as they are PtrAdds), so we omit it.
2315 return TypeSwitch<const VPSingleDefRecipe *, Type *>(R)
2316 .Case(caseFn: [](const VPReplicateRecipe *I) -> Type * {
2317 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: I->getUnderlyingValue()))
2318 return GEP->getSourceElementType();
2319 return nullptr;
2320 })
2321 .Case<VPVectorPointerRecipe, VPWidenGEPRecipe>(
2322 caseFn: [](auto *I) { return I->getSourceElementType(); })
2323 .Default(defaultFn: [](auto *) { return nullptr; });
2324 }
2325
2326 /// Returns true if recipe \p Def can be safely handed for CSE.
2327 static bool canHandle(const VPSingleDefRecipe *Def) {
2328 // We can extend the list of handled recipes in the future,
2329 // provided we account for the data embedded in them while checking for
2330 // equality or hashing.
2331 auto C = vputils::getOpcodeOrIntrinsicID(V: Def);
2332
2333 // Allocas must not be merged, as each creates a distinct allocation.
2334 if (!C || (!C->first && C->second == Instruction::Alloca))
2335 return false;
2336
2337 // Loads are handled as cse() only reuses them within a block with no
2338 // intervening memory write. Any other memory access is rejected.
2339 if (Def->mayWriteToMemory())
2340 return false;
2341 return !Def->mayReadFromMemory() ||
2342 (!C->first && C->second == Instruction::Load);
2343 }
2344
2345 /// Hash the underlying data of \p Def.
2346 static unsigned getHashValue(const VPSingleDefRecipe *Def) {
2347 hash_code Result = hash_combine(
2348 args: Def->getVPRecipeID(), args: vputils::getOpcodeOrIntrinsicID(V: Def),
2349 args: getGEPSourceElementType(R: Def), args: Def->getScalarType(),
2350 args: vputils::isSingleScalar(VPV: Def), args: hash_combine_range(R: Def->operands()));
2351 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: Def))
2352 if (RFlags->hasPredicate())
2353 return hash_combine(args: Result, args: RFlags->getPredicate());
2354 if (auto *SIVSteps = dyn_cast<VPScalarIVStepsRecipe>(Val: Def))
2355 return hash_combine(args: Result, args: SIVSteps->getInductionOpcode());
2356 if (auto *DIV = dyn_cast<VPDerivedIVRecipe>(Val: Def))
2357 return hash_combine(args: Result, args: DIV->getInductionKind(), args: DIV->getFPBinOp());
2358 // Fold in the separately stored consecutive flag. Alignment is left out and
2359 // handled by cse.
2360 if (auto *Load = dyn_cast<VPWidenMemoryRecipe>(Val: Def))
2361 return hash_combine(args: Result, args: Load->isConsecutive());
2362 if (auto *ExpSCEV = dyn_cast<VPExpandSCEVRecipe>(Val: Def))
2363 return hash_combine(args: Result, args: ExpSCEV->getSCEV());
2364 return Result;
2365 }
2366
2367 /// Check equality of underlying data of \p L and \p R.
2368 static bool isEqual(const VPSingleDefRecipe *L, const VPSingleDefRecipe *R) {
2369 if (L->getVPRecipeID() != R->getVPRecipeID() ||
2370 vputils::getOpcodeOrIntrinsicID(V: L) !=
2371 vputils::getOpcodeOrIntrinsicID(V: R) ||
2372 getGEPSourceElementType(R: L) != getGEPSourceElementType(R) ||
2373 vputils::isSingleScalar(VPV: L) != vputils::isSingleScalar(VPV: R) ||
2374 !equal(LRange: L->operands(), RRange: R->operands()))
2375 return false;
2376 assert(vputils::getOpcodeOrIntrinsicID(L) &&
2377 vputils::getOpcodeOrIntrinsicID(R) &&
2378 "must have valid opcode info for both recipes");
2379 if (auto *LFlags = dyn_cast<VPRecipeWithIRFlags>(Val: L))
2380 if (LFlags->hasPredicate() &&
2381 LFlags->getPredicate() !=
2382 cast<VPRecipeWithIRFlags>(Val: R)->getPredicate())
2383 return false;
2384 if (auto *LSIV = dyn_cast<VPScalarIVStepsRecipe>(Val: L))
2385 if (LSIV->getInductionOpcode() !=
2386 cast<VPScalarIVStepsRecipe>(Val: R)->getInductionOpcode())
2387 return false;
2388 if (auto *LDIV = dyn_cast<VPDerivedIVRecipe>(Val: L)) {
2389 auto *RDIV = cast<VPDerivedIVRecipe>(Val: R);
2390 if (LDIV->getInductionKind() != RDIV->getInductionKind() ||
2391 LDIV->getFPBinOp() != RDIV->getFPBinOp())
2392 return false;
2393 }
2394 // Compare the separately stored consecutive flag. Alignment is left out and
2395 // handled by cse.
2396 if (auto *LL = dyn_cast<VPWidenMemoryRecipe>(Val: L))
2397 if (LL->isConsecutive() != cast<VPWidenMemoryRecipe>(Val: R)->isConsecutive())
2398 return false;
2399 // Whether to expand in IR is left out and handled by cse.
2400 if (auto *LExp = dyn_cast<VPExpandSCEVRecipe>(Val: L))
2401 if (LExp->getSCEV() != cast<VPExpandSCEVRecipe>(Val: R)->getSCEV())
2402 return false;
2403 // Phi recipes can only be equal if they are in the same VPBB, as they
2404 // implicitly depend on their predecessors.
2405 if (isa<VPWidenPHIRecipe>(Val: L) && L->getParent() != R->getParent())
2406 return false;
2407 // Recipes in replicate regions implicitly depend on predicate. If either
2408 // recipe is in a replicate region, only consider them equal if both have
2409 // the same parent.
2410 const VPRegionBlock *RegionL = L->getRegion();
2411 const VPRegionBlock *RegionR = R->getRegion();
2412 if (((RegionL && RegionL->isReplicator()) ||
2413 (RegionR && RegionR->isReplicator())) &&
2414 L->getParent() != R->getParent())
2415 return false;
2416 return L->getScalarType() == R->getScalarType();
2417 }
2418};
2419} // end anonymous namespace
2420
2421/// Perform a common-subexpression-elimination of VPSingleDefRecipes on the \p
2422/// Plan.
2423void VPlanTransforms::cse(VPlan &Plan, const VPDominatorTree &VPDT) {
2424 DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo> CSEMap;
2425 // CSE map for loads. Must be cleared on recipes that may write to memory, and
2426 // at the end of each VPBB.
2427 DenseMap<VPSingleDefRecipe *, VPSingleDefRecipe *, VPCSEDenseMapInfo>
2428 LoadCSEMap;
2429
2430 ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
2431 Plan.getEntry());
2432 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: RPOT)) {
2433 for (VPRecipeBase &R : *VPBB) {
2434 if (R.mayWriteToMemory())
2435 LoadCSEMap.clear();
2436 auto *Def = dyn_cast<VPSingleDefRecipe>(Val: &R);
2437 if (!Def || !VPCSEDenseMapInfo::canHandle(Def))
2438 continue;
2439 bool IsLoad = vputils::getOpcode(V: Def) == Instruction::Load;
2440 auto [It, Inserted] =
2441 (IsLoad ? LoadCSEMap : CSEMap).try_emplace(Key: Def, Args&: Def);
2442 if (Inserted)
2443 continue;
2444 VPSingleDefRecipe *V = It->second;
2445 // V must dominate Def for a valid replacement.
2446 if (!VPDT.dominates(A: V->getParent(), B: VPBB))
2447 continue;
2448 if (IsLoad) {
2449 if (getLoadAlign(R: V) < getLoadAlign(R: Def)) {
2450 // Record Def as the candidate for subsequent loads, as it may be
2451 // reusable where V is not.
2452 It->second = Def;
2453 continue;
2454 }
2455 // Keep only metadata common to both loads on the survivor.
2456 cast<VPIRMetadata>(Val: V)->intersect(MD: *cast<VPIRMetadata>(Val: Def));
2457 }
2458 // Only expansions in IR can replace expansions in IR.
2459 auto *DefExp = dyn_cast<VPExpandSCEVRecipe>(Val: Def);
2460 if (DefExp && DefExp->expandInIR() &&
2461 !cast<VPExpandSCEVRecipe>(Val: V)->expandInIR()) {
2462 It->second = Def;
2463 continue;
2464 }
2465 // Only keep flags present on both V and Def.
2466 if (auto *RFlags = dyn_cast<VPRecipeWithIRFlags>(Val: V))
2467 RFlags->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Def));
2468 // Only keep call attributes valid for both V and Def.
2469 if (auto *Call = dyn_cast<VPWidenCallRecipe>(Val: V))
2470 Call->intersectAttrs(Other: *cast<VPWidenCallRecipe>(Val: Def), Ctx&: Plan.getContext());
2471 Def->replaceAllUsesWith(New: V);
2472 // Reset Plan's TC when TC expressions are CSE'd.
2473 if (Def == Plan.getTripCount())
2474 Plan.resetTripCount(NewTripCount: V);
2475 }
2476 LoadCSEMap.clear();
2477 }
2478}
2479
2480/// Return true if we do not know how to (mechanically) hoist or sink a
2481/// non-memory or memory recipe \p R out of a loop region. When sinking, passing
2482/// \p Sinking = true ensures that assumes aren't sunk.
2483static bool cannotHoistOrSinkRecipe(VPRecipeBase &R, VPBasicBlock *FirstBB,
2484 VPBasicBlock *LastBB,
2485 bool Sinking = false) {
2486 if (!isa<VPReplicateRecipe>(Val: R) || !R.mayReadOrWriteMemory() ||
2487 match(V: &R, P: m_Intrinsic<Intrinsic::assume>()))
2488 return vputils::cannotHoistOrSinkRecipe(R, Sinking);
2489
2490 // Check that the memory operation doesn't alias between FirstBB and LastBB.
2491 auto MemLoc = vputils::getMemoryLocation(R);
2492
2493 // TODO: Could make use of SinkStoreInfo::isNoAliasViaDistance by collecting
2494 // stores upfront, and constructing a full SinkStoreInfo.
2495 auto SinkInfo =
2496 Sinking ? std::make_optional(t: SinkStoreInfo(cast<VPReplicateRecipe>(Val&: R)))
2497 : std::nullopt;
2498
2499 return !MemLoc ||
2500 !canHoistOrSinkWithNoAliasCheck(MemLoc: *MemLoc, FirstBB, LastBB, SinkInfo);
2501}
2502
2503/// Move loop-invariant recipes out of the vector loop region in \p Plan.
2504static void licm(VPlan &Plan, const VPDominatorTree &VPDT) {
2505 VPBasicBlock *Preheader = Plan.getVectorPreheader();
2506
2507 // Hoist any loop invariant recipes from the vector loop region to the
2508 // preheader. Perform a shallow traversal of the vector loop region, to
2509 // exclude recipes in replicate regions. Only hoist from blocks dominating the
2510 // exiting block, which are guaranteed to execute if the vector pre-header is,
2511 // so we don't need to check speculation safety.
2512 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2513 assert(Preheader->getSingleSuccessor() == LoopRegion &&
2514 "Expected vector prehader's successor to be the vector loop region");
2515 VPBasicBlock *ExitingVPBB = LoopRegion->getExitingBasicBlock();
2516 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2517 Range: vp_depth_first_shallow(G: LoopRegion->getEntry()))) {
2518 if (!VPDT.dominates(A: VPBB, B: ExitingVPBB))
2519 continue;
2520 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2521 if (any_of(Range: R.operands(), P: [](VPValue *Op) {
2522 return !Op->isDefinedOutsideLoopRegions();
2523 }))
2524 continue;
2525 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2526 LastBB: LoopRegion->getExitingBasicBlock()))
2527 continue;
2528 R.moveBefore(I: Preheader->end());
2529 }
2530 }
2531
2532 // Sink recipes with no users inside the vector loop region if all users are
2533 // in the same exit block of the region.
2534 // TODO: Extend to sink recipes from inner loops.
2535 PostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> POT(
2536 LoopRegion->getEntry());
2537 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
2538 for (VPRecipeBase &R : make_early_inc_range(Range: reverse(C&: *VPBB))) {
2539 // TODO: Use R.definedValues() instead of casting to VPSingleDefRecipe to
2540 // support recipes with multiple defined values (e.g., interleaved loads).
2541 auto *Def = dyn_cast<VPSingleDefRecipe>(Val: &R);
2542 if (!Def)
2543 continue;
2544
2545 if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R)) {
2546 assert(!RepR->isPredicated() &&
2547 "Expected prior transformation of predicated replicates to "
2548 "replicate regions");
2549 // narrowToSingleScalarRecipes should have already maximally narrowed
2550 // replicates to single-scalar replicates.
2551 // TODO: When unrolling, replicateByVF doesn't handle sunk
2552 // non-single-scalar replicates correctly.
2553 if (!RepR->isSingleScalar())
2554 continue;
2555
2556 // The pointer operand of stores must be loop-invariant.
2557 if (RepR->getOpcode() == Instruction::Store &&
2558 !RepR->getOperand(N: 1)->isDefinedOutsideLoopRegions())
2559 continue;
2560 }
2561
2562 // Cannot sink the recipe if the user is defined in a loop region or a
2563 // non-successor of the vector loop region. Cannot sink if user is a phi
2564 // either.
2565 VPBasicBlock *SinkBB = nullptr;
2566 if (any_of(Range: Def->users(), P: [&SinkBB, &LoopRegion](VPUser *U) {
2567 auto *UserR = cast<VPRecipeBase>(Val: U);
2568 VPBasicBlock *Parent = UserR->getParent();
2569 // TODO: Support sinking when users are in multiple blocks.
2570 if (SinkBB && SinkBB != Parent)
2571 return true;
2572 SinkBB = Parent;
2573 // TODO: If the user is a PHI node, we should check the block of
2574 // incoming value. Support PHI node users if needed.
2575 return UserR->isPhi() || Parent->getEnclosingLoopRegion() ||
2576 Parent->getSinglePredecessor() != LoopRegion;
2577 }))
2578 continue;
2579
2580 if (cannotHoistOrSinkRecipe(R, FirstBB: LoopRegion->getEntryBasicBlock(),
2581 LastBB: LoopRegion->getExitingBasicBlock(),
2582 /*Sinking=*/true))
2583 continue;
2584
2585 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: &R);
2586 assert((!R.mayWriteToMemory() ||
2587 (RepR && RepR->getOpcode() == Instruction::Store &&
2588 RepR->getOperand(1)->isDefinedOutsideLoopRegions())) &&
2589 "The only recipes that may write to memory are expected to be "
2590 "stores with invariant pointer-operand");
2591
2592 if (!SinkBB)
2593 SinkBB = cast<VPBasicBlock>(Val: LoopRegion->getSingleSuccessor());
2594
2595 // TODO: This will need to be a check instead of a assert after
2596 // conditional branches in vectorized loops are supported.
2597 assert(VPDT.properlyDominates(VPBB, SinkBB) &&
2598 "Defining block must dominate sink block");
2599 // TODO: Clone the recipe if users are on multiple exit paths, instead of
2600 // just moving.
2601 Def->moveBefore(I: SinkBB->getFirstNonPhi());
2602 }
2603 }
2604}
2605
2606void VPlanTransforms::truncateToMinimalBitwidths(
2607 VPlan &Plan, const MapVector<Instruction *, uint64_t> &MinBWs) {
2608 if (Plan.hasScalarVFOnly())
2609 return;
2610 // Keep track of created truncates, so they can be re-used. Note that we
2611 // cannot use RAUW after creating a new truncate, as this would could make
2612 // other uses have different types for their operands, making them invalidly
2613 // typed.
2614 // An operand may be shared by recipes narrowed to different widths, so the
2615 // truncates are keyed by operand and narrowed type.
2616 DenseMap<std::pair<VPValue *, Type *>, VPWidenCastRecipe *> ProcessedTruncs;
2617 VPBasicBlock *PH = Plan.getVectorPreheader();
2618 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
2619 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
2620 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
2621 if (!isa<VPWidenRecipe, VPWidenCastRecipe, VPReplicateRecipe,
2622 VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R))
2623 continue;
2624
2625 VPValue *ResultVPV = R.getVPSingleValue();
2626 auto *UI = cast_or_null<Instruction>(Val: ResultVPV->getUnderlyingValue());
2627 unsigned NewResSizeInBits = MinBWs.lookup(Key: UI);
2628 if (!NewResSizeInBits)
2629 continue;
2630
2631 // If the value wasn't vectorized, we must maintain the original scalar
2632 // type. Skip those here, after incrementing NumProcessedRecipes. Also
2633 // skip casts which do not need to be handled explicitly here, as
2634 // redundant casts will be removed during recipe simplification.
2635 if (isa<VPReplicateRecipe, VPWidenCastRecipe>(Val: &R))
2636 continue;
2637
2638 Type *OldResTy = ResultVPV->getScalarType();
2639 unsigned OldResSizeInBits = OldResTy->getScalarSizeInBits();
2640 assert(OldResTy->isIntegerTy() && "only integer types supported");
2641 (void)OldResSizeInBits;
2642
2643 auto *NewResTy = IntegerType::get(C&: Plan.getContext(), NumBits: NewResSizeInBits);
2644
2645 // Any wrapping introduced by shrinking this operation shouldn't be
2646 // considered undefined behavior. So, we can't unconditionally copy
2647 // arithmetic wrapping flags to VPW.
2648 if (auto *VPW = dyn_cast<VPRecipeWithIRFlags>(Val: &R))
2649 VPW->dropPoisonGeneratingFlags();
2650
2651 assert((OldResSizeInBits != NewResSizeInBits ||
2652 match(&R, m_ICmp(m_VPValue(), m_VPValue()))) &&
2653 "Only ICmps should not need extending the result.");
2654 assert(!isa<VPWidenStoreRecipe>(&R) && "stores cannot be narrowed");
2655
2656 // Loads/intrinsics are not recreated; they keep producing their original
2657 // wide result and narrowed users will truncate it as needed below.
2658 if (isa<VPWidenLoadRecipe, VPWidenIntrinsicRecipe>(Val: &R))
2659 continue;
2660
2661 // Shrink operands by introducing truncates as needed.
2662 unsigned StartIdx =
2663 match(V: &R, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(), Op2: m_VPValue())) ? 1 : 0;
2664 SmallVector<VPValue *> NewOperands(R.operands());
2665 for (VPValue *&Op : drop_begin(RangeOrContainer&: NewOperands, N: StartIdx)) {
2666 unsigned OpSizeInBits = Op->getScalarType()->getScalarSizeInBits();
2667 if (OpSizeInBits == NewResSizeInBits)
2668 continue;
2669 assert(OpSizeInBits > NewResSizeInBits && "nothing to truncate");
2670 auto [ProcessedIter, Inserted] =
2671 ProcessedTruncs.try_emplace(Key: {Op, NewResTy});
2672 if (Inserted) {
2673 VPBuilder Builder;
2674 if (isa<VPIRValue>(Val: Op))
2675 Builder.setInsertPoint(PH);
2676 else
2677 Builder.setInsertPoint(&R);
2678 ProcessedIter->second =
2679 Builder.createWidenCast(Opcode: Instruction::Trunc, Op, ResultTy: NewResTy);
2680 }
2681 Op = ProcessedIter->second;
2682 }
2683
2684 auto *NWR = cast<VPWidenRecipe>(Val: &R)->cloneWithOperands(NewOperands);
2685 NWR->insertBefore(InsertPos: &R);
2686
2687 // Wrap NWR in a ZExt to preserve the original wide type for downstream
2688 // users. Not needed for ICmps, whose result type is i1 irrespective of
2689 // the narrowing of their operands.
2690 VPValue *Replacement = NWR->getVPSingleValue();
2691 if (Replacement->getScalarType() != OldResTy)
2692 Replacement =
2693 VPBuilder::getToInsertAfter(R: NWR)
2694 .createWidenCast(Opcode: Instruction::ZExt, Op: Replacement, ResultTy: OldResTy)
2695 ->getVPSingleValue();
2696 ResultVPV->replaceAllUsesWith(New: Replacement);
2697 R.eraseFromParent();
2698 }
2699 }
2700}
2701
2702bool VPlanTransforms::removeBranchOnConst(VPlan &Plan, bool OnlyLatches) {
2703 std::optional<VPDominatorTree> VPDT;
2704 if (OnlyLatches)
2705 VPDT.emplace(args&: Plan);
2706
2707 // Collect all blocks before modifying the CFG so we can identify unreachable
2708 // ones after constant branch removal.
2709 SmallVector<VPBlockBase *> AllBlocks(vp_depth_first_shallow(G: Plan.getEntry()));
2710
2711 bool SimplifiedPhi = false;
2712 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: AllBlocks)) {
2713 VPValue *Cond;
2714 // Skip blocks that are not terminated by BranchOnCond.
2715 if (VPBB->empty() || !match(V: &VPBB->back(), P: m_BranchOnCond(Op0: m_VPValue(V&: Cond))))
2716 continue;
2717
2718 if (OnlyLatches && !VPBlockUtils::isLatch(VPB: VPBB, VPDT: *VPDT))
2719 continue;
2720
2721 assert(VPBB->getNumSuccessors() == 2 &&
2722 "Two successors expected for BranchOnCond");
2723 unsigned RemovedIdx;
2724 if (match(V: Cond, P: m_True()))
2725 RemovedIdx = 1;
2726 else if (match(V: Cond, P: m_False()))
2727 RemovedIdx = 0;
2728 else
2729 continue;
2730
2731 VPBasicBlock *RemovedSucc =
2732 cast<VPBasicBlock>(Val: VPBB->getSuccessors()[RemovedIdx]);
2733 assert(count(RemovedSucc->getPredecessors(), VPBB) == 1 &&
2734 "There must be a single edge between VPBB and its successor");
2735 // Values coming from VPBB into phi recipes of RemovedSucc are removed from
2736 // these recipes and single-entry header phis are removed.
2737 for (VPRecipeBase &R : make_early_inc_range(Range: RemovedSucc->phis())) {
2738 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: VPBB);
2739 SimplifiedPhi = true;
2740 // Remove now invalid header phis that are left single-entry after
2741 // removing their backedges.
2742 auto *PhiR = dyn_cast<VPHeaderPHIRecipe>(Val: &R);
2743 if (!PhiR || PhiR->getNumIncoming() != 1)
2744 continue;
2745 PhiR->replaceAllUsesWith(New: PhiR->getOperand(N: 0));
2746 PhiR->eraseFromParent();
2747 }
2748
2749 // Disconnect blocks and remove the terminator.
2750 VPBlockUtils::disconnectBlocks(From: VPBB, To: RemovedSucc);
2751 VPBB->back().eraseFromParent();
2752 }
2753
2754 // Compute which blocks are still reachable from the entry after constant
2755 // branch removal.
2756 SmallPtrSet<VPBlockBase *, 16> Reachable(
2757 llvm::from_range, vp_depth_first_shallow(G: Plan.getEntry()));
2758
2759 // Detach all unreachable blocks from their successors, removing their recipes
2760 // and incoming values from phi recipes.
2761 VPSymbolicValue Tmp(nullptr);
2762 for (VPBlockBase *B : AllBlocks) {
2763 if (Reachable.contains(Ptr: B))
2764 continue;
2765 for (VPBlockBase *Succ : to_vector(Range: B->successors())) {
2766 if (auto *SuccBB = dyn_cast<VPBasicBlock>(Val: Succ))
2767 for (VPRecipeBase &R : SuccBB->phis())
2768 cast<VPPhiAccessors>(Val: &R)->removeIncomingValueFor(IncomingBlock: B);
2769 VPBlockUtils::disconnectBlocks(From: B, To: Succ);
2770 }
2771 for (VPBasicBlock *DeadBB :
2772 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_deep(G: B))) {
2773 for (VPRecipeBase &R : make_early_inc_range(Range&: *DeadBB)) {
2774 for (VPValue *Def : R.definedValues())
2775 Def->replaceAllUsesWith(New: &Tmp);
2776 R.eraseFromParent();
2777 }
2778 }
2779 }
2780 return SimplifiedPhi;
2781}
2782
2783void VPlanTransforms::optimize(VPlan &Plan) {
2784 RUN_VPLAN_PASS(removeRedundantInductionCasts, Plan);
2785
2786 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2787 RUN_VPLAN_PASS(combineRecipes, Plan);
2788 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2789 RUN_VPLAN_PASS(simplifyBlends, Plan);
2790 RUN_VPLAN_PASS(legalizeAndOptimizeInductions, Plan);
2791 RUN_VPLAN_PASS(narrowToSingleScalarRecipes, Plan);
2792 RUN_VPLAN_PASS(reassociateHeaderMask, Plan);
2793 RUN_VPLAN_PASS(combineRecipes, Plan);
2794 RUN_VPLAN_PASS(removeBranchOnConst, Plan, /*OnlyLatches=*/false);
2795 RUN_VPLAN_PASS(simplifyReverses, Plan);
2796 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2797
2798 RUN_VPLAN_PASS(createAndOptimizeReplicateRegions, Plan);
2799 RUN_VPLAN_PASS(mergeBlocksIntoPredecessors, Plan);
2800 VPDominatorTree VPDT(Plan);
2801 RUN_VPLAN_PASS(licm, Plan, VPDT);
2802 RUN_VPLAN_PASS(cse, Plan, VPDT);
2803 RUN_VPLAN_PASS(removeDeadRecipes, Plan);
2804}
2805
2806void VPlanTransforms::simplifyLiveInsWithSCEV(VPlan &Plan,
2807 PredicatedScalarEvolution &PSE) {
2808 auto GetSimplifiedLiveInViaSCEV = [&](VPValue *VPV) -> VPValue * {
2809 const SCEV *Expr = vputils::getSCEVExprForVPValue(V: VPV, PSE);
2810 const APInt *C;
2811 if (match(S: Expr, P: m_scev_APInt(C)))
2812 return Plan.getConstantInt(Val: *C);
2813 return nullptr;
2814 };
2815
2816 for (VPValue *LiveIn : to_vector(Range: Plan.getLiveIns())) {
2817 if (VPValue *SimplifiedLiveIn = GetSimplifiedLiveInViaSCEV(LiveIn))
2818 LiveIn->replaceAllUsesWith(New: SimplifiedLiveIn);
2819 }
2820}
2821
2822void VPlanTransforms::replaceSymbolicStrides(
2823 VPlan &Plan, PredicatedScalarEvolution &PSE,
2824 const SymbolicStrideMap &StridesMap, const VPDominatorTree &VPDT) {
2825 // Replace VPValues for known constant strides guaranteed by predicated scalar
2826 // evolution that are guaranteed to be guarded by the runtime checks; that is,
2827 // blocks dominated by the vector header.
2828 assert(!Plan.getVectorLoopRegion() &&
2829 "expected to run before loop regions are created");
2830 const auto &[Header, _] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
2831 auto CanUseVersionedStride = [&VPDT, Header = Header, &Plan](VPUser *U) {
2832 auto *R = cast<VPRecipeBase>(Val: U);
2833 // Skip phis if the loop if loop is not yet guarded.
2834 if (isa<VPPhiAccessors>(Val: R) &&
2835 Header == Plan.getEntry()->getSingleSuccessor())
2836 return false;
2837 return VPDT.dominates(A: Header, B: R->getParent());
2838 };
2839 ValueToSCEVMapTy RewriteMap;
2840 for (const SCEVUnknown *Stride : StridesMap.values()) {
2841 Value *StrideV = Stride->getValue();
2842 const APInt *StrideConst;
2843 const SCEV *StrideExpr = PSE.getSCEV(V: StrideV);
2844 if (!match(S: StrideExpr, P: m_scev_APInt(C&: StrideConst)))
2845 // Only handle constant strides for now.
2846 continue;
2847 if (VPValue *StrideVPV = Plan.getLiveIn(V: StrideV))
2848 StrideVPV->replaceUsesWithIf(New: Plan.getConstantInt(Val: *StrideConst),
2849 ShouldReplace: CanUseVersionedStride);
2850
2851 // The versioned value may not be used in the loop directly but through an
2852 // integral cast (sext/zext/trunc). Add new live-ins in those cases.
2853 for (Value *U : StrideV->users()) {
2854 if (!isa<SExtInst, ZExtInst, TruncInst>(Val: U))
2855 continue;
2856 VPValue *StrideVPV = Plan.getLiveIn(V: U);
2857 if (!StrideVPV)
2858 continue;
2859 unsigned BW = U->getType()->getScalarSizeInBits();
2860 APInt C = isa<SExtInst>(Val: U) ? StrideConst->sext(width: BW)
2861 : StrideConst->zextOrTrunc(width: BW);
2862 StrideVPV->replaceUsesWithIf(New: Plan.getConstantInt(Val: C),
2863 ShouldReplace: CanUseVersionedStride);
2864 }
2865 RewriteMap[StrideV] = StrideExpr;
2866 }
2867
2868 ScalarEvolution &SE = *PSE.getSE();
2869 for (VPExpandSCEVRecipe &ExpSCEV :
2870 make_isa_range<VPExpandSCEVRecipe>(Range&: *Plan.getEntry())) {
2871 const SCEV *ScevExpr = ExpSCEV.getSCEV();
2872 const SCEV *NewSCEV =
2873 SCEVParameterRewriter::rewrite(Scev: ScevExpr, SE, Map&: RewriteMap);
2874 if (NewSCEV == ScevExpr || !SE.isGuaranteedNotToCauseUB(Op: NewSCEV))
2875 continue;
2876 VPValue *NewExp = vputils::getOrCreateVPValueForSCEVExpr(Plan, Expr: NewSCEV);
2877 ExpSCEV.replaceAllUsesWith(New: NewExp);
2878 if (Plan.getTripCount() == &ExpSCEV)
2879 Plan.resetTripCount(NewTripCount: NewExp);
2880 }
2881}
2882
2883void VPlanTransforms::dropPoisonGeneratingRecipes(VPlan &Plan) {
2884 // Collect recipes in the backward slice of `Root` that may generate a poison
2885 // value that is used after vectorization.
2886 SmallPtrSet<VPRecipeBase *, 16> Visited;
2887 auto CollectPoisonGeneratingInstrsInBackwardSlice([&](VPRecipeBase *Root) {
2888 SmallVector<VPRecipeBase *, 16> Worklist;
2889 Worklist.push_back(Elt: Root);
2890
2891 // Traverse the backward slice of Root through its use-def chain.
2892 while (!Worklist.empty()) {
2893 VPRecipeBase *CurRec = Worklist.pop_back_val();
2894
2895 if (!Visited.insert(Ptr: CurRec).second)
2896 continue;
2897
2898 // Prune search if we find another recipe generating a widen memory
2899 // instruction. Widen memory instructions involved in address computation
2900 // will lead to gather/scatter instructions, which don't need to be
2901 // handled.
2902 if (isa<VPWidenMemoryRecipe, VPInterleaveRecipe, VPScalarIVStepsRecipe,
2903 VPHeaderPHIRecipe>(Val: CurRec))
2904 continue;
2905
2906 // This recipe contributes to the address computation of a widen
2907 // load/store. If the underlying instruction has poison-generating flags,
2908 // drop them directly.
2909 if (auto *RecWithFlags = dyn_cast<VPRecipeWithIRFlags>(Val: CurRec)) {
2910 VPValue *A, *B;
2911 // Dropping disjoint from an OR may yield incorrect results, as some
2912 // analysis may have converted it to an Add implicitly (e.g. SCEV used
2913 // for dependence analysis). Instead, replace it with an equivalent Add.
2914 // This is possible as all users of the disjoint OR only access lanes
2915 // where the operands are disjoint or poison otherwise.
2916 if (match(V: RecWithFlags, P: m_BinaryOr(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))) &&
2917 RecWithFlags->isDisjoint()) {
2918 VPBuilder Builder(RecWithFlags);
2919 VPInstruction *New =
2920 Builder.createAdd(LHS: A, RHS: B, DL: RecWithFlags->getDebugLoc());
2921 New->setUnderlyingValue(RecWithFlags->getUnderlyingValue());
2922 RecWithFlags->replaceAllUsesWith(New);
2923 RecWithFlags->eraseFromParent();
2924 CurRec = New;
2925 } else
2926 RecWithFlags->dropPoisonGeneratingFlags();
2927 } else {
2928 Instruction *Instr = dyn_cast_or_null<Instruction>(
2929 Val: CurRec->getVPSingleValue()->getUnderlyingValue());
2930 (void)Instr;
2931 assert((!Instr || !Instr->hasPoisonGeneratingFlags()) &&
2932 "found instruction with poison generating flags not covered by "
2933 "VPRecipeWithIRFlags");
2934 }
2935
2936 // Add new definitions to the worklist.
2937 for (VPValue *Operand : CurRec->operands())
2938 if (VPRecipeBase *OpDef = Operand->getDefiningRecipe())
2939 Worklist.push_back(Elt: OpDef);
2940 }
2941 });
2942
2943 // We want to exclude the tail folding case, as we don't need to drop flags
2944 // for operations computing the first lane in this case: the first lane of the
2945 // header mask must always be true. For reverse memory accesses, the mask is
2946 // wrapped in a Reverse, which is just a permutation of the header mask, so
2947 // peel it off before checking. The header mask is still the abstract region
2948 // value at this point (materialization happens later).
2949 auto m_UnlessHdrMask = m_Unless( // NOLINT
2950 P: m_CombineOr(Ps: m_HeaderMask(), Ps: m_Reverse(Op0: m_HeaderMask())));
2951
2952 // Traverse all the recipes in the VPlan and collect the poison-generating
2953 // recipes in the backward slice starting at the address of a VPWidenRecipe or
2954 // VPInterleaveRecipe.
2955 auto Iter =
2956 vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntryBasicBlock());
2957 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: Iter)) {
2958 for (VPRecipeBase &Recipe : *VPBB) {
2959 if (auto *WidenRec = dyn_cast<VPWidenMemoryRecipe>(Val: &Recipe)) {
2960 VPRecipeBase *AddrDef = WidenRec->getAddr()->getDefiningRecipe();
2961 if (AddrDef && WidenRec->isConsecutive() && WidenRec->getMask() &&
2962 match(V: WidenRec->getMask(), P: m_UnlessHdrMask))
2963 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2964 } else if (auto *InterleaveRec = dyn_cast<VPInterleaveRecipe>(Val: &Recipe)) {
2965 VPRecipeBase *AddrDef = InterleaveRec->getAddr()->getDefiningRecipe();
2966 if (AddrDef && InterleaveRec->getMask() &&
2967 match(V: InterleaveRec->getMask(), P: m_UnlessHdrMask))
2968 CollectPoisonGeneratingInstrsInBackwardSlice(AddrDef);
2969 }
2970 }
2971 }
2972}
2973
2974void VPlanTransforms::createInterleaveGroups(
2975 VPlan &Plan,
2976 const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
2977 &InterleaveGroups,
2978 const bool &EpilogueAllowed) {
2979 if (InterleaveGroups.empty())
2980 return;
2981
2982 DenseMap<Instruction *, VPWidenMemoryRecipe *> IRMemberToRecipe;
2983 for (VPBasicBlock *VPBB :
2984 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
2985 G: Plan.getVectorLoopRegion()->getEntryBasicBlock())))
2986 for (VPRecipeBase &R : make_filter_range(Range&: *VPBB, Pred: [](VPRecipeBase &R) {
2987 return isa<VPWidenMemoryRecipe>(Val: &R);
2988 })) {
2989 auto *MemR = cast<VPWidenMemoryRecipe>(Val: &R);
2990 IRMemberToRecipe[&MemR->getIngredient()] = MemR;
2991 }
2992
2993 // Interleave memory: for each Interleave Group we marked earlier as relevant
2994 // for this VPlan, replace the Recipes widening its memory instructions with a
2995 // single VPInterleaveRecipe at its insertion point.
2996 VPDominatorTree VPDT(Plan);
2997 for (const auto *IG : InterleaveGroups) {
2998 VPWidenMemoryRecipe *Start = nullptr;
2999 Instruction *StartMember = nullptr;
3000 for (auto *Member : IG->members())
3001 if (VPWidenMemoryRecipe *R = IRMemberToRecipe.lookup(Val: Member)) {
3002 StartMember = Member;
3003 Start = R;
3004 break;
3005 }
3006 if (!StartMember) // All member recipes are dead, so the group is dead.
3007 continue;
3008 VPIRMetadata InterleaveMD(*Start);
3009 SmallVector<VPValue *, 4> StoredValues;
3010 for (unsigned I = 0; I < IG->getFactor(); ++I) {
3011 Instruction *MemberI = IG->getMember(Index: I);
3012 if (!MemberI)
3013 continue;
3014 if (VPWidenMemoryRecipe *MemoryR = IRMemberToRecipe.lookup(Val: MemberI)) {
3015 if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Val: MemoryR->getAsRecipe()))
3016 StoredValues.push_back(Elt: StoreR->getStoredValue());
3017 InterleaveMD.intersect(MD: *MemoryR);
3018 } else {
3019 InterleaveMD.intersect(MD: VPIRMetadata(*MemberI));
3020 }
3021 }
3022
3023 bool NeedsMaskForGaps =
3024 (IG->requiresScalarEpilogue() && !EpilogueAllowed) ||
3025 (!StoredValues.empty() && !IG->isFull());
3026
3027 Instruction *IRInsertPos = IG->getInsertPos();
3028 auto *InsertPos = IRMemberToRecipe.lookup(Val: IRInsertPos);
3029 if (!InsertPos) {
3030 // InsertPos member is dead: find a new member that is alive.
3031 assert(isa<VPWidenLoadRecipe>(Start->getAsRecipe()) &&
3032 "Dead member in non-load group?");
3033 InsertPos = Start;
3034 for (Instruction *Member : IG->members())
3035 if (VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member))
3036 if (VPDT.properlyDominates(A: MemberR->getAsRecipe(),
3037 B: InsertPos->getAsRecipe()))
3038 InsertPos = MemberR;
3039 IRInsertPos = &InsertPos->getIngredient();
3040 }
3041 VPRecipeBase *InsertPosR = InsertPos->getAsRecipe();
3042
3043 GEPNoWrapFlags NW = GEPNoWrapFlags::none();
3044 if (auto *Gep = dyn_cast<GetElementPtrInst>(
3045 Val: getLoadStorePointerOperand(V: IRInsertPos)->stripPointerCasts()))
3046 NW = Gep->getNoWrapFlags().withoutNoUnsignedWrap();
3047
3048 // Get or create the start address for the interleave group.
3049 VPValue *Addr = Start->getAddr();
3050 VPRecipeBase *AddrDef = Addr->getDefiningRecipe();
3051 if (IG->getIndex(Instr: StartMember) != 0 ||
3052 (AddrDef && !VPDT.properlyDominates(A: AddrDef, B: InsertPosR))) {
3053 // Either member zero's recipe is dead, or we cannot re-use the address of
3054 // member zero because it does not dominate the insert position. Instead,
3055 // use the address of the insert position and create a PtrAdd adjusting it
3056 // to the address of member zero.
3057 // TODO: Hoist Addr's defining recipe (and any operands as needed) to
3058 // InsertPos or sink loads above zero members to join it.
3059 assert(IG->getIndex(IRInsertPos) != 0 &&
3060 "index of insert position shouldn't be zero");
3061 auto &DL = IRInsertPos->getDataLayout();
3062 APInt Offset(32,
3063 DL.getTypeAllocSize(Ty: getLoadStoreType(I: IRInsertPos)) *
3064 IG->getIndex(Instr: IRInsertPos),
3065 /*IsSigned=*/true);
3066 VPValue *OffsetVPV = Plan.getConstantInt(Val: -Offset);
3067 VPBuilder B(InsertPosR);
3068 Addr = B.createNoWrapPtrAdd(Ptr: InsertPos->getAddr(), Offset: OffsetVPV, GEPFlags: NW);
3069 }
3070 // If the group is reverse, adjust the index to refer to the last vector
3071 // lane instead of the first. We adjust the index from the first vector
3072 // lane, rather than directly getting the pointer for lane VF - 1, because
3073 // the pointer operand of the interleaved access is supposed to be uniform.
3074 if (IG->isReverse()) {
3075 auto *ReversePtr = new VPVectorEndPointerRecipe(
3076 Addr, &Plan.getVF(), getLoadStoreType(I: IRInsertPos),
3077 -(int64_t)IG->getFactor(), NW, InsertPosR->getDebugLoc());
3078 ReversePtr->insertBefore(InsertPos: InsertPosR);
3079 Addr = ReversePtr;
3080 }
3081 auto *VPIG = new VPInterleaveRecipe(
3082 IG, Addr, StoredValues, InsertPos->getMask(), NeedsMaskForGaps,
3083 InterleaveMD, InsertPosR->getDebugLoc());
3084 VPIG->insertBefore(InsertPos: InsertPosR);
3085
3086 unsigned J = 0;
3087 for (unsigned i = 0; i < IG->getFactor(); ++i)
3088 if (Instruction *Member = IG->getMember(Index: i)) {
3089 VPWidenMemoryRecipe *MemberR = IRMemberToRecipe.lookup(Val: Member);
3090 if (!Member->getType()->isVoidTy()) {
3091 if (MemberR) {
3092 VPValue *OriginalV = MemberR->getAsRecipe()->getVPSingleValue();
3093 OriginalV->replaceAllUsesWith(New: VPIG->getVPValue(I: J));
3094 }
3095 J++;
3096 }
3097 if (MemberR)
3098 MemberR->getAsRecipe()->eraseFromParent();
3099 }
3100 }
3101}
3102
3103/// Matches an exit condition formed by comparing a value loaded from memory
3104/// with a loop-invariant term. Binds the comparison for the condition.
3105static auto m_Uncountable(VPValue *&Cond) {
3106 return m_VPValue(
3107 V&: Cond,
3108 Op: m_c_Cmp(Op0: m_VPInstruction<Instruction::Load>(Ops: m_VPValue()), Op1: m_LiveIn()));
3109}
3110
3111namespace {
3112struct CountableConditionMatch {
3113 VPValue *&Cmp;
3114 PredicatedScalarEvolution &PSE;
3115 Loop *L;
3116
3117 CountableConditionMatch(VPValue *&Cmp, PredicatedScalarEvolution &PSE,
3118 Loop *L)
3119 : Cmp(Cmp), PSE(PSE), L(L) {}
3120
3121 template <typename ITy> bool match(ITy *V) const {
3122 VPValue *Update;
3123 if (!VPlanPatternMatch::match(
3124 V, m_VPValue(V&: Cmp, Op: m_c_ICmp(Op0: m_VPValue(V&: Update, Op: m_Add(Op0: m_VPValue(),
3125 Op1: m_VPValue())),
3126 Op1: m_LiveIn()))))
3127 return false;
3128
3129 const SCEV *S = vputils::getSCEVExprForVPValue(V: Update, PSE, L);
3130 return SCEVPatternMatch::match(
3131 S, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_scev_One(), L: m_SpecificLoop(L)));
3132 }
3133};
3134} // end anonymous namespace
3135
3136/// Matches an exit condition formed by comparing the current value of a
3137/// affine add recurrence in the given loop with a stride of 1 against a
3138/// loop-invariant term. Binds the comparison for the condition.
3139static auto m_Countable(VPValue *&Cmp, PredicatedScalarEvolution &PSE,
3140 Loop *L) {
3141 return CountableConditionMatch(Cmp, PSE, L);
3142}
3143
3144bool VPlanTransforms::splitCombinedExits(VPlan &Plan,
3145 PredicatedScalarEvolution &PSE,
3146 Loop *L) {
3147 // Check for a single combined exit in the latch block.
3148 // TODO: Generalize to other blocks besides the latch.
3149 // If we don't find a combined condition in the latch, just return true
3150 // to proceed with vectorization.
3151 auto [_, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
3152
3153 // We're looking for a conditional branch...
3154 auto *Term = dyn_cast<VPInstruction>(Val: LatchVPBB->getTerminator());
3155 if (!Term || Term->getOpcode() != VPInstruction::BranchOnCond)
3156 return true;
3157
3158 // ...where the condition is a combination of both a countable and an
3159 // uncountable comparison.
3160 VPValue *Uncountable = nullptr;
3161 VPValue *Countable = nullptr;
3162 VPValue *Cond = Term->getOperand(N: 0);
3163 if (!match(V: Cond, P: m_CombineOr(Ps: m_c_LogicalOr(Op0: m_Uncountable(Cond&: Uncountable),
3164 Op1: m_Countable(Cmp&: Countable, PSE, L)),
3165 Ps: m_c_BinaryOr(Op0: m_Uncountable(Cond&: Uncountable),
3166 Op1: m_Countable(Cmp&: Countable, PSE, L)))))
3167 return true;
3168
3169 // If the conditions are combined with a logical or (select), then we'll
3170 // need to freeze the individual terms when splitting.
3171 bool NeedsFreeze = match(V: Cond, P: m_LogicalOr(Op0: m_VPValue(), Op1: m_VPValue()));
3172
3173 // If we do have a combined exit condition, bail out if there's more than
3174 // one exit block.
3175 // TODO: Support additional exits.
3176 ArrayRef<VPIRBasicBlock *> ExitBlocks = Plan.getExitBlocks();
3177 if (ExitBlocks.size() != 1)
3178 return false;
3179
3180 // If there are any live-outs, bail out. The exit block is an existing IR
3181 // block, and if we split the exiting block then the incoming blocks and
3182 // values won't be correct.
3183 // TODO: Support live-outs with combined exits.
3184 if (!ExitBlocks.front()->phis().empty())
3185 return false;
3186
3187 // Split the latch block just before the terminator.
3188 VPBasicBlock *NewLatch = LatchVPBB->splitAt(SplitAt: Term->getIterator());
3189
3190 // Create new terminator for uncountable condition.
3191 VPBuilder EEBuilder(LatchVPBB);
3192 if (NeedsFreeze)
3193 Uncountable = EEBuilder.createFreeze(Op: Uncountable);
3194 EEBuilder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {Uncountable});
3195
3196 // We need to connect the uncountable exit to the sole exit block. The
3197 // latch is expected to connect to the middle block instead.
3198 // In canonical form, the backedge is the last successor for the latch. So
3199 // the first successor (true path) should be the exit for both conditions.
3200 LatchVPBB->clearSuccessors();
3201 NewLatch->clearPredecessors();
3202 VPBlockUtils::connectBlocks(From: LatchVPBB, To: ExitBlocks.front());
3203 VPBlockUtils::connectBlocks(From: LatchVPBB, To: NewLatch);
3204
3205 // Set condition for latch block to countable condition.
3206 if (NeedsFreeze) {
3207 VPBuilder NewLatchBuilder(Term);
3208 Countable = NewLatchBuilder.createFreeze(Op: Countable);
3209 }
3210 Term->setOperand(I: 0, New: Countable);
3211
3212 return true;
3213}
3214
3215/// Returns the VPValue representing the uncountable exit comparison used by
3216/// AnyOf if the recipes it depends on can be traced back to live-ins and
3217/// the addresses (in GEP/PtrAdd form) of any (non-masked) load used in
3218/// generating the values for the comparison. The recipes are stored in
3219/// \p Recipes.
3220static VPValue *
3221getRecipesForUncountableExit(SmallVectorImpl<VPInstruction *> &Recipes,
3222 VPBasicBlock *LatchVPBB) {
3223 // Given a plain CFG VPlan loop with countable latch exiting block
3224 // \p LatchVPBB, we're looking to match the recipes contributing to the
3225 // uncountable exit condition comparison (here, vp<%4>) back to either
3226 // live-ins or the address nodes for the load used as part of the uncountable
3227 // exit comparison so that we can either move them within the loop, or copy
3228 // them to the preheader depending on the chosen method for dealing with
3229 // stores in uncountable exit loops.
3230 //
3231 // Currently, the address of the load is restricted to a GEP with 2 operands
3232 // and a live-in base address. This constraint may be relaxed later.
3233 //
3234 // VPlan ' for UF>=1' {
3235 // Live-in vp<%0> = VF * UF
3236 // Live-in vp<%1> = vector-trip-count
3237 // Live-in ir<20> = original trip-count
3238 //
3239 // ir-bb<entry>:
3240 // Successor(s): scalar.ph, vector.ph
3241 //
3242 // vector.ph:
3243 // Successor(s): for.body
3244 //
3245 // for.body:
3246 // EMIT vp<%2> = phi ir<0>, vp<%index.next>
3247 // EMIT-SCALAR ir<%iv> = phi [ ir<0>, vector.ph ], [ ir<%iv.next>, for.inc ]
3248 // EMIT ir<%uncountable.addr> = getelementptr inbounds nuw ir<%pred>,ir<%iv>
3249 // EMIT ir<%uncountable.val> = load ir<%uncountable.addr>
3250 // EMIT ir<%uncountable.cond> = icmp sgt ir<%uncountable.val>, ir<500>
3251 // EMIT vp<%3> = masked-cond ir<%uncountable.cond>
3252 // Successor(s): for.inc
3253 //
3254 // for.inc:
3255 // EMIT ir<%iv.next> = add nuw nsw ir<%iv>, ir<1>
3256 // EMIT ir<%countable.cond> = icmp eq ir<%iv.next>, ir<20>
3257 // EMIT vp<%index.next> = add nuw vp<%2>, vp<%0>
3258 // EMIT vp<%freeze> = freeze ir<%3>
3259 // EMIT vp<%4> = any-of ir<%freeze>
3260 // EMIT vp<%5> = icmp eq vp<%index.next>, vp<%1>
3261 // EMIT branch-on-two-conds vp<%4>, vp<%5>
3262 // Successor(s): middle.block, middle.block, for.body
3263 //
3264 // middle.block:
3265 // Successor(s): ir-bb<exit>, scalar.ph
3266 //
3267 // ir-bb<exit>:
3268 // No successors
3269 //
3270 // scalar.ph:
3271 // }
3272
3273 // Find the uncountable loop exit condition.
3274 VPValue *UncountableCondition = nullptr;
3275 if (!match(V: LatchVPBB->getTerminator(),
3276 P: m_BranchOnTwoConds(Op0: m_AnyOf(Op0: m_VPValue(V&: UncountableCondition)),
3277 Op1: m_VPValue())))
3278 return nullptr;
3279
3280 SmallVector<VPValue *, 4> Worklist;
3281 Worklist.push_back(Elt: UncountableCondition);
3282 while (!Worklist.empty()) {
3283 VPValue *V = Worklist.pop_back_val();
3284
3285 // Any value defined outside the loop does not need to be copied.
3286 if (V->isDefinedOutsideLoopRegions())
3287 continue;
3288
3289 VPValue *Op1, *Op2;
3290 // Walk back through recipes until we find at least one load from memory.
3291 if (match(V, P: m_Cmp(Op0: m_VPValue(V&: Op1), Op1: m_VPValue(V&: Op2)))) {
3292 Worklist.push_back(Elt: Op1);
3293 Worklist.push_back(Elt: Op2);
3294 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3295 } else if (match(V, P: m_VPInstruction<Instruction::Load>(Ops: m_VPValue(V&: Op1)))) {
3296 VPRecipeBase *GepR = Op1->getDefiningRecipe();
3297 // Only matching base + single offset term for now.
3298 if (GepR->getNumOperands() != 2)
3299 return nullptr;
3300 // Matching a GEP with a loop-invariant base ptr.
3301 if (!match(V: GepR, P: m_VPInstruction<Instruction::GetElementPtr>(
3302 Ops: m_LiveIn(), Ops: m_VPValue())))
3303 return nullptr;
3304 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3305 Recipes.push_back(Elt: cast<VPInstruction>(Val: GepR));
3306 } else if (match(V, P: m_Freeze(Op0: m_VPValue(V&: Op1)))) {
3307 Worklist.push_back(Elt: Op1);
3308 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3309 } else if (match(V, P: m_VPInstruction<VPInstruction::MaskedCond>(
3310 Ops: m_VPValue(V&: Op1)))) {
3311 Worklist.push_back(Elt: Op1);
3312 Recipes.push_back(Elt: cast<VPInstruction>(Val: V->getDefiningRecipe()));
3313 } else
3314 return nullptr;
3315 }
3316
3317 // If we couldn't match anything, don't return the condition. It may be
3318 // defined outside the loop.
3319 if (Recipes.empty() ||
3320 none_of(Range&: Recipes, P: match_fn(P: m_VPInstruction<Instruction::GetElementPtr>())))
3321 return nullptr;
3322
3323 return UncountableCondition;
3324}
3325
3326struct EarlyExitInfo {
3327 VPBasicBlock *EarlyExitingVPBB;
3328 VPIRBasicBlock *EarlyExitVPBB;
3329 VPValue *CondToExit;
3330};
3331
3332/// Update \p Plan to mask memory operations in the loop based on whether the
3333/// early exit is taken or not.
3334///
3335/// We're currently expecting to find a loop with properties similar to the
3336/// following:
3337///
3338/// for.body:
3339/// ir<%indvars.iv> = WIDEN-INDUCTION nuw nsw ir<0>, ir<1>, vp<%0>
3340/// EMIT ir<%arrayidx> = getelementptr inbounds nuw ir<@c>, ir<%indvars.iv>
3341/// EMIT-SCALAR ir<%0> = load ir<%arrayidx>
3342/// EMIT ir<%cmp1> = icmp sgt ir<%0>, ir<5>
3343/// EMIT vp<%1> = masked-cond ir<%cmp1>
3344/// Successor(s): if.end
3345///
3346/// if.end:
3347/// EMIT ir<%arrayidx3> = getelementptr inbounds nuw ir<@src>, ir<%indvars.iv>
3348/// EMIT-SCALAR ir<%2> = load ir<%arrayidx3>
3349/// EMIT ir<%add> = add nsw ir<%2>, ir<42>
3350/// EMIT ir<%arrayidx5> = getelementptr inbounds nuw ir<@dst>, ir<%indvars.iv>
3351/// EMIT store ir<%add>, ir<%arrayidx5>
3352/// EMIT ir<%indvars.iv.next> = add nuw nsw ir<%indvars.iv>, ir<1>
3353/// EMIT vp<%freeze> = freeze ir<%1>
3354/// EMIT vp<%3> = any-of ir<%freeze>
3355/// EMIT ir<%exitcond.not> = icmp eq ir<%indvars.iv.next>, ir<10000>
3356/// EMIT branch-on-two-conds vp<%3>, ir<%exitcond.not>
3357/// Successor(s): middle.block, middle.block, for.body
3358///
3359/// We currently expect LoopVectorizationLegality to ensure that:
3360/// * There must also be a counted exit. We will need to support speculative
3361/// or first-faulting loads before we can remove this restriction.
3362/// * Any stores within the loop must not alias with the load used for the
3363/// uncountable exit. We can relax this a bit with runtime aliasing checks.
3364/// * Other memory operations in the loop can take place before or after the
3365/// uncountable exit, but must also be unconditional. We need to support
3366/// combining the conditions in VPlanPredicator.
3367/// * The loop must have a single unconditional load contributing to the
3368/// uncountable exit comparison, and the other term must be loop-invariant.
3369/// Improving upon this requires work in getRecipesForUncountableExit to
3370/// handle more complex recipe graphs.
3371static bool handleUncountableExitsWithSideEffects(
3372 VPlan &Plan, SmallVectorImpl<EarlyExitInfo> &Exits,
3373 VPBasicBlock *HeaderVPBB, VPBasicBlock *LatchVPBB, VPBasicBlock *MiddleVPBB,
3374 OptimizationRemarkEmitter *ORE, Loop *TheLoop,
3375 PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC) {
3376
3377 // Disconnect early exiting blocks from successors, remove branches.
3378 for (auto &Exit : Exits) {
3379 if (Exit.EarlyExitingVPBB == LatchVPBB)
3380 continue;
3381
3382 for (VPRecipeBase &R : Exit.EarlyExitVPBB->phis())
3383 cast<VPIRPhi>(Val: &R)->removeIncomingValueFor(IncomingBlock: Exit.EarlyExitingVPBB);
3384 Exit.EarlyExitingVPBB->getTerminator()->eraseFromParent();
3385 VPBlockUtils::disconnectBlocks(From: Exit.EarlyExitingVPBB, To: Exit.EarlyExitVPBB);
3386 }
3387
3388 VPDominatorTree VPDT(Plan);
3389
3390 // We can abandon a VPlan entirely if we return false here, so we shouldn't
3391 // crash if some earlier assumptions on scalar IR don't hold for the vplan
3392 // version of the loop.
3393 SmallVector<VPInstruction *, 8> ConditionRecipes;
3394
3395 VPValue *Cond = getRecipesForUncountableExit(Recipes&: ConditionRecipes, LatchVPBB);
3396 if (!Cond) {
3397 reportVectorizationFailure(DebugMsg: "Unable to determine early exit condition for "
3398 "loop with side effects",
3399 ORETag: "EarlyExitSideEffectsCond", ORE, TheLoop);
3400 return false;
3401 }
3402
3403 // Find load contributing to condition.
3404 // At the moment LoopVectorizationLegality only supports a single
3405 // early-exit expression with a compare and a single load that must
3406 // be unconditional.
3407 // TODO: Support more than one load.
3408 auto *Load =
3409 find_singleton<VPInstruction>(Range&: ConditionRecipes, P: [](auto *I, bool _) {
3410 return match(I, m_VPInstruction<Instruction::Load>(Ops: m_VPValue()))
3411 ? I
3412 : nullptr;
3413 });
3414 assert(Load && "Couldn't find exactly one load");
3415 // TODO: Support conditional loads for uncountable exits.
3416 assert(VPDT.dominates(Load->getParent(), LatchVPBB) &&
3417 "Uncountable exit condition load is conditional.");
3418 VPInstruction *Ptr = cast<VPInstruction>(Val: Load->getOperand(N: 0));
3419
3420 // Ensure that we are guaranteed to be able to dereference the memory used
3421 // for determining the uncountable exit for the maximum possible number of
3422 // scalar iterations of the loop.
3423 //
3424 // TODO: Support first-faulting loads in cases where we don't know whether
3425 // all possible addresses are dereferenceable.
3426 {
3427 SmallVector<const SCEVPredicate *, 4> Predicates;
3428 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: TheLoop);
3429 const DataLayout &DL = Plan.getDataLayout();
3430 APInt EltSize(DL.getIndexTypeSizeInBits(Ty: Ptr->getScalarType()),
3431 DL.getTypeStoreSize(Ty: Load->getScalarType()).getFixedValue());
3432 if (!isDereferenceableAndAlignedInLoop(
3433 PtrSCEV, Alignment: cast<LoadInst>(Val: Load->getUnderlyingInstr())->getAlign(),
3434 EltSizeSCEV: PSE.getSE()->getConstant(Val: EltSize), L: TheLoop, SE&: *PSE.getSE(), DT, AC,
3435 Predicates: &Predicates)) {
3436 reportVectorizationFailure(DebugMsg: "Early exit loop with side effects contains "
3437 "load used by the exit condition that may "
3438 "fault",
3439 ORETag: "EarlyExitSideEffectsFaultingLoad", ORE,
3440 TheLoop);
3441 return false;
3442 }
3443 }
3444
3445 // Check for a single GEP for the condition load to see if we can link it to
3446 // a widen IV recipe with a step of 1; we're only interested in contiguous
3447 // accesses for the condition load right now.
3448 auto *IV = cast<VPWidenInductionRecipe>(Val: &HeaderVPBB->front());
3449 if (!match(V: IV->getStartValue(), P: m_SpecificInt(V: 0)) ||
3450 !match(V: IV->getStepValue(), P: m_SpecificInt(V: 1))) {
3451 reportVectorizationFailure(DebugMsg: "Early exit loop with side effects contains "
3452 "load used by the exit condition with an "
3453 "unsupported memory access pattern",
3454 ORETag: "EarlyExitSideEffectsBadLoadAccessPattern", ORE,
3455 TheLoop);
3456 return false;
3457 }
3458
3459 if (!match(V: Ptr, P: m_VPInstruction<Instruction::GetElementPtr>(
3460 Ops: m_LiveIn(), Ops: m_Specific(VPV: IV)))) {
3461 reportVectorizationFailure(DebugMsg: "Early exit loop with side effects contains "
3462 "load used by the exit condition with an "
3463 "unsupported memory access pattern",
3464 ORETag: "EarlyExitSideEffectsBadLoadAccessPattern", ORE,
3465 TheLoop);
3466 return false;
3467 }
3468
3469 // We want to guarantee that the uncountable exit condition (and the mask
3470 // we will generate from it) are available for all operations in the loop
3471 // that need to be masked. If the condition recipes are not already the first
3472 // recipes in the header after the last phi, move them there.
3473 auto InsertIt = HeaderVPBB->getFirstNonPhi();
3474 while (InsertIt != HeaderVPBB->end() &&
3475 is_contained(Range&: ConditionRecipes, Element: &*InsertIt)) {
3476 erase(C&: ConditionRecipes, V: &*InsertIt);
3477 InsertIt++;
3478 }
3479 for (auto *Recipe : reverse(C&: ConditionRecipes))
3480 Recipe->moveBefore(I: InsertIt);
3481
3482 // Create a mask to represent all lanes that fully execute in the vector loop,
3483 // stopping short of any early exit.
3484 VPBuilder MaskBuilder(InsertIt);
3485 VPValue *Mask = MaskBuilder.createWidenIntrinsic(VectorIntrinsicID: Intrinsic::mask_beforefirst,
3486 CallArguments: Cond, Ty: Cond->getScalarType());
3487
3488 // Convert all other memory operations to use the mask.
3489 for (VPBasicBlock *VPBB : vp_rpo_plain_cfg_loop_body(Header: HeaderVPBB))
3490 for (VPRecipeBase &R : *VPBB)
3491 if (R.mayReadOrWriteMemory() && &R != Load) {
3492 // TODO: Handle conditional memory operations in the loop.
3493 if (!VPDT.dominates(A: R.getParent(), B: LatchVPBB)) {
3494 reportVectorizationFailure(
3495 DebugMsg: "Early exit loop with side effects contains unsupported "
3496 "conditional memory operations",
3497 ORETag: "EarlyExitSideEffectsUnsupportedConditionalMemOps", ORE, TheLoop);
3498 return false;
3499 }
3500 cast<VPInstruction>(Val: &R)->addMask(Mask);
3501 }
3502
3503 // Update middle block branch to compare (IV + however many lanes were active)
3504 // against the full trip count, since we may be exiting the vector loop early.
3505 // If we didn't take an early exit, we should get the equivalent of VF from
3506 // the FirstActiveLane.
3507 assert(match(MiddleVPBB->getTerminator(), m_BranchOnCond()) &&
3508 "Expected BranchOnCond terminator for MiddleVPBB");
3509 VPBuilder MiddleBuilder(MiddleVPBB->getTerminator());
3510 Type *IVScalarTy = IV->getScalarType();
3511 VPValue *Zero = Plan.getZero(Ty: IVScalarTy);
3512 VPValue *FirstActive = MiddleBuilder.createScalarZExtOrTrunc(
3513 Op: MiddleBuilder.createFirstActiveLane(Masks: Cond), ResultTy: IVScalarTy, DL: DebugLoc());
3514 VPValue *ScalarIV = MiddleBuilder.createNaryOp(Opcode: VPInstruction::ExtractLane,
3515 Operands: {Zero, IV}, DL: DebugLoc());
3516 VPValue *ExitIV = MiddleBuilder.createAdd(LHS: ScalarIV, RHS: FirstActive);
3517 VPValue *FullTC =
3518 MiddleBuilder.createICmp(Pred: CmpInst::ICMP_EQ, A: ExitIV, B: Plan.getTripCount());
3519 MiddleVPBB->getTerminator()->setOperand(I: 0, New: FullTC);
3520
3521 // Update resume phi in scalar.ph.
3522 VPBasicBlock *ScalarPH = Plan.getScalarPreheader();
3523 auto Phis = ScalarPH->phis();
3524 // TODO: Handle more than one Phi; re-derive from IV.
3525 // TODO: Handle reductions.
3526 if (range_size(Range&: Phis) != 1) {
3527 reportVectorizationFailure(
3528 DebugMsg: "Early exit loop with side effects contains "
3529 "unsupported reductions, inductions or recurrences",
3530 ORETag: "EarlyExitSideEffectsReductions", ORE, TheLoop);
3531 return false;
3532 }
3533 VPPhi *ContinueIV = cast<VPPhi>(Val: Phis.begin());
3534 // Make sure we're referring to the same IV.
3535 assert(
3536 match(ContinueIV->getOperand(0),
3537 m_VPInstruction<VPInstruction::ExitingIVValue>(m_Specific(IV))) &&
3538 "Continuing from different IV");
3539 ContinueIV->setOperand(I: 0, New: ExitIV);
3540 return true;
3541}
3542
3543// Returns true if the loop contains stores, the only writes legality accepts in
3544// early-exit loops.
3545static bool loopHasStores(VPBasicBlock *HeaderVPBB) {
3546 return any_of(Range: vp_rpo_plain_cfg_loop_body(Header: HeaderVPBB), P: [](VPBasicBlock *VPBB) {
3547 return any_of(Range&: *VPBB, P: [VPBB](VPRecipeBase &R) {
3548 (void)VPBB;
3549 if (match(V: &R, P: m_Intrinsic<Intrinsic::pseudoprobe>()))
3550 return false;
3551 if (!R.mayWriteToMemory()) {
3552 assert((!R.mayHaveSideEffects() || &R == VPBB->getTerminator() ||
3553 R.isSafeToSpeculativelyExecute()) &&
3554 "legality only accepts recipes with side effects that are "
3555 "safe to speculate");
3556 return false;
3557 }
3558 assert(match(&R, m_VPInstruction<Instruction::Store>()) &&
3559 "legality only accepts stores as writes in early-exit loops");
3560 return true;
3561 });
3562 });
3563}
3564
3565bool VPlanTransforms::handleUncountableEarlyExits(
3566 VPlan &Plan, OptimizationRemarkEmitter *ORE, Loop *TheLoop,
3567 PredicatedScalarEvolution &PSE, DominatorTree &DT, AssumptionCache *AC,
3568 UncountableExitStyle Style) {
3569#ifndef NDEBUG
3570 VPDominatorTree VPDT(Plan);
3571#endif
3572
3573 auto *MiddleVPBB = VPBlockUtils::getPlainCFGMiddleBlock(Plan);
3574 auto [HeaderVPBB, LatchVPBB] = VPBlockUtils::getPlainCFGHeaderAndLatch(Plan);
3575
3576 if (loopHasStores(HeaderVPBB))
3577 Style = UncountableExitStyle::MaskedHandleExitInScalarLoop;
3578
3579 // Dereferenceability is checked separately for uncountable exit loops with
3580 // stores, as only the loads contributing to the exit condition need to
3581 // be checked.
3582 if (Style == UncountableExitStyle::ReadOnly &&
3583 !areAllLoadsDereferenceable(HeaderVPBB, TheLoop, PSE, DT, AC)) {
3584 reportVectorizationFailure(
3585 DebugMsg: "Auto-vectorization of early exit loops with potentially "
3586 "faulting loads is not supported",
3587 ORETag: "EarlyExitFaultingLoads", ORE, TheLoop);
3588 return false;
3589 }
3590
3591 VPBuilder LatchBuilder(LatchVPBB->getTerminator());
3592 SmallVector<EarlyExitInfo> Exits;
3593 for (auto [EarlyExitingVPBB, ExitBlock] :
3594 vputils::getEarlyExits(Plan, MiddleVPBB)) {
3595 // The final value of a reduction is computed in the middle block, which is
3596 // only reached when leaving the vector loop via the latch.
3597 // TODO: Support reductions used on early exits.
3598 for (VPRecipeBase &R : ExitBlock->phis()) {
3599 VPValue *ExitV =
3600 cast<VPIRPhi>(Val: &R)->getIncomingValueForBlock(VPBB: EarlyExitingVPBB);
3601 for (VPReductionPHIRecipe &PhiR :
3602 make_isa_range<VPReductionPHIRecipe>(Range: HeaderVPBB->phis())) {
3603 if (ExitV != PhiR.getBackedgeValue())
3604 continue;
3605 reportVectorizationFailure(
3606 DebugMsg: "Found reduction used on an early exit",
3607 OREMsg: "Cannot vectorize early exit loop with reduction used on an "
3608 "early exit",
3609 ORETag: "ReductionUsedOnEarlyExit", ORE, TheLoop);
3610 return false;
3611 }
3612 }
3613
3614 // Collect condition for this early exit.
3615 VPBlockBase *TrueSucc = EarlyExitingVPBB->getSuccessors()[0];
3616 VPValue *CondOfEarlyExitingVPBB;
3617 [[maybe_unused]] bool Matched =
3618 match(V: EarlyExitingVPBB->getTerminator(),
3619 P: m_BranchOnCond(Op0: m_VPValue(V&: CondOfEarlyExitingVPBB)));
3620 assert(Matched && "Terminator must be BranchOnCond");
3621
3622 // Insert the MaskedCond in the EarlyExitingVPBB so the predicator adds
3623 // the correct block mask.
3624 VPBuilder EarlyExitingBuilder(EarlyExitingVPBB->getTerminator());
3625 auto *CondToEarlyExit = EarlyExitingBuilder.createNaryOp(
3626 Opcode: VPInstruction::MaskedCond,
3627 Operands: TrueSucc == ExitBlock
3628 ? CondOfEarlyExitingVPBB
3629 : EarlyExitingBuilder.createNot(Operand: CondOfEarlyExitingVPBB));
3630 assert((isa<VPIRValue>(CondOfEarlyExitingVPBB) ||
3631 !VPDT.properlyDominates(EarlyExitingVPBB, LatchVPBB) ||
3632 VPDT.properlyDominates(
3633 CondOfEarlyExitingVPBB->getDefiningRecipe()->getParent(),
3634 LatchVPBB)) &&
3635 "exit condition must dominate the latch");
3636 Exits.push_back(Elt: {
3637 .EarlyExitingVPBB: EarlyExitingVPBB,
3638 .EarlyExitVPBB: ExitBlock,
3639 .CondToExit: CondToEarlyExit,
3640 });
3641 }
3642
3643 assert(!Exits.empty() && "must have at least one early exit");
3644 // Sort exits by RPO order to get correct program order. RPO gives a
3645 // topological ordering of the CFG, ensuring upstream exits are checked
3646 // before downstream exits in the dispatch chain.
3647 ReversePostOrderTraversal<VPBlockShallowTraversalWrapper<VPBlockBase *>> RPOT(
3648 HeaderVPBB);
3649 DenseMap<VPBlockBase *, unsigned> RPOIdx;
3650 for (const auto &[Num, VPB] : enumerate(First&: RPOT))
3651 RPOIdx[VPB] = Num;
3652 llvm::sort(C&: Exits, Comp: [&RPOIdx](const EarlyExitInfo &A, const EarlyExitInfo &B) {
3653 return RPOIdx[A.EarlyExitingVPBB] < RPOIdx[B.EarlyExitingVPBB];
3654 });
3655#ifndef NDEBUG
3656 // After RPO sorting, verify that for any pair where one exit dominates
3657 // another, the dominating exit comes first. This is guaranteed by RPO
3658 // (topological order) and is required for the dispatch chain correctness.
3659 for (unsigned I = 0; I + 1 < Exits.size(); ++I)
3660 for (unsigned J = I + 1; J < Exits.size(); ++J)
3661 assert(!VPDT.properlyDominates(Exits[J].EarlyExitingVPBB,
3662 Exits[I].EarlyExitingVPBB) &&
3663 "RPO sort must place dominating exits before dominated ones");
3664#endif
3665
3666 // Build the AnyOf condition for the latch terminator using logical OR
3667 // to avoid poison propagation from later exit conditions when an earlier
3668 // exit is taken.
3669 VPValue *Combined = Exits[0].CondToExit;
3670 for (const EarlyExitInfo &Info : drop_begin(RangeOrContainer&: Exits))
3671 Combined = LatchBuilder.createLogicalOr(LHS: Combined, RHS: Info.CondToExit);
3672 Combined = LatchBuilder.createFreeze(Op: Combined);
3673
3674 // Even though the logical or prevents posion propagation, we need to freeze
3675 // Combined to prevent poisoning the entire AnyOf result:
3676 //
3677 // Exits[0].CondToExit = [0,1,0,0]
3678 // Exits[1].CondToExit = [0,0,p,p]
3679 // Combined = [0,1,p,p]
3680 // freeze(Combined) = [0,1,?,?]
3681 // AnyOf = 1
3682 VPValue *IsAnyExitTaken =
3683 LatchBuilder.createNaryOp(Opcode: VPInstruction::AnyOf, Operands: Combined);
3684
3685 // Create a comparison for the latch exit condition and replace the
3686 // BranchOnCond with a BranchOnTwoConds. The original BranchOnCond's condition
3687 // is used as the latch-exit condition; canonical IV recipes have not been
3688 // introduced yet, so there is no BranchOnCount to derive the condition from.
3689 auto *LatchExitingBranch = cast<VPInstruction>(Val: LatchVPBB->getTerminator());
3690 assert(LatchExitingBranch->getOpcode() == VPInstruction::BranchOnCond &&
3691 "Unexpected terminator");
3692 VPValue *IsLatchExitTaken = LatchExitingBranch->getOperand(N: 0);
3693 DebugLoc LatchDL = LatchExitingBranch->getDebugLoc();
3694 LatchExitingBranch->eraseFromParent();
3695 LatchBuilder.setInsertPoint(LatchVPBB);
3696 LatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnTwoConds,
3697 Operands: {IsAnyExitTaken, IsLatchExitTaken}, DL: LatchDL);
3698 LatchVPBB->clearSuccessors();
3699
3700 if (Style == UncountableExitStyle::MaskedHandleExitInScalarLoop) {
3701 // If handling the exiting lane in the scalar loop, combine the exit
3702 // conditions into a single BranchOnCond.
3703 LatchVPBB->setSuccessors({MiddleVPBB, MiddleVPBB, HeaderVPBB});
3704 MiddleVPBB->clearPredecessors();
3705 MiddleVPBB->setPredecessors({LatchVPBB, LatchVPBB});
3706 return handleUncountableExitsWithSideEffects(Plan, Exits, HeaderVPBB,
3707 LatchVPBB, MiddleVPBB, ORE,
3708 TheLoop, PSE, DT, AC);
3709 }
3710
3711 // Create the vector.early.exit blocks.
3712 SmallVector<VPBasicBlock *> VectorEarlyExitVPBBs(Exits.size());
3713 for (unsigned Idx = 0; Idx != Exits.size(); ++Idx) {
3714 Twine BlockSuffix = Exits.size() == 1 ? "" : Twine(".") + Twine(Idx);
3715 VPBasicBlock *VectorEarlyExitVPBB =
3716 Plan.createVPBasicBlock(Name: "vector.early.exit" + BlockSuffix);
3717 VectorEarlyExitVPBBs[Idx] = VectorEarlyExitVPBB;
3718 }
3719
3720 // Create the dispatch block (or reuse the single exit block if only one
3721 // exit). The dispatch block computes the first active lane of the combined
3722 // condition and, for multiple exits, chains through conditions to determine
3723 // which exit to take.
3724 VPBasicBlock *DispatchVPBB =
3725 Exits.size() == 1 ? VectorEarlyExitVPBBs[0]
3726 : Plan.createVPBasicBlock(Name: "vector.early.exit.check");
3727 DispatchVPBB->setPredecessors({LatchVPBB});
3728 LatchVPBB->setSuccessors({DispatchVPBB, MiddleVPBB, HeaderVPBB});
3729 VPBuilder DispatchBuilder(DispatchVPBB->begin());
3730 VPValue *FirstActiveLane = DispatchBuilder.createFirstActiveLane(
3731 Masks: {Combined}, DL: DebugLoc::getUnknown(), Name: "first.active.lane");
3732
3733 // For each early exit, disconnect the original exiting block
3734 // (early.exiting.I) from the exit block (ir-bb<exit.I>) and route through a
3735 // new vector.early.exit block. Update ir-bb<exit.I>'s phis to extract their
3736 // values at the first active lane:
3737 //
3738 // Input:
3739 // early.exiting.I:
3740 // ...
3741 // EMIT branch-on-cond vp<%cond.I>
3742 // Successor(s): in.loop.succ, ir-bb<exit.I>
3743 //
3744 // ir-bb<exit.I>:
3745 // IR %phi = phi [ vp<%incoming.I>, early.exiting.I ], ...
3746 //
3747 // Output:
3748 // early.exiting.I:
3749 // ...
3750 // Successor(s): in.loop.succ
3751 //
3752 // vector.early.exit.I:
3753 // EMIT vp<%exit.val> = extract-lane vp<%first.lane>, vp<%incoming.I>
3754 // Successor(s): ir-bb<exit.I>
3755 //
3756 // ir-bb<exit.I>:
3757 // IR %phi = phi ... (extra operand: vp<%exit.val> from
3758 // vector.early.exit.I)
3759 //
3760 for (auto [Exit, VectorEarlyExitVPBB] :
3761 zip_equal(t&: Exits, u&: VectorEarlyExitVPBBs)) {
3762 auto &[EarlyExitingVPBB, EarlyExitVPBB, _] = Exit;
3763 // Adjust the phi nodes in EarlyExitVPBB.
3764 // 1. remove incoming values from EarlyExitingVPBB,
3765 // 2. extract the incoming value at FirstActiveLane
3766 // 3. add back the extracts as last operands for the phis
3767 // Then adjust the CFG, removing the edge between EarlyExitingVPBB and
3768 // EarlyExitVPBB and adding a new edge between VectorEarlyExitVPBB and
3769 // EarlyExitVPBB. The extracts at FirstActiveLane are now the incoming
3770 // values from VectorEarlyExitVPBB.
3771 for (VPRecipeBase &R : EarlyExitVPBB->phis()) {
3772 auto *ExitIRI = cast<VPIRPhi>(Val: &R);
3773 VPValue *IncomingVal =
3774 ExitIRI->getIncomingValueForBlock(VPBB: EarlyExitingVPBB);
3775 VPValue *NewIncoming = IncomingVal;
3776 if (!isa<VPIRValue>(Val: IncomingVal)) {
3777 VPBuilder EarlyExitBuilder(VectorEarlyExitVPBB);
3778 NewIncoming = EarlyExitBuilder.createNaryOp(
3779 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, IncomingVal},
3780 DL: DebugLoc::getUnknown(), Name: "early.exit.value");
3781 }
3782 ExitIRI->removeIncomingValueFor(IncomingBlock: EarlyExitingVPBB);
3783 ExitIRI->addIncoming(IncomingV: NewIncoming);
3784 }
3785
3786 EarlyExitingVPBB->getTerminator()->eraseFromParent();
3787 VPBlockUtils::disconnectBlocks(From: EarlyExitingVPBB, To: EarlyExitVPBB);
3788 VPBlockUtils::connectBlocks(From: VectorEarlyExitVPBB, To: EarlyExitVPBB);
3789 }
3790
3791 // Chain through exits: for each exit, check if its condition is true at
3792 // the first active lane. If so, take that exit; otherwise, try the next.
3793 // The last exit needs no check since it must be taken if all others fail.
3794 //
3795 // For 3 exits (cond.0, cond.1, cond.2), this creates:
3796 //
3797 // latch:
3798 // ...
3799 // EMIT vp<%combined> = logical-or vp<%cond.0>, vp<%cond.1>, vp<%cond.2>
3800 // EMIT vp<%combined.freeze> = freeze vp<%combined>
3801 // ...
3802 //
3803 // vector.early.exit.check:
3804 // EMIT vp<%first.lane> = first-active-lane vp<%combined.freeze>
3805 // EMIT vp<%at.cond.0> = extract-lane vp<%first.lane>, vp<%cond.0>
3806 // EMIT branch-on-cond vp<%at.cond.0>
3807 // Successor(s): vector.early.exit.0, vector.early.exit.check.0
3808 //
3809 // vector.early.exit.check.0:
3810 // EMIT vp<%at.cond.1> = extract-lane vp<%first.lane>, vp<%cond.1>
3811 // EMIT branch-on-cond vp<%at.cond.1>
3812 // Successor(s): vector.early.exit.1, vector.early.exit.2
3813 VPBasicBlock *CurrentBB = DispatchVPBB;
3814 for (auto [I, Exit] : enumerate(First: ArrayRef(Exits).drop_back())) {
3815 VPValue *LaneVal = DispatchBuilder.createNaryOp(
3816 Opcode: VPInstruction::ExtractLane, Operands: {FirstActiveLane, Exit.CondToExit},
3817 DL: DebugLoc::getUnknown(), Name: "exit.cond.at.lane");
3818
3819 // For the last dispatch, branch directly to the last exit on false;
3820 // otherwise, create a new check block.
3821 bool IsLastDispatch = (I + 2 == Exits.size());
3822 VPBasicBlock *FalseBB =
3823 IsLastDispatch ? VectorEarlyExitVPBBs.back()
3824 : Plan.createVPBasicBlock(
3825 Name: Twine("vector.early.exit.check.") + Twine(I));
3826
3827 DispatchBuilder.createNaryOp(Opcode: VPInstruction::BranchOnCond, Operands: {LaneVal});
3828 CurrentBB->setSuccessors({VectorEarlyExitVPBBs[I], FalseBB});
3829 VectorEarlyExitVPBBs[I]->setPredecessors({CurrentBB});
3830 FalseBB->setPredecessors({CurrentBB});
3831
3832 CurrentBB = FalseBB;
3833 DispatchBuilder.setInsertPoint(CurrentBB);
3834 }
3835
3836 return true;
3837}
3838
3839/// This function tries convert extended in-loop reductions to
3840/// VPExpressionRecipe and clamp the \p Range if it is beneficial and
3841/// valid. The created recipe must be decomposed to its constituent
3842/// recipes before execution.
3843static VPExpressionRecipe *
3844tryToMatchAndCreateExtendedReduction(VPReductionRecipe *Red, VPCostContext &Ctx,
3845 VFRange &Range) {
3846 Type *RedTy = Red->getScalarType();
3847 VPValue *VecOp = Red->getVecOp();
3848
3849 // We don't handle partial reductions here.
3850 if (Red->isPartialReduction())
3851 return nullptr;
3852
3853 // Clamp the range if using extended-reduction is profitable.
3854 auto IsExtendedRedValidAndClampRange =
3855 [&](unsigned Opcode, Instruction::CastOps ExtOpc, Type *SrcTy) -> bool {
3856 return LoopVectorizationPlanner::getDecisionAndClampRange(
3857 Predicate: [&](ElementCount VF) {
3858 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
3859 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
3860
3861 InstructionCost ExtRedCost = InstructionCost::getInvalid();
3862 InstructionCost ExtCost =
3863 cast<VPWidenCastRecipe>(Val: VecOp)->computeCost(VF, Ctx);
3864 InstructionCost RedCost = Red->computeCost(VF, Ctx);
3865
3866 assert(!RedTy->isFloatingPointTy() &&
3867 "getExtendedReductionCost only supports integer types");
3868 ExtRedCost = Ctx.TTI.getExtendedReductionCost(
3869 Opcode, IsUnsigned: ExtOpc == Instruction::CastOps::ZExt, ResTy: RedTy, Ty: SrcVecTy,
3870 FMF: Red->getFastMathFlagsOrNone(), CostKind);
3871 return ExtRedCost.isValid() && ExtRedCost < ExtCost + RedCost;
3872 },
3873 Range);
3874 };
3875
3876 VPValue *A;
3877 // Match reduce(ext)).
3878 if (match(V: VecOp, P: m_Isa<VPWidenCastRecipe>(P: m_ZExtOrSExt(Op0: m_VPValue(V&: A)))) &&
3879 IsExtendedRedValidAndClampRange(
3880 RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind()),
3881 cast<VPWidenCastRecipe>(Val: VecOp)->getOpcode(), A->getScalarType()))
3882 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
3883
3884 return nullptr;
3885}
3886
3887using ExtendKind = TTI::PartialReductionExtendKind;
3888
3889static ExtendKind getPartialReductionExtendKind(VPWidenCastRecipe *Cast) {
3890 return TTI::getPartialReductionExtendKind(CastOpc: Cast->getOpcode());
3891}
3892
3893/// Returns true if the narrow \p Mul in ext(mul(ext(A), ext(B))) or
3894/// ext(mul(ext(A), C)) cannot wrap. The kind of the inner extends selects
3895/// whether nuw (zext) or nsw (sext) is required on \p Mul.
3896static bool canFoldOuterExtendIntoMul(VPWidenRecipe *Mul) {
3897 auto *LHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
3898 auto *RHS = dyn_cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
3899 Type *LHSSrcTy = LHS->getOperand(N: 0)->getScalarType();
3900 // A constant RHS must fit in the source type of the LHS.
3901 Type *RHSSrcTy = RHS ? RHS->getOperand(N: 0)->getScalarType() : LHSSrcTy;
3902 ExtendKind ExtKind = getPartialReductionExtendKind(Cast: LHS);
3903 if (Mul->getScalarType()->getScalarSizeInBits() >=
3904 LHSSrcTy->getScalarSizeInBits() + RHSSrcTy->getScalarSizeInBits())
3905 return true;
3906 if (ExtKind == ExtendKind::PR_ZeroExtend)
3907 return Mul->hasNoUnsignedWrap();
3908 return ExtKind == ExtendKind::PR_SignExtend && Mul->hasNoSignedWrap();
3909}
3910
3911/// This function tries convert extended in-loop reductions to
3912/// VPExpressionRecipe and clamp the \p Range if it is beneficial
3913/// and valid. The created VPExpressionRecipe must be decomposed to its
3914/// constituent recipes before execution. Patterns of the
3915/// VPExpressionRecipe:
3916/// reduce.add(mul(...)),
3917/// reduce.add(mul(ext(A), ext(B))),
3918/// reduce.add(ext(mul(ext(A), ext(B)))).
3919/// reduce.fadd(fmul(ext(A), ext(B)))
3920static VPExpressionRecipe *
3921tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
3922 VPCostContext &Ctx, VFRange &Range) {
3923 unsigned Opcode = RecurrenceDescriptor::getOpcode(Kind: Red->getRecurrenceKind());
3924 if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
3925 Opcode != Instruction::FAdd)
3926 return nullptr;
3927
3928 // We don't handle partial reductions here.
3929 if (Red->isPartialReduction())
3930 return nullptr;
3931
3932 Type *RedTy = Red->getScalarType();
3933
3934 // Clamp the range if using multiply-accumulate-reduction is profitable.
3935 auto IsMulAccValidAndClampRange =
3936 [&](VPWidenRecipe *Mul, VPWidenCastRecipe *Ext0, VPWidenCastRecipe *Ext1,
3937 VPWidenCastRecipe *OuterExt) -> bool {
3938 return LoopVectorizationPlanner::getDecisionAndClampRange(
3939 Predicate: [&](ElementCount VF) {
3940 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
3941 Type *SrcTy = Ext0 ? Ext0->getOperand(N: 0)->getScalarType() : RedTy;
3942 InstructionCost MulAccCost;
3943
3944 // getMulAccReductionCost for in-loop reductions does not support
3945 // mixed or floating-point extends.
3946 if (Ext0 && Ext1 &&
3947 (Ext0->getOpcode() != Ext1->getOpcode() ||
3948 Ext0->getOpcode() == Instruction::CastOps::FPExt))
3949 return false;
3950
3951 bool IsZExt =
3952 !Ext0 || Ext0->getOpcode() == Instruction::CastOps::ZExt;
3953 auto *SrcVecTy = cast<VectorType>(Val: toVectorTy(Scalar: SrcTy, EC: VF));
3954 MulAccCost = Ctx.TTI.getMulAccReductionCost(IsUnsigned: IsZExt, RedOpcode: Opcode, ResTy: RedTy,
3955 Ty: SrcVecTy, CostKind);
3956
3957 InstructionCost MulCost = Mul->computeCost(VF, Ctx);
3958 InstructionCost RedCost = Red->computeCost(VF, Ctx);
3959 InstructionCost ExtCost = 0;
3960 if (Ext0)
3961 ExtCost += Ext0->computeCost(VF, Ctx);
3962 if (Ext1)
3963 ExtCost += Ext1->computeCost(VF, Ctx);
3964 if (OuterExt)
3965 ExtCost += OuterExt->computeCost(VF, Ctx);
3966
3967 return MulAccCost.isValid() &&
3968 MulAccCost < ExtCost + MulCost + RedCost;
3969 },
3970 Range);
3971 };
3972
3973 VPValue *VecOp = Red->getVecOp();
3974 VPRecipeBase *Sub = nullptr;
3975 VPValue *A, *B;
3976 VPValue *Tmp = nullptr;
3977
3978 if (RedTy->isFloatingPointTy())
3979 return nullptr;
3980
3981 // Sub reductions could have a sub between the add reduction and vec op.
3982 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_VPValue(V&: Tmp)))) {
3983 Sub = VecOp->getDefiningRecipe();
3984 VecOp = Tmp;
3985 }
3986
3987 // If ValB is a constant and can be safely extended, truncate it to the same
3988 // type as ExtA's operand, then extend it to the same type as ExtA. This
3989 // creates two uniform extends that can more easily be matched by the rest of
3990 // the bundling code. The ExtB reference, ValB and operand 1 of Mul are all
3991 // replaced with the new extend of the constant.
3992 auto ExtendAndReplaceConstantOp = [](VPWidenCastRecipe *ExtA,
3993 VPWidenCastRecipe *&ExtB, VPValue *&ValB,
3994 VPWidenRecipe *Mul) {
3995 if (!ExtA || ExtB || !isa<VPIRValue>(Val: ValB))
3996 return;
3997 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
3998 Instruction::CastOps ExtOpc = ExtA->getOpcode();
3999 const APInt *Const;
4000 if (!match(V: ValB, P: m_APInt(C&: Const)) ||
4001 !llvm::canConstantBeExtended(
4002 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
4003 return;
4004 // The truncate ensures that the type of each extended operand is the
4005 // same, and it's been proven that the constant can be extended from
4006 // NarrowTy safely. Necessary since ExtA's extended operand would be
4007 // e.g. an i8, while the const will likely be an i32. This will be
4008 // elided by later optimisations.
4009 VPBuilder Builder(Mul);
4010 auto *Trunc =
4011 Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc, Op: ValB, ResultTy: NarrowTy);
4012 Type *WideTy = ExtA->getScalarType();
4013 ValB = ExtB = Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy);
4014 Mul->setOperand(I: 1, New: ExtB);
4015 };
4016
4017 // Try to match reduce.add(mul(...)).
4018 if (match(V: VecOp, P: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B)))) {
4019 auto *RecipeA = dyn_cast<VPWidenCastRecipe>(Val: A);
4020 auto *RecipeB = dyn_cast<VPWidenCastRecipe>(Val: B);
4021 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
4022
4023 // Convert reduce.add(mul(ext, const)) to reduce.add(mul(ext, ext(const)))
4024 ExtendAndReplaceConstantOp(RecipeA, RecipeB, B, Mul);
4025
4026 // Match reduce.add/sub(mul(ext, ext)).
4027 if (RecipeA && RecipeB && match(V: RecipeA, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
4028 match(V: RecipeB, P: m_ZExtOrSExt(Op0: m_VPValue())) &&
4029 IsMulAccValidAndClampRange(Mul, RecipeA, RecipeB, nullptr)) {
4030 if (Sub)
4031 return new VPExpressionRecipe(RecipeA, RecipeB, Mul,
4032 cast<VPWidenRecipe>(Val: Sub), Red);
4033 return new VPExpressionRecipe(RecipeA, RecipeB, Mul, Red);
4034 }
4035 // TODO: Add an expression type for this variant with a negated mul
4036 if (!Sub && IsMulAccValidAndClampRange(Mul, nullptr, nullptr, nullptr))
4037 return new VPExpressionRecipe(Mul, Red);
4038 }
4039 // TODO: Add an expression type for negated versions of other expression
4040 // variants.
4041 if (Sub)
4042 return nullptr;
4043
4044 // Match reduce.add(ext(mul(A, B))).
4045 if (match(V: VecOp, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_VPValue(V&: A), Op1: m_VPValue(V&: B))))) {
4046 auto *Ext = cast<VPWidenCastRecipe>(Val: VecOp);
4047 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
4048 auto *Ext0 = dyn_cast<VPWidenCastRecipe>(Val: A);
4049 auto *Ext1 = dyn_cast<VPWidenCastRecipe>(Val: B);
4050
4051 // reduce.add(ext(mul(ext, const)))
4052 // -> reduce.add(ext(mul(ext, ext(const))))
4053 ExtendAndReplaceConstantOp(Ext0, Ext1, B, Mul);
4054
4055 // reduce.add(ext(mul(ext(A), ext(B))))
4056 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
4057 // The inner extends must either have the same opcode as the outer extend or
4058 // be the same sext, in which case the multiply cannot result in a negative
4059 // value without wrapping. If the narrow mul cannot wrap, the outer extend
4060 // can be folded away by doing wider extends for the operands of the mul.
4061 if (Ext0 && Ext1 &&
4062 (Ext->getOpcode() == Ext0->getOpcode() ||
4063 (Ext0 == Ext1 && Ext0->getOpcode() == Instruction::SExt)) &&
4064 Ext0->getOpcode() == Ext1->getOpcode() && Mul->hasOneUse() &&
4065 canFoldOuterExtendIntoMul(Mul) &&
4066 IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext)) {
4067 auto *NewExt0 = new VPWidenCastRecipe(
4068 Ext0->getOpcode(), Ext0->getOperand(N: 0), Ext->getScalarType(), nullptr,
4069 *Ext0, *Ext0, Ext0->getDebugLoc());
4070 NewExt0->insertBefore(InsertPos: Ext0);
4071
4072 VPWidenCastRecipe *NewExt1 = NewExt0;
4073 if (Ext0 != Ext1) {
4074 NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(N: 0),
4075 Ext->getScalarType(), nullptr, *Ext1,
4076 *Ext1, Ext1->getDebugLoc());
4077 NewExt1->insertBefore(InsertPos: Ext1);
4078 }
4079 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewExt0, NewExt1});
4080 NewMul->insertBefore(InsertPos: Mul);
4081 Ext->replaceAllUsesWith(New: NewMul);
4082 Ext->eraseFromParent();
4083 Mul->eraseFromParent();
4084 return new VPExpressionRecipe(NewExt0, NewExt1, NewMul, Red);
4085 }
4086 }
4087 return nullptr;
4088}
4089
4090/// This function tries to create abstract recipes from the reduction recipe for
4091/// following optimizations and cost estimation.
4092static void tryToCreateAbstractReductionRecipe(VPReductionRecipe *Red,
4093 VPCostContext &Ctx,
4094 VFRange &Range) {
4095 // Creation of VPExpressions for partial reductions is entirely handled in
4096 // transformToPartialReduction.
4097 if (Red->isPartialReduction())
4098 return;
4099
4100 VPExpressionRecipe *AbstractR = nullptr;
4101 auto IP = std::next(x: Red->getIterator());
4102 if (auto *MulAcc = tryToMatchAndCreateMulAccumulateReduction(Red, Ctx, Range))
4103 AbstractR = MulAcc;
4104 else if (auto *ExtRed = tryToMatchAndCreateExtendedReduction(Red, Ctx, Range))
4105 AbstractR = ExtRed;
4106 // Cannot create abstract inloop reduction recipes.
4107 if (!AbstractR)
4108 return;
4109
4110 AbstractR->insertBefore(IP);
4111 Red->replaceAllUsesWith(New: AbstractR);
4112}
4113
4114void VPlanTransforms::convertToAbstractRecipes(VPlan &Plan, VPCostContext &Ctx,
4115 VFRange &Range) {
4116 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
4117 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()))) {
4118 for (VPReductionRecipe &Red :
4119 make_early_inc_range(Range: make_isa_range<VPReductionRecipe>(Range&: *VPBB)))
4120 tryToCreateAbstractReductionRecipe(Red: &Red, Ctx, Range);
4121 }
4122}
4123
4124// Collect common metadata from a group of replicate recipes by intersecting
4125// metadata from all recipes in the group.
4126static VPIRMetadata getCommonMetadata(ArrayRef<VPReplicateRecipe *> Recipes) {
4127 VPIRMetadata CommonMetadata = *Recipes.front();
4128 for (VPReplicateRecipe *Recipe : drop_begin(RangeOrContainer&: Recipes))
4129 CommonMetadata.intersect(MD: *Recipe);
4130 // The recipe using the common metadata is not predicated, so it does not
4131 // share the group's execution frequency.
4132 CommonMetadata.clearExecutionFrequency();
4133 return CommonMetadata;
4134}
4135
4136template <unsigned Opcode>
4137static SmallVector<SmallVector<VPReplicateRecipe *, 4>>
4138collectComplementaryPredicatedMemOps(VPlan &Plan,
4139 PredicatedScalarEvolution &PSE,
4140 const Loop *L) {
4141 static_assert(Opcode == Instruction::Load || Opcode == Instruction::Store,
4142 "Only Load and Store opcodes supported");
4143
4144 // For each address, collect operations with the same or complementary masks.
4145 SmallVector<SmallVector<VPReplicateRecipe *, 4>> AllGroups;
4146 auto Groups = collectGroupedReplicateMemOps<Opcode>(
4147 Plan, PSE, L,
4148 [](VPReplicateRecipe *RepR) { return RepR->isPredicated(); });
4149 for (auto Recipes : Groups) {
4150 if (Recipes.size() < 2)
4151 continue;
4152
4153 assert(all_equal(map_range(Recipes, vputils::getLoadStoreValueType)) &&
4154 "Expected all recipes in group to have the same load-store type");
4155
4156 // Collect groups with the same or complementary masks.
4157 for (VPReplicateRecipe *&RecipeI : Recipes) {
4158 if (!RecipeI)
4159 continue;
4160
4161 VPValue *MaskI = RecipeI->getMask();
4162 SmallVector<VPReplicateRecipe *, 4> Group;
4163 Group.push_back(Elt: RecipeI);
4164 RecipeI = nullptr;
4165
4166 // Find all operations with the same or complementary masks.
4167 bool HasComplementaryMask = false;
4168 for (VPReplicateRecipe *&RecipeJ : Recipes) {
4169 if (!RecipeJ)
4170 continue;
4171
4172 VPValue *MaskJ = RecipeJ->getMask();
4173 // Check if any operation in the group has a complementary mask with
4174 // another, that is M1 == NOT(M2) or M2 == NOT(M1).
4175 HasComplementaryMask |= match(V: MaskI, P: m_Not(Op0: m_Specific(VPV: MaskJ))) ||
4176 match(V: MaskJ, P: m_Not(Op0: m_Specific(VPV: MaskI)));
4177 Group.push_back(Elt: RecipeJ);
4178 RecipeJ = nullptr;
4179 }
4180
4181 if (HasComplementaryMask) {
4182 assert(Group.size() >= 2 && "must have at least 2 entries");
4183 AllGroups.push_back(Elt: std::move(Group));
4184 }
4185 }
4186 }
4187
4188 return AllGroups;
4189}
4190
4191// Find the recipe with minimum alignment in the group.
4192template <typename InstType>
4193static VPReplicateRecipe *
4194findRecipeWithMinAlign(ArrayRef<VPReplicateRecipe *> Group) {
4195 return *min_element(Group, [](VPReplicateRecipe *A, VPReplicateRecipe *B) {
4196 return cast<InstType>(A->getUnderlyingInstr())->getAlign() <
4197 cast<InstType>(B->getUnderlyingInstr())->getAlign();
4198 });
4199}
4200
4201void VPlanTransforms::hoistPredicatedLoads(VPlan &Plan,
4202 PredicatedScalarEvolution &PSE,
4203 const Loop *L) {
4204 auto Groups =
4205 collectComplementaryPredicatedMemOps<Instruction::Load>(Plan, PSE, L);
4206 if (Groups.empty())
4207 return;
4208
4209 // Process each group of loads.
4210 for (auto &Group : Groups) {
4211 // Try to use the earliest (most dominating) load to replace all others.
4212 VPReplicateRecipe *EarliestLoad = Group[0];
4213 VPBasicBlock *FirstBB = EarliestLoad->getParent();
4214 VPBasicBlock *LastBB = Group.back()->getParent();
4215
4216 // Check that the load doesn't alias with stores between first and last.
4217 auto LoadLoc = vputils::getMemoryLocation(R: *EarliestLoad);
4218 if (!LoadLoc || !canHoistOrSinkWithNoAliasCheck(MemLoc: *LoadLoc, FirstBB, LastBB))
4219 continue;
4220
4221 // Collect common metadata from all loads in the group.
4222 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
4223
4224 // Find the load with minimum alignment to use.
4225 auto *LoadWithMinAlign = findRecipeWithMinAlign<LoadInst>(Group);
4226
4227 bool IsSingleScalar = EarliestLoad->isSingleScalar();
4228 assert(all_of(Group,
4229 [IsSingleScalar](VPReplicateRecipe *R) {
4230 return R->isSingleScalar() == IsSingleScalar;
4231 }) &&
4232 "all members in group must agree on IsSingleScalar");
4233
4234 // Create an unpredicated version of the earliest load with common
4235 // metadata.
4236 auto *UnpredicatedLoad = new VPReplicateRecipe(
4237 LoadWithMinAlign->getUnderlyingInstr(), {EarliestLoad->getOperand(N: 0)},
4238 IsSingleScalar, /*Mask=*/nullptr, *EarliestLoad, CommonMetadata);
4239
4240 UnpredicatedLoad->insertBefore(InsertPos: EarliestLoad);
4241
4242 // Replace all loads in the group with the unpredicated load.
4243 for (VPReplicateRecipe *Load : Group) {
4244 Load->replaceAllUsesWith(New: UnpredicatedLoad);
4245 Load->eraseFromParent();
4246 }
4247 }
4248}
4249
4250static bool
4251canSinkStoreWithNoAliasCheck(ArrayRef<VPReplicateRecipe *> StoresToSink,
4252 PredicatedScalarEvolution &PSE, const Loop &L) {
4253 auto StoreLoc = vputils::getMemoryLocation(R: *StoresToSink.front());
4254 if (!StoreLoc || !StoreLoc->AATags.Scope)
4255 return false;
4256
4257 // When sinking a group of stores, all members of the group alias each other.
4258 // Skip them during the alias checks.
4259 VPBasicBlock *FirstBB = StoresToSink.front()->getParent();
4260 VPBasicBlock *LastBB = StoresToSink.back()->getParent();
4261 SinkStoreInfo SinkInfo(StoresToSink, *StoresToSink[0], PSE, L);
4262 return canHoistOrSinkWithNoAliasCheck(MemLoc: *StoreLoc, FirstBB, LastBB, SinkInfo);
4263}
4264
4265void VPlanTransforms::sinkPredicatedStores(VPlan &Plan,
4266 PredicatedScalarEvolution &PSE,
4267 const Loop *L) {
4268 auto Groups =
4269 collectComplementaryPredicatedMemOps<Instruction::Store>(Plan, PSE, L);
4270 if (Groups.empty())
4271 return;
4272
4273 for (auto &Group : Groups) {
4274 if (!canSinkStoreWithNoAliasCheck(StoresToSink: Group, PSE, L: *L))
4275 continue;
4276
4277 // Use the last (most dominated) store's location for the unconditional
4278 // store.
4279 VPReplicateRecipe *LastStore = Group.back();
4280
4281 // Collect common alias metadata from all stores in the group.
4282 VPIRMetadata CommonMetadata = getCommonMetadata(Recipes: Group);
4283
4284 // Build select chain for stored values.
4285 VPValue *SelectedValue = Group[0]->getOperand(N: 0);
4286 VPBuilder Builder(LastStore);
4287
4288 bool IsSingleScalar = Group[0]->isSingleScalar();
4289 for (unsigned I = 1; I < Group.size(); ++I) {
4290 assert(IsSingleScalar == Group[I]->isSingleScalar() &&
4291 "all members in group must agree on IsSingleScalar");
4292 VPValue *Mask = Group[I]->getMask();
4293 VPValue *Value = Group[I]->getOperand(N: 0);
4294 SelectedValue = Builder.createSelect(
4295 Cond: Mask, TrueVal: Value, FalseVal: SelectedValue, DL: Group[I]->getDebugLoc(), Name: "",
4296 Flags: VPIRFlags::getDefaultFlags(Opcode: Instruction::Select,
4297 ResultTy: Value->getScalarType()));
4298 }
4299
4300 // Find the store with minimum alignment to use.
4301 auto *StoreWithMinAlign = findRecipeWithMinAlign<StoreInst>(Group);
4302
4303 // Create unconditional store with selected value and common metadata.
4304 auto *UnpredicatedStore = new VPReplicateRecipe(
4305 StoreWithMinAlign->getUnderlyingInstr(),
4306 {SelectedValue, LastStore->getOperand(N: 1)}, IsSingleScalar,
4307 /*Mask=*/nullptr, *LastStore, CommonMetadata);
4308 UnpredicatedStore->insertBefore(InsertPos: LastStore);
4309
4310 // Remove all predicated stores from the group.
4311 for (VPReplicateRecipe *Store : Group)
4312 Store->eraseFromParent();
4313 }
4314}
4315
4316void VPlanTransforms::widenMemoryAccessesByUF(VPlan &Plan, ElementCount VF,
4317 unsigned UF,
4318 const TargetTransformInfo &TTI) {
4319 assert(UF > 1 && "Expected plan to have an UF > 1");
4320
4321 auto m_ContiguousVecPtr = m_VecPtr(Op0: m_VPValue(), Op1: m_One());
4322 VPValue *Multiplier = Plan.getConstantInt(BitWidth: 64, Val: 1);
4323
4324 VPActiveLaneMaskPHIRecipe *MaskPhi = nullptr;
4325 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
4326 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
4327 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
4328 VPValue *Mask = nullptr;
4329 VPValue *StoredValue = nullptr;
4330 if (!match(V: &R,
4331 P: m_CombineOr(
4332 Ps: m_WidenLoad(Op0: m_ContiguousVecPtr),
4333 Ps: m_WidenStore(Op0: m_ContiguousVecPtr, Op1: m_VPValue(V&: StoredValue)),
4334 Ps: m_MaskedLoad(Addr: m_ContiguousVecPtr, Mask: m_VPValue(V&: Mask)),
4335 Ps: m_MaskedStore(Addr: m_ContiguousVecPtr, Val: m_VPValue(V&: StoredValue),
4336 Mask: m_VPValue(V&: Mask)))))
4337 continue;
4338
4339 auto *MemOp = cast<VPWidenMemoryRecipe>(Val: &R);
4340 assert(MemOp->isConsecutive() && "Expected consecutive load/store");
4341
4342 // If there's a mask, ensure it's a wide active-lane-mask.
4343 if (Mask && !match(V: Mask, P: m_ActiveLaneMaskPhi(Op0: m_ExtractVectorForPart(),
4344 Op1: m_ExtractVectorForPart())))
4345 continue;
4346
4347 auto MaskSource = MemOp->isMasked()
4348 ? TargetTransformInfo::MaskSource::ActiveLaneMask
4349 : TargetTransformInfo::MaskSource::None;
4350
4351 Type *AccessType = StoredValue ? StoredValue->getScalarType()
4352 : R.getVPSingleValue()->getScalarType();
4353 bool IsStore = isa<VPWidenStoreRecipe>(Val: MemOp->getAsRecipe());
4354 std::optional<Instruction::CastOps> CastHint;
4355 VPUser *MaybeCast = IsStore ? StoredValue->getDefiningRecipe()
4356 : R.getVPSingleValue()->getSingleUser();
4357 if (auto *Cast = dyn_cast_if_present<VPWidenCastRecipe>(Val: MaybeCast))
4358 CastHint = Cast->getOpcode();
4359
4360 VectorType *VectorAccessType = VectorType::get(ElementType: AccessType, EC: VF);
4361 if (!TTI.hasMultiVectorLoadStore(/*NumVectors=*/UF, Mask: MaskSource,
4362 VectorTy: VectorAccessType, IsStore, CastHint))
4363 continue;
4364
4365 DebugLoc DL = R.getDebugLoc();
4366 VPValue *Ptr = MemOp->getAddr();
4367 VPValue *Align = Plan.getConstantInt(BitWidth: 64, Val: MemOp->getAlign().value());
4368
4369 VPBuilder Builder(&R);
4370 VPInstruction *NewMemOp;
4371 if (IsStore) {
4372 VPValue *WideStoredValue = Builder.createNaryOp(
4373 Opcode: VPInstruction::ConcatVectors, Operands: {StoredValue}, DL);
4374 NewMemOp = Builder.createNaryOp(
4375 Opcode: VPInstruction::WideVectorStore,
4376 Operands: {Multiplier, Ptr, Align, WideStoredValue}, Inst: nullptr, Flags: {}, MD: *MemOp, DL);
4377 } else {
4378 VPValue *OldLoad = R.getVPSingleValue();
4379 NewMemOp = Builder.createNaryOp(
4380 Opcode: VPInstruction::WideVectorLoad, Operands: {Multiplier, Ptr, Align}, Inst: nullptr,
4381 Flags: {}, MD: *MemOp, DL, Name: "", ResultTy: OldLoad->getScalarType());
4382 VPValue *Extract =
4383 Builder.createNaryOp(Opcode: VPInstruction::ExtractVectorForPart,
4384 Operands: {NewMemOp, Plan.getConstantInt(BitWidth: 64, Val: 0)}, DL);
4385 OldLoad->replaceAllUsesWith(New: Extract);
4386 }
4387
4388 R.eraseFromParent();
4389 if (Mask) {
4390 assert((!MaskPhi || Mask == MaskPhi) && "Expected a single mask phi");
4391 NewMemOp->addMask(Mask);
4392 MaskPhi = cast<VPActiveLaneMaskPHIRecipe>(Val: Mask);
4393 }
4394 }
4395 }
4396
4397 if (!MaskPhi)
4398 return;
4399
4400 // Widen the mask phi to match the width of the masked loads/stores. This
4401 // effectively means removing the `extract-vector-for-part` from the preheader
4402 // and latch instead placing a single extract in the header. The
4403 // wide-vector-load/store operations can use the mask phi directly.
4404 auto *EntryMask = cast<VPInstruction>(Val: MaskPhi->getStartValue());
4405 auto *LoopMask = cast<VPInstruction>(Val: MaskPhi->getBackedgeValue());
4406 auto *EntryALM = cast<VPInstruction>(Val: EntryMask->getOperand(N: 0));
4407 auto *LoopALM = cast<VPInstruction>(Val: LoopMask->getOperand(N: 0));
4408
4409 assert((EntryALM->getOpcode() == VPInstruction::WideActiveLaneMask &&
4410 LoopALM->getOpcode() == VPInstruction::WideActiveLaneMask) &&
4411 "Expected incoming values of Phi to be WideActiveLaneMask");
4412
4413 auto Builder = VPBuilder(&*MaskPhi->getParent()->getFirstNonPhi());
4414 auto *MaskPart = Builder.createNaryOp(Opcode: VPInstruction::ExtractVectorForPart,
4415 Operands: {MaskPhi, Plan.getConstantInt(BitWidth: 64, Val: 0)});
4416
4417 MaskPhi->replaceUsesWithIf(
4418 New: MaskPart, ShouldReplace: match_fn(P: m_Unless(P: m_CombineOr(
4419 Ps: m_Specific(VPV: MaskPart),
4420 Ps: m_VPInstruction<VPInstruction::WideVectorLoad>(),
4421 Ps: m_VPInstruction<VPInstruction::WideVectorStore>()))));
4422
4423 // Remove extra ExtractVectorForPart operations.
4424 assert(EntryMask->getOpcode() == VPInstruction::ExtractVectorForPart &&
4425 LoopMask->getOpcode() == VPInstruction::ExtractVectorForPart &&
4426 "Expected entry/loop extract-vector-for-part ops");
4427 EntryMask->replaceAllUsesWith(New: EntryALM);
4428 LoopMask->replaceAllUsesWith(New: LoopALM);
4429}
4430
4431/// Returns true if \p V is VPWidenLoadRecipe or VPInterleaveRecipe that can be
4432/// converted to a narrower recipe. \p V is used by a wide recipe that feeds a
4433/// store interleave group at index \p Idx, \p WideMember0 is the recipe feeding
4434/// the same interleave group at index 0. A VPWidenLoadRecipe can be narrowed to
4435/// an index-independent load if it feeds all wide ops at all indices (\p OpV
4436/// must be the operand at index \p OpIdx for both the recipe at lane 0, \p
4437/// WideMember0). A VPInterleaveRecipe can be narrowed to a wide load, if \p V
4438/// is defined at \p Idx of a load interleave group.
4439/// A live-in or recipe defined outside the loop region can be converted, if it
4440/// is the same across all lanes, or we can create a BuildVector for it.
4441static bool canNarrowLoad(VPSingleDefRecipe *WideMember0, unsigned OpIdx,
4442 VPValue *OpV, unsigned Idx, bool IsScalable) {
4443 VPValue *Member0Op = WideMember0->getOperand(N: OpIdx);
4444 if (Member0Op->isDefinedOutsideLoopRegions()) {
4445 // Operand matches Member0, broadcast across all fields for both live-ins
4446 // and recipes.
4447 if (Member0Op == OpV)
4448 return true;
4449 // Otherwise distinct per-field VPValues are assembled into a BuildVector.
4450 return !IsScalable && OpV->isDefinedOutsideLoopRegions() &&
4451 OpV->getScalarType() == Member0Op->getScalarType();
4452 }
4453 VPRecipeBase *Member0OpR = Member0Op->getDefiningRecipe();
4454 if (auto *W = dyn_cast<VPWidenLoadRecipe>(Val: Member0OpR))
4455 // For scalable VFs, the narrowed plan processes vscale iterations at once,
4456 // so a shared wide load cannot be narrowed to a uniform scalar; bail out.
4457 return !IsScalable && !W->getMask() && W->isConsecutive() &&
4458 Member0Op == OpV;
4459 if (auto *IR = dyn_cast<VPInterleaveRecipe>(Val: Member0OpR))
4460 return IR->getInterleaveGroup()->isFull() && IR->getVPValue(I: Idx) == OpV;
4461 return false;
4462}
4463
4464/// Returns true if the wide recipes in \p Ops and their operand trees can be
4465/// narrowed. \p FirstMembersOf maps each in-loop value to the first member
4466/// list it was seen in, across all store groups.
4467static bool
4468canNarrowOps(ArrayRef<VPValue *> Ops, bool IsScalable,
4469 DenseMap<VPValue *, SmallVector<VPValue *>> &FirstMembersOf) {
4470 auto *WideMember0 = dyn_cast<VPRecipeWithIRFlags>(Val: Ops[0]);
4471 if (!WideMember0)
4472 return false;
4473 for (VPValue *V : Ops) {
4474 if (!isa<VPWidenRecipe, VPWidenCastRecipe>(Val: V))
4475 return false;
4476 auto *R = cast<VPRecipeWithIRFlags>(Val: V);
4477 if (vputils::getOpcode(V: R) != vputils::getOpcode(V: WideMember0))
4478 return false;
4479 if (R->getScalarType() != WideMember0->getScalarType())
4480 return false;
4481 if (R->hasPredicate() && R->getPredicate() != WideMember0->getPredicate())
4482 return false;
4483 }
4484
4485 // The first member is narrowed in place using the other members' operands.
4486 // Bail out if a value is the first member of one list and also part of a
4487 // different list.
4488 for (VPValue *V : Ops) {
4489 if (V->isDefinedOutsideLoopRegions())
4490 continue;
4491 auto [It, Inserted] = FirstMembersOf.try_emplace(Key: V, Args&: Ops);
4492 if (!Inserted && (It->second.front() == V || V == WideMember0) &&
4493 !equal(LRange&: It->second, RRange&: Ops))
4494 return false;
4495 }
4496
4497 for (unsigned Idx = 0; Idx != WideMember0->getNumOperands(); ++Idx) {
4498 SmallVector<VPValue *> OpsI;
4499 for (VPValue *Op : Ops)
4500 OpsI.push_back(Elt: Op->getDefiningRecipe()->getOperand(N: Idx));
4501
4502 if (canNarrowOps(Ops: OpsI, IsScalable, FirstMembersOf))
4503 continue;
4504
4505 if (any_of(Range: enumerate(First&: OpsI), P: [WideMember0, Idx, IsScalable](const auto &P) {
4506 const auto &[OpIdx, OpV] = P;
4507 return !canNarrowLoad(WideMember0, Idx, OpV, OpIdx, IsScalable);
4508 }))
4509 return false;
4510 }
4511
4512 return true;
4513}
4514
4515/// Returns VF from \p VFs if \p IR is a full interleave group with factor and
4516/// number of members both equal to VF. The interleave group must also access
4517/// the full vector width.
4518static std::optional<ElementCount>
4519isConsecutiveInterleaveGroup(VPInterleaveRecipe *InterleaveR,
4520 ArrayRef<ElementCount> VFs,
4521 const TargetTransformInfo &TTI) {
4522 if (!InterleaveR || InterleaveR->getMask())
4523 return std::nullopt;
4524
4525 Type *GroupElementTy = nullptr;
4526 if (InterleaveR->getStoredValues().empty()) {
4527 GroupElementTy = InterleaveR->getVPValue(I: 0)->getScalarType();
4528 if (!all_of(Range: InterleaveR->definedValues(), P: [GroupElementTy](VPValue *Op) {
4529 return Op->getScalarType() == GroupElementTy;
4530 }))
4531 return std::nullopt;
4532 } else {
4533 GroupElementTy = InterleaveR->getStoredValues()[0]->getScalarType();
4534 if (!all_of(Range: InterleaveR->getStoredValues(), P: [GroupElementTy](VPValue *Op) {
4535 return Op->getScalarType() == GroupElementTy;
4536 }))
4537 return std::nullopt;
4538 }
4539
4540 auto IG = InterleaveR->getInterleaveGroup();
4541 if (IG->getFactor() != IG->getNumMembers() || IG->isReverse())
4542 return std::nullopt;
4543
4544 auto GetVectorBitWidthForVF = [&TTI](ElementCount VF) {
4545 TypeSize Size = TTI.getRegisterBitWidth(
4546 K: VF.isFixed() ? TargetTransformInfo::RGK_FixedWidthVector
4547 : TargetTransformInfo::RGK_ScalableVector);
4548 assert(Size.isScalable() == VF.isScalable() &&
4549 "if Size is scalable, VF must be scalable and vice versa");
4550 return Size.getKnownMinValue();
4551 };
4552
4553 for (ElementCount VF : VFs) {
4554 unsigned MinVal = VF.getKnownMinValue();
4555 unsigned GroupSize = GroupElementTy->getScalarSizeInBits() * MinVal;
4556 if (IG->getFactor() == MinVal && GroupSize == GetVectorBitWidthForVF(VF))
4557 return {VF};
4558 }
4559 return std::nullopt;
4560}
4561
4562/// Returns true if \p VPValue is a narrow VPValue.
4563static bool isAlreadyNarrow(VPValue *VPV) {
4564 if (isa<VPIRValue>(Val: VPV))
4565 return true;
4566 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: VPV);
4567 return RepR && RepR->isSingleScalar();
4568}
4569
4570// Convert the wide recipes defining the VPValues in \p Members feeding an
4571// interleave group to a single narrow variant. The first member is reused as
4572// the narrowed recipe. BuildVectors for live-in operands are inserted into \p
4573// Preheader.
4574static VPValue *narrowInterleaveGroupOp(ArrayRef<VPValue *> Members,
4575 SmallPtrSetImpl<VPValue *> &NarrowedOps,
4576 VPBasicBlock *Preheader) {
4577 VPValue *V = Members.front();
4578 if (NarrowedOps.contains(Ptr: V))
4579 return V;
4580
4581 if (V->isDefinedOutsideLoopRegions()) {
4582 assert(all_of(Members,
4583 [V](VPValue *M) {
4584 return M->isDefinedOutsideLoopRegions() &&
4585 M->getScalarType() == V->getScalarType();
4586 }) &&
4587 "expected distinct loop-invariant values of matching scalar type");
4588 auto *BV = new VPInstruction(VPInstruction::BuildVector, Members);
4589 Preheader->appendRecipe(Recipe: BV);
4590 NarrowedOps.insert(Ptr: BV);
4591 return BV;
4592 }
4593
4594 if (isAlreadyNarrow(VPV: V))
4595 return V;
4596
4597 VPRecipeBase *R = V->getDefiningRecipe();
4598 if (isa<VPWidenRecipe, VPWidenCastRecipe>(Val: R)) {
4599 auto *WideMember0 = cast<VPRecipeWithIRFlags>(Val: R);
4600 for (VPValue *Member : Members.drop_front())
4601 WideMember0->intersectFlags(Other: *cast<VPRecipeWithIRFlags>(Val: Member));
4602 for (unsigned Idx = 0, E = WideMember0->getNumOperands(); Idx != E; ++Idx) {
4603 SmallVector<VPValue *> OpsI;
4604 for (VPValue *Member : Members)
4605 OpsI.push_back(Elt: Member->getDefiningRecipe()->getOperand(N: Idx));
4606 WideMember0->setOperand(
4607 I: Idx, New: narrowInterleaveGroupOp(Members: OpsI, NarrowedOps, Preheader));
4608 }
4609 return V;
4610 }
4611
4612 if (auto *LoadGroup = dyn_cast<VPInterleaveRecipe>(Val: R)) {
4613 // Narrow interleave group to wide load, as transformed VPlan will only
4614 // process one original iteration.
4615 const InterleaveGroup<Instruction> *IG = LoadGroup->getInterleaveGroup();
4616 auto *LI = cast<LoadInst>(Val: IG->getInsertPos());
4617 auto *L = VPBuilder(LoadGroup).createWidenLoad(
4618 Load&: *LI, Addr: LoadGroup->getAddr(), Mask: LoadGroup->getMask(), /*Consecutive=*/true,
4619 Alignment: IG->getAlign(), Metadata: *LoadGroup, DL: LoadGroup->getDebugLoc());
4620 NarrowedOps.insert(Ptr: L);
4621 return L;
4622 }
4623
4624 auto *WideLoad = cast<VPWidenLoadRecipe>(Val: R);
4625 VPValue *PtrOp = WideLoad->getAddr();
4626 if (auto *VecPtr = dyn_cast<VPVectorPointerRecipe>(Val: PtrOp))
4627 PtrOp = VecPtr->getOperand(N: 0);
4628 // Narrow wide load to uniform scalar load, as transformed VPlan will only
4629 // process one original iteration.
4630 auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(), {PtrOp},
4631 /*IsUniform*/ true,
4632 /*Mask*/ nullptr, {}, *WideLoad);
4633 N->insertBefore(InsertPos: WideLoad);
4634 NarrowedOps.insert(Ptr: N);
4635 return N;
4636}
4637
4638std::unique_ptr<VPlan>
4639VPlanTransforms::narrowInterleaveGroups(VPlan &Plan,
4640 const TargetTransformInfo &TTI) {
4641 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
4642
4643 if (!VectorLoop)
4644 return nullptr;
4645
4646 // Only handle single-block loops for now.
4647 if (VectorLoop->getEntryBasicBlock() != VectorLoop->getExitingBasicBlock())
4648 return nullptr;
4649
4650 // Skip plans when we may not be able to properly narrow.
4651 VPBasicBlock *Exiting = VectorLoop->getExitingBasicBlock();
4652 if (!match(V: &Exiting->back(), P: m_BranchOnCount()))
4653 return nullptr;
4654
4655 assert(match(&Exiting->back(),
4656 m_BranchOnCount(m_Add(m_VPValue(), m_Specific(&Plan.getVFxUF())),
4657 m_Specific(&Plan.getVectorTripCount()))) &&
4658 "unexpected branch-on-count");
4659
4660 SmallVector<VPInterleaveRecipe *> StoreGroups;
4661 DenseMap<VPValue *, SmallVector<VPValue *>> FirstMembersOf;
4662 std::optional<ElementCount> VFToOptimize;
4663 for (auto &R : *VectorLoop->getEntryBasicBlock()) {
4664 if (isa<VPDerivedIVRecipe, VPScalarIVStepsRecipe>(Val: &R) &&
4665 vputils::onlyFirstLaneUsed(Def: cast<VPSingleDefRecipe>(Val: &R)))
4666 continue;
4667
4668 // Bail out on recipes not supported at the moment:
4669 // * phi recipes other than the canonical induction
4670 // * recipes writing to memory except interleave groups
4671 // Only support plans with a canonical induction phi.
4672 if (R.isPhi())
4673 return nullptr;
4674
4675 auto *InterleaveR = dyn_cast<VPInterleaveRecipe>(Val: &R);
4676 if (R.mayWriteToMemory() && !InterleaveR)
4677 return nullptr;
4678
4679 // Bail out if any recipe defines a vector value used outside the
4680 // vector loop region.
4681 if (any_of(Range: R.definedValues(), P: [&](VPValue *V) {
4682 return any_of(Range: V->users(), P: [&](VPUser *U) {
4683 auto *UR = cast<VPRecipeBase>(Val: U);
4684 return UR->getParent()->getParent() != VectorLoop;
4685 });
4686 }))
4687 return nullptr;
4688
4689 // All other ops are allowed, but we reject uses that cannot be converted
4690 // when checking all allowed consumers (store interleave groups) below.
4691 if (!InterleaveR)
4692 continue;
4693
4694 // Try to find a single VF, where all interleave groups are consecutive and
4695 // saturate the full vector width. If we already have a candidate VF, check
4696 // if it is applicable for the current InterleaveR, otherwise look for a
4697 // suitable VF across the Plan's VFs.
4698 SmallVector<ElementCount> VFs =
4699 VFToOptimize ? SmallVector<ElementCount>({*VFToOptimize})
4700 : to_vector(Range: Plan.vectorFactors());
4701 std::optional<ElementCount> NarrowedVF =
4702 isConsecutiveInterleaveGroup(InterleaveR, VFs, TTI);
4703 if (!NarrowedVF || (VFToOptimize && NarrowedVF != VFToOptimize))
4704 return nullptr;
4705 VFToOptimize = NarrowedVF;
4706
4707 // Skip read interleave groups.
4708 if (InterleaveR->getStoredValues().empty())
4709 continue;
4710
4711 // Narrow interleave groups, if all operands are already matching narrow
4712 // ops.
4713 auto *Member0 = InterleaveR->getStoredValues()[0];
4714 if (isAlreadyNarrow(VPV: Member0) &&
4715 all_of(Range: InterleaveR->getStoredValues(), P: equal_to(Arg&: Member0))) {
4716 StoreGroups.push_back(Elt: InterleaveR);
4717 continue;
4718 }
4719
4720 // For now, we only support full interleave groups storing load interleave
4721 // groups.
4722 if (all_of(Range: enumerate(First: InterleaveR->getStoredValues()), P: [](auto Op) {
4723 VPRecipeBase *DefR = Op.value()->getDefiningRecipe();
4724 if (!DefR)
4725 return false;
4726 auto *IR = dyn_cast<VPInterleaveRecipe>(Val: DefR);
4727 return IR && IR->getInterleaveGroup()->isFull() &&
4728 IR->getVPValue(Op.index()) == Op.value();
4729 })) {
4730 StoreGroups.push_back(Elt: InterleaveR);
4731 continue;
4732 }
4733
4734 // Check if all values feeding InterleaveR are matching wide recipes whose
4735 // operands can be narrowed, without conflicting uses across groups.
4736 if (!canNarrowOps(Ops: InterleaveR->getStoredValues(),
4737 IsScalable: VFToOptimize->isScalable(), FirstMembersOf))
4738 return nullptr;
4739 StoreGroups.push_back(Elt: InterleaveR);
4740 }
4741
4742 if (StoreGroups.empty())
4743 return nullptr;
4744
4745 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
4746 bool RequiresScalarEpilogue =
4747 MiddleVPBB->getNumSuccessors() == 1 &&
4748 MiddleVPBB->getSingleSuccessor() == Plan.getScalarPreheader();
4749 // Bail out for tail-folding (middle block with a single successor to exit).
4750 if (MiddleVPBB->getNumSuccessors() != 2 && !RequiresScalarEpilogue)
4751 return nullptr;
4752
4753 // All interleave groups in Plan can be narrowed for VFToOptimize. Split the
4754 // original Plan into 2: a) a new clone which contains all VFs of Plan, except
4755 // VFToOptimize, and b) the original Plan with VFToOptimize as single VF.
4756 // TODO: Handle cases where only some interleave groups can be narrowed.
4757 std::unique_ptr<VPlan> NewPlan;
4758 if (size(Range: Plan.vectorFactors()) != 1) {
4759 NewPlan = std::unique_ptr<VPlan>(Plan.duplicate());
4760 Plan.setVF(*VFToOptimize);
4761 NewPlan->removeVF(VF: *VFToOptimize);
4762 }
4763
4764 // Convert InterleaveGroup \p R to a single VPWidenLoadRecipe.
4765 SmallPtrSet<VPValue *, 4> NarrowedOps;
4766 VPBasicBlock *Preheader = Plan.getVectorPreheader();
4767 // Narrow operation tree rooted at store groups.
4768 for (auto *StoreGroup : StoreGroups) {
4769 VPValue *Res = narrowInterleaveGroupOp(Members: StoreGroup->getStoredValues(),
4770 NarrowedOps, Preheader);
4771 const InterleaveGroup<Instruction> *IG = StoreGroup->getInterleaveGroup();
4772 auto *SI = cast<StoreInst>(Val: IG->getInsertPos());
4773 VPBuilder(StoreGroup)
4774 .createWidenStore(Store&: *SI, Addr: StoreGroup->getAddr(), StoredVal: Res, Mask: nullptr,
4775 /*Consecutive=*/true, Alignment: IG->getAlign(), Metadata: *StoreGroup,
4776 DL: StoreGroup->getDebugLoc());
4777 StoreGroup->eraseFromParent();
4778 }
4779
4780 // Adjust induction to reflect that the transformed plan only processes one
4781 // original iteration.
4782 VPInstruction *CanIVInc = vputils::findCanonicalIVIncrement(Plan);
4783 Type *CanIVTy = VectorLoop->getCanonicalIVType();
4784 VPBasicBlock *VectorPH = Plan.getVectorPreheader();
4785 VPBuilder PHBuilder(VectorPH->getFirstNonPhi());
4786
4787 VPValue *UF = &Plan.getUF();
4788 VPValue *Step;
4789 if (VFToOptimize->isScalable()) {
4790 VPValue *VScale =
4791 PHBuilder.createElementCount(Ty: CanIVTy, EC: ElementCount::getScalable(MinVal: 1));
4792 Step = PHBuilder.createOverflowingOp(Opcode: Instruction::Mul, Operands: {VScale, UF},
4793 WrapFlags: {true, false});
4794 Plan.getVF().replaceAllUsesWith(New: VScale);
4795 } else {
4796 Step = UF;
4797 Plan.getVF().replaceAllUsesWith(New: Plan.getConstantInt(Ty: CanIVTy, Val: 1));
4798 }
4799 // Materialize vector trip count with the narrowed step.
4800 materializeVectorTripCount(Plan, VectorPHVPBB: VectorPH, /*TailByMasking=*/false,
4801 RequiresScalarEpilogue, Step);
4802
4803 CanIVInc->setOperand(I: 1, New: Step);
4804 Plan.getVFxUF().replaceAllUsesWith(New: Step);
4805
4806 removeDeadRecipes(Plan);
4807 assert(none_of(*VectorLoop->getEntryBasicBlock(),
4808 IsaPred<VPVectorPointerRecipe>) &&
4809 "All VPVectorPointerRecipes should have been removed");
4810 return NewPlan;
4811}
4812
4813void VPlanTransforms::adjustFirstOrderRecurrenceMiddleUsers(VPlan &Plan,
4814 VFRange &Range) {
4815 VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
4816 auto *MiddleVPBB = Plan.getMiddleBlock();
4817 VPBuilder MiddleBuilder(MiddleVPBB->getFirstNonPhi());
4818
4819 auto IsScalableOne = [](ElementCount VF) -> bool {
4820 return VF == ElementCount::getScalable(MinVal: 1);
4821 };
4822
4823 for (VPFirstOrderRecurrencePHIRecipe &FOR :
4824 make_isa_range<VPFirstOrderRecurrencePHIRecipe>(
4825 Range: VectorRegion->getEntryBasicBlock()->phis())) {
4826 assert(VectorRegion->getSingleSuccessor() == Plan.getMiddleBlock() &&
4827 "Cannot handle loops with uncountable early exits");
4828
4829 // Find the existing splice for this FOR, created in
4830 // createHeaderPhiRecipes. All uses of FOR have already been replaced with
4831 // RecurSplice there; only RecurSplice itself still references FOR.
4832 auto *RecurSplice =
4833 findSingleUser<VPInstruction::FirstOrderRecurrenceSplice>(V: &FOR);
4834 assert(RecurSplice && "expected FirstOrderRecurrenceSplice");
4835
4836 // For VF vscale x 1, if vscale = 1, we are unable to extract the
4837 // penultimate value of the recurrence. Instead we rely on the existing
4838 // extract of the last element from the result of
4839 // VPInstruction::FirstOrderRecurrenceSplice.
4840 // TODO: Consider vscale_range info and UF.
4841 if (any_of(Range: RecurSplice->users(),
4842 P: [](VPUser *U) { return !cast<VPRecipeBase>(Val: U)->getRegion(); }) &&
4843 LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsScalableOne,
4844 Range))
4845 return;
4846
4847 // This is the second phase of vectorizing first-order recurrences, creating
4848 // extracts for users outside the loop. An overview of the transformation is
4849 // described below. Suppose we have the following loop with some use after
4850 // the loop of the last a[i-1],
4851 //
4852 // for (int i = 0; i < n; ++i) {
4853 // t = a[i - 1];
4854 // b[i] = a[i] - t;
4855 // }
4856 // use t;
4857 //
4858 // There is a first-order recurrence on "a". For this loop, the shorthand
4859 // scalar IR looks like:
4860 //
4861 // scalar.ph:
4862 // s.init = a[-1]
4863 // br scalar.body
4864 //
4865 // scalar.body:
4866 // i = phi [0, scalar.ph], [i+1, scalar.body]
4867 // s1 = phi [s.init, scalar.ph], [s2, scalar.body]
4868 // s2 = a[i]
4869 // b[i] = s2 - s1
4870 // br cond, scalar.body, exit.block
4871 //
4872 // exit.block:
4873 // use = lcssa.phi [s1, scalar.body]
4874 //
4875 // In this example, s1 is a recurrence because it's value depends on the
4876 // previous iteration. In the first phase of vectorization, we created a
4877 // VPFirstOrderRecurrencePHIRecipe v1 for s1. Now we create the extracts
4878 // for users in the scalar preheader and exit block.
4879 //
4880 // vector.ph:
4881 // v_init = vector(..., ..., ..., a[-1])
4882 // br vector.body
4883 //
4884 // vector.body
4885 // i = phi [0, vector.ph], [i+4, vector.body]
4886 // v1 = phi [v_init, vector.ph], [v2, vector.body]
4887 // v2 = a[i, i+1, i+2, i+3]
4888 // v1' = splice(v1(3), v2(0, 1, 2))
4889 // b[i, i+1, i+2, i+3] = v2 - v1'
4890 // br cond, vector.body, middle.block
4891 //
4892 // middle.block:
4893 // vector.recur.extract.for.phi = v2(2)
4894 // vector.recur.extract = v2(3)
4895 // br cond, scalar.ph, exit.block
4896 //
4897 // scalar.ph:
4898 // scalar.recur.init = phi [vector.recur.extract, middle.block],
4899 // [s.init, otherwise]
4900 // br scalar.body
4901 //
4902 // scalar.body:
4903 // i = phi [0, scalar.ph], [i+1, scalar.body]
4904 // s1 = phi [scalar.recur.init, scalar.ph], [s2, scalar.body]
4905 // s2 = a[i]
4906 // b[i] = s2 - s1
4907 // br cond, scalar.body, exit.block
4908 //
4909 // exit.block:
4910 // lo = lcssa.phi [s1, scalar.body],
4911 // [vector.recur.extract.for.phi, middle.block]
4912 //
4913 // Update extracts of the splice in the middle block: they extract the
4914 // penultimate element of the recurrence.
4915 for (VPRecipeBase &R : make_early_inc_range(
4916 Range: make_range(x: MiddleVPBB->getFirstNonPhi(), y: MiddleVPBB->end()))) {
4917 if (!match(V: &R, P: m_ExtractLastLaneOfLastPart(Op0: m_Specific(VPV: RecurSplice))))
4918 continue;
4919
4920 auto *ExtractR = cast<VPInstruction>(Val: &R);
4921 VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
4922 Opcode: VPInstruction::ExtractPenultimateElement, Operands: RecurSplice->getOperand(N: 1),
4923 DL: {}, Name: "vector.recur.extract.for.phi");
4924 for (VPUser *ExitU : to_vector(Range: ExtractR->users())) {
4925 if (auto *ExitPhi = dyn_cast<VPIRPhi>(Val: ExitU))
4926 ExitPhi->replaceUsesOfWith(From: ExtractR, To: PenultimateElement);
4927 }
4928 }
4929 }
4930}
4931
4932/// Check if \p V is a binary expression of a widened IV and a loop-invariant
4933/// value. Returns the widened IV if found, nullptr otherwise.
4934static VPWidenIntOrFpInductionRecipe *getExpressionIV(VPValue *V) {
4935 auto *BinOp = dyn_cast<VPWidenRecipe>(Val: V);
4936 if (!BinOp || !Instruction::isBinaryOp(Opcode: BinOp->getOpcode()) ||
4937 Instruction::isIntDivRem(Opcode: BinOp->getOpcode()))
4938 return nullptr;
4939
4940 VPValue *WidenIVCandidate = BinOp->getOperand(N: 0);
4941 VPValue *InvariantCandidate = BinOp->getOperand(N: 1);
4942 if (!isa<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate))
4943 std::swap(a&: WidenIVCandidate, b&: InvariantCandidate);
4944
4945 if (!InvariantCandidate->isDefinedOutsideLoopRegions())
4946 return nullptr;
4947
4948 return dyn_cast<VPWidenIntOrFpInductionRecipe>(Val: WidenIVCandidate);
4949}
4950
4951/// Create a scalar version of \p BinOp, with its \p WidenIV operand replaced
4952/// by \p ScalarIV, and place it after \p ScalarIV's defining recipe.
4953static VPValue *cloneBinOpForScalarIV(VPWidenRecipe *BinOp, VPValue *ScalarIV,
4954 VPWidenIntOrFpInductionRecipe *WidenIV) {
4955 assert(Instruction::isBinaryOp(BinOp->getOpcode()) &&
4956 BinOp->getNumOperands() == 2 && "BinOp must have 2 operands");
4957 auto *ClonedOp = BinOp->clone();
4958 if (ClonedOp->getOperand(N: 0) == WidenIV) {
4959 ClonedOp->setOperand(I: 0, New: ScalarIV);
4960 } else {
4961 assert(ClonedOp->getOperand(1) == WidenIV && "one operand must be WideIV");
4962 ClonedOp->setOperand(I: 1, New: ScalarIV);
4963 }
4964 ClonedOp->insertAfter(InsertPos: ScalarIV->getDefiningRecipe());
4965 return ClonedOp;
4966}
4967
4968/// If \p S is an affine AddRec, returns true if its step is known to be
4969/// positive and false if it is known to be negative. Returns std::nullopt if
4970/// \p S is not an affine AddRec, or if the sign of its step cannot be
4971/// determined.
4972static std::optional<bool> getStepDirection(const SCEV *S,
4973 ScalarEvolution &SE) {
4974 const SCEV *Step;
4975 if (!match(S, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV(V&: Step))))
4976 return std::nullopt;
4977 if (SE.isKnownPositive(S: Step))
4978 return true;
4979 if (SE.isKnownNegative(S: Step))
4980 return false;
4981 return std::nullopt;
4982}
4983
4984void VPlanTransforms::optimizeFindIVReductions(VPlan &Plan,
4985 PredicatedScalarEvolution &PSE,
4986 Loop &L) {
4987 ScalarEvolution &SE = *PSE.getSE();
4988 VPRegionBlock *VectorLoopRegion = Plan.getVectorLoopRegion();
4989
4990 // Helper lambda to check if the IV range excludes the sentinel value. Try
4991 // signed first, then unsigned. Return an excluded sentinel if found,
4992 // otherwise return std::nullopt.
4993 auto CheckSentinel = [&SE](const SCEV *IVSCEV,
4994 bool UseMax) -> std::optional<APSInt> {
4995 unsigned BW = IVSCEV->getType()->getScalarSizeInBits();
4996 for (bool Signed : {true, false}) {
4997 APSInt Sentinel = UseMax ? APSInt::getMinValue(numBits: BW, /*Unsigned=*/!Signed)
4998 : APSInt::getMaxValue(numBits: BW, /*Unsigned=*/!Signed);
4999
5000 ConstantRange IVRange =
5001 Signed ? SE.getSignedRange(S: IVSCEV) : SE.getUnsignedRange(S: IVSCEV);
5002 if (!IVRange.contains(Val: Sentinel))
5003 return Sentinel;
5004 }
5005 return std::nullopt;
5006 };
5007
5008 VPValue *HeaderMask = VectorLoopRegion->getHeaderMask();
5009 for (VPRecipeBase &Phi :
5010 make_early_inc_range(Range: VectorLoopRegion->getEntryBasicBlock()->phis())) {
5011 auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: &Phi);
5012 if (!PhiR || !RecurrenceDescriptor::isFindLastRecurrenceKind(
5013 Kind: PhiR->getRecurrenceKind()))
5014 continue;
5015
5016 Type *PhiTy = PhiR->getScalarType();
5017 if (PhiTy->isPointerTy() || PhiTy->isFloatingPointTy())
5018 continue;
5019
5020 // If there's a header mask, the backedge select will not be the find-last
5021 // select.
5022 VPValue *BackedgeVal = PhiR->getBackedgeValue();
5023 auto *FindLastSelect = cast<VPSingleDefRecipe>(Val: BackedgeVal);
5024 if (HeaderMask &&
5025 !match(V: BackedgeVal,
5026 P: m_Select(Op0: m_Specific(VPV: HeaderMask),
5027 Op1: m_VPSingleDefRecipe(V&: FindLastSelect), Op2: m_Specific(VPV: PhiR))))
5028 continue;
5029
5030 // Get the find-last expression from the find-last select of the reduction
5031 // phi. The find-last select should be a select between the phi and the
5032 // find-last expression.
5033 VPValue *Cond, *FindLastExpression;
5034 if (!match(R: FindLastSelect, P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: PhiR),
5035 Op2: m_VPValue(V&: FindLastExpression))) &&
5036 !match(R: FindLastSelect,
5037 P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_VPValue(V&: FindLastExpression),
5038 Op2: m_Specific(VPV: PhiR))))
5039 continue;
5040
5041 // Check if FindLastExpression is a simple expression of a widened IV. If
5042 // so, we can track the underlying IV instead and sink the expression.
5043 auto *IVOfExpressionToSink = getExpressionIV(V: FindLastExpression);
5044 const SCEV *IVSCEV = vputils::getSCEVExprForVPValue(
5045 V: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression, PSE,
5046 L: &L);
5047 if (!match(S: IVSCEV, P: m_scev_AffineAddRec(Op0: m_SCEV(), Op1: m_SCEV()))) {
5048 assert(!match(vputils::getSCEVExprForVPValue(FindLastExpression, PSE, &L),
5049 m_scev_AffineAddRec(m_SCEV(), m_SCEV())) &&
5050 "IVOfExpressionToSink not being an AddRec must imply "
5051 "FindLastExpression not being an AddRec.");
5052 continue;
5053 }
5054
5055 // Determine direction from the step of IVSCEV, if possible.
5056 std::optional<bool> StepDirection = getStepDirection(S: IVSCEV, SE);
5057 if (!StepDirection)
5058 continue;
5059
5060 bool UseMax = *StepDirection;
5061 std::optional<APSInt> SentinelVal = CheckSentinel(IVSCEV, UseMax);
5062 bool UseSigned = SentinelVal && SentinelVal->isSigned();
5063
5064 // Sinking an expression will disable epilogue vectorization. Only use it,
5065 // if FindLastExpression cannot be vectorized via a sentinel. Sinking may
5066 // also prevent vectorizing using a sentinel (e.g., if the expression is a
5067 // multiply or divide by large constant, respectively), which also makes
5068 // sinking undesirable.
5069 if (IVOfExpressionToSink) {
5070 const SCEV *FindLastExpressionSCEV =
5071 vputils::getSCEVExprForVPValue(V: FindLastExpression, PSE, L: &L);
5072 if (std::optional<bool> NewUseMax =
5073 getStepDirection(S: FindLastExpressionSCEV, SE)) {
5074 if (auto NewSentinel =
5075 CheckSentinel(FindLastExpressionSCEV, *NewUseMax)) {
5076 // The original expression already has a sentinel, so prefer not
5077 // sinking to keep epilogue vectorization possible.
5078 SentinelVal = *NewSentinel;
5079 UseSigned = NewSentinel->isSigned();
5080 UseMax = *NewUseMax;
5081 IVSCEV = FindLastExpressionSCEV;
5082 IVOfExpressionToSink = nullptr;
5083 }
5084 }
5085 }
5086
5087 // If no sentinel was found, fall back to a boolean AnyOf reduction to track
5088 // if the condition was ever true. Requires the IV to not wrap, otherwise we
5089 // cannot use min/max.
5090 if (!SentinelVal) {
5091 auto *AR = cast<SCEVAddRecExpr>(Val: IVSCEV);
5092 if (AR->hasNoSignedWrap())
5093 UseSigned = true;
5094 else if (AR->hasNoUnsignedWrap())
5095 UseSigned = false;
5096 else
5097 continue;
5098 }
5099
5100 VPInstruction *RdxResult = cast<VPInstruction>(Val: vputils::findRecipe(
5101 Start: BackedgeVal,
5102 Pred: match_fn(P: m_VPInstruction<VPInstruction::ComputeReductionResult>())));
5103
5104 VPValue *NewFindLastSelect = BackedgeVal;
5105 VPValue *SelectCond = Cond;
5106 if (!SentinelVal || IVOfExpressionToSink) {
5107 // When we need to create a new select, normalize the condition so that
5108 // PhiR is the last operand and include the header mask if needed.
5109 DebugLoc DL = FindLastSelect->getDefiningRecipe()->getDebugLoc();
5110 VPBuilder LoopBuilder(FindLastSelect->getDefiningRecipe());
5111 if (match(R: FindLastSelect,
5112 P: m_SelectLike(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: PhiR), Op2: m_VPValue())))
5113 SelectCond = LoopBuilder.createNot(Operand: SelectCond);
5114
5115 // When tail folding, mask the condition with the header mask to prevent
5116 // propagating poison from inactive lanes in the last vector iteration.
5117 if (HeaderMask)
5118 SelectCond = LoopBuilder.createLogicalAnd(LHS: HeaderMask, RHS: SelectCond);
5119
5120 if (SelectCond != Cond || IVOfExpressionToSink) {
5121 NewFindLastSelect = LoopBuilder.createSelect(
5122 Cond: SelectCond,
5123 TrueVal: IVOfExpressionToSink ? IVOfExpressionToSink : FindLastExpression,
5124 FalseVal: PhiR, DL);
5125 }
5126 }
5127
5128 // Create the reduction result in the middle block using sentinel directly.
5129 RecurKind MinMaxKind =
5130 UseMax ? (UseSigned ? RecurKind::SMax : RecurKind::UMax)
5131 : (UseSigned ? RecurKind::SMin : RecurKind::UMin);
5132 VPIRFlags Flags(MinMaxKind, /*IsOrdered=*/false, /*IsInLoop=*/false,
5133 FastMathFlags());
5134 DebugLoc ExitDL = RdxResult->getDebugLoc();
5135 VPBuilder MiddleBuilder(RdxResult);
5136 VPValue *ReducedIV =
5137 MiddleBuilder.createNaryOp(Opcode: VPInstruction::ComputeReductionResult,
5138 Operands: NewFindLastSelect, Flags, DL: ExitDL);
5139
5140 // If IVOfExpressionToSink is an expression to sink, sink it now.
5141 VPValue *VectorRegionExitingVal = ReducedIV;
5142 bool SunkExpression = false;
5143 if (IVOfExpressionToSink) {
5144 VectorRegionExitingVal =
5145 cloneBinOpForScalarIV(BinOp: cast<VPWidenRecipe>(Val: FindLastExpression),
5146 ScalarIV: ReducedIV, WidenIV: IVOfExpressionToSink);
5147 SunkExpression = true;
5148 }
5149
5150 VPValue *NewRdxResult;
5151 VPValue *StartVPV = PhiR->getStartValue();
5152 if (SentinelVal) {
5153 // Sentinel-based approach: reduce IVs with min/max, compare against
5154 // sentinel to detect if condition was ever true, select accordingly.
5155 VPValue *Sentinel = Plan.getConstantInt(Val: *SentinelVal);
5156 auto *Cmp = MiddleBuilder.createICmp(Pred: CmpInst::ICMP_NE, A: ReducedIV,
5157 B: Sentinel, DL: ExitDL);
5158 NewRdxResult = MiddleBuilder.createSelect(Cond: Cmp, TrueVal: VectorRegionExitingVal,
5159 FalseVal: StartVPV, DL: ExitDL);
5160 StartVPV = Sentinel;
5161 } else {
5162 // Introduce a boolean AnyOf reduction to track if the condition was ever
5163 // true in the loop. Use it to select the initial start value, if it was
5164 // never true.
5165 auto *AnyOfPhi = new VPReductionPHIRecipe(
5166 /*Phi=*/nullptr, RecurKind::Or, *Plan.getFalse(), *Plan.getFalse(),
5167 RdxUnordered{.VFScaleFactor: 1}, {}, /*HasUsesOutsideReductionChain=*/false);
5168 AnyOfPhi->insertAfter(InsertPos: PhiR);
5169
5170 VPBuilder LoopBuilder(BackedgeVal->getDefiningRecipe());
5171 VPValue *OrVal = LoopBuilder.createOr(LHS: AnyOfPhi, RHS: SelectCond);
5172 AnyOfPhi->setOperand(I: 1, New: OrVal);
5173
5174 NewRdxResult = MiddleBuilder.createAnyOfReduction(
5175 ChainOp: OrVal, TrueVal: VectorRegionExitingVal, FalseVal: StartVPV, DL: ExitDL);
5176
5177 // Initialize the IV reduction phi with the neutral element, not the
5178 // original start value, to ensure correct min/max reduction results.
5179 StartVPV = Plan.getOrAddLiveIn(
5180 V: getRecurrenceIdentity(K: MinMaxKind, Tp: IVSCEV->getType(), FMF: {}));
5181 }
5182 RdxResult->replaceAllUsesWith(New: NewRdxResult);
5183 RdxResult->eraseFromParent();
5184
5185 auto *NewPhiR = new VPReductionPHIRecipe(
5186 cast<PHINode>(Val: PhiR->getUnderlyingInstr()), RecurKind::FindIV, *StartVPV,
5187 *NewFindLastSelect, RdxUnordered{.VFScaleFactor: 1}, {},
5188 PhiR->hasUsesOutsideReductionChain());
5189 if (SunkExpression)
5190 NewPhiR->setExpressionSunk();
5191 NewPhiR->insertBefore(InsertPos: PhiR);
5192 PhiR->replaceAllUsesWith(New: NewPhiR);
5193 PhiR->eraseFromParent();
5194 }
5195}
5196
5197namespace {
5198
5199struct ReductionExtend {
5200 Type *SrcType = nullptr;
5201 ExtendKind Kind = ExtendKind::PR_None;
5202};
5203
5204/// Describes the extends used to compute the extended reduction operand.
5205/// ExtendB is optional. If ExtendB is present, ExtendsUser is a binary
5206/// operation.
5207struct ExtendedReductionOperand {
5208 /// The recipe that consumes the extends.
5209 VPWidenRecipe *ExtendsUser = nullptr;
5210 /// Extend descriptions (inputs to getPartialReductionCost).
5211 ReductionExtend ExtendA, ExtendB;
5212};
5213
5214/// A collection of recipes that describe a partial reduction. Matches either
5215/// reduction_bin_op (extended op, accumulator), or
5216/// reduction_bin_op (accumulator, extended op).
5217/// The possible forms of the "extended op" are listed in
5218/// matchExtendedReductionOperand.
5219struct PartialReductionDescriptor {
5220 /// The top-level binary operation that forms the reduction to a scalar
5221 /// after the loop body.
5222 VPWidenRecipe *ReductionBinOp = nullptr;
5223 /// The user of the extends that is then reduced.
5224 ExtendedReductionOperand ExtendedOp;
5225 /// The recurrence kind for the entire partial reduction chain.
5226 /// This allows distinguishing between Sub and AddWithSub recurrences,
5227 /// when the ReductionBinOp is a Instruction::Sub.
5228 RecurKind RK;
5229 /// The index of the accumulator operand of ReductionBinOp. The extended op
5230 /// is `1 - AccumulatorOpIdx`.
5231 unsigned AccumulatorOpIdx;
5232 unsigned ScaleFactor;
5233 /// Optional blend to represent predication for the block that updates the
5234 /// reduction.
5235 VPBlendRecipe *Blend = nullptr;
5236};
5237
5238// Return the incoming index of the single-use value in the blend, which is
5239// expected to be the predicated reduction update.
5240static std::optional<unsigned>
5241getBlendReductionUpdateValueIdx(VPBlendRecipe *Blend) {
5242 assert(Blend && !Blend->isNormalized() &&
5243 Blend->getNumIncomingValues() == 2 &&
5244 "Expected a non-normalized blend with two incoming values");
5245 bool FirstIncomingHasOneUse = Blend->getIncomingValue(Idx: 0)->hasOneUse();
5246
5247 // Only the update value should have one use (the blend). The previous
5248 // value should always have at least two uses, the blend and the reduction.
5249 if (FirstIncomingHasOneUse == Blend->getIncomingValue(Idx: 1)->hasOneUse())
5250 return std::nullopt;
5251 return FirstIncomingHasOneUse ? 0 : 1;
5252}
5253
5254static VPSingleDefRecipe *
5255optimizeExtendsForPartialReduction(VPSingleDefRecipe *Op) {
5256 // reduce.add(mul(ext(A), C))
5257 // -> reduce.add(mul(ext(A), ext(trunc(C))))
5258 const APInt *Const;
5259 if (match(R: Op, P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_APInt(C&: Const)))) {
5260 auto *ExtA = cast<VPWidenCastRecipe>(Val: Op->getOperand(N: 0));
5261 Instruction::CastOps ExtOpc = ExtA->getOpcode();
5262 Type *NarrowTy = ExtA->getOperand(N: 0)->getScalarType();
5263 if (!Op->hasOneUse() ||
5264 !llvm::canConstantBeExtended(
5265 C: Const, NarrowType: NarrowTy, ExtKind: TTI::getPartialReductionExtendKind(CastOpc: ExtOpc)))
5266 return Op;
5267
5268 VPBuilder Builder(Op);
5269 auto *Trunc = Builder.createWidenCast(Opcode: Instruction::CastOps::Trunc,
5270 Op: Op->getOperand(N: 1), ResultTy: NarrowTy);
5271 Type *WideTy = ExtA->getScalarType();
5272 Op->setOperand(I: 1, New: Builder.createWidenCast(Opcode: ExtOpc, Op: Trunc, ResultTy: WideTy));
5273 return Op;
5274 }
5275
5276 // reduce.add(abs(sub(ext(A), ext(B))))
5277 // -> reduce.add(ext(absolute-difference(A, B)))
5278 VPValue *X, *Y;
5279 if (match(R: Op, P: m_WidenIntrinsic<Intrinsic::abs>(Ops: m_Sub(
5280 Op0: m_ZExtOrSExt(Op0: m_VPValue(V&: X)), Op1: m_ZExtOrSExt(Op0: m_VPValue(V&: Y)))))) {
5281 auto *Sub = Op->getOperand(N: 0)->getDefiningRecipe();
5282 auto *Ext = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
5283 assert(Ext->getOpcode() ==
5284 cast<VPWidenCastRecipe>(Sub->getOperand(1))->getOpcode() &&
5285 "Expected both the LHS and RHS extends to be the same");
5286 bool IsSigned = Ext->getOpcode() == Instruction::SExt;
5287 VPBuilder Builder(Op);
5288 Type *SrcTy = X->getScalarType();
5289 auto *FreezeX = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {X}));
5290 auto *FreezeY = Builder.insert(R: new VPWidenRecipe(Instruction::Freeze, {Y}));
5291 auto *Max = Builder.insert(
5292 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smax : Intrinsic::umax,
5293 {FreezeX, FreezeY}, SrcTy));
5294 auto *Min = Builder.insert(
5295 R: new VPWidenIntrinsicRecipe(IsSigned ? Intrinsic::smin : Intrinsic::umin,
5296 {FreezeX, FreezeY}, SrcTy));
5297 auto *AbsDiff = Builder.insert(
5298 R: new VPWidenRecipe(Instruction::Sub, {Max, Min},
5299 VPIRFlags::getDefaultFlags(Opcode: Instruction::Sub)));
5300 return Builder.createWidenCast(Opcode: Instruction::CastOps::ZExt, Op: AbsDiff,
5301 ResultTy: Op->getScalarType());
5302 }
5303
5304 // reduce.add(ext(mul(ext(A), ext(B))))
5305 // -> reduce.add(mul(wider_ext(A), wider_ext(B)))
5306 // TODO: Support this optimization for float types.
5307 if (match(R: Op, P: m_ZExtOrSExt(Op0: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
5308 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
5309 auto *Ext = cast<VPWidenCastRecipe>(Val: Op);
5310 auto *Mul = cast<VPWidenRecipe>(Val: Ext->getOperand(N: 0));
5311 auto *MulLHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5312 auto *MulRHS = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5313 assert(Mul->hasOneUse() && Ext->getOpcode() == MulLHS->getOpcode() &&
5314 MulLHS->getOpcode() == MulRHS->getOpcode() &&
5315 canFoldOuterExtendIntoMul(Mul) &&
5316 "Expected the outer extend to be foldable");
5317 VPBuilder Builder(Mul);
5318 auto *NewLHS = Builder.createWidenCast(
5319 Opcode: MulLHS->getOpcode(), Op: MulLHS->getOperand(N: 0), ResultTy: Ext->getScalarType());
5320 auto *NewRHS = MulLHS == MulRHS
5321 ? NewLHS
5322 : Builder.createWidenCast(Opcode: MulRHS->getOpcode(),
5323 Op: MulRHS->getOperand(N: 0),
5324 ResultTy: Ext->getScalarType());
5325 auto *NewMul = Mul->cloneWithOperands(NewOperands: {NewLHS, NewRHS});
5326 Builder.insert(R: NewMul);
5327 Op->replaceAllUsesWith(New: NewMul);
5328 Op->eraseFromParent();
5329 Mul->eraseFromParent();
5330 return NewMul;
5331 }
5332
5333 return Op;
5334}
5335
5336static VPExpressionRecipe *
5337createPartialReductionExpression(VPReductionRecipe *Red) {
5338 VPValue *VecOp = Red->getVecOp();
5339
5340 // reduce.[f]add(ext(op))
5341 // -> VPExpressionRecipe(op, red)
5342 if (match(V: VecOp, P: m_WidenAnyExtend(Op0: m_VPValue())))
5343 return new VPExpressionRecipe(cast<VPWidenCastRecipe>(Val: VecOp), Red);
5344
5345 // reduce.[f]add(neg(ext(op)))
5346 // -> VPExpressionRecipe(op, sub/neg, red)
5347 if (match(V: VecOp, P: m_AnyNeg(Op0: m_WidenAnyExtend(Op0: m_VPValue())))) {
5348 auto *Neg = cast<VPWidenRecipe>(Val: VecOp);
5349 auto *Ext = cast<VPWidenCastRecipe>(Val: Neg->getLastOperand());
5350 return new VPExpressionRecipe(Ext, Neg, Red);
5351 }
5352
5353 // reduce.[f]add([f]mul(ext(a), ext(b)))
5354 // -> VPExpressionRecipe(a, b, mul, red)
5355 if (match(V: VecOp, P: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))) ||
5356 match(V: VecOp,
5357 P: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()), Op1: m_ZExtOrSExt(Op0: m_VPValue())))) {
5358 auto *Mul = cast<VPWidenRecipe>(Val: VecOp);
5359 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5360 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5361 return new VPExpressionRecipe(ExtA, ExtB, Mul, Red);
5362 }
5363
5364 // reduce.fadd(fneg(fmul(fpext(a), fpext(b))))
5365 // -> VPExpressionRecipe(a, b, fmul, fsub, red)
5366 if (match(V: VecOp,
5367 P: m_FNeg(Op0: m_FMul(Op0: m_FPExt(Op0: m_VPValue()), Op1: m_FPExt(Op0: m_VPValue()))))) {
5368 auto *FNeg = cast<VPWidenRecipe>(Val: VecOp);
5369 auto *FMul = cast<VPWidenRecipe>(Val: FNeg->getOperand(N: 0));
5370 auto *ExtA = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 0));
5371 auto *ExtB = cast<VPWidenCastRecipe>(Val: FMul->getOperand(N: 1));
5372 return new VPExpressionRecipe(ExtA, ExtB, FMul, FNeg, Red);
5373 }
5374
5375 // reduce.add(neg(mul(ext(a), ext(b))))
5376 // -> VPExpressionRecipe(a, b, mul, sub, red)
5377 if (match(V: VecOp, P: m_Sub(Op0: m_ZeroInt(), Op1: m_Mul(Op0: m_ZExtOrSExt(Op0: m_VPValue()),
5378 Op1: m_ZExtOrSExt(Op0: m_VPValue()))))) {
5379 auto *Sub = cast<VPWidenRecipe>(Val: VecOp);
5380 auto *Mul = cast<VPWidenRecipe>(Val: Sub->getOperand(N: 1));
5381 auto *ExtA = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 0));
5382 auto *ExtB = cast<VPWidenCastRecipe>(Val: Mul->getOperand(N: 1));
5383 return new VPExpressionRecipe(ExtA, ExtB, Mul, Sub, Red);
5384 }
5385
5386 llvm_unreachable("Unsupported expression");
5387}
5388
5389// Helper to transform a PartialReductionDescriptor into a partial reduction
5390// recipe. Assumes profitability has been checked.
5391static void transformToPartialReduction(const PartialReductionDescriptor &Link,
5392 VPlan &Plan,
5393 VPReductionPHIRecipe *RdxPhi) {
5394 VPWidenRecipe *WidenRecipe = Link.ReductionBinOp;
5395 assert(WidenRecipe->getNumOperands() == 2 && "Expected binary operation");
5396
5397 VPValue *Accumulator = WidenRecipe->getOperand(N: Link.AccumulatorOpIdx);
5398 auto *ExtendedOp = cast<VPSingleDefRecipe>(
5399 Val: WidenRecipe->getOperand(N: 1 - Link.AccumulatorOpIdx));
5400
5401 // FIXME: Do these transforms before invoking the cost-model.
5402 ExtendedOp = optimizeExtendsForPartialReduction(Op: ExtendedOp);
5403
5404 // Sub-reductions can be implemented in two ways:
5405 // (1) negate the operand in the vector loop (the default way).
5406 // (2) subtract the reduced value from the init value in the middle block.
5407 // Both ways keep the reduction itself as an 'add' reduction.
5408 //
5409 // The ISD nodes for partial reductions don't support folding the
5410 // sub/negation into its operands because the following is not a valid
5411 // transformation:
5412 // sub(0, mul(ext(a), ext(b)))
5413 // -> mul(ext(a), ext(sub(0, b)))
5414 //
5415 // It's therefore better to choose option (2) such that the partial
5416 // reduction is always positive (starting at '0') and to do a final
5417 // subtract in the middle block.
5418 if ((WidenRecipe->getOpcode() == Instruction::Sub &&
5419 Link.RK != RecurKind::Sub) ||
5420 (WidenRecipe->getOpcode() == Instruction::FSub &&
5421 Link.RK != RecurKind::FSub)) {
5422 VPBuilder Builder(WidenRecipe);
5423 Type *ElemTy = ExtendedOp->getScalarType();
5424 VPWidenRecipe *NegRecipe;
5425 if (WidenRecipe->getOpcode() == Instruction::FSub) {
5426 NegRecipe =
5427 new VPWidenRecipe(Instruction::FNeg, {ExtendedOp},
5428 VPIRFlags::getDefaultFlags(Opcode: Instruction::FNeg),
5429 VPIRMetadata(), DebugLoc::getUnknown());
5430 } else {
5431 auto *Zero = Plan.getZero(Ty: ElemTy);
5432 NegRecipe =
5433 new VPWidenRecipe(Instruction::Sub, {Zero, ExtendedOp},
5434 VPIRFlags::getDefaultFlags(Opcode: Instruction::Sub),
5435 VPIRMetadata(), DebugLoc::getUnknown());
5436 }
5437 Builder.insert(R: NegRecipe);
5438 ExtendedOp = NegRecipe;
5439 }
5440
5441 // Check if WidenRecipe is the final result of the reduction. If so, look
5442 // through the Select recipe introduced by tail-folding, otherwise look
5443 // through any Blend recipe introduced by predication for the block.
5444 VPValue *ExitSearch =
5445 Link.Blend ? cast<VPValue>(Val: Link.Blend) : cast<VPValue>(Val: WidenRecipe);
5446
5447 VPValue *Cond = nullptr;
5448 VPValue *ExitValue = findSingleUser<VPInstruction>(
5449 V: ExitSearch,
5450 P: m_Select(Op0: m_VPValue(V&: Cond), Op1: m_Specific(VPV: ExitSearch), Op2: m_Specific(VPV: RdxPhi)));
5451
5452 if (Link.Blend) {
5453 std::optional<unsigned> BlendReductionIdx =
5454 getBlendReductionUpdateValueIdx(Blend: Link.Blend);
5455 assert(BlendReductionIdx &&
5456 Link.Blend->getIncomingValue(*BlendReductionIdx) == WidenRecipe &&
5457 "Expected blend to contain the reduction update");
5458 VPValue *BlendCond = Link.Blend->getMask(Idx: *BlendReductionIdx);
5459 Cond = ExitValue ? VPBuilder(WidenRecipe)
5460 .createLogicalAnd(LHS: Cond, RHS: BlendCond,
5461 DL: WidenRecipe->getDebugLoc())
5462 : BlendCond;
5463 }
5464
5465 // When folding the tail, the inactive lanes of the reduction update are
5466 // computed from values that do not correspond to any scalar iteration
5467 // and must not be accumulated.
5468 if (!Cond)
5469 Cond = Plan.getVectorLoopRegion()->getHeaderMask();
5470
5471 [[maybe_unused]] bool IsLastInChain =
5472 RdxPhi->getBackedgeValue() == WidenRecipe ||
5473 RdxPhi->getBackedgeValue() == ExitValue ||
5474 RdxPhi->getBackedgeValue() == Link.Blend;
5475 assert((!ExitValue || IsLastInChain) &&
5476 "if we found ExitValue, it must match RdxPhi's backedge value");
5477
5478 Type *PhiType = RdxPhi->getScalarType();
5479 RecurKind RdxKind =
5480 PhiType->isFloatingPointTy() ? RecurKind::FAdd : RecurKind::Add;
5481 auto *PartialRed = new VPReductionRecipe(
5482 RdxKind,
5483 RdxKind == RecurKind::FAdd ? WidenRecipe->getFastMathFlagsOrNone()
5484 : FastMathFlags(),
5485 WidenRecipe->getUnderlyingInstr(), Accumulator, ExtendedOp, Cond,
5486 RdxUnordered{/*VFScaleFactor=*/Link.ScaleFactor});
5487 PartialRed->insertBefore(InsertPos: WidenRecipe);
5488
5489 if (ExitValue)
5490 ExitValue->replaceAllUsesWith(New: PartialRed);
5491 if (Link.Blend)
5492 Link.Blend->replaceAllUsesWith(New: PartialRed);
5493 WidenRecipe->replaceAllUsesWith(New: PartialRed);
5494
5495 // For cost-model purposes, fold this into a VPExpression.
5496 VPExpressionRecipe *E = createPartialReductionExpression(Red: PartialRed);
5497 E->insertBefore(InsertPos: WidenRecipe);
5498 PartialRed->replaceAllUsesWith(New: E);
5499}
5500
5501/// Returns the cost of a link in a partial-reduction chain for a given VF.
5502static InstructionCost
5503getPartialReductionLinkCost(VPCostContext &CostCtx,
5504 const PartialReductionDescriptor &Link,
5505 ElementCount VF) {
5506 Type *RdxType = Link.ReductionBinOp->getScalarType();
5507 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
5508 std::optional<unsigned> BinOpc = std::nullopt;
5509 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
5510 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
5511 BinOpc = ExtendedOp.ExtendsUser->getOpcode();
5512
5513 std::optional<llvm::FastMathFlags> Flags;
5514 if (RdxType->isFloatingPointTy())
5515 Flags = Link.ReductionBinOp->getFastMathFlagsOrNone();
5516
5517 auto GetLinkOpcode = [&Link]() -> unsigned {
5518 switch (Link.RK) {
5519 case RecurKind::Sub:
5520 return Instruction::Add;
5521 case RecurKind::FSub:
5522 return Instruction::FAdd;
5523 default:
5524 return Link.ReductionBinOp->getOpcode();
5525 }
5526 };
5527
5528 return CostCtx.TTI.getPartialReductionCost(
5529 Opcode: GetLinkOpcode(), InputTypeA: ExtendedOp.ExtendA.SrcType, InputTypeB: ExtendedOp.ExtendB.SrcType,
5530 AccumType: RdxType, VF, OpAExtend: ExtendedOp.ExtendA.Kind, OpBExtend: ExtendedOp.ExtendB.Kind, BinOp: BinOpc,
5531 CostKind: CostCtx.CostKind, FMF: Flags);
5532}
5533
5534/// Checks if \p Op (which is an operand of \p UpdateR) is an extended reduction
5535/// operand. This is an operand where the source of the value (e.g. a load) has
5536/// been extended (sext, zext, or fpext) before it is used in the reduction.
5537///
5538/// Possible forms matched by this function:
5539/// - UpdateR(PrevValue, ext(...))
5540/// - UpdateR(PrevValue, mul(ext(...), ext(...)))
5541/// - UpdateR(PrevValue, mul(ext(...), Constant))
5542/// - UpdateR(PrevValue, ext(mul(ext(...), ext(...))))
5543/// - UpdateR(PrevValue, ext(mul(ext(...), Constant)))
5544/// - UpdateR(PrevValue, abs(sub(ext(...), ext(...)))
5545///
5546/// Note: The second operand of UpdateR corresponds to \p Op in the examples.
5547static std::optional<ExtendedReductionOperand>
5548matchExtendedReductionOperand(VPWidenRecipe *UpdateR, VPValue *Op) {
5549 assert(is_contained(UpdateR->operands(), Op) &&
5550 "Op should be operand of UpdateR");
5551
5552 // Try matching an absolute difference operand of the form
5553 // `abs(sub(ext(A), ext(B)))`. This will be later transformed into
5554 // `ext(absolute-difference(A, B))`. This allows us to perform the absolute
5555 // difference on a wider type and get the extend for "free" from the partial
5556 // reduction.
5557 VPValue *X, *Y;
5558 if (Op->hasOneUse() &&
5559 match(V: Op, P: m_WidenIntrinsic<Intrinsic::abs>(
5560 Ops: m_OneUse(SubPattern: m_Sub(Op0: m_WidenAnyExtend(Op0: m_VPValue(V&: X)),
5561 Op1: m_WidenAnyExtend(Op0: m_VPValue(V&: Y))))))) {
5562 auto *Abs = cast<VPWidenIntrinsicRecipe>(Val: Op);
5563 auto *Sub = cast<VPWidenRecipe>(Val: Abs->getOperand(N: 0));
5564 auto *LHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 0));
5565 auto *RHSExt = cast<VPWidenCastRecipe>(Val: Sub->getOperand(N: 1));
5566 Type *LHSInputType = X->getScalarType();
5567 Type *RHSInputType = Y->getScalarType();
5568 if (LHSInputType != RHSInputType ||
5569 LHSExt->getOpcode() != RHSExt->getOpcode())
5570 return std::nullopt;
5571 // Note: This is essentially the same as matching ext(...) as we will
5572 // rewrite this operand to ext(absolute-difference(A, B)).
5573 return ExtendedReductionOperand{
5574 .ExtendsUser: Sub,
5575 /*ExtendA=*/{.SrcType: LHSInputType, .Kind: getPartialReductionExtendKind(Cast: LHSExt)},
5576 /*ExtendB=*/{}};
5577 }
5578
5579 std::optional<TTI::PartialReductionExtendKind> OuterExtKind;
5580 if (match(V: Op, P: m_WidenAnyExtend(Op0: m_VPValue()))) {
5581 auto *CastRecipe = cast<VPWidenCastRecipe>(Val: Op);
5582 VPValue *CastSource = CastRecipe->getOperand(N: 0);
5583 OuterExtKind = getPartialReductionExtendKind(Cast: CastRecipe);
5584 if (match(V: CastSource, P: m_Mul(Op0: m_VPValue(), Op1: m_VPValue())) ||
5585 match(V: CastSource, P: m_FMul(Op0: m_VPValue(), Op1: m_VPValue()))) {
5586 // Match: ext(mul(...))
5587 // Record the outer extend kind and set `Op` to the mul. We can then match
5588 // this as a binary operation. Note: We can optimize out the outer extend
5589 // by widening the inner extends to match it. See
5590 // optimizeExtendsForPartialReduction.
5591 Op = CastSource;
5592 } else {
5593 return ExtendedReductionOperand{
5594 .ExtendsUser: UpdateR,
5595 /*ExtendA=*/{.SrcType: CastSource->getScalarType(), .Kind: *OuterExtKind},
5596 /*ExtendB=*/{}};
5597 }
5598 }
5599
5600 if (!Op->hasOneUse())
5601 return std::nullopt;
5602
5603 VPWidenRecipe *MulOp = dyn_cast<VPWidenRecipe>(Val: Op);
5604 if (!MulOp ||
5605 !is_contained(Set: {Instruction::Mul, Instruction::FMul}, Element: MulOp->getOpcode()))
5606 return std::nullopt;
5607
5608 // The rest of the matching assumes `Op` is a (possibly extended) mul
5609 // operation.
5610
5611 VPValue *LHS = MulOp->getOperand(N: 0);
5612 VPValue *RHS = MulOp->getOperand(N: 1);
5613
5614 // The LHS of the operation must always be an extend.
5615 if (!match(V: LHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
5616 return std::nullopt;
5617
5618 auto *LHSCast = cast<VPWidenCastRecipe>(Val: LHS);
5619 Type *LHSInputType = LHSCast->getOperand(N: 0)->getScalarType();
5620 ExtendKind LHSExtendKind = getPartialReductionExtendKind(Cast: LHSCast);
5621
5622 // The RHS of the operation can be an extend or a constant integer.
5623 const APInt *RHSConst = nullptr;
5624 VPWidenCastRecipe *RHSCast = nullptr;
5625 if (match(V: RHS, P: m_WidenAnyExtend(Op0: m_VPValue())))
5626 RHSCast = cast<VPWidenCastRecipe>(Val: RHS);
5627 else if (!match(V: RHS, P: m_APInt(C&: RHSConst)) ||
5628 !canConstantBeExtended(C: RHSConst, NarrowType: LHSInputType, ExtKind: LHSExtendKind))
5629 return std::nullopt;
5630
5631 // The outer extend kind must match the inner extends for folding.
5632 for (VPWidenCastRecipe *Cast : {LHSCast, RHSCast})
5633 if (Cast && OuterExtKind &&
5634 getPartialReductionExtendKind(Cast) != OuterExtKind)
5635 return std::nullopt;
5636
5637 Type *RHSInputType = LHSInputType;
5638 ExtendKind RHSExtendKind = LHSExtendKind;
5639 if (RHSCast) {
5640 RHSInputType = RHSCast->getOperand(N: 0)->getScalarType();
5641 RHSExtendKind = getPartialReductionExtendKind(Cast: RHSCast);
5642 }
5643
5644 // The outer extend can only be treated as extending the operands of the mul
5645 // if the narrow mul cannot wrap.
5646 if (OuterExtKind && MulOp->getOpcode() == Instruction::Mul &&
5647 !canFoldOuterExtendIntoMul(Mul: MulOp))
5648 return std::nullopt;
5649
5650 return ExtendedReductionOperand{
5651 .ExtendsUser: MulOp, .ExtendA: {.SrcType: LHSInputType, .Kind: LHSExtendKind}, .ExtendB: {.SrcType: RHSInputType, .Kind: RHSExtendKind}};
5652}
5653
5654/// Examines each operation in the reduction chain corresponding to \p RedPhiR,
5655/// and determines if the target can use a cheaper operation with a wider
5656/// per-iteration input VF and narrower PHI VF. If successful, returns the chain
5657/// of partial reduction descriptors (that are links in the reduction chain).
5658static std::optional<SmallVector<PartialReductionDescriptor>>
5659getScaledReductionChain(VPReductionPHIRecipe *RedPhiR) {
5660 // Get the backedge value from the reduction PHI and find the
5661 // ComputeReductionResult that uses it (directly or through a select for
5662 // predicated reductions).
5663 auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR);
5664 if (!RdxResult)
5665 return std::nullopt;
5666 VPValue *ExitValue = RdxResult->getOperand(N: 0);
5667 match(V: ExitValue, P: m_Select(Op0: m_VPValue(), Op1: m_VPValue(V&: ExitValue), Op2: m_VPValue()));
5668
5669 SmallVector<PartialReductionDescriptor> Chain;
5670 RecurKind RK = RedPhiR->getRecurrenceKind();
5671 Type *PhiType = RedPhiR->getScalarType();
5672 TypeSize PHISize = PhiType->getPrimitiveSizeInBits();
5673
5674 // Work backwards from the ExitValue examining each reduction operation.
5675 VPValue *CurrentValue = ExitValue;
5676 while (CurrentValue != RedPhiR) {
5677 VPBlendRecipe *Blend = dyn_cast<VPBlendRecipe>(Val: CurrentValue);
5678 std::optional<unsigned> BlendReductionIdx;
5679 if (Blend) {
5680 assert(!Blend->isNormalized() && "Expect Blend not to be normalized.");
5681 if (Blend->getNumIncomingValues() != 2)
5682 return std::nullopt;
5683
5684 BlendReductionIdx = getBlendReductionUpdateValueIdx(Blend);
5685 if (!BlendReductionIdx)
5686 return std::nullopt;
5687
5688 CurrentValue = Blend->getIncomingValue(Idx: *BlendReductionIdx);
5689 }
5690
5691 auto *UpdateR = dyn_cast<VPWidenRecipe>(Val: CurrentValue);
5692 if (!UpdateR || !Instruction::isBinaryOp(Opcode: UpdateR->getOpcode()))
5693 return std::nullopt;
5694
5695 VPValue *Op = UpdateR->getOperand(N: 1);
5696 VPValue *PrevValue = UpdateR->getOperand(N: 0);
5697
5698 // Find the extended operand. The other operand (PrevValue) is the next link
5699 // in the reduction chain.
5700 std::optional<ExtendedReductionOperand> ExtendedOp =
5701 matchExtendedReductionOperand(UpdateR, Op);
5702 if (!ExtendedOp) {
5703 ExtendedOp = matchExtendedReductionOperand(UpdateR, Op: PrevValue);
5704 if (!ExtendedOp)
5705 return std::nullopt;
5706 std::swap(a&: Op, b&: PrevValue);
5707 }
5708
5709 // Look for VPBlend(reduce(PrevValue, Op), PrevValue), where
5710 // reduce is equal to CurrentValue. This can be lowered as
5711 // a conditional reduction by hoisting the select to the inputs.
5712 if (Blend && Blend->getIncomingValue(Idx: 1 - *BlendReductionIdx) != PrevValue)
5713 return std::nullopt;
5714
5715 Type *ExtSrcType = ExtendedOp->ExtendA.SrcType;
5716 TypeSize ExtSrcSize = ExtSrcType->getPrimitiveSizeInBits();
5717 if (!PHISize.hasKnownScalarFactor(RHS: ExtSrcSize))
5718 return std::nullopt;
5719
5720 PartialReductionDescriptor Link(
5721 {.ReductionBinOp: UpdateR, .ExtendedOp: *ExtendedOp, .RK: RK,
5722 .AccumulatorOpIdx: PrevValue == UpdateR->getOperand(N: 0) ? 0U : 1U,
5723 .ScaleFactor: static_cast<unsigned>(PHISize.getKnownScalarFactor(RHS: ExtSrcSize)),
5724 .Blend: Blend});
5725 Chain.push_back(Elt: Link);
5726 CurrentValue = PrevValue;
5727 }
5728
5729 // The chain links were collected by traversing backwards from the exit value.
5730 // Reverse the chain so the links follow program order.
5731 std::reverse(first: Chain.begin(), last: Chain.end());
5732 return Chain;
5733}
5734} // namespace
5735
5736// Scale the PHI and ReductionStartVector by \p Factor and if the recurrence is
5737// a sub-recurrence, negate the reduction result.
5738static void updatePartialReductionPhiAndResult(VPlan &Plan,
5739 VPReductionPHIRecipe *Phi,
5740 unsigned Factor, RecurKind RK) {
5741 assert(Phi->getVFScaleFactor() == 1 && "scale factor must not be set");
5742 Phi->setVFScaleFactor(Factor);
5743
5744 auto *StartInst = cast<VPInstruction>(Val: Phi->getStartValue());
5745 assert(StartInst->getOpcode() == VPInstruction::ReductionStartVector);
5746 auto *NewScaleFactor = Plan.getConstantInt(BitWidth: 32, Val: Factor);
5747 StartInst->setOperand(I: 2, New: NewScaleFactor);
5748
5749 if (RK != RecurKind::Sub && RK != RecurKind::FSub)
5750 return;
5751
5752 // Update the PHI node to start at `0` and update the reduction-result
5753 // to subtract from the PHI's start value.
5754 VPValue *OldStartValue = StartInst->getOperand(N: 0);
5755 StartInst->setOperand(I: 0, New: StartInst->getOperand(N: 1));
5756
5757 // Replace reduction_result by 'sub (startval, reductionresult)'.
5758 VPInstruction *RdxResult = vputils::findComputeReductionResult(PhiR: Phi);
5759 assert(RdxResult && "Could not find reduction result");
5760
5761 VPBuilder Builder = VPBuilder::getToInsertAfter(R: RdxResult);
5762 unsigned SubOpc = RK == RecurKind::FSub ? Instruction::BinaryOps::FSub
5763 : Instruction::BinaryOps::Sub;
5764 VPInstruction *NewResult = Builder.createNaryOp(
5765 Opcode: SubOpc, Operands: {OldStartValue, RdxResult}, Flags: VPIRFlags::getDefaultFlags(Opcode: SubOpc),
5766 DL: Phi->getDebugLoc());
5767 RdxResult->replaceUsesWithIf(New: NewResult, ShouldReplace: not_equal_to(Arg&: NewResult));
5768}
5769
5770void VPlanTransforms::createPartialReductions(VPlan &Plan,
5771 VPCostContext &CostCtx,
5772 VFRange &Range) {
5773 // Find all possible valid partial reductions, grouping chains by their PHI.
5774 // This grouping allows invalidating the whole chain, if any link is not a
5775 // valid partial reduction.
5776 MapVector<VPReductionPHIRecipe *, SmallVector<PartialReductionDescriptor>>
5777 PhiToChain;
5778 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
5779 SmallVector<VPReductionPHIRecipe *, 4> UnorderedReductions;
5780 for (VPReductionPHIRecipe &RedPhiR :
5781 make_isa_range<VPReductionPHIRecipe>(Range: HeaderVPBB->phis())) {
5782 if (auto Chain = getScaledReductionChain(RedPhiR: &RedPhiR))
5783 PhiToChain.try_emplace(Key: &RedPhiR, Args: std::move(*Chain));
5784 else if (CostCtx.Opts.use_partial_reductions_by_default &&
5785 (RedPhiR.getRecurrenceKind() == RecurKind::Add ||
5786 (RedPhiR.getRecurrenceKind() == RecurKind::FAdd &&
5787 !RedPhiR.isOrdered() && !RedPhiR.isInLoop())))
5788 UnorderedReductions.push_back(Elt: &RedPhiR);
5789 }
5790
5791 // For general unordered reductions which aren't part of a candidate chain for
5792 // a scaled partial reduction, we can still use the intrinsic to allow for
5793 // more optimization later on.
5794 for (auto *Rdx : UnorderedReductions) {
5795 auto *Backedge = dyn_cast<VPWidenRecipe>(Val: Rdx->getBackedgeValue());
5796 VPValue *OtherOp;
5797 if (!Backedge ||
5798 !match(V: Backedge,
5799 P: m_CombineOr(Ps: m_c_FAdd(Op0: m_Specific(VPV: Rdx), Op1: m_VPValue(V&: OtherOp)),
5800 Ps: m_c_Add(Op0: m_Specific(VPV: Rdx), Op1: m_VPValue(V&: OtherOp)))))
5801 continue;
5802
5803 // If the target indicates that the intrinsic is as cheap as (or cheaper
5804 // than) the add, then prefer the intrinsic.
5805 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
5806 Predicate: [&CostCtx, Rdx, Backedge](ElementCount VF) {
5807 InstructionCost CurrentCost = Backedge->computeCost(VF, Ctx&: CostCtx);
5808 Type *ScalarTy = Backedge->getScalarType();
5809 auto FMF = ScalarTy->isFloatingPointTy()
5810 ? std::make_optional(t: Rdx->getFastMathFlagsOrNone())
5811 : std::nullopt;
5812
5813 InstructionCost PRCost = CostCtx.TTI.getPartialReductionCost(
5814 Opcode: Backedge->getOpcode(), InputTypeA: ScalarTy, /*InputTypeB=*/nullptr,
5815 AccumType: ScalarTy, VF, OpAExtend: TTI::PR_None, OpBExtend: TTI::PR_None,
5816 /*BinOp=*/std::nullopt, CostKind: CostCtx.CostKind, FMF);
5817 return PRCost <= CurrentCost;
5818 },
5819 Range))
5820 continue;
5821
5822 auto *Partial = new VPReductionRecipe(
5823 Rdx->getRecurrenceKind(), Rdx->getFastMathFlagsOrNone(),
5824 Backedge->getUnderlyingInstr(), Rdx, OtherOp, nullptr,
5825 getReductionStyle(/*InLoop=*/false, /*Ordered=*/false,
5826 /*ScaleFactor=*/1));
5827 Partial->insertBefore(InsertPos: Backedge);
5828 Backedge->replaceAllUsesWith(New: Partial);
5829 Backedge->eraseFromParent();
5830 }
5831
5832 if (PhiToChain.empty())
5833 return;
5834
5835 // Build set of partial reduction operations and blends for user validation
5836 // and a map of reduction bin ops to their scale factors for scale validation.
5837 SmallPtrSet<VPRecipeBase *, 4> PartialReductionOps;
5838 SmallPtrSet<VPBlendRecipe *, 4> PartialReductionBlends;
5839 DenseMap<VPSingleDefRecipe *, unsigned> ScaledReductionMap;
5840 for (auto &[_, Chain] : PhiToChain)
5841 for (const PartialReductionDescriptor &Link : Chain) {
5842 PartialReductionOps.insert(Ptr: Link.ExtendedOp.ExtendsUser);
5843 if (Link.Blend)
5844 PartialReductionBlends.insert(Ptr: Link.Blend);
5845 ScaledReductionMap[Link.ReductionBinOp] = Link.ScaleFactor;
5846 }
5847
5848 // A partial reduction is invalid if any of its extends are used by
5849 // something that isn't another partial reduction. This is because the
5850 // extends are intended to be lowered along with the reduction itself.
5851 auto ExtendUsersValid = [&](VPValue *Ext) {
5852 return !isa<VPWidenCastRecipe>(Val: Ext) || all_of(Range: Ext->users(), P: [&](VPUser *U) {
5853 return PartialReductionOps.contains(Ptr: cast<VPRecipeBase>(Val: U));
5854 });
5855 };
5856
5857 auto IsProfitablePartialReductionChainForVF =
5858 [&](ArrayRef<PartialReductionDescriptor> Chain, ElementCount VF) -> bool {
5859 InstructionCost PartialCost = 0, RegularCost = 0;
5860
5861 // The chain is a profitable partial reduction chain if the cost of handling
5862 // the entire chain is cheaper when using partial reductions than when
5863 // handling the entire chain using regular reductions.
5864 for (const PartialReductionDescriptor &Link : Chain) {
5865 const ExtendedReductionOperand &ExtendedOp = Link.ExtendedOp;
5866 InstructionCost LinkCost = getPartialReductionLinkCost(CostCtx, Link, VF);
5867 if (!LinkCost.isValid())
5868 return false;
5869
5870 PartialCost += LinkCost;
5871 RegularCost += Link.ReductionBinOp->computeCost(VF, Ctx&: CostCtx);
5872 // If ExtendB is not none, then the "ExtendsUser" is the binary operation.
5873 if (ExtendedOp.ExtendB.Kind != ExtendKind::PR_None)
5874 RegularCost += ExtendedOp.ExtendsUser->computeCost(VF, Ctx&: CostCtx);
5875 for (VPValue *Op : ExtendedOp.ExtendsUser->operands())
5876 if (auto *Extend = dyn_cast<VPWidenCastRecipe>(Val: Op))
5877 RegularCost += Extend->computeCost(VF, Ctx&: CostCtx);
5878 }
5879 return PartialCost.isValid() && PartialCost < RegularCost;
5880 };
5881
5882 // Validate chains: check that extends are only used by partial reductions,
5883 // and that reduction bin ops are only used by other partial reductions with
5884 // matching scale factors, are outside the loop region or the select
5885 // introduced by tail-folding. Otherwise we would create users of scaled
5886 // reductions where the types of the other operands don't match.
5887 for (auto &[RedPhiR, Chain] : PhiToChain) {
5888 for (const PartialReductionDescriptor &Link : Chain) {
5889 if (!all_of(Range: Link.ExtendedOp.ExtendsUser->operands(), P: ExtendUsersValid)) {
5890 Chain.clear();
5891 break;
5892 }
5893 auto UseIsValid = [&, RedPhiR = RedPhiR](VPUser *U) {
5894 if (auto *PhiR = dyn_cast<VPReductionPHIRecipe>(Val: U))
5895 return PhiR == RedPhiR;
5896 auto *R = cast<VPSingleDefRecipe>(Val: U);
5897
5898 if (auto *Blend = dyn_cast<VPBlendRecipe>(Val: R))
5899 return Blend == Link.Blend || PartialReductionBlends.contains(Ptr: Blend);
5900
5901 return Link.ScaleFactor == ScaledReductionMap.lookup_or(Val: R, Default: 0) ||
5902 match(R, P: m_ComputeReductionResult(
5903 Op0: m_Specific(VPV: Link.ReductionBinOp))) ||
5904 match(R, P: m_Select(Op0: m_VPValue(), Op1: m_Specific(VPV: Link.ReductionBinOp),
5905 Op2: m_Specific(VPV: RedPhiR)));
5906 };
5907 if (!all_of(Range: Link.ReductionBinOp->users(), P: UseIsValid)) {
5908 Chain.clear();
5909 break;
5910 }
5911
5912 // Check if the compute-reduction-result is used by a sunk store.
5913 // TODO: Also form partial reductions in those cases.
5914 if (auto *RdxResult = vputils::findComputeReductionResult(PhiR: RedPhiR)) {
5915 if (any_of(Range: RdxResult->users(), P: [](VPUser *U) {
5916 auto *RepR = dyn_cast<VPReplicateRecipe>(Val: U);
5917 return RepR && RepR->getOpcode() == Instruction::Store;
5918 })) {
5919 Chain.clear();
5920 break;
5921 }
5922 }
5923 }
5924
5925 // Clear the chain if it is not profitable.
5926 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
5927 Predicate: [&, &Chain = Chain](ElementCount VF) {
5928 return IsProfitablePartialReductionChainForVF(Chain, VF);
5929 },
5930 Range))
5931 Chain.clear();
5932 }
5933
5934 for (auto &[Phi, Chain] : PhiToChain) {
5935 if (Chain.empty())
5936 continue;
5937
5938 for (const PartialReductionDescriptor &Link : Chain)
5939 transformToPartialReduction(Link, Plan, RdxPhi: Phi);
5940
5941 // After transforming all links in the chain, the PHI node and result need
5942 // updating. Note that we can pick any link in the chain for this, as the
5943 // ScaleFactor and RecurKind must match for all links in the chain.
5944 const PartialReductionDescriptor &Link = Chain[0];
5945 updatePartialReductionPhiAndResult(Plan, Phi, Factor: Link.ScaleFactor, RK: Link.RK);
5946 }
5947}
5948
5949void VPlanTransforms::makeMemOpWideningDecisions(VPlan &Plan, VFRange &Range,
5950 VPRecipeBuilder &RecipeBuilder,
5951 VPCostContext &CostCtx) {
5952 // Collect all loads/stores first. We will start with ones having simpler
5953 // decisions followed by more complex ones that are potentially
5954 // guided/dependent on the simpler ones.
5955 SmallVector<VPInstruction *> MemOps;
5956 for (VPBasicBlock *VPBB :
5957 VPBlockUtils::blocksOnly<VPBasicBlock>(Range: vp_depth_first_shallow(
5958 G: Plan.getVectorLoopRegion()->getEntryBasicBlock()))) {
5959 for (VPInstruction &VPI : make_isa_range<VPInstruction>(Range&: *VPBB)) {
5960 if (VPI.getUnderlyingValue() &&
5961 is_contained(Set: {Instruction::Load, Instruction::Store},
5962 Element: VPI.getOpcode()))
5963 MemOps.push_back(Elt: &VPI);
5964 }
5965 }
5966
5967 // Few helpers to process different kinds of memory operations.
5968
5969 // To be used as argument to `VPlanTransforms::runPass` which explicitly
5970 // specified pass name, hence `VPlan &` parameter.
5971 auto ProcessSubset = [&](VPlan &, auto ProcessVPInst) {
5972 SmallVector<VPInstruction *> RemainingMemOps;
5973 for (VPInstruction *VPI : MemOps) {
5974 if (!ProcessVPInst(VPI))
5975 RemainingMemOps.push_back(Elt: VPI);
5976 }
5977
5978 MemOps.clear();
5979 std::swap(LHS&: MemOps, RHS&: RemainingMemOps);
5980 };
5981
5982 auto ReplaceWith = [&](VPInstruction *VPI, VPRecipeBase *New) {
5983 assert(New->getParent() && "New recipe must have been inserted");
5984 if (VPI->getOpcode() == Instruction::Load)
5985 VPI->replaceAllUsesWith(New: New->getVPSingleValue());
5986 VPI->eraseFromParent();
5987
5988 // VPI has been processed.
5989 return true;
5990 };
5991
5992 auto Scalarize = [&](VPInstruction *VPI) {
5993 return ReplaceWith(VPI, VPBuilder(VPI).insert(
5994 R: RecipeBuilder.handleReplication(VPI, Range)));
5995 };
5996
5997 VPBasicBlock *MiddleVPBB = Plan.getMiddleBlock();
5998 VPBuilder FinalRedStoresBuilder(MiddleVPBB->getFirstNonPhi());
5999 VPlanTransforms::runPass(
6000 PassName: "lowerMemoryIdioms", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
6001 if (RecipeBuilder.replaceWithFinalIfReductionStore(
6002 VPI, FinalRedStoresBuilder))
6003 return true;
6004
6005 // Filter out scalar VPlan for the remaining idioms.
6006 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6007 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
6008 return false;
6009
6010 if (VPHistogramRecipe *Histogram = RecipeBuilder.widenIfHistogram(VPI))
6011 return ReplaceWith(VPI, VPBuilder(VPI).insert(R: Histogram));
6012
6013 return false;
6014 });
6015
6016 // Filter out scalar VPlan for the remaining memory operations.
6017 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6018 Predicate: [](ElementCount VF) { return VF.isScalar(); }, Range))
6019 return;
6020
6021 // If the instruction's allocated size doesn't equal it's type size, it
6022 // requires padding and will be scalarized.
6023 VPlanTransforms::runPass(
6024 PassName: "scalarizeMemOpsWithIrregularTypes", Pass&: ProcessSubset, Plan,
6025 Args: [&](VPInstruction *VPI) {
6026 Instruction *I = VPI->getUnderlyingInstr();
6027 if (hasIrregularType(Ty: getLoadStoreType(I), DL: I->getDataLayout()))
6028 return Scalarize(VPI);
6029
6030 return false;
6031 });
6032
6033 if (!RecipeBuilder.prefersVectorizedAddressing()) {
6034 VPlanTransforms::runPass(
6035 PassName: "makeVPlanMemOpDecision", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
6036 Instruction *I = VPI->getUnderlyingInstr();
6037 bool IsLoad = VPI->getOpcode() == Instruction::Load;
6038 if (RecipeBuilder.isPredicatedInst(I) || !IsLoad ||
6039 !vputils::isUsedByLoadStoreAddress(V: VPI))
6040 return false;
6041
6042 // Scalarize loads used as addresses, matching the legacy CM. The load
6043 // is single-scalar if the pointer is loop-invariant, otherwise it is
6044 // replicated per-lane. No mask is needed as the load is not
6045 // predicated.
6046 VPValue *Ptr = VPI->getOperand(N: 0);
6047 const SCEV *PtrSCEV =
6048 vputils::getSCEVExprForVPValue(V: Ptr, PSE&: CostCtx.PSE, L: CostCtx.L);
6049 bool IsSingleScalarLoad =
6050 !isa<SCEVCouldNotCompute>(Val: PtrSCEV) &&
6051 CostCtx.PSE.getSE()->isLoopInvariant(S: PtrSCEV, L: CostCtx.L);
6052
6053 ReplaceWith(VPI,
6054 VPBuilder(VPI).insert(R: new VPReplicateRecipe(
6055 I, Ptr, /*IsSingleScalar=*/IsSingleScalarLoad,
6056 /*Mask=*/nullptr, *VPI, *VPI, VPI->getDebugLoc())));
6057 return true;
6058 });
6059 }
6060
6061 // Widen unit-stride consecutive accesses, matching the legacy CM. Both
6062 // forward (stride +1) and reverse (stride -1) accesses are handled.
6063 VPlanTransforms::runPass(
6064 PassName: "widenConsecutiveMemOps", Pass&: ProcessSubset, Plan, Args: [&](VPInstruction *VPI) {
6065 Instruction *I = VPI->getUnderlyingInstr();
6066 bool IsLoad = VPI->getOpcode() == Instruction::Load;
6067 VPValue *Ptr = VPI->getOperand(N: !IsLoad);
6068 Type *ScalarTy =
6069 IsLoad ? VPI->getScalarType() : VPI->getOperand(N: 0)->getScalarType();
6070 std::optional<int64_t> Stride =
6071 vputils::getConstantStride(Addr: Ptr, AccessTy: ScalarTy, PSE&: CostCtx.PSE, L: CostCtx.L);
6072 if (Stride != 1 && Stride != -1)
6073 return false;
6074 bool Reverse = Stride == -1;
6075
6076 // A predicated access can only be widened (rather than scalarized) if
6077 // the target supports a masked load/store for it.
6078 // TODO: Determine if a load/store needs predication directly in VPlan.
6079 bool IsPredicated = RecipeBuilder.isPredicatedInst(I);
6080 Align Alignment = getLoadStoreAlignment(I);
6081 if (IsPredicated &&
6082 !CostCtx.Config.isLegalMaskedLoadOrStore(
6083 IsLoad, ScalarTy, Alignment, AddressSpace: getLoadStoreAddressSpace(I)))
6084 return false;
6085
6086 VPBuilder Builder(VPI);
6087 VPSingleDefRecipe *VectorPtr = Builder.createConsecutiveVectorPointer(
6088 Ptr, SourceElementTy: ScalarTy, Reverse, DL: VPI->getDebugLoc());
6089
6090 VPValue *Mask = IsPredicated ? VPI->getMask() : nullptr;
6091 // Reverse the mask so it matches the reversed access order.
6092 if (Reverse && Mask)
6093 Mask = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Mask,
6094 DL: VPI->getDebugLoc());
6095
6096 if (IsLoad) {
6097 VPSingleDefRecipe *Load = Builder.createWidenLoad(
6098 Load&: *cast<LoadInst>(Val: I), Addr: VectorPtr, Mask,
6099 /*Consecutive=*/true, Alignment, Metadata: *VPI, DL: VPI->getDebugLoc());
6100 // Reverse the loaded values back into program order.
6101 if (Reverse)
6102 Load = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: Load,
6103 DL: VPI->getDebugLoc());
6104 return ReplaceWith(VPI, Load);
6105 }
6106
6107 VPValue *StoredVal = VPI->getOperand(N: 0);
6108 if (Reverse)
6109 // Reverse the stored values so they are written in descending order.
6110 StoredVal = Builder.createNaryOp(Opcode: VPInstruction::Reverse, Operands: StoredVal,
6111 DL: VPI->getDebugLoc());
6112
6113 auto *StoreR = Builder.createWidenStore(
6114 Store&: *cast<StoreInst>(Val: I), Addr: VectorPtr, StoredVal, Mask,
6115 /*Consecutive=*/true, Alignment, Metadata: *VPI, DL: VPI->getDebugLoc());
6116 return ReplaceWith(VPI, StoreR);
6117 });
6118
6119 VPlanTransforms::runPass(PassName: "delegateMemOpWideningToLegacyCM", Pass&: ProcessSubset,
6120 Plan, Args: [&](VPInstruction *VPI) {
6121 if (VPRecipeBase *Recipe =
6122 RecipeBuilder.tryToWidenMemory(VPI, Range))
6123 return ReplaceWith(VPI, Recipe);
6124
6125 return Scalarize(VPI);
6126 });
6127}
6128
6129void VPlanTransforms::makeScalarizationDecisions(VPlan &Plan, VFRange &Range) {
6130 if (LoopVectorizationPlanner::getDecisionAndClampRange(
6131 Predicate: [&](ElementCount VF) { return VF.isScalar(); }, Range))
6132 return;
6133
6134 PostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> POT(
6135 Plan.getEntry());
6136 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Range&: POT)) {
6137 for (VPInstruction &VPI :
6138 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range: reverse(C&: *VPBB)))) {
6139 auto *I = cast_or_null<Instruction>(Val: VPI.getUnderlyingValue());
6140 // Wouldn't be able to create a `VPReplicateRecipe` anyway.
6141 if (!I)
6142 continue;
6143
6144 // If executing other lanes produces side-effects we can't avoid them.
6145 if (VPI.mayHaveSideEffects())
6146 continue;
6147
6148 // ExtractValue takes the shape of its aggregate, decided in
6149 // tryToCreateWidenNonPhiRecipe.
6150 if (VPI.getOpcode() == Instruction::ExtractValue)
6151 continue;
6152
6153 // We want to drop the mask operand, verify we can safely do that.
6154 if (VPI.isMasked() && !VPI.isSafeToSpeculativelyExecute())
6155 continue;
6156
6157 // Other lanes are needed - can't drop them.
6158 if (!vputils::onlyFirstLaneUsed(Def: &VPI))
6159 continue;
6160
6161 auto *Recipe = VPBuilder::createSingleScalarOp(
6162 Opcode: VPI.getOpcode(), Operands: VPI.operandsWithoutMask(), /*Mask=*/nullptr, Flags: VPI,
6163 Metadata: VPI, DL: VPI.getDebugLoc(), ResultTy: VPI.getScalarType(), UV: I);
6164 Recipe->insertBefore(InsertPos: &VPI);
6165 VPI.replaceAllUsesWith(New: Recipe);
6166 VPI.eraseFromParent();
6167 }
6168 }
6169}
6170
6171/// Returns true if \p Info's parameter kinds are compatible with \p Args.
6172static bool areVFParamsOk(const VFInfo &Info, ArrayRef<VPValue *> Args,
6173 PredicatedScalarEvolution &PSE, const Loop *L) {
6174 ScalarEvolution *SE = PSE.getSE();
6175 return all_of(Range: Info.Shape.Parameters, P: [&](VFParameter Param) {
6176 switch (Param.ParamKind) {
6177 case VFParamKind::Vector:
6178 case VFParamKind::GlobalPredicate:
6179 return true;
6180 case VFParamKind::OMP_Uniform:
6181 return SE->isSCEVable(Ty: Args[Param.ParamPos]->getScalarType()) &&
6182 SE->isLoopInvariant(
6183 S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
6184 L);
6185 case VFParamKind::OMP_Linear:
6186 return match(S: vputils::getSCEVExprForVPValue(V: Args[Param.ParamPos], PSE, L),
6187 P: m_scev_AffineAddRec(
6188 Op0: m_SCEV(), Op1: m_scev_SpecificSInt(V: Param.LinearStepOrPos),
6189 L: m_SpecificLoop(L)));
6190 default:
6191 return false;
6192 }
6193 });
6194}
6195
6196/// Find a vector variant of \p CI for \p VF, respecting \p MaskRequired.
6197/// Returns the variant function, or nullptr. Masked variants are assumed to
6198/// take the mask as a trailing parameter.
6199static Function *findVectorVariant(CallInst *CI, ArrayRef<VPValue *> Args,
6200 ElementCount VF, bool MaskRequired,
6201 PredicatedScalarEvolution &PSE,
6202 const Loop *L) {
6203 if (CI->isNoBuiltin())
6204 return nullptr;
6205 auto Mappings = VFDatabase::getMappings(CI: *CI);
6206 const auto *It = find_if(Range&: Mappings, P: [&](const VFInfo &Info) {
6207 return Info.Shape.VF == VF && (!MaskRequired || Info.isMasked()) &&
6208 areVFParamsOk(Info, Args, PSE, L);
6209 });
6210 if (It == Mappings.end())
6211 return nullptr;
6212 return CI->getModule()->getFunction(Name: It->VectorName);
6213}
6214
6215namespace {
6216/// The outcome of choosing how to widen a call at a given VF.
6217struct CallWideningDecision {
6218 enum class KindTy { Scalarize, Intrinsic, VectorVariant };
6219 CallWideningDecision(KindTy Kind, Function *Variant = nullptr)
6220 : Kind(Kind), Variant(Variant) {}
6221 KindTy Kind;
6222
6223 /// Set when Kind == VectorVariant.
6224 Function *Variant;
6225
6226 bool operator==(const CallWideningDecision &Other) const {
6227 return Kind == Other.Kind && Variant == Other.Variant;
6228 }
6229};
6230} // namespace
6231
6232/// Pick the cheapest widening for the call \p VPI at \p VF among scalarization,
6233/// vector intrinsic, and vector library variant.
6234static CallWideningDecision decideCallWidening(VPInstruction &VPI,
6235 ArrayRef<VPValue *> Ops,
6236 ElementCount VF,
6237 VPCostContext &CostCtx) {
6238 auto *CI = cast<CallInst>(Val: VPI.getUnderlyingInstr());
6239
6240 // Scalar VFs and calls forced or known to scalarize always replicate.
6241 if (VF.isScalar() || CostCtx.willBeScalarized(I: CI, VF))
6242 return CallWideningDecision::KindTy::Scalarize;
6243
6244 auto *CalledFn = cast<Function>(
6245 Val: VPI.getOperand(N: VPI.getNumOperandsWithoutMask() - 1)->getLiveInIRValue());
6246 Type *ResultTy = VPI.getScalarType();
6247 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
6248 bool MaskRequired = CostCtx.isMaskRequired(I: CI);
6249
6250 // Pseudo intrinsics (assume, lifetime, ...) are always scalarized.
6251 if (ID && VPCostContext::isFreeScalarIntrinsic(ID))
6252 return CallWideningDecision::KindTy::Scalarize;
6253
6254 InstructionCost ScalarCost =
6255 VPReplicateRecipe::computeCallCost(CalledFn, ResultTy, ArgOps: Ops,
6256 /*IsSingleScalar=*/false, VF, Ctx&: CostCtx);
6257
6258 Function *VecFunc =
6259 findVectorVariant(CI, Args: Ops, VF, MaskRequired, PSE&: CostCtx.PSE, L: CostCtx.L);
6260 InstructionCost VecCallCost = InstructionCost::getInvalid();
6261 if (VecFunc)
6262 VecCallCost = VPWidenCallRecipe::computeCallCost(Variant: VecFunc, Ctx&: CostCtx);
6263
6264 // Prefer the intrinsic if it is at least as cheap as scalarizing and any
6265 // available vector variant.
6266 if (ID) {
6267 InstructionCost IntrinsicCost =
6268 VPWidenIntrinsicRecipe::computeCallCost(ID, Operands: Ops, R: VPI, VF, Ctx&: CostCtx);
6269 if (IntrinsicCost.isValid() && ScalarCost >= IntrinsicCost &&
6270 (!VecFunc || VecCallCost >= IntrinsicCost))
6271 return CallWideningDecision::KindTy::Intrinsic;
6272 }
6273
6274 // Otherwise, use a vector library variant when it beats scalarizing.
6275 if (VecFunc && ScalarCost >= VecCallCost)
6276 return {CallWideningDecision::KindTy::VectorVariant, VecFunc};
6277
6278 return CallWideningDecision::KindTy::Scalarize;
6279}
6280
6281bool VPlanTransforms::makeCallWideningDecisions(VPlan &Plan, VFRange &Range,
6282 VPRecipeBuilder &RecipeBuilder,
6283 VPCostContext &CostCtx) {
6284 bool Widened = false;
6285 for (VPBasicBlock *VPBB : VPBlockUtils::blocksAs<VPBasicBlock>(
6286 Range: vp_depth_first_shallow(G: Plan.getVectorLoopRegion()->getEntry()))) {
6287 for (VPInstruction &VPI :
6288 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range&: *VPBB))) {
6289 if (!VPI.getUnderlyingValue() || VPI.getOpcode() != Instruction::Call)
6290 continue;
6291
6292 auto *CI = cast<CallInst>(Val: VPI.getUnderlyingInstr());
6293 SmallVector<VPValue *, 4> Ops(VPI.op_begin(),
6294 VPI.op_begin() + CI->arg_size());
6295
6296 CallWideningDecision Decision =
6297 decideCallWidening(VPI, Ops, VF: Range.Start, CostCtx);
6298 LoopVectorizationPlanner::getDecisionAndClampRange(
6299 Predicate: [&](ElementCount VF) {
6300 return Decision == decideCallWidening(VPI, Ops, VF, CostCtx);
6301 },
6302 Range);
6303
6304 assert((Decision.Kind == CallWideningDecision::KindTy::Scalarize ||
6305 !CI->hasOperandBundles()) &&
6306 "calls with operand bundles must not be widened");
6307 VPSingleDefRecipe *Replacement = nullptr;
6308 switch (Decision.Kind) {
6309 case CallWideningDecision::KindTy::Intrinsic: {
6310 Intrinsic::ID ID = getVectorIntrinsicIDForCall(CI, TLI: &CostCtx.TLI);
6311 Type *ResultTy = VPI.getScalarType();
6312 Replacement = new VPWidenIntrinsicRecipe(*CI, ID, Ops, ResultTy, VPI,
6313 VPI, VPI.getDebugLoc());
6314 Widened = true;
6315 break;
6316 }
6317 case CallWideningDecision::KindTy::VectorVariant: {
6318 // Masked variants take the mask as a trailing parameter, so they have
6319 // one more parameter than the original call's arguments.
6320 if (Decision.Variant->arg_size() > Ops.size()) {
6321 VPValue *Mask = VPI.isMasked() ? VPI.getMask() : Plan.getTrue();
6322 Ops.push_back(Elt: Mask);
6323 }
6324 Ops.push_back(Elt: VPI.getOperand(N: VPI.getNumOperandsWithoutMask() - 1));
6325 VPIRAttributes Attrs(VPI, Decision.Variant->getFunctionType());
6326 Replacement = new VPWidenCallRecipe(CI, Decision.Variant, Ops, VPI, VPI,
6327 Attrs, VPI.getDebugLoc());
6328 Widened = true;
6329 break;
6330 }
6331 case CallWideningDecision::KindTy::Scalarize:
6332 Replacement = RecipeBuilder.handleReplication(VPI: &VPI, Range);
6333 break;
6334 }
6335
6336 Replacement->insertBefore(InsertPos: &VPI);
6337 VPI.replaceAllUsesWith(New: Replacement);
6338 VPI.eraseFromParent();
6339 }
6340 }
6341 return Widened;
6342}
6343
6344void VPlanTransforms::narrowInductionTruncates(VPlan &Plan, VFRange &Range,
6345 const TargetTransformInfo &TTI,
6346 PredicatedScalarEvolution &PSE) {
6347 VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
6348 VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
6349 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6350 Range: vp_depth_first_shallow(G: LoopRegion->getEntry()))) {
6351 for (VPInstruction &VPI :
6352 make_early_inc_range(Range: make_isa_range<VPInstruction>(Range&: *VPBB))) {
6353 // Only truncates are handled, as sext/zext may wrap, FP conversions lose
6354 // precision and other casts depend on the pointer size.
6355 if (VPI.getOpcode() != Instruction::Trunc)
6356 continue;
6357
6358 VPValue *Op = VPI.getOperand(N: 0);
6359 auto *WideIV = getOptimizableIVOf(VPV: Op, PSE);
6360 if (!WideIV)
6361 continue;
6362
6363 // getOptimizableIVOf also matches an add of the IV and its step, which
6364 // is not handled here.
6365 // TODO: Also narrow truncates of the incremented IV.
6366 if (Op != WideIV)
6367 continue;
6368
6369 // A truncate that is not widened is left to the scalarization decisions
6370 // made earlier.
6371 if (vputils::onlyFirstLaneUsed(Def: &VPI))
6372 continue;
6373
6374 // Replacing a free truncate would add an induction update instruction to
6375 // each iteration of the loop. The canonical induction is exempt, as it
6376 // needs an update instruction regardless.
6377 auto IsNarrowingProfitable = [&](ElementCount VF) {
6378 return match(V: WideIV, P: m_CanonicalWidenIV()) ||
6379 !TTI.isTruncateFree(
6380 Ty1: toVectorTy(Scalar: VPI.getOperand(N: 0)->getScalarType(), EC: VF),
6381 Ty2: toVectorTy(Scalar: VPI.getScalarType(), EC: VF));
6382 };
6383 if (!LoopVectorizationPlanner::getDecisionAndClampRange(
6384 Predicate: IsNarrowingProfitable, Range))
6385 continue;
6386
6387 // Wrap flags of the original induction do not hold in the truncated
6388 // type, so do not propagate them.
6389 auto *NarrowIV = new VPWidenIntOrFpInductionRecipe(
6390 WideIV->getPHINode(), WideIV->getStartValue(), WideIV->getStepValue(),
6391 WideIV->getVFValue(), WideIV->getInductionDescriptor(),
6392 VPI.getScalarType(), VPIRFlags::WrapFlagsTy(false, false),
6393 VPI.getDebugLoc());
6394 NarrowIV->insertBefore(IP: HeaderVPBB->getFirstNonPhi());
6395 VPI.replaceAllUsesWith(New: NarrowIV);
6396 VPI.eraseFromParent();
6397 }
6398 }
6399}
6400
6401void VPlanTransforms::convertToStridedAccesses(VPlan &Plan,
6402 PredicatedScalarEvolution &PSE,
6403 Loop &L, VPCostContext &Ctx,
6404 VFRange &Range) {
6405 if (Plan.hasScalarVFOnly())
6406 return;
6407
6408 VPRegionBlock *VectorLoop = Plan.getVectorLoopRegion();
6409 VPValue *I32VF = nullptr;
6410 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6411 Range: vp_depth_first_shallow(G: VectorLoop->getEntry()))) {
6412 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
6413 auto *MemR = dyn_cast<VPWidenMemoryRecipe>(Val: &R);
6414 // TODO: Transform reverse access into strided access with -1 stride.
6415 // TODO: Transform gather/scatter with uniform address into strided access
6416 // with 0 stride.
6417 // TODO: Transform interleave access into multiple strided accesses.
6418 if (!MemR || MemR->isConsecutive())
6419 continue;
6420
6421 VPValue *Ptr = MemR->getAddr();
6422 // Check if this is a strided access by analyzing the address SCEV for an
6423 // affine addRec.
6424 const SCEV *PtrSCEV = vputils::getSCEVExprForVPValue(V: Ptr, PSE, L: &L);
6425 const SCEV *Start;
6426 const SCEVConstant *Step;
6427 // TODO: Support non-constant loop invariant stride.
6428 if (!match(S: PtrSCEV,
6429 P: m_scev_AffineAddRec(Op0: m_SCEV(V&: Start), Op1: m_SCEVConstant(V&: Step),
6430 L: m_SpecificLoop(L: &L))))
6431 continue;
6432
6433 VPValue *StoredValue = nullptr;
6434 Type *DataTy;
6435 Intrinsic::ID IntrinID;
6436 if (auto *StoreR = dyn_cast<VPWidenStoreRecipe>(Val: &R)) {
6437 StoredValue = StoreR->getStoredValue();
6438 DataTy = StoredValue->getScalarType();
6439 IntrinID = Intrinsic::experimental_vp_strided_store;
6440 } else {
6441 auto *LoadR = cast<VPWidenLoadRecipe>(Val: &R);
6442 DataTy = LoadR->getScalarType();
6443 IntrinID = Intrinsic::experimental_vp_strided_load;
6444 }
6445
6446 Align Alignment = MemR->getAlign();
6447 auto IsProfitable = [&](ElementCount VF) {
6448 Type *VectorTy = toVectorTy(Scalar: DataTy, EC: VF);
6449 if (!Ctx.TTI.isLegalStridedLoadStore(DataType: VectorTy, Alignment))
6450 return false;
6451 const InstructionCost CurrentCost = MemR->computeCost(VF, Ctx);
6452 const InstructionCost StridedLoadStoreCost =
6453 VPWidenMemIntrinsicRecipe::computeMemIntrinsicCost(
6454 IID: IntrinID, Ty: VectorTy, IsMasked: MemR->isMasked(), Alignment, Ctx);
6455 return StridedLoadStoreCost < CurrentCost;
6456 };
6457
6458 if (!LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsProfitable,
6459 Range))
6460 continue;
6461
6462 // Invalidate the legacy widening decision so the cost of replaced load is
6463 // not counted during precomputeCosts.
6464 // TODO: Remove once the legacy exit cost computation is retired.
6465 for (ElementCount VF : Range)
6466 Ctx.invalidateWideningDecision(I: &MemR->getIngredient(), VF);
6467
6468 // Get VF as i32 for the vector length operand.
6469 if (!I32VF) {
6470 VPBuilder Builder(Plan.getVectorPreheader());
6471 I32VF = Builder.createScalarZExtOrTrunc(
6472 Op: &Plan.getVF(), ResultTy: Type::getInt32Ty(C&: Plan.getContext()),
6473 DL: DebugLoc::getUnknown());
6474 }
6475
6476 VPBuilder Builder(&R);
6477 // Create the base pointer of strided access.
6478 // TODO: reuse VPDerivedIVRecipe for base pointer computation when it
6479 // supports a general VPValue as the start value.
6480 VPValue *StartVPV =
6481 VPSCEVExpander(Builder, *PSE.getSE(), R.getDebugLoc()).expand(S: Start);
6482 VPValue *StrideInBytes = Plan.getOrAddLiveIn(V: Step->getValue());
6483 Type *IndexTy = Plan.getDataLayout().getIndexType(PtrTy: Ptr->getScalarType());
6484 assert(IndexTy == StrideInBytes->getScalarType() &&
6485 "Stride type from SCEV must match the index type");
6486 VPValue *CanIV = Builder.createScalarZExtOrTrunc(
6487 Op: VectorLoop->getCanonicalIV(), ResultTy: IndexTy, DL: DebugLoc::getUnknown());
6488 auto *AddRecPtr = cast<SCEVAddRecExpr>(Val: PtrSCEV);
6489 auto *Offset = Builder.createOverflowingOp(
6490 Opcode: Instruction::Mul, Operands: {CanIV, StrideInBytes},
6491 WrapFlags: {AddRecPtr->hasNoUnsignedWrap(), /*HasNSW=*/false});
6492 GEPNoWrapFlags NWFlags = AddRecPtr->hasNoUnsignedWrap()
6493 ? GEPNoWrapFlags::noUnsignedWrap()
6494 : GEPNoWrapFlags::none();
6495 VPValue *BasePtr = Builder.createNoWrapPtrAdd(Ptr: StartVPV, Offset, GEPFlags: NWFlags);
6496
6497 // Create a new vector pointer for strided access.
6498 VPValue *NewPtr = Builder.createVectorPointer(
6499 Ptr: BasePtr, SourceElementTy: Type::getInt8Ty(C&: Plan.getContext()), Stride: StrideInBytes, GEPFlags: NWFlags,
6500 DL: R.getDebugLoc());
6501
6502 VPValue *Mask = MemR->getMask();
6503 if (!Mask)
6504 Mask = Plan.getTrue();
6505 SmallVector<VPValue *, 5> Ops;
6506 if (StoredValue)
6507 Ops.push_back(Elt: StoredValue);
6508 Ops.append(IL: {NewPtr, StrideInBytes, Mask, I32VF});
6509
6510 auto *StridedR = Builder.createWidenMemIntrinsic(
6511 VectorIntrinsicID: IntrinID, CallArguments: Ops,
6512 Ty: StoredValue ? Type::getVoidTy(C&: Plan.getContext()) : DataTy, Alignment,
6513 MD: *MemR, DL: R.getDebugLoc());
6514 if (!StoredValue)
6515 cast<VPWidenLoadRecipe>(Val: &R)->replaceAllUsesWith(New: StridedR);
6516 R.eraseFromParent();
6517 }
6518 }
6519}
6520
6521void VPlanTransforms::expandMaskBeforeFirst(VPlan &Plan, VPCostContext &Ctx,
6522 VFRange &Range) {
6523 if (Range.Start.isScalar())
6524 return;
6525
6526 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6527 Range: vp_depth_first_deep(G: Plan.getVectorLoopRegion()->getEntry()))) {
6528 for (VPRecipeBase &R : make_early_inc_range(Range&: *VPBB)) {
6529 VPValue *X;
6530 if (!match(V: &R,
6531 P: m_WidenIntrinsic<Intrinsic::mask_beforefirst>(Ops: m_VPValue(V&: X))))
6532 continue;
6533 auto *BeforeFirst = cast<VPWidenIntrinsicRecipe>(Val: &R);
6534 Type *Ty = Plan.getVF().getScalarType();
6535 auto IsCheap = [&](ElementCount VF) {
6536 Type *MaskTy = toVectorTy(Scalar: X->getScalarType(), EC: VF);
6537 Type *IdxTy = Plan.getDataLayout().getIndexType(C&: Plan.getContext(), AddressSpace: 0);
6538 IntrinsicCostAttributes CttzElts(
6539 Intrinsic::experimental_cttz_elts, IdxTy,
6540 {MaskTy, Type::getInt1Ty(C&: Ctx.LLVMCtx)});
6541 IntrinsicCostAttributes ActiveLaneMask(Intrinsic::get_active_lane_mask,
6542 MaskTy, {Ty, Ty});
6543 return BeforeFirst->computeCost(VF, Ctx) <
6544 Ctx.TTI.getIntrinsicInstrCost(ICA: CttzElts, CostKind: Ctx.CostKind) +
6545 Ctx.TTI.getIntrinsicInstrCost(ICA: ActiveLaneMask, CostKind: Ctx.CostKind);
6546 };
6547 if (LoopVectorizationPlanner::getDecisionAndClampRange(Predicate: IsCheap, Range))
6548 continue;
6549
6550 VPBuilder Builder(BeforeFirst);
6551 DebugLoc DL = BeforeFirst->getDebugLoc();
6552 VPValue *FirstActive = Builder.createScalarZExtOrTrunc(
6553 Op: Builder.createFirstActiveLane(Masks: X, DL), ResultTy: Ty, DL);
6554 VPValue *PartStart = Builder.createOverflowingOp(
6555 Opcode: VPInstruction::CanonicalIVIncrementForPart,
6556 Operands: {Plan.getZero(Ty), &Plan.getVF()});
6557 BeforeFirst->replaceAllUsesWith(New: Builder.createNaryOp(
6558 Opcode: VPInstruction::ActiveLaneMask, Operands: {PartStart, FirstActive}, DL));
6559 BeforeFirst->eraseFromParent();
6560 }
6561 }
6562}
6563
6564bool VPlanTransforms::widenCompressingPatterns(VPlan &Plan,
6565 VPRecipeBuilder &RecipeBuilder,
6566 ScalarEvolution &SE) {
6567 VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
6568 SmallDenseMap<PHINode *, SmallVector<VPInstruction *>> CondIndToMemOps;
6569 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
6570 Range: vp_depth_first_shallow(G: HeaderVPBB))) {
6571 for (VPInstruction &VPI : make_isa_range<VPInstruction>(Range&: *VPBB)) {
6572 if (std::optional<CompressedPtrInfo> Info =
6573 RecipeBuilder.getCompressedPtrInfo(VPI: &VPI))
6574 CondIndToMemOps[Info->ConditionalInductionPHI].push_back(Elt: &VPI);
6575 }
6576 }
6577
6578 VPBuilder Builder;
6579 for (auto &CondIndPhi :
6580 make_isa_range<VPConditionalInductionPHIRecipe>(Range: HeaderVPBB->phis())) {
6581 // Obtain the mask for the conditional update from the VPBlendRecipe.
6582 VPValue *Mask = nullptr;
6583 auto *BlendR = cast<VPBlendRecipe>(Val: CondIndPhi.getBackedgeValue());
6584 assert(!BlendR->isNormalized() && "Expected to run before simpifyBlends");
6585 for (unsigned ValIdx : seq(Size: BlendR->getNumIncomingValues()))
6586 if (VPValue *IncomingValue = BlendR->getIncomingValue(Idx: ValIdx);
6587 IncomingValue != &CondIndPhi) {
6588 Mask = BlendR->getMask(Idx: ValIdx);
6589 break;
6590 }
6591 assert(Mask && "Expected mask for conditional induction update");
6592
6593 // Replace all "compressed" loads and stores with expandload and
6594 // compressstore respectively.
6595 for (VPInstruction *VPI : CondIndToMemOps[CondIndPhi.getPHINode()]) {
6596 // Bail out if the mask for the memory op does not match the condition
6597 // used to update the conditional induction.
6598 if (VPI->getMask() != Mask)
6599 return false;
6600
6601 Builder.setInsertPoint(VPI);
6602 auto *CompressedMemOp =
6603 Builder.insert(R: RecipeBuilder.widenCompressedLoadOrStore(VPI));
6604
6605 VPI->replaceAllUsesWith(New: CompressedMemOp);
6606 VPI->eraseFromParent();
6607 }
6608
6609 // Update the conditional induction to increment by the number of active
6610 // lanes in the mask.
6611 Type *UpdateType = CondIndPhi.getScalarType();
6612 if (UpdateType->isPointerTy())
6613 UpdateType = Plan.getDataLayout().getIndexType(PtrTy: UpdateType);
6614
6615 DebugLoc DL = CondIndPhi.getDebugLoc();
6616 const SCEV *StepSCEV = CondIndPhi.getStepSCEV();
6617
6618 Builder.setInsertPoint(BlendR);
6619 VPValue *HandledLanes =
6620 Builder.createNaryOp(Opcode: VPInstruction::NumActiveLanes, Operands: Mask, ResultTy: UpdateType,
6621 Flags: {}, DL, Name: "handled.lanes");
6622 VPValue *Step = VPSCEVExpander(Builder, SE, DL).expand(S: StepSCEV);
6623 if (!isa<VPConstantInt>(Val: Step))
6624 Step = Builder.createFreeze(Op: Step);
6625
6626 VPValue *Offset = Builder.createOverflowingOp(
6627 Opcode: Instruction::Mul, Operands: {Step, HandledLanes}, WrapFlags: {false, false}, DL);
6628 VPValue *Update =
6629 CondIndPhi.getScalarType()->isPointerTy()
6630 ? Builder.createPtrAdd(Ptr: &CondIndPhi, Offset, DL, Name: "conditional.step")
6631 : Builder.createAdd(LHS: &CondIndPhi, RHS: Offset, DL, Name: "conditional.step");
6632 CondIndPhi.getBackedgeValue()->replaceAllUsesWith(New: Update);
6633 }
6634
6635 return true;
6636}
6637