1//===- AArch64PredicateAsCounterLoopRewrites.cpp --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM
4// Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9//
10// Rewrites IR for loop-carried wide masks that can be represented as
11// predicate-as-counter values. This applies when the mask is used by load/store
12// operations that can be mapped to multi-vector instructions (with +sve2p1).
13//
14// For example, a loop like:
15//
16// entry:
17// %step = vscale x 64
18// %mask.entry = @get.active.lane.mask(0, %n)
19//
20// loop:
21// %iv = phi i64 [0, entry], [%iv.next, loop]
22// %mask = phi <vscale x 64 x i1> [%mask.entry, entry],
23// [%mask.next, loop]
24//
25// %load = load <vscale x 64 x i8> %src[%iv], %mask
26// store <vscale x 64 x i8> %load, %dst[%iv], %mask
27//
28// %iv.next = %iv + %step
29// %mask.next = @get.active.lane.mask(%iv.next, %n)
30// br first.active(%mask.next), loop, exit
31//
32// Could be rewritten to:
33//
34// entry:
35// %step = vscale x 64
36// %mask.entry = @whilelo.c8(0, %n, VLx4)
37//
38// loop:
39// %iv = phi i64 [0, entry], [%iv.next, loop]
40// %mask = phi target("aarch64.svcount") [%mask.entry, entry],
41// [%mask.next, loop]
42//
43// %load = @ld1.pn.x4 <4 x <vscale x 16 x i8>> %src[%iv], %mask
44// @st1.pn.x4 <4 x <vscale x 16 x i8>> %load, %dest[%iv], %mask
45//
46// %iv.next = %iv + %step
47// %mask.next = @whilelo.c8(%iv.next, %n, VLx4)
48// br first.active(@pext(%mask.next, 0)), loop, exit
49//
50// This replaces the `get.active.lane.mask` intrinsics with AArch64
51// predicate-as-counter `whilelo` intrinsics and updates the mask phi to use the
52// `aarch64.svcount` target type. Within the loop, load/store users are mapped
53// to multi-vector load/store intrinsics where possible. Users that cannot be
54// mapped to multi-vector instructions materialize vector masks using the `pext`
55// intrinsic (which extracts vector predicates from a predicate-as-counter).
56//
57// This pass may be a temporary solution that is removed if we gain support
58// for target-specific VPlan transforms in the loop vectorizer.
59//
60//===----------------------------------------------------------------------===//
61
62#include "AArch64.h"
63#include "AArch64Subtarget.h"
64#include "AArch64TargetMachine.h"
65#include "llvm/ADT/SmallVector.h"
66#include "llvm/ADT/Statistic.h"
67#include "llvm/Analysis/LoopInfo.h"
68#include "llvm/Analysis/LoopPass.h"
69#include "llvm/CodeGen/TargetPassConfig.h"
70#include "llvm/IR/Attributes.h"
71#include "llvm/IR/DataLayout.h"
72#include "llvm/IR/IRBuilder.h"
73#include "llvm/IR/IntrinsicInst.h"
74#include "llvm/IR/Intrinsics.h"
75#include "llvm/IR/IntrinsicsAArch64.h"
76#include "llvm/IR/Module.h"
77#include "llvm/InitializePasses.h"
78#include "llvm/Pass.h"
79#include "llvm/Support/Debug.h"
80#include "llvm/Transforms/Utils.h"
81#include "llvm/Transforms/Utils/Local.h"
82#include <optional>
83
84using namespace llvm;
85
86#define DEBUG_TYPE "aarch64-pn-loop-rewrites"
87namespace {
88
89STATISTIC(LoopsRewritten, "Number of loops rewritten");
90
91struct MaskRewriteCandidate {
92 /// The mask phi node (used by masked operations within the loop).
93 PHINode *MaskPhi = nullptr;
94 /// The initial value for the mask (incoming value from the preheader).
95 IntrinsicInst *StartMask = nullptr;
96 /// The updated value for the mask (incoming value from the loop latch).
97 IntrinsicInst *NextMask = nullptr;
98 /// The multi-vector scale for the predicate-as-counter (2 or 4).
99 unsigned VectorScale = 0;
100 /// The element size (in bits) for the predicate-as-counter.
101 unsigned ElementSizeInBits = 0;
102};
103
104static void logLoopBailout(const Loop &L, const Twine &Reason) {
105 LLVM_DEBUG({
106 dbgs() << "PN loop rewrite: skipping loop with header ";
107 L.getHeader()->printAsOperand(dbgs(), /*PrintType=*/false);
108 dbgs() << ": " << Reason << "\n";
109 });
110}
111
112static void logMatchFailure(const PHINode &Phi, const Twine &Reason) {
113 LLVM_DEBUG({
114 dbgs() << "PN loop rewrite: failed to match mask phi ";
115 Phi.printAsOperand(dbgs(), /*PrintType=*/false);
116 dbgs() << ": " << Reason << "\n";
117 });
118}
119
120/// Returns the scalar size in bits for a type according to the data layout.
121static unsigned getScalarSizeInBits(const DataLayout &DL, Type *Ty) {
122 return DL.getTypeSizeInBits(Ty: Ty->getScalarType()).getFixedValue();
123}
124
125/// Returns the SVE element count for \p ElementSizeInBits.
126static ElementCount getSVEElementCount(unsigned ElementSizeInBits) {
127 return ElementCount::getScalable(MinVal: AArch64::SVEBitsPerBlock /
128 ElementSizeInBits);
129}
130
131/// Returns the largest scalar access size of masked loads/stores in loop \p L
132/// where \p MaskPhi is used as the mask. TODO: This heuristic may need
133/// refinement based on the frequency of different access sizes.
134static unsigned getLargestMaskedMemAccessSizeInBits(const Loop &L,
135 PHINode &MaskPhi) {
136 const DataLayout &DL = MaskPhi.getModule()->getDataLayout();
137 unsigned LargestAccessSizeInBits = 0;
138
139 for (User *U : MaskPhi.users()) {
140 auto *II = dyn_cast<IntrinsicInst>(Val: U);
141 if (!II || !L.contains(Inst: II))
142 continue;
143
144 Intrinsic::ID IID = II->getIntrinsicID();
145 if (IID != Intrinsic::masked_load && IID != Intrinsic::masked_store)
146 continue;
147
148 unsigned MaskOpIdx = IID == Intrinsic::masked_load ? 1 : 2;
149 if (II->getArgOperand(i: MaskOpIdx) != &MaskPhi)
150 continue;
151
152 unsigned AccessSizeInBits = getScalarSizeInBits(DL, Ty: II->getAccessType());
153 if (AccessSizeInBits > LargestAccessSizeInBits)
154 LargestAccessSizeInBits = AccessSizeInBits;
155 }
156
157 return LargestAccessSizeInBits;
158}
159
160/// Returns the predicate-as-counter whilelo intrinsic ID for
161/// \p ElementSizeInBits.
162static Intrinsic::ID getWhileLOIntrinsic(unsigned ElementSizeInBits) {
163 switch (ElementSizeInBits) {
164 case 8:
165 return Intrinsic::aarch64_sve_whilelo_c8;
166 case 16:
167 return Intrinsic::aarch64_sve_whilelo_c16;
168 case 32:
169 return Intrinsic::aarch64_sve_whilelo_c32;
170 case 64:
171 return Intrinsic::aarch64_sve_whilelo_c64;
172 default:
173 llvm_unreachable("unsupported predicate-as-counter element size");
174 }
175}
176
177/// Expands the predicate-as-counter mask \p Count into a wide vector mask.
178/// Masks are extracted from the counter using the paired pext intrinsics,
179/// then concatenated to form the wide mask value.
180static Value *buildWideMask(IRBuilder<> &Builder, const MaskRewriteCandidate &C,
181 Value *Count) {
182 ElementCount LegalEC = getSVEElementCount(ElementSizeInBits: C.ElementSizeInBits);
183 Type *LegalMaskTy = VectorType::get(ElementType: Builder.getInt1Ty(), EC: LegalEC);
184
185 Value *WideMask = PoisonValue::get(T: C.MaskPhi->getType());
186 for (unsigned PairOffset = 0; PairOffset != C.VectorScale / 2; ++PairOffset) {
187 auto *Pair =
188 Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_pext_x2, OverloadTypes: {LegalMaskTy},
189 Args: {Count, Builder.getInt32(C: PairOffset)},
190 /*FMFSource=*/{}, Name: "pn.pext.pair");
191 for (unsigned SliceInPair = 0; SliceInPair != 2; ++SliceInPair) {
192 Value *Part = Builder.CreateExtractValue(Agg: Pair, Idxs: SliceInPair, Name: "pn.pext");
193 unsigned Slice = PairOffset * 2 + SliceInPair;
194 WideMask = Builder.CreateInsertVector(
195 DstType: C.MaskPhi->getType(), SrcVec: WideMask, SubVec: Part,
196 Idx: Slice * LegalEC.getKnownMinValue(), Name: "pn.mask");
197 }
198 }
199
200 return WideMask;
201}
202
203/// Creates a predicate-as-counter whilelo for \p ElementSizeInBits between
204/// \p Start and \p End with multi-vector \p VectorScale (2 or 4).
205static Value *createWhileLO(IRBuilder<> &Builder, unsigned ElementSizeInBits,
206 Value *Start, Value *End, unsigned VectorScale) {
207 if (Start->getType()->getIntegerBitWidth() < 64) {
208 Start = Builder.CreateZExt(V: Start, DestTy: Builder.getInt64Ty());
209 End = Builder.CreateZExt(V: End, DestTy: Builder.getInt64Ty());
210 }
211
212 Intrinsic::ID WhileLO = getWhileLOIntrinsic(ElementSizeInBits);
213 return Builder.CreateIntrinsic(ID: WhileLO,
214 Args: {Start, End, Builder.getInt32(C: VectorScale)},
215 /*FMFSource=*/{}, Name: "pn.mask");
216}
217
218/// If \p UserI is a masked-load user of the original loop mask, rewrite it to
219/// a masked multi-vector load (using the predicate-as-counter) if the load
220/// access size matches the predicate-as-counter element size.
221static bool tryRewriteMaskedLoad(Instruction &UserI,
222 const MaskRewriteCandidate &C, Value *Count) {
223 auto *II = dyn_cast<IntrinsicInst>(Val: &UserI);
224 if (!II || II->getIntrinsicID() != Intrinsic::masked_load)
225 return false;
226
227 unsigned ElementSizeInBits =
228 getScalarSizeInBits(DL: II->getModule()->getDataLayout(), Ty: II->getType());
229 if (ElementSizeInBits != C.ElementSizeInBits)
230 return false;
231
232 if (!isa<PoisonValue, UndefValue>(Val: II->getArgOperand(i: 2)))
233 return false;
234
235 IRBuilder<> Builder(II);
236
237 ElementCount LegalEC = getSVEElementCount(ElementSizeInBits: C.ElementSizeInBits);
238 Type *ScalarType = II->getType()->getScalarType();
239 auto *WideDataTy = VectorType::get(ElementType: ScalarType, EC: LegalEC * C.VectorScale);
240 auto *LegalDataTy = VectorType::get(ElementType: ScalarType, EC: LegalEC);
241
242 auto *PNLoad = cast<IntrinsicInst>(Val: Builder.CreateIntrinsic(
243 ID: C.VectorScale == 4 ? Intrinsic::aarch64_sve_ld1_pn_x4
244 : Intrinsic::aarch64_sve_ld1_pn_x2,
245 OverloadTypes: {LegalDataTy, II->getArgOperand(i: 0)->getType()},
246 Args: {Count, II->getArgOperand(i: 0)},
247 /*FMFSource=*/{}, Name: "pn.ld1"));
248
249 // Copy pointer parameter attributes.
250 for (Attribute ParamAttr : II->getParamAttributes(ArgNo: 0))
251 PNLoad->addParamAttr(ArgNo: 1, Attr: ParamAttr);
252
253 PNLoad->copyMetadata(SrcInst: *II);
254
255 // Concatenate all results into the original wide value.
256 Value *WideData = PoisonValue::get(T: WideDataTy);
257 for (unsigned Slice = 0; Slice != C.VectorScale; ++Slice) {
258 Value *Part = Builder.CreateExtractValue(Agg: PNLoad, Idxs: Slice, Name: "pn.data");
259 WideData = Builder.CreateInsertVector(DstType: WideDataTy, SrcVec: WideData, SubVec: Part,
260 Idx: Slice * LegalEC.getKnownMinValue(),
261 Name: "pn.vec");
262 }
263
264 II->replaceAllUsesWith(V: WideData);
265 II->eraseFromParent();
266 return true;
267}
268
269/// If \p UserI is a masked-store user of the original loop mask, rewrite it to
270/// a masked multi-vector store (using the predicate-as-counter) if the store
271/// access size matches the predicate-as-counter element size.
272static bool tryRewriteMaskedStore(Instruction &UserI,
273 const MaskRewriteCandidate &C, Value *Count) {
274 auto *II = dyn_cast<IntrinsicInst>(Val: &UserI);
275 if (!II || II->getIntrinsicID() != Intrinsic::masked_store)
276 return false;
277
278 unsigned ElementSizeInBits = getScalarSizeInBits(
279 DL: II->getModule()->getDataLayout(), Ty: II->getArgOperand(i: 0)->getType());
280 if (ElementSizeInBits != C.ElementSizeInBits)
281 return false;
282
283 IRBuilder<> Builder(II);
284
285 Type *ScalarType = II->getArgOperand(i: 0)->getType()->getScalarType();
286 ElementCount LegalEC = getSVEElementCount(ElementSizeInBits: C.ElementSizeInBits);
287 auto *LegalDataTy = VectorType::get(ElementType: ScalarType, EC: LegalEC);
288
289 SmallVector<Value *, 6> StoreArgs;
290 for (unsigned Slice = 0; Slice != C.VectorScale; ++Slice)
291 StoreArgs.push_back(Elt: Builder.CreateExtractVector(
292 DstType: LegalDataTy, SrcVec: II->getArgOperand(i: 0), Idx: Slice * LegalEC.getKnownMinValue(),
293 Name: "pn.data"));
294 StoreArgs.push_back(Elt: Count);
295 StoreArgs.push_back(Elt: II->getArgOperand(i: 1));
296
297 auto *PNStore = cast<IntrinsicInst>(Val: Builder.CreateIntrinsic(
298 ID: C.VectorScale == 4 ? Intrinsic::aarch64_sve_st1_pn_x4
299 : Intrinsic::aarch64_sve_st1_pn_x2,
300 OverloadTypes: {LegalDataTy, II->getArgOperand(i: 1)->getType()}, Args: StoreArgs));
301
302 PNStore->copyMetadata(SrcInst: *II);
303
304 // Copy pointer parameter attributes.
305 for (Attribute ParamAttr : II->getParamAttributes(ArgNo: 1))
306 PNStore->addParamAttr(ArgNo: C.VectorScale + 1, Attr: ParamAttr);
307
308 II->eraseFromParent();
309 return true;
310}
311
312/// If \p UserI is an extractelement use of the original loop mask, attempt to
313/// rewrite it to `extractelement(pext(count, 0), idx)` if the extract index is
314/// known to be within the first mask section (which means pext index = 0). If
315/// the user of the extract is a branch and the source of the mask is a whilelo,
316/// this form can be optimized into checking the status flags.
317static bool tryRewriteExtractElement(Instruction &UserI,
318 const MaskRewriteCandidate &C,
319 Value *Count) {
320 auto *EEI = dyn_cast<ExtractElementInst>(Val: &UserI);
321 if (!EEI)
322 return false;
323
324 ElementCount LegalEC = getSVEElementCount(ElementSizeInBits: C.ElementSizeInBits);
325 auto *Idx = dyn_cast<ConstantInt>(Val: EEI->getIndexOperand());
326 if (!Idx || Idx->getValue().uge(RHS: LegalEC.getKnownMinValue()))
327 return false;
328
329 IRBuilder<> Builder(EEI);
330
331 auto *ExtractMask = Builder.CreateIntrinsic(
332 ID: Intrinsic::aarch64_sve_pext,
333 OverloadTypes: {VectorType::get(ElementType: Builder.getInt1Ty(), EC: LegalEC)},
334 Args: {Count, Builder.getInt32(C: 0)}, /*FMFSource=*/{}, Name: "pn.pext");
335
336 Value *Extracted = Builder.CreateExtractElement(
337 Vec: ExtractMask, Idx: EEI->getIndexOperand(), Name: EEI->getName() + ".pn");
338
339 EEI->replaceAllUsesWith(V: Extracted);
340 EEI->eraseFromParent();
341 return true;
342}
343
344class AArch64PredicateAsCounterLoopRewrites : public LoopPass {
345public:
346 static char ID;
347
348 AArch64PredicateAsCounterLoopRewrites() : LoopPass(ID) {}
349
350 void getAnalysisUsage(AnalysisUsage &AU) const override {
351 AU.addRequired<TargetPassConfig>();
352 // Require loop simplify to ensure loops have a preheader.
353 AU.addRequiredID(ID&: LoopSimplifyID);
354 AU.addPreservedID(ID&: LoopSimplifyID);
355 AU.setPreservesCFG();
356 }
357
358 bool runOnLoop(Loop *L, LPPassManager &) override;
359
360private:
361 std::optional<MaskRewriteCandidate> matchMaskPhi(Loop &L, PHINode &Phi) const;
362 bool rewriteCandidate(const MaskRewriteCandidate &C, Loop &L) const;
363};
364
365} // end anonymous namespace
366
367char AArch64PredicateAsCounterLoopRewrites::ID = 0;
368
369INITIALIZE_PASS_BEGIN(AArch64PredicateAsCounterLoopRewrites, DEBUG_TYPE,
370 "AArch64 Predicate As Counter Loop Rewrites", false,
371 false)
372INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
373INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
374INITIALIZE_PASS_END(AArch64PredicateAsCounterLoopRewrites, DEBUG_TYPE,
375 "AArch64 Predicate As Counter Loop Rewrites", false, false)
376
377Pass *llvm::createAArch64PredicateAsCounterLoopRewritesPass() {
378 return new AArch64PredicateAsCounterLoopRewrites();
379}
380
381bool AArch64PredicateAsCounterLoopRewrites::runOnLoop(Loop *L,
382 LPPassManager &) {
383 if (skipLoop(L)) {
384 logLoopBailout(L: *L, Reason: "skipLoop requested the loop to be skipped");
385 return false;
386 }
387
388 Function &F = *L->getHeader()->getParent();
389 auto &TPC = getAnalysis<TargetPassConfig>();
390 const AArch64Subtarget *ST =
391 TPC.getTM<AArch64TargetMachine>().getSubtargetImpl(F);
392 if (!ST->hasSVE2p1() && !(ST->hasSME2() && ST->isStreaming())) {
393 logLoopBailout(L: *L, Reason: "neither SVE2.1 nor SME2 is available");
394 return false;
395 }
396
397 if (!L->getLoopPreheader()) {
398 logLoopBailout(L: *L, Reason: "loop has no preheader");
399 return false;
400 }
401 if (!L->getLoopLatch()) {
402 logLoopBailout(L: *L, Reason: "loop has no latch");
403 return false;
404 }
405
406 bool Changed = false;
407 BasicBlock *Header = L->getHeader();
408 for (PHINode &Phi : make_early_inc_range(Range: Header->phis())) {
409 if (std::optional<MaskRewriteCandidate> Candidate = matchMaskPhi(L&: *L, Phi))
410 Changed |= rewriteCandidate(C: *Candidate, L&: *L);
411 }
412
413 if (Changed)
414 ++LoopsRewritten;
415
416 return Changed;
417}
418
419static IntrinsicInst *getGetActiveLaneMask(Value *V) {
420 auto *II = dyn_cast<IntrinsicInst>(Val: V);
421 return II && II->getIntrinsicID() == Intrinsic::get_active_lane_mask
422 ? II
423 : nullptr;
424}
425
426std::optional<MaskRewriteCandidate>
427AArch64PredicateAsCounterLoopRewrites::matchMaskPhi(Loop &L,
428 PHINode &Phi) const {
429 auto *PhiTy = dyn_cast<ScalableVectorType>(Val: Phi.getType());
430 if (!PhiTy || !PhiTy->getElementType()->isIntegerTy(BitWidth: 1))
431 return std::nullopt;
432
433 if (Phi.getNumIncomingValues() != 2) {
434 logMatchFailure(Phi, Reason: Twine("phi has ") + Twine(Phi.getNumIncomingValues()) +
435 " incoming values; expected 2");
436 return std::nullopt;
437 }
438
439 Value *StartValue = Phi.getIncomingValueForBlock(BB: L.getLoopPreheader());
440 Value *NextValue = Phi.getIncomingValueForBlock(BB: L.getLoopLatch());
441 IntrinsicInst *StartMask = getGetActiveLaneMask(V: StartValue);
442 IntrinsicInst *NextMask = getGetActiveLaneMask(V: NextValue);
443 if (!StartMask) {
444 logMatchFailure(Phi,
445 Reason: "preheader incoming value is not get_active_lane_mask");
446 return std::nullopt;
447 }
448 if (!NextMask) {
449 logMatchFailure(Phi, Reason: "latch incoming value is not get_active_lane_mask");
450 return std::nullopt;
451 }
452
453 unsigned WideMaskElements = PhiTy->getMinNumElements();
454 if (!isPowerOf2_32(Value: WideMaskElements)) {
455 logMatchFailure(Phi,
456 Reason: Twine("wide mask element count is not a power of 2: ") +
457 Twine(WideMaskElements));
458 return std::nullopt;
459 }
460
461 if (StartMask->getArgOperand(i: 0)->getType()->getIntegerBitWidth() > 64) {
462 logMatchFailure(Phi, Reason: "start mask induction operand is wider than i64");
463 return std::nullopt;
464 }
465 if (NextMask->getArgOperand(i: 0)->getType()->getIntegerBitWidth() > 64) {
466 logMatchFailure(Phi, Reason: "next mask induction operand is wider than i64");
467 return std::nullopt;
468 }
469
470 unsigned PreferredMaskElementSizeInBits =
471 getLargestMaskedMemAccessSizeInBits(L, MaskPhi&: Phi);
472
473 if (!is_contained(Set: {8u, 16u, 32u, 64u}, Element: PreferredMaskElementSizeInBits)) {
474 logMatchFailure(Phi, Reason: Twine("unsupported element size in bits: ") +
475 Twine(PreferredMaskElementSizeInBits));
476 return std::nullopt;
477 }
478
479 unsigned SVEMaskElements =
480 AArch64::SVEBitsPerBlock / PreferredMaskElementSizeInBits;
481 if (WideMaskElements <= SVEMaskElements) {
482 logMatchFailure(Phi, Reason: Twine("wide mask element count ") +
483 Twine(WideMaskElements) +
484 " is not wider than the legal mask width " +
485 Twine(SVEMaskElements));
486 return std::nullopt;
487 }
488
489 unsigned VectorScale = WideMaskElements / SVEMaskElements;
490 if (VectorScale != 2 && VectorScale != 4) {
491 logMatchFailure(Phi, Reason: Twine("unsupported predicate-as-counter scale: ") +
492 Twine(VectorScale));
493 return std::nullopt;
494 }
495
496 return MaskRewriteCandidate{.MaskPhi: &Phi, .StartMask: StartMask, .NextMask: NextMask, .VectorScale: VectorScale,
497 .ElementSizeInBits: PreferredMaskElementSizeInBits};
498}
499
500bool AArch64PredicateAsCounterLoopRewrites::rewriteCandidate(
501 const MaskRewriteCandidate &C, Loop &L) const {
502 IRBuilder<> Builder(C.StartMask);
503 Value *NewStart =
504 createWhileLO(Builder, ElementSizeInBits: C.ElementSizeInBits, Start: C.StartMask->getArgOperand(i: 0),
505 End: C.StartMask->getArgOperand(i: 1), VectorScale: C.VectorScale);
506 Builder.SetInsertPoint(C.NextMask);
507 Value *NewNext =
508 createWhileLO(Builder, ElementSizeInBits: C.ElementSizeInBits, Start: C.NextMask->getArgOperand(i: 0),
509 End: C.NextMask->getArgOperand(i: 1), VectorScale: C.VectorScale);
510
511 Builder.SetInsertPoint(C.MaskPhi);
512 auto *NewPhi =
513 Builder.CreatePHI(Ty: NewStart->getType(), NumReservedValues: 2, Name: C.MaskPhi->getName() + ".pn");
514 NewPhi->addIncoming(V: NewStart, BB: L.getLoopPreheader());
515 NewPhi->addIncoming(V: NewNext, BB: L.getLoopLatch());
516
517 auto RewriteUses = [&](Instruction *OldMask, Value *Count,
518 function_ref<bool(Use &U)> Predicate = nullptr) {
519 SmallVector<Use *, 8> UsesToRewrite;
520 for (Use &U : OldMask->uses()) {
521 if (!Predicate || Predicate(U))
522 UsesToRewrite.push_back(Elt: &U);
523 }
524
525 Value *WideMask = nullptr;
526 for (Use *U : UsesToRewrite) {
527 auto *UserI = cast<Instruction>(Val: U->getUser());
528 if (tryRewriteMaskedLoad(UserI&: *UserI, C, Count) ||
529 tryRewriteMaskedStore(UserI&: *UserI, C, Count) ||
530 tryRewriteExtractElement(UserI&: *UserI, C, Count))
531 continue;
532
533 if (!WideMask) {
534 BasicBlock::iterator InsertPt = OldMask->getIterator();
535 if (isa<PHINode>(Val: OldMask))
536 InsertPt = OldMask->getParent()->getFirstNonPHIIt();
537
538 Builder.SetInsertPoint(InsertPt);
539 Builder.SetCurrentDebugLocation(OldMask->getDebugLoc());
540 WideMask = buildWideMask(Builder, C, Count);
541 }
542
543 U->set(WideMask);
544 }
545 };
546
547 // For the start/next mask, we only replace non-phi in-loop users. Due to CSE,
548 // these masks could be used by other loops, and replacing them with
549 // predicate-as-counter masks could prevent matching those loops.
550 auto IsNonPhiUseInLoop = [&](Use &U) {
551 auto *UseInst = cast<Instruction>(Val: U.getUser());
552 return !isa<PHINode>(Val: UseInst) && L.contains(Inst: UseInst);
553 };
554
555 RewriteUses(C.MaskPhi, NewPhi);
556 RewriteUses(C.StartMask, NewStart, IsNonPhiUseInLoop);
557 RewriteUses(C.NextMask, NewNext, IsNonPhiUseInLoop);
558
559 RecursivelyDeleteTriviallyDeadInstructions(V: C.MaskPhi);
560 return true;
561}
562