1//===- VPlanPatternMatch.h - Match on VPValues and recipes ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides a simple and efficient mechanism for performing general
10// tree-based pattern matches on the VPlan values and recipes, based on
11// LLVM's IR pattern matchers.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
16#define LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
17
18#include "VPlan.h"
19#include "VPlanUtils.h"
20#include "llvm/Support/PatternMatchHelpers.h"
21#include <utility>
22
23using namespace llvm::PatternMatchHelpers;
24
25namespace llvm::VPlanPatternMatch {
26
27template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
28 return P.match(V);
29}
30
31template <typename Pattern> bool match(VPUser *U, const Pattern &P) {
32 auto *R = dyn_cast<VPRecipeBase>(Val: U);
33 return R && match(R, P);
34}
35
36template <typename Pattern> bool match(VPSingleDefRecipe *R, const Pattern &P) {
37 return P.match(static_cast<const VPRecipeBase *>(R));
38}
39
40/// A match functor that can be used as a UnaryPredicate in functional
41/// algorithms like all_of.
42template <typename Pattern> auto match_fn(const Pattern &P) {
43 return [&P](auto *V) { return match(V, P); };
44}
45
46/// Match an arbitrary VPValue and ignore it.
47inline auto m_VPValue() { return m_Isa<VPValue>(); }
48
49/// Match a specified VPValue.
50struct specificval_ty {
51 const VPValue *Val;
52
53 specificval_ty(const VPValue *V) : Val(V) {}
54
55 bool match(const VPValue *VPV) const { return VPV == Val; }
56};
57
58inline specificval_ty m_Specific(const VPValue *VPV) { return VPV; }
59
60/// Like m_Specific(), but works if the specific value to match is determined
61/// as part of the same match() expression. For example:
62/// m_Mul(m_VPValue(X), m_Specific(X)) is incorrect, because m_Specific() will
63/// bind X before the pattern match starts.
64/// m_Mul(m_VPValue(X), m_Deferred(X)) is correct, and will check against
65/// whichever value m_VPValue(X) populated.
66inline match_deferred<VPValue> m_Deferred(VPValue *const &V) { return V; }
67
68/// Match an integer constant if Pred::isValue returns true for the APInt. \p
69/// BitWidth optionally specifies the bitwidth the matched constant must have.
70/// If it is 0, the matched constant can have any bitwidth.
71template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty {
72 Pred P;
73
74 int_pred_ty(Pred P) : P(std::move(P)) {}
75 int_pred_ty() : P() {}
76
77 bool match(const VPValue *VPV) const {
78 auto *VPI = dyn_cast<VPInstruction>(Val: VPV);
79 if (VPI && VPI->getOpcode() == VPInstruction::Broadcast)
80 VPV = VPI->getOperand(N: 0);
81 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
82 if (!CI)
83 return false;
84
85 if (BitWidth != 0 && CI->getBitWidth() != BitWidth)
86 return false;
87 return P.isValue(CI->getAPInt());
88 }
89};
90
91/// Match a specified signed or unsigned integer value.
92struct is_specific_int {
93 APInt Val;
94 bool IsSigned;
95
96 is_specific_int(APInt Val, bool IsSigned = false)
97 : Val(std::move(Val)), IsSigned(IsSigned) {}
98
99 bool isValue(const APInt &C) const {
100 return APInt::isSameValue(I1: Val, I2: C, SignedCompare: IsSigned);
101 }
102};
103
104template <unsigned Bitwidth = 0>
105using specific_intval = int_pred_ty<is_specific_int, Bitwidth>;
106
107inline specific_intval<0> m_SpecificInt(uint64_t V) {
108 return specific_intval<0>(is_specific_int(APInt(64, V)));
109}
110
111inline specific_intval<0> m_SpecificSInt(int64_t V) {
112 return specific_intval<0>(
113 is_specific_int(APInt(64, V, /*isSigned=*/true), /*IsSigned=*/true));
114}
115
116inline specific_intval<1> m_False() {
117 return specific_intval<1>(is_specific_int(APInt(64, 0)));
118}
119
120inline specific_intval<1> m_True() {
121 return specific_intval<1>(is_specific_int(APInt(64, 1)));
122}
123
124struct is_all_ones {
125 bool isValue(const APInt &C) const { return C.isAllOnes(); }
126};
127
128/// Match an integer or vector with all bits set.
129/// For vectors, this includes constants with undefined elements.
130inline int_pred_ty<is_all_ones> m_AllOnes() {
131 return int_pred_ty<is_all_ones>();
132}
133
134struct is_zero_int {
135 bool isValue(const APInt &C) const { return C.isZero(); }
136};
137
138struct is_one {
139 bool isValue(const APInt &C) const { return C.isOne(); }
140};
141
142/// Match an integer 0 or a vector with all elements equal to 0.
143/// For vectors, this includes constants with undefined elements.
144inline int_pred_ty<is_zero_int> m_ZeroInt() {
145 return int_pred_ty<is_zero_int>();
146}
147
148/// Match an integer 1 or a vector with all elements equal to 1.
149/// For vectors, this includes constants with undefined elements.
150inline int_pred_ty<is_one> m_One() { return int_pred_ty<is_one>(); }
151
152struct bind_apint {
153 const APInt *&Res;
154
155 bind_apint(const APInt *&Res) : Res(Res) {}
156
157 bool match(const VPValue *VPV) const {
158 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
159 if (!CI)
160 return false;
161 Res = &CI->getAPInt();
162 return true;
163 }
164};
165
166inline bind_apint m_APInt(const APInt *&C) { return C; }
167
168struct bind_const_int {
169 uint64_t &Res;
170
171 bind_const_int(uint64_t &Res) : Res(Res) {}
172
173 bool match(const VPValue *VPV) const {
174 const APInt *APConst;
175 if (!bind_apint(APConst).match(VPV))
176 return false;
177 if (auto C = APConst->tryZExtValue()) {
178 Res = *C;
179 return true;
180 }
181 return false;
182 }
183};
184
185struct match_poison {
186 bool match(const VPValue *V) const {
187 return isa<VPIRValue>(Val: V) &&
188 isa<PoisonValue>(Val: cast<VPIRValue>(Val: V)->getValue());
189 }
190};
191
192/// Match a VPIRValue that's poison.
193inline match_poison m_Poison() { return match_poison(); }
194
195/// Match a plain integer constant no wider than 64-bits, capturing it if we
196/// match.
197inline bind_const_int m_ConstantInt(uint64_t &C) { return C; }
198
199/// Match a VPValue, capturing it if we match.
200inline match_bind<VPValue> m_VPValue(VPValue *&V) { return V; }
201
202/// Match a VPIRValue.
203inline match_bind<VPIRValue> m_VPIRValue(VPIRValue *&V) { return V; }
204
205/// Match a VPSingleDefRecipe, capturing if we match.
206inline match_bind<VPSingleDefRecipe>
207m_VPSingleDefRecipe(VPSingleDefRecipe *&V) {
208 return V;
209}
210
211/// Match a VPInstruction, capturing if we match.
212inline match_bind<VPInstruction> m_VPInstruction(VPInstruction *&V) {
213 return V;
214}
215
216template <typename Ops_t, unsigned Opcode, bool Commutative,
217 typename... RecipeTys>
218struct Recipe_match {
219 Ops_t Ops;
220
221 template <typename... OpTy> Recipe_match(OpTy... Ops) : Ops(Ops...) {
222 static_assert(std::tuple_size<Ops_t>::value == sizeof...(Ops) &&
223 "number of operands in constructor doesn't match Ops_t");
224 static_assert((!Commutative || std::tuple_size<Ops_t>::value == 2) &&
225 "only binary ops can be commutative");
226 }
227
228 bool match(const VPValue *V) const {
229 auto *DefR = V->getDefiningRecipe();
230 return DefR && match(DefR);
231 }
232
233 bool match(const VPSingleDefRecipe *R) const {
234 return match(static_cast<const VPRecipeBase *>(R));
235 }
236
237 bool match(const VPRecipeBase *R) const {
238 if (std::tuple_size_v<Ops_t> == 0) {
239 auto *VPI = dyn_cast<VPInstruction>(Val: R);
240 return VPI && VPI->getOpcode() == Opcode;
241 }
242
243 if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...))
244 return false;
245
246 if (R->getNumOperands() < std::tuple_size<Ops_t>::value) {
247 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R);
248 assert(((isa<VPInstruction>(R) &&
249 cast<VPInstruction>(R)->getNumOperandsForOpcode() == -1u) ||
250 (RepR && std::tuple_size_v<Ops_t> ==
251 RepR->getNumOperandsWithoutMask())) &&
252 "non-variadic recipe with matched opcode does not have the "
253 "expected number of operands");
254 return false;
255 }
256
257 // If the recipe has more operands than expected, we only support matching
258 // masked VPInstructions or predicated VPReplicateRecipes, where the number
259 // of operands of the matcher matches the number of operands excluding the
260 // mask.
261 if (R->getNumOperands() > std::tuple_size<Ops_t>::value) {
262 if (auto *VPI = dyn_cast<VPInstruction>(Val: R)) {
263 if (!VPI->isMasked() ||
264 VPI->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
265 return false;
266 } else if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R)) {
267 if (!RepR->isPredicated() ||
268 RepR->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
269 return false;
270 } else {
271 return false;
272 }
273 }
274
275 auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>();
276 if (all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
277 return Op.match(R->getOperand(N: Idx));
278 }))
279 return true;
280
281 return Commutative &&
282 all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
283 return Op.match(R->getOperand(N: R->getNumOperands() - Idx - 1));
284 });
285 }
286
287private:
288 template <typename RecipeTy>
289 static bool matchRecipeAndOpcode(const VPRecipeBase *R) {
290 auto *DefR = dyn_cast<RecipeTy>(R);
291 // Check for recipes that do not have opcodes.
292 if constexpr (std::is_same_v<RecipeTy, VPScalarIVStepsRecipe> ||
293 std::is_same_v<RecipeTy, VPDerivedIVRecipe> ||
294 std::is_same_v<RecipeTy, VPVectorEndPointerRecipe>)
295 return DefR;
296 else
297 return DefR && DefR->getOpcode() == Opcode;
298 }
299
300 /// Helper to check if predicate \p P holds on all tuple elements in Ops using
301 /// the provided index sequence.
302 template <typename Fn, std::size_t... Is>
303 bool all_of_tuple_elements(std::index_sequence<Is...>,
304 [[maybe_unused]] Fn P) const {
305 return (P(std::get<Is>(Ops), Is) && ...);
306 }
307};
308
309template <unsigned Opcode, typename... OpTys>
310using AllRecipe_match =
311 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ false,
312 VPWidenRecipe, VPReplicateRecipe, VPWidenCastRecipe,
313 VPInstruction>;
314
315template <unsigned Opcode, typename... OpTys>
316using AllRecipe_commutative_match =
317 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ true,
318 VPWidenRecipe, VPReplicateRecipe, VPInstruction>;
319
320template <unsigned Opcode, typename... OpTys>
321using VPInstruction_match = Recipe_match<std::tuple<OpTys...>, Opcode,
322 /*Commutative*/ false, VPInstruction>;
323
324template <unsigned Opcode, typename... OpTys>
325using VPInstruction_commutative_match =
326 Recipe_match<std::tuple<OpTys...>, Opcode,
327 /*Commutative*/ true, VPInstruction>;
328
329template <unsigned Opcode, typename... OpTys>
330inline VPInstruction_match<Opcode, OpTys...>
331m_VPInstruction(const OpTys &...Ops) {
332 return VPInstruction_match<Opcode, OpTys...>(Ops...);
333}
334
335template <unsigned Opcode, typename Op0_t, typename Op1_t>
336inline VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>
337m_c_VPInstruction(const Op0_t &Op0, const Op1_t &Op1) {
338 return VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
339}
340
341/// BuildVector is matches only its opcode, w/o matching its operands as the
342/// number of operands is not fixed.
343inline VPInstruction_match<VPInstruction::BuildVector> m_BuildVector() {
344 return m_VPInstruction<VPInstruction::BuildVector>();
345}
346
347/// BuildStructVector matches only its opcode, w/o matching its operands as the
348/// number of operands is not fixed.
349inline VPInstruction_match<VPInstruction::BuildStructVector>
350m_BuildStructVector() {
351 return m_VPInstruction<VPInstruction::BuildStructVector>();
352}
353
354template <typename Op0_t>
355inline VPInstruction_match<Instruction::Freeze, Op0_t>
356m_Freeze(const Op0_t &Op0) {
357 return m_VPInstruction<Instruction::Freeze>(Op0);
358}
359
360inline VPInstruction_match<VPInstruction::BranchOnCond> m_BranchOnCond() {
361 return m_VPInstruction<VPInstruction::BranchOnCond>();
362}
363
364template <typename Op0_t>
365inline VPInstruction_match<VPInstruction::BranchOnCond, Op0_t>
366m_BranchOnCond(const Op0_t &Op0) {
367 return m_VPInstruction<VPInstruction::BranchOnCond>(Op0);
368}
369
370inline VPInstruction_match<VPInstruction::BranchOnTwoConds>
371m_BranchOnTwoConds() {
372 return m_VPInstruction<VPInstruction::BranchOnTwoConds>();
373}
374
375template <typename Op0_t, typename Op1_t>
376inline VPInstruction_match<VPInstruction::BranchOnTwoConds, Op0_t, Op1_t>
377m_BranchOnTwoConds(const Op0_t &Op0, const Op1_t &Op1) {
378 return m_VPInstruction<VPInstruction::BranchOnTwoConds>(Op0, Op1);
379}
380
381inline VPInstruction_match<VPInstruction::BranchOnCount> m_BranchOnCount() {
382 return m_VPInstruction<VPInstruction::BranchOnCount>();
383}
384
385template <typename Op0_t, typename Op1_t>
386inline VPInstruction_match<VPInstruction::BranchOnCount, Op0_t, Op1_t>
387m_BranchOnCount(const Op0_t &Op0, const Op1_t &Op1) {
388 return m_VPInstruction<VPInstruction::BranchOnCount>(Op0, Op1);
389}
390
391inline auto m_Branch() {
392 return m_CombineOr(Ps: m_BranchOnCond(), Ps: m_BranchOnCount(), Ps: m_BranchOnTwoConds());
393}
394
395template <typename Op0_t>
396inline VPInstruction_match<VPInstruction::Broadcast, Op0_t>
397m_Broadcast(const Op0_t &Op0) {
398 return m_VPInstruction<VPInstruction::Broadcast>(Op0);
399}
400
401template <typename Op0_t>
402inline VPInstruction_match<VPInstruction::ExplicitVectorLength, Op0_t>
403m_EVL(const Op0_t &Op0) {
404 return m_VPInstruction<VPInstruction::ExplicitVectorLength>(Op0);
405}
406
407template <typename Op0_t>
408inline VPInstruction_match<VPInstruction::ExtractLastLane, Op0_t>
409m_ExtractLastLane(const Op0_t &Op0) {
410 return m_VPInstruction<VPInstruction::ExtractLastLane>(Op0);
411}
412
413template <typename Op0_t, typename Op1_t>
414inline VPInstruction_match<Instruction::ExtractElement, Op0_t, Op1_t>
415m_ExtractElement(const Op0_t &Op0, const Op1_t &Op1) {
416 return m_VPInstruction<Instruction::ExtractElement>(Op0, Op1);
417}
418
419template <typename Op0_t, typename Op1_t, typename Op2_t>
420inline VPInstruction_match<Instruction::InsertElement, Op0_t, Op1_t, Op2_t>
421m_InsertElement(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
422 return m_VPInstruction<Instruction::InsertElement>(Op0, Op1, Op2);
423}
424
425template <typename Op0_t, typename Op1_t>
426inline VPInstruction_match<VPInstruction::ExtractLane, Op0_t, Op1_t>
427m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1) {
428 return m_VPInstruction<VPInstruction::ExtractLane>(Op0, Op1);
429}
430
431template <typename Op0_t>
432inline VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>
433m_ExtractLastPart(const Op0_t &Op0) {
434 return m_VPInstruction<VPInstruction::ExtractLastPart>(Op0);
435}
436
437template <typename Op0_t>
438inline VPInstruction_match<
439 VPInstruction::ExtractLastLane,
440 VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>>
441m_ExtractLastLaneOfLastPart(const Op0_t &Op0) {
442 return m_ExtractLastLane(m_ExtractLastPart(Op0));
443}
444
445template <typename Op0_t>
446inline VPInstruction_match<VPInstruction::ExtractPenultimateElement, Op0_t>
447m_ExtractPenultimateElement(const Op0_t &Op0) {
448 return m_VPInstruction<VPInstruction::ExtractPenultimateElement>(Op0);
449}
450
451template <typename Op0_t, typename Op1_t, typename Op2_t>
452inline VPInstruction_match<VPInstruction::ActiveLaneMask, Op0_t, Op1_t, Op2_t>
453m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
454 return m_VPInstruction<VPInstruction::ActiveLaneMask>(Op0, Op1, Op2);
455}
456
457inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() {
458 return m_VPInstruction<VPInstruction::AnyOf>();
459}
460
461template <typename Op0_t>
462inline VPInstruction_match<VPInstruction::AnyOf, Op0_t>
463m_AnyOf(const Op0_t &Op0) {
464 return m_VPInstruction<VPInstruction::AnyOf>(Op0);
465}
466
467template <typename Op0_t>
468inline VPInstruction_match<VPInstruction::FirstActiveLane, Op0_t>
469m_FirstActiveLane(const Op0_t &Op0) {
470 return m_VPInstruction<VPInstruction::FirstActiveLane>(Op0);
471}
472
473template <typename Op0_t>
474inline VPInstruction_match<VPInstruction::LastActiveLane, Op0_t>
475m_LastActiveLane(const Op0_t &Op0) {
476 return m_VPInstruction<VPInstruction::LastActiveLane>(Op0);
477}
478
479template <typename Op0_t, typename Op1_t, typename Op2_t>
480inline VPInstruction_match<VPInstruction::ExtractLastActive, Op0_t, Op1_t,
481 Op2_t>
482m_ExtractLastActive(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
483 return m_VPInstruction<VPInstruction::ExtractLastActive>(Op0, Op1, Op2);
484}
485
486template <typename Op0_t>
487inline VPInstruction_match<VPInstruction::ComputeReductionResult, Op0_t>
488m_ComputeReductionResult(const Op0_t &Op0) {
489 return m_VPInstruction<VPInstruction::ComputeReductionResult>(Op0);
490}
491
492/// Match FindIV result pattern:
493/// select(icmp ne ComputeReductionResult(ReducedIV), Sentinel),
494/// ComputeReductionResult(ReducedIV), Start.
495template <typename Op0_t, typename Op1_t>
496inline bool matchFindIVResult(VPInstruction *VPI, Op0_t ReducedIV, Op1_t Start) {
497 return match(VPI, m_Select(m_SpecificICmp(ICmpInst::ICMP_NE,
498 m_ComputeReductionResult(ReducedIV),
499 m_VPValue()),
500 m_ComputeReductionResult(ReducedIV), Start));
501}
502
503template <typename Op0_t>
504inline VPInstruction_match<VPInstruction::Reverse, Op0_t>
505m_Reverse(const Op0_t &Op0) {
506 return m_VPInstruction<VPInstruction::Reverse>(Op0);
507}
508
509inline VPInstruction_match<VPInstruction::StepVector> m_StepVector() {
510 return m_VPInstruction<VPInstruction::StepVector>();
511}
512
513template <typename Op0_t>
514inline VPInstruction_match<VPInstruction::ExitingIVValue, Op0_t>
515m_ExitingIVValue(const Op0_t &Op0) {
516 return m_VPInstruction<VPInstruction::ExitingIVValue>(Op0);
517}
518
519template <unsigned Opcode, typename Op0_t>
520inline AllRecipe_match<Opcode, Op0_t> m_Unary(const Op0_t &Op0) {
521 return AllRecipe_match<Opcode, Op0_t>(Op0);
522}
523
524template <typename Op0_t>
525inline AllRecipe_match<Instruction::Trunc, Op0_t> m_Trunc(const Op0_t &Op0) {
526 return m_Unary<Instruction::Trunc, Op0_t>(Op0);
527}
528
529template <typename Op0_t>
530inline match_combine_or<AllRecipe_match<Instruction::Trunc, Op0_t>, Op0_t>
531m_TruncOrSelf(const Op0_t &Op0) {
532 return m_CombineOr(m_Trunc(Op0), Op0);
533}
534
535template <typename Op0_t>
536inline AllRecipe_match<Instruction::ZExt, Op0_t> m_ZExt(const Op0_t &Op0) {
537 return m_Unary<Instruction::ZExt, Op0_t>(Op0);
538}
539
540template <typename Op0_t>
541inline AllRecipe_match<Instruction::SExt, Op0_t> m_SExt(const Op0_t &Op0) {
542 return m_Unary<Instruction::SExt, Op0_t>(Op0);
543}
544
545template <typename Op0_t>
546inline AllRecipe_match<Instruction::FPExt, Op0_t> m_FPExt(const Op0_t &Op0) {
547 return m_Unary<Instruction::FPExt, Op0_t>(Op0);
548}
549
550template <typename Op0_t>
551inline AllRecipe_match<Instruction::FNeg, Op0_t> m_FNeg(const Op0_t &Op0) {
552 return m_Unary<Instruction::FNeg, Op0_t>(Op0);
553}
554
555template <typename Op0_t>
556inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>,
557 AllRecipe_match<Instruction::SExt, Op0_t>>
558m_ZExtOrSExt(const Op0_t &Op0) {
559 return m_CombineOr(m_ZExt(Op0), m_SExt(Op0));
560}
561
562template <typename Op0_t> inline auto m_WidenAnyExtend(const Op0_t &Op0) {
563 return m_Isa<VPWidenCastRecipe>(m_CombineOr(m_ZExtOrSExt(Op0), m_FPExt(Op0)));
564}
565
566template <typename Op0_t> inline auto m_AnyNeg(const Op0_t &Op0) {
567 return m_CombineOr(m_Sub(m_ZeroInt(), Op0), m_FNeg(Op0));
568}
569
570template <typename Op0_t>
571inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>, Op0_t>
572m_ZExtOrSelf(const Op0_t &Op0) {
573 return m_CombineOr(m_ZExt(Op0), Op0);
574}
575
576template <typename Op0_t> inline auto m_ZExtOrTruncOrSelf(const Op0_t &Op0) {
577 return m_CombineOr(m_ZExt(Op0), m_Trunc(Op0), Op0);
578}
579
580template <unsigned Opcode, typename Op0_t, typename Op1_t>
581inline AllRecipe_match<Opcode, Op0_t, Op1_t> m_Binary(const Op0_t &Op0,
582 const Op1_t &Op1) {
583 return AllRecipe_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
584}
585
586template <unsigned Opcode, typename Op0_t, typename Op1_t>
587inline AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>
588m_c_Binary(const Op0_t &Op0, const Op1_t &Op1) {
589 return AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
590}
591
592template <typename Op0_t, typename Op1_t>
593inline AllRecipe_match<Instruction::Add, Op0_t, Op1_t> m_Add(const Op0_t &Op0,
594 const Op1_t &Op1) {
595 return m_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
596}
597
598template <typename Op0_t, typename Op1_t>
599inline AllRecipe_commutative_match<Instruction::Add, Op0_t, Op1_t>
600m_c_Add(const Op0_t &Op0, const Op1_t &Op1) {
601 return m_c_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
602}
603
604template <typename Op0_t, typename Op1_t>
605inline AllRecipe_match<Instruction::Sub, Op0_t, Op1_t> m_Sub(const Op0_t &Op0,
606 const Op1_t &Op1) {
607 return m_Binary<Instruction::Sub, Op0_t, Op1_t>(Op0, Op1);
608}
609
610template <typename Op0_t, typename Op1_t>
611inline AllRecipe_match<Instruction::Mul, Op0_t, Op1_t> m_Mul(const Op0_t &Op0,
612 const Op1_t &Op1) {
613 return m_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
614}
615
616template <typename Op0_t, typename Op1_t>
617inline AllRecipe_commutative_match<Instruction::Mul, Op0_t, Op1_t>
618m_c_Mul(const Op0_t &Op0, const Op1_t &Op1) {
619 return m_c_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
620}
621
622template <typename Op0_t, typename Op1_t>
623inline AllRecipe_match<Instruction::Shl, Op0_t, Op1_t> m_Shl(const Op0_t &Op0,
624 const Op1_t &Op1) {
625 return m_Binary<Instruction::Shl, Op0_t, Op1_t>(Op0, Op1);
626}
627
628template <typename Op0_t, typename Op1_t>
629inline AllRecipe_match<Instruction::LShr, Op0_t, Op1_t>
630m_LShr(const Op0_t &Op0, const Op1_t &Op1) {
631 return m_Binary<Instruction::LShr, Op0_t, Op1_t>(Op0, Op1);
632}
633
634template <typename Op0_t, typename Op1_t>
635inline AllRecipe_match<Instruction::FMul, Op0_t, Op1_t>
636m_FMul(const Op0_t &Op0, const Op1_t &Op1) {
637 return m_Binary<Instruction::FMul, Op0_t, Op1_t>(Op0, Op1);
638}
639
640template <typename Op0_t, typename Op1_t>
641inline AllRecipe_match<Instruction::FAdd, Op0_t, Op1_t>
642m_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
643 return m_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
644}
645
646template <typename Op0_t, typename Op1_t>
647inline AllRecipe_commutative_match<Instruction::FAdd, Op0_t, Op1_t>
648m_c_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
649 return m_c_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
650}
651
652template <typename Op0_t, typename Op1_t>
653inline AllRecipe_match<Instruction::UDiv, Op0_t, Op1_t>
654m_UDiv(const Op0_t &Op0, const Op1_t &Op1) {
655 return m_Binary<Instruction::UDiv, Op0_t, Op1_t>(Op0, Op1);
656}
657
658template <typename Op0_t, typename Op1_t>
659inline AllRecipe_match<Instruction::URem, Op0_t, Op1_t>
660m_URem(const Op0_t &Op0, const Op1_t &Op1) {
661 return m_Binary<Instruction::URem, Op0_t, Op1_t>(Op0, Op1);
662}
663
664template <typename Op0_t, typename Op1_t>
665inline AllRecipe_match<Instruction::SRem, Op0_t, Op1_t>
666m_SRem(const Op0_t &Op0, const Op1_t &Op1) {
667 return m_Binary<Instruction::SRem, Op0_t, Op1_t>(Op0, Op1);
668}
669
670/// Match a binary AND operation.
671template <typename Op0_t, typename Op1_t>
672inline AllRecipe_commutative_match<Instruction::And, Op0_t, Op1_t>
673m_c_BinaryAnd(const Op0_t &Op0, const Op1_t &Op1) {
674 return m_c_Binary<Instruction::And, Op0_t, Op1_t>(Op0, Op1);
675}
676
677/// Match a binary OR operation. Note that while conceptually the operands can
678/// be matched commutatively, \p Commutative defaults to false in line with the
679/// IR-based pattern matching infrastructure. Use m_c_BinaryOr for a commutative
680/// version of the matcher.
681template <typename Op0_t, typename Op1_t>
682inline AllRecipe_match<Instruction::Or, Op0_t, Op1_t>
683m_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
684 return m_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
685}
686
687template <typename Op0_t, typename Op1_t>
688inline AllRecipe_commutative_match<Instruction::Or, Op0_t, Op1_t>
689m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
690 return m_c_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
691}
692
693/// Cmp_match is a variant of BinaryRecipe_match that also binds the comparison
694/// predicate. Opcodes must either be Instruction::ICmp or Instruction::FCmp, or
695/// both.
696template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
697struct Cmp_match {
698 static_assert((sizeof...(Opcodes) == 1 || sizeof...(Opcodes) == 2) &&
699 "Expected one or two opcodes");
700 static_assert(
701 ((Opcodes == Instruction::ICmp || Opcodes == Instruction::FCmp) && ...) &&
702 "Expected a compare instruction opcode");
703
704 CmpPredicate *Predicate = nullptr;
705 Op0_t Op0;
706 Op1_t Op1;
707
708 Cmp_match(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1)
709 : Predicate(&Pred), Op0(Op0), Op1(Op1) {}
710 Cmp_match(const Op0_t &Op0, const Op1_t &Op1) : Op0(Op0), Op1(Op1) {}
711
712 bool match(const VPValue *V) const {
713 auto *DefR = V->getDefiningRecipe();
714 return DefR && match(DefR);
715 }
716
717 bool match(const VPRecipeBase *V) const {
718 if ((m_Binary<Opcodes>(Op0, Op1).match(V) || ...)) {
719 if (Predicate)
720 *Predicate = cast<VPRecipeWithIRFlags>(Val: V)->getPredicate();
721 return true;
722 }
723 return false;
724 }
725};
726
727/// SpecificCmp_match is a variant of Cmp_match that matches the comparison
728/// predicate, instead of binding it.
729template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
730struct SpecificCmp_match {
731 const CmpPredicate Predicate;
732 Op0_t Op0;
733 Op1_t Op1;
734
735 SpecificCmp_match(CmpPredicate Pred, const Op0_t &LHS, const Op1_t &RHS)
736 : Predicate(Pred), Op0(LHS), Op1(RHS) {}
737
738 bool match(const VPValue *V) const {
739 auto *DefR = V->getDefiningRecipe();
740 return DefR && match(DefR);
741 }
742
743 bool match(const VPRecipeBase *V) const {
744 CmpPredicate CurrentPred;
745 return Cmp_match<Op0_t, Op1_t, Opcodes...>(CurrentPred, Op0, Op1)
746 .match(V) &&
747 CmpPredicate::getMatching(A: CurrentPred, B: Predicate);
748 }
749};
750
751template <typename Op0_t, typename Op1_t>
752inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp> m_ICmp(const Op0_t &Op0,
753 const Op1_t &Op1) {
754 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Op0, Op1);
755}
756
757template <typename Op0_t, typename Op1_t>
758inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp>
759m_ICmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
760 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Pred, Op0, Op1);
761}
762
763template <typename Op0_t, typename Op1_t>
764inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>
765m_SpecificICmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
766 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>(MatchPred, Op0,
767 Op1);
768}
769
770template <typename Op0_t, typename Op1_t>
771inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
772m_Cmp(const Op0_t &Op0, const Op1_t &Op1) {
773 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(Op0,
774 Op1);
775}
776
777template <typename Op0_t, typename Op1_t>
778inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
779m_Cmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
780 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
781 Pred, Op0, Op1);
782}
783
784template <typename Op0_t, typename Op1_t>
785inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
786m_SpecificCmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
787 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
788 MatchPred, Op0, Op1);
789}
790
791template <typename Op0_t, typename Op1_t>
792inline auto m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1) {
793 return m_CombineOr(
794 Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr,
795 /*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>(
796 Op0, Op1),
797 VPInstruction_match<VPInstruction::PtrAdd, Op0_t, Op1_t>(Op0, Op1),
798 VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>(Op0, Op1));
799}
800
801/// Match a VPBlendRecipe with 2 incoming values ([I0, I1, M1] ==
802/// normalized([I0, M0, I1, M1])) as select(M1, I1, I0), mirroring how it is
803/// lowered.
804template <typename Op0_t, typename Op1_t, typename Op2_t> struct Blend2_match {
805 Op0_t MaskOp;
806 Op1_t TrueOp;
807 Op2_t FalseOp;
808
809 Blend2_match(const Op0_t &MaskOp, const Op1_t &TrueOp, const Op2_t &FalseOp)
810 : MaskOp(MaskOp), TrueOp(TrueOp), FalseOp(FalseOp) {}
811
812 template <typename T> bool match(const T *Val) const {
813 auto *Blend = dyn_cast<VPBlendRecipe>(Val);
814 if (!Blend || Blend->getNumIncomingValues() != 2)
815 return false;
816 return MaskOp.match(Blend->getMask(1)) &&
817 TrueOp.match(Blend->getIncomingValue(1)) &&
818 FalseOp.match(Blend->getIncomingValue(0));
819 }
820};
821
822/// Match recipe recipe with Select opcode, i.e. excluding VPBlendRecipe.
823template <typename Op0_t, typename Op1_t, typename Op2_t>
824inline AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>
825m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
826 return AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>(
827 {Op0, Op1, Op2});
828}
829
830/// Match recipe with Select opcode or an equivalent VPBlendRecipe with 2
831/// incoming values.
832template <typename Op0_t, typename Op1_t, typename Op2_t>
833inline auto m_SelectLike(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
834 return m_CombineOr(m_Select(Op0, Op1, Op2),
835 Blend2_match<Op0_t, Op1_t, Op2_t>(Op0, Op1, Op2));
836}
837
838template <typename Op0_t> inline auto m_Not(const Op0_t &Op0) {
839 return m_CombineOr(m_VPInstruction<VPInstruction::Not>(Op0),
840 m_c_Binary<Instruction::Xor>(m_AllOnes(), Op0));
841}
842
843template <typename Op0_t, typename Op1_t, typename Op2_t>
844inline auto m_c_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
845 return m_CombineOr(m_Select(Op0, Op1, Op2), m_Select(m_Not(Op0), Op2, Op1));
846}
847
848template <typename Op0_t, typename Op1_t>
849inline auto m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
850 return m_CombineOr(
851 m_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
852 m_Select(Op0, Op1, m_False()));
853}
854
855template <typename Op0_t, typename Op1_t>
856inline auto m_c_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
857 return m_CombineOr(
858 m_c_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
859 m_c_Select(Op0, Op1, m_False()));
860}
861
862template <typename Op0_t, typename Op1_t>
863inline auto m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
864 return m_CombineOr(
865 m_c_VPInstruction<VPInstruction::LogicalOr, Op0_t, Op1_t>(Op0, Op1),
866 m_Select(Op0, m_True(), Op1));
867}
868
869template <typename Op0_t, typename Op1_t>
870inline auto m_c_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
871 return m_c_Select(Op0, m_True(), Op1);
872}
873
874/// Match the canonical induction variable (IV) of any loop region.
875struct canonical_iv_match {
876 template <typename ArgTy> bool match(const ArgTy *V) const {
877 const auto *RV = dyn_cast<VPRegionValue>(V);
878 return RV && RV->getDefiningRegion()->getCanonicalIV() == RV;
879 }
880};
881
882inline canonical_iv_match m_CanonicalIV() { return {}; }
883
884/// Match the abstract header mask of any loop region.
885struct header_mask_match {
886 template <typename ArgTy> bool match(const ArgTy *V) const {
887 const auto *RV = dyn_cast<VPRegionValue>(V);
888 return RV && RV->getDefiningRegion()->getHeaderMask() == RV;
889 }
890};
891
892inline header_mask_match m_HeaderMask() { return {}; }
893
894/// Match a canonical VPWidenIntOrFpInductionRecipe optionally capturing it.
895struct canonical_widen_iv_match {
896 VPWidenIntOrFpInductionRecipe **Capture = nullptr;
897
898 canonical_widen_iv_match() = default;
899 canonical_widen_iv_match(VPWidenIntOrFpInductionRecipe *&V) : Capture(&V) {}
900
901 template <typename ArgTy> bool match(ArgTy *V) const {
902 auto *WidenIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(V);
903 if (!WidenIV || !WidenIV->isCanonical())
904 return false;
905 if (Capture)
906 *Capture = WidenIV;
907 return true;
908 }
909};
910
911inline canonical_widen_iv_match m_CanonicalWidenIV() { return {}; }
912
913/// Match a canonical VPWidenIntOrFpInductionRecipe, capturing it.
914inline canonical_widen_iv_match
915m_CanonicalWidenIV(VPWidenIntOrFpInductionRecipe *&V) {
916 return canonical_widen_iv_match(V);
917}
918
919template <typename Op0_t, typename Op1_t, typename Op2_t>
920inline auto m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1,
921 const Op2_t &Op2) {
922 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
923 VPScalarIVStepsRecipe>({Op0, Op1, Op2});
924}
925
926template <typename Op0_t, typename Op1_t, typename Op2_t>
927inline auto m_DerivedIV(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
928 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
929 VPDerivedIVRecipe>({Op0, Op1, Op2});
930}
931
932template <typename Addr_t, typename Mask_t> struct Load_match {
933 Addr_t Addr;
934 Mask_t Mask;
935
936 Load_match(Addr_t Addr, Mask_t Mask) : Addr(Addr), Mask(Mask) {}
937
938 template <typename OpTy> bool match(const OpTy *V) const {
939 auto *Load = dyn_cast<VPWidenLoadRecipe>(V);
940 if (!Load || !Addr.match(Load->getAddr()) || !Load->isMasked() ||
941 !Mask.match(Load->getMask()))
942 return false;
943 return true;
944 }
945};
946
947/// Match a (possibly reversed) masked load.
948template <typename Addr_t, typename Mask_t>
949inline Load_match<Addr_t, Mask_t> m_MaskedLoad(const Addr_t &Addr,
950 const Mask_t &Mask) {
951 return Load_match<Addr_t, Mask_t>(Addr, Mask);
952}
953
954template <typename Addr_t, typename Val_t, typename Mask_t> struct Store_match {
955 Addr_t Addr;
956 Val_t Val;
957 Mask_t Mask;
958
959 Store_match(Addr_t Addr, Val_t Val, Mask_t Mask)
960 : Addr(Addr), Val(Val), Mask(Mask) {}
961
962 template <typename OpTy> bool match(const OpTy *V) const {
963 auto *Store = dyn_cast<VPWidenStoreRecipe>(V);
964 if (!Store || !Addr.match(Store->getAddr()) ||
965 !Val.match(Store->getStoredValue()) || !Store->isMasked() ||
966 !Mask.match(Store->getMask()))
967 return false;
968 return true;
969 }
970};
971
972/// Match a (possibly reversed) masked store.
973template <typename Addr_t, typename Val_t, typename Mask_t>
974inline Store_match<Addr_t, Val_t, Mask_t>
975m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) {
976 return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask);
977}
978
979template <typename Op0_t, typename Op1_t>
980using VectorEndPointerRecipe_match =
981 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
982 /*Commutative*/ false, VPVectorEndPointerRecipe>;
983
984template <typename Op0_t, typename Op1_t>
985VectorEndPointerRecipe_match<Op0_t, Op1_t> m_VecEndPtr(const Op0_t &Op0,
986 const Op1_t &Op1) {
987 return VectorEndPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
988}
989
990/// Match a call argument at a given argument index.
991template <typename Opnd_t> struct Argument_match {
992 /// Call argument index to match.
993 unsigned OpI;
994 Opnd_t Val;
995
996 Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}
997
998 template <typename OpTy> bool match(OpTy *V) const {
999 if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V))
1000 return Val.match(R->getOperand(OpI));
1001 if (const auto *R = dyn_cast<VPWidenCallRecipe>(V))
1002 return Val.match(R->getOperand(OpI));
1003 if (const auto *R = dyn_cast<VPReplicateRecipe>(V))
1004 if (R->getOpcode() == Instruction::Call)
1005 return Val.match(R->getOperand(OpI));
1006 if (const auto *R = dyn_cast<VPInstruction>(V))
1007 if (R->getOpcode() == Instruction::Call ||
1008 R->getOpcode() == VPInstruction::Intrinsic)
1009 return Val.match(R->getOperand(OpI));
1010 return false;
1011 }
1012};
1013
1014/// Match a call argument.
1015template <unsigned OpI, typename Opnd_t>
1016inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) {
1017 return Argument_match<Opnd_t>(OpI, Op);
1018}
1019
1020/// Intrinsic matchers.
1021struct IntrinsicID_match {
1022 unsigned ID;
1023
1024 IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {}
1025
1026 template <typename OpTy> bool match(OpTy *V) const {
1027 return vputils::getIntrinsicID(V) == ID;
1028 }
1029};
1030
1031/// Match intrinsic calls with a runtime intrinsic ID.
1032inline IntrinsicID_match m_Intrinsic(Intrinsic::ID IntrID) {
1033 return IntrinsicID_match(IntrID);
1034}
1035
1036struct IntrinsicMatchImpl {
1037 template <Intrinsic::ID IntrID, typename... Ts, size_t... Is>
1038 static auto impl(std::index_sequence<Is...>, const Ts &...Ops) {
1039 return m_CombineAnd(IntrinsicID_match(IntrID), m_Argument<Is>(Ops)...);
1040 }
1041};
1042
1043/// Match intrinsic calls like this:
1044/// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...)
1045template <Intrinsic::ID IntrID, typename... Ts>
1046inline auto m_Intrinsic(const Ts &...Ops) {
1047 return IntrinsicMatchImpl::impl<IntrID>(
1048 std::make_index_sequence<sizeof...(Ts)>{}, Ops...);
1049}
1050
1051template <Intrinsic::ID IntrID, typename... T>
1052inline auto m_WidenIntrinsic(const T &...Ops) {
1053 return m_Isa<VPWidenIntrinsicRecipe>(m_Intrinsic<IntrID>(Ops...));
1054}
1055
1056/// Match VPValues that represent live-ins: VPIRValues and (plain)
1057/// VPSymbolicValues. VPRegionValues (which inherit from VPSymbolicValue) are
1058/// not live-ins and are excluded.
1059struct LiveIn_match {
1060 template <typename ITy> bool match(ITy *V) const {
1061 return isa<VPIRValue>(V) ||
1062 (isa<VPSymbolicValue>(V) && !isa<VPRegionValue>(V));
1063 }
1064};
1065
1066inline auto m_VScale() { return m_Intrinsic<Intrinsic::vscale>(); }
1067
1068inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
1069
1070/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
1071/// and bind the source element type and operands.
1072struct GetElementPtr_match {
1073 Type *&SourceElementType;
1074 ArrayRef<VPValue *> &Operands;
1075
1076 GetElementPtr_match(Type *&SourceElementType, ArrayRef<VPValue *> &Operands)
1077 : SourceElementType(SourceElementType), Operands(Operands) {}
1078
1079 template <typename ITy> bool match(ITy *V) const {
1080 return matchRecipeAndBind<VPWidenGEPRecipe>(V) ||
1081 matchRecipeAndBind<VPInstruction>(V) ||
1082 matchRecipeAndBind<VPReplicateRecipe>(V);
1083 }
1084
1085private:
1086 template <typename RecipeTy> bool matchRecipeAndBind(const VPValue *V) const {
1087 auto *DefR = dyn_cast<RecipeTy>(V);
1088 if (!DefR)
1089 return false;
1090
1091 if constexpr (std::is_same_v<RecipeTy, VPWidenGEPRecipe>) {
1092 SourceElementType = DefR->getSourceElementType();
1093 } else if (DefR->getOpcode() == Instruction::GetElementPtr) {
1094 SourceElementType = cast<GetElementPtrInst>(DefR->getUnderlyingInstr())
1095 ->getSourceElementType();
1096 } else if constexpr (std::is_same_v<RecipeTy, VPInstruction>) {
1097 if (DefR->getOpcode() == VPInstruction::PtrAdd) {
1098 // PtrAdd is a byte-offset GEP with i8 element type.
1099 LLVMContext &Ctx = DefR->getParent()->getPlan()->getContext();
1100 SourceElementType = Type::getInt8Ty(C&: Ctx);
1101 } else {
1102 return false;
1103 }
1104 } else {
1105 return false;
1106 }
1107
1108 Operands = ArrayRef<VPValue *>(DefR->op_begin(), DefR->op_end());
1109 return true;
1110 }
1111};
1112
1113/// Match a GEP recipe with any number of operands and bind source element type
1114/// and operands.
1115inline GetElementPtr_match m_GetElementPtr(Type *&SourceElementType,
1116 ArrayRef<VPValue *> &Operands) {
1117 return GetElementPtr_match(SourceElementType, Operands);
1118}
1119
1120template <typename SubPattern_t> struct OneUse_match {
1121 SubPattern_t SubPattern;
1122
1123 OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}
1124
1125 template <typename OpTy> bool match(OpTy *V) const {
1126 return V->hasOneUse() && SubPattern.match(V);
1127 }
1128};
1129
1130template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) {
1131 return SubPattern;
1132}
1133
1134inline match_bind<VPReductionPHIRecipe>
1135m_ReductionPhi(VPReductionPHIRecipe *&V) {
1136 return V;
1137}
1138
1139template <typename Op0_t, typename Op1_t>
1140inline auto m_VPPhi(const Op0_t &Op0, const Op1_t &Op1) {
1141 return Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::PHI,
1142 /*Commutative*/ false, VPInstruction>({Op0, Op1});
1143}
1144
1145/// If \p V is used by a recipe matching pattern \p P, return it. Otherwise
1146/// return nullptr;
1147template <typename MatchT>
1148VPRecipeBase *findUserOf(VPValue *V, const MatchT &P) {
1149 auto It = find_if(V->users(), match_fn(P));
1150 return It == V->user_end() ? nullptr : cast<VPRecipeBase>(*It);
1151}
1152
1153/// If \p V is used by a VPInstruction with \p Opcode, return it. Otherwise
1154/// return nullptr.
1155template <unsigned Opcode> VPInstruction *findUserOf(VPValue *V) {
1156 return cast_or_null<VPInstruction>(findUserOf(V, m_VPInstruction<Opcode>()));
1157}
1158
1159template <typename RecipeTy> RecipeTy *findUserOf(VPValue *V) {
1160 return cast_or_null<RecipeTy>(findUserOf(V, m_Isa<RecipeTy>()));
1161}
1162} // namespace llvm::VPlanPatternMatch
1163
1164#endif
1165