1//===- VPlanPatternMatch.h - Match on VPValues and recipes ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides a simple and efficient mechanism for performing general
10// tree-based pattern matches on the VPlan values and recipes, based on
11// LLVM's IR pattern matchers.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
16#define LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
17
18#include "VPlan.h"
19#include "VPlanUtils.h"
20#include "llvm/Support/PatternMatchHelpers.h"
21
22using namespace llvm::PatternMatchHelpers;
23
24namespace llvm::VPlanPatternMatch {
25
26template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
27 return P.match(V);
28}
29
30template <typename Pattern> bool match(VPUser *U, const Pattern &P) {
31 auto *R = dyn_cast<VPRecipeBase>(Val: U);
32 return R && match(R, P);
33}
34
35template <typename Pattern> bool match(VPSingleDefRecipe *R, const Pattern &P) {
36 return P.match(static_cast<const VPRecipeBase *>(R));
37}
38
39/// A match functor that can be used as a UnaryPredicate in functional
40/// algorithms like all_of.
41template <typename Pattern> auto match_fn(const Pattern &P) {
42 return [&P](auto *V) { return match(V, P); };
43}
44
45/// Match an arbitrary VPValue and ignore it.
46inline auto m_VPValue() { return m_Isa<VPValue>(); }
47
48/// Match a specified VPValue.
49struct specificval_ty {
50 const VPValue *Val;
51
52 specificval_ty(const VPValue *V) : Val(V) {}
53
54 bool match(const VPValue *VPV) const { return VPV == Val; }
55};
56
57inline specificval_ty m_Specific(const VPValue *VPV) { return VPV; }
58
59/// Like m_Specific(), but works if the specific value to match is determined
60/// as part of the same match() expression. For example:
61/// m_Mul(m_VPValue(X), m_Specific(X)) is incorrect, because m_Specific() will
62/// bind X before the pattern match starts.
63/// m_Mul(m_VPValue(X), m_Deferred(X)) is correct, and will check against
64/// whichever value m_VPValue(X) populated.
65inline match_deferred<VPValue> m_Deferred(VPValue *const &V) { return V; }
66
67/// Match an integer constant if Pred::isValue returns true for the APInt. \p
68/// BitWidth optionally specifies the bitwidth the matched constant must have.
69/// If it is 0, the matched constant can have any bitwidth.
70template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty {
71 Pred P;
72
73 int_pred_ty(Pred P) : P(std::move(P)) {}
74 int_pred_ty() : P() {}
75
76 bool match(const VPValue *VPV) const {
77 auto *VPI = dyn_cast<VPInstruction>(Val: VPV);
78 if (VPI && VPI->getOpcode() == VPInstruction::Broadcast)
79 VPV = VPI->getOperand(N: 0);
80 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
81 if (!CI)
82 return false;
83
84 if (BitWidth != 0 && CI->getBitWidth() != BitWidth)
85 return false;
86 return P.isValue(CI->getAPInt());
87 }
88};
89
90/// Match a specified signed or unsigned integer value.
91struct is_specific_int {
92 APInt Val;
93 bool IsSigned;
94
95 is_specific_int(APInt Val, bool IsSigned = false)
96 : Val(std::move(Val)), IsSigned(IsSigned) {}
97
98 bool isValue(const APInt &C) const {
99 return APInt::isSameValue(I1: Val, I2: C, SignedCompare: IsSigned);
100 }
101};
102
103template <unsigned Bitwidth = 0>
104using specific_intval = int_pred_ty<is_specific_int, Bitwidth>;
105
106inline specific_intval<0> m_SpecificInt(uint64_t V) {
107 return specific_intval<0>(is_specific_int(APInt(64, V)));
108}
109
110inline specific_intval<0> m_SpecificSInt(int64_t V) {
111 return specific_intval<0>(
112 is_specific_int(APInt(64, V, /*isSigned=*/true), /*IsSigned=*/true));
113}
114
115inline specific_intval<1> m_False() {
116 return specific_intval<1>(is_specific_int(APInt(64, 0)));
117}
118
119inline specific_intval<1> m_True() {
120 return specific_intval<1>(is_specific_int(APInt(64, 1)));
121}
122
123struct is_all_ones {
124 bool isValue(const APInt &C) const { return C.isAllOnes(); }
125};
126
127/// Match an integer or vector with all bits set.
128/// For vectors, this includes constants with undefined elements.
129inline int_pred_ty<is_all_ones> m_AllOnes() {
130 return int_pred_ty<is_all_ones>();
131}
132
133struct is_zero_int {
134 bool isValue(const APInt &C) const { return C.isZero(); }
135};
136
137struct is_one {
138 bool isValue(const APInt &C) const { return C.isOne(); }
139};
140
141/// Match an integer 0 or a vector with all elements equal to 0.
142/// For vectors, this includes constants with undefined elements.
143inline int_pred_ty<is_zero_int> m_ZeroInt() {
144 return int_pred_ty<is_zero_int>();
145}
146
147/// Match an integer 1 or a vector with all elements equal to 1.
148/// For vectors, this includes constants with undefined elements.
149inline int_pred_ty<is_one> m_One() { return int_pred_ty<is_one>(); }
150
151struct bind_apint {
152 const APInt *&Res;
153
154 bind_apint(const APInt *&Res) : Res(Res) {}
155
156 bool match(const VPValue *VPV) const {
157 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
158 if (!CI)
159 return false;
160 Res = &CI->getAPInt();
161 return true;
162 }
163};
164
165inline bind_apint m_APInt(const APInt *&C) { return C; }
166
167struct bind_const_int {
168 uint64_t &Res;
169
170 bind_const_int(uint64_t &Res) : Res(Res) {}
171
172 bool match(const VPValue *VPV) const {
173 const APInt *APConst;
174 if (!bind_apint(APConst).match(VPV))
175 return false;
176 if (auto C = APConst->tryZExtValue()) {
177 Res = *C;
178 return true;
179 }
180 return false;
181 }
182};
183
184struct match_poison {
185 bool match(const VPValue *V) const {
186 return isa<VPIRValue>(Val: V) &&
187 isa<PoisonValue>(Val: cast<VPIRValue>(Val: V)->getValue());
188 }
189};
190
191/// Match a VPIRValue that's poison.
192inline match_poison m_Poison() { return match_poison(); }
193
194/// Match a plain integer constant no wider than 64-bits, capturing it if we
195/// match.
196inline bind_const_int m_ConstantInt(uint64_t &C) { return C; }
197
198/// Match a VPValue, capturing it if we match.
199inline match_bind<VPValue> m_VPValue(VPValue *&V) { return V; }
200
201/// Match a VPIRValue.
202inline match_bind<VPIRValue> m_VPIRValue(VPIRValue *&V) { return V; }
203
204/// Match a VPSingleDefRecipe, capturing if we match.
205inline match_bind<VPSingleDefRecipe>
206m_VPSingleDefRecipe(VPSingleDefRecipe *&V) {
207 return V;
208}
209
210/// Match a VPInstruction, capturing if we match.
211inline match_bind<VPInstruction> m_VPInstruction(VPInstruction *&V) {
212 return V;
213}
214
215template <typename Ops_t, unsigned Opcode, bool Commutative,
216 typename... RecipeTys>
217struct Recipe_match {
218 Ops_t Ops;
219
220 template <typename... OpTy> Recipe_match(OpTy... Ops) : Ops(Ops...) {
221 static_assert(std::tuple_size<Ops_t>::value == sizeof...(Ops) &&
222 "number of operands in constructor doesn't match Ops_t");
223 static_assert((!Commutative || std::tuple_size<Ops_t>::value == 2) &&
224 "only binary ops can be commutative");
225 }
226
227 bool match(const VPValue *V) const {
228 auto *DefR = V->getDefiningRecipe();
229 return DefR && match(DefR);
230 }
231
232 bool match(const VPSingleDefRecipe *R) const {
233 return match(static_cast<const VPRecipeBase *>(R));
234 }
235
236 bool match(const VPRecipeBase *R) const {
237 if (std::tuple_size_v<Ops_t> == 0) {
238 auto *VPI = dyn_cast<VPInstruction>(Val: R);
239 return VPI && VPI->getOpcode() == Opcode;
240 }
241
242 if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...))
243 return false;
244
245 if (R->getNumOperands() < std::tuple_size<Ops_t>::value) {
246 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R);
247 assert(((isa<VPInstruction>(R) &&
248 cast<VPInstruction>(R)->getNumOperandsForOpcode() == -1u) ||
249 (RepR && std::tuple_size_v<Ops_t> ==
250 RepR->getNumOperandsWithoutMask())) &&
251 "non-variadic recipe with matched opcode does not have the "
252 "expected number of operands");
253 return false;
254 }
255
256 // If the recipe has more operands than expected, we only support matching
257 // masked VPInstructions or predicated VPReplicateRecipes, where the number
258 // of operands of the matcher matches the number of operands excluding the
259 // mask.
260 if (R->getNumOperands() > std::tuple_size<Ops_t>::value) {
261 if (auto *VPI = dyn_cast<VPInstruction>(Val: R)) {
262 if (!VPI->isMasked() ||
263 VPI->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
264 return false;
265 } else if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R)) {
266 if (!RepR->isPredicated() ||
267 RepR->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
268 return false;
269 } else {
270 return false;
271 }
272 }
273
274 auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>();
275 if (all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
276 return Op.match(R->getOperand(N: Idx));
277 }))
278 return true;
279
280 return Commutative &&
281 all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
282 return Op.match(R->getOperand(N: R->getNumOperands() - Idx - 1));
283 });
284 }
285
286private:
287 template <typename RecipeTy>
288 static bool matchRecipeAndOpcode(const VPRecipeBase *R) {
289 auto *DefR = dyn_cast<RecipeTy>(R);
290 // Check for recipes that do not have opcodes.
291 if constexpr (std::is_same_v<RecipeTy, VPScalarIVStepsRecipe> ||
292 std::is_same_v<RecipeTy, VPDerivedIVRecipe> ||
293 std::is_same_v<RecipeTy, VPVectorEndPointerRecipe>)
294 return DefR;
295 else
296 return DefR && DefR->getOpcode() == Opcode;
297 }
298
299 /// Helper to check if predicate \p P holds on all tuple elements in Ops using
300 /// the provided index sequence.
301 template <typename Fn, std::size_t... Is>
302 bool all_of_tuple_elements(std::index_sequence<Is...>,
303 [[maybe_unused]] Fn P) const {
304 return (P(std::get<Is>(Ops), Is) && ...);
305 }
306};
307
308template <unsigned Opcode, typename... OpTys>
309using AllRecipe_match =
310 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ false,
311 VPWidenRecipe, VPReplicateRecipe, VPWidenCastRecipe,
312 VPInstruction>;
313
314template <unsigned Opcode, typename... OpTys>
315using AllRecipe_commutative_match =
316 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ true,
317 VPWidenRecipe, VPReplicateRecipe, VPInstruction>;
318
319template <unsigned Opcode, typename... OpTys>
320using VPInstruction_match = Recipe_match<std::tuple<OpTys...>, Opcode,
321 /*Commutative*/ false, VPInstruction>;
322
323template <unsigned Opcode, typename... OpTys>
324using VPInstruction_commutative_match =
325 Recipe_match<std::tuple<OpTys...>, Opcode,
326 /*Commutative*/ true, VPInstruction>;
327
328template <unsigned Opcode, typename... OpTys>
329inline VPInstruction_match<Opcode, OpTys...>
330m_VPInstruction(const OpTys &...Ops) {
331 return VPInstruction_match<Opcode, OpTys...>(Ops...);
332}
333
334template <unsigned Opcode, typename Op0_t, typename Op1_t>
335inline VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>
336m_c_VPInstruction(const Op0_t &Op0, const Op1_t &Op1) {
337 return VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
338}
339
340/// BuildVector is matches only its opcode, w/o matching its operands as the
341/// number of operands is not fixed.
342inline VPInstruction_match<VPInstruction::BuildVector> m_BuildVector() {
343 return m_VPInstruction<VPInstruction::BuildVector>();
344}
345
346/// BuildStructVector matches only its opcode, w/o matching its operands as the
347/// number of operands is not fixed.
348inline VPInstruction_match<VPInstruction::BuildStructVector>
349m_BuildStructVector() {
350 return m_VPInstruction<VPInstruction::BuildStructVector>();
351}
352
353template <typename Op0_t>
354inline VPInstruction_match<Instruction::Freeze, Op0_t>
355m_Freeze(const Op0_t &Op0) {
356 return m_VPInstruction<Instruction::Freeze>(Op0);
357}
358
359inline VPInstruction_match<VPInstruction::BranchOnCond> m_BranchOnCond() {
360 return m_VPInstruction<VPInstruction::BranchOnCond>();
361}
362
363template <typename Op0_t>
364inline VPInstruction_match<VPInstruction::BranchOnCond, Op0_t>
365m_BranchOnCond(const Op0_t &Op0) {
366 return m_VPInstruction<VPInstruction::BranchOnCond>(Op0);
367}
368
369inline VPInstruction_match<VPInstruction::BranchOnTwoConds>
370m_BranchOnTwoConds() {
371 return m_VPInstruction<VPInstruction::BranchOnTwoConds>();
372}
373
374template <typename Op0_t, typename Op1_t>
375inline VPInstruction_match<VPInstruction::BranchOnTwoConds, Op0_t, Op1_t>
376m_BranchOnTwoConds(const Op0_t &Op0, const Op1_t &Op1) {
377 return m_VPInstruction<VPInstruction::BranchOnTwoConds>(Op0, Op1);
378}
379
380template <typename Op0_t>
381inline VPInstruction_match<VPInstruction::Broadcast, Op0_t>
382m_Broadcast(const Op0_t &Op0) {
383 return m_VPInstruction<VPInstruction::Broadcast>(Op0);
384}
385
386template <typename Op0_t>
387inline VPInstruction_match<VPInstruction::ExplicitVectorLength, Op0_t>
388m_EVL(const Op0_t &Op0) {
389 return m_VPInstruction<VPInstruction::ExplicitVectorLength>(Op0);
390}
391
392template <typename Op0_t>
393inline VPInstruction_match<VPInstruction::ExtractLastLane, Op0_t>
394m_ExtractLastLane(const Op0_t &Op0) {
395 return m_VPInstruction<VPInstruction::ExtractLastLane>(Op0);
396}
397
398template <typename Op0_t, typename Op1_t>
399inline VPInstruction_match<Instruction::ExtractElement, Op0_t, Op1_t>
400m_ExtractElement(const Op0_t &Op0, const Op1_t &Op1) {
401 return m_VPInstruction<Instruction::ExtractElement>(Op0, Op1);
402}
403
404template <typename Op0_t, typename Op1_t, typename Op2_t>
405inline VPInstruction_match<Instruction::InsertElement, Op0_t, Op1_t, Op2_t>
406m_InsertElement(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
407 return m_VPInstruction<Instruction::InsertElement>(Op0, Op1, Op2);
408}
409
410template <typename Op0_t, typename Op1_t>
411inline VPInstruction_match<VPInstruction::ExtractLane, Op0_t, Op1_t>
412m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1) {
413 return m_VPInstruction<VPInstruction::ExtractLane>(Op0, Op1);
414}
415
416template <typename Op0_t>
417inline VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>
418m_ExtractLastPart(const Op0_t &Op0) {
419 return m_VPInstruction<VPInstruction::ExtractLastPart>(Op0);
420}
421
422template <typename Op0_t>
423inline VPInstruction_match<
424 VPInstruction::ExtractLastLane,
425 VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>>
426m_ExtractLastLaneOfLastPart(const Op0_t &Op0) {
427 return m_ExtractLastLane(m_ExtractLastPart(Op0));
428}
429
430template <typename Op0_t>
431inline VPInstruction_match<VPInstruction::ExtractPenultimateElement, Op0_t>
432m_ExtractPenultimateElement(const Op0_t &Op0) {
433 return m_VPInstruction<VPInstruction::ExtractPenultimateElement>(Op0);
434}
435
436template <typename Op0_t, typename Op1_t, typename Op2_t>
437inline VPInstruction_match<VPInstruction::ActiveLaneMask, Op0_t, Op1_t, Op2_t>
438m_ActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
439 return m_VPInstruction<VPInstruction::ActiveLaneMask>(Op0, Op1, Op2);
440}
441
442inline VPInstruction_match<VPInstruction::BranchOnCount> m_BranchOnCount() {
443 return m_VPInstruction<VPInstruction::BranchOnCount>();
444}
445
446template <typename Op0_t, typename Op1_t>
447inline VPInstruction_match<VPInstruction::BranchOnCount, Op0_t, Op1_t>
448m_BranchOnCount(const Op0_t &Op0, const Op1_t &Op1) {
449 return m_VPInstruction<VPInstruction::BranchOnCount>(Op0, Op1);
450}
451
452inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() {
453 return m_VPInstruction<VPInstruction::AnyOf>();
454}
455
456template <typename Op0_t>
457inline VPInstruction_match<VPInstruction::AnyOf, Op0_t>
458m_AnyOf(const Op0_t &Op0) {
459 return m_VPInstruction<VPInstruction::AnyOf>(Op0);
460}
461
462template <typename Op0_t>
463inline VPInstruction_match<VPInstruction::FirstActiveLane, Op0_t>
464m_FirstActiveLane(const Op0_t &Op0) {
465 return m_VPInstruction<VPInstruction::FirstActiveLane>(Op0);
466}
467
468template <typename Op0_t>
469inline VPInstruction_match<VPInstruction::LastActiveLane, Op0_t>
470m_LastActiveLane(const Op0_t &Op0) {
471 return m_VPInstruction<VPInstruction::LastActiveLane>(Op0);
472}
473
474template <typename Op0_t, typename Op1_t, typename Op2_t>
475inline VPInstruction_match<VPInstruction::ExtractLastActive, Op0_t, Op1_t,
476 Op2_t>
477m_ExtractLastActive(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
478 return m_VPInstruction<VPInstruction::ExtractLastActive>(Op0, Op1, Op2);
479}
480
481template <typename Op0_t>
482inline VPInstruction_match<VPInstruction::ComputeReductionResult, Op0_t>
483m_ComputeReductionResult(const Op0_t &Op0) {
484 return m_VPInstruction<VPInstruction::ComputeReductionResult>(Op0);
485}
486
487/// Match FindIV result pattern:
488/// select(icmp ne ComputeReductionResult(ReducedIV), Sentinel),
489/// ComputeReductionResult(ReducedIV), Start.
490template <typename Op0_t, typename Op1_t>
491inline bool matchFindIVResult(VPInstruction *VPI, Op0_t ReducedIV, Op1_t Start) {
492 return match(VPI, m_Select(m_SpecificICmp(ICmpInst::ICMP_NE,
493 m_ComputeReductionResult(ReducedIV),
494 m_VPValue()),
495 m_ComputeReductionResult(ReducedIV), Start));
496}
497
498template <typename Op0_t>
499inline VPInstruction_match<VPInstruction::Reverse, Op0_t>
500m_Reverse(const Op0_t &Op0) {
501 return m_VPInstruction<VPInstruction::Reverse>(Op0);
502}
503
504inline VPInstruction_match<VPInstruction::StepVector> m_StepVector() {
505 return m_VPInstruction<VPInstruction::StepVector>();
506}
507
508template <typename Op0_t>
509inline VPInstruction_match<VPInstruction::ExitingIVValue, Op0_t>
510m_ExitingIVValue(const Op0_t &Op0) {
511 return m_VPInstruction<VPInstruction::ExitingIVValue>(Op0);
512}
513
514template <unsigned Opcode, typename Op0_t>
515inline AllRecipe_match<Opcode, Op0_t> m_Unary(const Op0_t &Op0) {
516 return AllRecipe_match<Opcode, Op0_t>(Op0);
517}
518
519template <typename Op0_t>
520inline AllRecipe_match<Instruction::Trunc, Op0_t> m_Trunc(const Op0_t &Op0) {
521 return m_Unary<Instruction::Trunc, Op0_t>(Op0);
522}
523
524template <typename Op0_t>
525inline match_combine_or<AllRecipe_match<Instruction::Trunc, Op0_t>, Op0_t>
526m_TruncOrSelf(const Op0_t &Op0) {
527 return m_CombineOr(m_Trunc(Op0), Op0);
528}
529
530template <typename Op0_t>
531inline AllRecipe_match<Instruction::ZExt, Op0_t> m_ZExt(const Op0_t &Op0) {
532 return m_Unary<Instruction::ZExt, Op0_t>(Op0);
533}
534
535template <typename Op0_t>
536inline AllRecipe_match<Instruction::SExt, Op0_t> m_SExt(const Op0_t &Op0) {
537 return m_Unary<Instruction::SExt, Op0_t>(Op0);
538}
539
540template <typename Op0_t>
541inline AllRecipe_match<Instruction::FPExt, Op0_t> m_FPExt(const Op0_t &Op0) {
542 return m_Unary<Instruction::FPExt, Op0_t>(Op0);
543}
544
545template <typename Op0_t>
546inline AllRecipe_match<Instruction::FNeg, Op0_t> m_FNeg(const Op0_t &Op0) {
547 return m_Unary<Instruction::FNeg, Op0_t>(Op0);
548}
549
550template <typename Op0_t>
551inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>,
552 AllRecipe_match<Instruction::SExt, Op0_t>>
553m_ZExtOrSExt(const Op0_t &Op0) {
554 return m_CombineOr(m_ZExt(Op0), m_SExt(Op0));
555}
556
557template <typename Op0_t> inline auto m_WidenAnyExtend(const Op0_t &Op0) {
558 return m_Isa<VPWidenCastRecipe>(m_CombineOr(m_ZExtOrSExt(Op0), m_FPExt(Op0)));
559}
560
561template <typename Op0_t> inline auto m_AnyNeg(const Op0_t &Op0) {
562 return m_CombineOr(m_Sub(m_ZeroInt(), Op0), m_FNeg(Op0));
563}
564
565template <typename Op0_t>
566inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>, Op0_t>
567m_ZExtOrSelf(const Op0_t &Op0) {
568 return m_CombineOr(m_ZExt(Op0), Op0);
569}
570
571template <typename Op0_t> inline auto m_ZExtOrTruncOrSelf(const Op0_t &Op0) {
572 return m_CombineOr(m_ZExt(Op0), m_Trunc(Op0), Op0);
573}
574
575template <unsigned Opcode, typename Op0_t, typename Op1_t>
576inline AllRecipe_match<Opcode, Op0_t, Op1_t> m_Binary(const Op0_t &Op0,
577 const Op1_t &Op1) {
578 return AllRecipe_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
579}
580
581template <unsigned Opcode, typename Op0_t, typename Op1_t>
582inline AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>
583m_c_Binary(const Op0_t &Op0, const Op1_t &Op1) {
584 return AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
585}
586
587template <typename Op0_t, typename Op1_t>
588inline AllRecipe_match<Instruction::Add, Op0_t, Op1_t> m_Add(const Op0_t &Op0,
589 const Op1_t &Op1) {
590 return m_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
591}
592
593template <typename Op0_t, typename Op1_t>
594inline AllRecipe_commutative_match<Instruction::Add, Op0_t, Op1_t>
595m_c_Add(const Op0_t &Op0, const Op1_t &Op1) {
596 return m_c_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
597}
598
599template <typename Op0_t, typename Op1_t>
600inline AllRecipe_match<Instruction::Sub, Op0_t, Op1_t> m_Sub(const Op0_t &Op0,
601 const Op1_t &Op1) {
602 return m_Binary<Instruction::Sub, Op0_t, Op1_t>(Op0, Op1);
603}
604
605template <typename Op0_t, typename Op1_t>
606inline AllRecipe_match<Instruction::Mul, Op0_t, Op1_t> m_Mul(const Op0_t &Op0,
607 const Op1_t &Op1) {
608 return m_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
609}
610
611template <typename Op0_t, typename Op1_t>
612inline AllRecipe_commutative_match<Instruction::Mul, Op0_t, Op1_t>
613m_c_Mul(const Op0_t &Op0, const Op1_t &Op1) {
614 return m_c_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
615}
616
617template <typename Op0_t, typename Op1_t>
618inline AllRecipe_match<Instruction::Shl, Op0_t, Op1_t> m_Shl(const Op0_t &Op0,
619 const Op1_t &Op1) {
620 return m_Binary<Instruction::Shl, Op0_t, Op1_t>(Op0, Op1);
621}
622
623template <typename Op0_t, typename Op1_t>
624inline AllRecipe_match<Instruction::LShr, Op0_t, Op1_t>
625m_LShr(const Op0_t &Op0, const Op1_t &Op1) {
626 return m_Binary<Instruction::LShr, Op0_t, Op1_t>(Op0, Op1);
627}
628
629template <typename Op0_t, typename Op1_t>
630inline AllRecipe_match<Instruction::FMul, Op0_t, Op1_t>
631m_FMul(const Op0_t &Op0, const Op1_t &Op1) {
632 return m_Binary<Instruction::FMul, Op0_t, Op1_t>(Op0, Op1);
633}
634
635template <typename Op0_t, typename Op1_t>
636inline AllRecipe_match<Instruction::FAdd, Op0_t, Op1_t>
637m_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
638 return m_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
639}
640
641template <typename Op0_t, typename Op1_t>
642inline AllRecipe_commutative_match<Instruction::FAdd, Op0_t, Op1_t>
643m_c_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
644 return m_c_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
645}
646
647template <typename Op0_t, typename Op1_t>
648inline AllRecipe_match<Instruction::UDiv, Op0_t, Op1_t>
649m_UDiv(const Op0_t &Op0, const Op1_t &Op1) {
650 return m_Binary<Instruction::UDiv, Op0_t, Op1_t>(Op0, Op1);
651}
652
653template <typename Op0_t, typename Op1_t>
654inline AllRecipe_match<Instruction::URem, Op0_t, Op1_t>
655m_URem(const Op0_t &Op0, const Op1_t &Op1) {
656 return m_Binary<Instruction::URem, Op0_t, Op1_t>(Op0, Op1);
657}
658
659template <typename Op0_t, typename Op1_t>
660inline AllRecipe_match<Instruction::SRem, Op0_t, Op1_t>
661m_SRem(const Op0_t &Op0, const Op1_t &Op1) {
662 return m_Binary<Instruction::SRem, Op0_t, Op1_t>(Op0, Op1);
663}
664
665/// Match a binary AND operation.
666template <typename Op0_t, typename Op1_t>
667inline AllRecipe_commutative_match<Instruction::And, Op0_t, Op1_t>
668m_c_BinaryAnd(const Op0_t &Op0, const Op1_t &Op1) {
669 return m_c_Binary<Instruction::And, Op0_t, Op1_t>(Op0, Op1);
670}
671
672/// Match a binary OR operation. Note that while conceptually the operands can
673/// be matched commutatively, \p Commutative defaults to false in line with the
674/// IR-based pattern matching infrastructure. Use m_c_BinaryOr for a commutative
675/// version of the matcher.
676template <typename Op0_t, typename Op1_t>
677inline AllRecipe_match<Instruction::Or, Op0_t, Op1_t>
678m_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
679 return m_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
680}
681
682template <typename Op0_t, typename Op1_t>
683inline AllRecipe_commutative_match<Instruction::Or, Op0_t, Op1_t>
684m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
685 return m_c_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
686}
687
688/// Cmp_match is a variant of BinaryRecipe_match that also binds the comparison
689/// predicate. Opcodes must either be Instruction::ICmp or Instruction::FCmp, or
690/// both.
691template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
692struct Cmp_match {
693 static_assert((sizeof...(Opcodes) == 1 || sizeof...(Opcodes) == 2) &&
694 "Expected one or two opcodes");
695 static_assert(
696 ((Opcodes == Instruction::ICmp || Opcodes == Instruction::FCmp) && ...) &&
697 "Expected a compare instruction opcode");
698
699 CmpPredicate *Predicate = nullptr;
700 Op0_t Op0;
701 Op1_t Op1;
702
703 Cmp_match(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1)
704 : Predicate(&Pred), Op0(Op0), Op1(Op1) {}
705 Cmp_match(const Op0_t &Op0, const Op1_t &Op1) : Op0(Op0), Op1(Op1) {}
706
707 bool match(const VPValue *V) const {
708 auto *DefR = V->getDefiningRecipe();
709 return DefR && match(DefR);
710 }
711
712 bool match(const VPRecipeBase *V) const {
713 if ((m_Binary<Opcodes>(Op0, Op1).match(V) || ...)) {
714 if (Predicate)
715 *Predicate = cast<VPRecipeWithIRFlags>(Val: V)->getPredicate();
716 return true;
717 }
718 return false;
719 }
720};
721
722/// SpecificCmp_match is a variant of Cmp_match that matches the comparison
723/// predicate, instead of binding it.
724template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
725struct SpecificCmp_match {
726 const CmpPredicate Predicate;
727 Op0_t Op0;
728 Op1_t Op1;
729
730 SpecificCmp_match(CmpPredicate Pred, const Op0_t &LHS, const Op1_t &RHS)
731 : Predicate(Pred), Op0(LHS), Op1(RHS) {}
732
733 bool match(const VPValue *V) const {
734 auto *DefR = V->getDefiningRecipe();
735 return DefR && match(DefR);
736 }
737
738 bool match(const VPRecipeBase *V) const {
739 CmpPredicate CurrentPred;
740 return Cmp_match<Op0_t, Op1_t, Opcodes...>(CurrentPred, Op0, Op1)
741 .match(V) &&
742 CmpPredicate::getMatching(A: CurrentPred, B: Predicate);
743 }
744};
745
746template <typename Op0_t, typename Op1_t>
747inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp> m_ICmp(const Op0_t &Op0,
748 const Op1_t &Op1) {
749 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Op0, Op1);
750}
751
752template <typename Op0_t, typename Op1_t>
753inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp>
754m_ICmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
755 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Pred, Op0, Op1);
756}
757
758template <typename Op0_t, typename Op1_t>
759inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>
760m_SpecificICmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
761 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>(MatchPred, Op0,
762 Op1);
763}
764
765template <typename Op0_t, typename Op1_t>
766inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
767m_Cmp(const Op0_t &Op0, const Op1_t &Op1) {
768 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(Op0,
769 Op1);
770}
771
772template <typename Op0_t, typename Op1_t>
773inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
774m_Cmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
775 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
776 Pred, Op0, Op1);
777}
778
779template <typename Op0_t, typename Op1_t>
780inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
781m_SpecificCmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
782 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
783 MatchPred, Op0, Op1);
784}
785
786template <typename Op0_t, typename Op1_t>
787inline auto m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1) {
788 return m_CombineOr(
789 Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr,
790 /*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>(
791 Op0, Op1),
792 VPInstruction_match<VPInstruction::PtrAdd, Op0_t, Op1_t>(Op0, Op1),
793 VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>(Op0, Op1));
794}
795
796/// Match a VPBlendRecipe with 2 incoming values ([I0, I1, M1] ==
797/// normalized([I0, M0, I1, M1])) as select(M1, I1, I0), mirroring how it is
798/// lowered.
799template <typename Op0_t, typename Op1_t, typename Op2_t> struct Blend2_match {
800 Op0_t MaskOp;
801 Op1_t TrueOp;
802 Op2_t FalseOp;
803
804 Blend2_match(const Op0_t &MaskOp, const Op1_t &TrueOp, const Op2_t &FalseOp)
805 : MaskOp(MaskOp), TrueOp(TrueOp), FalseOp(FalseOp) {}
806
807 template <typename T> bool match(const T *Val) const {
808 auto *Blend = dyn_cast<VPBlendRecipe>(Val);
809 if (!Blend || Blend->getNumIncomingValues() != 2)
810 return false;
811 return MaskOp.match(Blend->getMask(1)) &&
812 TrueOp.match(Blend->getIncomingValue(1)) &&
813 FalseOp.match(Blend->getIncomingValue(0));
814 }
815};
816
817/// Match recipe recipe with Select opcode, i.e. excluding VPBlendRecipe.
818template <typename Op0_t, typename Op1_t, typename Op2_t>
819inline AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>
820m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
821 return AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>(
822 {Op0, Op1, Op2});
823}
824
825/// Match recipe with Select opcode or an equivalent VPBlendRecipe with 2
826/// incoming values.
827template <typename Op0_t, typename Op1_t, typename Op2_t>
828inline auto m_SelectLike(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
829 return m_CombineOr(m_Select(Op0, Op1, Op2),
830 Blend2_match<Op0_t, Op1_t, Op2_t>(Op0, Op1, Op2));
831}
832
833template <typename Op0_t> inline auto m_Not(const Op0_t &Op0) {
834 return m_CombineOr(m_VPInstruction<VPInstruction::Not>(Op0),
835 m_c_Binary<Instruction::Xor>(m_AllOnes(), Op0));
836}
837
838template <typename Op0_t, typename Op1_t, typename Op2_t>
839inline auto m_c_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
840 return m_CombineOr(m_Select(Op0, Op1, Op2), m_Select(m_Not(Op0), Op2, Op1));
841}
842
843template <typename Op0_t, typename Op1_t>
844inline auto m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
845 return m_CombineOr(
846 m_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
847 m_Select(Op0, Op1, m_False()));
848}
849
850template <typename Op0_t, typename Op1_t>
851inline auto m_c_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
852 return m_CombineOr(
853 m_c_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
854 m_c_Select(Op0, Op1, m_False()));
855}
856
857template <typename Op0_t, typename Op1_t>
858inline auto m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
859 return m_CombineOr(
860 m_c_VPInstruction<VPInstruction::LogicalOr, Op0_t, Op1_t>(Op0, Op1),
861 m_Select(Op0, m_True(), Op1));
862}
863
864template <typename Op0_t, typename Op1_t>
865inline auto m_c_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
866 return m_c_Select(Op0, m_True(), Op1);
867}
868
869/// Match the canonical induction variable (IV) of any loop region.
870struct canonical_iv_match {
871 template <typename ArgTy> bool match(const ArgTy *V) const {
872 const auto *RV = dyn_cast<VPRegionValue>(V);
873 return RV && RV->getDefiningRegion()->getCanonicalIV() == RV;
874 }
875};
876
877inline canonical_iv_match m_CanonicalIV() { return {}; }
878
879/// Match a canonical VPWidenIntOrFpInductionRecipe optionally capturing it.
880struct canonical_widen_iv_match {
881 VPWidenIntOrFpInductionRecipe **Capture = nullptr;
882
883 canonical_widen_iv_match() = default;
884 canonical_widen_iv_match(VPWidenIntOrFpInductionRecipe *&V) : Capture(&V) {}
885
886 template <typename ArgTy> bool match(ArgTy *V) const {
887 auto *WidenIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(V);
888 if (!WidenIV || !WidenIV->isCanonical())
889 return false;
890 if (Capture)
891 *Capture = WidenIV;
892 return true;
893 }
894};
895
896inline canonical_widen_iv_match m_CanonicalWidenIV() { return {}; }
897
898/// Match a canonical VPWidenIntOrFpInductionRecipe, capturing it.
899inline canonical_widen_iv_match
900m_CanonicalWidenIV(VPWidenIntOrFpInductionRecipe *&V) {
901 return canonical_widen_iv_match(V);
902}
903
904template <typename Op0_t, typename Op1_t, typename Op2_t>
905inline auto m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1,
906 const Op2_t &Op2) {
907 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
908 VPScalarIVStepsRecipe>({Op0, Op1, Op2});
909}
910
911template <typename Op0_t, typename Op1_t, typename Op2_t>
912inline auto m_DerivedIV(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
913 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
914 VPDerivedIVRecipe>({Op0, Op1, Op2});
915}
916
917template <typename Addr_t, typename Mask_t> struct Load_match {
918 Addr_t Addr;
919 Mask_t Mask;
920
921 Load_match(Addr_t Addr, Mask_t Mask) : Addr(Addr), Mask(Mask) {}
922
923 template <typename OpTy> bool match(const OpTy *V) const {
924 auto *Load = dyn_cast<VPWidenLoadRecipe>(V);
925 if (!Load || !Addr.match(Load->getAddr()) || !Load->isMasked() ||
926 !Mask.match(Load->getMask()))
927 return false;
928 return true;
929 }
930};
931
932/// Match a (possibly reversed) masked load.
933template <typename Addr_t, typename Mask_t>
934inline Load_match<Addr_t, Mask_t> m_MaskedLoad(const Addr_t &Addr,
935 const Mask_t &Mask) {
936 return Load_match<Addr_t, Mask_t>(Addr, Mask);
937}
938
939template <typename Addr_t, typename Val_t, typename Mask_t> struct Store_match {
940 Addr_t Addr;
941 Val_t Val;
942 Mask_t Mask;
943
944 Store_match(Addr_t Addr, Val_t Val, Mask_t Mask)
945 : Addr(Addr), Val(Val), Mask(Mask) {}
946
947 template <typename OpTy> bool match(const OpTy *V) const {
948 auto *Store = dyn_cast<VPWidenStoreRecipe>(V);
949 if (!Store || !Addr.match(Store->getAddr()) ||
950 !Val.match(Store->getStoredValue()) || !Store->isMasked() ||
951 !Mask.match(Store->getMask()))
952 return false;
953 return true;
954 }
955};
956
957/// Match a (possibly reversed) masked store.
958template <typename Addr_t, typename Val_t, typename Mask_t>
959inline Store_match<Addr_t, Val_t, Mask_t>
960m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) {
961 return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask);
962}
963
964template <typename Op0_t, typename Op1_t>
965using VectorEndPointerRecipe_match =
966 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
967 /*Commutative*/ false, VPVectorEndPointerRecipe>;
968
969template <typename Op0_t, typename Op1_t>
970VectorEndPointerRecipe_match<Op0_t, Op1_t> m_VecEndPtr(const Op0_t &Op0,
971 const Op1_t &Op1) {
972 return VectorEndPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
973}
974
975/// Match a call argument at a given argument index.
976template <typename Opnd_t> struct Argument_match {
977 /// Call argument index to match.
978 unsigned OpI;
979 Opnd_t Val;
980
981 Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}
982
983 template <typename OpTy> bool match(OpTy *V) const {
984 if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V))
985 return Val.match(R->getOperand(OpI));
986 if (const auto *R = dyn_cast<VPWidenCallRecipe>(V))
987 return Val.match(R->getOperand(OpI));
988 if (const auto *R = dyn_cast<VPReplicateRecipe>(V))
989 if (R->getOpcode() == Instruction::Call)
990 return Val.match(R->getOperand(OpI));
991 if (const auto *R = dyn_cast<VPInstruction>(V))
992 if (R->getOpcode() == Instruction::Call)
993 return Val.match(R->getOperand(OpI));
994 return false;
995 }
996};
997
998/// Match a call argument.
999template <unsigned OpI, typename Opnd_t>
1000inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) {
1001 return Argument_match<Opnd_t>(OpI, Op);
1002}
1003
1004/// Intrinsic matchers.
1005struct IntrinsicID_match {
1006 unsigned ID;
1007
1008 IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {}
1009
1010 template <typename OpTy> bool match(OpTy *V) const {
1011 return vputils::getIntrinsicID(V) == ID;
1012 }
1013};
1014
1015/// Intrinsic matches are combinations of ID matchers, and argument
1016/// matchers. Higher arity matcher are defined recursively in terms of and-ing
1017/// them with lower arity matchers. Here's some convenient typedefs for up to
1018/// several arguments, and more can be added as needed
1019template <typename T0 = void, typename T1 = void, typename T2 = void,
1020 typename T3 = void>
1021struct m_Intrinsic_Ty;
1022template <typename T0> struct m_Intrinsic_Ty<T0> {
1023 using Ty = match_combine_and<IntrinsicID_match, Argument_match<T0>>;
1024};
1025template <typename T0, typename T1> struct m_Intrinsic_Ty<T0, T1> {
1026 using Ty =
1027 match_combine_and<typename m_Intrinsic_Ty<T0>::Ty, Argument_match<T1>>;
1028};
1029template <typename T0, typename T1, typename T2>
1030struct m_Intrinsic_Ty<T0, T1, T2> {
1031 using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1>::Ty,
1032 Argument_match<T2>>;
1033};
1034template <typename T0, typename T1, typename T2, typename T3>
1035struct m_Intrinsic_Ty {
1036 using Ty = match_combine_and<typename m_Intrinsic_Ty<T0, T1, T2>::Ty,
1037 Argument_match<T3>>;
1038};
1039
1040/// Match intrinsic calls like this:
1041/// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...)
1042template <Intrinsic::ID IntrID> inline IntrinsicID_match m_Intrinsic() {
1043 return IntrinsicID_match(IntrID);
1044}
1045
1046/// Match intrinsic calls with a runtime intrinsic ID.
1047inline IntrinsicID_match m_Intrinsic(Intrinsic::ID IntrID) {
1048 return IntrinsicID_match(IntrID);
1049}
1050
1051template <Intrinsic::ID IntrID, typename T0>
1052inline typename m_Intrinsic_Ty<T0>::Ty m_Intrinsic(const T0 &Op0) {
1053 return m_CombineAnd(m_Intrinsic<IntrID>(), m_Argument<0>(Op0));
1054}
1055
1056template <Intrinsic::ID IntrID, typename T0, typename T1>
1057inline typename m_Intrinsic_Ty<T0, T1>::Ty m_Intrinsic(const T0 &Op0,
1058 const T1 &Op1) {
1059 return m_CombineAnd(m_Intrinsic<IntrID>(Op0), m_Argument<1>(Op1));
1060}
1061
1062template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2>
1063inline typename m_Intrinsic_Ty<T0, T1, T2>::Ty
1064m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2) {
1065 return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1), m_Argument<2>(Op2));
1066}
1067
1068template <Intrinsic::ID IntrID, typename T0, typename T1, typename T2,
1069 typename T3>
1070inline typename m_Intrinsic_Ty<T0, T1, T2, T3>::Ty
1071m_Intrinsic(const T0 &Op0, const T1 &Op1, const T2 &Op2, const T3 &Op3) {
1072 return m_CombineAnd(m_Intrinsic<IntrID>(Op0, Op1, Op2), m_Argument<3>(Op3));
1073}
1074
1075template <Intrinsic::ID IntrID, typename... T>
1076inline auto m_WidenIntrinsic(const T &...Ops) {
1077 return m_Isa<VPWidenIntrinsicRecipe>(m_Intrinsic<IntrID>(Ops...));
1078}
1079
1080inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
1081
1082/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
1083/// and bind the source element type and operands.
1084struct GetElementPtr_match {
1085 Type *&SourceElementType;
1086 ArrayRef<VPValue *> &Operands;
1087
1088 GetElementPtr_match(Type *&SourceElementType, ArrayRef<VPValue *> &Operands)
1089 : SourceElementType(SourceElementType), Operands(Operands) {}
1090
1091 template <typename ITy> bool match(ITy *V) const {
1092 return matchRecipeAndBind<VPWidenGEPRecipe>(V) ||
1093 matchRecipeAndBind<VPInstruction>(V) ||
1094 matchRecipeAndBind<VPReplicateRecipe>(V);
1095 }
1096
1097private:
1098 template <typename RecipeTy> bool matchRecipeAndBind(const VPValue *V) const {
1099 auto *DefR = dyn_cast<RecipeTy>(V);
1100 if (!DefR)
1101 return false;
1102
1103 if constexpr (std::is_same_v<RecipeTy, VPWidenGEPRecipe>) {
1104 SourceElementType = DefR->getSourceElementType();
1105 } else if (DefR->getOpcode() == Instruction::GetElementPtr) {
1106 SourceElementType = cast<GetElementPtrInst>(DefR->getUnderlyingInstr())
1107 ->getSourceElementType();
1108 } else if constexpr (std::is_same_v<RecipeTy, VPInstruction>) {
1109 if (DefR->getOpcode() == VPInstruction::PtrAdd) {
1110 // PtrAdd is a byte-offset GEP with i8 element type.
1111 LLVMContext &Ctx = DefR->getParent()->getPlan()->getContext();
1112 SourceElementType = Type::getInt8Ty(C&: Ctx);
1113 } else {
1114 return false;
1115 }
1116 } else {
1117 return false;
1118 }
1119
1120 Operands = ArrayRef<VPValue *>(DefR->op_begin(), DefR->op_end());
1121 return true;
1122 }
1123};
1124
1125/// Match a GEP recipe with any number of operands and bind source element type
1126/// and operands.
1127inline GetElementPtr_match m_GetElementPtr(Type *&SourceElementType,
1128 ArrayRef<VPValue *> &Operands) {
1129 return GetElementPtr_match(SourceElementType, Operands);
1130}
1131
1132template <typename SubPattern_t> struct OneUse_match {
1133 SubPattern_t SubPattern;
1134
1135 OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}
1136
1137 template <typename OpTy> bool match(OpTy *V) const {
1138 return V->hasOneUse() && SubPattern.match(V);
1139 }
1140};
1141
1142template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) {
1143 return SubPattern;
1144}
1145
1146inline match_bind<VPReductionPHIRecipe>
1147m_ReductionPhi(VPReductionPHIRecipe *&V) {
1148 return V;
1149}
1150
1151template <typename Op0_t, typename Op1_t>
1152inline auto m_VPPhi(const Op0_t &Op0, const Op1_t &Op1) {
1153 return Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::PHI,
1154 /*Commutative*/ false, VPInstruction>({Op0, Op1});
1155}
1156
1157/// If \p V is used by a recipe matching pattern \p P, return it. Otherwise
1158/// return nullptr;
1159template <typename MatchT>
1160VPRecipeBase *findUserOf(VPValue *V, const MatchT &P) {
1161 auto It = find_if(V->users(), match_fn(P));
1162 return It == V->user_end() ? nullptr : cast<VPRecipeBase>(*It);
1163}
1164
1165/// If \p V is used by a VPInstruction with \p Opcode, return it. Otherwise
1166/// return nullptr.
1167template <unsigned Opcode> VPInstruction *findUserOf(VPValue *V) {
1168 return cast_or_null<VPInstruction>(findUserOf(V, m_VPInstruction<Opcode>()));
1169}
1170
1171template <typename RecipeTy> RecipeTy *findUserOf(VPValue *V) {
1172 return cast_or_null<RecipeTy>(findUserOf(V, m_Isa<RecipeTy>()));
1173}
1174} // namespace llvm::VPlanPatternMatch
1175
1176#endif
1177