1//===- VPlanPatternMatch.h - Match on VPValues and recipes ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides a simple and efficient mechanism for performing general
10// tree-based pattern matches on the VPlan values and recipes, based on
11// LLVM's IR pattern matchers.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
16#define LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
17
18#include "VPlan.h"
19#include "VPlanUtils.h"
20#include "llvm/Support/PatternMatchHelpers.h"
21#include <utility>
22
23namespace llvm::VPlanPatternMatch {
24
25using namespace llvm::PatternMatchHelpers;
26
27template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
28 return P.match(V);
29}
30
31template <typename Pattern> bool match(VPUser *U, const Pattern &P) {
32 auto *R = dyn_cast<VPRecipeBase>(Val: U);
33 return R && match(R, P);
34}
35
36template <typename Pattern> bool match(VPSingleDefRecipe *R, const Pattern &P) {
37 return P.match(static_cast<const VPRecipeBase *>(R));
38}
39
40/// A match functor that can be used as a UnaryPredicate in functional
41/// algorithms like all_of.
42template <typename Pattern> auto match_fn(const Pattern &P) {
43 return [&P](auto *V) { return match(V, P); };
44}
45
46/// Match an arbitrary VPValue and ignore it.
47inline auto m_VPValue() { return m_Isa<VPValue>(); }
48
49/// Match a specified VPValue.
50struct specificval_ty {
51 const VPValue *Val;
52
53 specificval_ty(const VPValue *V) : Val(V) {}
54
55 bool match(const VPValue *VPV) const { return VPV == Val; }
56};
57
58inline specificval_ty m_Specific(const VPValue *VPV) { return VPV; }
59
60/// Like m_Specific(), but works if the specific value to match is determined
61/// as part of the same match() expression. For example:
62/// m_Mul(m_VPValue(X), m_Specific(X)) is incorrect, because m_Specific() will
63/// bind X before the pattern match starts.
64/// m_Mul(m_VPValue(X), m_Deferred(X)) is correct, and will check against
65/// whichever value m_VPValue(X) populated.
66inline match_deferred<VPValue> m_Deferred(VPValue *const &V) { return V; }
67
68/// Match an integer constant if Pred::isValue returns true for the APInt. \p
69/// BitWidth optionally specifies the bitwidth the matched constant must have.
70/// If it is 0, the matched constant can have any bitwidth.
71template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty {
72 Pred P;
73
74 int_pred_ty(Pred P) : P(std::move(P)) {}
75 int_pred_ty() : P() {}
76
77 bool match(const VPValue *VPV) const {
78 auto *VPI = dyn_cast<VPInstruction>(Val: VPV);
79 if (VPI && VPI->getOpcode() == VPInstruction::Broadcast)
80 VPV = VPI->getOperand(N: 0);
81 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
82 if (!CI)
83 return false;
84
85 if (BitWidth != 0 && CI->getBitWidth() != BitWidth)
86 return false;
87 return P.isValue(CI->getAPInt());
88 }
89};
90
91/// Match a specified signed or unsigned integer value.
92struct is_specific_int {
93 APInt Val;
94 bool IsSigned;
95
96 is_specific_int(APInt Val, bool IsSigned = false)
97 : Val(std::move(Val)), IsSigned(IsSigned) {}
98
99 bool isValue(const APInt &C) const {
100 return APInt::isSameValue(I1: Val, I2: C, SignedCompare: IsSigned);
101 }
102};
103
104template <unsigned Bitwidth = 0>
105using specific_intval = int_pred_ty<is_specific_int, Bitwidth>;
106
107inline specific_intval<0> m_SpecificInt(uint64_t V) {
108 return specific_intval<0>(is_specific_int(APInt(64, V)));
109}
110
111inline specific_intval<0> m_SpecificSInt(int64_t V) {
112 return specific_intval<0>(
113 is_specific_int(APInt(64, V, /*isSigned=*/true), /*IsSigned=*/true));
114}
115
116inline specific_intval<1> m_False() {
117 return specific_intval<1>(is_specific_int(APInt(64, 0)));
118}
119
120inline specific_intval<1> m_True() {
121 return specific_intval<1>(is_specific_int(APInt(64, 1)));
122}
123
124struct is_all_ones {
125 bool isValue(const APInt &C) const { return C.isAllOnes(); }
126};
127
128/// Match an integer or vector with all bits set.
129/// For vectors, this includes constants with undefined elements.
130inline int_pred_ty<is_all_ones> m_AllOnes() {
131 return int_pred_ty<is_all_ones>();
132}
133
134struct is_zero_int {
135 bool isValue(const APInt &C) const { return C.isZero(); }
136};
137
138struct is_one {
139 bool isValue(const APInt &C) const { return C.isOne(); }
140};
141
142/// Match an integer 0 or a vector with all elements equal to 0.
143/// For vectors, this includes constants with undefined elements.
144inline int_pred_ty<is_zero_int> m_ZeroInt() {
145 return int_pred_ty<is_zero_int>();
146}
147
148/// Match an integer 1 or a vector with all elements equal to 1.
149/// For vectors, this includes constants with undefined elements.
150inline int_pred_ty<is_one> m_One() { return int_pred_ty<is_one>(); }
151
152struct bind_apint {
153 const APInt *&Res;
154
155 bind_apint(const APInt *&Res) : Res(Res) {}
156
157 bool match(const VPValue *VPV) const {
158 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
159 if (!CI)
160 return false;
161 Res = &CI->getAPInt();
162 return true;
163 }
164};
165
166inline bind_apint m_APInt(const APInt *&C) { return C; }
167
168struct bind_const_int {
169 uint64_t &Res;
170
171 bind_const_int(uint64_t &Res) : Res(Res) {}
172
173 bool match(const VPValue *VPV) const {
174 const APInt *APConst;
175 if (!bind_apint(APConst).match(VPV))
176 return false;
177 if (auto C = APConst->tryZExtValue()) {
178 Res = *C;
179 return true;
180 }
181 return false;
182 }
183};
184
185struct match_poison {
186 bool match(const VPValue *V) const {
187 return isa<VPIRValue>(Val: V) &&
188 isa<PoisonValue>(Val: cast<VPIRValue>(Val: V)->getValue());
189 }
190};
191
192/// Match a VPIRValue that's poison.
193inline match_poison m_Poison() { return match_poison(); }
194
195/// Match a plain integer constant no wider than 64-bits, capturing it if we
196/// match.
197inline bind_const_int m_ConstantInt(uint64_t &C) { return C; }
198
199/// Match a VPValue, capturing it if we match.
200inline match_bind<VPValue> m_VPValue(VPValue *&V) { return V; }
201
202/// Match against the nested pattern, and capture the value if we match.
203template <typename Op_t> inline auto m_VPValue(VPValue *&V, const Op_t &Op) {
204 return m_CombineAnd(Op, m_VPValue(V));
205}
206
207/// Match a VPIRValue.
208inline match_bind<VPIRValue> m_VPIRValue(VPIRValue *&V) { return V; }
209
210/// Match a VPSingleDefRecipe, capturing if we match.
211inline match_bind<VPSingleDefRecipe>
212m_VPSingleDefRecipe(VPSingleDefRecipe *&V) {
213 return V;
214}
215
216/// Match a VPInstruction, capturing if we match.
217inline match_bind<VPInstruction> m_VPInstruction(VPInstruction *&V) {
218 return V;
219}
220
221template <typename Ops_t, unsigned Opcode, bool Commutative,
222 typename... RecipeTys>
223struct Recipe_match {
224 Ops_t Ops;
225
226 template <typename... OpTy> Recipe_match(OpTy... Ops) : Ops(Ops...) {
227 static_assert(std::tuple_size<Ops_t>::value == sizeof...(Ops) &&
228 "number of operands in constructor doesn't match Ops_t");
229 static_assert((!Commutative || std::tuple_size<Ops_t>::value == 2) &&
230 "only binary ops can be commutative");
231 }
232
233 bool match(const VPValue *V) const {
234 auto *DefR = V->getDefiningRecipe();
235 return DefR && match(DefR);
236 }
237
238 bool match(const VPSingleDefRecipe *R) const {
239 return match(static_cast<const VPRecipeBase *>(R));
240 }
241
242 bool match(const VPRecipeBase *R) const {
243 if (std::tuple_size_v<Ops_t> == 0) {
244 auto *VPI = dyn_cast<VPInstruction>(Val: R);
245 return VPI && VPI->getOpcode() == Opcode;
246 }
247
248 if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...))
249 return false;
250
251 if (R->getNumOperands() < std::tuple_size<Ops_t>::value) {
252 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R);
253 assert(((isa<VPInstruction>(R) &&
254 cast<VPInstruction>(R)->getNumOperandsForOpcode() == -1u) ||
255 (RepR && std::tuple_size_v<Ops_t> ==
256 RepR->getNumOperandsWithoutMask())) &&
257 "non-variadic recipe with matched opcode does not have the "
258 "expected number of operands");
259 return false;
260 }
261
262 // If the recipe has more operands than expected, we only support matching
263 // masked VPInstructions or predicated VPReplicateRecipes, where the number
264 // of operands of the matcher matches the number of operands excluding the
265 // mask.
266 if (R->getNumOperands() > std::tuple_size<Ops_t>::value) {
267 if (auto *VPI = dyn_cast<VPInstruction>(Val: R)) {
268 if (!VPI->isMasked() ||
269 VPI->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
270 return false;
271 } else if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R)) {
272 if (!RepR->isPredicated() ||
273 RepR->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
274 return false;
275 } else {
276 return false;
277 }
278 }
279
280 auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>();
281 if (all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
282 return Op.match(R->getOperand(N: Idx));
283 }))
284 return true;
285
286 return Commutative &&
287 all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
288 return Op.match(R->getOperand(N: R->getNumOperands() - Idx - 1));
289 });
290 }
291
292private:
293 template <typename RecipeTy>
294 static bool matchRecipeAndOpcode(const VPRecipeBase *R) {
295 auto *DefR = dyn_cast<RecipeTy>(R);
296 // Check for recipes that do not have opcodes.
297 if constexpr (std::is_same_v<RecipeTy, VPScalarIVStepsRecipe> ||
298 std::is_same_v<RecipeTy, VPDerivedIVRecipe> ||
299 std::is_same_v<RecipeTy, VPVectorEndPointerRecipe>)
300 return DefR;
301 else
302 return DefR && DefR->getOpcode() == Opcode;
303 }
304
305 /// Helper to check if predicate \p P holds on all tuple elements in Ops using
306 /// the provided index sequence.
307 template <typename Fn, std::size_t... Is>
308 bool all_of_tuple_elements(std::index_sequence<Is...>,
309 [[maybe_unused]] Fn P) const {
310 return (P(std::get<Is>(Ops), Is) && ...);
311 }
312};
313
314template <unsigned Opcode, typename... OpTys>
315using AllRecipe_match =
316 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ false,
317 VPWidenRecipe, VPReplicateRecipe, VPWidenCastRecipe,
318 VPInstruction>;
319
320template <unsigned Opcode, typename... OpTys>
321using AllRecipe_commutative_match =
322 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ true,
323 VPWidenRecipe, VPReplicateRecipe, VPInstruction>;
324
325template <unsigned Opcode, typename... OpTys>
326using VPInstruction_match = Recipe_match<std::tuple<OpTys...>, Opcode,
327 /*Commutative*/ false, VPInstruction>;
328
329template <unsigned Opcode, typename... OpTys>
330using VPInstruction_commutative_match =
331 Recipe_match<std::tuple<OpTys...>, Opcode,
332 /*Commutative*/ true, VPInstruction>;
333
334template <unsigned Opcode, typename... OpTys>
335inline VPInstruction_match<Opcode, OpTys...>
336m_VPInstruction(const OpTys &...Ops) {
337 return VPInstruction_match<Opcode, OpTys...>(Ops...);
338}
339
340template <unsigned Opcode, typename Op0_t, typename Op1_t>
341inline VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>
342m_c_VPInstruction(const Op0_t &Op0, const Op1_t &Op1) {
343 return VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
344}
345
346/// BuildVector is matches only its opcode, w/o matching its operands as the
347/// number of operands is not fixed.
348inline VPInstruction_match<VPInstruction::BuildVector> m_BuildVector() {
349 return m_VPInstruction<VPInstruction::BuildVector>();
350}
351
352/// BuildStructVector matches only its opcode, w/o matching its operands as the
353/// number of operands is not fixed.
354inline VPInstruction_match<VPInstruction::BuildStructVector>
355m_BuildStructVector() {
356 return m_VPInstruction<VPInstruction::BuildStructVector>();
357}
358
359template <typename Op0_t>
360inline VPInstruction_match<Instruction::Freeze, Op0_t>
361m_Freeze(const Op0_t &Op0) {
362 return m_VPInstruction<Instruction::Freeze>(Op0);
363}
364
365inline VPInstruction_match<VPInstruction::BranchOnCond> m_BranchOnCond() {
366 return m_VPInstruction<VPInstruction::BranchOnCond>();
367}
368
369template <typename Op0_t>
370inline VPInstruction_match<VPInstruction::BranchOnCond, Op0_t>
371m_BranchOnCond(const Op0_t &Op0) {
372 return m_VPInstruction<VPInstruction::BranchOnCond>(Op0);
373}
374
375inline VPInstruction_match<VPInstruction::BranchOnTwoConds>
376m_BranchOnTwoConds() {
377 return m_VPInstruction<VPInstruction::BranchOnTwoConds>();
378}
379
380template <typename Op0_t, typename Op1_t>
381inline VPInstruction_match<VPInstruction::BranchOnTwoConds, Op0_t, Op1_t>
382m_BranchOnTwoConds(const Op0_t &Op0, const Op1_t &Op1) {
383 return m_VPInstruction<VPInstruction::BranchOnTwoConds>(Op0, Op1);
384}
385
386inline VPInstruction_match<VPInstruction::BranchOnCount> m_BranchOnCount() {
387 return m_VPInstruction<VPInstruction::BranchOnCount>();
388}
389
390template <typename Op0_t, typename Op1_t>
391inline VPInstruction_match<VPInstruction::BranchOnCount, Op0_t, Op1_t>
392m_BranchOnCount(const Op0_t &Op0, const Op1_t &Op1) {
393 return m_VPInstruction<VPInstruction::BranchOnCount>(Op0, Op1);
394}
395
396inline auto m_Branch() {
397 return m_CombineOr(Ps: m_BranchOnCond(), Ps: m_BranchOnCount(), Ps: m_BranchOnTwoConds());
398}
399
400template <typename Op0_t>
401inline VPInstruction_match<VPInstruction::Broadcast, Op0_t>
402m_Broadcast(const Op0_t &Op0) {
403 return m_VPInstruction<VPInstruction::Broadcast>(Op0);
404}
405
406template <typename Op0_t>
407inline VPInstruction_match<VPInstruction::ExplicitVectorLength, Op0_t>
408m_EVL(const Op0_t &Op0) {
409 return m_VPInstruction<VPInstruction::ExplicitVectorLength>(Op0);
410}
411
412template <typename Op0_t>
413inline VPInstruction_match<VPInstruction::ExtractLastLane, Op0_t>
414m_ExtractLastLane(const Op0_t &Op0) {
415 return m_VPInstruction<VPInstruction::ExtractLastLane>(Op0);
416}
417
418template <typename Op0_t, typename Op1_t>
419inline VPInstruction_match<Instruction::ExtractElement, Op0_t, Op1_t>
420m_ExtractElement(const Op0_t &Op0, const Op1_t &Op1) {
421 return m_VPInstruction<Instruction::ExtractElement>(Op0, Op1);
422}
423
424template <typename Op0_t, typename Op1_t, typename Op2_t>
425inline VPInstruction_match<Instruction::InsertElement, Op0_t, Op1_t, Op2_t>
426m_InsertElement(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
427 return m_VPInstruction<Instruction::InsertElement>(Op0, Op1, Op2);
428}
429
430template <typename Op0_t, typename Op1_t>
431inline VPInstruction_match<VPInstruction::ExtractLane, Op0_t, Op1_t>
432m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1) {
433 return m_VPInstruction<VPInstruction::ExtractLane>(Op0, Op1);
434}
435
436template <typename Op0_t>
437inline VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>
438m_ExtractLastPart(const Op0_t &Op0) {
439 return m_VPInstruction<VPInstruction::ExtractLastPart>(Op0);
440}
441
442template <typename Op0_t>
443inline VPInstruction_match<
444 VPInstruction::ExtractLastLane,
445 VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>>
446m_ExtractLastLaneOfLastPart(const Op0_t &Op0) {
447 return m_ExtractLastLane(m_ExtractLastPart(Op0));
448}
449
450template <typename Op0_t, typename Op1_t>
451inline VPInstruction_match<VPInstruction::ExtractVectorForPart, Op0_t, Op1_t>
452m_ExtractVectorForPart(const Op0_t &Op0, const Op1_t &Op1) {
453 return m_VPInstruction<VPInstruction::ExtractVectorForPart>(Op0, Op1);
454}
455
456template <typename Op0_t>
457inline VPInstruction_match<VPInstruction::ExtractPenultimateElement, Op0_t>
458m_ExtractPenultimateElement(const Op0_t &Op0) {
459 return m_VPInstruction<VPInstruction::ExtractPenultimateElement>(Op0);
460}
461
462template <typename Op0_t, typename Op1_t, typename Op2_t>
463inline VPInstruction_match<VPInstruction::WideActiveLaneMask, Op0_t, Op1_t,
464 Op2_t>
465m_WideActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
466 return m_VPInstruction<VPInstruction::WideActiveLaneMask>(Op0, Op1, Op2);
467}
468
469inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() {
470 return m_VPInstruction<VPInstruction::AnyOf>();
471}
472
473template <typename Op0_t>
474inline VPInstruction_match<VPInstruction::AnyOf, Op0_t>
475m_AnyOf(const Op0_t &Op0) {
476 return m_VPInstruction<VPInstruction::AnyOf>(Op0);
477}
478
479template <typename Op0_t>
480inline VPInstruction_match<VPInstruction::FirstActiveLane, Op0_t>
481m_FirstActiveLane(const Op0_t &Op0) {
482 return m_VPInstruction<VPInstruction::FirstActiveLane>(Op0);
483}
484
485template <typename Op0_t>
486inline VPInstruction_match<VPInstruction::LastActiveLane, Op0_t>
487m_LastActiveLane(const Op0_t &Op0) {
488 return m_VPInstruction<VPInstruction::LastActiveLane>(Op0);
489}
490
491template <typename Op0_t, typename Op1_t, typename Op2_t>
492inline VPInstruction_match<VPInstruction::ExtractLastActive, Op0_t, Op1_t,
493 Op2_t>
494m_ExtractLastActive(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
495 return m_VPInstruction<VPInstruction::ExtractLastActive>(Op0, Op1, Op2);
496}
497
498template <typename Op0_t>
499inline VPInstruction_match<VPInstruction::ComputeReductionResult, Op0_t>
500m_ComputeReductionResult(const Op0_t &Op0) {
501 return m_VPInstruction<VPInstruction::ComputeReductionResult>(Op0);
502}
503
504/// Match FindIV result pattern:
505/// select(icmp ne ComputeReductionResult(ReducedIV), Sentinel),
506/// ComputeReductionResult(ReducedIV), Start.
507template <typename Op0_t, typename Op1_t>
508inline bool matchFindIVResult(VPInstruction *VPI, Op0_t ReducedIV, Op1_t Start) {
509 return match(VPI, m_Select(m_SpecificICmp(ICmpInst::ICMP_NE,
510 m_ComputeReductionResult(ReducedIV),
511 m_VPValue()),
512 m_ComputeReductionResult(ReducedIV), Start));
513}
514
515template <typename Op0_t>
516inline VPInstruction_match<VPInstruction::Reverse, Op0_t>
517m_Reverse(const Op0_t &Op0) {
518 return m_VPInstruction<VPInstruction::Reverse>(Op0);
519}
520
521inline VPInstruction_match<VPInstruction::StepVector> m_StepVector() {
522 return m_VPInstruction<VPInstruction::StepVector>();
523}
524
525template <typename Op0_t>
526inline VPInstruction_match<VPInstruction::ExitingIVValue, Op0_t>
527m_ExitingIVValue(const Op0_t &Op0) {
528 return m_VPInstruction<VPInstruction::ExitingIVValue>(Op0);
529}
530
531template <unsigned Opcode, typename Op0_t>
532inline AllRecipe_match<Opcode, Op0_t> m_Unary(const Op0_t &Op0) {
533 return AllRecipe_match<Opcode, Op0_t>(Op0);
534}
535
536template <typename Op0_t>
537inline AllRecipe_match<Instruction::Trunc, Op0_t> m_Trunc(const Op0_t &Op0) {
538 return m_Unary<Instruction::Trunc, Op0_t>(Op0);
539}
540
541template <typename Op0_t>
542inline match_combine_or<AllRecipe_match<Instruction::Trunc, Op0_t>, Op0_t>
543m_TruncOrSelf(const Op0_t &Op0) {
544 return m_CombineOr(m_Trunc(Op0), Op0);
545}
546
547template <typename Op0_t>
548inline AllRecipe_match<Instruction::ZExt, Op0_t> m_ZExt(const Op0_t &Op0) {
549 return m_Unary<Instruction::ZExt, Op0_t>(Op0);
550}
551
552template <typename Op0_t>
553inline AllRecipe_match<Instruction::SExt, Op0_t> m_SExt(const Op0_t &Op0) {
554 return m_Unary<Instruction::SExt, Op0_t>(Op0);
555}
556
557template <typename Op0_t>
558inline AllRecipe_match<Instruction::FPExt, Op0_t> m_FPExt(const Op0_t &Op0) {
559 return m_Unary<Instruction::FPExt, Op0_t>(Op0);
560}
561
562template <typename Op0_t>
563inline AllRecipe_match<Instruction::FNeg, Op0_t> m_FNeg(const Op0_t &Op0) {
564 return m_Unary<Instruction::FNeg, Op0_t>(Op0);
565}
566
567template <typename Op0_t>
568inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>,
569 AllRecipe_match<Instruction::SExt, Op0_t>>
570m_ZExtOrSExt(const Op0_t &Op0) {
571 return m_CombineOr(m_ZExt(Op0), m_SExt(Op0));
572}
573
574template <typename Op0_t> inline auto m_WidenAnyExtend(const Op0_t &Op0) {
575 return m_Isa<VPWidenCastRecipe>(m_CombineOr(m_ZExtOrSExt(Op0), m_FPExt(Op0)));
576}
577
578template <typename Op0_t> inline auto m_AnyNeg(const Op0_t &Op0) {
579 return m_CombineOr(m_Sub(m_ZeroInt(), Op0), m_FNeg(Op0));
580}
581
582template <typename Op0_t>
583inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>, Op0_t>
584m_ZExtOrSelf(const Op0_t &Op0) {
585 return m_CombineOr(m_ZExt(Op0), Op0);
586}
587
588template <typename Op0_t> inline auto m_ZExtOrTruncOrSelf(const Op0_t &Op0) {
589 return m_CombineOr(m_ZExt(Op0), m_Trunc(Op0), Op0);
590}
591
592template <unsigned Opcode, typename Op0_t, typename Op1_t>
593inline AllRecipe_match<Opcode, Op0_t, Op1_t> m_Binary(const Op0_t &Op0,
594 const Op1_t &Op1) {
595 return AllRecipe_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
596}
597
598template <unsigned Opcode, typename Op0_t, typename Op1_t>
599inline AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>
600m_c_Binary(const Op0_t &Op0, const Op1_t &Op1) {
601 return AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
602}
603
604template <typename Op0_t, typename Op1_t>
605inline AllRecipe_match<Instruction::Add, Op0_t, Op1_t> m_Add(const Op0_t &Op0,
606 const Op1_t &Op1) {
607 return m_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
608}
609
610template <typename Op0_t, typename Op1_t>
611inline AllRecipe_commutative_match<Instruction::Add, Op0_t, Op1_t>
612m_c_Add(const Op0_t &Op0, const Op1_t &Op1) {
613 return m_c_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
614}
615
616template <typename Op0_t, typename Op1_t>
617inline AllRecipe_match<Instruction::Sub, Op0_t, Op1_t> m_Sub(const Op0_t &Op0,
618 const Op1_t &Op1) {
619 return m_Binary<Instruction::Sub, Op0_t, Op1_t>(Op0, Op1);
620}
621
622template <typename Op0_t, typename Op1_t>
623inline AllRecipe_match<Instruction::Mul, Op0_t, Op1_t> m_Mul(const Op0_t &Op0,
624 const Op1_t &Op1) {
625 return m_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
626}
627
628template <typename Op0_t, typename Op1_t>
629inline AllRecipe_commutative_match<Instruction::Mul, Op0_t, Op1_t>
630m_c_Mul(const Op0_t &Op0, const Op1_t &Op1) {
631 return m_c_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
632}
633
634template <typename Op0_t, typename Op1_t>
635inline AllRecipe_match<Instruction::Shl, Op0_t, Op1_t> m_Shl(const Op0_t &Op0,
636 const Op1_t &Op1) {
637 return m_Binary<Instruction::Shl, Op0_t, Op1_t>(Op0, Op1);
638}
639
640template <typename Op0_t, typename Op1_t>
641inline AllRecipe_match<Instruction::LShr, Op0_t, Op1_t>
642m_LShr(const Op0_t &Op0, const Op1_t &Op1) {
643 return m_Binary<Instruction::LShr, Op0_t, Op1_t>(Op0, Op1);
644}
645
646template <typename Op0_t, typename Op1_t>
647inline AllRecipe_match<Instruction::FMul, Op0_t, Op1_t>
648m_FMul(const Op0_t &Op0, const Op1_t &Op1) {
649 return m_Binary<Instruction::FMul, Op0_t, Op1_t>(Op0, Op1);
650}
651
652template <typename Op0_t, typename Op1_t>
653inline AllRecipe_match<Instruction::FAdd, Op0_t, Op1_t>
654m_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
655 return m_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
656}
657
658template <typename Op0_t, typename Op1_t>
659inline AllRecipe_commutative_match<Instruction::FAdd, Op0_t, Op1_t>
660m_c_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
661 return m_c_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
662}
663
664template <typename Op0_t, typename Op1_t>
665inline AllRecipe_match<Instruction::UDiv, Op0_t, Op1_t>
666m_UDiv(const Op0_t &Op0, const Op1_t &Op1) {
667 return m_Binary<Instruction::UDiv, Op0_t, Op1_t>(Op0, Op1);
668}
669
670template <typename Op0_t, typename Op1_t>
671inline AllRecipe_match<Instruction::URem, Op0_t, Op1_t>
672m_URem(const Op0_t &Op0, const Op1_t &Op1) {
673 return m_Binary<Instruction::URem, Op0_t, Op1_t>(Op0, Op1);
674}
675
676template <typename Op0_t, typename Op1_t>
677inline AllRecipe_match<Instruction::SDiv, Op0_t, Op1_t>
678m_SDiv(const Op0_t &Op0, const Op1_t &Op1) {
679 return m_Binary<Instruction::SDiv, Op0_t, Op1_t>(Op0, Op1);
680}
681
682template <typename Op0_t, typename Op1_t>
683inline AllRecipe_match<Instruction::SRem, Op0_t, Op1_t>
684m_SRem(const Op0_t &Op0, const Op1_t &Op1) {
685 return m_Binary<Instruction::SRem, Op0_t, Op1_t>(Op0, Op1);
686}
687
688/// Match a binary AND operation.
689template <typename Op0_t, typename Op1_t>
690inline AllRecipe_commutative_match<Instruction::And, Op0_t, Op1_t>
691m_c_BinaryAnd(const Op0_t &Op0, const Op1_t &Op1) {
692 return m_c_Binary<Instruction::And, Op0_t, Op1_t>(Op0, Op1);
693}
694
695/// Match a binary OR operation. Note that while conceptually the operands can
696/// be matched commutatively, \p Commutative defaults to false in line with the
697/// IR-based pattern matching infrastructure. Use m_c_BinaryOr for a commutative
698/// version of the matcher.
699template <typename Op0_t, typename Op1_t>
700inline AllRecipe_match<Instruction::Or, Op0_t, Op1_t>
701m_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
702 return m_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
703}
704
705template <typename Op0_t, typename Op1_t>
706inline AllRecipe_commutative_match<Instruction::Or, Op0_t, Op1_t>
707m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
708 return m_c_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
709}
710
711/// Cmp_match is a variant of BinaryRecipe_match that also binds the comparison
712/// predicate. Opcodes must either be Instruction::ICmp or Instruction::FCmp, or
713/// both.
714template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
715struct Cmp_match {
716 static_assert((sizeof...(Opcodes) == 1 || sizeof...(Opcodes) == 2) &&
717 "Expected one or two opcodes");
718 static_assert(
719 ((Opcodes == Instruction::ICmp || Opcodes == Instruction::FCmp) && ...) &&
720 "Expected a compare instruction opcode");
721
722 CmpPredicate *Predicate = nullptr;
723 Op0_t Op0;
724 Op1_t Op1;
725
726 Cmp_match(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1)
727 : Predicate(&Pred), Op0(Op0), Op1(Op1) {}
728 Cmp_match(const Op0_t &Op0, const Op1_t &Op1) : Op0(Op0), Op1(Op1) {}
729
730 bool match(const VPValue *V) const {
731 auto *DefR = V->getDefiningRecipe();
732 return DefR && match(DefR);
733 }
734
735 bool match(const VPRecipeBase *V) const {
736 if ((m_Binary<Opcodes>(Op0, Op1).match(V) || ...)) {
737 if (Predicate)
738 *Predicate = cast<VPRecipeWithIRFlags>(Val: V)->getPredicate();
739 return true;
740 }
741 return false;
742 }
743};
744
745/// SpecificCmp_match is a variant of Cmp_match that matches the comparison
746/// predicate, instead of binding it.
747template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
748struct SpecificCmp_match {
749 const CmpPredicate Predicate;
750 Op0_t Op0;
751 Op1_t Op1;
752
753 SpecificCmp_match(CmpPredicate Pred, const Op0_t &LHS, const Op1_t &RHS)
754 : Predicate(Pred), Op0(LHS), Op1(RHS) {}
755
756 bool match(const VPValue *V) const {
757 auto *DefR = V->getDefiningRecipe();
758 return DefR && match(DefR);
759 }
760
761 bool match(const VPRecipeBase *V) const {
762 CmpPredicate CurrentPred;
763 return Cmp_match<Op0_t, Op1_t, Opcodes...>(CurrentPred, Op0, Op1)
764 .match(V) &&
765 CmpPredicate::getMatching(A: CurrentPred, B: Predicate);
766 }
767};
768
769template <typename Op0_t, typename Op1_t>
770inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp> m_ICmp(const Op0_t &Op0,
771 const Op1_t &Op1) {
772 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Op0, Op1);
773}
774
775template <typename Op0_t, typename Op1_t>
776inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp>
777m_ICmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
778 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Pred, Op0, Op1);
779}
780
781template <typename Op0_t, typename Op1_t>
782inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>
783m_SpecificICmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
784 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>(MatchPred, Op0,
785 Op1);
786}
787
788template <typename Op0_t, typename Op1_t>
789inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
790m_Cmp(const Op0_t &Op0, const Op1_t &Op1) {
791 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(Op0,
792 Op1);
793}
794
795template <typename Op0_t, typename Op1_t>
796inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
797m_Cmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
798 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
799 Pred, Op0, Op1);
800}
801
802template <typename Op0_t, typename Op1_t>
803inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
804m_SpecificCmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
805 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
806 MatchPred, Op0, Op1);
807}
808
809template <typename Op0_t, typename Op1_t>
810inline auto m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1) {
811 return m_CombineOr(
812 Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr,
813 /*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>(
814 Op0, Op1),
815 VPInstruction_match<VPInstruction::PtrAdd, Op0_t, Op1_t>(Op0, Op1),
816 VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>(Op0, Op1));
817}
818
819/// Match a VPBlendRecipe with 2 incoming values ([I0, I1, M1] ==
820/// normalized([I0, M0, I1, M1])) as select(M1, I1, I0), mirroring how it is
821/// lowered.
822template <typename Op0_t, typename Op1_t, typename Op2_t> struct Blend2_match {
823 Op0_t MaskOp;
824 Op1_t TrueOp;
825 Op2_t FalseOp;
826
827 Blend2_match(const Op0_t &MaskOp, const Op1_t &TrueOp, const Op2_t &FalseOp)
828 : MaskOp(MaskOp), TrueOp(TrueOp), FalseOp(FalseOp) {}
829
830 template <typename T> bool match(const T *Val) const {
831 auto *Blend = dyn_cast<VPBlendRecipe>(Val);
832 if (!Blend || Blend->getNumIncomingValues() != 2)
833 return false;
834 return MaskOp.match(Blend->getMask(1)) &&
835 TrueOp.match(Blend->getIncomingValue(1)) &&
836 FalseOp.match(Blend->getIncomingValue(0));
837 }
838};
839
840/// Match recipe recipe with Select opcode, i.e. excluding VPBlendRecipe.
841template <typename Op0_t, typename Op1_t, typename Op2_t>
842inline AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>
843m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
844 return AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>(
845 {Op0, Op1, Op2});
846}
847
848/// Match recipe with Select opcode or an equivalent VPBlendRecipe with 2
849/// incoming values.
850template <typename Op0_t, typename Op1_t, typename Op2_t>
851inline auto m_SelectLike(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
852 return m_CombineOr(m_Select(Op0, Op1, Op2),
853 Blend2_match<Op0_t, Op1_t, Op2_t>(Op0, Op1, Op2));
854}
855
856template <typename Op0_t> inline auto m_Not(const Op0_t &Op0) {
857 return m_CombineOr(m_VPInstruction<VPInstruction::Not>(Op0),
858 m_c_Binary<Instruction::Xor>(m_AllOnes(), Op0));
859}
860
861template <typename Op0_t, typename Op1_t, typename Op2_t>
862inline auto m_c_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
863 return m_CombineOr(m_Select(Op0, Op1, Op2), m_Select(m_Not(Op0), Op2, Op1));
864}
865
866template <typename Op0_t, typename Op1_t>
867inline auto m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
868 return m_CombineOr(
869 m_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
870 m_Select(Op0, Op1, m_False()));
871}
872
873template <typename Op0_t, typename Op1_t>
874inline auto m_c_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
875 return m_CombineOr(
876 m_c_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
877 m_c_Select(Op0, Op1, m_False()));
878}
879
880template <typename Op0_t, typename Op1_t>
881inline auto m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
882 return m_CombineOr(
883 m_c_VPInstruction<VPInstruction::LogicalOr, Op0_t, Op1_t>(Op0, Op1),
884 m_Select(Op0, m_True(), Op1));
885}
886
887template <typename Op0_t, typename Op1_t>
888inline auto m_c_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
889 return m_c_Select(Op0, m_True(), Op1);
890}
891
892/// Match the canonical induction variable (IV) of any loop region.
893struct canonical_iv_match {
894 template <typename ArgTy> bool match(const ArgTy *V) const {
895 const auto *RV = dyn_cast<VPRegionValue>(V);
896 return RV && RV->getDefiningRegion()->getCanonicalIV() == RV;
897 }
898};
899
900inline canonical_iv_match m_CanonicalIV() { return {}; }
901
902/// Match the abstract header mask of any loop region.
903struct header_mask_match {
904 template <typename ArgTy> bool match(const ArgTy *V) const {
905 const auto *RV = dyn_cast<VPRegionValue>(V);
906 return RV && RV->getDefiningRegion()->getHeaderMask() == RV;
907 }
908};
909
910inline header_mask_match m_HeaderMask() { return {}; }
911
912/// Match a canonical VPWidenIntOrFpInductionRecipe optionally capturing it.
913struct canonical_widen_iv_match {
914 VPWidenIntOrFpInductionRecipe **Capture = nullptr;
915
916 canonical_widen_iv_match() = default;
917 canonical_widen_iv_match(VPWidenIntOrFpInductionRecipe *&V) : Capture(&V) {}
918
919 template <typename ArgTy> bool match(ArgTy *V) const {
920 auto *WidenIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(V);
921 if (!WidenIV || !WidenIV->isCanonical())
922 return false;
923 if (Capture)
924 *Capture = WidenIV;
925 return true;
926 }
927};
928
929inline canonical_widen_iv_match m_CanonicalWidenIV() { return {}; }
930
931/// Match a canonical VPWidenIntOrFpInductionRecipe, capturing it.
932inline canonical_widen_iv_match
933m_CanonicalWidenIV(VPWidenIntOrFpInductionRecipe *&V) {
934 return canonical_widen_iv_match(V);
935}
936
937template <typename Op0_t, typename Op1_t, typename Op2_t>
938inline auto m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1,
939 const Op2_t &Op2) {
940 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
941 VPScalarIVStepsRecipe>({Op0, Op1, Op2});
942}
943
944template <typename Op0_t, typename Op1_t, typename Op2_t>
945inline auto m_DerivedIV(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
946 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
947 VPDerivedIVRecipe>({Op0, Op1, Op2});
948}
949
950template <typename Addr_t, typename Mask_t> struct Load_match {
951 Addr_t Addr;
952 Mask_t Mask;
953
954 Load_match(Addr_t Addr, Mask_t Mask) : Addr(Addr), Mask(Mask) {}
955
956 template <typename OpTy> bool match(const OpTy *V) const {
957 auto *Load = dyn_cast<VPWidenLoadRecipe>(V);
958 if (!Load || !Addr.match(Load->getAddr()) || !Load->isMasked() ||
959 !Mask.match(Load->getMask()))
960 return false;
961 return true;
962 }
963};
964
965/// Match a (possibly reversed) masked load.
966template <typename Addr_t, typename Mask_t>
967inline Load_match<Addr_t, Mask_t> m_MaskedLoad(const Addr_t &Addr,
968 const Mask_t &Mask) {
969 return Load_match<Addr_t, Mask_t>(Addr, Mask);
970}
971
972template <typename Addr_t, typename Val_t, typename Mask_t> struct Store_match {
973 Addr_t Addr;
974 Val_t Val;
975 Mask_t Mask;
976
977 Store_match(Addr_t Addr, Val_t Val, Mask_t Mask)
978 : Addr(Addr), Val(Val), Mask(Mask) {}
979
980 template <typename OpTy> bool match(const OpTy *V) const {
981 auto *Store = dyn_cast<VPWidenStoreRecipe>(V);
982 if (!Store || !Addr.match(Store->getAddr()) ||
983 !Val.match(Store->getStoredValue()) || !Store->isMasked() ||
984 !Mask.match(Store->getMask()))
985 return false;
986 return true;
987 }
988};
989
990/// Match a (possibly reversed) masked store.
991template <typename Addr_t, typename Val_t, typename Mask_t>
992inline Store_match<Addr_t, Val_t, Mask_t>
993m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) {
994 return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask);
995}
996
997template <typename Op0_t, typename Op1_t>
998using VectorEndPointerRecipe_match =
999 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
1000 /*Commutative*/ false, VPVectorEndPointerRecipe>;
1001
1002template <typename Op0_t, typename Op1_t>
1003VectorEndPointerRecipe_match<Op0_t, Op1_t> m_VecEndPtr(const Op0_t &Op0,
1004 const Op1_t &Op1) {
1005 return VectorEndPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
1006}
1007
1008/// Match a call argument at a given argument index.
1009template <typename Opnd_t> struct Argument_match {
1010 /// Call argument index to match.
1011 unsigned OpI;
1012 Opnd_t Val;
1013
1014 Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}
1015
1016 template <typename OpTy> bool match(OpTy *V) const {
1017 if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V))
1018 return Val.match(R->getOperand(OpI));
1019 if (const auto *R = dyn_cast<VPWidenCallRecipe>(V))
1020 return Val.match(R->getOperand(OpI));
1021 if (const auto *R = dyn_cast<VPReplicateRecipe>(V))
1022 if (R->getOpcode() == Instruction::Call)
1023 return Val.match(R->getOperand(OpI));
1024 if (const auto *R = dyn_cast<VPInstruction>(V))
1025 if (R->getOpcode() == Instruction::Call ||
1026 R->getOpcode() == VPInstruction::Intrinsic)
1027 return Val.match(R->getOperand(OpI));
1028 return false;
1029 }
1030};
1031
1032/// Match a call argument.
1033template <unsigned OpI, typename Opnd_t>
1034inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) {
1035 return Argument_match<Opnd_t>(OpI, Op);
1036}
1037
1038/// Intrinsic matchers.
1039struct IntrinsicID_match {
1040 unsigned ID;
1041
1042 IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {}
1043
1044 template <typename OpTy> bool match(OpTy *V) const {
1045 return vputils::getIntrinsicID(V) == ID;
1046 }
1047};
1048
1049/// Match intrinsic calls with a runtime intrinsic ID.
1050inline IntrinsicID_match m_Intrinsic(Intrinsic::ID IntrID) {
1051 return IntrinsicID_match(IntrID);
1052}
1053
1054struct IntrinsicMatchImpl {
1055 template <Intrinsic::ID IntrID, typename... Ts, size_t... Is>
1056 static auto impl(std::index_sequence<Is...>, const Ts &...Ops) {
1057 return m_CombineAnd(IntrinsicID_match(IntrID), m_Argument<Is>(Ops)...);
1058 }
1059};
1060
1061/// Match intrinsic calls like this:
1062/// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...)
1063template <Intrinsic::ID IntrID, typename... Ts>
1064inline auto m_Intrinsic(const Ts &...Ops) {
1065 return IntrinsicMatchImpl::impl<IntrID>(
1066 std::make_index_sequence<sizeof...(Ts)>{}, Ops...);
1067}
1068
1069template <Intrinsic::ID IntrID, typename... T>
1070inline auto m_WidenIntrinsic(const T &...Ops) {
1071 return m_Isa<VPWidenIntrinsicRecipe>(m_Intrinsic<IntrID>(Ops...));
1072}
1073
1074/// Match VPValues that represent live-ins: VPIRValues and (plain)
1075/// VPSymbolicValues. VPRegionValues (which inherit from VPSymbolicValue) are
1076/// not live-ins and are excluded.
1077struct LiveIn_match {
1078 template <typename ITy> bool match(ITy *V) const {
1079 return isa<VPIRValue>(V) ||
1080 (isa<VPSymbolicValue>(V) && !isa<VPRegionValue>(V));
1081 }
1082};
1083
1084inline auto m_VScale() { return m_Intrinsic<Intrinsic::vscale>(); }
1085
1086inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
1087
1088/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
1089/// and bind the source element type and operands.
1090struct GetElementPtr_match {
1091 Type *&SourceElementType;
1092 ArrayRef<VPValue *> &Operands;
1093
1094 GetElementPtr_match(Type *&SourceElementType, ArrayRef<VPValue *> &Operands)
1095 : SourceElementType(SourceElementType), Operands(Operands) {}
1096
1097 template <typename ITy> bool match(ITy *V) const {
1098 return matchRecipeAndBind<VPWidenGEPRecipe>(V) ||
1099 matchRecipeAndBind<VPInstruction>(V) ||
1100 matchRecipeAndBind<VPReplicateRecipe>(V);
1101 }
1102
1103private:
1104 template <typename RecipeTy> bool matchRecipeAndBind(const VPValue *V) const {
1105 auto *DefR = dyn_cast<RecipeTy>(V);
1106 if (!DefR)
1107 return false;
1108
1109 if constexpr (std::is_same_v<RecipeTy, VPWidenGEPRecipe>) {
1110 SourceElementType = DefR->getSourceElementType();
1111 } else if (DefR->getOpcode() == Instruction::GetElementPtr) {
1112 SourceElementType = cast<GetElementPtrInst>(DefR->getUnderlyingInstr())
1113 ->getSourceElementType();
1114 } else if constexpr (std::is_same_v<RecipeTy, VPInstruction>) {
1115 if (DefR->getOpcode() == VPInstruction::PtrAdd) {
1116 // PtrAdd is a byte-offset GEP with i8 element type.
1117 LLVMContext &Ctx = DefR->getParent()->getPlan()->getContext();
1118 SourceElementType = Type::getInt8Ty(C&: Ctx);
1119 } else {
1120 return false;
1121 }
1122 } else {
1123 return false;
1124 }
1125
1126 Operands = ArrayRef<VPValue *>(DefR->op_begin(), DefR->op_end());
1127 return true;
1128 }
1129};
1130
1131/// Match a GEP recipe with any number of operands and bind source element type
1132/// and operands.
1133inline GetElementPtr_match m_GetElementPtr(Type *&SourceElementType,
1134 ArrayRef<VPValue *> &Operands) {
1135 return GetElementPtr_match(SourceElementType, Operands);
1136}
1137
1138template <typename SubPattern_t> struct OneUse_match {
1139 SubPattern_t SubPattern;
1140
1141 OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}
1142
1143 template <typename OpTy> bool match(OpTy *V) const {
1144 return V->hasOneUse() && SubPattern.match(V);
1145 }
1146};
1147
1148template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) {
1149 return SubPattern;
1150}
1151
1152inline match_bind<VPReductionPHIRecipe>
1153m_ReductionPhi(VPReductionPHIRecipe *&V) {
1154 return V;
1155}
1156
1157template <typename Op0_t, typename Op1_t>
1158inline auto m_VPPhi(const Op0_t &Op0, const Op1_t &Op1) {
1159 return Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::PHI,
1160 /*Commutative*/ false, VPInstruction>({Op0, Op1});
1161}
1162
1163/// If \p V is used by a recipe matching pattern \p P, return it. Otherwise
1164/// return nullptr;
1165template <typename MatchT>
1166VPRecipeBase *findUserOf(VPValue *V, const MatchT &P) {
1167 auto It = find_if(V->users(), match_fn(P));
1168 return It == V->user_end() ? nullptr : cast<VPRecipeBase>(*It);
1169}
1170
1171/// If \p V is used by a VPInstruction with \p Opcode, return it. Otherwise
1172/// return nullptr.
1173template <unsigned Opcode> VPInstruction *findUserOf(VPValue *V) {
1174 return cast_or_null<VPInstruction>(findUserOf(V, m_VPInstruction<Opcode>()));
1175}
1176
1177template <typename RecipeTy> RecipeTy *findUserOf(VPValue *V) {
1178 return cast_or_null<RecipeTy>(findUserOf(V, m_Isa<RecipeTy>()));
1179}
1180} // namespace llvm::VPlanPatternMatch
1181
1182#endif
1183