1//===- VPlanPatternMatch.h - Match on VPValues and recipes ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides a simple and efficient mechanism for performing general
10// tree-based pattern matches on the VPlan values and recipes, based on
11// LLVM's IR pattern matchers.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
16#define LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
17
18#include "VPlan.h"
19#include "VPlanUtils.h"
20#include "llvm/Support/PatternMatchHelpers.h"
21#include <utility>
22
23namespace llvm::VPlanPatternMatch {
24
25using namespace llvm::PatternMatchHelpers;
26
27template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
28 return P.match(V);
29}
30
31template <typename Pattern> bool match(VPUser *U, const Pattern &P) {
32 auto *R = dyn_cast<VPRecipeBase>(Val: U);
33 return R && match(R, P);
34}
35
36template <typename Pattern> bool match(VPSingleDefRecipe *R, const Pattern &P) {
37 return P.match(static_cast<const VPRecipeBase *>(R));
38}
39
40/// A match functor that can be used as a UnaryPredicate in functional
41/// algorithms like all_of.
42template <typename Pattern> auto match_fn(const Pattern &P) {
43 return [&P](auto *V) { return match(V, P); };
44}
45
46/// Match an arbitrary VPValue and ignore it.
47inline auto m_VPValue() { return m_Isa<VPValue>(); }
48
49/// Match a specified VPValue.
50struct specificval_ty {
51 const VPValue *Val;
52
53 specificval_ty(const VPValue *V) : Val(V) {}
54
55 bool match(const VPValue *VPV) const { return VPV == Val; }
56};
57
58inline specificval_ty m_Specific(const VPValue *VPV) { return VPV; }
59
60/// Like m_Specific(), but works if the specific value to match is determined
61/// as part of the same match() expression. For example:
62/// m_Mul(m_VPValue(X), m_Specific(X)) is incorrect, because m_Specific() will
63/// bind X before the pattern match starts.
64/// m_Mul(m_VPValue(X), m_Deferred(X)) is correct, and will check against
65/// whichever value m_VPValue(X) populated.
66inline match_deferred<VPValue> m_Deferred(VPValue *const &V) { return V; }
67
68/// Match an integer constant if Pred::isValue returns true for the APInt. \p
69/// BitWidth optionally specifies the bitwidth the matched constant must have.
70/// If it is 0, the matched constant can have any bitwidth.
71template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty {
72 Pred P;
73
74 int_pred_ty(Pred P) : P(std::move(P)) {}
75 int_pred_ty() : P() {}
76
77 bool match(const VPValue *VPV) const {
78 auto *VPI = dyn_cast<VPInstruction>(Val: VPV);
79 if (VPI && VPI->getOpcode() == VPInstruction::Broadcast)
80 VPV = VPI->getOperand(N: 0);
81 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
82 if (!CI)
83 return false;
84
85 if (BitWidth != 0 && CI->getBitWidth() != BitWidth)
86 return false;
87 return P.isValue(CI->getAPInt());
88 }
89};
90
91/// Match a specified signed or unsigned integer value.
92struct is_specific_int {
93 APInt Val;
94 bool IsSigned;
95
96 is_specific_int(APInt Val, bool IsSigned = false)
97 : Val(std::move(Val)), IsSigned(IsSigned) {}
98
99 bool isValue(const APInt &C) const {
100 return APInt::isSameValue(I1: Val, I2: C, SignedCompare: IsSigned);
101 }
102};
103
104template <unsigned Bitwidth = 0>
105using specific_intval = int_pred_ty<is_specific_int, Bitwidth>;
106
107inline specific_intval<0> m_SpecificInt(uint64_t V) {
108 return specific_intval<0>(is_specific_int(APInt(64, V)));
109}
110
111inline specific_intval<0> m_SpecificSInt(int64_t V) {
112 return specific_intval<0>(
113 is_specific_int(APInt(64, V, /*isSigned=*/true), /*IsSigned=*/true));
114}
115
116struct is_all_ones {
117 bool isValue(const APInt &C) const { return C.isAllOnes(); }
118};
119
120/// Match an integer or vector with all bits set.
121/// For vectors, this includes constants with undefined elements.
122inline int_pred_ty<is_all_ones> m_AllOnes() {
123 return int_pred_ty<is_all_ones>();
124}
125
126struct is_zero_int {
127 bool isValue(const APInt &C) const { return C.isZero(); }
128};
129
130struct is_one {
131 bool isValue(const APInt &C) const { return C.isOne(); }
132};
133
134/// Match an integer 0 or a vector with all elements equal to 0.
135/// For vectors, this includes constants with undefined elements.
136inline int_pred_ty<is_zero_int> m_ZeroInt() {
137 return int_pred_ty<is_zero_int>();
138}
139
140/// Match an integer 1 or a vector with all elements equal to 1.
141/// For vectors, this includes constants with undefined elements.
142inline int_pred_ty<is_one> m_One() { return int_pred_ty<is_one>(); }
143
144inline int_pred_ty<is_zero_int, 1> m_False() { return {}; }
145
146inline int_pred_ty<is_one, 1> m_True() { return {}; }
147
148struct bind_apint {
149 const APInt *&Res;
150
151 bind_apint(const APInt *&Res) : Res(Res) {}
152
153 bool match(const VPValue *VPV) const {
154 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
155 if (!CI)
156 return false;
157 Res = &CI->getAPInt();
158 return true;
159 }
160};
161
162inline bind_apint m_APInt(const APInt *&C) { return C; }
163
164struct bind_const_int {
165 uint64_t &Res;
166
167 bind_const_int(uint64_t &Res) : Res(Res) {}
168
169 bool match(const VPValue *VPV) const {
170 const APInt *APConst;
171 if (!bind_apint(APConst).match(VPV))
172 return false;
173 if (auto C = APConst->tryZExtValue()) {
174 Res = *C;
175 return true;
176 }
177 return false;
178 }
179};
180
181struct match_poison {
182 bool match(const VPValue *V) const {
183 return isa<VPIRValue>(Val: V) &&
184 isa<PoisonValue>(Val: cast<VPIRValue>(Val: V)->getValue());
185 }
186};
187
188/// Match a VPIRValue that's poison.
189inline match_poison m_Poison() { return match_poison(); }
190
191/// Match a plain integer constant no wider than 64-bits, capturing it if we
192/// match.
193inline bind_const_int m_ConstantInt(uint64_t &C) { return C; }
194
195/// Match a VPValue, capturing it if we match.
196inline match_bind<VPValue> m_VPValue(VPValue *&V) { return V; }
197
198/// Match against the nested pattern, and capture the value if we match.
199template <typename Op_t> inline auto m_VPValue(VPValue *&V, const Op_t &Op) {
200 return m_CombineAnd(Op, m_VPValue(V));
201}
202
203/// Match a VPIRValue.
204inline match_bind<VPIRValue> m_VPIRValue(VPIRValue *&V) { return V; }
205
206/// Match a VPSingleDefRecipe, capturing if we match.
207inline match_bind<VPSingleDefRecipe>
208m_VPSingleDefRecipe(VPSingleDefRecipe *&V) {
209 return V;
210}
211
212/// Match a VPInstruction, capturing if we match.
213inline match_bind<VPInstruction> m_VPInstruction(VPInstruction *&V) {
214 return V;
215}
216
217template <typename Ops_t, unsigned Opcode, bool Commutative,
218 typename... RecipeTys>
219struct Recipe_match {
220 Ops_t Ops;
221
222 template <typename... OpTy> Recipe_match(OpTy... Ops) : Ops(Ops...) {
223 static_assert(std::tuple_size<Ops_t>::value == sizeof...(Ops) &&
224 "number of operands in constructor doesn't match Ops_t");
225 static_assert((!Commutative || std::tuple_size<Ops_t>::value == 2) &&
226 "only binary ops can be commutative");
227 }
228
229 bool match(const VPValue *V) const {
230 auto *DefR = V->getDefiningRecipe();
231 return DefR && match(DefR);
232 }
233
234 bool match(const VPSingleDefRecipe *R) const {
235 return match(static_cast<const VPRecipeBase *>(R));
236 }
237
238 bool match(const VPRecipeBase *R) const {
239 if (std::tuple_size_v<Ops_t> == 0) {
240 auto *VPI = dyn_cast<VPInstruction>(Val: R);
241 return VPI && VPI->getOpcode() == Opcode;
242 }
243
244 if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...))
245 return false;
246
247 if (R->getNumOperands() < std::tuple_size<Ops_t>::value) {
248 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R);
249 assert(((isa<VPInstruction>(R) &&
250 cast<VPInstruction>(R)->getNumOperandsForOpcode() == -1u) ||
251 (RepR && std::tuple_size_v<Ops_t> ==
252 RepR->getNumOperandsWithoutMask())) &&
253 "non-variadic recipe with matched opcode does not have the "
254 "expected number of operands");
255 return false;
256 }
257
258 // If the recipe has more operands than expected, we only support matching
259 // masked VPInstructions or predicated VPReplicateRecipes, where the number
260 // of operands of the matcher matches the number of operands excluding the
261 // mask.
262 if (R->getNumOperands() > std::tuple_size<Ops_t>::value) {
263 if (auto *VPI = dyn_cast<VPInstruction>(Val: R)) {
264 if (!VPI->isMasked() ||
265 VPI->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
266 return false;
267 } else if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R)) {
268 if (!RepR->isPredicated() ||
269 RepR->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
270 return false;
271 } else {
272 return false;
273 }
274 }
275
276 auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>();
277 if (all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
278 return Op.match(R->getOperand(N: Idx));
279 }))
280 return true;
281
282 return Commutative &&
283 all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
284 return Op.match(R->getOperand(N: R->getNumOperands() - Idx - 1));
285 });
286 }
287
288private:
289 template <typename RecipeTy>
290 static bool matchRecipeAndOpcode(const VPRecipeBase *R) {
291 auto *DefR = dyn_cast<RecipeTy>(R);
292 // Check for recipes that do not have opcodes.
293 if constexpr (std::is_same_v<RecipeTy, VPScalarIVStepsRecipe> ||
294 std::is_same_v<RecipeTy, VPDerivedIVRecipe> ||
295 std::is_same_v<RecipeTy, VPVectorEndPointerRecipe>)
296 return DefR;
297 else
298 return DefR && DefR->getOpcode() == Opcode;
299 }
300
301 /// Helper to check if predicate \p P holds on all tuple elements in Ops using
302 /// the provided index sequence.
303 template <typename Fn, std::size_t... Is>
304 bool all_of_tuple_elements(std::index_sequence<Is...>,
305 [[maybe_unused]] Fn P) const {
306 return (P(std::get<Is>(Ops), Is) && ...);
307 }
308};
309
310template <unsigned Opcode, typename... OpTys>
311using AllRecipe_match =
312 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ false,
313 VPWidenRecipe, VPReplicateRecipe, VPWidenCastRecipe,
314 VPInstruction>;
315
316template <unsigned Opcode, typename... OpTys>
317using AllRecipe_commutative_match =
318 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ true,
319 VPWidenRecipe, VPReplicateRecipe, VPInstruction>;
320
321template <unsigned Opcode, typename... OpTys>
322using VPInstruction_match = Recipe_match<std::tuple<OpTys...>, Opcode,
323 /*Commutative*/ false, VPInstruction>;
324
325template <unsigned Opcode, typename... OpTys>
326using VPInstruction_commutative_match =
327 Recipe_match<std::tuple<OpTys...>, Opcode,
328 /*Commutative*/ true, VPInstruction>;
329
330template <unsigned Opcode, typename... OpTys>
331inline VPInstruction_match<Opcode, OpTys...>
332m_VPInstruction(const OpTys &...Ops) {
333 return VPInstruction_match<Opcode, OpTys...>(Ops...);
334}
335
336template <unsigned Opcode, typename Op0_t, typename Op1_t>
337inline VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>
338m_c_VPInstruction(const Op0_t &Op0, const Op1_t &Op1) {
339 return VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
340}
341
342/// BuildVector is matches only its opcode, w/o matching its operands as the
343/// number of operands is not fixed.
344inline VPInstruction_match<VPInstruction::BuildVector> m_BuildVector() {
345 return m_VPInstruction<VPInstruction::BuildVector>();
346}
347
348/// BuildStructVector matches only its opcode, w/o matching its operands as the
349/// number of operands is not fixed.
350inline VPInstruction_match<VPInstruction::BuildStructVector>
351m_BuildStructVector() {
352 return m_VPInstruction<VPInstruction::BuildStructVector>();
353}
354
355template <typename Op0_t>
356inline VPInstruction_match<Instruction::Freeze, Op0_t>
357m_Freeze(const Op0_t &Op0) {
358 return m_VPInstruction<Instruction::Freeze>(Op0);
359}
360
361inline VPInstruction_match<VPInstruction::BranchOnCond> m_BranchOnCond() {
362 return m_VPInstruction<VPInstruction::BranchOnCond>();
363}
364
365template <typename Op0_t>
366inline VPInstruction_match<VPInstruction::BranchOnCond, Op0_t>
367m_BranchOnCond(const Op0_t &Op0) {
368 return m_VPInstruction<VPInstruction::BranchOnCond>(Op0);
369}
370
371inline VPInstruction_match<VPInstruction::BranchOnTwoConds>
372m_BranchOnTwoConds() {
373 return m_VPInstruction<VPInstruction::BranchOnTwoConds>();
374}
375
376template <typename Op0_t, typename Op1_t>
377inline VPInstruction_match<VPInstruction::BranchOnTwoConds, Op0_t, Op1_t>
378m_BranchOnTwoConds(const Op0_t &Op0, const Op1_t &Op1) {
379 return m_VPInstruction<VPInstruction::BranchOnTwoConds>(Op0, Op1);
380}
381
382inline VPInstruction_match<VPInstruction::BranchOnCount> m_BranchOnCount() {
383 return m_VPInstruction<VPInstruction::BranchOnCount>();
384}
385
386template <typename Op0_t, typename Op1_t>
387inline VPInstruction_match<VPInstruction::BranchOnCount, Op0_t, Op1_t>
388m_BranchOnCount(const Op0_t &Op0, const Op1_t &Op1) {
389 return m_VPInstruction<VPInstruction::BranchOnCount>(Op0, Op1);
390}
391
392inline auto m_Branch() {
393 return m_CombineOr(Ps: m_BranchOnCond(), Ps: m_BranchOnCount(), Ps: m_BranchOnTwoConds());
394}
395
396template <typename Op0_t>
397inline VPInstruction_match<VPInstruction::Broadcast, Op0_t>
398m_Broadcast(const Op0_t &Op0) {
399 return m_VPInstruction<VPInstruction::Broadcast>(Op0);
400}
401
402template <typename Op0_t>
403inline VPInstruction_match<VPInstruction::ExplicitVectorLength, Op0_t>
404m_EVL(const Op0_t &Op0) {
405 return m_VPInstruction<VPInstruction::ExplicitVectorLength>(Op0);
406}
407
408template <typename Op0_t>
409inline VPInstruction_match<VPInstruction::ExtractLastLane, Op0_t>
410m_ExtractLastLane(const Op0_t &Op0) {
411 return m_VPInstruction<VPInstruction::ExtractLastLane>(Op0);
412}
413
414template <typename Op0_t, typename Op1_t>
415inline VPInstruction_match<Instruction::ExtractElement, Op0_t, Op1_t>
416m_ExtractElement(const Op0_t &Op0, const Op1_t &Op1) {
417 return m_VPInstruction<Instruction::ExtractElement>(Op0, Op1);
418}
419
420template <typename Op0_t, typename Op1_t, typename Op2_t>
421inline VPInstruction_match<Instruction::InsertElement, Op0_t, Op1_t, Op2_t>
422m_InsertElement(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
423 return m_VPInstruction<Instruction::InsertElement>(Op0, Op1, Op2);
424}
425
426template <typename Op0_t, typename Op1_t>
427inline VPInstruction_match<VPInstruction::ExtractLane, Op0_t, Op1_t>
428m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1) {
429 return m_VPInstruction<VPInstruction::ExtractLane>(Op0, Op1);
430}
431
432template <typename Op0_t>
433inline VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>
434m_ExtractLastPart(const Op0_t &Op0) {
435 return m_VPInstruction<VPInstruction::ExtractLastPart>(Op0);
436}
437
438template <typename Op0_t>
439inline VPInstruction_match<
440 VPInstruction::ExtractLastLane,
441 VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>>
442m_ExtractLastLaneOfLastPart(const Op0_t &Op0) {
443 return m_ExtractLastLane(m_ExtractLastPart(Op0));
444}
445
446template <typename Op0_t, typename Op1_t>
447inline VPInstruction_match<VPInstruction::ExtractVectorForPart, Op0_t, Op1_t>
448m_ExtractVectorForPart(const Op0_t &Op0, const Op1_t &Op1) {
449 return m_VPInstruction<VPInstruction::ExtractVectorForPart>(Op0, Op1);
450}
451
452template <typename Op0_t>
453inline VPInstruction_match<VPInstruction::ExtractPenultimateElement, Op0_t>
454m_ExtractPenultimateElement(const Op0_t &Op0) {
455 return m_VPInstruction<VPInstruction::ExtractPenultimateElement>(Op0);
456}
457
458template <typename Op0_t, typename Op1_t, typename Op2_t>
459inline VPInstruction_match<VPInstruction::WideActiveLaneMask, Op0_t, Op1_t,
460 Op2_t>
461m_WideActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
462 return m_VPInstruction<VPInstruction::WideActiveLaneMask>(Op0, Op1, Op2);
463}
464
465inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() {
466 return m_VPInstruction<VPInstruction::AnyOf>();
467}
468
469template <typename Op0_t>
470inline VPInstruction_match<VPInstruction::AnyOf, Op0_t>
471m_AnyOf(const Op0_t &Op0) {
472 return m_VPInstruction<VPInstruction::AnyOf>(Op0);
473}
474
475template <typename Op0_t>
476inline VPInstruction_match<VPInstruction::FirstActiveLane, Op0_t>
477m_FirstActiveLane(const Op0_t &Op0) {
478 return m_VPInstruction<VPInstruction::FirstActiveLane>(Op0);
479}
480
481template <typename Op0_t>
482inline VPInstruction_match<VPInstruction::LastActiveLane, Op0_t>
483m_LastActiveLane(const Op0_t &Op0) {
484 return m_VPInstruction<VPInstruction::LastActiveLane>(Op0);
485}
486
487template <typename Op0_t, typename Op1_t, typename Op2_t>
488inline VPInstruction_match<VPInstruction::ExtractLastActive, Op0_t, Op1_t,
489 Op2_t>
490m_ExtractLastActive(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
491 return m_VPInstruction<VPInstruction::ExtractLastActive>(Op0, Op1, Op2);
492}
493
494template <typename Op0_t>
495inline VPInstruction_match<VPInstruction::ComputeReductionResult, Op0_t>
496m_ComputeReductionResult(const Op0_t &Op0) {
497 return m_VPInstruction<VPInstruction::ComputeReductionResult>(Op0);
498}
499
500/// Match FindIV result pattern:
501/// select(icmp ne ComputeReductionResult(ReducedIV), Sentinel),
502/// ComputeReductionResult(ReducedIV), Start.
503template <typename Op0_t, typename Op1_t>
504inline bool matchFindIVResult(VPInstruction *VPI, Op0_t ReducedIV, Op1_t Start) {
505 return match(VPI, m_Select(m_SpecificICmp(ICmpInst::ICMP_NE,
506 m_ComputeReductionResult(ReducedIV),
507 m_VPValue()),
508 m_ComputeReductionResult(ReducedIV), Start));
509}
510
511template <typename Op0_t>
512inline VPInstruction_match<VPInstruction::Reverse, Op0_t>
513m_Reverse(const Op0_t &Op0) {
514 return m_VPInstruction<VPInstruction::Reverse>(Op0);
515}
516
517inline VPInstruction_match<VPInstruction::StepVector> m_StepVector() {
518 return m_VPInstruction<VPInstruction::StepVector>();
519}
520
521template <typename Op0_t>
522inline VPInstruction_match<VPInstruction::ExitingIVValue, Op0_t>
523m_ExitingIVValue(const Op0_t &Op0) {
524 return m_VPInstruction<VPInstruction::ExitingIVValue>(Op0);
525}
526
527template <unsigned Opcode, typename Op0_t>
528inline AllRecipe_match<Opcode, Op0_t> m_Unary(const Op0_t &Op0) {
529 return AllRecipe_match<Opcode, Op0_t>(Op0);
530}
531
532template <typename Op0_t>
533inline AllRecipe_match<Instruction::Trunc, Op0_t> m_Trunc(const Op0_t &Op0) {
534 return m_Unary<Instruction::Trunc, Op0_t>(Op0);
535}
536
537template <typename Op0_t>
538inline match_combine_or<AllRecipe_match<Instruction::Trunc, Op0_t>, Op0_t>
539m_TruncOrSelf(const Op0_t &Op0) {
540 return m_CombineOr(m_Trunc(Op0), Op0);
541}
542
543template <typename Op0_t>
544inline AllRecipe_match<Instruction::ZExt, Op0_t> m_ZExt(const Op0_t &Op0) {
545 return m_Unary<Instruction::ZExt, Op0_t>(Op0);
546}
547
548template <typename Op0_t>
549inline AllRecipe_match<Instruction::SExt, Op0_t> m_SExt(const Op0_t &Op0) {
550 return m_Unary<Instruction::SExt, Op0_t>(Op0);
551}
552
553template <typename Op0_t>
554inline AllRecipe_match<Instruction::FPExt, Op0_t> m_FPExt(const Op0_t &Op0) {
555 return m_Unary<Instruction::FPExt, Op0_t>(Op0);
556}
557
558template <typename Op0_t>
559inline AllRecipe_match<Instruction::BitCast, Op0_t>
560m_BitCast(const Op0_t &Op0) {
561 return m_Unary<Instruction::BitCast, Op0_t>(Op0);
562}
563
564template <typename Op0_t>
565inline AllRecipe_match<Instruction::PtrToAddr, Op0_t>
566m_PtrToAddr(const Op0_t &Op0) {
567 return m_Unary<Instruction::PtrToAddr, Op0_t>(Op0);
568}
569
570template <typename Op0_t>
571inline AllRecipe_match<Instruction::FNeg, Op0_t> m_FNeg(const Op0_t &Op0) {
572 return m_Unary<Instruction::FNeg, Op0_t>(Op0);
573}
574
575template <typename Op0_t>
576inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>,
577 AllRecipe_match<Instruction::SExt, Op0_t>>
578m_ZExtOrSExt(const Op0_t &Op0) {
579 return m_CombineOr(m_ZExt(Op0), m_SExt(Op0));
580}
581
582template <typename Op0_t> inline auto m_WidenAnyExtend(const Op0_t &Op0) {
583 return m_Isa<VPWidenCastRecipe>(m_CombineOr(m_ZExtOrSExt(Op0), m_FPExt(Op0)));
584}
585
586template <typename Op0_t> inline auto m_AnyNeg(const Op0_t &Op0) {
587 return m_CombineOr(m_Sub(m_ZeroInt(), Op0), m_FNeg(Op0));
588}
589
590template <typename Op0_t>
591inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>, Op0_t>
592m_ZExtOrSelf(const Op0_t &Op0) {
593 return m_CombineOr(m_ZExt(Op0), Op0);
594}
595
596template <typename Op0_t> inline auto m_ZExtOrTruncOrSelf(const Op0_t &Op0) {
597 return m_CombineOr(m_ZExt(Op0), m_Trunc(Op0), Op0);
598}
599
600template <unsigned Opcode, typename Op0_t, typename Op1_t>
601inline AllRecipe_match<Opcode, Op0_t, Op1_t> m_Binary(const Op0_t &Op0,
602 const Op1_t &Op1) {
603 return AllRecipe_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
604}
605
606template <unsigned Opcode, typename Op0_t, typename Op1_t>
607inline AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>
608m_c_Binary(const Op0_t &Op0, const Op1_t &Op1) {
609 return AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
610}
611
612template <typename Op0_t, typename Op1_t>
613inline AllRecipe_match<Instruction::Add, Op0_t, Op1_t> m_Add(const Op0_t &Op0,
614 const Op1_t &Op1) {
615 return m_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
616}
617
618template <typename Op0_t, typename Op1_t>
619inline AllRecipe_commutative_match<Instruction::Add, Op0_t, Op1_t>
620m_c_Add(const Op0_t &Op0, const Op1_t &Op1) {
621 return m_c_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
622}
623
624template <typename Op0_t, typename Op1_t>
625inline AllRecipe_match<Instruction::Sub, Op0_t, Op1_t> m_Sub(const Op0_t &Op0,
626 const Op1_t &Op1) {
627 return m_Binary<Instruction::Sub, Op0_t, Op1_t>(Op0, Op1);
628}
629
630template <typename Op0_t, typename Op1_t>
631inline AllRecipe_match<Instruction::Mul, Op0_t, Op1_t> m_Mul(const Op0_t &Op0,
632 const Op1_t &Op1) {
633 return m_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
634}
635
636template <typename Op0_t, typename Op1_t>
637inline AllRecipe_commutative_match<Instruction::Mul, Op0_t, Op1_t>
638m_c_Mul(const Op0_t &Op0, const Op1_t &Op1) {
639 return m_c_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
640}
641
642template <typename Op0_t, typename Op1_t>
643inline AllRecipe_match<Instruction::Shl, Op0_t, Op1_t> m_Shl(const Op0_t &Op0,
644 const Op1_t &Op1) {
645 return m_Binary<Instruction::Shl, Op0_t, Op1_t>(Op0, Op1);
646}
647
648template <typename Op0_t, typename Op1_t>
649inline AllRecipe_match<Instruction::LShr, Op0_t, Op1_t>
650m_LShr(const Op0_t &Op0, const Op1_t &Op1) {
651 return m_Binary<Instruction::LShr, Op0_t, Op1_t>(Op0, Op1);
652}
653
654template <typename Op0_t, typename Op1_t>
655inline AllRecipe_match<Instruction::AShr, Op0_t, Op1_t>
656m_AShr(const Op0_t &Op0, const Op1_t &Op1) {
657 return m_Binary<Instruction::AShr, Op0_t, Op1_t>(Op0, Op1);
658}
659
660template <typename Op0_t, typename Op1_t>
661inline AllRecipe_match<Instruction::FMul, Op0_t, Op1_t>
662m_FMul(const Op0_t &Op0, const Op1_t &Op1) {
663 return m_Binary<Instruction::FMul, Op0_t, Op1_t>(Op0, Op1);
664}
665
666template <typename Op0_t, typename Op1_t>
667inline AllRecipe_match<Instruction::FAdd, Op0_t, Op1_t>
668m_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
669 return m_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
670}
671
672template <typename Op0_t, typename Op1_t>
673inline AllRecipe_commutative_match<Instruction::FAdd, Op0_t, Op1_t>
674m_c_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
675 return m_c_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
676}
677
678template <typename Op0_t, typename Op1_t>
679inline AllRecipe_match<Instruction::UDiv, Op0_t, Op1_t>
680m_UDiv(const Op0_t &Op0, const Op1_t &Op1) {
681 return m_Binary<Instruction::UDiv, Op0_t, Op1_t>(Op0, Op1);
682}
683
684template <typename Op0_t, typename Op1_t>
685inline AllRecipe_match<Instruction::URem, Op0_t, Op1_t>
686m_URem(const Op0_t &Op0, const Op1_t &Op1) {
687 return m_Binary<Instruction::URem, Op0_t, Op1_t>(Op0, Op1);
688}
689
690template <typename Op0_t, typename Op1_t>
691inline AllRecipe_match<Instruction::SDiv, Op0_t, Op1_t>
692m_SDiv(const Op0_t &Op0, const Op1_t &Op1) {
693 return m_Binary<Instruction::SDiv, Op0_t, Op1_t>(Op0, Op1);
694}
695
696template <typename Op0_t, typename Op1_t>
697inline AllRecipe_match<Instruction::SRem, Op0_t, Op1_t>
698m_SRem(const Op0_t &Op0, const Op1_t &Op1) {
699 return m_Binary<Instruction::SRem, Op0_t, Op1_t>(Op0, Op1);
700}
701
702/// Match a binary AND operation.
703template <typename Op0_t, typename Op1_t>
704inline AllRecipe_commutative_match<Instruction::And, Op0_t, Op1_t>
705m_c_BinaryAnd(const Op0_t &Op0, const Op1_t &Op1) {
706 return m_c_Binary<Instruction::And, Op0_t, Op1_t>(Op0, Op1);
707}
708
709/// Match a binary OR operation. Note that while conceptually the operands can
710/// be matched commutatively, \p Commutative defaults to false in line with the
711/// IR-based pattern matching infrastructure. Use m_c_BinaryOr for a commutative
712/// version of the matcher.
713template <typename Op0_t, typename Op1_t>
714inline AllRecipe_match<Instruction::Or, Op0_t, Op1_t>
715m_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
716 return m_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
717}
718
719template <typename Op0_t, typename Op1_t>
720inline AllRecipe_commutative_match<Instruction::Or, Op0_t, Op1_t>
721m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
722 return m_c_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
723}
724
725/// Cmp_match is a variant of BinaryRecipe_match that also binds the comparison
726/// predicate. Opcodes must either be Instruction::ICmp or Instruction::FCmp, or
727/// both.
728template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
729struct Cmp_match {
730 static_assert((sizeof...(Opcodes) == 1 || sizeof...(Opcodes) == 2) &&
731 "Expected one or two opcodes");
732 static_assert(
733 ((Opcodes == Instruction::ICmp || Opcodes == Instruction::FCmp) && ...) &&
734 "Expected a compare instruction opcode");
735
736 CmpPredicate *Predicate = nullptr;
737 Op0_t Op0;
738 Op1_t Op1;
739
740 Cmp_match(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1)
741 : Predicate(&Pred), Op0(Op0), Op1(Op1) {}
742 Cmp_match(const Op0_t &Op0, const Op1_t &Op1) : Op0(Op0), Op1(Op1) {}
743
744 bool match(const VPValue *V) const {
745 auto *DefR = V->getDefiningRecipe();
746 return DefR && match(DefR);
747 }
748
749 bool match(const VPRecipeBase *V) const {
750 if ((m_Binary<Opcodes>(Op0, Op1).match(V) || ...)) {
751 if (Predicate)
752 *Predicate = cast<VPRecipeWithIRFlags>(Val: V)->getPredicate();
753 return true;
754 }
755 return false;
756 }
757};
758
759/// SpecificCmp_match is a variant of Cmp_match that matches the comparison
760/// predicate, instead of binding it.
761template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
762struct SpecificCmp_match {
763 const CmpPredicate Predicate;
764 Op0_t Op0;
765 Op1_t Op1;
766
767 SpecificCmp_match(CmpPredicate Pred, const Op0_t &LHS, const Op1_t &RHS)
768 : Predicate(Pred), Op0(LHS), Op1(RHS) {}
769
770 bool match(const VPValue *V) const {
771 auto *DefR = V->getDefiningRecipe();
772 return DefR && match(DefR);
773 }
774
775 bool match(const VPRecipeBase *V) const {
776 CmpPredicate CurrentPred;
777 return Cmp_match<Op0_t, Op1_t, Opcodes...>(CurrentPred, Op0, Op1)
778 .match(V) &&
779 CmpPredicate::getMatching(A: CurrentPred, B: Predicate);
780 }
781};
782
783template <typename Op0_t, typename Op1_t>
784inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp> m_ICmp(const Op0_t &Op0,
785 const Op1_t &Op1) {
786 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Op0, Op1);
787}
788
789template <typename Op0_t, typename Op1_t>
790inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp>
791m_ICmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
792 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Pred, Op0, Op1);
793}
794
795template <typename Op0_t, typename Op1_t>
796inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>
797m_SpecificICmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
798 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>(MatchPred, Op0,
799 Op1);
800}
801
802template <typename Op0_t, typename Op1_t>
803inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
804m_Cmp(const Op0_t &Op0, const Op1_t &Op1) {
805 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(Op0,
806 Op1);
807}
808
809template <typename Op0_t, typename Op1_t>
810inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
811m_Cmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
812 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
813 Pred, Op0, Op1);
814}
815
816template <typename Op0_t, typename Op1_t>
817inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
818m_SpecificCmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
819 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
820 MatchPred, Op0, Op1);
821}
822
823template <typename Op0_t, typename Op1_t>
824inline auto m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1) {
825 return m_CombineOr(
826 Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr,
827 /*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>(
828 Op0, Op1),
829 VPInstruction_match<VPInstruction::PtrAdd, Op0_t, Op1_t>(Op0, Op1),
830 VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>(Op0, Op1));
831}
832
833/// Match a VPBlendRecipe with 2 incoming values ([I0, I1, M1] ==
834/// normalized([I0, M0, I1, M1])) as select(M1, I1, I0), mirroring how it is
835/// lowered.
836template <typename Op0_t, typename Op1_t, typename Op2_t> struct Blend2_match {
837 Op0_t MaskOp;
838 Op1_t TrueOp;
839 Op2_t FalseOp;
840
841 Blend2_match(const Op0_t &MaskOp, const Op1_t &TrueOp, const Op2_t &FalseOp)
842 : MaskOp(MaskOp), TrueOp(TrueOp), FalseOp(FalseOp) {}
843
844 template <typename T> bool match(const T *Val) const {
845 auto *Blend = dyn_cast<VPBlendRecipe>(Val);
846 if (!Blend || Blend->getNumIncomingValues() != 2)
847 return false;
848 return MaskOp.match(Blend->getMask(1)) &&
849 TrueOp.match(Blend->getIncomingValue(1)) &&
850 FalseOp.match(Blend->getIncomingValue(0));
851 }
852};
853
854/// Match recipe recipe with Select opcode, i.e. excluding VPBlendRecipe.
855template <typename Op0_t, typename Op1_t, typename Op2_t>
856inline AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>
857m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
858 return AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>(
859 {Op0, Op1, Op2});
860}
861
862/// Match recipe with Select opcode or an equivalent VPBlendRecipe with 2
863/// incoming values.
864template <typename Op0_t, typename Op1_t, typename Op2_t>
865inline auto m_SelectLike(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
866 return m_CombineOr(m_Select(Op0, Op1, Op2),
867 Blend2_match<Op0_t, Op1_t, Op2_t>(Op0, Op1, Op2));
868}
869
870template <typename Op0_t> inline auto m_Not(const Op0_t &Op0) {
871 return m_CombineOr(m_VPInstruction<VPInstruction::Not>(Op0),
872 m_c_Binary<Instruction::Xor>(m_AllOnes(), Op0));
873}
874
875template <typename Op0_t, typename Op1_t, typename Op2_t>
876inline auto m_c_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
877 return m_CombineOr(m_Select(Op0, Op1, Op2), m_Select(m_Not(Op0), Op2, Op1));
878}
879
880template <typename Op0_t, typename Op1_t>
881inline auto m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
882 return m_CombineOr(
883 m_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
884 m_Select(Op0, Op1, m_False()));
885}
886
887template <typename Op0_t, typename Op1_t> struct RemoveMask_match {
888 Op0_t In;
889 Op1_t &Out;
890
891 RemoveMask_match(const Op0_t &In, Op1_t &Out) : In(In), Out(Out) {}
892
893 template <typename OpTy> bool match(OpTy *V) const {
894 if (m_Specific(In).match(V)) {
895 Out = nullptr;
896 return true;
897 }
898 return m_LogicalAnd(m_Specific(In), m_VPValue(Out)).match(V);
899 }
900};
901
902/// Match a specific mask \p In, or a combination of it (logical-and In, Out).
903/// Returns the remaining part \p Out if so, or nullptr otherwise.
904template <typename Op0_t, typename Op1_t>
905inline RemoveMask_match<Op0_t, Op1_t> m_RemoveMask(const Op0_t &In,
906 Op1_t &Out) {
907 return RemoveMask_match<Op0_t, Op1_t>(In, Out);
908}
909
910template <typename Op0_t, typename Op1_t>
911inline auto m_c_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
912 return m_CombineOr(
913 m_c_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
914 m_c_Select(Op0, Op1, m_False()));
915}
916
917template <typename Op0_t, typename Op1_t>
918inline auto m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
919 return m_CombineOr(
920 m_c_VPInstruction<VPInstruction::LogicalOr, Op0_t, Op1_t>(Op0, Op1),
921 m_Select(Op0, m_True(), Op1));
922}
923
924template <typename Op0_t, typename Op1_t>
925inline auto m_c_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
926 return m_c_Select(Op0, m_True(), Op1);
927}
928
929/// Match the canonical induction variable (IV) of any loop region.
930struct canonical_iv_match {
931 template <typename ArgTy> bool match(const ArgTy *V) const {
932 const auto *RV = dyn_cast<VPRegionValue>(V);
933 return RV && RV->getDefiningRegion()->getCanonicalIV() == RV;
934 }
935};
936
937inline canonical_iv_match m_CanonicalIV() { return {}; }
938
939/// Match the abstract header mask of any loop region.
940struct header_mask_match {
941 template <typename ArgTy> bool match(const ArgTy *V) const {
942 const auto *RV = dyn_cast<VPRegionValue>(V);
943 return RV && RV->getDefiningRegion()->getHeaderMask() == RV;
944 }
945};
946
947inline header_mask_match m_HeaderMask() { return {}; }
948
949/// Match a canonical VPWidenIntOrFpInductionRecipe optionally capturing it.
950struct canonical_widen_iv_match {
951 VPWidenIntOrFpInductionRecipe **Capture = nullptr;
952
953 canonical_widen_iv_match() = default;
954 canonical_widen_iv_match(VPWidenIntOrFpInductionRecipe *&V) : Capture(&V) {}
955
956 template <typename ArgTy> bool match(ArgTy *V) const {
957 auto *WidenIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(V);
958 if (!WidenIV || !WidenIV->isCanonical())
959 return false;
960 if (Capture)
961 *Capture = WidenIV;
962 return true;
963 }
964};
965
966inline canonical_widen_iv_match m_CanonicalWidenIV() { return {}; }
967
968/// Match a canonical VPWidenIntOrFpInductionRecipe, capturing it.
969inline canonical_widen_iv_match
970m_CanonicalWidenIV(VPWidenIntOrFpInductionRecipe *&V) {
971 return canonical_widen_iv_match(V);
972}
973
974template <typename Op0_t, typename Op1_t, typename Op2_t>
975inline auto m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1,
976 const Op2_t &Op2) {
977 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
978 VPScalarIVStepsRecipe>({Op0, Op1, Op2});
979}
980
981template <typename Op0_t, typename Op1_t, typename Op2_t>
982inline auto m_DerivedIV(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
983 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
984 VPDerivedIVRecipe>({Op0, Op1, Op2});
985}
986
987template <typename Addr_t, typename Mask_t> struct Load_match {
988 Addr_t Addr;
989 Mask_t Mask;
990
991 Load_match(Addr_t Addr, Mask_t Mask) : Addr(Addr), Mask(Mask) {}
992
993 template <typename OpTy> bool match(const OpTy *V) const {
994 auto *Load = dyn_cast<VPWidenLoadRecipe>(V);
995 if (!Load || !Addr.match(Load->getAddr()) || !Load->isMasked() ||
996 !Mask.match(Load->getMask()))
997 return false;
998 return true;
999 }
1000};
1001
1002/// Match a (possibly reversed) masked load.
1003template <typename Addr_t, typename Mask_t>
1004inline Load_match<Addr_t, Mask_t> m_MaskedLoad(const Addr_t &Addr,
1005 const Mask_t &Mask) {
1006 return Load_match<Addr_t, Mask_t>(Addr, Mask);
1007}
1008
1009template <typename Addr_t, typename Val_t, typename Mask_t> struct Store_match {
1010 Addr_t Addr;
1011 Val_t Val;
1012 Mask_t Mask;
1013
1014 Store_match(Addr_t Addr, Val_t Val, Mask_t Mask)
1015 : Addr(Addr), Val(Val), Mask(Mask) {}
1016
1017 template <typename OpTy> bool match(const OpTy *V) const {
1018 auto *Store = dyn_cast<VPWidenStoreRecipe>(V);
1019 if (!Store || !Addr.match(Store->getAddr()) ||
1020 !Val.match(Store->getStoredValue()) || !Store->isMasked() ||
1021 !Mask.match(Store->getMask()))
1022 return false;
1023 return true;
1024 }
1025};
1026
1027/// Match a (possibly reversed) masked store.
1028template <typename Addr_t, typename Val_t, typename Mask_t>
1029inline Store_match<Addr_t, Val_t, Mask_t>
1030m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) {
1031 return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask);
1032}
1033
1034template <typename Op0_t, typename Op1_t>
1035using VectorEndPointerRecipe_match =
1036 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
1037 /*Commutative*/ false, VPVectorEndPointerRecipe>;
1038
1039template <typename Op0_t, typename Op1_t>
1040VectorEndPointerRecipe_match<Op0_t, Op1_t> m_VecEndPtr(const Op0_t &Op0,
1041 const Op1_t &Op1) {
1042 return VectorEndPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
1043}
1044
1045/// Match a call argument at a given argument index.
1046template <typename Opnd_t> struct Argument_match {
1047 /// Call argument index to match.
1048 unsigned OpI;
1049 Opnd_t Val;
1050
1051 Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}
1052
1053 template <typename OpTy> bool match(OpTy *V) const {
1054 if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V))
1055 return Val.match(R->getOperand(OpI));
1056 if (const auto *R = dyn_cast<VPWidenCallRecipe>(V))
1057 return Val.match(R->getOperand(OpI));
1058 if (const auto *R = dyn_cast<VPReplicateRecipe>(V))
1059 if (R->getOpcode() == Instruction::Call)
1060 return Val.match(R->getOperand(OpI));
1061 if (const auto *R = dyn_cast<VPInstruction>(V))
1062 if (R->getOpcode() == Instruction::Call ||
1063 R->getOpcode() == VPInstruction::Intrinsic)
1064 return Val.match(R->getOperand(OpI));
1065 return false;
1066 }
1067};
1068
1069/// Match a call argument.
1070template <unsigned OpI, typename Opnd_t>
1071inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) {
1072 return Argument_match<Opnd_t>(OpI, Op);
1073}
1074
1075/// Intrinsic matchers.
1076struct IntrinsicID_match {
1077 unsigned ID;
1078
1079 IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {}
1080
1081 template <typename OpTy> bool match(OpTy *V) const {
1082 return vputils::getIntrinsicID(V) == ID;
1083 }
1084};
1085
1086/// Match intrinsic calls with a runtime intrinsic ID.
1087inline IntrinsicID_match m_Intrinsic(Intrinsic::ID IntrID) {
1088 return IntrinsicID_match(IntrID);
1089}
1090
1091struct IntrinsicMatchImpl {
1092 template <Intrinsic::ID IntrID, typename... Ts, size_t... Is>
1093 static auto impl(std::index_sequence<Is...>, const Ts &...Ops) {
1094 return m_CombineAnd(IntrinsicID_match(IntrID), m_Argument<Is>(Ops)...);
1095 }
1096};
1097
1098/// Match intrinsic calls like this:
1099/// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...)
1100template <Intrinsic::ID IntrID, typename... Ts>
1101inline auto m_Intrinsic(const Ts &...Ops) {
1102 return IntrinsicMatchImpl::impl<IntrID>(
1103 std::make_index_sequence<sizeof...(Ts)>{}, Ops...);
1104}
1105
1106template <Intrinsic::ID IntrID, typename... T>
1107inline auto m_WidenIntrinsic(const T &...Ops) {
1108 return m_Isa<VPWidenIntrinsicRecipe>(m_Intrinsic<IntrID>(Ops...));
1109}
1110
1111/// Match VPValues that represent live-ins: VPIRValues and (plain)
1112/// VPSymbolicValues. VPRegionValues (which inherit from VPSymbolicValue) are
1113/// not live-ins and are excluded.
1114struct LiveIn_match {
1115 template <typename ITy> bool match(ITy *V) const {
1116 return isa<VPIRValue>(V) ||
1117 (isa<VPSymbolicValue>(V) && !isa<VPRegionValue>(V));
1118 }
1119};
1120
1121inline auto m_VScale() { return m_Intrinsic<Intrinsic::vscale>(); }
1122
1123inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
1124
1125/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
1126/// and bind the source element type and operands.
1127struct GetElementPtr_match {
1128 Type *&SourceElementType;
1129 ArrayRef<VPValue *> &Operands;
1130
1131 GetElementPtr_match(Type *&SourceElementType, ArrayRef<VPValue *> &Operands)
1132 : SourceElementType(SourceElementType), Operands(Operands) {}
1133
1134 template <typename ITy> bool match(ITy *V) const {
1135 return matchRecipeAndBind<VPWidenGEPRecipe>(V) ||
1136 matchRecipeAndBind<VPInstruction>(V) ||
1137 matchRecipeAndBind<VPReplicateRecipe>(V);
1138 }
1139
1140private:
1141 template <typename RecipeTy> bool matchRecipeAndBind(const VPValue *V) const {
1142 auto *DefR = dyn_cast<RecipeTy>(V);
1143 if (!DefR)
1144 return false;
1145
1146 if constexpr (std::is_same_v<RecipeTy, VPWidenGEPRecipe>) {
1147 SourceElementType = DefR->getSourceElementType();
1148 } else if (DefR->getOpcode() == Instruction::GetElementPtr) {
1149 SourceElementType = cast<GetElementPtrInst>(DefR->getUnderlyingInstr())
1150 ->getSourceElementType();
1151 } else if constexpr (std::is_same_v<RecipeTy, VPInstruction>) {
1152 if (DefR->getOpcode() == VPInstruction::PtrAdd) {
1153 // PtrAdd is a byte-offset GEP with i8 element type.
1154 LLVMContext &Ctx = DefR->getParent()->getPlan()->getContext();
1155 SourceElementType = Type::getInt8Ty(C&: Ctx);
1156 } else {
1157 return false;
1158 }
1159 } else {
1160 return false;
1161 }
1162
1163 Operands = ArrayRef<VPValue *>(DefR->op_begin(), DefR->op_end());
1164 return true;
1165 }
1166};
1167
1168/// Match a GEP recipe with any number of operands and bind source element type
1169/// and operands.
1170inline GetElementPtr_match m_GetElementPtr(Type *&SourceElementType,
1171 ArrayRef<VPValue *> &Operands) {
1172 return GetElementPtr_match(SourceElementType, Operands);
1173}
1174
1175template <typename SubPattern_t> struct OneUse_match {
1176 SubPattern_t SubPattern;
1177
1178 OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}
1179
1180 template <typename OpTy> bool match(OpTy *V) const {
1181 return V->hasOneUse() && SubPattern.match(V);
1182 }
1183};
1184
1185template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) {
1186 return SubPattern;
1187}
1188
1189inline match_bind<VPReductionPHIRecipe>
1190m_ReductionPhi(VPReductionPHIRecipe *&V) {
1191 return V;
1192}
1193
1194template <typename Op0_t, typename Op1_t>
1195inline auto m_VPPhi(const Op0_t &Op0, const Op1_t &Op1) {
1196 return Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::PHI,
1197 /*Commutative*/ false, VPInstruction>({Op0, Op1});
1198}
1199
1200/// If \p V is used by a recipe matching pattern \p P, return it. Otherwise
1201/// return nullptr;
1202template <typename MatchT>
1203VPRecipeBase *findUserOf(VPValue *V, const MatchT &P) {
1204 auto It = find_if(V->users(), match_fn(P));
1205 return It == V->user_end() ? nullptr : cast<VPRecipeBase>(*It);
1206}
1207
1208/// If \p V is used by a VPInstruction with \p Opcode, return it. Otherwise
1209/// return nullptr.
1210template <unsigned Opcode> VPInstruction *findUserOf(VPValue *V) {
1211 return cast_or_null<VPInstruction>(findUserOf(V, m_VPInstruction<Opcode>()));
1212}
1213
1214template <typename RecipeTy> RecipeTy *findUserOf(VPValue *V) {
1215 return cast_or_null<RecipeTy>(findUserOf(V, m_Isa<RecipeTy>()));
1216}
1217} // namespace llvm::VPlanPatternMatch
1218
1219#endif
1220