1//===- VPlanPatternMatch.h - Match on VPValues and recipes ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides a simple and efficient mechanism for performing general
10// tree-based pattern matches on the VPlan values and recipes, based on
11// LLVM's IR pattern matchers.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
16#define LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
17
18#include "VPlan.h"
19#include "VPlanUtils.h"
20#include "llvm/ADT/STLForwardCompat.h"
21#include "llvm/Support/PatternMatchHelpers.h"
22#include <utility>
23
24namespace llvm::VPlanPatternMatch {
25
26using namespace llvm::PatternMatchHelpers;
27
28template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
29 return P.match(V);
30}
31
32template <typename Pattern> bool match(VPUser *U, const Pattern &P) {
33 auto *R = dyn_cast<VPRecipeBase>(Val: U);
34 return R && match(R, P);
35}
36
37template <typename Pattern> bool match(VPSingleDefRecipe *R, const Pattern &P) {
38 return P.match(static_cast<const VPRecipeBase *>(R));
39}
40
41/// A match functor that can be used as a UnaryPredicate in functional
42/// algorithms like all_of.
43template <typename Pattern> auto match_fn(const Pattern &P) {
44 return [&P](auto *V) { return match(V, P); };
45}
46
47/// Match an arbitrary VPValue and ignore it.
48inline auto m_VPValue() { return m_Isa<VPValue>(); }
49
50/// Match a specified VPValue.
51struct specificval_ty {
52 const VPValue *Val;
53
54 specificval_ty(const VPValue *V) : Val(V) {}
55
56 bool match(const VPValue *VPV) const { return VPV == Val; }
57 bool match(const VPRecipeBase *R) const {
58 return R->getNumDefinedValues() == 1 && R->getVPSingleValue() == Val;
59 }
60};
61
62inline specificval_ty m_Specific(const VPValue *VPV) { return VPV; }
63
64/// Like m_Specific(), but works if the specific value to match is determined
65/// as part of the same match() expression. For example:
66/// m_Mul(m_VPValue(X), m_Specific(X)) is incorrect, because m_Specific() will
67/// bind X before the pattern match starts.
68/// m_Mul(m_VPValue(X), m_Deferred(X)) is correct, and will check against
69/// whichever value m_VPValue(X) populated.
70inline match_deferred<VPValue> m_Deferred(VPValue *const &V) { return V; }
71
72/// Match an integer constant if Pred::isValue returns true for the APInt. \p
73/// BitWidth optionally specifies the bitwidth the matched constant must have.
74/// If it is 0, the matched constant can have any bitwidth.
75template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty {
76 Pred P;
77
78 int_pred_ty(Pred P) : P(std::move(P)) {}
79 int_pred_ty() : P() {}
80
81 bool match(const VPValue *VPV) const {
82 auto *VPI = dyn_cast<VPInstruction>(Val: VPV);
83 if (VPI && VPI->getOpcode() == VPInstruction::Broadcast)
84 VPV = VPI->getOperand(N: 0);
85 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
86 if (!CI)
87 return false;
88
89 if (BitWidth != 0 && CI->getBitWidth() != BitWidth)
90 return false;
91 return P.isValue(CI->getAPInt());
92 }
93};
94
95/// Match a specified signed or unsigned integer value.
96struct is_specific_int {
97 APInt Val;
98 bool IsSigned;
99
100 is_specific_int(APInt Val, bool IsSigned = false)
101 : Val(std::move(Val)), IsSigned(IsSigned) {}
102
103 bool isValue(const APInt &C) const {
104 return APInt::isSameValue(I1: Val, I2: C, SignedCompare: IsSigned);
105 }
106};
107
108template <unsigned Bitwidth = 0>
109using specific_intval = int_pred_ty<is_specific_int, Bitwidth>;
110
111inline specific_intval<0> m_SpecificInt(uint64_t V) {
112 return specific_intval<0>(is_specific_int(APInt(64, V)));
113}
114
115inline specific_intval<0> m_SpecificSInt(int64_t V) {
116 return specific_intval<0>(
117 is_specific_int(APInt(64, V, /*isSigned=*/true), /*IsSigned=*/true));
118}
119
120struct is_all_ones {
121 bool isValue(const APInt &C) const { return C.isAllOnes(); }
122};
123
124/// Match an integer or vector with all bits set.
125/// For vectors, this includes constants with undefined elements.
126inline int_pred_ty<is_all_ones> m_AllOnes() {
127 return int_pred_ty<is_all_ones>();
128}
129
130struct is_zero_int {
131 bool isValue(const APInt &C) const { return C.isZero(); }
132};
133
134struct is_one {
135 bool isValue(const APInt &C) const { return C.isOne(); }
136};
137
138/// Match an integer 0 or a vector with all elements equal to 0.
139/// For vectors, this includes constants with undefined elements.
140inline int_pred_ty<is_zero_int> m_ZeroInt() {
141 return int_pred_ty<is_zero_int>();
142}
143
144/// Match an integer 1 or a vector with all elements equal to 1.
145/// For vectors, this includes constants with undefined elements.
146inline int_pred_ty<is_one> m_One() { return int_pred_ty<is_one>(); }
147
148inline int_pred_ty<is_zero_int, 1> m_False() { return {}; }
149
150inline int_pred_ty<is_one, 1> m_True() { return {}; }
151
152struct bind_apint {
153 const APInt *&Res;
154
155 bind_apint(const APInt *&Res) : Res(Res) {}
156
157 bool match(const VPValue *VPV) const {
158 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
159 if (!CI)
160 return false;
161 Res = &CI->getAPInt();
162 return true;
163 }
164};
165
166inline bind_apint m_APInt(const APInt *&C) { return C; }
167
168struct bind_const_int {
169 uint64_t &Res;
170
171 bind_const_int(uint64_t &Res) : Res(Res) {}
172
173 bool match(const VPValue *VPV) const {
174 const APInt *APConst;
175 if (!bind_apint(APConst).match(VPV))
176 return false;
177 if (auto C = APConst->tryZExtValue()) {
178 Res = *C;
179 return true;
180 }
181 return false;
182 }
183};
184
185struct match_poison {
186 bool match(const VPValue *V) const {
187 return isa<VPIRValue>(Val: V) &&
188 isa<PoisonValue>(Val: cast<VPIRValue>(Val: V)->getValue());
189 }
190};
191
192/// Match a VPIRValue that's poison.
193inline match_poison m_Poison() { return match_poison(); }
194
195/// Match a plain integer constant no wider than 64-bits, capturing it if we
196/// match.
197inline bind_const_int m_ConstantInt(uint64_t &C) { return C; }
198
199/// Match a VPValue, capturing it if we match.
200inline match_bind<VPValue> m_VPValue(VPValue *&V) { return V; }
201
202/// Match against the nested pattern, and capture the value if we match.
203template <typename Op_t> inline auto m_VPValue(VPValue *&V, const Op_t &Op) {
204 return m_CombineAnd(Op, m_VPValue(V));
205}
206
207/// Match a VPIRValue.
208inline match_bind<VPIRValue> m_VPIRValue(VPIRValue *&V) { return V; }
209
210/// Match a VPSingleDefRecipe, capturing if we match.
211inline match_bind<VPSingleDefRecipe>
212m_VPSingleDefRecipe(VPSingleDefRecipe *&V) {
213 return V;
214}
215
216/// Match a VPInstruction, capturing if we match.
217inline match_bind<VPInstruction> m_VPInstruction(VPInstruction *&V) {
218 return V;
219}
220
221template <typename T>
222using hasOpcode_t = decltype(std::declval<T &>().getOpcode()); // NOLINT
223
224template <typename Ops_t, unsigned Opcode, bool Commutative,
225 typename... RecipeTys>
226struct Recipe_match {
227 Ops_t Ops;
228
229 template <typename... OpTy> Recipe_match(OpTy... Ops) : Ops(Ops...) {
230 static_assert(std::tuple_size<Ops_t>::value == sizeof...(Ops) &&
231 "number of operands in constructor doesn't match Ops_t");
232 static_assert((!Commutative || std::tuple_size<Ops_t>::value == 2) &&
233 "only binary ops can be commutative");
234 }
235
236 bool match(const VPValue *V) const {
237 auto *DefR = V->getDefiningRecipe();
238 return DefR && match(DefR);
239 }
240
241 bool match(const VPSingleDefRecipe *R) const {
242 return match(static_cast<const VPRecipeBase *>(R));
243 }
244
245 bool match(const VPRecipeBase *R) const {
246 if (std::tuple_size_v<Ops_t> == 0) {
247 auto *VPI = dyn_cast<VPInstruction>(Val: R);
248 return VPI && VPI->getOpcode() == Opcode;
249 }
250
251 if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...))
252 return false;
253
254 if (R->getNumOperands() < std::tuple_size<Ops_t>::value) {
255 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R);
256 assert(((isa<VPInstruction>(R) &&
257 cast<VPInstruction>(R)->getNumOperandsForOpcode() == -1u) ||
258 (RepR && std::tuple_size_v<Ops_t> ==
259 RepR->getNumOperandsWithoutMask())) &&
260 "non-variadic recipe with matched opcode does not have the "
261 "expected number of operands");
262 return false;
263 }
264
265 // If the recipe has more operands than expected, we only support matching
266 // masked VPInstructions or predicated VPReplicateRecipes, where the number
267 // of operands of the matcher matches the number of operands excluding the
268 // mask.
269 if (R->getNumOperands() > std::tuple_size<Ops_t>::value) {
270 if (auto *VPI = dyn_cast<VPInstruction>(Val: R)) {
271 if (!VPI->isMasked() ||
272 VPI->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
273 return false;
274 } else if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R)) {
275 if (!RepR->isPredicated() ||
276 RepR->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
277 return false;
278 } else {
279 return false;
280 }
281 }
282
283 auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>();
284 if (all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
285 return Op.match(R->getOperand(N: Idx));
286 }))
287 return true;
288
289 return Commutative &&
290 all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
291 return Op.match(R->getOperand(N: R->getNumOperands() - Idx - 1));
292 });
293 }
294
295private:
296 template <typename RecipeTy>
297 static bool matchRecipeAndOpcode(const VPRecipeBase *R) {
298 auto *DefR = dyn_cast<RecipeTy>(R);
299 if constexpr (Opcode && is_detected<hasOpcode_t, RecipeTy>::value)
300 return DefR && DefR->getOpcode() == Opcode;
301 return DefR;
302 }
303
304 /// Helper to check if predicate \p P holds on all tuple elements in Ops using
305 /// the provided index sequence.
306 template <typename Fn, std::size_t... Is>
307 bool all_of_tuple_elements(std::index_sequence<Is...>,
308 [[maybe_unused]] Fn P) const {
309 return (P(std::get<Is>(Ops), Is) && ...);
310 }
311};
312
313template <unsigned Opcode, typename... OpTys>
314using AllRecipe_match =
315 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ false,
316 VPWidenRecipe, VPReplicateRecipe, VPWidenCastRecipe,
317 VPInstruction>;
318
319template <unsigned Opcode, typename... OpTys>
320using AllRecipe_commutative_match =
321 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ true,
322 VPWidenRecipe, VPReplicateRecipe, VPInstruction>;
323
324template <unsigned Opcode, typename... OpTys>
325using VPInstruction_match = Recipe_match<std::tuple<OpTys...>, Opcode,
326 /*Commutative*/ false, VPInstruction>;
327
328template <unsigned Opcode, typename... OpTys>
329using VPInstruction_commutative_match =
330 Recipe_match<std::tuple<OpTys...>, Opcode,
331 /*Commutative*/ true, VPInstruction>;
332
333template <unsigned Opcode, typename... OpTys>
334inline VPInstruction_match<Opcode, OpTys...>
335m_VPInstruction(const OpTys &...Ops) {
336 return VPInstruction_match<Opcode, OpTys...>(Ops...);
337}
338
339template <unsigned Opcode, typename Op0_t, typename Op1_t>
340inline VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>
341m_c_VPInstruction(const Op0_t &Op0, const Op1_t &Op1) {
342 return VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
343}
344
345/// BuildVector is matches only its opcode, w/o matching its operands as the
346/// number of operands is not fixed.
347inline VPInstruction_match<VPInstruction::BuildVector> m_BuildVector() {
348 return m_VPInstruction<VPInstruction::BuildVector>();
349}
350
351template <typename Op0_t>
352inline VPInstruction_match<Instruction::Freeze, Op0_t>
353m_Freeze(const Op0_t &Op0) {
354 return m_VPInstruction<Instruction::Freeze>(Op0);
355}
356
357inline VPInstruction_match<VPInstruction::BranchOnCond> m_BranchOnCond() {
358 return m_VPInstruction<VPInstruction::BranchOnCond>();
359}
360
361template <typename Op0_t>
362inline VPInstruction_match<VPInstruction::BranchOnCond, Op0_t>
363m_BranchOnCond(const Op0_t &Op0) {
364 return m_VPInstruction<VPInstruction::BranchOnCond>(Op0);
365}
366
367inline VPInstruction_match<VPInstruction::BranchOnTwoConds>
368m_BranchOnTwoConds() {
369 return m_VPInstruction<VPInstruction::BranchOnTwoConds>();
370}
371
372template <typename Op0_t, typename Op1_t>
373inline VPInstruction_match<VPInstruction::BranchOnTwoConds, Op0_t, Op1_t>
374m_BranchOnTwoConds(const Op0_t &Op0, const Op1_t &Op1) {
375 return m_VPInstruction<VPInstruction::BranchOnTwoConds>(Op0, Op1);
376}
377
378inline VPInstruction_match<VPInstruction::BranchOnCount> m_BranchOnCount() {
379 return m_VPInstruction<VPInstruction::BranchOnCount>();
380}
381
382template <typename Op0_t, typename Op1_t>
383inline VPInstruction_match<VPInstruction::BranchOnCount, Op0_t, Op1_t>
384m_BranchOnCount(const Op0_t &Op0, const Op1_t &Op1) {
385 return m_VPInstruction<VPInstruction::BranchOnCount>(Op0, Op1);
386}
387
388inline auto m_Branch() {
389 return m_CombineOr(Ps: m_BranchOnCond(), Ps: m_BranchOnCount(), Ps: m_BranchOnTwoConds());
390}
391
392template <typename Op0_t>
393inline VPInstruction_match<VPInstruction::Broadcast, Op0_t>
394m_Broadcast(const Op0_t &Op0) {
395 return m_VPInstruction<VPInstruction::Broadcast>(Op0);
396}
397
398template <typename Op0_t>
399inline VPInstruction_match<VPInstruction::ExplicitVectorLength, Op0_t>
400m_EVL(const Op0_t &Op0) {
401 return m_VPInstruction<VPInstruction::ExplicitVectorLength>(Op0);
402}
403
404template <typename Op0_t>
405inline VPInstruction_match<VPInstruction::ExtractLastLane, Op0_t>
406m_ExtractLastLane(const Op0_t &Op0) {
407 return m_VPInstruction<VPInstruction::ExtractLastLane>(Op0);
408}
409
410template <typename Op0_t, typename Op1_t>
411inline VPInstruction_match<Instruction::ExtractElement, Op0_t, Op1_t>
412m_ExtractElement(const Op0_t &Op0, const Op1_t &Op1) {
413 return m_VPInstruction<Instruction::ExtractElement>(Op0, Op1);
414}
415
416template <typename Op0_t, typename Op1_t, typename Op2_t>
417inline VPInstruction_match<Instruction::InsertElement, Op0_t, Op1_t, Op2_t>
418m_InsertElement(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
419 return m_VPInstruction<Instruction::InsertElement>(Op0, Op1, Op2);
420}
421
422template <typename Op0_t, typename Op1_t>
423inline VPInstruction_match<VPInstruction::ExtractLane, Op0_t, Op1_t>
424m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1) {
425 return m_VPInstruction<VPInstruction::ExtractLane>(Op0, Op1);
426}
427
428template <typename Op0_t>
429inline VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>
430m_ExtractLastPart(const Op0_t &Op0) {
431 return m_VPInstruction<VPInstruction::ExtractLastPart>(Op0);
432}
433
434template <typename Op0_t>
435inline VPInstruction_match<
436 VPInstruction::ExtractLastLane,
437 VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>>
438m_ExtractLastLaneOfLastPart(const Op0_t &Op0) {
439 return m_ExtractLastLane(m_ExtractLastPart(Op0));
440}
441
442template <typename... Op_t>
443inline VPInstruction_match<VPInstruction::ExtractVectorForPart, Op_t...>
444m_ExtractVectorForPart(const Op_t &...Op) {
445 static_assert(sizeof...(Op) == 2 || sizeof...(Op) == 0,
446 "Expected two operands (or none for a wildcard match)");
447 return m_VPInstruction<VPInstruction::ExtractVectorForPart>(Op...);
448}
449
450template <typename Op0_t>
451inline VPInstruction_match<VPInstruction::ExtractPenultimateElement, Op0_t>
452m_ExtractPenultimateElement(const Op0_t &Op0) {
453 return m_VPInstruction<VPInstruction::ExtractPenultimateElement>(Op0);
454}
455
456template <typename... Op_t>
457inline VPInstruction_match<VPInstruction::WideActiveLaneMask, Op_t...>
458m_WideActiveLaneMask(const Op_t &...Op) {
459 static_assert(sizeof...(Op) == 3 || sizeof...(Op) == 0,
460 "Expected three operands (or none for a wildcard match)");
461 return m_VPInstruction<VPInstruction::WideActiveLaneMask>(Op...);
462}
463
464inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() {
465 return m_VPInstruction<VPInstruction::AnyOf>();
466}
467
468template <typename Op0_t>
469inline VPInstruction_match<VPInstruction::AnyOf, Op0_t>
470m_AnyOf(const Op0_t &Op0) {
471 return m_VPInstruction<VPInstruction::AnyOf>(Op0);
472}
473
474template <typename Op0_t>
475inline VPInstruction_match<VPInstruction::FirstActiveLane, Op0_t>
476m_FirstActiveLane(const Op0_t &Op0) {
477 return m_VPInstruction<VPInstruction::FirstActiveLane>(Op0);
478}
479
480template <typename Op0_t>
481inline VPInstruction_match<VPInstruction::LastActiveLane, Op0_t>
482m_LastActiveLane(const Op0_t &Op0) {
483 return m_VPInstruction<VPInstruction::LastActiveLane>(Op0);
484}
485
486template <typename Op0_t, typename Op1_t, typename Op2_t>
487inline VPInstruction_match<VPInstruction::ExtractLastActive, Op0_t, Op1_t,
488 Op2_t>
489m_ExtractLastActive(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
490 return m_VPInstruction<VPInstruction::ExtractLastActive>(Op0, Op1, Op2);
491}
492
493template <typename Op0_t>
494inline VPInstruction_match<VPInstruction::ComputeReductionResult, Op0_t>
495m_ComputeReductionResult(const Op0_t &Op0) {
496 return m_VPInstruction<VPInstruction::ComputeReductionResult>(Op0);
497}
498
499/// Match FindIV result pattern:
500/// select(icmp ne ComputeReductionResult(ReducedIV), Sentinel),
501/// ComputeReductionResult(ReducedIV), Start.
502template <typename Op0_t, typename Op1_t>
503inline bool matchFindIVResult(VPInstruction *VPI, Op0_t ReducedIV, Op1_t Start) {
504 return match(VPI, m_Select(m_SpecificICmp(ICmpInst::ICMP_NE,
505 m_ComputeReductionResult(ReducedIV),
506 m_VPValue()),
507 m_ComputeReductionResult(ReducedIV), Start));
508}
509
510template <typename Op0_t>
511inline VPInstruction_match<VPInstruction::Reverse, Op0_t>
512m_Reverse(const Op0_t &Op0) {
513 return m_VPInstruction<VPInstruction::Reverse>(Op0);
514}
515
516inline VPInstruction_match<VPInstruction::StepVector> m_StepVector() {
517 return m_VPInstruction<VPInstruction::StepVector>();
518}
519
520template <typename Op0_t>
521inline VPInstruction_match<VPInstruction::ExitingIVValue, Op0_t>
522m_ExitingIVValue(const Op0_t &Op0) {
523 return m_VPInstruction<VPInstruction::ExitingIVValue>(Op0);
524}
525
526template <unsigned Opcode, typename Op0_t>
527inline AllRecipe_match<Opcode, Op0_t> m_Unary(const Op0_t &Op0) {
528 return AllRecipe_match<Opcode, Op0_t>(Op0);
529}
530
531template <typename Op0_t>
532inline AllRecipe_match<Instruction::Trunc, Op0_t> m_Trunc(const Op0_t &Op0) {
533 return m_Unary<Instruction::Trunc, Op0_t>(Op0);
534}
535
536template <typename Op0_t>
537inline match_combine_or<AllRecipe_match<Instruction::Trunc, Op0_t>, Op0_t>
538m_TruncOrSelf(const Op0_t &Op0) {
539 return m_CombineOr(m_Trunc(Op0), Op0);
540}
541
542template <typename Op0_t>
543inline AllRecipe_match<Instruction::ZExt, Op0_t> m_ZExt(const Op0_t &Op0) {
544 return m_Unary<Instruction::ZExt, Op0_t>(Op0);
545}
546
547template <typename Op0_t>
548inline AllRecipe_match<Instruction::SExt, Op0_t> m_SExt(const Op0_t &Op0) {
549 return m_Unary<Instruction::SExt, Op0_t>(Op0);
550}
551
552template <typename Op0_t>
553inline AllRecipe_match<Instruction::FPExt, Op0_t> m_FPExt(const Op0_t &Op0) {
554 return m_Unary<Instruction::FPExt, Op0_t>(Op0);
555}
556
557template <typename Op0_t>
558inline AllRecipe_match<Instruction::BitCast, Op0_t>
559m_BitCast(const Op0_t &Op0) {
560 return m_Unary<Instruction::BitCast, Op0_t>(Op0);
561}
562
563template <typename Op0_t>
564inline AllRecipe_match<Instruction::PtrToAddr, Op0_t>
565m_PtrToAddr(const Op0_t &Op0) {
566 return m_Unary<Instruction::PtrToAddr, Op0_t>(Op0);
567}
568
569template <typename Op0_t>
570inline AllRecipe_match<Instruction::FNeg, Op0_t> m_FNeg(const Op0_t &Op0) {
571 return m_Unary<Instruction::FNeg, Op0_t>(Op0);
572}
573
574template <typename Op0_t>
575inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>,
576 AllRecipe_match<Instruction::SExt, Op0_t>>
577m_ZExtOrSExt(const Op0_t &Op0) {
578 return m_CombineOr(m_ZExt(Op0), m_SExt(Op0));
579}
580
581template <typename Op0_t> inline auto m_WidenAnyExtend(const Op0_t &Op0) {
582 return m_Isa<VPWidenCastRecipe>(m_CombineOr(m_ZExtOrSExt(Op0), m_FPExt(Op0)));
583}
584
585template <typename Op0_t> inline auto m_AnyNeg(const Op0_t &Op0) {
586 return m_CombineOr(m_Sub(m_ZeroInt(), Op0), m_FNeg(Op0));
587}
588
589template <typename Op0_t>
590inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>, Op0_t>
591m_ZExtOrSelf(const Op0_t &Op0) {
592 return m_CombineOr(m_ZExt(Op0), Op0);
593}
594
595template <typename Op0_t> inline auto m_ZExtOrTrunc(const Op0_t &Op0) {
596 return m_CombineOr(m_ZExt(Op0), m_Trunc(Op0));
597}
598
599template <typename Op0_t> inline auto m_ZExtOrTruncOrSelf(const Op0_t &Op0) {
600 return m_CombineOr(m_ZExtOrTrunc(Op0), Op0);
601}
602
603template <unsigned Opcode, typename Op0_t, typename Op1_t>
604inline AllRecipe_match<Opcode, Op0_t, Op1_t> m_Binary(const Op0_t &Op0,
605 const Op1_t &Op1) {
606 return AllRecipe_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
607}
608
609template <unsigned Opcode, typename Op0_t, typename Op1_t>
610inline AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>
611m_c_Binary(const Op0_t &Op0, const Op1_t &Op1) {
612 return AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
613}
614
615template <typename Op0_t, typename Op1_t>
616inline AllRecipe_match<Instruction::Add, Op0_t, Op1_t> m_Add(const Op0_t &Op0,
617 const Op1_t &Op1) {
618 return m_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
619}
620
621template <typename Op0_t, typename Op1_t>
622inline AllRecipe_commutative_match<Instruction::Add, Op0_t, Op1_t>
623m_c_Add(const Op0_t &Op0, const Op1_t &Op1) {
624 return m_c_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
625}
626
627template <typename Op0_t, typename Op1_t>
628inline AllRecipe_match<Instruction::Sub, Op0_t, Op1_t> m_Sub(const Op0_t &Op0,
629 const Op1_t &Op1) {
630 return m_Binary<Instruction::Sub, Op0_t, Op1_t>(Op0, Op1);
631}
632
633template <typename Op0_t, typename Op1_t>
634inline AllRecipe_match<Instruction::Mul, Op0_t, Op1_t> m_Mul(const Op0_t &Op0,
635 const Op1_t &Op1) {
636 return m_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
637}
638
639template <typename Op0_t, typename Op1_t>
640inline AllRecipe_commutative_match<Instruction::Mul, Op0_t, Op1_t>
641m_c_Mul(const Op0_t &Op0, const Op1_t &Op1) {
642 return m_c_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
643}
644
645template <typename Op0_t, typename Op1_t>
646inline AllRecipe_match<Instruction::Shl, Op0_t, Op1_t> m_Shl(const Op0_t &Op0,
647 const Op1_t &Op1) {
648 return m_Binary<Instruction::Shl, Op0_t, Op1_t>(Op0, Op1);
649}
650
651template <typename Op0_t, typename Op1_t>
652inline AllRecipe_match<Instruction::LShr, Op0_t, Op1_t>
653m_LShr(const Op0_t &Op0, const Op1_t &Op1) {
654 return m_Binary<Instruction::LShr, Op0_t, Op1_t>(Op0, Op1);
655}
656
657template <typename Op0_t, typename Op1_t>
658inline AllRecipe_match<Instruction::AShr, Op0_t, Op1_t>
659m_AShr(const Op0_t &Op0, const Op1_t &Op1) {
660 return m_Binary<Instruction::AShr, Op0_t, Op1_t>(Op0, Op1);
661}
662
663template <typename Op0_t, typename Op1_t>
664inline AllRecipe_match<Instruction::FMul, Op0_t, Op1_t>
665m_FMul(const Op0_t &Op0, const Op1_t &Op1) {
666 return m_Binary<Instruction::FMul, Op0_t, Op1_t>(Op0, Op1);
667}
668
669template <typename Op0_t, typename Op1_t>
670inline AllRecipe_match<Instruction::FAdd, Op0_t, Op1_t>
671m_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
672 return m_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
673}
674
675template <typename Op0_t, typename Op1_t>
676inline AllRecipe_commutative_match<Instruction::FAdd, Op0_t, Op1_t>
677m_c_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
678 return m_c_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
679}
680
681template <typename Op0_t, typename Op1_t>
682inline AllRecipe_match<Instruction::UDiv, Op0_t, Op1_t>
683m_UDiv(const Op0_t &Op0, const Op1_t &Op1) {
684 return m_Binary<Instruction::UDiv, Op0_t, Op1_t>(Op0, Op1);
685}
686
687template <typename Op0_t, typename Op1_t>
688inline AllRecipe_match<Instruction::URem, Op0_t, Op1_t>
689m_URem(const Op0_t &Op0, const Op1_t &Op1) {
690 return m_Binary<Instruction::URem, Op0_t, Op1_t>(Op0, Op1);
691}
692
693template <typename Op0_t, typename Op1_t>
694inline AllRecipe_match<Instruction::SDiv, Op0_t, Op1_t>
695m_SDiv(const Op0_t &Op0, const Op1_t &Op1) {
696 return m_Binary<Instruction::SDiv, Op0_t, Op1_t>(Op0, Op1);
697}
698
699template <typename Op0_t, typename Op1_t>
700inline AllRecipe_match<Instruction::SRem, Op0_t, Op1_t>
701m_SRem(const Op0_t &Op0, const Op1_t &Op1) {
702 return m_Binary<Instruction::SRem, Op0_t, Op1_t>(Op0, Op1);
703}
704
705/// Match a binary AND operation.
706template <typename Op0_t, typename Op1_t>
707inline AllRecipe_commutative_match<Instruction::And, Op0_t, Op1_t>
708m_c_BinaryAnd(const Op0_t &Op0, const Op1_t &Op1) {
709 return m_c_Binary<Instruction::And, Op0_t, Op1_t>(Op0, Op1);
710}
711
712/// Match a binary OR operation. Note that while conceptually the operands can
713/// be matched commutatively, \p Commutative defaults to false in line with the
714/// IR-based pattern matching infrastructure. Use m_c_BinaryOr for a commutative
715/// version of the matcher.
716template <typename Op0_t, typename Op1_t>
717inline AllRecipe_match<Instruction::Or, Op0_t, Op1_t>
718m_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
719 return m_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
720}
721
722template <typename Op0_t, typename Op1_t>
723inline AllRecipe_commutative_match<Instruction::Or, Op0_t, Op1_t>
724m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
725 return m_c_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
726}
727
728/// Cmp_match is a variant of BinaryRecipe_match that also binds the comparison
729/// predicate. Opcodes must either be Instruction::ICmp or Instruction::FCmp, or
730/// both.
731template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
732struct Cmp_match {
733 static_assert((sizeof...(Opcodes) == 1 || sizeof...(Opcodes) == 2) &&
734 "Expected one or two opcodes");
735 static_assert(
736 ((Opcodes == Instruction::ICmp || Opcodes == Instruction::FCmp) && ...) &&
737 "Expected a compare instruction opcode");
738
739 CmpPredicate *Predicate = nullptr;
740 Op0_t Op0;
741 Op1_t Op1;
742
743 Cmp_match(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1)
744 : Predicate(&Pred), Op0(Op0), Op1(Op1) {}
745 Cmp_match(const Op0_t &Op0, const Op1_t &Op1) : Op0(Op0), Op1(Op1) {}
746
747 bool match(const VPValue *V) const {
748 auto *DefR = V->getDefiningRecipe();
749 return DefR && match(DefR);
750 }
751
752 bool match(const VPRecipeBase *V) const {
753 if ((m_Binary<Opcodes>(Op0, Op1).match(V) || ...)) {
754 if (Predicate)
755 *Predicate = cast<VPRecipeWithIRFlags>(Val: V)->getPredicate();
756 return true;
757 }
758 return false;
759 }
760};
761
762/// SpecificCmp_match is a variant of Cmp_match that matches the comparison
763/// predicate, instead of binding it.
764template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
765struct SpecificCmp_match {
766 const CmpPredicate Predicate;
767 Op0_t Op0;
768 Op1_t Op1;
769
770 SpecificCmp_match(CmpPredicate Pred, const Op0_t &LHS, const Op1_t &RHS)
771 : Predicate(Pred), Op0(LHS), Op1(RHS) {}
772
773 bool match(const VPValue *V) const {
774 auto *DefR = V->getDefiningRecipe();
775 return DefR && match(DefR);
776 }
777
778 bool match(const VPRecipeBase *V) const {
779 CmpPredicate CurrentPred;
780 return Cmp_match<Op0_t, Op1_t, Opcodes...>(CurrentPred, Op0, Op1)
781 .match(V) &&
782 CmpPredicate::getMatching(A: CurrentPred, B: Predicate);
783 }
784};
785
786template <typename Op0_t, typename Op1_t>
787inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp> m_ICmp(const Op0_t &Op0,
788 const Op1_t &Op1) {
789 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Op0, Op1);
790}
791
792template <typename Op0_t, typename Op1_t>
793inline auto m_c_ICmp(const Op0_t &Op0, const Op1_t &Op1) {
794 return m_CombineOr(Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Op0, Op1),
795 Cmp_match<Op1_t, Op0_t, Instruction::ICmp>(Op1, Op0));
796}
797
798template <typename Op0_t, typename Op1_t>
799inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp>
800m_ICmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
801 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Pred, Op0, Op1);
802}
803
804template <typename Op0_t, typename Op1_t>
805inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>
806m_SpecificICmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
807 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>(MatchPred, Op0,
808 Op1);
809}
810
811template <typename Op0_t, typename Op1_t>
812inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
813m_Cmp(const Op0_t &Op0, const Op1_t &Op1) {
814 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(Op0,
815 Op1);
816}
817
818template <typename Op0_t, typename Op1_t>
819inline auto m_c_Cmp(const Op0_t &Op0, const Op1_t &Op1) {
820 return m_CombineOr(
821 Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(Op0, Op1),
822 Cmp_match<Op1_t, Op0_t, Instruction::ICmp, Instruction::FCmp>(Op1, Op0));
823}
824
825template <typename Op0_t, typename Op1_t>
826inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
827m_Cmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
828 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
829 Pred, Op0, Op1);
830}
831
832template <typename Op0_t, typename Op1_t>
833inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
834m_SpecificCmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
835 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
836 MatchPred, Op0, Op1);
837}
838
839template <typename Op0_t, typename Op1_t>
840inline auto m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1) {
841 return m_CombineOr(
842 Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr,
843 /*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>(
844 Op0, Op1),
845 VPInstruction_match<VPInstruction::PtrAdd, Op0_t, Op1_t>(Op0, Op1),
846 VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>(Op0, Op1));
847}
848
849/// Match a VPBlendRecipe with 2 incoming values ([I0, I1, M1] ==
850/// normalized([I0, M0, I1, M1])) as select(M1, I1, I0), mirroring how it is
851/// lowered.
852template <typename Op0_t, typename Op1_t, typename Op2_t> struct Blend2_match {
853 Op0_t MaskOp;
854 Op1_t TrueOp;
855 Op2_t FalseOp;
856
857 Blend2_match(const Op0_t &MaskOp, const Op1_t &TrueOp, const Op2_t &FalseOp)
858 : MaskOp(MaskOp), TrueOp(TrueOp), FalseOp(FalseOp) {}
859
860 template <typename T> bool match(const T *Val) const {
861 auto *Blend = dyn_cast<VPBlendRecipe>(Val);
862 if (!Blend || Blend->getNumIncomingValues() != 2)
863 return false;
864 return MaskOp.match(Blend->getMask(1)) &&
865 TrueOp.match(Blend->getIncomingValue(1)) &&
866 FalseOp.match(Blend->getIncomingValue(0));
867 }
868};
869
870/// Match recipe recipe with Select opcode, i.e. excluding VPBlendRecipe.
871template <typename Op0_t, typename Op1_t, typename Op2_t>
872inline AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>
873m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
874 return AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>(
875 {Op0, Op1, Op2});
876}
877
878/// Match recipe with Select opcode or an equivalent VPBlendRecipe with 2
879/// incoming values.
880template <typename Op0_t, typename Op1_t, typename Op2_t>
881inline auto m_SelectLike(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
882 return m_CombineOr(m_Select(Op0, Op1, Op2),
883 Blend2_match<Op0_t, Op1_t, Op2_t>(Op0, Op1, Op2));
884}
885
886template <typename Op0_t> inline auto m_Not(const Op0_t &Op0) {
887 return m_CombineOr(m_VPInstruction<VPInstruction::Not>(Op0),
888 m_c_Binary<Instruction::Xor>(m_AllOnes(), Op0));
889}
890
891template <typename Op0_t, typename Op1_t, typename Op2_t>
892inline auto m_c_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
893 return m_CombineOr(m_Select(Op0, Op1, Op2), m_Select(m_Not(Op0), Op2, Op1));
894}
895
896template <typename Op0_t, typename Op1_t>
897inline auto m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
898 return m_CombineOr(
899 m_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
900 m_Select(Op0, Op1, m_False()));
901}
902
903template <typename Op0_t, typename Op1_t> struct RemoveMask_match {
904 Op0_t In;
905 Op1_t &Out;
906
907 RemoveMask_match(const Op0_t &In, Op1_t &Out) : In(In), Out(Out) {}
908
909 template <typename OpTy> bool match(OpTy *V) const {
910 if (m_Specific(In).match(V)) {
911 Out = nullptr;
912 return true;
913 }
914 return m_LogicalAnd(m_Specific(In), m_VPValue(Out)).match(V);
915 }
916};
917
918/// Match a specific mask \p In, or a combination of it (logical-and In, Out).
919/// Returns the remaining part \p Out if so, or nullptr otherwise.
920template <typename Op0_t, typename Op1_t>
921inline RemoveMask_match<Op0_t, Op1_t> m_RemoveMask(const Op0_t &In,
922 Op1_t &Out) {
923 return RemoveMask_match<Op0_t, Op1_t>(In, Out);
924}
925
926template <typename Op0_t, typename Op1_t>
927inline auto m_c_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
928 return m_CombineOr(
929 m_c_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
930 m_c_Select(Op0, Op1, m_False()));
931}
932
933template <typename Op0_t, typename Op1_t>
934inline auto m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
935 return m_CombineOr(
936 m_c_VPInstruction<VPInstruction::LogicalOr, Op0_t, Op1_t>(Op0, Op1),
937 m_Select(Op0, m_True(), Op1));
938}
939
940template <typename Op0_t, typename Op1_t>
941inline auto m_c_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
942 return m_CombineOr(m_c_Select(Op0, m_True(), Op1),
943 m_c_Select(Op1, m_True(), Op0));
944}
945
946/// Match the canonical induction variable (IV) of any loop region.
947struct canonical_iv_match {
948 template <typename ArgTy> bool match(const ArgTy *V) const {
949 const auto *RV = dyn_cast<VPRegionValue>(V);
950 return RV && RV->getDefiningRegion()->getCanonicalIV() == RV;
951 }
952};
953
954inline canonical_iv_match m_CanonicalIV() { return {}; }
955
956/// Match the abstract header mask of any loop region.
957struct header_mask_match {
958 template <typename ArgTy> bool match(const ArgTy *V) const {
959 const auto *RV = dyn_cast<VPRegionValue>(V);
960 return RV && RV->getDefiningRegion()->getHeaderMask() == RV;
961 }
962};
963
964inline header_mask_match m_HeaderMask() { return {}; }
965
966/// Match a canonical VPWidenIntOrFpInductionRecipe optionally capturing it.
967struct canonical_widen_iv_match {
968 VPWidenIntOrFpInductionRecipe **Capture = nullptr;
969
970 canonical_widen_iv_match() = default;
971 canonical_widen_iv_match(VPWidenIntOrFpInductionRecipe *&V) : Capture(&V) {}
972
973 template <typename ArgTy> bool match(ArgTy *V) const {
974 auto *WidenIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(V);
975 if (!WidenIV || !WidenIV->isCanonical())
976 return false;
977 if (Capture)
978 *Capture = WidenIV;
979 return true;
980 }
981};
982
983inline canonical_widen_iv_match m_CanonicalWidenIV() { return {}; }
984
985/// Match a canonical VPWidenIntOrFpInductionRecipe, capturing it.
986inline canonical_widen_iv_match
987m_CanonicalWidenIV(VPWidenIntOrFpInductionRecipe *&V) {
988 return canonical_widen_iv_match(V);
989}
990
991template <typename Op0_t, typename Op1_t, typename Op2_t>
992inline auto m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1,
993 const Op2_t &Op2) {
994 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
995 VPScalarIVStepsRecipe>({Op0, Op1, Op2});
996}
997
998template <typename Op0_t, typename Op1_t, typename Op2_t>
999inline auto m_DerivedIV(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
1000 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
1001 VPDerivedIVRecipe>({Op0, Op1, Op2});
1002}
1003
1004template <typename Addr_t, typename Mask_t> struct Load_match {
1005 Addr_t Addr;
1006 Mask_t Mask;
1007
1008 Load_match(Addr_t Addr, Mask_t Mask) : Addr(Addr), Mask(Mask) {}
1009
1010 template <typename OpTy> bool match(const OpTy *V) const {
1011 auto *Load = dyn_cast<VPWidenLoadRecipe>(V);
1012 if (!Load || !Addr.match(Load->getAddr()) || !Load->isMasked() ||
1013 !Mask.match(Load->getMask()))
1014 return false;
1015 return true;
1016 }
1017};
1018
1019/// Match a (possibly reversed) masked load.
1020template <typename Addr_t, typename Mask_t>
1021inline Load_match<Addr_t, Mask_t> m_MaskedLoad(const Addr_t &Addr,
1022 const Mask_t &Mask) {
1023 return Load_match<Addr_t, Mask_t>(Addr, Mask);
1024}
1025
1026template <typename Addr_t, typename Val_t, typename Mask_t> struct Store_match {
1027 Addr_t Addr;
1028 Val_t Val;
1029 Mask_t Mask;
1030
1031 Store_match(Addr_t Addr, Val_t Val, Mask_t Mask)
1032 : Addr(Addr), Val(Val), Mask(Mask) {}
1033
1034 template <typename OpTy> bool match(const OpTy *V) const {
1035 auto *Store = dyn_cast<VPWidenStoreRecipe>(V);
1036 if (!Store || !Addr.match(Store->getAddr()) ||
1037 !Val.match(Store->getStoredValue()) || !Store->isMasked() ||
1038 !Mask.match(Store->getMask()))
1039 return false;
1040 return true;
1041 }
1042};
1043
1044/// Match a (possibly reversed) masked store.
1045template <typename Addr_t, typename Val_t, typename Mask_t>
1046inline Store_match<Addr_t, Val_t, Mask_t>
1047m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) {
1048 return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask);
1049}
1050
1051template <typename Op0_t, typename Op1_t>
1052using VectorPointerRecipe_match =
1053 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
1054 /*Commutative*/ false, VPVectorPointerRecipe>;
1055
1056template <typename Op0_t, typename Op1_t>
1057VectorPointerRecipe_match<Op0_t, Op1_t> m_VecPtr(const Op0_t &Op0,
1058 const Op1_t &Op1) {
1059 return VectorPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
1060}
1061
1062template <typename Op0_t, typename Op1_t>
1063using ActiveLaneMaskPHIRecipe_match =
1064 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
1065 /*Commutative*/ false, VPActiveLaneMaskPHIRecipe>;
1066
1067template <typename Op0_t, typename Op1_t>
1068ActiveLaneMaskPHIRecipe_match<Op0_t, Op1_t>
1069m_ActiveLaneMaskPhi(const Op0_t &Op0, const Op1_t &Op1) {
1070 return ActiveLaneMaskPHIRecipe_match<Op0_t, Op1_t>(Op0, Op1);
1071}
1072
1073template <typename Op0_t>
1074using VPWidenLoadRecipe_match =
1075 Recipe_match<std::tuple<Op0_t>, 0,
1076 /*Commutative*/ false, VPWidenLoadRecipe>;
1077
1078template <typename Op0_t>
1079VPWidenLoadRecipe_match<Op0_t> m_WidenLoad(const Op0_t &Op0) {
1080 return VPWidenLoadRecipe_match<Op0_t>(Op0);
1081}
1082
1083template <typename Op0_t, typename Op1_t>
1084using VPWidenStoreRecipe_match =
1085 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
1086 /*Commutative*/ false, VPWidenStoreRecipe>;
1087
1088template <typename Op0_t, typename Op1_t>
1089VPWidenStoreRecipe_match<Op0_t, Op1_t> m_WidenStore(const Op0_t &Op0,
1090 const Op1_t &Op1) {
1091 return VPWidenStoreRecipe_match<Op0_t, Op1_t>(Op0, Op1);
1092}
1093
1094template <typename Op0_t, typename Op1_t>
1095using VectorEndPointerRecipe_match =
1096 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
1097 /*Commutative*/ false, VPVectorEndPointerRecipe>;
1098
1099template <typename Op0_t, typename Op1_t>
1100VectorEndPointerRecipe_match<Op0_t, Op1_t> m_VecEndPtr(const Op0_t &Op0,
1101 const Op1_t &Op1) {
1102 return VectorEndPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
1103}
1104
1105/// Match a call argument at a given argument index.
1106template <typename Opnd_t> struct Argument_match {
1107 /// Call argument index to match.
1108 unsigned OpI;
1109 Opnd_t Val;
1110
1111 Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}
1112
1113 template <typename OpTy> bool match(OpTy *V) const {
1114 if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V))
1115 return Val.match(R->getOperand(OpI));
1116 if (const auto *R = dyn_cast<VPWidenCallRecipe>(V))
1117 return Val.match(R->getOperand(OpI));
1118 if (const auto *R = dyn_cast<VPReplicateRecipe>(V))
1119 if (R->getOpcode() == Instruction::Call)
1120 return Val.match(R->getOperand(OpI));
1121 if (const auto *R = dyn_cast<VPInstruction>(V))
1122 if (R->getOpcode() == Instruction::Call ||
1123 R->getOpcode() == VPInstruction::Intrinsic)
1124 return Val.match(R->getOperand(OpI));
1125 return false;
1126 }
1127};
1128
1129/// Match a call argument.
1130template <unsigned OpI, typename Opnd_t>
1131inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) {
1132 return Argument_match<Opnd_t>(OpI, Op);
1133}
1134
1135/// Intrinsic matchers.
1136struct IntrinsicID_match {
1137 unsigned ID;
1138
1139 IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {}
1140
1141 template <typename OpTy> bool match(OpTy *V) const {
1142 return vputils::getIntrinsicID(V) == ID;
1143 }
1144};
1145
1146/// Match intrinsic calls with a runtime intrinsic ID.
1147inline IntrinsicID_match m_Intrinsic(Intrinsic::ID IntrID) {
1148 return IntrinsicID_match(IntrID);
1149}
1150
1151struct IntrinsicMatchImpl {
1152 template <Intrinsic::ID IntrID, typename... Ts, size_t... Is>
1153 static auto impl(std::index_sequence<Is...>, const Ts &...Ops) {
1154 return m_CombineAnd(IntrinsicID_match(IntrID), m_Argument<Is>(Ops)...);
1155 }
1156};
1157
1158/// Match intrinsic calls like this:
1159/// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...)
1160template <Intrinsic::ID IntrID, typename... Ts>
1161inline auto m_Intrinsic(const Ts &...Ops) {
1162 return IntrinsicMatchImpl::impl<IntrID>(
1163 std::make_index_sequence<sizeof...(Ts)>{}, Ops...);
1164}
1165
1166template <Intrinsic::ID IntrID, typename... T>
1167inline auto m_WidenIntrinsic(const T &...Ops) {
1168 return m_Isa<VPWidenIntrinsicRecipe>(m_Intrinsic<IntrID>(Ops...));
1169}
1170
1171/// Match VPValues that represent live-ins: VPIRValues and (plain)
1172/// VPSymbolicValues. VPRegionValues (which inherit from VPSymbolicValue) are
1173/// not live-ins and are excluded.
1174struct LiveIn_match {
1175 template <typename ITy> bool match(ITy *V) const {
1176 return isa<VPIRValue>(V) ||
1177 (isa<VPSymbolicValue>(V) && !isa<VPRegionValue>(V));
1178 }
1179};
1180
1181inline auto m_VScale() { return m_Intrinsic<Intrinsic::vscale>(); }
1182
1183inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
1184
1185/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
1186/// and bind the source element type and operands.
1187struct GetElementPtr_match {
1188 Type *&SourceElementType;
1189 ArrayRef<VPValue *> &Operands;
1190
1191 GetElementPtr_match(Type *&SourceElementType, ArrayRef<VPValue *> &Operands)
1192 : SourceElementType(SourceElementType), Operands(Operands) {}
1193
1194 template <typename ITy> bool match(ITy *V) const {
1195 return matchRecipeAndBind<VPWidenGEPRecipe>(V) ||
1196 matchRecipeAndBind<VPInstruction>(V) ||
1197 matchRecipeAndBind<VPReplicateRecipe>(V);
1198 }
1199
1200private:
1201 template <typename RecipeTy> bool matchRecipeAndBind(const VPValue *V) const {
1202 auto *DefR = dyn_cast<RecipeTy>(V);
1203 if (!DefR)
1204 return false;
1205
1206 if constexpr (std::is_same_v<RecipeTy, VPWidenGEPRecipe>) {
1207 SourceElementType = DefR->getSourceElementType();
1208 } else if (DefR->getOpcode() == Instruction::GetElementPtr) {
1209 SourceElementType = cast<GetElementPtrInst>(DefR->getUnderlyingInstr())
1210 ->getSourceElementType();
1211 } else if constexpr (std::is_same_v<RecipeTy, VPInstruction>) {
1212 if (DefR->getOpcode() == VPInstruction::PtrAdd) {
1213 // PtrAdd is a byte-offset GEP with i8 element type.
1214 LLVMContext &Ctx = DefR->getParent()->getPlan()->getContext();
1215 SourceElementType = Type::getInt8Ty(C&: Ctx);
1216 } else {
1217 return false;
1218 }
1219 } else {
1220 return false;
1221 }
1222
1223 Operands = ArrayRef<VPValue *>(DefR->op_begin(), DefR->op_end());
1224 return true;
1225 }
1226};
1227
1228/// Match a GEP recipe with any number of operands and bind source element type
1229/// and operands.
1230inline GetElementPtr_match m_GetElementPtr(Type *&SourceElementType,
1231 ArrayRef<VPValue *> &Operands) {
1232 return GetElementPtr_match(SourceElementType, Operands);
1233}
1234
1235template <typename SubPattern_t> struct OneUse_match {
1236 SubPattern_t SubPattern;
1237
1238 OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}
1239
1240 template <typename OpTy> bool match(OpTy *V) const {
1241 return V->hasOneUse() && SubPattern.match(V);
1242 }
1243};
1244
1245template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) {
1246 return SubPattern;
1247}
1248
1249inline match_bind<VPReductionPHIRecipe>
1250m_ReductionPhi(VPReductionPHIRecipe *&V) {
1251 return V;
1252}
1253
1254template <typename Op0_t, typename Op1_t>
1255inline auto m_VPPhi(const Op0_t &Op0, const Op1_t &Op1) {
1256 return Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::PHI,
1257 /*Commutative*/ false, VPInstruction>({Op0, Op1});
1258}
1259
1260/// Return the sole user of \p V matching pattern \p P, casting it to \p
1261/// RecipeTy, or nullptr.
1262template <typename RecipeTy, typename MatchT>
1263RecipeTy *findSingleUser(VPValue *V, const MatchT &P) {
1264 return find_singleton<RecipeTy>(V->users(), [&P](VPUser *U, bool) {
1265 return match(U, P) ? cast<RecipeTy>(U) : nullptr;
1266 });
1267}
1268
1269/// Return the sole user of \p V that is a VPInstruction with \p Opcode, or
1270/// nullptr.
1271template <unsigned Opcode> VPInstruction *findSingleUser(VPValue *V) {
1272 return findSingleUser<VPInstruction>(V, m_VPInstruction<Opcode>());
1273}
1274
1275/// Return the sole user of \p V that is a \p RecipeTy, or nullptr.
1276template <typename RecipeTy> RecipeTy *findSingleUser(VPValue *V) {
1277 return findSingleUser<RecipeTy>(V, m_Isa<RecipeTy>());
1278}
1279} // namespace llvm::VPlanPatternMatch
1280
1281#endif
1282