1//===- VPlanPatternMatch.h - Match on VPValues and recipes ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file provides a simple and efficient mechanism for performing general
10// tree-based pattern matches on the VPlan values and recipes, based on
11// LLVM's IR pattern matchers.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
16#define LLVM_TRANSFORM_VECTORIZE_VPLANPATTERNMATCH_H
17
18#include "VPlan.h"
19#include "VPlanUtils.h"
20#include "llvm/ADT/STLForwardCompat.h"
21#include "llvm/Support/PatternMatchHelpers.h"
22#include <utility>
23
24namespace llvm::VPlanPatternMatch {
25
26using namespace llvm::PatternMatchHelpers;
27
28template <typename Val, typename Pattern> bool match(Val *V, const Pattern &P) {
29 return P.match(V);
30}
31
32template <typename Pattern> bool match(VPUser *U, const Pattern &P) {
33 auto *R = dyn_cast<VPRecipeBase>(Val: U);
34 return R && match(R, P);
35}
36
37template <typename Pattern> bool match(VPSingleDefRecipe *R, const Pattern &P) {
38 return P.match(static_cast<const VPRecipeBase *>(R));
39}
40
41/// A match functor that can be used as a UnaryPredicate in functional
42/// algorithms like all_of.
43template <typename Pattern> auto match_fn(const Pattern &P) {
44 return [&P](auto *V) { return match(V, P); };
45}
46
47/// Match an arbitrary VPValue and ignore it.
48inline auto m_VPValue() { return m_Isa<VPValue>(); }
49
50/// Match a specified VPValue.
51struct specificval_ty {
52 const VPValue *Val;
53
54 specificval_ty(const VPValue *V) : Val(V) {}
55
56 bool match(const VPValue *VPV) const { return VPV == Val; }
57};
58
59inline specificval_ty m_Specific(const VPValue *VPV) { return VPV; }
60
61/// Like m_Specific(), but works if the specific value to match is determined
62/// as part of the same match() expression. For example:
63/// m_Mul(m_VPValue(X), m_Specific(X)) is incorrect, because m_Specific() will
64/// bind X before the pattern match starts.
65/// m_Mul(m_VPValue(X), m_Deferred(X)) is correct, and will check against
66/// whichever value m_VPValue(X) populated.
67inline match_deferred<VPValue> m_Deferred(VPValue *const &V) { return V; }
68
69/// Match an integer constant if Pred::isValue returns true for the APInt. \p
70/// BitWidth optionally specifies the bitwidth the matched constant must have.
71/// If it is 0, the matched constant can have any bitwidth.
72template <typename Pred, unsigned BitWidth = 0> struct int_pred_ty {
73 Pred P;
74
75 int_pred_ty(Pred P) : P(std::move(P)) {}
76 int_pred_ty() : P() {}
77
78 bool match(const VPValue *VPV) const {
79 auto *VPI = dyn_cast<VPInstruction>(Val: VPV);
80 if (VPI && VPI->getOpcode() == VPInstruction::Broadcast)
81 VPV = VPI->getOperand(N: 0);
82 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
83 if (!CI)
84 return false;
85
86 if (BitWidth != 0 && CI->getBitWidth() != BitWidth)
87 return false;
88 return P.isValue(CI->getAPInt());
89 }
90};
91
92/// Match a specified signed or unsigned integer value.
93struct is_specific_int {
94 APInt Val;
95 bool IsSigned;
96
97 is_specific_int(APInt Val, bool IsSigned = false)
98 : Val(std::move(Val)), IsSigned(IsSigned) {}
99
100 bool isValue(const APInt &C) const {
101 return APInt::isSameValue(I1: Val, I2: C, SignedCompare: IsSigned);
102 }
103};
104
105template <unsigned Bitwidth = 0>
106using specific_intval = int_pred_ty<is_specific_int, Bitwidth>;
107
108inline specific_intval<0> m_SpecificInt(uint64_t V) {
109 return specific_intval<0>(is_specific_int(APInt(64, V)));
110}
111
112inline specific_intval<0> m_SpecificSInt(int64_t V) {
113 return specific_intval<0>(
114 is_specific_int(APInt(64, V, /*isSigned=*/true), /*IsSigned=*/true));
115}
116
117struct is_all_ones {
118 bool isValue(const APInt &C) const { return C.isAllOnes(); }
119};
120
121/// Match an integer or vector with all bits set.
122/// For vectors, this includes constants with undefined elements.
123inline int_pred_ty<is_all_ones> m_AllOnes() {
124 return int_pred_ty<is_all_ones>();
125}
126
127struct is_zero_int {
128 bool isValue(const APInt &C) const { return C.isZero(); }
129};
130
131struct is_one {
132 bool isValue(const APInt &C) const { return C.isOne(); }
133};
134
135/// Match an integer 0 or a vector with all elements equal to 0.
136/// For vectors, this includes constants with undefined elements.
137inline int_pred_ty<is_zero_int> m_ZeroInt() {
138 return int_pred_ty<is_zero_int>();
139}
140
141/// Match an integer 1 or a vector with all elements equal to 1.
142/// For vectors, this includes constants with undefined elements.
143inline int_pred_ty<is_one> m_One() { return int_pred_ty<is_one>(); }
144
145inline int_pred_ty<is_zero_int, 1> m_False() { return {}; }
146
147inline int_pred_ty<is_one, 1> m_True() { return {}; }
148
149struct bind_apint {
150 const APInt *&Res;
151
152 bind_apint(const APInt *&Res) : Res(Res) {}
153
154 bool match(const VPValue *VPV) const {
155 auto *CI = dyn_cast<VPConstantInt>(Val: VPV);
156 if (!CI)
157 return false;
158 Res = &CI->getAPInt();
159 return true;
160 }
161};
162
163inline bind_apint m_APInt(const APInt *&C) { return C; }
164
165struct bind_const_int {
166 uint64_t &Res;
167
168 bind_const_int(uint64_t &Res) : Res(Res) {}
169
170 bool match(const VPValue *VPV) const {
171 const APInt *APConst;
172 if (!bind_apint(APConst).match(VPV))
173 return false;
174 if (auto C = APConst->tryZExtValue()) {
175 Res = *C;
176 return true;
177 }
178 return false;
179 }
180};
181
182struct match_poison {
183 bool match(const VPValue *V) const {
184 return isa<VPIRValue>(Val: V) &&
185 isa<PoisonValue>(Val: cast<VPIRValue>(Val: V)->getValue());
186 }
187};
188
189/// Match a VPIRValue that's poison.
190inline match_poison m_Poison() { return match_poison(); }
191
192/// Match a plain integer constant no wider than 64-bits, capturing it if we
193/// match.
194inline bind_const_int m_ConstantInt(uint64_t &C) { return C; }
195
196/// Match a VPValue, capturing it if we match.
197inline match_bind<VPValue> m_VPValue(VPValue *&V) { return V; }
198
199/// Match against the nested pattern, and capture the value if we match.
200template <typename Op_t> inline auto m_VPValue(VPValue *&V, const Op_t &Op) {
201 return m_CombineAnd(Op, m_VPValue(V));
202}
203
204/// Match a VPIRValue.
205inline match_bind<VPIRValue> m_VPIRValue(VPIRValue *&V) { return V; }
206
207/// Match a VPSingleDefRecipe, capturing if we match.
208inline match_bind<VPSingleDefRecipe>
209m_VPSingleDefRecipe(VPSingleDefRecipe *&V) {
210 return V;
211}
212
213/// Match a VPInstruction, capturing if we match.
214inline match_bind<VPInstruction> m_VPInstruction(VPInstruction *&V) {
215 return V;
216}
217
218template <typename T>
219using hasOpcode_t = decltype(std::declval<T &>().getOpcode()); // NOLINT
220
221template <typename Ops_t, unsigned Opcode, bool Commutative,
222 typename... RecipeTys>
223struct Recipe_match {
224 Ops_t Ops;
225
226 template <typename... OpTy> Recipe_match(OpTy... Ops) : Ops(Ops...) {
227 static_assert(std::tuple_size<Ops_t>::value == sizeof...(Ops) &&
228 "number of operands in constructor doesn't match Ops_t");
229 static_assert((!Commutative || std::tuple_size<Ops_t>::value == 2) &&
230 "only binary ops can be commutative");
231 }
232
233 bool match(const VPValue *V) const {
234 auto *DefR = V->getDefiningRecipe();
235 return DefR && match(DefR);
236 }
237
238 bool match(const VPSingleDefRecipe *R) const {
239 return match(static_cast<const VPRecipeBase *>(R));
240 }
241
242 bool match(const VPRecipeBase *R) const {
243 if (std::tuple_size_v<Ops_t> == 0) {
244 auto *VPI = dyn_cast<VPInstruction>(Val: R);
245 return VPI && VPI->getOpcode() == Opcode;
246 }
247
248 if ((!matchRecipeAndOpcode<RecipeTys>(R) && ...))
249 return false;
250
251 if (R->getNumOperands() < std::tuple_size<Ops_t>::value) {
252 [[maybe_unused]] auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R);
253 assert(((isa<VPInstruction>(R) &&
254 cast<VPInstruction>(R)->getNumOperandsForOpcode() == -1u) ||
255 (RepR && std::tuple_size_v<Ops_t> ==
256 RepR->getNumOperandsWithoutMask())) &&
257 "non-variadic recipe with matched opcode does not have the "
258 "expected number of operands");
259 return false;
260 }
261
262 // If the recipe has more operands than expected, we only support matching
263 // masked VPInstructions or predicated VPReplicateRecipes, where the number
264 // of operands of the matcher matches the number of operands excluding the
265 // mask.
266 if (R->getNumOperands() > std::tuple_size<Ops_t>::value) {
267 if (auto *VPI = dyn_cast<VPInstruction>(Val: R)) {
268 if (!VPI->isMasked() ||
269 VPI->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
270 return false;
271 } else if (auto *RepR = dyn_cast<VPReplicateRecipe>(Val: R)) {
272 if (!RepR->isPredicated() ||
273 RepR->getNumOperandsWithoutMask() != std::tuple_size<Ops_t>::value)
274 return false;
275 } else {
276 return false;
277 }
278 }
279
280 auto IdxSeq = std::make_index_sequence<std::tuple_size<Ops_t>::value>();
281 if (all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
282 return Op.match(R->getOperand(N: Idx));
283 }))
284 return true;
285
286 return Commutative &&
287 all_of_tuple_elements(IdxSeq, [R](auto Op, unsigned Idx) {
288 return Op.match(R->getOperand(N: R->getNumOperands() - Idx - 1));
289 });
290 }
291
292private:
293 template <typename RecipeTy>
294 static bool matchRecipeAndOpcode(const VPRecipeBase *R) {
295 auto *DefR = dyn_cast<RecipeTy>(R);
296 if constexpr (Opcode && is_detected<hasOpcode_t, RecipeTy>::value)
297 return DefR && DefR->getOpcode() == Opcode;
298 return DefR;
299 }
300
301 /// Helper to check if predicate \p P holds on all tuple elements in Ops using
302 /// the provided index sequence.
303 template <typename Fn, std::size_t... Is>
304 bool all_of_tuple_elements(std::index_sequence<Is...>,
305 [[maybe_unused]] Fn P) const {
306 return (P(std::get<Is>(Ops), Is) && ...);
307 }
308};
309
310template <unsigned Opcode, typename... OpTys>
311using AllRecipe_match =
312 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ false,
313 VPWidenRecipe, VPReplicateRecipe, VPWidenCastRecipe,
314 VPInstruction>;
315
316template <unsigned Opcode, typename... OpTys>
317using AllRecipe_commutative_match =
318 Recipe_match<std::tuple<OpTys...>, Opcode, /*Commutative*/ true,
319 VPWidenRecipe, VPReplicateRecipe, VPInstruction>;
320
321template <unsigned Opcode, typename... OpTys>
322using VPInstruction_match = Recipe_match<std::tuple<OpTys...>, Opcode,
323 /*Commutative*/ false, VPInstruction>;
324
325template <unsigned Opcode, typename... OpTys>
326using VPInstruction_commutative_match =
327 Recipe_match<std::tuple<OpTys...>, Opcode,
328 /*Commutative*/ true, VPInstruction>;
329
330template <unsigned Opcode, typename... OpTys>
331inline VPInstruction_match<Opcode, OpTys...>
332m_VPInstruction(const OpTys &...Ops) {
333 return VPInstruction_match<Opcode, OpTys...>(Ops...);
334}
335
336template <unsigned Opcode, typename Op0_t, typename Op1_t>
337inline VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>
338m_c_VPInstruction(const Op0_t &Op0, const Op1_t &Op1) {
339 return VPInstruction_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
340}
341
342/// BuildVector is matches only its opcode, w/o matching its operands as the
343/// number of operands is not fixed.
344inline VPInstruction_match<VPInstruction::BuildVector> m_BuildVector() {
345 return m_VPInstruction<VPInstruction::BuildVector>();
346}
347
348template <typename Op0_t>
349inline VPInstruction_match<Instruction::Freeze, Op0_t>
350m_Freeze(const Op0_t &Op0) {
351 return m_VPInstruction<Instruction::Freeze>(Op0);
352}
353
354inline VPInstruction_match<VPInstruction::BranchOnCond> m_BranchOnCond() {
355 return m_VPInstruction<VPInstruction::BranchOnCond>();
356}
357
358template <typename Op0_t>
359inline VPInstruction_match<VPInstruction::BranchOnCond, Op0_t>
360m_BranchOnCond(const Op0_t &Op0) {
361 return m_VPInstruction<VPInstruction::BranchOnCond>(Op0);
362}
363
364inline VPInstruction_match<VPInstruction::BranchOnTwoConds>
365m_BranchOnTwoConds() {
366 return m_VPInstruction<VPInstruction::BranchOnTwoConds>();
367}
368
369template <typename Op0_t, typename Op1_t>
370inline VPInstruction_match<VPInstruction::BranchOnTwoConds, Op0_t, Op1_t>
371m_BranchOnTwoConds(const Op0_t &Op0, const Op1_t &Op1) {
372 return m_VPInstruction<VPInstruction::BranchOnTwoConds>(Op0, Op1);
373}
374
375inline VPInstruction_match<VPInstruction::BranchOnCount> m_BranchOnCount() {
376 return m_VPInstruction<VPInstruction::BranchOnCount>();
377}
378
379template <typename Op0_t, typename Op1_t>
380inline VPInstruction_match<VPInstruction::BranchOnCount, Op0_t, Op1_t>
381m_BranchOnCount(const Op0_t &Op0, const Op1_t &Op1) {
382 return m_VPInstruction<VPInstruction::BranchOnCount>(Op0, Op1);
383}
384
385inline auto m_Branch() {
386 return m_CombineOr(Ps: m_BranchOnCond(), Ps: m_BranchOnCount(), Ps: m_BranchOnTwoConds());
387}
388
389template <typename Op0_t>
390inline VPInstruction_match<VPInstruction::Broadcast, Op0_t>
391m_Broadcast(const Op0_t &Op0) {
392 return m_VPInstruction<VPInstruction::Broadcast>(Op0);
393}
394
395template <typename Op0_t>
396inline VPInstruction_match<VPInstruction::ExplicitVectorLength, Op0_t>
397m_EVL(const Op0_t &Op0) {
398 return m_VPInstruction<VPInstruction::ExplicitVectorLength>(Op0);
399}
400
401template <typename Op0_t>
402inline VPInstruction_match<VPInstruction::ExtractLastLane, Op0_t>
403m_ExtractLastLane(const Op0_t &Op0) {
404 return m_VPInstruction<VPInstruction::ExtractLastLane>(Op0);
405}
406
407template <typename Op0_t, typename Op1_t>
408inline VPInstruction_match<Instruction::ExtractElement, Op0_t, Op1_t>
409m_ExtractElement(const Op0_t &Op0, const Op1_t &Op1) {
410 return m_VPInstruction<Instruction::ExtractElement>(Op0, Op1);
411}
412
413template <typename Op0_t, typename Op1_t, typename Op2_t>
414inline VPInstruction_match<Instruction::InsertElement, Op0_t, Op1_t, Op2_t>
415m_InsertElement(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
416 return m_VPInstruction<Instruction::InsertElement>(Op0, Op1, Op2);
417}
418
419template <typename Op0_t, typename Op1_t>
420inline VPInstruction_match<VPInstruction::ExtractLane, Op0_t, Op1_t>
421m_ExtractLane(const Op0_t &Op0, const Op1_t &Op1) {
422 return m_VPInstruction<VPInstruction::ExtractLane>(Op0, Op1);
423}
424
425template <typename Op0_t>
426inline VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>
427m_ExtractLastPart(const Op0_t &Op0) {
428 return m_VPInstruction<VPInstruction::ExtractLastPart>(Op0);
429}
430
431template <typename Op0_t>
432inline VPInstruction_match<
433 VPInstruction::ExtractLastLane,
434 VPInstruction_match<VPInstruction::ExtractLastPart, Op0_t>>
435m_ExtractLastLaneOfLastPart(const Op0_t &Op0) {
436 return m_ExtractLastLane(m_ExtractLastPart(Op0));
437}
438
439template <typename Op0_t, typename Op1_t>
440inline VPInstruction_match<VPInstruction::ExtractVectorForPart, Op0_t, Op1_t>
441m_ExtractVectorForPart(const Op0_t &Op0, const Op1_t &Op1) {
442 return m_VPInstruction<VPInstruction::ExtractVectorForPart>(Op0, Op1);
443}
444
445template <typename Op0_t>
446inline VPInstruction_match<VPInstruction::ExtractPenultimateElement, Op0_t>
447m_ExtractPenultimateElement(const Op0_t &Op0) {
448 return m_VPInstruction<VPInstruction::ExtractPenultimateElement>(Op0);
449}
450
451template <typename Op0_t, typename Op1_t, typename Op2_t>
452inline VPInstruction_match<VPInstruction::WideActiveLaneMask, Op0_t, Op1_t,
453 Op2_t>
454m_WideActiveLaneMask(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
455 return m_VPInstruction<VPInstruction::WideActiveLaneMask>(Op0, Op1, Op2);
456}
457
458inline VPInstruction_match<VPInstruction::AnyOf> m_AnyOf() {
459 return m_VPInstruction<VPInstruction::AnyOf>();
460}
461
462template <typename Op0_t>
463inline VPInstruction_match<VPInstruction::AnyOf, Op0_t>
464m_AnyOf(const Op0_t &Op0) {
465 return m_VPInstruction<VPInstruction::AnyOf>(Op0);
466}
467
468template <typename Op0_t>
469inline VPInstruction_match<VPInstruction::FirstActiveLane, Op0_t>
470m_FirstActiveLane(const Op0_t &Op0) {
471 return m_VPInstruction<VPInstruction::FirstActiveLane>(Op0);
472}
473
474template <typename Op0_t>
475inline VPInstruction_match<VPInstruction::LastActiveLane, Op0_t>
476m_LastActiveLane(const Op0_t &Op0) {
477 return m_VPInstruction<VPInstruction::LastActiveLane>(Op0);
478}
479
480template <typename Op0_t, typename Op1_t, typename Op2_t>
481inline VPInstruction_match<VPInstruction::ExtractLastActive, Op0_t, Op1_t,
482 Op2_t>
483m_ExtractLastActive(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
484 return m_VPInstruction<VPInstruction::ExtractLastActive>(Op0, Op1, Op2);
485}
486
487template <typename Op0_t>
488inline VPInstruction_match<VPInstruction::ComputeReductionResult, Op0_t>
489m_ComputeReductionResult(const Op0_t &Op0) {
490 return m_VPInstruction<VPInstruction::ComputeReductionResult>(Op0);
491}
492
493/// Match FindIV result pattern:
494/// select(icmp ne ComputeReductionResult(ReducedIV), Sentinel),
495/// ComputeReductionResult(ReducedIV), Start.
496template <typename Op0_t, typename Op1_t>
497inline bool matchFindIVResult(VPInstruction *VPI, Op0_t ReducedIV, Op1_t Start) {
498 return match(VPI, m_Select(m_SpecificICmp(ICmpInst::ICMP_NE,
499 m_ComputeReductionResult(ReducedIV),
500 m_VPValue()),
501 m_ComputeReductionResult(ReducedIV), Start));
502}
503
504template <typename Op0_t>
505inline VPInstruction_match<VPInstruction::Reverse, Op0_t>
506m_Reverse(const Op0_t &Op0) {
507 return m_VPInstruction<VPInstruction::Reverse>(Op0);
508}
509
510inline VPInstruction_match<VPInstruction::StepVector> m_StepVector() {
511 return m_VPInstruction<VPInstruction::StepVector>();
512}
513
514template <typename Op0_t>
515inline VPInstruction_match<VPInstruction::ExitingIVValue, Op0_t>
516m_ExitingIVValue(const Op0_t &Op0) {
517 return m_VPInstruction<VPInstruction::ExitingIVValue>(Op0);
518}
519
520template <unsigned Opcode, typename Op0_t>
521inline AllRecipe_match<Opcode, Op0_t> m_Unary(const Op0_t &Op0) {
522 return AllRecipe_match<Opcode, Op0_t>(Op0);
523}
524
525template <typename Op0_t>
526inline AllRecipe_match<Instruction::Trunc, Op0_t> m_Trunc(const Op0_t &Op0) {
527 return m_Unary<Instruction::Trunc, Op0_t>(Op0);
528}
529
530template <typename Op0_t>
531inline match_combine_or<AllRecipe_match<Instruction::Trunc, Op0_t>, Op0_t>
532m_TruncOrSelf(const Op0_t &Op0) {
533 return m_CombineOr(m_Trunc(Op0), Op0);
534}
535
536template <typename Op0_t>
537inline AllRecipe_match<Instruction::ZExt, Op0_t> m_ZExt(const Op0_t &Op0) {
538 return m_Unary<Instruction::ZExt, Op0_t>(Op0);
539}
540
541template <typename Op0_t>
542inline AllRecipe_match<Instruction::SExt, Op0_t> m_SExt(const Op0_t &Op0) {
543 return m_Unary<Instruction::SExt, Op0_t>(Op0);
544}
545
546template <typename Op0_t>
547inline AllRecipe_match<Instruction::FPExt, Op0_t> m_FPExt(const Op0_t &Op0) {
548 return m_Unary<Instruction::FPExt, Op0_t>(Op0);
549}
550
551template <typename Op0_t>
552inline AllRecipe_match<Instruction::BitCast, Op0_t>
553m_BitCast(const Op0_t &Op0) {
554 return m_Unary<Instruction::BitCast, Op0_t>(Op0);
555}
556
557template <typename Op0_t>
558inline AllRecipe_match<Instruction::PtrToAddr, Op0_t>
559m_PtrToAddr(const Op0_t &Op0) {
560 return m_Unary<Instruction::PtrToAddr, Op0_t>(Op0);
561}
562
563template <typename Op0_t>
564inline AllRecipe_match<Instruction::FNeg, Op0_t> m_FNeg(const Op0_t &Op0) {
565 return m_Unary<Instruction::FNeg, Op0_t>(Op0);
566}
567
568template <typename Op0_t>
569inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>,
570 AllRecipe_match<Instruction::SExt, Op0_t>>
571m_ZExtOrSExt(const Op0_t &Op0) {
572 return m_CombineOr(m_ZExt(Op0), m_SExt(Op0));
573}
574
575template <typename Op0_t> inline auto m_WidenAnyExtend(const Op0_t &Op0) {
576 return m_Isa<VPWidenCastRecipe>(m_CombineOr(m_ZExtOrSExt(Op0), m_FPExt(Op0)));
577}
578
579template <typename Op0_t> inline auto m_AnyNeg(const Op0_t &Op0) {
580 return m_CombineOr(m_Sub(m_ZeroInt(), Op0), m_FNeg(Op0));
581}
582
583template <typename Op0_t>
584inline match_combine_or<AllRecipe_match<Instruction::ZExt, Op0_t>, Op0_t>
585m_ZExtOrSelf(const Op0_t &Op0) {
586 return m_CombineOr(m_ZExt(Op0), Op0);
587}
588
589template <typename Op0_t> inline auto m_ZExtOrTrunc(const Op0_t &Op0) {
590 return m_CombineOr(m_ZExt(Op0), m_Trunc(Op0));
591}
592
593template <typename Op0_t> inline auto m_ZExtOrTruncOrSelf(const Op0_t &Op0) {
594 return m_CombineOr(m_ZExtOrTrunc(Op0), Op0);
595}
596
597template <unsigned Opcode, typename Op0_t, typename Op1_t>
598inline AllRecipe_match<Opcode, Op0_t, Op1_t> m_Binary(const Op0_t &Op0,
599 const Op1_t &Op1) {
600 return AllRecipe_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
601}
602
603template <unsigned Opcode, typename Op0_t, typename Op1_t>
604inline AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>
605m_c_Binary(const Op0_t &Op0, const Op1_t &Op1) {
606 return AllRecipe_commutative_match<Opcode, Op0_t, Op1_t>(Op0, Op1);
607}
608
609template <typename Op0_t, typename Op1_t>
610inline AllRecipe_match<Instruction::Add, Op0_t, Op1_t> m_Add(const Op0_t &Op0,
611 const Op1_t &Op1) {
612 return m_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
613}
614
615template <typename Op0_t, typename Op1_t>
616inline AllRecipe_commutative_match<Instruction::Add, Op0_t, Op1_t>
617m_c_Add(const Op0_t &Op0, const Op1_t &Op1) {
618 return m_c_Binary<Instruction::Add, Op0_t, Op1_t>(Op0, Op1);
619}
620
621template <typename Op0_t, typename Op1_t>
622inline AllRecipe_match<Instruction::Sub, Op0_t, Op1_t> m_Sub(const Op0_t &Op0,
623 const Op1_t &Op1) {
624 return m_Binary<Instruction::Sub, Op0_t, Op1_t>(Op0, Op1);
625}
626
627template <typename Op0_t, typename Op1_t>
628inline AllRecipe_match<Instruction::Mul, Op0_t, Op1_t> m_Mul(const Op0_t &Op0,
629 const Op1_t &Op1) {
630 return m_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
631}
632
633template <typename Op0_t, typename Op1_t>
634inline AllRecipe_commutative_match<Instruction::Mul, Op0_t, Op1_t>
635m_c_Mul(const Op0_t &Op0, const Op1_t &Op1) {
636 return m_c_Binary<Instruction::Mul, Op0_t, Op1_t>(Op0, Op1);
637}
638
639template <typename Op0_t, typename Op1_t>
640inline AllRecipe_match<Instruction::Shl, Op0_t, Op1_t> m_Shl(const Op0_t &Op0,
641 const Op1_t &Op1) {
642 return m_Binary<Instruction::Shl, Op0_t, Op1_t>(Op0, Op1);
643}
644
645template <typename Op0_t, typename Op1_t>
646inline AllRecipe_match<Instruction::LShr, Op0_t, Op1_t>
647m_LShr(const Op0_t &Op0, const Op1_t &Op1) {
648 return m_Binary<Instruction::LShr, Op0_t, Op1_t>(Op0, Op1);
649}
650
651template <typename Op0_t, typename Op1_t>
652inline AllRecipe_match<Instruction::AShr, Op0_t, Op1_t>
653m_AShr(const Op0_t &Op0, const Op1_t &Op1) {
654 return m_Binary<Instruction::AShr, Op0_t, Op1_t>(Op0, Op1);
655}
656
657template <typename Op0_t, typename Op1_t>
658inline AllRecipe_match<Instruction::FMul, Op0_t, Op1_t>
659m_FMul(const Op0_t &Op0, const Op1_t &Op1) {
660 return m_Binary<Instruction::FMul, Op0_t, Op1_t>(Op0, Op1);
661}
662
663template <typename Op0_t, typename Op1_t>
664inline AllRecipe_match<Instruction::FAdd, Op0_t, Op1_t>
665m_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
666 return m_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
667}
668
669template <typename Op0_t, typename Op1_t>
670inline AllRecipe_commutative_match<Instruction::FAdd, Op0_t, Op1_t>
671m_c_FAdd(const Op0_t &Op0, const Op1_t &Op1) {
672 return m_c_Binary<Instruction::FAdd, Op0_t, Op1_t>(Op0, Op1);
673}
674
675template <typename Op0_t, typename Op1_t>
676inline AllRecipe_match<Instruction::UDiv, Op0_t, Op1_t>
677m_UDiv(const Op0_t &Op0, const Op1_t &Op1) {
678 return m_Binary<Instruction::UDiv, Op0_t, Op1_t>(Op0, Op1);
679}
680
681template <typename Op0_t, typename Op1_t>
682inline AllRecipe_match<Instruction::URem, Op0_t, Op1_t>
683m_URem(const Op0_t &Op0, const Op1_t &Op1) {
684 return m_Binary<Instruction::URem, Op0_t, Op1_t>(Op0, Op1);
685}
686
687template <typename Op0_t, typename Op1_t>
688inline AllRecipe_match<Instruction::SDiv, Op0_t, Op1_t>
689m_SDiv(const Op0_t &Op0, const Op1_t &Op1) {
690 return m_Binary<Instruction::SDiv, Op0_t, Op1_t>(Op0, Op1);
691}
692
693template <typename Op0_t, typename Op1_t>
694inline AllRecipe_match<Instruction::SRem, Op0_t, Op1_t>
695m_SRem(const Op0_t &Op0, const Op1_t &Op1) {
696 return m_Binary<Instruction::SRem, Op0_t, Op1_t>(Op0, Op1);
697}
698
699/// Match a binary AND operation.
700template <typename Op0_t, typename Op1_t>
701inline AllRecipe_commutative_match<Instruction::And, Op0_t, Op1_t>
702m_c_BinaryAnd(const Op0_t &Op0, const Op1_t &Op1) {
703 return m_c_Binary<Instruction::And, Op0_t, Op1_t>(Op0, Op1);
704}
705
706/// Match a binary OR operation. Note that while conceptually the operands can
707/// be matched commutatively, \p Commutative defaults to false in line with the
708/// IR-based pattern matching infrastructure. Use m_c_BinaryOr for a commutative
709/// version of the matcher.
710template <typename Op0_t, typename Op1_t>
711inline AllRecipe_match<Instruction::Or, Op0_t, Op1_t>
712m_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
713 return m_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
714}
715
716template <typename Op0_t, typename Op1_t>
717inline AllRecipe_commutative_match<Instruction::Or, Op0_t, Op1_t>
718m_c_BinaryOr(const Op0_t &Op0, const Op1_t &Op1) {
719 return m_c_Binary<Instruction::Or, Op0_t, Op1_t>(Op0, Op1);
720}
721
722/// Cmp_match is a variant of BinaryRecipe_match that also binds the comparison
723/// predicate. Opcodes must either be Instruction::ICmp or Instruction::FCmp, or
724/// both.
725template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
726struct Cmp_match {
727 static_assert((sizeof...(Opcodes) == 1 || sizeof...(Opcodes) == 2) &&
728 "Expected one or two opcodes");
729 static_assert(
730 ((Opcodes == Instruction::ICmp || Opcodes == Instruction::FCmp) && ...) &&
731 "Expected a compare instruction opcode");
732
733 CmpPredicate *Predicate = nullptr;
734 Op0_t Op0;
735 Op1_t Op1;
736
737 Cmp_match(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1)
738 : Predicate(&Pred), Op0(Op0), Op1(Op1) {}
739 Cmp_match(const Op0_t &Op0, const Op1_t &Op1) : Op0(Op0), Op1(Op1) {}
740
741 bool match(const VPValue *V) const {
742 auto *DefR = V->getDefiningRecipe();
743 return DefR && match(DefR);
744 }
745
746 bool match(const VPRecipeBase *V) const {
747 if ((m_Binary<Opcodes>(Op0, Op1).match(V) || ...)) {
748 if (Predicate)
749 *Predicate = cast<VPRecipeWithIRFlags>(Val: V)->getPredicate();
750 return true;
751 }
752 return false;
753 }
754};
755
756/// SpecificCmp_match is a variant of Cmp_match that matches the comparison
757/// predicate, instead of binding it.
758template <typename Op0_t, typename Op1_t, unsigned... Opcodes>
759struct SpecificCmp_match {
760 const CmpPredicate Predicate;
761 Op0_t Op0;
762 Op1_t Op1;
763
764 SpecificCmp_match(CmpPredicate Pred, const Op0_t &LHS, const Op1_t &RHS)
765 : Predicate(Pred), Op0(LHS), Op1(RHS) {}
766
767 bool match(const VPValue *V) const {
768 auto *DefR = V->getDefiningRecipe();
769 return DefR && match(DefR);
770 }
771
772 bool match(const VPRecipeBase *V) const {
773 CmpPredicate CurrentPred;
774 return Cmp_match<Op0_t, Op1_t, Opcodes...>(CurrentPred, Op0, Op1)
775 .match(V) &&
776 CmpPredicate::getMatching(A: CurrentPred, B: Predicate);
777 }
778};
779
780template <typename Op0_t, typename Op1_t>
781inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp> m_ICmp(const Op0_t &Op0,
782 const Op1_t &Op1) {
783 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Op0, Op1);
784}
785
786template <typename Op0_t, typename Op1_t>
787inline auto m_c_ICmp(const Op0_t &Op0, const Op1_t &Op1) {
788 return m_CombineOr(Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Op0, Op1),
789 Cmp_match<Op1_t, Op0_t, Instruction::ICmp>(Op1, Op0));
790}
791
792template <typename Op0_t, typename Op1_t>
793inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp>
794m_ICmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
795 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp>(Pred, Op0, Op1);
796}
797
798template <typename Op0_t, typename Op1_t>
799inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>
800m_SpecificICmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
801 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp>(MatchPred, Op0,
802 Op1);
803}
804
805template <typename Op0_t, typename Op1_t>
806inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
807m_Cmp(const Op0_t &Op0, const Op1_t &Op1) {
808 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(Op0,
809 Op1);
810}
811
812template <typename Op0_t, typename Op1_t>
813inline auto m_c_Cmp(const Op0_t &Op0, const Op1_t &Op1) {
814 return m_CombineOr(
815 Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(Op0, Op1),
816 Cmp_match<Op1_t, Op0_t, Instruction::ICmp, Instruction::FCmp>(Op1, Op0));
817}
818
819template <typename Op0_t, typename Op1_t>
820inline Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
821m_Cmp(CmpPredicate &Pred, const Op0_t &Op0, const Op1_t &Op1) {
822 return Cmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
823 Pred, Op0, Op1);
824}
825
826template <typename Op0_t, typename Op1_t>
827inline SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>
828m_SpecificCmp(CmpPredicate MatchPred, const Op0_t &Op0, const Op1_t &Op1) {
829 return SpecificCmp_match<Op0_t, Op1_t, Instruction::ICmp, Instruction::FCmp>(
830 MatchPred, Op0, Op1);
831}
832
833template <typename Op0_t, typename Op1_t>
834inline auto m_GetElementPtr(const Op0_t &Op0, const Op1_t &Op1) {
835 return m_CombineOr(
836 Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::GetElementPtr,
837 /*Commutative*/ false, VPReplicateRecipe, VPWidenGEPRecipe>(
838 Op0, Op1),
839 VPInstruction_match<VPInstruction::PtrAdd, Op0_t, Op1_t>(Op0, Op1),
840 VPInstruction_match<VPInstruction::WidePtrAdd, Op0_t, Op1_t>(Op0, Op1));
841}
842
843/// Match a VPBlendRecipe with 2 incoming values ([I0, I1, M1] ==
844/// normalized([I0, M0, I1, M1])) as select(M1, I1, I0), mirroring how it is
845/// lowered.
846template <typename Op0_t, typename Op1_t, typename Op2_t> struct Blend2_match {
847 Op0_t MaskOp;
848 Op1_t TrueOp;
849 Op2_t FalseOp;
850
851 Blend2_match(const Op0_t &MaskOp, const Op1_t &TrueOp, const Op2_t &FalseOp)
852 : MaskOp(MaskOp), TrueOp(TrueOp), FalseOp(FalseOp) {}
853
854 template <typename T> bool match(const T *Val) const {
855 auto *Blend = dyn_cast<VPBlendRecipe>(Val);
856 if (!Blend || Blend->getNumIncomingValues() != 2)
857 return false;
858 return MaskOp.match(Blend->getMask(1)) &&
859 TrueOp.match(Blend->getIncomingValue(1)) &&
860 FalseOp.match(Blend->getIncomingValue(0));
861 }
862};
863
864/// Match recipe recipe with Select opcode, i.e. excluding VPBlendRecipe.
865template <typename Op0_t, typename Op1_t, typename Op2_t>
866inline AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>
867m_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
868 return AllRecipe_match<Instruction::Select, Op0_t, Op1_t, Op2_t>(
869 {Op0, Op1, Op2});
870}
871
872/// Match recipe with Select opcode or an equivalent VPBlendRecipe with 2
873/// incoming values.
874template <typename Op0_t, typename Op1_t, typename Op2_t>
875inline auto m_SelectLike(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
876 return m_CombineOr(m_Select(Op0, Op1, Op2),
877 Blend2_match<Op0_t, Op1_t, Op2_t>(Op0, Op1, Op2));
878}
879
880template <typename Op0_t> inline auto m_Not(const Op0_t &Op0) {
881 return m_CombineOr(m_VPInstruction<VPInstruction::Not>(Op0),
882 m_c_Binary<Instruction::Xor>(m_AllOnes(), Op0));
883}
884
885template <typename Op0_t, typename Op1_t, typename Op2_t>
886inline auto m_c_Select(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
887 return m_CombineOr(m_Select(Op0, Op1, Op2), m_Select(m_Not(Op0), Op2, Op1));
888}
889
890template <typename Op0_t, typename Op1_t>
891inline auto m_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
892 return m_CombineOr(
893 m_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
894 m_Select(Op0, Op1, m_False()));
895}
896
897template <typename Op0_t, typename Op1_t> struct RemoveMask_match {
898 Op0_t In;
899 Op1_t &Out;
900
901 RemoveMask_match(const Op0_t &In, Op1_t &Out) : In(In), Out(Out) {}
902
903 template <typename OpTy> bool match(OpTy *V) const {
904 if (m_Specific(In).match(V)) {
905 Out = nullptr;
906 return true;
907 }
908 return m_LogicalAnd(m_Specific(In), m_VPValue(Out)).match(V);
909 }
910};
911
912/// Match a specific mask \p In, or a combination of it (logical-and In, Out).
913/// Returns the remaining part \p Out if so, or nullptr otherwise.
914template <typename Op0_t, typename Op1_t>
915inline RemoveMask_match<Op0_t, Op1_t> m_RemoveMask(const Op0_t &In,
916 Op1_t &Out) {
917 return RemoveMask_match<Op0_t, Op1_t>(In, Out);
918}
919
920template <typename Op0_t, typename Op1_t>
921inline auto m_c_LogicalAnd(const Op0_t &Op0, const Op1_t &Op1) {
922 return m_CombineOr(
923 m_c_VPInstruction<VPInstruction::LogicalAnd, Op0_t, Op1_t>(Op0, Op1),
924 m_c_Select(Op0, Op1, m_False()));
925}
926
927template <typename Op0_t, typename Op1_t>
928inline auto m_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
929 return m_CombineOr(
930 m_c_VPInstruction<VPInstruction::LogicalOr, Op0_t, Op1_t>(Op0, Op1),
931 m_Select(Op0, m_True(), Op1));
932}
933
934template <typename Op0_t, typename Op1_t>
935inline auto m_c_LogicalOr(const Op0_t &Op0, const Op1_t &Op1) {
936 return m_CombineOr(m_c_Select(Op0, m_True(), Op1),
937 m_c_Select(Op1, m_True(), Op0));
938}
939
940/// Match the canonical induction variable (IV) of any loop region.
941struct canonical_iv_match {
942 template <typename ArgTy> bool match(const ArgTy *V) const {
943 const auto *RV = dyn_cast<VPRegionValue>(V);
944 return RV && RV->getDefiningRegion()->getCanonicalIV() == RV;
945 }
946};
947
948inline canonical_iv_match m_CanonicalIV() { return {}; }
949
950/// Match the abstract header mask of any loop region.
951struct header_mask_match {
952 template <typename ArgTy> bool match(const ArgTy *V) const {
953 const auto *RV = dyn_cast<VPRegionValue>(V);
954 return RV && RV->getDefiningRegion()->getHeaderMask() == RV;
955 }
956};
957
958inline header_mask_match m_HeaderMask() { return {}; }
959
960/// Match a canonical VPWidenIntOrFpInductionRecipe optionally capturing it.
961struct canonical_widen_iv_match {
962 VPWidenIntOrFpInductionRecipe **Capture = nullptr;
963
964 canonical_widen_iv_match() = default;
965 canonical_widen_iv_match(VPWidenIntOrFpInductionRecipe *&V) : Capture(&V) {}
966
967 template <typename ArgTy> bool match(ArgTy *V) const {
968 auto *WidenIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(V);
969 if (!WidenIV || !WidenIV->isCanonical())
970 return false;
971 if (Capture)
972 *Capture = WidenIV;
973 return true;
974 }
975};
976
977inline canonical_widen_iv_match m_CanonicalWidenIV() { return {}; }
978
979/// Match a canonical VPWidenIntOrFpInductionRecipe, capturing it.
980inline canonical_widen_iv_match
981m_CanonicalWidenIV(VPWidenIntOrFpInductionRecipe *&V) {
982 return canonical_widen_iv_match(V);
983}
984
985template <typename Op0_t, typename Op1_t, typename Op2_t>
986inline auto m_ScalarIVSteps(const Op0_t &Op0, const Op1_t &Op1,
987 const Op2_t &Op2) {
988 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
989 VPScalarIVStepsRecipe>({Op0, Op1, Op2});
990}
991
992template <typename Op0_t, typename Op1_t, typename Op2_t>
993inline auto m_DerivedIV(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
994 return Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t>, 0, false,
995 VPDerivedIVRecipe>({Op0, Op1, Op2});
996}
997
998template <typename Addr_t, typename Mask_t> struct Load_match {
999 Addr_t Addr;
1000 Mask_t Mask;
1001
1002 Load_match(Addr_t Addr, Mask_t Mask) : Addr(Addr), Mask(Mask) {}
1003
1004 template <typename OpTy> bool match(const OpTy *V) const {
1005 auto *Load = dyn_cast<VPWidenLoadRecipe>(V);
1006 if (!Load || !Addr.match(Load->getAddr()) || !Load->isMasked() ||
1007 !Mask.match(Load->getMask()))
1008 return false;
1009 return true;
1010 }
1011};
1012
1013/// Match a (possibly reversed) masked load.
1014template <typename Addr_t, typename Mask_t>
1015inline Load_match<Addr_t, Mask_t> m_MaskedLoad(const Addr_t &Addr,
1016 const Mask_t &Mask) {
1017 return Load_match<Addr_t, Mask_t>(Addr, Mask);
1018}
1019
1020template <typename Addr_t, typename Val_t, typename Mask_t> struct Store_match {
1021 Addr_t Addr;
1022 Val_t Val;
1023 Mask_t Mask;
1024
1025 Store_match(Addr_t Addr, Val_t Val, Mask_t Mask)
1026 : Addr(Addr), Val(Val), Mask(Mask) {}
1027
1028 template <typename OpTy> bool match(const OpTy *V) const {
1029 auto *Store = dyn_cast<VPWidenStoreRecipe>(V);
1030 if (!Store || !Addr.match(Store->getAddr()) ||
1031 !Val.match(Store->getStoredValue()) || !Store->isMasked() ||
1032 !Mask.match(Store->getMask()))
1033 return false;
1034 return true;
1035 }
1036};
1037
1038/// Match a (possibly reversed) masked store.
1039template <typename Addr_t, typename Val_t, typename Mask_t>
1040inline Store_match<Addr_t, Val_t, Mask_t>
1041m_MaskedStore(const Addr_t &Addr, const Val_t &Val, const Mask_t &Mask) {
1042 return Store_match<Addr_t, Val_t, Mask_t>(Addr, Val, Mask);
1043}
1044
1045template <typename Op0_t, typename Op1_t>
1046using VectorPointerRecipe_match =
1047 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
1048 /*Commutative*/ false, VPVectorPointerRecipe>;
1049
1050template <typename Op0_t, typename Op1_t>
1051VectorPointerRecipe_match<Op0_t, Op1_t> m_VecPtr(const Op0_t &Op0,
1052 const Op1_t &Op1) {
1053 return VectorPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
1054}
1055
1056template <typename Op0_t>
1057using VPWidenLoadRecipe_match =
1058 Recipe_match<std::tuple<Op0_t>, 0,
1059 /*Commutative*/ false, VPWidenLoadRecipe>;
1060
1061template <typename Op0_t>
1062VPWidenLoadRecipe_match<Op0_t> m_WidenLoad(const Op0_t &Op0) {
1063 return VPWidenLoadRecipe_match<Op0_t>(Op0);
1064}
1065
1066template <typename Op0_t, typename Op1_t>
1067using VPWidenStoreRecipe_match =
1068 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
1069 /*Commutative*/ false, VPWidenStoreRecipe>;
1070
1071template <typename Op0_t, typename Op1_t>
1072VPWidenStoreRecipe_match<Op0_t, Op1_t> m_WidenStore(const Op0_t &Op0,
1073 const Op1_t &Op1) {
1074 return VPWidenStoreRecipe_match<Op0_t, Op1_t>(Op0, Op1);
1075}
1076
1077template <typename Op0_t, typename Op1_t>
1078using VectorEndPointerRecipe_match =
1079 Recipe_match<std::tuple<Op0_t, Op1_t>, 0,
1080 /*Commutative*/ false, VPVectorEndPointerRecipe>;
1081
1082template <typename Op0_t, typename Op1_t>
1083VectorEndPointerRecipe_match<Op0_t, Op1_t> m_VecEndPtr(const Op0_t &Op0,
1084 const Op1_t &Op1) {
1085 return VectorEndPointerRecipe_match<Op0_t, Op1_t>(Op0, Op1);
1086}
1087
1088/// Match a call argument at a given argument index.
1089template <typename Opnd_t> struct Argument_match {
1090 /// Call argument index to match.
1091 unsigned OpI;
1092 Opnd_t Val;
1093
1094 Argument_match(unsigned OpIdx, const Opnd_t &V) : OpI(OpIdx), Val(V) {}
1095
1096 template <typename OpTy> bool match(OpTy *V) const {
1097 if (const auto *R = dyn_cast<VPWidenIntrinsicRecipe>(V))
1098 return Val.match(R->getOperand(OpI));
1099 if (const auto *R = dyn_cast<VPWidenCallRecipe>(V))
1100 return Val.match(R->getOperand(OpI));
1101 if (const auto *R = dyn_cast<VPReplicateRecipe>(V))
1102 if (R->getOpcode() == Instruction::Call)
1103 return Val.match(R->getOperand(OpI));
1104 if (const auto *R = dyn_cast<VPInstruction>(V))
1105 if (R->getOpcode() == Instruction::Call ||
1106 R->getOpcode() == VPInstruction::Intrinsic)
1107 return Val.match(R->getOperand(OpI));
1108 return false;
1109 }
1110};
1111
1112/// Match a call argument.
1113template <unsigned OpI, typename Opnd_t>
1114inline Argument_match<Opnd_t> m_Argument(const Opnd_t &Op) {
1115 return Argument_match<Opnd_t>(OpI, Op);
1116}
1117
1118/// Intrinsic matchers.
1119struct IntrinsicID_match {
1120 unsigned ID;
1121
1122 IntrinsicID_match(Intrinsic::ID IntrID) : ID(IntrID) {}
1123
1124 template <typename OpTy> bool match(OpTy *V) const {
1125 return vputils::getIntrinsicID(V) == ID;
1126 }
1127};
1128
1129/// Match intrinsic calls with a runtime intrinsic ID.
1130inline IntrinsicID_match m_Intrinsic(Intrinsic::ID IntrID) {
1131 return IntrinsicID_match(IntrID);
1132}
1133
1134struct IntrinsicMatchImpl {
1135 template <Intrinsic::ID IntrID, typename... Ts, size_t... Is>
1136 static auto impl(std::index_sequence<Is...>, const Ts &...Ops) {
1137 return m_CombineAnd(IntrinsicID_match(IntrID), m_Argument<Is>(Ops)...);
1138 }
1139};
1140
1141/// Match intrinsic calls like this:
1142/// m_Intrinsic<Intrinsic::fabs>(m_VPValue(X), ...)
1143template <Intrinsic::ID IntrID, typename... Ts>
1144inline auto m_Intrinsic(const Ts &...Ops) {
1145 return IntrinsicMatchImpl::impl<IntrID>(
1146 std::make_index_sequence<sizeof...(Ts)>{}, Ops...);
1147}
1148
1149template <Intrinsic::ID IntrID, typename... T>
1150inline auto m_WidenIntrinsic(const T &...Ops) {
1151 return m_Isa<VPWidenIntrinsicRecipe>(m_Intrinsic<IntrID>(Ops...));
1152}
1153
1154/// Match VPValues that represent live-ins: VPIRValues and (plain)
1155/// VPSymbolicValues. VPRegionValues (which inherit from VPSymbolicValue) are
1156/// not live-ins and are excluded.
1157struct LiveIn_match {
1158 template <typename ITy> bool match(ITy *V) const {
1159 return isa<VPIRValue>(V) ||
1160 (isa<VPSymbolicValue>(V) && !isa<VPRegionValue>(V));
1161 }
1162};
1163
1164inline auto m_VScale() { return m_Intrinsic<Intrinsic::vscale>(); }
1165
1166inline auto m_LiveIn() { return m_Isa<VPIRValue, VPSymbolicValue>(); }
1167
1168/// Match a GEP recipe (VPWidenGEPRecipe, VPInstruction, or VPReplicateRecipe)
1169/// and bind the source element type and operands.
1170struct GetElementPtr_match {
1171 Type *&SourceElementType;
1172 ArrayRef<VPValue *> &Operands;
1173
1174 GetElementPtr_match(Type *&SourceElementType, ArrayRef<VPValue *> &Operands)
1175 : SourceElementType(SourceElementType), Operands(Operands) {}
1176
1177 template <typename ITy> bool match(ITy *V) const {
1178 return matchRecipeAndBind<VPWidenGEPRecipe>(V) ||
1179 matchRecipeAndBind<VPInstruction>(V) ||
1180 matchRecipeAndBind<VPReplicateRecipe>(V);
1181 }
1182
1183private:
1184 template <typename RecipeTy> bool matchRecipeAndBind(const VPValue *V) const {
1185 auto *DefR = dyn_cast<RecipeTy>(V);
1186 if (!DefR)
1187 return false;
1188
1189 if constexpr (std::is_same_v<RecipeTy, VPWidenGEPRecipe>) {
1190 SourceElementType = DefR->getSourceElementType();
1191 } else if (DefR->getOpcode() == Instruction::GetElementPtr) {
1192 SourceElementType = cast<GetElementPtrInst>(DefR->getUnderlyingInstr())
1193 ->getSourceElementType();
1194 } else if constexpr (std::is_same_v<RecipeTy, VPInstruction>) {
1195 if (DefR->getOpcode() == VPInstruction::PtrAdd) {
1196 // PtrAdd is a byte-offset GEP with i8 element type.
1197 LLVMContext &Ctx = DefR->getParent()->getPlan()->getContext();
1198 SourceElementType = Type::getInt8Ty(C&: Ctx);
1199 } else {
1200 return false;
1201 }
1202 } else {
1203 return false;
1204 }
1205
1206 Operands = ArrayRef<VPValue *>(DefR->op_begin(), DefR->op_end());
1207 return true;
1208 }
1209};
1210
1211/// Match a GEP recipe with any number of operands and bind source element type
1212/// and operands.
1213inline GetElementPtr_match m_GetElementPtr(Type *&SourceElementType,
1214 ArrayRef<VPValue *> &Operands) {
1215 return GetElementPtr_match(SourceElementType, Operands);
1216}
1217
1218template <typename SubPattern_t> struct OneUse_match {
1219 SubPattern_t SubPattern;
1220
1221 OneUse_match(const SubPattern_t &SP) : SubPattern(SP) {}
1222
1223 template <typename OpTy> bool match(OpTy *V) const {
1224 return V->hasOneUse() && SubPattern.match(V);
1225 }
1226};
1227
1228template <typename T> inline OneUse_match<T> m_OneUse(const T &SubPattern) {
1229 return SubPattern;
1230}
1231
1232inline match_bind<VPReductionPHIRecipe>
1233m_ReductionPhi(VPReductionPHIRecipe *&V) {
1234 return V;
1235}
1236
1237template <typename Op0_t, typename Op1_t>
1238inline auto m_VPPhi(const Op0_t &Op0, const Op1_t &Op1) {
1239 return Recipe_match<std::tuple<Op0_t, Op1_t>, Instruction::PHI,
1240 /*Commutative*/ false, VPInstruction>({Op0, Op1});
1241}
1242
1243/// If \p V is used by a recipe matching pattern \p P, return it. Otherwise
1244/// return nullptr;
1245template <typename MatchT>
1246VPRecipeBase *findUserOf(VPValue *V, const MatchT &P) {
1247 auto It = find_if(V->users(), match_fn(P));
1248 return It == V->user_end() ? nullptr : cast<VPRecipeBase>(*It);
1249}
1250
1251/// If \p V is used by a VPInstruction with \p Opcode, return it. Otherwise
1252/// return nullptr.
1253template <unsigned Opcode> VPInstruction *findUserOf(VPValue *V) {
1254 return cast_or_null<VPInstruction>(findUserOf(V, m_VPInstruction<Opcode>()));
1255}
1256
1257template <typename RecipeTy> RecipeTy *findUserOf(VPValue *V) {
1258 return cast_or_null<RecipeTy>(findUserOf(V, m_Isa<RecipeTy>()));
1259}
1260} // namespace llvm::VPlanPatternMatch
1261
1262#endif
1263