1//==--------------- llvm/CodeGen/SDPatternMatch.h ---------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// Contains matchers for matching SelectionDAG nodes and values.
10///
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_CODEGEN_SDPATTERNMATCH_H
14#define LLVM_CODEGEN_SDPATTERNMATCH_H
15
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallBitVector.h"
20#include "llvm/ADT/bit.h"
21#include "llvm/CodeGen/SelectionDAG.h"
22#include "llvm/CodeGen/SelectionDAGNodes.h"
23#include "llvm/CodeGen/TargetLowering.h"
24#include "llvm/Support/KnownBits.h"
25
26#include <type_traits>
27
28namespace llvm {
29namespace SDPatternMatch {
30
31/// MatchContext can repurpose existing patterns to behave differently under
32/// a certain context. For instance, `m_SpecificOpc(ISD::ADD)` matches plain ADD
33/// nodes in normal circumstances, but matches VP_ADD nodes under a custom
34/// VPMatchContext. This design is meant to facilitate code / pattern reusing.
35class BasicMatchContext {
36 const SelectionDAG *DAG;
37 const TargetLowering *TLI;
38
39public:
40 explicit BasicMatchContext(const SelectionDAG *DAG)
41 : DAG(DAG), TLI(DAG ? &DAG->getTargetLoweringInfo() : nullptr) {}
42
43 explicit BasicMatchContext(const TargetLowering *TLI)
44 : DAG(nullptr), TLI(TLI) {}
45
46 // A valid MatchContext has to implement the following functions.
47
48 const SelectionDAG *getDAG() const { return DAG; }
49
50 const TargetLowering *getTLI() const { return TLI; }
51
52 /// Return true if N effectively has opcode Opcode.
53 bool match(SDValue N, unsigned Opcode) const {
54 return N->getOpcode() == Opcode;
55 }
56
57 unsigned getNumOperands(SDValue N) const { return N->getNumOperands(); }
58};
59
60template <typename Pattern, typename MatchContext>
61[[nodiscard]] bool sd_context_match(SDValue N, const MatchContext &Ctx,
62 Pattern &&P) {
63 return P.match(Ctx, N);
64}
65
66template <typename Pattern, typename MatchContext>
67[[nodiscard]] bool sd_context_match(SDNode *N, const MatchContext &Ctx,
68 Pattern &&P) {
69 return sd_context_match(SDValue(N, 0), Ctx, P);
70}
71
72template <typename Pattern>
73[[nodiscard]] bool sd_match(SDNode *N, const SelectionDAG *DAG, Pattern &&P) {
74 return sd_context_match(N, BasicMatchContext(DAG), P);
75}
76
77template <typename Pattern>
78[[nodiscard]] bool sd_match(SDValue N, const SelectionDAG *DAG, Pattern &&P) {
79 return sd_context_match(N, BasicMatchContext(DAG), P);
80}
81
82template <typename Pattern>
83[[nodiscard]] bool sd_match(SDNode *N, Pattern &&P) {
84 return sd_match(N, nullptr, P);
85}
86
87template <typename Pattern>
88[[nodiscard]] bool sd_match(SDValue N, Pattern &&P) {
89 return sd_match(N, nullptr, P);
90}
91
92// === Utilities ===
93struct Value_match {
94 SDValue MatchVal;
95
96 Value_match() = default;
97
98 explicit Value_match(SDValue Match) : MatchVal(Match) {}
99
100 template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
101 if (MatchVal)
102 return MatchVal == N;
103 return N.getNode();
104 }
105};
106
107/// Match any valid SDValue.
108inline Value_match m_Value() { return Value_match(); }
109
110inline Value_match m_Specific(SDValue N) {
111 assert(N);
112 return Value_match(N);
113}
114
115template <unsigned ResNo, typename Pattern> struct Result_match {
116 Pattern P;
117
118 explicit Result_match(const Pattern &P) : P(P) {}
119
120 template <typename MatchContext>
121 bool match(const MatchContext &Ctx, SDValue N) {
122 return N.getResNo() == ResNo && P.match(Ctx, N);
123 }
124};
125
126/// Match only if the SDValue is a certain result at ResNo.
127template <unsigned ResNo, typename Pattern>
128inline Result_match<ResNo, Pattern> m_Result(const Pattern &P) {
129 return Result_match<ResNo, Pattern>(P);
130}
131
132struct DeferredValue_match {
133 SDValue &MatchVal;
134
135 explicit DeferredValue_match(SDValue &Match) : MatchVal(Match) {}
136
137 template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
138 return N == MatchVal;
139 }
140};
141
142/// Similar to m_Specific, but the specific value to match is determined by
143/// another sub-pattern in the same sd_match() expression. For instance,
144/// We cannot match `(add V, V)` with `m_Add(m_Value(X), m_Specific(X))` since
145/// `X` is not initialized at the time it got copied into `m_Specific`. Instead,
146/// we should use `m_Add(m_Value(X), m_Deferred(X))`.
147inline DeferredValue_match m_Deferred(SDValue &V) {
148 return DeferredValue_match(V);
149}
150
151struct Opcode_match {
152 unsigned Opcode;
153
154 explicit Opcode_match(unsigned Opc) : Opcode(Opc) {}
155
156 template <typename MatchContext>
157 bool match(const MatchContext &Ctx, SDValue N) {
158 return Ctx.match(N, Opcode);
159 }
160};
161
162// === Patterns combinators ===
163template <typename... Preds> struct And {
164 template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
165 return true;
166 }
167};
168
169template <typename Pred, typename... Preds>
170struct And<Pred, Preds...> : And<Preds...> {
171 Pred P;
172 And(const Pred &p, const Preds &...preds) : And<Preds...>(preds...), P(p) {}
173
174 template <typename MatchContext>
175 bool match(const MatchContext &Ctx, SDValue N) {
176 return P.match(Ctx, N) && And<Preds...>::match(Ctx, N);
177 }
178};
179
180template <typename... Preds> struct Or {
181 template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
182 return false;
183 }
184};
185
186template <typename Pred, typename... Preds>
187struct Or<Pred, Preds...> : Or<Preds...> {
188 Pred P;
189 Or(const Pred &p, const Preds &...preds) : Or<Preds...>(preds...), P(p) {}
190
191 template <typename MatchContext>
192 bool match(const MatchContext &Ctx, SDValue N) {
193 return P.match(Ctx, N) || Or<Preds...>::match(Ctx, N);
194 }
195};
196
197template <typename Pred> struct Not {
198 Pred P;
199
200 explicit Not(const Pred &P) : P(P) {}
201
202 template <typename MatchContext>
203 bool match(const MatchContext &Ctx, SDValue N) {
204 return !P.match(Ctx, N);
205 }
206};
207// Explicit deduction guide.
208template <typename Pred> Not(const Pred &P) -> Not<Pred>;
209
210/// Match if the inner pattern does NOT match.
211template <typename Pred> inline Not<Pred> m_Unless(const Pred &P) {
212 return Not{P};
213}
214
215template <typename... Preds> And<Preds...> m_AllOf(const Preds &...preds) {
216 return And<Preds...>(preds...);
217}
218
219template <typename... Preds> Or<Preds...> m_AnyOf(const Preds &...preds) {
220 return Or<Preds...>(preds...);
221}
222
223template <typename... Preds> auto m_NoneOf(const Preds &...preds) {
224 return m_Unless(m_AnyOf(preds...));
225}
226
227inline Opcode_match m_SpecificOpc(unsigned Opcode) {
228 return Opcode_match(Opcode);
229}
230
231inline auto m_Undef() {
232 return m_AnyOf(preds: Opcode_match(ISD::UNDEF), preds: Opcode_match(ISD::POISON));
233}
234
235inline Opcode_match m_Poison() { return Opcode_match(ISD::POISON); }
236
237template <unsigned NumUses, typename Pattern> struct NUses_match {
238 Pattern P;
239
240 explicit NUses_match(const Pattern &P) : P(P) {}
241
242 template <typename MatchContext>
243 bool match(const MatchContext &Ctx, SDValue N) {
244 // SDNode::hasNUsesOfValue is pretty expensive when the SDNode produces
245 // multiple results, hence we check the subsequent pattern here before
246 // checking the number of value users.
247 return P.match(Ctx, N) && N->hasNUsesOfValue(NUses: NumUses, Value: N.getResNo());
248 }
249};
250
251template <typename Pattern>
252inline NUses_match<1, Pattern> m_OneUse(const Pattern &P) {
253 return NUses_match<1, Pattern>(P);
254}
255template <unsigned N, typename Pattern>
256inline NUses_match<N, Pattern> m_NUses(const Pattern &P) {
257 return NUses_match<N, Pattern>(P);
258}
259
260inline NUses_match<1, Value_match> m_OneUse() {
261 return NUses_match<1, Value_match>(m_Value());
262}
263template <unsigned N> inline NUses_match<N, Value_match> m_NUses() {
264 return NUses_match<N, Value_match>(m_Value());
265}
266
267template <typename PredPattern> struct Value_bind {
268 SDValue &BindVal;
269 PredPattern Pred;
270
271 Value_bind(SDValue &N, const PredPattern &P) : BindVal(N), Pred(P) {}
272
273 template <typename MatchContext>
274 bool match(const MatchContext &Ctx, SDValue N) {
275 if (!Pred.match(Ctx, N))
276 return false;
277
278 BindVal = N;
279 return true;
280 }
281};
282
283inline auto m_Value(SDValue &N) {
284 return Value_bind<Value_match>(N, m_Value());
285}
286/// Conditionally bind an SDValue based on the predicate.
287template <typename PredPattern>
288inline auto m_Value(SDValue &N, const PredPattern &P) {
289 return Value_bind<PredPattern>(N, P);
290}
291
292template <typename Pattern, typename PredFuncT> struct TLI_pred_match {
293 Pattern P;
294 PredFuncT PredFunc;
295
296 TLI_pred_match(const PredFuncT &Pred, const Pattern &P)
297 : P(P), PredFunc(Pred) {}
298
299 template <typename MatchContext>
300 bool match(const MatchContext &Ctx, SDValue N) {
301 assert(Ctx.getTLI() && "TargetLowering is required for this pattern.");
302 return PredFunc(*Ctx.getTLI(), N) && P.match(Ctx, N);
303 }
304};
305
306// Explicit deduction guide.
307template <typename PredFuncT, typename Pattern>
308TLI_pred_match(const PredFuncT &Pred, const Pattern &P)
309 -> TLI_pred_match<Pattern, PredFuncT>;
310
311/// Match legal SDNodes based on the information provided by TargetLowering.
312template <typename Pattern> inline auto m_LegalOp(const Pattern &P) {
313 return TLI_pred_match{[](const TargetLowering &TLI, SDValue N) {
314 return TLI.isOperationLegal(Op: N->getOpcode(),
315 VT: N.getValueType());
316 },
317 P};
318}
319
320/// Switch to a different MatchContext for subsequent patterns.
321template <typename NewMatchContext, typename Pattern> struct SwitchContext {
322 const NewMatchContext &Ctx;
323 Pattern P;
324
325 template <typename OrigMatchContext>
326 bool match(const OrigMatchContext &, SDValue N) {
327 return P.match(Ctx, N);
328 }
329};
330
331template <typename MatchContext, typename Pattern>
332inline SwitchContext<MatchContext, Pattern> m_Context(const MatchContext &Ctx,
333 Pattern &&P) {
334 return SwitchContext<MatchContext, Pattern>{Ctx, std::move(P)};
335}
336
337// === Value type ===
338
339template <typename Pattern> struct ValueType_bind {
340 EVT &BindVT;
341 Pattern P;
342
343 explicit ValueType_bind(EVT &Bind, const Pattern &P) : BindVT(Bind), P(P) {}
344
345 template <typename MatchContext>
346 bool match(const MatchContext &Ctx, SDValue N) {
347 BindVT = N.getValueType();
348 return P.match(Ctx, N);
349 }
350};
351
352template <typename Pattern>
353ValueType_bind(const Pattern &P) -> ValueType_bind<Pattern>;
354
355/// Retreive the ValueType of the current SDValue.
356inline auto m_VT(EVT &VT) { return ValueType_bind(VT, m_Value()); }
357
358template <typename Pattern> inline auto m_VT(EVT &VT, const Pattern &P) {
359 return ValueType_bind(VT, P);
360}
361
362template <typename Pattern, typename PredFuncT> struct ValueType_match {
363 PredFuncT PredFunc;
364 Pattern P;
365
366 ValueType_match(const PredFuncT &Pred, const Pattern &P)
367 : PredFunc(Pred), P(P) {}
368
369 template <typename MatchContext>
370 bool match(const MatchContext &Ctx, SDValue N) {
371 return PredFunc(N.getValueType()) && P.match(Ctx, N);
372 }
373};
374
375// Explicit deduction guide.
376template <typename PredFuncT, typename Pattern>
377ValueType_match(const PredFuncT &Pred, const Pattern &P)
378 -> ValueType_match<Pattern, PredFuncT>;
379
380/// Match a specific ValueType.
381template <typename Pattern>
382inline auto m_SpecificVT(EVT RefVT, const Pattern &P) {
383 return ValueType_match{[=](EVT VT) { return VT == RefVT; }, P};
384}
385inline auto m_SpecificVT(EVT RefVT) {
386 return ValueType_match{[=](EVT VT) { return VT == RefVT; }, m_Value()};
387}
388
389inline auto m_Glue() { return m_SpecificVT(RefVT: MVT::Glue); }
390inline auto m_OtherVT() { return m_SpecificVT(RefVT: MVT::Other); }
391
392/// Match a scalar ValueType.
393template <typename Pattern>
394inline auto m_SpecificScalarVT(EVT RefVT, const Pattern &P) {
395 return ValueType_match{[=](EVT VT) { return VT.getScalarType() == RefVT; },
396 P};
397}
398inline auto m_SpecificScalarVT(EVT RefVT) {
399 return ValueType_match{[=](EVT VT) { return VT.getScalarType() == RefVT; },
400 m_Value()};
401}
402
403/// Match a vector ValueType.
404template <typename Pattern>
405inline auto m_SpecificVectorElementVT(EVT RefVT, const Pattern &P) {
406 return ValueType_match{[=](EVT VT) {
407 return VT.isVector() &&
408 VT.getVectorElementType() == RefVT;
409 },
410 P};
411}
412inline auto m_SpecificVectorElementVT(EVT RefVT) {
413 return ValueType_match{[=](EVT VT) {
414 return VT.isVector() &&
415 VT.getVectorElementType() == RefVT;
416 },
417 m_Value()};
418}
419
420/// Match any integer ValueTypes.
421template <typename Pattern> inline auto m_IntegerVT(const Pattern &P) {
422 return ValueType_match{[](EVT VT) { return VT.isInteger(); }, P};
423}
424inline auto m_IntegerVT() {
425 return ValueType_match{[](EVT VT) { return VT.isInteger(); }, m_Value()};
426}
427
428/// Match any floating point ValueTypes.
429template <typename Pattern> inline auto m_FloatingPointVT(const Pattern &P) {
430 return ValueType_match{[](EVT VT) { return VT.isFloatingPoint(); }, P};
431}
432inline auto m_FloatingPointVT() {
433 return ValueType_match{[](EVT VT) { return VT.isFloatingPoint(); },
434 m_Value()};
435}
436
437/// Match any vector ValueTypes.
438template <typename Pattern> inline auto m_VectorVT(const Pattern &P) {
439 return ValueType_match{[](EVT VT) { return VT.isVector(); }, P};
440}
441inline auto m_VectorVT() {
442 return ValueType_match{[](EVT VT) { return VT.isVector(); }, m_Value()};
443}
444
445/// Match fixed-length vector ValueTypes.
446template <typename Pattern> inline auto m_FixedVectorVT(const Pattern &P) {
447 return ValueType_match{[](EVT VT) { return VT.isFixedLengthVector(); }, P};
448}
449inline auto m_FixedVectorVT() {
450 return ValueType_match{[](EVT VT) { return VT.isFixedLengthVector(); },
451 m_Value()};
452}
453
454/// Match scalable vector ValueTypes.
455template <typename Pattern> inline auto m_ScalableVectorVT(const Pattern &P) {
456 return ValueType_match{[](EVT VT) { return VT.isScalableVector(); }, P};
457}
458inline auto m_ScalableVectorVT() {
459 return ValueType_match{[](EVT VT) { return VT.isScalableVector(); },
460 m_Value()};
461}
462
463/// Match legal ValueTypes based on the information provided by TargetLowering.
464template <typename Pattern> inline auto m_LegalType(const Pattern &P) {
465 return TLI_pred_match{[](const TargetLowering &TLI, SDValue N) {
466 return TLI.isTypeLegal(VT: N.getValueType());
467 },
468 P};
469}
470
471// === Generic node matching ===
472template <unsigned OpIdx, typename... OpndPreds> struct Operands_match {
473 template <typename MatchContext>
474 bool match(const MatchContext &Ctx, SDValue N) {
475 // Returns false if there are more operands than predicates;
476 // Ignores the last two operands if both the Context and the Node are VP
477 return Ctx.getNumOperands(N) == OpIdx;
478 }
479};
480
481template <unsigned OpIdx, typename OpndPred, typename... OpndPreds>
482struct Operands_match<OpIdx, OpndPred, OpndPreds...>
483 : Operands_match<OpIdx + 1, OpndPreds...> {
484 OpndPred P;
485
486 Operands_match(const OpndPred &p, const OpndPreds &...preds)
487 : Operands_match<OpIdx + 1, OpndPreds...>(preds...), P(p) {}
488
489 template <typename MatchContext>
490 bool match(const MatchContext &Ctx, SDValue N) {
491 if (OpIdx < N->getNumOperands())
492 return P.match(Ctx, N->getOperand(Num: OpIdx)) &&
493 Operands_match<OpIdx + 1, OpndPreds...>::match(Ctx, N);
494
495 // This is the case where there are more predicates than operands.
496 return false;
497 }
498};
499
500template <typename... OpndPreds>
501auto m_Node(unsigned Opcode, const OpndPreds &...preds) {
502 return m_AllOf(m_SpecificOpc(Opcode),
503 Operands_match<0, OpndPreds...>(preds...));
504}
505
506/// Provide number of operands that are not chain or glue, as well as the first
507/// index of such operand.
508template <bool ExcludeChain> struct EffectiveOperands {
509 unsigned Size = 0;
510 unsigned FirstIndex = 0;
511
512 template <typename MatchContext>
513 explicit EffectiveOperands(SDValue N, const MatchContext &Ctx) {
514 const unsigned TotalNumOps = Ctx.getNumOperands(N);
515 FirstIndex = TotalNumOps;
516 for (unsigned I = 0; I < TotalNumOps; ++I) {
517 // Count the number of non-chain and non-glue nodes (we ignore chain
518 // and glue by default) and retreive the operand index offset.
519 EVT VT = N->getOperand(Num: I).getValueType();
520 if (VT != MVT::Glue && VT != MVT::Other) {
521 ++Size;
522 if (FirstIndex == TotalNumOps)
523 FirstIndex = I;
524 }
525 }
526 }
527};
528
529template <> struct EffectiveOperands<false> {
530 unsigned Size = 0;
531 unsigned FirstIndex = 0;
532
533 template <typename MatchContext>
534 explicit EffectiveOperands(SDValue N, const MatchContext &Ctx)
535 : Size(Ctx.getNumOperands(N)) {}
536};
537
538// === Ternary operations ===
539template <typename T0_P, typename T1_P, typename T2_P, bool Commutable = false,
540 bool ExcludeChain = false>
541struct TernaryOpc_match {
542 unsigned Opcode;
543 T0_P Op0;
544 T1_P Op1;
545 T2_P Op2;
546
547 TernaryOpc_match(unsigned Opc, const T0_P &Op0, const T1_P &Op1,
548 const T2_P &Op2)
549 : Opcode(Opc), Op0(Op0), Op1(Op1), Op2(Op2) {}
550
551 template <typename MatchContext>
552 bool match(const MatchContext &Ctx, SDValue N) {
553 if (sd_context_match(N, Ctx, m_SpecificOpc(Opcode))) {
554 EffectiveOperands<ExcludeChain> EO(N, Ctx);
555 assert(EO.Size == 3);
556 return ((Op0.match(Ctx, N->getOperand(Num: EO.FirstIndex)) &&
557 Op1.match(Ctx, N->getOperand(Num: EO.FirstIndex + 1))) ||
558 (Commutable && Op0.match(Ctx, N->getOperand(Num: EO.FirstIndex + 1)) &&
559 Op1.match(Ctx, N->getOperand(Num: EO.FirstIndex)))) &&
560 Op2.match(Ctx, N->getOperand(Num: EO.FirstIndex + 2));
561 }
562
563 return false;
564 }
565};
566
567template <typename T0_P, typename T1_P, typename T2_P>
568inline TernaryOpc_match<T0_P, T1_P, T2_P>
569m_SetCC(const T0_P &LHS, const T1_P &RHS, const T2_P &CC) {
570 return TernaryOpc_match<T0_P, T1_P, T2_P>(ISD::SETCC, LHS, RHS, CC);
571}
572
573template <typename T0_P, typename T1_P, typename T2_P>
574inline TernaryOpc_match<T0_P, T1_P, T2_P, true, false>
575m_c_SetCC(const T0_P &LHS, const T1_P &RHS, const T2_P &CC) {
576 return TernaryOpc_match<T0_P, T1_P, T2_P, true, false>(ISD::SETCC, LHS, RHS,
577 CC);
578}
579
580template <typename T0_P, typename T1_P, typename T2_P>
581inline TernaryOpc_match<T0_P, T1_P, T2_P>
582m_Select(const T0_P &Cond, const T1_P &T, const T2_P &F) {
583 return TernaryOpc_match<T0_P, T1_P, T2_P>(ISD::SELECT, Cond, T, F);
584}
585
586template <typename T0_P, typename T1_P, typename T2_P>
587inline TernaryOpc_match<T0_P, T1_P, T2_P>
588m_VSelect(const T0_P &Cond, const T1_P &T, const T2_P &F) {
589 return TernaryOpc_match<T0_P, T1_P, T2_P>(ISD::VSELECT, Cond, T, F);
590}
591
592template <typename T0_P, typename T1_P, typename T2_P>
593inline auto m_SelectLike(const T0_P &Cond, const T1_P &T, const T2_P &F) {
594 return m_AnyOf(m_Select(Cond, T, F), m_VSelect(Cond, T, F));
595}
596
597template <typename T0_P, typename T1_P, typename T2_P>
598inline Result_match<0, TernaryOpc_match<T0_P, T1_P, T2_P>>
599m_Load(const T0_P &Ch, const T1_P &Ptr, const T2_P &Offset) {
600 return m_Result<0>(
601 TernaryOpc_match<T0_P, T1_P, T2_P>(ISD::LOAD, Ch, Ptr, Offset));
602}
603
604template <typename T0_P, typename T1_P, typename T2_P>
605inline TernaryOpc_match<T0_P, T1_P, T2_P>
606m_InsertElt(const T0_P &Vec, const T1_P &Val, const T2_P &Idx) {
607 return TernaryOpc_match<T0_P, T1_P, T2_P>(ISD::INSERT_VECTOR_ELT, Vec, Val,
608 Idx);
609}
610
611template <typename LHS, typename RHS, typename IDX>
612inline TernaryOpc_match<LHS, RHS, IDX>
613m_InsertSubvector(const LHS &Base, const RHS &Sub, const IDX &Idx) {
614 return TernaryOpc_match<LHS, RHS, IDX>(ISD::INSERT_SUBVECTOR, Base, Sub, Idx);
615}
616
617template <typename T0_P, typename T1_P, typename T2_P>
618inline TernaryOpc_match<T0_P, T1_P, T2_P>
619m_SpliceRight(const T0_P &V1, const T1_P &V2, const T2_P &Offset) {
620 return TernaryOpc_match<T0_P, T1_P, T2_P>(ISD::VECTOR_SPLICE_RIGHT, V1, V2,
621 Offset);
622}
623
624template <typename T0_P, typename T1_P, typename T2_P>
625inline TernaryOpc_match<T0_P, T1_P, T2_P>
626m_TernaryOp(unsigned Opc, const T0_P &Op0, const T1_P &Op1, const T2_P &Op2) {
627 return TernaryOpc_match<T0_P, T1_P, T2_P>(Opc, Op0, Op1, Op2);
628}
629
630template <typename T0_P, typename T1_P, typename T2_P>
631inline TernaryOpc_match<T0_P, T1_P, T2_P, true>
632m_c_TernaryOp(unsigned Opc, const T0_P &Op0, const T1_P &Op1, const T2_P &Op2) {
633 return TernaryOpc_match<T0_P, T1_P, T2_P, true>(Opc, Op0, Op1, Op2);
634}
635
636template <typename LTy, typename RTy, typename TTy, typename FTy, typename CCTy>
637inline auto m_SelectCC(const LTy &L, const RTy &R, const TTy &T, const FTy &F,
638 const CCTy &CC) {
639 return m_Node(ISD::SELECT_CC, L, R, T, F, CC);
640}
641
642template <typename LTy, typename RTy, typename TTy, typename FTy, typename CCTy>
643inline auto m_SelectCCLike(const LTy &L, const RTy &R, const TTy &T,
644 const FTy &F, const CCTy &CC) {
645 return m_AnyOf(m_Select(m_SetCC(L, R, CC), T, F), m_SelectCC(L, R, T, F, CC));
646}
647
648// === Binary operations ===
649template <typename LHS_P, typename RHS_P, bool Commutable = false,
650 bool ExcludeChain = false>
651struct BinaryOpc_match {
652 unsigned Opcode;
653 LHS_P LHS;
654 RHS_P RHS;
655 SDNodeFlags Flags;
656 BinaryOpc_match(unsigned Opc, const LHS_P &L, const RHS_P &R,
657 SDNodeFlags Flgs = SDNodeFlags())
658 : Opcode(Opc), LHS(L), RHS(R), Flags(Flgs) {}
659
660 template <typename MatchContext>
661 bool match(const MatchContext &Ctx, SDValue N) {
662 if (sd_context_match(N, Ctx, m_SpecificOpc(Opcode))) {
663 EffectiveOperands<ExcludeChain> EO(N, Ctx);
664 assert(EO.Size == 2);
665 if (!((LHS.match(Ctx, N->getOperand(Num: EO.FirstIndex)) &&
666 RHS.match(Ctx, N->getOperand(Num: EO.FirstIndex + 1))) ||
667 (Commutable && LHS.match(Ctx, N->getOperand(Num: EO.FirstIndex + 1)) &&
668 RHS.match(Ctx, N->getOperand(Num: EO.FirstIndex)))))
669 return false;
670
671 return (Flags & N->getFlags()) == Flags;
672 }
673
674 return false;
675 }
676};
677
678/// Matching while capturing mask
679template <typename T0, typename T1, typename T2> struct SDShuffle_match {
680 T0 Op1;
681 T1 Op2;
682 T2 Mask;
683
684 SDShuffle_match(const T0 &Op1, const T1 &Op2, const T2 &Mask)
685 : Op1(Op1), Op2(Op2), Mask(Mask) {}
686
687 template <typename MatchContext>
688 bool match(const MatchContext &Ctx, SDValue N) {
689 if (auto *I = dyn_cast<ShuffleVectorSDNode>(Val&: N)) {
690 return Op1.match(Ctx, I->getOperand(Num: 0)) &&
691 Op2.match(Ctx, I->getOperand(Num: 1)) && Mask.match(I->getMask());
692 }
693 return false;
694 }
695};
696struct m_Mask {
697 ArrayRef<int> &MaskRef;
698 m_Mask(ArrayRef<int> &MaskRef) : MaskRef(MaskRef) {}
699 bool match(ArrayRef<int> Mask) {
700 MaskRef = Mask;
701 return true;
702 }
703};
704
705struct m_SpecificMask {
706 ArrayRef<int> MaskRef;
707 m_SpecificMask(ArrayRef<int> MaskRef) : MaskRef(MaskRef) {}
708 bool match(ArrayRef<int> Mask) { return MaskRef == Mask; }
709};
710
711template <typename LHS_P, typename RHS_P, typename Pred_t,
712 bool Commutable = false, bool ExcludeChain = false>
713struct MaxMin_match {
714 using PredType = Pred_t;
715 LHS_P LHS;
716 RHS_P RHS;
717
718 MaxMin_match(const LHS_P &L, const RHS_P &R) : LHS(L), RHS(R) {}
719
720 template <typename MatchContext>
721 bool match(const MatchContext &Ctx, SDValue N) {
722 auto MatchMinMax = [&](SDValue L, SDValue R, SDValue TrueValue,
723 SDValue FalseValue, ISD::CondCode CC) {
724 if ((TrueValue != L || FalseValue != R) &&
725 (TrueValue != R || FalseValue != L))
726 return false;
727
728 ISD::CondCode Cond =
729 TrueValue == L ? CC : getSetCCInverse(Operation: CC, Type: L.getValueType());
730 if (!Pred_t::match(Cond))
731 return false;
732
733 return (LHS.match(Ctx, L) && RHS.match(Ctx, R)) ||
734 (Commutable && LHS.match(Ctx, R) && RHS.match(Ctx, L));
735 };
736
737 if (sd_context_match(N, Ctx, m_SpecificOpc(Opcode: ISD::SELECT)) ||
738 sd_context_match(N, Ctx, m_SpecificOpc(Opcode: ISD::VSELECT))) {
739 EffectiveOperands<ExcludeChain> EO_SELECT(N, Ctx);
740 assert(EO_SELECT.Size == 3);
741 SDValue Cond = N->getOperand(Num: EO_SELECT.FirstIndex);
742 SDValue TrueValue = N->getOperand(Num: EO_SELECT.FirstIndex + 1);
743 SDValue FalseValue = N->getOperand(Num: EO_SELECT.FirstIndex + 2);
744
745 if (sd_context_match(Cond, Ctx, m_SpecificOpc(Opcode: ISD::SETCC))) {
746 EffectiveOperands<ExcludeChain> EO_SETCC(Cond, Ctx);
747 assert(EO_SETCC.Size == 3);
748 SDValue L = Cond->getOperand(Num: EO_SETCC.FirstIndex);
749 SDValue R = Cond->getOperand(Num: EO_SETCC.FirstIndex + 1);
750 auto *CondNode =
751 cast<CondCodeSDNode>(Cond->getOperand(Num: EO_SETCC.FirstIndex + 2));
752 return MatchMinMax(L, R, TrueValue, FalseValue, CondNode->get());
753 }
754 }
755
756 if (sd_context_match(N, Ctx, m_SpecificOpc(Opcode: ISD::SELECT_CC))) {
757 EffectiveOperands<ExcludeChain> EO_SELECT(N, Ctx);
758 assert(EO_SELECT.Size == 5);
759 SDValue L = N->getOperand(Num: EO_SELECT.FirstIndex);
760 SDValue R = N->getOperand(Num: EO_SELECT.FirstIndex + 1);
761 SDValue TrueValue = N->getOperand(Num: EO_SELECT.FirstIndex + 2);
762 SDValue FalseValue = N->getOperand(Num: EO_SELECT.FirstIndex + 3);
763 auto *CondNode =
764 cast<CondCodeSDNode>(N->getOperand(Num: EO_SELECT.FirstIndex + 4));
765 return MatchMinMax(L, R, TrueValue, FalseValue, CondNode->get());
766 }
767
768 return false;
769 }
770};
771
772// Helper class for identifying signed max predicates.
773struct smax_pred_ty {
774 static bool match(ISD::CondCode Cond) {
775 return Cond == ISD::CondCode::SETGT || Cond == ISD::CondCode::SETGE;
776 }
777};
778
779// Helper class for identifying unsigned max predicates.
780struct umax_pred_ty {
781 static bool match(ISD::CondCode Cond) {
782 return Cond == ISD::CondCode::SETUGT || Cond == ISD::CondCode::SETUGE;
783 }
784};
785
786// Helper class for identifying signed min predicates.
787struct smin_pred_ty {
788 static bool match(ISD::CondCode Cond) {
789 return Cond == ISD::CondCode::SETLT || Cond == ISD::CondCode::SETLE;
790 }
791};
792
793// Helper class for identifying unsigned min predicates.
794struct umin_pred_ty {
795 static bool match(ISD::CondCode Cond) {
796 return Cond == ISD::CondCode::SETULT || Cond == ISD::CondCode::SETULE;
797 }
798};
799
800template <typename LHS, typename RHS>
801inline BinaryOpc_match<LHS, RHS> m_BinOp(unsigned Opc, const LHS &L,
802 const RHS &R,
803 SDNodeFlags Flgs = SDNodeFlags()) {
804 return BinaryOpc_match<LHS, RHS>(Opc, L, R, Flgs);
805}
806template <typename LHS, typename RHS>
807inline BinaryOpc_match<LHS, RHS, true>
808m_c_BinOp(unsigned Opc, const LHS &L, const RHS &R,
809 SDNodeFlags Flgs = SDNodeFlags()) {
810 return BinaryOpc_match<LHS, RHS, true>(Opc, L, R, Flgs);
811}
812
813template <typename LHS, typename RHS>
814inline BinaryOpc_match<LHS, RHS, false, true>
815m_ChainedBinOp(unsigned Opc, const LHS &L, const RHS &R) {
816 return BinaryOpc_match<LHS, RHS, false, true>(Opc, L, R);
817}
818template <typename LHS, typename RHS>
819inline BinaryOpc_match<LHS, RHS, true, true>
820m_c_ChainedBinOp(unsigned Opc, const LHS &L, const RHS &R) {
821 return BinaryOpc_match<LHS, RHS, true, true>(Opc, L, R);
822}
823
824// Common binary operations
825template <typename LHS, typename RHS>
826inline BinaryOpc_match<LHS, RHS, true> m_Add(const LHS &L, const RHS &R) {
827 return BinaryOpc_match<LHS, RHS, true>(ISD::ADD, L, R);
828}
829
830template <typename LHS, typename RHS>
831inline BinaryOpc_match<LHS, RHS> m_Sub(const LHS &L, const RHS &R) {
832 return BinaryOpc_match<LHS, RHS>(ISD::SUB, L, R);
833}
834
835template <typename LHS, typename RHS>
836inline BinaryOpc_match<LHS, RHS, true> m_Mul(const LHS &L, const RHS &R) {
837 return BinaryOpc_match<LHS, RHS, true>(ISD::MUL, L, R);
838}
839
840template <typename LHS, typename RHS>
841inline BinaryOpc_match<LHS, RHS, true> m_And(const LHS &L, const RHS &R) {
842 return BinaryOpc_match<LHS, RHS, true>(ISD::AND, L, R);
843}
844
845template <typename LHS, typename RHS>
846inline BinaryOpc_match<LHS, RHS, true> m_Or(const LHS &L, const RHS &R) {
847 return BinaryOpc_match<LHS, RHS, true>(ISD::OR, L, R);
848}
849
850template <typename LHS, typename RHS>
851inline BinaryOpc_match<LHS, RHS, true> m_DisjointOr(const LHS &L,
852 const RHS &R) {
853 return BinaryOpc_match<LHS, RHS, true>(ISD::OR, L, R, SDNodeFlags::Disjoint);
854}
855
856template <typename LHS, typename RHS>
857inline auto m_AddLike(const LHS &L, const RHS &R) {
858 return m_AnyOf(m_Add(L, R), m_DisjointOr(L, R));
859}
860
861template <typename LHS, typename RHS>
862inline BinaryOpc_match<LHS, RHS, true> m_Xor(const LHS &L, const RHS &R) {
863 return BinaryOpc_match<LHS, RHS, true>(ISD::XOR, L, R);
864}
865
866template <typename LHS, typename RHS>
867inline auto m_BitwiseLogic(const LHS &L, const RHS &R) {
868 return m_AnyOf(m_And(L, R), m_Or(L, R), m_Xor(L, R));
869}
870
871template <unsigned Opc, typename Pred, typename LHS, typename RHS>
872inline auto m_MaxMinLike(const LHS &L, const RHS &R) {
873 return m_AnyOf(BinaryOpc_match<LHS, RHS, true>(Opc, L, R),
874 MaxMin_match<LHS, RHS, Pred, true>(L, R));
875}
876
877template <typename LHS, typename RHS>
878inline BinaryOpc_match<LHS, RHS, true> m_SMin(const LHS &L, const RHS &R) {
879 return BinaryOpc_match<LHS, RHS, true>(ISD::SMIN, L, R);
880}
881
882template <typename LHS, typename RHS>
883inline auto m_SMinLike(const LHS &L, const RHS &R) {
884 return m_AnyOf(
885 m_MaxMinLike<ISD::SMIN, smin_pred_ty>(L, R),
886 m_MaxMinLike<ISD::UMIN, umin_pred_ty>(m_NonNegative(L), m_NonNegative(R)),
887 m_MaxMinLike<ISD::UMIN, umin_pred_ty>(m_Negative(L), m_Negative(R)));
888}
889
890template <typename LHS, typename RHS>
891inline BinaryOpc_match<LHS, RHS, true> m_SMax(const LHS &L, const RHS &R) {
892 return BinaryOpc_match<LHS, RHS, true>(ISD::SMAX, L, R);
893}
894
895template <typename LHS, typename RHS>
896inline auto m_SMaxLike(const LHS &L, const RHS &R) {
897 return m_AnyOf(
898 m_MaxMinLike<ISD::SMAX, smax_pred_ty>(L, R),
899 m_MaxMinLike<ISD::UMAX, umax_pred_ty>(m_NonNegative(L), m_NonNegative(R)),
900 m_MaxMinLike<ISD::UMAX, umax_pred_ty>(m_Negative(L), m_Negative(R)));
901}
902
903template <typename LHS, typename RHS>
904inline BinaryOpc_match<LHS, RHS, true> m_UMin(const LHS &L, const RHS &R) {
905 return BinaryOpc_match<LHS, RHS, true>(ISD::UMIN, L, R);
906}
907
908template <typename LHS, typename RHS>
909inline auto m_UMinLike(const LHS &L, const RHS &R) {
910 return m_AnyOf(
911 m_MaxMinLike<ISD::UMIN, umin_pred_ty>(L, R),
912 m_MaxMinLike<ISD::SMIN, smin_pred_ty>(m_NonNegative(L), m_NonNegative(R)),
913 m_MaxMinLike<ISD::SMIN, smin_pred_ty>(m_Negative(L), m_Negative(R)));
914}
915
916template <typename LHS, typename RHS>
917inline BinaryOpc_match<LHS, RHS, true> m_UMax(const LHS &L, const RHS &R) {
918 return BinaryOpc_match<LHS, RHS, true>(ISD::UMAX, L, R);
919}
920
921template <typename LHS, typename RHS>
922inline auto m_UMaxLike(const LHS &L, const RHS &R) {
923 return m_AnyOf(
924 m_MaxMinLike<ISD::UMAX, umax_pred_ty>(L, R),
925 m_MaxMinLike<ISD::SMAX, smax_pred_ty>(m_NonNegative(L), m_NonNegative(R)),
926 m_MaxMinLike<ISD::SMAX, smax_pred_ty>(m_Negative(L), m_Negative(R)));
927}
928
929template <typename LHS, typename RHS>
930inline BinaryOpc_match<LHS, RHS> m_UDiv(const LHS &L, const RHS &R) {
931 return BinaryOpc_match<LHS, RHS>(ISD::UDIV, L, R);
932}
933template <typename LHS, typename RHS>
934inline BinaryOpc_match<LHS, RHS> m_SDiv(const LHS &L, const RHS &R) {
935 return BinaryOpc_match<LHS, RHS>(ISD::SDIV, L, R);
936}
937
938template <typename LHS, typename RHS>
939inline BinaryOpc_match<LHS, RHS> m_URem(const LHS &L, const RHS &R) {
940 return BinaryOpc_match<LHS, RHS>(ISD::UREM, L, R);
941}
942template <typename LHS, typename RHS>
943inline BinaryOpc_match<LHS, RHS> m_SRem(const LHS &L, const RHS &R) {
944 return BinaryOpc_match<LHS, RHS>(ISD::SREM, L, R);
945}
946
947template <typename LHS, typename RHS>
948inline BinaryOpc_match<LHS, RHS> m_Shl(const LHS &L, const RHS &R) {
949 return BinaryOpc_match<LHS, RHS>(ISD::SHL, L, R);
950}
951
952template <typename LHS, typename RHS>
953inline BinaryOpc_match<LHS, RHS> m_Sra(const LHS &L, const RHS &R) {
954 return BinaryOpc_match<LHS, RHS>(ISD::SRA, L, R);
955}
956template <typename LHS, typename RHS>
957inline BinaryOpc_match<LHS, RHS> m_Srl(const LHS &L, const RHS &R) {
958 return BinaryOpc_match<LHS, RHS>(ISD::SRL, L, R);
959}
960template <typename LHS, typename RHS>
961inline auto m_ExactSr(const LHS &L, const RHS &R) {
962 return m_AnyOf(BinaryOpc_match<LHS, RHS>(ISD::SRA, L, R, SDNodeFlags::Exact),
963 BinaryOpc_match<LHS, RHS>(ISD::SRL, L, R, SDNodeFlags::Exact));
964}
965
966template <typename LHS, typename RHS>
967inline BinaryOpc_match<LHS, RHS> m_Rotl(const LHS &L, const RHS &R) {
968 return BinaryOpc_match<LHS, RHS>(ISD::ROTL, L, R);
969}
970
971template <typename LHS, typename RHS>
972inline BinaryOpc_match<LHS, RHS> m_Rotr(const LHS &L, const RHS &R) {
973 return BinaryOpc_match<LHS, RHS>(ISD::ROTR, L, R);
974}
975
976template <typename T0_P, typename T1_P, typename T2_P>
977inline TernaryOpc_match<T0_P, T1_P, T2_P>
978m_FShL(const T0_P &Op0, const T1_P &Op1, const T2_P &Op2) {
979 return m_TernaryOp(ISD::FSHL, Op0, Op1, Op2);
980}
981
982template <typename T0_P, typename T1_P, typename T2_P>
983inline TernaryOpc_match<T0_P, T1_P, T2_P>
984m_FShR(const T0_P &Op0, const T1_P &Op1, const T2_P &Op2) {
985 return m_TernaryOp(ISD::FSHR, Op0, Op1, Op2);
986}
987
988template <typename T0_P, typename T1_P, typename T2_P, bool Left>
989struct FunnelShiftLike_match {
990 T0_P Op0;
991 T1_P Op1;
992 T2_P Op2;
993
994 FunnelShiftLike_match(const T0_P &Op0, const T1_P &Op1, const T2_P &Op2)
995 : Op0(Op0), Op1(Op1), Op2(Op2) {}
996
997 static bool hasComplementaryConstantShifts(const APInt &ShlV,
998 const APInt &SrlV,
999 unsigned BitWidth) {
1000 unsigned SumWidth = std::max(a: ShlV.getBitWidth(), b: SrlV.getBitWidth()) + 1;
1001 unsigned BitWidthBits = llvm::bit_width(Value: BitWidth);
1002 if (BitWidthBits > SumWidth)
1003 return false;
1004
1005 return ShlV.zext(width: SumWidth) + SrlV.zext(width: SumWidth) ==
1006 APInt(SumWidth, BitWidth);
1007 }
1008
1009 template <typename MatchContext>
1010 bool matchOperands(const MatchContext &Ctx, SDValue X, SDValue Y, SDValue Z) {
1011 return Op0.match(Ctx, X) && Op1.match(Ctx, Y) && Op2.match(Ctx, Z);
1012 }
1013
1014 template <typename MatchContext>
1015 bool matchShiftOr(const MatchContext &Ctx, SDValue N, unsigned BitWidth);
1016
1017 template <typename MatchContext>
1018 bool match(const MatchContext &Ctx, SDValue N) {
1019 if (sd_context_match(N, Ctx,
1020 Left ? m_FShL(Op0, Op1, Op2) : m_FShR(Op0, Op1, Op2)))
1021 return true;
1022
1023 SDValue X, Z;
1024 if (sd_context_match(N, Ctx,
1025 Left ? m_Rotl(L: m_Value(N&: X), R: m_Value(N&: Z))
1026 : m_Rotr(L: m_Value(N&: X), R: m_Value(N&: Z))))
1027 return matchOperands(Ctx, X, X, Z);
1028
1029 return matchShiftOr(Ctx, N, N.getValueType().getScalarSizeInBits());
1030 }
1031};
1032
1033template <typename T0_P, typename T1_P, typename T2_P>
1034inline FunnelShiftLike_match<T0_P, T1_P, T2_P, true>
1035m_FShLLike(const T0_P &Op0, const T1_P &Op1, const T2_P &Op2) {
1036 return FunnelShiftLike_match<T0_P, T1_P, T2_P, true>(Op0, Op1, Op2);
1037}
1038
1039template <typename T0_P, typename T1_P, typename T2_P>
1040inline FunnelShiftLike_match<T0_P, T1_P, T2_P, false>
1041m_FShRLike(const T0_P &Op0, const T1_P &Op1, const T2_P &Op2) {
1042 return FunnelShiftLike_match<T0_P, T1_P, T2_P, false>(Op0, Op1, Op2);
1043}
1044
1045template <typename LHS, typename RHS>
1046inline BinaryOpc_match<LHS, RHS, true> m_Clmul(const LHS &L, const RHS &R) {
1047 return BinaryOpc_match<LHS, RHS, true>(ISD::CLMUL, L, R);
1048}
1049
1050template <typename LHS, typename RHS>
1051inline BinaryOpc_match<LHS, RHS, true> m_FAdd(const LHS &L, const RHS &R) {
1052 return BinaryOpc_match<LHS, RHS, true>(ISD::FADD, L, R);
1053}
1054
1055template <typename LHS, typename RHS>
1056inline BinaryOpc_match<LHS, RHS> m_FSub(const LHS &L, const RHS &R) {
1057 return BinaryOpc_match<LHS, RHS>(ISD::FSUB, L, R);
1058}
1059
1060template <typename LHS, typename RHS>
1061inline BinaryOpc_match<LHS, RHS, true> m_FMul(const LHS &L, const RHS &R) {
1062 return BinaryOpc_match<LHS, RHS, true>(ISD::FMUL, L, R);
1063}
1064
1065template <typename LHS, typename RHS>
1066inline BinaryOpc_match<LHS, RHS> m_FDiv(const LHS &L, const RHS &R) {
1067 return BinaryOpc_match<LHS, RHS>(ISD::FDIV, L, R);
1068}
1069
1070template <typename LHS, typename RHS>
1071inline BinaryOpc_match<LHS, RHS> m_FRem(const LHS &L, const RHS &R) {
1072 return BinaryOpc_match<LHS, RHS>(ISD::FREM, L, R);
1073}
1074
1075template <typename V1_t, typename V2_t>
1076inline BinaryOpc_match<V1_t, V2_t> m_Shuffle(const V1_t &v1, const V2_t &v2) {
1077 return BinaryOpc_match<V1_t, V2_t>(ISD::VECTOR_SHUFFLE, v1, v2);
1078}
1079
1080template <typename V1_t, typename V2_t, typename Mask_t>
1081inline SDShuffle_match<V1_t, V2_t, Mask_t>
1082m_Shuffle(const V1_t &v1, const V2_t &v2, const Mask_t &mask) {
1083 return SDShuffle_match<V1_t, V2_t, Mask_t>(v1, v2, mask);
1084}
1085
1086template <typename LHS, typename RHS>
1087inline BinaryOpc_match<LHS, RHS> m_ExtractElt(const LHS &Vec, const RHS &Idx) {
1088 return BinaryOpc_match<LHS, RHS>(ISD::EXTRACT_VECTOR_ELT, Vec, Idx);
1089}
1090
1091template <typename LHS, typename RHS>
1092inline BinaryOpc_match<LHS, RHS> m_ExtractSubvector(const LHS &Vec,
1093 const RHS &Idx) {
1094 return BinaryOpc_match<LHS, RHS>(ISD::EXTRACT_SUBVECTOR, Vec, Idx);
1095}
1096
1097// === Unary operations ===
1098template <typename Opnd_P, bool ExcludeChain = false> struct UnaryOpc_match {
1099 unsigned Opcode;
1100 Opnd_P Opnd;
1101 SDNodeFlags Flags;
1102 UnaryOpc_match(unsigned Opc, const Opnd_P &Op,
1103 SDNodeFlags Flgs = SDNodeFlags())
1104 : Opcode(Opc), Opnd(Op), Flags(Flgs) {}
1105
1106 template <typename MatchContext>
1107 bool match(const MatchContext &Ctx, SDValue N) {
1108 if (sd_context_match(N, Ctx, m_SpecificOpc(Opcode))) {
1109 EffectiveOperands<ExcludeChain> EO(N, Ctx);
1110 assert(EO.Size == 1);
1111 if (!Opnd.match(Ctx, N->getOperand(Num: EO.FirstIndex)))
1112 return false;
1113
1114 return (Flags & N->getFlags()) == Flags;
1115 }
1116
1117 return false;
1118 }
1119};
1120
1121template <typename Opnd>
1122inline UnaryOpc_match<Opnd> m_UnaryOp(unsigned Opc, const Opnd &Op) {
1123 return UnaryOpc_match<Opnd>(Opc, Op);
1124}
1125template <typename Opnd>
1126inline UnaryOpc_match<Opnd, true> m_ChainedUnaryOp(unsigned Opc,
1127 const Opnd &Op) {
1128 return UnaryOpc_match<Opnd, true>(Opc, Op);
1129}
1130
1131template <typename Opnd> inline UnaryOpc_match<Opnd> m_BitCast(const Opnd &Op) {
1132 return UnaryOpc_match<Opnd>(ISD::BITCAST, Op);
1133}
1134
1135template <typename Opnd>
1136inline UnaryOpc_match<Opnd> m_BSwap(const Opnd &Op) {
1137 return UnaryOpc_match<Opnd>(ISD::BSWAP, Op);
1138}
1139
1140template <typename Opnd>
1141inline UnaryOpc_match<Opnd> m_BitReverse(const Opnd &Op) {
1142 return UnaryOpc_match<Opnd>(ISD::BITREVERSE, Op);
1143}
1144
1145template <typename Opnd> inline UnaryOpc_match<Opnd> m_ZExt(const Opnd &Op) {
1146 return UnaryOpc_match<Opnd>(ISD::ZERO_EXTEND, Op);
1147}
1148
1149template <typename Opnd>
1150inline UnaryOpc_match<Opnd> m_NNegZExt(const Opnd &Op) {
1151 return UnaryOpc_match<Opnd>(ISD::ZERO_EXTEND, Op, SDNodeFlags::NonNeg);
1152}
1153
1154template <typename Opnd> inline auto m_SExt(const Opnd &Op) {
1155 return UnaryOpc_match<Opnd>(ISD::SIGN_EXTEND, Op);
1156}
1157
1158template <typename Opnd> inline UnaryOpc_match<Opnd> m_AnyExt(const Opnd &Op) {
1159 return UnaryOpc_match<Opnd>(ISD::ANY_EXTEND, Op);
1160}
1161
1162template <typename Opnd> inline UnaryOpc_match<Opnd> m_Trunc(const Opnd &Op) {
1163 return UnaryOpc_match<Opnd>(ISD::TRUNCATE, Op);
1164}
1165
1166template <typename Opnd> inline auto m_Abs(const Opnd &Op) {
1167 return m_AnyOf(UnaryOpc_match<Opnd>(ISD::ABS, Op),
1168 UnaryOpc_match<Opnd>(ISD::ABS_MIN_POISON, Op));
1169}
1170
1171template <typename Opnd> inline UnaryOpc_match<Opnd> m_FAbs(const Opnd &Op) {
1172 return UnaryOpc_match<Opnd>(ISD::FABS, Op);
1173}
1174
1175/// Match a zext or identity
1176/// Allows to peek through optional extensions
1177template <typename Opnd> inline auto m_ZExtOrSelf(const Opnd &Op) {
1178 return m_AnyOf(m_ZExt(Op), Op);
1179}
1180
1181/// Match a sext or identity
1182/// Allows to peek through optional extensions
1183template <typename Opnd> inline auto m_SExtOrSelf(const Opnd &Op) {
1184 return m_AnyOf(m_SExt(Op), Op);
1185}
1186
1187template <typename Opnd> inline auto m_SExtLike(const Opnd &Op) {
1188 return m_AnyOf(m_SExt(Op), m_NNegZExt(Op));
1189}
1190
1191/// Match a aext or identity
1192/// Allows to peek through optional extensions
1193template <typename Opnd>
1194inline Or<UnaryOpc_match<Opnd>, Opnd> m_AExtOrSelf(const Opnd &Op) {
1195 return Or<UnaryOpc_match<Opnd>, Opnd>(m_AnyExt(Op), Op);
1196}
1197
1198/// Match a trunc or identity
1199/// Allows to peek through optional truncations
1200template <typename Opnd>
1201inline Or<UnaryOpc_match<Opnd>, Opnd> m_TruncOrSelf(const Opnd &Op) {
1202 return Or<UnaryOpc_match<Opnd>, Opnd>(m_Trunc(Op), Op);
1203}
1204
1205template <typename Opnd> inline UnaryOpc_match<Opnd> m_VScale(const Opnd &Op) {
1206 return UnaryOpc_match<Opnd>(ISD::VSCALE, Op);
1207}
1208
1209template <typename Opnd> inline UnaryOpc_match<Opnd> m_FPToUI(const Opnd &Op) {
1210 return UnaryOpc_match<Opnd>(ISD::FP_TO_UINT, Op);
1211}
1212
1213template <typename Opnd> inline UnaryOpc_match<Opnd> m_FPToSI(const Opnd &Op) {
1214 return UnaryOpc_match<Opnd>(ISD::FP_TO_SINT, Op);
1215}
1216
1217template <typename Opnd> inline UnaryOpc_match<Opnd> m_Ctpop(const Opnd &Op) {
1218 return UnaryOpc_match<Opnd>(ISD::CTPOP, Op);
1219}
1220
1221template <typename Opnd> inline UnaryOpc_match<Opnd> m_Ctlz(const Opnd &Op) {
1222 return UnaryOpc_match<Opnd>(ISD::CTLZ, Op);
1223}
1224
1225template <typename Opnd> inline UnaryOpc_match<Opnd> m_Cttz(const Opnd &Op) {
1226 return UnaryOpc_match<Opnd>(ISD::CTTZ, Op);
1227}
1228
1229template <typename Opnd> inline UnaryOpc_match<Opnd> m_FNeg(const Opnd &Op) {
1230 return UnaryOpc_match<Opnd>(ISD::FNEG, Op);
1231}
1232
1233template <typename Opnd>
1234inline UnaryOpc_match<Opnd> m_VectorReverse(const Opnd &Op) {
1235 return UnaryOpc_match<Opnd>(ISD::VECTOR_REVERSE, Op);
1236}
1237
1238// === Constants ===
1239struct ConstantInt_match {
1240 APInt *BindVal;
1241
1242 explicit ConstantInt_match(APInt *V) : BindVal(V) {}
1243
1244 template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
1245 // The logics here are similar to that in
1246 // SelectionDAG::isConstantIntBuildVectorOrConstantInt, but the latter also
1247 // treats GlobalAddressSDNode as a constant, which is difficult to turn into
1248 // APInt.
1249 if (auto *C = dyn_cast_or_null<ConstantSDNode>(Val: N.getNode())) {
1250 if (BindVal)
1251 *BindVal = C->getAPIntValue();
1252 return true;
1253 }
1254
1255 APInt Discard;
1256 return ISD::isConstantSplatVector(N: N.getNode(),
1257 SplatValue&: BindVal ? *BindVal : Discard);
1258 }
1259};
1260
1261template <typename T> struct Constant64_match {
1262 static_assert(sizeof(T) == 8, "T must be 64 bits wide");
1263
1264 T &BindVal;
1265
1266 explicit Constant64_match(T &V) : BindVal(V) {}
1267
1268 template <typename MatchContext>
1269 bool match(const MatchContext &Ctx, SDValue N) {
1270 APInt V;
1271 if (!ConstantInt_match(&V).match(Ctx, N))
1272 return false;
1273
1274 if constexpr (std::is_signed_v<T>) {
1275 if (std::optional<int64_t> TrySExt = V.trySExtValue()) {
1276 BindVal = *TrySExt;
1277 return true;
1278 }
1279 }
1280
1281 if constexpr (std::is_unsigned_v<T>) {
1282 if (std::optional<uint64_t> TryZExt = V.tryZExtValue()) {
1283 BindVal = *TryZExt;
1284 return true;
1285 }
1286 }
1287
1288 return false;
1289 }
1290};
1291
1292/// Match any integer constants or splat of an integer constant.
1293inline ConstantInt_match m_ConstInt() { return ConstantInt_match(nullptr); }
1294/// Match any integer constants or splat of an integer constant; return the
1295/// specific constant or constant splat value.
1296inline ConstantInt_match m_ConstInt(APInt &V) { return ConstantInt_match(&V); }
1297/// Match any integer constants or splat of an integer constant that can fit in
1298/// 64 bits; return the specific constant or constant splat value, zero-extended
1299/// to 64 bits.
1300inline Constant64_match<uint64_t> m_ConstInt(uint64_t &V) {
1301 return Constant64_match<uint64_t>(V);
1302}
1303/// Match any integer constants or splat of an integer constant that can fit in
1304/// 64 bits; return the specific constant or constant splat value, sign-extended
1305/// to 64 bits.
1306inline Constant64_match<int64_t> m_ConstInt(int64_t &V) {
1307 return Constant64_match<int64_t>(V);
1308}
1309
1310template <typename T0_P, typename T1_P, typename T2_P, bool Left>
1311template <typename MatchContext>
1312bool FunnelShiftLike_match<T0_P, T1_P, T2_P, Left>::matchShiftOr(
1313 const MatchContext &Ctx, SDValue N, unsigned BitWidth) {
1314 SDValue X, Y, ShlAmt, SrlAmt;
1315 APInt ShlConst, SrlConst;
1316 if (!sd_context_match(
1317 N, Ctx,
1318 m_Or(L: m_Shl(L: m_Value(N&: X), R: m_Value(N&: ShlAmt, P: m_ConstInt(V&: ShlConst))),
1319 R: m_Srl(L: m_Value(N&: Y), R: m_Value(N&: SrlAmt, P: m_ConstInt(V&: SrlConst))))) ||
1320 !hasComplementaryConstantShifts(ShlV: ShlConst, SrlV: SrlConst, BitWidth))
1321 return false;
1322
1323 return matchOperands(Ctx, X, Y, Left ? ShlAmt : SrlAmt);
1324}
1325
1326struct SpecificInt_match {
1327 APInt IntVal;
1328
1329 explicit SpecificInt_match(APInt APV) : IntVal(std::move(APV)) {}
1330
1331 template <typename MatchContext>
1332 bool match(const MatchContext &Ctx, SDValue N) {
1333 APInt ConstInt;
1334 if (sd_context_match(N, Ctx, m_ConstInt(V&: ConstInt)))
1335 return APInt::isSameValue(I1: IntVal, I2: ConstInt);
1336 return false;
1337 }
1338};
1339
1340/// Match a specific integer constant or constant splat value.
1341inline SpecificInt_match m_SpecificInt(APInt V) {
1342 return SpecificInt_match(std::move(V));
1343}
1344inline SpecificInt_match m_SpecificInt(uint64_t V) {
1345 return SpecificInt_match(APInt(64, V));
1346}
1347
1348struct SpecificFP_match {
1349 APFloat Val;
1350
1351 explicit SpecificFP_match(APFloat V) : Val(V) {}
1352
1353 template <typename MatchContext>
1354 bool match(const MatchContext &Ctx, SDValue V) {
1355 if (const auto *CFP = dyn_cast<ConstantFPSDNode>(Val: V.getNode()))
1356 return CFP->isExactlyValue(V: Val);
1357 if (ConstantFPSDNode *C = isConstOrConstSplatFP(N: V, /*AllowUndefs=*/AllowUndefs: true))
1358 return C->getValueAPF().compare(RHS: Val) == APFloat::cmpEqual;
1359 return false;
1360 }
1361};
1362
1363/// Match a specific float constant.
1364inline SpecificFP_match m_SpecificFP(APFloat V) { return SpecificFP_match(V); }
1365
1366inline SpecificFP_match m_SpecificFP(double V) {
1367 return SpecificFP_match(APFloat(V));
1368}
1369
1370struct Negative_match {
1371 template <typename MatchContext>
1372 bool match(const MatchContext &Ctx, SDValue N) {
1373 const SelectionDAG *DAG = Ctx.getDAG();
1374 return DAG && DAG->computeKnownBits(Op: N).isNegative();
1375 }
1376};
1377
1378struct NonNegative_match {
1379 template <typename MatchContext>
1380 bool match(const MatchContext &Ctx, SDValue N) {
1381 const SelectionDAG *DAG = Ctx.getDAG();
1382 return DAG && DAG->computeKnownBits(Op: N).isNonNegative();
1383 }
1384};
1385
1386struct StrictlyPositive_match {
1387 template <typename MatchContext>
1388 bool match(const MatchContext &Ctx, SDValue N) {
1389 const SelectionDAG *DAG = Ctx.getDAG();
1390 return DAG && DAG->computeKnownBits(Op: N).isStrictlyPositive();
1391 }
1392};
1393
1394struct NonPositive_match {
1395 template <typename MatchContext>
1396 bool match(const MatchContext &Ctx, SDValue N) {
1397 const SelectionDAG *DAG = Ctx.getDAG();
1398 return DAG && DAG->computeKnownBits(Op: N).isNonPositive();
1399 }
1400};
1401
1402struct NonZero_match {
1403 template <typename MatchContext>
1404 bool match(const MatchContext &Ctx, SDValue N) {
1405 const SelectionDAG *DAG = Ctx.getDAG();
1406 return DAG && DAG->computeKnownBits(Op: N).isNonZero();
1407 }
1408};
1409
1410struct Zero_match {
1411 bool AllowUndefs;
1412
1413 explicit Zero_match(bool AllowUndefs) : AllowUndefs(AllowUndefs) {}
1414
1415 template <typename MatchContext>
1416 bool match(const MatchContext &, SDValue N) const {
1417 return isZeroOrZeroSplat(N, AllowUndefs);
1418 }
1419};
1420
1421struct Ones_match {
1422 bool AllowUndefs;
1423
1424 Ones_match(bool AllowUndefs) : AllowUndefs(AllowUndefs) {}
1425
1426 template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
1427 return isOnesOrOnesSplat(N, AllowUndefs);
1428 }
1429};
1430
1431struct AllOnes_match {
1432 bool AllowUndefs;
1433
1434 AllOnes_match(bool AllowUndefs) : AllowUndefs(AllowUndefs) {}
1435
1436 template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
1437 return isAllOnesOrAllOnesSplat(V: N, AllowUndefs);
1438 }
1439};
1440
1441inline Negative_match m_Negative() { return Negative_match(); }
1442template <typename Pattern> inline auto m_Negative(const Pattern &P) {
1443 return m_AllOf(m_Negative(), P);
1444}
1445inline NonNegative_match m_NonNegative() { return NonNegative_match(); }
1446template <typename Pattern> inline auto m_NonNegative(const Pattern &P) {
1447 return m_AllOf(m_NonNegative(), P);
1448}
1449inline StrictlyPositive_match m_StrictlyPositive() {
1450 return StrictlyPositive_match();
1451}
1452template <typename Pattern> inline auto m_StrictlyPositive(const Pattern &P) {
1453 return m_AllOf(m_StrictlyPositive(), P);
1454}
1455inline NonPositive_match m_NonPositive() { return NonPositive_match(); }
1456template <typename Pattern> inline auto m_NonPositive(const Pattern &P) {
1457 return m_AllOf(m_NonPositive(), P);
1458}
1459inline NonZero_match m_NonZero() { return NonZero_match(); }
1460template <typename Pattern> inline auto m_NonZero(const Pattern &P) {
1461 return m_AllOf(m_NonZero(), P);
1462}
1463inline Ones_match m_One(bool AllowUndefs = false) {
1464 return Ones_match(AllowUndefs);
1465}
1466inline Zero_match m_Zero(bool AllowUndefs = false) {
1467 return Zero_match(AllowUndefs);
1468}
1469inline AllOnes_match m_AllOnes(bool AllowUndefs = false) {
1470 return AllOnes_match(AllowUndefs);
1471}
1472
1473/// Match true boolean value based on the information provided by
1474/// TargetLowering.
1475inline auto m_True() {
1476 return TLI_pred_match{
1477 [](const TargetLowering &TLI, SDValue N) {
1478 APInt ConstVal;
1479 if (sd_match(N, P: m_ConstInt(V&: ConstVal)))
1480 switch (TLI.getBooleanContents(Type: N.getValueType())) {
1481 case TargetLowering::ZeroOrOneBooleanContent:
1482 return ConstVal.isOne();
1483 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1484 return ConstVal.isAllOnes();
1485 case TargetLowering::UndefinedBooleanContent:
1486 return (ConstVal & 0x01) == 1;
1487 }
1488
1489 return false;
1490 },
1491 m_Value()};
1492}
1493/// Match false boolean value based on the information provided by
1494/// TargetLowering.
1495inline auto m_False() {
1496 return TLI_pred_match{
1497 [](const TargetLowering &TLI, SDValue N) {
1498 APInt ConstVal;
1499 if (sd_match(N, P: m_ConstInt(V&: ConstVal)))
1500 switch (TLI.getBooleanContents(Type: N.getValueType())) {
1501 case TargetLowering::ZeroOrOneBooleanContent:
1502 case TargetLowering::ZeroOrNegativeOneBooleanContent:
1503 return ConstVal.isZero();
1504 case TargetLowering::UndefinedBooleanContent:
1505 return (ConstVal & 0x01) == 0;
1506 }
1507
1508 return false;
1509 },
1510 m_Value()};
1511}
1512
1513struct CondCode_match {
1514 std::optional<ISD::CondCode> CCToMatch;
1515 ISD::CondCode *BindCC = nullptr;
1516
1517 explicit CondCode_match(ISD::CondCode CC) : CCToMatch(CC) {}
1518
1519 explicit CondCode_match(ISD::CondCode *CC) : BindCC(CC) {}
1520
1521 template <typename MatchContext> bool match(const MatchContext &, SDValue N) {
1522 if (auto *CC = dyn_cast<CondCodeSDNode>(Val: N.getNode())) {
1523 if (CCToMatch && *CCToMatch != CC->get())
1524 return false;
1525
1526 if (BindCC)
1527 *BindCC = CC->get();
1528 return true;
1529 }
1530
1531 return false;
1532 }
1533};
1534
1535/// Match any conditional code SDNode.
1536inline CondCode_match m_CondCode() { return CondCode_match(nullptr); }
1537/// Match any conditional code SDNode and return its ISD::CondCode value.
1538inline CondCode_match m_CondCode(ISD::CondCode &CC) {
1539 return CondCode_match(&CC);
1540}
1541/// Match a conditional code SDNode with a specific ISD::CondCode.
1542inline CondCode_match m_SpecificCondCode(ISD::CondCode CC) {
1543 return CondCode_match(CC);
1544}
1545
1546/// Match a negate as a sub(0, v)
1547template <typename ValTy>
1548inline BinaryOpc_match<Zero_match, ValTy, false> m_Neg(const ValTy &V) {
1549 return m_Sub(m_Zero(), V);
1550}
1551
1552/// Match a Not as a xor(v, -1) or xor(-1, v)
1553template <typename ValTy>
1554inline BinaryOpc_match<ValTy, AllOnes_match, true> m_Not(const ValTy &V) {
1555 return m_Xor(V, m_AllOnes());
1556}
1557
1558template <unsigned IntrinsicId, typename... OpndPreds>
1559inline auto m_IntrinsicWOChain(const OpndPreds &...Opnds) {
1560 return m_Node(ISD::INTRINSIC_WO_CHAIN, m_SpecificInt(V: IntrinsicId), Opnds...);
1561}
1562
1563struct SpecificNeg_match {
1564 SDValue V;
1565
1566 explicit SpecificNeg_match(SDValue V) : V(V) {}
1567
1568 template <typename MatchContext>
1569 bool match(const MatchContext &Ctx, SDValue N) {
1570 if (sd_context_match(N, Ctx, m_Neg(V: m_Specific(N: V))))
1571 return true;
1572
1573 return ISD::matchBinaryPredicate(
1574 LHS: V, RHS: N, Match: [](ConstantSDNode *LHS, ConstantSDNode *RHS) {
1575 return LHS->getAPIntValue() == -RHS->getAPIntValue();
1576 });
1577 }
1578};
1579
1580/// Match a negation of a specific value V, either as sub(0, V) or as
1581/// constant(s) that are the negation of V's constant(s).
1582inline SpecificNeg_match m_SpecificNeg(SDValue V) {
1583 return SpecificNeg_match(V);
1584}
1585
1586template <typename... PatternTs> struct ReassociatableOpc_match {
1587 unsigned Opcode;
1588 std::tuple<PatternTs...> Patterns;
1589 constexpr static size_t NumPatterns =
1590 std::tuple_size_v<std::tuple<PatternTs...>>;
1591
1592 SDNodeFlags Flags;
1593
1594 ReassociatableOpc_match(unsigned Opcode, const PatternTs &...Patterns)
1595 : Opcode(Opcode), Patterns(Patterns...) {}
1596
1597 ReassociatableOpc_match(unsigned Opcode, SDNodeFlags Flags,
1598 const PatternTs &...Patterns)
1599 : Opcode(Opcode), Patterns(Patterns...), Flags(Flags) {}
1600
1601 template <typename MatchContext>
1602 bool match(const MatchContext &Ctx, SDValue N) {
1603 std::array<SDValue, NumPatterns> Leaves;
1604 size_t LeavesIdx = 0;
1605 if (!(collectLeaves(V: N, Leaves, LeafIdx&: LeavesIdx) && (LeavesIdx == NumPatterns)))
1606 return false;
1607
1608 Bitset<NumPatterns> Used;
1609 return std::apply(
1610 [&](auto &...P) -> bool {
1611 return reassociatableMatchHelper(Ctx, Leaves, Used, P...);
1612 },
1613 Patterns);
1614 }
1615
1616 bool collectLeaves(SDValue V, std::array<SDValue, NumPatterns> &Leaves,
1617 std::size_t &LeafIdx) {
1618 if (V->getOpcode() == Opcode && (Flags & V->getFlags()) == Flags) {
1619 for (size_t I = 0, N = V->getNumOperands(); I < N; I++)
1620 if ((LeafIdx == NumPatterns) ||
1621 !collectLeaves(V: V->getOperand(Num: I), Leaves, LeafIdx))
1622 return false;
1623 } else {
1624 Leaves[LeafIdx] = V;
1625 LeafIdx++;
1626 }
1627 return true;
1628 }
1629
1630 // Searchs for a matching leaf for every sub-pattern.
1631 template <typename MatchContext, typename PatternHd, typename... PatternTl>
1632 [[nodiscard]] inline bool
1633 reassociatableMatchHelper(const MatchContext &Ctx, ArrayRef<SDValue> Leaves,
1634 Bitset<NumPatterns> &Used, PatternHd &HeadPattern,
1635 PatternTl &...TailPatterns) {
1636 for (size_t Match = 0, N = Used.size(); Match < N; Match++) {
1637 if (Used[Match] || !(sd_context_match(Leaves[Match], Ctx, HeadPattern)))
1638 continue;
1639 Used.set(Match);
1640 if (reassociatableMatchHelper(Ctx, Leaves, Used, TailPatterns...))
1641 return true;
1642 Used.reset(Match);
1643 }
1644 return false;
1645 }
1646
1647 template <typename MatchContext>
1648 [[nodiscard]] inline bool
1649 reassociatableMatchHelper(const MatchContext &Ctx, ArrayRef<SDValue> Leaves,
1650 Bitset<NumPatterns> &Used) {
1651 return true;
1652 }
1653};
1654
1655template <typename... PatternTs>
1656inline ReassociatableOpc_match<PatternTs...>
1657m_ReassociatableAdd(const PatternTs &...Patterns) {
1658 return ReassociatableOpc_match<PatternTs...>(ISD::ADD, Patterns...);
1659}
1660
1661template <typename... PatternTs>
1662inline ReassociatableOpc_match<PatternTs...>
1663m_ReassociatableOr(const PatternTs &...Patterns) {
1664 return ReassociatableOpc_match<PatternTs...>(ISD::OR, Patterns...);
1665}
1666
1667template <typename... PatternTs>
1668inline ReassociatableOpc_match<PatternTs...>
1669m_ReassociatableAnd(const PatternTs &...Patterns) {
1670 return ReassociatableOpc_match<PatternTs...>(ISD::AND, Patterns...);
1671}
1672
1673template <typename... PatternTs>
1674inline ReassociatableOpc_match<PatternTs...>
1675m_ReassociatableMul(const PatternTs &...Patterns) {
1676 return ReassociatableOpc_match<PatternTs...>(ISD::MUL, Patterns...);
1677}
1678
1679template <typename... PatternTs>
1680inline ReassociatableOpc_match<PatternTs...>
1681m_ReassociatableNSWAdd(const PatternTs &...Patterns) {
1682 return ReassociatableOpc_match<PatternTs...>(
1683 ISD::ADD, SDNodeFlags::NoSignedWrap, Patterns...);
1684}
1685
1686template <typename... PatternTs>
1687inline ReassociatableOpc_match<PatternTs...>
1688m_ReassociatableNUWAdd(const PatternTs &...Patterns) {
1689 return ReassociatableOpc_match<PatternTs...>(
1690 ISD::ADD, SDNodeFlags::NoUnsignedWrap, Patterns...);
1691}
1692
1693} // namespace SDPatternMatch
1694} // namespace llvm
1695#endif
1696