1//===- DAGISelMatcherGen.cpp - Matcher generator --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Basic/SDNodeProperties.h"
10#include "Common/CodeGenDAGPatterns.h"
11#include "Common/CodeGenInstruction.h"
12#include "Common/CodeGenRegisters.h"
13#include "Common/CodeGenTarget.h"
14#include "Common/InfoByHwMode.h"
15#include "DAGISelMatcher.h"
16#include "llvm/ADT/SmallVector.h"
17#include "llvm/ADT/StringMap.h"
18#include "llvm/TableGen/Error.h"
19#include "llvm/TableGen/Record.h"
20#include <utility>
21using namespace llvm;
22
23/// getRegisterValueType - Look up and return the ValueType of the specified
24/// register. If the register is a member of multiple register classes, they
25/// must all have the same type.
26static MVT getRegisterValueType(const Record *R, const CodeGenTarget &T) {
27 bool FoundRC = false;
28 MVT VT = MVT::Other;
29 const CodeGenRegister *Reg = T.getRegBank().getReg(R);
30
31 for (const auto &RC : T.getRegBank().getRegClasses()) {
32 if (!RC.contains(Reg))
33 continue;
34
35 if (!FoundRC) {
36 FoundRC = true;
37 const ValueTypeByHwMode &VVT = RC.getValueTypeNum(VTNum: 0);
38 assert(VVT.isSimple());
39 VT = VVT.getSimple();
40 continue;
41 }
42
43#ifndef NDEBUG
44 // If this occurs in multiple register classes, they all have to agree.
45 const ValueTypeByHwMode &VVT = RC.getValueTypeNum(0);
46 assert(VVT.isSimple() && VVT.getSimple() == VT &&
47 "ValueType mismatch between register classes for this register");
48#endif
49 }
50 return VT;
51}
52
53namespace {
54class MatcherGen {
55 const PatternToMatch &Pattern;
56 const CodeGenDAGPatterns &CGP;
57
58 /// PatWithNoTypes - This is a clone of Pattern.getSrcPattern() that starts
59 /// out with all of the types removed. This allows us to insert type checks
60 /// as we scan the tree.
61 TreePatternNodePtr PatWithNoTypes;
62
63 /// VariableMap - A map from variable names ('$dst') to the recorded operand
64 /// number that they were captured as. These are biased by 1 to make
65 /// insertion easier.
66 StringMap<unsigned> VariableMap;
67
68 /// This maintains the recorded operand number that OPC_CheckComplexPattern
69 /// drops each sub-operand into. We don't want to insert these into
70 /// VariableMap because that leads to identity checking if they are
71 /// encountered multiple times. Biased by 1 like VariableMap for
72 /// consistency.
73 StringMap<unsigned> NamedComplexPatternOperands;
74
75 /// NextRecordedOperandNo - As we emit opcodes to record matched values in
76 /// the RecordedNodes array, this keeps track of which slot will be next to
77 /// record into.
78 unsigned NextRecordedOperandNo = 0;
79
80 /// MatchedChainNodes - This maintains the position in the recorded nodes
81 /// array of all of the recorded input nodes that have chains.
82 SmallVector<unsigned, 2> MatchedChainNodes;
83
84 /// MatchedComplexPatterns - This maintains a list of all of the
85 /// ComplexPatterns that we need to check. The second element of each pair
86 /// is the recorded operand number of the input node.
87 SmallVector<std::pair<const TreePatternNode *, unsigned>, 2>
88 MatchedComplexPatterns;
89
90 /// PhysRegInputs - List list has an entry for each explicitly specified
91 /// physreg input to the pattern. The first elt is the Register node, the
92 /// second is the recorded slot number the input pattern match saved it in.
93 SmallVector<std::pair<const Record *, unsigned>, 2> PhysRegInputs;
94
95 /// This is the top level of the generated matcher, the result.
96 MatcherList TheMatcherList;
97
98 /// As we emit matcher nodes, this points to the latest check which should
99 /// have future checks inserted after it.
100 MatcherList::iterator InsertPt;
101
102public:
103 MatcherGen(const PatternToMatch &pattern, const CodeGenDAGPatterns &cgp);
104
105 bool EmitMatcherCode(unsigned Variant);
106 void EmitResultCode();
107
108 MatcherList GetMatcherList() { return std::move(TheMatcherList); }
109
110private:
111 void AddMatcher(Matcher *NewNode);
112 void InferPossibleTypes();
113
114 // Matcher Generation.
115 void EmitMatchCode(const TreePatternNode &N, TreePatternNode &NodeNoTypes);
116 void EmitLeafMatchCode(const TreePatternNode &N);
117 void EmitOperatorMatchCode(const TreePatternNode &N,
118 TreePatternNode &NodeNoTypes);
119
120 /// If this is the first time a node with unique identifier Name has been
121 /// seen, record it. Otherwise, emit a check to make sure this is the same
122 /// node. Returns true if this is the first encounter.
123 bool recordUniqueNode(ArrayRef<std::string> Names);
124
125 // Result Code Generation.
126 unsigned getNamedArgumentSlot(StringRef Name) {
127 unsigned VarMapEntry = VariableMap[Name];
128 assert(VarMapEntry != 0 &&
129 "Variable referenced but not defined and not caught earlier!");
130 return VarMapEntry - 1;
131 }
132
133 void EmitResultOperand(const TreePatternNode &N,
134 SmallVectorImpl<unsigned> &ResultOps);
135 void EmitResultOfNamedOperand(const TreePatternNode &N,
136 SmallVectorImpl<unsigned> &ResultOps);
137 void EmitResultLeafAsOperand(const TreePatternNode &N,
138 SmallVectorImpl<unsigned> &ResultOps);
139 void EmitResultInstructionAsOperand(const TreePatternNode &N,
140 SmallVectorImpl<unsigned> &ResultOps);
141 void EmitResultSDNodeXFormAsOperand(const TreePatternNode &N,
142 SmallVectorImpl<unsigned> &ResultOps);
143};
144
145} // end anonymous namespace
146
147MatcherGen::MatcherGen(const PatternToMatch &pattern,
148 const CodeGenDAGPatterns &cgp)
149 : Pattern(pattern), CGP(cgp), InsertPt(TheMatcherList.before_begin()) {
150 // We need to produce the matcher tree for the patterns source pattern. To
151 // do this we need to match the structure as well as the types. To do the
152 // type matching, we want to figure out the fewest number of type checks we
153 // need to emit. For example, if there is only one integer type supported
154 // by a target, there should be no type comparisons at all for integer
155 // patterns!
156 //
157 // To figure out the fewest number of type checks needed, clone the pattern,
158 // remove the types, then perform type inference on the pattern as a whole.
159 // If there are unresolved types, emit an explicit check for those types,
160 // apply the type to the tree, then rerun type inference. Iterate until all
161 // types are resolved.
162 //
163 PatWithNoTypes = Pattern.getSrcPattern().clone();
164 PatWithNoTypes->RemoveAllTypes();
165
166 // If there are types that are manifestly known, infer them.
167 InferPossibleTypes();
168}
169
170/// InferPossibleTypes - As we emit the pattern, we end up generating type
171/// checks and applying them to the 'PatWithNoTypes' tree. As we do this, we
172/// want to propagate implied types as far throughout the tree as possible so
173/// that we avoid doing redundant type checks. This does the type propagation.
174void MatcherGen::InferPossibleTypes() {
175 // TP - Get *SOME* tree pattern, we don't care which. It is only used for
176 // diagnostics, which we know are impossible at this point.
177 TreePattern &TP = *CGP.pf_begin()->second;
178
179 bool MadeChange = true;
180 while (MadeChange)
181 MadeChange = PatWithNoTypes->ApplyTypeConstraints(
182 TP, NotRegisters: true /*Ignore reg constraints*/);
183}
184
185/// AddMatcher - Add a matcher node to the current graph we're building.
186void MatcherGen::AddMatcher(Matcher *NewNode) {
187 InsertPt = TheMatcherList.insert_after(Pos: InsertPt, N: NewNode);
188}
189
190//===----------------------------------------------------------------------===//
191// Pattern Match Generation
192//===----------------------------------------------------------------------===//
193
194/// EmitLeafMatchCode - Generate matching code for leaf nodes.
195void MatcherGen::EmitLeafMatchCode(const TreePatternNode &N) {
196 assert(N.isLeaf() && "Not a leaf?");
197
198 // Direct match against an integer constant.
199 if (const IntInit *II = dyn_cast<IntInit>(Val: N.getLeafValue())) {
200 // If this is the root of the dag we're matching, we emit a redundant opcode
201 // check to ensure that this gets folded into the normal top-level
202 // OpcodeSwitch.
203 if (&N == &Pattern.getSrcPattern()) {
204 const SDNodeInfo &NI = CGP.getSDNodeInfo(R: CGP.getSDNodeNamed(Name: "imm"));
205 AddMatcher(NewNode: new CheckOpcodeMatcher(NI));
206 }
207
208 return AddMatcher(NewNode: new CheckIntegerMatcher(II->getValue()));
209 }
210
211 // An UnsetInit represents a named node without any constraints.
212 if (isa<UnsetInit>(Val: N.getLeafValue())) {
213 assert(N.hasName() && "Unnamed ? leaf");
214 return;
215 }
216
217 const DefInit *DI = dyn_cast<DefInit>(Val: N.getLeafValue());
218 if (!DI) {
219 errs() << "Unknown leaf kind: " << N << "\n";
220 abort();
221 }
222
223 const Record *LeafRec = DI->getDef();
224
225 // A ValueType leaf node can represent a register when named, or itself when
226 // unnamed.
227 if (LeafRec->isSubClassOf(Name: "ValueType")) {
228 // A named ValueType leaf always matches: (add i32:$a, i32:$b).
229 if (N.hasName())
230 return;
231 // An unnamed ValueType as in (sext_inreg GPR:$foo, i8).
232 return AddMatcher(NewNode: new CheckValueTypeMatcher(llvm::getValueType(Rec: LeafRec)));
233 }
234
235 if ( // Handle register references. Nothing to do here, they always match.
236 LeafRec->isSubClassOf(Name: "RegisterClassLike") ||
237 LeafRec->isSubClassOf(Name: "RegisterOperand") ||
238 LeafRec->isSubClassOf(Name: "SubRegIndex") ||
239 // Place holder for SRCVALUE nodes. Nothing to do here.
240 LeafRec->getName() == "srcvalue")
241 return;
242
243 // If we have a physreg reference like (mul gpr:$src, EAX) then we need to
244 // record the register
245 if (LeafRec->isSubClassOf(Name: "Register")) {
246 AddMatcher(NewNode: new RecordMatcher("physreg input " + LeafRec->getName().str(),
247 NextRecordedOperandNo));
248 PhysRegInputs.emplace_back(Args&: LeafRec, Args: NextRecordedOperandNo++);
249 return;
250 }
251
252 if (LeafRec->isSubClassOf(Name: "CondCode"))
253 return AddMatcher(NewNode: new CheckCondCodeMatcher(LeafRec->getName()));
254
255 if (LeafRec->isSubClassOf(Name: "ComplexPattern")) {
256 // We can't model ComplexPattern uses that don't have their name taken yet.
257 // The OPC_CheckComplexPattern operation implicitly records the results.
258 if (N.getName().empty()) {
259 std::string S;
260 raw_string_ostream OS(S);
261 OS << "We expect complex pattern uses to have names: " << N;
262 PrintFatalError(Msg: S);
263 }
264
265 // Remember this ComplexPattern so that we can emit it after all the other
266 // structural matches are done.
267 unsigned InputOperand = VariableMap[N.getName()] - 1;
268 MatchedComplexPatterns.emplace_back(Args: &N, Args&: InputOperand);
269 return;
270 }
271
272 if (LeafRec->getName() == "immAllOnesV" ||
273 LeafRec->getName() == "immAllZerosV") {
274 // If this is the root of the dag we're matching, we emit a redundant opcode
275 // check to ensure that this gets folded into the normal top-level
276 // OpcodeSwitch.
277 if (&N == &Pattern.getSrcPattern()) {
278 MVT VT = N.getSimpleType(ResNo: 0);
279 StringRef Name = VT.isScalableVector() ? "splat_vector" : "build_vector";
280 const SDNodeInfo &NI = CGP.getSDNodeInfo(R: CGP.getSDNodeNamed(Name));
281 AddMatcher(NewNode: new CheckOpcodeMatcher(NI));
282 }
283 if (LeafRec->getName() == "immAllOnesV")
284 AddMatcher(NewNode: new CheckImmAllOnesVMatcher());
285 else
286 AddMatcher(NewNode: new CheckImmAllZerosVMatcher());
287 return;
288 }
289
290 errs() << "Unknown leaf kind: " << N << "\n";
291 abort();
292}
293
294void MatcherGen::EmitOperatorMatchCode(const TreePatternNode &N,
295 TreePatternNode &NodeNoTypes) {
296 assert(!N.isLeaf() && "Not an operator?");
297
298 if (N.getOperator()->isSubClassOf(Name: "ComplexPattern")) {
299 // The "name" of a non-leaf complex pattern (MY_PAT $op1, $op2) is
300 // "MY_PAT:op1:op2". We should already have validated that the uses are
301 // consistent.
302 std::string PatternName = N.getOperator()->getName().str();
303 for (const TreePatternNode &Child : N.children()) {
304 PatternName += ":";
305 PatternName += Child.getName();
306 }
307
308 if (recordUniqueNode(Names: PatternName)) {
309 auto NodeAndOpNum = std::pair(&N, NextRecordedOperandNo - 1);
310 MatchedComplexPatterns.push_back(Elt: NodeAndOpNum);
311 }
312
313 return;
314 }
315
316 const SDNodeInfo &CInfo = CGP.getSDNodeInfo(R: N.getOperator());
317
318 // If this is an 'and R, 1234' where the operation is AND/OR and the RHS is
319 // a constant without a predicate fn that has more than one bit set, handle
320 // this as a special case. This is usually for targets that have special
321 // handling of certain large constants (e.g. alpha with it's 8/16/32-bit
322 // handling stuff). Using these instructions is often far more efficient
323 // than materializing the constant. Unfortunately, both the instcombiner
324 // and the dag combiner can often infer that bits are dead, and thus drop
325 // them from the mask in the dag. For example, it might turn 'AND X, 255'
326 // into 'AND X, 254' if it knows the low bit is set. Emit code that checks
327 // to handle this.
328 if ((N.getOperator()->getName() == "and" ||
329 N.getOperator()->getName() == "or") &&
330 N.getChild(N: 1).isLeaf() && N.getChild(N: 1).getPredicateCalls().empty() &&
331 N.getPredicateCalls().empty()) {
332 if (const IntInit *II = dyn_cast<IntInit>(Val: N.getChild(N: 1).getLeafValue())) {
333 if (!llvm::has_single_bit<uint32_t>(
334 Value: II->getValue())) { // Don't bother with single bits.
335 // If this is at the root of the pattern, we emit a redundant
336 // CheckOpcode so that the following checks get factored properly under
337 // a single opcode check.
338 if (&N == &Pattern.getSrcPattern())
339 AddMatcher(NewNode: new CheckOpcodeMatcher(CInfo));
340
341 // Emit the CheckAndImm/CheckOrImm node.
342 if (N.getOperator()->getName() == "and")
343 AddMatcher(NewNode: new CheckAndImmMatcher(II->getValue()));
344 else
345 AddMatcher(NewNode: new CheckOrImmMatcher(II->getValue()));
346
347 // Match the LHS of the AND as appropriate.
348 AddMatcher(NewNode: new MoveChildMatcher(0));
349 EmitMatchCode(N: N.getChild(N: 0), NodeNoTypes&: NodeNoTypes.getChild(N: 0));
350 AddMatcher(NewNode: new MoveParentMatcher());
351 return;
352 }
353 }
354 }
355
356 // Check that the current opcode lines up.
357 AddMatcher(NewNode: new CheckOpcodeMatcher(CInfo));
358
359 // If this node has memory references (i.e. is a load or store), tell the
360 // interpreter to capture them in the memref array.
361 if (N.NodeHasProperty(Property: SDNPMemOperand, CGP))
362 AddMatcher(NewNode: new RecordMemRefMatcher());
363
364 // If this node has a chain, then the chain is operand #0 is the SDNode, and
365 // the child numbers of the node are all offset by one.
366 unsigned OpNo = 0;
367 if (N.NodeHasProperty(Property: SDNPHasChain, CGP)) {
368 // Record the node and remember it in our chained nodes list.
369 AddMatcher(NewNode: new RecordMatcher("'" + N.getOperator()->getName().str() +
370 "' chained node",
371 NextRecordedOperandNo));
372 // Remember all of the input chains our pattern will match.
373 MatchedChainNodes.push_back(Elt: NextRecordedOperandNo++);
374
375 // Don't look at the input chain when matching the tree pattern to the
376 // SDNode.
377 OpNo = 1;
378
379 // If this node is not the root and the subtree underneath it produces a
380 // chain, then the result of matching the node is also produce a chain.
381 // Beyond that, this means that we're also folding (at least) the root node
382 // into the node that produce the chain (for example, matching
383 // "(add reg, (load ptr))" as a add_with_memory on X86). This is
384 // problematic, if the 'reg' node also uses the load (say, its chain).
385 // Graphically:
386 //
387 // [LD]
388 // ^ ^
389 // | \ DAG's like cheese.
390 // / |
391 // / [YY]
392 // | ^
393 // [XX]--/
394 //
395 // It would be invalid to fold XX and LD. In this case, folding the two
396 // nodes together would induce a cycle in the DAG, making it a 'cyclic DAG'
397 // To prevent this, we emit a dynamic check for legality before allowing
398 // this to be folded.
399 //
400 const TreePatternNode &Root = Pattern.getSrcPattern();
401 if (&N != &Root) { // Not the root of the pattern.
402 // If there is a node between the root and this node, then we definitely
403 // need to emit the check.
404 bool NeedCheck = !Root.hasChild(N: &N);
405
406 // If it *is* an immediate child of the root, we can still need a check if
407 // the root SDNode has multiple inputs. For us, this means that it is an
408 // intrinsic, has multiple operands, or has other inputs like chain or
409 // glue).
410 if (!NeedCheck) {
411 const SDNodeInfo &PInfo = CGP.getSDNodeInfo(R: Root.getOperator());
412 NeedCheck =
413 Root.getOperator() == CGP.get_intrinsic_void_sdnode() ||
414 Root.getOperator() == CGP.get_intrinsic_w_chain_sdnode() ||
415 Root.getOperator() == CGP.get_intrinsic_wo_chain_sdnode() ||
416 PInfo.getNumOperands() > 1 || PInfo.hasProperty(Prop: SDNPHasChain) ||
417 PInfo.hasProperty(Prop: SDNPInGlue) || PInfo.hasProperty(Prop: SDNPOptInGlue);
418 }
419
420 if (NeedCheck)
421 AddMatcher(NewNode: new CheckFoldableChainNodeMatcher());
422 }
423 }
424
425 // If this node has an output glue and isn't the root, remember it.
426 if (N.NodeHasProperty(Property: SDNPOutGlue, CGP) && &N != &Pattern.getSrcPattern()) {
427 // TODO: This redundantly records nodes with both glues and chains.
428
429 // Record the node and remember it in our chained nodes list.
430 AddMatcher(NewNode: new RecordMatcher("'" + N.getOperator()->getName().str() +
431 "' glue output node",
432 NextRecordedOperandNo));
433 }
434
435 // If this node is known to have an input glue or if it *might* have an input
436 // glue, capture it as the glue input of the pattern.
437 if (N.NodeHasProperty(Property: SDNPOptInGlue, CGP) ||
438 N.NodeHasProperty(Property: SDNPInGlue, CGP))
439 AddMatcher(NewNode: new CaptureGlueInputMatcher());
440
441 for (unsigned i = 0, e = N.getNumChildren(); i != e; ++i, ++OpNo) {
442 // Get the code suitable for matching this child. Move to the child, check
443 // it then move back to the parent.
444 AddMatcher(NewNode: new MoveChildMatcher(OpNo));
445 EmitMatchCode(N: N.getChild(N: i), NodeNoTypes&: NodeNoTypes.getChild(N: i));
446 AddMatcher(NewNode: new MoveParentMatcher());
447 }
448}
449
450bool MatcherGen::recordUniqueNode(ArrayRef<std::string> Names) {
451 unsigned Entry = 0;
452 for (const std::string &Name : Names) {
453 unsigned &VarMapEntry = VariableMap[Name];
454 if (!Entry)
455 Entry = VarMapEntry;
456 assert(Entry == VarMapEntry);
457 }
458
459 bool NewRecord = false;
460 if (Entry == 0) {
461 // If it is a named node, we must emit a 'Record' opcode.
462 std::string WhatFor;
463 for (const std::string &Name : Names) {
464 if (!WhatFor.empty())
465 WhatFor += ',';
466 WhatFor += "$" + Name;
467 }
468 AddMatcher(NewNode: new RecordMatcher(WhatFor, NextRecordedOperandNo));
469 Entry = ++NextRecordedOperandNo;
470 NewRecord = true;
471 } else {
472 // If we get here, this is a second reference to a specific name. Since
473 // we already have checked that the first reference is valid, we don't
474 // have to recursively match it, just check that it's the same as the
475 // previously named thing.
476 AddMatcher(NewNode: new CheckSameMatcher(Entry - 1));
477 }
478
479 for (const std::string &Name : Names)
480 VariableMap[Name] = Entry;
481
482 return NewRecord;
483}
484
485void MatcherGen::EmitMatchCode(const TreePatternNode &N,
486 TreePatternNode &NodeNoTypes) {
487 // If N and NodeNoTypes don't agree on a type, then this is a case where we
488 // need to do a type check. Emit the check, apply the type to NodeNoTypes and
489 // reinfer any correlated types.
490 SmallVector<unsigned, 2> ResultsToTypeCheck;
491
492 for (unsigned i = 0, e = NodeNoTypes.getNumTypes(); i != e; ++i) {
493 if (NodeNoTypes.getExtType(ResNo: i) == N.getExtType(ResNo: i))
494 continue;
495 NodeNoTypes.setType(ResNo: i, T: N.getExtType(ResNo: i));
496 InferPossibleTypes();
497 ResultsToTypeCheck.push_back(Elt: i);
498 }
499
500 // If this node has a name associated with it, capture it in VariableMap. If
501 // we already saw this in the pattern, emit code to verify dagness.
502 SmallVector<std::string, 4> Names;
503 if (!N.getName().empty())
504 Names.push_back(Elt: N.getName().str());
505
506 for (const ScopedName &Name : N.getNamesAsPredicateArg()) {
507 Names.push_back(
508 Elt: ("pred:" + Twine(Name.getScope()) + ":" + Name.getIdentifier()).str());
509 }
510
511 if (!Names.empty()) {
512 if (!recordUniqueNode(Names))
513 return;
514 }
515
516 if (N.isLeaf())
517 EmitLeafMatchCode(N);
518 else
519 EmitOperatorMatchCode(N, NodeNoTypes);
520
521 // If there are node predicates for this node, generate their checks.
522 for (const TreePredicateCall &Pred : N.getPredicateCalls()) {
523 SmallVector<unsigned, 4> Operands;
524 if (Pred.Fn.usesOperands()) {
525 TreePattern *TP = Pred.Fn.getOrigPatFragRecord();
526 for (const std::string &Arg : TP->getArgList()) {
527 std::string Name = ("pred:" + Twine(Pred.Scope) + ":" + Arg).str();
528 Operands.push_back(Elt: getNamedArgumentSlot(Name));
529 }
530 }
531 AddMatcher(NewNode: new CheckPredicateMatcher(Pred.Fn, Operands));
532 }
533
534 for (unsigned I : ResultsToTypeCheck)
535 AddMatcher(NewNode: new CheckTypeMatcher(N.getType(ResNo: I), I));
536}
537
538/// EmitMatcherCode - Generate the code that matches the predicate of this
539/// pattern for the specified Variant. If the variant is invalid this returns
540/// true and does not generate code, if it is valid, it returns false.
541bool MatcherGen::EmitMatcherCode(unsigned Variant) {
542 // If the root of the pattern is a ComplexPattern and if it is specified to
543 // match some number of root opcodes, these are considered to be our variants.
544 // Depending on which variant we're generating code for, emit the root opcode
545 // check.
546 if (const ComplexPattern *CP =
547 Pattern.getSrcPattern().getComplexPatternInfo(CGP)) {
548 ArrayRef<const Record *> OpNodes = CP->getRootNodes();
549 assert(!OpNodes.empty() &&
550 "Complex Pattern must specify what it can match");
551 if (Variant >= OpNodes.size())
552 return true;
553
554 AddMatcher(NewNode: new CheckOpcodeMatcher(CGP.getSDNodeInfo(R: OpNodes[Variant])));
555 } else {
556 if (Variant != 0)
557 return true;
558 }
559
560 // Emit the matcher for the pattern structure and types.
561 EmitMatchCode(N: Pattern.getSrcPattern(), NodeNoTypes&: *PatWithNoTypes);
562
563 // If the pattern has a predicate on it (e.g. only enabled when a subtarget
564 // feature is around, do the check).
565 std::string PredicateCheck = Pattern.getPredicateCheck();
566 if (!PredicateCheck.empty())
567 AddMatcher(NewNode: new CheckPatternPredicateMatcher(PredicateCheck));
568
569 // Now that we've completed the structural type match, emit any ComplexPattern
570 // checks (e.g. addrmode matches). We emit this after the structural match
571 // because they are generally more expensive to evaluate and more difficult to
572 // factor.
573 for (const auto &MCP : MatchedComplexPatterns) {
574 auto &N = *MCP.first;
575
576 // Remember where the results of this match get stuck.
577 if (N.isLeaf()) {
578 NamedComplexPatternOperands[N.getName()] = NextRecordedOperandNo + 1;
579 } else {
580 unsigned CurOp = NextRecordedOperandNo;
581 for (const TreePatternNode &Child : N.children()) {
582 NamedComplexPatternOperands[Child.getName()] = CurOp + 1;
583 CurOp += Child.getNumMIResults(CGP);
584 }
585 }
586
587 // Get the slot we recorded the value in from the name on the node.
588 unsigned RecNodeEntry = MCP.second;
589
590 const ComplexPattern *CP = N.getComplexPatternInfo(CGP);
591 assert(CP && "Not a valid ComplexPattern!");
592
593 // Emit a CheckComplexPat operation, which does the match (aborting if it
594 // fails) and pushes the matched operands onto the recorded nodes list.
595 AddMatcher(NewNode: new CheckComplexPatMatcher(*CP, RecNodeEntry, N.getName(),
596 NextRecordedOperandNo));
597
598 // Record the right number of operands.
599 NextRecordedOperandNo += CP->getNumOperands();
600 if (CP->hasProperty(Prop: SDNPHasChain)) {
601 // If the complex pattern has a chain, then we need to keep track of the
602 // fact that we just recorded a chain input. The chain input will be
603 // matched as the last operand of the predicate if it was successful.
604 ++NextRecordedOperandNo; // Chained node operand.
605
606 // It is the last operand recorded.
607 assert(NextRecordedOperandNo > 1 &&
608 "Should have recorded input/result chains at least!");
609 MatchedChainNodes.push_back(Elt: NextRecordedOperandNo - 1);
610 }
611
612 // TODO: Complex patterns can't have output glues, if they did, we'd want
613 // to record them.
614 }
615
616 return false;
617}
618
619//===----------------------------------------------------------------------===//
620// Node Result Generation
621//===----------------------------------------------------------------------===//
622
623void MatcherGen::EmitResultOfNamedOperand(
624 const TreePatternNode &N, SmallVectorImpl<unsigned> &ResultOps) {
625 assert(!N.getName().empty() && "Operand not named!");
626
627 if (unsigned SlotNo = NamedComplexPatternOperands[N.getName()]) {
628 // Complex operands have already been completely selected, just find the
629 // right slot ant add the arguments directly.
630 for (unsigned i = 0; i < N.getNumMIResults(CGP); ++i)
631 ResultOps.push_back(Elt: SlotNo - 1 + i);
632
633 return;
634 }
635
636 unsigned SlotNo = getNamedArgumentSlot(Name: N.getName());
637
638 // If this is an 'imm' or 'fpimm' node, make sure to convert it to the target
639 // version of the immediate so that it doesn't get selected due to some other
640 // node use.
641 if (!N.isLeaf()) {
642 StringRef OperatorName = N.getOperator()->getName();
643 if (OperatorName == "imm" || OperatorName == "fpimm") {
644 AddMatcher(NewNode: new EmitConvertToTargetMatcher(SlotNo, NextRecordedOperandNo));
645 ResultOps.push_back(Elt: NextRecordedOperandNo++);
646 return;
647 }
648 }
649
650 for (unsigned i = 0; i < N.getNumMIResults(CGP); ++i)
651 ResultOps.push_back(Elt: SlotNo + i);
652}
653
654void MatcherGen::EmitResultLeafAsOperand(const TreePatternNode &N,
655 SmallVectorImpl<unsigned> &ResultOps) {
656 assert(N.isLeaf() && "Must be a leaf");
657
658 if (const IntInit *II = dyn_cast<IntInit>(Val: N.getLeafValue())) {
659 AddMatcher(NewNode: new EmitIntegerMatcher(II->getValue(), N.getType(ResNo: 0),
660 NextRecordedOperandNo));
661 ResultOps.push_back(Elt: NextRecordedOperandNo++);
662 return;
663 }
664
665 // If this is an explicit register reference, handle it.
666 if (const DefInit *DI = dyn_cast<DefInit>(Val: N.getLeafValue())) {
667 const Record *Def = DI->getDef();
668 if (Def->isSubClassOf(Name: "Register")) {
669 const CodeGenRegister *Reg = CGP.getTargetInfo().getRegBank().getReg(Def);
670 AddMatcher(
671 NewNode: new EmitRegisterMatcher(Reg, N.getType(ResNo: 0), NextRecordedOperandNo));
672 ResultOps.push_back(Elt: NextRecordedOperandNo++);
673 return;
674 } else if (Def->isSubClassOf(Name: "RegisterByHwMode")) {
675 PrintFatalError(ErrorLoc: Def->getLoc() /* TODO: N.getLoc() */,
676 Msg: "RegisterByHwMode in SelectionDAG patterns "
677 "not yet supported!");
678 }
679
680 if (Def->getName() == "zero_reg") {
681 AddMatcher(NewNode: new EmitRegisterMatcher(nullptr, N.getType(ResNo: 0),
682 NextRecordedOperandNo));
683 ResultOps.push_back(Elt: NextRecordedOperandNo++);
684 return;
685 }
686
687 if (Def->getName() == "undef_tied_input") {
688 ValueTypeByHwMode ResultVT = N.getType(ResNo: 0);
689 auto IDOperandNo = NextRecordedOperandNo++;
690 const Record *ImpDef = Def->getRecords().getDef(Name: "IMPLICIT_DEF");
691 const CodeGenInstruction &II = CGP.getTargetInfo().getInstruction(InstRec: ImpDef);
692 AddMatcher(NewNode: new EmitNodeMatcher(II, ResultVT, {}, false, false, false,
693 false, -1, IDOperandNo));
694 ResultOps.push_back(Elt: IDOperandNo);
695 return;
696 }
697
698 // Handle a reference to a register class. This is used
699 // in COPY_TO_SUBREG instructions.
700 if (Def->isSubClassOf(Name: "RegisterOperand"))
701 Def = Def->getValueAsDef(FieldName: "RegClass");
702 if (Def->isSubClassOf(Name: "RegisterClass")) {
703 // If the register class has an enum integer value greater than 127, the
704 // encoding overflows the limit of 7 bits, which precludes the use of
705 // StringIntegerMatcher. In this case, fallback to using IntegerMatcher.
706 const CodeGenRegisterClass &RC =
707 CGP.getTargetInfo().getRegisterClass(R: Def);
708 std::string Name = RC.getQualifiedIdName();
709 AddMatcher(NewNode: new EmitIntegerMatcher(Name, RC.EnumValue, MVT::i32,
710 NextRecordedOperandNo));
711 ResultOps.push_back(Elt: NextRecordedOperandNo++);
712 return;
713 }
714
715 // Handle a subregister index. This is used for INSERT_SUBREG etc.
716 if (Def->isSubClassOf(Name: "SubRegIndex")) {
717 const CodeGenRegBank &RB = CGP.getTargetInfo().getRegBank();
718 const CodeGenSubRegIndex *I = RB.findSubRegIdx(Def);
719 std::string Name = getQualifiedName(R: Def);
720 AddMatcher(NewNode: new EmitIntegerMatcher(Name, I->EnumValue, MVT::i32,
721 NextRecordedOperandNo));
722 ResultOps.push_back(Elt: NextRecordedOperandNo++);
723 return;
724 }
725 }
726
727 errs() << "unhandled leaf node:\n";
728 N.dump();
729}
730
731static bool mayInstNodeLoadOrStore(const TreePatternNode &N,
732 const CodeGenDAGPatterns &CGP) {
733 const Record *Op = N.getOperator();
734 const CodeGenTarget &CGT = CGP.getTargetInfo();
735 const CodeGenInstruction &II = CGT.getInstruction(InstRec: Op);
736 return II.mayLoad || II.mayStore;
737}
738
739static unsigned numNodesThatMayLoadOrStore(const TreePatternNode &N,
740 const CodeGenDAGPatterns &CGP) {
741 if (N.isLeaf())
742 return 0;
743
744 const Record *OpRec = N.getOperator();
745 if (!OpRec->isSubClassOf(Name: "Instruction"))
746 return 0;
747
748 unsigned Count = 0;
749 if (mayInstNodeLoadOrStore(N, CGP))
750 ++Count;
751
752 for (const TreePatternNode &Child : N.children())
753 Count += numNodesThatMayLoadOrStore(N: Child, CGP);
754
755 return Count;
756}
757
758void MatcherGen::EmitResultInstructionAsOperand(
759 const TreePatternNode &N, SmallVectorImpl<unsigned> &OutputOps) {
760 const Record *Op = N.getOperator();
761 const CodeGenTarget &CGT = CGP.getTargetInfo();
762 const CodeGenInstruction &II = CGT.getInstruction(InstRec: Op);
763 const DAGInstruction &Inst = CGP.getInstruction(R: Op);
764
765 bool isRoot = &N == &Pattern.getDstPattern();
766
767 // TreeHasOutGlue - True if this tree has glue.
768 bool TreeHasInGlue = false, TreeHasOutGlue = false;
769 if (isRoot) {
770 const TreePatternNode &SrcPat = Pattern.getSrcPattern();
771 TreeHasInGlue = SrcPat.TreeHasProperty(Property: SDNPOptInGlue, CGP) ||
772 SrcPat.TreeHasProperty(Property: SDNPInGlue, CGP);
773
774 // FIXME2: this is checking the entire pattern, not just the node in
775 // question, doing this just for the root seems like a total hack.
776 TreeHasOutGlue = SrcPat.TreeHasProperty(Property: SDNPOutGlue, CGP);
777 }
778
779 // NumResults - This is the number of results produced by the instruction in
780 // the "outs" list.
781 unsigned NumResults = Inst.getNumResults();
782
783 // Number of operands we know the output instruction must have. If it is
784 // variadic, we could have more operands.
785 unsigned NumFixedOperands = II.Operands.size();
786
787 SmallVector<unsigned, 8> InstOps;
788
789 // Loop over all of the fixed operands of the instruction pattern, emitting
790 // code to fill them all in. The node 'N' usually has number children equal to
791 // the number of input operands of the instruction. However, in cases where
792 // there are predicate operands for an instruction, we need to fill in the
793 // 'execute always' values. Match up the node operands to the instruction
794 // operands to do this.
795 unsigned ChildNo = 0;
796
797 // Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the
798 // number of operands at the end of the list which have default values.
799 // Those can come from the pattern if it provides enough arguments, or be
800 // filled in with the default if the pattern hasn't provided them. But any
801 // operand with a default value _before_ the last mandatory one will be
802 // filled in with their defaults unconditionally.
803 unsigned NonOverridableOperands = NumFixedOperands;
804 while (NonOverridableOperands > NumResults &&
805 CGP.operandHasDefault(Op: II.Operands[NonOverridableOperands - 1].Rec))
806 --NonOverridableOperands;
807
808 for (unsigned InstOpNo = NumResults, e = NumFixedOperands; InstOpNo != e;
809 ++InstOpNo) {
810 // Determine what to emit for this operand.
811 const Record *OperandNode = II.Operands[InstOpNo].Rec;
812 if (CGP.operandHasDefault(Op: OperandNode) &&
813 (InstOpNo < NonOverridableOperands || ChildNo >= N.getNumChildren())) {
814 // This is a predicate or optional def operand which the pattern has not
815 // overridden, or which we aren't letting it override; emit the 'default
816 // ops' operands.
817 const DAGDefaultOperand &DefaultOp = CGP.getDefaultOperand(R: OperandNode);
818 for (const TreePatternNodePtr &Op : DefaultOp.DefaultOps)
819 EmitResultOperand(N: *Op, ResultOps&: InstOps);
820 continue;
821 }
822
823 // Otherwise this is a normal operand or a predicate operand without
824 // 'execute always'; emit it.
825
826 // For operands with multiple sub-operands we may need to emit
827 // multiple child patterns to cover them all. However, ComplexPattern
828 // children may themselves emit multiple MI operands.
829 unsigned NumSubOps = 1;
830 if (OperandNode->isSubClassOf(Name: "Operand")) {
831 const DagInit *MIOpInfo = OperandNode->getValueAsDag(FieldName: "MIOperandInfo");
832 if (unsigned NumArgs = MIOpInfo->getNumArgs())
833 NumSubOps = NumArgs;
834 }
835
836 unsigned FinalNumOps = InstOps.size() + NumSubOps;
837 while (InstOps.size() < FinalNumOps) {
838 const TreePatternNode &Child = N.getChild(N: ChildNo);
839 unsigned BeforeAddingNumOps = InstOps.size();
840 EmitResultOperand(N: Child, ResultOps&: InstOps);
841 assert(InstOps.size() > BeforeAddingNumOps && "Didn't add any operands");
842
843 // If the operand is an instruction and it produced multiple results, just
844 // take the first one.
845 if (!Child.isLeaf() && Child.getOperator()->isSubClassOf(Name: "Instruction"))
846 InstOps.resize(N: BeforeAddingNumOps + 1);
847
848 ++ChildNo;
849 }
850 }
851
852 // If this is a variadic output instruction (i.e. REG_SEQUENCE), we can't
853 // expand suboperands, use default operands, or other features determined from
854 // the CodeGenInstruction after the fixed operands, which were handled
855 // above. Emit the remaining instructions implicitly added by the use for
856 // variable_ops.
857 if (II.Operands.isVariadic) {
858 for (unsigned I = ChildNo, E = N.getNumChildren(); I < E; ++I)
859 EmitResultOperand(N: N.getChild(N: I), ResultOps&: InstOps);
860 }
861
862 // If this node has input glue or explicitly specified input physregs, we
863 // need to add chained and glued copyfromreg nodes and materialize the glue
864 // input.
865 if (isRoot && !PhysRegInputs.empty()) {
866 // Emit all of the CopyToReg nodes for the input physical registers. These
867 // occur in patterns like (mul:i8 AL:i8, GR8:i8:$src).
868 for (const auto &PhysRegInput : PhysRegInputs) {
869 const CodeGenRegister *Reg =
870 CGP.getTargetInfo().getRegBank().getReg(PhysRegInput.first);
871 AddMatcher(NewNode: new EmitCopyToRegMatcher(PhysRegInput.second, Reg));
872 }
873
874 // Even if the node has no other glue inputs, the resultant node must be
875 // glued to the CopyFromReg nodes we just generated.
876 TreeHasInGlue = true;
877 }
878
879 // Result order: node results, chain, glue
880
881 // Determine the result types.
882 SmallVector<ValueTypeByHwMode, 4> ResultVTs;
883 for (unsigned i = 0, e = N.getNumTypes(); i != e; ++i)
884 ResultVTs.push_back(Elt: N.getType(ResNo: i));
885
886 // If this is the root instruction of a pattern that has physical registers in
887 // its result pattern, add output VTs for them. For example, X86 has:
888 // (set AL, (mul ...))
889 if (isRoot && !Pattern.getDstRegs().empty()) {
890 // If the root came from an implicit def in the instruction handling stuff,
891 // don't re-add it.
892 const Record *HandledReg = nullptr;
893 if (II.HasOneImplicitDefWithKnownVT(TargetInfo: CGT) != MVT::Other)
894 HandledReg = II.ImplicitDefs[0];
895
896 for (const Record *Reg : Pattern.getDstRegs()) {
897 if (!Reg->isSubClassOf(Name: "Register") || Reg == HandledReg)
898 continue;
899 ResultVTs.push_back(Elt: getRegisterValueType(R: Reg, T: CGT));
900 }
901 }
902
903 // If this is the root of the pattern and the pattern we're matching includes
904 // a node that is variadic, mark the generated node as variadic so that it
905 // gets the excess operands from the input DAG.
906 int NumFixedArityOperands = -1;
907 if (isRoot && Pattern.getSrcPattern().NodeHasProperty(Property: SDNPVariadic, CGP))
908 NumFixedArityOperands = Pattern.getSrcPattern().getNumChildren();
909
910 // If this is the root node and multiple matched nodes in the input pattern
911 // have MemRefs in them, have the interpreter collect them and plop them onto
912 // this node. If there is just one node with MemRefs, leave them on that node
913 // even if it is not the root.
914 //
915 // FIXME3: This is actively incorrect for result patterns with multiple
916 // memory-referencing instructions.
917 bool PatternHasMemOperands =
918 Pattern.getSrcPattern().TreeHasProperty(Property: SDNPMemOperand, CGP);
919
920 bool NodeHasMemRefs = false;
921 if (PatternHasMemOperands) {
922 unsigned NumNodesThatLoadOrStore =
923 numNodesThatMayLoadOrStore(N: Pattern.getDstPattern(), CGP);
924 bool NodeIsUniqueLoadOrStore =
925 mayInstNodeLoadOrStore(N, CGP) && NumNodesThatLoadOrStore == 1;
926 NodeHasMemRefs =
927 NodeIsUniqueLoadOrStore || (isRoot && (mayInstNodeLoadOrStore(N, CGP) ||
928 NumNodesThatLoadOrStore != 1));
929 }
930
931 // Determine whether we need to attach a chain to this node.
932 bool NodeHasChain = false;
933 if (Pattern.getSrcPattern().TreeHasProperty(Property: SDNPHasChain, CGP)) {
934 // For some instructions, we were able to infer from the pattern whether
935 // they should have a chain. Otherwise, attach the chain to the root.
936 //
937 // FIXME2: This is extremely dubious for several reasons, not the least of
938 // which it gives special status to instructions with patterns that Pat<>
939 // nodes can't duplicate.
940 if (II.hasChain_Inferred)
941 NodeHasChain = II.hasChain;
942 else
943 NodeHasChain = isRoot;
944 // Instructions which load and store from memory should have a chain,
945 // regardless of whether they happen to have a pattern saying so.
946 if (II.hasCtrlDep || II.mayLoad || II.mayStore || II.canFoldAsLoad ||
947 II.hasSideEffects)
948 NodeHasChain = true;
949 }
950
951 assert((!ResultVTs.empty() || TreeHasOutGlue || NodeHasChain) &&
952 "Node has no result");
953
954 AddMatcher(NewNode: new EmitNodeMatcher(II, ResultVTs, InstOps, NodeHasChain,
955 TreeHasInGlue, TreeHasOutGlue, NodeHasMemRefs,
956 NumFixedArityOperands, NextRecordedOperandNo));
957
958 // The non-chain and non-glue results of the newly emitted node get recorded.
959 for (const ValueTypeByHwMode &ResultVT : ResultVTs) {
960 if (ResultVT.isSimple() && (ResultVT.getSimple() == MVT::Other ||
961 ResultVT.getSimple() == MVT::Glue))
962 break;
963 OutputOps.push_back(Elt: NextRecordedOperandNo++);
964 }
965}
966
967void MatcherGen::EmitResultSDNodeXFormAsOperand(
968 const TreePatternNode &N, SmallVectorImpl<unsigned> &ResultOps) {
969 assert(N.getOperator()->isSubClassOf("SDNodeXForm") && "Not SDNodeXForm?");
970
971 // Emit the operand.
972 SmallVector<unsigned, 8> InputOps;
973
974 // FIXME2: Could easily generalize this to support multiple inputs and outputs
975 // to the SDNodeXForm. For now we just support one input and one output like
976 // the old instruction selector.
977 assert(N.getNumChildren() == 1);
978 EmitResultOperand(N: N.getChild(N: 0), ResultOps&: InputOps);
979
980 // The input currently must have produced exactly one result.
981 assert(InputOps.size() == 1 && "Unexpected input to SDNodeXForm");
982
983 AddMatcher(NewNode: new EmitNodeXFormMatcher(InputOps[0], N.getOperator(),
984 NextRecordedOperandNo));
985 ResultOps.push_back(Elt: NextRecordedOperandNo++);
986}
987
988void MatcherGen::EmitResultOperand(const TreePatternNode &N,
989 SmallVectorImpl<unsigned> &ResultOps) {
990 // This is something selected from the pattern we matched.
991 if (!N.getName().empty())
992 return EmitResultOfNamedOperand(N, ResultOps);
993
994 if (N.isLeaf())
995 return EmitResultLeafAsOperand(N, ResultOps);
996
997 const Record *OpRec = N.getOperator();
998 if (OpRec->isSubClassOf(Name: "Instruction"))
999 return EmitResultInstructionAsOperand(N, OutputOps&: ResultOps);
1000 if (OpRec->isSubClassOf(Name: "SDNodeXForm"))
1001 return EmitResultSDNodeXFormAsOperand(N, ResultOps);
1002 errs() << "Unknown result node to emit code for: " << N << '\n';
1003 PrintFatalError(Msg: "Unknown node in result pattern!");
1004}
1005
1006void MatcherGen::EmitResultCode() {
1007 // Patterns that match nodes with (potentially multiple) chain inputs have to
1008 // merge them together into a token factor. This informs the generated code
1009 // what all the chained nodes are.
1010 if (!MatchedChainNodes.empty())
1011 AddMatcher(NewNode: new EmitMergeInputChainsMatcher(MatchedChainNodes));
1012
1013 // Codegen the root of the result pattern, capturing the resulting values.
1014 SmallVector<unsigned, 8> Ops;
1015 EmitResultOperand(N: Pattern.getDstPattern(), ResultOps&: Ops);
1016
1017 // At this point, we have however many values the result pattern produces.
1018 // However, the input pattern might not need all of these. If there are
1019 // excess values at the end (such as implicit defs of condition codes etc)
1020 // just lop them off. This doesn't need to worry about glue or chains, just
1021 // explicit results.
1022 //
1023 unsigned NumSrcResults = Pattern.getSrcPattern().getNumTypes();
1024
1025 // If the pattern also has implicit results, count them as well.
1026 if (!Pattern.getDstRegs().empty()) {
1027 // If the root came from an implicit def in the instruction handling stuff,
1028 // don't re-add it.
1029 const Record *HandledReg = nullptr;
1030 const TreePatternNode &DstPat = Pattern.getDstPattern();
1031 if (!DstPat.isLeaf() && DstPat.getOperator()->isSubClassOf(Name: "Instruction")) {
1032 const CodeGenTarget &CGT = CGP.getTargetInfo();
1033 const CodeGenInstruction &II = CGT.getInstruction(InstRec: DstPat.getOperator());
1034
1035 if (II.HasOneImplicitDefWithKnownVT(TargetInfo: CGT) != MVT::Other)
1036 HandledReg = II.ImplicitDefs[0];
1037 }
1038
1039 for (const Record *Reg : Pattern.getDstRegs()) {
1040 if (!Reg->isSubClassOf(Name: "Register") || Reg == HandledReg)
1041 continue;
1042 ++NumSrcResults;
1043 }
1044 }
1045
1046 SmallVector<unsigned, 8> Results(Ops);
1047
1048 // Apply result permutation.
1049 for (unsigned ResNo = 0; ResNo < Pattern.getDstPattern().getNumResults();
1050 ++ResNo) {
1051 Results[ResNo] = Ops[Pattern.getDstPattern().getResultIndex(ResNo)];
1052 }
1053
1054 Results.resize(N: NumSrcResults);
1055 AddMatcher(NewNode: new CompleteMatchMatcher(Results, Pattern));
1056}
1057
1058/// Create the matcher for the specified pattern with the specified variant.
1059/// If the variant number is invalid, this returns an empty MatcherList.
1060MatcherList llvm::ConvertPatternToMatcherList(const PatternToMatch &Pattern,
1061 unsigned Variant,
1062 const CodeGenDAGPatterns &CGP) {
1063 MatcherGen Gen(Pattern, CGP);
1064
1065 // Generate the code for the matcher.
1066 if (Gen.EmitMatcherCode(Variant))
1067 return MatcherList();
1068
1069 // FIXME2: Kill extra MoveParent commands at the end of the matcher sequence.
1070 // FIXME2: Split result code out to another table, and make the matcher end
1071 // with an "Emit <index>" command. This allows result generation stuff to be
1072 // shared and factored?
1073
1074 // If the match succeeds, then we generate Pattern.
1075 Gen.EmitResultCode();
1076
1077 // Unconditional match.
1078 return Gen.GetMatcherList();
1079}
1080