1//===- GlobalISelMatchTable.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "GlobalISelMatchTable.h"
10#include "Common/CodeGenInstruction.h"
11#include "Common/CodeGenRegisters.h"
12#include "llvm/ADT/Statistic.h"
13#include "llvm/Support/CommandLine.h"
14#include "llvm/Support/Debug.h"
15#include "llvm/Support/LEB128.h"
16#include "llvm/Support/ScopedPrinter.h"
17#include "llvm/Support/raw_ostream.h"
18#include "llvm/TableGen/Error.h"
19
20#define DEBUG_TYPE "gi-match-table"
21
22STATISTIC(NumPatternEmitted, "Number of patterns emitted");
23
24using namespace llvm;
25using namespace gi;
26
27// FIXME: Use createStringError instead.
28static Error failUnsupported(const Twine &Reason) {
29 return make_error<StringError>(Args: Reason, Args: inconvertibleErrorCode());
30}
31
32/// Get the name of the enum value used to number the predicate function.
33static std::string getEnumNameForPredicate(const TreePredicateFn &Predicate) {
34 if (Predicate.hasGISelPredicateCode())
35 return "GICXXPred_MI_" + Predicate.getFnName();
36 if (Predicate.hasGISelLeafPredicateCode())
37 return "GICXXPred_MO_" + Predicate.getFnName();
38 return "GICXXPred_" + Predicate.getImmTypeIdentifier().str() + "_" +
39 Predicate.getFnName();
40}
41
42static std::string
43getMatchOpcodeForImmPredicate(const TreePredicateFn &Predicate) {
44 return "GIM_Check" + Predicate.getImmTypeIdentifier().str() + "ImmPredicate";
45}
46
47// GIMT_Encode2/4/8
48constexpr StringLiteral EncodeMacroName = "GIMT_Encode";
49
50//===- Helpers ------------------------------------------------------------===//
51
52void llvm::gi::emitEncodingMacrosDef(raw_ostream &OS) {
53 OS << "#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__\n"
54 << "#define " << EncodeMacroName << "2(Val)"
55 << " uint8_t(Val), uint8_t((Val) >> 8)\n"
56 << "#define " << EncodeMacroName << "4(Val)"
57 << " uint8_t(Val), uint8_t((Val) >> 8), "
58 "uint8_t((Val) >> 16), uint8_t((Val) >> 24)\n"
59 << "#define " << EncodeMacroName << "8(Val)"
60 << " uint8_t(Val), uint8_t((Val) >> 8), "
61 "uint8_t((Val) >> 16), uint8_t((Val) >> 24), "
62 "uint8_t(uint64_t(Val) >> 32), uint8_t(uint64_t(Val) >> 40), "
63 "uint8_t(uint64_t(Val) >> 48), uint8_t(uint64_t(Val) >> 56)\n"
64 << "#else\n"
65 << "#define " << EncodeMacroName << "2(Val)"
66 << " uint8_t((Val) >> 8), uint8_t(Val)\n"
67 << "#define " << EncodeMacroName << "4(Val)"
68 << " uint8_t((Val) >> 24), uint8_t((Val) >> 16), "
69 "uint8_t((Val) >> 8), uint8_t(Val)\n"
70 << "#define " << EncodeMacroName << "8(Val)"
71 << " uint8_t(uint64_t(Val) >> 56), uint8_t(uint64_t(Val) >> 48), "
72 "uint8_t(uint64_t(Val) >> 40), uint8_t(uint64_t(Val) >> 32), "
73 "uint8_t((Val) >> 24), uint8_t((Val) >> 16), "
74 "uint8_t((Val) >> 8), uint8_t(Val)\n"
75 << "#endif\n";
76}
77
78void llvm::gi::emitEncodingMacrosUndef(raw_ostream &OS) {
79 OS << "#undef " << EncodeMacroName << "2\n"
80 << "#undef " << EncodeMacroName << "4\n"
81 << "#undef " << EncodeMacroName << "8\n";
82}
83
84std::string
85llvm::gi::getNameForFeatureBitset(ArrayRef<const Record *> FeatureBitset,
86 int HwModeIdx) {
87 std::string Name = "GIFBS";
88 for (const Record *Feature : FeatureBitset)
89 Name += ("_" + Feature->getName()).str();
90 if (HwModeIdx >= 0)
91 Name += ("_HwMode" + std::to_string(val: HwModeIdx));
92 return Name;
93}
94
95template <class GroupT>
96std::vector<Matcher *>
97llvm::gi::optimizeRules(ArrayRef<Matcher *> Rules,
98 std::vector<std::unique_ptr<Matcher>> &MatcherStorage) {
99
100 std::vector<Matcher *> OptRules;
101 std::unique_ptr<GroupT> CurrentGroup = std::make_unique<GroupT>();
102 assert(CurrentGroup->empty() && "Newly created group isn't empty!");
103 unsigned NumGroups = 0;
104
105 auto ProcessCurrentGroup = [&]() {
106 if (CurrentGroup->empty())
107 // An empty group is good to be reused:
108 return;
109
110 // If the group isn't large enough to provide any benefit, move all the
111 // added rules out of it and make sure to re-create the group to properly
112 // re-initialize it:
113 if (CurrentGroup->size() < 2)
114 append_range(OptRules, CurrentGroup->matchers());
115 else {
116 CurrentGroup->finalize();
117 OptRules.push_back(CurrentGroup.get());
118 MatcherStorage.emplace_back(std::move(CurrentGroup));
119 ++NumGroups;
120 }
121 CurrentGroup = std::make_unique<GroupT>();
122 };
123 for (Matcher *Rule : Rules) {
124 // Greedily add as many matchers as possible to the current group:
125 if (CurrentGroup->addMatcher(*Rule))
126 continue;
127
128 ProcessCurrentGroup();
129 assert(CurrentGroup->empty() && "A group wasn't properly re-initialized");
130
131 // Try to add the pending matcher to a newly created empty group:
132 if (!CurrentGroup->addMatcher(*Rule))
133 // If we couldn't add the matcher to an empty group, that group type
134 // doesn't support that kind of matchers at all, so just skip it:
135 OptRules.push_back(x: Rule);
136 }
137 ProcessCurrentGroup();
138
139 LLVM_DEBUG(dbgs() << "NumGroups: " << NumGroups << "\n");
140 (void)NumGroups;
141 assert(CurrentGroup->empty() && "The last group wasn't properly processed");
142 return OptRules;
143}
144
145template std::vector<Matcher *> llvm::gi::optimizeRules<GroupMatcher>(
146 ArrayRef<Matcher *> Rules,
147 std::vector<std::unique_ptr<Matcher>> &MatcherStorage);
148
149template std::vector<Matcher *> llvm::gi::optimizeRules<SwitchMatcher>(
150 ArrayRef<Matcher *> Rules,
151 std::vector<std::unique_ptr<Matcher>> &MatcherStorage);
152
153static std::string getEncodedEmitStr(StringRef NamedValue, unsigned NumBytes) {
154 if (NumBytes == 2 || NumBytes == 4 || NumBytes == 8)
155 return (EncodeMacroName + Twine(NumBytes) + "(" + NamedValue + ")").str();
156 llvm_unreachable("Unsupported number of bytes!");
157}
158
159template <class Range> static bool matchersRecordOperand(Range &&R) {
160 return any_of(R, [](const auto &I) { return I->recordsOperand(); });
161}
162
163//===- Global Data --------------------------------------------------------===//
164
165std::set<LLTCodeGen> llvm::gi::KnownTypes;
166
167//===- MatchTableRecord ---------------------------------------------------===//
168
169void MatchTableRecord::emit(raw_ostream &OS, bool LineBreakIsNextAfterThis,
170 const MatchTable &Table) const {
171 bool UseLineComment =
172 LineBreakIsNextAfterThis || (Flags & MTRF_LineBreakFollows);
173 if (Flags & (MTRF_JumpTarget | MTRF_CommaFollows))
174 UseLineComment = false;
175
176 if (Flags & MTRF_Comment)
177 OS << (UseLineComment ? "// " : "/*");
178
179 if (NumElements > 1 && !(Flags & (MTRF_PreEncoded | MTRF_Comment)))
180 OS << getEncodedEmitStr(NamedValue: EmitStr, NumBytes: NumElements);
181 else
182 OS << EmitStr;
183
184 if (Flags & MTRF_Label)
185 OS << ": @" << Table.getLabelIndex(LabelID);
186
187 if ((Flags & MTRF_Comment) && !UseLineComment)
188 OS << "*/";
189
190 if (Flags & MTRF_JumpTarget) {
191 if (Flags & MTRF_Comment)
192 OS << " ";
193 // TODO: Could encode this AOT to speed up build of generated file
194 OS << getEncodedEmitStr(NamedValue: llvm::to_string(Value: Table.getLabelIndex(LabelID)),
195 NumBytes: NumElements);
196 }
197
198 if (Flags & MTRF_CommaFollows) {
199 OS << ",";
200 if (!LineBreakIsNextAfterThis && !(Flags & MTRF_LineBreakFollows))
201 OS << " ";
202 }
203
204 if (Flags & MTRF_LineBreakFollows)
205 OS << "\n";
206}
207
208//===- MatchTable ---------------------------------------------------------===//
209
210MatchTableRecord MatchTable::LineBreak = {
211 std::nullopt, "" /* Emit String */, 0 /* Elements */,
212 MatchTableRecord::MTRF_LineBreakFollows};
213
214MatchTableRecord MatchTable::Comment(StringRef Comment) {
215 return MatchTableRecord(std::nullopt, Comment, 0,
216 MatchTableRecord::MTRF_Comment);
217}
218
219MatchTableRecord MatchTable::Opcode(StringRef Opcode, int IndentAdjust) {
220 unsigned ExtraFlags = 0;
221 if (IndentAdjust > 0)
222 ExtraFlags |= MatchTableRecord::MTRF_Indent;
223 if (IndentAdjust < 0)
224 ExtraFlags |= MatchTableRecord::MTRF_Outdent;
225
226 return MatchTableRecord(std::nullopt, Opcode, 1,
227 MatchTableRecord::MTRF_CommaFollows | ExtraFlags);
228}
229
230MatchTableRecord MatchTable::NamedValue(unsigned NumBytes,
231 StringRef NamedValue) {
232 return MatchTableRecord(std::nullopt, NamedValue, NumBytes,
233 MatchTableRecord::MTRF_CommaFollows);
234}
235
236MatchTableRecord MatchTable::NamedValue(unsigned NumBytes, StringRef Namespace,
237 StringRef NamedValue) {
238 return MatchTableRecord(std::nullopt, (Namespace + "::" + NamedValue).str(),
239 NumBytes, MatchTableRecord::MTRF_CommaFollows);
240}
241
242MatchTableRecord MatchTable::IntValue(unsigned NumBytes, int64_t IntValue) {
243 assert(isUIntN(NumBytes * 8, IntValue) || isIntN(NumBytes * 8, IntValue));
244 uint64_t UIntValue = IntValue;
245 if (NumBytes < 8)
246 UIntValue &= (UINT64_C(1) << NumBytes * 8) - 1;
247 std::string Str = llvm::to_string(Value: UIntValue);
248 if (UIntValue > INT64_MAX)
249 Str += 'u';
250 // TODO: Could optimize this directly to save the compiler some work when
251 // building the file
252 return MatchTableRecord(std::nullopt, Str, NumBytes,
253 MatchTableRecord::MTRF_CommaFollows);
254}
255
256MatchTableRecord MatchTable::ULEB128Value(uint64_t IntValue) {
257 uint8_t Buffer[10];
258 unsigned Len = encodeULEB128(Value: IntValue, p: Buffer);
259
260 // Simple case (most common)
261 if (Len == 1) {
262 return MatchTableRecord(std::nullopt, llvm::to_string(Value: (unsigned)Buffer[0]),
263 1, MatchTableRecord::MTRF_CommaFollows);
264 }
265
266 // Print it as, e.g. /* -123456 (*/, 0xC0, 0xBB, 0x78 /*)*/
267 std::string Str;
268 raw_string_ostream OS(Str);
269 OS << "/* " << llvm::to_string(Value: IntValue) << "(*/";
270 for (unsigned K = 0; K < Len; ++K) {
271 if (K)
272 OS << ", ";
273 OS << "0x" << llvm::toHex(Input: {Buffer[K]});
274 }
275 OS << "/*)*/";
276 return MatchTableRecord(std::nullopt, Str, Len,
277 MatchTableRecord::MTRF_CommaFollows |
278 MatchTableRecord::MTRF_PreEncoded);
279}
280
281MatchTableRecord MatchTable::Label(unsigned LabelID) {
282 return MatchTableRecord(LabelID, "Label " + llvm::to_string(Value: LabelID), 0,
283 MatchTableRecord::MTRF_Label |
284 MatchTableRecord::MTRF_Comment |
285 MatchTableRecord::MTRF_LineBreakFollows);
286}
287
288MatchTableRecord MatchTable::JumpTarget(unsigned LabelID) {
289 return MatchTableRecord(LabelID, "Label " + llvm::to_string(Value: LabelID), 4,
290 MatchTableRecord::MTRF_JumpTarget |
291 MatchTableRecord::MTRF_Comment |
292 MatchTableRecord::MTRF_CommaFollows);
293}
294
295void MatchTable::emitUse(raw_ostream &OS) const { OS << "MatchTable" << ID; }
296
297void MatchTable::emitDeclaration(raw_ostream &OS) const {
298 static constexpr unsigned BaseIndent = 4;
299 unsigned Indentation = 0;
300 OS << " constexpr static uint8_t MatchTable" << ID << "[] = {";
301 LineBreak.emit(OS, LineBreakIsNextAfterThis: true, Table: *this);
302
303 // We want to display the table index of each line in a consistent
304 // manner. It has to appear as a column on the left side of the table.
305 // To determine how wide the column needs to be, check how many characters
306 // we need to fit the largest possible index in the current table.
307 const unsigned NumColsForIdx = llvm::to_string(Value: CurrentSize).size();
308
309 unsigned CurIndex = 0;
310 const auto BeginLine = [&]() {
311 OS.indent(NumSpaces: BaseIndent);
312 std::string IdxStr = llvm::to_string(Value: CurIndex);
313 // Pad the string with spaces to keep the size of the prefix consistent.
314 OS << " /* ";
315 OS.indent(NumSpaces: NumColsForIdx - IdxStr.size()) << IdxStr << " */ ";
316 OS.indent(NumSpaces: Indentation);
317 };
318
319 BeginLine();
320 for (auto I = Contents.begin(), E = Contents.end(); I != E; ++I) {
321 bool LineBreakIsNext = false;
322 const auto &NextI = std::next(x: I);
323
324 if (NextI != E) {
325 if (NextI->EmitStr == "" &&
326 NextI->Flags == MatchTableRecord::MTRF_LineBreakFollows)
327 LineBreakIsNext = true;
328 }
329
330 if (I->Flags & MatchTableRecord::MTRF_Indent)
331 Indentation += 2;
332
333 I->emit(OS, LineBreakIsNextAfterThis: LineBreakIsNext, Table: *this);
334 if (I->Flags & MatchTableRecord::MTRF_LineBreakFollows)
335 BeginLine();
336
337 if (I->Flags & MatchTableRecord::MTRF_Outdent)
338 Indentation -= 2;
339
340 CurIndex += I->size();
341 }
342 assert(CurIndex == CurrentSize);
343 OS << "}; // Size: " << CurrentSize << " bytes\n";
344}
345
346MatchTable MatchTable::buildTable(ArrayRef<Matcher *> Rules, bool WithCoverage,
347 bool IsCombiner) {
348 MatchTable Table(WithCoverage, IsCombiner);
349 for (Matcher *Rule : Rules)
350 Rule->emit(Table);
351
352 return Table << MatchTable::Opcode(Opcode: "GIM_Reject") << MatchTable::LineBreak;
353}
354
355//===- LLTCodeGen ---------------------------------------------------------===//
356
357std::string LLTCodeGen::getCxxEnumValue() const {
358 std::string Str;
359 raw_string_ostream OS(Str);
360
361 emitCxxEnumValue(OS);
362 return Str;
363}
364
365void LLTCodeGen::emitCxxEnumValue(raw_ostream &OS) const {
366 if (Ty.isScalar()) {
367 if (Ty.isBFloat16())
368 OS << "GILLT_bf16";
369 else if (Ty.isPPCF128())
370 OS << "GILLT_ppcf128";
371 else if (Ty.isX86FP80())
372 OS << "GILLT_x86fp80";
373 else if (Ty.isFloat())
374 OS << "GILLT_f" << Ty.getSizeInBits();
375 else if (Ty.isInteger())
376 OS << "GILLT_i" << Ty.getSizeInBits();
377 else
378 OS << "GILLT_s" << Ty.getSizeInBits();
379 return;
380 }
381 if (Ty.isVector()) {
382 OS << (Ty.isScalable() ? "GILLT_nxv" : "GILLT_v")
383 << Ty.getElementCount().getKnownMinValue();
384
385 LLT ElemTy = Ty.getElementType();
386 if (ElemTy.isBFloat16())
387 OS << "bf16";
388 else if (ElemTy.isPPCF128())
389 OS << "ppcf128";
390 else if (ElemTy.isX86FP80())
391 OS << "x86fp80";
392 else if (ElemTy.isFloat())
393 OS << "f" << ElemTy.getSizeInBits();
394 else if (ElemTy.isInteger())
395 OS << "i" << ElemTy.getSizeInBits();
396 else
397 OS << "s" << ElemTy.getSizeInBits();
398 return;
399 }
400
401 if (Ty.isPointer()) {
402 OS << "GILLT_p" << Ty.getAddressSpace();
403 if (Ty.getSizeInBits() > 0)
404 OS << "s" << Ty.getSizeInBits();
405 return;
406 }
407
408 llvm_unreachable("Unhandled LLT");
409}
410
411void LLTCodeGen::emitCxxConstructorCall(raw_ostream &OS) const {
412 if (Ty.isScalar()) {
413 if (Ty.isInteger())
414 OS << "LLT::integer(" << Ty.getScalarSizeInBits() << ")";
415 else if (Ty.isBFloat16())
416 OS << "LLT::bfloat16()";
417 else if (Ty.isPPCF128())
418 OS << "LLT::ppcf128()";
419 else if (Ty.isX86FP80())
420 OS << "LLT::x86fp80()";
421 else if (Ty.isFloat())
422 OS << "LLT::floatIEEE(" << Ty.getScalarSizeInBits() << ")";
423 else
424 OS << "LLT::scalar(" << Ty.getScalarSizeInBits() << ")";
425 return;
426 }
427
428 if (Ty.isVector()) {
429 OS << "LLT::vector("
430 << (Ty.isScalable() ? "ElementCount::getScalable("
431 : "ElementCount::getFixed(")
432 << Ty.getElementCount().getKnownMinValue() << "), ";
433
434 LLT ElemTy = Ty.getElementType();
435 if (ElemTy.isInteger())
436 OS << "LLT::integer(" << ElemTy.getScalarSizeInBits() << ")";
437 else if (ElemTy.isBFloat16())
438 OS << "LLT::bfloat16()";
439 else if (ElemTy.isPPCF128())
440 OS << "LLT::ppcf128()";
441 else if (ElemTy.isX86FP80())
442 OS << "LLT::x86fp80()";
443 else if (ElemTy.isFloat())
444 OS << "LLT::floatIEEE(" << ElemTy.getScalarSizeInBits() << ")";
445 else
446 OS << "LLT::scalar(" << Ty.getScalarSizeInBits() << ")";
447 OS << ")";
448 return;
449 }
450
451 if (Ty.isPointer() && Ty.getSizeInBits() > 0) {
452 OS << "LLT::pointer(" << Ty.getAddressSpace() << ", " << Ty.getSizeInBits()
453 << ")";
454 return;
455 }
456
457 llvm_unreachable("Unhandled LLT");
458}
459
460/// This ordering is used for std::unique() and llvm::sort(). There's no
461/// particular logic behind the order but either A < B or B < A must be
462/// true if A != B.
463bool LLTCodeGen::operator<(const LLTCodeGen &Other) const {
464 return Ty.getUniqueRAWLLTData() < Other.Ty.getUniqueRAWLLTData();
465}
466
467//===- LLTCodeGen Helpers -------------------------------------------------===//
468
469std::optional<LLTCodeGen> llvm::gi::MVTToLLT(MVT VT) {
470 if (VT.isVector() && !VT.getVectorElementCount().isScalar())
471 return LLTCodeGen(LLT(VT));
472
473 if (VT.isInteger() || VT.isFloatingPoint())
474 return LLTCodeGen(LLT(VT));
475
476 return std::nullopt;
477}
478
479//===- Matcher ------------------------------------------------------------===//
480
481void Matcher::optimize() {}
482
483Matcher::~Matcher() = default;
484
485//===- GroupMatcher -------------------------------------------------------===//
486
487bool GroupMatcher::recordsOperand() const {
488 return matchersRecordOperand(R: Conditions) || matchersRecordOperand(R: Matchers);
489}
490
491bool GroupMatcher::candidateConditionMatches(
492 const PredicateMatcher &Predicate) const {
493
494 if (empty()) {
495 // Sharing predicates for nested instructions is not supported yet as we
496 // currently don't hoist the GIM_RecordInsn's properly, therefore we can
497 // only work on the original root instruction (InsnVarID == 0):
498 if (Predicate.getInsnVarID() != 0)
499 return false;
500 // ... otherwise an empty group can handle any predicate with no specific
501 // requirements:
502 return true;
503 }
504
505 const Matcher &Representative = **Matchers.begin();
506 const auto &RepresentativeCondition = Representative.getFirstCondition();
507 // ... if not empty, the group can only accomodate matchers with the exact
508 // same first condition:
509 return Predicate.isIdentical(B: RepresentativeCondition);
510}
511
512std::unique_ptr<PredicateMatcher> GroupMatcher::popFirstCondition() {
513 assert(!Conditions.empty() &&
514 "Trying to pop a condition from a condition-less group");
515 std::unique_ptr<PredicateMatcher> P = std::move(Conditions.front());
516 Conditions.erase(CI: Conditions.begin());
517 return P;
518}
519
520/// Check if the Condition, which is a predicate of M, cannot be hoisted outside
521/// of (i.e., checked before) M.
522static bool cannotHoistCondition(const PredicateMatcher &Condition,
523 const Matcher &M) {
524 // The condition can't be hoisted if it is a C++ predicate that refers to
525 // operands and the operands are registered within the matcher.
526 return Condition.dependsOnOperands() && M.recordsOperand();
527}
528
529bool GroupMatcher::addMatcher(Matcher &Candidate) {
530 if (!Candidate.hasFirstCondition())
531 return false;
532
533 // Only add candidates that have a matching first condition that can be
534 // hoisted into the GroupMatcher.
535 const PredicateMatcher &Predicate = Candidate.getFirstCondition();
536 if (!candidateConditionMatches(Predicate) ||
537 cannotHoistCondition(Condition: Predicate, M: Candidate))
538 return false;
539
540 Matchers.push_back(x: &Candidate);
541 return true;
542}
543
544void GroupMatcher::finalize() {
545 assert(Conditions.empty() && "Already finalized?");
546 if (empty())
547 return;
548
549 Matcher &FirstRule = **Matchers.begin();
550 for (;;) {
551 // All the checks are expected to succeed during the first iteration:
552 for (const auto &Rule : Matchers)
553 if (!Rule->hasFirstCondition())
554 return;
555 // Hoist the first condition if it is identical in all matchers in the group
556 // and it can be hoisted in every matcher.
557 const auto &FirstCondition = FirstRule.getFirstCondition();
558 if (cannotHoistCondition(Condition: FirstCondition, M: FirstRule))
559 return;
560 for (unsigned I = 1, E = Matchers.size(); I < E; ++I) {
561 const auto &OtherFirstCondition = Matchers[I]->getFirstCondition();
562 if (!OtherFirstCondition.isIdentical(B: FirstCondition) ||
563 cannotHoistCondition(Condition: OtherFirstCondition, M: *Matchers[I]))
564 return;
565 }
566
567 Conditions.push_back(Elt: FirstRule.popFirstCondition());
568 for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
569 Matchers[I]->popFirstCondition();
570 }
571}
572
573void GroupMatcher::emit(MatchTable &Table) {
574 unsigned LabelID = ~0U;
575 if (!Conditions.empty()) {
576 LabelID = Table.allocateLabelID();
577 Table << MatchTable::Opcode(Opcode: "GIM_Try", IndentAdjust: +1)
578 << MatchTable::Comment(Comment: "On fail goto")
579 << MatchTable::JumpTarget(LabelID) << MatchTable::LineBreak;
580 }
581 for (auto &Condition : Conditions)
582 Condition->emitPredicateOpcodes(
583 Table, Rule&: *static_cast<RuleMatcher *>(*Matchers.begin()));
584
585 for (const auto &M : Matchers)
586 M->emit(Table);
587
588 // Exit the group
589 if (!Conditions.empty())
590 Table << MatchTable::Opcode(Opcode: "GIM_Reject", IndentAdjust: -1) << MatchTable::LineBreak
591 << MatchTable::Label(LabelID);
592}
593
594void GroupMatcher::optimize() {
595 // Make sure we only sort by a specific predicate within a range of rules that
596 // all have that predicate checked against a specific value (not a wildcard):
597 auto F = Matchers.begin();
598 auto T = F;
599 auto E = Matchers.end();
600 while (T != E) {
601 while (T != E) {
602 auto *R = static_cast<RuleMatcher *>(*T);
603 if (!R->getFirstConditionAsRootType().get().isValid())
604 break;
605 ++T;
606 }
607 std::stable_sort(first: F, last: T, comp: [](Matcher *A, Matcher *B) {
608 auto *L = static_cast<RuleMatcher *>(A);
609 auto *R = static_cast<RuleMatcher *>(B);
610 return L->getFirstConditionAsRootType() <
611 R->getFirstConditionAsRootType();
612 });
613 if (T != E)
614 F = ++T;
615 }
616 Matchers = optimizeRules<GroupMatcher>(Rules: Matchers, MatcherStorage);
617 Matchers = optimizeRules<SwitchMatcher>(Rules: Matchers, MatcherStorage);
618}
619
620//===- SwitchMatcher ------------------------------------------------------===//
621
622bool SwitchMatcher::recordsOperand() const {
623 assert(!isa_and_present<RecordNamedOperandMatcher>(Condition.get()) &&
624 "Switch conditions should not record named operands");
625 return matchersRecordOperand(R: Matchers);
626}
627
628bool SwitchMatcher::isSupportedPredicateType(const PredicateMatcher &P) {
629 return isa<InstructionOpcodeMatcher>(Val: P) || isa<LLTOperandMatcher>(Val: P);
630}
631
632bool SwitchMatcher::candidateConditionMatches(
633 const PredicateMatcher &Predicate) const {
634
635 if (empty()) {
636 // Sharing predicates for nested instructions is not supported yet as we
637 // currently don't hoist the GIM_RecordInsn's properly, therefore we can
638 // only work on the original root instruction (InsnVarID == 0):
639 if (Predicate.getInsnVarID() != 0)
640 return false;
641 // ... while an attempt to add even a root matcher to an empty SwitchMatcher
642 // could fail as not all the types of conditions are supported:
643 if (!isSupportedPredicateType(P: Predicate))
644 return false;
645 // ... or the condition might not have a proper implementation of
646 // getValue() / isIdenticalDownToValue() yet:
647 if (!Predicate.hasValue())
648 return false;
649 // ... otherwise an empty Switch can accomodate the condition with no
650 // further requirements:
651 return true;
652 }
653
654 const Matcher &CaseRepresentative = **Matchers.begin();
655 const auto &RepresentativeCondition = CaseRepresentative.getFirstCondition();
656 // Switch-cases must share the same kind of condition and path to the value it
657 // checks:
658 if (!Predicate.isIdenticalDownToValue(B: RepresentativeCondition))
659 return false;
660
661 const auto Value = Predicate.getValue();
662 // ... but be unique with respect to the actual value they check:
663 return Values.count(x: Value) == 0;
664}
665
666bool SwitchMatcher::addMatcher(Matcher &Candidate) {
667 if (!Candidate.hasFirstCondition())
668 return false;
669
670 const PredicateMatcher &Predicate = Candidate.getFirstCondition();
671 if (!candidateConditionMatches(Predicate))
672 return false;
673 const auto Value = Predicate.getValue();
674 Values.insert(x: Value);
675
676 Matchers.push_back(x: &Candidate);
677 return true;
678}
679
680void SwitchMatcher::finalize() {
681 assert(Condition == nullptr && "Already finalized");
682 assert(Values.size() == Matchers.size() && "Broken SwitchMatcher");
683 if (empty())
684 return;
685
686 llvm::stable_sort(Range&: Matchers, C: [](const Matcher *L, const Matcher *R) {
687 return L->getFirstCondition().getValue() <
688 R->getFirstCondition().getValue();
689 });
690 Condition = Matchers[0]->popFirstCondition();
691 for (unsigned I = 1, E = Values.size(); I < E; ++I)
692 Matchers[I]->popFirstCondition();
693}
694
695void SwitchMatcher::emitPredicateSpecificOpcodes(const PredicateMatcher &P,
696 MatchTable &Table) {
697 assert(isSupportedPredicateType(P) && "Predicate type is not supported");
698
699 if (const auto *Condition = dyn_cast<InstructionOpcodeMatcher>(Val: &P)) {
700 Table << MatchTable::Opcode(Opcode: "GIM_SwitchOpcode") << MatchTable::Comment(Comment: "MI")
701 << MatchTable::ULEB128Value(IntValue: Condition->getInsnVarID());
702 return;
703 }
704 if (const auto *Condition = dyn_cast<LLTOperandMatcher>(Val: &P)) {
705 Table << MatchTable::Opcode(Opcode: "GIM_SwitchType") << MatchTable::Comment(Comment: "MI")
706 << MatchTable::ULEB128Value(IntValue: Condition->getInsnVarID())
707 << MatchTable::Comment(Comment: "Op")
708 << MatchTable::ULEB128Value(IntValue: Condition->getOpIdx());
709 return;
710 }
711
712 llvm_unreachable("emitPredicateSpecificOpcodes is broken: can not handle a "
713 "predicate type that is claimed to be supported");
714}
715
716void SwitchMatcher::emit(MatchTable &Table) {
717 assert(Values.size() == Matchers.size() && "Broken SwitchMatcher");
718 if (empty())
719 return;
720 assert(Condition != nullptr &&
721 "Broken SwitchMatcher, hasn't been finalized?");
722
723 std::vector<unsigned> LabelIDs(Values.size());
724 std::generate(first: LabelIDs.begin(), last: LabelIDs.end(),
725 gen: [&Table]() { return Table.allocateLabelID(); });
726 const unsigned Default = Table.allocateLabelID();
727
728 const int64_t LowerBound = Values.begin()->RawValue;
729 const int64_t UpperBound = Values.rbegin()->RawValue + 1;
730
731 emitPredicateSpecificOpcodes(P: *Condition, Table);
732
733 Table << MatchTable::Comment(Comment: "[") << MatchTable::IntValue(NumBytes: 2, IntValue: LowerBound)
734 << MatchTable::IntValue(NumBytes: 2, IntValue: UpperBound) << MatchTable::Comment(Comment: ")")
735 << MatchTable::Comment(Comment: "default:") << MatchTable::JumpTarget(LabelID: Default);
736
737 int64_t J = LowerBound;
738 auto VI = Values.begin();
739 for (unsigned I = 0, E = Values.size(); I < E; ++I) {
740 auto V = *VI++;
741 while (J++ < V.RawValue)
742 Table << MatchTable::IntValue(NumBytes: 4, IntValue: 0);
743 V.Record.turnIntoComment();
744 Table << MatchTable::LineBreak << V.Record
745 << MatchTable::JumpTarget(LabelID: LabelIDs[I]);
746 }
747 Table << MatchTable::LineBreak;
748
749 for (unsigned I = 0, E = Values.size(); I < E; ++I) {
750 Table << MatchTable::Label(LabelID: LabelIDs[I]);
751 Matchers[I]->emit(Table);
752 Table << MatchTable::Opcode(Opcode: "GIM_Reject") << MatchTable::LineBreak;
753 }
754 Table << MatchTable::Label(LabelID: Default);
755}
756
757//===- RuleMatcher --------------------------------------------------------===//
758
759RuleMatcher::RuleMatcher(ArrayRef<SMLoc> SrcLoc)
760 : SrcLoc(SrcLoc), RuleID(NextRuleID++) {}
761
762uint64_t RuleMatcher::NextRuleID = 0;
763
764StringRef RuleMatcher::getOpcode() const {
765 return Matchers.front()->getOpcode();
766}
767
768bool RuleMatcher::recordsOperand() const {
769 return matchersRecordOperand(R: Matchers);
770}
771
772LLTCodeGen RuleMatcher::getFirstConditionAsRootType() {
773 InstructionMatcher &InsnMatcher = *Matchers.front();
774 if (!InsnMatcher.predicates_empty())
775 if (const auto *TM =
776 dyn_cast<LLTOperandMatcher>(Val: &**InsnMatcher.predicates_begin()))
777 if (TM->getInsnVarID() == 0 && TM->getOpIdx() == 0)
778 return TM->getTy();
779 return {};
780}
781
782void RuleMatcher::optimize() {
783 for (auto &Item : InsnVariableIDs) {
784 InstructionMatcher &InsnMatcher = *Item.first;
785 for (auto &OM : InsnMatcher.operands()) {
786 // Complex Patterns are usually expensive and they relatively rarely fail
787 // on their own: more often we end up throwing away all the work done by a
788 // matching part of a complex pattern because some other part of the
789 // enclosing pattern didn't match. All of this makes it beneficial to
790 // delay complex patterns until the very end of the rule matching,
791 // especially for targets having lots of complex patterns.
792 for (auto &OP : OM->predicates())
793 if (isa<ComplexPatternOperandMatcher>(Val: OP))
794 EpilogueMatchers.emplace_back(args: std::move(OP));
795 OM->eraseNullPredicates();
796 }
797 InsnMatcher.optimize();
798 }
799 llvm::sort(C&: EpilogueMatchers, Comp: [](const std::unique_ptr<PredicateMatcher> &L,
800 const std::unique_ptr<PredicateMatcher> &R) {
801 return std::tuple(L->getKind(), L->getInsnVarID(), L->getOpIdx()) <
802 std::tuple(R->getKind(), R->getInsnVarID(), R->getOpIdx());
803 });
804
805 // Deduplicate EraseInst actions, and if an EraseInst erases the root, place
806 // it at the end to favor generation of GIR_EraseRootFromParent_Done
807 DenseSet<unsigned> AlreadySeenEraseInsts;
808 auto EraseRootIt = Actions.end();
809 auto It = Actions.begin();
810 while (It != Actions.end()) {
811 if (const auto *EI = dyn_cast<EraseInstAction>(Val: It->get())) {
812 unsigned InstID = EI->getInsnID();
813 if (!AlreadySeenEraseInsts.insert(V: InstID).second) {
814 It = Actions.erase(position: It);
815 continue;
816 }
817
818 if (InstID == 0)
819 EraseRootIt = It;
820 }
821
822 ++It;
823 }
824
825 if (EraseRootIt != Actions.end())
826 Actions.splice(position: Actions.end(), x&: Actions, i: EraseRootIt);
827}
828
829bool RuleMatcher::hasFirstCondition() const {
830 if (insnmatchers_empty())
831 return false;
832 InstructionMatcher &Matcher = insnmatchers_front();
833 if (!Matcher.predicates_empty())
834 return true;
835 for (auto &OM : Matcher.operands())
836 for (auto &OP : OM->predicates())
837 if (!isa<InstructionOperandMatcher>(Val: OP))
838 return true;
839 return false;
840}
841
842const PredicateMatcher &RuleMatcher::getFirstCondition() const {
843 assert(!insnmatchers_empty() &&
844 "Trying to get a condition from an empty RuleMatcher");
845
846 InstructionMatcher &Matcher = insnmatchers_front();
847 if (!Matcher.predicates_empty())
848 return **Matcher.predicates_begin();
849 // If there is no more predicate on the instruction itself, look at its
850 // operands.
851 for (auto &OM : Matcher.operands())
852 for (auto &OP : OM->predicates())
853 if (!isa<InstructionOperandMatcher>(Val: OP))
854 return *OP;
855
856 llvm_unreachable("Trying to get a condition from an InstructionMatcher with "
857 "no conditions");
858}
859
860std::unique_ptr<PredicateMatcher> RuleMatcher::popFirstCondition() {
861 assert(!insnmatchers_empty() &&
862 "Trying to pop a condition from an empty RuleMatcher");
863
864 InstructionMatcher &Matcher = insnmatchers_front();
865 if (!Matcher.predicates_empty())
866 return Matcher.predicates_pop_front();
867 // If there is no more predicate on the instruction itself, look at its
868 // operands.
869 for (auto &OM : Matcher.operands())
870 for (auto &OP : OM->predicates())
871 if (!isa<InstructionOperandMatcher>(Val: OP)) {
872 std::unique_ptr<PredicateMatcher> Result = std::move(OP);
873 OM->eraseNullPredicates();
874 return Result;
875 }
876
877 llvm_unreachable("Trying to pop a condition from an InstructionMatcher with "
878 "no conditions");
879}
880
881GISelFlags RuleMatcher::updateGISelFlag(GISelFlags CurFlags, const Record *R,
882 StringRef FlagName,
883 GISelFlags FlagBit) {
884 // If the value of a flag is unset, ignore it.
885 // If it's set, it always takes precedence over the existing value so
886 // clear/set the corresponding bit.
887 bool Unset = false;
888 bool Value = R->getValueAsBitOrUnset(FieldName: "GIIgnoreCopies", Unset);
889 if (!Unset)
890 return Value ? (CurFlags | FlagBit) : (CurFlags & ~FlagBit);
891 return CurFlags;
892}
893
894SaveAndRestore<GISelFlags> RuleMatcher::setGISelFlags(const Record *R) {
895 if (!R || !R->isSubClassOf(Name: "GISelFlags"))
896 return {Flags, Flags};
897
898 assert((R->isSubClassOf("PatFrags") || R->isSubClassOf("Pattern")) &&
899 "GISelFlags is only expected on Pattern/PatFrags!");
900
901 GISelFlags NewFlags =
902 updateGISelFlag(CurFlags: Flags, R, FlagName: "GIIgnoreCopies", FlagBit: GISF_IgnoreCopies);
903 return {Flags, NewFlags};
904}
905
906Error RuleMatcher::defineComplexSubOperand(StringRef SymbolicName,
907 const Record *ComplexPattern,
908 unsigned RendererID,
909 unsigned SubOperandID,
910 StringRef ParentSymbolicName) {
911 std::string ParentName(ParentSymbolicName);
912 auto [It, Inserted] = ComplexSubOperands.try_emplace(
913 Key: SymbolicName, Args&: ComplexPattern, Args&: RendererID, Args&: SubOperandID);
914 if (!Inserted) {
915 const std::string &RecordedParentName =
916 ComplexSubOperandsParentName[SymbolicName];
917 if (RecordedParentName != ParentName)
918 return failUnsupported(Reason: "Error: Complex suboperand " + SymbolicName +
919 " referenced by different operands: " +
920 RecordedParentName + " and " + ParentName + ".");
921 // Complex suboperand referenced more than once from same the operand is
922 // used to generate 'same operand check'. Emitting of
923 // GIR_ComplexSubOperandRenderer for them is already handled.
924 return Error::success();
925 }
926
927 ComplexSubOperandsParentName[SymbolicName] = std::move(ParentName);
928
929 return Error::success();
930}
931
932InstructionMatcher &RuleMatcher::addInstructionMatcher(StringRef SymbolicName) {
933 Matchers.emplace_back(args: new InstructionMatcher(*this, SymbolicName));
934 MutatableInsns.insert(Ptr: Matchers.back().get());
935 return *Matchers.back();
936}
937
938void RuleMatcher::addRequiredSimplePredicate(StringRef PredName) {
939 RequiredSimplePredicates.push_back(x: PredName.str());
940}
941
942const std::vector<std::string> &RuleMatcher::getRequiredSimplePredicates() {
943 return RequiredSimplePredicates;
944}
945
946unsigned RuleMatcher::implicitlyDefineInsnVar(InstructionMatcher &Matcher) {
947 unsigned NewInsnVarID = NextInsnVarID++;
948 InsnVariableIDs[&Matcher] = NewInsnVarID;
949 return NewInsnVarID;
950}
951
952unsigned RuleMatcher::getInsnVarID(InstructionMatcher &InsnMatcher) const {
953 const auto &I = InsnVariableIDs.find(x: &InsnMatcher);
954 if (I != InsnVariableIDs.end())
955 return I->second;
956 llvm_unreachable("Matched Insn was not captured in a local variable");
957}
958
959void RuleMatcher::defineOperand(StringRef SymbolicName, OperandMatcher &OM) {
960 if (DefinedOperands.try_emplace(Key: SymbolicName, Args: &OM).second)
961 return;
962
963 // If the operand is already defined, then we must ensure both references in
964 // the matcher have the exact same node.
965 RuleMatcher &RM = OM.getInstructionMatcher().getRuleMatcher();
966 OM.addPredicate<SameOperandMatcher>(
967 args: OM.getSymbolicName(), args: getOperandMatcher(Name: OM.getSymbolicName()).getOpIdx(),
968 args: RM.getGISelFlags());
969}
970
971void RuleMatcher::definePhysRegOperand(const Record *Reg, OperandMatcher &OM) {
972 PhysRegOperands.try_emplace(Key: Reg, Args: &OM);
973}
974
975InstructionMatcher &
976RuleMatcher::getInstructionMatcher(StringRef SymbolicName) const {
977 for (const auto &I : InsnVariableIDs)
978 if (I.first->getSymbolicName() == SymbolicName)
979 return *I.first;
980 llvm_unreachable(
981 ("Failed to lookup instruction " + SymbolicName).str().c_str());
982}
983
984const OperandMatcher &
985RuleMatcher::getPhysRegOperandMatcher(const Record *Reg) const {
986 const auto &I = PhysRegOperands.find(Key: Reg);
987
988 if (I == PhysRegOperands.end()) {
989 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Register " + Reg->getName() +
990 " was not declared in matcher");
991 }
992
993 return *I->second;
994}
995
996OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) {
997 const auto &I = DefinedOperands.find(Key: Name);
998
999 if (I == DefinedOperands.end())
1000 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Operand " + Name + " was not declared in matcher");
1001
1002 return *I->second;
1003}
1004
1005const OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) const {
1006 const auto &I = DefinedOperands.find(Key: Name);
1007
1008 if (I == DefinedOperands.end())
1009 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Operand " + Name + " was not declared in matcher");
1010
1011 return *I->second;
1012}
1013
1014void RuleMatcher::emit(MatchTable &Table) {
1015 if (Matchers.empty())
1016 llvm_unreachable("Unexpected empty matcher!");
1017
1018 // The representation supports rules that require multiple roots such as:
1019 // %ptr(p0) = ...
1020 // %elt0(s32) = G_LOAD %ptr
1021 // %1(p0) = G_ADD %ptr, 4
1022 // %elt1(s32) = G_LOAD p0 %1
1023 // which could be usefully folded into:
1024 // %ptr(p0) = ...
1025 // %elt0(s32), %elt1(s32) = TGT_LOAD_PAIR %ptr
1026 // on some targets but we don't need to make use of that yet.
1027 assert(Matchers.size() == 1 && "Cannot handle multi-root matchers yet");
1028
1029 unsigned LabelID = Table.allocateLabelID();
1030 Table << MatchTable::Opcode(Opcode: "GIM_Try", IndentAdjust: +1)
1031 << MatchTable::Comment(Comment: "On fail goto")
1032 << MatchTable::JumpTarget(LabelID)
1033 << MatchTable::Comment(Comment: ("Rule ID " + Twine(RuleID) + " //").str())
1034 << MatchTable::LineBreak;
1035
1036 if (!RequiredFeatures.empty() || HwModeIdx >= 0) {
1037 Table << MatchTable::Opcode(Opcode: "GIM_CheckFeatures")
1038 << MatchTable::NamedValue(
1039 NumBytes: 2, NamedValue: getNameForFeatureBitset(FeatureBitset: RequiredFeatures, HwModeIdx))
1040 << MatchTable::LineBreak;
1041 }
1042
1043 if (!RequiredSimplePredicates.empty()) {
1044 for (const auto &Pred : RequiredSimplePredicates) {
1045 Table << MatchTable::Opcode(Opcode: "GIM_CheckSimplePredicate")
1046 << MatchTable::NamedValue(NumBytes: 2, NamedValue: Pred) << MatchTable::LineBreak;
1047 }
1048 }
1049
1050 Matchers.front()->emitPredicateOpcodes(Table, Rule&: *this);
1051
1052 // Check if it's safe to replace registers.
1053 for (const auto &MA : Actions)
1054 MA->emitAdditionalPredicates(Table, Rule&: *this);
1055
1056 // We must also check if it's safe to fold the matched instructions.
1057 if (InsnVariableIDs.size() >= 2) {
1058
1059 // FIXME: Emit checks to determine it's _actually_ safe to fold and/or
1060 // account for unsafe cases.
1061 //
1062 // Example:
1063 // MI1--> %0 = ...
1064 // %1 = ... %0
1065 // MI0--> %2 = ... %0
1066 // It's not safe to erase MI1. We currently handle this by not
1067 // erasing %0 (even when it's dead).
1068 //
1069 // Example:
1070 // MI1--> %0 = load volatile @a
1071 // %1 = load volatile @a
1072 // MI0--> %2 = ... %0
1073 // It's not safe to sink %0's def past %1. We currently handle
1074 // this by rejecting all loads.
1075 //
1076 // Example:
1077 // MI1--> %0 = load @a
1078 // %1 = store @a
1079 // MI0--> %2 = ... %0
1080 // It's not safe to sink %0's def past %1. We currently handle
1081 // this by rejecting all loads.
1082 //
1083 // Example:
1084 // G_CONDBR %cond, @BB1
1085 // BB0:
1086 // MI1--> %0 = load @a
1087 // G_BR @BB1
1088 // BB1:
1089 // MI0--> %2 = ... %0
1090 // It's not always safe to sink %0 across control flow. In this
1091 // case it may introduce a memory fault. We currentl handle
1092 // this by rejecting all loads.
1093
1094 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsSafeToFold")
1095 << MatchTable::Comment(Comment: "NumInsns")
1096 << MatchTable::IntValue(NumBytes: 1, IntValue: InsnVariableIDs.size() - 1)
1097 << MatchTable::LineBreak;
1098 }
1099
1100 for (const auto &PM : EpilogueMatchers)
1101 PM->emitPredicateOpcodes(Table, Rule&: *this);
1102
1103 if (!CustomCXXAction.empty()) {
1104 /// Handle combiners relying on custom C++ code instead of actions.
1105 assert(Table.isCombiner() && "CustomCXXAction is only for combiners!");
1106 // We cannot have actions other than debug comments.
1107 assert(none_of(Actions, [](auto &A) {
1108 return A->getKind() != MatchAction::AK_DebugComment;
1109 }));
1110 for (const auto &MA : Actions)
1111 MA->emitActionOpcodes(Table, Rule&: *this);
1112 Table << MatchTable::Opcode(Opcode: "GIR_DoneWithCustomAction", IndentAdjust: -1)
1113 << MatchTable::Comment(Comment: "Fn")
1114 << MatchTable::NamedValue(NumBytes: 2, NamedValue: CustomCXXAction)
1115 << MatchTable::LineBreak;
1116 } else {
1117 // Emit all actions except the last one, then emit coverage and emit the
1118 // final action.
1119 //
1120 // This is because some actions, such as GIR_EraseRootFromParent_Done, also
1121 // double as a GIR_Done and terminate execution of the rule.
1122 if (!Actions.empty()) {
1123 for (const auto &MA : drop_end(RangeOrContainer&: Actions))
1124 MA->emitActionOpcodes(Table, Rule&: *this);
1125 }
1126
1127 assert((Table.isWithCoverage() ? !Table.isCombiner() : true) &&
1128 "Combiner tables don't support coverage!");
1129 if (Table.isWithCoverage())
1130 Table << MatchTable::Opcode(Opcode: "GIR_Coverage")
1131 << MatchTable::IntValue(NumBytes: 4, IntValue: RuleID) << MatchTable::LineBreak;
1132 else if (!Table.isCombiner())
1133 Table << MatchTable::Comment(
1134 Comment: ("GIR_Coverage, " + Twine(RuleID) + ",").str())
1135 << MatchTable::LineBreak;
1136
1137 if (Actions.empty() ||
1138 !Actions.back()->emitActionOpcodesAndDone(Table, Rule&: *this)) {
1139 Table << MatchTable::Opcode(Opcode: "GIR_Done", IndentAdjust: -1) << MatchTable::LineBreak;
1140 }
1141 }
1142
1143 Table << MatchTable::Label(LabelID);
1144 ++NumPatternEmitted;
1145}
1146
1147bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const {
1148 // Rules involving more match roots have higher priority.
1149 if (Matchers.size() > B.Matchers.size())
1150 return true;
1151 if (Matchers.size() < B.Matchers.size())
1152 return false;
1153
1154 for (auto Matcher : zip(t: Matchers, u: B.Matchers)) {
1155 if (std::get<0>(t&: Matcher)->isHigherPriorityThan(B&: *std::get<1>(t&: Matcher)))
1156 return true;
1157 if (std::get<1>(t&: Matcher)->isHigherPriorityThan(B&: *std::get<0>(t&: Matcher)))
1158 return false;
1159 }
1160
1161 return false;
1162}
1163
1164unsigned RuleMatcher::countRendererFns() const {
1165 return std::accumulate(
1166 first: Matchers.begin(), last: Matchers.end(), init: 0,
1167 binary_op: [](unsigned A, const std::unique_ptr<InstructionMatcher> &Matcher) {
1168 return A + Matcher->countRendererFns();
1169 });
1170}
1171
1172void RuleMatcher::insnmatchers_pop_front() { Matchers.erase(position: Matchers.begin()); }
1173
1174//===- PredicateMatcher ---------------------------------------------------===//
1175
1176PredicateMatcher::~PredicateMatcher() = default;
1177
1178//===- OperandPredicateMatcher --------------------------------------------===//
1179
1180OperandPredicateMatcher::~OperandPredicateMatcher() = default;
1181
1182bool OperandPredicateMatcher::isHigherPriorityThan(
1183 const OperandPredicateMatcher &B) const {
1184 // Generally speaking, an instruction is more important than an Int or a
1185 // LiteralInt because it can cover more nodes but there's an exception to
1186 // this. G_CONSTANT's are less important than either of those two because they
1187 // are more permissive.
1188
1189 const auto *AOM = dyn_cast<InstructionOperandMatcher>(Val: this);
1190 const auto *BOM = dyn_cast<InstructionOperandMatcher>(Val: &B);
1191 bool AIsConstantInsn = AOM && AOM->getInsnMatcher().isConstantInstruction();
1192 bool BIsConstantInsn = BOM && BOM->getInsnMatcher().isConstantInstruction();
1193
1194 // The relative priorities between a G_CONSTANT and any other instruction
1195 // don't actually matter but this code is needed to ensure a strict weak
1196 // ordering. This is particularly important on Windows where the rules will
1197 // be incorrectly sorted without it.
1198 if (AOM && BOM)
1199 return !AIsConstantInsn && BIsConstantInsn;
1200
1201 if (AIsConstantInsn && (B.Kind == OPM_Int || B.Kind == OPM_LiteralInt))
1202 return false;
1203 if (BIsConstantInsn && (Kind == OPM_Int || Kind == OPM_LiteralInt))
1204 return true;
1205
1206 return Kind < B.Kind;
1207}
1208
1209//===- SameOperandMatcher -------------------------------------------------===//
1210
1211void SameOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1212 RuleMatcher &Rule) const {
1213 const OperandMatcher &OtherOM = Rule.getOperandMatcher(Name: MatchingName);
1214 unsigned OtherInsnVarID = Rule.getInsnVarID(InsnMatcher&: OtherOM.getInstructionMatcher());
1215 assert(OtherInsnVarID == OtherOM.getInstructionMatcher().getInsnVarID());
1216 const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
1217 Table << MatchTable::Opcode(Opcode: IgnoreCopies
1218 ? "GIM_CheckIsSameOperandIgnoreCopies"
1219 : "GIM_CheckIsSameOperand")
1220 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1221 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OpIdx)
1222 << MatchTable::Comment(Comment: "OtherMI")
1223 << MatchTable::ULEB128Value(IntValue: OtherInsnVarID)
1224 << MatchTable::Comment(Comment: "OtherOpIdx")
1225 << MatchTable::ULEB128Value(IntValue: OtherOM.getOpIdx())
1226 << MatchTable::LineBreak;
1227}
1228
1229//===- LLTOperandMatcher --------------------------------------------------===//
1230
1231std::map<LLTCodeGen, unsigned> LLTOperandMatcher::TypeIDValues;
1232
1233RecordAndValue LLTOperandMatcher::getValue() const {
1234 const auto VI = TypeIDValues.find(x: Ty);
1235 if (VI == TypeIDValues.end())
1236 return MatchTable::NamedValue(NumBytes: 1, NamedValue: getTy().getCxxEnumValue());
1237 return {MatchTable::NamedValue(NumBytes: 1, NamedValue: getTy().getCxxEnumValue()), VI->second};
1238}
1239
1240bool LLTOperandMatcher::hasValue() const {
1241 if (TypeIDValues.size() != KnownTypes.size())
1242 initTypeIDValuesMap();
1243 return TypeIDValues.count(x: Ty);
1244}
1245
1246void LLTOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1247 RuleMatcher &Rule) const {
1248 if (InsnVarID == 0) {
1249 Table << MatchTable::Opcode(Opcode: "GIM_RootCheckType");
1250 } else {
1251 Table << MatchTable::Opcode(Opcode: "GIM_CheckType") << MatchTable::Comment(Comment: "MI")
1252 << MatchTable::ULEB128Value(IntValue: InsnVarID);
1253 }
1254 Table << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1255 << MatchTable::Comment(Comment: "Type") << getValue().Record
1256 << MatchTable::LineBreak;
1257}
1258
1259//===- PointerToAnyOperandMatcher -----------------------------------------===//
1260
1261void PointerToAnyOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1262 RuleMatcher &Rule) const {
1263 Table << MatchTable::Opcode(Opcode: "GIM_CheckPointerToAny")
1264 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1265 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1266 << MatchTable::Comment(Comment: "SizeInBits")
1267 << MatchTable::ULEB128Value(IntValue: SizeInBits) << MatchTable::LineBreak;
1268}
1269
1270//===- RecordNamedOperandMatcher ------------------------------------------===//
1271
1272void RecordNamedOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1273 RuleMatcher &Rule) const {
1274 Table << MatchTable::Opcode(Opcode: "GIM_RecordNamedOperand")
1275 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1276 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1277 << MatchTable::Comment(Comment: "StoreIdx") << MatchTable::ULEB128Value(IntValue: StoreIdx)
1278 << MatchTable::Comment(Comment: "Name : " + Name) << MatchTable::LineBreak;
1279}
1280
1281//===- RecordRegisterType ------------------------------------------===//
1282
1283void RecordRegisterType::emitPredicateOpcodes(MatchTable &Table,
1284 RuleMatcher &Rule) const {
1285 assert(Idx < 0 && "Temp types always have negative indexes!");
1286 Table << MatchTable::Opcode(Opcode: "GIM_RecordRegType") << MatchTable::Comment(Comment: "MI")
1287 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
1288 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::Comment(Comment: "TempTypeIdx")
1289 << MatchTable::IntValue(NumBytes: 1, IntValue: Idx) << MatchTable::LineBreak;
1290}
1291
1292//===- ComplexPatternOperandMatcher ---------------------------------------===//
1293
1294void ComplexPatternOperandMatcher::emitPredicateOpcodes(
1295 MatchTable &Table, RuleMatcher &Rule) const {
1296 unsigned ID = getAllocatedTemporariesBaseID();
1297 Table << MatchTable::Opcode(Opcode: "GIM_CheckComplexPattern")
1298 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1299 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1300 << MatchTable::Comment(Comment: "Renderer") << MatchTable::IntValue(NumBytes: 2, IntValue: ID)
1301 << MatchTable::NamedValue(NumBytes: 2, NamedValue: ("GICP_" + TheDef.getName()).str())
1302 << MatchTable::LineBreak;
1303}
1304
1305unsigned ComplexPatternOperandMatcher::getAllocatedTemporariesBaseID() const {
1306 return Operand.getAllocatedTemporariesBaseID();
1307}
1308
1309//===- RegisterBankOperandMatcher -----------------------------------------===//
1310
1311bool RegisterBankOperandMatcher::isIdentical(const PredicateMatcher &B) const {
1312 return OperandPredicateMatcher::isIdentical(B) &&
1313 RC.getDef() == cast<RegisterBankOperandMatcher>(Val: &B)->RC.getDef();
1314}
1315
1316void RegisterBankOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1317 RuleMatcher &Rule) const {
1318 if (InsnVarID == 0) {
1319 Table << MatchTable::Opcode(Opcode: "GIM_RootCheckRegBankForClass");
1320 } else {
1321 Table << MatchTable::Opcode(Opcode: "GIM_CheckRegBankForClass")
1322 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID);
1323 }
1324
1325 Table << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1326 << MatchTable::Comment(Comment: "RC")
1327 << MatchTable::NamedValue(NumBytes: 2, NamedValue: RC.getQualifiedIdName())
1328 << MatchTable::LineBreak;
1329}
1330
1331//===- MBBOperandMatcher --------------------------------------------------===//
1332
1333void MBBOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1334 RuleMatcher &Rule) const {
1335 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsMBB") << MatchTable::Comment(Comment: "MI")
1336 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
1337 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
1338}
1339
1340//===- ImmOperandMatcher --------------------------------------------------===//
1341
1342void ImmOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1343 RuleMatcher &Rule) const {
1344 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsImm") << MatchTable::Comment(Comment: "MI")
1345 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
1346 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
1347}
1348
1349//===- ConstantIntOperandMatcher ------------------------------------------===//
1350
1351void ConstantIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1352 RuleMatcher &Rule) const {
1353 const bool IsInt8 = isInt<8>(x: Value);
1354 Table << MatchTable::Opcode(Opcode: IsInt8 ? "GIM_CheckConstantInt8"
1355 : "GIM_CheckConstantInt")
1356 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1357 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1358 << MatchTable::IntValue(NumBytes: IsInt8 ? 1 : 8, IntValue: Value) << MatchTable::LineBreak;
1359}
1360
1361//===- LiteralIntOperandMatcher -------------------------------------------===//
1362
1363void LiteralIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1364 RuleMatcher &Rule) const {
1365 Table << MatchTable::Opcode(Opcode: "GIM_CheckLiteralInt")
1366 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1367 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1368 << MatchTable::IntValue(NumBytes: 8, IntValue: Value) << MatchTable::LineBreak;
1369}
1370
1371//===- CmpPredicateOperandMatcher -----------------------------------------===//
1372
1373void CmpPredicateOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1374 RuleMatcher &Rule) const {
1375 Table << MatchTable::Opcode(Opcode: "GIM_CheckCmpPredicate")
1376 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1377 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1378 << MatchTable::Comment(Comment: "Predicate")
1379 << MatchTable::NamedValue(NumBytes: 2, Namespace: "CmpInst", NamedValue: PredName)
1380 << MatchTable::LineBreak;
1381}
1382
1383//===- IntrinsicIDOperandMatcher ------------------------------------------===//
1384
1385void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1386 RuleMatcher &Rule) const {
1387 Table << MatchTable::Opcode(Opcode: "GIM_CheckIntrinsicID")
1388 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1389 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1390 << MatchTable::NamedValue(NumBytes: 2, NamedValue: "Intrinsic::" + II->EnumName.str())
1391 << MatchTable::LineBreak;
1392}
1393
1394//===- OperandImmPredicateMatcher -----------------------------------------===//
1395
1396void OperandImmPredicateMatcher::emitPredicateOpcodes(MatchTable &Table,
1397 RuleMatcher &Rule) const {
1398 Table << MatchTable::Opcode(Opcode: "GIM_CheckImmOperandPredicate")
1399 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1400 << MatchTable::Comment(Comment: "MO") << MatchTable::ULEB128Value(IntValue: OpIdx)
1401 << MatchTable::Comment(Comment: "Predicate")
1402 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1403 << MatchTable::LineBreak;
1404}
1405
1406//===- OperandLeafPredicateMatcher
1407//-----------------------------------------===//
1408
1409void OperandLeafPredicateMatcher::emitPredicateOpcodes(
1410 MatchTable &Table, RuleMatcher &Rule) const {
1411 Table << MatchTable::Opcode(Opcode: "GIM_CheckLeafOperandPredicate")
1412 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1413 << MatchTable::Comment(Comment: "MO") << MatchTable::ULEB128Value(IntValue: OpIdx)
1414 << MatchTable::Comment(Comment: "Predicate")
1415 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1416 << MatchTable::LineBreak;
1417}
1418
1419//===- OperandMatcher -----------------------------------------------------===//
1420
1421std::string OperandMatcher::getOperandExpr(unsigned InsnVarID) const {
1422 return "State.MIs[" + llvm::to_string(Value: InsnVarID) + "]->getOperand(" +
1423 llvm::to_string(Value: OpIdx) + ")";
1424}
1425
1426unsigned OperandMatcher::getInsnVarID() const { return Insn.getInsnVarID(); }
1427
1428TempTypeIdx OperandMatcher::getTempTypeIdx(RuleMatcher &Rule) {
1429 assert(!IsVariadic && "Cannot use this on variadic operands!");
1430 if (TTIdx >= 0) {
1431 // Temp type index not assigned yet, so assign one and add the necessary
1432 // predicate.
1433 TTIdx = Rule.getNextTempTypeIdx();
1434 assert(TTIdx < 0);
1435 addPredicate<RecordRegisterType>(args&: TTIdx);
1436 return TTIdx;
1437 }
1438 return TTIdx;
1439}
1440
1441bool OperandMatcher::recordsOperand() const {
1442 return matchersRecordOperand(R: Predicates);
1443}
1444
1445void OperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1446 RuleMatcher &Rule) {
1447 if (!Optimized) {
1448 std::string Comment;
1449 raw_string_ostream CommentOS(Comment);
1450 CommentOS << "MIs[" << getInsnVarID() << "] ";
1451 if (SymbolicName.empty())
1452 CommentOS << "Operand " << OpIdx;
1453 else
1454 CommentOS << SymbolicName;
1455 Table << MatchTable::Comment(Comment) << MatchTable::LineBreak;
1456 }
1457
1458 emitPredicateListOpcodes(Table, args&: Rule);
1459}
1460
1461bool OperandMatcher::isHigherPriorityThan(OperandMatcher &B) {
1462 // Operand matchers involving more predicates have higher priority.
1463 if (predicates_size() > B.predicates_size())
1464 return true;
1465 if (predicates_size() < B.predicates_size())
1466 return false;
1467
1468 // This assumes that predicates are added in a consistent order.
1469 for (auto &&Predicate : zip(t: predicates(), u: B.predicates())) {
1470 if (std::get<0>(t&: Predicate)->isHigherPriorityThan(B: *std::get<1>(t&: Predicate)))
1471 return true;
1472 if (std::get<1>(t&: Predicate)->isHigherPriorityThan(B: *std::get<0>(t&: Predicate)))
1473 return false;
1474 }
1475
1476 return false;
1477}
1478
1479unsigned OperandMatcher::countRendererFns() {
1480 return std::accumulate(
1481 first: predicates().begin(), last: predicates().end(), init: 0,
1482 binary_op: [](unsigned A,
1483 const std::unique_ptr<OperandPredicateMatcher> &Predicate) {
1484 return A + Predicate->countRendererFns();
1485 });
1486}
1487
1488Error OperandMatcher::addTypeCheckPredicate(const TypeSetByHwMode &VTy,
1489 bool OperandIsAPointer) {
1490 if (!VTy.isMachineValueType())
1491 return failUnsupported(Reason: "unsupported typeset");
1492
1493 if ((VTy.getMachineValueType() == MVT::iPTR ||
1494 VTy.getMachineValueType() == MVT::cPTR) &&
1495 OperandIsAPointer) {
1496 addPredicate<PointerToAnyOperandMatcher>(args: 0);
1497 return Error::success();
1498 }
1499
1500 auto OpTyOrNone = MVTToLLT(VT: VTy.getMachineValueType().SimpleTy);
1501 if (!OpTyOrNone)
1502 return failUnsupported(Reason: "unsupported type");
1503
1504 if (OperandIsAPointer)
1505 addPredicate<PointerToAnyOperandMatcher>(args: OpTyOrNone->get().getSizeInBits());
1506 else if (VTy.isPointer())
1507 addPredicate<LLTOperandMatcher>(
1508 args: LLT::pointer(AddressSpace: VTy.getPtrAddrSpace(), SizeInBits: OpTyOrNone->get().getSizeInBits()));
1509 else
1510 addPredicate<LLTOperandMatcher>(args&: *OpTyOrNone);
1511 return Error::success();
1512}
1513
1514//===- InstructionOpcodeMatcher -------------------------------------------===//
1515
1516DenseMap<const CodeGenInstruction *, unsigned>
1517 InstructionOpcodeMatcher::OpcodeValues;
1518
1519RecordAndValue
1520InstructionOpcodeMatcher::getInstValue(const CodeGenInstruction *I) const {
1521 const auto VI = OpcodeValues.find(Val: I);
1522 if (VI != OpcodeValues.end())
1523 return {MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName()), VI->second};
1524 return MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName());
1525}
1526
1527void InstructionOpcodeMatcher::initOpcodeValuesMap(
1528 const CodeGenTarget &Target) {
1529 OpcodeValues.clear();
1530
1531 for (const CodeGenInstruction *I : Target.getInstructions())
1532 OpcodeValues[I] = Target.getInstrIntValue(R: I->TheDef);
1533}
1534
1535RecordAndValue InstructionOpcodeMatcher::getValue() const {
1536 assert(Insts.size() == 1);
1537
1538 const CodeGenInstruction *I = Insts[0];
1539 const auto VI = OpcodeValues.find(Val: I);
1540 if (VI != OpcodeValues.end())
1541 return {MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName()), VI->second};
1542 return MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName());
1543}
1544
1545void InstructionOpcodeMatcher::emitPredicateOpcodes(MatchTable &Table,
1546 RuleMatcher &Rule) const {
1547 StringRef CheckType =
1548 Insts.size() == 1 ? "GIM_CheckOpcode" : "GIM_CheckOpcodeIsEither";
1549 Table << MatchTable::Opcode(Opcode: CheckType) << MatchTable::Comment(Comment: "MI")
1550 << MatchTable::ULEB128Value(IntValue: InsnVarID);
1551
1552 for (const CodeGenInstruction *I : Insts)
1553 Table << getInstValue(I).Record;
1554 Table << MatchTable::LineBreak;
1555}
1556
1557bool InstructionOpcodeMatcher::isHigherPriorityThan(
1558 const InstructionPredicateMatcher &B) const {
1559 if (InstructionPredicateMatcher::isHigherPriorityThan(B))
1560 return true;
1561 if (B.InstructionPredicateMatcher::isHigherPriorityThan(B: *this))
1562 return false;
1563
1564 // Prioritize opcodes for cosmetic reasons in the generated source. Although
1565 // this is cosmetic at the moment, we may want to drive a similar ordering
1566 // using instruction frequency information to improve compile time.
1567 if (const InstructionOpcodeMatcher *BO =
1568 dyn_cast<InstructionOpcodeMatcher>(Val: &B))
1569 return Insts[0]->getName() < BO->Insts[0]->getName();
1570
1571 return false;
1572}
1573
1574bool InstructionOpcodeMatcher::isConstantInstruction() const {
1575 return Insts.size() == 1 && Insts[0]->getName() == "G_CONSTANT";
1576}
1577
1578StringRef InstructionOpcodeMatcher::getOpcode() const {
1579 return Insts[0]->getName();
1580}
1581
1582bool InstructionOpcodeMatcher::isVariadicNumOperands() const {
1583 // If one is variadic, they all should be.
1584 return Insts[0]->Operands.isVariadic;
1585}
1586
1587StringRef InstructionOpcodeMatcher::getOperandType(unsigned OpIdx) const {
1588 // Types expected to be uniform for all alternatives.
1589 return Insts[0]->Operands[OpIdx].OperandType;
1590}
1591
1592//===- InstructionNumOperandsMatcher --------------------------------------===//
1593
1594void InstructionNumOperandsMatcher::emitPredicateOpcodes(
1595 MatchTable &Table, RuleMatcher &Rule) const {
1596 StringRef Opc;
1597 switch (CK) {
1598 case CheckKind::Eq:
1599 Opc = "GIM_CheckNumOperands";
1600 break;
1601 case CheckKind::GE:
1602 Opc = "GIM_CheckNumOperandsGE";
1603 break;
1604 case CheckKind::LE:
1605 Opc = "GIM_CheckNumOperandsLE";
1606 break;
1607 }
1608 Table << MatchTable::Opcode(Opcode: Opc) << MatchTable::Comment(Comment: "MI")
1609 << MatchTable::ULEB128Value(IntValue: InsnVarID)
1610 << MatchTable::Comment(Comment: "Expected")
1611 << MatchTable::ULEB128Value(IntValue: NumOperands) << MatchTable::LineBreak;
1612}
1613
1614//===- InstructionImmPredicateMatcher -------------------------------------===//
1615
1616bool InstructionImmPredicateMatcher::isIdentical(
1617 const PredicateMatcher &B) const {
1618 return InstructionPredicateMatcher::isIdentical(B) &&
1619 Predicate.getOrigPatFragRecord() ==
1620 cast<InstructionImmPredicateMatcher>(Val: &B)
1621 ->Predicate.getOrigPatFragRecord();
1622}
1623
1624void InstructionImmPredicateMatcher::emitPredicateOpcodes(
1625 MatchTable &Table, RuleMatcher &Rule) const {
1626 Table << MatchTable::Opcode(Opcode: getMatchOpcodeForImmPredicate(Predicate))
1627 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1628 << MatchTable::Comment(Comment: "Predicate")
1629 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1630 << MatchTable::LineBreak;
1631}
1632
1633//===- AtomicOrderingMMOPredicateMatcher ----------------------------------===//
1634
1635bool AtomicOrderingMMOPredicateMatcher::isIdentical(
1636 const PredicateMatcher &B) const {
1637 if (!InstructionPredicateMatcher::isIdentical(B))
1638 return false;
1639 const auto &R = *cast<AtomicOrderingMMOPredicateMatcher>(Val: &B);
1640 return Order == R.Order && Comparator == R.Comparator;
1641}
1642
1643void AtomicOrderingMMOPredicateMatcher::emitPredicateOpcodes(
1644 MatchTable &Table, RuleMatcher &Rule) const {
1645 StringRef Opcode = "GIM_CheckAtomicOrdering";
1646
1647 if (Comparator == AO_OrStronger)
1648 Opcode = "GIM_CheckAtomicOrderingOrStrongerThan";
1649 if (Comparator == AO_WeakerThan)
1650 Opcode = "GIM_CheckAtomicOrderingWeakerThan";
1651
1652 Table << MatchTable::Opcode(Opcode) << MatchTable::Comment(Comment: "MI")
1653 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Order")
1654 << MatchTable::NamedValue(NumBytes: 1,
1655 NamedValue: ("(uint8_t)AtomicOrdering::" + Order).str())
1656 << MatchTable::LineBreak;
1657}
1658
1659//===- MemorySizePredicateMatcher -----------------------------------------===//
1660
1661void MemorySizePredicateMatcher::emitPredicateOpcodes(MatchTable &Table,
1662 RuleMatcher &Rule) const {
1663 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemorySizeEqualTo")
1664 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1665 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1666 << MatchTable::Comment(Comment: "Size") << MatchTable::IntValue(NumBytes: 4, IntValue: Size)
1667 << MatchTable::LineBreak;
1668}
1669
1670//===- MemoryAddressSpacePredicateMatcher ---------------------------------===//
1671
1672bool MemoryAddressSpacePredicateMatcher::isIdentical(
1673 const PredicateMatcher &B) const {
1674 if (!InstructionPredicateMatcher::isIdentical(B))
1675 return false;
1676 auto *Other = cast<MemoryAddressSpacePredicateMatcher>(Val: &B);
1677 return MMOIdx == Other->MMOIdx && AddrSpaces == Other->AddrSpaces;
1678}
1679
1680void MemoryAddressSpacePredicateMatcher::emitPredicateOpcodes(
1681 MatchTable &Table, RuleMatcher &Rule) const {
1682 assert(AddrSpaces.size() < 256);
1683 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemoryAddressSpace")
1684 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1685 << MatchTable::Comment(Comment: "MMO")
1686 << MatchTable::ULEB128Value(IntValue: MMOIdx)
1687 // Encode number of address spaces to expect.
1688 << MatchTable::Comment(Comment: "NumAddrSpace")
1689 << MatchTable::IntValue(NumBytes: 1, IntValue: AddrSpaces.size());
1690 for (unsigned AS : AddrSpaces)
1691 Table << MatchTable::Comment(Comment: "AddrSpace") << MatchTable::ULEB128Value(IntValue: AS);
1692
1693 Table << MatchTable::LineBreak;
1694}
1695
1696//===- MemoryAlignmentPredicateMatcher ------------------------------------===//
1697
1698bool MemoryAlignmentPredicateMatcher::isIdentical(
1699 const PredicateMatcher &B) const {
1700 if (!InstructionPredicateMatcher::isIdentical(B))
1701 return false;
1702 auto *Other = cast<MemoryAlignmentPredicateMatcher>(Val: &B);
1703 return MMOIdx == Other->MMOIdx && MinAlign == Other->MinAlign;
1704}
1705
1706void MemoryAlignmentPredicateMatcher::emitPredicateOpcodes(
1707 MatchTable &Table, RuleMatcher &Rule) const {
1708 // TODO: we could support more, just need to emit the right opcode or switch
1709 // to log alignment.
1710 assert(MinAlign < 256);
1711 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemoryAlignment")
1712 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1713 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1714 << MatchTable::Comment(Comment: "MinAlign") << MatchTable::IntValue(NumBytes: 1, IntValue: MinAlign)
1715 << MatchTable::LineBreak;
1716}
1717
1718//===- MemoryVsLLTSizePredicateMatcher ------------------------------------===//
1719
1720bool MemoryVsLLTSizePredicateMatcher::isIdentical(
1721 const PredicateMatcher &B) const {
1722 return InstructionPredicateMatcher::isIdentical(B) &&
1723 MMOIdx == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->MMOIdx &&
1724 Relation == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->Relation &&
1725 OpIdx == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->OpIdx;
1726}
1727
1728void MemoryVsLLTSizePredicateMatcher::emitPredicateOpcodes(
1729 MatchTable &Table, RuleMatcher &Rule) const {
1730 Table << MatchTable::Opcode(
1731 Opcode: Relation == EqualTo ? "GIM_CheckMemorySizeEqualToLLT"
1732 : Relation == GreaterThan ? "GIM_CheckMemorySizeGreaterThanLLT"
1733 : "GIM_CheckMemorySizeLessThanLLT")
1734 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1735 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1736 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OpIdx)
1737 << MatchTable::LineBreak;
1738}
1739
1740//===- VectorSplatImmPredicateMatcher -------------------------------------===//
1741
1742void VectorSplatImmPredicateMatcher::emitPredicateOpcodes(
1743 MatchTable &Table, RuleMatcher &Rule) const {
1744 if (Kind == AllOnes)
1745 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsBuildVectorAllOnes");
1746 else
1747 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsBuildVectorAllZeros");
1748
1749 Table << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID);
1750 Table << MatchTable::LineBreak;
1751}
1752
1753//===- GenericInstructionPredicateMatcher ---------------------------------===//
1754
1755GenericInstructionPredicateMatcher::GenericInstructionPredicateMatcher(
1756 unsigned InsnVarID, TreePredicateFn Predicate)
1757 : GenericInstructionPredicateMatcher(InsnVarID,
1758 getEnumNameForPredicate(Predicate)) {}
1759
1760bool GenericInstructionPredicateMatcher::isIdentical(
1761 const PredicateMatcher &B) const {
1762 return InstructionPredicateMatcher::isIdentical(B) &&
1763 EnumVal ==
1764 static_cast<const GenericInstructionPredicateMatcher &>(B).EnumVal;
1765}
1766void GenericInstructionPredicateMatcher::emitPredicateOpcodes(
1767 MatchTable &Table, RuleMatcher &Rule) const {
1768 Table << MatchTable::Opcode(Opcode: "GIM_CheckCxxInsnPredicate")
1769 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1770 << MatchTable::Comment(Comment: "FnId") << MatchTable::NamedValue(NumBytes: 2, NamedValue: EnumVal)
1771 << MatchTable::LineBreak;
1772}
1773
1774//===- MIFlagsInstructionPredicateMatcher ---------------------------------===//
1775
1776bool MIFlagsInstructionPredicateMatcher::isIdentical(
1777 const PredicateMatcher &B) const {
1778 if (!InstructionPredicateMatcher::isIdentical(B))
1779 return false;
1780 const auto &Other =
1781 static_cast<const MIFlagsInstructionPredicateMatcher &>(B);
1782 return Flags == Other.Flags && CheckNot == Other.CheckNot;
1783}
1784
1785void MIFlagsInstructionPredicateMatcher::emitPredicateOpcodes(
1786 MatchTable &Table, RuleMatcher &Rule) const {
1787 Table << MatchTable::Opcode(Opcode: CheckNot ? "GIM_MIFlagsNot" : "GIM_MIFlags")
1788 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1789 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: Flags, Separator: " | "))
1790 << MatchTable::LineBreak;
1791}
1792
1793//===- InstructionMatcher -------------------------------------------------===//
1794
1795OperandMatcher &
1796InstructionMatcher::addOperand(unsigned OpIdx, const std::string &SymbolicName,
1797 unsigned AllocatedTemporariesBaseID,
1798 bool IsVariadic) {
1799 assert((Operands.empty() || !Operands.back()->isVariadic()) &&
1800 "Cannot add more operands after a variadic operand");
1801 Operands.emplace_back(args: new OperandMatcher(
1802 *this, OpIdx, SymbolicName, AllocatedTemporariesBaseID, IsVariadic));
1803 if (!SymbolicName.empty())
1804 Rule.defineOperand(SymbolicName, OM&: *Operands.back());
1805 return *Operands.back();
1806}
1807
1808OperandMatcher &InstructionMatcher::getOperand(unsigned OpIdx) {
1809 auto I = llvm::find_if(Range&: Operands,
1810 P: [&OpIdx](const std::unique_ptr<OperandMatcher> &X) {
1811 return X->getOpIdx() == OpIdx;
1812 });
1813 if (I != Operands.end())
1814 return **I;
1815 llvm_unreachable("Failed to lookup operand");
1816}
1817
1818OperandMatcher &InstructionMatcher::addPhysRegInput(const Record *Reg,
1819 unsigned OpIdx,
1820 unsigned TempOpIdx) {
1821 assert(SymbolicName.empty());
1822 OperandMatcher *OM = new OperandMatcher(*this, OpIdx, "", TempOpIdx);
1823 Operands.emplace_back(args&: OM);
1824 Rule.definePhysRegOperand(Reg, OM&: *OM);
1825 return *OM;
1826}
1827
1828bool InstructionMatcher::recordsOperand() const {
1829 return matchersRecordOperand(R: Predicates) || matchersRecordOperand(R: operands());
1830}
1831
1832void InstructionMatcher::emitPredicateOpcodes(MatchTable &Table,
1833 RuleMatcher &Rule) {
1834 if (canAddNumOperandsCheck()) {
1835 InstructionNumOperandsMatcher(InsnVarID, getNumOperandMatchers())
1836 .emitPredicateOpcodes(Table, Rule);
1837 }
1838
1839 // First emit all instruction level predicates need to be verified before we
1840 // can verify operands.
1841 emitFilteredPredicateListOpcodes(
1842 ShouldEmitPredicate: [](const PredicateMatcher &P) { return !P.dependsOnOperands(); }, Table,
1843 args&: Rule);
1844
1845 // Emit all operand constraints.
1846 for (const auto &Operand : Operands)
1847 Operand->emitPredicateOpcodes(Table, Rule);
1848
1849 // All of the tablegen defined predicates should now be matched. Now emit
1850 // any custom predicates that rely on all generated checks.
1851 emitFilteredPredicateListOpcodes(
1852 ShouldEmitPredicate: [](const PredicateMatcher &P) { return P.dependsOnOperands(); }, Table,
1853 args&: Rule);
1854}
1855
1856bool InstructionMatcher::isHigherPriorityThan(InstructionMatcher &B) {
1857 // Instruction matchers involving more operands have higher priority.
1858 if (Operands.size() > B.Operands.size())
1859 return true;
1860 if (Operands.size() < B.Operands.size())
1861 return false;
1862
1863 for (auto &&P : zip(t: predicates(), u: B.predicates())) {
1864 auto L = static_cast<InstructionPredicateMatcher *>(std::get<0>(t&: P).get());
1865 auto R = static_cast<InstructionPredicateMatcher *>(std::get<1>(t&: P).get());
1866 if (L->isHigherPriorityThan(B: *R))
1867 return true;
1868 if (R->isHigherPriorityThan(B: *L))
1869 return false;
1870 }
1871
1872 for (auto Operand : zip(t&: Operands, u&: B.Operands)) {
1873 if (std::get<0>(t&: Operand)->isHigherPriorityThan(B&: *std::get<1>(t&: Operand)))
1874 return true;
1875 if (std::get<1>(t&: Operand)->isHigherPriorityThan(B&: *std::get<0>(t&: Operand)))
1876 return false;
1877 }
1878
1879 return false;
1880}
1881
1882unsigned InstructionMatcher::countRendererFns() {
1883 return std::accumulate(
1884 first: predicates().begin(), last: predicates().end(), init: 0,
1885 binary_op: [](unsigned A,
1886 const std::unique_ptr<PredicateMatcher> &Predicate) {
1887 return A + Predicate->countRendererFns();
1888 }) +
1889 std::accumulate(
1890 first: Operands.begin(), last: Operands.end(), init: 0,
1891 binary_op: [](unsigned A, const std::unique_ptr<OperandMatcher> &Operand) {
1892 return A + Operand->countRendererFns();
1893 });
1894}
1895
1896void InstructionMatcher::optimize() {
1897 SmallVector<std::unique_ptr<PredicateMatcher>, 8> Stash;
1898 const auto &OpcMatcher = getOpcodeMatcher();
1899
1900 Stash.push_back(Elt: predicates_pop_front());
1901 if (Stash.back().get() == &OpcMatcher) {
1902 // FIXME: Is this even needed still? Why the isVariadicNumOperands check?
1903 if (canAddNumOperandsCheck() && OpcMatcher.isVariadicNumOperands() &&
1904 getNumOperandMatchers() != 0) {
1905 Stash.emplace_back(Args: new InstructionNumOperandsMatcher(
1906 InsnVarID, getNumOperandMatchers()));
1907 }
1908 AllowNumOpsCheck = false;
1909
1910 for (auto &OM : Operands)
1911 for (auto &OP : OM->predicates())
1912 if (isa<IntrinsicIDOperandMatcher>(Val: OP)) {
1913 Stash.push_back(Elt: std::move(OP));
1914 OM->eraseNullPredicates();
1915 break;
1916 }
1917 }
1918
1919 if (InsnVarID > 0) {
1920 assert(!Operands.empty() && "Nested instruction is expected to def a vreg");
1921 for (auto &OP : Operands[0]->predicates())
1922 OP.reset();
1923 Operands[0]->eraseNullPredicates();
1924 }
1925 for (auto &OM : Operands) {
1926 for (auto &OP : OM->predicates())
1927 if (isa<LLTOperandMatcher>(Val: OP))
1928 Stash.push_back(Elt: std::move(OP));
1929 OM->eraseNullPredicates();
1930 }
1931 while (!Stash.empty())
1932 prependPredicate(Predicate: Stash.pop_back_val());
1933}
1934
1935//===- InstructionOperandMatcher ------------------------------------------===//
1936
1937void InstructionOperandMatcher::emitCaptureOpcodes(MatchTable &Table,
1938 RuleMatcher &Rule) const {
1939 const unsigned NewInsnVarID = InsnMatcher->getInsnVarID();
1940 const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
1941 Table << MatchTable::Opcode(Opcode: IgnoreCopies ? "GIM_RecordInsnIgnoreCopies"
1942 : "GIM_RecordInsn")
1943 << MatchTable::Comment(Comment: "DefineMI")
1944 << MatchTable::ULEB128Value(IntValue: NewInsnVarID) << MatchTable::Comment(Comment: "MI")
1945 << MatchTable::ULEB128Value(IntValue: getInsnVarID())
1946 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: getOpIdx())
1947 << MatchTable::Comment(Comment: "MIs[" + llvm::to_string(Value: NewInsnVarID) + "]")
1948 << MatchTable::LineBreak;
1949}
1950
1951bool InstructionOperandMatcher::isHigherPriorityThan(
1952 const OperandPredicateMatcher &B) const {
1953 if (OperandPredicateMatcher::isHigherPriorityThan(B))
1954 return true;
1955 if (B.OperandPredicateMatcher::isHigherPriorityThan(B: *this))
1956 return false;
1957
1958 if (const InstructionOperandMatcher *BP =
1959 dyn_cast<InstructionOperandMatcher>(Val: &B))
1960 if (InsnMatcher->isHigherPriorityThan(B&: *BP->InsnMatcher))
1961 return true;
1962 return false;
1963}
1964
1965//===- OperandRenderer ----------------------------------------------------===//
1966
1967OperandRenderer::~OperandRenderer() = default;
1968
1969//===- CopyRenderer -------------------------------------------------------===//
1970
1971void CopyRenderer::emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule,
1972 unsigned NewInsnID, unsigned OldInsnID,
1973 unsigned OpIdx, StringRef Name,
1974 bool ForVariadic) {
1975 if (!ForVariadic && NewInsnID == 0 && OldInsnID == 0) {
1976 Table << MatchTable::Opcode(Opcode: "GIR_RootToRootCopy");
1977 } else {
1978 Table << MatchTable::Opcode(Opcode: ForVariadic ? "GIR_CopyRemaining" : "GIR_Copy")
1979 << MatchTable::Comment(Comment: "NewInsnID")
1980 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1981 << MatchTable::Comment(Comment: "OldInsnID")
1982 << MatchTable::ULEB128Value(IntValue: OldInsnID);
1983 }
1984
1985 Table << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OpIdx)
1986 << MatchTable::Comment(Comment: Name) << MatchTable::LineBreak;
1987}
1988
1989void CopyRenderer::emitRenderOpcodes(MatchTable &Table,
1990 RuleMatcher &Rule) const {
1991 const OperandMatcher &Operand = Rule.getOperandMatcher(Name: SymbolicName);
1992 unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher&: Operand.getInstructionMatcher());
1993
1994 emitRenderOpcodes(Table, Rule, NewInsnID, OldInsnID: OldInsnVarID, OpIdx: Operand.getOpIdx(),
1995 Name: SymbolicName, ForVariadic: Operand.isVariadic());
1996}
1997
1998//===- CopyPhysRegRenderer ------------------------------------------------===//
1999
2000void CopyPhysRegRenderer::emitRenderOpcodes(MatchTable &Table,
2001 RuleMatcher &Rule) const {
2002 const OperandMatcher &Operand = Rule.getPhysRegOperandMatcher(Reg: PhysReg);
2003 unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher&: Operand.getInstructionMatcher());
2004 CopyRenderer::emitRenderOpcodes(Table, Rule, NewInsnID, OldInsnID: OldInsnVarID,
2005 OpIdx: Operand.getOpIdx(), Name: PhysReg->getName());
2006}
2007
2008//===- CopyOrAddZeroRegRenderer -------------------------------------------===//
2009
2010void CopyOrAddZeroRegRenderer::emitRenderOpcodes(MatchTable &Table,
2011 RuleMatcher &Rule) const {
2012 const OperandMatcher &Operand = Rule.getOperandMatcher(Name: SymbolicName);
2013 unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher&: Operand.getInstructionMatcher());
2014 Table << MatchTable::Opcode(Opcode: "GIR_CopyOrAddZeroReg")
2015 << MatchTable::Comment(Comment: "NewInsnID")
2016 << MatchTable::ULEB128Value(IntValue: NewInsnID)
2017 << MatchTable::Comment(Comment: "OldInsnID")
2018 << MatchTable::ULEB128Value(IntValue: OldInsnVarID)
2019 << MatchTable::Comment(Comment: "OpIdx")
2020 << MatchTable::ULEB128Value(IntValue: Operand.getOpIdx())
2021 << MatchTable::NamedValue(
2022 NumBytes: 2,
2023 Namespace: (ZeroRegisterDef->getValue(Name: "Namespace")
2024 ? ZeroRegisterDef->getValueAsString(FieldName: "Namespace")
2025 : ""),
2026 NamedValue: ZeroRegisterDef->getName())
2027 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
2028}
2029
2030//===- CopyConstantAsImmRenderer ------------------------------------------===//
2031
2032void CopyConstantAsImmRenderer::emitRenderOpcodes(MatchTable &Table,
2033 RuleMatcher &Rule) const {
2034 InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
2035 unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
2036 Table << MatchTable::Opcode(Opcode: Signed ? "GIR_CopyConstantAsSImm"
2037 : "GIR_CopyConstantAsUImm")
2038 << MatchTable::Comment(Comment: "NewInsnID")
2039 << MatchTable::ULEB128Value(IntValue: NewInsnID)
2040 << MatchTable::Comment(Comment: "OldInsnID")
2041 << MatchTable::ULEB128Value(IntValue: OldInsnVarID)
2042 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
2043}
2044
2045//===- CopyFConstantAsFPImmRenderer ---------------------------------------===//
2046
2047void CopyFConstantAsFPImmRenderer::emitRenderOpcodes(MatchTable &Table,
2048 RuleMatcher &Rule) const {
2049 InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
2050 unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
2051 Table << MatchTable::Opcode(Opcode: "GIR_CopyFConstantAsFPImm")
2052 << MatchTable::Comment(Comment: "NewInsnID")
2053 << MatchTable::ULEB128Value(IntValue: NewInsnID)
2054 << MatchTable::Comment(Comment: "OldInsnID")
2055 << MatchTable::ULEB128Value(IntValue: OldInsnVarID)
2056 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
2057}
2058
2059//===- CopySubRegRenderer -------------------------------------------------===//
2060
2061void CopySubRegRenderer::emitRenderOpcodes(MatchTable &Table,
2062 RuleMatcher &Rule) const {
2063 const OperandMatcher &Operand = Rule.getOperandMatcher(Name: SymbolicName);
2064 unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher&: Operand.getInstructionMatcher());
2065 Table << MatchTable::Opcode(Opcode: "GIR_CopySubReg")
2066 << MatchTable::Comment(Comment: "NewInsnID")
2067 << MatchTable::ULEB128Value(IntValue: NewInsnID)
2068 << MatchTable::Comment(Comment: "OldInsnID")
2069 << MatchTable::ULEB128Value(IntValue: OldInsnVarID)
2070 << MatchTable::Comment(Comment: "OpIdx")
2071 << MatchTable::ULEB128Value(IntValue: Operand.getOpIdx())
2072 << MatchTable::Comment(Comment: "SubRegIdx")
2073 << MatchTable::IntValue(NumBytes: 2, IntValue: SubReg->EnumValue)
2074 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
2075}
2076
2077//===- AddRegisterRenderer ------------------------------------------------===//
2078
2079void AddRegisterRenderer::emitRenderOpcodes(MatchTable &Table,
2080 RuleMatcher &Rule) const {
2081 Table << MatchTable::Opcode(Opcode: "GIR_AddRegister")
2082 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID);
2083 if (RegisterDef->getName() != "zero_reg") {
2084 Table << MatchTable::NamedValue(
2085 NumBytes: 2,
2086 Namespace: (RegisterDef->getValue(Name: "Namespace")
2087 ? RegisterDef->getValueAsString(FieldName: "Namespace")
2088 : ""),
2089 NamedValue: RegisterDef->getName());
2090 } else {
2091 Table << MatchTable::NamedValue(NumBytes: 2, Namespace: Target.getRegNamespace(), NamedValue: "NoRegister");
2092 }
2093 Table << MatchTable::Comment(Comment: "AddRegisterRegFlags");
2094
2095 // TODO: This is encoded as a 64-bit element, but only 16 or 32-bits are
2096 // really needed for a physical register reference. We can pack the
2097 // register and flags in a single field.
2098 if (IsDef) {
2099 Table << MatchTable::NamedValue(
2100 NumBytes: 2, NamedValue: IsDead ? "static_cast<uint16_t>(RegState::Define|RegState::Dead)"
2101 : "static_cast<uint16_t>(RegState::Define)");
2102 } else {
2103 assert(!IsDead && "A use cannot be dead");
2104 Table << MatchTable::IntValue(NumBytes: 2, IntValue: 0);
2105 }
2106 Table << MatchTable::LineBreak;
2107}
2108
2109//===- TempRegRenderer ----------------------------------------------------===//
2110
2111void TempRegRenderer::emitRenderOpcodes(MatchTable &Table,
2112 RuleMatcher &Rule) const {
2113 const bool NeedsFlags = (SubRegIdx || IsDef);
2114 if (SubRegIdx) {
2115 assert(!IsDef);
2116 Table << MatchTable::Opcode(Opcode: "GIR_AddTempSubRegister");
2117 } else {
2118 Table << MatchTable::Opcode(Opcode: NeedsFlags ? "GIR_AddTempRegister"
2119 : "GIR_AddSimpleTempRegister");
2120 }
2121
2122 Table << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2123 << MatchTable::Comment(Comment: "TempRegID")
2124 << MatchTable::ULEB128Value(IntValue: TempRegID);
2125
2126 if (!NeedsFlags) {
2127 Table << MatchTable::LineBreak;
2128 return;
2129 }
2130
2131 Table << MatchTable::Comment(Comment: "TempRegFlags");
2132 if (IsDef) {
2133 SmallString<32> RegFlags;
2134 RegFlags += "static_cast<uint16_t>(RegState::Define";
2135 if (IsDead)
2136 RegFlags += "|RegState::Dead";
2137 RegFlags += ")";
2138 Table << MatchTable::NamedValue(NumBytes: 2, NamedValue: RegFlags);
2139 } else {
2140 Table << MatchTable::IntValue(NumBytes: 2, IntValue: 0);
2141 }
2142
2143 if (SubRegIdx)
2144 Table << MatchTable::NamedValue(NumBytes: 2, NamedValue: SubRegIdx->getQualifiedName());
2145 Table << MatchTable::LineBreak;
2146}
2147
2148//===- ImmRenderer --------------------------------------------------------===//
2149
2150void ImmRenderer::emitAddImm(MatchTable &Table, RuleMatcher &RM,
2151 unsigned InsnID, int64_t Imm, StringRef ImmName) {
2152 const bool IsInt8 = isInt<8>(x: Imm);
2153
2154 Table << MatchTable::Opcode(Opcode: IsInt8 ? "GIR_AddImm8" : "GIR_AddImm")
2155 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2156 << MatchTable::Comment(Comment: ImmName)
2157 << MatchTable::IntValue(NumBytes: IsInt8 ? 1 : 8, IntValue: Imm) << MatchTable::LineBreak;
2158}
2159
2160void ImmRenderer::emitRenderOpcodes(MatchTable &Table,
2161 RuleMatcher &Rule) const {
2162 if (CImmLLT) {
2163 assert(Table.isCombiner() &&
2164 "ConstantInt immediate are only for combiners!");
2165 Table << MatchTable::Opcode(Opcode: "GIR_AddCImm") << MatchTable::Comment(Comment: "InsnID")
2166 << MatchTable::ULEB128Value(IntValue: InsnID) << MatchTable::Comment(Comment: "Type")
2167 << *CImmLLT << MatchTable::Comment(Comment: "Imm")
2168 << MatchTable::IntValue(NumBytes: 8, IntValue: Imm) << MatchTable::LineBreak;
2169 } else {
2170 emitAddImm(Table, RM&: Rule, InsnID, Imm);
2171 }
2172}
2173
2174//===- SubRegIndexRenderer ------------------------------------------------===//
2175
2176void SubRegIndexRenderer::emitRenderOpcodes(MatchTable &Table,
2177 RuleMatcher &Rule) const {
2178 ImmRenderer::emitAddImm(Table, RM&: Rule, InsnID, Imm: SubRegIdx->EnumValue,
2179 ImmName: "SubRegIndex");
2180}
2181
2182//===- RenderComplexPatternOperand ----------------------------------------===//
2183
2184void RenderComplexPatternOperand::emitRenderOpcodes(MatchTable &Table,
2185 RuleMatcher &Rule) const {
2186 Table << MatchTable::Opcode(
2187 Opcode: SubOperand ? (SubReg ? "GIR_ComplexSubOperandSubRegRenderer"
2188 : "GIR_ComplexSubOperandRenderer")
2189 : "GIR_ComplexRenderer")
2190 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2191 << MatchTable::Comment(Comment: "RendererID")
2192 << MatchTable::IntValue(NumBytes: 2, IntValue: RendererID);
2193 if (SubOperand)
2194 Table << MatchTable::Comment(Comment: "SubOperand")
2195 << MatchTable::ULEB128Value(IntValue: *SubOperand);
2196 if (SubReg)
2197 Table << MatchTable::Comment(Comment: "SubRegIdx")
2198 << MatchTable::IntValue(NumBytes: 2, IntValue: SubReg->EnumValue);
2199 Table << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
2200}
2201
2202//===- IntrinsicIDRenderer ------------------------------------------------===//
2203
2204void IntrinsicIDRenderer::emitRenderOpcodes(MatchTable &Table,
2205 RuleMatcher &Rule) const {
2206 Table << MatchTable::Opcode(Opcode: "GIR_AddIntrinsicID") << MatchTable::Comment(Comment: "MI")
2207 << MatchTable::ULEB128Value(IntValue: InsnID)
2208 << MatchTable::NamedValue(NumBytes: 2, NamedValue: "Intrinsic::" + II->EnumName.str())
2209 << MatchTable::LineBreak;
2210}
2211
2212//===- CustomRenderer -----------------------------------------------------===//
2213
2214void CustomRenderer::emitRenderOpcodes(MatchTable &Table,
2215 RuleMatcher &Rule) const {
2216 InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
2217 unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
2218 Table << MatchTable::Opcode(Opcode: "GIR_CustomRenderer")
2219 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2220 << MatchTable::Comment(Comment: "OldInsnID")
2221 << MatchTable::ULEB128Value(IntValue: OldInsnVarID)
2222 << MatchTable::Comment(Comment: "Renderer")
2223 << MatchTable::NamedValue(
2224 NumBytes: 2, NamedValue: "GICR_" + Renderer.getValueAsString(FieldName: "RendererFn").str())
2225 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
2226}
2227
2228//===- CustomOperandRenderer ----------------------------------------------===//
2229
2230void CustomOperandRenderer::emitRenderOpcodes(MatchTable &Table,
2231 RuleMatcher &Rule) const {
2232 const OperandMatcher &OpdMatcher = Rule.getOperandMatcher(Name: SymbolicName);
2233 Table << MatchTable::Opcode(Opcode: "GIR_CustomOperandRenderer")
2234 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2235 << MatchTable::Comment(Comment: "OldInsnID")
2236 << MatchTable::ULEB128Value(IntValue: OpdMatcher.getInsnVarID())
2237 << MatchTable::Comment(Comment: "OpIdx")
2238 << MatchTable::ULEB128Value(IntValue: OpdMatcher.getOpIdx())
2239 << MatchTable::Comment(Comment: "OperandRenderer")
2240 << MatchTable::NamedValue(
2241 NumBytes: 2, NamedValue: "GICR_" + Renderer.getValueAsString(FieldName: "RendererFn").str())
2242 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
2243}
2244
2245//===- BuildMIAction ------------------------------------------------------===//
2246
2247bool BuildMIAction::canMutate(RuleMatcher &Rule,
2248 const InstructionMatcher *Insn) const {
2249 if (!Insn || Insn->hasVariadicMatcher())
2250 return false;
2251
2252 if (OperandRenderers.size() != Insn->getNumOperandMatchers())
2253 return false;
2254
2255 for (const auto &Renderer : enumerate(First: OperandRenderers)) {
2256 if (const auto *Copy = dyn_cast<CopyRenderer>(Val: &*Renderer.value())) {
2257 const OperandMatcher &OM =
2258 Rule.getOperandMatcher(Name: Copy->getSymbolicName());
2259 if (Insn != &OM.getInstructionMatcher() ||
2260 OM.getOpIdx() != Renderer.index())
2261 return false;
2262 } else {
2263 return false;
2264 }
2265 }
2266
2267 return true;
2268}
2269
2270void BuildMIAction::chooseInsnToMutate(RuleMatcher &Rule) {
2271 for (auto *MutateCandidate : Rule.mutatable_insns()) {
2272 if (canMutate(Rule, Insn: MutateCandidate)) {
2273 // Take the first one we're offered that we're able to mutate.
2274 Rule.reserveInsnMatcherForMutation(InsnMatcher: MutateCandidate);
2275 Matched = MutateCandidate;
2276 return;
2277 }
2278 }
2279}
2280
2281void BuildMIAction::emitActionOpcodes(MatchTable &Table,
2282 RuleMatcher &Rule) const {
2283 const auto AddMIFlags = [&]() {
2284 for (const InstructionMatcher *IM : CopiedFlags) {
2285 Table << MatchTable::Opcode(Opcode: "GIR_CopyMIFlags")
2286 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2287 << MatchTable::Comment(Comment: "OldInsnID")
2288 << MatchTable::ULEB128Value(IntValue: IM->getInsnVarID())
2289 << MatchTable::LineBreak;
2290 }
2291
2292 if (!SetFlags.empty()) {
2293 Table << MatchTable::Opcode(Opcode: "GIR_SetMIFlags")
2294 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2295 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: SetFlags, Separator: " | "))
2296 << MatchTable::LineBreak;
2297 }
2298
2299 if (!UnsetFlags.empty()) {
2300 Table << MatchTable::Opcode(Opcode: "GIR_UnsetMIFlags")
2301 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2302 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: UnsetFlags, Separator: " | "))
2303 << MatchTable::LineBreak;
2304 }
2305 };
2306
2307 if (Matched) {
2308 assert(canMutate(Rule, Matched) &&
2309 "Arranged to mutate an insn that isn't mutatable");
2310
2311 unsigned RecycleInsnID = Rule.getInsnVarID(InsnMatcher&: *Matched);
2312 Table << MatchTable::Opcode(Opcode: "GIR_MutateOpcode")
2313 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2314 << MatchTable::Comment(Comment: "RecycleInsnID")
2315 << MatchTable::ULEB128Value(IntValue: RecycleInsnID)
2316 << MatchTable::Comment(Comment: "Opcode")
2317 << MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName())
2318 << MatchTable::LineBreak;
2319
2320 if (!I->ImplicitDefs.empty() || !I->ImplicitUses.empty()) {
2321 for (auto *Def : I->ImplicitDefs) {
2322 auto Namespace = Def->getValue(Name: "Namespace")
2323 ? Def->getValueAsString(FieldName: "Namespace")
2324 : "";
2325 const bool IsDead = DeadImplicitDefs.contains(Ptr: Def);
2326 Table << MatchTable::Opcode(Opcode: "GIR_AddImplicitDef")
2327 << MatchTable::Comment(Comment: "InsnID")
2328 << MatchTable::ULEB128Value(IntValue: InsnID)
2329 << MatchTable::NamedValue(NumBytes: 2, Namespace, NamedValue: Def->getName())
2330 << (IsDead ? MatchTable::NamedValue(
2331 NumBytes: 2, NamedValue: "static_cast<unsigned>(RegState::Dead)")
2332 : MatchTable::IntValue(NumBytes: 2, IntValue: 0))
2333 << MatchTable::LineBreak;
2334 }
2335 for (auto *Use : I->ImplicitUses) {
2336 auto Namespace = Use->getValue(Name: "Namespace")
2337 ? Use->getValueAsString(FieldName: "Namespace")
2338 : "";
2339 Table << MatchTable::Opcode(Opcode: "GIR_AddImplicitUse")
2340 << MatchTable::Comment(Comment: "InsnID")
2341 << MatchTable::ULEB128Value(IntValue: InsnID)
2342 << MatchTable::NamedValue(NumBytes: 2, Namespace, NamedValue: Use->getName())
2343 << MatchTable::LineBreak;
2344 }
2345 }
2346
2347 AddMIFlags();
2348
2349 // Mark the mutated instruction as erased.
2350 Rule.tryEraseInsnID(ID: RecycleInsnID);
2351 return;
2352 }
2353
2354 // TODO: Simple permutation looks like it could be almost as common as
2355 // mutation due to commutative operations.
2356
2357 if (InsnID == 0) {
2358 Table << MatchTable::Opcode(Opcode: "GIR_BuildRootMI");
2359 } else {
2360 Table << MatchTable::Opcode(Opcode: "GIR_BuildMI") << MatchTable::Comment(Comment: "InsnID")
2361 << MatchTable::ULEB128Value(IntValue: InsnID);
2362 }
2363
2364 Table << MatchTable::Comment(Comment: "Opcode")
2365 << MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName())
2366 << MatchTable::LineBreak;
2367
2368 for (const auto &Renderer : OperandRenderers)
2369 Renderer->emitRenderOpcodes(Table, Rule);
2370
2371 for (auto [OpIdx, Def] : enumerate(First: I->ImplicitDefs)) {
2372 auto Namespace =
2373 Def->getValue(Name: "Namespace") ? Def->getValueAsString(FieldName: "Namespace") : "";
2374 if (DeadImplicitDefs.contains(Ptr: Def)) {
2375 Table
2376 << MatchTable::Opcode(Opcode: "GIR_SetImplicitDefDead")
2377 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2378 << MatchTable::Comment(
2379 Comment: ("OpIdx for " + Namespace + "::" + Def->getName() + "").str())
2380 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
2381 }
2382 }
2383
2384 if (I->mayLoad || I->mayStore) {
2385 // Emit the ID's for all the instructions that are matched by this rule.
2386 // TODO: Limit this to matched instructions that mayLoad/mayStore or have
2387 // some other means of having a memoperand. Also limit this to
2388 // emitted instructions that expect to have a memoperand too. For
2389 // example, (G_SEXT (G_LOAD x)) that results in separate load and
2390 // sign-extend instructions shouldn't put the memoperand on the
2391 // sign-extend since it has no effect there.
2392
2393 std::vector<unsigned> MergeInsnIDs;
2394 for (const auto &IDMatcherPair : Rule.defined_insn_vars())
2395 MergeInsnIDs.push_back(x: IDMatcherPair.second);
2396 llvm::sort(C&: MergeInsnIDs);
2397
2398 Table << MatchTable::Opcode(Opcode: "GIR_MergeMemOperands")
2399 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2400 << MatchTable::Comment(Comment: "NumInsns")
2401 << MatchTable::IntValue(NumBytes: 1, IntValue: MergeInsnIDs.size())
2402 << MatchTable::Comment(Comment: "MergeInsnID's");
2403 for (const auto &MergeInsnID : MergeInsnIDs)
2404 Table << MatchTable::ULEB128Value(IntValue: MergeInsnID);
2405 Table << MatchTable::LineBreak;
2406 }
2407
2408 AddMIFlags();
2409}
2410
2411//===- BuildConstantAction ------------------------------------------------===//
2412
2413void BuildConstantAction::emitActionOpcodes(MatchTable &Table,
2414 RuleMatcher &Rule) const {
2415 Table << MatchTable::Opcode(Opcode: "GIR_BuildConstant")
2416 << MatchTable::Comment(Comment: "TempRegID")
2417 << MatchTable::ULEB128Value(IntValue: TempRegID) << MatchTable::Comment(Comment: "Val")
2418 << MatchTable::IntValue(NumBytes: 8, IntValue: Val) << MatchTable::LineBreak;
2419}
2420
2421//===- EraseInstAction ----------------------------------------------------===//
2422
2423void EraseInstAction::emitActionOpcodes(MatchTable &Table,
2424 RuleMatcher &Rule) const {
2425 // Avoid erasing the same inst twice.
2426 if (!Rule.tryEraseInsnID(ID: InsnID))
2427 return;
2428
2429 Table << MatchTable::Opcode(Opcode: "GIR_EraseFromParent")
2430 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2431 << MatchTable::LineBreak;
2432}
2433
2434bool EraseInstAction::emitActionOpcodesAndDone(MatchTable &Table,
2435 RuleMatcher &Rule) const {
2436 if (InsnID != 0) {
2437 emitActionOpcodes(Table, Rule);
2438 return false;
2439 }
2440
2441 if (!Rule.tryEraseInsnID(ID: 0))
2442 return false;
2443
2444 Table << MatchTable::Opcode(Opcode: "GIR_EraseRootFromParent_Done", IndentAdjust: -1)
2445 << MatchTable::LineBreak;
2446 return true;
2447}
2448
2449//===- ReplaceRegAction ---------------------------------------------------===//
2450
2451void ReplaceRegAction::emitAdditionalPredicates(MatchTable &Table,
2452 RuleMatcher &Rule) const {
2453 if (TempRegID != (unsigned)-1)
2454 return;
2455
2456 Table << MatchTable::Opcode(Opcode: "GIM_CheckCanReplaceReg")
2457 << MatchTable::Comment(Comment: "OldInsnID")
2458 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2459 << MatchTable::Comment(Comment: "OldOpIdx") << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2460 << MatchTable::Comment(Comment: "NewInsnId")
2461 << MatchTable::ULEB128Value(IntValue: NewInsnId)
2462 << MatchTable::Comment(Comment: "NewOpIdx") << MatchTable::ULEB128Value(IntValue: NewOpIdx)
2463 << MatchTable::LineBreak;
2464}
2465
2466void ReplaceRegAction::emitActionOpcodes(MatchTable &Table,
2467 RuleMatcher &Rule) const {
2468 if (TempRegID != (unsigned)-1) {
2469 Table << MatchTable::Opcode(Opcode: "GIR_ReplaceRegWithTempReg")
2470 << MatchTable::Comment(Comment: "OldInsnID")
2471 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2472 << MatchTable::Comment(Comment: "OldOpIdx")
2473 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2474 << MatchTable::Comment(Comment: "TempRegID")
2475 << MatchTable::ULEB128Value(IntValue: TempRegID) << MatchTable::LineBreak;
2476 } else {
2477 Table << MatchTable::Opcode(Opcode: "GIR_ReplaceReg")
2478 << MatchTable::Comment(Comment: "OldInsnID")
2479 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2480 << MatchTable::Comment(Comment: "OldOpIdx")
2481 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2482 << MatchTable::Comment(Comment: "NewInsnId")
2483 << MatchTable::ULEB128Value(IntValue: NewInsnId)
2484 << MatchTable::Comment(Comment: "NewOpIdx")
2485 << MatchTable::ULEB128Value(IntValue: NewOpIdx) << MatchTable::LineBreak;
2486 }
2487}
2488
2489//===- ConstrainOperandToRegClassAction -----------------------------------===//
2490
2491void ConstrainOperandToRegClassAction::emitActionOpcodes(
2492 MatchTable &Table, RuleMatcher &Rule) const {
2493 Table << MatchTable::Opcode(Opcode: "GIR_ConstrainOperandRC")
2494 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2495 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
2496 << MatchTable::NamedValue(NumBytes: 2, NamedValue: RC.getQualifiedIdName())
2497 << MatchTable::LineBreak;
2498}
2499
2500//===- MakeTempRegisterAction ---------------------------------------------===//
2501
2502void MakeTempRegisterAction::emitActionOpcodes(MatchTable &Table,
2503 RuleMatcher &Rule) const {
2504 Table << MatchTable::Opcode(Opcode: "GIR_MakeTempReg")
2505 << MatchTable::Comment(Comment: "TempRegID")
2506 << MatchTable::ULEB128Value(IntValue: TempRegID) << MatchTable::Comment(Comment: "TypeID")
2507 << Ty << MatchTable::LineBreak;
2508}
2509