1//===- Matchers.cpp -------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Matchers.h"
10#include "Common/CodeGenInstruction.h"
11#include "Common/CodeGenRegisters.h"
12#include "llvm/ADT/Statistic.h"
13#include "llvm/Support/CommandLine.h"
14#include "llvm/Support/Debug.h"
15#include "llvm/Support/ScopedPrinter.h"
16#include "llvm/Support/raw_ostream.h"
17#include "llvm/TableGen/Error.h"
18
19#define DEBUG_TYPE "gi-match-table-matchers"
20
21STATISTIC(NumPatternEmitted, "Number of patterns emitted");
22
23using namespace llvm;
24using namespace gi;
25
26// FIXME: Use createStringError instead.
27static Error failUnsupported(const Twine &Reason) {
28 return make_error<StringError>(Args: Reason, Args: inconvertibleErrorCode());
29}
30
31/// Get the name of the enum value used to number the predicate function.
32static std::string getEnumNameForPredicate(const TreePredicateFn &Predicate) {
33 if (Predicate.hasGISelPredicateCode())
34 return "GICXXPred_MI_" + Predicate.getFnName();
35 if (Predicate.hasGISelLeafPredicateCode())
36 return "GICXXPred_MO_" + Predicate.getFnName();
37 return "GICXXPred_" + Predicate.getImmTypeIdentifier().str() + "_" +
38 Predicate.getFnName();
39}
40
41static std::string
42getMatchOpcodeForImmPredicate(const TreePredicateFn &Predicate) {
43 return "GIM_Check" + Predicate.getImmTypeIdentifier().str() + "ImmPredicate";
44}
45
46//===- Helpers ------------------------------------------------------------===//
47
48template <class GroupT>
49static std::vector<Matcher *>
50optimizeRules(ArrayRef<Matcher *> Rules,
51 std::vector<std::unique_ptr<Matcher>> &MatcherStorage) {
52
53 std::vector<Matcher *> Worklist(Rules.begin(), Rules.end());
54 std::vector<Matcher *> OptRules;
55 std::unique_ptr<GroupT> CurrentGroup = std::make_unique<GroupT>();
56 assert(CurrentGroup->empty() && "Newly created group isn't empty!");
57 unsigned NumGroups = 0;
58
59 auto ProcessCurrentGroup = [&]() {
60 if (CurrentGroup->empty())
61 // An empty group is good to be reused:
62 return;
63
64 // If the group isn't large enough to provide any benefit, move all the
65 // added rules out of it and make sure to re-create the group to properly
66 // re-initialize it:
67 if (CurrentGroup->size() < 2)
68 append_range(OptRules, CurrentGroup->matchers());
69 else {
70 CurrentGroup->finalize();
71 CurrentGroup->optimize();
72 OptRules.push_back(CurrentGroup.get());
73 MatcherStorage.emplace_back(std::move(CurrentGroup));
74 ++NumGroups;
75 }
76 CurrentGroup = std::make_unique<GroupT>();
77 };
78
79 for (Matcher *Rule : Worklist) {
80 // Greedily add as many matchers as possible to the current group:
81 if (CurrentGroup->addMatcher(*Rule))
82 continue;
83
84 ProcessCurrentGroup();
85 assert(CurrentGroup->empty() && "A group wasn't properly re-initialized");
86
87 // Try to add the pending matcher to a newly created empty group:
88 if (!CurrentGroup->addMatcher(*Rule))
89 // If we couldn't add the matcher to an empty group, that group type
90 // doesn't support that kind of matchers at all, so just skip it:
91 OptRules.push_back(x: Rule);
92 }
93 ProcessCurrentGroup();
94
95 assert(OptRules.size() <= Worklist.size() && "Optimization added rules?");
96 LLVM_DEBUG(dbgs() << "NumGroups: " << NumGroups << "\n");
97 (void)NumGroups;
98 assert(CurrentGroup->empty() && "The last group wasn't properly processed");
99 return OptRules;
100}
101
102std::vector<Matcher *> llvm::gi::optimizeRuleset(
103 MutableArrayRef<RuleMatcher> Rules,
104 std::vector<std::unique_ptr<Matcher>> &MatcherStorage) {
105 SmallVector<Matcher *> InputRules(make_pointer_range(Range&: Rules));
106
107 // Now sort the Rules.
108 unsigned CurrentOrdering = 0;
109 StringMap<unsigned> OpcodeOrder;
110 for (RuleMatcher &Rule : Rules) {
111 const StringRef Opcode = Rule.getOpcode();
112 assert(!Opcode.empty() && "Didn't expect an undefined opcode");
113 if (OpcodeOrder.try_emplace(Key: Opcode, Args&: CurrentOrdering).second)
114 ++CurrentOrdering;
115 }
116
117 llvm::stable_sort(
118 Range&: InputRules, C: [&OpcodeOrder](const Matcher *A, const Matcher *B) {
119 const auto *L = cast<RuleMatcher>(Val: A);
120 const auto *R = cast<RuleMatcher>(Val: B);
121 return std::tuple(OpcodeOrder[L->getOpcode()],
122 L->roots_front().getNumOperandMatchers()) <
123 std::tuple(OpcodeOrder[R->getOpcode()],
124 R->roots_front().getNumOperandMatchers());
125 });
126
127 for (Matcher *R : InputRules)
128 R->optimize();
129
130 // Then form groups, and switches in that order.
131 std::vector<Matcher *> OptRules =
132 optimizeRules<GroupMatcher>(Rules: InputRules, MatcherStorage);
133 OptRules = optimizeRules<SwitchMatcher>(Rules: OptRules, MatcherStorage);
134 return OptRules;
135}
136
137MatchTable llvm::gi::buildMatchTable(ArrayRef<Matcher *> Rules,
138 bool WithCoverage, bool IsCombiner) {
139 MatchTable Table(WithCoverage, IsCombiner);
140 for (Matcher *Rule : Rules)
141 Rule->emit(Table);
142
143 return Table << MatchTable::Opcode(Opcode: "GIM_Reject") << MatchTable::LineBreak;
144}
145
146template <class Range> static bool matchersRecordOperand(Range &&R) {
147 return any_of(R, [](const auto &I) { return I->recordsOperand(); });
148}
149
150static void emitType(MatchTable &Table, const LLTCodeGenOrTempType &Ty) {
151 if (Ty.isLLTCodeGen())
152 Table << MatchTable::NamedValue(NumBytes: 1, NamedValue: Ty.getLLTCodeGen().getCxxEnumValue());
153 else if (Ty.isLLTByHwMode())
154 Table << MatchTable::IntValue(NumBytes: 1, IntValue: getLLTByHwModeID(Ty: Ty.getLLTByHwMode()));
155 else
156 Table << MatchTable::IntValue(NumBytes: 1, IntValue: Ty.getTempTypeIdx());
157}
158
159//===- Matcher ------------------------------------------------------------===//
160
161void Matcher::optimize() {}
162
163Matcher::~Matcher() = default;
164
165//===- GroupMatcher -------------------------------------------------------===//
166
167bool GroupMatcher::recordsOperand() const {
168 return matchersRecordOperand(R: Conditions) || matchersRecordOperand(R: Matchers);
169}
170
171bool GroupMatcher::candidateConditionMatches(
172 const PredicateMatcher &Predicate) const {
173
174 if (empty()) {
175 // Sharing predicates for nested instructions is not supported yet as we
176 // currently don't hoist the GIM_RecordInsn's properly, therefore we can
177 // only work on the original root instruction (InsnVarID == 0):
178 if (Predicate.getInsnVarID() != 0)
179 return false;
180 // ... otherwise an empty group can handle any predicate with no specific
181 // requirements:
182 return true;
183 }
184
185 const Matcher &Representative = **Matchers.begin();
186 const auto &RepresentativeCondition = Representative.getFirstCondition();
187 // ... if not empty, the group can only accomodate matchers with the exact
188 // same first condition:
189 return Predicate.isIdentical(B: RepresentativeCondition);
190}
191
192std::unique_ptr<PredicateMatcher> GroupMatcher::popFirstCondition() {
193 assert(!Conditions.empty() &&
194 "Trying to pop a condition from a condition-less group");
195 std::unique_ptr<PredicateMatcher> P = std::move(Conditions.front());
196 Conditions.erase(CI: Conditions.begin());
197 return P;
198}
199
200bool GroupMatcher::addMatcher(Matcher &Candidate) {
201 if (!Candidate.hasFirstCondition())
202 return false;
203
204 // Only add candidates that have a matching first condition that can be
205 // hoisted into the GroupMatcher.
206 const PredicateMatcher &Predicate = Candidate.getFirstCondition();
207 if (!candidateConditionMatches(Predicate) ||
208 !Predicate.canHoistOutsideOf(M: Candidate))
209 return false;
210
211 Matchers.push_back(x: &Candidate);
212 return true;
213}
214
215void GroupMatcher::finalize() {
216 assert(Conditions.empty() && "Already finalized?");
217 if (empty())
218 return;
219
220 Matcher &FirstRule = **Matchers.begin();
221 for (;;) {
222 // All the checks are expected to succeed during the first iteration:
223 for (const auto &Rule : Matchers)
224 if (!Rule->hasFirstCondition())
225 return;
226 // Hoist the first condition if it is identical in all matchers in the group
227 // and it can be hoisted in every matcher.
228 const auto &FirstCondition = FirstRule.getFirstCondition();
229 if (!FirstCondition.canHoistOutsideOf(M: FirstRule))
230 return;
231 for (unsigned I = 1, E = Matchers.size(); I < E; ++I) {
232 const auto &OtherFirstCondition = Matchers[I]->getFirstCondition();
233 if (!OtherFirstCondition.isIdentical(B: FirstCondition) ||
234 !OtherFirstCondition.canHoistOutsideOf(M: *Matchers[I]))
235 return;
236 }
237
238 Conditions.push_back(Elt: FirstRule.popFirstCondition());
239 for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
240 Matchers[I]->popFirstCondition();
241 }
242}
243
244void GroupMatcher::emit(MatchTable &Table) {
245 unsigned LabelID = ~0U;
246 if (!Conditions.empty()) {
247 LabelID = Table.allocateLabelID();
248 Table << MatchTable::Opcode(Opcode: "GIM_Try", IndentAdjust: +1)
249 << MatchTable::Comment(Comment: "On fail goto")
250 << MatchTable::JumpTarget(LabelID) << MatchTable::LineBreak;
251 }
252 for (auto &Condition : Conditions)
253 Condition->emitPredicateOpcodes(Table);
254
255 for (const auto &M : Matchers)
256 M->emit(Table);
257
258 // Exit the group
259 if (!Conditions.empty())
260 Table << MatchTable::Opcode(Opcode: "GIM_Reject", IndentAdjust: -1) << MatchTable::LineBreak
261 << MatchTable::Label(LabelID);
262}
263
264void GroupMatcher::optimize() {
265 // Make sure we only sort by a specific predicate within a range of rules that
266 // all have that predicate checked against a specific value (not a wildcard):
267 // TODO: Is this even relevant ? Check diffs w/ just using a simple sort
268 // instead of this.
269 auto F = Matchers.begin();
270 auto T = F;
271 auto E = Matchers.end();
272 while (T != E) {
273 while (T != E) {
274 if (!(*T)->getFirstConditionAsRootType().get().isValid())
275 break;
276 ++T;
277 }
278 std::stable_sort(first: F, last: T, comp: [](Matcher *A, Matcher *B) {
279 return A->getFirstConditionAsRootType() <
280 B->getFirstConditionAsRootType();
281 });
282 if (T != E)
283 F = ++T;
284 }
285 Matchers = optimizeRules<GroupMatcher>(Rules: Matchers, MatcherStorage);
286 Matchers = optimizeRules<SwitchMatcher>(Rules: Matchers, MatcherStorage);
287}
288
289LLTCodeGen GroupMatcher::getFirstConditionAsRootType() const {
290 if (!hasFirstCondition())
291 return {};
292
293 const PredicateMatcher &PM = *Conditions.front();
294 if (const auto *TM = dyn_cast<LLTOperandMatcher>(Val: &PM)) {
295 if (TM->getInsnVarID() == 0 && TM->getOpIdx() == 0)
296 return TM->getTy();
297 }
298
299 return {};
300}
301
302//===- SwitchMatcher ------------------------------------------------------===//
303
304SwitchMatcher::SwitchMatcher() : Matcher(MK_Switch) {}
305SwitchMatcher::~SwitchMatcher() = default;
306
307bool SwitchMatcher::recordsOperand() const {
308 assert(!isa_and_present<RecordNamedOperandMatcher>(Condition.get()) &&
309 "Switch conditions should not record named operands");
310 return matchersRecordOperand(R: Matchers);
311}
312
313bool SwitchMatcher::isSupportedPredicateType(const PredicateMatcher &P) {
314 return isa<InstructionOpcodeMatcher>(Val: P) || isa<LLTOperandShapeMatcher>(Val: P) ||
315 isa<LLTOperandMatcher>(Val: P);
316}
317
318bool SwitchMatcher::candidateConditionMatches(
319 const PredicateMatcher &Predicate) const {
320
321 if (empty()) {
322 // Sharing predicates for nested instructions is not supported yet as we
323 // currently don't hoist the GIM_RecordInsn's properly, therefore we can
324 // only work on the original root instruction (InsnVarID == 0):
325 if (Predicate.getInsnVarID() != 0)
326 return false;
327 // ... while an attempt to add even a root matcher to an empty SwitchMatcher
328 // could fail as not all the types of conditions are supported:
329 if (!isSupportedPredicateType(P: Predicate))
330 return false;
331 // ... or the condition might not have a proper implementation of
332 // getValue() / isIdenticalDownToValue() yet:
333 if (!Predicate.hasValue())
334 return false;
335 // ... otherwise an empty Switch can accomodate the condition with no
336 // further requirements:
337 return true;
338 }
339
340 const Matcher &CaseRepresentative = **Matchers.begin();
341 const auto &RepresentativeCondition = CaseRepresentative.getFirstCondition();
342 // Switch-cases must share the same kind of condition and path to the value it
343 // checks:
344 if (!Predicate.isIdenticalDownToValue(B: RepresentativeCondition))
345 return false;
346
347 return true;
348}
349
350bool SwitchMatcher::addMatcher(Matcher &Candidate) {
351 if (!Candidate.hasFirstCondition())
352 return false;
353
354 const PredicateMatcher &Predicate = Candidate.getFirstCondition();
355 if (!candidateConditionMatches(Predicate))
356 return false;
357 const auto Value = Predicate.getValue();
358 auto It = Buckets.find(x: Value.RawValue);
359 if (It == Buckets.end())
360 It = Buckets.emplace(args: Value.RawValue, args: Bucket(Value)).first;
361#ifndef NDEBUG
362 else
363 assert(It->second.Value.Record.EmitStr == Value.Record.EmitStr &&
364 "Mismatched records for identical switch value");
365#endif
366 It->second.Matchers.push_back(x: &Candidate);
367 Matchers.push_back(x: &Candidate);
368 return true;
369}
370
371void SwitchMatcher::finalize() {
372 assert(Condition == nullptr && "Already finalized");
373#ifndef NDEBUG
374 unsigned NumBucketedMatchers = 0;
375 for (const auto &Entry : Buckets)
376 NumBucketedMatchers += Entry.second.Matchers.size();
377 assert(NumBucketedMatchers == Matchers.size() && "Broken SwitchMatcher");
378#endif
379 if (empty())
380 return;
381
382 Condition = Matchers.front()->popFirstCondition();
383 for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
384 Matchers[I]->popFirstCondition();
385
386 // After removing the switch condition, try to hoist any shared predicates
387 // within each switch bucket.
388 for (auto &Entry : Buckets) {
389 auto &BucketMatchers = Entry.second.Matchers;
390 BucketMatchers =
391 optimizeRules<GroupMatcher>(Rules: BucketMatchers, MatcherStorage);
392 }
393
394 Matchers.clear();
395 for (auto &Entry : Buckets)
396 append_range(C&: Matchers, R&: Entry.second.Matchers);
397}
398
399void SwitchMatcher::emitPredicateSpecificOpcodes(const PredicateMatcher &P,
400 MatchTable &Table) {
401 assert(isSupportedPredicateType(P) && "Predicate type is not supported");
402
403 if (const auto *Condition = dyn_cast<InstructionOpcodeMatcher>(Val: &P)) {
404 Table << MatchTable::Opcode(Opcode: "GIM_SwitchOpcode") << MatchTable::Comment(Comment: "MI")
405 << MatchTable::ULEB128Value(IntValue: Condition->getInsnVarID());
406 return;
407 }
408 if (const auto *Condition = dyn_cast<LLTOperandShapeMatcher>(Val: &P)) {
409 Table << MatchTable::Opcode(Opcode: "GIM_SwitchTypeShape")
410 << MatchTable::Comment(Comment: "MI")
411 << MatchTable::ULEB128Value(IntValue: Condition->getInsnVarID())
412 << MatchTable::Comment(Comment: "Op")
413 << MatchTable::ULEB128Value(IntValue: Condition->getOpIdx());
414 return;
415 }
416 if (const auto *Condition = dyn_cast<LLTOperandMatcher>(Val: &P)) {
417 Table << MatchTable::Opcode(Opcode: "GIM_SwitchType") << MatchTable::Comment(Comment: "MI")
418 << MatchTable::ULEB128Value(IntValue: Condition->getInsnVarID())
419 << MatchTable::Comment(Comment: "Op")
420 << MatchTable::ULEB128Value(IntValue: Condition->getOpIdx());
421 return;
422 }
423
424 llvm_unreachable("emitPredicateSpecificOpcodes is broken: can not handle a "
425 "predicate type that is claimed to be supported");
426}
427
428void SwitchMatcher::emit(MatchTable &Table) {
429#ifndef NDEBUG
430 unsigned NumBucketedMatchers = 0;
431 for (const auto &Entry : Buckets)
432 NumBucketedMatchers += Entry.second.Matchers.size();
433 assert(NumBucketedMatchers == Matchers.size() && "Broken SwitchMatcher");
434#endif
435 if (empty())
436 return;
437 assert(Condition != nullptr &&
438 "Broken SwitchMatcher, hasn't been finalized?");
439
440 std::vector<unsigned> LabelIDs(Buckets.size());
441 std::generate(first: LabelIDs.begin(), last: LabelIDs.end(),
442 gen: [&Table]() { return Table.allocateLabelID(); });
443 const unsigned Default = Table.allocateLabelID();
444
445 const int64_t LowerBound = Buckets.begin()->second.Value.RawValue;
446 const int64_t UpperBound = Buckets.rbegin()->second.Value.RawValue + 1;
447
448 emitPredicateSpecificOpcodes(P: *Condition, Table);
449
450 Table << MatchTable::Comment(Comment: "[") << MatchTable::IntValue(NumBytes: 2, IntValue: LowerBound)
451 << MatchTable::IntValue(NumBytes: 2, IntValue: UpperBound) << MatchTable::Comment(Comment: ")")
452 << MatchTable::Comment(Comment: "default:") << MatchTable::JumpTarget(LabelID: Default);
453
454 int64_t J = LowerBound;
455 unsigned CaseIdx = 0;
456 for (auto &Entry : Buckets) {
457 auto &V = Entry.second.Value;
458 while (J++ < V.RawValue)
459 Table << MatchTable::IntValue(NumBytes: 4, IntValue: 0);
460 V.Record.turnIntoComment();
461 Table << MatchTable::LineBreak << V.Record
462 << MatchTable::JumpTarget(LabelID: LabelIDs[CaseIdx]);
463 ++CaseIdx;
464 }
465 Table << MatchTable::LineBreak;
466
467 CaseIdx = 0;
468 for (auto &Entry : Buckets) {
469 Table << MatchTable::Label(LabelID: LabelIDs[CaseIdx]);
470 for (Matcher *M : Entry.second.Matchers)
471 M->emit(Table);
472 Table << MatchTable::Opcode(Opcode: "GIM_Reject") << MatchTable::LineBreak;
473 ++CaseIdx;
474 }
475 Table << MatchTable::Label(LabelID: Default);
476}
477
478//===- RuleMatcher --------------------------------------------------------===//
479
480RuleMatcher::RuleMatcher(ArrayRef<SMLoc> SrcLoc, bool UsesRecordOperand)
481 : Matcher(Matcher::MK_Rule), UsesRecordOperand(UsesRecordOperand),
482 SrcLoc(SrcLoc), RuleID(NextRuleID++) {}
483
484uint64_t RuleMatcher::NextRuleID = 0;
485
486StringRef RuleMatcher::getOpcode() const { return Roots.front()->getOpcode(); }
487
488bool RuleMatcher::recordsOperand() const {
489 return !usesRecordOperand() || matchersRecordOperand(R: InsnMatchers);
490}
491
492LLTCodeGen RuleMatcher::getFirstConditionAsRootType() const {
493 InstructionMatcher &InsnMatcher = *Roots.front();
494 if (!InsnMatcher.predicates_empty()) {
495 if (const auto *TM =
496 dyn_cast<LLTOperandMatcher>(Val: &**InsnMatcher.predicates_begin())) {
497 if (TM->getInsnVarID() == 0 && TM->getOpIdx() == 0)
498 return TM->getTy();
499 }
500 }
501 return {};
502}
503
504void RuleMatcher::optimize() {
505 for (const auto &InsnMatcher : InsnMatchers) {
506 for (auto &OM : InsnMatcher->operands()) {
507 // Complex Patterns are usually expensive and they relatively rarely fail
508 // on their own: more often we end up throwing away all the work done by a
509 // matching part of a complex pattern because some other part of the
510 // enclosing pattern didn't match. All of this makes it beneficial to
511 // delay complex patterns until the very end of the rule matching,
512 // especially for targets having lots of complex patterns.
513 for (auto &OP : OM->predicates())
514 if (isa<ComplexPatternOperandMatcher>(Val: OP))
515 EpilogueMatchers.emplace_back(args: std::move(OP));
516 OM->eraseNullPredicates();
517 }
518 InsnMatcher->optimize();
519 }
520 llvm::sort(C&: EpilogueMatchers, Comp: [](const std::unique_ptr<PredicateMatcher> &L,
521 const std::unique_ptr<PredicateMatcher> &R) {
522 return std::tuple(L->getKind(), L->getInsnVarID(), L->getOpIdx()) <
523 std::tuple(R->getKind(), R->getInsnVarID(), R->getOpIdx());
524 });
525
526 // Deduplicate EraseInst actions, and if an EraseInst erases the root, place
527 // it at the end to favor generation of GIR_EraseRootFromParent_Done
528 DenseSet<unsigned> AlreadySeenEraseInsts;
529 auto EraseRootIt = Actions.end();
530 auto It = Actions.begin();
531 while (It != Actions.end()) {
532 if (const auto *EI = dyn_cast<EraseInstAction>(Val: It->get())) {
533 unsigned InstID = EI->getInsnID();
534 if (!AlreadySeenEraseInsts.insert(V: InstID).second) {
535 It = Actions.erase(position: It);
536 continue;
537 }
538
539 if (InstID == 0)
540 EraseRootIt = It;
541 }
542
543 ++It;
544 }
545
546 if (EraseRootIt != Actions.end())
547 Actions.splice(position: Actions.end(), x&: Actions, i: EraseRootIt);
548}
549
550bool RuleMatcher::hasFirstCondition() const {
551 if (roots_empty())
552 return false;
553 InstructionMatcher &Matcher = roots_front();
554 if (!Matcher.predicates_empty())
555 return true;
556 for (auto &OM : Matcher.operands())
557 for (auto &OP : OM->predicates())
558 if (!isa<InstructionOperandMatcher>(Val: OP))
559 return true;
560 return false;
561}
562
563const PredicateMatcher &RuleMatcher::getFirstCondition() const {
564 assert(!roots_empty() &&
565 "Trying to get a condition from an empty RuleMatcher");
566
567 InstructionMatcher &Matcher = roots_front();
568 if (!Matcher.predicates_empty())
569 return **Matcher.predicates_begin();
570 // If there is no more predicate on the instruction itself, look at its
571 // operands.
572 for (auto &OM : Matcher.operands())
573 for (auto &OP : OM->predicates())
574 if (!isa<InstructionOperandMatcher>(Val: OP))
575 return *OP;
576
577 llvm_unreachable("Trying to get a condition from an InstructionMatcher with "
578 "no conditions");
579}
580
581std::unique_ptr<PredicateMatcher> RuleMatcher::popFirstCondition() {
582 assert(!roots_empty() &&
583 "Trying to pop a condition from an empty RuleMatcher");
584
585 InstructionMatcher &Matcher = roots_front();
586 if (!Matcher.predicates_empty())
587 return Matcher.predicates_pop_front();
588 // If there is no more predicate on the instruction itself, look at its
589 // operands.
590 for (auto &OM : Matcher.operands())
591 for (auto &OP : OM->predicates())
592 if (!isa<InstructionOperandMatcher>(Val: OP)) {
593 std::unique_ptr<PredicateMatcher> Result = std::move(OP);
594 OM->eraseNullPredicates();
595 return Result;
596 }
597
598 llvm_unreachable("Trying to pop a condition from an InstructionMatcher with "
599 "no conditions");
600}
601
602GISelFlags RuleMatcher::updateGISelFlag(GISelFlags CurFlags, const Record *R,
603 StringRef FlagName,
604 GISelFlags FlagBit) {
605 // If the value of a flag is unset, ignore it.
606 // If it's set, it always takes precedence over the existing value so
607 // clear/set the corresponding bit.
608 bool Unset = false;
609 bool Value = R->getValueAsBitOrUnset(FieldName: "GIIgnoreCopies", Unset);
610 if (!Unset)
611 return Value ? (CurFlags | FlagBit) : (CurFlags & ~FlagBit);
612 return CurFlags;
613}
614
615SaveAndRestore<GISelFlags> RuleMatcher::setGISelFlags(const Record *R) {
616 if (!R || !R->isSubClassOf(Name: "GISelFlags"))
617 return {Flags, Flags};
618
619 assert((R->isSubClassOf("PatFrags") || R->isSubClassOf("Pattern")) &&
620 "GISelFlags is only expected on Pattern/PatFrags!");
621
622 GISelFlags NewFlags =
623 updateGISelFlag(CurFlags: Flags, R, FlagName: "GIIgnoreCopies", FlagBit: GISF_IgnoreCopies);
624 return {Flags, NewFlags};
625}
626
627Error RuleMatcher::defineComplexSubOperand(StringRef SymbolicName,
628 const Record *ComplexPattern,
629 unsigned RendererID,
630 unsigned SubOperandID,
631 StringRef ParentSymbolicName) {
632 std::string ParentName(ParentSymbolicName);
633 auto [It, Inserted] = ComplexSubOperands.try_emplace(
634 Key: SymbolicName, Args&: ComplexPattern, Args&: RendererID, Args&: SubOperandID);
635 if (!Inserted) {
636 const std::string &RecordedParentName =
637 ComplexSubOperandsParentName[SymbolicName];
638 if (RecordedParentName != ParentName)
639 return failUnsupported(Reason: "Error: Complex suboperand " + SymbolicName +
640 " referenced by different operands: " +
641 RecordedParentName + " and " + ParentName + ".");
642 // Complex suboperand referenced more than once from same the operand is
643 // used to generate 'same operand check'. Emitting of
644 // GIR_ComplexSubOperandRenderer for them is already handled.
645 return Error::success();
646 }
647
648 ComplexSubOperandsParentName[SymbolicName] = std::move(ParentName);
649
650 return Error::success();
651}
652
653InstructionMatcher &
654RuleMatcher::allocateInstructionMatcher(StringRef SymbolicName,
655 bool AllowNumOpsCheck) {
656 return *InsnMatchers.emplace_back(args: std::make_unique<InstructionMatcher>(
657 args&: *this, args: InsnMatchers.size(), args&: SymbolicName, args&: AllowNumOpsCheck));
658}
659
660InstructionMatcher &RuleMatcher::addInstructionMatcher(StringRef SymbolicName) {
661 auto &Res = allocateInstructionMatcher(SymbolicName);
662 Roots.push_back(Elt: &Res);
663 MutatableInsns.insert(Ptr: &Res);
664 return Res;
665}
666
667void RuleMatcher::addRequiredSimplePredicate(StringRef PredName) {
668 RequiredSimplePredicates.push_back(x: PredName.str());
669}
670
671const std::vector<std::string> &RuleMatcher::getRequiredSimplePredicates() {
672 return RequiredSimplePredicates;
673}
674
675void RuleMatcher::defineOperand(StringRef SymbolicName, OperandMatcher &OM) {
676 if (DefinedOperands.try_emplace(Key: SymbolicName, Args: &OM).second)
677 return;
678
679 // If the operand is already defined, then we must ensure both references in
680 // the matcher have the exact same node.
681 RuleMatcher &RM = OM.getInstructionMatcher().getRuleMatcher();
682 auto &OtherOM = getOperandMatcher(Name: OM.getSymbolicName());
683 OM.addPredicate<SameOperandMatcher>(args: OtherOM.getInsnVarID(),
684 args: OtherOM.getOpIdx(), args: RM.getGISelFlags());
685}
686
687void RuleMatcher::definePhysRegOperand(const Record *Reg, OperandMatcher &OM) {
688 PhysRegOperands.try_emplace(Key: Reg, Args: &OM);
689}
690
691InstructionMatcher &
692RuleMatcher::getInstructionMatcher(StringRef SymbolicName) const {
693 for (const auto &InsnMatcher : InsnMatchers) {
694 if (InsnMatcher->getSymbolicName() == SymbolicName)
695 return *InsnMatcher;
696 }
697 llvm_unreachable(
698 ("Failed to lookup instruction " + SymbolicName).str().c_str());
699}
700
701const OperandMatcher &
702RuleMatcher::getPhysRegOperandMatcher(const Record *Reg) const {
703 const auto &I = PhysRegOperands.find(Key: Reg);
704
705 if (I == PhysRegOperands.end()) {
706 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Register " + Reg->getName() +
707 " was not declared in matcher");
708 }
709
710 return *I->second;
711}
712
713OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) {
714 const auto &I = DefinedOperands.find(Key: Name);
715
716 if (I == DefinedOperands.end())
717 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Operand " + Name + " was not declared in matcher");
718
719 return *I->second;
720}
721
722const OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) const {
723 const auto &I = DefinedOperands.find(Key: Name);
724
725 if (I == DefinedOperands.end())
726 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Operand " + Name + " was not declared in matcher");
727
728 return *I->second;
729}
730
731void RuleMatcher::emit(MatchTable &Table) {
732 if (Roots.empty())
733 llvm_unreachable("Unexpected empty matcher!");
734
735 // The representation supports rules that require multiple roots such as:
736 // %ptr(p0) = ...
737 // %elt0(s32) = G_LOAD %ptr
738 // %1(p0) = G_ADD %ptr, 4
739 // %elt1(s32) = G_LOAD p0 %1
740 // which could be usefully folded into:
741 // %ptr(p0) = ...
742 // %elt0(s32), %elt1(s32) = TGT_LOAD_PAIR %ptr
743 // on some targets but we don't need to make use of that yet.
744 assert(Roots.size() == 1 && "Cannot handle multi-root matchers yet");
745
746 unsigned LabelID = Table.allocateLabelID();
747
748 if (!RequiredFeatures.empty() || HwModeIdx >= 0) {
749 Table << MatchTable::Opcode(Opcode: "GIM_Try_CheckFeatures", IndentAdjust: +1)
750 << MatchTable::Comment(Comment: "On fail goto")
751 << MatchTable::JumpTarget(LabelID)
752 << MatchTable::NamedValue(
753 NumBytes: 2, NamedValue: getNameForFeatureBitset(FeatureBitset: RequiredFeatures, HwModeIdx));
754 } else {
755 Table << MatchTable::Opcode(Opcode: "GIM_Try", IndentAdjust: +1)
756 << MatchTable::Comment(Comment: "On fail goto")
757 << MatchTable::JumpTarget(LabelID);
758 }
759 Table << MatchTable::Comment(Comment: ("Rule ID " + Twine(RuleID) + " //").str())
760 << MatchTable::LineBreak;
761
762 if (!RequiredSimplePredicates.empty()) {
763 for (const auto &Pred : RequiredSimplePredicates) {
764 Table << MatchTable::Opcode(Opcode: "GIM_CheckSimplePredicate")
765 << MatchTable::NamedValue(NumBytes: 2, NamedValue: Pred) << MatchTable::LineBreak;
766 }
767 }
768
769 Roots.front()->emitPredicateOpcodes(Table);
770
771 // Check if it's safe to replace registers.
772 for (const auto &MA : Actions)
773 MA->emitAdditionalPredicates(Table);
774
775 // We must also check if it's safe to fold the matched instructions.
776 if (InsnMatchers.size() >= 2) {
777
778 // FIXME: Emit checks to determine it's _actually_ safe to fold and/or
779 // account for unsafe cases.
780 //
781 // Example:
782 // MI1--> %0 = ...
783 // %1 = ... %0
784 // MI0--> %2 = ... %0
785 // It's not safe to erase MI1. We currently handle this by not
786 // erasing %0 (even when it's dead).
787 //
788 // Example:
789 // MI1--> %0 = load volatile @a
790 // %1 = load volatile @a
791 // MI0--> %2 = ... %0
792 // It's not safe to sink %0's def past %1. We currently handle
793 // this by rejecting all loads.
794 //
795 // Example:
796 // MI1--> %0 = load @a
797 // %1 = store @a
798 // MI0--> %2 = ... %0
799 // It's not safe to sink %0's def past %1. We currently handle
800 // this by rejecting all loads.
801 //
802 // Example:
803 // G_CONDBR %cond, @BB1
804 // BB0:
805 // MI1--> %0 = load @a
806 // G_BR @BB1
807 // BB1:
808 // MI0--> %2 = ... %0
809 // It's not always safe to sink %0 across control flow. In this
810 // case it may introduce a memory fault. We currentl handle
811 // this by rejecting all loads.
812
813 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsSafeToFold")
814 << MatchTable::Comment(Comment: "NumInsns")
815 << MatchTable::IntValue(NumBytes: 1, IntValue: InsnMatchers.size() - 1)
816 << MatchTable::LineBreak;
817 }
818
819 for (const auto &PM : EpilogueMatchers)
820 PM->emitPredicateOpcodes(Table);
821
822 if (!CustomCXXAction.empty()) {
823 /// Handle combiners relying on custom C++ code instead of actions.
824 assert(Table.isCombiner() && "CustomCXXAction is only for combiners!");
825 // We cannot have actions other than debug comments.
826 assert(none_of(Actions, [](auto &A) {
827 return A->getKind() != MatchAction::AK_DebugComment;
828 }));
829 for (const auto &MA : Actions)
830 MA->emitActionOpcodes(Table);
831 Table << MatchTable::Opcode(Opcode: "GIR_DoneWithCustomAction", IndentAdjust: -1)
832 << MatchTable::Comment(Comment: "Fn")
833 << MatchTable::NamedValue(NumBytes: 2, NamedValue: CustomCXXAction)
834 << MatchTable::LineBreak;
835 } else {
836 // Emit all actions except the last one, then emit coverage and emit the
837 // final action.
838 //
839 // This is because some actions, such as GIR_EraseRootFromParent_Done, also
840 // double as a GIR_Done and terminate execution of the rule.
841 if (!Actions.empty()) {
842 for (const auto &MA : drop_end(RangeOrContainer&: Actions))
843 MA->emitActionOpcodes(Table);
844 }
845
846 // Emit coverage right before the Done opcode>
847 auto EmitCoverage = [&] {
848 assert((Table.isWithCoverage() ? !Table.isCombiner() : true) &&
849 "Combiner tables don't support coverage!");
850 if (Table.isWithCoverage())
851 Table << MatchTable::Opcode(Opcode: "GIR_Coverage")
852 << MatchTable::IntValue(NumBytes: 4, IntValue: RuleID) << MatchTable::LineBreak;
853 else if (!Table.isCombiner())
854 Table << MatchTable::Comment(
855 Comment: ("GIR_Coverage, " + Twine(RuleID) + ",").str())
856 << MatchTable::LineBreak;
857 };
858
859 if (Actions.empty() ||
860 !Actions.back()->emitActionOpcodesAndDone(Table, OnDone: EmitCoverage)) {
861 EmitCoverage();
862 Table << MatchTable::Opcode(Opcode: "GIR_Done", IndentAdjust: -1) << MatchTable::LineBreak;
863 }
864 }
865
866 Table << MatchTable::Label(LabelID);
867 ++NumPatternEmitted;
868}
869
870bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const {
871 // Rules involving more match roots have higher priority.
872 if (Roots.size() > B.Roots.size())
873 return true;
874 if (Roots.size() < B.Roots.size())
875 return false;
876
877 for (auto Matcher : zip(t: Roots, u: B.Roots)) {
878 if (std::get<0>(t&: Matcher)->isHigherPriorityThan(B&: *std::get<1>(t&: Matcher)))
879 return true;
880 if (std::get<1>(t&: Matcher)->isHigherPriorityThan(B&: *std::get<0>(t&: Matcher)))
881 return false;
882 }
883
884 return false;
885}
886
887unsigned RuleMatcher::countRendererFns() const {
888 return std::accumulate(first: Roots.begin(), last: Roots.end(), init: 0,
889 binary_op: [](unsigned A, InstructionMatcher *Matcher) {
890 return A + Matcher->countRendererFns();
891 });
892}
893
894void RuleMatcher::roots_pop_front() { Roots.erase(CI: Roots.begin()); }
895
896//===- PredicateMatcher ---------------------------------------------------===//
897
898PredicateMatcher::~PredicateMatcher() = default;
899
900//===- OperandPredicateMatcher --------------------------------------------===//
901
902OperandPredicateMatcher::~OperandPredicateMatcher() = default;
903
904bool OperandPredicateMatcher::isHigherPriorityThan(
905 const OperandPredicateMatcher &B) const {
906 // Generally speaking, an instruction is more important than an Int or a
907 // LiteralInt because it can cover more nodes but there's an exception to
908 // this. G_CONSTANT's are less important than either of those two because they
909 // are more permissive.
910
911 const auto *AOM = dyn_cast<InstructionOperandMatcher>(Val: this);
912 const auto *BOM = dyn_cast<InstructionOperandMatcher>(Val: &B);
913 bool AIsConstantInsn = AOM && AOM->getInsnMatcher().isConstantInstruction();
914 bool BIsConstantInsn = BOM && BOM->getInsnMatcher().isConstantInstruction();
915
916 // The relative priorities between a G_CONSTANT and any other instruction
917 // don't actually matter but this code is needed to ensure a strict weak
918 // ordering. This is particularly important on Windows where the rules will
919 // be incorrectly sorted without it.
920 if (AOM && BOM)
921 return !AIsConstantInsn && BIsConstantInsn;
922
923 if (AIsConstantInsn && (B.Kind == OPM_Int || B.Kind == OPM_LiteralInt))
924 return false;
925 if (BIsConstantInsn && (Kind == OPM_Int || Kind == OPM_LiteralInt))
926 return true;
927
928 return Kind < B.Kind;
929}
930
931//===- SameOperandMatcher -------------------------------------------------===//
932
933void SameOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
934 const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
935 Table << MatchTable::Opcode(Opcode: IgnoreCopies
936 ? "GIM_CheckIsSameOperandIgnoreCopies"
937 : "GIM_CheckIsSameOperand")
938 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
939 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OpIdx)
940 << MatchTable::Comment(Comment: "OtherMI")
941 << MatchTable::ULEB128Value(IntValue: OtherInsnID)
942 << MatchTable::Comment(Comment: "OtherOpIdx")
943 << MatchTable::ULEB128Value(IntValue: OtherOpIdx) << MatchTable::LineBreak;
944}
945
946//===- LLTOperandMatcher --------------------------------------------------===//
947
948std::map<LLTCodeGen, unsigned> LLTOperandMatcher::TypeIDValues;
949
950RecordAndValue LLTOperandMatcher::getValue() const {
951 assert(!HwModeTy && "HwMode dependent types don't have a value");
952 const auto VI = TypeIDValues.find(x: Ty);
953 if (VI == TypeIDValues.end())
954 return MatchTable::NamedValue(NumBytes: 1, NamedValue: getTy().getCxxEnumValue());
955 return {MatchTable::NamedValue(NumBytes: 1, NamedValue: getTy().getCxxEnumValue()), VI->second};
956}
957
958bool LLTOperandMatcher::hasValue() const {
959 // HwMode dependent types can't be used in a switch.
960 if (HwModeTy)
961 return false;
962 if (TypeIDValues.size() != KnownTypes.size())
963 initTypeIDValuesMap();
964 return TypeIDValues.count(x: Ty);
965}
966
967void LLTOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
968 if (HwModeTy) {
969 if (InsnVarID == 0) {
970 Table << MatchTable::Opcode(Opcode: "GIM_RootCheckTypeByHwMode");
971 } else {
972 Table << MatchTable::Opcode(Opcode: "GIM_CheckTypeByHwMode")
973 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID);
974 }
975 Table << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
976 << MatchTable::Comment(Comment: "TypeByHwMode")
977 << MatchTable::IntValue(NumBytes: 1, IntValue: getLLTByHwModeID(Ty: *HwModeTy))
978 << MatchTable::LineBreak;
979 return;
980 }
981
982 if (InsnVarID == 0) {
983 Table << MatchTable::Opcode(Opcode: "GIM_RootCheckType");
984 } else {
985 Table << MatchTable::Opcode(Opcode: "GIM_CheckType") << MatchTable::Comment(Comment: "MI")
986 << MatchTable::ULEB128Value(IntValue: InsnVarID);
987 }
988 Table << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
989 << MatchTable::Comment(Comment: "Type") << getValue().Record
990 << MatchTable::LineBreak;
991}
992
993//===- PointerToAnyOperandMatcher -----------------------------------------===//
994
995void PointerToAnyOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
996 if (HwModeTy) {
997 Table << MatchTable::Opcode(Opcode: "GIM_CheckPointerToAnyByHwMode")
998 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
999 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1000 << MatchTable::Comment(Comment: "TypeByHwMode")
1001 << MatchTable::IntValue(NumBytes: 1, IntValue: getLLTByHwModeID(Ty: *HwModeTy))
1002 << MatchTable::LineBreak;
1003 return;
1004 }
1005
1006 Table << MatchTable::Opcode(Opcode: "GIM_CheckPointerToAny")
1007 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1008 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1009 << MatchTable::Comment(Comment: "SizeInBits")
1010 << MatchTable::ULEB128Value(IntValue: SizeInBits) << MatchTable::LineBreak;
1011}
1012
1013//===- RecordNamedOperandMatcher ------------------------------------------===//
1014
1015void RecordNamedOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1016 Table << MatchTable::Opcode(Opcode: "GIM_RecordNamedOperand")
1017 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1018 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1019 << MatchTable::Comment(Comment: "StoreIdx") << MatchTable::ULEB128Value(IntValue: StoreIdx)
1020 << MatchTable::Comment(Comment: "Name : " + Name) << MatchTable::LineBreak;
1021}
1022
1023//===- RecordRegisterType ------------------------------------------===//
1024
1025void RecordRegisterType::emitPredicateOpcodes(MatchTable &Table) const {
1026 assert(Idx < 0 && "Temp types always have negative indexes!");
1027 Table << MatchTable::Opcode(Opcode: "GIM_RecordRegType") << MatchTable::Comment(Comment: "MI")
1028 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
1029 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::Comment(Comment: "TempTypeIdx")
1030 << MatchTable::IntValue(NumBytes: 1, IntValue: Idx) << MatchTable::LineBreak;
1031}
1032
1033//===- ComplexPatternOperandMatcher ---------------------------------------===//
1034
1035void ComplexPatternOperandMatcher::emitPredicateOpcodes(
1036 MatchTable &Table) const {
1037 unsigned ID = getAllocatedTemporariesBaseID();
1038 Table << MatchTable::Opcode(Opcode: "GIM_CheckComplexPattern")
1039 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1040 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1041 << MatchTable::Comment(Comment: "Renderer") << MatchTable::IntValue(NumBytes: 2, IntValue: ID)
1042 << MatchTable::NamedValue(NumBytes: 2, NamedValue: ("GICP_" + TheDef.getName()).str())
1043 << MatchTable::LineBreak;
1044}
1045
1046unsigned ComplexPatternOperandMatcher::getAllocatedTemporariesBaseID() const {
1047 return Operand.getAllocatedTemporariesBaseID();
1048}
1049
1050//===- RegisterBankOperandMatcher -----------------------------------------===//
1051
1052bool RegisterBankOperandMatcher::isIdentical(const PredicateMatcher &B) const {
1053 return OperandPredicateMatcher::isIdentical(B) &&
1054 RC.getDef() == cast<RegisterBankOperandMatcher>(Val: &B)->RC.getDef();
1055}
1056
1057void RegisterBankOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1058 if (InsnVarID == 0) {
1059 Table << MatchTable::Opcode(Opcode: "GIM_RootCheckRegBankForClass");
1060 } else {
1061 Table << MatchTable::Opcode(Opcode: "GIM_CheckRegBankForClass")
1062 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID);
1063 }
1064
1065 Table << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1066 << MatchTable::Comment(Comment: "RC")
1067 << MatchTable::NamedValue(NumBytes: 2, NamedValue: RC.getQualifiedIdName())
1068 << MatchTable::LineBreak;
1069}
1070
1071//===- MBBOperandMatcher --------------------------------------------------===//
1072
1073void MBBOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1074 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsMBB") << MatchTable::Comment(Comment: "MI")
1075 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
1076 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
1077}
1078
1079//===- ImmOperandMatcher --------------------------------------------------===//
1080
1081void ImmOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1082 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsImm") << MatchTable::Comment(Comment: "MI")
1083 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
1084 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
1085}
1086
1087//===- ConstantIntOperandMatcher ------------------------------------------===//
1088
1089void ConstantIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1090 const bool IsInt8 = isInt<8>(x: Value);
1091 Table << MatchTable::Opcode(Opcode: IsInt8 ? "GIM_CheckConstantInt8"
1092 : "GIM_CheckConstantInt")
1093 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1094 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1095 << MatchTable::IntValue(NumBytes: IsInt8 ? 1 : 8, IntValue: Value) << MatchTable::LineBreak;
1096}
1097
1098//===- LiteralIntOperandMatcher -------------------------------------------===//
1099
1100void LiteralIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1101 Table << MatchTable::Opcode(Opcode: "GIM_CheckLiteralInt")
1102 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1103 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1104 << MatchTable::IntValue(NumBytes: 8, IntValue: Value) << MatchTable::LineBreak;
1105}
1106
1107//===- CmpPredicateOperandMatcher -----------------------------------------===//
1108
1109void CmpPredicateOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1110 Table << MatchTable::Opcode(Opcode: "GIM_CheckCmpPredicate")
1111 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1112 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1113 << MatchTable::Comment(Comment: "Predicate")
1114 << MatchTable::NamedValue(NumBytes: 2, Namespace: "CmpInst", NamedValue: PredName)
1115 << MatchTable::LineBreak;
1116}
1117
1118//===- IntrinsicIDOperandMatcher ------------------------------------------===//
1119
1120void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1121 Table << MatchTable::Opcode(Opcode: "GIM_CheckIntrinsicID")
1122 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1123 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1124 << MatchTable::NamedValue(NumBytes: 2, NamedValue: "Intrinsic::" + II->EnumName.str())
1125 << MatchTable::LineBreak;
1126}
1127
1128//===- OperandImmPredicateMatcher -----------------------------------------===//
1129
1130void OperandImmPredicateMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1131 Table << MatchTable::Opcode(Opcode: "GIM_CheckImmOperandPredicate")
1132 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1133 << MatchTable::Comment(Comment: "MO") << MatchTable::ULEB128Value(IntValue: OpIdx)
1134 << MatchTable::Comment(Comment: "Predicate")
1135 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1136 << MatchTable::LineBreak;
1137}
1138
1139//===- OperandLeafPredicateMatcher ----------------------------------------===//
1140
1141void OperandLeafPredicateMatcher::emitPredicateOpcodes(
1142 MatchTable &Table) const {
1143 Table << MatchTable::Opcode(Opcode: "GIM_CheckLeafOperandPredicate")
1144 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1145 << MatchTable::Comment(Comment: "MO") << MatchTable::ULEB128Value(IntValue: OpIdx)
1146 << MatchTable::Comment(Comment: "Predicate")
1147 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1148 << MatchTable::LineBreak;
1149}
1150
1151//===- OperandMatcher -----------------------------------------------------===//
1152
1153std::string OperandMatcher::getOperandExpr(unsigned InsnVarID) const {
1154 return "State.MIs[" + llvm::to_string(Value: InsnVarID) + "]->getOperand(" +
1155 llvm::to_string(Value: OpIdx) + ")";
1156}
1157
1158unsigned OperandMatcher::getInsnVarID() const { return Insn.getInsnVarID(); }
1159
1160TempTypeIdx OperandMatcher::getTempTypeIdx(RuleMatcher &Rule) {
1161 assert(!IsVariadic && "Cannot use this on variadic operands!");
1162 if (TTIdx >= 0) {
1163 // Temp type index not assigned yet, so assign one and add the necessary
1164 // predicate.
1165 TTIdx = Rule.getNextTempTypeIdx();
1166 assert(TTIdx < 0);
1167 addPredicate<RecordRegisterType>(args&: TTIdx);
1168 return TTIdx;
1169 }
1170 return TTIdx;
1171}
1172
1173bool OperandMatcher::recordsOperand() const {
1174 return matchersRecordOperand(R: Predicates);
1175}
1176
1177void OperandMatcher::emitPredicateOpcodes(MatchTable &Table) {
1178 if (!Optimized) {
1179 std::string Comment;
1180 raw_string_ostream CommentOS(Comment);
1181 CommentOS << "MIs[" << getInsnVarID() << "] ";
1182 if (SymbolicName.empty())
1183 CommentOS << "Operand " << OpIdx;
1184 else
1185 CommentOS << SymbolicName;
1186 Table << MatchTable::Comment(Comment) << MatchTable::LineBreak;
1187 }
1188
1189 emitPredicateListOpcodes(Table);
1190}
1191
1192bool OperandMatcher::isHigherPriorityThan(OperandMatcher &B) {
1193 // Operand matchers involving more predicates have higher priority.
1194 if (predicates_size() > B.predicates_size())
1195 return true;
1196 if (predicates_size() < B.predicates_size())
1197 return false;
1198
1199 // This assumes that predicates are added in a consistent order.
1200 for (auto &&Predicate : zip(t: predicates(), u: B.predicates())) {
1201 if (std::get<0>(t&: Predicate)->isHigherPriorityThan(B: *std::get<1>(t&: Predicate)))
1202 return true;
1203 if (std::get<1>(t&: Predicate)->isHigherPriorityThan(B: *std::get<0>(t&: Predicate)))
1204 return false;
1205 }
1206
1207 return false;
1208}
1209
1210unsigned OperandMatcher::countRendererFns() {
1211 return std::accumulate(
1212 first: predicates().begin(), last: predicates().end(), init: 0,
1213 binary_op: [](unsigned A,
1214 const std::unique_ptr<OperandPredicateMatcher> &Predicate) {
1215 return A + Predicate->countRendererFns();
1216 });
1217}
1218
1219Error OperandMatcher::addTypeCheckPredicate(const TypeSetByHwMode &VTy,
1220 bool OperandIsAPointer) {
1221 if (!VTy.isMachineValueType()) {
1222 // The type depends on the HwMode.
1223 std::optional<LLTOrLLTByHwMode> Ty = getLLTOrLLTByHwMode(VTy);
1224 if (!Ty)
1225 return failUnsupported(Reason: "unsupported typeset");
1226
1227 if (const auto *LLT = std::get_if<LLTCodeGen>(ptr: &*Ty)) {
1228 if (OperandIsAPointer)
1229 addPredicate<PointerToAnyOperandMatcher>(args: LLT->get().getSizeInBits());
1230 else
1231 addPredicate<LLTOperandMatcher>(args: *LLT);
1232 } else if (OperandIsAPointer) {
1233 addPredicate<PointerToAnyOperandMatcher>(args&: std::get<LLTByHwMode>(v&: *Ty));
1234 } else {
1235 addPredicate<LLTOperandMatcher>(args&: std::get<LLTByHwMode>(v&: *Ty));
1236 }
1237 return Error::success();
1238 }
1239
1240 if ((VTy.getMachineValueType() == MVT::iPTR ||
1241 VTy.getMachineValueType() == MVT::cPTR) &&
1242 OperandIsAPointer) {
1243 addPredicate<PointerToAnyOperandMatcher>(args: 0);
1244 return Error::success();
1245 }
1246
1247 // Metadata operands have no LLT representation and no runtime type check is
1248 // needed — they are guaranteed to be MO_Metadata by the IRTranslator. This
1249 // mirrors how srcvalue is handled in importChildMatcher.
1250 if (VTy.getMachineValueType() == MVT::Metadata)
1251 return Error::success();
1252
1253 auto OpTyOrNone = MVTToLLT(VT: VTy.getMachineValueType().SimpleTy);
1254 if (!OpTyOrNone)
1255 return failUnsupported(Reason: "unsupported type");
1256
1257 if (OperandIsAPointer)
1258 addPredicate<PointerToAnyOperandMatcher>(args: OpTyOrNone->get().getSizeInBits());
1259 else if (VTy.isPointer())
1260 addPredicate<LLTOperandMatcher>(
1261 args: LLT::pointer(AddressSpace: VTy.getPtrAddrSpace(), SizeInBits: OpTyOrNone->get().getSizeInBits()));
1262 else
1263 addPredicate<LLTOperandMatcher>(args&: *OpTyOrNone);
1264 return Error::success();
1265}
1266
1267//===- InstructionOpcodeMatcher -------------------------------------------===//
1268
1269DenseMap<const CodeGenInstruction *, unsigned>
1270 InstructionOpcodeMatcher::OpcodeValues;
1271
1272RecordAndValue
1273InstructionOpcodeMatcher::getInstValue(const CodeGenInstruction *I) const {
1274 const auto VI = OpcodeValues.find(Val: I);
1275 if (VI != OpcodeValues.end())
1276 return {MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName()), VI->second};
1277 return MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName());
1278}
1279
1280void InstructionOpcodeMatcher::initOpcodeValuesMap(
1281 const CodeGenTarget &Target) {
1282 OpcodeValues.clear();
1283
1284 for (const CodeGenInstruction *I : Target.getInstructions())
1285 OpcodeValues[I] = Target.getInstrIntValue(R: I->TheDef);
1286}
1287
1288RecordAndValue InstructionOpcodeMatcher::getValue() const {
1289 assert(Insts.size() == 1);
1290
1291 const CodeGenInstruction *I = Insts[0];
1292 const auto VI = OpcodeValues.find(Val: I);
1293 if (VI != OpcodeValues.end())
1294 return {MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName()), VI->second};
1295 return MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName());
1296}
1297
1298void InstructionOpcodeMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1299 StringRef CheckType =
1300 Insts.size() == 1 ? "GIM_CheckOpcode" : "GIM_CheckOpcodeIsEither";
1301 Table << MatchTable::Opcode(Opcode: CheckType) << MatchTable::Comment(Comment: "MI")
1302 << MatchTable::ULEB128Value(IntValue: InsnVarID);
1303
1304 for (const CodeGenInstruction *I : Insts)
1305 Table << getInstValue(I).Record;
1306 Table << MatchTable::LineBreak;
1307}
1308
1309bool InstructionOpcodeMatcher::isHigherPriorityThan(
1310 const InstructionPredicateMatcher &B) const {
1311 if (InstructionPredicateMatcher::isHigherPriorityThan(B))
1312 return true;
1313 if (B.InstructionPredicateMatcher::isHigherPriorityThan(B: *this))
1314 return false;
1315
1316 // Prioritize opcodes for cosmetic reasons in the generated source. Although
1317 // this is cosmetic at the moment, we may want to drive a similar ordering
1318 // using instruction frequency information to improve compile time.
1319 if (const InstructionOpcodeMatcher *BO =
1320 dyn_cast<InstructionOpcodeMatcher>(Val: &B))
1321 return Insts[0]->getName() < BO->Insts[0]->getName();
1322
1323 return false;
1324}
1325
1326bool InstructionOpcodeMatcher::isConstantInstruction() const {
1327 return Insts.size() == 1 && Insts[0]->getName() == "G_CONSTANT";
1328}
1329
1330StringRef InstructionOpcodeMatcher::getOpcode() const {
1331 return Insts[0]->getName();
1332}
1333
1334bool InstructionOpcodeMatcher::isVariadicNumOperands() const {
1335 // If one is variadic, they all should be.
1336 return Insts[0]->Operands.isVariadic;
1337}
1338
1339StringRef InstructionOpcodeMatcher::getOperandType(unsigned OpIdx) const {
1340 // Types expected to be uniform for all alternatives.
1341 return Insts[0]->Operands[OpIdx].OperandType;
1342}
1343
1344//===- InstructionNumOperandsMatcher --------------------------------------===//
1345
1346void InstructionNumOperandsMatcher::emitPredicateOpcodes(
1347 MatchTable &Table) const {
1348 StringRef Opc;
1349 switch (CK) {
1350 case CheckKind::Eq:
1351 Opc = "GIM_CheckNumOperands";
1352 break;
1353 case CheckKind::GE:
1354 Opc = "GIM_CheckNumOperandsGE";
1355 break;
1356 case CheckKind::LE:
1357 Opc = "GIM_CheckNumOperandsLE";
1358 break;
1359 }
1360 Table << MatchTable::Opcode(Opcode: Opc) << MatchTable::Comment(Comment: "MI")
1361 << MatchTable::ULEB128Value(IntValue: InsnVarID)
1362 << MatchTable::Comment(Comment: "Expected")
1363 << MatchTable::ULEB128Value(IntValue: NumOperands) << MatchTable::LineBreak;
1364}
1365
1366//===- InstructionImmPredicateMatcher -------------------------------------===//
1367
1368bool InstructionImmPredicateMatcher::isIdentical(
1369 const PredicateMatcher &B) const {
1370 return InstructionPredicateMatcher::isIdentical(B) &&
1371 Predicate.getOrigPatFragRecord() ==
1372 cast<InstructionImmPredicateMatcher>(Val: &B)
1373 ->Predicate.getOrigPatFragRecord();
1374}
1375
1376void InstructionImmPredicateMatcher::emitPredicateOpcodes(
1377 MatchTable &Table) const {
1378 Table << MatchTable::Opcode(Opcode: getMatchOpcodeForImmPredicate(Predicate))
1379 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1380 << MatchTable::Comment(Comment: "Predicate")
1381 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1382 << MatchTable::LineBreak;
1383}
1384
1385//===- AtomicOrderingMMOPredicateMatcher ----------------------------------===//
1386
1387bool AtomicOrderingMMOPredicateMatcher::isIdentical(
1388 const PredicateMatcher &B) const {
1389 if (!InstructionPredicateMatcher::isIdentical(B))
1390 return false;
1391 const auto &R = *cast<AtomicOrderingMMOPredicateMatcher>(Val: &B);
1392 return Order == R.Order && Comparator == R.Comparator;
1393}
1394
1395void AtomicOrderingMMOPredicateMatcher::emitPredicateOpcodes(
1396 MatchTable &Table) const {
1397 StringRef Opcode = "GIM_CheckAtomicOrdering";
1398
1399 if (Comparator == AO_OrStronger)
1400 Opcode = "GIM_CheckAtomicOrderingOrStrongerThan";
1401 if (Comparator == AO_WeakerThan)
1402 Opcode = "GIM_CheckAtomicOrderingWeakerThan";
1403
1404 Table << MatchTable::Opcode(Opcode) << MatchTable::Comment(Comment: "MI")
1405 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Order")
1406 << MatchTable::NamedValue(NumBytes: 1,
1407 NamedValue: ("(uint8_t)AtomicOrdering::" + Order).str())
1408 << MatchTable::LineBreak;
1409}
1410
1411//===- MemorySizePredicateMatcher -----------------------------------------===//
1412
1413void MemorySizePredicateMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1414 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemorySizeEqualTo")
1415 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1416 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1417 << MatchTable::Comment(Comment: "Size") << MatchTable::IntValue(NumBytes: 4, IntValue: Size)
1418 << MatchTable::LineBreak;
1419}
1420
1421//===- MemoryAddressSpacePredicateMatcher ---------------------------------===//
1422
1423bool MemoryAddressSpacePredicateMatcher::isIdentical(
1424 const PredicateMatcher &B) const {
1425 if (!InstructionPredicateMatcher::isIdentical(B))
1426 return false;
1427 auto *Other = cast<MemoryAddressSpacePredicateMatcher>(Val: &B);
1428 return MMOIdx == Other->MMOIdx && AddrSpaces == Other->AddrSpaces;
1429}
1430
1431void MemoryAddressSpacePredicateMatcher::emitPredicateOpcodes(
1432 MatchTable &Table) const {
1433 assert(AddrSpaces.size() < 256);
1434 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemoryAddressSpace")
1435 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1436 << MatchTable::Comment(Comment: "MMO")
1437 << MatchTable::ULEB128Value(IntValue: MMOIdx)
1438 // Encode number of address spaces to expect.
1439 << MatchTable::Comment(Comment: "NumAddrSpace")
1440 << MatchTable::IntValue(NumBytes: 1, IntValue: AddrSpaces.size());
1441 for (unsigned AS : AddrSpaces)
1442 Table << MatchTable::Comment(Comment: "AddrSpace") << MatchTable::ULEB128Value(IntValue: AS);
1443
1444 Table << MatchTable::LineBreak;
1445}
1446
1447//===- MemoryAlignmentPredicateMatcher ------------------------------------===//
1448
1449bool MemoryAlignmentPredicateMatcher::isIdentical(
1450 const PredicateMatcher &B) const {
1451 if (!InstructionPredicateMatcher::isIdentical(B))
1452 return false;
1453 auto *Other = cast<MemoryAlignmentPredicateMatcher>(Val: &B);
1454 return MMOIdx == Other->MMOIdx && MinAlign == Other->MinAlign;
1455}
1456
1457void MemoryAlignmentPredicateMatcher::emitPredicateOpcodes(
1458 MatchTable &Table) const {
1459 // TODO: we could support more, just need to emit the right opcode or switch
1460 // to log alignment.
1461 assert(MinAlign < 256);
1462 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemoryAlignment")
1463 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1464 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1465 << MatchTable::Comment(Comment: "MinAlign") << MatchTable::IntValue(NumBytes: 1, IntValue: MinAlign)
1466 << MatchTable::LineBreak;
1467}
1468
1469//===- MemoryVsLLTSizePredicateMatcher ------------------------------------===//
1470
1471bool MemoryVsLLTSizePredicateMatcher::isIdentical(
1472 const PredicateMatcher &B) const {
1473 return InstructionPredicateMatcher::isIdentical(B) &&
1474 MMOIdx == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->MMOIdx &&
1475 Relation == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->Relation &&
1476 OpIdx == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->OpIdx;
1477}
1478
1479void MemoryVsLLTSizePredicateMatcher::emitPredicateOpcodes(
1480 MatchTable &Table) const {
1481 Table << MatchTable::Opcode(
1482 Opcode: Relation == EqualTo ? "GIM_CheckMemorySizeEqualToLLT"
1483 : Relation == GreaterThan ? "GIM_CheckMemorySizeGreaterThanLLT"
1484 : "GIM_CheckMemorySizeLessThanLLT")
1485 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1486 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1487 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OpIdx)
1488 << MatchTable::LineBreak;
1489}
1490
1491//===- VectorSplatImmPredicateMatcher -------------------------------------===//
1492
1493void VectorSplatImmPredicateMatcher::emitPredicateOpcodes(
1494 MatchTable &Table) const {
1495 if (Kind == AllOnes)
1496 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsBuildVectorAllOnes");
1497 else
1498 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsBuildVectorAllZeros");
1499
1500 Table << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID);
1501 Table << MatchTable::LineBreak;
1502}
1503
1504//===- GenericInstructionPredicateMatcher ---------------------------------===//
1505
1506GenericInstructionPredicateMatcher::GenericInstructionPredicateMatcher(
1507 unsigned InsnVarID, TreePredicateFn Predicate)
1508 : GenericInstructionPredicateMatcher(InsnVarID,
1509 getEnumNameForPredicate(Predicate)) {}
1510
1511bool GenericInstructionPredicateMatcher::isIdentical(
1512 const PredicateMatcher &B) const {
1513 return InstructionPredicateMatcher::isIdentical(B) &&
1514 EnumVal ==
1515 static_cast<const GenericInstructionPredicateMatcher &>(B).EnumVal;
1516}
1517void GenericInstructionPredicateMatcher::emitPredicateOpcodes(
1518 MatchTable &Table) const {
1519 Table << MatchTable::Opcode(Opcode: "GIM_CheckCxxInsnPredicate")
1520 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1521 << MatchTable::Comment(Comment: "FnId") << MatchTable::NamedValue(NumBytes: 2, NamedValue: EnumVal)
1522 << MatchTable::LineBreak;
1523}
1524
1525//===- MIFlagsInstructionPredicateMatcher ---------------------------------===//
1526
1527bool MIFlagsInstructionPredicateMatcher::isIdentical(
1528 const PredicateMatcher &B) const {
1529 if (!InstructionPredicateMatcher::isIdentical(B))
1530 return false;
1531 const auto &Other =
1532 static_cast<const MIFlagsInstructionPredicateMatcher &>(B);
1533 return Flags == Other.Flags && CheckNot == Other.CheckNot;
1534}
1535
1536void MIFlagsInstructionPredicateMatcher::emitPredicateOpcodes(
1537 MatchTable &Table) const {
1538 Table << MatchTable::Opcode(Opcode: CheckNot ? "GIM_MIFlagsNot" : "GIM_MIFlags")
1539 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1540 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: Flags, Separator: " | "))
1541 << MatchTable::LineBreak;
1542}
1543
1544//===- InstructionMatcher -------------------------------------------------===//
1545
1546OperandMatcher &
1547InstructionMatcher::addOperand(unsigned OpIdx, const std::string &SymbolicName,
1548 unsigned AllocatedTemporariesBaseID,
1549 bool IsVariadic) {
1550 assert((Operands.empty() || !Operands.back()->isVariadic()) &&
1551 "Cannot add more operands after a variadic operand");
1552 Operands.emplace_back(args: new OperandMatcher(
1553 *this, OpIdx, SymbolicName, AllocatedTemporariesBaseID, IsVariadic));
1554 if (!SymbolicName.empty())
1555 Rule.defineOperand(SymbolicName, OM&: *Operands.back());
1556 return *Operands.back();
1557}
1558
1559OperandMatcher &InstructionMatcher::getOperand(unsigned OpIdx) {
1560 auto I = llvm::find_if(Range&: Operands,
1561 P: [&OpIdx](const std::unique_ptr<OperandMatcher> &X) {
1562 return X->getOpIdx() == OpIdx;
1563 });
1564 if (I != Operands.end())
1565 return **I;
1566 llvm_unreachable("Failed to lookup operand");
1567}
1568
1569OperandMatcher &InstructionMatcher::addPhysRegInput(const Record *Reg,
1570 unsigned OpIdx,
1571 unsigned TempOpIdx) {
1572 assert(SymbolicName.empty());
1573 OperandMatcher *OM = new OperandMatcher(*this, OpIdx, "", TempOpIdx);
1574 Operands.emplace_back(args&: OM);
1575 Rule.definePhysRegOperand(Reg, OM&: *OM);
1576 return *OM;
1577}
1578
1579bool InstructionMatcher::recordsOperand() const {
1580 return matchersRecordOperand(R: Predicates) || matchersRecordOperand(R: operands());
1581}
1582
1583void InstructionMatcher::emitPredicateOpcodes(MatchTable &Table) {
1584 if (canAddNumOperandsCheck()) {
1585 InstructionNumOperandsMatcher(InsnVarID, getNumOperandMatchers())
1586 .emitPredicateOpcodes(Table);
1587 }
1588
1589 // First emit all instruction level predicates need to be verified before we
1590 // can verify operands.
1591 emitFilteredPredicateListOpcodes(
1592 ShouldEmitPredicate: [](const PredicateMatcher &P) { return !P.dependsOnRecordedOperands(); },
1593 Table);
1594
1595 // Emit all operand constraints.
1596 for (const auto &Operand : Operands)
1597 Operand->emitPredicateOpcodes(Table);
1598
1599 // All of the tablegen defined predicates should now be matched. Now emit
1600 // any custom predicates that rely on all generated checks.
1601 emitFilteredPredicateListOpcodes(
1602 ShouldEmitPredicate: [](const PredicateMatcher &P) { return P.dependsOnRecordedOperands(); },
1603 Table);
1604}
1605
1606bool InstructionMatcher::isHigherPriorityThan(InstructionMatcher &B) {
1607 // Instruction matchers involving more operands have higher priority.
1608 if (Operands.size() > B.Operands.size())
1609 return true;
1610 if (Operands.size() < B.Operands.size())
1611 return false;
1612
1613 // Instruction matchers involving more predicates have higher priority.
1614 if (predicates_size() > B.predicates_size())
1615 return true;
1616 if (predicates_size() < B.predicates_size())
1617 return false;
1618
1619 for (auto &&P : zip(t: predicates(), u: B.predicates())) {
1620 auto L = static_cast<InstructionPredicateMatcher *>(std::get<0>(t&: P).get());
1621 auto R = static_cast<InstructionPredicateMatcher *>(std::get<1>(t&: P).get());
1622 if (L->isHigherPriorityThan(B: *R))
1623 return true;
1624 if (R->isHigherPriorityThan(B: *L))
1625 return false;
1626 }
1627
1628 for (auto Operand : zip(t&: Operands, u&: B.Operands)) {
1629 if (std::get<0>(t&: Operand)->isHigherPriorityThan(B&: *std::get<1>(t&: Operand)))
1630 return true;
1631 if (std::get<1>(t&: Operand)->isHigherPriorityThan(B&: *std::get<0>(t&: Operand)))
1632 return false;
1633 }
1634
1635 return false;
1636}
1637
1638unsigned InstructionMatcher::countRendererFns() {
1639 return std::accumulate(
1640 first: predicates().begin(), last: predicates().end(), init: 0,
1641 binary_op: [](unsigned A,
1642 const std::unique_ptr<PredicateMatcher> &Predicate) {
1643 return A + Predicate->countRendererFns();
1644 }) +
1645 std::accumulate(
1646 first: Operands.begin(), last: Operands.end(), init: 0,
1647 binary_op: [](unsigned A, const std::unique_ptr<OperandMatcher> &Operand) {
1648 return A + Operand->countRendererFns();
1649 });
1650}
1651
1652void InstructionMatcher::optimize() {
1653 SmallVector<std::unique_ptr<PredicateMatcher>, 8> Stash;
1654 const auto &OpcMatcher = getOpcodeMatcher();
1655
1656 Stash.push_back(Elt: predicates_pop_front());
1657 if (Stash.back().get() == &OpcMatcher) {
1658 // FIXME: Is this even needed still? Why the isVariadicNumOperands check?
1659 if (canAddNumOperandsCheck() && OpcMatcher.isVariadicNumOperands() &&
1660 getNumOperandMatchers() != 0) {
1661 Stash.emplace_back(Args: new InstructionNumOperandsMatcher(
1662 InsnVarID, getNumOperandMatchers()));
1663 }
1664 AllowNumOpsCheck = false;
1665
1666 for (auto &OM : Operands)
1667 for (auto &OP : OM->predicates())
1668 if (isa<IntrinsicIDOperandMatcher>(Val: OP)) {
1669 Stash.push_back(Elt: std::move(OP));
1670 OM->eraseNullPredicates();
1671 break;
1672 }
1673 }
1674
1675 if (InsnVarID > 0) {
1676 assert(!Operands.empty() && "Nested instruction is expected to def a vreg");
1677 for (auto &OP : Operands[0]->predicates()) {
1678 // LLTOperandMatcher need to be kept as they may have a more specific type
1679 // than the parent instruction type.
1680 if (const auto *LLTPred = dyn_cast<LLTOperandMatcher>(Val: &*OP)) {
1681 if (LLTPred->isHwModeDependent() ||
1682 (!LLTPred->getTy().get().isAnyScalar() &&
1683 !LLTPred->getTy().get().isAnyVector()))
1684 continue;
1685 }
1686 OP.reset();
1687 }
1688 Operands[0]->eraseNullPredicates();
1689 }
1690 for (auto &OM : Operands) {
1691 for (auto &OP : OM->predicates())
1692 if (isa<LLTOperandMatcher>(Val: OP) || isa<LLTOperandShapeMatcher>(Val: OP))
1693 Stash.push_back(Elt: std::move(OP));
1694 OM->eraseNullPredicates();
1695 }
1696 while (!Stash.empty())
1697 prependPredicate(Predicate: Stash.pop_back_val());
1698}
1699
1700//===- InstructionOperandMatcher ------------------------------------------===//
1701
1702void InstructionOperandMatcher::emitCaptureOpcodes(MatchTable &Table) const {
1703 const unsigned NewInsnVarID = InsnMatcher.getInsnVarID();
1704 const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
1705 Table << MatchTable::Opcode(Opcode: IgnoreCopies ? "GIM_RecordInsnIgnoreCopies"
1706 : "GIM_RecordInsn")
1707 << MatchTable::Comment(Comment: "DefineMI")
1708 << MatchTable::ULEB128Value(IntValue: NewInsnVarID) << MatchTable::Comment(Comment: "MI")
1709 << MatchTable::ULEB128Value(IntValue: getInsnVarID())
1710 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: getOpIdx())
1711 << MatchTable::Comment(Comment: "MIs[" + llvm::to_string(Value: NewInsnVarID) + "]")
1712 << MatchTable::LineBreak;
1713}
1714
1715bool InstructionOperandMatcher::isHigherPriorityThan(
1716 const OperandPredicateMatcher &B) const {
1717 if (OperandPredicateMatcher::isHigherPriorityThan(B))
1718 return true;
1719 if (B.OperandPredicateMatcher::isHigherPriorityThan(B: *this))
1720 return false;
1721
1722 if (const InstructionOperandMatcher *BP =
1723 dyn_cast<InstructionOperandMatcher>(Val: &B))
1724 if (InsnMatcher.isHigherPriorityThan(B&: BP->InsnMatcher))
1725 return true;
1726 return false;
1727}
1728
1729//===- OperandRenderer ----------------------------------------------------===//
1730
1731OperandRenderer::~OperandRenderer() = default;
1732
1733//===- CopyRenderer -------------------------------------------------------===//
1734
1735void CopyRenderer::emitRenderOpcodes(MatchTable &Table, unsigned NewInsnID,
1736 unsigned OldInsnID, unsigned OldOpIdx,
1737 StringRef Name, bool ForVariadic) {
1738 if (!ForVariadic && NewInsnID == 0 && OldInsnID == 0) {
1739 Table << MatchTable::Opcode(Opcode: "GIR_RootToRootCopy");
1740 } else {
1741 Table << MatchTable::Opcode(Opcode: ForVariadic ? "GIR_CopyRemaining" : "GIR_Copy")
1742 << MatchTable::Comment(Comment: "NewInsnID")
1743 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1744 << MatchTable::Comment(Comment: "OldInsnID")
1745 << MatchTable::ULEB128Value(IntValue: OldInsnID);
1746 }
1747
1748 Table << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1749 << MatchTable::Comment(Comment: Name) << MatchTable::LineBreak;
1750}
1751
1752void CopyRenderer::emitRenderOpcodes(MatchTable &Table) const {
1753 emitRenderOpcodes(Table, NewInsnID, OldInsnID, OldOpIdx, Name: SymbolicName,
1754 ForVariadic: OldOpIsVariadic);
1755}
1756
1757//===- CopyPhysRegRenderer ------------------------------------------------===//
1758
1759void CopyPhysRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1760 CopyRenderer::emitRenderOpcodes(Table, NewInsnID, OldInsnID, OldOpIdx,
1761 Name: PhysReg->getName());
1762}
1763
1764//===- CopyOrAddZeroRegRenderer -------------------------------------------===//
1765
1766void CopyOrAddZeroRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1767 Table << MatchTable::Opcode(Opcode: "GIR_CopyOrAddZeroReg")
1768 << MatchTable::Comment(Comment: "NewInsnID")
1769 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1770 << MatchTable::Comment(Comment: "OldInsnID")
1771 << MatchTable::ULEB128Value(IntValue: OldInsnID) << MatchTable::Comment(Comment: "OpIdx")
1772 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1773 << MatchTable::NamedValue(
1774 NumBytes: 2,
1775 Namespace: (ZeroRegisterDef->getValue(Name: "Namespace")
1776 ? ZeroRegisterDef->getValueAsString(FieldName: "Namespace")
1777 : ""),
1778 NamedValue: ZeroRegisterDef->getName())
1779 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1780}
1781
1782//===- CopyConstantAsImmRenderer ------------------------------------------===//
1783
1784void CopyConstantAsImmRenderer::emitRenderOpcodes(MatchTable &Table) const {
1785 Table << MatchTable::Opcode(Opcode: Signed ? "GIR_CopyConstantAsSImm"
1786 : "GIR_CopyConstantAsUImm")
1787 << MatchTable::Comment(Comment: "NewInsnID")
1788 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1789 << MatchTable::Comment(Comment: "OldInsnID")
1790 << MatchTable::ULEB128Value(IntValue: OldInsnID)
1791 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1792}
1793
1794//===- CopyFConstantAsFPImmRenderer ---------------------------------------===//
1795
1796void CopyFConstantAsFPImmRenderer::emitRenderOpcodes(MatchTable &Table) const {
1797 Table << MatchTable::Opcode(Opcode: "GIR_CopyFConstantAsFPImm")
1798 << MatchTable::Comment(Comment: "NewInsnID")
1799 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1800 << MatchTable::Comment(Comment: "OldInsnID")
1801 << MatchTable::ULEB128Value(IntValue: OldInsnID)
1802 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1803}
1804
1805//===- CopySubRegRenderer -------------------------------------------------===//
1806
1807void CopySubRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1808 Table << MatchTable::Opcode(Opcode: "GIR_CopySubReg")
1809 << MatchTable::Comment(Comment: "NewInsnID")
1810 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1811 << MatchTable::Comment(Comment: "OldInsnID")
1812 << MatchTable::ULEB128Value(IntValue: OldInsnID) << MatchTable::Comment(Comment: "OpIdx")
1813 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1814 << MatchTable::Comment(Comment: "SubRegIdx")
1815 << MatchTable::IntValue(NumBytes: 2, IntValue: SubReg->EnumValue)
1816 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1817}
1818
1819//===- AddRegisterRenderer ------------------------------------------------===//
1820
1821void AddRegisterRenderer::emitRenderOpcodes(MatchTable &Table) const {
1822 Table << MatchTable::Opcode(Opcode: "GIR_AddRegister")
1823 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID);
1824 if (RegisterDef->getName() != "zero_reg") {
1825 Table << MatchTable::NamedValue(
1826 NumBytes: 2,
1827 Namespace: (RegisterDef->getValue(Name: "Namespace")
1828 ? RegisterDef->getValueAsString(FieldName: "Namespace")
1829 : ""),
1830 NamedValue: RegisterDef->getName());
1831 } else {
1832 Table << MatchTable::NamedValue(NumBytes: 2, Namespace: Target.getRegNamespace(), NamedValue: "NoRegister");
1833 }
1834 Table << MatchTable::Comment(Comment: "AddRegisterRegFlags");
1835
1836 // TODO: This is encoded as a 64-bit element, but only 16 or 32-bits are
1837 // really needed for a physical register reference. We can pack the
1838 // register and flags in a single field.
1839 if (IsDef) {
1840 Table << MatchTable::NamedValue(
1841 NumBytes: 2, NamedValue: IsDead ? "static_cast<uint16_t>(RegState::Define|RegState::Dead)"
1842 : "static_cast<uint16_t>(RegState::Define)");
1843 } else {
1844 assert(!IsDead && "A use cannot be dead");
1845 Table << MatchTable::IntValue(NumBytes: 2, IntValue: 0);
1846 }
1847 Table << MatchTable::LineBreak;
1848}
1849
1850//===- TempRegRenderer ----------------------------------------------------===//
1851
1852void TempRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1853 const bool NeedsFlags = (SubRegIdx || IsDef);
1854 if (SubRegIdx) {
1855 assert(!IsDef);
1856 Table << MatchTable::Opcode(Opcode: "GIR_AddTempSubRegister");
1857 } else {
1858 Table << MatchTable::Opcode(Opcode: NeedsFlags ? "GIR_AddTempRegister"
1859 : "GIR_AddSimpleTempRegister");
1860 }
1861
1862 Table << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1863 << MatchTable::Comment(Comment: "TempRegID")
1864 << MatchTable::ULEB128Value(IntValue: TempRegID);
1865
1866 if (!NeedsFlags) {
1867 Table << MatchTable::LineBreak;
1868 return;
1869 }
1870
1871 Table << MatchTable::Comment(Comment: "TempRegFlags");
1872 if (IsDef) {
1873 SmallString<32> RegFlags;
1874 RegFlags += "static_cast<uint16_t>(RegState::Define";
1875 if (IsDead)
1876 RegFlags += "|RegState::Dead";
1877 RegFlags += ")";
1878 Table << MatchTable::NamedValue(NumBytes: 2, NamedValue: RegFlags);
1879 } else {
1880 Table << MatchTable::IntValue(NumBytes: 2, IntValue: 0);
1881 }
1882
1883 if (SubRegIdx)
1884 Table << MatchTable::NamedValue(NumBytes: 2, NamedValue: SubRegIdx->getQualifiedName());
1885 Table << MatchTable::LineBreak;
1886}
1887
1888//===- ImmRenderer --------------------------------------------------------===//
1889
1890void ImmRenderer::emitAddImm(MatchTable &Table, unsigned InsnID, int64_t Imm,
1891 StringRef ImmName) {
1892 const bool IsInt8 = isInt<8>(x: Imm);
1893
1894 Table << MatchTable::Opcode(Opcode: IsInt8 ? "GIR_AddImm8" : "GIR_AddImm")
1895 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1896 << MatchTable::Comment(Comment: ImmName)
1897 << MatchTable::IntValue(NumBytes: IsInt8 ? 1 : 8, IntValue: Imm) << MatchTable::LineBreak;
1898}
1899
1900void ImmRenderer::emitRenderOpcodes(MatchTable &Table) const {
1901 if (ConstantLLT) {
1902 assert(Table.isCombiner() &&
1903 "ConstantInt/ConstantFP immediates are only for combiners!");
1904 Table << MatchTable::Opcode(Opcode: IsFP ? "GIR_AddCFPImm" : "GIR_AddCImm")
1905 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1906 << MatchTable::Comment(Comment: "Type");
1907 emitType(Table, Ty: *ConstantLLT);
1908 Table << MatchTable::Comment(Comment: "Imm") << MatchTable::IntValue(NumBytes: 8, IntValue: Imm)
1909 << MatchTable::LineBreak;
1910 } else {
1911 emitAddImm(Table, InsnID, Imm);
1912 }
1913}
1914
1915//===- SubRegIndexRenderer ------------------------------------------------===//
1916
1917void SubRegIndexRenderer::emitRenderOpcodes(MatchTable &Table) const {
1918 ImmRenderer::emitAddImm(Table, InsnID, Imm: SubRegIdx->EnumValue, ImmName: "SubRegIndex");
1919}
1920
1921//===- RenderComplexPatternOperand ----------------------------------------===//
1922
1923void RenderComplexPatternOperand::emitRenderOpcodes(MatchTable &Table) const {
1924 Table << MatchTable::Opcode(
1925 Opcode: SubOperand ? (SubReg ? "GIR_ComplexSubOperandSubRegRenderer"
1926 : "GIR_ComplexSubOperandRenderer")
1927 : "GIR_ComplexRenderer")
1928 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1929 << MatchTable::Comment(Comment: "RendererID")
1930 << MatchTable::IntValue(NumBytes: 2, IntValue: RendererID);
1931 if (SubOperand)
1932 Table << MatchTable::Comment(Comment: "SubOperand")
1933 << MatchTable::ULEB128Value(IntValue: *SubOperand);
1934 if (SubReg)
1935 Table << MatchTable::Comment(Comment: "SubRegIdx")
1936 << MatchTable::IntValue(NumBytes: 2, IntValue: SubReg->EnumValue);
1937 Table << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1938}
1939
1940//===- IntrinsicIDRenderer ------------------------------------------------===//
1941
1942void IntrinsicIDRenderer::emitRenderOpcodes(MatchTable &Table) const {
1943 Table << MatchTable::Opcode(Opcode: "GIR_AddIntrinsicID") << MatchTable::Comment(Comment: "MI")
1944 << MatchTable::ULEB128Value(IntValue: InsnID)
1945 << MatchTable::NamedValue(NumBytes: 2, NamedValue: "Intrinsic::" + II->EnumName.str())
1946 << MatchTable::LineBreak;
1947}
1948
1949//===- CustomRenderer -----------------------------------------------------===//
1950
1951void CustomRenderer::emitRenderOpcodes(MatchTable &Table) const {
1952 Table << MatchTable::Opcode(Opcode: "GIR_CustomRenderer")
1953 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1954 << MatchTable::Comment(Comment: "OldInsnID")
1955 << MatchTable::ULEB128Value(IntValue: OldInsnID)
1956 << MatchTable::Comment(Comment: "Renderer")
1957 << MatchTable::NamedValue(
1958 NumBytes: 2, NamedValue: "GICR_" + Renderer.getValueAsString(FieldName: "RendererFn").str())
1959 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1960}
1961
1962//===- CustomOperandRenderer ----------------------------------------------===//
1963
1964void CustomOperandRenderer::emitRenderOpcodes(MatchTable &Table) const {
1965 Table << MatchTable::Opcode(Opcode: "GIR_CustomOperandRenderer")
1966 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1967 << MatchTable::Comment(Comment: "OldInsnID")
1968 << MatchTable::ULEB128Value(IntValue: OldInsnID) << MatchTable::Comment(Comment: "OpIdx")
1969 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1970 << MatchTable::Comment(Comment: "OperandRenderer")
1971 << MatchTable::NamedValue(
1972 NumBytes: 2, NamedValue: "GICR_" + Renderer.getValueAsString(FieldName: "RendererFn").str())
1973 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1974}
1975
1976//===- BuildMIAction ------------------------------------------------------===//
1977
1978bool BuildMIAction::canMutate(RuleMatcher &Rule,
1979 const InstructionMatcher *Insn) const {
1980 if (!Insn || Insn->hasVariadicMatcher())
1981 return false;
1982
1983 if (OperandRenderers.size() != Insn->getNumOperandMatchers())
1984 return false;
1985
1986 for (const auto &Renderer : enumerate(First: OperandRenderers)) {
1987 if (const auto *Copy = dyn_cast<CopyRenderer>(Val: &*Renderer.value())) {
1988 const OperandMatcher &OM =
1989 Rule.getOperandMatcher(Name: Copy->getSymbolicName());
1990 if (Insn != &OM.getInstructionMatcher() ||
1991 OM.getOpIdx() != Renderer.index())
1992 return false;
1993 } else {
1994 return false;
1995 }
1996 }
1997
1998 return true;
1999}
2000
2001void BuildMIAction::chooseInsnToMutate(RuleMatcher &Rule) {
2002 for (auto *MutateCandidate : Rule.mutatable_insns()) {
2003 if (canMutate(Rule, Insn: MutateCandidate)) {
2004 // Take the first one we're offered that we're able to mutate.
2005 Rule.reserveInsnMatcherForMutation(InsnMatcher: MutateCandidate);
2006 Matched = MutateCandidate;
2007 Rule.tryEraseInsnID(ID: MutateCandidate->getInsnVarID());
2008 return;
2009 }
2010 }
2011}
2012
2013void BuildMIAction::emitActionOpcodes(MatchTable &Table) const {
2014 const auto AddMIFlags = [&]() {
2015 for (const InstructionMatcher *IM : CopiedFlags) {
2016 Table << MatchTable::Opcode(Opcode: "GIR_CopyMIFlags")
2017 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2018 << MatchTable::Comment(Comment: "OldInsnID")
2019 << MatchTable::ULEB128Value(IntValue: IM->getInsnVarID())
2020 << MatchTable::LineBreak;
2021 }
2022
2023 if (!SetFlags.empty()) {
2024 Table << MatchTable::Opcode(Opcode: "GIR_SetMIFlags")
2025 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2026 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: SetFlags, Separator: " | "))
2027 << MatchTable::LineBreak;
2028 }
2029
2030 if (!UnsetFlags.empty()) {
2031 Table << MatchTable::Opcode(Opcode: "GIR_UnsetMIFlags")
2032 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2033 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: UnsetFlags, Separator: " | "))
2034 << MatchTable::LineBreak;
2035 }
2036 };
2037
2038 if (Matched) {
2039 unsigned RecycleInsnID = Matched->getInsnVarID();
2040 Table << MatchTable::Opcode(Opcode: "GIR_MutateOpcode")
2041 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2042 << MatchTable::Comment(Comment: "RecycleInsnID")
2043 << MatchTable::ULEB128Value(IntValue: RecycleInsnID)
2044 << MatchTable::Comment(Comment: "Opcode")
2045 << MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName())
2046 << MatchTable::LineBreak;
2047
2048 if (!I->ImplicitDefs.empty() || !I->ImplicitUses.empty()) {
2049 for (auto *Def : I->ImplicitDefs) {
2050 auto Namespace = Def->getValue(Name: "Namespace")
2051 ? Def->getValueAsString(FieldName: "Namespace")
2052 : "";
2053 const bool IsDead = DeadImplicitDefs.contains(Ptr: Def);
2054 Table << MatchTable::Opcode(Opcode: "GIR_AddImplicitDef")
2055 << MatchTable::Comment(Comment: "InsnID")
2056 << MatchTable::ULEB128Value(IntValue: InsnID)
2057 << MatchTable::NamedValue(NumBytes: 2, Namespace, NamedValue: Def->getName())
2058 << (IsDead ? MatchTable::NamedValue(
2059 NumBytes: 2, NamedValue: "static_cast<unsigned>(RegState::Dead)")
2060 : MatchTable::IntValue(NumBytes: 2, IntValue: 0))
2061 << MatchTable::LineBreak;
2062 }
2063 for (auto *Use : I->ImplicitUses) {
2064 auto Namespace = Use->getValue(Name: "Namespace")
2065 ? Use->getValueAsString(FieldName: "Namespace")
2066 : "";
2067 Table << MatchTable::Opcode(Opcode: "GIR_AddImplicitUse")
2068 << MatchTable::Comment(Comment: "InsnID")
2069 << MatchTable::ULEB128Value(IntValue: InsnID)
2070 << MatchTable::NamedValue(NumBytes: 2, Namespace, NamedValue: Use->getName())
2071 << MatchTable::LineBreak;
2072 }
2073 }
2074
2075 AddMIFlags();
2076
2077 // Mark the mutated instruction as erased.
2078 return;
2079 }
2080
2081 // TODO: Simple permutation looks like it could be almost as common as
2082 // mutation due to commutative operations.
2083
2084 if (InsnID == 0) {
2085 Table << MatchTable::Opcode(Opcode: "GIR_BuildRootMI");
2086 } else {
2087 Table << MatchTable::Opcode(Opcode: "GIR_BuildMI") << MatchTable::Comment(Comment: "InsnID")
2088 << MatchTable::ULEB128Value(IntValue: InsnID);
2089 }
2090
2091 Table << MatchTable::Comment(Comment: "Opcode")
2092 << MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName())
2093 << MatchTable::LineBreak;
2094
2095 for (const auto &Renderer : OperandRenderers)
2096 Renderer->emitRenderOpcodes(Table);
2097
2098 for (auto [OpIdx, Def] : enumerate(First: I->ImplicitDefs)) {
2099 auto Namespace =
2100 Def->getValue(Name: "Namespace") ? Def->getValueAsString(FieldName: "Namespace") : "";
2101 if (DeadImplicitDefs.contains(Ptr: Def)) {
2102 Table
2103 << MatchTable::Opcode(Opcode: "GIR_SetImplicitDefDead")
2104 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2105 << MatchTable::Comment(
2106 Comment: ("OpIdx for " + Namespace + "::" + Def->getName() + "").str())
2107 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
2108 }
2109 }
2110
2111 if (!MergeInsnIDs.empty()) {
2112 assert(I->mayLoad || I->mayStore);
2113 Table << MatchTable::Opcode(Opcode: "GIR_MergeMemOperands")
2114 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2115 << MatchTable::Comment(Comment: "NumInsns")
2116 << MatchTable::IntValue(NumBytes: 1, IntValue: MergeInsnIDs.size())
2117 << MatchTable::Comment(Comment: "MergeInsnID's");
2118 for (const auto &MergeInsnID : MergeInsnIDs)
2119 Table << MatchTable::ULEB128Value(IntValue: MergeInsnID);
2120 Table << MatchTable::LineBreak;
2121 }
2122
2123 AddMIFlags();
2124}
2125
2126//===- BuildConstantAction ------------------------------------------------===//
2127
2128void BuildConstantAction::emitActionOpcodes(MatchTable &Table) const {
2129 Table << MatchTable::Opcode(Opcode: "GIR_BuildConstant")
2130 << MatchTable::Comment(Comment: "TempRegID")
2131 << MatchTable::ULEB128Value(IntValue: TempRegID) << MatchTable::Comment(Comment: "Val")
2132 << MatchTable::IntValue(NumBytes: 8, IntValue: Val) << MatchTable::LineBreak;
2133}
2134
2135//===- EraseInstAction ----------------------------------------------------===//
2136
2137void EraseInstAction::emitActionOpcodes(MatchTable &Table) const {
2138 Table << MatchTable::Opcode(Opcode: "GIR_EraseFromParent")
2139 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2140 << MatchTable::LineBreak;
2141}
2142
2143bool EraseInstAction::emitActionOpcodesAndDone(
2144 MatchTable &Table, function_ref<void()> OnDone) const {
2145 if (InsnID != 0) {
2146 emitActionOpcodes(Table);
2147 return false;
2148 }
2149
2150 OnDone();
2151 Table << MatchTable::Opcode(Opcode: "GIR_EraseRootFromParent_Done", IndentAdjust: -1)
2152 << MatchTable::LineBreak;
2153 return true;
2154}
2155
2156//===- ReplaceRegAction ---------------------------------------------------===//
2157
2158void ReplaceRegAction::emitAdditionalPredicates(MatchTable &Table) const {
2159 if (TempRegID != (unsigned)-1)
2160 return;
2161
2162 Table << MatchTable::Opcode(Opcode: "GIM_CheckCanReplaceReg")
2163 << MatchTable::Comment(Comment: "OldInsnID")
2164 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2165 << MatchTable::Comment(Comment: "OldOpIdx") << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2166 << MatchTable::Comment(Comment: "NewInsnId")
2167 << MatchTable::ULEB128Value(IntValue: NewInsnId)
2168 << MatchTable::Comment(Comment: "NewOpIdx") << MatchTable::ULEB128Value(IntValue: NewOpIdx)
2169 << MatchTable::LineBreak;
2170}
2171
2172void ReplaceRegAction::emitActionOpcodes(MatchTable &Table) const {
2173 if (TempRegID != (unsigned)-1) {
2174 Table << MatchTable::Opcode(Opcode: "GIR_ReplaceRegWithTempReg")
2175 << MatchTable::Comment(Comment: "OldInsnID")
2176 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2177 << MatchTable::Comment(Comment: "OldOpIdx")
2178 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2179 << MatchTable::Comment(Comment: "TempRegID")
2180 << MatchTable::ULEB128Value(IntValue: TempRegID) << MatchTable::LineBreak;
2181 } else {
2182 Table << MatchTable::Opcode(Opcode: "GIR_ReplaceReg")
2183 << MatchTable::Comment(Comment: "OldInsnID")
2184 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2185 << MatchTable::Comment(Comment: "OldOpIdx")
2186 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2187 << MatchTable::Comment(Comment: "NewInsnId")
2188 << MatchTable::ULEB128Value(IntValue: NewInsnId)
2189 << MatchTable::Comment(Comment: "NewOpIdx")
2190 << MatchTable::ULEB128Value(IntValue: NewOpIdx) << MatchTable::LineBreak;
2191 }
2192}
2193
2194//===- ConstrainOperandToRegClassAction -----------------------------------===//
2195
2196void ConstrainOperandToRegClassAction::emitActionOpcodes(
2197 MatchTable &Table) const {
2198 Table << MatchTable::Opcode(Opcode: "GIR_ConstrainOperandRC")
2199 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2200 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
2201 << MatchTable::NamedValue(NumBytes: 2, NamedValue: RC.getQualifiedIdName())
2202 << MatchTable::LineBreak;
2203}
2204
2205//===- MakeTempRegisterAction ---------------------------------------------===//
2206
2207void MakeTempRegisterAction::emitActionOpcodes(MatchTable &Table) const {
2208 Table << MatchTable::Opcode(Opcode: Ty.isLLTByHwMode() ? "GIR_MakeTempRegByHwMode"
2209 : "GIR_MakeTempReg")
2210 << MatchTable::Comment(Comment: "TempRegID")
2211 << MatchTable::ULEB128Value(IntValue: TempRegID)
2212 << MatchTable::Comment(Comment: Ty.isLLTByHwMode() ? "TypeByHwMode" : "TypeID");
2213 emitType(Table, Ty);
2214 Table << MatchTable::LineBreak;
2215}
2216