1//===- Matchers.cpp -------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Matchers.h"
10#include "Common/CodeGenInstruction.h"
11#include "Common/CodeGenRegisters.h"
12#include "llvm/ADT/Statistic.h"
13#include "llvm/Support/CommandLine.h"
14#include "llvm/Support/Debug.h"
15#include "llvm/Support/LEB128.h"
16#include "llvm/Support/ScopedPrinter.h"
17#include "llvm/Support/raw_ostream.h"
18#include "llvm/TableGen/Error.h"
19
20#define DEBUG_TYPE "gi-match-table-matchers"
21
22STATISTIC(NumPatternEmitted, "Number of patterns emitted");
23
24using namespace llvm;
25using namespace gi;
26
27// FIXME: Use createStringError instead.
28static Error failUnsupported(const Twine &Reason) {
29 return make_error<StringError>(Args: Reason, Args: inconvertibleErrorCode());
30}
31
32/// Get the name of the enum value used to number the predicate function.
33static std::string getEnumNameForPredicate(const TreePredicateFn &Predicate) {
34 if (Predicate.hasGISelPredicateCode())
35 return "GICXXPred_MI_" + Predicate.getFnName();
36 if (Predicate.hasGISelLeafPredicateCode())
37 return "GICXXPred_MO_" + Predicate.getFnName();
38 return "GICXXPred_" + Predicate.getImmTypeIdentifier().str() + "_" +
39 Predicate.getFnName();
40}
41
42static std::string
43getMatchOpcodeForImmPredicate(const TreePredicateFn &Predicate) {
44 return "GIM_Check" + Predicate.getImmTypeIdentifier().str() + "ImmPredicate";
45}
46
47//===- Helpers ------------------------------------------------------------===//
48
49template <class GroupT>
50static std::vector<Matcher *>
51optimizeRules(ArrayRef<Matcher *> Rules,
52 std::vector<std::unique_ptr<Matcher>> &MatcherStorage) {
53
54 std::vector<Matcher *> Worklist(Rules.begin(), Rules.end());
55 std::vector<Matcher *> OptRules;
56 std::unique_ptr<GroupT> CurrentGroup = std::make_unique<GroupT>();
57 assert(CurrentGroup->empty() && "Newly created group isn't empty!");
58 unsigned NumGroups = 0;
59
60 auto ProcessCurrentGroup = [&]() {
61 if (CurrentGroup->empty())
62 // An empty group is good to be reused:
63 return;
64
65 // If the group isn't large enough to provide any benefit, move all the
66 // added rules out of it and make sure to re-create the group to properly
67 // re-initialize it:
68 if (CurrentGroup->size() < 2)
69 append_range(OptRules, CurrentGroup->matchers());
70 else {
71 CurrentGroup->finalize();
72 CurrentGroup->optimize();
73 OptRules.push_back(CurrentGroup.get());
74 MatcherStorage.emplace_back(std::move(CurrentGroup));
75 ++NumGroups;
76 }
77 CurrentGroup = std::make_unique<GroupT>();
78 };
79
80 for (Matcher *Rule : Worklist) {
81 // Greedily add as many matchers as possible to the current group:
82 if (CurrentGroup->addMatcher(*Rule))
83 continue;
84
85 ProcessCurrentGroup();
86 assert(CurrentGroup->empty() && "A group wasn't properly re-initialized");
87
88 // Try to add the pending matcher to a newly created empty group:
89 if (!CurrentGroup->addMatcher(*Rule))
90 // If we couldn't add the matcher to an empty group, that group type
91 // doesn't support that kind of matchers at all, so just skip it:
92 OptRules.push_back(x: Rule);
93 }
94 ProcessCurrentGroup();
95
96 assert(OptRules.size() <= Worklist.size() && "Optimization added rules?");
97 LLVM_DEBUG(dbgs() << "NumGroups: " << NumGroups << "\n");
98 (void)NumGroups;
99 assert(CurrentGroup->empty() && "The last group wasn't properly processed");
100 return OptRules;
101}
102
103std::vector<Matcher *> llvm::gi::optimizeRuleset(
104 MutableArrayRef<RuleMatcher> Rules,
105 std::vector<std::unique_ptr<Matcher>> &MatcherStorage) {
106 SmallVector<Matcher *> InputRules(make_pointer_range(Range&: Rules));
107
108 // Now sort the Rules.
109 unsigned CurrentOrdering = 0;
110 StringMap<unsigned> OpcodeOrder;
111 for (RuleMatcher &Rule : Rules) {
112 const StringRef Opcode = Rule.getOpcode();
113 assert(!Opcode.empty() && "Didn't expect an undefined opcode");
114 if (OpcodeOrder.try_emplace(Key: Opcode, Args&: CurrentOrdering).second)
115 ++CurrentOrdering;
116 }
117
118 llvm::stable_sort(
119 Range&: InputRules, C: [&OpcodeOrder](const Matcher *A, const Matcher *B) {
120 const auto *L = cast<RuleMatcher>(Val: A);
121 const auto *R = cast<RuleMatcher>(Val: B);
122 return std::tuple(OpcodeOrder[L->getOpcode()],
123 L->roots_front().getNumOperandMatchers()) <
124 std::tuple(OpcodeOrder[R->getOpcode()],
125 R->roots_front().getNumOperandMatchers());
126 });
127
128 for (Matcher *R : InputRules)
129 R->optimize();
130
131 // Then form groups, and switches in that order.
132 std::vector<Matcher *> OptRules =
133 optimizeRules<GroupMatcher>(Rules: InputRules, MatcherStorage);
134 OptRules = optimizeRules<SwitchMatcher>(Rules: OptRules, MatcherStorage);
135 return OptRules;
136}
137
138MatchTable llvm::gi::buildMatchTable(ArrayRef<Matcher *> Rules,
139 bool WithCoverage, bool IsCombiner) {
140 MatchTable Table(WithCoverage, IsCombiner);
141 for (Matcher *Rule : Rules)
142 Rule->emit(Table);
143
144 return Table << MatchTable::Opcode(Opcode: "GIM_Reject") << MatchTable::LineBreak;
145}
146
147template <class Range> static bool matchersRecordOperand(Range &&R) {
148 return any_of(R, [](const auto &I) { return I->recordsOperand(); });
149}
150
151static void emitType(MatchTable &Table, const LLTCodeGenOrTempType &Ty) {
152 if (Ty.isLLTCodeGen())
153 Table << MatchTable::NamedValue(NumBytes: 1, NamedValue: Ty.getLLTCodeGen().getCxxEnumValue());
154 else
155 Table << MatchTable::IntValue(NumBytes: 1, IntValue: Ty.getTempTypeIdx());
156}
157
158//===- Matcher ------------------------------------------------------------===//
159
160void Matcher::optimize() {}
161
162Matcher::~Matcher() = default;
163
164//===- GroupMatcher -------------------------------------------------------===//
165
166bool GroupMatcher::recordsOperand() const {
167 return matchersRecordOperand(R: Conditions) || matchersRecordOperand(R: Matchers);
168}
169
170bool GroupMatcher::candidateConditionMatches(
171 const PredicateMatcher &Predicate) const {
172
173 if (empty()) {
174 // Sharing predicates for nested instructions is not supported yet as we
175 // currently don't hoist the GIM_RecordInsn's properly, therefore we can
176 // only work on the original root instruction (InsnVarID == 0):
177 if (Predicate.getInsnVarID() != 0)
178 return false;
179 // ... otherwise an empty group can handle any predicate with no specific
180 // requirements:
181 return true;
182 }
183
184 const Matcher &Representative = **Matchers.begin();
185 const auto &RepresentativeCondition = Representative.getFirstCondition();
186 // ... if not empty, the group can only accomodate matchers with the exact
187 // same first condition:
188 return Predicate.isIdentical(B: RepresentativeCondition);
189}
190
191std::unique_ptr<PredicateMatcher> GroupMatcher::popFirstCondition() {
192 assert(!Conditions.empty() &&
193 "Trying to pop a condition from a condition-less group");
194 std::unique_ptr<PredicateMatcher> P = std::move(Conditions.front());
195 Conditions.erase(CI: Conditions.begin());
196 return P;
197}
198
199bool GroupMatcher::addMatcher(Matcher &Candidate) {
200 if (!Candidate.hasFirstCondition())
201 return false;
202
203 // Only add candidates that have a matching first condition that can be
204 // hoisted into the GroupMatcher.
205 const PredicateMatcher &Predicate = Candidate.getFirstCondition();
206 if (!candidateConditionMatches(Predicate) ||
207 !Predicate.canHoistOutsideOf(M: Candidate))
208 return false;
209
210 Matchers.push_back(x: &Candidate);
211 return true;
212}
213
214void GroupMatcher::finalize() {
215 assert(Conditions.empty() && "Already finalized?");
216 if (empty())
217 return;
218
219 Matcher &FirstRule = **Matchers.begin();
220 for (;;) {
221 // All the checks are expected to succeed during the first iteration:
222 for (const auto &Rule : Matchers)
223 if (!Rule->hasFirstCondition())
224 return;
225 // Hoist the first condition if it is identical in all matchers in the group
226 // and it can be hoisted in every matcher.
227 const auto &FirstCondition = FirstRule.getFirstCondition();
228 if (!FirstCondition.canHoistOutsideOf(M: FirstRule))
229 return;
230 for (unsigned I = 1, E = Matchers.size(); I < E; ++I) {
231 const auto &OtherFirstCondition = Matchers[I]->getFirstCondition();
232 if (!OtherFirstCondition.isIdentical(B: FirstCondition) ||
233 !OtherFirstCondition.canHoistOutsideOf(M: *Matchers[I]))
234 return;
235 }
236
237 Conditions.push_back(Elt: FirstRule.popFirstCondition());
238 for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
239 Matchers[I]->popFirstCondition();
240 }
241}
242
243void GroupMatcher::emit(MatchTable &Table) {
244 unsigned LabelID = ~0U;
245 if (!Conditions.empty()) {
246 LabelID = Table.allocateLabelID();
247 Table << MatchTable::Opcode(Opcode: "GIM_Try", IndentAdjust: +1)
248 << MatchTable::Comment(Comment: "On fail goto")
249 << MatchTable::JumpTarget(LabelID) << MatchTable::LineBreak;
250 }
251 for (auto &Condition : Conditions)
252 Condition->emitPredicateOpcodes(Table);
253
254 for (const auto &M : Matchers)
255 M->emit(Table);
256
257 // Exit the group
258 if (!Conditions.empty())
259 Table << MatchTable::Opcode(Opcode: "GIM_Reject", IndentAdjust: -1) << MatchTable::LineBreak
260 << MatchTable::Label(LabelID);
261}
262
263void GroupMatcher::optimize() {
264 // Make sure we only sort by a specific predicate within a range of rules that
265 // all have that predicate checked against a specific value (not a wildcard):
266 // TODO: Is this even relevant ? Check diffs w/ just using a simple sort
267 // instead of this.
268 auto F = Matchers.begin();
269 auto T = F;
270 auto E = Matchers.end();
271 while (T != E) {
272 while (T != E) {
273 if (!(*T)->getFirstConditionAsRootType().get().isValid())
274 break;
275 ++T;
276 }
277 std::stable_sort(first: F, last: T, comp: [](Matcher *A, Matcher *B) {
278 return A->getFirstConditionAsRootType() <
279 B->getFirstConditionAsRootType();
280 });
281 if (T != E)
282 F = ++T;
283 }
284 Matchers = optimizeRules<GroupMatcher>(Rules: Matchers, MatcherStorage);
285 Matchers = optimizeRules<SwitchMatcher>(Rules: Matchers, MatcherStorage);
286}
287
288LLTCodeGen GroupMatcher::getFirstConditionAsRootType() const {
289 if (!hasFirstCondition())
290 return {};
291
292 const PredicateMatcher &PM = *Conditions.front();
293 if (const auto *TM = dyn_cast<LLTOperandMatcher>(Val: &PM)) {
294 if (TM->getInsnVarID() == 0 && TM->getOpIdx() == 0)
295 return TM->getTy();
296 }
297
298 return {};
299}
300
301//===- SwitchMatcher ------------------------------------------------------===//
302
303SwitchMatcher::SwitchMatcher() : Matcher(MK_Switch) {}
304SwitchMatcher::~SwitchMatcher() = default;
305
306bool SwitchMatcher::recordsOperand() const {
307 assert(!isa_and_present<RecordNamedOperandMatcher>(Condition.get()) &&
308 "Switch conditions should not record named operands");
309 return matchersRecordOperand(R: Matchers);
310}
311
312bool SwitchMatcher::isSupportedPredicateType(const PredicateMatcher &P) {
313 return isa<InstructionOpcodeMatcher>(Val: P) || isa<LLTOperandShapeMatcher>(Val: P) ||
314 isa<LLTOperandMatcher>(Val: P);
315}
316
317bool SwitchMatcher::candidateConditionMatches(
318 const PredicateMatcher &Predicate) const {
319
320 if (empty()) {
321 // Sharing predicates for nested instructions is not supported yet as we
322 // currently don't hoist the GIM_RecordInsn's properly, therefore we can
323 // only work on the original root instruction (InsnVarID == 0):
324 if (Predicate.getInsnVarID() != 0)
325 return false;
326 // ... while an attempt to add even a root matcher to an empty SwitchMatcher
327 // could fail as not all the types of conditions are supported:
328 if (!isSupportedPredicateType(P: Predicate))
329 return false;
330 // ... or the condition might not have a proper implementation of
331 // getValue() / isIdenticalDownToValue() yet:
332 if (!Predicate.hasValue())
333 return false;
334 // ... otherwise an empty Switch can accomodate the condition with no
335 // further requirements:
336 return true;
337 }
338
339 const Matcher &CaseRepresentative = **Matchers.begin();
340 const auto &RepresentativeCondition = CaseRepresentative.getFirstCondition();
341 // Switch-cases must share the same kind of condition and path to the value it
342 // checks:
343 if (!Predicate.isIdenticalDownToValue(B: RepresentativeCondition))
344 return false;
345
346 return true;
347}
348
349bool SwitchMatcher::addMatcher(Matcher &Candidate) {
350 if (!Candidate.hasFirstCondition())
351 return false;
352
353 const PredicateMatcher &Predicate = Candidate.getFirstCondition();
354 if (!candidateConditionMatches(Predicate))
355 return false;
356 const auto Value = Predicate.getValue();
357 auto It = Buckets.find(x: Value.RawValue);
358 if (It == Buckets.end())
359 It = Buckets.emplace(args: Value.RawValue, args: Bucket(Value)).first;
360#ifndef NDEBUG
361 else
362 assert(It->second.Value.Record.EmitStr == Value.Record.EmitStr &&
363 "Mismatched records for identical switch value");
364#endif
365 It->second.Matchers.push_back(x: &Candidate);
366 Matchers.push_back(x: &Candidate);
367 return true;
368}
369
370void SwitchMatcher::finalize() {
371 assert(Condition == nullptr && "Already finalized");
372#ifndef NDEBUG
373 unsigned NumBucketedMatchers = 0;
374 for (const auto &Entry : Buckets)
375 NumBucketedMatchers += Entry.second.Matchers.size();
376 assert(NumBucketedMatchers == Matchers.size() && "Broken SwitchMatcher");
377#endif
378 if (empty())
379 return;
380
381 Condition = Matchers.front()->popFirstCondition();
382 for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
383 Matchers[I]->popFirstCondition();
384
385 // After removing the switch condition, try to hoist any shared predicates
386 // within each switch bucket.
387 for (auto &Entry : Buckets) {
388 auto &BucketMatchers = Entry.second.Matchers;
389 BucketMatchers =
390 optimizeRules<GroupMatcher>(Rules: BucketMatchers, MatcherStorage);
391 }
392
393 Matchers.clear();
394 for (auto &Entry : Buckets)
395 append_range(C&: Matchers, R&: Entry.second.Matchers);
396}
397
398void SwitchMatcher::emitPredicateSpecificOpcodes(const PredicateMatcher &P,
399 MatchTable &Table) {
400 assert(isSupportedPredicateType(P) && "Predicate type is not supported");
401
402 if (const auto *Condition = dyn_cast<InstructionOpcodeMatcher>(Val: &P)) {
403 Table << MatchTable::Opcode(Opcode: "GIM_SwitchOpcode") << MatchTable::Comment(Comment: "MI")
404 << MatchTable::ULEB128Value(IntValue: Condition->getInsnVarID());
405 return;
406 }
407 if (const auto *Condition = dyn_cast<LLTOperandShapeMatcher>(Val: &P)) {
408 Table << MatchTable::Opcode(Opcode: "GIM_SwitchTypeShape")
409 << MatchTable::Comment(Comment: "MI")
410 << MatchTable::ULEB128Value(IntValue: Condition->getInsnVarID())
411 << MatchTable::Comment(Comment: "Op")
412 << MatchTable::ULEB128Value(IntValue: Condition->getOpIdx());
413 return;
414 }
415 if (const auto *Condition = dyn_cast<LLTOperandMatcher>(Val: &P)) {
416 Table << MatchTable::Opcode(Opcode: "GIM_SwitchType") << MatchTable::Comment(Comment: "MI")
417 << MatchTable::ULEB128Value(IntValue: Condition->getInsnVarID())
418 << MatchTable::Comment(Comment: "Op")
419 << MatchTable::ULEB128Value(IntValue: Condition->getOpIdx());
420 return;
421 }
422
423 llvm_unreachable("emitPredicateSpecificOpcodes is broken: can not handle a "
424 "predicate type that is claimed to be supported");
425}
426
427void SwitchMatcher::emit(MatchTable &Table) {
428#ifndef NDEBUG
429 unsigned NumBucketedMatchers = 0;
430 for (const auto &Entry : Buckets)
431 NumBucketedMatchers += Entry.second.Matchers.size();
432 assert(NumBucketedMatchers == Matchers.size() && "Broken SwitchMatcher");
433#endif
434 if (empty())
435 return;
436 assert(Condition != nullptr &&
437 "Broken SwitchMatcher, hasn't been finalized?");
438
439 std::vector<unsigned> LabelIDs(Buckets.size());
440 std::generate(first: LabelIDs.begin(), last: LabelIDs.end(),
441 gen: [&Table]() { return Table.allocateLabelID(); });
442 const unsigned Default = Table.allocateLabelID();
443
444 const int64_t LowerBound = Buckets.begin()->second.Value.RawValue;
445 const int64_t UpperBound = Buckets.rbegin()->second.Value.RawValue + 1;
446
447 emitPredicateSpecificOpcodes(P: *Condition, Table);
448
449 Table << MatchTable::Comment(Comment: "[") << MatchTable::IntValue(NumBytes: 2, IntValue: LowerBound)
450 << MatchTable::IntValue(NumBytes: 2, IntValue: UpperBound) << MatchTable::Comment(Comment: ")")
451 << MatchTable::Comment(Comment: "default:") << MatchTable::JumpTarget(LabelID: Default);
452
453 int64_t J = LowerBound;
454 unsigned CaseIdx = 0;
455 for (auto &Entry : Buckets) {
456 auto &V = Entry.second.Value;
457 while (J++ < V.RawValue)
458 Table << MatchTable::IntValue(NumBytes: 4, IntValue: 0);
459 V.Record.turnIntoComment();
460 Table << MatchTable::LineBreak << V.Record
461 << MatchTable::JumpTarget(LabelID: LabelIDs[CaseIdx]);
462 ++CaseIdx;
463 }
464 Table << MatchTable::LineBreak;
465
466 CaseIdx = 0;
467 for (auto &Entry : Buckets) {
468 Table << MatchTable::Label(LabelID: LabelIDs[CaseIdx]);
469 for (Matcher *M : Entry.second.Matchers)
470 M->emit(Table);
471 Table << MatchTable::Opcode(Opcode: "GIM_Reject") << MatchTable::LineBreak;
472 ++CaseIdx;
473 }
474 Table << MatchTable::Label(LabelID: Default);
475}
476
477//===- RuleMatcher --------------------------------------------------------===//
478
479RuleMatcher::RuleMatcher(ArrayRef<SMLoc> SrcLoc, bool UsesRecordOperand)
480 : Matcher(Matcher::MK_Rule), UsesRecordOperand(UsesRecordOperand),
481 SrcLoc(SrcLoc), RuleID(NextRuleID++) {}
482
483uint64_t RuleMatcher::NextRuleID = 0;
484
485StringRef RuleMatcher::getOpcode() const { return Roots.front()->getOpcode(); }
486
487bool RuleMatcher::recordsOperand() const {
488 return !usesRecordOperand() || matchersRecordOperand(R: InsnMatchers);
489}
490
491LLTCodeGen RuleMatcher::getFirstConditionAsRootType() const {
492 InstructionMatcher &InsnMatcher = *Roots.front();
493 if (!InsnMatcher.predicates_empty()) {
494 if (const auto *TM =
495 dyn_cast<LLTOperandMatcher>(Val: &**InsnMatcher.predicates_begin())) {
496 if (TM->getInsnVarID() == 0 && TM->getOpIdx() == 0)
497 return TM->getTy();
498 }
499 }
500 return {};
501}
502
503void RuleMatcher::optimize() {
504 for (const auto &InsnMatcher : InsnMatchers) {
505 for (auto &OM : InsnMatcher->operands()) {
506 // Complex Patterns are usually expensive and they relatively rarely fail
507 // on their own: more often we end up throwing away all the work done by a
508 // matching part of a complex pattern because some other part of the
509 // enclosing pattern didn't match. All of this makes it beneficial to
510 // delay complex patterns until the very end of the rule matching,
511 // especially for targets having lots of complex patterns.
512 for (auto &OP : OM->predicates())
513 if (isa<ComplexPatternOperandMatcher>(Val: OP))
514 EpilogueMatchers.emplace_back(args: std::move(OP));
515 OM->eraseNullPredicates();
516 }
517 InsnMatcher->optimize();
518 }
519 llvm::sort(C&: EpilogueMatchers, Comp: [](const std::unique_ptr<PredicateMatcher> &L,
520 const std::unique_ptr<PredicateMatcher> &R) {
521 return std::tuple(L->getKind(), L->getInsnVarID(), L->getOpIdx()) <
522 std::tuple(R->getKind(), R->getInsnVarID(), R->getOpIdx());
523 });
524
525 // Deduplicate EraseInst actions, and if an EraseInst erases the root, place
526 // it at the end to favor generation of GIR_EraseRootFromParent_Done
527 DenseSet<unsigned> AlreadySeenEraseInsts;
528 auto EraseRootIt = Actions.end();
529 auto It = Actions.begin();
530 while (It != Actions.end()) {
531 if (const auto *EI = dyn_cast<EraseInstAction>(Val: It->get())) {
532 unsigned InstID = EI->getInsnID();
533 if (!AlreadySeenEraseInsts.insert(V: InstID).second) {
534 It = Actions.erase(position: It);
535 continue;
536 }
537
538 if (InstID == 0)
539 EraseRootIt = It;
540 }
541
542 ++It;
543 }
544
545 if (EraseRootIt != Actions.end())
546 Actions.splice(position: Actions.end(), x&: Actions, i: EraseRootIt);
547}
548
549bool RuleMatcher::hasFirstCondition() const {
550 if (roots_empty())
551 return false;
552 InstructionMatcher &Matcher = roots_front();
553 if (!Matcher.predicates_empty())
554 return true;
555 for (auto &OM : Matcher.operands())
556 for (auto &OP : OM->predicates())
557 if (!isa<InstructionOperandMatcher>(Val: OP))
558 return true;
559 return false;
560}
561
562const PredicateMatcher &RuleMatcher::getFirstCondition() const {
563 assert(!roots_empty() &&
564 "Trying to get a condition from an empty RuleMatcher");
565
566 InstructionMatcher &Matcher = roots_front();
567 if (!Matcher.predicates_empty())
568 return **Matcher.predicates_begin();
569 // If there is no more predicate on the instruction itself, look at its
570 // operands.
571 for (auto &OM : Matcher.operands())
572 for (auto &OP : OM->predicates())
573 if (!isa<InstructionOperandMatcher>(Val: OP))
574 return *OP;
575
576 llvm_unreachable("Trying to get a condition from an InstructionMatcher with "
577 "no conditions");
578}
579
580std::unique_ptr<PredicateMatcher> RuleMatcher::popFirstCondition() {
581 assert(!roots_empty() &&
582 "Trying to pop a condition from an empty RuleMatcher");
583
584 InstructionMatcher &Matcher = roots_front();
585 if (!Matcher.predicates_empty())
586 return Matcher.predicates_pop_front();
587 // If there is no more predicate on the instruction itself, look at its
588 // operands.
589 for (auto &OM : Matcher.operands())
590 for (auto &OP : OM->predicates())
591 if (!isa<InstructionOperandMatcher>(Val: OP)) {
592 std::unique_ptr<PredicateMatcher> Result = std::move(OP);
593 OM->eraseNullPredicates();
594 return Result;
595 }
596
597 llvm_unreachable("Trying to pop a condition from an InstructionMatcher with "
598 "no conditions");
599}
600
601GISelFlags RuleMatcher::updateGISelFlag(GISelFlags CurFlags, const Record *R,
602 StringRef FlagName,
603 GISelFlags FlagBit) {
604 // If the value of a flag is unset, ignore it.
605 // If it's set, it always takes precedence over the existing value so
606 // clear/set the corresponding bit.
607 bool Unset = false;
608 bool Value = R->getValueAsBitOrUnset(FieldName: "GIIgnoreCopies", Unset);
609 if (!Unset)
610 return Value ? (CurFlags | FlagBit) : (CurFlags & ~FlagBit);
611 return CurFlags;
612}
613
614SaveAndRestore<GISelFlags> RuleMatcher::setGISelFlags(const Record *R) {
615 if (!R || !R->isSubClassOf(Name: "GISelFlags"))
616 return {Flags, Flags};
617
618 assert((R->isSubClassOf("PatFrags") || R->isSubClassOf("Pattern")) &&
619 "GISelFlags is only expected on Pattern/PatFrags!");
620
621 GISelFlags NewFlags =
622 updateGISelFlag(CurFlags: Flags, R, FlagName: "GIIgnoreCopies", FlagBit: GISF_IgnoreCopies);
623 return {Flags, NewFlags};
624}
625
626Error RuleMatcher::defineComplexSubOperand(StringRef SymbolicName,
627 const Record *ComplexPattern,
628 unsigned RendererID,
629 unsigned SubOperandID,
630 StringRef ParentSymbolicName) {
631 std::string ParentName(ParentSymbolicName);
632 auto [It, Inserted] = ComplexSubOperands.try_emplace(
633 Key: SymbolicName, Args&: ComplexPattern, Args&: RendererID, Args&: SubOperandID);
634 if (!Inserted) {
635 const std::string &RecordedParentName =
636 ComplexSubOperandsParentName[SymbolicName];
637 if (RecordedParentName != ParentName)
638 return failUnsupported(Reason: "Error: Complex suboperand " + SymbolicName +
639 " referenced by different operands: " +
640 RecordedParentName + " and " + ParentName + ".");
641 // Complex suboperand referenced more than once from same the operand is
642 // used to generate 'same operand check'. Emitting of
643 // GIR_ComplexSubOperandRenderer for them is already handled.
644 return Error::success();
645 }
646
647 ComplexSubOperandsParentName[SymbolicName] = std::move(ParentName);
648
649 return Error::success();
650}
651
652InstructionMatcher &RuleMatcher::addInstructionMatcher(StringRef SymbolicName) {
653 auto &Res = allocateInstructionMatcher(SymbolicName);
654 Roots.push_back(Elt: &Res);
655 MutatableInsns.insert(Ptr: &Res);
656 return Res;
657}
658
659void RuleMatcher::addRequiredSimplePredicate(StringRef PredName) {
660 RequiredSimplePredicates.push_back(x: PredName.str());
661}
662
663const std::vector<std::string> &RuleMatcher::getRequiredSimplePredicates() {
664 return RequiredSimplePredicates;
665}
666
667void RuleMatcher::defineOperand(StringRef SymbolicName, OperandMatcher &OM) {
668 if (DefinedOperands.try_emplace(Key: SymbolicName, Args: &OM).second)
669 return;
670
671 // If the operand is already defined, then we must ensure both references in
672 // the matcher have the exact same node.
673 RuleMatcher &RM = OM.getInstructionMatcher().getRuleMatcher();
674 auto &OtherOM = getOperandMatcher(Name: OM.getSymbolicName());
675 OM.addPredicate<SameOperandMatcher>(args: OtherOM.getInsnVarID(),
676 args: OtherOM.getOpIdx(), args: RM.getGISelFlags());
677}
678
679void RuleMatcher::definePhysRegOperand(const Record *Reg, OperandMatcher &OM) {
680 PhysRegOperands.try_emplace(Key: Reg, Args: &OM);
681}
682
683InstructionMatcher &
684RuleMatcher::getInstructionMatcher(StringRef SymbolicName) const {
685 for (const auto &InsnMatcher : InsnMatchers) {
686 if (InsnMatcher->getSymbolicName() == SymbolicName)
687 return *InsnMatcher;
688 }
689 llvm_unreachable(
690 ("Failed to lookup instruction " + SymbolicName).str().c_str());
691}
692
693const OperandMatcher &
694RuleMatcher::getPhysRegOperandMatcher(const Record *Reg) const {
695 const auto &I = PhysRegOperands.find(Key: Reg);
696
697 if (I == PhysRegOperands.end()) {
698 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Register " + Reg->getName() +
699 " was not declared in matcher");
700 }
701
702 return *I->second;
703}
704
705OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) {
706 const auto &I = DefinedOperands.find(Key: Name);
707
708 if (I == DefinedOperands.end())
709 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Operand " + Name + " was not declared in matcher");
710
711 return *I->second;
712}
713
714const OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) const {
715 const auto &I = DefinedOperands.find(Key: Name);
716
717 if (I == DefinedOperands.end())
718 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Operand " + Name + " was not declared in matcher");
719
720 return *I->second;
721}
722
723void RuleMatcher::emit(MatchTable &Table) {
724 if (Roots.empty())
725 llvm_unreachable("Unexpected empty matcher!");
726
727 // The representation supports rules that require multiple roots such as:
728 // %ptr(p0) = ...
729 // %elt0(s32) = G_LOAD %ptr
730 // %1(p0) = G_ADD %ptr, 4
731 // %elt1(s32) = G_LOAD p0 %1
732 // which could be usefully folded into:
733 // %ptr(p0) = ...
734 // %elt0(s32), %elt1(s32) = TGT_LOAD_PAIR %ptr
735 // on some targets but we don't need to make use of that yet.
736 assert(Roots.size() == 1 && "Cannot handle multi-root matchers yet");
737
738 unsigned LabelID = Table.allocateLabelID();
739
740 if (!RequiredFeatures.empty() || HwModeIdx >= 0) {
741 Table << MatchTable::Opcode(Opcode: "GIM_Try_CheckFeatures", IndentAdjust: +1)
742 << MatchTable::Comment(Comment: "On fail goto")
743 << MatchTable::JumpTarget(LabelID)
744 << MatchTable::NamedValue(
745 NumBytes: 2, NamedValue: getNameForFeatureBitset(FeatureBitset: RequiredFeatures, HwModeIdx));
746 } else {
747 Table << MatchTable::Opcode(Opcode: "GIM_Try", IndentAdjust: +1)
748 << MatchTable::Comment(Comment: "On fail goto")
749 << MatchTable::JumpTarget(LabelID);
750 }
751 Table << MatchTable::Comment(Comment: ("Rule ID " + Twine(RuleID) + " //").str())
752 << MatchTable::LineBreak;
753
754 if (!RequiredSimplePredicates.empty()) {
755 for (const auto &Pred : RequiredSimplePredicates) {
756 Table << MatchTable::Opcode(Opcode: "GIM_CheckSimplePredicate")
757 << MatchTable::NamedValue(NumBytes: 2, NamedValue: Pred) << MatchTable::LineBreak;
758 }
759 }
760
761 Roots.front()->emitPredicateOpcodes(Table);
762
763 // Check if it's safe to replace registers.
764 for (const auto &MA : Actions)
765 MA->emitAdditionalPredicates(Table);
766
767 // We must also check if it's safe to fold the matched instructions.
768 if (InsnMatchers.size() >= 2) {
769
770 // FIXME: Emit checks to determine it's _actually_ safe to fold and/or
771 // account for unsafe cases.
772 //
773 // Example:
774 // MI1--> %0 = ...
775 // %1 = ... %0
776 // MI0--> %2 = ... %0
777 // It's not safe to erase MI1. We currently handle this by not
778 // erasing %0 (even when it's dead).
779 //
780 // Example:
781 // MI1--> %0 = load volatile @a
782 // %1 = load volatile @a
783 // MI0--> %2 = ... %0
784 // It's not safe to sink %0's def past %1. We currently handle
785 // this by rejecting all loads.
786 //
787 // Example:
788 // MI1--> %0 = load @a
789 // %1 = store @a
790 // MI0--> %2 = ... %0
791 // It's not safe to sink %0's def past %1. We currently handle
792 // this by rejecting all loads.
793 //
794 // Example:
795 // G_CONDBR %cond, @BB1
796 // BB0:
797 // MI1--> %0 = load @a
798 // G_BR @BB1
799 // BB1:
800 // MI0--> %2 = ... %0
801 // It's not always safe to sink %0 across control flow. In this
802 // case it may introduce a memory fault. We currentl handle
803 // this by rejecting all loads.
804
805 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsSafeToFold")
806 << MatchTable::Comment(Comment: "NumInsns")
807 << MatchTable::IntValue(NumBytes: 1, IntValue: InsnMatchers.size() - 1)
808 << MatchTable::LineBreak;
809 }
810
811 for (const auto &PM : EpilogueMatchers)
812 PM->emitPredicateOpcodes(Table);
813
814 if (!CustomCXXAction.empty()) {
815 /// Handle combiners relying on custom C++ code instead of actions.
816 assert(Table.isCombiner() && "CustomCXXAction is only for combiners!");
817 // We cannot have actions other than debug comments.
818 assert(none_of(Actions, [](auto &A) {
819 return A->getKind() != MatchAction::AK_DebugComment;
820 }));
821 for (const auto &MA : Actions)
822 MA->emitActionOpcodes(Table);
823 Table << MatchTable::Opcode(Opcode: "GIR_DoneWithCustomAction", IndentAdjust: -1)
824 << MatchTable::Comment(Comment: "Fn")
825 << MatchTable::NamedValue(NumBytes: 2, NamedValue: CustomCXXAction)
826 << MatchTable::LineBreak;
827 } else {
828 // Emit all actions except the last one, then emit coverage and emit the
829 // final action.
830 //
831 // This is because some actions, such as GIR_EraseRootFromParent_Done, also
832 // double as a GIR_Done and terminate execution of the rule.
833 if (!Actions.empty()) {
834 for (const auto &MA : drop_end(RangeOrContainer&: Actions))
835 MA->emitActionOpcodes(Table);
836 }
837
838 // Emit coverage right before the Done opcode>
839 auto EmitCoverage = [&] {
840 assert((Table.isWithCoverage() ? !Table.isCombiner() : true) &&
841 "Combiner tables don't support coverage!");
842 if (Table.isWithCoverage())
843 Table << MatchTable::Opcode(Opcode: "GIR_Coverage")
844 << MatchTable::IntValue(NumBytes: 4, IntValue: RuleID) << MatchTable::LineBreak;
845 else if (!Table.isCombiner())
846 Table << MatchTable::Comment(
847 Comment: ("GIR_Coverage, " + Twine(RuleID) + ",").str())
848 << MatchTable::LineBreak;
849 };
850
851 if (Actions.empty() ||
852 !Actions.back()->emitActionOpcodesAndDone(Table, OnDone: EmitCoverage)) {
853 EmitCoverage();
854 Table << MatchTable::Opcode(Opcode: "GIR_Done", IndentAdjust: -1) << MatchTable::LineBreak;
855 }
856 }
857
858 Table << MatchTable::Label(LabelID);
859 ++NumPatternEmitted;
860}
861
862bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const {
863 // Rules involving more match roots have higher priority.
864 if (Roots.size() > B.Roots.size())
865 return true;
866 if (Roots.size() < B.Roots.size())
867 return false;
868
869 for (auto Matcher : zip(t: Roots, u: B.Roots)) {
870 if (std::get<0>(t&: Matcher)->isHigherPriorityThan(B&: *std::get<1>(t&: Matcher)))
871 return true;
872 if (std::get<1>(t&: Matcher)->isHigherPriorityThan(B&: *std::get<0>(t&: Matcher)))
873 return false;
874 }
875
876 return false;
877}
878
879unsigned RuleMatcher::countRendererFns() const {
880 return std::accumulate(first: Roots.begin(), last: Roots.end(), init: 0,
881 binary_op: [](unsigned A, InstructionMatcher *Matcher) {
882 return A + Matcher->countRendererFns();
883 });
884}
885
886void RuleMatcher::roots_pop_front() { Roots.erase(CI: Roots.begin()); }
887
888//===- PredicateMatcher ---------------------------------------------------===//
889
890PredicateMatcher::~PredicateMatcher() = default;
891
892//===- OperandPredicateMatcher --------------------------------------------===//
893
894OperandPredicateMatcher::~OperandPredicateMatcher() = default;
895
896bool OperandPredicateMatcher::isHigherPriorityThan(
897 const OperandPredicateMatcher &B) const {
898 // Generally speaking, an instruction is more important than an Int or a
899 // LiteralInt because it can cover more nodes but there's an exception to
900 // this. G_CONSTANT's are less important than either of those two because they
901 // are more permissive.
902
903 const auto *AOM = dyn_cast<InstructionOperandMatcher>(Val: this);
904 const auto *BOM = dyn_cast<InstructionOperandMatcher>(Val: &B);
905 bool AIsConstantInsn = AOM && AOM->getInsnMatcher().isConstantInstruction();
906 bool BIsConstantInsn = BOM && BOM->getInsnMatcher().isConstantInstruction();
907
908 // The relative priorities between a G_CONSTANT and any other instruction
909 // don't actually matter but this code is needed to ensure a strict weak
910 // ordering. This is particularly important on Windows where the rules will
911 // be incorrectly sorted without it.
912 if (AOM && BOM)
913 return !AIsConstantInsn && BIsConstantInsn;
914
915 if (AIsConstantInsn && (B.Kind == OPM_Int || B.Kind == OPM_LiteralInt))
916 return false;
917 if (BIsConstantInsn && (Kind == OPM_Int || Kind == OPM_LiteralInt))
918 return true;
919
920 return Kind < B.Kind;
921}
922
923//===- SameOperandMatcher -------------------------------------------------===//
924
925void SameOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
926 const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
927 Table << MatchTable::Opcode(Opcode: IgnoreCopies
928 ? "GIM_CheckIsSameOperandIgnoreCopies"
929 : "GIM_CheckIsSameOperand")
930 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
931 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OpIdx)
932 << MatchTable::Comment(Comment: "OtherMI")
933 << MatchTable::ULEB128Value(IntValue: OtherInsnID)
934 << MatchTable::Comment(Comment: "OtherOpIdx")
935 << MatchTable::ULEB128Value(IntValue: OtherOpIdx) << MatchTable::LineBreak;
936}
937
938//===- LLTOperandMatcher --------------------------------------------------===//
939
940std::map<LLTCodeGen, unsigned> LLTOperandMatcher::TypeIDValues;
941
942RecordAndValue LLTOperandMatcher::getValue() const {
943 const auto VI = TypeIDValues.find(x: Ty);
944 if (VI == TypeIDValues.end())
945 return MatchTable::NamedValue(NumBytes: 1, NamedValue: getTy().getCxxEnumValue());
946 return {MatchTable::NamedValue(NumBytes: 1, NamedValue: getTy().getCxxEnumValue()), VI->second};
947}
948
949bool LLTOperandMatcher::hasValue() const {
950 if (TypeIDValues.size() != KnownTypes.size())
951 initTypeIDValuesMap();
952 return TypeIDValues.count(x: Ty);
953}
954
955void LLTOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
956 if (InsnVarID == 0) {
957 Table << MatchTable::Opcode(Opcode: "GIM_RootCheckType");
958 } else {
959 Table << MatchTable::Opcode(Opcode: "GIM_CheckType") << MatchTable::Comment(Comment: "MI")
960 << MatchTable::ULEB128Value(IntValue: InsnVarID);
961 }
962 Table << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
963 << MatchTable::Comment(Comment: "Type") << getValue().Record
964 << MatchTable::LineBreak;
965}
966
967//===- PointerToAnyOperandMatcher -----------------------------------------===//
968
969void PointerToAnyOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
970 Table << MatchTable::Opcode(Opcode: "GIM_CheckPointerToAny")
971 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
972 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
973 << MatchTable::Comment(Comment: "SizeInBits")
974 << MatchTable::ULEB128Value(IntValue: SizeInBits) << MatchTable::LineBreak;
975}
976
977//===- RecordNamedOperandMatcher ------------------------------------------===//
978
979void RecordNamedOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
980 Table << MatchTable::Opcode(Opcode: "GIM_RecordNamedOperand")
981 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
982 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
983 << MatchTable::Comment(Comment: "StoreIdx") << MatchTable::ULEB128Value(IntValue: StoreIdx)
984 << MatchTable::Comment(Comment: "Name : " + Name) << MatchTable::LineBreak;
985}
986
987//===- RecordRegisterType ------------------------------------------===//
988
989void RecordRegisterType::emitPredicateOpcodes(MatchTable &Table) const {
990 assert(Idx < 0 && "Temp types always have negative indexes!");
991 Table << MatchTable::Opcode(Opcode: "GIM_RecordRegType") << MatchTable::Comment(Comment: "MI")
992 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
993 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::Comment(Comment: "TempTypeIdx")
994 << MatchTable::IntValue(NumBytes: 1, IntValue: Idx) << MatchTable::LineBreak;
995}
996
997//===- ComplexPatternOperandMatcher ---------------------------------------===//
998
999void ComplexPatternOperandMatcher::emitPredicateOpcodes(
1000 MatchTable &Table) const {
1001 unsigned ID = getAllocatedTemporariesBaseID();
1002 Table << MatchTable::Opcode(Opcode: "GIM_CheckComplexPattern")
1003 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1004 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1005 << MatchTable::Comment(Comment: "Renderer") << MatchTable::IntValue(NumBytes: 2, IntValue: ID)
1006 << MatchTable::NamedValue(NumBytes: 2, NamedValue: ("GICP_" + TheDef.getName()).str())
1007 << MatchTable::LineBreak;
1008}
1009
1010unsigned ComplexPatternOperandMatcher::getAllocatedTemporariesBaseID() const {
1011 return Operand.getAllocatedTemporariesBaseID();
1012}
1013
1014//===- RegisterBankOperandMatcher -----------------------------------------===//
1015
1016bool RegisterBankOperandMatcher::isIdentical(const PredicateMatcher &B) const {
1017 return OperandPredicateMatcher::isIdentical(B) &&
1018 RC.getDef() == cast<RegisterBankOperandMatcher>(Val: &B)->RC.getDef();
1019}
1020
1021void RegisterBankOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1022 if (InsnVarID == 0) {
1023 Table << MatchTable::Opcode(Opcode: "GIM_RootCheckRegBankForClass");
1024 } else {
1025 Table << MatchTable::Opcode(Opcode: "GIM_CheckRegBankForClass")
1026 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID);
1027 }
1028
1029 Table << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1030 << MatchTable::Comment(Comment: "RC")
1031 << MatchTable::NamedValue(NumBytes: 2, NamedValue: RC.getQualifiedIdName())
1032 << MatchTable::LineBreak;
1033}
1034
1035//===- MBBOperandMatcher --------------------------------------------------===//
1036
1037void MBBOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1038 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsMBB") << MatchTable::Comment(Comment: "MI")
1039 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
1040 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
1041}
1042
1043//===- ImmOperandMatcher --------------------------------------------------===//
1044
1045void ImmOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1046 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsImm") << MatchTable::Comment(Comment: "MI")
1047 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
1048 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
1049}
1050
1051//===- ConstantIntOperandMatcher ------------------------------------------===//
1052
1053void ConstantIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1054 const bool IsInt8 = isInt<8>(x: Value);
1055 Table << MatchTable::Opcode(Opcode: IsInt8 ? "GIM_CheckConstantInt8"
1056 : "GIM_CheckConstantInt")
1057 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1058 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1059 << MatchTable::IntValue(NumBytes: IsInt8 ? 1 : 8, IntValue: Value) << MatchTable::LineBreak;
1060}
1061
1062//===- LiteralIntOperandMatcher -------------------------------------------===//
1063
1064void LiteralIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1065 Table << MatchTable::Opcode(Opcode: "GIM_CheckLiteralInt")
1066 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1067 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1068 << MatchTable::IntValue(NumBytes: 8, IntValue: Value) << MatchTable::LineBreak;
1069}
1070
1071//===- CmpPredicateOperandMatcher -----------------------------------------===//
1072
1073void CmpPredicateOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1074 Table << MatchTable::Opcode(Opcode: "GIM_CheckCmpPredicate")
1075 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1076 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1077 << MatchTable::Comment(Comment: "Predicate")
1078 << MatchTable::NamedValue(NumBytes: 2, Namespace: "CmpInst", NamedValue: PredName)
1079 << MatchTable::LineBreak;
1080}
1081
1082//===- IntrinsicIDOperandMatcher ------------------------------------------===//
1083
1084void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1085 Table << MatchTable::Opcode(Opcode: "GIM_CheckIntrinsicID")
1086 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1087 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1088 << MatchTable::NamedValue(NumBytes: 2, NamedValue: "Intrinsic::" + II->EnumName.str())
1089 << MatchTable::LineBreak;
1090}
1091
1092//===- OperandImmPredicateMatcher -----------------------------------------===//
1093
1094void OperandImmPredicateMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1095 Table << MatchTable::Opcode(Opcode: "GIM_CheckImmOperandPredicate")
1096 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1097 << MatchTable::Comment(Comment: "MO") << MatchTable::ULEB128Value(IntValue: OpIdx)
1098 << MatchTable::Comment(Comment: "Predicate")
1099 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1100 << MatchTable::LineBreak;
1101}
1102
1103//===- OperandLeafPredicateMatcher ----------------------------------------===//
1104
1105void OperandLeafPredicateMatcher::emitPredicateOpcodes(
1106 MatchTable &Table) const {
1107 Table << MatchTable::Opcode(Opcode: "GIM_CheckLeafOperandPredicate")
1108 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1109 << MatchTable::Comment(Comment: "MO") << MatchTable::ULEB128Value(IntValue: OpIdx)
1110 << MatchTable::Comment(Comment: "Predicate")
1111 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1112 << MatchTable::LineBreak;
1113}
1114
1115//===- OperandMatcher -----------------------------------------------------===//
1116
1117std::string OperandMatcher::getOperandExpr(unsigned InsnVarID) const {
1118 return "State.MIs[" + llvm::to_string(Value: InsnVarID) + "]->getOperand(" +
1119 llvm::to_string(Value: OpIdx) + ")";
1120}
1121
1122unsigned OperandMatcher::getInsnVarID() const { return Insn.getInsnVarID(); }
1123
1124TempTypeIdx OperandMatcher::getTempTypeIdx(RuleMatcher &Rule) {
1125 assert(!IsVariadic && "Cannot use this on variadic operands!");
1126 if (TTIdx >= 0) {
1127 // Temp type index not assigned yet, so assign one and add the necessary
1128 // predicate.
1129 TTIdx = Rule.getNextTempTypeIdx();
1130 assert(TTIdx < 0);
1131 addPredicate<RecordRegisterType>(args&: TTIdx);
1132 return TTIdx;
1133 }
1134 return TTIdx;
1135}
1136
1137bool OperandMatcher::recordsOperand() const {
1138 return matchersRecordOperand(R: Predicates);
1139}
1140
1141void OperandMatcher::emitPredicateOpcodes(MatchTable &Table) {
1142 if (!Optimized) {
1143 std::string Comment;
1144 raw_string_ostream CommentOS(Comment);
1145 CommentOS << "MIs[" << getInsnVarID() << "] ";
1146 if (SymbolicName.empty())
1147 CommentOS << "Operand " << OpIdx;
1148 else
1149 CommentOS << SymbolicName;
1150 Table << MatchTable::Comment(Comment) << MatchTable::LineBreak;
1151 }
1152
1153 emitPredicateListOpcodes(Table);
1154}
1155
1156bool OperandMatcher::isHigherPriorityThan(OperandMatcher &B) {
1157 // Operand matchers involving more predicates have higher priority.
1158 if (predicates_size() > B.predicates_size())
1159 return true;
1160 if (predicates_size() < B.predicates_size())
1161 return false;
1162
1163 // This assumes that predicates are added in a consistent order.
1164 for (auto &&Predicate : zip(t: predicates(), u: B.predicates())) {
1165 if (std::get<0>(t&: Predicate)->isHigherPriorityThan(B: *std::get<1>(t&: Predicate)))
1166 return true;
1167 if (std::get<1>(t&: Predicate)->isHigherPriorityThan(B: *std::get<0>(t&: Predicate)))
1168 return false;
1169 }
1170
1171 return false;
1172}
1173
1174unsigned OperandMatcher::countRendererFns() {
1175 return std::accumulate(
1176 first: predicates().begin(), last: predicates().end(), init: 0,
1177 binary_op: [](unsigned A,
1178 const std::unique_ptr<OperandPredicateMatcher> &Predicate) {
1179 return A + Predicate->countRendererFns();
1180 });
1181}
1182
1183Error OperandMatcher::addTypeCheckPredicate(const TypeSetByHwMode &VTy,
1184 bool OperandIsAPointer) {
1185 if (!VTy.isMachineValueType())
1186 return failUnsupported(Reason: "unsupported typeset");
1187
1188 if ((VTy.getMachineValueType() == MVT::iPTR ||
1189 VTy.getMachineValueType() == MVT::cPTR) &&
1190 OperandIsAPointer) {
1191 addPredicate<PointerToAnyOperandMatcher>(args: 0);
1192 return Error::success();
1193 }
1194
1195 // Metadata operands have no LLT representation and no runtime type check is
1196 // needed — they are guaranteed to be MO_Metadata by the IRTranslator. This
1197 // mirrors how srcvalue is handled in importChildMatcher.
1198 if (VTy.getMachineValueType() == MVT::Metadata)
1199 return Error::success();
1200
1201 auto OpTyOrNone = MVTToLLT(VT: VTy.getMachineValueType().SimpleTy);
1202 if (!OpTyOrNone)
1203 return failUnsupported(Reason: "unsupported type");
1204
1205 if (OperandIsAPointer)
1206 addPredicate<PointerToAnyOperandMatcher>(args: OpTyOrNone->get().getSizeInBits());
1207 else if (VTy.isPointer())
1208 addPredicate<LLTOperandMatcher>(
1209 args: LLT::pointer(AddressSpace: VTy.getPtrAddrSpace(), SizeInBits: OpTyOrNone->get().getSizeInBits()));
1210 else
1211 addPredicate<LLTOperandMatcher>(args&: *OpTyOrNone);
1212 return Error::success();
1213}
1214
1215//===- InstructionOpcodeMatcher -------------------------------------------===//
1216
1217DenseMap<const CodeGenInstruction *, unsigned>
1218 InstructionOpcodeMatcher::OpcodeValues;
1219
1220RecordAndValue
1221InstructionOpcodeMatcher::getInstValue(const CodeGenInstruction *I) const {
1222 const auto VI = OpcodeValues.find(Val: I);
1223 if (VI != OpcodeValues.end())
1224 return {MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName()), VI->second};
1225 return MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName());
1226}
1227
1228void InstructionOpcodeMatcher::initOpcodeValuesMap(
1229 const CodeGenTarget &Target) {
1230 OpcodeValues.clear();
1231
1232 for (const CodeGenInstruction *I : Target.getInstructions())
1233 OpcodeValues[I] = Target.getInstrIntValue(R: I->TheDef);
1234}
1235
1236RecordAndValue InstructionOpcodeMatcher::getValue() const {
1237 assert(Insts.size() == 1);
1238
1239 const CodeGenInstruction *I = Insts[0];
1240 const auto VI = OpcodeValues.find(Val: I);
1241 if (VI != OpcodeValues.end())
1242 return {MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName()), VI->second};
1243 return MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName());
1244}
1245
1246void InstructionOpcodeMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1247 StringRef CheckType =
1248 Insts.size() == 1 ? "GIM_CheckOpcode" : "GIM_CheckOpcodeIsEither";
1249 Table << MatchTable::Opcode(Opcode: CheckType) << MatchTable::Comment(Comment: "MI")
1250 << MatchTable::ULEB128Value(IntValue: InsnVarID);
1251
1252 for (const CodeGenInstruction *I : Insts)
1253 Table << getInstValue(I).Record;
1254 Table << MatchTable::LineBreak;
1255}
1256
1257bool InstructionOpcodeMatcher::isHigherPriorityThan(
1258 const InstructionPredicateMatcher &B) const {
1259 if (InstructionPredicateMatcher::isHigherPriorityThan(B))
1260 return true;
1261 if (B.InstructionPredicateMatcher::isHigherPriorityThan(B: *this))
1262 return false;
1263
1264 // Prioritize opcodes for cosmetic reasons in the generated source. Although
1265 // this is cosmetic at the moment, we may want to drive a similar ordering
1266 // using instruction frequency information to improve compile time.
1267 if (const InstructionOpcodeMatcher *BO =
1268 dyn_cast<InstructionOpcodeMatcher>(Val: &B))
1269 return Insts[0]->getName() < BO->Insts[0]->getName();
1270
1271 return false;
1272}
1273
1274bool InstructionOpcodeMatcher::isConstantInstruction() const {
1275 return Insts.size() == 1 && Insts[0]->getName() == "G_CONSTANT";
1276}
1277
1278StringRef InstructionOpcodeMatcher::getOpcode() const {
1279 return Insts[0]->getName();
1280}
1281
1282bool InstructionOpcodeMatcher::isVariadicNumOperands() const {
1283 // If one is variadic, they all should be.
1284 return Insts[0]->Operands.isVariadic;
1285}
1286
1287StringRef InstructionOpcodeMatcher::getOperandType(unsigned OpIdx) const {
1288 // Types expected to be uniform for all alternatives.
1289 return Insts[0]->Operands[OpIdx].OperandType;
1290}
1291
1292//===- InstructionNumOperandsMatcher --------------------------------------===//
1293
1294void InstructionNumOperandsMatcher::emitPredicateOpcodes(
1295 MatchTable &Table) const {
1296 StringRef Opc;
1297 switch (CK) {
1298 case CheckKind::Eq:
1299 Opc = "GIM_CheckNumOperands";
1300 break;
1301 case CheckKind::GE:
1302 Opc = "GIM_CheckNumOperandsGE";
1303 break;
1304 case CheckKind::LE:
1305 Opc = "GIM_CheckNumOperandsLE";
1306 break;
1307 }
1308 Table << MatchTable::Opcode(Opcode: Opc) << MatchTable::Comment(Comment: "MI")
1309 << MatchTable::ULEB128Value(IntValue: InsnVarID)
1310 << MatchTable::Comment(Comment: "Expected")
1311 << MatchTable::ULEB128Value(IntValue: NumOperands) << MatchTable::LineBreak;
1312}
1313
1314//===- InstructionImmPredicateMatcher -------------------------------------===//
1315
1316bool InstructionImmPredicateMatcher::isIdentical(
1317 const PredicateMatcher &B) const {
1318 return InstructionPredicateMatcher::isIdentical(B) &&
1319 Predicate.getOrigPatFragRecord() ==
1320 cast<InstructionImmPredicateMatcher>(Val: &B)
1321 ->Predicate.getOrigPatFragRecord();
1322}
1323
1324void InstructionImmPredicateMatcher::emitPredicateOpcodes(
1325 MatchTable &Table) const {
1326 Table << MatchTable::Opcode(Opcode: getMatchOpcodeForImmPredicate(Predicate))
1327 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1328 << MatchTable::Comment(Comment: "Predicate")
1329 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1330 << MatchTable::LineBreak;
1331}
1332
1333//===- AtomicOrderingMMOPredicateMatcher ----------------------------------===//
1334
1335bool AtomicOrderingMMOPredicateMatcher::isIdentical(
1336 const PredicateMatcher &B) const {
1337 if (!InstructionPredicateMatcher::isIdentical(B))
1338 return false;
1339 const auto &R = *cast<AtomicOrderingMMOPredicateMatcher>(Val: &B);
1340 return Order == R.Order && Comparator == R.Comparator;
1341}
1342
1343void AtomicOrderingMMOPredicateMatcher::emitPredicateOpcodes(
1344 MatchTable &Table) const {
1345 StringRef Opcode = "GIM_CheckAtomicOrdering";
1346
1347 if (Comparator == AO_OrStronger)
1348 Opcode = "GIM_CheckAtomicOrderingOrStrongerThan";
1349 if (Comparator == AO_WeakerThan)
1350 Opcode = "GIM_CheckAtomicOrderingWeakerThan";
1351
1352 Table << MatchTable::Opcode(Opcode) << MatchTable::Comment(Comment: "MI")
1353 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Order")
1354 << MatchTable::NamedValue(NumBytes: 1,
1355 NamedValue: ("(uint8_t)AtomicOrdering::" + Order).str())
1356 << MatchTable::LineBreak;
1357}
1358
1359//===- MemorySizePredicateMatcher -----------------------------------------===//
1360
1361void MemorySizePredicateMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1362 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemorySizeEqualTo")
1363 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1364 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1365 << MatchTable::Comment(Comment: "Size") << MatchTable::IntValue(NumBytes: 4, IntValue: Size)
1366 << MatchTable::LineBreak;
1367}
1368
1369//===- MemoryAddressSpacePredicateMatcher ---------------------------------===//
1370
1371bool MemoryAddressSpacePredicateMatcher::isIdentical(
1372 const PredicateMatcher &B) const {
1373 if (!InstructionPredicateMatcher::isIdentical(B))
1374 return false;
1375 auto *Other = cast<MemoryAddressSpacePredicateMatcher>(Val: &B);
1376 return MMOIdx == Other->MMOIdx && AddrSpaces == Other->AddrSpaces;
1377}
1378
1379void MemoryAddressSpacePredicateMatcher::emitPredicateOpcodes(
1380 MatchTable &Table) const {
1381 assert(AddrSpaces.size() < 256);
1382 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemoryAddressSpace")
1383 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1384 << MatchTable::Comment(Comment: "MMO")
1385 << MatchTable::ULEB128Value(IntValue: MMOIdx)
1386 // Encode number of address spaces to expect.
1387 << MatchTable::Comment(Comment: "NumAddrSpace")
1388 << MatchTable::IntValue(NumBytes: 1, IntValue: AddrSpaces.size());
1389 for (unsigned AS : AddrSpaces)
1390 Table << MatchTable::Comment(Comment: "AddrSpace") << MatchTable::ULEB128Value(IntValue: AS);
1391
1392 Table << MatchTable::LineBreak;
1393}
1394
1395//===- MemoryAlignmentPredicateMatcher ------------------------------------===//
1396
1397bool MemoryAlignmentPredicateMatcher::isIdentical(
1398 const PredicateMatcher &B) const {
1399 if (!InstructionPredicateMatcher::isIdentical(B))
1400 return false;
1401 auto *Other = cast<MemoryAlignmentPredicateMatcher>(Val: &B);
1402 return MMOIdx == Other->MMOIdx && MinAlign == Other->MinAlign;
1403}
1404
1405void MemoryAlignmentPredicateMatcher::emitPredicateOpcodes(
1406 MatchTable &Table) const {
1407 // TODO: we could support more, just need to emit the right opcode or switch
1408 // to log alignment.
1409 assert(MinAlign < 256);
1410 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemoryAlignment")
1411 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1412 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1413 << MatchTable::Comment(Comment: "MinAlign") << MatchTable::IntValue(NumBytes: 1, IntValue: MinAlign)
1414 << MatchTable::LineBreak;
1415}
1416
1417//===- MemoryVsLLTSizePredicateMatcher ------------------------------------===//
1418
1419bool MemoryVsLLTSizePredicateMatcher::isIdentical(
1420 const PredicateMatcher &B) const {
1421 return InstructionPredicateMatcher::isIdentical(B) &&
1422 MMOIdx == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->MMOIdx &&
1423 Relation == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->Relation &&
1424 OpIdx == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->OpIdx;
1425}
1426
1427void MemoryVsLLTSizePredicateMatcher::emitPredicateOpcodes(
1428 MatchTable &Table) const {
1429 Table << MatchTable::Opcode(
1430 Opcode: Relation == EqualTo ? "GIM_CheckMemorySizeEqualToLLT"
1431 : Relation == GreaterThan ? "GIM_CheckMemorySizeGreaterThanLLT"
1432 : "GIM_CheckMemorySizeLessThanLLT")
1433 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1434 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1435 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OpIdx)
1436 << MatchTable::LineBreak;
1437}
1438
1439//===- VectorSplatImmPredicateMatcher -------------------------------------===//
1440
1441void VectorSplatImmPredicateMatcher::emitPredicateOpcodes(
1442 MatchTable &Table) const {
1443 if (Kind == AllOnes)
1444 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsBuildVectorAllOnes");
1445 else
1446 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsBuildVectorAllZeros");
1447
1448 Table << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID);
1449 Table << MatchTable::LineBreak;
1450}
1451
1452//===- GenericInstructionPredicateMatcher ---------------------------------===//
1453
1454GenericInstructionPredicateMatcher::GenericInstructionPredicateMatcher(
1455 unsigned InsnVarID, TreePredicateFn Predicate)
1456 : GenericInstructionPredicateMatcher(InsnVarID,
1457 getEnumNameForPredicate(Predicate)) {}
1458
1459bool GenericInstructionPredicateMatcher::isIdentical(
1460 const PredicateMatcher &B) const {
1461 return InstructionPredicateMatcher::isIdentical(B) &&
1462 EnumVal ==
1463 static_cast<const GenericInstructionPredicateMatcher &>(B).EnumVal;
1464}
1465void GenericInstructionPredicateMatcher::emitPredicateOpcodes(
1466 MatchTable &Table) const {
1467 Table << MatchTable::Opcode(Opcode: "GIM_CheckCxxInsnPredicate")
1468 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1469 << MatchTable::Comment(Comment: "FnId") << MatchTable::NamedValue(NumBytes: 2, NamedValue: EnumVal)
1470 << MatchTable::LineBreak;
1471}
1472
1473//===- MIFlagsInstructionPredicateMatcher ---------------------------------===//
1474
1475bool MIFlagsInstructionPredicateMatcher::isIdentical(
1476 const PredicateMatcher &B) const {
1477 if (!InstructionPredicateMatcher::isIdentical(B))
1478 return false;
1479 const auto &Other =
1480 static_cast<const MIFlagsInstructionPredicateMatcher &>(B);
1481 return Flags == Other.Flags && CheckNot == Other.CheckNot;
1482}
1483
1484void MIFlagsInstructionPredicateMatcher::emitPredicateOpcodes(
1485 MatchTable &Table) const {
1486 Table << MatchTable::Opcode(Opcode: CheckNot ? "GIM_MIFlagsNot" : "GIM_MIFlags")
1487 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1488 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: Flags, Separator: " | "))
1489 << MatchTable::LineBreak;
1490}
1491
1492//===- InstructionMatcher -------------------------------------------------===//
1493
1494OperandMatcher &
1495InstructionMatcher::addOperand(unsigned OpIdx, const std::string &SymbolicName,
1496 unsigned AllocatedTemporariesBaseID,
1497 bool IsVariadic) {
1498 assert((Operands.empty() || !Operands.back()->isVariadic()) &&
1499 "Cannot add more operands after a variadic operand");
1500 Operands.emplace_back(args: new OperandMatcher(
1501 *this, OpIdx, SymbolicName, AllocatedTemporariesBaseID, IsVariadic));
1502 if (!SymbolicName.empty())
1503 Rule.defineOperand(SymbolicName, OM&: *Operands.back());
1504 return *Operands.back();
1505}
1506
1507OperandMatcher &InstructionMatcher::getOperand(unsigned OpIdx) {
1508 auto I = llvm::find_if(Range&: Operands,
1509 P: [&OpIdx](const std::unique_ptr<OperandMatcher> &X) {
1510 return X->getOpIdx() == OpIdx;
1511 });
1512 if (I != Operands.end())
1513 return **I;
1514 llvm_unreachable("Failed to lookup operand");
1515}
1516
1517OperandMatcher &InstructionMatcher::addPhysRegInput(const Record *Reg,
1518 unsigned OpIdx,
1519 unsigned TempOpIdx) {
1520 assert(SymbolicName.empty());
1521 OperandMatcher *OM = new OperandMatcher(*this, OpIdx, "", TempOpIdx);
1522 Operands.emplace_back(args&: OM);
1523 Rule.definePhysRegOperand(Reg, OM&: *OM);
1524 return *OM;
1525}
1526
1527bool InstructionMatcher::recordsOperand() const {
1528 return matchersRecordOperand(R: Predicates) || matchersRecordOperand(R: operands());
1529}
1530
1531void InstructionMatcher::emitPredicateOpcodes(MatchTable &Table) {
1532 if (canAddNumOperandsCheck()) {
1533 InstructionNumOperandsMatcher(InsnVarID, getNumOperandMatchers())
1534 .emitPredicateOpcodes(Table);
1535 }
1536
1537 // First emit all instruction level predicates need to be verified before we
1538 // can verify operands.
1539 emitFilteredPredicateListOpcodes(
1540 ShouldEmitPredicate: [](const PredicateMatcher &P) { return !P.dependsOnRecordedOperands(); },
1541 Table);
1542
1543 // Emit all operand constraints.
1544 for (const auto &Operand : Operands)
1545 Operand->emitPredicateOpcodes(Table);
1546
1547 // All of the tablegen defined predicates should now be matched. Now emit
1548 // any custom predicates that rely on all generated checks.
1549 emitFilteredPredicateListOpcodes(
1550 ShouldEmitPredicate: [](const PredicateMatcher &P) { return P.dependsOnRecordedOperands(); },
1551 Table);
1552}
1553
1554bool InstructionMatcher::isHigherPriorityThan(InstructionMatcher &B) {
1555 // Instruction matchers involving more operands have higher priority.
1556 if (Operands.size() > B.Operands.size())
1557 return true;
1558 if (Operands.size() < B.Operands.size())
1559 return false;
1560
1561 for (auto &&P : zip(t: predicates(), u: B.predicates())) {
1562 auto L = static_cast<InstructionPredicateMatcher *>(std::get<0>(t&: P).get());
1563 auto R = static_cast<InstructionPredicateMatcher *>(std::get<1>(t&: P).get());
1564 if (L->isHigherPriorityThan(B: *R))
1565 return true;
1566 if (R->isHigherPriorityThan(B: *L))
1567 return false;
1568 }
1569
1570 for (auto Operand : zip(t&: Operands, u&: B.Operands)) {
1571 if (std::get<0>(t&: Operand)->isHigherPriorityThan(B&: *std::get<1>(t&: Operand)))
1572 return true;
1573 if (std::get<1>(t&: Operand)->isHigherPriorityThan(B&: *std::get<0>(t&: Operand)))
1574 return false;
1575 }
1576 // Instruction matchers involving more predicates have higher priority.
1577 if (predicates_size() > B.predicates_size())
1578 return true;
1579 if (predicates_size() < B.predicates_size())
1580 return false;
1581
1582 return false;
1583}
1584
1585unsigned InstructionMatcher::countRendererFns() {
1586 return std::accumulate(
1587 first: predicates().begin(), last: predicates().end(), init: 0,
1588 binary_op: [](unsigned A,
1589 const std::unique_ptr<PredicateMatcher> &Predicate) {
1590 return A + Predicate->countRendererFns();
1591 }) +
1592 std::accumulate(
1593 first: Operands.begin(), last: Operands.end(), init: 0,
1594 binary_op: [](unsigned A, const std::unique_ptr<OperandMatcher> &Operand) {
1595 return A + Operand->countRendererFns();
1596 });
1597}
1598
1599void InstructionMatcher::optimize() {
1600 SmallVector<std::unique_ptr<PredicateMatcher>, 8> Stash;
1601 const auto &OpcMatcher = getOpcodeMatcher();
1602
1603 Stash.push_back(Elt: predicates_pop_front());
1604 if (Stash.back().get() == &OpcMatcher) {
1605 // FIXME: Is this even needed still? Why the isVariadicNumOperands check?
1606 if (canAddNumOperandsCheck() && OpcMatcher.isVariadicNumOperands() &&
1607 getNumOperandMatchers() != 0) {
1608 Stash.emplace_back(Args: new InstructionNumOperandsMatcher(
1609 InsnVarID, getNumOperandMatchers()));
1610 }
1611 AllowNumOpsCheck = false;
1612
1613 for (auto &OM : Operands)
1614 for (auto &OP : OM->predicates())
1615 if (isa<IntrinsicIDOperandMatcher>(Val: OP)) {
1616 Stash.push_back(Elt: std::move(OP));
1617 OM->eraseNullPredicates();
1618 break;
1619 }
1620 }
1621
1622 if (InsnVarID > 0) {
1623 assert(!Operands.empty() && "Nested instruction is expected to def a vreg");
1624 for (auto &OP : Operands[0]->predicates())
1625 OP.reset();
1626 Operands[0]->eraseNullPredicates();
1627 }
1628 for (auto &OM : Operands) {
1629 for (auto &OP : OM->predicates())
1630 if (isa<LLTOperandMatcher>(Val: OP) || isa<LLTOperandShapeMatcher>(Val: OP))
1631 Stash.push_back(Elt: std::move(OP));
1632 OM->eraseNullPredicates();
1633 }
1634 while (!Stash.empty())
1635 prependPredicate(Predicate: Stash.pop_back_val());
1636}
1637
1638//===- InstructionOperandMatcher ------------------------------------------===//
1639
1640void InstructionOperandMatcher::emitCaptureOpcodes(MatchTable &Table) const {
1641 const unsigned NewInsnVarID = InsnMatcher.getInsnVarID();
1642 const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
1643 Table << MatchTable::Opcode(Opcode: IgnoreCopies ? "GIM_RecordInsnIgnoreCopies"
1644 : "GIM_RecordInsn")
1645 << MatchTable::Comment(Comment: "DefineMI")
1646 << MatchTable::ULEB128Value(IntValue: NewInsnVarID) << MatchTable::Comment(Comment: "MI")
1647 << MatchTable::ULEB128Value(IntValue: getInsnVarID())
1648 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: getOpIdx())
1649 << MatchTable::Comment(Comment: "MIs[" + llvm::to_string(Value: NewInsnVarID) + "]")
1650 << MatchTable::LineBreak;
1651}
1652
1653bool InstructionOperandMatcher::isHigherPriorityThan(
1654 const OperandPredicateMatcher &B) const {
1655 if (OperandPredicateMatcher::isHigherPriorityThan(B))
1656 return true;
1657 if (B.OperandPredicateMatcher::isHigherPriorityThan(B: *this))
1658 return false;
1659
1660 if (const InstructionOperandMatcher *BP =
1661 dyn_cast<InstructionOperandMatcher>(Val: &B))
1662 if (InsnMatcher.isHigherPriorityThan(B&: BP->InsnMatcher))
1663 return true;
1664 return false;
1665}
1666
1667//===- OperandRenderer ----------------------------------------------------===//
1668
1669OperandRenderer::~OperandRenderer() = default;
1670
1671//===- CopyRenderer -------------------------------------------------------===//
1672
1673void CopyRenderer::emitRenderOpcodes(MatchTable &Table, unsigned NewInsnID,
1674 unsigned OldInsnID, unsigned OldOpIdx,
1675 StringRef Name, bool ForVariadic) {
1676 if (!ForVariadic && NewInsnID == 0 && OldInsnID == 0) {
1677 Table << MatchTable::Opcode(Opcode: "GIR_RootToRootCopy");
1678 } else {
1679 Table << MatchTable::Opcode(Opcode: ForVariadic ? "GIR_CopyRemaining" : "GIR_Copy")
1680 << MatchTable::Comment(Comment: "NewInsnID")
1681 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1682 << MatchTable::Comment(Comment: "OldInsnID")
1683 << MatchTable::ULEB128Value(IntValue: OldInsnID);
1684 }
1685
1686 Table << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1687 << MatchTable::Comment(Comment: Name) << MatchTable::LineBreak;
1688}
1689
1690void CopyRenderer::emitRenderOpcodes(MatchTable &Table) const {
1691 emitRenderOpcodes(Table, NewInsnID, OldInsnID, OldOpIdx, Name: SymbolicName,
1692 ForVariadic: OldOpIsVariadic);
1693}
1694
1695//===- CopyPhysRegRenderer ------------------------------------------------===//
1696
1697void CopyPhysRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1698 CopyRenderer::emitRenderOpcodes(Table, NewInsnID, OldInsnID, OldOpIdx,
1699 Name: PhysReg->getName());
1700}
1701
1702//===- CopyOrAddZeroRegRenderer -------------------------------------------===//
1703
1704void CopyOrAddZeroRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1705 Table << MatchTable::Opcode(Opcode: "GIR_CopyOrAddZeroReg")
1706 << MatchTable::Comment(Comment: "NewInsnID")
1707 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1708 << MatchTable::Comment(Comment: "OldInsnID")
1709 << MatchTable::ULEB128Value(IntValue: OldInsnID) << MatchTable::Comment(Comment: "OpIdx")
1710 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1711 << MatchTable::NamedValue(
1712 NumBytes: 2,
1713 Namespace: (ZeroRegisterDef->getValue(Name: "Namespace")
1714 ? ZeroRegisterDef->getValueAsString(FieldName: "Namespace")
1715 : ""),
1716 NamedValue: ZeroRegisterDef->getName())
1717 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1718}
1719
1720//===- CopyConstantAsImmRenderer ------------------------------------------===//
1721
1722void CopyConstantAsImmRenderer::emitRenderOpcodes(MatchTable &Table) const {
1723 Table << MatchTable::Opcode(Opcode: Signed ? "GIR_CopyConstantAsSImm"
1724 : "GIR_CopyConstantAsUImm")
1725 << MatchTable::Comment(Comment: "NewInsnID")
1726 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1727 << MatchTable::Comment(Comment: "OldInsnID")
1728 << MatchTable::ULEB128Value(IntValue: OldInsnID)
1729 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1730}
1731
1732//===- CopyFConstantAsFPImmRenderer ---------------------------------------===//
1733
1734void CopyFConstantAsFPImmRenderer::emitRenderOpcodes(MatchTable &Table) const {
1735 Table << MatchTable::Opcode(Opcode: "GIR_CopyFConstantAsFPImm")
1736 << MatchTable::Comment(Comment: "NewInsnID")
1737 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1738 << MatchTable::Comment(Comment: "OldInsnID")
1739 << MatchTable::ULEB128Value(IntValue: OldInsnID)
1740 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1741}
1742
1743//===- CopySubRegRenderer -------------------------------------------------===//
1744
1745void CopySubRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1746 Table << MatchTable::Opcode(Opcode: "GIR_CopySubReg")
1747 << MatchTable::Comment(Comment: "NewInsnID")
1748 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1749 << MatchTable::Comment(Comment: "OldInsnID")
1750 << MatchTable::ULEB128Value(IntValue: OldInsnID) << MatchTable::Comment(Comment: "OpIdx")
1751 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1752 << MatchTable::Comment(Comment: "SubRegIdx")
1753 << MatchTable::IntValue(NumBytes: 2, IntValue: SubReg->EnumValue)
1754 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1755}
1756
1757//===- AddRegisterRenderer ------------------------------------------------===//
1758
1759void AddRegisterRenderer::emitRenderOpcodes(MatchTable &Table) const {
1760 Table << MatchTable::Opcode(Opcode: "GIR_AddRegister")
1761 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID);
1762 if (RegisterDef->getName() != "zero_reg") {
1763 Table << MatchTable::NamedValue(
1764 NumBytes: 2,
1765 Namespace: (RegisterDef->getValue(Name: "Namespace")
1766 ? RegisterDef->getValueAsString(FieldName: "Namespace")
1767 : ""),
1768 NamedValue: RegisterDef->getName());
1769 } else {
1770 Table << MatchTable::NamedValue(NumBytes: 2, Namespace: Target.getRegNamespace(), NamedValue: "NoRegister");
1771 }
1772 Table << MatchTable::Comment(Comment: "AddRegisterRegFlags");
1773
1774 // TODO: This is encoded as a 64-bit element, but only 16 or 32-bits are
1775 // really needed for a physical register reference. We can pack the
1776 // register and flags in a single field.
1777 if (IsDef) {
1778 Table << MatchTable::NamedValue(
1779 NumBytes: 2, NamedValue: IsDead ? "static_cast<uint16_t>(RegState::Define|RegState::Dead)"
1780 : "static_cast<uint16_t>(RegState::Define)");
1781 } else {
1782 assert(!IsDead && "A use cannot be dead");
1783 Table << MatchTable::IntValue(NumBytes: 2, IntValue: 0);
1784 }
1785 Table << MatchTable::LineBreak;
1786}
1787
1788//===- TempRegRenderer ----------------------------------------------------===//
1789
1790void TempRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1791 const bool NeedsFlags = (SubRegIdx || IsDef);
1792 if (SubRegIdx) {
1793 assert(!IsDef);
1794 Table << MatchTable::Opcode(Opcode: "GIR_AddTempSubRegister");
1795 } else {
1796 Table << MatchTable::Opcode(Opcode: NeedsFlags ? "GIR_AddTempRegister"
1797 : "GIR_AddSimpleTempRegister");
1798 }
1799
1800 Table << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1801 << MatchTable::Comment(Comment: "TempRegID")
1802 << MatchTable::ULEB128Value(IntValue: TempRegID);
1803
1804 if (!NeedsFlags) {
1805 Table << MatchTable::LineBreak;
1806 return;
1807 }
1808
1809 Table << MatchTable::Comment(Comment: "TempRegFlags");
1810 if (IsDef) {
1811 SmallString<32> RegFlags;
1812 RegFlags += "static_cast<uint16_t>(RegState::Define";
1813 if (IsDead)
1814 RegFlags += "|RegState::Dead";
1815 RegFlags += ")";
1816 Table << MatchTable::NamedValue(NumBytes: 2, NamedValue: RegFlags);
1817 } else {
1818 Table << MatchTable::IntValue(NumBytes: 2, IntValue: 0);
1819 }
1820
1821 if (SubRegIdx)
1822 Table << MatchTable::NamedValue(NumBytes: 2, NamedValue: SubRegIdx->getQualifiedName());
1823 Table << MatchTable::LineBreak;
1824}
1825
1826//===- ImmRenderer --------------------------------------------------------===//
1827
1828void ImmRenderer::emitAddImm(MatchTable &Table, unsigned InsnID, int64_t Imm,
1829 StringRef ImmName) {
1830 const bool IsInt8 = isInt<8>(x: Imm);
1831
1832 Table << MatchTable::Opcode(Opcode: IsInt8 ? "GIR_AddImm8" : "GIR_AddImm")
1833 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1834 << MatchTable::Comment(Comment: ImmName)
1835 << MatchTable::IntValue(NumBytes: IsInt8 ? 1 : 8, IntValue: Imm) << MatchTable::LineBreak;
1836}
1837
1838void ImmRenderer::emitRenderOpcodes(MatchTable &Table) const {
1839 if (CImmLLT) {
1840 assert(Table.isCombiner() &&
1841 "ConstantInt immediate are only for combiners!");
1842 Table << MatchTable::Opcode(Opcode: "GIR_AddCImm") << MatchTable::Comment(Comment: "InsnID")
1843 << MatchTable::ULEB128Value(IntValue: InsnID) << MatchTable::Comment(Comment: "Type");
1844 emitType(Table, Ty: *CImmLLT);
1845 Table << MatchTable::Comment(Comment: "Imm") << MatchTable::IntValue(NumBytes: 8, IntValue: Imm)
1846 << MatchTable::LineBreak;
1847 } else {
1848 emitAddImm(Table, InsnID, Imm);
1849 }
1850}
1851
1852//===- SubRegIndexRenderer ------------------------------------------------===//
1853
1854void SubRegIndexRenderer::emitRenderOpcodes(MatchTable &Table) const {
1855 ImmRenderer::emitAddImm(Table, InsnID, Imm: SubRegIdx->EnumValue, ImmName: "SubRegIndex");
1856}
1857
1858//===- RenderComplexPatternOperand ----------------------------------------===//
1859
1860void RenderComplexPatternOperand::emitRenderOpcodes(MatchTable &Table) const {
1861 Table << MatchTable::Opcode(
1862 Opcode: SubOperand ? (SubReg ? "GIR_ComplexSubOperandSubRegRenderer"
1863 : "GIR_ComplexSubOperandRenderer")
1864 : "GIR_ComplexRenderer")
1865 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1866 << MatchTable::Comment(Comment: "RendererID")
1867 << MatchTable::IntValue(NumBytes: 2, IntValue: RendererID);
1868 if (SubOperand)
1869 Table << MatchTable::Comment(Comment: "SubOperand")
1870 << MatchTable::ULEB128Value(IntValue: *SubOperand);
1871 if (SubReg)
1872 Table << MatchTable::Comment(Comment: "SubRegIdx")
1873 << MatchTable::IntValue(NumBytes: 2, IntValue: SubReg->EnumValue);
1874 Table << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1875}
1876
1877//===- IntrinsicIDRenderer ------------------------------------------------===//
1878
1879void IntrinsicIDRenderer::emitRenderOpcodes(MatchTable &Table) const {
1880 Table << MatchTable::Opcode(Opcode: "GIR_AddIntrinsicID") << MatchTable::Comment(Comment: "MI")
1881 << MatchTable::ULEB128Value(IntValue: InsnID)
1882 << MatchTable::NamedValue(NumBytes: 2, NamedValue: "Intrinsic::" + II->EnumName.str())
1883 << MatchTable::LineBreak;
1884}
1885
1886//===- CustomRenderer -----------------------------------------------------===//
1887
1888void CustomRenderer::emitRenderOpcodes(MatchTable &Table) const {
1889 Table << MatchTable::Opcode(Opcode: "GIR_CustomRenderer")
1890 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1891 << MatchTable::Comment(Comment: "OldInsnID")
1892 << MatchTable::ULEB128Value(IntValue: OldInsnID)
1893 << MatchTable::Comment(Comment: "Renderer")
1894 << MatchTable::NamedValue(
1895 NumBytes: 2, NamedValue: "GICR_" + Renderer.getValueAsString(FieldName: "RendererFn").str())
1896 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1897}
1898
1899//===- CustomOperandRenderer ----------------------------------------------===//
1900
1901void CustomOperandRenderer::emitRenderOpcodes(MatchTable &Table) const {
1902 Table << MatchTable::Opcode(Opcode: "GIR_CustomOperandRenderer")
1903 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1904 << MatchTable::Comment(Comment: "OldInsnID")
1905 << MatchTable::ULEB128Value(IntValue: OldInsnID) << MatchTable::Comment(Comment: "OpIdx")
1906 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1907 << MatchTable::Comment(Comment: "OperandRenderer")
1908 << MatchTable::NamedValue(
1909 NumBytes: 2, NamedValue: "GICR_" + Renderer.getValueAsString(FieldName: "RendererFn").str())
1910 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1911}
1912
1913//===- BuildMIAction ------------------------------------------------------===//
1914
1915bool BuildMIAction::canMutate(RuleMatcher &Rule,
1916 const InstructionMatcher *Insn) const {
1917 if (!Insn || Insn->hasVariadicMatcher())
1918 return false;
1919
1920 if (OperandRenderers.size() != Insn->getNumOperandMatchers())
1921 return false;
1922
1923 for (const auto &Renderer : enumerate(First: OperandRenderers)) {
1924 if (const auto *Copy = dyn_cast<CopyRenderer>(Val: &*Renderer.value())) {
1925 const OperandMatcher &OM =
1926 Rule.getOperandMatcher(Name: Copy->getSymbolicName());
1927 if (Insn != &OM.getInstructionMatcher() ||
1928 OM.getOpIdx() != Renderer.index())
1929 return false;
1930 } else {
1931 return false;
1932 }
1933 }
1934
1935 return true;
1936}
1937
1938void BuildMIAction::chooseInsnToMutate(RuleMatcher &Rule) {
1939 for (auto *MutateCandidate : Rule.mutatable_insns()) {
1940 if (canMutate(Rule, Insn: MutateCandidate)) {
1941 // Take the first one we're offered that we're able to mutate.
1942 Rule.reserveInsnMatcherForMutation(InsnMatcher: MutateCandidate);
1943 Matched = MutateCandidate;
1944 Rule.tryEraseInsnID(ID: MutateCandidate->getInsnVarID());
1945 return;
1946 }
1947 }
1948}
1949
1950void BuildMIAction::emitActionOpcodes(MatchTable &Table) const {
1951 const auto AddMIFlags = [&]() {
1952 for (const InstructionMatcher *IM : CopiedFlags) {
1953 Table << MatchTable::Opcode(Opcode: "GIR_CopyMIFlags")
1954 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1955 << MatchTable::Comment(Comment: "OldInsnID")
1956 << MatchTable::ULEB128Value(IntValue: IM->getInsnVarID())
1957 << MatchTable::LineBreak;
1958 }
1959
1960 if (!SetFlags.empty()) {
1961 Table << MatchTable::Opcode(Opcode: "GIR_SetMIFlags")
1962 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1963 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: SetFlags, Separator: " | "))
1964 << MatchTable::LineBreak;
1965 }
1966
1967 if (!UnsetFlags.empty()) {
1968 Table << MatchTable::Opcode(Opcode: "GIR_UnsetMIFlags")
1969 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1970 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: UnsetFlags, Separator: " | "))
1971 << MatchTable::LineBreak;
1972 }
1973 };
1974
1975 if (Matched) {
1976 unsigned RecycleInsnID = Matched->getInsnVarID();
1977 Table << MatchTable::Opcode(Opcode: "GIR_MutateOpcode")
1978 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1979 << MatchTable::Comment(Comment: "RecycleInsnID")
1980 << MatchTable::ULEB128Value(IntValue: RecycleInsnID)
1981 << MatchTable::Comment(Comment: "Opcode")
1982 << MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName())
1983 << MatchTable::LineBreak;
1984
1985 if (!I->ImplicitDefs.empty() || !I->ImplicitUses.empty()) {
1986 for (auto *Def : I->ImplicitDefs) {
1987 auto Namespace = Def->getValue(Name: "Namespace")
1988 ? Def->getValueAsString(FieldName: "Namespace")
1989 : "";
1990 const bool IsDead = DeadImplicitDefs.contains(Ptr: Def);
1991 Table << MatchTable::Opcode(Opcode: "GIR_AddImplicitDef")
1992 << MatchTable::Comment(Comment: "InsnID")
1993 << MatchTable::ULEB128Value(IntValue: InsnID)
1994 << MatchTable::NamedValue(NumBytes: 2, Namespace, NamedValue: Def->getName())
1995 << (IsDead ? MatchTable::NamedValue(
1996 NumBytes: 2, NamedValue: "static_cast<unsigned>(RegState::Dead)")
1997 : MatchTable::IntValue(NumBytes: 2, IntValue: 0))
1998 << MatchTable::LineBreak;
1999 }
2000 for (auto *Use : I->ImplicitUses) {
2001 auto Namespace = Use->getValue(Name: "Namespace")
2002 ? Use->getValueAsString(FieldName: "Namespace")
2003 : "";
2004 Table << MatchTable::Opcode(Opcode: "GIR_AddImplicitUse")
2005 << MatchTable::Comment(Comment: "InsnID")
2006 << MatchTable::ULEB128Value(IntValue: InsnID)
2007 << MatchTable::NamedValue(NumBytes: 2, Namespace, NamedValue: Use->getName())
2008 << MatchTable::LineBreak;
2009 }
2010 }
2011
2012 AddMIFlags();
2013
2014 // Mark the mutated instruction as erased.
2015 return;
2016 }
2017
2018 // TODO: Simple permutation looks like it could be almost as common as
2019 // mutation due to commutative operations.
2020
2021 if (InsnID == 0) {
2022 Table << MatchTable::Opcode(Opcode: "GIR_BuildRootMI");
2023 } else {
2024 Table << MatchTable::Opcode(Opcode: "GIR_BuildMI") << MatchTable::Comment(Comment: "InsnID")
2025 << MatchTable::ULEB128Value(IntValue: InsnID);
2026 }
2027
2028 Table << MatchTable::Comment(Comment: "Opcode")
2029 << MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName())
2030 << MatchTable::LineBreak;
2031
2032 for (const auto &Renderer : OperandRenderers)
2033 Renderer->emitRenderOpcodes(Table);
2034
2035 for (auto [OpIdx, Def] : enumerate(First: I->ImplicitDefs)) {
2036 auto Namespace =
2037 Def->getValue(Name: "Namespace") ? Def->getValueAsString(FieldName: "Namespace") : "";
2038 if (DeadImplicitDefs.contains(Ptr: Def)) {
2039 Table
2040 << MatchTable::Opcode(Opcode: "GIR_SetImplicitDefDead")
2041 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2042 << MatchTable::Comment(
2043 Comment: ("OpIdx for " + Namespace + "::" + Def->getName() + "").str())
2044 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
2045 }
2046 }
2047
2048 if (!MergeInsnIDs.empty()) {
2049 assert(I->mayLoad || I->mayStore);
2050 Table << MatchTable::Opcode(Opcode: "GIR_MergeMemOperands")
2051 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2052 << MatchTable::Comment(Comment: "NumInsns")
2053 << MatchTable::IntValue(NumBytes: 1, IntValue: MergeInsnIDs.size())
2054 << MatchTable::Comment(Comment: "MergeInsnID's");
2055 for (const auto &MergeInsnID : MergeInsnIDs)
2056 Table << MatchTable::ULEB128Value(IntValue: MergeInsnID);
2057 Table << MatchTable::LineBreak;
2058 }
2059
2060 AddMIFlags();
2061}
2062
2063//===- BuildConstantAction ------------------------------------------------===//
2064
2065void BuildConstantAction::emitActionOpcodes(MatchTable &Table) const {
2066 Table << MatchTable::Opcode(Opcode: "GIR_BuildConstant")
2067 << MatchTable::Comment(Comment: "TempRegID")
2068 << MatchTable::ULEB128Value(IntValue: TempRegID) << MatchTable::Comment(Comment: "Val")
2069 << MatchTable::IntValue(NumBytes: 8, IntValue: Val) << MatchTable::LineBreak;
2070}
2071
2072//===- EraseInstAction ----------------------------------------------------===//
2073
2074void EraseInstAction::emitActionOpcodes(MatchTable &Table) const {
2075 Table << MatchTable::Opcode(Opcode: "GIR_EraseFromParent")
2076 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2077 << MatchTable::LineBreak;
2078}
2079
2080bool EraseInstAction::emitActionOpcodesAndDone(
2081 MatchTable &Table, function_ref<void()> OnDone) const {
2082 if (InsnID != 0) {
2083 emitActionOpcodes(Table);
2084 return false;
2085 }
2086
2087 OnDone();
2088 Table << MatchTable::Opcode(Opcode: "GIR_EraseRootFromParent_Done", IndentAdjust: -1)
2089 << MatchTable::LineBreak;
2090 return true;
2091}
2092
2093//===- ReplaceRegAction ---------------------------------------------------===//
2094
2095void ReplaceRegAction::emitAdditionalPredicates(MatchTable &Table) const {
2096 if (TempRegID != (unsigned)-1)
2097 return;
2098
2099 Table << MatchTable::Opcode(Opcode: "GIM_CheckCanReplaceReg")
2100 << MatchTable::Comment(Comment: "OldInsnID")
2101 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2102 << MatchTable::Comment(Comment: "OldOpIdx") << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2103 << MatchTable::Comment(Comment: "NewInsnId")
2104 << MatchTable::ULEB128Value(IntValue: NewInsnId)
2105 << MatchTable::Comment(Comment: "NewOpIdx") << MatchTable::ULEB128Value(IntValue: NewOpIdx)
2106 << MatchTable::LineBreak;
2107}
2108
2109void ReplaceRegAction::emitActionOpcodes(MatchTable &Table) const {
2110 if (TempRegID != (unsigned)-1) {
2111 Table << MatchTable::Opcode(Opcode: "GIR_ReplaceRegWithTempReg")
2112 << MatchTable::Comment(Comment: "OldInsnID")
2113 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2114 << MatchTable::Comment(Comment: "OldOpIdx")
2115 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2116 << MatchTable::Comment(Comment: "TempRegID")
2117 << MatchTable::ULEB128Value(IntValue: TempRegID) << MatchTable::LineBreak;
2118 } else {
2119 Table << MatchTable::Opcode(Opcode: "GIR_ReplaceReg")
2120 << MatchTable::Comment(Comment: "OldInsnID")
2121 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2122 << MatchTable::Comment(Comment: "OldOpIdx")
2123 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2124 << MatchTable::Comment(Comment: "NewInsnId")
2125 << MatchTable::ULEB128Value(IntValue: NewInsnId)
2126 << MatchTable::Comment(Comment: "NewOpIdx")
2127 << MatchTable::ULEB128Value(IntValue: NewOpIdx) << MatchTable::LineBreak;
2128 }
2129}
2130
2131//===- ConstrainOperandToRegClassAction -----------------------------------===//
2132
2133void ConstrainOperandToRegClassAction::emitActionOpcodes(
2134 MatchTable &Table) const {
2135 Table << MatchTable::Opcode(Opcode: "GIR_ConstrainOperandRC")
2136 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2137 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
2138 << MatchTable::NamedValue(NumBytes: 2, NamedValue: RC.getQualifiedIdName())
2139 << MatchTable::LineBreak;
2140}
2141
2142//===- MakeTempRegisterAction ---------------------------------------------===//
2143
2144void MakeTempRegisterAction::emitActionOpcodes(MatchTable &Table) const {
2145 Table << MatchTable::Opcode(Opcode: "GIR_MakeTempReg")
2146 << MatchTable::Comment(Comment: "TempRegID")
2147 << MatchTable::ULEB128Value(IntValue: TempRegID) << MatchTable::Comment(Comment: "TypeID");
2148 emitType(Table, Ty);
2149 Table << MatchTable::LineBreak;
2150}
2151