1//===- Matchers.cpp -------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "Matchers.h"
10#include "Common/CodeGenInstruction.h"
11#include "Common/CodeGenRegisters.h"
12#include "llvm/ADT/Statistic.h"
13#include "llvm/Support/CommandLine.h"
14#include "llvm/Support/Debug.h"
15#include "llvm/Support/LEB128.h"
16#include "llvm/Support/ScopedPrinter.h"
17#include "llvm/Support/raw_ostream.h"
18#include "llvm/TableGen/Error.h"
19
20#define DEBUG_TYPE "gi-match-table-matchers"
21
22STATISTIC(NumPatternEmitted, "Number of patterns emitted");
23
24using namespace llvm;
25using namespace gi;
26
27// FIXME: Use createStringError instead.
28static Error failUnsupported(const Twine &Reason) {
29 return make_error<StringError>(Args: Reason, Args: inconvertibleErrorCode());
30}
31
32/// Get the name of the enum value used to number the predicate function.
33static std::string getEnumNameForPredicate(const TreePredicateFn &Predicate) {
34 if (Predicate.hasGISelPredicateCode())
35 return "GICXXPred_MI_" + Predicate.getFnName();
36 if (Predicate.hasGISelLeafPredicateCode())
37 return "GICXXPred_MO_" + Predicate.getFnName();
38 return "GICXXPred_" + Predicate.getImmTypeIdentifier().str() + "_" +
39 Predicate.getFnName();
40}
41
42static std::string
43getMatchOpcodeForImmPredicate(const TreePredicateFn &Predicate) {
44 return "GIM_Check" + Predicate.getImmTypeIdentifier().str() + "ImmPredicate";
45}
46
47//===- Helpers ------------------------------------------------------------===//
48
49template <class GroupT>
50static std::vector<Matcher *>
51optimizeRules(ArrayRef<Matcher *> Rules,
52 std::vector<std::unique_ptr<Matcher>> &MatcherStorage) {
53
54 std::vector<Matcher *> Worklist(Rules.begin(), Rules.end());
55 std::vector<Matcher *> OptRules;
56 std::unique_ptr<GroupT> CurrentGroup = std::make_unique<GroupT>();
57 assert(CurrentGroup->empty() && "Newly created group isn't empty!");
58 unsigned NumGroups = 0;
59
60 auto ProcessCurrentGroup = [&]() {
61 if (CurrentGroup->empty())
62 // An empty group is good to be reused:
63 return;
64
65 // If the group isn't large enough to provide any benefit, move all the
66 // added rules out of it and make sure to re-create the group to properly
67 // re-initialize it:
68 if (CurrentGroup->size() < 2)
69 append_range(OptRules, CurrentGroup->matchers());
70 else {
71 CurrentGroup->finalize();
72 CurrentGroup->optimize();
73 OptRules.push_back(CurrentGroup.get());
74 MatcherStorage.emplace_back(std::move(CurrentGroup));
75 ++NumGroups;
76 }
77 CurrentGroup = std::make_unique<GroupT>();
78 };
79
80 for (Matcher *Rule : Worklist) {
81 // Greedily add as many matchers as possible to the current group:
82 if (CurrentGroup->addMatcher(*Rule))
83 continue;
84
85 ProcessCurrentGroup();
86 assert(CurrentGroup->empty() && "A group wasn't properly re-initialized");
87
88 // Try to add the pending matcher to a newly created empty group:
89 if (!CurrentGroup->addMatcher(*Rule))
90 // If we couldn't add the matcher to an empty group, that group type
91 // doesn't support that kind of matchers at all, so just skip it:
92 OptRules.push_back(x: Rule);
93 }
94 ProcessCurrentGroup();
95
96 assert(OptRules.size() <= Worklist.size() && "Optimization added rules?");
97 LLVM_DEBUG(dbgs() << "NumGroups: " << NumGroups << "\n");
98 (void)NumGroups;
99 assert(CurrentGroup->empty() && "The last group wasn't properly processed");
100 return OptRules;
101}
102
103std::vector<Matcher *> llvm::gi::optimizeRuleset(
104 MutableArrayRef<RuleMatcher> Rules,
105 std::vector<std::unique_ptr<Matcher>> &MatcherStorage) {
106 SmallVector<Matcher *> InputRules(make_pointer_range(Range&: Rules));
107
108 // Now sort the Rules.
109 unsigned CurrentOrdering = 0;
110 StringMap<unsigned> OpcodeOrder;
111 for (RuleMatcher &Rule : Rules) {
112 const StringRef Opcode = Rule.getOpcode();
113 assert(!Opcode.empty() && "Didn't expect an undefined opcode");
114 if (OpcodeOrder.try_emplace(Key: Opcode, Args&: CurrentOrdering).second)
115 ++CurrentOrdering;
116 }
117
118 llvm::stable_sort(
119 Range&: InputRules, C: [&OpcodeOrder](const Matcher *A, const Matcher *B) {
120 const auto *L = cast<RuleMatcher>(Val: A);
121 const auto *R = cast<RuleMatcher>(Val: B);
122 return std::tuple(OpcodeOrder[L->getOpcode()],
123 L->roots_front().getNumOperandMatchers()) <
124 std::tuple(OpcodeOrder[R->getOpcode()],
125 R->roots_front().getNumOperandMatchers());
126 });
127
128 for (Matcher *R : InputRules)
129 R->optimize();
130
131 // Then form groups, and switches in that order.
132 std::vector<Matcher *> OptRules =
133 optimizeRules<GroupMatcher>(Rules: InputRules, MatcherStorage);
134 OptRules = optimizeRules<SwitchMatcher>(Rules: OptRules, MatcherStorage);
135 return OptRules;
136}
137
138MatchTable llvm::gi::buildMatchTable(ArrayRef<Matcher *> Rules,
139 bool WithCoverage, bool IsCombiner) {
140 MatchTable Table(WithCoverage, IsCombiner);
141 for (Matcher *Rule : Rules)
142 Rule->emit(Table);
143
144 return Table << MatchTable::Opcode(Opcode: "GIM_Reject") << MatchTable::LineBreak;
145}
146
147template <class Range> static bool matchersRecordOperand(Range &&R) {
148 return any_of(R, [](const auto &I) { return I->recordsOperand(); });
149}
150
151static void emitType(MatchTable &Table, const LLTCodeGenOrTempType &Ty) {
152 if (Ty.isLLTCodeGen())
153 Table << MatchTable::NamedValue(NumBytes: 1, NamedValue: Ty.getLLTCodeGen().getCxxEnumValue());
154 else
155 Table << MatchTable::IntValue(NumBytes: 1, IntValue: Ty.getTempTypeIdx());
156}
157
158//===- Matcher ------------------------------------------------------------===//
159
160void Matcher::optimize() {}
161
162Matcher::~Matcher() = default;
163
164//===- GroupMatcher -------------------------------------------------------===//
165
166bool GroupMatcher::recordsOperand() const {
167 return matchersRecordOperand(R: Conditions) || matchersRecordOperand(R: Matchers);
168}
169
170bool GroupMatcher::candidateConditionMatches(
171 const PredicateMatcher &Predicate) const {
172
173 if (empty()) {
174 // Sharing predicates for nested instructions is not supported yet as we
175 // currently don't hoist the GIM_RecordInsn's properly, therefore we can
176 // only work on the original root instruction (InsnVarID == 0):
177 if (Predicate.getInsnVarID() != 0)
178 return false;
179 // ... otherwise an empty group can handle any predicate with no specific
180 // requirements:
181 return true;
182 }
183
184 const Matcher &Representative = **Matchers.begin();
185 const auto &RepresentativeCondition = Representative.getFirstCondition();
186 // ... if not empty, the group can only accomodate matchers with the exact
187 // same first condition:
188 return Predicate.isIdentical(B: RepresentativeCondition);
189}
190
191std::unique_ptr<PredicateMatcher> GroupMatcher::popFirstCondition() {
192 assert(!Conditions.empty() &&
193 "Trying to pop a condition from a condition-less group");
194 std::unique_ptr<PredicateMatcher> P = std::move(Conditions.front());
195 Conditions.erase(CI: Conditions.begin());
196 return P;
197}
198
199bool GroupMatcher::addMatcher(Matcher &Candidate) {
200 if (!Candidate.hasFirstCondition())
201 return false;
202
203 // Only add candidates that have a matching first condition that can be
204 // hoisted into the GroupMatcher.
205 const PredicateMatcher &Predicate = Candidate.getFirstCondition();
206 if (!candidateConditionMatches(Predicate) ||
207 !Predicate.canHoistOutsideOf(M: Candidate))
208 return false;
209
210 Matchers.push_back(x: &Candidate);
211 return true;
212}
213
214void GroupMatcher::finalize() {
215 assert(Conditions.empty() && "Already finalized?");
216 if (empty())
217 return;
218
219 Matcher &FirstRule = **Matchers.begin();
220 for (;;) {
221 // All the checks are expected to succeed during the first iteration:
222 for (const auto &Rule : Matchers)
223 if (!Rule->hasFirstCondition())
224 return;
225 // Hoist the first condition if it is identical in all matchers in the group
226 // and it can be hoisted in every matcher.
227 const auto &FirstCondition = FirstRule.getFirstCondition();
228 if (!FirstCondition.canHoistOutsideOf(M: FirstRule))
229 return;
230 for (unsigned I = 1, E = Matchers.size(); I < E; ++I) {
231 const auto &OtherFirstCondition = Matchers[I]->getFirstCondition();
232 if (!OtherFirstCondition.isIdentical(B: FirstCondition) ||
233 !OtherFirstCondition.canHoistOutsideOf(M: *Matchers[I]))
234 return;
235 }
236
237 Conditions.push_back(Elt: FirstRule.popFirstCondition());
238 for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
239 Matchers[I]->popFirstCondition();
240 }
241}
242
243void GroupMatcher::emit(MatchTable &Table) {
244 unsigned LabelID = ~0U;
245 if (!Conditions.empty()) {
246 LabelID = Table.allocateLabelID();
247 Table << MatchTable::Opcode(Opcode: "GIM_Try", IndentAdjust: +1)
248 << MatchTable::Comment(Comment: "On fail goto")
249 << MatchTable::JumpTarget(LabelID) << MatchTable::LineBreak;
250 }
251 for (auto &Condition : Conditions)
252 Condition->emitPredicateOpcodes(Table);
253
254 for (const auto &M : Matchers)
255 M->emit(Table);
256
257 // Exit the group
258 if (!Conditions.empty())
259 Table << MatchTable::Opcode(Opcode: "GIM_Reject", IndentAdjust: -1) << MatchTable::LineBreak
260 << MatchTable::Label(LabelID);
261}
262
263void GroupMatcher::optimize() {
264 // Make sure we only sort by a specific predicate within a range of rules that
265 // all have that predicate checked against a specific value (not a wildcard):
266 // TODO: Is this even relevant ? Check diffs w/ just using a simple sort
267 // instead of this.
268 auto F = Matchers.begin();
269 auto T = F;
270 auto E = Matchers.end();
271 while (T != E) {
272 while (T != E) {
273 if (!(*T)->getFirstConditionAsRootType().get().isValid())
274 break;
275 ++T;
276 }
277 std::stable_sort(first: F, last: T, comp: [](Matcher *A, Matcher *B) {
278 return A->getFirstConditionAsRootType() <
279 B->getFirstConditionAsRootType();
280 });
281 if (T != E)
282 F = ++T;
283 }
284 Matchers = optimizeRules<GroupMatcher>(Rules: Matchers, MatcherStorage);
285 Matchers = optimizeRules<SwitchMatcher>(Rules: Matchers, MatcherStorage);
286}
287
288LLTCodeGen GroupMatcher::getFirstConditionAsRootType() const {
289 if (!hasFirstCondition())
290 return {};
291
292 const PredicateMatcher &PM = *Conditions.front();
293 if (const auto *TM = dyn_cast<LLTOperandMatcher>(Val: &PM)) {
294 if (TM->getInsnVarID() == 0 && TM->getOpIdx() == 0)
295 return TM->getTy();
296 }
297
298 return {};
299}
300
301//===- SwitchMatcher ------------------------------------------------------===//
302
303SwitchMatcher::SwitchMatcher() : Matcher(MK_Switch) {}
304SwitchMatcher::~SwitchMatcher() = default;
305
306bool SwitchMatcher::recordsOperand() const {
307 assert(!isa_and_present<RecordNamedOperandMatcher>(Condition.get()) &&
308 "Switch conditions should not record named operands");
309 return matchersRecordOperand(R: Matchers);
310}
311
312bool SwitchMatcher::isSupportedPredicateType(const PredicateMatcher &P) {
313 return isa<InstructionOpcodeMatcher>(Val: P) || isa<LLTOperandShapeMatcher>(Val: P) ||
314 isa<LLTOperandMatcher>(Val: P);
315}
316
317bool SwitchMatcher::candidateConditionMatches(
318 const PredicateMatcher &Predicate) const {
319
320 if (empty()) {
321 // Sharing predicates for nested instructions is not supported yet as we
322 // currently don't hoist the GIM_RecordInsn's properly, therefore we can
323 // only work on the original root instruction (InsnVarID == 0):
324 if (Predicate.getInsnVarID() != 0)
325 return false;
326 // ... while an attempt to add even a root matcher to an empty SwitchMatcher
327 // could fail as not all the types of conditions are supported:
328 if (!isSupportedPredicateType(P: Predicate))
329 return false;
330 // ... or the condition might not have a proper implementation of
331 // getValue() / isIdenticalDownToValue() yet:
332 if (!Predicate.hasValue())
333 return false;
334 // ... otherwise an empty Switch can accomodate the condition with no
335 // further requirements:
336 return true;
337 }
338
339 const Matcher &CaseRepresentative = **Matchers.begin();
340 const auto &RepresentativeCondition = CaseRepresentative.getFirstCondition();
341 // Switch-cases must share the same kind of condition and path to the value it
342 // checks:
343 if (!Predicate.isIdenticalDownToValue(B: RepresentativeCondition))
344 return false;
345
346 return true;
347}
348
349bool SwitchMatcher::addMatcher(Matcher &Candidate) {
350 if (!Candidate.hasFirstCondition())
351 return false;
352
353 const PredicateMatcher &Predicate = Candidate.getFirstCondition();
354 if (!candidateConditionMatches(Predicate))
355 return false;
356 const auto Value = Predicate.getValue();
357 auto It = Buckets.find(x: Value.RawValue);
358 if (It == Buckets.end())
359 It = Buckets.emplace(args: Value.RawValue, args: Bucket(Value)).first;
360#ifndef NDEBUG
361 else
362 assert(It->second.Value.Record.EmitStr == Value.Record.EmitStr &&
363 "Mismatched records for identical switch value");
364#endif
365 It->second.Matchers.push_back(x: &Candidate);
366 Matchers.push_back(x: &Candidate);
367 return true;
368}
369
370void SwitchMatcher::finalize() {
371 assert(Condition == nullptr && "Already finalized");
372#ifndef NDEBUG
373 unsigned NumBucketedMatchers = 0;
374 for (const auto &Entry : Buckets)
375 NumBucketedMatchers += Entry.second.Matchers.size();
376 assert(NumBucketedMatchers == Matchers.size() && "Broken SwitchMatcher");
377#endif
378 if (empty())
379 return;
380
381 Condition = Matchers.front()->popFirstCondition();
382 for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
383 Matchers[I]->popFirstCondition();
384
385 // After removing the switch condition, try to hoist any shared predicates
386 // within each switch bucket.
387 for (auto &Entry : Buckets) {
388 auto &BucketMatchers = Entry.second.Matchers;
389 BucketMatchers =
390 optimizeRules<GroupMatcher>(Rules: BucketMatchers, MatcherStorage);
391 }
392
393 Matchers.clear();
394 for (auto &Entry : Buckets)
395 append_range(C&: Matchers, R&: Entry.second.Matchers);
396}
397
398void SwitchMatcher::emitPredicateSpecificOpcodes(const PredicateMatcher &P,
399 MatchTable &Table) {
400 assert(isSupportedPredicateType(P) && "Predicate type is not supported");
401
402 if (const auto *Condition = dyn_cast<InstructionOpcodeMatcher>(Val: &P)) {
403 Table << MatchTable::Opcode(Opcode: "GIM_SwitchOpcode") << MatchTable::Comment(Comment: "MI")
404 << MatchTable::ULEB128Value(IntValue: Condition->getInsnVarID());
405 return;
406 }
407 if (const auto *Condition = dyn_cast<LLTOperandShapeMatcher>(Val: &P)) {
408 Table << MatchTable::Opcode(Opcode: "GIM_SwitchTypeShape")
409 << MatchTable::Comment(Comment: "MI")
410 << MatchTable::ULEB128Value(IntValue: Condition->getInsnVarID())
411 << MatchTable::Comment(Comment: "Op")
412 << MatchTable::ULEB128Value(IntValue: Condition->getOpIdx());
413 return;
414 }
415 if (const auto *Condition = dyn_cast<LLTOperandMatcher>(Val: &P)) {
416 Table << MatchTable::Opcode(Opcode: "GIM_SwitchType") << MatchTable::Comment(Comment: "MI")
417 << MatchTable::ULEB128Value(IntValue: Condition->getInsnVarID())
418 << MatchTable::Comment(Comment: "Op")
419 << MatchTable::ULEB128Value(IntValue: Condition->getOpIdx());
420 return;
421 }
422
423 llvm_unreachable("emitPredicateSpecificOpcodes is broken: can not handle a "
424 "predicate type that is claimed to be supported");
425}
426
427void SwitchMatcher::emit(MatchTable &Table) {
428#ifndef NDEBUG
429 unsigned NumBucketedMatchers = 0;
430 for (const auto &Entry : Buckets)
431 NumBucketedMatchers += Entry.second.Matchers.size();
432 assert(NumBucketedMatchers == Matchers.size() && "Broken SwitchMatcher");
433#endif
434 if (empty())
435 return;
436 assert(Condition != nullptr &&
437 "Broken SwitchMatcher, hasn't been finalized?");
438
439 std::vector<unsigned> LabelIDs(Buckets.size());
440 std::generate(first: LabelIDs.begin(), last: LabelIDs.end(),
441 gen: [&Table]() { return Table.allocateLabelID(); });
442 const unsigned Default = Table.allocateLabelID();
443
444 const int64_t LowerBound = Buckets.begin()->second.Value.RawValue;
445 const int64_t UpperBound = Buckets.rbegin()->second.Value.RawValue + 1;
446
447 emitPredicateSpecificOpcodes(P: *Condition, Table);
448
449 Table << MatchTable::Comment(Comment: "[") << MatchTable::IntValue(NumBytes: 2, IntValue: LowerBound)
450 << MatchTable::IntValue(NumBytes: 2, IntValue: UpperBound) << MatchTable::Comment(Comment: ")")
451 << MatchTable::Comment(Comment: "default:") << MatchTable::JumpTarget(LabelID: Default);
452
453 int64_t J = LowerBound;
454 unsigned CaseIdx = 0;
455 for (auto &Entry : Buckets) {
456 auto &V = Entry.second.Value;
457 while (J++ < V.RawValue)
458 Table << MatchTable::IntValue(NumBytes: 4, IntValue: 0);
459 V.Record.turnIntoComment();
460 Table << MatchTable::LineBreak << V.Record
461 << MatchTable::JumpTarget(LabelID: LabelIDs[CaseIdx]);
462 ++CaseIdx;
463 }
464 Table << MatchTable::LineBreak;
465
466 CaseIdx = 0;
467 for (auto &Entry : Buckets) {
468 Table << MatchTable::Label(LabelID: LabelIDs[CaseIdx]);
469 for (Matcher *M : Entry.second.Matchers)
470 M->emit(Table);
471 Table << MatchTable::Opcode(Opcode: "GIM_Reject") << MatchTable::LineBreak;
472 ++CaseIdx;
473 }
474 Table << MatchTable::Label(LabelID: Default);
475}
476
477//===- RuleMatcher --------------------------------------------------------===//
478
479RuleMatcher::RuleMatcher(ArrayRef<SMLoc> SrcLoc, bool UsesRecordOperand)
480 : Matcher(Matcher::MK_Rule), UsesRecordOperand(UsesRecordOperand),
481 SrcLoc(SrcLoc), RuleID(NextRuleID++) {}
482
483uint64_t RuleMatcher::NextRuleID = 0;
484
485StringRef RuleMatcher::getOpcode() const { return Roots.front()->getOpcode(); }
486
487bool RuleMatcher::recordsOperand() const {
488 return !usesRecordOperand() || matchersRecordOperand(R: InsnMatchers);
489}
490
491LLTCodeGen RuleMatcher::getFirstConditionAsRootType() const {
492 InstructionMatcher &InsnMatcher = *Roots.front();
493 if (!InsnMatcher.predicates_empty()) {
494 if (const auto *TM =
495 dyn_cast<LLTOperandMatcher>(Val: &**InsnMatcher.predicates_begin())) {
496 if (TM->getInsnVarID() == 0 && TM->getOpIdx() == 0)
497 return TM->getTy();
498 }
499 }
500 return {};
501}
502
503void RuleMatcher::optimize() {
504 for (const auto &InsnMatcher : InsnMatchers) {
505 for (auto &OM : InsnMatcher->operands()) {
506 // Complex Patterns are usually expensive and they relatively rarely fail
507 // on their own: more often we end up throwing away all the work done by a
508 // matching part of a complex pattern because some other part of the
509 // enclosing pattern didn't match. All of this makes it beneficial to
510 // delay complex patterns until the very end of the rule matching,
511 // especially for targets having lots of complex patterns.
512 for (auto &OP : OM->predicates())
513 if (isa<ComplexPatternOperandMatcher>(Val: OP))
514 EpilogueMatchers.emplace_back(args: std::move(OP));
515 OM->eraseNullPredicates();
516 }
517 InsnMatcher->optimize();
518 }
519 llvm::sort(C&: EpilogueMatchers, Comp: [](const std::unique_ptr<PredicateMatcher> &L,
520 const std::unique_ptr<PredicateMatcher> &R) {
521 return std::tuple(L->getKind(), L->getInsnVarID(), L->getOpIdx()) <
522 std::tuple(R->getKind(), R->getInsnVarID(), R->getOpIdx());
523 });
524
525 // Deduplicate EraseInst actions, and if an EraseInst erases the root, place
526 // it at the end to favor generation of GIR_EraseRootFromParent_Done
527 DenseSet<unsigned> AlreadySeenEraseInsts;
528 auto EraseRootIt = Actions.end();
529 auto It = Actions.begin();
530 while (It != Actions.end()) {
531 if (const auto *EI = dyn_cast<EraseInstAction>(Val: It->get())) {
532 unsigned InstID = EI->getInsnID();
533 if (!AlreadySeenEraseInsts.insert(V: InstID).second) {
534 It = Actions.erase(position: It);
535 continue;
536 }
537
538 if (InstID == 0)
539 EraseRootIt = It;
540 }
541
542 ++It;
543 }
544
545 if (EraseRootIt != Actions.end())
546 Actions.splice(position: Actions.end(), x&: Actions, i: EraseRootIt);
547}
548
549bool RuleMatcher::hasFirstCondition() const {
550 if (roots_empty())
551 return false;
552 InstructionMatcher &Matcher = roots_front();
553 if (!Matcher.predicates_empty())
554 return true;
555 for (auto &OM : Matcher.operands())
556 for (auto &OP : OM->predicates())
557 if (!isa<InstructionOperandMatcher>(Val: OP))
558 return true;
559 return false;
560}
561
562const PredicateMatcher &RuleMatcher::getFirstCondition() const {
563 assert(!roots_empty() &&
564 "Trying to get a condition from an empty RuleMatcher");
565
566 InstructionMatcher &Matcher = roots_front();
567 if (!Matcher.predicates_empty())
568 return **Matcher.predicates_begin();
569 // If there is no more predicate on the instruction itself, look at its
570 // operands.
571 for (auto &OM : Matcher.operands())
572 for (auto &OP : OM->predicates())
573 if (!isa<InstructionOperandMatcher>(Val: OP))
574 return *OP;
575
576 llvm_unreachable("Trying to get a condition from an InstructionMatcher with "
577 "no conditions");
578}
579
580std::unique_ptr<PredicateMatcher> RuleMatcher::popFirstCondition() {
581 assert(!roots_empty() &&
582 "Trying to pop a condition from an empty RuleMatcher");
583
584 InstructionMatcher &Matcher = roots_front();
585 if (!Matcher.predicates_empty())
586 return Matcher.predicates_pop_front();
587 // If there is no more predicate on the instruction itself, look at its
588 // operands.
589 for (auto &OM : Matcher.operands())
590 for (auto &OP : OM->predicates())
591 if (!isa<InstructionOperandMatcher>(Val: OP)) {
592 std::unique_ptr<PredicateMatcher> Result = std::move(OP);
593 OM->eraseNullPredicates();
594 return Result;
595 }
596
597 llvm_unreachable("Trying to pop a condition from an InstructionMatcher with "
598 "no conditions");
599}
600
601GISelFlags RuleMatcher::updateGISelFlag(GISelFlags CurFlags, const Record *R,
602 StringRef FlagName,
603 GISelFlags FlagBit) {
604 // If the value of a flag is unset, ignore it.
605 // If it's set, it always takes precedence over the existing value so
606 // clear/set the corresponding bit.
607 bool Unset = false;
608 bool Value = R->getValueAsBitOrUnset(FieldName: "GIIgnoreCopies", Unset);
609 if (!Unset)
610 return Value ? (CurFlags | FlagBit) : (CurFlags & ~FlagBit);
611 return CurFlags;
612}
613
614SaveAndRestore<GISelFlags> RuleMatcher::setGISelFlags(const Record *R) {
615 if (!R || !R->isSubClassOf(Name: "GISelFlags"))
616 return {Flags, Flags};
617
618 assert((R->isSubClassOf("PatFrags") || R->isSubClassOf("Pattern")) &&
619 "GISelFlags is only expected on Pattern/PatFrags!");
620
621 GISelFlags NewFlags =
622 updateGISelFlag(CurFlags: Flags, R, FlagName: "GIIgnoreCopies", FlagBit: GISF_IgnoreCopies);
623 return {Flags, NewFlags};
624}
625
626Error RuleMatcher::defineComplexSubOperand(StringRef SymbolicName,
627 const Record *ComplexPattern,
628 unsigned RendererID,
629 unsigned SubOperandID,
630 StringRef ParentSymbolicName) {
631 std::string ParentName(ParentSymbolicName);
632 auto [It, Inserted] = ComplexSubOperands.try_emplace(
633 Key: SymbolicName, Args&: ComplexPattern, Args&: RendererID, Args&: SubOperandID);
634 if (!Inserted) {
635 const std::string &RecordedParentName =
636 ComplexSubOperandsParentName[SymbolicName];
637 if (RecordedParentName != ParentName)
638 return failUnsupported(Reason: "Error: Complex suboperand " + SymbolicName +
639 " referenced by different operands: " +
640 RecordedParentName + " and " + ParentName + ".");
641 // Complex suboperand referenced more than once from same the operand is
642 // used to generate 'same operand check'. Emitting of
643 // GIR_ComplexSubOperandRenderer for them is already handled.
644 return Error::success();
645 }
646
647 ComplexSubOperandsParentName[SymbolicName] = std::move(ParentName);
648
649 return Error::success();
650}
651
652InstructionMatcher &
653RuleMatcher::allocateInstructionMatcher(StringRef SymbolicName,
654 bool AllowNumOpsCheck) {
655 return *InsnMatchers.emplace_back(args: std::make_unique<InstructionMatcher>(
656 args&: *this, args: InsnMatchers.size(), args&: SymbolicName, args&: AllowNumOpsCheck));
657}
658
659InstructionMatcher &RuleMatcher::addInstructionMatcher(StringRef SymbolicName) {
660 auto &Res = allocateInstructionMatcher(SymbolicName);
661 Roots.push_back(Elt: &Res);
662 MutatableInsns.insert(Ptr: &Res);
663 return Res;
664}
665
666void RuleMatcher::addRequiredSimplePredicate(StringRef PredName) {
667 RequiredSimplePredicates.push_back(x: PredName.str());
668}
669
670const std::vector<std::string> &RuleMatcher::getRequiredSimplePredicates() {
671 return RequiredSimplePredicates;
672}
673
674void RuleMatcher::defineOperand(StringRef SymbolicName, OperandMatcher &OM) {
675 if (DefinedOperands.try_emplace(Key: SymbolicName, Args: &OM).second)
676 return;
677
678 // If the operand is already defined, then we must ensure both references in
679 // the matcher have the exact same node.
680 RuleMatcher &RM = OM.getInstructionMatcher().getRuleMatcher();
681 auto &OtherOM = getOperandMatcher(Name: OM.getSymbolicName());
682 OM.addPredicate<SameOperandMatcher>(args: OtherOM.getInsnVarID(),
683 args: OtherOM.getOpIdx(), args: RM.getGISelFlags());
684}
685
686void RuleMatcher::definePhysRegOperand(const Record *Reg, OperandMatcher &OM) {
687 PhysRegOperands.try_emplace(Key: Reg, Args: &OM);
688}
689
690InstructionMatcher &
691RuleMatcher::getInstructionMatcher(StringRef SymbolicName) const {
692 for (const auto &InsnMatcher : InsnMatchers) {
693 if (InsnMatcher->getSymbolicName() == SymbolicName)
694 return *InsnMatcher;
695 }
696 llvm_unreachable(
697 ("Failed to lookup instruction " + SymbolicName).str().c_str());
698}
699
700const OperandMatcher &
701RuleMatcher::getPhysRegOperandMatcher(const Record *Reg) const {
702 const auto &I = PhysRegOperands.find(Key: Reg);
703
704 if (I == PhysRegOperands.end()) {
705 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Register " + Reg->getName() +
706 " was not declared in matcher");
707 }
708
709 return *I->second;
710}
711
712OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) {
713 const auto &I = DefinedOperands.find(Key: Name);
714
715 if (I == DefinedOperands.end())
716 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Operand " + Name + " was not declared in matcher");
717
718 return *I->second;
719}
720
721const OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) const {
722 const auto &I = DefinedOperands.find(Key: Name);
723
724 if (I == DefinedOperands.end())
725 PrintFatalError(ErrorLoc: SrcLoc, Msg: "Operand " + Name + " was not declared in matcher");
726
727 return *I->second;
728}
729
730void RuleMatcher::emit(MatchTable &Table) {
731 if (Roots.empty())
732 llvm_unreachable("Unexpected empty matcher!");
733
734 // The representation supports rules that require multiple roots such as:
735 // %ptr(p0) = ...
736 // %elt0(s32) = G_LOAD %ptr
737 // %1(p0) = G_ADD %ptr, 4
738 // %elt1(s32) = G_LOAD p0 %1
739 // which could be usefully folded into:
740 // %ptr(p0) = ...
741 // %elt0(s32), %elt1(s32) = TGT_LOAD_PAIR %ptr
742 // on some targets but we don't need to make use of that yet.
743 assert(Roots.size() == 1 && "Cannot handle multi-root matchers yet");
744
745 unsigned LabelID = Table.allocateLabelID();
746
747 if (!RequiredFeatures.empty() || HwModeIdx >= 0) {
748 Table << MatchTable::Opcode(Opcode: "GIM_Try_CheckFeatures", IndentAdjust: +1)
749 << MatchTable::Comment(Comment: "On fail goto")
750 << MatchTable::JumpTarget(LabelID)
751 << MatchTable::NamedValue(
752 NumBytes: 2, NamedValue: getNameForFeatureBitset(FeatureBitset: RequiredFeatures, HwModeIdx));
753 } else {
754 Table << MatchTable::Opcode(Opcode: "GIM_Try", IndentAdjust: +1)
755 << MatchTable::Comment(Comment: "On fail goto")
756 << MatchTable::JumpTarget(LabelID);
757 }
758 Table << MatchTable::Comment(Comment: ("Rule ID " + Twine(RuleID) + " //").str())
759 << MatchTable::LineBreak;
760
761 if (!RequiredSimplePredicates.empty()) {
762 for (const auto &Pred : RequiredSimplePredicates) {
763 Table << MatchTable::Opcode(Opcode: "GIM_CheckSimplePredicate")
764 << MatchTable::NamedValue(NumBytes: 2, NamedValue: Pred) << MatchTable::LineBreak;
765 }
766 }
767
768 Roots.front()->emitPredicateOpcodes(Table);
769
770 // Check if it's safe to replace registers.
771 for (const auto &MA : Actions)
772 MA->emitAdditionalPredicates(Table);
773
774 // We must also check if it's safe to fold the matched instructions.
775 if (InsnMatchers.size() >= 2) {
776
777 // FIXME: Emit checks to determine it's _actually_ safe to fold and/or
778 // account for unsafe cases.
779 //
780 // Example:
781 // MI1--> %0 = ...
782 // %1 = ... %0
783 // MI0--> %2 = ... %0
784 // It's not safe to erase MI1. We currently handle this by not
785 // erasing %0 (even when it's dead).
786 //
787 // Example:
788 // MI1--> %0 = load volatile @a
789 // %1 = load volatile @a
790 // MI0--> %2 = ... %0
791 // It's not safe to sink %0's def past %1. We currently handle
792 // this by rejecting all loads.
793 //
794 // Example:
795 // MI1--> %0 = load @a
796 // %1 = store @a
797 // MI0--> %2 = ... %0
798 // It's not safe to sink %0's def past %1. We currently handle
799 // this by rejecting all loads.
800 //
801 // Example:
802 // G_CONDBR %cond, @BB1
803 // BB0:
804 // MI1--> %0 = load @a
805 // G_BR @BB1
806 // BB1:
807 // MI0--> %2 = ... %0
808 // It's not always safe to sink %0 across control flow. In this
809 // case it may introduce a memory fault. We currentl handle
810 // this by rejecting all loads.
811
812 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsSafeToFold")
813 << MatchTable::Comment(Comment: "NumInsns")
814 << MatchTable::IntValue(NumBytes: 1, IntValue: InsnMatchers.size() - 1)
815 << MatchTable::LineBreak;
816 }
817
818 for (const auto &PM : EpilogueMatchers)
819 PM->emitPredicateOpcodes(Table);
820
821 if (!CustomCXXAction.empty()) {
822 /// Handle combiners relying on custom C++ code instead of actions.
823 assert(Table.isCombiner() && "CustomCXXAction is only for combiners!");
824 // We cannot have actions other than debug comments.
825 assert(none_of(Actions, [](auto &A) {
826 return A->getKind() != MatchAction::AK_DebugComment;
827 }));
828 for (const auto &MA : Actions)
829 MA->emitActionOpcodes(Table);
830 Table << MatchTable::Opcode(Opcode: "GIR_DoneWithCustomAction", IndentAdjust: -1)
831 << MatchTable::Comment(Comment: "Fn")
832 << MatchTable::NamedValue(NumBytes: 2, NamedValue: CustomCXXAction)
833 << MatchTable::LineBreak;
834 } else {
835 // Emit all actions except the last one, then emit coverage and emit the
836 // final action.
837 //
838 // This is because some actions, such as GIR_EraseRootFromParent_Done, also
839 // double as a GIR_Done and terminate execution of the rule.
840 if (!Actions.empty()) {
841 for (const auto &MA : drop_end(RangeOrContainer&: Actions))
842 MA->emitActionOpcodes(Table);
843 }
844
845 // Emit coverage right before the Done opcode>
846 auto EmitCoverage = [&] {
847 assert((Table.isWithCoverage() ? !Table.isCombiner() : true) &&
848 "Combiner tables don't support coverage!");
849 if (Table.isWithCoverage())
850 Table << MatchTable::Opcode(Opcode: "GIR_Coverage")
851 << MatchTable::IntValue(NumBytes: 4, IntValue: RuleID) << MatchTable::LineBreak;
852 else if (!Table.isCombiner())
853 Table << MatchTable::Comment(
854 Comment: ("GIR_Coverage, " + Twine(RuleID) + ",").str())
855 << MatchTable::LineBreak;
856 };
857
858 if (Actions.empty() ||
859 !Actions.back()->emitActionOpcodesAndDone(Table, OnDone: EmitCoverage)) {
860 EmitCoverage();
861 Table << MatchTable::Opcode(Opcode: "GIR_Done", IndentAdjust: -1) << MatchTable::LineBreak;
862 }
863 }
864
865 Table << MatchTable::Label(LabelID);
866 ++NumPatternEmitted;
867}
868
869bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const {
870 // Rules involving more match roots have higher priority.
871 if (Roots.size() > B.Roots.size())
872 return true;
873 if (Roots.size() < B.Roots.size())
874 return false;
875
876 for (auto Matcher : zip(t: Roots, u: B.Roots)) {
877 if (std::get<0>(t&: Matcher)->isHigherPriorityThan(B&: *std::get<1>(t&: Matcher)))
878 return true;
879 if (std::get<1>(t&: Matcher)->isHigherPriorityThan(B&: *std::get<0>(t&: Matcher)))
880 return false;
881 }
882
883 return false;
884}
885
886unsigned RuleMatcher::countRendererFns() const {
887 return std::accumulate(first: Roots.begin(), last: Roots.end(), init: 0,
888 binary_op: [](unsigned A, InstructionMatcher *Matcher) {
889 return A + Matcher->countRendererFns();
890 });
891}
892
893void RuleMatcher::roots_pop_front() { Roots.erase(CI: Roots.begin()); }
894
895//===- PredicateMatcher ---------------------------------------------------===//
896
897PredicateMatcher::~PredicateMatcher() = default;
898
899//===- OperandPredicateMatcher --------------------------------------------===//
900
901OperandPredicateMatcher::~OperandPredicateMatcher() = default;
902
903bool OperandPredicateMatcher::isHigherPriorityThan(
904 const OperandPredicateMatcher &B) const {
905 // Generally speaking, an instruction is more important than an Int or a
906 // LiteralInt because it can cover more nodes but there's an exception to
907 // this. G_CONSTANT's are less important than either of those two because they
908 // are more permissive.
909
910 const auto *AOM = dyn_cast<InstructionOperandMatcher>(Val: this);
911 const auto *BOM = dyn_cast<InstructionOperandMatcher>(Val: &B);
912 bool AIsConstantInsn = AOM && AOM->getInsnMatcher().isConstantInstruction();
913 bool BIsConstantInsn = BOM && BOM->getInsnMatcher().isConstantInstruction();
914
915 // The relative priorities between a G_CONSTANT and any other instruction
916 // don't actually matter but this code is needed to ensure a strict weak
917 // ordering. This is particularly important on Windows where the rules will
918 // be incorrectly sorted without it.
919 if (AOM && BOM)
920 return !AIsConstantInsn && BIsConstantInsn;
921
922 if (AIsConstantInsn && (B.Kind == OPM_Int || B.Kind == OPM_LiteralInt))
923 return false;
924 if (BIsConstantInsn && (Kind == OPM_Int || Kind == OPM_LiteralInt))
925 return true;
926
927 return Kind < B.Kind;
928}
929
930//===- SameOperandMatcher -------------------------------------------------===//
931
932void SameOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
933 const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
934 Table << MatchTable::Opcode(Opcode: IgnoreCopies
935 ? "GIM_CheckIsSameOperandIgnoreCopies"
936 : "GIM_CheckIsSameOperand")
937 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
938 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OpIdx)
939 << MatchTable::Comment(Comment: "OtherMI")
940 << MatchTable::ULEB128Value(IntValue: OtherInsnID)
941 << MatchTable::Comment(Comment: "OtherOpIdx")
942 << MatchTable::ULEB128Value(IntValue: OtherOpIdx) << MatchTable::LineBreak;
943}
944
945//===- LLTOperandMatcher --------------------------------------------------===//
946
947std::map<LLTCodeGen, unsigned> LLTOperandMatcher::TypeIDValues;
948
949RecordAndValue LLTOperandMatcher::getValue() const {
950 const auto VI = TypeIDValues.find(x: Ty);
951 if (VI == TypeIDValues.end())
952 return MatchTable::NamedValue(NumBytes: 1, NamedValue: getTy().getCxxEnumValue());
953 return {MatchTable::NamedValue(NumBytes: 1, NamedValue: getTy().getCxxEnumValue()), VI->second};
954}
955
956bool LLTOperandMatcher::hasValue() const {
957 if (TypeIDValues.size() != KnownTypes.size())
958 initTypeIDValuesMap();
959 return TypeIDValues.count(x: Ty);
960}
961
962void LLTOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
963 if (InsnVarID == 0) {
964 Table << MatchTable::Opcode(Opcode: "GIM_RootCheckType");
965 } else {
966 Table << MatchTable::Opcode(Opcode: "GIM_CheckType") << MatchTable::Comment(Comment: "MI")
967 << MatchTable::ULEB128Value(IntValue: InsnVarID);
968 }
969 Table << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
970 << MatchTable::Comment(Comment: "Type") << getValue().Record
971 << MatchTable::LineBreak;
972}
973
974//===- PointerToAnyOperandMatcher -----------------------------------------===//
975
976void PointerToAnyOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
977 Table << MatchTable::Opcode(Opcode: "GIM_CheckPointerToAny")
978 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
979 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
980 << MatchTable::Comment(Comment: "SizeInBits")
981 << MatchTable::ULEB128Value(IntValue: SizeInBits) << MatchTable::LineBreak;
982}
983
984//===- RecordNamedOperandMatcher ------------------------------------------===//
985
986void RecordNamedOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
987 Table << MatchTable::Opcode(Opcode: "GIM_RecordNamedOperand")
988 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
989 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
990 << MatchTable::Comment(Comment: "StoreIdx") << MatchTable::ULEB128Value(IntValue: StoreIdx)
991 << MatchTable::Comment(Comment: "Name : " + Name) << MatchTable::LineBreak;
992}
993
994//===- RecordRegisterType ------------------------------------------===//
995
996void RecordRegisterType::emitPredicateOpcodes(MatchTable &Table) const {
997 assert(Idx < 0 && "Temp types always have negative indexes!");
998 Table << MatchTable::Opcode(Opcode: "GIM_RecordRegType") << MatchTable::Comment(Comment: "MI")
999 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
1000 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::Comment(Comment: "TempTypeIdx")
1001 << MatchTable::IntValue(NumBytes: 1, IntValue: Idx) << MatchTable::LineBreak;
1002}
1003
1004//===- ComplexPatternOperandMatcher ---------------------------------------===//
1005
1006void ComplexPatternOperandMatcher::emitPredicateOpcodes(
1007 MatchTable &Table) const {
1008 unsigned ID = getAllocatedTemporariesBaseID();
1009 Table << MatchTable::Opcode(Opcode: "GIM_CheckComplexPattern")
1010 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1011 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1012 << MatchTable::Comment(Comment: "Renderer") << MatchTable::IntValue(NumBytes: 2, IntValue: ID)
1013 << MatchTable::NamedValue(NumBytes: 2, NamedValue: ("GICP_" + TheDef.getName()).str())
1014 << MatchTable::LineBreak;
1015}
1016
1017unsigned ComplexPatternOperandMatcher::getAllocatedTemporariesBaseID() const {
1018 return Operand.getAllocatedTemporariesBaseID();
1019}
1020
1021//===- RegisterBankOperandMatcher -----------------------------------------===//
1022
1023bool RegisterBankOperandMatcher::isIdentical(const PredicateMatcher &B) const {
1024 return OperandPredicateMatcher::isIdentical(B) &&
1025 RC.getDef() == cast<RegisterBankOperandMatcher>(Val: &B)->RC.getDef();
1026}
1027
1028void RegisterBankOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1029 if (InsnVarID == 0) {
1030 Table << MatchTable::Opcode(Opcode: "GIM_RootCheckRegBankForClass");
1031 } else {
1032 Table << MatchTable::Opcode(Opcode: "GIM_CheckRegBankForClass")
1033 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID);
1034 }
1035
1036 Table << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1037 << MatchTable::Comment(Comment: "RC")
1038 << MatchTable::NamedValue(NumBytes: 2, NamedValue: RC.getQualifiedIdName())
1039 << MatchTable::LineBreak;
1040}
1041
1042//===- MBBOperandMatcher --------------------------------------------------===//
1043
1044void MBBOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1045 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsMBB") << MatchTable::Comment(Comment: "MI")
1046 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
1047 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
1048}
1049
1050//===- ImmOperandMatcher --------------------------------------------------===//
1051
1052void ImmOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1053 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsImm") << MatchTable::Comment(Comment: "MI")
1054 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Op")
1055 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
1056}
1057
1058//===- ConstantIntOperandMatcher ------------------------------------------===//
1059
1060void ConstantIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1061 const bool IsInt8 = isInt<8>(x: Value);
1062 Table << MatchTable::Opcode(Opcode: IsInt8 ? "GIM_CheckConstantInt8"
1063 : "GIM_CheckConstantInt")
1064 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1065 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1066 << MatchTable::IntValue(NumBytes: IsInt8 ? 1 : 8, IntValue: Value) << MatchTable::LineBreak;
1067}
1068
1069//===- LiteralIntOperandMatcher -------------------------------------------===//
1070
1071void LiteralIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1072 Table << MatchTable::Opcode(Opcode: "GIM_CheckLiteralInt")
1073 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1074 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1075 << MatchTable::IntValue(NumBytes: 8, IntValue: Value) << MatchTable::LineBreak;
1076}
1077
1078//===- CmpPredicateOperandMatcher -----------------------------------------===//
1079
1080void CmpPredicateOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1081 Table << MatchTable::Opcode(Opcode: "GIM_CheckCmpPredicate")
1082 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1083 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1084 << MatchTable::Comment(Comment: "Predicate")
1085 << MatchTable::NamedValue(NumBytes: 2, Namespace: "CmpInst", NamedValue: PredName)
1086 << MatchTable::LineBreak;
1087}
1088
1089//===- IntrinsicIDOperandMatcher ------------------------------------------===//
1090
1091void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1092 Table << MatchTable::Opcode(Opcode: "GIM_CheckIntrinsicID")
1093 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1094 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
1095 << MatchTable::NamedValue(NumBytes: 2, NamedValue: "Intrinsic::" + II->EnumName.str())
1096 << MatchTable::LineBreak;
1097}
1098
1099//===- OperandImmPredicateMatcher -----------------------------------------===//
1100
1101void OperandImmPredicateMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1102 Table << MatchTable::Opcode(Opcode: "GIM_CheckImmOperandPredicate")
1103 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1104 << MatchTable::Comment(Comment: "MO") << MatchTable::ULEB128Value(IntValue: OpIdx)
1105 << MatchTable::Comment(Comment: "Predicate")
1106 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1107 << MatchTable::LineBreak;
1108}
1109
1110//===- OperandLeafPredicateMatcher ----------------------------------------===//
1111
1112void OperandLeafPredicateMatcher::emitPredicateOpcodes(
1113 MatchTable &Table) const {
1114 Table << MatchTable::Opcode(Opcode: "GIM_CheckLeafOperandPredicate")
1115 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1116 << MatchTable::Comment(Comment: "MO") << MatchTable::ULEB128Value(IntValue: OpIdx)
1117 << MatchTable::Comment(Comment: "Predicate")
1118 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1119 << MatchTable::LineBreak;
1120}
1121
1122//===- OperandMatcher -----------------------------------------------------===//
1123
1124std::string OperandMatcher::getOperandExpr(unsigned InsnVarID) const {
1125 return "State.MIs[" + llvm::to_string(Value: InsnVarID) + "]->getOperand(" +
1126 llvm::to_string(Value: OpIdx) + ")";
1127}
1128
1129unsigned OperandMatcher::getInsnVarID() const { return Insn.getInsnVarID(); }
1130
1131TempTypeIdx OperandMatcher::getTempTypeIdx(RuleMatcher &Rule) {
1132 assert(!IsVariadic && "Cannot use this on variadic operands!");
1133 if (TTIdx >= 0) {
1134 // Temp type index not assigned yet, so assign one and add the necessary
1135 // predicate.
1136 TTIdx = Rule.getNextTempTypeIdx();
1137 assert(TTIdx < 0);
1138 addPredicate<RecordRegisterType>(args&: TTIdx);
1139 return TTIdx;
1140 }
1141 return TTIdx;
1142}
1143
1144bool OperandMatcher::recordsOperand() const {
1145 return matchersRecordOperand(R: Predicates);
1146}
1147
1148void OperandMatcher::emitPredicateOpcodes(MatchTable &Table) {
1149 if (!Optimized) {
1150 std::string Comment;
1151 raw_string_ostream CommentOS(Comment);
1152 CommentOS << "MIs[" << getInsnVarID() << "] ";
1153 if (SymbolicName.empty())
1154 CommentOS << "Operand " << OpIdx;
1155 else
1156 CommentOS << SymbolicName;
1157 Table << MatchTable::Comment(Comment) << MatchTable::LineBreak;
1158 }
1159
1160 emitPredicateListOpcodes(Table);
1161}
1162
1163bool OperandMatcher::isHigherPriorityThan(OperandMatcher &B) {
1164 // Operand matchers involving more predicates have higher priority.
1165 if (predicates_size() > B.predicates_size())
1166 return true;
1167 if (predicates_size() < B.predicates_size())
1168 return false;
1169
1170 // This assumes that predicates are added in a consistent order.
1171 for (auto &&Predicate : zip(t: predicates(), u: B.predicates())) {
1172 if (std::get<0>(t&: Predicate)->isHigherPriorityThan(B: *std::get<1>(t&: Predicate)))
1173 return true;
1174 if (std::get<1>(t&: Predicate)->isHigherPriorityThan(B: *std::get<0>(t&: Predicate)))
1175 return false;
1176 }
1177
1178 return false;
1179}
1180
1181unsigned OperandMatcher::countRendererFns() {
1182 return std::accumulate(
1183 first: predicates().begin(), last: predicates().end(), init: 0,
1184 binary_op: [](unsigned A,
1185 const std::unique_ptr<OperandPredicateMatcher> &Predicate) {
1186 return A + Predicate->countRendererFns();
1187 });
1188}
1189
1190Error OperandMatcher::addTypeCheckPredicate(const TypeSetByHwMode &VTy,
1191 bool OperandIsAPointer) {
1192 if (!VTy.isMachineValueType())
1193 return failUnsupported(Reason: "unsupported typeset");
1194
1195 if ((VTy.getMachineValueType() == MVT::iPTR ||
1196 VTy.getMachineValueType() == MVT::cPTR) &&
1197 OperandIsAPointer) {
1198 addPredicate<PointerToAnyOperandMatcher>(args: 0);
1199 return Error::success();
1200 }
1201
1202 // Metadata operands have no LLT representation and no runtime type check is
1203 // needed — they are guaranteed to be MO_Metadata by the IRTranslator. This
1204 // mirrors how srcvalue is handled in importChildMatcher.
1205 if (VTy.getMachineValueType() == MVT::Metadata)
1206 return Error::success();
1207
1208 auto OpTyOrNone = MVTToLLT(VT: VTy.getMachineValueType().SimpleTy);
1209 if (!OpTyOrNone)
1210 return failUnsupported(Reason: "unsupported type");
1211
1212 if (OperandIsAPointer)
1213 addPredicate<PointerToAnyOperandMatcher>(args: OpTyOrNone->get().getSizeInBits());
1214 else if (VTy.isPointer())
1215 addPredicate<LLTOperandMatcher>(
1216 args: LLT::pointer(AddressSpace: VTy.getPtrAddrSpace(), SizeInBits: OpTyOrNone->get().getSizeInBits()));
1217 else
1218 addPredicate<LLTOperandMatcher>(args&: *OpTyOrNone);
1219 return Error::success();
1220}
1221
1222//===- InstructionOpcodeMatcher -------------------------------------------===//
1223
1224DenseMap<const CodeGenInstruction *, unsigned>
1225 InstructionOpcodeMatcher::OpcodeValues;
1226
1227RecordAndValue
1228InstructionOpcodeMatcher::getInstValue(const CodeGenInstruction *I) const {
1229 const auto VI = OpcodeValues.find(Val: I);
1230 if (VI != OpcodeValues.end())
1231 return {MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName()), VI->second};
1232 return MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName());
1233}
1234
1235void InstructionOpcodeMatcher::initOpcodeValuesMap(
1236 const CodeGenTarget &Target) {
1237 OpcodeValues.clear();
1238
1239 for (const CodeGenInstruction *I : Target.getInstructions())
1240 OpcodeValues[I] = Target.getInstrIntValue(R: I->TheDef);
1241}
1242
1243RecordAndValue InstructionOpcodeMatcher::getValue() const {
1244 assert(Insts.size() == 1);
1245
1246 const CodeGenInstruction *I = Insts[0];
1247 const auto VI = OpcodeValues.find(Val: I);
1248 if (VI != OpcodeValues.end())
1249 return {MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName()), VI->second};
1250 return MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName());
1251}
1252
1253void InstructionOpcodeMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1254 StringRef CheckType =
1255 Insts.size() == 1 ? "GIM_CheckOpcode" : "GIM_CheckOpcodeIsEither";
1256 Table << MatchTable::Opcode(Opcode: CheckType) << MatchTable::Comment(Comment: "MI")
1257 << MatchTable::ULEB128Value(IntValue: InsnVarID);
1258
1259 for (const CodeGenInstruction *I : Insts)
1260 Table << getInstValue(I).Record;
1261 Table << MatchTable::LineBreak;
1262}
1263
1264bool InstructionOpcodeMatcher::isHigherPriorityThan(
1265 const InstructionPredicateMatcher &B) const {
1266 if (InstructionPredicateMatcher::isHigherPriorityThan(B))
1267 return true;
1268 if (B.InstructionPredicateMatcher::isHigherPriorityThan(B: *this))
1269 return false;
1270
1271 // Prioritize opcodes for cosmetic reasons in the generated source. Although
1272 // this is cosmetic at the moment, we may want to drive a similar ordering
1273 // using instruction frequency information to improve compile time.
1274 if (const InstructionOpcodeMatcher *BO =
1275 dyn_cast<InstructionOpcodeMatcher>(Val: &B))
1276 return Insts[0]->getName() < BO->Insts[0]->getName();
1277
1278 return false;
1279}
1280
1281bool InstructionOpcodeMatcher::isConstantInstruction() const {
1282 return Insts.size() == 1 && Insts[0]->getName() == "G_CONSTANT";
1283}
1284
1285StringRef InstructionOpcodeMatcher::getOpcode() const {
1286 return Insts[0]->getName();
1287}
1288
1289bool InstructionOpcodeMatcher::isVariadicNumOperands() const {
1290 // If one is variadic, they all should be.
1291 return Insts[0]->Operands.isVariadic;
1292}
1293
1294StringRef InstructionOpcodeMatcher::getOperandType(unsigned OpIdx) const {
1295 // Types expected to be uniform for all alternatives.
1296 return Insts[0]->Operands[OpIdx].OperandType;
1297}
1298
1299//===- InstructionNumOperandsMatcher --------------------------------------===//
1300
1301void InstructionNumOperandsMatcher::emitPredicateOpcodes(
1302 MatchTable &Table) const {
1303 StringRef Opc;
1304 switch (CK) {
1305 case CheckKind::Eq:
1306 Opc = "GIM_CheckNumOperands";
1307 break;
1308 case CheckKind::GE:
1309 Opc = "GIM_CheckNumOperandsGE";
1310 break;
1311 case CheckKind::LE:
1312 Opc = "GIM_CheckNumOperandsLE";
1313 break;
1314 }
1315 Table << MatchTable::Opcode(Opcode: Opc) << MatchTable::Comment(Comment: "MI")
1316 << MatchTable::ULEB128Value(IntValue: InsnVarID)
1317 << MatchTable::Comment(Comment: "Expected")
1318 << MatchTable::ULEB128Value(IntValue: NumOperands) << MatchTable::LineBreak;
1319}
1320
1321//===- InstructionImmPredicateMatcher -------------------------------------===//
1322
1323bool InstructionImmPredicateMatcher::isIdentical(
1324 const PredicateMatcher &B) const {
1325 return InstructionPredicateMatcher::isIdentical(B) &&
1326 Predicate.getOrigPatFragRecord() ==
1327 cast<InstructionImmPredicateMatcher>(Val: &B)
1328 ->Predicate.getOrigPatFragRecord();
1329}
1330
1331void InstructionImmPredicateMatcher::emitPredicateOpcodes(
1332 MatchTable &Table) const {
1333 Table << MatchTable::Opcode(Opcode: getMatchOpcodeForImmPredicate(Predicate))
1334 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1335 << MatchTable::Comment(Comment: "Predicate")
1336 << MatchTable::NamedValue(NumBytes: 2, NamedValue: getEnumNameForPredicate(Predicate))
1337 << MatchTable::LineBreak;
1338}
1339
1340//===- AtomicOrderingMMOPredicateMatcher ----------------------------------===//
1341
1342bool AtomicOrderingMMOPredicateMatcher::isIdentical(
1343 const PredicateMatcher &B) const {
1344 if (!InstructionPredicateMatcher::isIdentical(B))
1345 return false;
1346 const auto &R = *cast<AtomicOrderingMMOPredicateMatcher>(Val: &B);
1347 return Order == R.Order && Comparator == R.Comparator;
1348}
1349
1350void AtomicOrderingMMOPredicateMatcher::emitPredicateOpcodes(
1351 MatchTable &Table) const {
1352 StringRef Opcode = "GIM_CheckAtomicOrdering";
1353
1354 if (Comparator == AO_OrStronger)
1355 Opcode = "GIM_CheckAtomicOrderingOrStrongerThan";
1356 if (Comparator == AO_WeakerThan)
1357 Opcode = "GIM_CheckAtomicOrderingWeakerThan";
1358
1359 Table << MatchTable::Opcode(Opcode) << MatchTable::Comment(Comment: "MI")
1360 << MatchTable::ULEB128Value(IntValue: InsnVarID) << MatchTable::Comment(Comment: "Order")
1361 << MatchTable::NamedValue(NumBytes: 1,
1362 NamedValue: ("(uint8_t)AtomicOrdering::" + Order).str())
1363 << MatchTable::LineBreak;
1364}
1365
1366//===- MemorySizePredicateMatcher -----------------------------------------===//
1367
1368void MemorySizePredicateMatcher::emitPredicateOpcodes(MatchTable &Table) const {
1369 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemorySizeEqualTo")
1370 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1371 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1372 << MatchTable::Comment(Comment: "Size") << MatchTable::IntValue(NumBytes: 4, IntValue: Size)
1373 << MatchTable::LineBreak;
1374}
1375
1376//===- MemoryAddressSpacePredicateMatcher ---------------------------------===//
1377
1378bool MemoryAddressSpacePredicateMatcher::isIdentical(
1379 const PredicateMatcher &B) const {
1380 if (!InstructionPredicateMatcher::isIdentical(B))
1381 return false;
1382 auto *Other = cast<MemoryAddressSpacePredicateMatcher>(Val: &B);
1383 return MMOIdx == Other->MMOIdx && AddrSpaces == Other->AddrSpaces;
1384}
1385
1386void MemoryAddressSpacePredicateMatcher::emitPredicateOpcodes(
1387 MatchTable &Table) const {
1388 assert(AddrSpaces.size() < 256);
1389 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemoryAddressSpace")
1390 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1391 << MatchTable::Comment(Comment: "MMO")
1392 << MatchTable::ULEB128Value(IntValue: MMOIdx)
1393 // Encode number of address spaces to expect.
1394 << MatchTable::Comment(Comment: "NumAddrSpace")
1395 << MatchTable::IntValue(NumBytes: 1, IntValue: AddrSpaces.size());
1396 for (unsigned AS : AddrSpaces)
1397 Table << MatchTable::Comment(Comment: "AddrSpace") << MatchTable::ULEB128Value(IntValue: AS);
1398
1399 Table << MatchTable::LineBreak;
1400}
1401
1402//===- MemoryAlignmentPredicateMatcher ------------------------------------===//
1403
1404bool MemoryAlignmentPredicateMatcher::isIdentical(
1405 const PredicateMatcher &B) const {
1406 if (!InstructionPredicateMatcher::isIdentical(B))
1407 return false;
1408 auto *Other = cast<MemoryAlignmentPredicateMatcher>(Val: &B);
1409 return MMOIdx == Other->MMOIdx && MinAlign == Other->MinAlign;
1410}
1411
1412void MemoryAlignmentPredicateMatcher::emitPredicateOpcodes(
1413 MatchTable &Table) const {
1414 // TODO: we could support more, just need to emit the right opcode or switch
1415 // to log alignment.
1416 assert(MinAlign < 256);
1417 Table << MatchTable::Opcode(Opcode: "GIM_CheckMemoryAlignment")
1418 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1419 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1420 << MatchTable::Comment(Comment: "MinAlign") << MatchTable::IntValue(NumBytes: 1, IntValue: MinAlign)
1421 << MatchTable::LineBreak;
1422}
1423
1424//===- MemoryVsLLTSizePredicateMatcher ------------------------------------===//
1425
1426bool MemoryVsLLTSizePredicateMatcher::isIdentical(
1427 const PredicateMatcher &B) const {
1428 return InstructionPredicateMatcher::isIdentical(B) &&
1429 MMOIdx == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->MMOIdx &&
1430 Relation == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->Relation &&
1431 OpIdx == cast<MemoryVsLLTSizePredicateMatcher>(Val: &B)->OpIdx;
1432}
1433
1434void MemoryVsLLTSizePredicateMatcher::emitPredicateOpcodes(
1435 MatchTable &Table) const {
1436 Table << MatchTable::Opcode(
1437 Opcode: Relation == EqualTo ? "GIM_CheckMemorySizeEqualToLLT"
1438 : Relation == GreaterThan ? "GIM_CheckMemorySizeGreaterThanLLT"
1439 : "GIM_CheckMemorySizeLessThanLLT")
1440 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1441 << MatchTable::Comment(Comment: "MMO") << MatchTable::ULEB128Value(IntValue: MMOIdx)
1442 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OpIdx)
1443 << MatchTable::LineBreak;
1444}
1445
1446//===- VectorSplatImmPredicateMatcher -------------------------------------===//
1447
1448void VectorSplatImmPredicateMatcher::emitPredicateOpcodes(
1449 MatchTable &Table) const {
1450 if (Kind == AllOnes)
1451 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsBuildVectorAllOnes");
1452 else
1453 Table << MatchTable::Opcode(Opcode: "GIM_CheckIsBuildVectorAllZeros");
1454
1455 Table << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID);
1456 Table << MatchTable::LineBreak;
1457}
1458
1459//===- GenericInstructionPredicateMatcher ---------------------------------===//
1460
1461GenericInstructionPredicateMatcher::GenericInstructionPredicateMatcher(
1462 unsigned InsnVarID, TreePredicateFn Predicate)
1463 : GenericInstructionPredicateMatcher(InsnVarID,
1464 getEnumNameForPredicate(Predicate)) {}
1465
1466bool GenericInstructionPredicateMatcher::isIdentical(
1467 const PredicateMatcher &B) const {
1468 return InstructionPredicateMatcher::isIdentical(B) &&
1469 EnumVal ==
1470 static_cast<const GenericInstructionPredicateMatcher &>(B).EnumVal;
1471}
1472void GenericInstructionPredicateMatcher::emitPredicateOpcodes(
1473 MatchTable &Table) const {
1474 Table << MatchTable::Opcode(Opcode: "GIM_CheckCxxInsnPredicate")
1475 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1476 << MatchTable::Comment(Comment: "FnId") << MatchTable::NamedValue(NumBytes: 2, NamedValue: EnumVal)
1477 << MatchTable::LineBreak;
1478}
1479
1480//===- MIFlagsInstructionPredicateMatcher ---------------------------------===//
1481
1482bool MIFlagsInstructionPredicateMatcher::isIdentical(
1483 const PredicateMatcher &B) const {
1484 if (!InstructionPredicateMatcher::isIdentical(B))
1485 return false;
1486 const auto &Other =
1487 static_cast<const MIFlagsInstructionPredicateMatcher &>(B);
1488 return Flags == Other.Flags && CheckNot == Other.CheckNot;
1489}
1490
1491void MIFlagsInstructionPredicateMatcher::emitPredicateOpcodes(
1492 MatchTable &Table) const {
1493 Table << MatchTable::Opcode(Opcode: CheckNot ? "GIM_MIFlagsNot" : "GIM_MIFlags")
1494 << MatchTable::Comment(Comment: "MI") << MatchTable::ULEB128Value(IntValue: InsnVarID)
1495 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: Flags, Separator: " | "))
1496 << MatchTable::LineBreak;
1497}
1498
1499//===- InstructionMatcher -------------------------------------------------===//
1500
1501OperandMatcher &
1502InstructionMatcher::addOperand(unsigned OpIdx, const std::string &SymbolicName,
1503 unsigned AllocatedTemporariesBaseID,
1504 bool IsVariadic) {
1505 assert((Operands.empty() || !Operands.back()->isVariadic()) &&
1506 "Cannot add more operands after a variadic operand");
1507 Operands.emplace_back(args: new OperandMatcher(
1508 *this, OpIdx, SymbolicName, AllocatedTemporariesBaseID, IsVariadic));
1509 if (!SymbolicName.empty())
1510 Rule.defineOperand(SymbolicName, OM&: *Operands.back());
1511 return *Operands.back();
1512}
1513
1514OperandMatcher &InstructionMatcher::getOperand(unsigned OpIdx) {
1515 auto I = llvm::find_if(Range&: Operands,
1516 P: [&OpIdx](const std::unique_ptr<OperandMatcher> &X) {
1517 return X->getOpIdx() == OpIdx;
1518 });
1519 if (I != Operands.end())
1520 return **I;
1521 llvm_unreachable("Failed to lookup operand");
1522}
1523
1524OperandMatcher &InstructionMatcher::addPhysRegInput(const Record *Reg,
1525 unsigned OpIdx,
1526 unsigned TempOpIdx) {
1527 assert(SymbolicName.empty());
1528 OperandMatcher *OM = new OperandMatcher(*this, OpIdx, "", TempOpIdx);
1529 Operands.emplace_back(args&: OM);
1530 Rule.definePhysRegOperand(Reg, OM&: *OM);
1531 return *OM;
1532}
1533
1534bool InstructionMatcher::recordsOperand() const {
1535 return matchersRecordOperand(R: Predicates) || matchersRecordOperand(R: operands());
1536}
1537
1538void InstructionMatcher::emitPredicateOpcodes(MatchTable &Table) {
1539 if (canAddNumOperandsCheck()) {
1540 InstructionNumOperandsMatcher(InsnVarID, getNumOperandMatchers())
1541 .emitPredicateOpcodes(Table);
1542 }
1543
1544 // First emit all instruction level predicates need to be verified before we
1545 // can verify operands.
1546 emitFilteredPredicateListOpcodes(
1547 ShouldEmitPredicate: [](const PredicateMatcher &P) { return !P.dependsOnRecordedOperands(); },
1548 Table);
1549
1550 // Emit all operand constraints.
1551 for (const auto &Operand : Operands)
1552 Operand->emitPredicateOpcodes(Table);
1553
1554 // All of the tablegen defined predicates should now be matched. Now emit
1555 // any custom predicates that rely on all generated checks.
1556 emitFilteredPredicateListOpcodes(
1557 ShouldEmitPredicate: [](const PredicateMatcher &P) { return P.dependsOnRecordedOperands(); },
1558 Table);
1559}
1560
1561bool InstructionMatcher::isHigherPriorityThan(InstructionMatcher &B) {
1562 // Instruction matchers involving more operands have higher priority.
1563 if (Operands.size() > B.Operands.size())
1564 return true;
1565 if (Operands.size() < B.Operands.size())
1566 return false;
1567
1568 for (auto &&P : zip(t: predicates(), u: B.predicates())) {
1569 auto L = static_cast<InstructionPredicateMatcher *>(std::get<0>(t&: P).get());
1570 auto R = static_cast<InstructionPredicateMatcher *>(std::get<1>(t&: P).get());
1571 if (L->isHigherPriorityThan(B: *R))
1572 return true;
1573 if (R->isHigherPriorityThan(B: *L))
1574 return false;
1575 }
1576
1577 for (auto Operand : zip(t&: Operands, u&: B.Operands)) {
1578 if (std::get<0>(t&: Operand)->isHigherPriorityThan(B&: *std::get<1>(t&: Operand)))
1579 return true;
1580 if (std::get<1>(t&: Operand)->isHigherPriorityThan(B&: *std::get<0>(t&: Operand)))
1581 return false;
1582 }
1583 // Instruction matchers involving more predicates have higher priority.
1584 if (predicates_size() > B.predicates_size())
1585 return true;
1586 if (predicates_size() < B.predicates_size())
1587 return false;
1588
1589 return false;
1590}
1591
1592unsigned InstructionMatcher::countRendererFns() {
1593 return std::accumulate(
1594 first: predicates().begin(), last: predicates().end(), init: 0,
1595 binary_op: [](unsigned A,
1596 const std::unique_ptr<PredicateMatcher> &Predicate) {
1597 return A + Predicate->countRendererFns();
1598 }) +
1599 std::accumulate(
1600 first: Operands.begin(), last: Operands.end(), init: 0,
1601 binary_op: [](unsigned A, const std::unique_ptr<OperandMatcher> &Operand) {
1602 return A + Operand->countRendererFns();
1603 });
1604}
1605
1606void InstructionMatcher::optimize() {
1607 SmallVector<std::unique_ptr<PredicateMatcher>, 8> Stash;
1608 const auto &OpcMatcher = getOpcodeMatcher();
1609
1610 Stash.push_back(Elt: predicates_pop_front());
1611 if (Stash.back().get() == &OpcMatcher) {
1612 // FIXME: Is this even needed still? Why the isVariadicNumOperands check?
1613 if (canAddNumOperandsCheck() && OpcMatcher.isVariadicNumOperands() &&
1614 getNumOperandMatchers() != 0) {
1615 Stash.emplace_back(Args: new InstructionNumOperandsMatcher(
1616 InsnVarID, getNumOperandMatchers()));
1617 }
1618 AllowNumOpsCheck = false;
1619
1620 for (auto &OM : Operands)
1621 for (auto &OP : OM->predicates())
1622 if (isa<IntrinsicIDOperandMatcher>(Val: OP)) {
1623 Stash.push_back(Elt: std::move(OP));
1624 OM->eraseNullPredicates();
1625 break;
1626 }
1627 }
1628
1629 if (InsnVarID > 0) {
1630 assert(!Operands.empty() && "Nested instruction is expected to def a vreg");
1631 for (auto &OP : Operands[0]->predicates())
1632 OP.reset();
1633 Operands[0]->eraseNullPredicates();
1634 }
1635 for (auto &OM : Operands) {
1636 for (auto &OP : OM->predicates())
1637 if (isa<LLTOperandMatcher>(Val: OP) || isa<LLTOperandShapeMatcher>(Val: OP))
1638 Stash.push_back(Elt: std::move(OP));
1639 OM->eraseNullPredicates();
1640 }
1641 while (!Stash.empty())
1642 prependPredicate(Predicate: Stash.pop_back_val());
1643}
1644
1645//===- InstructionOperandMatcher ------------------------------------------===//
1646
1647void InstructionOperandMatcher::emitCaptureOpcodes(MatchTable &Table) const {
1648 const unsigned NewInsnVarID = InsnMatcher.getInsnVarID();
1649 const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
1650 Table << MatchTable::Opcode(Opcode: IgnoreCopies ? "GIM_RecordInsnIgnoreCopies"
1651 : "GIM_RecordInsn")
1652 << MatchTable::Comment(Comment: "DefineMI")
1653 << MatchTable::ULEB128Value(IntValue: NewInsnVarID) << MatchTable::Comment(Comment: "MI")
1654 << MatchTable::ULEB128Value(IntValue: getInsnVarID())
1655 << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: getOpIdx())
1656 << MatchTable::Comment(Comment: "MIs[" + llvm::to_string(Value: NewInsnVarID) + "]")
1657 << MatchTable::LineBreak;
1658}
1659
1660bool InstructionOperandMatcher::isHigherPriorityThan(
1661 const OperandPredicateMatcher &B) const {
1662 if (OperandPredicateMatcher::isHigherPriorityThan(B))
1663 return true;
1664 if (B.OperandPredicateMatcher::isHigherPriorityThan(B: *this))
1665 return false;
1666
1667 if (const InstructionOperandMatcher *BP =
1668 dyn_cast<InstructionOperandMatcher>(Val: &B))
1669 if (InsnMatcher.isHigherPriorityThan(B&: BP->InsnMatcher))
1670 return true;
1671 return false;
1672}
1673
1674//===- OperandRenderer ----------------------------------------------------===//
1675
1676OperandRenderer::~OperandRenderer() = default;
1677
1678//===- CopyRenderer -------------------------------------------------------===//
1679
1680void CopyRenderer::emitRenderOpcodes(MatchTable &Table, unsigned NewInsnID,
1681 unsigned OldInsnID, unsigned OldOpIdx,
1682 StringRef Name, bool ForVariadic) {
1683 if (!ForVariadic && NewInsnID == 0 && OldInsnID == 0) {
1684 Table << MatchTable::Opcode(Opcode: "GIR_RootToRootCopy");
1685 } else {
1686 Table << MatchTable::Opcode(Opcode: ForVariadic ? "GIR_CopyRemaining" : "GIR_Copy")
1687 << MatchTable::Comment(Comment: "NewInsnID")
1688 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1689 << MatchTable::Comment(Comment: "OldInsnID")
1690 << MatchTable::ULEB128Value(IntValue: OldInsnID);
1691 }
1692
1693 Table << MatchTable::Comment(Comment: "OpIdx") << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1694 << MatchTable::Comment(Comment: Name) << MatchTable::LineBreak;
1695}
1696
1697void CopyRenderer::emitRenderOpcodes(MatchTable &Table) const {
1698 emitRenderOpcodes(Table, NewInsnID, OldInsnID, OldOpIdx, Name: SymbolicName,
1699 ForVariadic: OldOpIsVariadic);
1700}
1701
1702//===- CopyPhysRegRenderer ------------------------------------------------===//
1703
1704void CopyPhysRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1705 CopyRenderer::emitRenderOpcodes(Table, NewInsnID, OldInsnID, OldOpIdx,
1706 Name: PhysReg->getName());
1707}
1708
1709//===- CopyOrAddZeroRegRenderer -------------------------------------------===//
1710
1711void CopyOrAddZeroRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1712 Table << MatchTable::Opcode(Opcode: "GIR_CopyOrAddZeroReg")
1713 << MatchTable::Comment(Comment: "NewInsnID")
1714 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1715 << MatchTable::Comment(Comment: "OldInsnID")
1716 << MatchTable::ULEB128Value(IntValue: OldInsnID) << MatchTable::Comment(Comment: "OpIdx")
1717 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1718 << MatchTable::NamedValue(
1719 NumBytes: 2,
1720 Namespace: (ZeroRegisterDef->getValue(Name: "Namespace")
1721 ? ZeroRegisterDef->getValueAsString(FieldName: "Namespace")
1722 : ""),
1723 NamedValue: ZeroRegisterDef->getName())
1724 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1725}
1726
1727//===- CopyConstantAsImmRenderer ------------------------------------------===//
1728
1729void CopyConstantAsImmRenderer::emitRenderOpcodes(MatchTable &Table) const {
1730 Table << MatchTable::Opcode(Opcode: Signed ? "GIR_CopyConstantAsSImm"
1731 : "GIR_CopyConstantAsUImm")
1732 << MatchTable::Comment(Comment: "NewInsnID")
1733 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1734 << MatchTable::Comment(Comment: "OldInsnID")
1735 << MatchTable::ULEB128Value(IntValue: OldInsnID)
1736 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1737}
1738
1739//===- CopyFConstantAsFPImmRenderer ---------------------------------------===//
1740
1741void CopyFConstantAsFPImmRenderer::emitRenderOpcodes(MatchTable &Table) const {
1742 Table << MatchTable::Opcode(Opcode: "GIR_CopyFConstantAsFPImm")
1743 << MatchTable::Comment(Comment: "NewInsnID")
1744 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1745 << MatchTable::Comment(Comment: "OldInsnID")
1746 << MatchTable::ULEB128Value(IntValue: OldInsnID)
1747 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1748}
1749
1750//===- CopySubRegRenderer -------------------------------------------------===//
1751
1752void CopySubRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1753 Table << MatchTable::Opcode(Opcode: "GIR_CopySubReg")
1754 << MatchTable::Comment(Comment: "NewInsnID")
1755 << MatchTable::ULEB128Value(IntValue: NewInsnID)
1756 << MatchTable::Comment(Comment: "OldInsnID")
1757 << MatchTable::ULEB128Value(IntValue: OldInsnID) << MatchTable::Comment(Comment: "OpIdx")
1758 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1759 << MatchTable::Comment(Comment: "SubRegIdx")
1760 << MatchTable::IntValue(NumBytes: 2, IntValue: SubReg->EnumValue)
1761 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1762}
1763
1764//===- AddRegisterRenderer ------------------------------------------------===//
1765
1766void AddRegisterRenderer::emitRenderOpcodes(MatchTable &Table) const {
1767 Table << MatchTable::Opcode(Opcode: "GIR_AddRegister")
1768 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID);
1769 if (RegisterDef->getName() != "zero_reg") {
1770 Table << MatchTable::NamedValue(
1771 NumBytes: 2,
1772 Namespace: (RegisterDef->getValue(Name: "Namespace")
1773 ? RegisterDef->getValueAsString(FieldName: "Namespace")
1774 : ""),
1775 NamedValue: RegisterDef->getName());
1776 } else {
1777 Table << MatchTable::NamedValue(NumBytes: 2, Namespace: Target.getRegNamespace(), NamedValue: "NoRegister");
1778 }
1779 Table << MatchTable::Comment(Comment: "AddRegisterRegFlags");
1780
1781 // TODO: This is encoded as a 64-bit element, but only 16 or 32-bits are
1782 // really needed for a physical register reference. We can pack the
1783 // register and flags in a single field.
1784 if (IsDef) {
1785 Table << MatchTable::NamedValue(
1786 NumBytes: 2, NamedValue: IsDead ? "static_cast<uint16_t>(RegState::Define|RegState::Dead)"
1787 : "static_cast<uint16_t>(RegState::Define)");
1788 } else {
1789 assert(!IsDead && "A use cannot be dead");
1790 Table << MatchTable::IntValue(NumBytes: 2, IntValue: 0);
1791 }
1792 Table << MatchTable::LineBreak;
1793}
1794
1795//===- TempRegRenderer ----------------------------------------------------===//
1796
1797void TempRegRenderer::emitRenderOpcodes(MatchTable &Table) const {
1798 const bool NeedsFlags = (SubRegIdx || IsDef);
1799 if (SubRegIdx) {
1800 assert(!IsDef);
1801 Table << MatchTable::Opcode(Opcode: "GIR_AddTempSubRegister");
1802 } else {
1803 Table << MatchTable::Opcode(Opcode: NeedsFlags ? "GIR_AddTempRegister"
1804 : "GIR_AddSimpleTempRegister");
1805 }
1806
1807 Table << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1808 << MatchTable::Comment(Comment: "TempRegID")
1809 << MatchTable::ULEB128Value(IntValue: TempRegID);
1810
1811 if (!NeedsFlags) {
1812 Table << MatchTable::LineBreak;
1813 return;
1814 }
1815
1816 Table << MatchTable::Comment(Comment: "TempRegFlags");
1817 if (IsDef) {
1818 SmallString<32> RegFlags;
1819 RegFlags += "static_cast<uint16_t>(RegState::Define";
1820 if (IsDead)
1821 RegFlags += "|RegState::Dead";
1822 RegFlags += ")";
1823 Table << MatchTable::NamedValue(NumBytes: 2, NamedValue: RegFlags);
1824 } else {
1825 Table << MatchTable::IntValue(NumBytes: 2, IntValue: 0);
1826 }
1827
1828 if (SubRegIdx)
1829 Table << MatchTable::NamedValue(NumBytes: 2, NamedValue: SubRegIdx->getQualifiedName());
1830 Table << MatchTable::LineBreak;
1831}
1832
1833//===- ImmRenderer --------------------------------------------------------===//
1834
1835void ImmRenderer::emitAddImm(MatchTable &Table, unsigned InsnID, int64_t Imm,
1836 StringRef ImmName) {
1837 const bool IsInt8 = isInt<8>(x: Imm);
1838
1839 Table << MatchTable::Opcode(Opcode: IsInt8 ? "GIR_AddImm8" : "GIR_AddImm")
1840 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1841 << MatchTable::Comment(Comment: ImmName)
1842 << MatchTable::IntValue(NumBytes: IsInt8 ? 1 : 8, IntValue: Imm) << MatchTable::LineBreak;
1843}
1844
1845void ImmRenderer::emitRenderOpcodes(MatchTable &Table) const {
1846 if (CImmLLT) {
1847 assert(Table.isCombiner() &&
1848 "ConstantInt immediate are only for combiners!");
1849 Table << MatchTable::Opcode(Opcode: "GIR_AddCImm") << MatchTable::Comment(Comment: "InsnID")
1850 << MatchTable::ULEB128Value(IntValue: InsnID) << MatchTable::Comment(Comment: "Type");
1851 emitType(Table, Ty: *CImmLLT);
1852 Table << MatchTable::Comment(Comment: "Imm") << MatchTable::IntValue(NumBytes: 8, IntValue: Imm)
1853 << MatchTable::LineBreak;
1854 } else {
1855 emitAddImm(Table, InsnID, Imm);
1856 }
1857}
1858
1859//===- SubRegIndexRenderer ------------------------------------------------===//
1860
1861void SubRegIndexRenderer::emitRenderOpcodes(MatchTable &Table) const {
1862 ImmRenderer::emitAddImm(Table, InsnID, Imm: SubRegIdx->EnumValue, ImmName: "SubRegIndex");
1863}
1864
1865//===- RenderComplexPatternOperand ----------------------------------------===//
1866
1867void RenderComplexPatternOperand::emitRenderOpcodes(MatchTable &Table) const {
1868 Table << MatchTable::Opcode(
1869 Opcode: SubOperand ? (SubReg ? "GIR_ComplexSubOperandSubRegRenderer"
1870 : "GIR_ComplexSubOperandRenderer")
1871 : "GIR_ComplexRenderer")
1872 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1873 << MatchTable::Comment(Comment: "RendererID")
1874 << MatchTable::IntValue(NumBytes: 2, IntValue: RendererID);
1875 if (SubOperand)
1876 Table << MatchTable::Comment(Comment: "SubOperand")
1877 << MatchTable::ULEB128Value(IntValue: *SubOperand);
1878 if (SubReg)
1879 Table << MatchTable::Comment(Comment: "SubRegIdx")
1880 << MatchTable::IntValue(NumBytes: 2, IntValue: SubReg->EnumValue);
1881 Table << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1882}
1883
1884//===- IntrinsicIDRenderer ------------------------------------------------===//
1885
1886void IntrinsicIDRenderer::emitRenderOpcodes(MatchTable &Table) const {
1887 Table << MatchTable::Opcode(Opcode: "GIR_AddIntrinsicID") << MatchTable::Comment(Comment: "MI")
1888 << MatchTable::ULEB128Value(IntValue: InsnID)
1889 << MatchTable::NamedValue(NumBytes: 2, NamedValue: "Intrinsic::" + II->EnumName.str())
1890 << MatchTable::LineBreak;
1891}
1892
1893//===- CustomRenderer -----------------------------------------------------===//
1894
1895void CustomRenderer::emitRenderOpcodes(MatchTable &Table) const {
1896 Table << MatchTable::Opcode(Opcode: "GIR_CustomRenderer")
1897 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1898 << MatchTable::Comment(Comment: "OldInsnID")
1899 << MatchTable::ULEB128Value(IntValue: OldInsnID)
1900 << MatchTable::Comment(Comment: "Renderer")
1901 << MatchTable::NamedValue(
1902 NumBytes: 2, NamedValue: "GICR_" + Renderer.getValueAsString(FieldName: "RendererFn").str())
1903 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1904}
1905
1906//===- CustomOperandRenderer ----------------------------------------------===//
1907
1908void CustomOperandRenderer::emitRenderOpcodes(MatchTable &Table) const {
1909 Table << MatchTable::Opcode(Opcode: "GIR_CustomOperandRenderer")
1910 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1911 << MatchTable::Comment(Comment: "OldInsnID")
1912 << MatchTable::ULEB128Value(IntValue: OldInsnID) << MatchTable::Comment(Comment: "OpIdx")
1913 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
1914 << MatchTable::Comment(Comment: "OperandRenderer")
1915 << MatchTable::NamedValue(
1916 NumBytes: 2, NamedValue: "GICR_" + Renderer.getValueAsString(FieldName: "RendererFn").str())
1917 << MatchTable::Comment(Comment: SymbolicName) << MatchTable::LineBreak;
1918}
1919
1920//===- BuildMIAction ------------------------------------------------------===//
1921
1922bool BuildMIAction::canMutate(RuleMatcher &Rule,
1923 const InstructionMatcher *Insn) const {
1924 if (!Insn || Insn->hasVariadicMatcher())
1925 return false;
1926
1927 if (OperandRenderers.size() != Insn->getNumOperandMatchers())
1928 return false;
1929
1930 for (const auto &Renderer : enumerate(First: OperandRenderers)) {
1931 if (const auto *Copy = dyn_cast<CopyRenderer>(Val: &*Renderer.value())) {
1932 const OperandMatcher &OM =
1933 Rule.getOperandMatcher(Name: Copy->getSymbolicName());
1934 if (Insn != &OM.getInstructionMatcher() ||
1935 OM.getOpIdx() != Renderer.index())
1936 return false;
1937 } else {
1938 return false;
1939 }
1940 }
1941
1942 return true;
1943}
1944
1945void BuildMIAction::chooseInsnToMutate(RuleMatcher &Rule) {
1946 for (auto *MutateCandidate : Rule.mutatable_insns()) {
1947 if (canMutate(Rule, Insn: MutateCandidate)) {
1948 // Take the first one we're offered that we're able to mutate.
1949 Rule.reserveInsnMatcherForMutation(InsnMatcher: MutateCandidate);
1950 Matched = MutateCandidate;
1951 Rule.tryEraseInsnID(ID: MutateCandidate->getInsnVarID());
1952 return;
1953 }
1954 }
1955}
1956
1957void BuildMIAction::emitActionOpcodes(MatchTable &Table) const {
1958 const auto AddMIFlags = [&]() {
1959 for (const InstructionMatcher *IM : CopiedFlags) {
1960 Table << MatchTable::Opcode(Opcode: "GIR_CopyMIFlags")
1961 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1962 << MatchTable::Comment(Comment: "OldInsnID")
1963 << MatchTable::ULEB128Value(IntValue: IM->getInsnVarID())
1964 << MatchTable::LineBreak;
1965 }
1966
1967 if (!SetFlags.empty()) {
1968 Table << MatchTable::Opcode(Opcode: "GIR_SetMIFlags")
1969 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1970 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: SetFlags, Separator: " | "))
1971 << MatchTable::LineBreak;
1972 }
1973
1974 if (!UnsetFlags.empty()) {
1975 Table << MatchTable::Opcode(Opcode: "GIR_UnsetMIFlags")
1976 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1977 << MatchTable::NamedValue(NumBytes: 4, NamedValue: join(R: UnsetFlags, Separator: " | "))
1978 << MatchTable::LineBreak;
1979 }
1980 };
1981
1982 if (Matched) {
1983 unsigned RecycleInsnID = Matched->getInsnVarID();
1984 Table << MatchTable::Opcode(Opcode: "GIR_MutateOpcode")
1985 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
1986 << MatchTable::Comment(Comment: "RecycleInsnID")
1987 << MatchTable::ULEB128Value(IntValue: RecycleInsnID)
1988 << MatchTable::Comment(Comment: "Opcode")
1989 << MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName())
1990 << MatchTable::LineBreak;
1991
1992 if (!I->ImplicitDefs.empty() || !I->ImplicitUses.empty()) {
1993 for (auto *Def : I->ImplicitDefs) {
1994 auto Namespace = Def->getValue(Name: "Namespace")
1995 ? Def->getValueAsString(FieldName: "Namespace")
1996 : "";
1997 const bool IsDead = DeadImplicitDefs.contains(Ptr: Def);
1998 Table << MatchTable::Opcode(Opcode: "GIR_AddImplicitDef")
1999 << MatchTable::Comment(Comment: "InsnID")
2000 << MatchTable::ULEB128Value(IntValue: InsnID)
2001 << MatchTable::NamedValue(NumBytes: 2, Namespace, NamedValue: Def->getName())
2002 << (IsDead ? MatchTable::NamedValue(
2003 NumBytes: 2, NamedValue: "static_cast<unsigned>(RegState::Dead)")
2004 : MatchTable::IntValue(NumBytes: 2, IntValue: 0))
2005 << MatchTable::LineBreak;
2006 }
2007 for (auto *Use : I->ImplicitUses) {
2008 auto Namespace = Use->getValue(Name: "Namespace")
2009 ? Use->getValueAsString(FieldName: "Namespace")
2010 : "";
2011 Table << MatchTable::Opcode(Opcode: "GIR_AddImplicitUse")
2012 << MatchTable::Comment(Comment: "InsnID")
2013 << MatchTable::ULEB128Value(IntValue: InsnID)
2014 << MatchTable::NamedValue(NumBytes: 2, Namespace, NamedValue: Use->getName())
2015 << MatchTable::LineBreak;
2016 }
2017 }
2018
2019 AddMIFlags();
2020
2021 // Mark the mutated instruction as erased.
2022 return;
2023 }
2024
2025 // TODO: Simple permutation looks like it could be almost as common as
2026 // mutation due to commutative operations.
2027
2028 if (InsnID == 0) {
2029 Table << MatchTable::Opcode(Opcode: "GIR_BuildRootMI");
2030 } else {
2031 Table << MatchTable::Opcode(Opcode: "GIR_BuildMI") << MatchTable::Comment(Comment: "InsnID")
2032 << MatchTable::ULEB128Value(IntValue: InsnID);
2033 }
2034
2035 Table << MatchTable::Comment(Comment: "Opcode")
2036 << MatchTable::NamedValue(NumBytes: 2, Namespace: I->Namespace, NamedValue: I->getName())
2037 << MatchTable::LineBreak;
2038
2039 for (const auto &Renderer : OperandRenderers)
2040 Renderer->emitRenderOpcodes(Table);
2041
2042 for (auto [OpIdx, Def] : enumerate(First: I->ImplicitDefs)) {
2043 auto Namespace =
2044 Def->getValue(Name: "Namespace") ? Def->getValueAsString(FieldName: "Namespace") : "";
2045 if (DeadImplicitDefs.contains(Ptr: Def)) {
2046 Table
2047 << MatchTable::Opcode(Opcode: "GIR_SetImplicitDefDead")
2048 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2049 << MatchTable::Comment(
2050 Comment: ("OpIdx for " + Namespace + "::" + Def->getName() + "").str())
2051 << MatchTable::ULEB128Value(IntValue: OpIdx) << MatchTable::LineBreak;
2052 }
2053 }
2054
2055 if (!MergeInsnIDs.empty()) {
2056 assert(I->mayLoad || I->mayStore);
2057 Table << MatchTable::Opcode(Opcode: "GIR_MergeMemOperands")
2058 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2059 << MatchTable::Comment(Comment: "NumInsns")
2060 << MatchTable::IntValue(NumBytes: 1, IntValue: MergeInsnIDs.size())
2061 << MatchTable::Comment(Comment: "MergeInsnID's");
2062 for (const auto &MergeInsnID : MergeInsnIDs)
2063 Table << MatchTable::ULEB128Value(IntValue: MergeInsnID);
2064 Table << MatchTable::LineBreak;
2065 }
2066
2067 AddMIFlags();
2068}
2069
2070//===- BuildConstantAction ------------------------------------------------===//
2071
2072void BuildConstantAction::emitActionOpcodes(MatchTable &Table) const {
2073 Table << MatchTable::Opcode(Opcode: "GIR_BuildConstant")
2074 << MatchTable::Comment(Comment: "TempRegID")
2075 << MatchTable::ULEB128Value(IntValue: TempRegID) << MatchTable::Comment(Comment: "Val")
2076 << MatchTable::IntValue(NumBytes: 8, IntValue: Val) << MatchTable::LineBreak;
2077}
2078
2079//===- EraseInstAction ----------------------------------------------------===//
2080
2081void EraseInstAction::emitActionOpcodes(MatchTable &Table) const {
2082 Table << MatchTable::Opcode(Opcode: "GIR_EraseFromParent")
2083 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2084 << MatchTable::LineBreak;
2085}
2086
2087bool EraseInstAction::emitActionOpcodesAndDone(
2088 MatchTable &Table, function_ref<void()> OnDone) const {
2089 if (InsnID != 0) {
2090 emitActionOpcodes(Table);
2091 return false;
2092 }
2093
2094 OnDone();
2095 Table << MatchTable::Opcode(Opcode: "GIR_EraseRootFromParent_Done", IndentAdjust: -1)
2096 << MatchTable::LineBreak;
2097 return true;
2098}
2099
2100//===- ReplaceRegAction ---------------------------------------------------===//
2101
2102void ReplaceRegAction::emitAdditionalPredicates(MatchTable &Table) const {
2103 if (TempRegID != (unsigned)-1)
2104 return;
2105
2106 Table << MatchTable::Opcode(Opcode: "GIM_CheckCanReplaceReg")
2107 << MatchTable::Comment(Comment: "OldInsnID")
2108 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2109 << MatchTable::Comment(Comment: "OldOpIdx") << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2110 << MatchTable::Comment(Comment: "NewInsnId")
2111 << MatchTable::ULEB128Value(IntValue: NewInsnId)
2112 << MatchTable::Comment(Comment: "NewOpIdx") << MatchTable::ULEB128Value(IntValue: NewOpIdx)
2113 << MatchTable::LineBreak;
2114}
2115
2116void ReplaceRegAction::emitActionOpcodes(MatchTable &Table) const {
2117 if (TempRegID != (unsigned)-1) {
2118 Table << MatchTable::Opcode(Opcode: "GIR_ReplaceRegWithTempReg")
2119 << MatchTable::Comment(Comment: "OldInsnID")
2120 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2121 << MatchTable::Comment(Comment: "OldOpIdx")
2122 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2123 << MatchTable::Comment(Comment: "TempRegID")
2124 << MatchTable::ULEB128Value(IntValue: TempRegID) << MatchTable::LineBreak;
2125 } else {
2126 Table << MatchTable::Opcode(Opcode: "GIR_ReplaceReg")
2127 << MatchTable::Comment(Comment: "OldInsnID")
2128 << MatchTable::ULEB128Value(IntValue: OldInsnID)
2129 << MatchTable::Comment(Comment: "OldOpIdx")
2130 << MatchTable::ULEB128Value(IntValue: OldOpIdx)
2131 << MatchTable::Comment(Comment: "NewInsnId")
2132 << MatchTable::ULEB128Value(IntValue: NewInsnId)
2133 << MatchTable::Comment(Comment: "NewOpIdx")
2134 << MatchTable::ULEB128Value(IntValue: NewOpIdx) << MatchTable::LineBreak;
2135 }
2136}
2137
2138//===- ConstrainOperandToRegClassAction -----------------------------------===//
2139
2140void ConstrainOperandToRegClassAction::emitActionOpcodes(
2141 MatchTable &Table) const {
2142 Table << MatchTable::Opcode(Opcode: "GIR_ConstrainOperandRC")
2143 << MatchTable::Comment(Comment: "InsnID") << MatchTable::ULEB128Value(IntValue: InsnID)
2144 << MatchTable::Comment(Comment: "Op") << MatchTable::ULEB128Value(IntValue: OpIdx)
2145 << MatchTable::NamedValue(NumBytes: 2, NamedValue: RC.getQualifiedIdName())
2146 << MatchTable::LineBreak;
2147}
2148
2149//===- MakeTempRegisterAction ---------------------------------------------===//
2150
2151void MakeTempRegisterAction::emitActionOpcodes(MatchTable &Table) const {
2152 Table << MatchTable::Opcode(Opcode: "GIR_MakeTempReg")
2153 << MatchTable::Comment(Comment: "TempRegID")
2154 << MatchTable::ULEB128Value(IntValue: TempRegID) << MatchTable::Comment(Comment: "TypeID");
2155 emitType(Table, Ty);
2156 Table << MatchTable::LineBreak;
2157}
2158