1
2//===- GlobalISelEmitter.cpp - Generate an instruction selector -----------===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// This tablegen backend emits code for use by the GlobalISel instruction
12/// selector. See include/llvm/Target/GlobalISel/Target.td.
13///
14/// This file analyzes the patterns recognized by the SelectionDAGISel tablegen
15/// backend, filters out the ones that are unsupported, maps
16/// SelectionDAG-specific constructs to their GlobalISel counterpart
17/// (when applicable: MVT to LLT; SDNode to generic Instruction).
18///
19/// Not all patterns are supported: pass the tablegen invocation
20/// "-warn-on-skipped-patterns" to emit a warning when a pattern is skipped,
21/// as well as why.
22///
23/// The generated file defines a single method:
24/// bool <Target>InstructionSelector::selectImpl(MachineInstr &I) const;
25/// intended to be used in InstructionSelector::select as the first-step
26/// selector for the patterns that don't require complex C++.
27///
28/// FIXME: We'll probably want to eventually define a base
29/// "TargetGenInstructionSelector" class.
30///
31//===----------------------------------------------------------------------===//
32
33#include "Basic/CodeGenIntrinsics.h"
34#include "Common/CodeGenDAGPatterns.h"
35#include "Common/CodeGenInstruction.h"
36#include "Common/CodeGenRegisters.h"
37#include "Common/CodeGenTarget.h"
38#include "Common/GlobalISel/GlobalISelMatchTable.h"
39#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
40#include "Common/InfoByHwMode.h"
41#include "Common/SubtargetFeatureInfo.h"
42#include "llvm/ADT/Statistic.h"
43#include "llvm/CodeGenTypes/LowLevelType.h"
44#include "llvm/CodeGenTypes/MachineValueType.h"
45#include "llvm/Support/CodeGenCoverage.h"
46#include "llvm/Support/CommandLine.h"
47#include "llvm/Support/Error.h"
48#include "llvm/Support/ScopedPrinter.h"
49#include "llvm/TableGen/Error.h"
50#include "llvm/TableGen/Record.h"
51#include "llvm/TableGen/TableGenBackend.h"
52#include <string>
53
54using namespace llvm;
55using namespace llvm::gi;
56
57using action_iterator = RuleMatcher::action_iterator;
58
59#define DEBUG_TYPE "gisel-emitter"
60
61STATISTIC(NumPatternTotal, "Total number of patterns");
62STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG");
63STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped");
64STATISTIC(NumPatternsTested,
65 "Number of patterns executed according to coverage information");
66
67cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel");
68
69static cl::opt<bool> WarnOnSkippedPatterns(
70 "warn-on-skipped-patterns",
71 cl::desc("Explain why a pattern was skipped for inclusion "
72 "in the GlobalISel selector"),
73 cl::init(Val: false), cl::cat(GlobalISelEmitterCat));
74
75static cl::opt<bool> GenerateCoverage(
76 "instrument-gisel-coverage",
77 cl::desc("Generate coverage instrumentation for GlobalISel"),
78 cl::init(Val: false), cl::cat(GlobalISelEmitterCat));
79
80static cl::opt<std::string> UseCoverageFile(
81 "gisel-coverage-file", cl::init(Val: ""),
82 cl::desc("Specify file to retrieve coverage information from"),
83 cl::cat(GlobalISelEmitterCat));
84
85static cl::opt<bool> OptimizeMatchTable(
86 "optimize-match-table",
87 cl::desc("Generate an optimized version of the match table"),
88 cl::init(Val: true), cl::cat(GlobalISelEmitterCat));
89
90namespace {
91
92static std::string explainPredicates(const TreePatternNode &N) {
93 std::string Explanation;
94 StringRef Separator = "";
95 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
96 const TreePredicateFn &P = Call.Fn;
97 Explanation +=
98 (Separator + P.getOrigPatFragRecord()->getRecord()->getName()).str();
99 Separator = ", ";
100
101 if (P.isAlwaysTrue())
102 Explanation += " always-true";
103 if (P.isImmediatePattern())
104 Explanation += " immediate";
105
106 if (P.isUnindexed())
107 Explanation += " unindexed";
108
109 if (P.isNonExtLoad())
110 Explanation += " non-extload";
111 if (P.isAnyExtLoad())
112 Explanation += " extload";
113 if (P.isSignExtLoad())
114 Explanation += " sextload";
115 if (P.isZeroExtLoad())
116 Explanation += " zextload";
117
118 if (P.isNonTruncStore())
119 Explanation += " non-truncstore";
120 if (P.isTruncStore())
121 Explanation += " truncstore";
122
123 if (Record *VT = P.getMemoryVT())
124 Explanation += (" MemVT=" + VT->getName()).str();
125 if (Record *VT = P.getScalarMemoryVT())
126 Explanation += (" ScalarVT(MemVT)=" + VT->getName()).str();
127
128 if (ListInit *AddrSpaces = P.getAddressSpaces()) {
129 raw_string_ostream OS(Explanation);
130 OS << " AddressSpaces=[";
131
132 StringRef AddrSpaceSeparator;
133 for (Init *Val : AddrSpaces->getValues()) {
134 IntInit *IntVal = dyn_cast<IntInit>(Val);
135 if (!IntVal)
136 continue;
137
138 OS << AddrSpaceSeparator << IntVal->getValue();
139 AddrSpaceSeparator = ", ";
140 }
141
142 OS << ']';
143 }
144
145 int64_t MinAlign = P.getMinAlignment();
146 if (MinAlign > 0)
147 Explanation += " MinAlign=" + utostr(X: MinAlign);
148
149 if (P.isAtomicOrderingMonotonic())
150 Explanation += " monotonic";
151 if (P.isAtomicOrderingAcquire())
152 Explanation += " acquire";
153 if (P.isAtomicOrderingRelease())
154 Explanation += " release";
155 if (P.isAtomicOrderingAcquireRelease())
156 Explanation += " acq_rel";
157 if (P.isAtomicOrderingSequentiallyConsistent())
158 Explanation += " seq_cst";
159 if (P.isAtomicOrderingAcquireOrStronger())
160 Explanation += " >=acquire";
161 if (P.isAtomicOrderingWeakerThanAcquire())
162 Explanation += " <acquire";
163 if (P.isAtomicOrderingReleaseOrStronger())
164 Explanation += " >=release";
165 if (P.isAtomicOrderingWeakerThanRelease())
166 Explanation += " <release";
167 }
168 return Explanation;
169}
170
171std::string explainOperator(Record *Operator) {
172 if (Operator->isSubClassOf(Name: "SDNode"))
173 return (" (" + Operator->getValueAsString(FieldName: "Opcode") + ")").str();
174
175 if (Operator->isSubClassOf(Name: "Intrinsic"))
176 return (" (Operator is an Intrinsic, " + Operator->getName() + ")").str();
177
178 if (Operator->isSubClassOf(Name: "ComplexPattern"))
179 return (" (Operator is an unmapped ComplexPattern, " + Operator->getName() +
180 ")")
181 .str();
182
183 if (Operator->isSubClassOf(Name: "SDNodeXForm"))
184 return (" (Operator is an unmapped SDNodeXForm, " + Operator->getName() +
185 ")")
186 .str();
187
188 return (" (Operator " + Operator->getName() + " not understood)").str();
189}
190
191/// Helper function to let the emitter report skip reason error messages.
192static Error failedImport(const Twine &Reason) {
193 return make_error<StringError>(Args: Reason, Args: inconvertibleErrorCode());
194}
195
196static Error isTrivialOperatorNode(const TreePatternNode &N) {
197 std::string Explanation;
198 std::string Separator;
199
200 bool HasUnsupportedPredicate = false;
201 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
202 const TreePredicateFn &Predicate = Call.Fn;
203
204 if (Predicate.isAlwaysTrue())
205 continue;
206
207 if (Predicate.isImmediatePattern())
208 continue;
209
210 if (Predicate.hasNoUse() || Predicate.hasOneUse())
211 continue;
212
213 if (Predicate.isNonExtLoad() || Predicate.isAnyExtLoad() ||
214 Predicate.isSignExtLoad() || Predicate.isZeroExtLoad())
215 continue;
216
217 if (Predicate.isNonTruncStore() || Predicate.isTruncStore())
218 continue;
219
220 if (Predicate.isLoad() && Predicate.getMemoryVT())
221 continue;
222
223 if (Predicate.isLoad() || Predicate.isStore()) {
224 if (Predicate.isUnindexed())
225 continue;
226 }
227
228 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
229 const ListInit *AddrSpaces = Predicate.getAddressSpaces();
230 if (AddrSpaces && !AddrSpaces->empty())
231 continue;
232
233 if (Predicate.getMinAlignment() > 0)
234 continue;
235 }
236
237 if (Predicate.isAtomic() && Predicate.getMemoryVT())
238 continue;
239
240 if (Predicate.isAtomic() &&
241 (Predicate.isAtomicOrderingMonotonic() ||
242 Predicate.isAtomicOrderingAcquire() ||
243 Predicate.isAtomicOrderingRelease() ||
244 Predicate.isAtomicOrderingAcquireRelease() ||
245 Predicate.isAtomicOrderingSequentiallyConsistent() ||
246 Predicate.isAtomicOrderingAcquireOrStronger() ||
247 Predicate.isAtomicOrderingWeakerThanAcquire() ||
248 Predicate.isAtomicOrderingReleaseOrStronger() ||
249 Predicate.isAtomicOrderingWeakerThanRelease()))
250 continue;
251
252 if (Predicate.hasGISelPredicateCode())
253 continue;
254
255 HasUnsupportedPredicate = true;
256 Explanation = Separator + "Has a predicate (" + explainPredicates(N) + ")";
257 Separator = ", ";
258 Explanation += (Separator + "first-failing:" +
259 Predicate.getOrigPatFragRecord()->getRecord()->getName())
260 .str();
261 break;
262 }
263
264 if (!HasUnsupportedPredicate)
265 return Error::success();
266
267 return failedImport(Reason: Explanation);
268}
269
270static Record *getInitValueAsRegClass(Init *V) {
271 if (DefInit *VDefInit = dyn_cast<DefInit>(Val: V)) {
272 if (VDefInit->getDef()->isSubClassOf(Name: "RegisterOperand"))
273 return VDefInit->getDef()->getValueAsDef(FieldName: "RegClass");
274 if (VDefInit->getDef()->isSubClassOf(Name: "RegisterClass"))
275 return VDefInit->getDef();
276 }
277 return nullptr;
278}
279
280static std::string getScopedName(unsigned Scope, const std::string &Name) {
281 return ("pred:" + Twine(Scope) + ":" + Name).str();
282}
283
284static std::string getMangledRootDefName(StringRef DefOperandName) {
285 return ("DstI[" + DefOperandName + "]").str();
286}
287
288//===- GlobalISelEmitter class --------------------------------------------===//
289
290static Expected<LLTCodeGen> getInstResultType(const TreePatternNode &Dst,
291 const CodeGenTarget &Target) {
292 // While we allow more than one output (both implicit and explicit defs)
293 // below, we only expect one explicit def here.
294 assert(Dst.getOperator()->isSubClassOf("Instruction"));
295 CodeGenInstruction &InstInfo = Target.getInstruction(InstRec: Dst.getOperator());
296 if (!InstInfo.Operands.NumDefs)
297 return failedImport(Reason: "Dst pattern child needs a def");
298
299 ArrayRef<TypeSetByHwMode> ChildTypes = Dst.getExtTypes();
300 if (ChildTypes.size() < 1)
301 return failedImport(Reason: "Dst pattern child has no result");
302
303 // If there are multiple results, just take the first one (this is how
304 // SelectionDAG does it).
305 std::optional<LLTCodeGen> MaybeOpTy;
306 if (ChildTypes.front().isMachineValueType()) {
307 MaybeOpTy = MVTToLLT(SVT: ChildTypes.front().getMachineValueType().SimpleTy);
308 }
309
310 if (!MaybeOpTy)
311 return failedImport(Reason: "Dst operand has an unsupported type");
312 return *MaybeOpTy;
313}
314
315class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter {
316public:
317 explicit GlobalISelEmitter(RecordKeeper &RK);
318
319 void emitAdditionalImpl(raw_ostream &OS) override;
320
321 void emitMIPredicateFns(raw_ostream &OS) override;
322 void emitI64ImmPredicateFns(raw_ostream &OS) override;
323 void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
324 void emitAPIntImmPredicateFns(raw_ostream &OS) override;
325 void emitTestSimplePredicate(raw_ostream &OS) override;
326 void emitRunCustomAction(raw_ostream &OS) override;
327
328 void postProcessRule(RuleMatcher &M);
329
330 const CodeGenTarget &getTarget() const override { return Target; }
331 StringRef getClassName() const override { return ClassName; }
332
333 void run(raw_ostream &OS);
334
335private:
336 std::string ClassName;
337
338 const RecordKeeper &RK;
339 const CodeGenDAGPatterns CGP;
340 const CodeGenTarget &Target;
341 CodeGenRegBank &CGRegs;
342
343 std::vector<Record *> AllPatFrags;
344
345 /// Keep track of the equivalence between SDNodes and Instruction by mapping
346 /// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to
347 /// check for attributes on the relation such as CheckMMOIsNonAtomic.
348 /// This is defined using 'GINodeEquiv' in the target description.
349 DenseMap<Record *, Record *> NodeEquivs;
350
351 /// Keep track of the equivalence between ComplexPattern's and
352 /// GIComplexOperandMatcher. Map entries are specified by subclassing
353 /// GIComplexPatternEquiv.
354 DenseMap<const Record *, const Record *> ComplexPatternEquivs;
355
356 /// Keep track of the equivalence between SDNodeXForm's and
357 /// GICustomOperandRenderer. Map entries are specified by subclassing
358 /// GISDNodeXFormEquiv.
359 DenseMap<const Record *, const Record *> SDNodeXFormEquivs;
360
361 /// Keep track of Scores of PatternsToMatch similar to how the DAG does.
362 /// This adds compatibility for RuleMatchers to use this for ordering rules.
363 DenseMap<uint64_t, int> RuleMatcherScores;
364
365 // Rule coverage information.
366 std::optional<CodeGenCoverage> RuleCoverage;
367
368 /// Variables used to help with collecting of named operands for predicates
369 /// with 'let PredicateCodeUsesOperands = 1'. WaitingForNamedOperands is set
370 /// to the number of named operands that predicate expects. Store locations in
371 /// StoreIdxForName correspond to the order in which operand names appear in
372 /// predicate's argument list.
373 /// When we visit named operand and WaitingForNamedOperands is not zero, add
374 /// matcher that will record operand and decrease counter.
375 unsigned WaitingForNamedOperands = 0;
376 StringMap<unsigned> StoreIdxForName;
377
378 void gatherOpcodeValues();
379 void gatherTypeIDValues();
380 void gatherNodeEquivs();
381
382 Record *findNodeEquiv(Record *N) const;
383 const CodeGenInstruction *getEquivNode(Record &Equiv,
384 const TreePatternNode &N) const;
385
386 Error importRulePredicates(RuleMatcher &M, ArrayRef<Record *> Predicates);
387 Expected<InstructionMatcher &>
388 createAndImportSelDAGMatcher(RuleMatcher &Rule,
389 InstructionMatcher &InsnMatcher,
390 const TreePatternNode &Src, unsigned &TempOpIdx);
391 Error importComplexPatternOperandMatcher(OperandMatcher &OM, Record *R,
392 unsigned &TempOpIdx) const;
393 Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
394 const TreePatternNode &SrcChild,
395 bool OperandIsAPointer, bool OperandIsImmArg,
396 unsigned OpIdx, unsigned &TempOpIdx);
397
398 Expected<BuildMIAction &> createAndImportInstructionRenderer(
399 RuleMatcher &M, InstructionMatcher &InsnMatcher,
400 const TreePatternNode &Src, const TreePatternNode &Dst);
401 Expected<action_iterator> createAndImportSubInstructionRenderer(
402 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
403 const TreePatternNode &Src, unsigned TempReg);
404 Expected<action_iterator>
405 createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M,
406 const TreePatternNode &Dst);
407
408 Expected<action_iterator>
409 importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M,
410 BuildMIAction &DstMIBuilder,
411 const TreePatternNode &Src,
412 const TreePatternNode &Dst, unsigned Start = 0);
413
414 Expected<action_iterator> importExplicitUseRenderers(
415 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
416 const llvm::TreePatternNode &Dst, const TreePatternNode &Src);
417 Expected<action_iterator> importExplicitUseRenderer(
418 action_iterator InsertPt, RuleMatcher &Rule, BuildMIAction &DstMIBuilder,
419 const TreePatternNode &DstChild, const TreePatternNode &Src);
420 Error importDefaultOperandRenderers(action_iterator InsertPt, RuleMatcher &M,
421 BuildMIAction &DstMIBuilder,
422 const DAGDefaultOperand &DefaultOp) const;
423 Error
424 importImplicitDefRenderers(BuildMIAction &DstMIBuilder,
425 const std::vector<Record *> &ImplicitDefs) const;
426
427 /// Analyze pattern \p P, returning a matcher for it if possible.
428 /// Otherwise, return an Error explaining why we don't support it.
429 Expected<RuleMatcher> runOnPattern(const PatternToMatch &P);
430
431 void declareSubtargetFeature(Record *Predicate);
432
433 unsigned declareHwModeCheck(StringRef HwModeFeatures);
434
435 MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize,
436 bool WithCoverage);
437
438 /// Infer a CodeGenRegisterClass for the type of \p SuperRegNode. The returned
439 /// CodeGenRegisterClass will support the CodeGenRegisterClass of
440 /// \p SubRegNode, and the subregister index defined by \p SubRegIdxNode.
441 /// If no register class is found, return std::nullopt.
442 std::optional<const CodeGenRegisterClass *>
443 inferSuperRegisterClassForNode(const TypeSetByHwMode &Ty,
444 const TreePatternNode &SuperRegNode,
445 const TreePatternNode &SubRegIdxNode);
446 std::optional<CodeGenSubRegIndex *>
447 inferSubRegIndexForNode(const TreePatternNode &SubRegIdxNode);
448
449 /// Infer a CodeGenRegisterClass which suppoorts \p Ty and \p SubRegIdxNode.
450 /// Return std::nullopt if no such class exists.
451 std::optional<const CodeGenRegisterClass *>
452 inferSuperRegisterClass(const TypeSetByHwMode &Ty,
453 const TreePatternNode &SubRegIdxNode);
454
455 /// Return the CodeGenRegisterClass associated with \p Leaf if it has one.
456 std::optional<const CodeGenRegisterClass *>
457 getRegClassFromLeaf(const TreePatternNode &Leaf);
458
459 /// Return a CodeGenRegisterClass for \p N if one can be found. Return
460 /// std::nullopt otherwise.
461 std::optional<const CodeGenRegisterClass *>
462 inferRegClassFromPattern(const TreePatternNode &N);
463
464 /// Return the size of the MemoryVT in this predicate, if possible.
465 std::optional<unsigned>
466 getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate);
467
468 // Add builtin predicates.
469 Expected<InstructionMatcher &>
470 addBuiltinPredicates(const Record *SrcGIEquivOrNull,
471 const TreePredicateFn &Predicate,
472 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher);
473};
474
475StringRef getPatFragPredicateEnumName(Record *R) { return R->getName(); }
476
477void GlobalISelEmitter::gatherOpcodeValues() {
478 InstructionOpcodeMatcher::initOpcodeValuesMap(Target);
479}
480
481void GlobalISelEmitter::gatherTypeIDValues() {
482 LLTOperandMatcher::initTypeIDValuesMap();
483}
484
485void GlobalISelEmitter::gatherNodeEquivs() {
486 assert(NodeEquivs.empty());
487 for (Record *Equiv : RK.getAllDerivedDefinitions(ClassName: "GINodeEquiv"))
488 NodeEquivs[Equiv->getValueAsDef(FieldName: "Node")] = Equiv;
489
490 assert(ComplexPatternEquivs.empty());
491 for (Record *Equiv : RK.getAllDerivedDefinitions(ClassName: "GIComplexPatternEquiv")) {
492 Record *SelDAGEquiv = Equiv->getValueAsDef(FieldName: "SelDAGEquivalent");
493 if (!SelDAGEquiv)
494 continue;
495 ComplexPatternEquivs[SelDAGEquiv] = Equiv;
496 }
497
498 assert(SDNodeXFormEquivs.empty());
499 for (Record *Equiv : RK.getAllDerivedDefinitions(ClassName: "GISDNodeXFormEquiv")) {
500 Record *SelDAGEquiv = Equiv->getValueAsDef(FieldName: "SelDAGEquivalent");
501 if (!SelDAGEquiv)
502 continue;
503 SDNodeXFormEquivs[SelDAGEquiv] = Equiv;
504 }
505}
506
507Record *GlobalISelEmitter::findNodeEquiv(Record *N) const {
508 return NodeEquivs.lookup(Val: N);
509}
510
511const CodeGenInstruction *
512GlobalISelEmitter::getEquivNode(Record &Equiv, const TreePatternNode &N) const {
513 if (N.getNumChildren() >= 1) {
514 // setcc operation maps to two different G_* instructions based on the type.
515 if (!Equiv.isValueUnset(FieldName: "IfFloatingPoint") &&
516 MVT(N.getChild(N: 0).getSimpleType(ResNo: 0)).isFloatingPoint())
517 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfFloatingPoint"));
518 }
519
520 if (!Equiv.isValueUnset(FieldName: "IfConvergent") &&
521 N.getIntrinsicInfo(CDP: CGP)->isConvergent)
522 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfConvergent"));
523
524 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
525 const TreePredicateFn &Predicate = Call.Fn;
526 if (!Equiv.isValueUnset(FieldName: "IfSignExtend") &&
527 (Predicate.isLoad() || Predicate.isAtomic()) &&
528 Predicate.isSignExtLoad())
529 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfSignExtend"));
530 if (!Equiv.isValueUnset(FieldName: "IfZeroExtend") &&
531 (Predicate.isLoad() || Predicate.isAtomic()) &&
532 Predicate.isZeroExtLoad())
533 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfZeroExtend"));
534 }
535
536 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "I"));
537}
538
539GlobalISelEmitter::GlobalISelEmitter(RecordKeeper &RK)
540 : GlobalISelMatchTableExecutorEmitter(), RK(RK), CGP(RK),
541 Target(CGP.getTargetInfo()), CGRegs(Target.getRegBank()) {
542 ClassName = Target.getName().str() + "InstructionSelector";
543}
544
545//===- Emitter ------------------------------------------------------------===//
546
547Error GlobalISelEmitter::importRulePredicates(RuleMatcher &M,
548 ArrayRef<Record *> Predicates) {
549 for (Record *Pred : Predicates) {
550 if (Pred->getValueAsString(FieldName: "CondString").empty())
551 continue;
552 declareSubtargetFeature(Predicate: Pred);
553 M.addRequiredFeature(Feature: Pred);
554 }
555
556 return Error::success();
557}
558
559std::optional<unsigned> GlobalISelEmitter::getMemSizeBitsFromPredicate(
560 const TreePredicateFn &Predicate) {
561 std::optional<LLTCodeGen> MemTyOrNone =
562 MVTToLLT(SVT: getValueType(Rec: Predicate.getMemoryVT()));
563
564 if (!MemTyOrNone)
565 return std::nullopt;
566
567 // Align so unusual types like i1 don't get rounded down.
568 return llvm::alignTo(
569 Value: static_cast<unsigned>(MemTyOrNone->get().getSizeInBits()), Align: 8);
570}
571
572Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
573 const Record *SrcGIEquivOrNull, const TreePredicateFn &Predicate,
574 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher) {
575 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
576 if (const ListInit *AddrSpaces = Predicate.getAddressSpaces()) {
577 SmallVector<unsigned, 4> ParsedAddrSpaces;
578
579 for (Init *Val : AddrSpaces->getValues()) {
580 IntInit *IntVal = dyn_cast<IntInit>(Val);
581 if (!IntVal)
582 return failedImport(Reason: "Address space is not an integer");
583 ParsedAddrSpaces.push_back(Elt: IntVal->getValue());
584 }
585
586 if (!ParsedAddrSpaces.empty()) {
587 InsnMatcher.addPredicate<MemoryAddressSpacePredicateMatcher>(
588 args: 0, args&: ParsedAddrSpaces);
589 return InsnMatcher;
590 }
591 }
592
593 int64_t MinAlign = Predicate.getMinAlignment();
594 if (MinAlign > 0) {
595 InsnMatcher.addPredicate<MemoryAlignmentPredicateMatcher>(args: 0, args&: MinAlign);
596 return InsnMatcher;
597 }
598 }
599
600 // G_LOAD is used for both non-extending and any-extending loads.
601 if (Predicate.isLoad() && Predicate.isNonExtLoad()) {
602 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
603 args: 0, args: MemoryVsLLTSizePredicateMatcher::EqualTo, args: 0);
604 return InsnMatcher;
605 }
606 if (Predicate.isLoad() && Predicate.isAnyExtLoad()) {
607 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
608 args: 0, args: MemoryVsLLTSizePredicateMatcher::LessThan, args: 0);
609 return InsnMatcher;
610 }
611
612 if (Predicate.isStore()) {
613 if (Predicate.isTruncStore()) {
614 if (Predicate.getMemoryVT() != nullptr) {
615 // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size.
616 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
617 if (!MemSizeInBits)
618 return failedImport(Reason: "MemVT could not be converted to LLT");
619
620 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(args: 0, args: *MemSizeInBits /
621 8);
622 } else {
623 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
624 args: 0, args: MemoryVsLLTSizePredicateMatcher::LessThan, args: 0);
625 }
626 return InsnMatcher;
627 }
628 if (Predicate.isNonTruncStore()) {
629 // We need to check the sizes match here otherwise we could incorrectly
630 // match truncating stores with non-truncating ones.
631 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
632 args: 0, args: MemoryVsLLTSizePredicateMatcher::EqualTo, args: 0);
633 }
634 }
635
636 assert(SrcGIEquivOrNull != nullptr && "Invalid SrcGIEquivOrNull value");
637 // No check required. We already did it by swapping the opcode.
638 if (!SrcGIEquivOrNull->isValueUnset(FieldName: "IfSignExtend") &&
639 Predicate.isSignExtLoad())
640 return InsnMatcher;
641
642 // No check required. We already did it by swapping the opcode.
643 if (!SrcGIEquivOrNull->isValueUnset(FieldName: "IfZeroExtend") &&
644 Predicate.isZeroExtLoad())
645 return InsnMatcher;
646
647 // No check required. G_STORE by itself is a non-extending store.
648 if (Predicate.isNonTruncStore())
649 return InsnMatcher;
650
651 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
652 if (Predicate.getMemoryVT() != nullptr) {
653 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
654 if (!MemSizeInBits)
655 return failedImport(Reason: "MemVT could not be converted to LLT");
656
657 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(args: 0,
658 args: *MemSizeInBits / 8);
659 return InsnMatcher;
660 }
661 }
662
663 if (Predicate.isLoad() || Predicate.isStore()) {
664 // No check required. A G_LOAD/G_STORE is an unindexed load.
665 if (Predicate.isUnindexed())
666 return InsnMatcher;
667 }
668
669 if (Predicate.isAtomic()) {
670 if (Predicate.isAtomicOrderingMonotonic()) {
671 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Monotonic");
672 return InsnMatcher;
673 }
674 if (Predicate.isAtomicOrderingAcquire()) {
675 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Acquire");
676 return InsnMatcher;
677 }
678 if (Predicate.isAtomicOrderingRelease()) {
679 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Release");
680 return InsnMatcher;
681 }
682 if (Predicate.isAtomicOrderingAcquireRelease()) {
683 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
684 args: "AcquireRelease");
685 return InsnMatcher;
686 }
687 if (Predicate.isAtomicOrderingSequentiallyConsistent()) {
688 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
689 args: "SequentiallyConsistent");
690 return InsnMatcher;
691 }
692 }
693
694 if (Predicate.isAtomicOrderingAcquireOrStronger()) {
695 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
696 args: "Acquire", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
697 return InsnMatcher;
698 }
699 if (Predicate.isAtomicOrderingWeakerThanAcquire()) {
700 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
701 args: "Acquire", args: AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
702 return InsnMatcher;
703 }
704
705 if (Predicate.isAtomicOrderingReleaseOrStronger()) {
706 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
707 args: "Release", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
708 return InsnMatcher;
709 }
710 if (Predicate.isAtomicOrderingWeakerThanRelease()) {
711 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
712 args: "Release", args: AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
713 return InsnMatcher;
714 }
715 HasAddedMatcher = false;
716 return InsnMatcher;
717}
718
719Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
720 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
721 const TreePatternNode &Src, unsigned &TempOpIdx) {
722 const auto SavedFlags = Rule.setGISelFlags(Src.getGISelFlagsRecord());
723
724 Record *SrcGIEquivOrNull = nullptr;
725 const CodeGenInstruction *SrcGIOrNull = nullptr;
726
727 // Start with the defined operands (i.e., the results of the root operator).
728 if (Src.isLeaf()) {
729 Init *SrcInit = Src.getLeafValue();
730 if (isa<IntInit>(Val: SrcInit)) {
731 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(
732 args: &Target.getInstruction(InstRec: RK.getDef(Name: "G_CONSTANT")));
733 } else
734 return failedImport(
735 Reason: "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
736 } else {
737 SrcGIEquivOrNull = findNodeEquiv(N: Src.getOperator());
738 if (!SrcGIEquivOrNull)
739 return failedImport(Reason: "Pattern operator lacks an equivalent Instruction" +
740 explainOperator(Operator: Src.getOperator()));
741 SrcGIOrNull = getEquivNode(Equiv&: *SrcGIEquivOrNull, N: Src);
742
743 // The operators look good: match the opcode
744 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(args&: SrcGIOrNull);
745 }
746
747 unsigned OpIdx = 0;
748 for (const TypeSetByHwMode &VTy : Src.getExtTypes()) {
749 // Results don't have a name unless they are the root node. The caller will
750 // set the name if appropriate.
751 const bool OperandIsAPointer =
752 SrcGIOrNull && SrcGIOrNull->isOutOperandAPointer(i: OpIdx);
753 OperandMatcher &OM = InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
754 if (auto Error = OM.addTypeCheckPredicate(VTy, OperandIsAPointer))
755 return failedImport(Reason: toString(E: std::move(Error)) +
756 " for result of Src pattern operator");
757 }
758
759 for (const TreePredicateCall &Call : Src.getPredicateCalls()) {
760 const TreePredicateFn &Predicate = Call.Fn;
761 bool HasAddedBuiltinMatcher = true;
762 if (Predicate.isAlwaysTrue())
763 continue;
764
765 if (Predicate.isImmediatePattern()) {
766 InsnMatcher.addPredicate<InstructionImmPredicateMatcher>(args: Predicate);
767 continue;
768 }
769
770 auto InsnMatcherOrError = addBuiltinPredicates(
771 SrcGIEquivOrNull, Predicate, InsnMatcher, HasAddedMatcher&: HasAddedBuiltinMatcher);
772 if (auto Error = InsnMatcherOrError.takeError())
773 return std::move(Error);
774
775 // FIXME: This should be part of addBuiltinPredicates(). If we add this at
776 // the start of addBuiltinPredicates() without returning, then there might
777 // be cases where we hit the last return before which the
778 // HasAddedBuiltinMatcher will be set to false. The predicate could be
779 // missed if we add it in the middle or at the end due to return statements
780 // after the addPredicate<>() calls.
781 if (Predicate.hasNoUse()) {
782 InsnMatcher.addPredicate<NoUsePredicateMatcher>();
783 HasAddedBuiltinMatcher = true;
784 }
785 if (Predicate.hasOneUse()) {
786 InsnMatcher.addPredicate<OneUsePredicateMatcher>();
787 HasAddedBuiltinMatcher = true;
788 }
789
790 if (Predicate.hasGISelPredicateCode()) {
791 if (Predicate.usesOperands()) {
792 assert(WaitingForNamedOperands == 0 &&
793 "previous predicate didn't find all operands or "
794 "nested predicate that uses operands");
795 TreePattern *TP = Predicate.getOrigPatFragRecord();
796 WaitingForNamedOperands = TP->getNumArgs();
797 for (unsigned I = 0; I < WaitingForNamedOperands; ++I)
798 StoreIdxForName[getScopedName(Scope: Call.Scope, Name: TP->getArgName(i: I))] = I;
799 }
800 InsnMatcher.addPredicate<GenericInstructionPredicateMatcher>(args: Predicate);
801 continue;
802 }
803 if (!HasAddedBuiltinMatcher) {
804 return failedImport(Reason: "Src pattern child has predicate (" +
805 explainPredicates(N: Src) + ")");
806 }
807 }
808
809 if (SrcGIEquivOrNull &&
810 SrcGIEquivOrNull->getValueAsBit(FieldName: "CheckMMOIsNonAtomic"))
811 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "NotAtomic");
812 else if (SrcGIEquivOrNull &&
813 SrcGIEquivOrNull->getValueAsBit(FieldName: "CheckMMOIsAtomic")) {
814 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
815 args: "Unordered", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
816 }
817
818 if (Src.isLeaf()) {
819 Init *SrcInit = Src.getLeafValue();
820 if (IntInit *SrcIntInit = dyn_cast<IntInit>(Val: SrcInit)) {
821 OperandMatcher &OM =
822 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: Src.getName(), AllocatedTemporariesBaseID: TempOpIdx);
823 OM.addPredicate<LiteralIntOperandMatcher>(args: SrcIntInit->getValue());
824 } else
825 return failedImport(
826 Reason: "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
827 } else {
828 assert(SrcGIOrNull &&
829 "Expected to have already found an equivalent Instruction");
830 if (SrcGIOrNull->TheDef->getName() == "G_CONSTANT" ||
831 SrcGIOrNull->TheDef->getName() == "G_FCONSTANT") {
832 // imm/fpimm still have operands but we don't need to do anything with it
833 // here since we don't support ImmLeaf predicates yet. However, we still
834 // need to note the hidden operand to get GIM_CheckNumOperands correct.
835 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
836 return InsnMatcher;
837 }
838
839 if (SrcGIOrNull->TheDef->getName() == "G_FRAME_INDEX") {
840 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: Src.getName(), AllocatedTemporariesBaseID: TempOpIdx);
841 return InsnMatcher;
842 }
843
844 // Special case because the operand order is changed from setcc. The
845 // predicate operand needs to be swapped from the last operand to the first
846 // source.
847
848 unsigned NumChildren = Src.getNumChildren();
849 bool IsFCmp = SrcGIOrNull->TheDef->getName() == "G_FCMP";
850
851 if (IsFCmp || SrcGIOrNull->TheDef->getName() == "G_ICMP") {
852 const TreePatternNode &SrcChild = Src.getChild(N: NumChildren - 1);
853 if (SrcChild.isLeaf()) {
854 DefInit *DI = dyn_cast<DefInit>(Val: SrcChild.getLeafValue());
855 Record *CCDef = DI ? DI->getDef() : nullptr;
856 if (!CCDef || !CCDef->isSubClassOf(Name: "CondCode"))
857 return failedImport(Reason: "Unable to handle CondCode");
858
859 OperandMatcher &OM =
860 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: SrcChild.getName(), AllocatedTemporariesBaseID: TempOpIdx);
861 StringRef PredType = IsFCmp ? CCDef->getValueAsString(FieldName: "FCmpPredicate")
862 : CCDef->getValueAsString(FieldName: "ICmpPredicate");
863
864 if (!PredType.empty()) {
865 OM.addPredicate<CmpPredicateOperandMatcher>(args: std::string(PredType));
866 // Process the other 2 operands normally.
867 --NumChildren;
868 }
869 }
870 }
871
872 // Match the used operands (i.e. the children of the operator).
873 bool IsIntrinsic =
874 SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" ||
875 SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_W_SIDE_EFFECTS" ||
876 SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_CONVERGENT" ||
877 SrcGIOrNull->TheDef->getName() ==
878 "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS";
879 const CodeGenIntrinsic *II = Src.getIntrinsicInfo(CDP: CGP);
880 if (IsIntrinsic && !II)
881 return failedImport(Reason: "Expected IntInit containing intrinsic ID)");
882
883 for (unsigned I = 0; I != NumChildren; ++I) {
884 const TreePatternNode &SrcChild = Src.getChild(N: I);
885
886 // We need to determine the meaning of a literal integer based on the
887 // context. If this is a field required to be an immediate (such as an
888 // immarg intrinsic argument), the required predicates are different than
889 // a constant which may be materialized in a register. If we have an
890 // argument that is required to be an immediate, we should not emit an LLT
891 // type check, and should not be looking for a G_CONSTANT defined
892 // register.
893 bool OperandIsImmArg = SrcGIOrNull->isInOperandImmArg(i: I);
894
895 // SelectionDAG allows pointers to be represented with iN since it doesn't
896 // distinguish between pointers and integers but they are different types
897 // in GlobalISel. Coerce integers to pointers to address space 0 if the
898 // context indicates a pointer.
899 //
900 bool OperandIsAPointer = SrcGIOrNull->isInOperandAPointer(i: I);
901
902 if (IsIntrinsic) {
903 // For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately
904 // following the defs is an intrinsic ID.
905 if (I == 0) {
906 OperandMatcher &OM =
907 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: SrcChild.getName(), AllocatedTemporariesBaseID: TempOpIdx);
908 OM.addPredicate<IntrinsicIDOperandMatcher>(args&: II);
909 continue;
910 }
911
912 // We have to check intrinsics for llvm_anyptr_ty and immarg parameters.
913 //
914 // Note that we have to look at the i-1th parameter, because we don't
915 // have the intrinsic ID in the intrinsic's parameter list.
916 OperandIsAPointer |= II->isParamAPointer(ParamIdx: I - 1);
917 OperandIsImmArg |= II->isParamImmArg(ParamIdx: I - 1);
918 }
919
920 if (auto Error =
921 importChildMatcher(Rule, InsnMatcher, SrcChild, OperandIsAPointer,
922 OperandIsImmArg, OpIdx: OpIdx++, TempOpIdx))
923 return std::move(Error);
924 }
925 }
926
927 return InsnMatcher;
928}
929
930Error GlobalISelEmitter::importComplexPatternOperandMatcher(
931 OperandMatcher &OM, Record *R, unsigned &TempOpIdx) const {
932 const auto &ComplexPattern = ComplexPatternEquivs.find(Val: R);
933 if (ComplexPattern == ComplexPatternEquivs.end())
934 return failedImport(Reason: "SelectionDAG ComplexPattern (" + R->getName() +
935 ") not mapped to GlobalISel");
936
937 OM.addPredicate<ComplexPatternOperandMatcher>(args&: OM, args: *ComplexPattern->second);
938 TempOpIdx++;
939 return Error::success();
940}
941
942// Get the name to use for a pattern operand. For an anonymous physical register
943// input, this should use the register name.
944static StringRef getSrcChildName(const TreePatternNode &SrcChild,
945 Record *&PhysReg) {
946 StringRef SrcChildName = SrcChild.getName();
947 if (SrcChildName.empty() && SrcChild.isLeaf()) {
948 if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
949 auto *ChildRec = ChildDefInit->getDef();
950 if (ChildRec->isSubClassOf(Name: "Register")) {
951 SrcChildName = ChildRec->getName();
952 PhysReg = ChildRec;
953 }
954 }
955 }
956
957 return SrcChildName;
958}
959
960Error GlobalISelEmitter::importChildMatcher(
961 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
962 const TreePatternNode &SrcChild, bool OperandIsAPointer,
963 bool OperandIsImmArg, unsigned OpIdx, unsigned &TempOpIdx) {
964
965 Record *PhysReg = nullptr;
966 std::string SrcChildName = std::string(getSrcChildName(SrcChild, PhysReg));
967 if (!SrcChild.isLeaf() &&
968 SrcChild.getOperator()->isSubClassOf(Name: "ComplexPattern")) {
969 // The "name" of a non-leaf complex pattern (MY_PAT $op1, $op2) is
970 // "MY_PAT:op1:op2" and the ones with same "name" represent same operand.
971 std::string PatternName = std::string(SrcChild.getOperator()->getName());
972 for (unsigned I = 0; I < SrcChild.getNumChildren(); ++I) {
973 PatternName += ":";
974 PatternName += SrcChild.getChild(N: I).getName();
975 }
976 SrcChildName = PatternName;
977 }
978
979 OperandMatcher &OM =
980 PhysReg ? InsnMatcher.addPhysRegInput(Reg: PhysReg, OpIdx, TempOpIdx)
981 : InsnMatcher.addOperand(OpIdx, SymbolicName: SrcChildName, AllocatedTemporariesBaseID: TempOpIdx);
982 if (OM.isSameAsAnotherOperand())
983 return Error::success();
984
985 ArrayRef<TypeSetByHwMode> ChildTypes = SrcChild.getExtTypes();
986 if (ChildTypes.size() != 1)
987 return failedImport(Reason: "Src pattern child has multiple results");
988
989 // Check MBB's before the type check since they are not a known type.
990 if (!SrcChild.isLeaf()) {
991 if (SrcChild.getOperator()->isSubClassOf(Name: "SDNode")) {
992 auto &ChildSDNI = CGP.getSDNodeInfo(R: SrcChild.getOperator());
993 if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") {
994 OM.addPredicate<MBBOperandMatcher>();
995 return Error::success();
996 }
997 if (SrcChild.getOperator()->getName() == "timm") {
998 OM.addPredicate<ImmOperandMatcher>();
999
1000 // Add predicates, if any
1001 for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {
1002 const TreePredicateFn &Predicate = Call.Fn;
1003
1004 // Only handle immediate patterns for now
1005 if (Predicate.isImmediatePattern()) {
1006 OM.addPredicate<OperandImmPredicateMatcher>(args: Predicate);
1007 }
1008 }
1009
1010 return Error::success();
1011 }
1012 }
1013 }
1014
1015 // Immediate arguments have no meaningful type to check as they don't have
1016 // registers.
1017 if (!OperandIsImmArg) {
1018 if (auto Error =
1019 OM.addTypeCheckPredicate(VTy: ChildTypes.front(), OperandIsAPointer))
1020 return failedImport(Reason: toString(E: std::move(Error)) + " for Src operand (" +
1021 to_string(Value: SrcChild) + ")");
1022 }
1023
1024 // Try look up SrcChild for a (named) predicate operand if there is any.
1025 if (WaitingForNamedOperands) {
1026 auto &ScopedNames = SrcChild.getNamesAsPredicateArg();
1027 if (!ScopedNames.empty()) {
1028 auto PA = ScopedNames.begin();
1029 std::string Name = getScopedName(Scope: PA->getScope(), Name: PA->getIdentifier());
1030 OM.addPredicate<RecordNamedOperandMatcher>(args&: StoreIdxForName[Name], args&: Name);
1031 --WaitingForNamedOperands;
1032 }
1033 }
1034
1035 // Check for nested instructions.
1036 if (!SrcChild.isLeaf()) {
1037 if (SrcChild.getOperator()->isSubClassOf(Name: "ComplexPattern")) {
1038 // When a ComplexPattern is used as an operator, it should do the same
1039 // thing as when used as a leaf. However, the children of the operator
1040 // name the sub-operands that make up the complex operand and we must
1041 // prepare to reference them in the renderer too.
1042 unsigned RendererID = TempOpIdx;
1043 if (auto Error = importComplexPatternOperandMatcher(
1044 OM, R: SrcChild.getOperator(), TempOpIdx))
1045 return Error;
1046
1047 for (unsigned I = 0, E = SrcChild.getNumChildren(); I != E; ++I) {
1048 auto &SubOperand = SrcChild.getChild(N: I);
1049 if (!SubOperand.getName().empty()) {
1050 if (auto Error = Rule.defineComplexSubOperand(
1051 SymbolicName: SubOperand.getName(), ComplexPattern: SrcChild.getOperator(), RendererID, SubOperandID: I,
1052 ParentSymbolicName: SrcChildName))
1053 return Error;
1054 }
1055 }
1056
1057 return Error::success();
1058 }
1059
1060 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
1061 args&: InsnMatcher.getRuleMatcher(), args: SrcChild.getName());
1062 if (!MaybeInsnOperand) {
1063 // This isn't strictly true. If the user were to provide exactly the same
1064 // matchers as the original operand then we could allow it. However, it's
1065 // simpler to not permit the redundant specification.
1066 return failedImport(
1067 Reason: "Nested instruction cannot be the same as another operand");
1068 }
1069
1070 // Map the node to a gMIR instruction.
1071 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
1072 auto InsnMatcherOrError = createAndImportSelDAGMatcher(
1073 Rule, InsnMatcher&: InsnOperand.getInsnMatcher(), Src: SrcChild, TempOpIdx);
1074 if (auto Error = InsnMatcherOrError.takeError())
1075 return Error;
1076
1077 return Error::success();
1078 }
1079
1080 if (SrcChild.hasAnyPredicate())
1081 return failedImport(Reason: "Src pattern child has unsupported predicate");
1082
1083 // Check for constant immediates.
1084 if (auto *ChildInt = dyn_cast<IntInit>(Val: SrcChild.getLeafValue())) {
1085 if (OperandIsImmArg) {
1086 // Checks for argument directly in operand list
1087 OM.addPredicate<LiteralIntOperandMatcher>(args: ChildInt->getValue());
1088 } else {
1089 // Checks for materialized constant
1090 OM.addPredicate<ConstantIntOperandMatcher>(args: ChildInt->getValue());
1091 }
1092 return Error::success();
1093 }
1094
1095 // Check for def's like register classes or ComplexPattern's.
1096 if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
1097 auto *ChildRec = ChildDefInit->getDef();
1098
1099 // Check for register classes.
1100 if (ChildRec->isSubClassOf(Name: "RegisterClass") ||
1101 ChildRec->isSubClassOf(Name: "RegisterOperand")) {
1102 OM.addPredicate<RegisterBankOperandMatcher>(
1103 args: Target.getRegisterClass(R: getInitValueAsRegClass(V: ChildDefInit)));
1104 return Error::success();
1105 }
1106
1107 if (ChildRec->isSubClassOf(Name: "Register")) {
1108 // This just be emitted as a copy to the specific register.
1109 ValueTypeByHwMode VT = ChildTypes.front().getValueTypeByHwMode();
1110 const CodeGenRegisterClass *RC =
1111 CGRegs.getMinimalPhysRegClass(RegRecord: ChildRec, VT: &VT);
1112 if (!RC) {
1113 return failedImport(
1114 Reason: "Could not determine physical register class of pattern source");
1115 }
1116
1117 OM.addPredicate<RegisterBankOperandMatcher>(args: *RC);
1118 return Error::success();
1119 }
1120
1121 // Check for ValueType.
1122 if (ChildRec->isSubClassOf(Name: "ValueType")) {
1123 // We already added a type check as standard practice so this doesn't need
1124 // to do anything.
1125 return Error::success();
1126 }
1127
1128 // Check for ComplexPattern's.
1129 if (ChildRec->isSubClassOf(Name: "ComplexPattern"))
1130 return importComplexPatternOperandMatcher(OM, R: ChildRec, TempOpIdx);
1131
1132 if (ChildRec->isSubClassOf(Name: "ImmLeaf")) {
1133 return failedImport(
1134 Reason: "Src pattern child def is an unsupported tablegen class (ImmLeaf)");
1135 }
1136
1137 // Place holder for SRCVALUE nodes. Nothing to do here.
1138 if (ChildRec->getName() == "srcvalue")
1139 return Error::success();
1140
1141 const bool ImmAllOnesV = ChildRec->getName() == "immAllOnesV";
1142 if (ImmAllOnesV || ChildRec->getName() == "immAllZerosV") {
1143 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
1144 args&: InsnMatcher.getRuleMatcher(), args: SrcChild.getName(), args: false);
1145 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
1146
1147 ValueTypeByHwMode VTy = ChildTypes.front().getValueTypeByHwMode();
1148
1149 const CodeGenInstruction &BuildVector =
1150 Target.getInstruction(InstRec: RK.getDef(Name: "G_BUILD_VECTOR"));
1151 const CodeGenInstruction &BuildVectorTrunc =
1152 Target.getInstruction(InstRec: RK.getDef(Name: "G_BUILD_VECTOR_TRUNC"));
1153
1154 // Treat G_BUILD_VECTOR as the canonical opcode, and G_BUILD_VECTOR_TRUNC
1155 // as an alternative.
1156 InsnOperand.getInsnMatcher().addPredicate<InstructionOpcodeMatcher>(
1157 args: ArrayRef({&BuildVector, &BuildVectorTrunc}));
1158
1159 // TODO: Handle both G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC We could
1160 // theoretically not emit any opcode check, but getOpcodeMatcher currently
1161 // has to succeed.
1162 OperandMatcher &OM =
1163 InsnOperand.getInsnMatcher().addOperand(OpIdx: 0, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
1164 if (auto Error =
1165 OM.addTypeCheckPredicate(VTy, OperandIsAPointer: false /* OperandIsAPointer */))
1166 return failedImport(Reason: toString(E: std::move(Error)) +
1167 " for result of Src pattern operator");
1168
1169 InsnOperand.getInsnMatcher().addPredicate<VectorSplatImmPredicateMatcher>(
1170 args: ImmAllOnesV ? VectorSplatImmPredicateMatcher::AllOnes
1171 : VectorSplatImmPredicateMatcher::AllZeros);
1172 return Error::success();
1173 }
1174
1175 return failedImport(
1176 Reason: "Src pattern child def is an unsupported tablegen class");
1177 }
1178
1179 return failedImport(Reason: "Src pattern child is an unsupported kind");
1180}
1181
1182Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderer(
1183 action_iterator InsertPt, RuleMatcher &Rule, BuildMIAction &DstMIBuilder,
1184 const TreePatternNode &DstChild, const TreePatternNode &Src) {
1185
1186 const auto &SubOperand = Rule.getComplexSubOperand(SymbolicName: DstChild.getName());
1187 if (SubOperand) {
1188 DstMIBuilder.addRenderer<RenderComplexPatternOperand>(
1189 args&: *std::get<0>(t: *SubOperand), args: DstChild.getName(), args: std::get<1>(t: *SubOperand),
1190 args: std::get<2>(t: *SubOperand));
1191 return InsertPt;
1192 }
1193
1194 if (!DstChild.isLeaf()) {
1195 if (DstChild.getOperator()->isSubClassOf(Name: "SDNodeXForm")) {
1196 auto &Child = DstChild.getChild(N: 0);
1197 auto I = SDNodeXFormEquivs.find(Val: DstChild.getOperator());
1198 if (I != SDNodeXFormEquivs.end()) {
1199 Record *XFormOpc = DstChild.getOperator()->getValueAsDef(FieldName: "Opcode");
1200 if (XFormOpc->getName() == "timm") {
1201 // If this is a TargetConstant, there won't be a corresponding
1202 // instruction to transform. Instead, this will refer directly to an
1203 // operand in an instruction's operand list.
1204 DstMIBuilder.addRenderer<CustomOperandRenderer>(args: *I->second,
1205 args: Child.getName());
1206 } else {
1207 DstMIBuilder.addRenderer<CustomRenderer>(args: *I->second, args: Child.getName());
1208 }
1209
1210 return InsertPt;
1211 }
1212 return failedImport(Reason: "SDNodeXForm " + Child.getName() +
1213 " has no custom renderer");
1214 }
1215
1216 // We accept 'bb' here. It's an operator because BasicBlockSDNode isn't
1217 // inline, but in MI it's just another operand.
1218 if (DstChild.getOperator()->isSubClassOf(Name: "SDNode")) {
1219 auto &ChildSDNI = CGP.getSDNodeInfo(R: DstChild.getOperator());
1220 if (ChildSDNI.getSDClassName() == "BasicBlockSDNode") {
1221 DstMIBuilder.addRenderer<CopyRenderer>(args: DstChild.getName());
1222 return InsertPt;
1223 }
1224 }
1225
1226 // Similarly, imm is an operator in TreePatternNode's view but must be
1227 // rendered as operands.
1228 // FIXME: The target should be able to choose sign-extended when appropriate
1229 // (e.g. on Mips).
1230 if (DstChild.getOperator()->getName() == "timm") {
1231 DstMIBuilder.addRenderer<CopyRenderer>(args: DstChild.getName());
1232 return InsertPt;
1233 }
1234 if (DstChild.getOperator()->getName() == "tframeindex") {
1235 DstMIBuilder.addRenderer<CopyRenderer>(args: DstChild.getName());
1236 return InsertPt;
1237 }
1238 if (DstChild.getOperator()->getName() == "imm") {
1239 DstMIBuilder.addRenderer<CopyConstantAsImmRenderer>(args: DstChild.getName());
1240 return InsertPt;
1241 }
1242 if (DstChild.getOperator()->getName() == "fpimm") {
1243 DstMIBuilder.addRenderer<CopyFConstantAsFPImmRenderer>(
1244 args: DstChild.getName());
1245 return InsertPt;
1246 }
1247
1248 if (DstChild.getOperator()->isSubClassOf(Name: "Instruction")) {
1249 auto OpTy = getInstResultType(Dst: DstChild, Target);
1250 if (!OpTy)
1251 return OpTy.takeError();
1252
1253 unsigned TempRegID = Rule.allocateTempRegID();
1254 InsertPt =
1255 Rule.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTy, args&: TempRegID);
1256 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID);
1257
1258 auto InsertPtOrError = createAndImportSubInstructionRenderer(
1259 InsertPt: ++InsertPt, M&: Rule, Dst: DstChild, Src, TempReg: TempRegID);
1260 if (auto Error = InsertPtOrError.takeError())
1261 return std::move(Error);
1262 return InsertPtOrError.get();
1263 }
1264
1265 return failedImport(Reason: "Dst pattern child isn't a leaf node or an MBB" +
1266 llvm::to_string(Value: DstChild));
1267 }
1268
1269 // It could be a specific immediate in which case we should just check for
1270 // that immediate.
1271 if (const IntInit *ChildIntInit =
1272 dyn_cast<IntInit>(Val: DstChild.getLeafValue())) {
1273 DstMIBuilder.addRenderer<ImmRenderer>(args: ChildIntInit->getValue());
1274 return InsertPt;
1275 }
1276
1277 // Otherwise, we're looking for a bog-standard RegisterClass operand.
1278 if (auto *ChildDefInit = dyn_cast<DefInit>(Val: DstChild.getLeafValue())) {
1279 auto *ChildRec = ChildDefInit->getDef();
1280
1281 ArrayRef<TypeSetByHwMode> ChildTypes = DstChild.getExtTypes();
1282 if (ChildTypes.size() != 1)
1283 return failedImport(Reason: "Dst pattern child has multiple results");
1284
1285 std::optional<LLTCodeGen> OpTyOrNone;
1286 if (ChildTypes.front().isMachineValueType())
1287 OpTyOrNone = MVTToLLT(SVT: ChildTypes.front().getMachineValueType().SimpleTy);
1288 if (!OpTyOrNone)
1289 return failedImport(Reason: "Dst operand has an unsupported type");
1290
1291 if (ChildRec->isSubClassOf(Name: "Register")) {
1292 DstMIBuilder.addRenderer<AddRegisterRenderer>(args: Target, args&: ChildRec);
1293 return InsertPt;
1294 }
1295
1296 if (ChildRec->isSubClassOf(Name: "RegisterClass") ||
1297 ChildRec->isSubClassOf(Name: "RegisterOperand") ||
1298 ChildRec->isSubClassOf(Name: "ValueType")) {
1299 if (ChildRec->isSubClassOf(Name: "RegisterOperand") &&
1300 !ChildRec->isValueUnset(FieldName: "GIZeroRegister")) {
1301 DstMIBuilder.addRenderer<CopyOrAddZeroRegRenderer>(
1302 args: DstChild.getName(), args: ChildRec->getValueAsDef(FieldName: "GIZeroRegister"));
1303 return InsertPt;
1304 }
1305
1306 DstMIBuilder.addRenderer<CopyRenderer>(args: DstChild.getName());
1307 return InsertPt;
1308 }
1309
1310 if (ChildRec->isSubClassOf(Name: "SubRegIndex")) {
1311 CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(ChildRec);
1312 DstMIBuilder.addRenderer<ImmRenderer>(args: SubIdx->EnumValue);
1313 return InsertPt;
1314 }
1315
1316 if (ChildRec->isSubClassOf(Name: "ComplexPattern")) {
1317 const auto &ComplexPattern = ComplexPatternEquivs.find(Val: ChildRec);
1318 if (ComplexPattern == ComplexPatternEquivs.end())
1319 return failedImport(
1320 Reason: "SelectionDAG ComplexPattern not mapped to GlobalISel");
1321
1322 const OperandMatcher &OM = Rule.getOperandMatcher(Name: DstChild.getName());
1323 DstMIBuilder.addRenderer<RenderComplexPatternOperand>(
1324 args: *ComplexPattern->second, args: DstChild.getName(),
1325 args: OM.getAllocatedTemporariesBaseID());
1326 return InsertPt;
1327 }
1328
1329 return failedImport(
1330 Reason: "Dst pattern child def is an unsupported tablegen class");
1331 }
1332
1333 // Handle the case where the MVT/register class is omitted in the dest pattern
1334 // but MVT exists in the source pattern.
1335 if (isa<UnsetInit>(Val: DstChild.getLeafValue())) {
1336 for (unsigned NumOp = 0; NumOp < Src.getNumChildren(); NumOp++)
1337 if (Src.getChild(N: NumOp).getName() == DstChild.getName()) {
1338 DstMIBuilder.addRenderer<CopyRenderer>(args: Src.getChild(N: NumOp).getName());
1339 return InsertPt;
1340 }
1341 }
1342 return failedImport(Reason: "Dst pattern child is an unsupported kind");
1343}
1344
1345Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
1346 RuleMatcher &M, InstructionMatcher &InsnMatcher, const TreePatternNode &Src,
1347 const TreePatternNode &Dst) {
1348 auto InsertPtOrError = createInstructionRenderer(InsertPt: M.actions_end(), M, Dst);
1349 if (auto Error = InsertPtOrError.takeError())
1350 return std::move(Error);
1351
1352 action_iterator InsertPt = InsertPtOrError.get();
1353 BuildMIAction &DstMIBuilder = *static_cast<BuildMIAction *>(InsertPt->get());
1354
1355 for (auto PhysInput : InsnMatcher.getPhysRegInputs()) {
1356 InsertPt = M.insertAction<BuildMIAction>(
1357 InsertPt, args: M.allocateOutputInsnID(),
1358 args: &Target.getInstruction(InstRec: RK.getDef(Name: "COPY")));
1359 BuildMIAction &CopyToPhysRegMIBuilder =
1360 *static_cast<BuildMIAction *>(InsertPt->get());
1361 CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>(
1362 args: Target, args&: PhysInput.first, args: true);
1363 CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(args&: PhysInput.first);
1364 }
1365
1366 if (auto Error =
1367 importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Src, Dst)
1368 .takeError())
1369 return std::move(Error);
1370
1371 if (auto Error =
1372 importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst, Src)
1373 .takeError())
1374 return std::move(Error);
1375
1376 return DstMIBuilder;
1377}
1378
1379Expected<action_iterator>
1380GlobalISelEmitter::createAndImportSubInstructionRenderer(
1381 const action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
1382 const TreePatternNode &Src, unsigned TempRegID) {
1383 auto InsertPtOrError = createInstructionRenderer(InsertPt, M, Dst);
1384
1385 // TODO: Assert there's exactly one result.
1386
1387 if (auto Error = InsertPtOrError.takeError())
1388 return std::move(Error);
1389
1390 BuildMIAction &DstMIBuilder =
1391 *static_cast<BuildMIAction *>(InsertPtOrError.get()->get());
1392
1393 // Assign the result to TempReg.
1394 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, args: true);
1395
1396 // Handle additional (ignored) results.
1397 if (DstMIBuilder.getCGI()->Operands.NumDefs > 1) {
1398 InsertPtOrError = importExplicitDefRenderers(
1399 InsertPt: std::prev(x: *InsertPtOrError), M, DstMIBuilder, Src, Dst, /*Start=*/1);
1400 if (auto Error = InsertPtOrError.takeError())
1401 return std::move(Error);
1402 }
1403
1404 InsertPtOrError = importExplicitUseRenderers(InsertPt: InsertPtOrError.get(), M,
1405 DstMIBuilder, Dst, Src);
1406 if (auto Error = InsertPtOrError.takeError())
1407 return std::move(Error);
1408
1409 // We need to make sure that when we import an INSERT_SUBREG as a
1410 // subinstruction that it ends up being constrained to the correct super
1411 // register and subregister classes.
1412 auto OpName = Target.getInstruction(InstRec: Dst.getOperator()).TheDef->getName();
1413 if (OpName == "INSERT_SUBREG") {
1414 auto SubClass = inferRegClassFromPattern(N: Dst.getChild(N: 1));
1415 if (!SubClass)
1416 return failedImport(
1417 Reason: "Cannot infer register class from INSERT_SUBREG operand #1");
1418 std::optional<const CodeGenRegisterClass *> SuperClass =
1419 inferSuperRegisterClassForNode(Ty: Dst.getExtType(ResNo: 0), SuperRegNode: Dst.getChild(N: 0),
1420 SubRegIdxNode: Dst.getChild(N: 2));
1421 if (!SuperClass)
1422 return failedImport(
1423 Reason: "Cannot infer register class for INSERT_SUBREG operand #0");
1424 // The destination and the super register source of an INSERT_SUBREG must
1425 // be the same register class.
1426 M.insertAction<ConstrainOperandToRegClassAction>(
1427 InsertPt, args: DstMIBuilder.getInsnID(), args: 0, args: **SuperClass);
1428 M.insertAction<ConstrainOperandToRegClassAction>(
1429 InsertPt, args: DstMIBuilder.getInsnID(), args: 1, args: **SuperClass);
1430 M.insertAction<ConstrainOperandToRegClassAction>(
1431 InsertPt, args: DstMIBuilder.getInsnID(), args: 2, args: **SubClass);
1432 return InsertPtOrError.get();
1433 }
1434
1435 if (OpName == "EXTRACT_SUBREG") {
1436 // EXTRACT_SUBREG selects into a subregister COPY but unlike most
1437 // instructions, the result register class is controlled by the
1438 // subregisters of the operand. As a result, we must constrain the result
1439 // class rather than check that it's already the right one.
1440 auto SuperClass = inferRegClassFromPattern(N: Dst.getChild(N: 0));
1441 if (!SuperClass)
1442 return failedImport(
1443 Reason: "Cannot infer register class from EXTRACT_SUBREG operand #0");
1444
1445 auto SubIdx = inferSubRegIndexForNode(SubRegIdxNode: Dst.getChild(N: 1));
1446 if (!SubIdx)
1447 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a subreg index");
1448
1449 const auto SrcRCDstRCPair =
1450 (*SuperClass)->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx: *SubIdx);
1451 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
1452 M.insertAction<ConstrainOperandToRegClassAction>(
1453 InsertPt, args: DstMIBuilder.getInsnID(), args: 0, args&: *SrcRCDstRCPair->second);
1454 M.insertAction<ConstrainOperandToRegClassAction>(
1455 InsertPt, args: DstMIBuilder.getInsnID(), args: 1, args&: *SrcRCDstRCPair->first);
1456
1457 // We're done with this pattern! It's eligible for GISel emission; return
1458 // it.
1459 return InsertPtOrError.get();
1460 }
1461
1462 // Similar to INSERT_SUBREG, we also have to handle SUBREG_TO_REG as a
1463 // subinstruction.
1464 if (OpName == "SUBREG_TO_REG") {
1465 auto SubClass = inferRegClassFromPattern(N: Dst.getChild(N: 1));
1466 if (!SubClass)
1467 return failedImport(
1468 Reason: "Cannot infer register class from SUBREG_TO_REG child #1");
1469 auto SuperClass =
1470 inferSuperRegisterClass(Ty: Dst.getExtType(ResNo: 0), SubRegIdxNode: Dst.getChild(N: 2));
1471 if (!SuperClass)
1472 return failedImport(
1473 Reason: "Cannot infer register class for SUBREG_TO_REG operand #0");
1474 M.insertAction<ConstrainOperandToRegClassAction>(
1475 InsertPt, args: DstMIBuilder.getInsnID(), args: 0, args: **SuperClass);
1476 M.insertAction<ConstrainOperandToRegClassAction>(
1477 InsertPt, args: DstMIBuilder.getInsnID(), args: 2, args: **SubClass);
1478 return InsertPtOrError.get();
1479 }
1480
1481 if (OpName == "REG_SEQUENCE") {
1482 auto SuperClass = inferRegClassFromPattern(N: Dst.getChild(N: 0));
1483 M.insertAction<ConstrainOperandToRegClassAction>(
1484 InsertPt, args: DstMIBuilder.getInsnID(), args: 0, args: **SuperClass);
1485
1486 unsigned Num = Dst.getNumChildren();
1487 for (unsigned I = 1; I != Num; I += 2) {
1488 const TreePatternNode &SubRegChild = Dst.getChild(N: I + 1);
1489
1490 auto SubIdx = inferSubRegIndexForNode(SubRegIdxNode: SubRegChild);
1491 if (!SubIdx)
1492 return failedImport(Reason: "REG_SEQUENCE child is not a subreg index");
1493
1494 const auto SrcRCDstRCPair =
1495 (*SuperClass)->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx: *SubIdx);
1496 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
1497 M.insertAction<ConstrainOperandToRegClassAction>(
1498 InsertPt, args: DstMIBuilder.getInsnID(), args&: I, args&: *SrcRCDstRCPair->second);
1499 }
1500
1501 return InsertPtOrError.get();
1502 }
1503
1504 M.insertAction<ConstrainOperandsToDefinitionAction>(InsertPt,
1505 args: DstMIBuilder.getInsnID());
1506 return InsertPtOrError.get();
1507}
1508
1509Expected<action_iterator> GlobalISelEmitter::createInstructionRenderer(
1510 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst) {
1511 Record *DstOp = Dst.getOperator();
1512 if (!DstOp->isSubClassOf(Name: "Instruction")) {
1513 if (DstOp->isSubClassOf(Name: "ValueType"))
1514 return failedImport(
1515 Reason: "Pattern operator isn't an instruction (it's a ValueType)");
1516 return failedImport(Reason: "Pattern operator isn't an instruction");
1517 }
1518 CodeGenInstruction *DstI = &Target.getInstruction(InstRec: DstOp);
1519
1520 // COPY_TO_REGCLASS is just a copy with a ConstrainOperandToRegClassAction
1521 // attached. Similarly for EXTRACT_SUBREG except that's a subregister copy.
1522 StringRef Name = DstI->TheDef->getName();
1523 if (Name == "COPY_TO_REGCLASS" || Name == "EXTRACT_SUBREG")
1524 DstI = &Target.getInstruction(InstRec: RK.getDef(Name: "COPY"));
1525
1526 return M.insertAction<BuildMIAction>(InsertPt, args: M.allocateOutputInsnID(),
1527 args&: DstI);
1528}
1529
1530Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
1531 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
1532 const TreePatternNode &Src, const TreePatternNode &Dst, unsigned Start) {
1533 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
1534 const unsigned SrcNumDefs = Src.getExtTypes().size();
1535 const unsigned DstNumDefs = DstI->Operands.NumDefs;
1536 if (DstNumDefs == 0)
1537 return InsertPt;
1538
1539 for (unsigned I = Start; I < SrcNumDefs; ++I) {
1540 std::string OpName = getMangledRootDefName(DefOperandName: DstI->Operands[I].Name);
1541 // CopyRenderer saves a StringRef, so cannot pass OpName itself -
1542 // let's use a string with an appropriate lifetime.
1543 StringRef PermanentRef = M.getOperandMatcher(Name: OpName).getSymbolicName();
1544 DstMIBuilder.addRenderer<CopyRenderer>(args&: PermanentRef);
1545 }
1546
1547 // Some instructions have multiple defs, but are missing a type entry
1548 // (e.g. s_cc_out operands).
1549 if (Dst.getExtTypes().size() < DstNumDefs)
1550 return failedImport(Reason: "unhandled discarded def");
1551
1552 for (unsigned I = SrcNumDefs; I < DstNumDefs; ++I) {
1553 const TypeSetByHwMode &ExtTy = Dst.getExtType(ResNo: I);
1554 if (!ExtTy.isMachineValueType())
1555 return failedImport(Reason: "unsupported typeset");
1556
1557 auto OpTy = MVTToLLT(SVT: ExtTy.getMachineValueType().SimpleTy);
1558 if (!OpTy)
1559 return failedImport(Reason: "unsupported type");
1560
1561 unsigned TempRegID = M.allocateTempRegID();
1562 InsertPt =
1563 M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTy, args&: TempRegID);
1564 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, args: true, args: nullptr, args: true);
1565 }
1566
1567 return InsertPt;
1568}
1569
1570Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
1571 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
1572 const llvm::TreePatternNode &Dst, const llvm::TreePatternNode &Src) {
1573 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
1574 CodeGenInstruction *OrigDstI = &Target.getInstruction(InstRec: Dst.getOperator());
1575
1576 StringRef Name = OrigDstI->TheDef->getName();
1577 unsigned ExpectedDstINumUses = Dst.getNumChildren();
1578
1579 // EXTRACT_SUBREG needs to use a subregister COPY.
1580 if (Name == "EXTRACT_SUBREG") {
1581 if (!Dst.getChild(N: 1).isLeaf())
1582 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a leaf");
1583 DefInit *SubRegInit = dyn_cast<DefInit>(Val: Dst.getChild(N: 1).getLeafValue());
1584 if (!SubRegInit)
1585 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a subreg index");
1586
1587 CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
1588 const TreePatternNode &ValChild = Dst.getChild(N: 0);
1589 if (!ValChild.isLeaf()) {
1590 // We really have to handle the source instruction, and then insert a
1591 // copy from the subregister.
1592 auto ExtractSrcTy = getInstResultType(Dst: ValChild, Target);
1593 if (!ExtractSrcTy)
1594 return ExtractSrcTy.takeError();
1595
1596 unsigned TempRegID = M.allocateTempRegID();
1597 InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *ExtractSrcTy,
1598 args&: TempRegID);
1599
1600 auto InsertPtOrError = createAndImportSubInstructionRenderer(
1601 InsertPt: ++InsertPt, M, Dst: ValChild, Src, TempRegID);
1602 if (auto Error = InsertPtOrError.takeError())
1603 return std::move(Error);
1604
1605 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, args: false, args&: SubIdx);
1606 return InsertPt;
1607 }
1608
1609 // If this is a source operand, this is just a subregister copy.
1610 Record *RCDef = getInitValueAsRegClass(V: ValChild.getLeafValue());
1611 if (!RCDef)
1612 return failedImport(Reason: "EXTRACT_SUBREG child #0 could not "
1613 "be coerced to a register class");
1614
1615 CodeGenRegisterClass *RC = CGRegs.getRegClass(RCDef);
1616
1617 const auto SrcRCDstRCPair =
1618 RC->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1619 if (SrcRCDstRCPair) {
1620 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
1621 if (SrcRCDstRCPair->first != RC)
1622 return failedImport(Reason: "EXTRACT_SUBREG requires an additional COPY");
1623 }
1624
1625 StringRef RegOperandName = Dst.getChild(N: 0).getName();
1626 if (const auto &SubOperand = M.getComplexSubOperand(SymbolicName: RegOperandName)) {
1627 DstMIBuilder.addRenderer<RenderComplexPatternOperand>(
1628 args&: *std::get<0>(t: *SubOperand), args&: RegOperandName, args: std::get<1>(t: *SubOperand),
1629 args: std::get<2>(t: *SubOperand), args&: SubIdx);
1630 return InsertPt;
1631 }
1632
1633 DstMIBuilder.addRenderer<CopySubRegRenderer>(args&: RegOperandName, args&: SubIdx);
1634 return InsertPt;
1635 }
1636
1637 if (Name == "REG_SEQUENCE") {
1638 if (!Dst.getChild(N: 0).isLeaf())
1639 return failedImport(Reason: "REG_SEQUENCE child #0 is not a leaf");
1640
1641 Record *RCDef = getInitValueAsRegClass(V: Dst.getChild(N: 0).getLeafValue());
1642 if (!RCDef)
1643 return failedImport(Reason: "REG_SEQUENCE child #0 could not "
1644 "be coerced to a register class");
1645
1646 if ((ExpectedDstINumUses - 1) % 2 != 0)
1647 return failedImport(Reason: "Malformed REG_SEQUENCE");
1648
1649 for (unsigned I = 1; I != ExpectedDstINumUses; I += 2) {
1650 const TreePatternNode &ValChild = Dst.getChild(N: I);
1651 const TreePatternNode &SubRegChild = Dst.getChild(N: I + 1);
1652
1653 if (DefInit *SubRegInit = dyn_cast<DefInit>(Val: SubRegChild.getLeafValue())) {
1654 CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
1655
1656 auto InsertPtOrError =
1657 importExplicitUseRenderer(InsertPt, Rule&: M, DstMIBuilder, DstChild: ValChild, Src);
1658 if (auto Error = InsertPtOrError.takeError())
1659 return std::move(Error);
1660 InsertPt = InsertPtOrError.get();
1661 DstMIBuilder.addRenderer<SubRegIndexRenderer>(args&: SubIdx);
1662 }
1663 }
1664
1665 return InsertPt;
1666 }
1667
1668 // Render the explicit uses.
1669 unsigned DstINumUses = OrigDstI->Operands.size() - OrigDstI->Operands.NumDefs;
1670 if (Name == "COPY_TO_REGCLASS") {
1671 DstINumUses--; // Ignore the class constraint.
1672 ExpectedDstINumUses--;
1673 }
1674
1675 // NumResults - This is the number of results produced by the instruction in
1676 // the "outs" list.
1677 unsigned NumResults = OrigDstI->Operands.NumDefs;
1678
1679 // Number of operands we know the output instruction must have. If it is
1680 // variadic, we could have more operands.
1681 unsigned NumFixedOperands = DstI->Operands.size();
1682
1683 // Loop over all of the fixed operands of the instruction pattern, emitting
1684 // code to fill them all in. The node 'N' usually has number children equal to
1685 // the number of input operands of the instruction. However, in cases where
1686 // there are predicate operands for an instruction, we need to fill in the
1687 // 'execute always' values. Match up the node operands to the instruction
1688 // operands to do this.
1689 unsigned Child = 0;
1690
1691 // Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the
1692 // number of operands at the end of the list which have default values.
1693 // Those can come from the pattern if it provides enough arguments, or be
1694 // filled in with the default if the pattern hasn't provided them. But any
1695 // operand with a default value _before_ the last mandatory one will be
1696 // filled in with their defaults unconditionally.
1697 unsigned NonOverridableOperands = NumFixedOperands;
1698 while (NonOverridableOperands > NumResults &&
1699 CGP.operandHasDefault(Op: DstI->Operands[NonOverridableOperands - 1].Rec))
1700 --NonOverridableOperands;
1701
1702 unsigned NumDefaultOps = 0;
1703 for (unsigned I = 0; I != DstINumUses; ++I) {
1704 unsigned InstOpNo = DstI->Operands.NumDefs + I;
1705
1706 // Determine what to emit for this operand.
1707 Record *OperandNode = DstI->Operands[InstOpNo].Rec;
1708
1709 // If the operand has default values, introduce them now.
1710 if (CGP.operandHasDefault(Op: OperandNode) &&
1711 (InstOpNo < NonOverridableOperands || Child >= Dst.getNumChildren())) {
1712 // This is a predicate or optional def operand which the pattern has not
1713 // overridden, or which we aren't letting it override; emit the 'default
1714 // ops' operands.
1715
1716 Record *OperandNode = DstI->Operands[InstOpNo].Rec;
1717 if (auto Error = importDefaultOperandRenderers(
1718 InsertPt, M, DstMIBuilder, DefaultOp: CGP.getDefaultOperand(R: OperandNode)))
1719 return std::move(Error);
1720
1721 ++NumDefaultOps;
1722 continue;
1723 }
1724
1725 auto InsertPtOrError = importExplicitUseRenderer(InsertPt, Rule&: M, DstMIBuilder,
1726 DstChild: Dst.getChild(N: Child), Src);
1727 if (auto Error = InsertPtOrError.takeError())
1728 return std::move(Error);
1729 InsertPt = InsertPtOrError.get();
1730 ++Child;
1731 }
1732
1733 if (NumDefaultOps + ExpectedDstINumUses != DstINumUses)
1734 return failedImport(Reason: "Expected " + llvm::to_string(Value: DstINumUses) +
1735 " used operands but found " +
1736 llvm::to_string(Value: ExpectedDstINumUses) +
1737 " explicit ones and " + llvm::to_string(Value: NumDefaultOps) +
1738 " default ones");
1739
1740 return InsertPt;
1741}
1742
1743Error GlobalISelEmitter::importDefaultOperandRenderers(
1744 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
1745 const DAGDefaultOperand &DefaultOp) const {
1746 for (const auto &Op : DefaultOp.DefaultOps) {
1747 const auto &N = *Op;
1748 if (!N.isLeaf())
1749 return failedImport(Reason: "Could not add default op");
1750
1751 const auto *DefaultOp = N.getLeafValue();
1752
1753 if (const DefInit *DefaultDefOp = dyn_cast<DefInit>(Val: DefaultOp)) {
1754 std::optional<LLTCodeGen> OpTyOrNone = MVTToLLT(SVT: N.getSimpleType(ResNo: 0));
1755 auto *Def = DefaultDefOp->getDef();
1756 if (Def->getName() == "undef_tied_input") {
1757 unsigned TempRegID = M.allocateTempRegID();
1758 M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTyOrNone,
1759 args&: TempRegID);
1760 InsertPt = M.insertAction<BuildMIAction>(
1761 InsertPt, args: M.allocateOutputInsnID(),
1762 args: &Target.getInstruction(InstRec: RK.getDef(Name: "IMPLICIT_DEF")));
1763 BuildMIAction &IDMIBuilder =
1764 *static_cast<BuildMIAction *>(InsertPt->get());
1765 IDMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID);
1766 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID);
1767 } else {
1768 DstMIBuilder.addRenderer<AddRegisterRenderer>(args: Target, args&: Def);
1769 }
1770 continue;
1771 }
1772
1773 if (const IntInit *DefaultIntOp = dyn_cast<IntInit>(Val: DefaultOp)) {
1774 DstMIBuilder.addRenderer<ImmRenderer>(args: DefaultIntOp->getValue());
1775 continue;
1776 }
1777
1778 return failedImport(Reason: "Could not add default op");
1779 }
1780
1781 return Error::success();
1782}
1783
1784Error GlobalISelEmitter::importImplicitDefRenderers(
1785 BuildMIAction &DstMIBuilder,
1786 const std::vector<Record *> &ImplicitDefs) const {
1787 if (!ImplicitDefs.empty())
1788 return failedImport(Reason: "Pattern defines a physical register");
1789 return Error::success();
1790}
1791
1792std::optional<const CodeGenRegisterClass *>
1793GlobalISelEmitter::getRegClassFromLeaf(const TreePatternNode &Leaf) {
1794 assert(Leaf.isLeaf() && "Expected leaf?");
1795 Record *RCRec = getInitValueAsRegClass(V: Leaf.getLeafValue());
1796 if (!RCRec)
1797 return std::nullopt;
1798 CodeGenRegisterClass *RC = CGRegs.getRegClass(RCRec);
1799 if (!RC)
1800 return std::nullopt;
1801 return RC;
1802}
1803
1804std::optional<const CodeGenRegisterClass *>
1805GlobalISelEmitter::inferRegClassFromPattern(const TreePatternNode &N) {
1806 if (N.isLeaf())
1807 return getRegClassFromLeaf(Leaf: N);
1808
1809 // We don't have a leaf node, so we have to try and infer something. Check
1810 // that we have an instruction that we can infer something from.
1811
1812 // Only handle things that produce at least one value (if multiple values,
1813 // just take the first one).
1814 if (N.getNumTypes() < 1)
1815 return std::nullopt;
1816 Record *OpRec = N.getOperator();
1817
1818 // We only want instructions.
1819 if (!OpRec->isSubClassOf(Name: "Instruction"))
1820 return std::nullopt;
1821
1822 // Don't want to try and infer things when there could potentially be more
1823 // than one candidate register class.
1824 auto &Inst = Target.getInstruction(InstRec: OpRec);
1825
1826 // Handle any special-case instructions which we can safely infer register
1827 // classes from.
1828 StringRef InstName = Inst.TheDef->getName();
1829 bool IsRegSequence = InstName == "REG_SEQUENCE";
1830 if (IsRegSequence || InstName == "COPY_TO_REGCLASS") {
1831 // If we have a COPY_TO_REGCLASS, then we need to handle it specially. It
1832 // has the desired register class as the first child.
1833 const TreePatternNode &RCChild = N.getChild(N: IsRegSequence ? 0 : 1);
1834 if (!RCChild.isLeaf())
1835 return std::nullopt;
1836 return getRegClassFromLeaf(Leaf: RCChild);
1837 }
1838 if (InstName == "INSERT_SUBREG") {
1839 const TreePatternNode &Child0 = N.getChild(N: 0);
1840 assert(Child0.getNumTypes() == 1 && "Unexpected number of types!");
1841 const TypeSetByHwMode &VTy = Child0.getExtType(ResNo: 0);
1842 return inferSuperRegisterClassForNode(Ty: VTy, SuperRegNode: Child0, SubRegIdxNode: N.getChild(N: 2));
1843 }
1844 if (InstName == "EXTRACT_SUBREG") {
1845 assert(N.getNumTypes() == 1 && "Unexpected number of types!");
1846 const TypeSetByHwMode &VTy = N.getExtType(ResNo: 0);
1847 return inferSuperRegisterClass(Ty: VTy, SubRegIdxNode: N.getChild(N: 1));
1848 }
1849
1850 // Handle destination record types that we can safely infer a register class
1851 // from.
1852 const auto &DstIOperand = Inst.Operands[0];
1853 Record *DstIOpRec = DstIOperand.Rec;
1854 if (DstIOpRec->isSubClassOf(Name: "RegisterOperand")) {
1855 DstIOpRec = DstIOpRec->getValueAsDef(FieldName: "RegClass");
1856 const CodeGenRegisterClass &RC = Target.getRegisterClass(R: DstIOpRec);
1857 return &RC;
1858 }
1859
1860 if (DstIOpRec->isSubClassOf(Name: "RegisterClass")) {
1861 const CodeGenRegisterClass &RC = Target.getRegisterClass(R: DstIOpRec);
1862 return &RC;
1863 }
1864
1865 return std::nullopt;
1866}
1867
1868std::optional<const CodeGenRegisterClass *>
1869GlobalISelEmitter::inferSuperRegisterClass(
1870 const TypeSetByHwMode &Ty, const TreePatternNode &SubRegIdxNode) {
1871 // We need a ValueTypeByHwMode for getSuperRegForSubReg.
1872 if (!Ty.isValueTypeByHwMode(AllowEmpty: false))
1873 return std::nullopt;
1874 if (!SubRegIdxNode.isLeaf())
1875 return std::nullopt;
1876 DefInit *SubRegInit = dyn_cast<DefInit>(Val: SubRegIdxNode.getLeafValue());
1877 if (!SubRegInit)
1878 return std::nullopt;
1879 CodeGenSubRegIndex *SubIdx = CGRegs.getSubRegIdx(SubRegInit->getDef());
1880
1881 // Use the information we found above to find a minimal register class which
1882 // supports the subregister and type we want.
1883 auto RC =
1884 Target.getSuperRegForSubReg(Ty: Ty.getValueTypeByHwMode(), RegBank&: CGRegs, SubIdx,
1885 /* MustBeAllocatable */ true);
1886 if (!RC)
1887 return std::nullopt;
1888 return *RC;
1889}
1890
1891std::optional<const CodeGenRegisterClass *>
1892GlobalISelEmitter::inferSuperRegisterClassForNode(
1893 const TypeSetByHwMode &Ty, const TreePatternNode &SuperRegNode,
1894 const TreePatternNode &SubRegIdxNode) {
1895 // Check if we already have a defined register class for the super register
1896 // node. If we do, then we should preserve that rather than inferring anything
1897 // from the subregister index node. We can assume that whoever wrote the
1898 // pattern in the first place made sure that the super register and
1899 // subregister are compatible.
1900 if (std::optional<const CodeGenRegisterClass *> SuperRegisterClass =
1901 inferRegClassFromPattern(N: SuperRegNode))
1902 return *SuperRegisterClass;
1903 return inferSuperRegisterClass(Ty, SubRegIdxNode);
1904}
1905
1906std::optional<CodeGenSubRegIndex *> GlobalISelEmitter::inferSubRegIndexForNode(
1907 const TreePatternNode &SubRegIdxNode) {
1908 if (!SubRegIdxNode.isLeaf())
1909 return std::nullopt;
1910
1911 DefInit *SubRegInit = dyn_cast<DefInit>(Val: SubRegIdxNode.getLeafValue());
1912 if (!SubRegInit)
1913 return std::nullopt;
1914 return CGRegs.getSubRegIdx(SubRegInit->getDef());
1915}
1916
1917Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
1918 // Keep track of the matchers and actions to emit.
1919 int Score = P.getPatternComplexity(CGP);
1920 RuleMatcher M(P.getSrcRecord()->getLoc());
1921 RuleMatcherScores[M.getRuleID()] = Score;
1922 M.addAction<DebugCommentAction>(args: llvm::to_string(Value: P.getSrcPattern()) +
1923 " => " +
1924 llvm::to_string(Value: P.getDstPattern()));
1925
1926 SmallVector<Record *, 4> Predicates;
1927 P.getPredicateRecords(PredicateRecs&: Predicates);
1928 if (auto Error = importRulePredicates(M, Predicates))
1929 return std::move(Error);
1930
1931 if (!P.getHwModeFeatures().empty())
1932 M.addHwModeIdx(Idx: declareHwModeCheck(HwModeFeatures: P.getHwModeFeatures()));
1933
1934 // Next, analyze the pattern operators.
1935 TreePatternNode &Src = P.getSrcPattern();
1936 TreePatternNode &Dst = P.getDstPattern();
1937
1938 // If the root of either pattern isn't a simple operator, ignore it.
1939 if (auto Err = isTrivialOperatorNode(N: Dst))
1940 return failedImport(Reason: "Dst pattern root isn't a trivial operator (" +
1941 toString(E: std::move(Err)) + ")");
1942 if (auto Err = isTrivialOperatorNode(N: Src))
1943 return failedImport(Reason: "Src pattern root isn't a trivial operator (" +
1944 toString(E: std::move(Err)) + ")");
1945
1946 // The different predicates and matchers created during
1947 // addInstructionMatcher use the RuleMatcher M to set up their
1948 // instruction ID (InsnVarID) that are going to be used when
1949 // M is going to be emitted.
1950 // However, the code doing the emission still relies on the IDs
1951 // returned during that process by the RuleMatcher when issuing
1952 // the recordInsn opcodes.
1953 // Because of that:
1954 // 1. The order in which we created the predicates
1955 // and such must be the same as the order in which we emit them,
1956 // and
1957 // 2. We need to reset the generation of the IDs in M somewhere between
1958 // addInstructionMatcher and emit
1959 //
1960 // FIXME: Long term, we don't want to have to rely on this implicit
1961 // naming being the same. One possible solution would be to have
1962 // explicit operator for operation capture and reference those.
1963 // The plus side is that it would expose opportunities to share
1964 // the capture accross rules. The downside is that it would
1965 // introduce a dependency between predicates (captures must happen
1966 // before their first use.)
1967 InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(SymbolicName: Src.getName());
1968 unsigned TempOpIdx = 0;
1969
1970 const auto SavedFlags = M.setGISelFlags(P.getSrcRecord());
1971
1972 auto InsnMatcherOrError =
1973 createAndImportSelDAGMatcher(Rule&: M, InsnMatcher&: InsnMatcherTemp, Src, TempOpIdx);
1974 if (auto Error = InsnMatcherOrError.takeError())
1975 return std::move(Error);
1976 InstructionMatcher &InsnMatcher = InsnMatcherOrError.get();
1977
1978 if (Dst.isLeaf()) {
1979 Record *RCDef = getInitValueAsRegClass(V: Dst.getLeafValue());
1980 if (RCDef) {
1981 const CodeGenRegisterClass &RC = Target.getRegisterClass(R: RCDef);
1982
1983 // We need to replace the def and all its uses with the specified
1984 // operand. However, we must also insert COPY's wherever needed.
1985 // For now, emit a copy and let the register allocator clean up.
1986 auto &DstI = Target.getInstruction(InstRec: RK.getDef(Name: "COPY"));
1987 const auto &DstIOperand = DstI.Operands[0];
1988
1989 OperandMatcher &OM0 = InsnMatcher.getOperand(OpIdx: 0);
1990 OM0.setSymbolicName(DstIOperand.Name);
1991 M.defineOperand(SymbolicName: OM0.getSymbolicName(), OM&: OM0);
1992 OM0.addPredicate<RegisterBankOperandMatcher>(args: RC);
1993
1994 auto &DstMIBuilder =
1995 M.addAction<BuildMIAction>(args: M.allocateOutputInsnID(), args: &DstI);
1996 DstMIBuilder.addRenderer<CopyRenderer>(args: DstIOperand.Name);
1997 DstMIBuilder.addRenderer<CopyRenderer>(args: Dst.getName());
1998 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 0, args: RC);
1999
2000 // Erase the root.
2001 unsigned RootInsnID = M.getInsnVarID(InsnMatcher);
2002 M.addAction<EraseInstAction>(args&: RootInsnID);
2003
2004 // We're done with this pattern! It's eligible for GISel emission; return
2005 // it.
2006 ++NumPatternImported;
2007 return std::move(M);
2008 }
2009
2010 return failedImport(Reason: "Dst pattern root isn't a known leaf");
2011 }
2012
2013 // Start with the defined operands (i.e., the results of the root operator).
2014 Record *DstOp = Dst.getOperator();
2015 if (!DstOp->isSubClassOf(Name: "Instruction"))
2016 return failedImport(Reason: "Pattern operator isn't an instruction");
2017
2018 auto &DstI = Target.getInstruction(InstRec: DstOp);
2019 StringRef DstIName = DstI.TheDef->getName();
2020
2021 unsigned DstNumDefs = DstI.Operands.NumDefs,
2022 SrcNumDefs = Src.getExtTypes().size();
2023 if (DstNumDefs < SrcNumDefs) {
2024 if (DstNumDefs != 0)
2025 return failedImport(Reason: "Src pattern result has more defs than dst MI (" +
2026 to_string(Value: SrcNumDefs) + " def(s) vs " +
2027 to_string(Value: DstNumDefs) + " def(s))");
2028
2029 bool FoundNoUsePred = false;
2030 for (const auto &Pred : InsnMatcher.predicates()) {
2031 if ((FoundNoUsePred = isa<NoUsePredicateMatcher>(Val: Pred.get())))
2032 break;
2033 }
2034 if (!FoundNoUsePred)
2035 return failedImport(Reason: "Src pattern result has " + to_string(Value: SrcNumDefs) +
2036 " def(s) without the HasNoUse predicate set to true "
2037 "but Dst MI has no def");
2038 }
2039
2040 // The root of the match also has constraints on the register bank so that it
2041 // matches the result instruction.
2042 unsigned OpIdx = 0;
2043 unsigned N = std::min(a: DstNumDefs, b: SrcNumDefs);
2044 for (unsigned I = 0; I < N; ++I) {
2045 const TypeSetByHwMode &VTy = Src.getExtType(ResNo: I);
2046
2047 const auto &DstIOperand = DstI.Operands[OpIdx];
2048 PointerUnion<Record *, const CodeGenRegisterClass *> MatchedRC =
2049 DstIOperand.Rec;
2050 if (DstIName == "COPY_TO_REGCLASS") {
2051 MatchedRC = getInitValueAsRegClass(V: Dst.getChild(N: 1).getLeafValue());
2052
2053 if (MatchedRC.isNull())
2054 return failedImport(
2055 Reason: "COPY_TO_REGCLASS operand #1 isn't a register class");
2056 } else if (DstIName == "REG_SEQUENCE") {
2057 MatchedRC = getInitValueAsRegClass(V: Dst.getChild(N: 0).getLeafValue());
2058 if (MatchedRC.isNull())
2059 return failedImport(Reason: "REG_SEQUENCE operand #0 isn't a register class");
2060 } else if (DstIName == "EXTRACT_SUBREG") {
2061 auto InferredClass = inferRegClassFromPattern(N: Dst.getChild(N: 0));
2062 if (!InferredClass)
2063 return failedImport(
2064 Reason: "Could not infer class for EXTRACT_SUBREG operand #0");
2065
2066 // We can assume that a subregister is in the same bank as it's super
2067 // register.
2068 MatchedRC = (*InferredClass)->getDef();
2069 } else if (DstIName == "INSERT_SUBREG") {
2070 auto MaybeSuperClass =
2071 inferSuperRegisterClassForNode(Ty: VTy, SuperRegNode: Dst.getChild(N: 0), SubRegIdxNode: Dst.getChild(N: 2));
2072 if (!MaybeSuperClass)
2073 return failedImport(
2074 Reason: "Cannot infer register class for INSERT_SUBREG operand #0");
2075 // Move to the next pattern here, because the register class we found
2076 // doesn't necessarily have a record associated with it. So, we can't
2077 // set DstIOpRec using this.
2078 MatchedRC = *MaybeSuperClass;
2079 } else if (DstIName == "SUBREG_TO_REG") {
2080 auto MaybeRegClass = inferSuperRegisterClass(Ty: VTy, SubRegIdxNode: Dst.getChild(N: 2));
2081 if (!MaybeRegClass)
2082 return failedImport(
2083 Reason: "Cannot infer register class for SUBREG_TO_REG operand #0");
2084 MatchedRC = *MaybeRegClass;
2085 } else if (MatchedRC.get<Record *>()->isSubClassOf(Name: "RegisterOperand"))
2086 MatchedRC = MatchedRC.get<Record *>()->getValueAsDef(FieldName: "RegClass");
2087 else if (!MatchedRC.get<Record *>()->isSubClassOf(Name: "RegisterClass"))
2088 return failedImport(Reason: "Dst MI def isn't a register class" + to_string(Value: Dst));
2089
2090 OperandMatcher &OM = InsnMatcher.getOperand(OpIdx);
2091 // The operand names declared in the DstI instruction are unrelated to
2092 // those used in pattern's source and destination DAGs, so mangle the
2093 // former to prevent implicitly adding unexpected
2094 // GIM_CheckIsSameOperand predicates by the defineOperand method.
2095 OM.setSymbolicName(getMangledRootDefName(DefOperandName: DstIOperand.Name));
2096 M.defineOperand(SymbolicName: OM.getSymbolicName(), OM);
2097 if (MatchedRC.is<Record *>())
2098 MatchedRC = &Target.getRegisterClass(R: MatchedRC.get<Record *>());
2099 OM.addPredicate<RegisterBankOperandMatcher>(
2100 args: *MatchedRC.get<const CodeGenRegisterClass *>());
2101 ++OpIdx;
2102 }
2103
2104 auto DstMIBuilderOrError =
2105 createAndImportInstructionRenderer(M, InsnMatcher, Src, Dst);
2106 if (auto Error = DstMIBuilderOrError.takeError())
2107 return std::move(Error);
2108 BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get();
2109
2110 // Render the implicit defs.
2111 // These are only added to the root of the result.
2112 if (auto Error = importImplicitDefRenderers(DstMIBuilder, ImplicitDefs: P.getDstRegs()))
2113 return std::move(Error);
2114
2115 DstMIBuilder.chooseInsnToMutate(Rule&: M);
2116
2117 // Constrain the registers to classes. This is normally derived from the
2118 // emitted instruction but a few instructions require special handling.
2119 if (DstIName == "COPY_TO_REGCLASS") {
2120 // COPY_TO_REGCLASS does not provide operand constraints itself but the
2121 // result is constrained to the class given by the second child.
2122 Record *DstIOpRec = getInitValueAsRegClass(V: Dst.getChild(N: 1).getLeafValue());
2123
2124 if (DstIOpRec == nullptr)
2125 return failedImport(Reason: "COPY_TO_REGCLASS operand #1 isn't a register class");
2126
2127 M.addAction<ConstrainOperandToRegClassAction>(
2128 args: 0, args: 0, args: Target.getRegisterClass(R: DstIOpRec));
2129 } else if (DstIName == "EXTRACT_SUBREG") {
2130 auto SuperClass = inferRegClassFromPattern(N: Dst.getChild(N: 0));
2131 if (!SuperClass)
2132 return failedImport(
2133 Reason: "Cannot infer register class from EXTRACT_SUBREG operand #0");
2134
2135 auto SubIdx = inferSubRegIndexForNode(SubRegIdxNode: Dst.getChild(N: 1));
2136 if (!SubIdx)
2137 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a subreg index");
2138
2139 // It would be nice to leave this constraint implicit but we're required
2140 // to pick a register class so constrain the result to a register class
2141 // that can hold the correct MVT.
2142 //
2143 // FIXME: This may introduce an extra copy if the chosen class doesn't
2144 // actually contain the subregisters.
2145 assert(Src.getExtTypes().size() == 1 &&
2146 "Expected Src of EXTRACT_SUBREG to have one result type");
2147
2148 const auto SrcRCDstRCPair =
2149 (*SuperClass)->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx: *SubIdx);
2150 if (!SrcRCDstRCPair) {
2151 return failedImport(Reason: "subreg index is incompatible "
2152 "with inferred reg class");
2153 }
2154
2155 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
2156 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 0,
2157 args&: *SrcRCDstRCPair->second);
2158 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 1, args&: *SrcRCDstRCPair->first);
2159 } else if (DstIName == "INSERT_SUBREG") {
2160 assert(Src.getExtTypes().size() == 1 &&
2161 "Expected Src of INSERT_SUBREG to have one result type");
2162 // We need to constrain the destination, a super regsister source, and a
2163 // subregister source.
2164 auto SubClass = inferRegClassFromPattern(N: Dst.getChild(N: 1));
2165 if (!SubClass)
2166 return failedImport(
2167 Reason: "Cannot infer register class from INSERT_SUBREG operand #1");
2168 auto SuperClass = inferSuperRegisterClassForNode(
2169 Ty: Src.getExtType(ResNo: 0), SuperRegNode: Dst.getChild(N: 0), SubRegIdxNode: Dst.getChild(N: 2));
2170 if (!SuperClass)
2171 return failedImport(
2172 Reason: "Cannot infer register class for INSERT_SUBREG operand #0");
2173 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 0, args: **SuperClass);
2174 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 1, args: **SuperClass);
2175 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 2, args: **SubClass);
2176 } else if (DstIName == "SUBREG_TO_REG") {
2177 // We need to constrain the destination and subregister source.
2178 assert(Src.getExtTypes().size() == 1 &&
2179 "Expected Src of SUBREG_TO_REG to have one result type");
2180
2181 // Attempt to infer the subregister source from the first child. If it has
2182 // an explicitly given register class, we'll use that. Otherwise, we will
2183 // fail.
2184 auto SubClass = inferRegClassFromPattern(N: Dst.getChild(N: 1));
2185 if (!SubClass)
2186 return failedImport(
2187 Reason: "Cannot infer register class from SUBREG_TO_REG child #1");
2188 // We don't have a child to look at that might have a super register node.
2189 auto SuperClass =
2190 inferSuperRegisterClass(Ty: Src.getExtType(ResNo: 0), SubRegIdxNode: Dst.getChild(N: 2));
2191 if (!SuperClass)
2192 return failedImport(
2193 Reason: "Cannot infer register class for SUBREG_TO_REG operand #0");
2194 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 0, args: **SuperClass);
2195 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 2, args: **SubClass);
2196 } else if (DstIName == "REG_SEQUENCE") {
2197 auto SuperClass = inferRegClassFromPattern(N: Dst.getChild(N: 0));
2198
2199 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 0, args: **SuperClass);
2200
2201 unsigned Num = Dst.getNumChildren();
2202 for (unsigned I = 1; I != Num; I += 2) {
2203 TreePatternNode &SubRegChild = Dst.getChild(N: I + 1);
2204
2205 auto SubIdx = inferSubRegIndexForNode(SubRegIdxNode: SubRegChild);
2206 if (!SubIdx)
2207 return failedImport(Reason: "REG_SEQUENCE child is not a subreg index");
2208
2209 const auto SrcRCDstRCPair =
2210 (*SuperClass)->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx: *SubIdx);
2211
2212 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args&: I,
2213 args&: *SrcRCDstRCPair->second);
2214 }
2215 } else {
2216 M.addAction<ConstrainOperandsToDefinitionAction>(args: 0);
2217 }
2218
2219 // Erase the root.
2220 unsigned RootInsnID = M.getInsnVarID(InsnMatcher);
2221 M.addAction<EraseInstAction>(args&: RootInsnID);
2222
2223 // We're done with this pattern! It's eligible for GISel emission; return it.
2224 ++NumPatternImported;
2225 return std::move(M);
2226}
2227
2228MatchTable
2229GlobalISelEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules,
2230 bool Optimize, bool WithCoverage) {
2231 std::vector<Matcher *> InputRules;
2232 for (Matcher &Rule : Rules)
2233 InputRules.push_back(x: &Rule);
2234
2235 if (!Optimize)
2236 return MatchTable::buildTable(Rules: InputRules, WithCoverage);
2237
2238 unsigned CurrentOrdering = 0;
2239 StringMap<unsigned> OpcodeOrder;
2240 for (RuleMatcher &Rule : Rules) {
2241 const StringRef Opcode = Rule.getOpcode();
2242 assert(!Opcode.empty() && "Didn't expect an undefined opcode");
2243 if (OpcodeOrder.count(Key: Opcode) == 0)
2244 OpcodeOrder[Opcode] = CurrentOrdering++;
2245 }
2246
2247 llvm::stable_sort(Range&: InputRules, C: [&OpcodeOrder](const Matcher *A,
2248 const Matcher *B) {
2249 auto *L = static_cast<const RuleMatcher *>(A);
2250 auto *R = static_cast<const RuleMatcher *>(B);
2251 return std::tuple(OpcodeOrder[L->getOpcode()], L->getNumOperands()) <
2252 std::tuple(OpcodeOrder[R->getOpcode()], R->getNumOperands());
2253 });
2254
2255 for (Matcher *Rule : InputRules)
2256 Rule->optimize();
2257
2258 std::vector<std::unique_ptr<Matcher>> MatcherStorage;
2259 std::vector<Matcher *> OptRules =
2260 optimizeRules<GroupMatcher>(Rules: InputRules, MatcherStorage);
2261
2262 for (Matcher *Rule : OptRules)
2263 Rule->optimize();
2264
2265 OptRules = optimizeRules<SwitchMatcher>(Rules: OptRules, MatcherStorage);
2266
2267 return MatchTable::buildTable(Rules: OptRules, WithCoverage);
2268}
2269
2270void GlobalISelEmitter::emitAdditionalImpl(raw_ostream &OS) {
2271 OS << "bool " << getClassName()
2272 << "::selectImpl(MachineInstr &I, CodeGenCoverage "
2273 "&CoverageInfo) const {\n"
2274 << " const PredicateBitset AvailableFeatures = "
2275 "getAvailableFeatures();\n"
2276 << " MachineIRBuilder B(I);\n"
2277 << " State.MIs.clear();\n"
2278 << " State.MIs.push_back(&I);\n\n"
2279 << " if (executeMatchTable(*this, State, ExecInfo, B"
2280 << ", getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures"
2281 << ", &CoverageInfo)) {\n"
2282 << " return true;\n"
2283 << " }\n\n"
2284 << " return false;\n"
2285 << "}\n\n";
2286}
2287
2288void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) {
2289 std::vector<Record *> MatchedRecords;
2290 std::copy_if(first: AllPatFrags.begin(), last: AllPatFrags.end(),
2291 result: std::back_inserter(x&: MatchedRecords), pred: [&](Record *R) {
2292 return !R->getValueAsString(FieldName: "GISelPredicateCode").empty();
2293 });
2294 emitMIPredicateFnsImpl<Record *>(
2295 OS,
2296 AdditionalDecls: " const MachineFunction &MF = *MI.getParent()->getParent();\n"
2297 " const MachineRegisterInfo &MRI = MF.getRegInfo();\n"
2298 " const auto &Operands = State.RecordedOperands;\n"
2299 " (void)Operands;\n"
2300 " (void)MRI;",
2301 Predicates: ArrayRef<Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2302 GetPredCode: [&](Record *R) { return R->getValueAsString(FieldName: "GISelPredicateCode"); },
2303 Comment: "PatFrag predicates.");
2304}
2305
2306void GlobalISelEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
2307 std::vector<Record *> MatchedRecords;
2308 std::copy_if(first: AllPatFrags.begin(), last: AllPatFrags.end(),
2309 result: std::back_inserter(x&: MatchedRecords), pred: [&](Record *R) {
2310 bool Unset;
2311 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2312 !R->getValueAsBitOrUnset(FieldName: "IsAPFloat", Unset) &&
2313 !R->getValueAsBit(FieldName: "IsAPInt");
2314 });
2315 emitImmPredicateFnsImpl<Record *>(
2316 OS, TypeIdentifier: "I64", ArgType: "int64_t", Predicates: ArrayRef<Record *>(MatchedRecords),
2317 GetPredEnumName: &getPatFragPredicateEnumName,
2318 GetPredCode: [&](Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2319 Comment: "PatFrag predicates.");
2320}
2321
2322void GlobalISelEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) {
2323 std::vector<Record *> MatchedRecords;
2324 std::copy_if(first: AllPatFrags.begin(), last: AllPatFrags.end(),
2325 result: std::back_inserter(x&: MatchedRecords), pred: [&](Record *R) {
2326 bool Unset;
2327 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2328 R->getValueAsBitOrUnset(FieldName: "IsAPFloat", Unset);
2329 });
2330 emitImmPredicateFnsImpl<Record *>(
2331 OS, TypeIdentifier: "APFloat", ArgType: "const APFloat &", Predicates: ArrayRef<Record *>(MatchedRecords),
2332 GetPredEnumName: &getPatFragPredicateEnumName,
2333 GetPredCode: [&](Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2334 Comment: "PatFrag predicates.");
2335}
2336
2337void GlobalISelEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) {
2338 std::vector<Record *> MatchedRecords;
2339 std::copy_if(first: AllPatFrags.begin(), last: AllPatFrags.end(),
2340 result: std::back_inserter(x&: MatchedRecords), pred: [&](Record *R) {
2341 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2342 R->getValueAsBit(FieldName: "IsAPInt");
2343 });
2344 emitImmPredicateFnsImpl<Record *>(
2345 OS, TypeIdentifier: "APInt", ArgType: "const APInt &", Predicates: ArrayRef<Record *>(MatchedRecords),
2346 GetPredEnumName: &getPatFragPredicateEnumName,
2347 GetPredCode: [&](Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2348 Comment: "PatFrag predicates.");
2349}
2350
2351void GlobalISelEmitter::emitTestSimplePredicate(raw_ostream &OS) {
2352 OS << "bool " << getClassName() << "::testSimplePredicate(unsigned) const {\n"
2353 << " llvm_unreachable(\"" + getClassName() +
2354 " does not support simple predicates!\");\n"
2355 << " return false;\n"
2356 << "}\n";
2357}
2358
2359void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) {
2360 OS << "bool " << getClassName()
2361 << "::runCustomAction(unsigned, const MatcherState&, NewMIVector &) const "
2362 "{\n"
2363 << " llvm_unreachable(\"" + getClassName() +
2364 " does not support custom C++ actions!\");\n"
2365 << "}\n";
2366}
2367
2368void GlobalISelEmitter::postProcessRule(RuleMatcher &M) {
2369 SmallPtrSet<Record *, 16> UsedRegs;
2370
2371 // TODO: deal with subregs?
2372 for (auto &A : M.actions()) {
2373 auto *MI = dyn_cast<BuildMIAction>(Val: A.get());
2374 if (!MI)
2375 continue;
2376
2377 for (auto *Use : MI->getCGI()->ImplicitUses)
2378 UsedRegs.insert(Ptr: Use);
2379 }
2380
2381 for (auto &A : M.actions()) {
2382 auto *MI = dyn_cast<BuildMIAction>(Val: A.get());
2383 if (!MI)
2384 continue;
2385
2386 for (auto *Def : MI->getCGI()->ImplicitDefs) {
2387 if (!UsedRegs.contains(Ptr: Def))
2388 MI->setDeadImplicitDef(Def);
2389 }
2390 }
2391}
2392
2393void GlobalISelEmitter::run(raw_ostream &OS) {
2394 if (!UseCoverageFile.empty()) {
2395 RuleCoverage = CodeGenCoverage();
2396 auto RuleCoverageBufOrErr = MemoryBuffer::getFile(Filename: UseCoverageFile);
2397 if (!RuleCoverageBufOrErr) {
2398 PrintWarning(WarningLoc: SMLoc(), Msg: "Missing rule coverage data");
2399 RuleCoverage = std::nullopt;
2400 } else {
2401 if (!RuleCoverage->parse(Buffer&: *RuleCoverageBufOrErr.get(), BackendName: Target.getName())) {
2402 PrintWarning(WarningLoc: SMLoc(), Msg: "Ignoring invalid or missing rule coverage data");
2403 RuleCoverage = std::nullopt;
2404 }
2405 }
2406 }
2407
2408 // Track the run-time opcode values
2409 gatherOpcodeValues();
2410 // Track the run-time LLT ID values
2411 gatherTypeIDValues();
2412
2413 // Track the GINodeEquiv definitions.
2414 gatherNodeEquivs();
2415
2416 AllPatFrags = RK.getAllDerivedDefinitions(ClassName: "PatFrags");
2417
2418 emitSourceFileHeader(
2419 Desc: ("Global Instruction Selector for the " + Target.getName() + " target")
2420 .str(),
2421 OS);
2422 std::vector<RuleMatcher> Rules;
2423 // Look through the SelectionDAG patterns we found, possibly emitting some.
2424 for (const PatternToMatch &Pat : CGP.ptms()) {
2425 ++NumPatternTotal;
2426
2427 if (Pat.getGISelShouldIgnore())
2428 continue; // skip without warning
2429 auto MatcherOrErr = runOnPattern(P: Pat);
2430
2431 // The pattern analysis can fail, indicating an unsupported pattern.
2432 // Report that if we've been asked to do so.
2433 if (auto Err = MatcherOrErr.takeError()) {
2434 if (WarnOnSkippedPatterns) {
2435 PrintWarning(WarningLoc: Pat.getSrcRecord()->getLoc(),
2436 Msg: "Skipped pattern: " + toString(E: std::move(Err)));
2437 } else {
2438 consumeError(Err: std::move(Err));
2439 }
2440 ++NumPatternImportsSkipped;
2441 continue;
2442 }
2443
2444 if (RuleCoverage) {
2445 if (RuleCoverage->isCovered(RuleID: MatcherOrErr->getRuleID()))
2446 ++NumPatternsTested;
2447 else
2448 PrintWarning(WarningLoc: Pat.getSrcRecord()->getLoc(),
2449 Msg: "Pattern is not covered by a test");
2450 }
2451 Rules.push_back(x: std::move(MatcherOrErr.get()));
2452 postProcessRule(M&: Rules.back());
2453 }
2454
2455 // Comparison function to order records by name.
2456 auto OrderByName = [](const Record *A, const Record *B) {
2457 return A->getName() < B->getName();
2458 };
2459
2460 std::vector<Record *> ComplexPredicates =
2461 RK.getAllDerivedDefinitions(ClassName: "GIComplexOperandMatcher");
2462 llvm::sort(C&: ComplexPredicates, Comp: OrderByName);
2463
2464 std::vector<StringRef> CustomRendererFns;
2465 transform(Range: RK.getAllDerivedDefinitions(ClassName: "GICustomOperandRenderer"),
2466 d_first: std::back_inserter(x&: CustomRendererFns), F: [](const auto &Record) {
2467 return Record->getValueAsString("RendererFn");
2468 });
2469 // Sort and remove duplicates to get a list of unique renderer functions, in
2470 // case some were mentioned more than once.
2471 llvm::sort(C&: CustomRendererFns);
2472 CustomRendererFns.erase(first: llvm::unique(R&: CustomRendererFns),
2473 last: CustomRendererFns.end());
2474
2475 // Create a table containing the LLT objects needed by the matcher and an enum
2476 // for the matcher to reference them with.
2477 std::vector<LLTCodeGen> TypeObjects;
2478 append_range(C&: TypeObjects, R&: KnownTypes);
2479 llvm::sort(C&: TypeObjects);
2480
2481 // Sort rules.
2482 llvm::stable_sort(Range&: Rules, C: [&](const RuleMatcher &A, const RuleMatcher &B) {
2483 int ScoreA = RuleMatcherScores[A.getRuleID()];
2484 int ScoreB = RuleMatcherScores[B.getRuleID()];
2485 if (ScoreA > ScoreB)
2486 return true;
2487 if (ScoreB > ScoreA)
2488 return false;
2489 if (A.isHigherPriorityThan(B)) {
2490 assert(!B.isHigherPriorityThan(A) && "Cannot be more important "
2491 "and less important at "
2492 "the same time");
2493 return true;
2494 }
2495 return false;
2496 });
2497
2498 unsigned MaxTemporaries = 0;
2499 for (const auto &Rule : Rules)
2500 MaxTemporaries = std::max(a: MaxTemporaries, b: Rule.countRendererFns());
2501
2502 // Build match table
2503 const MatchTable Table =
2504 buildMatchTable(Rules, Optimize: OptimizeMatchTable, WithCoverage: GenerateCoverage);
2505
2506 emitPredicateBitset(OS, IfDefName: "GET_GLOBALISEL_PREDICATE_BITSET");
2507 emitTemporariesDecl(OS, IfDefName: "GET_GLOBALISEL_TEMPORARIES_DECL");
2508 emitTemporariesInit(OS, MaxTemporaries, IfDefName: "GET_GLOBALISEL_TEMPORARIES_INIT");
2509 emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexOperandMatchers: ComplexPredicates,
2510 CustomOperandRenderers: CustomRendererFns, IfDefName: "GET_GLOBALISEL_IMPL");
2511 emitPredicatesDecl(OS, IfDefName: "GET_GLOBALISEL_PREDICATES_DECL");
2512 emitPredicatesInit(OS, IfDefName: "GET_GLOBALISEL_PREDICATES_INIT");
2513}
2514
2515void GlobalISelEmitter::declareSubtargetFeature(Record *Predicate) {
2516 SubtargetFeatures.try_emplace(k: Predicate, args&: Predicate, args: SubtargetFeatures.size());
2517}
2518
2519unsigned GlobalISelEmitter::declareHwModeCheck(StringRef HwModeFeatures) {
2520 return HwModes.emplace(args: HwModeFeatures.str(), args: HwModes.size()).first->second;
2521}
2522
2523} // end anonymous namespace
2524
2525//===----------------------------------------------------------------------===//
2526
2527static TableGen::Emitter::OptClass<GlobalISelEmitter>
2528 X("gen-global-isel", "Generate GlobalISel selector");
2529