1
2//===- GlobalISelEmitter.cpp - Generate an instruction selector -----------===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// This tablegen backend emits code for use by the GlobalISel instruction
12/// selector. See include/llvm/Target/GlobalISel/Target.td.
13///
14/// This file analyzes the patterns recognized by the SelectionDAGISel tablegen
15/// backend, filters out the ones that are unsupported, maps
16/// SelectionDAG-specific constructs to their GlobalISel counterpart
17/// (when applicable: MVT to LLT; SDNode to generic Instruction).
18///
19/// Not all patterns are supported: pass the tablegen invocation
20/// "-warn-on-skipped-patterns" to emit a warning when a pattern is skipped,
21/// as well as why.
22///
23/// The generated file defines a single method:
24/// bool <Target>InstructionSelector::selectImpl(MachineInstr &I) const;
25/// intended to be used in InstructionSelector::select as the first-step
26/// selector for the patterns that don't require complex C++.
27///
28/// FIXME: We'll probably want to eventually define a base
29/// "TargetGenInstructionSelector" class.
30///
31//===----------------------------------------------------------------------===//
32
33#include "Basic/CodeGenIntrinsics.h"
34#include "Common/CodeGenDAGPatterns.h"
35#include "Common/CodeGenInstruction.h"
36#include "Common/CodeGenRegisters.h"
37#include "Common/CodeGenTarget.h"
38#include "Common/GlobalISel/GlobalISelMatchTable.h"
39#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
40#include "Common/InfoByHwMode.h"
41#include "llvm/ADT/Statistic.h"
42#include "llvm/CodeGenTypes/LowLevelType.h"
43#include "llvm/CodeGenTypes/MachineValueType.h"
44#include "llvm/Support/CodeGenCoverage.h"
45#include "llvm/Support/CommandLine.h"
46#include "llvm/Support/Error.h"
47#include "llvm/Support/ScopedPrinter.h"
48#include "llvm/TableGen/Error.h"
49#include "llvm/TableGen/Record.h"
50#include "llvm/TableGen/TableGenBackend.h"
51#include <string>
52
53using namespace llvm;
54using namespace llvm::gi;
55
56using action_iterator = RuleMatcher::action_iterator;
57
58#define DEBUG_TYPE "gisel-emitter"
59
60STATISTIC(NumPatternTotal, "Total number of patterns");
61STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG");
62STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped");
63STATISTIC(NumPatternsTested,
64 "Number of patterns executed according to coverage information");
65
66static cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel");
67
68static cl::opt<bool> WarnOnSkippedPatterns(
69 "warn-on-skipped-patterns",
70 cl::desc("Explain why a pattern was skipped for inclusion "
71 "in the GlobalISel selector"),
72 cl::init(Val: false), cl::cat(GlobalISelEmitterCat));
73
74static cl::opt<bool> GenerateCoverage(
75 "instrument-gisel-coverage",
76 cl::desc("Generate coverage instrumentation for GlobalISel"),
77 cl::init(Val: false), cl::cat(GlobalISelEmitterCat));
78
79static cl::opt<std::string> UseCoverageFile(
80 "gisel-coverage-file", cl::init(Val: ""),
81 cl::desc("Specify file to retrieve coverage information from"),
82 cl::cat(GlobalISelEmitterCat));
83
84static cl::opt<bool> OptimizeMatchTable(
85 "optimize-match-table",
86 cl::desc("Generate an optimized version of the match table"),
87 cl::init(Val: true), cl::cat(GlobalISelEmitterCat));
88
89static std::string explainPredicates(const TreePatternNode &N) {
90 std::string Explanation;
91 StringRef Separator = "";
92 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
93 const TreePredicateFn &P = Call.Fn;
94 Explanation +=
95 (Separator + P.getOrigPatFragRecord()->getRecord()->getName()).str();
96 Separator = ", ";
97
98 if (P.isAlwaysTrue())
99 Explanation += " always-true";
100 if (P.isImmediatePattern())
101 Explanation += " immediate";
102
103 if (P.isUnindexed())
104 Explanation += " unindexed";
105
106 if (P.isNonExtLoad())
107 Explanation += " non-extload";
108 if (P.isAnyExtLoad())
109 Explanation += " extload";
110 if (P.isSignExtLoad())
111 Explanation += " sextload";
112 if (P.isZeroExtLoad())
113 Explanation += " zextload";
114
115 if (P.isNonTruncStore())
116 Explanation += " non-truncstore";
117 if (P.isTruncStore())
118 Explanation += " truncstore";
119
120 if (const Record *VT = P.getMemoryVT())
121 Explanation += (" MemVT=" + VT->getName()).str();
122 if (const Record *VT = P.getScalarMemoryVT())
123 Explanation += (" ScalarVT(MemVT)=" + VT->getName()).str();
124
125 if (const ListInit *AddrSpaces = P.getAddressSpaces()) {
126 raw_string_ostream OS(Explanation);
127 OS << " AddressSpaces=[";
128
129 StringRef AddrSpaceSeparator;
130 for (const Init *Val : AddrSpaces->getElements()) {
131 const IntInit *IntVal = dyn_cast<IntInit>(Val);
132 if (!IntVal)
133 continue;
134
135 OS << AddrSpaceSeparator << IntVal->getValue();
136 AddrSpaceSeparator = ", ";
137 }
138
139 OS << ']';
140 }
141
142 int64_t MinAlign = P.getMinAlignment();
143 if (MinAlign > 0)
144 Explanation += " MinAlign=" + utostr(X: MinAlign);
145
146 if (P.isAtomicOrderingMonotonic())
147 Explanation += " monotonic";
148 if (P.isAtomicOrderingAcquire())
149 Explanation += " acquire";
150 if (P.isAtomicOrderingRelease())
151 Explanation += " release";
152 if (P.isAtomicOrderingAcquireRelease())
153 Explanation += " acq_rel";
154 if (P.isAtomicOrderingSequentiallyConsistent())
155 Explanation += " seq_cst";
156 if (P.isAtomicOrderingAcquireOrStronger())
157 Explanation += " >=acquire";
158 if (P.isAtomicOrderingWeakerThanAcquire())
159 Explanation += " <acquire";
160 if (P.isAtomicOrderingReleaseOrStronger())
161 Explanation += " >=release";
162 if (P.isAtomicOrderingWeakerThanRelease())
163 Explanation += " <release";
164 }
165 return Explanation;
166}
167
168static std::string explainOperator(const Record *Operator) {
169 if (Operator->isSubClassOf(Name: "SDNode"))
170 return (" (" + Operator->getValueAsString(FieldName: "Opcode") + ")").str();
171
172 if (Operator->isSubClassOf(Name: "Intrinsic"))
173 return (" (Operator is an Intrinsic, " + Operator->getName() + ")").str();
174
175 if (Operator->isSubClassOf(Name: "ComplexPattern"))
176 return (" (Operator is an unmapped ComplexPattern, " + Operator->getName() +
177 ")")
178 .str();
179
180 if (Operator->isSubClassOf(Name: "SDNodeXForm"))
181 return (" (Operator is an unmapped SDNodeXForm, " + Operator->getName() +
182 ")")
183 .str();
184
185 return (" (Operator " + Operator->getName() + " not understood)").str();
186}
187
188/// Helper function to let the emitter report skip reason error messages.
189static Error failedImport(const Twine &Reason) {
190 return make_error<StringError>(Args: Reason, Args: inconvertibleErrorCode());
191}
192
193static Error isTrivialOperatorNode(const TreePatternNode &N) {
194 std::string Explanation;
195 std::string Separator;
196
197 bool HasUnsupportedPredicate = false;
198 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
199 const TreePredicateFn &Predicate = Call.Fn;
200
201 if (Predicate.isAlwaysTrue())
202 continue;
203
204 if (Predicate.isImmediatePattern())
205 continue;
206
207 if (Predicate.hasNoUse() || Predicate.hasOneUse())
208 continue;
209
210 if (Predicate.isNonExtLoad() || Predicate.isAnyExtLoad() ||
211 Predicate.isSignExtLoad() || Predicate.isZeroExtLoad())
212 continue;
213
214 if (Predicate.isNonTruncStore() || Predicate.isTruncStore())
215 continue;
216
217 if (Predicate.isLoad() && Predicate.getMemoryVT())
218 continue;
219
220 if (Predicate.isStore() && Predicate.getMemoryVT())
221 continue;
222
223 if (Predicate.isLoad() || Predicate.isStore()) {
224 if (Predicate.isUnindexed())
225 continue;
226 }
227
228 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
229 const ListInit *AddrSpaces = Predicate.getAddressSpaces();
230 if (AddrSpaces && !AddrSpaces->empty())
231 continue;
232
233 if (Predicate.getMinAlignment() > 0)
234 continue;
235 }
236
237 if (Predicate.isAtomic() && Predicate.getMemoryVT())
238 continue;
239
240 if (Predicate.isAtomic() &&
241 (Predicate.isAtomicOrderingMonotonic() ||
242 Predicate.isAtomicOrderingAcquire() ||
243 Predicate.isAtomicOrderingRelease() ||
244 Predicate.isAtomicOrderingAcquireRelease() ||
245 Predicate.isAtomicOrderingSequentiallyConsistent() ||
246 Predicate.isAtomicOrderingAcquireOrStronger() ||
247 Predicate.isAtomicOrderingWeakerThanAcquire() ||
248 Predicate.isAtomicOrderingReleaseOrStronger() ||
249 Predicate.isAtomicOrderingWeakerThanRelease()))
250 continue;
251
252 if (Predicate.hasGISelPredicateCode())
253 continue;
254
255 HasUnsupportedPredicate = true;
256 Explanation = Separator + "Has a predicate (" + explainPredicates(N) + ")";
257 Separator = ", ";
258 Explanation += (Separator + "first-failing:" +
259 Predicate.getOrigPatFragRecord()->getRecord()->getName())
260 .str();
261 break;
262 }
263
264 if (!HasUnsupportedPredicate)
265 return Error::success();
266
267 return failedImport(Reason: Explanation);
268}
269
270static std::string getScopedName(unsigned Scope, const std::string &Name) {
271 return ("pred:" + Twine(Scope) + ":" + Name).str();
272}
273
274static std::string getMangledRootDefName(StringRef DefOperandName) {
275 return ("DstI[" + DefOperandName + "]").str();
276}
277
278//===- GlobalISelEmitter class --------------------------------------------===//
279
280static Expected<LLTCodeGen> getInstResultType(const TreePatternNode &Dst,
281 const CodeGenTarget &Target) {
282 // While we allow more than one output (both implicit and explicit defs)
283 // below, we only expect one explicit def here.
284 assert(Dst.getOperator()->isSubClassOf("Instruction"));
285 const CodeGenInstruction &InstInfo = Target.getInstruction(InstRec: Dst.getOperator());
286 if (!InstInfo.Operands.NumDefs)
287 return failedImport(Reason: "Dst pattern child needs a def");
288
289 ArrayRef<TypeSetByHwMode> ChildTypes = Dst.getExtTypes();
290 if (ChildTypes.size() < 1)
291 return failedImport(Reason: "Dst pattern child has no result");
292
293 // If there are multiple results, just take the first one (this is how
294 // SelectionDAG does it).
295 std::optional<LLTCodeGen> MaybeOpTy;
296 if (ChildTypes.front().isMachineValueType()) {
297 MaybeOpTy = MVTToLLT(VT: ChildTypes.front().getMachineValueType().SimpleTy);
298 }
299
300 if (!MaybeOpTy)
301 return failedImport(Reason: "Dst operand has an unsupported type");
302 return *MaybeOpTy;
303}
304
305namespace {
306class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter {
307public:
308 explicit GlobalISelEmitter(const RecordKeeper &RK);
309
310 void emitAdditionalImpl(raw_ostream &OS) override;
311
312 void emitMIPredicateFns(raw_ostream &OS) override;
313 void emitLeafPredicateFns(raw_ostream &OS) override;
314 void emitI64ImmPredicateFns(raw_ostream &OS) override;
315 void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
316 void emitAPIntImmPredicateFns(raw_ostream &OS) override;
317 void emitTestSimplePredicate(raw_ostream &OS) override;
318 void emitRunCustomAction(raw_ostream &OS) override;
319
320 const CodeGenTarget &getTarget() const override { return Target; }
321 StringRef getClassName() const override { return ClassName; }
322
323 void run(raw_ostream &OS);
324
325private:
326 std::string ClassName;
327
328 const RecordKeeper &RK;
329 const CodeGenDAGPatterns CGP;
330 const CodeGenTarget &Target;
331 CodeGenRegBank &CGRegs;
332
333 ArrayRef<const Record *> AllPatFrags;
334
335 /// Keep track of the equivalence between SDNodes and Instruction by mapping
336 /// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to
337 /// check for attributes on the relation such as CheckMMOIsNonAtomic.
338 /// This is defined using 'GINodeEquiv' in the target description.
339 DenseMap<const Record *, const Record *> NodeEquivs;
340
341 /// Keep track of the equivalence between ComplexPattern's and
342 /// GIComplexOperandMatcher. Map entries are specified by subclassing
343 /// GIComplexPatternEquiv.
344 DenseMap<const Record *, const Record *> ComplexPatternEquivs;
345
346 /// Keep track of the equivalence between SDNodeXForm's and
347 /// GICustomOperandRenderer. Map entries are specified by subclassing
348 /// GISDNodeXFormEquiv.
349 DenseMap<const Record *, const Record *> SDNodeXFormEquivs;
350
351 /// Keep track of Scores of PatternsToMatch similar to how the DAG does.
352 /// This adds compatibility for RuleMatchers to use this for ordering rules.
353 DenseMap<uint64_t, int> RuleMatcherScores;
354
355 // Rule coverage information.
356 std::optional<CodeGenCoverage> RuleCoverage;
357
358 /// Variables used to help with collecting of named operands for predicates
359 /// with 'let PredicateCodeUsesOperands = 1'. WaitingForNamedOperands is set
360 /// to the number of named operands that predicate expects. Store locations in
361 /// StoreIdxForName correspond to the order in which operand names appear in
362 /// predicate's argument list.
363 /// When we visit named operand and WaitingForNamedOperands is not zero, add
364 /// matcher that will record operand and decrease counter.
365 unsigned WaitingForNamedOperands = 0;
366 StringMap<unsigned> StoreIdxForName;
367
368 void gatherOpcodeValues();
369 void gatherTypeIDValues();
370 void gatherNodeEquivs();
371
372 const Record *findNodeEquiv(const Record *N) const;
373 const CodeGenInstruction *getEquivNode(const Record &Equiv,
374 const TreePatternNode &N) const;
375
376 Error importRulePredicates(RuleMatcher &M,
377 ArrayRef<const Record *> Predicates);
378 Expected<InstructionMatcher &>
379 createAndImportSelDAGMatcher(RuleMatcher &Rule,
380 InstructionMatcher &InsnMatcher,
381 const TreePatternNode &Src, unsigned &TempOpIdx);
382 Error importComplexPatternOperandMatcher(OperandMatcher &OM, const Record *R,
383 unsigned &TempOpIdx) const;
384 Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
385 const TreePatternNode &SrcChild,
386 bool OperandIsAPointer, bool OperandIsImmArg,
387 unsigned OpIdx, unsigned &TempOpIdx);
388
389 Expected<BuildMIAction &>
390 createAndImportInstructionRenderer(RuleMatcher &M,
391 InstructionMatcher &InsnMatcher,
392 const TreePatternNode &Dst) const;
393 Expected<action_iterator> createAndImportSubInstructionRenderer(
394 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
395 unsigned TempReg) const;
396 Expected<action_iterator>
397 createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M,
398 const TreePatternNode &Dst) const;
399
400 Expected<action_iterator>
401 importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M,
402 BuildMIAction &DstMIBuilder,
403 const TreePatternNode &Dst, bool IsRoot) const;
404
405 Expected<action_iterator>
406 importExplicitUseRenderers(action_iterator InsertPt, RuleMatcher &M,
407 BuildMIAction &DstMIBuilder,
408 const TreePatternNode &Dst) const;
409
410 Error importNamedNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
411 const TreePatternNode &N) const;
412
413 Error importLeafNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
414 const TreePatternNode &N,
415 action_iterator InsertPt) const;
416
417 Error importXFormNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
418 const TreePatternNode &N) const;
419
420 Error importInstructionNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
421 const TreePatternNode &N,
422 action_iterator &InsertPt) const;
423
424 Error importNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
425 const TreePatternNode &N,
426 action_iterator &InsertPt) const;
427
428 Error importImplicitDefRenderers(BuildMIAction &DstMIBuilder,
429 ArrayRef<const Record *> ImplicitDefs) const;
430
431 /// Analyze pattern \p P, returning a matcher for it if possible.
432 /// Otherwise, return an Error explaining why we don't support it.
433 Expected<RuleMatcher> runOnPattern(const PatternToMatch &P);
434
435 void declareSubtargetFeature(const Record *Predicate);
436
437 unsigned declareHwModeCheck(StringRef HwModeFeatures);
438
439 MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize,
440 bool WithCoverage);
441
442 /// Infer a CodeGenRegisterClass for the type of \p SuperRegNode. The returned
443 /// CodeGenRegisterClass will support the CodeGenRegisterClass of
444 /// \p SubRegNode, and the subregister index defined by \p SubRegIdxNode.
445 /// If no register class is found, return nullptr.
446 const CodeGenRegisterClass *
447 inferSuperRegisterClassForNode(const TypeSetByHwMode &Ty,
448 const TreePatternNode &SuperRegNode,
449 const TreePatternNode &SubRegIdxNode) const;
450 const CodeGenSubRegIndex *
451 inferSubRegIndexForNode(const TreePatternNode &SubRegIdxNode) const;
452
453 /// Infer a CodeGenRegisterClass which suppoorts \p Ty and \p SubRegIdxNode.
454 /// Return nullptr if no such class exists.
455 const CodeGenRegisterClass *
456 inferSuperRegisterClass(const TypeSetByHwMode &Ty,
457 const TreePatternNode &SubRegIdxNode) const;
458
459 /// Return the CodeGenRegisterClass associated with \p Leaf if it has one.
460 const CodeGenRegisterClass *
461 getRegClassFromLeaf(const TreePatternNode &Leaf) const;
462
463 /// Return a CodeGenRegisterClass for \p N if one can be found. Return
464 /// nullptr otherwise.
465 const CodeGenRegisterClass *
466 inferRegClassFromPattern(const TreePatternNode &N) const;
467
468 const CodeGenRegisterClass *
469 inferRegClassFromRegisterClassLike(const Record *RecClassLike) const;
470
471 const CodeGenRegisterClass *
472 inferRegClassFromInstructionPattern(const TreePatternNode &N,
473 unsigned ResIdx) const;
474
475 Error constrainOperands(action_iterator InsertPt, RuleMatcher &M,
476 unsigned InsnID, const TreePatternNode &Dst) const;
477
478 /// Return the size of the MemoryVT in this predicate, if possible.
479 std::optional<unsigned>
480 getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate);
481
482 // Add builtin predicates.
483 Expected<InstructionMatcher &>
484 addBuiltinPredicates(const Record *SrcGIEquivOrNull,
485 const TreePredicateFn &Predicate,
486 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher);
487};
488} // namespace
489
490static StringRef getPatFragPredicateEnumName(const Record *R) {
491 return R->getName();
492}
493
494void GlobalISelEmitter::gatherOpcodeValues() {
495 InstructionOpcodeMatcher::initOpcodeValuesMap(Target);
496}
497
498void GlobalISelEmitter::gatherTypeIDValues() {
499 LLTOperandMatcher::initTypeIDValuesMap();
500}
501
502void GlobalISelEmitter::gatherNodeEquivs() {
503 assert(NodeEquivs.empty());
504 for (const Record *Equiv : RK.getAllDerivedDefinitions(ClassName: "GINodeEquiv"))
505 NodeEquivs[Equiv->getValueAsDef(FieldName: "Node")] = Equiv;
506
507 assert(ComplexPatternEquivs.empty());
508 for (const Record *Equiv :
509 RK.getAllDerivedDefinitions(ClassName: "GIComplexPatternEquiv")) {
510 const Record *SelDAGEquiv = Equiv->getValueAsDef(FieldName: "SelDAGEquivalent");
511 if (!SelDAGEquiv)
512 continue;
513 ComplexPatternEquivs[SelDAGEquiv] = Equiv;
514 }
515
516 assert(SDNodeXFormEquivs.empty());
517 for (const Record *Equiv :
518 RK.getAllDerivedDefinitions(ClassName: "GISDNodeXFormEquiv")) {
519 const Record *SelDAGEquiv = Equiv->getValueAsDef(FieldName: "SelDAGEquivalent");
520 if (!SelDAGEquiv)
521 continue;
522 SDNodeXFormEquivs[SelDAGEquiv] = Equiv;
523 }
524}
525
526const Record *GlobalISelEmitter::findNodeEquiv(const Record *N) const {
527 return NodeEquivs.lookup(Val: N);
528}
529
530const CodeGenInstruction *
531GlobalISelEmitter::getEquivNode(const Record &Equiv,
532 const TreePatternNode &N) const {
533 if (N.getNumChildren() >= 1) {
534 // setcc operation maps to two different G_* instructions based on the type.
535 if (!Equiv.isValueUnset(FieldName: "IfFloatingPoint") &&
536 MVT(N.getChild(N: 0).getSimpleType(ResNo: 0)).isFloatingPoint())
537 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfFloatingPoint"));
538 }
539
540 if (!Equiv.isValueUnset(FieldName: "IfConvergent") &&
541 N.getIntrinsicInfo(CDP: CGP)->isConvergent)
542 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfConvergent"));
543
544 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
545 const TreePredicateFn &Predicate = Call.Fn;
546 if (!Equiv.isValueUnset(FieldName: "IfSignExtend") &&
547 (Predicate.isLoad() || Predicate.isAtomic()) &&
548 Predicate.isSignExtLoad())
549 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfSignExtend"));
550 if (!Equiv.isValueUnset(FieldName: "IfZeroExtend") &&
551 (Predicate.isLoad() || Predicate.isAtomic()) &&
552 Predicate.isZeroExtLoad())
553 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfZeroExtend"));
554 }
555
556 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "I"));
557}
558
559GlobalISelEmitter::GlobalISelEmitter(const RecordKeeper &RK)
560 : GlobalISelMatchTableExecutorEmitter(), RK(RK), CGP(RK),
561 Target(CGP.getTargetInfo()), CGRegs(Target.getRegBank()) {
562 ClassName = Target.getName().str() + "InstructionSelector";
563}
564
565//===- Emitter ------------------------------------------------------------===//
566
567Error GlobalISelEmitter::importRulePredicates(
568 RuleMatcher &M, ArrayRef<const Record *> Predicates) {
569 for (const Record *Pred : Predicates) {
570 if (Pred->getValueAsString(FieldName: "CondString").empty())
571 continue;
572 declareSubtargetFeature(Predicate: Pred);
573 M.addRequiredFeature(Feature: Pred);
574 }
575
576 return Error::success();
577}
578
579std::optional<unsigned> GlobalISelEmitter::getMemSizeBitsFromPredicate(
580 const TreePredicateFn &Predicate) {
581 std::optional<LLTCodeGen> MemTyOrNone =
582 MVTToLLT(VT: getValueType(Rec: Predicate.getMemoryVT()));
583
584 if (!MemTyOrNone)
585 return std::nullopt;
586
587 // Align so unusual types like i1 don't get rounded down.
588 return llvm::alignTo(
589 Value: static_cast<unsigned>(MemTyOrNone->get().getSizeInBits()), Align: 8);
590}
591
592Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
593 const Record *SrcGIEquivOrNull, const TreePredicateFn &Predicate,
594 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher) {
595 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
596 if (const ListInit *AddrSpaces = Predicate.getAddressSpaces()) {
597 SmallVector<unsigned, 4> ParsedAddrSpaces;
598
599 for (const Init *Val : AddrSpaces->getElements()) {
600 const IntInit *IntVal = dyn_cast<IntInit>(Val);
601 if (!IntVal)
602 return failedImport(Reason: "Address space is not an integer");
603 ParsedAddrSpaces.push_back(Elt: IntVal->getValue());
604 }
605
606 if (!ParsedAddrSpaces.empty()) {
607 InsnMatcher.addPredicate<MemoryAddressSpacePredicateMatcher>(
608 args: 0, args&: ParsedAddrSpaces);
609 return InsnMatcher;
610 }
611 }
612
613 int64_t MinAlign = Predicate.getMinAlignment();
614 if (MinAlign > 0) {
615 InsnMatcher.addPredicate<MemoryAlignmentPredicateMatcher>(args: 0, args&: MinAlign);
616 return InsnMatcher;
617 }
618 }
619
620 // G_LOAD is used for both non-extending and any-extending loads.
621 if (Predicate.isLoad() || Predicate.isAtomic()) {
622 if (Predicate.isNonExtLoad()) {
623 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
624 args: 0, args: MemoryVsLLTSizePredicateMatcher::EqualTo, args: 0);
625 return InsnMatcher;
626 }
627 if (Predicate.isAnyExtLoad()) {
628 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
629 args: 0, args: MemoryVsLLTSizePredicateMatcher::LessThan, args: 0);
630 return InsnMatcher;
631 }
632 }
633
634 if (Predicate.isStore()) {
635 if (Predicate.isTruncStore()) {
636 if (Predicate.getMemoryVT() != nullptr) {
637 // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size.
638 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
639 if (!MemSizeInBits)
640 return failedImport(Reason: "MemVT could not be converted to LLT");
641
642 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(args: 0, args: *MemSizeInBits /
643 8);
644 } else {
645 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
646 args: 0, args: MemoryVsLLTSizePredicateMatcher::LessThan, args: 0);
647 }
648 return InsnMatcher;
649 }
650 if (Predicate.isNonTruncStore()) {
651 // We need to check the sizes match here otherwise we could incorrectly
652 // match truncating stores with non-truncating ones.
653 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
654 args: 0, args: MemoryVsLLTSizePredicateMatcher::EqualTo, args: 0);
655 }
656 }
657
658 assert(SrcGIEquivOrNull != nullptr && "Invalid SrcGIEquivOrNull value");
659 // No check required. We already did it by swapping the opcode.
660 if (!SrcGIEquivOrNull->isValueUnset(FieldName: "IfSignExtend") &&
661 Predicate.isSignExtLoad())
662 return InsnMatcher;
663
664 // No check required. We already did it by swapping the opcode.
665 if (!SrcGIEquivOrNull->isValueUnset(FieldName: "IfZeroExtend") &&
666 Predicate.isZeroExtLoad())
667 return InsnMatcher;
668
669 // No check required. G_STORE by itself is a non-extending store.
670 if (Predicate.isNonTruncStore())
671 return InsnMatcher;
672
673 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
674 if (Predicate.getMemoryVT() != nullptr) {
675 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
676 if (!MemSizeInBits)
677 return failedImport(Reason: "MemVT could not be converted to LLT");
678
679 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(args: 0,
680 args: *MemSizeInBits / 8);
681 return InsnMatcher;
682 }
683 }
684
685 if (Predicate.isLoad() || Predicate.isStore()) {
686 // No check required. A G_LOAD/G_STORE is an unindexed load.
687 if (Predicate.isUnindexed())
688 return InsnMatcher;
689 }
690
691 if (Predicate.isAtomic()) {
692 if (Predicate.isAtomicOrderingMonotonic()) {
693 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Monotonic");
694 return InsnMatcher;
695 }
696 if (Predicate.isAtomicOrderingAcquire()) {
697 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Acquire");
698 return InsnMatcher;
699 }
700 if (Predicate.isAtomicOrderingRelease()) {
701 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Release");
702 return InsnMatcher;
703 }
704 if (Predicate.isAtomicOrderingAcquireRelease()) {
705 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
706 args: "AcquireRelease");
707 return InsnMatcher;
708 }
709 if (Predicate.isAtomicOrderingSequentiallyConsistent()) {
710 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
711 args: "SequentiallyConsistent");
712 return InsnMatcher;
713 }
714 }
715
716 if (Predicate.isAtomicOrderingAcquireOrStronger()) {
717 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
718 args: "Acquire", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
719 return InsnMatcher;
720 }
721 if (Predicate.isAtomicOrderingWeakerThanAcquire()) {
722 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
723 args: "Acquire", args: AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
724 return InsnMatcher;
725 }
726
727 if (Predicate.isAtomicOrderingReleaseOrStronger()) {
728 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
729 args: "Release", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
730 return InsnMatcher;
731 }
732 if (Predicate.isAtomicOrderingWeakerThanRelease()) {
733 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
734 args: "Release", args: AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
735 return InsnMatcher;
736 }
737 HasAddedMatcher = false;
738 return InsnMatcher;
739}
740
741Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
742 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
743 const TreePatternNode &Src, unsigned &TempOpIdx) {
744 const auto SavedFlags = Rule.setGISelFlags(Src.getGISelFlagsRecord());
745
746 const Record *SrcGIEquivOrNull = nullptr;
747 const CodeGenInstruction *SrcGIOrNull = nullptr;
748
749 // Start with the defined operands (i.e., the results of the root operator).
750 if (Src.isLeaf()) {
751 const Init *SrcInit = Src.getLeafValue();
752 if (isa<IntInit>(Val: SrcInit)) {
753 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(
754 args: &Target.getInstruction(InstRec: RK.getDef(Name: "G_CONSTANT")));
755 } else {
756 return failedImport(
757 Reason: "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
758 }
759 } else {
760 SrcGIEquivOrNull = findNodeEquiv(N: Src.getOperator());
761 if (!SrcGIEquivOrNull)
762 return failedImport(Reason: "Pattern operator lacks an equivalent Instruction" +
763 explainOperator(Operator: Src.getOperator()));
764 SrcGIOrNull = getEquivNode(Equiv: *SrcGIEquivOrNull, N: Src);
765
766 // The operators look good: match the opcode
767 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(args&: SrcGIOrNull);
768 }
769
770 // Since there are no opcodes for atomic loads and stores comparing to
771 // SelectionDAG, we add CheckMMOIsNonAtomic predicate immediately after the
772 // opcode predicate to make a logical combination of them.
773 if (SrcGIEquivOrNull &&
774 SrcGIEquivOrNull->getValueAsBit(FieldName: "CheckMMOIsNonAtomic"))
775 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "NotAtomic");
776 else if (SrcGIEquivOrNull &&
777 SrcGIEquivOrNull->getValueAsBit(FieldName: "CheckMMOIsAtomic")) {
778 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
779 args: "Unordered", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
780 }
781
782 unsigned OpIdx = 0;
783 for (const TypeSetByHwMode &VTy : Src.getExtTypes()) {
784 // Results don't have a name unless they are the root node. The caller will
785 // set the name if appropriate.
786 const bool OperandIsAPointer =
787 SrcGIOrNull && SrcGIOrNull->isOutOperandAPointer(i: OpIdx);
788 OperandMatcher &OM = InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
789 if (auto Error = OM.addTypeCheckPredicate(VTy, OperandIsAPointer))
790 return failedImport(Reason: toString(E: std::move(Error)) +
791 " for result of Src pattern operator");
792 }
793
794 for (const TreePredicateCall &Call : Src.getPredicateCalls()) {
795 const TreePredicateFn &Predicate = Call.Fn;
796 bool HasAddedBuiltinMatcher = true;
797 if (Predicate.isAlwaysTrue())
798 continue;
799
800 if (Predicate.isImmediatePattern()) {
801 InsnMatcher.addPredicate<InstructionImmPredicateMatcher>(args: Predicate);
802 continue;
803 }
804
805 auto InsnMatcherOrError = addBuiltinPredicates(
806 SrcGIEquivOrNull, Predicate, InsnMatcher, HasAddedMatcher&: HasAddedBuiltinMatcher);
807 if (auto Error = InsnMatcherOrError.takeError())
808 return std::move(Error);
809
810 // FIXME: This should be part of addBuiltinPredicates(). If we add this at
811 // the start of addBuiltinPredicates() without returning, then there might
812 // be cases where we hit the last return before which the
813 // HasAddedBuiltinMatcher will be set to false. The predicate could be
814 // missed if we add it in the middle or at the end due to return statements
815 // after the addPredicate<>() calls.
816 if (Predicate.hasNoUse()) {
817 InsnMatcher.addPredicate<NoUsePredicateMatcher>();
818 HasAddedBuiltinMatcher = true;
819 }
820 if (Predicate.hasOneUse()) {
821 InsnMatcher.addPredicate<OneUsePredicateMatcher>();
822 HasAddedBuiltinMatcher = true;
823 }
824
825 if (Predicate.hasGISelPredicateCode()) {
826 if (Predicate.usesOperands()) {
827 assert(WaitingForNamedOperands == 0 &&
828 "previous predicate didn't find all operands or "
829 "nested predicate that uses operands");
830 TreePattern *TP = Predicate.getOrigPatFragRecord();
831 WaitingForNamedOperands = TP->getNumArgs();
832 for (unsigned I = 0; I < WaitingForNamedOperands; ++I)
833 StoreIdxForName[getScopedName(Scope: Call.Scope, Name: TP->getArgName(i: I))] = I;
834 }
835 InsnMatcher.addPredicate<GenericInstructionPredicateMatcher>(args: Predicate);
836 continue;
837 }
838 if (!HasAddedBuiltinMatcher) {
839 return failedImport(Reason: "Src pattern child has predicate (" +
840 explainPredicates(N: Src) + ")");
841 }
842 }
843
844 if (Src.isLeaf()) {
845 const Init *SrcInit = Src.getLeafValue();
846 if (const IntInit *SrcIntInit = dyn_cast<IntInit>(Val: SrcInit)) {
847 OperandMatcher &OM =
848 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: Src.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
849 OM.addPredicate<LiteralIntOperandMatcher>(args: SrcIntInit->getValue());
850 } else {
851 return failedImport(
852 Reason: "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
853 }
854 } else {
855 assert(SrcGIOrNull &&
856 "Expected to have already found an equivalent Instruction");
857 if (SrcGIOrNull->getName() == "G_CONSTANT" ||
858 SrcGIOrNull->getName() == "G_FCONSTANT" ||
859 SrcGIOrNull->getName() == "G_FRAME_INDEX") {
860 // imm/fpimm still have operands but we don't need to do anything with it
861 // here since we don't support ImmLeaf predicates yet. However, we still
862 // need to note the hidden operand to get GIM_CheckNumOperands correct.
863 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
864 return InsnMatcher;
865 }
866
867 // Special case because the operand order is changed from setcc. The
868 // predicate operand needs to be swapped from the last operand to the first
869 // source.
870
871 unsigned NumChildren = Src.getNumChildren();
872 bool IsFCmp = SrcGIOrNull->getName() == "G_FCMP";
873
874 if (IsFCmp || SrcGIOrNull->getName() == "G_ICMP") {
875 const TreePatternNode &SrcChild = Src.getChild(N: NumChildren - 1);
876 if (SrcChild.isLeaf()) {
877 const DefInit *DI = dyn_cast<DefInit>(Val: SrcChild.getLeafValue());
878 const Record *CCDef = DI ? DI->getDef() : nullptr;
879 if (!CCDef || !CCDef->isSubClassOf(Name: "CondCode"))
880 return failedImport(Reason: "Unable to handle CondCode");
881
882 OperandMatcher &OM = InsnMatcher.addOperand(
883 OpIdx: OpIdx++, SymbolicName: SrcChild.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
884 StringRef PredType = IsFCmp ? CCDef->getValueAsString(FieldName: "FCmpPredicate")
885 : CCDef->getValueAsString(FieldName: "ICmpPredicate");
886
887 if (!PredType.empty()) {
888 OM.addPredicate<CmpPredicateOperandMatcher>(args: PredType.str());
889 // Process the other 2 operands normally.
890 --NumChildren;
891 }
892 }
893 }
894
895 // Match the used operands (i.e. the children of the operator).
896 bool IsIntrinsic =
897 SrcGIOrNull->getName() == "G_INTRINSIC" ||
898 SrcGIOrNull->getName() == "G_INTRINSIC_W_SIDE_EFFECTS" ||
899 SrcGIOrNull->getName() == "G_INTRINSIC_CONVERGENT" ||
900 SrcGIOrNull->getName() == "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS";
901 const CodeGenIntrinsic *II = Src.getIntrinsicInfo(CDP: CGP);
902 if (IsIntrinsic && !II)
903 return failedImport(Reason: "Expected IntInit containing intrinsic ID)");
904
905 for (unsigned I = 0; I != NumChildren; ++I) {
906 const TreePatternNode &SrcChild = Src.getChild(N: I);
907
908 // We need to determine the meaning of a literal integer based on the
909 // context. If this is a field required to be an immediate (such as an
910 // immarg intrinsic argument), the required predicates are different than
911 // a constant which may be materialized in a register. If we have an
912 // argument that is required to be an immediate, we should not emit an LLT
913 // type check, and should not be looking for a G_CONSTANT defined
914 // register.
915 bool OperandIsImmArg = SrcGIOrNull->isInOperandImmArg(i: I);
916
917 // SelectionDAG allows pointers to be represented with iN since it doesn't
918 // distinguish between pointers and integers but they are different types
919 // in GlobalISel. Coerce integers to pointers to address space 0 if the
920 // context indicates a pointer.
921 //
922 bool OperandIsAPointer = SrcGIOrNull->isInOperandAPointer(i: I);
923
924 if (IsIntrinsic) {
925 // For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately
926 // following the defs is an intrinsic ID.
927 if (I == 0) {
928 OperandMatcher &OM = InsnMatcher.addOperand(
929 OpIdx: OpIdx++, SymbolicName: SrcChild.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
930 OM.addPredicate<IntrinsicIDOperandMatcher>(args&: II);
931 continue;
932 }
933
934 // We have to check intrinsics for llvm_anyptr_ty and immarg parameters.
935 //
936 // Note that we have to look at the i-1th parameter, because we don't
937 // have the intrinsic ID in the intrinsic's parameter list.
938 OperandIsAPointer |= II->isParamAPointer(ParamIdx: I - 1);
939 OperandIsImmArg |= II->isParamImmArg(ParamIdx: I - 1);
940 }
941
942 if (auto Error =
943 importChildMatcher(Rule, InsnMatcher, SrcChild, OperandIsAPointer,
944 OperandIsImmArg, OpIdx: OpIdx++, TempOpIdx))
945 return std::move(Error);
946 }
947 }
948
949 return InsnMatcher;
950}
951
952Error GlobalISelEmitter::importComplexPatternOperandMatcher(
953 OperandMatcher &OM, const Record *R, unsigned &TempOpIdx) const {
954 const auto &ComplexPattern = ComplexPatternEquivs.find(Val: R);
955 if (ComplexPattern == ComplexPatternEquivs.end())
956 return failedImport(Reason: "SelectionDAG ComplexPattern (" + R->getName() +
957 ") not mapped to GlobalISel");
958
959 OM.addPredicate<ComplexPatternOperandMatcher>(args&: OM, args: *ComplexPattern->second);
960 TempOpIdx++;
961 return Error::success();
962}
963
964// Get the name to use for a pattern operand. For an anonymous physical register
965// input, this should use the register name.
966static StringRef getSrcChildName(const TreePatternNode &SrcChild,
967 const Record *&PhysReg) {
968 StringRef SrcChildName = SrcChild.getName();
969 if (SrcChildName.empty() && SrcChild.isLeaf()) {
970 if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
971 auto *ChildRec = ChildDefInit->getDef();
972 if (ChildRec->isSubClassOf(Name: "Register")) {
973 SrcChildName = ChildRec->getName();
974 PhysReg = ChildRec;
975 }
976 }
977 }
978
979 return SrcChildName;
980}
981
982Error GlobalISelEmitter::importChildMatcher(
983 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
984 const TreePatternNode &SrcChild, bool OperandIsAPointer,
985 bool OperandIsImmArg, unsigned OpIdx, unsigned &TempOpIdx) {
986
987 const Record *PhysReg = nullptr;
988 std::string SrcChildName = getSrcChildName(SrcChild, PhysReg).str();
989 if (!SrcChild.isLeaf() &&
990 SrcChild.getOperator()->isSubClassOf(Name: "ComplexPattern")) {
991 // The "name" of a non-leaf complex pattern (MY_PAT $op1, $op2) is
992 // "MY_PAT:op1:op2" and the ones with same "name" represent same operand.
993 std::string PatternName = SrcChild.getOperator()->getName().str();
994 for (const TreePatternNode &Child : SrcChild.children()) {
995 PatternName += ":";
996 PatternName += Child.getName();
997 }
998 SrcChildName = PatternName;
999 }
1000
1001 OperandMatcher &OM =
1002 PhysReg ? InsnMatcher.addPhysRegInput(Reg: PhysReg, OpIdx, TempOpIdx)
1003 : InsnMatcher.addOperand(OpIdx, SymbolicName: SrcChildName, AllocatedTemporariesBaseID: TempOpIdx);
1004 if (OM.isSameAsAnotherOperand())
1005 return Error::success();
1006
1007 ArrayRef<TypeSetByHwMode> ChildTypes = SrcChild.getExtTypes();
1008 if (ChildTypes.size() != 1)
1009 return failedImport(Reason: "Src pattern child has multiple results");
1010
1011 // Check MBB's before the type check since they are not a known type.
1012 if (!SrcChild.isLeaf()) {
1013 if (SrcChild.getOperator()->getName() == "bb") {
1014 OM.addPredicate<MBBOperandMatcher>();
1015 return Error::success();
1016 }
1017 if (SrcChild.getOperator()->getName() == "timm") {
1018 OM.addPredicate<ImmOperandMatcher>();
1019
1020 // Add predicates, if any
1021 for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {
1022 const TreePredicateFn &Predicate = Call.Fn;
1023
1024 // Only handle immediate patterns for now
1025 if (Predicate.isImmediatePattern()) {
1026 OM.addPredicate<OperandImmPredicateMatcher>(args: Predicate);
1027 }
1028 }
1029
1030 return Error::success();
1031 }
1032 } else if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
1033 auto *ChildRec = ChildDefInit->getDef();
1034 if (ChildRec->isSubClassOf(Name: "ValueType") && !SrcChild.hasName()) {
1035 // An unnamed ValueType as in (sext_inreg GPR:$foo, i8). GISel represents
1036 // this as a literal constant with the scalar size.
1037 MVT VT = llvm::getValueType(Rec: ChildRec);
1038 OM.addPredicate<LiteralIntOperandMatcher>(args: VT.getScalarSizeInBits());
1039 return Error::success();
1040 }
1041 }
1042
1043 // Immediate arguments have no meaningful type to check as they don't have
1044 // registers.
1045 if (!OperandIsImmArg) {
1046 if (auto Error =
1047 OM.addTypeCheckPredicate(VTy: ChildTypes.front(), OperandIsAPointer))
1048 return failedImport(Reason: toString(E: std::move(Error)) + " for Src operand (" +
1049 to_string(Value: SrcChild) + ")");
1050 }
1051
1052 // Try look up SrcChild for a (named) predicate operand if there is any.
1053 if (WaitingForNamedOperands) {
1054 auto &ScopedNames = SrcChild.getNamesAsPredicateArg();
1055 if (!ScopedNames.empty()) {
1056 auto PA = ScopedNames.begin();
1057 std::string Name = getScopedName(Scope: PA->getScope(), Name: PA->getIdentifier());
1058 OM.addPredicate<RecordNamedOperandMatcher>(args&: StoreIdxForName[Name], args&: Name);
1059 --WaitingForNamedOperands;
1060 }
1061 }
1062
1063 // Check for nested instructions.
1064 if (!SrcChild.isLeaf()) {
1065 if (SrcChild.getOperator()->isSubClassOf(Name: "ComplexPattern")) {
1066 // When a ComplexPattern is used as an operator, it should do the same
1067 // thing as when used as a leaf. However, the children of the operator
1068 // name the sub-operands that make up the complex operand and we must
1069 // prepare to reference them in the renderer too.
1070 unsigned RendererID = TempOpIdx;
1071 if (auto Error = importComplexPatternOperandMatcher(
1072 OM, R: SrcChild.getOperator(), TempOpIdx))
1073 return Error;
1074
1075 for (unsigned I = 0, E = SrcChild.getNumChildren(); I != E; ++I) {
1076 auto &SubOperand = SrcChild.getChild(N: I);
1077 if (!SubOperand.getName().empty()) {
1078 if (auto Error = Rule.defineComplexSubOperand(
1079 SymbolicName: SubOperand.getName(), ComplexPattern: SrcChild.getOperator(), RendererID, SubOperandID: I,
1080 ParentSymbolicName: SrcChildName))
1081 return Error;
1082 }
1083 }
1084
1085 return Error::success();
1086 }
1087
1088 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
1089 args&: InsnMatcher.getRuleMatcher(), args: SrcChild.getName());
1090 if (!MaybeInsnOperand) {
1091 // This isn't strictly true. If the user were to provide exactly the same
1092 // matchers as the original operand then we could allow it. However, it's
1093 // simpler to not permit the redundant specification.
1094 return failedImport(
1095 Reason: "Nested instruction cannot be the same as another operand");
1096 }
1097
1098 // Map the node to a gMIR instruction.
1099 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
1100 auto InsnMatcherOrError = createAndImportSelDAGMatcher(
1101 Rule, InsnMatcher&: InsnOperand.getInsnMatcher(), Src: SrcChild, TempOpIdx);
1102 if (auto Error = InsnMatcherOrError.takeError())
1103 return Error;
1104
1105 return Error::success();
1106 }
1107
1108 if (SrcChild.hasAnyPredicate()) {
1109 for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {
1110 const TreePredicateFn &Predicate = Call.Fn;
1111
1112 if (!Predicate.hasGISelLeafPredicateCode())
1113 return failedImport(Reason: "Src pattern child has unsupported predicate");
1114 OM.addPredicate<OperandLeafPredicateMatcher>(args: Predicate);
1115 }
1116 return Error::success();
1117 }
1118
1119 // Check for constant immediates.
1120 if (auto *ChildInt = dyn_cast<IntInit>(Val: SrcChild.getLeafValue())) {
1121 if (OperandIsImmArg) {
1122 // Checks for argument directly in operand list
1123 OM.addPredicate<LiteralIntOperandMatcher>(args: ChildInt->getValue());
1124 } else {
1125 // Checks for materialized constant
1126 OM.addPredicate<ConstantIntOperandMatcher>(args: ChildInt->getValue());
1127 }
1128 return Error::success();
1129 }
1130
1131 // Check for def's like register classes or ComplexPattern's.
1132 if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
1133 auto *ChildRec = ChildDefInit->getDef();
1134
1135 // Check for register classes.
1136 if (ChildRec->isSubClassOf(Name: "RegisterClassLike") ||
1137 ChildRec->isSubClassOf(Name: "RegisterOperand")) {
1138 OM.addPredicate<RegisterBankOperandMatcher>(
1139 args: Target.getRegisterClass(R: Target.getInitValueAsRegClass(
1140 V: ChildDefInit,
1141 /*AssumeRegClassByHwModeIsDefault=*/true)));
1142 return Error::success();
1143 }
1144
1145 if (ChildRec->isSubClassOf(Name: "Register")) {
1146 // This just be emitted as a copy to the specific register.
1147 ValueTypeByHwMode VT = ChildTypes.front().getValueTypeByHwMode();
1148 const CodeGenRegisterClass *RC =
1149 CGRegs.getMinimalPhysRegClass(RegRecord: ChildRec, VT: &VT);
1150 if (!RC) {
1151 return failedImport(
1152 Reason: "Could not determine physical register class of pattern source");
1153 }
1154
1155 OM.addPredicate<RegisterBankOperandMatcher>(args: *RC);
1156 return Error::success();
1157 }
1158
1159 // Check for ValueType.
1160 if (ChildRec->isSubClassOf(Name: "ValueType")) {
1161 // We already added a type check as standard practice so this doesn't need
1162 // to do anything.
1163 return Error::success();
1164 }
1165
1166 // Check for ComplexPattern's.
1167 if (ChildRec->isSubClassOf(Name: "ComplexPattern"))
1168 return importComplexPatternOperandMatcher(OM, R: ChildRec, TempOpIdx);
1169
1170 if (ChildRec->isSubClassOf(Name: "ImmLeaf")) {
1171 return failedImport(
1172 Reason: "Src pattern child def is an unsupported tablegen class (ImmLeaf)");
1173 }
1174
1175 // Place holder for SRCVALUE nodes. Nothing to do here.
1176 if (ChildRec->getName() == "srcvalue")
1177 return Error::success();
1178
1179 const bool ImmAllOnesV = ChildRec->getName() == "immAllOnesV";
1180 if (ImmAllOnesV || ChildRec->getName() == "immAllZerosV") {
1181 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
1182 args&: InsnMatcher.getRuleMatcher(), args: SrcChild.getName(), args: false);
1183 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
1184
1185 ValueTypeByHwMode VTy = ChildTypes.front().getValueTypeByHwMode();
1186
1187 const CodeGenInstruction &BuildVector =
1188 Target.getInstruction(InstRec: RK.getDef(Name: "G_BUILD_VECTOR"));
1189 const CodeGenInstruction &BuildVectorTrunc =
1190 Target.getInstruction(InstRec: RK.getDef(Name: "G_BUILD_VECTOR_TRUNC"));
1191
1192 // Treat G_BUILD_VECTOR as the canonical opcode, and G_BUILD_VECTOR_TRUNC
1193 // as an alternative.
1194 InsnOperand.getInsnMatcher().addPredicate<InstructionOpcodeMatcher>(
1195 args: ArrayRef({&BuildVector, &BuildVectorTrunc}));
1196
1197 // TODO: Handle both G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC We could
1198 // theoretically not emit any opcode check, but getOpcodeMatcher currently
1199 // has to succeed.
1200 OperandMatcher &OM =
1201 InsnOperand.getInsnMatcher().addOperand(OpIdx: 0, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
1202 if (auto Error = OM.addTypeCheckPredicate(VTy: TypeSetByHwMode(VTy),
1203 /*OperandIsAPointer=*/false))
1204 return failedImport(Reason: toString(E: std::move(Error)) +
1205 " for result of Src pattern operator");
1206
1207 InsnOperand.getInsnMatcher().addPredicate<VectorSplatImmPredicateMatcher>(
1208 args: ImmAllOnesV ? VectorSplatImmPredicateMatcher::AllOnes
1209 : VectorSplatImmPredicateMatcher::AllZeros);
1210 return Error::success();
1211 }
1212
1213 return failedImport(
1214 Reason: "Src pattern child def is an unsupported tablegen class");
1215 }
1216
1217 return failedImport(Reason: "Src pattern child is an unsupported kind");
1218}
1219
1220// Equivalent of MatcherGen::EmitResultOfNamedOperand.
1221Error GlobalISelEmitter::importNamedNodeRenderer(
1222 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N) const {
1223 StringRef NodeName = N.getName();
1224
1225 if (auto SubOperand = M.getComplexSubOperand(SymbolicName: NodeName)) {
1226 auto [ComplexPatternRec, RendererID, SubOperandIdx] = *SubOperand;
1227 MIBuilder.addRenderer<RenderComplexPatternOperand>(
1228 args: *ComplexPatternRec, args&: NodeName, args&: RendererID, args&: SubOperandIdx);
1229 return Error::success();
1230 }
1231
1232 if (!N.isLeaf()) {
1233 StringRef OperatorName = N.getOperator()->getName();
1234
1235 if (OperatorName == "imm") {
1236 MIBuilder.addRenderer<CopyConstantAsImmRenderer>(args&: NodeName);
1237 return Error::success();
1238 }
1239
1240 if (OperatorName == "fpimm") {
1241 MIBuilder.addRenderer<CopyFConstantAsFPImmRenderer>(args&: NodeName);
1242 return Error::success();
1243 }
1244
1245 // TODO: 'imm' and 'fpimm' are the only nodes that need special treatment.
1246 // Remove this check and add CopyRenderer unconditionally for other nodes.
1247 if (OperatorName == "bb" || OperatorName == "timm" ||
1248 OperatorName == "tframeindex") {
1249 MIBuilder.addRenderer<CopyRenderer>(args&: NodeName);
1250 return Error::success();
1251 }
1252
1253 return failedImport(Reason: "node has unsupported operator " + to_string(Value: N));
1254 }
1255
1256 if (const auto *DI = dyn_cast<DefInit>(Val: N.getLeafValue())) {
1257 const Record *R = DI->getDef();
1258
1259 if (N.getNumResults() != 1)
1260 return failedImport(Reason: "node does not have one result " + to_string(Value: N));
1261
1262 if (R->isSubClassOf(Name: "ComplexPattern")) {
1263 auto I = ComplexPatternEquivs.find(Val: R);
1264 if (I == ComplexPatternEquivs.end())
1265 return failedImport(Reason: "ComplexPattern " + R->getName() +
1266 " does not have GISel equivalent");
1267
1268 const OperandMatcher &OM = M.getOperandMatcher(Name: NodeName);
1269 MIBuilder.addRenderer<RenderComplexPatternOperand>(
1270 args: *I->second, args&: NodeName, args: OM.getAllocatedTemporariesBaseID());
1271 return Error::success();
1272 }
1273
1274 if (R->isSubClassOf(Name: "RegisterOperand") &&
1275 !R->isValueUnset(FieldName: "GIZeroRegister")) {
1276 MIBuilder.addRenderer<CopyOrAddZeroRegRenderer>(
1277 args&: NodeName, args: R->getValueAsDef(FieldName: "GIZeroRegister"));
1278 return Error::success();
1279 }
1280
1281 // TODO: All special cases are handled above. Remove this check and add
1282 // CopyRenderer unconditionally.
1283 if (R->isSubClassOf(Name: "RegisterClassLike") ||
1284 R->isSubClassOf(Name: "RegisterOperand") || R->isSubClassOf(Name: "ValueType")) {
1285 MIBuilder.addRenderer<CopyRenderer>(args&: NodeName);
1286 return Error::success();
1287 }
1288 }
1289
1290 // TODO: Change this to assert and move to the beginning of the function.
1291 if (!M.hasOperand(SymbolicName: NodeName))
1292 return failedImport(Reason: "could not find node $" + NodeName +
1293 " in the source DAG");
1294
1295 // TODO: Remove this check and add CopyRenderer unconditionally.
1296 // TODO: Handle nodes with multiple results (provided they can reach here).
1297 if (isa<UnsetInit>(Val: N.getLeafValue())) {
1298 MIBuilder.addRenderer<CopyRenderer>(args&: NodeName);
1299 return Error::success();
1300 }
1301
1302 return failedImport(Reason: "unsupported node " + to_string(Value: N));
1303}
1304
1305// Equivalent of MatcherGen::EmitResultLeafAsOperand.
1306Error GlobalISelEmitter::importLeafNodeRenderer(
1307 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N,
1308 action_iterator InsertPt) const {
1309 if (const auto *II = dyn_cast<IntInit>(Val: N.getLeafValue())) {
1310 MIBuilder.addRenderer<ImmRenderer>(args: II->getValue());
1311 return Error::success();
1312 }
1313
1314 if (const auto *DI = dyn_cast<DefInit>(Val: N.getLeafValue())) {
1315 const Record *R = DI->getDef();
1316
1317 if (R->isSubClassOf(Name: "Register") || R->getName() == "zero_reg") {
1318 MIBuilder.addRenderer<AddRegisterRenderer>(args: Target, args&: R);
1319 return Error::success();
1320 }
1321
1322 if (R->getName() == "undef_tied_input") {
1323 std::optional<LLTCodeGen> OpTyOrNone = MVTToLLT(VT: N.getSimpleType(ResNo: 0));
1324 if (!OpTyOrNone)
1325 return failedImport(Reason: "unsupported type");
1326
1327 unsigned TempRegID = M.allocateTempRegID();
1328 M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTyOrNone, args&: TempRegID);
1329
1330 auto I = M.insertAction<BuildMIAction>(
1331 InsertPt, args: M.allocateOutputInsnID(),
1332 args: &Target.getInstruction(InstRec: RK.getDef(Name: "IMPLICIT_DEF")));
1333 auto &ImpDefBuilder = static_cast<BuildMIAction &>(**I);
1334 ImpDefBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, /*IsDef=*/args: true);
1335
1336 MIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID);
1337 return Error::success();
1338 }
1339
1340 if (R->isSubClassOf(Name: "SubRegIndex")) {
1341 const CodeGenSubRegIndex *SubRegIndex = CGRegs.findSubRegIdx(Def: R);
1342 MIBuilder.addRenderer<ImmRenderer>(args: SubRegIndex->EnumValue);
1343 return Error::success();
1344 }
1345
1346 // There are also RegisterClass / RegisterOperand operands of REG_SEQUENCE /
1347 // COPY_TO_REGCLASS, but these instructions are currently handled elsewhere.
1348 }
1349
1350 return failedImport(Reason: "unrecognized node " + to_string(Value: N));
1351}
1352
1353// Equivalent of MatcherGen::EmitResultSDNodeXFormAsOperand.
1354Error GlobalISelEmitter::importXFormNodeRenderer(
1355 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N) const {
1356 const Record *XFormRec = N.getOperator();
1357 auto I = SDNodeXFormEquivs.find(Val: XFormRec);
1358 if (I == SDNodeXFormEquivs.end())
1359 return failedImport(Reason: "SDNodeXForm " + XFormRec->getName() +
1360 " does not have GISel equivalent");
1361
1362 // TODO: Fail to import if GISDNodeXForm does not have RendererFn.
1363 // This currently results in a fatal error in emitRenderOpcodes.
1364 const Record *XFormEquivRec = I->second;
1365
1366 // The node to apply the transformation function to.
1367 // FIXME: The node may not have a name and may be a leaf. It should be
1368 // rendered first, like any other nodes. This may or may not require
1369 // introducing a temporary register, and we can't tell that without
1370 // inspecting the node (possibly recursively). This is a general drawback
1371 // of appending renderers directly to BuildMIAction.
1372 const TreePatternNode &Node = N.getChild(N: 0);
1373 StringRef NodeName = Node.getName();
1374
1375 const Record *XFormOpc = CGP.getSDNodeTransform(R: XFormRec).first;
1376 if (XFormOpc->getName() == "timm") {
1377 // If this is a TargetConstant, there won't be a corresponding
1378 // instruction to transform. Instead, this will refer directly to an
1379 // operand in an instruction's operand list.
1380 MIBuilder.addRenderer<CustomOperandRenderer>(args: *XFormEquivRec, args&: NodeName);
1381 } else {
1382 MIBuilder.addRenderer<CustomRenderer>(args: *XFormEquivRec, args&: NodeName);
1383 }
1384
1385 return Error::success();
1386}
1387
1388// Equivalent of MatcherGen::EmitResultInstructionAsOperand.
1389Error GlobalISelEmitter::importInstructionNodeRenderer(
1390 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N,
1391 action_iterator &InsertPt) const {
1392 Expected<LLTCodeGen> OpTy = getInstResultType(Dst: N, Target);
1393 if (!OpTy)
1394 return OpTy.takeError();
1395
1396 // TODO: See the comment in importXFormNodeRenderer. We rely on the node
1397 // requiring a temporary register, which prevents us from using this
1398 // function on the root of the destination DAG.
1399 unsigned TempRegID = M.allocateTempRegID();
1400 InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTy, args&: TempRegID);
1401 MIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID);
1402
1403 auto InsertPtOrError =
1404 createAndImportSubInstructionRenderer(InsertPt: ++InsertPt, M, Dst: N, TempReg: TempRegID);
1405 if (!InsertPtOrError)
1406 return InsertPtOrError.takeError();
1407
1408 InsertPt = *InsertPtOrError;
1409 return Error::success();
1410}
1411
1412// Equivalent of MatcherGen::EmitResultOperand.
1413Error GlobalISelEmitter::importNodeRenderer(RuleMatcher &M,
1414 BuildMIAction &MIBuilder,
1415 const TreePatternNode &N,
1416 action_iterator &InsertPt) const {
1417 if (N.hasName())
1418 return importNamedNodeRenderer(M, MIBuilder, N);
1419
1420 if (N.isLeaf())
1421 return importLeafNodeRenderer(M, MIBuilder, N, InsertPt);
1422
1423 if (N.getOperator()->isSubClassOf(Name: "SDNodeXForm"))
1424 return importXFormNodeRenderer(M, MIBuilder, N);
1425
1426 if (N.getOperator()->isSubClassOf(Name: "Instruction"))
1427 return importInstructionNodeRenderer(M, MIBuilder, N, InsertPt);
1428
1429 // Should not reach here.
1430 return failedImport(Reason: "unrecognized node " + llvm::to_string(Value: N));
1431}
1432
1433/// Generates code that builds the resulting instruction(s) from the destination
1434/// DAG. Note that to do this we do not and should not need the source DAG.
1435/// We do need to know whether a generated instruction defines a result of the
1436/// source DAG; this information is available via RuleMatcher::hasOperand.
1437Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
1438 RuleMatcher &M, InstructionMatcher &InsnMatcher,
1439 const TreePatternNode &Dst) const {
1440 auto InsertPtOrError = createInstructionRenderer(InsertPt: M.actions_end(), M, Dst);
1441 if (auto Error = InsertPtOrError.takeError())
1442 return std::move(Error);
1443
1444 action_iterator InsertPt = InsertPtOrError.get();
1445 BuildMIAction &DstMIBuilder = *static_cast<BuildMIAction *>(InsertPt->get());
1446
1447 for (auto PhysOp : M.physoperands()) {
1448 InsertPt = M.insertAction<BuildMIAction>(
1449 InsertPt, args: M.allocateOutputInsnID(),
1450 args: &Target.getInstruction(InstRec: RK.getDef(Name: "COPY")));
1451 BuildMIAction &CopyToPhysRegMIBuilder =
1452 *static_cast<BuildMIAction *>(InsertPt->get());
1453 CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>(args: Target,
1454 args&: PhysOp.first, args: true);
1455 CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(args&: PhysOp.first);
1456 }
1457
1458 if (auto Error = importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Dst,
1459 /*IsRoot=*/true)
1460 .takeError())
1461 return std::move(Error);
1462
1463 if (auto Error = importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst)
1464 .takeError())
1465 return std::move(Error);
1466
1467 return DstMIBuilder;
1468}
1469
1470Expected<action_iterator>
1471GlobalISelEmitter::createAndImportSubInstructionRenderer(
1472 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
1473 unsigned TempRegID) const {
1474 auto InsertPtOrError = createInstructionRenderer(InsertPt, M, Dst);
1475
1476 // TODO: Assert there's exactly one result.
1477
1478 if (auto Error = InsertPtOrError.takeError())
1479 return std::move(Error);
1480
1481 BuildMIAction &DstMIBuilder =
1482 *static_cast<BuildMIAction *>(InsertPtOrError.get()->get());
1483
1484 // Assign the result to TempReg.
1485 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, args: true);
1486
1487 // Handle additional (ignored) results.
1488 InsertPtOrError = importExplicitDefRenderers(
1489 InsertPt: std::prev(x: *InsertPtOrError), M, DstMIBuilder, Dst, /*IsRoot=*/false);
1490 if (auto Error = InsertPtOrError.takeError())
1491 return std::move(Error);
1492
1493 InsertPtOrError =
1494 importExplicitUseRenderers(InsertPt: InsertPtOrError.get(), M, DstMIBuilder, Dst);
1495 if (auto Error = InsertPtOrError.takeError())
1496 return std::move(Error);
1497
1498 if (auto Error =
1499 constrainOperands(InsertPt, M, InsnID: DstMIBuilder.getInsnID(), Dst))
1500 return std::move(Error);
1501
1502 return InsertPtOrError.get();
1503}
1504
1505Expected<action_iterator>
1506GlobalISelEmitter::createInstructionRenderer(action_iterator InsertPt,
1507 RuleMatcher &M,
1508 const TreePatternNode &Dst) const {
1509 const Record *DstOp = Dst.getOperator();
1510 if (!DstOp->isSubClassOf(Name: "Instruction")) {
1511 if (DstOp->isSubClassOf(Name: "ValueType"))
1512 return failedImport(
1513 Reason: "Pattern operator isn't an instruction (it's a ValueType)");
1514 return failedImport(Reason: "Pattern operator isn't an instruction");
1515 }
1516 const CodeGenInstruction *DstI = &Target.getInstruction(InstRec: DstOp);
1517
1518 // COPY_TO_REGCLASS is just a copy with a ConstrainOperandToRegClassAction
1519 // attached. Similarly for EXTRACT_SUBREG except that's a subregister copy.
1520 StringRef Name = DstI->getName();
1521 if (Name == "COPY_TO_REGCLASS" || Name == "EXTRACT_SUBREG")
1522 DstI = &Target.getInstruction(InstRec: RK.getDef(Name: "COPY"));
1523
1524 return M.insertAction<BuildMIAction>(InsertPt, args: M.allocateOutputInsnID(),
1525 args&: DstI);
1526}
1527
1528Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
1529 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
1530 const TreePatternNode &Dst, bool IsRoot) const {
1531 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
1532
1533 // Process explicit defs. The caller may have already handled the first def.
1534 for (unsigned I = IsRoot ? 0 : 1, E = DstI->Operands.NumDefs; I != E; ++I) {
1535 const CGIOperandList::OperandInfo &OpInfo = DstI->Operands[I];
1536 std::string OpName = getMangledRootDefName(DefOperandName: OpInfo.Name);
1537
1538 // If the def is used in the source DAG, forward it.
1539 if (IsRoot && M.hasOperand(SymbolicName: OpName)) {
1540 // CopyRenderer saves a StringRef, so cannot pass OpName itself -
1541 // let's use a string with an appropriate lifetime.
1542 StringRef PermanentRef = M.getOperandMatcher(Name: OpName).getSymbolicName();
1543 DstMIBuilder.addRenderer<CopyRenderer>(args&: PermanentRef);
1544 continue;
1545 }
1546
1547 // A discarded explicit def may be an optional physical register.
1548 // If this is the case, add the default register and mark it as dead.
1549 if (OpInfo.Rec->isSubClassOf(Name: "OptionalDefOperand")) {
1550 for (const TreePatternNode &DefaultOp :
1551 make_pointee_range(Range: CGP.getDefaultOperand(R: OpInfo.Rec).DefaultOps)) {
1552 // TODO: Do these checks in ParseDefaultOperands.
1553 if (!DefaultOp.isLeaf())
1554 return failedImport(Reason: "optional def is not a leaf");
1555
1556 const auto *RegDI = dyn_cast<DefInit>(Val: DefaultOp.getLeafValue());
1557 if (!RegDI)
1558 return failedImport(Reason: "optional def is not a record");
1559
1560 const Record *Reg = RegDI->getDef();
1561 if (!Reg->isSubClassOf(Name: "Register") && Reg->getName() != "zero_reg")
1562 return failedImport(Reason: "optional def is not a register");
1563
1564 DstMIBuilder.addRenderer<AddRegisterRenderer>(
1565 args: Target, args&: Reg, /*IsDef=*/args: true, /*IsDead=*/args: true);
1566 }
1567 continue;
1568 }
1569
1570 // The def is discarded, create a dead virtual register for it.
1571 const TypeSetByHwMode &ExtTy = Dst.getExtType(ResNo: I);
1572 if (!ExtTy.isMachineValueType())
1573 return failedImport(Reason: "unsupported typeset");
1574
1575 auto OpTy = MVTToLLT(VT: ExtTy.getMachineValueType().SimpleTy);
1576 if (!OpTy)
1577 return failedImport(Reason: "unsupported type");
1578
1579 unsigned TempRegID = M.allocateTempRegID();
1580 InsertPt =
1581 M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTy, args&: TempRegID);
1582 DstMIBuilder.addRenderer<TempRegRenderer>(
1583 args&: TempRegID, /*IsDef=*/args: true, /*SubReg=*/args: nullptr, /*IsDead=*/args: true);
1584 }
1585
1586 // Implicit defs are not currently supported, mark all of them as dead.
1587 for (const Record *Reg : DstI->ImplicitDefs) {
1588 std::string OpName = getMangledRootDefName(DefOperandName: Reg->getName());
1589 assert(!M.hasOperand(OpName) && "The pattern should've been rejected");
1590 DstMIBuilder.setDeadImplicitDef(Reg);
1591 }
1592
1593 return InsertPt;
1594}
1595
1596Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
1597 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
1598 const TreePatternNode &Dst) const {
1599 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
1600 const CodeGenInstruction *OrigDstI =
1601 &Target.getInstruction(InstRec: Dst.getOperator());
1602
1603 StringRef Name = OrigDstI->getName();
1604 unsigned ExpectedDstINumUses = Dst.getNumChildren();
1605
1606 // EXTRACT_SUBREG needs to use a subregister COPY.
1607 if (Name == "EXTRACT_SUBREG") {
1608 if (!Dst.getChild(N: 1).isLeaf())
1609 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a leaf");
1610 const DefInit *SubRegInit =
1611 dyn_cast<DefInit>(Val: Dst.getChild(N: 1).getLeafValue());
1612 if (!SubRegInit)
1613 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a subreg index");
1614
1615 const CodeGenSubRegIndex *SubIdx =
1616 CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
1617 const TreePatternNode &ValChild = Dst.getChild(N: 0);
1618 if (!ValChild.isLeaf()) {
1619 // We really have to handle the source instruction, and then insert a
1620 // copy from the subregister.
1621 auto ExtractSrcTy = getInstResultType(Dst: ValChild, Target);
1622 if (!ExtractSrcTy)
1623 return ExtractSrcTy.takeError();
1624
1625 unsigned TempRegID = M.allocateTempRegID();
1626 InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *ExtractSrcTy,
1627 args&: TempRegID);
1628
1629 auto InsertPtOrError = createAndImportSubInstructionRenderer(
1630 InsertPt: ++InsertPt, M, Dst: ValChild, TempRegID);
1631 if (auto Error = InsertPtOrError.takeError())
1632 return std::move(Error);
1633
1634 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, args: false, args&: SubIdx);
1635 return InsertPt;
1636 }
1637
1638 // If this is a source operand, this is just a subregister copy.
1639 const Record *RCDef =
1640 Target.getInitValueAsRegClass(V: ValChild.getLeafValue());
1641 if (!RCDef)
1642 return failedImport(Reason: "EXTRACT_SUBREG child #0 could not "
1643 "be coerced to a register class");
1644
1645 CodeGenRegisterClass *RC = CGRegs.getRegClass(RCDef);
1646
1647 const auto SrcRCDstRCPair =
1648 RC->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1649 if (SrcRCDstRCPair) {
1650 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
1651 if (SrcRCDstRCPair->first != RC)
1652 return failedImport(Reason: "EXTRACT_SUBREG requires an additional COPY");
1653 }
1654
1655 StringRef RegOperandName = Dst.getChild(N: 0).getName();
1656 if (const auto &SubOperand = M.getComplexSubOperand(SymbolicName: RegOperandName)) {
1657 DstMIBuilder.addRenderer<RenderComplexPatternOperand>(
1658 args: *std::get<0>(t: *SubOperand), args&: RegOperandName, args: std::get<1>(t: *SubOperand),
1659 args: std::get<2>(t: *SubOperand), args&: SubIdx);
1660 return InsertPt;
1661 }
1662
1663 DstMIBuilder.addRenderer<CopySubRegRenderer>(args&: RegOperandName, args&: SubIdx);
1664 return InsertPt;
1665 }
1666
1667 if (Name == "REG_SEQUENCE") {
1668 if (!Dst.getChild(N: 0).isLeaf())
1669 return failedImport(Reason: "REG_SEQUENCE child #0 is not a leaf");
1670
1671 const Record *RCDef =
1672 Target.getInitValueAsRegClass(V: Dst.getChild(N: 0).getLeafValue());
1673 if (!RCDef)
1674 return failedImport(Reason: "REG_SEQUENCE child #0 could not "
1675 "be coerced to a register class");
1676
1677 if ((ExpectedDstINumUses - 1) % 2 != 0)
1678 return failedImport(Reason: "Malformed REG_SEQUENCE");
1679
1680 for (unsigned I = 1; I != ExpectedDstINumUses; I += 2) {
1681 const TreePatternNode &ValChild = Dst.getChild(N: I);
1682 const TreePatternNode &SubRegChild = Dst.getChild(N: I + 1);
1683
1684 if (const DefInit *SubRegInit =
1685 dyn_cast<DefInit>(Val: SubRegChild.getLeafValue())) {
1686 const CodeGenSubRegIndex *SubIdx =
1687 CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
1688
1689 if (Error Err = importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: ValChild, InsertPt))
1690 return Err;
1691
1692 DstMIBuilder.addRenderer<SubRegIndexRenderer>(args&: SubIdx);
1693 }
1694 }
1695
1696 return InsertPt;
1697 }
1698
1699 // Render the explicit uses.
1700 unsigned DstINumUses = OrigDstI->Operands.size() - OrigDstI->Operands.NumDefs;
1701 if (Name == "COPY_TO_REGCLASS") {
1702 DstINumUses--; // Ignore the class constraint.
1703 ExpectedDstINumUses--;
1704 }
1705
1706 // NumResults - This is the number of results produced by the instruction in
1707 // the "outs" list.
1708 unsigned NumResults = OrigDstI->Operands.NumDefs;
1709
1710 // Number of operands we know the output instruction must have. If it is
1711 // variadic, we could have more operands.
1712 unsigned NumFixedOperands = DstI->Operands.size();
1713
1714 // Loop over all of the fixed operands of the instruction pattern, emitting
1715 // code to fill them all in. The node 'N' usually has number children equal to
1716 // the number of input operands of the instruction. However, in cases where
1717 // there are predicate operands for an instruction, we need to fill in the
1718 // 'execute always' values. Match up the node operands to the instruction
1719 // operands to do this.
1720 unsigned Child = 0;
1721
1722 // Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the
1723 // number of operands at the end of the list which have default values.
1724 // Those can come from the pattern if it provides enough arguments, or be
1725 // filled in with the default if the pattern hasn't provided them. But any
1726 // operand with a default value _before_ the last mandatory one will be
1727 // filled in with their defaults unconditionally.
1728 unsigned NonOverridableOperands = NumFixedOperands;
1729 while (NonOverridableOperands > NumResults &&
1730 CGP.operandHasDefault(Op: DstI->Operands[NonOverridableOperands - 1].Rec))
1731 --NonOverridableOperands;
1732
1733 unsigned NumDefaultOps = 0;
1734 for (unsigned I = 0; I != DstINumUses; ++I) {
1735 unsigned InstOpNo = DstI->Operands.NumDefs + I;
1736
1737 // Determine what to emit for this operand.
1738 const Record *OperandNode = DstI->Operands[InstOpNo].Rec;
1739
1740 // If the operand has default values, introduce them now.
1741 if (CGP.operandHasDefault(Op: OperandNode) &&
1742 (InstOpNo < NonOverridableOperands || Child >= Dst.getNumChildren())) {
1743 // This is a predicate or optional def operand which the pattern has not
1744 // overridden, or which we aren't letting it override; emit the 'default
1745 // ops' operands.
1746 for (const TreePatternNode &OpNode :
1747 make_pointee_range(Range: CGP.getDefaultOperand(R: OperandNode).DefaultOps)) {
1748 if (Error Err = importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: OpNode, InsertPt))
1749 return Err;
1750 }
1751
1752 ++NumDefaultOps;
1753 continue;
1754 }
1755
1756 if (Error Err =
1757 importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: Dst.getChild(N: Child), InsertPt))
1758 return Err;
1759
1760 ++Child;
1761 }
1762
1763 if (NumDefaultOps + ExpectedDstINumUses != DstINumUses)
1764 return failedImport(Reason: "Expected " + llvm::to_string(Value: DstINumUses) +
1765 " used operands but found " +
1766 llvm::to_string(Value: ExpectedDstINumUses) +
1767 " explicit ones and " + llvm::to_string(Value: NumDefaultOps) +
1768 " default ones");
1769
1770 return InsertPt;
1771}
1772
1773Error GlobalISelEmitter::importImplicitDefRenderers(
1774 BuildMIAction &DstMIBuilder, ArrayRef<const Record *> ImplicitDefs) const {
1775 if (!ImplicitDefs.empty())
1776 return failedImport(Reason: "Pattern defines a physical register");
1777 return Error::success();
1778}
1779
1780Error GlobalISelEmitter::constrainOperands(action_iterator InsertPt,
1781 RuleMatcher &M, unsigned InsnID,
1782 const TreePatternNode &Dst) const {
1783 const Record *DstOp = Dst.getOperator();
1784 const CodeGenInstruction &DstI = Target.getInstruction(InstRec: DstOp);
1785 StringRef DstIName = DstI.getName();
1786
1787 if (DstIName == "COPY_TO_REGCLASS") {
1788 // COPY_TO_REGCLASS does not provide operand constraints itself but the
1789 // result is constrained to the class given by the second child.
1790 const Record *DstIOpRec =
1791 Target.getInitValueAsRegClass(V: Dst.getChild(N: 1).getLeafValue());
1792
1793 if (DstIOpRec == nullptr)
1794 return failedImport(Reason: "COPY_TO_REGCLASS operand #1 isn't a register class");
1795
1796 M.insertAction<ConstrainOperandToRegClassAction>(
1797 InsertPt, args&: InsnID, args: 0, args: Target.getRegisterClass(R: DstIOpRec));
1798 } else if (DstIName == "EXTRACT_SUBREG") {
1799 const CodeGenRegisterClass *SuperClass =
1800 inferRegClassFromPattern(N: Dst.getChild(N: 0));
1801 if (!SuperClass)
1802 return failedImport(
1803 Reason: "Cannot infer register class from EXTRACT_SUBREG operand #0");
1804
1805 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: Dst.getChild(N: 1));
1806 if (!SubIdx)
1807 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a subreg index");
1808
1809 // It would be nice to leave this constraint implicit but we're required
1810 // to pick a register class so constrain the result to a register class
1811 // that can hold the correct MVT.
1812 //
1813 // FIXME: This may introduce an extra copy if the chosen class doesn't
1814 // actually contain the subregisters.
1815 const auto SrcRCDstRCPair =
1816 SuperClass->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1817 if (!SrcRCDstRCPair) {
1818 return failedImport(Reason: "subreg index is incompatible "
1819 "with inferred reg class");
1820 }
1821
1822 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
1823 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1824 args&: *SrcRCDstRCPair->second);
1825 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 1,
1826 args&: *SrcRCDstRCPair->first);
1827 } else if (DstIName == "INSERT_SUBREG") {
1828 // We need to constrain the destination, a super regsister source, and a
1829 // subregister source.
1830 const CodeGenRegisterClass *SubClass =
1831 inferRegClassFromPattern(N: Dst.getChild(N: 1));
1832 if (!SubClass)
1833 return failedImport(
1834 Reason: "Cannot infer register class from INSERT_SUBREG operand #1");
1835 const CodeGenRegisterClass *SuperClass = inferSuperRegisterClassForNode(
1836 Ty: Dst.getExtType(ResNo: 0), SuperRegNode: Dst.getChild(N: 0), SubRegIdxNode: Dst.getChild(N: 2));
1837 if (!SuperClass)
1838 return failedImport(
1839 Reason: "Cannot infer register class for INSERT_SUBREG operand #0");
1840 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1841 args: *SuperClass);
1842 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 1,
1843 args: *SuperClass);
1844 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 2,
1845 args: *SubClass);
1846 } else if (DstIName == "SUBREG_TO_REG") {
1847 // We need to constrain the destination and subregister source.
1848 // Attempt to infer the subregister source from the first child. If it has
1849 // an explicitly given register class, we'll use that. Otherwise, we will
1850 // fail.
1851 const CodeGenRegisterClass *SubClass =
1852 inferRegClassFromPattern(N: Dst.getChild(N: 1));
1853 if (!SubClass)
1854 return failedImport(
1855 Reason: "Cannot infer register class from SUBREG_TO_REG child #1");
1856 // We don't have a child to look at that might have a super register node.
1857 const CodeGenRegisterClass *SuperClass =
1858 inferSuperRegisterClass(Ty: Dst.getExtType(ResNo: 0), SubRegIdxNode: Dst.getChild(N: 2));
1859 if (!SuperClass)
1860 return failedImport(
1861 Reason: "Cannot infer register class for SUBREG_TO_REG operand #0");
1862 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1863 args: *SuperClass);
1864 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 2,
1865 args: *SubClass);
1866 } else if (DstIName == "REG_SEQUENCE") {
1867 const CodeGenRegisterClass *SuperClass =
1868 inferRegClassFromPattern(N: Dst.getChild(N: 0));
1869
1870 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1871 args: *SuperClass);
1872
1873 unsigned Num = Dst.getNumChildren();
1874 for (unsigned I = 1; I != Num; I += 2) {
1875 const TreePatternNode &SubRegChild = Dst.getChild(N: I + 1);
1876
1877 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: SubRegChild);
1878 if (!SubIdx)
1879 return failedImport(Reason: "REG_SEQUENCE child is not a subreg index");
1880
1881 const auto SrcRCDstRCPair =
1882 SuperClass->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1883
1884 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args&: I,
1885 args&: *SrcRCDstRCPair->second);
1886 }
1887 } else {
1888 M.insertAction<ConstrainOperandsToDefinitionAction>(InsertPt, args&: InsnID);
1889 }
1890
1891 return Error::success();
1892}
1893
1894const CodeGenRegisterClass *
1895GlobalISelEmitter::getRegClassFromLeaf(const TreePatternNode &Leaf) const {
1896 assert(Leaf.isLeaf() && "Expected leaf?");
1897 const Record *RCRec = Target.getInitValueAsRegClass(V: Leaf.getLeafValue());
1898 if (!RCRec)
1899 return nullptr;
1900 return CGRegs.getRegClass(RCRec);
1901}
1902
1903const CodeGenRegisterClass *
1904GlobalISelEmitter::inferRegClassFromPattern(const TreePatternNode &N) const {
1905 if (N.isLeaf())
1906 return getRegClassFromLeaf(Leaf: N);
1907
1908 // We don't have a leaf node, so we have to try and infer something. Check
1909 // that we have an instruction that we can infer something from.
1910
1911 // Only handle things that produce at least one value (if multiple values,
1912 // just take the first one).
1913 if (N.getNumTypes() < 1)
1914 return nullptr;
1915 const Record *OpRec = N.getOperator();
1916
1917 // We only want instructions.
1918 if (!OpRec->isSubClassOf(Name: "Instruction"))
1919 return nullptr;
1920
1921 // Don't want to try and infer things when there could potentially be more
1922 // than one candidate register class.
1923 return inferRegClassFromInstructionPattern(N, /*ResIdx=*/0);
1924}
1925
1926const CodeGenRegisterClass *
1927GlobalISelEmitter::inferRegClassFromRegisterClassLike(
1928 const Record *RegClassDef) const {
1929 if (RegClassDef->isSubClassOf(Name: "RegClassByHwMode")) {
1930 // TODO: We only are trying to match the regbank, which we assume is the
1931 // same across modes, but we are just picking one class and assuming they
1932 // all have the same bank.
1933 //
1934 // We should verify there is a common regbank among the possible classes,
1935 // and return that instead of a concrete class.
1936 const HwModeSelect &ModeSelect =
1937 Target.getHwModes().getHwModeSelect(R: RegClassDef);
1938 if (ModeSelect.Items.empty())
1939 return nullptr;
1940 return Target.getRegBank().getRegClass(ModeSelect.Items.front().second);
1941 }
1942
1943 return &Target.getRegisterClass(R: RegClassDef);
1944}
1945
1946const CodeGenRegisterClass *
1947GlobalISelEmitter::inferRegClassFromInstructionPattern(const TreePatternNode &N,
1948 unsigned ResIdx) const {
1949 const CodeGenInstruction &Inst = Target.getInstruction(InstRec: N.getOperator());
1950 assert(ResIdx < Inst.Operands.NumDefs &&
1951 "Can only infer register class for explicit defs");
1952
1953 // Handle any special-case instructions which we can safely infer register
1954 // classes from.
1955 StringRef InstName = Inst.getName();
1956 if (InstName == "REG_SEQUENCE") {
1957 // (outs $super_dst), (ins $dst_regclass, variable_ops)
1958 // Destination register class is explicitly specified by the first operand.
1959 const TreePatternNode &RCChild = N.getChild(N: 0);
1960 if (!RCChild.isLeaf())
1961 return nullptr;
1962 return getRegClassFromLeaf(Leaf: RCChild);
1963 }
1964
1965 if (InstName == "COPY_TO_REGCLASS") {
1966 // (outs $dst), (ins $src, $dst_regclass)
1967 // Destination register class is explicitly specified by the second operand.
1968 const TreePatternNode &RCChild = N.getChild(N: 1);
1969 if (!RCChild.isLeaf())
1970 return nullptr;
1971 return getRegClassFromLeaf(Leaf: RCChild);
1972 }
1973
1974 if (InstName == "INSERT_SUBREG") {
1975 // (outs $super_dst), (ins $super_src, $sub_src, $sub_idx);
1976 // If we can infer the register class for the first operand, use that.
1977 // Otherwise, find a register class that supports both the specified
1978 // sub-register index and the type of the instruction's result.
1979 const TreePatternNode &Child0 = N.getChild(N: 0);
1980 assert(Child0.getNumTypes() == 1 && "Unexpected number of types!");
1981 return inferSuperRegisterClassForNode(Ty: N.getExtType(ResNo: 0), SuperRegNode: Child0,
1982 SubRegIdxNode: N.getChild(N: 2));
1983 }
1984
1985 if (InstName == "EXTRACT_SUBREG") {
1986 // (outs $sub_dst), (ins $super_src, $sub_idx)
1987 // Find a register class that can be used for a sub-register copy from
1988 // the specified source at the specified sub-register index.
1989 const CodeGenRegisterClass *SuperRC =
1990 inferRegClassFromPattern(N: N.getChild(N: 0));
1991 if (!SuperRC)
1992 return nullptr;
1993
1994 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: N.getChild(N: 1));
1995 if (!SubIdx)
1996 return nullptr;
1997
1998 const auto SubRCAndSubRegRC =
1999 SuperRC->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
2000 if (!SubRCAndSubRegRC)
2001 return nullptr;
2002
2003 return SubRCAndSubRegRC->second;
2004 }
2005
2006 if (InstName == "SUBREG_TO_REG") {
2007 // (outs $super_dst), (ins $super_src, $sub_src, $sub_idx)
2008 // Find a register class that supports both the specified sub-register
2009 // index and the type of the instruction's result.
2010 return inferSuperRegisterClass(Ty: N.getExtType(ResNo: 0), SubRegIdxNode: N.getChild(N: 2));
2011 }
2012
2013 // Handle destination record types that we can safely infer a register class
2014 // from.
2015 const auto &DstIOperand = Inst.Operands[ResIdx];
2016 const Record *DstIOpRec = DstIOperand.Rec;
2017
2018 if (DstIOpRec->isSubClassOf(Name: "RegisterOperand")) {
2019 const Record *RegClassDef = DstIOpRec->getValueAsDef(FieldName: "RegClass");
2020 return inferRegClassFromRegisterClassLike(RegClassDef);
2021 }
2022
2023 if (DstIOpRec->isSubClassOf(Name: "RegClassByHwMode"))
2024 return inferRegClassFromRegisterClassLike(RegClassDef: DstIOpRec);
2025
2026 if (DstIOpRec->isSubClassOf(Name: "RegisterClass"))
2027 return &Target.getRegisterClass(R: DstIOpRec);
2028
2029 return nullptr;
2030}
2031
2032const CodeGenRegisterClass *GlobalISelEmitter::inferSuperRegisterClass(
2033 const TypeSetByHwMode &Ty, const TreePatternNode &SubRegIdxNode) const {
2034 // We need a ValueTypeByHwMode for getSuperRegForSubReg.
2035 if (!Ty.isValueTypeByHwMode(AllowEmpty: false))
2036 return nullptr;
2037 if (!SubRegIdxNode.isLeaf())
2038 return nullptr;
2039 const DefInit *SubRegInit = dyn_cast<DefInit>(Val: SubRegIdxNode.getLeafValue());
2040 if (!SubRegInit)
2041 return nullptr;
2042 const CodeGenSubRegIndex *SubIdx = CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
2043
2044 // Use the information we found above to find a minimal register class which
2045 // supports the subregister and type we want.
2046 return CGRegs.getSuperRegForSubReg(Ty: Ty.getValueTypeByHwMode(), SubIdx,
2047 /*MustBeAllocatable=*/true);
2048}
2049
2050const CodeGenRegisterClass *GlobalISelEmitter::inferSuperRegisterClassForNode(
2051 const TypeSetByHwMode &Ty, const TreePatternNode &SuperRegNode,
2052 const TreePatternNode &SubRegIdxNode) const {
2053 // Check if we already have a defined register class for the super register
2054 // node. If we do, then we should preserve that rather than inferring anything
2055 // from the subregister index node. We can assume that whoever wrote the
2056 // pattern in the first place made sure that the super register and
2057 // subregister are compatible.
2058 if (const CodeGenRegisterClass *SuperRegisterClass =
2059 inferRegClassFromPattern(N: SuperRegNode))
2060 return SuperRegisterClass;
2061 return inferSuperRegisterClass(Ty, SubRegIdxNode);
2062}
2063
2064const CodeGenSubRegIndex *GlobalISelEmitter::inferSubRegIndexForNode(
2065 const TreePatternNode &SubRegIdxNode) const {
2066 if (!SubRegIdxNode.isLeaf())
2067 return nullptr;
2068
2069 const DefInit *SubRegInit = dyn_cast<DefInit>(Val: SubRegIdxNode.getLeafValue());
2070 if (!SubRegInit)
2071 return nullptr;
2072 return CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
2073}
2074
2075Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
2076 // Keep track of the matchers and actions to emit.
2077 int Score = P.getPatternComplexity(CGP);
2078 RuleMatcher M(P.getSrcRecord()->getLoc());
2079 RuleMatcherScores[M.getRuleID()] = Score;
2080 M.addAction<DebugCommentAction>(args: llvm::to_string(Value: P.getSrcPattern()) +
2081 " => " +
2082 llvm::to_string(Value: P.getDstPattern()));
2083
2084 SmallVector<const Record *, 4> Predicates;
2085 P.getPredicateRecords(PredicateRecs&: Predicates);
2086 if (auto Error = importRulePredicates(M, Predicates))
2087 return std::move(Error);
2088
2089 if (!P.getHwModeFeatures().empty())
2090 M.addHwModeIdx(Idx: declareHwModeCheck(HwModeFeatures: P.getHwModeFeatures()));
2091
2092 // Next, analyze the pattern operators.
2093 TreePatternNode &Src = P.getSrcPattern();
2094 TreePatternNode &Dst = P.getDstPattern();
2095
2096 // If the root of either pattern isn't a simple operator, ignore it.
2097 if (auto Err = isTrivialOperatorNode(N: Dst))
2098 return failedImport(Reason: "Dst pattern root isn't a trivial operator (" +
2099 toString(E: std::move(Err)) + ")");
2100 if (auto Err = isTrivialOperatorNode(N: Src))
2101 return failedImport(Reason: "Src pattern root isn't a trivial operator (" +
2102 toString(E: std::move(Err)) + ")");
2103
2104 // The different predicates and matchers created during
2105 // addInstructionMatcher use the RuleMatcher M to set up their
2106 // instruction ID (InsnVarID) that are going to be used when
2107 // M is going to be emitted.
2108 // However, the code doing the emission still relies on the IDs
2109 // returned during that process by the RuleMatcher when issuing
2110 // the recordInsn opcodes.
2111 // Because of that:
2112 // 1. The order in which we created the predicates
2113 // and such must be the same as the order in which we emit them,
2114 // and
2115 // 2. We need to reset the generation of the IDs in M somewhere between
2116 // addInstructionMatcher and emit
2117 //
2118 // FIXME: Long term, we don't want to have to rely on this implicit
2119 // naming being the same. One possible solution would be to have
2120 // explicit operator for operation capture and reference those.
2121 // The plus side is that it would expose opportunities to share
2122 // the capture accross rules. The downside is that it would
2123 // introduce a dependency between predicates (captures must happen
2124 // before their first use.)
2125 InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(SymbolicName: Src.getName());
2126 unsigned TempOpIdx = 0;
2127
2128 const auto SavedFlags = M.setGISelFlags(P.getSrcRecord());
2129
2130 auto InsnMatcherOrError =
2131 createAndImportSelDAGMatcher(Rule&: M, InsnMatcher&: InsnMatcherTemp, Src, TempOpIdx);
2132 if (auto Error = InsnMatcherOrError.takeError())
2133 return std::move(Error);
2134 InstructionMatcher &InsnMatcher = InsnMatcherOrError.get();
2135
2136 if (Dst.isLeaf()) {
2137 if (const Record *RCDef =
2138 Target.getInitValueAsRegClass(V: Dst.getLeafValue())) {
2139 const CodeGenRegisterClass &RC = Target.getRegisterClass(R: RCDef);
2140
2141 // We need to replace the def and all its uses with the specified
2142 // operand. However, we must also insert COPY's wherever needed.
2143 // For now, emit a copy and let the register allocator clean up.
2144 const CodeGenInstruction &DstI = Target.getInstruction(InstRec: RK.getDef(Name: "COPY"));
2145 const auto &DstIOperand = DstI.Operands[0];
2146
2147 OperandMatcher &OM0 = InsnMatcher.getOperand(OpIdx: 0);
2148 OM0.setSymbolicName(DstIOperand.Name);
2149 M.defineOperand(SymbolicName: OM0.getSymbolicName(), OM&: OM0);
2150 OM0.addPredicate<RegisterBankOperandMatcher>(args: RC);
2151
2152 auto &DstMIBuilder =
2153 M.addAction<BuildMIAction>(args: M.allocateOutputInsnID(), args: &DstI);
2154 DstMIBuilder.addRenderer<CopyRenderer>(args: DstIOperand.Name);
2155 DstMIBuilder.addRenderer<CopyRenderer>(args: Dst.getName());
2156 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 0, args: RC);
2157
2158 // Erase the root.
2159 unsigned RootInsnID = M.getInsnVarID(InsnMatcher);
2160 M.addAction<EraseInstAction>(args&: RootInsnID);
2161
2162 // We're done with this pattern! It's eligible for GISel emission; return
2163 // it.
2164 ++NumPatternImported;
2165 return std::move(M);
2166 }
2167
2168 return failedImport(Reason: "Dst pattern root isn't a known leaf");
2169 }
2170
2171 // Start with the defined operands (i.e., the results of the root operator).
2172 const Record *DstOp = Dst.getOperator();
2173 if (!DstOp->isSubClassOf(Name: "Instruction"))
2174 return failedImport(Reason: "Pattern operator isn't an instruction");
2175
2176 const CodeGenInstruction &DstI = Target.getInstruction(InstRec: DstOp);
2177
2178 // Count both implicit and explicit defs in the dst instruction.
2179 // This avoids errors importing patterns that have inherent implicit defs.
2180 unsigned DstExpDefs = DstI.Operands.NumDefs,
2181 DstNumDefs = DstI.ImplicitDefs.size() + DstExpDefs,
2182 SrcNumDefs = Src.getExtTypes().size();
2183 if (DstNumDefs < SrcNumDefs) {
2184 if (DstNumDefs != 0)
2185 return failedImport(Reason: "Src pattern result has more defs than dst MI (" +
2186 to_string(Value: SrcNumDefs) + " def(s) vs " +
2187 to_string(Value: DstNumDefs) + " def(s))");
2188
2189 bool FoundNoUsePred = false;
2190 for (const auto &Pred : InsnMatcher.predicates()) {
2191 if ((FoundNoUsePred = isa<NoUsePredicateMatcher>(Val: Pred.get())))
2192 break;
2193 }
2194 if (!FoundNoUsePred)
2195 return failedImport(Reason: "Src pattern result has " + to_string(Value: SrcNumDefs) +
2196 " def(s) without the HasNoUse predicate set to true "
2197 "but Dst MI has no def");
2198 }
2199
2200 // The root of the match also has constraints on the register bank so that it
2201 // matches the result instruction.
2202 unsigned N = std::min(a: DstExpDefs, b: SrcNumDefs);
2203 for (unsigned I = 0; I < N; ++I) {
2204 const auto &DstIOperand = DstI.Operands[I];
2205
2206 OperandMatcher &OM = InsnMatcher.getOperand(OpIdx: I);
2207 // The operand names declared in the DstI instruction are unrelated to
2208 // those used in pattern's source and destination DAGs, so mangle the
2209 // former to prevent implicitly adding unexpected
2210 // GIM_CheckIsSameOperand predicates by the defineOperand method.
2211 OM.setSymbolicName(getMangledRootDefName(DefOperandName: DstIOperand.Name));
2212 M.defineOperand(SymbolicName: OM.getSymbolicName(), OM);
2213
2214 const CodeGenRegisterClass *RC =
2215 inferRegClassFromInstructionPattern(N: Dst, ResIdx: I);
2216 if (!RC)
2217 return failedImport(Reason: "Could not infer register class for result #" +
2218 Twine(I) + " from pattern " + to_string(Value: Dst));
2219 OM.addPredicate<RegisterBankOperandMatcher>(args: *RC);
2220 }
2221
2222 auto DstMIBuilderOrError =
2223 createAndImportInstructionRenderer(M, InsnMatcher, Dst);
2224 if (auto Error = DstMIBuilderOrError.takeError())
2225 return std::move(Error);
2226 BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get();
2227
2228 // Render the implicit defs.
2229 // These are only added to the root of the result.
2230 if (auto Error = importImplicitDefRenderers(DstMIBuilder, ImplicitDefs: P.getDstRegs()))
2231 return std::move(Error);
2232
2233 DstMIBuilder.chooseInsnToMutate(Rule&: M);
2234
2235 // Constrain the registers to classes. This is normally derived from the
2236 // emitted instruction but a few instructions require special handling.
2237 if (auto Error =
2238 constrainOperands(InsertPt: M.actions_end(), M, InsnID: DstMIBuilder.getInsnID(), Dst))
2239 return std::move(Error);
2240
2241 // Erase the root.
2242 unsigned RootInsnID = M.getInsnVarID(InsnMatcher);
2243 M.addAction<EraseInstAction>(args&: RootInsnID);
2244
2245 // We're done with this pattern! It's eligible for GISel emission; return it.
2246 ++NumPatternImported;
2247 return std::move(M);
2248}
2249
2250MatchTable
2251GlobalISelEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules,
2252 bool Optimize, bool WithCoverage) {
2253 std::vector<Matcher *> InputRules;
2254 for (Matcher &Rule : Rules)
2255 InputRules.push_back(x: &Rule);
2256
2257 if (!Optimize)
2258 return MatchTable::buildTable(Rules: InputRules, WithCoverage);
2259
2260 unsigned CurrentOrdering = 0;
2261 StringMap<unsigned> OpcodeOrder;
2262 for (RuleMatcher &Rule : Rules) {
2263 const StringRef Opcode = Rule.getOpcode();
2264 assert(!Opcode.empty() && "Didn't expect an undefined opcode");
2265 if (OpcodeOrder.try_emplace(Key: Opcode, Args&: CurrentOrdering).second)
2266 ++CurrentOrdering;
2267 }
2268
2269 llvm::stable_sort(
2270 Range&: InputRules, C: [&OpcodeOrder](const Matcher *A, const Matcher *B) {
2271 auto *L = static_cast<const RuleMatcher *>(A);
2272 auto *R = static_cast<const RuleMatcher *>(B);
2273 return std::tuple(OpcodeOrder[L->getOpcode()],
2274 L->insnmatchers_front().getNumOperandMatchers()) <
2275 std::tuple(OpcodeOrder[R->getOpcode()],
2276 R->insnmatchers_front().getNumOperandMatchers());
2277 });
2278
2279 for (Matcher *Rule : InputRules)
2280 Rule->optimize();
2281
2282 std::vector<std::unique_ptr<Matcher>> MatcherStorage;
2283 std::vector<Matcher *> OptRules =
2284 optimizeRules<GroupMatcher>(Rules: InputRules, MatcherStorage);
2285
2286 for (Matcher *Rule : OptRules)
2287 Rule->optimize();
2288
2289 OptRules = optimizeRules<SwitchMatcher>(Rules: OptRules, MatcherStorage);
2290
2291 return MatchTable::buildTable(Rules: OptRules, WithCoverage);
2292}
2293
2294void GlobalISelEmitter::emitAdditionalImpl(raw_ostream &OS) {
2295 OS << "bool " << getClassName()
2296 << "::selectImpl(MachineInstr &I, CodeGenCoverage "
2297 "&CoverageInfo) const {\n"
2298 << " const PredicateBitset AvailableFeatures = "
2299 "getAvailableFeatures();\n"
2300 << " MachineIRBuilder B(I);\n"
2301 << " State.MIs.clear();\n"
2302 << " State.MIs.push_back(&I);\n\n"
2303 << " if (executeMatchTable(*this, State, ExecInfo, B"
2304 << ", getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures"
2305 << ", &CoverageInfo)) {\n"
2306 << " return true;\n"
2307 << " }\n\n"
2308 << " return false;\n"
2309 << "}\n\n";
2310}
2311
2312void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) {
2313 std::vector<const Record *> MatchedRecords;
2314 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2315 P: [](const Record *R) {
2316 return !R->getValueAsString(FieldName: "GISelPredicateCode").empty();
2317 });
2318 emitMIPredicateFnsImpl<const Record *>(
2319 OS,
2320 AdditionalDecls: " const MachineFunction &MF = *MI.getParent()->getParent();\n"
2321 " const MachineRegisterInfo &MRI = MF.getRegInfo();\n"
2322 " const auto &Operands = State.RecordedOperands;\n"
2323 " (void)Operands;\n"
2324 " (void)MRI;",
2325 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2326 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "GISelPredicateCode"); },
2327 Comment: "PatFrag predicates.");
2328}
2329
2330void GlobalISelEmitter::emitLeafPredicateFns(raw_ostream &OS) {
2331 std::vector<const Record *> MatchedRecords;
2332 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2333 P: [](const Record *R) {
2334 return (!R->getValueAsOptionalString(FieldName: "GISelLeafPredicateCode")
2335 .value_or(u: std::string())
2336 .empty());
2337 });
2338 emitLeafPredicateFnsImpl<const Record *>(
2339 OS,
2340 AdditionalDecls: " const auto &Operands = State.RecordedOperands;\n"
2341 " Register Reg = MO.getReg();\n"
2342 " (void)Operands;\n"
2343 " (void)Reg;",
2344 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2345 GetPredCode: [](const Record *R) {
2346 return R->getValueAsString(FieldName: "GISelLeafPredicateCode");
2347 },
2348 Comment: "PatFrag predicates.");
2349}
2350
2351void GlobalISelEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
2352 std::vector<const Record *> MatchedRecords;
2353 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2354 P: [](const Record *R) {
2355 bool Unset;
2356 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2357 !R->getValueAsBitOrUnset(FieldName: "IsAPFloat", Unset) &&
2358 !R->getValueAsBit(FieldName: "IsAPInt");
2359 });
2360 emitImmPredicateFnsImpl<const Record *>(
2361 OS, TypeIdentifier: "I64", ArgType: "int64_t", Predicates: ArrayRef<const Record *>(MatchedRecords),
2362 GetPredEnumName: &getPatFragPredicateEnumName,
2363 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2364 Comment: "PatFrag predicates.");
2365}
2366
2367void GlobalISelEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) {
2368 std::vector<const Record *> MatchedRecords;
2369 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2370 P: [](const Record *R) {
2371 bool Unset;
2372 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2373 R->getValueAsBitOrUnset(FieldName: "IsAPFloat", Unset);
2374 });
2375 emitImmPredicateFnsImpl<const Record *>(
2376 OS, TypeIdentifier: "APFloat", ArgType: "const APFloat &",
2377 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2378 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2379 Comment: "PatFrag predicates.");
2380}
2381
2382void GlobalISelEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) {
2383 std::vector<const Record *> MatchedRecords;
2384 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2385 P: [](const Record *R) {
2386 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2387 R->getValueAsBit(FieldName: "IsAPInt");
2388 });
2389 emitImmPredicateFnsImpl<const Record *>(
2390 OS, TypeIdentifier: "APInt", ArgType: "const APInt &", Predicates: ArrayRef<const Record *>(MatchedRecords),
2391 GetPredEnumName: &getPatFragPredicateEnumName,
2392 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2393 Comment: "PatFrag predicates.");
2394}
2395
2396void GlobalISelEmitter::emitTestSimplePredicate(raw_ostream &OS) {
2397 OS << "bool " << getClassName() << "::testSimplePredicate(unsigned) const {\n"
2398 << " llvm_unreachable(\"" + getClassName() +
2399 " does not support simple predicates!\");\n"
2400 << " return false;\n"
2401 << "}\n";
2402}
2403
2404void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) {
2405 OS << "bool " << getClassName()
2406 << "::runCustomAction(unsigned, const MatcherState&, NewMIVector &) const "
2407 "{\n"
2408 << " llvm_unreachable(\"" + getClassName() +
2409 " does not support custom C++ actions!\");\n"
2410 << "}\n";
2411}
2412
2413bool hasBFloatType(const TreePatternNode &Node) {
2414 for (unsigned I = 0, E = Node.getNumTypes(); I < E; I++) {
2415 MVT VT = Node.getSimpleType(ResNo: I);
2416 if (VT.getScalarType() == MVT::bf16)
2417 return true;
2418 }
2419 for (const TreePatternNode &C : Node.children())
2420 if (hasBFloatType(Node: C))
2421 return true;
2422 return false;
2423}
2424
2425void GlobalISelEmitter::run(raw_ostream &OS) {
2426 if (!UseCoverageFile.empty()) {
2427 RuleCoverage = CodeGenCoverage();
2428 auto RuleCoverageBufOrErr = MemoryBuffer::getFile(Filename: UseCoverageFile);
2429 if (!RuleCoverageBufOrErr) {
2430 PrintWarning(WarningLoc: SMLoc(), Msg: "Missing rule coverage data");
2431 RuleCoverage = std::nullopt;
2432 } else {
2433 if (!RuleCoverage->parse(Buffer&: *RuleCoverageBufOrErr.get(), BackendName: Target.getName())) {
2434 PrintWarning(WarningLoc: SMLoc(), Msg: "Ignoring invalid or missing rule coverage data");
2435 RuleCoverage = std::nullopt;
2436 }
2437 }
2438 }
2439
2440 // Track the run-time opcode values
2441 gatherOpcodeValues();
2442 // Track the run-time LLT ID values
2443 gatherTypeIDValues();
2444
2445 // Track the GINodeEquiv definitions.
2446 gatherNodeEquivs();
2447
2448 AllPatFrags = RK.getAllDerivedDefinitions(ClassName: "PatFrags");
2449
2450 emitSourceFileHeader(
2451 Desc: ("Global Instruction Selector for the " + Target.getName() + " target")
2452 .str(),
2453 OS);
2454 std::vector<RuleMatcher> Rules;
2455 // Look through the SelectionDAG patterns we found, possibly emitting some.
2456 for (const PatternToMatch &Pat : CGP.ptms()) {
2457 ++NumPatternTotal;
2458
2459 if (Pat.getGISelShouldIgnore())
2460 continue; // skip without warning
2461
2462 // Skip any patterns containing BF16 types, as GISel cannot currently tell
2463 // the difference between fp16 and bf16. FIXME: This can be removed once
2464 // BF16 is supported properly.
2465 if (hasBFloatType(Node: Pat.getSrcPattern()))
2466 continue;
2467
2468 auto MatcherOrErr = runOnPattern(P: Pat);
2469
2470 // The pattern analysis can fail, indicating an unsupported pattern.
2471 // Report that if we've been asked to do so.
2472 if (auto Err = MatcherOrErr.takeError()) {
2473 if (WarnOnSkippedPatterns) {
2474 PrintWarning(WarningLoc: Pat.getSrcRecord()->getLoc(),
2475 Msg: "Skipped pattern: " + toString(E: std::move(Err)));
2476 } else {
2477 consumeError(Err: std::move(Err));
2478 }
2479 ++NumPatternImportsSkipped;
2480 continue;
2481 }
2482
2483 if (RuleCoverage) {
2484 if (RuleCoverage->isCovered(RuleID: MatcherOrErr->getRuleID()))
2485 ++NumPatternsTested;
2486 else
2487 PrintWarning(WarningLoc: Pat.getSrcRecord()->getLoc(),
2488 Msg: "Pattern is not covered by a test");
2489 }
2490 Rules.push_back(x: std::move(MatcherOrErr.get()));
2491 }
2492
2493 // Comparison function to order records by name.
2494 auto OrderByName = [](const Record *A, const Record *B) {
2495 return A->getName() < B->getName();
2496 };
2497
2498 std::vector<const Record *> ComplexPredicates =
2499 RK.getAllDerivedDefinitions(ClassName: "GIComplexOperandMatcher");
2500 llvm::sort(C&: ComplexPredicates, Comp: OrderByName);
2501
2502 std::vector<StringRef> CustomRendererFns;
2503 transform(Range: RK.getAllDerivedDefinitions(ClassName: "GICustomOperandRenderer"),
2504 d_first: std::back_inserter(x&: CustomRendererFns), F: [](const auto &Record) {
2505 return Record->getValueAsString("RendererFn");
2506 });
2507 // Sort and remove duplicates to get a list of unique renderer functions, in
2508 // case some were mentioned more than once.
2509 llvm::sort(C&: CustomRendererFns);
2510 CustomRendererFns.erase(first: llvm::unique(R&: CustomRendererFns),
2511 last: CustomRendererFns.end());
2512
2513 // Create a table containing the LLT objects needed by the matcher and an enum
2514 // for the matcher to reference them with.
2515 std::vector<LLTCodeGen> TypeObjects;
2516 append_range(C&: TypeObjects, R&: KnownTypes);
2517 llvm::sort(C&: TypeObjects);
2518
2519 // Sort rules.
2520 llvm::stable_sort(Range&: Rules, C: [&](const RuleMatcher &A, const RuleMatcher &B) {
2521 int ScoreA = RuleMatcherScores[A.getRuleID()];
2522 int ScoreB = RuleMatcherScores[B.getRuleID()];
2523 if (ScoreA > ScoreB)
2524 return true;
2525 if (ScoreB > ScoreA)
2526 return false;
2527 if (A.isHigherPriorityThan(B)) {
2528 assert(!B.isHigherPriorityThan(A) && "Cannot be more important "
2529 "and less important at "
2530 "the same time");
2531 return true;
2532 }
2533 return false;
2534 });
2535
2536 unsigned MaxTemporaries = 0;
2537 for (const auto &Rule : Rules)
2538 MaxTemporaries = std::max(a: MaxTemporaries, b: Rule.countRendererFns());
2539
2540 // Build match table
2541 const MatchTable Table =
2542 buildMatchTable(Rules, Optimize: OptimizeMatchTable, WithCoverage: GenerateCoverage);
2543
2544 emitPredicateBitset(OS, IfDefName: "GET_GLOBALISEL_PREDICATE_BITSET");
2545 emitTemporariesDecl(OS, IfDefName: "GET_GLOBALISEL_TEMPORARIES_DECL");
2546 emitTemporariesInit(OS, MaxTemporaries, IfDefName: "GET_GLOBALISEL_TEMPORARIES_INIT");
2547 emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexOperandMatchers: ComplexPredicates,
2548 CustomOperandRenderers: CustomRendererFns, IfDefName: "GET_GLOBALISEL_IMPL");
2549 emitPredicatesDecl(OS, IfDefName: "GET_GLOBALISEL_PREDICATES_DECL");
2550 emitPredicatesInit(OS, IfDefName: "GET_GLOBALISEL_PREDICATES_INIT");
2551}
2552
2553void GlobalISelEmitter::declareSubtargetFeature(const Record *Predicate) {
2554 SubtargetFeatures.try_emplace(k: Predicate, args&: Predicate, args: SubtargetFeatures.size());
2555}
2556
2557unsigned GlobalISelEmitter::declareHwModeCheck(StringRef HwModeFeatures) {
2558 return HwModes.emplace(args: HwModeFeatures.str(), args: HwModes.size()).first->second;
2559}
2560
2561//===----------------------------------------------------------------------===//
2562
2563static TableGen::Emitter::OptClass<GlobalISelEmitter>
2564 X("gen-global-isel", "Generate GlobalISel selector");
2565