1
2//===- GlobalISelEmitter.cpp - Generate an instruction selector -----------===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// This tablegen backend emits code for use by the GlobalISel instruction
12/// selector. See include/llvm/Target/GlobalISel/Target.td.
13///
14/// This file analyzes the patterns recognized by the SelectionDAGISel tablegen
15/// backend, filters out the ones that are unsupported, maps
16/// SelectionDAG-specific constructs to their GlobalISel counterpart
17/// (when applicable: MVT to LLT; SDNode to generic Instruction).
18///
19/// Not all patterns are supported: pass the tablegen invocation
20/// "-warn-on-skipped-patterns" to emit a warning when a pattern is skipped,
21/// as well as why.
22///
23/// The generated file defines a single method:
24/// bool <Target>InstructionSelector::selectImpl(MachineInstr &I) const;
25/// intended to be used in InstructionSelector::select as the first-step
26/// selector for the patterns that don't require complex C++.
27///
28/// FIXME: We'll probably want to eventually define a base
29/// "TargetGenInstructionSelector" class.
30///
31//===----------------------------------------------------------------------===//
32
33#include "Basic/CodeGenIntrinsics.h"
34#include "Common/CodeGenDAGPatterns.h"
35#include "Common/CodeGenInstruction.h"
36#include "Common/CodeGenRegisters.h"
37#include "Common/CodeGenTarget.h"
38#include "Common/GlobalISel/GlobalISelMatchTable.h"
39#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
40#include "Common/InfoByHwMode.h"
41#include "llvm/ADT/Statistic.h"
42#include "llvm/CodeGenTypes/LowLevelType.h"
43#include "llvm/CodeGenTypes/MachineValueType.h"
44#include "llvm/Support/CodeGenCoverage.h"
45#include "llvm/Support/CommandLine.h"
46#include "llvm/Support/Error.h"
47#include "llvm/Support/ScopedPrinter.h"
48#include "llvm/TableGen/Error.h"
49#include "llvm/TableGen/Record.h"
50#include "llvm/TableGen/TableGenBackend.h"
51#include <string>
52
53using namespace llvm;
54using namespace llvm::gi;
55
56using action_iterator = RuleMatcher::action_iterator;
57
58#define DEBUG_TYPE "gisel-emitter"
59
60STATISTIC(NumPatternTotal, "Total number of patterns");
61STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG");
62STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped");
63STATISTIC(NumPatternsTested,
64 "Number of patterns executed according to coverage information");
65
66static cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel");
67
68static cl::opt<bool> WarnOnSkippedPatterns(
69 "warn-on-skipped-patterns",
70 cl::desc("Explain why a pattern was skipped for inclusion "
71 "in the GlobalISel selector"),
72 cl::init(Val: false), cl::cat(GlobalISelEmitterCat));
73
74static cl::opt<bool> GenerateCoverage(
75 "instrument-gisel-coverage",
76 cl::desc("Generate coverage instrumentation for GlobalISel"),
77 cl::init(Val: false), cl::cat(GlobalISelEmitterCat));
78
79static cl::opt<std::string> UseCoverageFile(
80 "gisel-coverage-file", cl::init(Val: ""),
81 cl::desc("Specify file to retrieve coverage information from"),
82 cl::cat(GlobalISelEmitterCat));
83
84static cl::opt<bool> OptimizeMatchTable(
85 "optimize-match-table",
86 cl::desc("Generate an optimized version of the match table"),
87 cl::init(Val: true), cl::cat(GlobalISelEmitterCat));
88
89namespace {
90
91static std::string explainPredicates(const TreePatternNode &N) {
92 std::string Explanation;
93 StringRef Separator = "";
94 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
95 const TreePredicateFn &P = Call.Fn;
96 Explanation +=
97 (Separator + P.getOrigPatFragRecord()->getRecord()->getName()).str();
98 Separator = ", ";
99
100 if (P.isAlwaysTrue())
101 Explanation += " always-true";
102 if (P.isImmediatePattern())
103 Explanation += " immediate";
104
105 if (P.isUnindexed())
106 Explanation += " unindexed";
107
108 if (P.isNonExtLoad())
109 Explanation += " non-extload";
110 if (P.isAnyExtLoad())
111 Explanation += " extload";
112 if (P.isSignExtLoad())
113 Explanation += " sextload";
114 if (P.isZeroExtLoad())
115 Explanation += " zextload";
116
117 if (P.isNonTruncStore())
118 Explanation += " non-truncstore";
119 if (P.isTruncStore())
120 Explanation += " truncstore";
121
122 if (const Record *VT = P.getMemoryVT())
123 Explanation += (" MemVT=" + VT->getName()).str();
124 if (const Record *VT = P.getScalarMemoryVT())
125 Explanation += (" ScalarVT(MemVT)=" + VT->getName()).str();
126
127 if (const ListInit *AddrSpaces = P.getAddressSpaces()) {
128 raw_string_ostream OS(Explanation);
129 OS << " AddressSpaces=[";
130
131 StringRef AddrSpaceSeparator;
132 for (const Init *Val : AddrSpaces->getElements()) {
133 const IntInit *IntVal = dyn_cast<IntInit>(Val);
134 if (!IntVal)
135 continue;
136
137 OS << AddrSpaceSeparator << IntVal->getValue();
138 AddrSpaceSeparator = ", ";
139 }
140
141 OS << ']';
142 }
143
144 int64_t MinAlign = P.getMinAlignment();
145 if (MinAlign > 0)
146 Explanation += " MinAlign=" + utostr(X: MinAlign);
147
148 if (P.isAtomicOrderingMonotonic())
149 Explanation += " monotonic";
150 if (P.isAtomicOrderingAcquire())
151 Explanation += " acquire";
152 if (P.isAtomicOrderingRelease())
153 Explanation += " release";
154 if (P.isAtomicOrderingAcquireRelease())
155 Explanation += " acq_rel";
156 if (P.isAtomicOrderingSequentiallyConsistent())
157 Explanation += " seq_cst";
158 if (P.isAtomicOrderingAcquireOrStronger())
159 Explanation += " >=acquire";
160 if (P.isAtomicOrderingWeakerThanAcquire())
161 Explanation += " <acquire";
162 if (P.isAtomicOrderingReleaseOrStronger())
163 Explanation += " >=release";
164 if (P.isAtomicOrderingWeakerThanRelease())
165 Explanation += " <release";
166 }
167 return Explanation;
168}
169
170std::string explainOperator(const Record *Operator) {
171 if (Operator->isSubClassOf(Name: "SDNode"))
172 return (" (" + Operator->getValueAsString(FieldName: "Opcode") + ")").str();
173
174 if (Operator->isSubClassOf(Name: "Intrinsic"))
175 return (" (Operator is an Intrinsic, " + Operator->getName() + ")").str();
176
177 if (Operator->isSubClassOf(Name: "ComplexPattern"))
178 return (" (Operator is an unmapped ComplexPattern, " + Operator->getName() +
179 ")")
180 .str();
181
182 if (Operator->isSubClassOf(Name: "SDNodeXForm"))
183 return (" (Operator is an unmapped SDNodeXForm, " + Operator->getName() +
184 ")")
185 .str();
186
187 return (" (Operator " + Operator->getName() + " not understood)").str();
188}
189
190/// Helper function to let the emitter report skip reason error messages.
191static Error failedImport(const Twine &Reason) {
192 return make_error<StringError>(Args: Reason, Args: inconvertibleErrorCode());
193}
194
195static Error isTrivialOperatorNode(const TreePatternNode &N) {
196 std::string Explanation;
197 std::string Separator;
198
199 bool HasUnsupportedPredicate = false;
200 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
201 const TreePredicateFn &Predicate = Call.Fn;
202
203 if (Predicate.isAlwaysTrue())
204 continue;
205
206 if (Predicate.isImmediatePattern())
207 continue;
208
209 if (Predicate.hasNoUse() || Predicate.hasOneUse())
210 continue;
211
212 if (Predicate.isNonExtLoad() || Predicate.isAnyExtLoad() ||
213 Predicate.isSignExtLoad() || Predicate.isZeroExtLoad())
214 continue;
215
216 if (Predicate.isNonTruncStore() || Predicate.isTruncStore())
217 continue;
218
219 if (Predicate.isLoad() && Predicate.getMemoryVT())
220 continue;
221
222 if (Predicate.isStore() && Predicate.getMemoryVT())
223 continue;
224
225 if (Predicate.isLoad() || Predicate.isStore()) {
226 if (Predicate.isUnindexed())
227 continue;
228 }
229
230 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
231 const ListInit *AddrSpaces = Predicate.getAddressSpaces();
232 if (AddrSpaces && !AddrSpaces->empty())
233 continue;
234
235 if (Predicate.getMinAlignment() > 0)
236 continue;
237 }
238
239 if (Predicate.isAtomic() && Predicate.getMemoryVT())
240 continue;
241
242 if (Predicate.isAtomic() &&
243 (Predicate.isAtomicOrderingMonotonic() ||
244 Predicate.isAtomicOrderingAcquire() ||
245 Predicate.isAtomicOrderingRelease() ||
246 Predicate.isAtomicOrderingAcquireRelease() ||
247 Predicate.isAtomicOrderingSequentiallyConsistent() ||
248 Predicate.isAtomicOrderingAcquireOrStronger() ||
249 Predicate.isAtomicOrderingWeakerThanAcquire() ||
250 Predicate.isAtomicOrderingReleaseOrStronger() ||
251 Predicate.isAtomicOrderingWeakerThanRelease()))
252 continue;
253
254 if (Predicate.hasGISelPredicateCode())
255 continue;
256
257 HasUnsupportedPredicate = true;
258 Explanation = Separator + "Has a predicate (" + explainPredicates(N) + ")";
259 Separator = ", ";
260 Explanation += (Separator + "first-failing:" +
261 Predicate.getOrigPatFragRecord()->getRecord()->getName())
262 .str();
263 break;
264 }
265
266 if (!HasUnsupportedPredicate)
267 return Error::success();
268
269 return failedImport(Reason: Explanation);
270}
271
272static const Record *getInitValueAsRegClass(const Init *V) {
273 if (const DefInit *VDefInit = dyn_cast<DefInit>(Val: V)) {
274 if (VDefInit->getDef()->isSubClassOf(Name: "RegisterOperand"))
275 return VDefInit->getDef()->getValueAsDef(FieldName: "RegClass");
276 if (VDefInit->getDef()->isSubClassOf(Name: "RegisterClass"))
277 return VDefInit->getDef();
278 }
279 return nullptr;
280}
281
282static std::string getScopedName(unsigned Scope, const std::string &Name) {
283 return ("pred:" + Twine(Scope) + ":" + Name).str();
284}
285
286static std::string getMangledRootDefName(StringRef DefOperandName) {
287 return ("DstI[" + DefOperandName + "]").str();
288}
289
290//===- GlobalISelEmitter class --------------------------------------------===//
291
292static Expected<LLTCodeGen> getInstResultType(const TreePatternNode &Dst,
293 const CodeGenTarget &Target) {
294 // While we allow more than one output (both implicit and explicit defs)
295 // below, we only expect one explicit def here.
296 assert(Dst.getOperator()->isSubClassOf("Instruction"));
297 CodeGenInstruction &InstInfo = Target.getInstruction(InstRec: Dst.getOperator());
298 if (!InstInfo.Operands.NumDefs)
299 return failedImport(Reason: "Dst pattern child needs a def");
300
301 ArrayRef<TypeSetByHwMode> ChildTypes = Dst.getExtTypes();
302 if (ChildTypes.size() < 1)
303 return failedImport(Reason: "Dst pattern child has no result");
304
305 // If there are multiple results, just take the first one (this is how
306 // SelectionDAG does it).
307 std::optional<LLTCodeGen> MaybeOpTy;
308 if (ChildTypes.front().isMachineValueType()) {
309 MaybeOpTy = MVTToLLT(SVT: ChildTypes.front().getMachineValueType().SimpleTy);
310 }
311
312 if (!MaybeOpTy)
313 return failedImport(Reason: "Dst operand has an unsupported type");
314 return *MaybeOpTy;
315}
316
317class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter {
318public:
319 explicit GlobalISelEmitter(const RecordKeeper &RK);
320
321 void emitAdditionalImpl(raw_ostream &OS) override;
322
323 void emitMIPredicateFns(raw_ostream &OS) override;
324 void emitLeafPredicateFns(raw_ostream &OS) override;
325 void emitI64ImmPredicateFns(raw_ostream &OS) override;
326 void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
327 void emitAPIntImmPredicateFns(raw_ostream &OS) override;
328 void emitTestSimplePredicate(raw_ostream &OS) override;
329 void emitRunCustomAction(raw_ostream &OS) override;
330
331 const CodeGenTarget &getTarget() const override { return Target; }
332 StringRef getClassName() const override { return ClassName; }
333
334 void run(raw_ostream &OS);
335
336private:
337 std::string ClassName;
338
339 const RecordKeeper &RK;
340 const CodeGenDAGPatterns CGP;
341 const CodeGenTarget &Target;
342 CodeGenRegBank &CGRegs;
343
344 ArrayRef<const Record *> AllPatFrags;
345
346 /// Keep track of the equivalence between SDNodes and Instruction by mapping
347 /// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to
348 /// check for attributes on the relation such as CheckMMOIsNonAtomic.
349 /// This is defined using 'GINodeEquiv' in the target description.
350 DenseMap<const Record *, const Record *> NodeEquivs;
351
352 /// Keep track of the equivalence between ComplexPattern's and
353 /// GIComplexOperandMatcher. Map entries are specified by subclassing
354 /// GIComplexPatternEquiv.
355 DenseMap<const Record *, const Record *> ComplexPatternEquivs;
356
357 /// Keep track of the equivalence between SDNodeXForm's and
358 /// GICustomOperandRenderer. Map entries are specified by subclassing
359 /// GISDNodeXFormEquiv.
360 DenseMap<const Record *, const Record *> SDNodeXFormEquivs;
361
362 /// Keep track of Scores of PatternsToMatch similar to how the DAG does.
363 /// This adds compatibility for RuleMatchers to use this for ordering rules.
364 DenseMap<uint64_t, int> RuleMatcherScores;
365
366 // Rule coverage information.
367 std::optional<CodeGenCoverage> RuleCoverage;
368
369 /// Variables used to help with collecting of named operands for predicates
370 /// with 'let PredicateCodeUsesOperands = 1'. WaitingForNamedOperands is set
371 /// to the number of named operands that predicate expects. Store locations in
372 /// StoreIdxForName correspond to the order in which operand names appear in
373 /// predicate's argument list.
374 /// When we visit named operand and WaitingForNamedOperands is not zero, add
375 /// matcher that will record operand and decrease counter.
376 unsigned WaitingForNamedOperands = 0;
377 StringMap<unsigned> StoreIdxForName;
378
379 void gatherOpcodeValues();
380 void gatherTypeIDValues();
381 void gatherNodeEquivs();
382
383 const Record *findNodeEquiv(const Record *N) const;
384 const CodeGenInstruction *getEquivNode(const Record &Equiv,
385 const TreePatternNode &N) const;
386
387 Error importRulePredicates(RuleMatcher &M,
388 ArrayRef<const Record *> Predicates);
389 Expected<InstructionMatcher &>
390 createAndImportSelDAGMatcher(RuleMatcher &Rule,
391 InstructionMatcher &InsnMatcher,
392 const TreePatternNode &Src, unsigned &TempOpIdx);
393 Error importComplexPatternOperandMatcher(OperandMatcher &OM, const Record *R,
394 unsigned &TempOpIdx) const;
395 Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
396 const TreePatternNode &SrcChild,
397 bool OperandIsAPointer, bool OperandIsImmArg,
398 unsigned OpIdx, unsigned &TempOpIdx);
399
400 Expected<BuildMIAction &>
401 createAndImportInstructionRenderer(RuleMatcher &M,
402 InstructionMatcher &InsnMatcher,
403 const TreePatternNode &Dst) const;
404 Expected<action_iterator> createAndImportSubInstructionRenderer(
405 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
406 unsigned TempReg) const;
407 Expected<action_iterator>
408 createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M,
409 const TreePatternNode &Dst) const;
410
411 Expected<action_iterator>
412 importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M,
413 BuildMIAction &DstMIBuilder,
414 const TreePatternNode &Dst, bool IsRoot) const;
415
416 Expected<action_iterator>
417 importExplicitUseRenderers(action_iterator InsertPt, RuleMatcher &M,
418 BuildMIAction &DstMIBuilder,
419 const TreePatternNode &Dst) const;
420
421 Error importNamedNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
422 const TreePatternNode &N) const;
423
424 Error importLeafNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
425 const TreePatternNode &N,
426 action_iterator InsertPt) const;
427
428 Error importXFormNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
429 const TreePatternNode &N) const;
430
431 Error importInstructionNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
432 const TreePatternNode &N,
433 action_iterator &InsertPt) const;
434
435 Error importNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
436 const TreePatternNode &N,
437 action_iterator &InsertPt) const;
438
439 Error importImplicitDefRenderers(BuildMIAction &DstMIBuilder,
440 ArrayRef<const Record *> ImplicitDefs) const;
441
442 /// Analyze pattern \p P, returning a matcher for it if possible.
443 /// Otherwise, return an Error explaining why we don't support it.
444 Expected<RuleMatcher> runOnPattern(const PatternToMatch &P);
445
446 void declareSubtargetFeature(const Record *Predicate);
447
448 unsigned declareHwModeCheck(StringRef HwModeFeatures);
449
450 MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize,
451 bool WithCoverage);
452
453 /// Infer a CodeGenRegisterClass for the type of \p SuperRegNode. The returned
454 /// CodeGenRegisterClass will support the CodeGenRegisterClass of
455 /// \p SubRegNode, and the subregister index defined by \p SubRegIdxNode.
456 /// If no register class is found, return nullptr.
457 const CodeGenRegisterClass *
458 inferSuperRegisterClassForNode(const TypeSetByHwMode &Ty,
459 const TreePatternNode &SuperRegNode,
460 const TreePatternNode &SubRegIdxNode) const;
461 const CodeGenSubRegIndex *
462 inferSubRegIndexForNode(const TreePatternNode &SubRegIdxNode) const;
463
464 /// Infer a CodeGenRegisterClass which suppoorts \p Ty and \p SubRegIdxNode.
465 /// Return nullptr if no such class exists.
466 const CodeGenRegisterClass *
467 inferSuperRegisterClass(const TypeSetByHwMode &Ty,
468 const TreePatternNode &SubRegIdxNode) const;
469
470 /// Return the CodeGenRegisterClass associated with \p Leaf if it has one.
471 const CodeGenRegisterClass *
472 getRegClassFromLeaf(const TreePatternNode &Leaf) const;
473
474 /// Return a CodeGenRegisterClass for \p N if one can be found. Return
475 /// nullptr otherwise.
476 const CodeGenRegisterClass *
477 inferRegClassFromPattern(const TreePatternNode &N) const;
478
479 const CodeGenRegisterClass *
480 inferRegClassFromInstructionPattern(const TreePatternNode &N,
481 unsigned ResIdx) const;
482
483 Error constrainOperands(action_iterator InsertPt, RuleMatcher &M,
484 unsigned InsnID, const TreePatternNode &Dst) const;
485
486 /// Return the size of the MemoryVT in this predicate, if possible.
487 std::optional<unsigned>
488 getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate);
489
490 // Add builtin predicates.
491 Expected<InstructionMatcher &>
492 addBuiltinPredicates(const Record *SrcGIEquivOrNull,
493 const TreePredicateFn &Predicate,
494 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher);
495};
496
497StringRef getPatFragPredicateEnumName(const Record *R) { return R->getName(); }
498
499void GlobalISelEmitter::gatherOpcodeValues() {
500 InstructionOpcodeMatcher::initOpcodeValuesMap(Target);
501}
502
503void GlobalISelEmitter::gatherTypeIDValues() {
504 LLTOperandMatcher::initTypeIDValuesMap();
505}
506
507void GlobalISelEmitter::gatherNodeEquivs() {
508 assert(NodeEquivs.empty());
509 for (const Record *Equiv : RK.getAllDerivedDefinitions(ClassName: "GINodeEquiv"))
510 NodeEquivs[Equiv->getValueAsDef(FieldName: "Node")] = Equiv;
511
512 assert(ComplexPatternEquivs.empty());
513 for (const Record *Equiv :
514 RK.getAllDerivedDefinitions(ClassName: "GIComplexPatternEquiv")) {
515 const Record *SelDAGEquiv = Equiv->getValueAsDef(FieldName: "SelDAGEquivalent");
516 if (!SelDAGEquiv)
517 continue;
518 ComplexPatternEquivs[SelDAGEquiv] = Equiv;
519 }
520
521 assert(SDNodeXFormEquivs.empty());
522 for (const Record *Equiv :
523 RK.getAllDerivedDefinitions(ClassName: "GISDNodeXFormEquiv")) {
524 const Record *SelDAGEquiv = Equiv->getValueAsDef(FieldName: "SelDAGEquivalent");
525 if (!SelDAGEquiv)
526 continue;
527 SDNodeXFormEquivs[SelDAGEquiv] = Equiv;
528 }
529}
530
531const Record *GlobalISelEmitter::findNodeEquiv(const Record *N) const {
532 return NodeEquivs.lookup(Val: N);
533}
534
535const CodeGenInstruction *
536GlobalISelEmitter::getEquivNode(const Record &Equiv,
537 const TreePatternNode &N) const {
538 if (N.getNumChildren() >= 1) {
539 // setcc operation maps to two different G_* instructions based on the type.
540 if (!Equiv.isValueUnset(FieldName: "IfFloatingPoint") &&
541 MVT(N.getChild(N: 0).getSimpleType(ResNo: 0)).isFloatingPoint())
542 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfFloatingPoint"));
543 }
544
545 if (!Equiv.isValueUnset(FieldName: "IfConvergent") &&
546 N.getIntrinsicInfo(CDP: CGP)->isConvergent)
547 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfConvergent"));
548
549 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
550 const TreePredicateFn &Predicate = Call.Fn;
551 if (!Equiv.isValueUnset(FieldName: "IfSignExtend") &&
552 (Predicate.isLoad() || Predicate.isAtomic()) &&
553 Predicate.isSignExtLoad())
554 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfSignExtend"));
555 if (!Equiv.isValueUnset(FieldName: "IfZeroExtend") &&
556 (Predicate.isLoad() || Predicate.isAtomic()) &&
557 Predicate.isZeroExtLoad())
558 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfZeroExtend"));
559 }
560
561 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "I"));
562}
563
564GlobalISelEmitter::GlobalISelEmitter(const RecordKeeper &RK)
565 : GlobalISelMatchTableExecutorEmitter(), RK(RK), CGP(RK),
566 Target(CGP.getTargetInfo()), CGRegs(Target.getRegBank()) {
567 ClassName = Target.getName().str() + "InstructionSelector";
568}
569
570//===- Emitter ------------------------------------------------------------===//
571
572Error GlobalISelEmitter::importRulePredicates(
573 RuleMatcher &M, ArrayRef<const Record *> Predicates) {
574 for (const Record *Pred : Predicates) {
575 if (Pred->getValueAsString(FieldName: "CondString").empty())
576 continue;
577 declareSubtargetFeature(Predicate: Pred);
578 M.addRequiredFeature(Feature: Pred);
579 }
580
581 return Error::success();
582}
583
584std::optional<unsigned> GlobalISelEmitter::getMemSizeBitsFromPredicate(
585 const TreePredicateFn &Predicate) {
586 std::optional<LLTCodeGen> MemTyOrNone =
587 MVTToLLT(SVT: getValueType(Rec: Predicate.getMemoryVT()));
588
589 if (!MemTyOrNone)
590 return std::nullopt;
591
592 // Align so unusual types like i1 don't get rounded down.
593 return llvm::alignTo(
594 Value: static_cast<unsigned>(MemTyOrNone->get().getSizeInBits()), Align: 8);
595}
596
597Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
598 const Record *SrcGIEquivOrNull, const TreePredicateFn &Predicate,
599 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher) {
600 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
601 if (const ListInit *AddrSpaces = Predicate.getAddressSpaces()) {
602 SmallVector<unsigned, 4> ParsedAddrSpaces;
603
604 for (const Init *Val : AddrSpaces->getElements()) {
605 const IntInit *IntVal = dyn_cast<IntInit>(Val);
606 if (!IntVal)
607 return failedImport(Reason: "Address space is not an integer");
608 ParsedAddrSpaces.push_back(Elt: IntVal->getValue());
609 }
610
611 if (!ParsedAddrSpaces.empty()) {
612 InsnMatcher.addPredicate<MemoryAddressSpacePredicateMatcher>(
613 args: 0, args&: ParsedAddrSpaces);
614 return InsnMatcher;
615 }
616 }
617
618 int64_t MinAlign = Predicate.getMinAlignment();
619 if (MinAlign > 0) {
620 InsnMatcher.addPredicate<MemoryAlignmentPredicateMatcher>(args: 0, args&: MinAlign);
621 return InsnMatcher;
622 }
623 }
624
625 // G_LOAD is used for both non-extending and any-extending loads.
626 if (Predicate.isLoad() || Predicate.isAtomic()) {
627 if (Predicate.isNonExtLoad()) {
628 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
629 args: 0, args: MemoryVsLLTSizePredicateMatcher::EqualTo, args: 0);
630 return InsnMatcher;
631 }
632 if (Predicate.isAnyExtLoad()) {
633 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
634 args: 0, args: MemoryVsLLTSizePredicateMatcher::LessThan, args: 0);
635 return InsnMatcher;
636 }
637 }
638
639 if (Predicate.isStore()) {
640 if (Predicate.isTruncStore()) {
641 if (Predicate.getMemoryVT() != nullptr) {
642 // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size.
643 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
644 if (!MemSizeInBits)
645 return failedImport(Reason: "MemVT could not be converted to LLT");
646
647 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(args: 0, args: *MemSizeInBits /
648 8);
649 } else {
650 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
651 args: 0, args: MemoryVsLLTSizePredicateMatcher::LessThan, args: 0);
652 }
653 return InsnMatcher;
654 }
655 if (Predicate.isNonTruncStore()) {
656 // We need to check the sizes match here otherwise we could incorrectly
657 // match truncating stores with non-truncating ones.
658 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
659 args: 0, args: MemoryVsLLTSizePredicateMatcher::EqualTo, args: 0);
660 }
661 }
662
663 assert(SrcGIEquivOrNull != nullptr && "Invalid SrcGIEquivOrNull value");
664 // No check required. We already did it by swapping the opcode.
665 if (!SrcGIEquivOrNull->isValueUnset(FieldName: "IfSignExtend") &&
666 Predicate.isSignExtLoad())
667 return InsnMatcher;
668
669 // No check required. We already did it by swapping the opcode.
670 if (!SrcGIEquivOrNull->isValueUnset(FieldName: "IfZeroExtend") &&
671 Predicate.isZeroExtLoad())
672 return InsnMatcher;
673
674 // No check required. G_STORE by itself is a non-extending store.
675 if (Predicate.isNonTruncStore())
676 return InsnMatcher;
677
678 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
679 if (Predicate.getMemoryVT() != nullptr) {
680 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
681 if (!MemSizeInBits)
682 return failedImport(Reason: "MemVT could not be converted to LLT");
683
684 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(args: 0,
685 args: *MemSizeInBits / 8);
686 return InsnMatcher;
687 }
688 }
689
690 if (Predicate.isLoad() || Predicate.isStore()) {
691 // No check required. A G_LOAD/G_STORE is an unindexed load.
692 if (Predicate.isUnindexed())
693 return InsnMatcher;
694 }
695
696 if (Predicate.isAtomic()) {
697 if (Predicate.isAtomicOrderingMonotonic()) {
698 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Monotonic");
699 return InsnMatcher;
700 }
701 if (Predicate.isAtomicOrderingAcquire()) {
702 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Acquire");
703 return InsnMatcher;
704 }
705 if (Predicate.isAtomicOrderingRelease()) {
706 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Release");
707 return InsnMatcher;
708 }
709 if (Predicate.isAtomicOrderingAcquireRelease()) {
710 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
711 args: "AcquireRelease");
712 return InsnMatcher;
713 }
714 if (Predicate.isAtomicOrderingSequentiallyConsistent()) {
715 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
716 args: "SequentiallyConsistent");
717 return InsnMatcher;
718 }
719 }
720
721 if (Predicate.isAtomicOrderingAcquireOrStronger()) {
722 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
723 args: "Acquire", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
724 return InsnMatcher;
725 }
726 if (Predicate.isAtomicOrderingWeakerThanAcquire()) {
727 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
728 args: "Acquire", args: AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
729 return InsnMatcher;
730 }
731
732 if (Predicate.isAtomicOrderingReleaseOrStronger()) {
733 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
734 args: "Release", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
735 return InsnMatcher;
736 }
737 if (Predicate.isAtomicOrderingWeakerThanRelease()) {
738 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
739 args: "Release", args: AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
740 return InsnMatcher;
741 }
742 HasAddedMatcher = false;
743 return InsnMatcher;
744}
745
746Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
747 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
748 const TreePatternNode &Src, unsigned &TempOpIdx) {
749 const auto SavedFlags = Rule.setGISelFlags(Src.getGISelFlagsRecord());
750
751 const Record *SrcGIEquivOrNull = nullptr;
752 const CodeGenInstruction *SrcGIOrNull = nullptr;
753
754 // Start with the defined operands (i.e., the results of the root operator).
755 if (Src.isLeaf()) {
756 const Init *SrcInit = Src.getLeafValue();
757 if (isa<IntInit>(Val: SrcInit)) {
758 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(
759 args: &Target.getInstruction(InstRec: RK.getDef(Name: "G_CONSTANT")));
760 } else {
761 return failedImport(
762 Reason: "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
763 }
764 } else {
765 SrcGIEquivOrNull = findNodeEquiv(N: Src.getOperator());
766 if (!SrcGIEquivOrNull)
767 return failedImport(Reason: "Pattern operator lacks an equivalent Instruction" +
768 explainOperator(Operator: Src.getOperator()));
769 SrcGIOrNull = getEquivNode(Equiv: *SrcGIEquivOrNull, N: Src);
770
771 // The operators look good: match the opcode
772 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(args&: SrcGIOrNull);
773 }
774
775 // Since there are no opcodes for atomic loads and stores comparing to
776 // SelectionDAG, we add CheckMMOIsNonAtomic predicate immediately after the
777 // opcode predicate to make a logical combination of them.
778 if (SrcGIEquivOrNull &&
779 SrcGIEquivOrNull->getValueAsBit(FieldName: "CheckMMOIsNonAtomic"))
780 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "NotAtomic");
781 else if (SrcGIEquivOrNull &&
782 SrcGIEquivOrNull->getValueAsBit(FieldName: "CheckMMOIsAtomic")) {
783 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
784 args: "Unordered", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
785 }
786
787 unsigned OpIdx = 0;
788 for (const TypeSetByHwMode &VTy : Src.getExtTypes()) {
789 // Results don't have a name unless they are the root node. The caller will
790 // set the name if appropriate.
791 const bool OperandIsAPointer =
792 SrcGIOrNull && SrcGIOrNull->isOutOperandAPointer(i: OpIdx);
793 OperandMatcher &OM = InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
794 if (auto Error = OM.addTypeCheckPredicate(VTy, OperandIsAPointer))
795 return failedImport(Reason: toString(E: std::move(Error)) +
796 " for result of Src pattern operator");
797 }
798
799 for (const TreePredicateCall &Call : Src.getPredicateCalls()) {
800 const TreePredicateFn &Predicate = Call.Fn;
801 bool HasAddedBuiltinMatcher = true;
802 if (Predicate.isAlwaysTrue())
803 continue;
804
805 if (Predicate.isImmediatePattern()) {
806 InsnMatcher.addPredicate<InstructionImmPredicateMatcher>(args: Predicate);
807 continue;
808 }
809
810 auto InsnMatcherOrError = addBuiltinPredicates(
811 SrcGIEquivOrNull, Predicate, InsnMatcher, HasAddedMatcher&: HasAddedBuiltinMatcher);
812 if (auto Error = InsnMatcherOrError.takeError())
813 return std::move(Error);
814
815 // FIXME: This should be part of addBuiltinPredicates(). If we add this at
816 // the start of addBuiltinPredicates() without returning, then there might
817 // be cases where we hit the last return before which the
818 // HasAddedBuiltinMatcher will be set to false. The predicate could be
819 // missed if we add it in the middle or at the end due to return statements
820 // after the addPredicate<>() calls.
821 if (Predicate.hasNoUse()) {
822 InsnMatcher.addPredicate<NoUsePredicateMatcher>();
823 HasAddedBuiltinMatcher = true;
824 }
825 if (Predicate.hasOneUse()) {
826 InsnMatcher.addPredicate<OneUsePredicateMatcher>();
827 HasAddedBuiltinMatcher = true;
828 }
829
830 if (Predicate.hasGISelPredicateCode()) {
831 if (Predicate.usesOperands()) {
832 assert(WaitingForNamedOperands == 0 &&
833 "previous predicate didn't find all operands or "
834 "nested predicate that uses operands");
835 TreePattern *TP = Predicate.getOrigPatFragRecord();
836 WaitingForNamedOperands = TP->getNumArgs();
837 for (unsigned I = 0; I < WaitingForNamedOperands; ++I)
838 StoreIdxForName[getScopedName(Scope: Call.Scope, Name: TP->getArgName(i: I))] = I;
839 }
840 InsnMatcher.addPredicate<GenericInstructionPredicateMatcher>(args: Predicate);
841 continue;
842 }
843 if (!HasAddedBuiltinMatcher) {
844 return failedImport(Reason: "Src pattern child has predicate (" +
845 explainPredicates(N: Src) + ")");
846 }
847 }
848
849 if (Src.isLeaf()) {
850 const Init *SrcInit = Src.getLeafValue();
851 if (const IntInit *SrcIntInit = dyn_cast<IntInit>(Val: SrcInit)) {
852 OperandMatcher &OM =
853 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: Src.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
854 OM.addPredicate<LiteralIntOperandMatcher>(args: SrcIntInit->getValue());
855 } else {
856 return failedImport(
857 Reason: "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
858 }
859 } else {
860 assert(SrcGIOrNull &&
861 "Expected to have already found an equivalent Instruction");
862 if (SrcGIOrNull->TheDef->getName() == "G_CONSTANT" ||
863 SrcGIOrNull->TheDef->getName() == "G_FCONSTANT" ||
864 SrcGIOrNull->TheDef->getName() == "G_FRAME_INDEX") {
865 // imm/fpimm still have operands but we don't need to do anything with it
866 // here since we don't support ImmLeaf predicates yet. However, we still
867 // need to note the hidden operand to get GIM_CheckNumOperands correct.
868 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
869 return InsnMatcher;
870 }
871
872 // Special case because the operand order is changed from setcc. The
873 // predicate operand needs to be swapped from the last operand to the first
874 // source.
875
876 unsigned NumChildren = Src.getNumChildren();
877 bool IsFCmp = SrcGIOrNull->TheDef->getName() == "G_FCMP";
878
879 if (IsFCmp || SrcGIOrNull->TheDef->getName() == "G_ICMP") {
880 const TreePatternNode &SrcChild = Src.getChild(N: NumChildren - 1);
881 if (SrcChild.isLeaf()) {
882 const DefInit *DI = dyn_cast<DefInit>(Val: SrcChild.getLeafValue());
883 const Record *CCDef = DI ? DI->getDef() : nullptr;
884 if (!CCDef || !CCDef->isSubClassOf(Name: "CondCode"))
885 return failedImport(Reason: "Unable to handle CondCode");
886
887 OperandMatcher &OM = InsnMatcher.addOperand(
888 OpIdx: OpIdx++, SymbolicName: SrcChild.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
889 StringRef PredType = IsFCmp ? CCDef->getValueAsString(FieldName: "FCmpPredicate")
890 : CCDef->getValueAsString(FieldName: "ICmpPredicate");
891
892 if (!PredType.empty()) {
893 OM.addPredicate<CmpPredicateOperandMatcher>(args: PredType.str());
894 // Process the other 2 operands normally.
895 --NumChildren;
896 }
897 }
898 }
899
900 // Match the used operands (i.e. the children of the operator).
901 bool IsIntrinsic =
902 SrcGIOrNull->TheDef->getName() == "G_INTRINSIC" ||
903 SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_W_SIDE_EFFECTS" ||
904 SrcGIOrNull->TheDef->getName() == "G_INTRINSIC_CONVERGENT" ||
905 SrcGIOrNull->TheDef->getName() ==
906 "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS";
907 const CodeGenIntrinsic *II = Src.getIntrinsicInfo(CDP: CGP);
908 if (IsIntrinsic && !II)
909 return failedImport(Reason: "Expected IntInit containing intrinsic ID)");
910
911 for (unsigned I = 0; I != NumChildren; ++I) {
912 const TreePatternNode &SrcChild = Src.getChild(N: I);
913
914 // We need to determine the meaning of a literal integer based on the
915 // context. If this is a field required to be an immediate (such as an
916 // immarg intrinsic argument), the required predicates are different than
917 // a constant which may be materialized in a register. If we have an
918 // argument that is required to be an immediate, we should not emit an LLT
919 // type check, and should not be looking for a G_CONSTANT defined
920 // register.
921 bool OperandIsImmArg = SrcGIOrNull->isInOperandImmArg(i: I);
922
923 // SelectionDAG allows pointers to be represented with iN since it doesn't
924 // distinguish between pointers and integers but they are different types
925 // in GlobalISel. Coerce integers to pointers to address space 0 if the
926 // context indicates a pointer.
927 //
928 bool OperandIsAPointer = SrcGIOrNull->isInOperandAPointer(i: I);
929
930 if (IsIntrinsic) {
931 // For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately
932 // following the defs is an intrinsic ID.
933 if (I == 0) {
934 OperandMatcher &OM = InsnMatcher.addOperand(
935 OpIdx: OpIdx++, SymbolicName: SrcChild.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
936 OM.addPredicate<IntrinsicIDOperandMatcher>(args&: II);
937 continue;
938 }
939
940 // We have to check intrinsics for llvm_anyptr_ty and immarg parameters.
941 //
942 // Note that we have to look at the i-1th parameter, because we don't
943 // have the intrinsic ID in the intrinsic's parameter list.
944 OperandIsAPointer |= II->isParamAPointer(ParamIdx: I - 1);
945 OperandIsImmArg |= II->isParamImmArg(ParamIdx: I - 1);
946 }
947
948 if (auto Error =
949 importChildMatcher(Rule, InsnMatcher, SrcChild, OperandIsAPointer,
950 OperandIsImmArg, OpIdx: OpIdx++, TempOpIdx))
951 return std::move(Error);
952 }
953 }
954
955 return InsnMatcher;
956}
957
958Error GlobalISelEmitter::importComplexPatternOperandMatcher(
959 OperandMatcher &OM, const Record *R, unsigned &TempOpIdx) const {
960 const auto &ComplexPattern = ComplexPatternEquivs.find(Val: R);
961 if (ComplexPattern == ComplexPatternEquivs.end())
962 return failedImport(Reason: "SelectionDAG ComplexPattern (" + R->getName() +
963 ") not mapped to GlobalISel");
964
965 OM.addPredicate<ComplexPatternOperandMatcher>(args&: OM, args: *ComplexPattern->second);
966 TempOpIdx++;
967 return Error::success();
968}
969
970// Get the name to use for a pattern operand. For an anonymous physical register
971// input, this should use the register name.
972static StringRef getSrcChildName(const TreePatternNode &SrcChild,
973 const Record *&PhysReg) {
974 StringRef SrcChildName = SrcChild.getName();
975 if (SrcChildName.empty() && SrcChild.isLeaf()) {
976 if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
977 auto *ChildRec = ChildDefInit->getDef();
978 if (ChildRec->isSubClassOf(Name: "Register")) {
979 SrcChildName = ChildRec->getName();
980 PhysReg = ChildRec;
981 }
982 }
983 }
984
985 return SrcChildName;
986}
987
988Error GlobalISelEmitter::importChildMatcher(
989 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
990 const TreePatternNode &SrcChild, bool OperandIsAPointer,
991 bool OperandIsImmArg, unsigned OpIdx, unsigned &TempOpIdx) {
992
993 const Record *PhysReg = nullptr;
994 std::string SrcChildName = getSrcChildName(SrcChild, PhysReg).str();
995 if (!SrcChild.isLeaf() &&
996 SrcChild.getOperator()->isSubClassOf(Name: "ComplexPattern")) {
997 // The "name" of a non-leaf complex pattern (MY_PAT $op1, $op2) is
998 // "MY_PAT:op1:op2" and the ones with same "name" represent same operand.
999 std::string PatternName = SrcChild.getOperator()->getName().str();
1000 for (const TreePatternNode &Child : SrcChild.children()) {
1001 PatternName += ":";
1002 PatternName += Child.getName();
1003 }
1004 SrcChildName = PatternName;
1005 }
1006
1007 OperandMatcher &OM =
1008 PhysReg ? InsnMatcher.addPhysRegInput(Reg: PhysReg, OpIdx, TempOpIdx)
1009 : InsnMatcher.addOperand(OpIdx, SymbolicName: SrcChildName, AllocatedTemporariesBaseID: TempOpIdx);
1010 if (OM.isSameAsAnotherOperand())
1011 return Error::success();
1012
1013 ArrayRef<TypeSetByHwMode> ChildTypes = SrcChild.getExtTypes();
1014 if (ChildTypes.size() != 1)
1015 return failedImport(Reason: "Src pattern child has multiple results");
1016
1017 // Check MBB's before the type check since they are not a known type.
1018 if (!SrcChild.isLeaf()) {
1019 if (SrcChild.getOperator()->getName() == "bb") {
1020 OM.addPredicate<MBBOperandMatcher>();
1021 return Error::success();
1022 }
1023 if (SrcChild.getOperator()->getName() == "timm") {
1024 OM.addPredicate<ImmOperandMatcher>();
1025
1026 // Add predicates, if any
1027 for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {
1028 const TreePredicateFn &Predicate = Call.Fn;
1029
1030 // Only handle immediate patterns for now
1031 if (Predicate.isImmediatePattern()) {
1032 OM.addPredicate<OperandImmPredicateMatcher>(args: Predicate);
1033 }
1034 }
1035
1036 return Error::success();
1037 }
1038 } else if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
1039 auto *ChildRec = ChildDefInit->getDef();
1040 if (ChildRec->isSubClassOf(Name: "ValueType") && !SrcChild.hasName()) {
1041 // An unnamed ValueType as in (sext_inreg GPR:$foo, i8). GISel represents
1042 // this as a literal constant with the scalar size.
1043 MVT::SimpleValueType VT = llvm::getValueType(Rec: ChildRec);
1044 OM.addPredicate<LiteralIntOperandMatcher>(args: MVT(VT).getScalarSizeInBits());
1045 return Error::success();
1046 }
1047 }
1048
1049 // Immediate arguments have no meaningful type to check as they don't have
1050 // registers.
1051 if (!OperandIsImmArg) {
1052 if (auto Error =
1053 OM.addTypeCheckPredicate(VTy: ChildTypes.front(), OperandIsAPointer))
1054 return failedImport(Reason: toString(E: std::move(Error)) + " for Src operand (" +
1055 to_string(Value: SrcChild) + ")");
1056 }
1057
1058 // Try look up SrcChild for a (named) predicate operand if there is any.
1059 if (WaitingForNamedOperands) {
1060 auto &ScopedNames = SrcChild.getNamesAsPredicateArg();
1061 if (!ScopedNames.empty()) {
1062 auto PA = ScopedNames.begin();
1063 std::string Name = getScopedName(Scope: PA->getScope(), Name: PA->getIdentifier());
1064 OM.addPredicate<RecordNamedOperandMatcher>(args&: StoreIdxForName[Name], args&: Name);
1065 --WaitingForNamedOperands;
1066 }
1067 }
1068
1069 // Check for nested instructions.
1070 if (!SrcChild.isLeaf()) {
1071 if (SrcChild.getOperator()->isSubClassOf(Name: "ComplexPattern")) {
1072 // When a ComplexPattern is used as an operator, it should do the same
1073 // thing as when used as a leaf. However, the children of the operator
1074 // name the sub-operands that make up the complex operand and we must
1075 // prepare to reference them in the renderer too.
1076 unsigned RendererID = TempOpIdx;
1077 if (auto Error = importComplexPatternOperandMatcher(
1078 OM, R: SrcChild.getOperator(), TempOpIdx))
1079 return Error;
1080
1081 for (unsigned I = 0, E = SrcChild.getNumChildren(); I != E; ++I) {
1082 auto &SubOperand = SrcChild.getChild(N: I);
1083 if (!SubOperand.getName().empty()) {
1084 if (auto Error = Rule.defineComplexSubOperand(
1085 SymbolicName: SubOperand.getName(), ComplexPattern: SrcChild.getOperator(), RendererID, SubOperandID: I,
1086 ParentSymbolicName: SrcChildName))
1087 return Error;
1088 }
1089 }
1090
1091 return Error::success();
1092 }
1093
1094 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
1095 args&: InsnMatcher.getRuleMatcher(), args: SrcChild.getName());
1096 if (!MaybeInsnOperand) {
1097 // This isn't strictly true. If the user were to provide exactly the same
1098 // matchers as the original operand then we could allow it. However, it's
1099 // simpler to not permit the redundant specification.
1100 return failedImport(
1101 Reason: "Nested instruction cannot be the same as another operand");
1102 }
1103
1104 // Map the node to a gMIR instruction.
1105 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
1106 auto InsnMatcherOrError = createAndImportSelDAGMatcher(
1107 Rule, InsnMatcher&: InsnOperand.getInsnMatcher(), Src: SrcChild, TempOpIdx);
1108 if (auto Error = InsnMatcherOrError.takeError())
1109 return Error;
1110
1111 return Error::success();
1112 }
1113
1114 if (SrcChild.hasAnyPredicate()) {
1115 for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {
1116 const TreePredicateFn &Predicate = Call.Fn;
1117
1118 if (!Predicate.hasGISelLeafPredicateCode())
1119 return failedImport(Reason: "Src pattern child has unsupported predicate");
1120 OM.addPredicate<OperandLeafPredicateMatcher>(args: Predicate);
1121 }
1122 return Error::success();
1123 }
1124
1125 // Check for constant immediates.
1126 if (auto *ChildInt = dyn_cast<IntInit>(Val: SrcChild.getLeafValue())) {
1127 if (OperandIsImmArg) {
1128 // Checks for argument directly in operand list
1129 OM.addPredicate<LiteralIntOperandMatcher>(args: ChildInt->getValue());
1130 } else {
1131 // Checks for materialized constant
1132 OM.addPredicate<ConstantIntOperandMatcher>(args: ChildInt->getValue());
1133 }
1134 return Error::success();
1135 }
1136
1137 // Check for def's like register classes or ComplexPattern's.
1138 if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
1139 auto *ChildRec = ChildDefInit->getDef();
1140
1141 // Check for register classes.
1142 if (ChildRec->isSubClassOf(Name: "RegisterClass") ||
1143 ChildRec->isSubClassOf(Name: "RegisterOperand")) {
1144 OM.addPredicate<RegisterBankOperandMatcher>(
1145 args: Target.getRegisterClass(R: getInitValueAsRegClass(V: ChildDefInit)));
1146 return Error::success();
1147 }
1148
1149 if (ChildRec->isSubClassOf(Name: "Register")) {
1150 // This just be emitted as a copy to the specific register.
1151 ValueTypeByHwMode VT = ChildTypes.front().getValueTypeByHwMode();
1152 const CodeGenRegisterClass *RC =
1153 CGRegs.getMinimalPhysRegClass(RegRecord: ChildRec, VT: &VT);
1154 if (!RC) {
1155 return failedImport(
1156 Reason: "Could not determine physical register class of pattern source");
1157 }
1158
1159 OM.addPredicate<RegisterBankOperandMatcher>(args: *RC);
1160 return Error::success();
1161 }
1162
1163 // Check for ValueType.
1164 if (ChildRec->isSubClassOf(Name: "ValueType")) {
1165 // We already added a type check as standard practice so this doesn't need
1166 // to do anything.
1167 return Error::success();
1168 }
1169
1170 // Check for ComplexPattern's.
1171 if (ChildRec->isSubClassOf(Name: "ComplexPattern"))
1172 return importComplexPatternOperandMatcher(OM, R: ChildRec, TempOpIdx);
1173
1174 if (ChildRec->isSubClassOf(Name: "ImmLeaf")) {
1175 return failedImport(
1176 Reason: "Src pattern child def is an unsupported tablegen class (ImmLeaf)");
1177 }
1178
1179 // Place holder for SRCVALUE nodes. Nothing to do here.
1180 if (ChildRec->getName() == "srcvalue")
1181 return Error::success();
1182
1183 const bool ImmAllOnesV = ChildRec->getName() == "immAllOnesV";
1184 if (ImmAllOnesV || ChildRec->getName() == "immAllZerosV") {
1185 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
1186 args&: InsnMatcher.getRuleMatcher(), args: SrcChild.getName(), args: false);
1187 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
1188
1189 ValueTypeByHwMode VTy = ChildTypes.front().getValueTypeByHwMode();
1190
1191 const CodeGenInstruction &BuildVector =
1192 Target.getInstruction(InstRec: RK.getDef(Name: "G_BUILD_VECTOR"));
1193 const CodeGenInstruction &BuildVectorTrunc =
1194 Target.getInstruction(InstRec: RK.getDef(Name: "G_BUILD_VECTOR_TRUNC"));
1195
1196 // Treat G_BUILD_VECTOR as the canonical opcode, and G_BUILD_VECTOR_TRUNC
1197 // as an alternative.
1198 InsnOperand.getInsnMatcher().addPredicate<InstructionOpcodeMatcher>(
1199 args: ArrayRef({&BuildVector, &BuildVectorTrunc}));
1200
1201 // TODO: Handle both G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC We could
1202 // theoretically not emit any opcode check, but getOpcodeMatcher currently
1203 // has to succeed.
1204 OperandMatcher &OM =
1205 InsnOperand.getInsnMatcher().addOperand(OpIdx: 0, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
1206 if (auto Error = OM.addTypeCheckPredicate(VTy: TypeSetByHwMode(VTy),
1207 /*OperandIsAPointer=*/false))
1208 return failedImport(Reason: toString(E: std::move(Error)) +
1209 " for result of Src pattern operator");
1210
1211 InsnOperand.getInsnMatcher().addPredicate<VectorSplatImmPredicateMatcher>(
1212 args: ImmAllOnesV ? VectorSplatImmPredicateMatcher::AllOnes
1213 : VectorSplatImmPredicateMatcher::AllZeros);
1214 return Error::success();
1215 }
1216
1217 return failedImport(
1218 Reason: "Src pattern child def is an unsupported tablegen class");
1219 }
1220
1221 return failedImport(Reason: "Src pattern child is an unsupported kind");
1222}
1223
1224// Equivalent of MatcherGen::EmitResultOfNamedOperand.
1225Error GlobalISelEmitter::importNamedNodeRenderer(
1226 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N) const {
1227 StringRef NodeName = N.getName();
1228
1229 if (auto SubOperand = M.getComplexSubOperand(SymbolicName: NodeName)) {
1230 auto [ComplexPatternRec, RendererID, SubOperandIdx] = *SubOperand;
1231 MIBuilder.addRenderer<RenderComplexPatternOperand>(
1232 args: *ComplexPatternRec, args&: NodeName, args&: RendererID, args&: SubOperandIdx);
1233 return Error::success();
1234 }
1235
1236 if (!N.isLeaf()) {
1237 StringRef OperatorName = N.getOperator()->getName();
1238
1239 if (OperatorName == "imm") {
1240 MIBuilder.addRenderer<CopyConstantAsImmRenderer>(args&: NodeName);
1241 return Error::success();
1242 }
1243
1244 if (OperatorName == "fpimm") {
1245 MIBuilder.addRenderer<CopyFConstantAsFPImmRenderer>(args&: NodeName);
1246 return Error::success();
1247 }
1248
1249 // TODO: 'imm' and 'fpimm' are the only nodes that need special treatment.
1250 // Remove this check and add CopyRenderer unconditionally for other nodes.
1251 if (OperatorName == "bb" || OperatorName == "timm" ||
1252 OperatorName == "tframeindex") {
1253 MIBuilder.addRenderer<CopyRenderer>(args&: NodeName);
1254 return Error::success();
1255 }
1256
1257 return failedImport(Reason: "node has unsupported operator " + to_string(Value: N));
1258 }
1259
1260 if (const auto *DI = dyn_cast<DefInit>(Val: N.getLeafValue())) {
1261 const Record *R = DI->getDef();
1262
1263 if (N.getNumResults() != 1)
1264 return failedImport(Reason: "node does not have one result " + to_string(Value: N));
1265
1266 if (R->isSubClassOf(Name: "ComplexPattern")) {
1267 auto I = ComplexPatternEquivs.find(Val: R);
1268 if (I == ComplexPatternEquivs.end())
1269 return failedImport(Reason: "ComplexPattern " + R->getName() +
1270 " does not have GISel equivalent");
1271
1272 const OperandMatcher &OM = M.getOperandMatcher(Name: NodeName);
1273 MIBuilder.addRenderer<RenderComplexPatternOperand>(
1274 args: *I->second, args&: NodeName, args: OM.getAllocatedTemporariesBaseID());
1275 return Error::success();
1276 }
1277
1278 if (R->isSubClassOf(Name: "RegisterOperand") &&
1279 !R->isValueUnset(FieldName: "GIZeroRegister")) {
1280 MIBuilder.addRenderer<CopyOrAddZeroRegRenderer>(
1281 args&: NodeName, args: R->getValueAsDef(FieldName: "GIZeroRegister"));
1282 return Error::success();
1283 }
1284
1285 // TODO: All special cases are handled above. Remove this check and add
1286 // CopyRenderer unconditionally.
1287 if (R->isSubClassOf(Name: "RegisterClass") ||
1288 R->isSubClassOf(Name: "RegisterOperand") || R->isSubClassOf(Name: "ValueType")) {
1289 MIBuilder.addRenderer<CopyRenderer>(args&: NodeName);
1290 return Error::success();
1291 }
1292 }
1293
1294 // TODO: Change this to assert and move to the beginning of the function.
1295 if (!M.hasOperand(SymbolicName: NodeName))
1296 return failedImport(Reason: "could not find node $" + NodeName +
1297 " in the source DAG");
1298
1299 // TODO: Remove this check and add CopyRenderer unconditionally.
1300 // TODO: Handle nodes with multiple results (provided they can reach here).
1301 if (isa<UnsetInit>(Val: N.getLeafValue())) {
1302 MIBuilder.addRenderer<CopyRenderer>(args&: NodeName);
1303 return Error::success();
1304 }
1305
1306 return failedImport(Reason: "unsupported node " + to_string(Value: N));
1307}
1308
1309// Equivalent of MatcherGen::EmitResultLeafAsOperand.
1310Error GlobalISelEmitter::importLeafNodeRenderer(
1311 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N,
1312 action_iterator InsertPt) const {
1313 if (const auto *II = dyn_cast<IntInit>(Val: N.getLeafValue())) {
1314 MIBuilder.addRenderer<ImmRenderer>(args: II->getValue());
1315 return Error::success();
1316 }
1317
1318 if (const auto *DI = dyn_cast<DefInit>(Val: N.getLeafValue())) {
1319 const Record *R = DI->getDef();
1320
1321 if (R->isSubClassOf(Name: "Register") || R->getName() == "zero_reg") {
1322 MIBuilder.addRenderer<AddRegisterRenderer>(args: Target, args&: R);
1323 return Error::success();
1324 }
1325
1326 if (R->getName() == "undef_tied_input") {
1327 std::optional<LLTCodeGen> OpTyOrNone = MVTToLLT(SVT: N.getSimpleType(ResNo: 0));
1328 if (!OpTyOrNone)
1329 return failedImport(Reason: "unsupported type");
1330
1331 unsigned TempRegID = M.allocateTempRegID();
1332 M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTyOrNone, args&: TempRegID);
1333
1334 auto I = M.insertAction<BuildMIAction>(
1335 InsertPt, args: M.allocateOutputInsnID(),
1336 args: &Target.getInstruction(InstRec: RK.getDef(Name: "IMPLICIT_DEF")));
1337 auto &ImpDefBuilder = static_cast<BuildMIAction &>(**I);
1338 ImpDefBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, /*IsDef=*/args: true);
1339
1340 MIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID);
1341 return Error::success();
1342 }
1343
1344 if (R->isSubClassOf(Name: "SubRegIndex")) {
1345 const CodeGenSubRegIndex *SubRegIndex = CGRegs.findSubRegIdx(Def: R);
1346 MIBuilder.addRenderer<ImmRenderer>(args: SubRegIndex->EnumValue);
1347 return Error::success();
1348 }
1349
1350 // There are also RegisterClass / RegisterOperand operands of REG_SEQUENCE /
1351 // COPY_TO_REGCLASS, but these instructions are currently handled elsewhere.
1352 }
1353
1354 return failedImport(Reason: "unrecognized node " + to_string(Value: N));
1355}
1356
1357// Equivalent of MatcherGen::EmitResultSDNodeXFormAsOperand.
1358Error GlobalISelEmitter::importXFormNodeRenderer(
1359 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N) const {
1360 const Record *XFormRec = N.getOperator();
1361 auto I = SDNodeXFormEquivs.find(Val: XFormRec);
1362 if (I == SDNodeXFormEquivs.end())
1363 return failedImport(Reason: "SDNodeXForm " + XFormRec->getName() +
1364 " does not have GISel equivalent");
1365
1366 // TODO: Fail to import if GISDNodeXForm does not have RendererFn.
1367 // This currently results in a fatal error in emitRenderOpcodes.
1368 const Record *XFormEquivRec = I->second;
1369
1370 // The node to apply the transformation function to.
1371 // FIXME: The node may not have a name and may be a leaf. It should be
1372 // rendered first, like any other nodes. This may or may not require
1373 // introducing a temporary register, and we can't tell that without
1374 // inspecting the node (possibly recursively). This is a general drawback
1375 // of appending renderers directly to BuildMIAction.
1376 const TreePatternNode &Node = N.getChild(N: 0);
1377 StringRef NodeName = Node.getName();
1378
1379 const Record *XFormOpc = CGP.getSDNodeTransform(R: XFormRec).first;
1380 if (XFormOpc->getName() == "timm") {
1381 // If this is a TargetConstant, there won't be a corresponding
1382 // instruction to transform. Instead, this will refer directly to an
1383 // operand in an instruction's operand list.
1384 MIBuilder.addRenderer<CustomOperandRenderer>(args: *XFormEquivRec, args&: NodeName);
1385 } else {
1386 MIBuilder.addRenderer<CustomRenderer>(args: *XFormEquivRec, args&: NodeName);
1387 }
1388
1389 return Error::success();
1390}
1391
1392// Equivalent of MatcherGen::EmitResultInstructionAsOperand.
1393Error GlobalISelEmitter::importInstructionNodeRenderer(
1394 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N,
1395 action_iterator &InsertPt) const {
1396 Expected<LLTCodeGen> OpTy = getInstResultType(Dst: N, Target);
1397 if (!OpTy)
1398 return OpTy.takeError();
1399
1400 // TODO: See the comment in importXFormNodeRenderer. We rely on the node
1401 // requiring a temporary register, which prevents us from using this
1402 // function on the root of the destination DAG.
1403 unsigned TempRegID = M.allocateTempRegID();
1404 InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTy, args&: TempRegID);
1405 MIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID);
1406
1407 auto InsertPtOrError =
1408 createAndImportSubInstructionRenderer(InsertPt: ++InsertPt, M, Dst: N, TempReg: TempRegID);
1409 if (!InsertPtOrError)
1410 return InsertPtOrError.takeError();
1411
1412 InsertPt = *InsertPtOrError;
1413 return Error::success();
1414}
1415
1416// Equivalent of MatcherGen::EmitResultOperand.
1417Error GlobalISelEmitter::importNodeRenderer(RuleMatcher &M,
1418 BuildMIAction &MIBuilder,
1419 const TreePatternNode &N,
1420 action_iterator &InsertPt) const {
1421 if (N.hasName())
1422 return importNamedNodeRenderer(M, MIBuilder, N);
1423
1424 if (N.isLeaf())
1425 return importLeafNodeRenderer(M, MIBuilder, N, InsertPt);
1426
1427 if (N.getOperator()->isSubClassOf(Name: "SDNodeXForm"))
1428 return importXFormNodeRenderer(M, MIBuilder, N);
1429
1430 if (N.getOperator()->isSubClassOf(Name: "Instruction"))
1431 return importInstructionNodeRenderer(M, MIBuilder, N, InsertPt);
1432
1433 // Should not reach here.
1434 return failedImport(Reason: "unrecognized node " + llvm::to_string(Value: N));
1435}
1436
1437/// Generates code that builds the resulting instruction(s) from the destination
1438/// DAG. Note that to do this we do not and should not need the source DAG.
1439/// We do need to know whether a generated instruction defines a result of the
1440/// source DAG; this information is available via RuleMatcher::hasOperand.
1441Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
1442 RuleMatcher &M, InstructionMatcher &InsnMatcher,
1443 const TreePatternNode &Dst) const {
1444 auto InsertPtOrError = createInstructionRenderer(InsertPt: M.actions_end(), M, Dst);
1445 if (auto Error = InsertPtOrError.takeError())
1446 return std::move(Error);
1447
1448 action_iterator InsertPt = InsertPtOrError.get();
1449 BuildMIAction &DstMIBuilder = *static_cast<BuildMIAction *>(InsertPt->get());
1450
1451 for (auto PhysOp : M.physoperands()) {
1452 InsertPt = M.insertAction<BuildMIAction>(
1453 InsertPt, args: M.allocateOutputInsnID(),
1454 args: &Target.getInstruction(InstRec: RK.getDef(Name: "COPY")));
1455 BuildMIAction &CopyToPhysRegMIBuilder =
1456 *static_cast<BuildMIAction *>(InsertPt->get());
1457 CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>(args: Target,
1458 args&: PhysOp.first, args: true);
1459 CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(args&: PhysOp.first);
1460 }
1461
1462 if (auto Error = importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Dst,
1463 /*IsRoot=*/true)
1464 .takeError())
1465 return std::move(Error);
1466
1467 if (auto Error = importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst)
1468 .takeError())
1469 return std::move(Error);
1470
1471 return DstMIBuilder;
1472}
1473
1474Expected<action_iterator>
1475GlobalISelEmitter::createAndImportSubInstructionRenderer(
1476 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
1477 unsigned TempRegID) const {
1478 auto InsertPtOrError = createInstructionRenderer(InsertPt, M, Dst);
1479
1480 // TODO: Assert there's exactly one result.
1481
1482 if (auto Error = InsertPtOrError.takeError())
1483 return std::move(Error);
1484
1485 BuildMIAction &DstMIBuilder =
1486 *static_cast<BuildMIAction *>(InsertPtOrError.get()->get());
1487
1488 // Assign the result to TempReg.
1489 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, args: true);
1490
1491 // Handle additional (ignored) results.
1492 InsertPtOrError = importExplicitDefRenderers(
1493 InsertPt: std::prev(x: *InsertPtOrError), M, DstMIBuilder, Dst, /*IsRoot=*/false);
1494 if (auto Error = InsertPtOrError.takeError())
1495 return std::move(Error);
1496
1497 InsertPtOrError =
1498 importExplicitUseRenderers(InsertPt: InsertPtOrError.get(), M, DstMIBuilder, Dst);
1499 if (auto Error = InsertPtOrError.takeError())
1500 return std::move(Error);
1501
1502 if (auto Error =
1503 constrainOperands(InsertPt, M, InsnID: DstMIBuilder.getInsnID(), Dst))
1504 return std::move(Error);
1505
1506 return InsertPtOrError.get();
1507}
1508
1509Expected<action_iterator>
1510GlobalISelEmitter::createInstructionRenderer(action_iterator InsertPt,
1511 RuleMatcher &M,
1512 const TreePatternNode &Dst) const {
1513 const Record *DstOp = Dst.getOperator();
1514 if (!DstOp->isSubClassOf(Name: "Instruction")) {
1515 if (DstOp->isSubClassOf(Name: "ValueType"))
1516 return failedImport(
1517 Reason: "Pattern operator isn't an instruction (it's a ValueType)");
1518 return failedImport(Reason: "Pattern operator isn't an instruction");
1519 }
1520 CodeGenInstruction *DstI = &Target.getInstruction(InstRec: DstOp);
1521
1522 // COPY_TO_REGCLASS is just a copy with a ConstrainOperandToRegClassAction
1523 // attached. Similarly for EXTRACT_SUBREG except that's a subregister copy.
1524 StringRef Name = DstI->TheDef->getName();
1525 if (Name == "COPY_TO_REGCLASS" || Name == "EXTRACT_SUBREG")
1526 DstI = &Target.getInstruction(InstRec: RK.getDef(Name: "COPY"));
1527
1528 return M.insertAction<BuildMIAction>(InsertPt, args: M.allocateOutputInsnID(),
1529 args&: DstI);
1530}
1531
1532Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
1533 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
1534 const TreePatternNode &Dst, bool IsRoot) const {
1535 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
1536
1537 // Process explicit defs. The caller may have already handled the first def.
1538 for (unsigned I = IsRoot ? 0 : 1, E = DstI->Operands.NumDefs; I != E; ++I) {
1539 const CGIOperandList::OperandInfo &OpInfo = DstI->Operands[I];
1540 std::string OpName = getMangledRootDefName(DefOperandName: OpInfo.Name);
1541
1542 // If the def is used in the source DAG, forward it.
1543 if (IsRoot && M.hasOperand(SymbolicName: OpName)) {
1544 // CopyRenderer saves a StringRef, so cannot pass OpName itself -
1545 // let's use a string with an appropriate lifetime.
1546 StringRef PermanentRef = M.getOperandMatcher(Name: OpName).getSymbolicName();
1547 DstMIBuilder.addRenderer<CopyRenderer>(args&: PermanentRef);
1548 continue;
1549 }
1550
1551 // A discarded explicit def may be an optional physical register.
1552 // If this is the case, add the default register and mark it as dead.
1553 if (OpInfo.Rec->isSubClassOf(Name: "OptionalDefOperand")) {
1554 for (const TreePatternNode &DefaultOp :
1555 make_pointee_range(Range: CGP.getDefaultOperand(R: OpInfo.Rec).DefaultOps)) {
1556 // TODO: Do these checks in ParseDefaultOperands.
1557 if (!DefaultOp.isLeaf())
1558 return failedImport(Reason: "optional def is not a leaf");
1559
1560 const auto *RegDI = dyn_cast<DefInit>(Val: DefaultOp.getLeafValue());
1561 if (!RegDI)
1562 return failedImport(Reason: "optional def is not a record");
1563
1564 const Record *Reg = RegDI->getDef();
1565 if (!Reg->isSubClassOf(Name: "Register") && Reg->getName() != "zero_reg")
1566 return failedImport(Reason: "optional def is not a register");
1567
1568 DstMIBuilder.addRenderer<AddRegisterRenderer>(
1569 args: Target, args&: Reg, /*IsDef=*/args: true, /*IsDead=*/args: true);
1570 }
1571 continue;
1572 }
1573
1574 // The def is discarded, create a dead virtual register for it.
1575 const TypeSetByHwMode &ExtTy = Dst.getExtType(ResNo: I);
1576 if (!ExtTy.isMachineValueType())
1577 return failedImport(Reason: "unsupported typeset");
1578
1579 auto OpTy = MVTToLLT(SVT: ExtTy.getMachineValueType().SimpleTy);
1580 if (!OpTy)
1581 return failedImport(Reason: "unsupported type");
1582
1583 unsigned TempRegID = M.allocateTempRegID();
1584 InsertPt =
1585 M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTy, args&: TempRegID);
1586 DstMIBuilder.addRenderer<TempRegRenderer>(
1587 args&: TempRegID, /*IsDef=*/args: true, /*SubReg=*/args: nullptr, /*IsDead=*/args: true);
1588 }
1589
1590 // Implicit defs are not currently supported, mark all of them as dead.
1591 for (const Record *Reg : DstI->ImplicitDefs) {
1592 std::string OpName = getMangledRootDefName(DefOperandName: Reg->getName());
1593 assert(!M.hasOperand(OpName) && "The pattern should've been rejected");
1594 DstMIBuilder.setDeadImplicitDef(Reg);
1595 }
1596
1597 return InsertPt;
1598}
1599
1600Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
1601 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
1602 const TreePatternNode &Dst) const {
1603 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
1604 CodeGenInstruction *OrigDstI = &Target.getInstruction(InstRec: Dst.getOperator());
1605
1606 StringRef Name = OrigDstI->TheDef->getName();
1607 unsigned ExpectedDstINumUses = Dst.getNumChildren();
1608
1609 // EXTRACT_SUBREG needs to use a subregister COPY.
1610 if (Name == "EXTRACT_SUBREG") {
1611 if (!Dst.getChild(N: 1).isLeaf())
1612 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a leaf");
1613 const DefInit *SubRegInit =
1614 dyn_cast<DefInit>(Val: Dst.getChild(N: 1).getLeafValue());
1615 if (!SubRegInit)
1616 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a subreg index");
1617
1618 const CodeGenSubRegIndex *SubIdx =
1619 CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
1620 const TreePatternNode &ValChild = Dst.getChild(N: 0);
1621 if (!ValChild.isLeaf()) {
1622 // We really have to handle the source instruction, and then insert a
1623 // copy from the subregister.
1624 auto ExtractSrcTy = getInstResultType(Dst: ValChild, Target);
1625 if (!ExtractSrcTy)
1626 return ExtractSrcTy.takeError();
1627
1628 unsigned TempRegID = M.allocateTempRegID();
1629 InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *ExtractSrcTy,
1630 args&: TempRegID);
1631
1632 auto InsertPtOrError = createAndImportSubInstructionRenderer(
1633 InsertPt: ++InsertPt, M, Dst: ValChild, TempRegID);
1634 if (auto Error = InsertPtOrError.takeError())
1635 return std::move(Error);
1636
1637 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, args: false, args&: SubIdx);
1638 return InsertPt;
1639 }
1640
1641 // If this is a source operand, this is just a subregister copy.
1642 const Record *RCDef = getInitValueAsRegClass(V: ValChild.getLeafValue());
1643 if (!RCDef)
1644 return failedImport(Reason: "EXTRACT_SUBREG child #0 could not "
1645 "be coerced to a register class");
1646
1647 CodeGenRegisterClass *RC = CGRegs.getRegClass(RCDef);
1648
1649 const auto SrcRCDstRCPair =
1650 RC->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1651 if (SrcRCDstRCPair) {
1652 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
1653 if (SrcRCDstRCPair->first != RC)
1654 return failedImport(Reason: "EXTRACT_SUBREG requires an additional COPY");
1655 }
1656
1657 StringRef RegOperandName = Dst.getChild(N: 0).getName();
1658 if (const auto &SubOperand = M.getComplexSubOperand(SymbolicName: RegOperandName)) {
1659 DstMIBuilder.addRenderer<RenderComplexPatternOperand>(
1660 args: *std::get<0>(t: *SubOperand), args&: RegOperandName, args: std::get<1>(t: *SubOperand),
1661 args: std::get<2>(t: *SubOperand), args&: SubIdx);
1662 return InsertPt;
1663 }
1664
1665 DstMIBuilder.addRenderer<CopySubRegRenderer>(args&: RegOperandName, args&: SubIdx);
1666 return InsertPt;
1667 }
1668
1669 if (Name == "REG_SEQUENCE") {
1670 if (!Dst.getChild(N: 0).isLeaf())
1671 return failedImport(Reason: "REG_SEQUENCE child #0 is not a leaf");
1672
1673 const Record *RCDef =
1674 getInitValueAsRegClass(V: Dst.getChild(N: 0).getLeafValue());
1675 if (!RCDef)
1676 return failedImport(Reason: "REG_SEQUENCE child #0 could not "
1677 "be coerced to a register class");
1678
1679 if ((ExpectedDstINumUses - 1) % 2 != 0)
1680 return failedImport(Reason: "Malformed REG_SEQUENCE");
1681
1682 for (unsigned I = 1; I != ExpectedDstINumUses; I += 2) {
1683 const TreePatternNode &ValChild = Dst.getChild(N: I);
1684 const TreePatternNode &SubRegChild = Dst.getChild(N: I + 1);
1685
1686 if (const DefInit *SubRegInit =
1687 dyn_cast<DefInit>(Val: SubRegChild.getLeafValue())) {
1688 const CodeGenSubRegIndex *SubIdx =
1689 CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
1690
1691 if (Error Err = importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: ValChild, InsertPt))
1692 return Err;
1693
1694 DstMIBuilder.addRenderer<SubRegIndexRenderer>(args&: SubIdx);
1695 }
1696 }
1697
1698 return InsertPt;
1699 }
1700
1701 // Render the explicit uses.
1702 unsigned DstINumUses = OrigDstI->Operands.size() - OrigDstI->Operands.NumDefs;
1703 if (Name == "COPY_TO_REGCLASS") {
1704 DstINumUses--; // Ignore the class constraint.
1705 ExpectedDstINumUses--;
1706 }
1707
1708 // NumResults - This is the number of results produced by the instruction in
1709 // the "outs" list.
1710 unsigned NumResults = OrigDstI->Operands.NumDefs;
1711
1712 // Number of operands we know the output instruction must have. If it is
1713 // variadic, we could have more operands.
1714 unsigned NumFixedOperands = DstI->Operands.size();
1715
1716 // Loop over all of the fixed operands of the instruction pattern, emitting
1717 // code to fill them all in. The node 'N' usually has number children equal to
1718 // the number of input operands of the instruction. However, in cases where
1719 // there are predicate operands for an instruction, we need to fill in the
1720 // 'execute always' values. Match up the node operands to the instruction
1721 // operands to do this.
1722 unsigned Child = 0;
1723
1724 // Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the
1725 // number of operands at the end of the list which have default values.
1726 // Those can come from the pattern if it provides enough arguments, or be
1727 // filled in with the default if the pattern hasn't provided them. But any
1728 // operand with a default value _before_ the last mandatory one will be
1729 // filled in with their defaults unconditionally.
1730 unsigned NonOverridableOperands = NumFixedOperands;
1731 while (NonOverridableOperands > NumResults &&
1732 CGP.operandHasDefault(Op: DstI->Operands[NonOverridableOperands - 1].Rec))
1733 --NonOverridableOperands;
1734
1735 unsigned NumDefaultOps = 0;
1736 for (unsigned I = 0; I != DstINumUses; ++I) {
1737 unsigned InstOpNo = DstI->Operands.NumDefs + I;
1738
1739 // Determine what to emit for this operand.
1740 const Record *OperandNode = DstI->Operands[InstOpNo].Rec;
1741
1742 // If the operand has default values, introduce them now.
1743 if (CGP.operandHasDefault(Op: OperandNode) &&
1744 (InstOpNo < NonOverridableOperands || Child >= Dst.getNumChildren())) {
1745 // This is a predicate or optional def operand which the pattern has not
1746 // overridden, or which we aren't letting it override; emit the 'default
1747 // ops' operands.
1748 for (const TreePatternNode &OpNode :
1749 make_pointee_range(Range: CGP.getDefaultOperand(R: OperandNode).DefaultOps)) {
1750 if (Error Err = importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: OpNode, InsertPt))
1751 return Err;
1752 }
1753
1754 ++NumDefaultOps;
1755 continue;
1756 }
1757
1758 if (Error Err =
1759 importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: Dst.getChild(N: Child), InsertPt))
1760 return Err;
1761
1762 ++Child;
1763 }
1764
1765 if (NumDefaultOps + ExpectedDstINumUses != DstINumUses)
1766 return failedImport(Reason: "Expected " + llvm::to_string(Value: DstINumUses) +
1767 " used operands but found " +
1768 llvm::to_string(Value: ExpectedDstINumUses) +
1769 " explicit ones and " + llvm::to_string(Value: NumDefaultOps) +
1770 " default ones");
1771
1772 return InsertPt;
1773}
1774
1775Error GlobalISelEmitter::importImplicitDefRenderers(
1776 BuildMIAction &DstMIBuilder, ArrayRef<const Record *> ImplicitDefs) const {
1777 if (!ImplicitDefs.empty())
1778 return failedImport(Reason: "Pattern defines a physical register");
1779 return Error::success();
1780}
1781
1782Error GlobalISelEmitter::constrainOperands(action_iterator InsertPt,
1783 RuleMatcher &M, unsigned InsnID,
1784 const TreePatternNode &Dst) const {
1785 const Record *DstOp = Dst.getOperator();
1786 const CodeGenInstruction &DstI = Target.getInstruction(InstRec: DstOp);
1787 StringRef DstIName = DstI.TheDef->getName();
1788
1789 if (DstIName == "COPY_TO_REGCLASS") {
1790 // COPY_TO_REGCLASS does not provide operand constraints itself but the
1791 // result is constrained to the class given by the second child.
1792 const Record *DstIOpRec =
1793 getInitValueAsRegClass(V: Dst.getChild(N: 1).getLeafValue());
1794
1795 if (DstIOpRec == nullptr)
1796 return failedImport(Reason: "COPY_TO_REGCLASS operand #1 isn't a register class");
1797
1798 M.insertAction<ConstrainOperandToRegClassAction>(
1799 InsertPt, args&: InsnID, args: 0, args: Target.getRegisterClass(R: DstIOpRec));
1800 } else if (DstIName == "EXTRACT_SUBREG") {
1801 const CodeGenRegisterClass *SuperClass =
1802 inferRegClassFromPattern(N: Dst.getChild(N: 0));
1803 if (!SuperClass)
1804 return failedImport(
1805 Reason: "Cannot infer register class from EXTRACT_SUBREG operand #0");
1806
1807 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: Dst.getChild(N: 1));
1808 if (!SubIdx)
1809 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a subreg index");
1810
1811 // It would be nice to leave this constraint implicit but we're required
1812 // to pick a register class so constrain the result to a register class
1813 // that can hold the correct MVT.
1814 //
1815 // FIXME: This may introduce an extra copy if the chosen class doesn't
1816 // actually contain the subregisters.
1817 const auto SrcRCDstRCPair =
1818 SuperClass->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1819 if (!SrcRCDstRCPair) {
1820 return failedImport(Reason: "subreg index is incompatible "
1821 "with inferred reg class");
1822 }
1823
1824 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
1825 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1826 args&: *SrcRCDstRCPair->second);
1827 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 1,
1828 args&: *SrcRCDstRCPair->first);
1829 } else if (DstIName == "INSERT_SUBREG") {
1830 // We need to constrain the destination, a super regsister source, and a
1831 // subregister source.
1832 const CodeGenRegisterClass *SubClass =
1833 inferRegClassFromPattern(N: Dst.getChild(N: 1));
1834 if (!SubClass)
1835 return failedImport(
1836 Reason: "Cannot infer register class from INSERT_SUBREG operand #1");
1837 const CodeGenRegisterClass *SuperClass = inferSuperRegisterClassForNode(
1838 Ty: Dst.getExtType(ResNo: 0), SuperRegNode: Dst.getChild(N: 0), SubRegIdxNode: Dst.getChild(N: 2));
1839 if (!SuperClass)
1840 return failedImport(
1841 Reason: "Cannot infer register class for INSERT_SUBREG operand #0");
1842 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1843 args: *SuperClass);
1844 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 1,
1845 args: *SuperClass);
1846 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 2,
1847 args: *SubClass);
1848 } else if (DstIName == "SUBREG_TO_REG") {
1849 // We need to constrain the destination and subregister source.
1850 // Attempt to infer the subregister source from the first child. If it has
1851 // an explicitly given register class, we'll use that. Otherwise, we will
1852 // fail.
1853 const CodeGenRegisterClass *SubClass =
1854 inferRegClassFromPattern(N: Dst.getChild(N: 1));
1855 if (!SubClass)
1856 return failedImport(
1857 Reason: "Cannot infer register class from SUBREG_TO_REG child #1");
1858 // We don't have a child to look at that might have a super register node.
1859 const CodeGenRegisterClass *SuperClass =
1860 inferSuperRegisterClass(Ty: Dst.getExtType(ResNo: 0), SubRegIdxNode: Dst.getChild(N: 2));
1861 if (!SuperClass)
1862 return failedImport(
1863 Reason: "Cannot infer register class for SUBREG_TO_REG operand #0");
1864 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1865 args: *SuperClass);
1866 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 2,
1867 args: *SubClass);
1868 } else if (DstIName == "REG_SEQUENCE") {
1869 const CodeGenRegisterClass *SuperClass =
1870 inferRegClassFromPattern(N: Dst.getChild(N: 0));
1871
1872 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1873 args: *SuperClass);
1874
1875 unsigned Num = Dst.getNumChildren();
1876 for (unsigned I = 1; I != Num; I += 2) {
1877 const TreePatternNode &SubRegChild = Dst.getChild(N: I + 1);
1878
1879 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: SubRegChild);
1880 if (!SubIdx)
1881 return failedImport(Reason: "REG_SEQUENCE child is not a subreg index");
1882
1883 const auto SrcRCDstRCPair =
1884 SuperClass->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1885
1886 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args&: I,
1887 args&: *SrcRCDstRCPair->second);
1888 }
1889 } else {
1890 M.insertAction<ConstrainOperandsToDefinitionAction>(InsertPt, args&: InsnID);
1891 }
1892
1893 return Error::success();
1894}
1895
1896const CodeGenRegisterClass *
1897GlobalISelEmitter::getRegClassFromLeaf(const TreePatternNode &Leaf) const {
1898 assert(Leaf.isLeaf() && "Expected leaf?");
1899 const Record *RCRec = getInitValueAsRegClass(V: Leaf.getLeafValue());
1900 if (!RCRec)
1901 return nullptr;
1902 return CGRegs.getRegClass(RCRec);
1903}
1904
1905const CodeGenRegisterClass *
1906GlobalISelEmitter::inferRegClassFromPattern(const TreePatternNode &N) const {
1907 if (N.isLeaf())
1908 return getRegClassFromLeaf(Leaf: N);
1909
1910 // We don't have a leaf node, so we have to try and infer something. Check
1911 // that we have an instruction that we can infer something from.
1912
1913 // Only handle things that produce at least one value (if multiple values,
1914 // just take the first one).
1915 if (N.getNumTypes() < 1)
1916 return nullptr;
1917 const Record *OpRec = N.getOperator();
1918
1919 // We only want instructions.
1920 if (!OpRec->isSubClassOf(Name: "Instruction"))
1921 return nullptr;
1922
1923 // Don't want to try and infer things when there could potentially be more
1924 // than one candidate register class.
1925 return inferRegClassFromInstructionPattern(N, /*ResIdx=*/0);
1926}
1927
1928const CodeGenRegisterClass *
1929GlobalISelEmitter::inferRegClassFromInstructionPattern(const TreePatternNode &N,
1930 unsigned ResIdx) const {
1931 const CodeGenInstruction &Inst = Target.getInstruction(InstRec: N.getOperator());
1932 assert(ResIdx < Inst.Operands.NumDefs &&
1933 "Can only infer register class for explicit defs");
1934
1935 // Handle any special-case instructions which we can safely infer register
1936 // classes from.
1937 StringRef InstName = Inst.TheDef->getName();
1938 if (InstName == "REG_SEQUENCE") {
1939 // (outs $super_dst), (ins $dst_regclass, variable_ops)
1940 // Destination register class is explicitly specified by the first operand.
1941 const TreePatternNode &RCChild = N.getChild(N: 0);
1942 if (!RCChild.isLeaf())
1943 return nullptr;
1944 return getRegClassFromLeaf(Leaf: RCChild);
1945 }
1946
1947 if (InstName == "COPY_TO_REGCLASS") {
1948 // (outs $dst), (ins $src, $dst_regclass)
1949 // Destination register class is explicitly specified by the second operand.
1950 const TreePatternNode &RCChild = N.getChild(N: 1);
1951 if (!RCChild.isLeaf())
1952 return nullptr;
1953 return getRegClassFromLeaf(Leaf: RCChild);
1954 }
1955
1956 if (InstName == "INSERT_SUBREG") {
1957 // (outs $super_dst), (ins $super_src, $sub_src, $sub_idx);
1958 // If we can infer the register class for the first operand, use that.
1959 // Otherwise, find a register class that supports both the specified
1960 // sub-register index and the type of the instruction's result.
1961 const TreePatternNode &Child0 = N.getChild(N: 0);
1962 assert(Child0.getNumTypes() == 1 && "Unexpected number of types!");
1963 return inferSuperRegisterClassForNode(Ty: N.getExtType(ResNo: 0), SuperRegNode: Child0,
1964 SubRegIdxNode: N.getChild(N: 2));
1965 }
1966
1967 if (InstName == "EXTRACT_SUBREG") {
1968 // (outs $sub_dst), (ins $super_src, $sub_idx)
1969 // Find a register class that can be used for a sub-register copy from
1970 // the specified source at the specified sub-register index.
1971 const CodeGenRegisterClass *SuperRC =
1972 inferRegClassFromPattern(N: N.getChild(N: 0));
1973 if (!SuperRC)
1974 return nullptr;
1975
1976 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: N.getChild(N: 1));
1977 if (!SubIdx)
1978 return nullptr;
1979
1980 const auto SubRCAndSubRegRC =
1981 SuperRC->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1982 if (!SubRCAndSubRegRC)
1983 return nullptr;
1984
1985 return SubRCAndSubRegRC->second;
1986 }
1987
1988 if (InstName == "SUBREG_TO_REG") {
1989 // (outs $super_dst), (ins $super_src, $sub_src, $sub_idx)
1990 // Find a register class that supports both the specified sub-register
1991 // index and the type of the instruction's result.
1992 return inferSuperRegisterClass(Ty: N.getExtType(ResNo: 0), SubRegIdxNode: N.getChild(N: 2));
1993 }
1994
1995 // Handle destination record types that we can safely infer a register class
1996 // from.
1997 const auto &DstIOperand = Inst.Operands[ResIdx];
1998 const Record *DstIOpRec = DstIOperand.Rec;
1999 if (DstIOpRec->isSubClassOf(Name: "RegisterOperand"))
2000 return &Target.getRegisterClass(R: DstIOpRec->getValueAsDef(FieldName: "RegClass"));
2001
2002 if (DstIOpRec->isSubClassOf(Name: "RegisterClass"))
2003 return &Target.getRegisterClass(R: DstIOpRec);
2004
2005 return nullptr;
2006}
2007
2008const CodeGenRegisterClass *GlobalISelEmitter::inferSuperRegisterClass(
2009 const TypeSetByHwMode &Ty, const TreePatternNode &SubRegIdxNode) const {
2010 // We need a ValueTypeByHwMode for getSuperRegForSubReg.
2011 if (!Ty.isValueTypeByHwMode(AllowEmpty: false))
2012 return nullptr;
2013 if (!SubRegIdxNode.isLeaf())
2014 return nullptr;
2015 const DefInit *SubRegInit = dyn_cast<DefInit>(Val: SubRegIdxNode.getLeafValue());
2016 if (!SubRegInit)
2017 return nullptr;
2018 const CodeGenSubRegIndex *SubIdx = CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
2019
2020 // Use the information we found above to find a minimal register class which
2021 // supports the subregister and type we want.
2022 return CGRegs.getSuperRegForSubReg(Ty: Ty.getValueTypeByHwMode(), SubIdx,
2023 /*MustBeAllocatable=*/true);
2024}
2025
2026const CodeGenRegisterClass *GlobalISelEmitter::inferSuperRegisterClassForNode(
2027 const TypeSetByHwMode &Ty, const TreePatternNode &SuperRegNode,
2028 const TreePatternNode &SubRegIdxNode) const {
2029 // Check if we already have a defined register class for the super register
2030 // node. If we do, then we should preserve that rather than inferring anything
2031 // from the subregister index node. We can assume that whoever wrote the
2032 // pattern in the first place made sure that the super register and
2033 // subregister are compatible.
2034 if (const CodeGenRegisterClass *SuperRegisterClass =
2035 inferRegClassFromPattern(N: SuperRegNode))
2036 return SuperRegisterClass;
2037 return inferSuperRegisterClass(Ty, SubRegIdxNode);
2038}
2039
2040const CodeGenSubRegIndex *GlobalISelEmitter::inferSubRegIndexForNode(
2041 const TreePatternNode &SubRegIdxNode) const {
2042 if (!SubRegIdxNode.isLeaf())
2043 return nullptr;
2044
2045 const DefInit *SubRegInit = dyn_cast<DefInit>(Val: SubRegIdxNode.getLeafValue());
2046 if (!SubRegInit)
2047 return nullptr;
2048 return CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
2049}
2050
2051Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
2052 // Keep track of the matchers and actions to emit.
2053 int Score = P.getPatternComplexity(CGP);
2054 RuleMatcher M(P.getSrcRecord()->getLoc());
2055 RuleMatcherScores[M.getRuleID()] = Score;
2056 M.addAction<DebugCommentAction>(args: llvm::to_string(Value: P.getSrcPattern()) +
2057 " => " +
2058 llvm::to_string(Value: P.getDstPattern()));
2059
2060 SmallVector<const Record *, 4> Predicates;
2061 P.getPredicateRecords(PredicateRecs&: Predicates);
2062 if (auto Error = importRulePredicates(M, Predicates))
2063 return std::move(Error);
2064
2065 if (!P.getHwModeFeatures().empty())
2066 M.addHwModeIdx(Idx: declareHwModeCheck(HwModeFeatures: P.getHwModeFeatures()));
2067
2068 // Next, analyze the pattern operators.
2069 TreePatternNode &Src = P.getSrcPattern();
2070 TreePatternNode &Dst = P.getDstPattern();
2071
2072 // If the root of either pattern isn't a simple operator, ignore it.
2073 if (auto Err = isTrivialOperatorNode(N: Dst))
2074 return failedImport(Reason: "Dst pattern root isn't a trivial operator (" +
2075 toString(E: std::move(Err)) + ")");
2076 if (auto Err = isTrivialOperatorNode(N: Src))
2077 return failedImport(Reason: "Src pattern root isn't a trivial operator (" +
2078 toString(E: std::move(Err)) + ")");
2079
2080 // The different predicates and matchers created during
2081 // addInstructionMatcher use the RuleMatcher M to set up their
2082 // instruction ID (InsnVarID) that are going to be used when
2083 // M is going to be emitted.
2084 // However, the code doing the emission still relies on the IDs
2085 // returned during that process by the RuleMatcher when issuing
2086 // the recordInsn opcodes.
2087 // Because of that:
2088 // 1. The order in which we created the predicates
2089 // and such must be the same as the order in which we emit them,
2090 // and
2091 // 2. We need to reset the generation of the IDs in M somewhere between
2092 // addInstructionMatcher and emit
2093 //
2094 // FIXME: Long term, we don't want to have to rely on this implicit
2095 // naming being the same. One possible solution would be to have
2096 // explicit operator for operation capture and reference those.
2097 // The plus side is that it would expose opportunities to share
2098 // the capture accross rules. The downside is that it would
2099 // introduce a dependency between predicates (captures must happen
2100 // before their first use.)
2101 InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(SymbolicName: Src.getName());
2102 unsigned TempOpIdx = 0;
2103
2104 const auto SavedFlags = M.setGISelFlags(P.getSrcRecord());
2105
2106 auto InsnMatcherOrError =
2107 createAndImportSelDAGMatcher(Rule&: M, InsnMatcher&: InsnMatcherTemp, Src, TempOpIdx);
2108 if (auto Error = InsnMatcherOrError.takeError())
2109 return std::move(Error);
2110 InstructionMatcher &InsnMatcher = InsnMatcherOrError.get();
2111
2112 if (Dst.isLeaf()) {
2113 if (const Record *RCDef = getInitValueAsRegClass(V: Dst.getLeafValue())) {
2114 const CodeGenRegisterClass &RC = Target.getRegisterClass(R: RCDef);
2115
2116 // We need to replace the def and all its uses with the specified
2117 // operand. However, we must also insert COPY's wherever needed.
2118 // For now, emit a copy and let the register allocator clean up.
2119 auto &DstI = Target.getInstruction(InstRec: RK.getDef(Name: "COPY"));
2120 const auto &DstIOperand = DstI.Operands[0];
2121
2122 OperandMatcher &OM0 = InsnMatcher.getOperand(OpIdx: 0);
2123 OM0.setSymbolicName(DstIOperand.Name);
2124 M.defineOperand(SymbolicName: OM0.getSymbolicName(), OM&: OM0);
2125 OM0.addPredicate<RegisterBankOperandMatcher>(args: RC);
2126
2127 auto &DstMIBuilder =
2128 M.addAction<BuildMIAction>(args: M.allocateOutputInsnID(), args: &DstI);
2129 DstMIBuilder.addRenderer<CopyRenderer>(args: DstIOperand.Name);
2130 DstMIBuilder.addRenderer<CopyRenderer>(args: Dst.getName());
2131 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 0, args: RC);
2132
2133 // Erase the root.
2134 unsigned RootInsnID = M.getInsnVarID(InsnMatcher);
2135 M.addAction<EraseInstAction>(args&: RootInsnID);
2136
2137 // We're done with this pattern! It's eligible for GISel emission; return
2138 // it.
2139 ++NumPatternImported;
2140 return std::move(M);
2141 }
2142
2143 return failedImport(Reason: "Dst pattern root isn't a known leaf");
2144 }
2145
2146 // Start with the defined operands (i.e., the results of the root operator).
2147 const Record *DstOp = Dst.getOperator();
2148 if (!DstOp->isSubClassOf(Name: "Instruction"))
2149 return failedImport(Reason: "Pattern operator isn't an instruction");
2150
2151 const CodeGenInstruction &DstI = Target.getInstruction(InstRec: DstOp);
2152
2153 // Count both implicit and explicit defs in the dst instruction.
2154 // This avoids errors importing patterns that have inherent implicit defs.
2155 unsigned DstExpDefs = DstI.Operands.NumDefs,
2156 DstNumDefs = DstI.ImplicitDefs.size() + DstExpDefs,
2157 SrcNumDefs = Src.getExtTypes().size();
2158 if (DstNumDefs < SrcNumDefs) {
2159 if (DstNumDefs != 0)
2160 return failedImport(Reason: "Src pattern result has more defs than dst MI (" +
2161 to_string(Value: SrcNumDefs) + " def(s) vs " +
2162 to_string(Value: DstNumDefs) + " def(s))");
2163
2164 bool FoundNoUsePred = false;
2165 for (const auto &Pred : InsnMatcher.predicates()) {
2166 if ((FoundNoUsePred = isa<NoUsePredicateMatcher>(Val: Pred.get())))
2167 break;
2168 }
2169 if (!FoundNoUsePred)
2170 return failedImport(Reason: "Src pattern result has " + to_string(Value: SrcNumDefs) +
2171 " def(s) without the HasNoUse predicate set to true "
2172 "but Dst MI has no def");
2173 }
2174
2175 // The root of the match also has constraints on the register bank so that it
2176 // matches the result instruction.
2177 unsigned N = std::min(a: DstExpDefs, b: SrcNumDefs);
2178 for (unsigned I = 0; I < N; ++I) {
2179 const auto &DstIOperand = DstI.Operands[I];
2180
2181 OperandMatcher &OM = InsnMatcher.getOperand(OpIdx: I);
2182 // The operand names declared in the DstI instruction are unrelated to
2183 // those used in pattern's source and destination DAGs, so mangle the
2184 // former to prevent implicitly adding unexpected
2185 // GIM_CheckIsSameOperand predicates by the defineOperand method.
2186 OM.setSymbolicName(getMangledRootDefName(DefOperandName: DstIOperand.Name));
2187 M.defineOperand(SymbolicName: OM.getSymbolicName(), OM);
2188
2189 const CodeGenRegisterClass *RC =
2190 inferRegClassFromInstructionPattern(N: Dst, ResIdx: I);
2191 if (!RC)
2192 return failedImport(Reason: "Could not infer register class for result #" +
2193 Twine(I) + " from pattern " + to_string(Value: Dst));
2194 OM.addPredicate<RegisterBankOperandMatcher>(args: *RC);
2195 }
2196
2197 auto DstMIBuilderOrError =
2198 createAndImportInstructionRenderer(M, InsnMatcher, Dst);
2199 if (auto Error = DstMIBuilderOrError.takeError())
2200 return std::move(Error);
2201 BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get();
2202
2203 // Render the implicit defs.
2204 // These are only added to the root of the result.
2205 if (auto Error = importImplicitDefRenderers(DstMIBuilder, ImplicitDefs: P.getDstRegs()))
2206 return std::move(Error);
2207
2208 DstMIBuilder.chooseInsnToMutate(Rule&: M);
2209
2210 // Constrain the registers to classes. This is normally derived from the
2211 // emitted instruction but a few instructions require special handling.
2212 if (auto Error =
2213 constrainOperands(InsertPt: M.actions_end(), M, InsnID: DstMIBuilder.getInsnID(), Dst))
2214 return std::move(Error);
2215
2216 // Erase the root.
2217 unsigned RootInsnID = M.getInsnVarID(InsnMatcher);
2218 M.addAction<EraseInstAction>(args&: RootInsnID);
2219
2220 // We're done with this pattern! It's eligible for GISel emission; return it.
2221 ++NumPatternImported;
2222 return std::move(M);
2223}
2224
2225MatchTable
2226GlobalISelEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules,
2227 bool Optimize, bool WithCoverage) {
2228 std::vector<Matcher *> InputRules;
2229 for (Matcher &Rule : Rules)
2230 InputRules.push_back(x: &Rule);
2231
2232 if (!Optimize)
2233 return MatchTable::buildTable(Rules: InputRules, WithCoverage);
2234
2235 unsigned CurrentOrdering = 0;
2236 StringMap<unsigned> OpcodeOrder;
2237 for (RuleMatcher &Rule : Rules) {
2238 const StringRef Opcode = Rule.getOpcode();
2239 assert(!Opcode.empty() && "Didn't expect an undefined opcode");
2240 if (OpcodeOrder.try_emplace(Key: Opcode, Args&: CurrentOrdering).second)
2241 ++CurrentOrdering;
2242 }
2243
2244 llvm::stable_sort(
2245 Range&: InputRules, C: [&OpcodeOrder](const Matcher *A, const Matcher *B) {
2246 auto *L = static_cast<const RuleMatcher *>(A);
2247 auto *R = static_cast<const RuleMatcher *>(B);
2248 return std::tuple(OpcodeOrder[L->getOpcode()],
2249 L->insnmatchers_front().getNumOperandMatchers()) <
2250 std::tuple(OpcodeOrder[R->getOpcode()],
2251 R->insnmatchers_front().getNumOperandMatchers());
2252 });
2253
2254 for (Matcher *Rule : InputRules)
2255 Rule->optimize();
2256
2257 std::vector<std::unique_ptr<Matcher>> MatcherStorage;
2258 std::vector<Matcher *> OptRules =
2259 optimizeRules<GroupMatcher>(Rules: InputRules, MatcherStorage);
2260
2261 for (Matcher *Rule : OptRules)
2262 Rule->optimize();
2263
2264 OptRules = optimizeRules<SwitchMatcher>(Rules: OptRules, MatcherStorage);
2265
2266 return MatchTable::buildTable(Rules: OptRules, WithCoverage);
2267}
2268
2269void GlobalISelEmitter::emitAdditionalImpl(raw_ostream &OS) {
2270 OS << "bool " << getClassName()
2271 << "::selectImpl(MachineInstr &I, CodeGenCoverage "
2272 "&CoverageInfo) const {\n"
2273 << " const PredicateBitset AvailableFeatures = "
2274 "getAvailableFeatures();\n"
2275 << " MachineIRBuilder B(I);\n"
2276 << " State.MIs.clear();\n"
2277 << " State.MIs.push_back(&I);\n\n"
2278 << " if (executeMatchTable(*this, State, ExecInfo, B"
2279 << ", getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures"
2280 << ", &CoverageInfo)) {\n"
2281 << " return true;\n"
2282 << " }\n\n"
2283 << " return false;\n"
2284 << "}\n\n";
2285}
2286
2287void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) {
2288 std::vector<const Record *> MatchedRecords;
2289 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2290 P: [](const Record *R) {
2291 return !R->getValueAsString(FieldName: "GISelPredicateCode").empty();
2292 });
2293 emitMIPredicateFnsImpl<const Record *>(
2294 OS,
2295 AdditionalDecls: " const MachineFunction &MF = *MI.getParent()->getParent();\n"
2296 " const MachineRegisterInfo &MRI = MF.getRegInfo();\n"
2297 " const auto &Operands = State.RecordedOperands;\n"
2298 " (void)Operands;\n"
2299 " (void)MRI;",
2300 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2301 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "GISelPredicateCode"); },
2302 Comment: "PatFrag predicates.");
2303}
2304
2305void GlobalISelEmitter::emitLeafPredicateFns(raw_ostream &OS) {
2306 std::vector<const Record *> MatchedRecords;
2307 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2308 P: [](const Record *R) {
2309 return (!R->getValueAsOptionalString(FieldName: "GISelLeafPredicateCode")
2310 .value_or(u: std::string())
2311 .empty());
2312 });
2313 emitLeafPredicateFnsImpl<const Record *>(
2314 OS,
2315 AdditionalDecls: " const auto &Operands = State.RecordedOperands;\n"
2316 " Register Reg = MO.getReg();\n"
2317 " (void)Operands;\n"
2318 " (void)Reg;",
2319 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2320 GetPredCode: [](const Record *R) {
2321 return R->getValueAsString(FieldName: "GISelLeafPredicateCode");
2322 },
2323 Comment: "PatFrag predicates.");
2324}
2325
2326void GlobalISelEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
2327 std::vector<const Record *> MatchedRecords;
2328 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2329 P: [](const Record *R) {
2330 bool Unset;
2331 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2332 !R->getValueAsBitOrUnset(FieldName: "IsAPFloat", Unset) &&
2333 !R->getValueAsBit(FieldName: "IsAPInt");
2334 });
2335 emitImmPredicateFnsImpl<const Record *>(
2336 OS, TypeIdentifier: "I64", ArgType: "int64_t", Predicates: ArrayRef<const Record *>(MatchedRecords),
2337 GetPredEnumName: &getPatFragPredicateEnumName,
2338 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2339 Comment: "PatFrag predicates.");
2340}
2341
2342void GlobalISelEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) {
2343 std::vector<const Record *> MatchedRecords;
2344 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2345 P: [](const Record *R) {
2346 bool Unset;
2347 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2348 R->getValueAsBitOrUnset(FieldName: "IsAPFloat", Unset);
2349 });
2350 emitImmPredicateFnsImpl<const Record *>(
2351 OS, TypeIdentifier: "APFloat", ArgType: "const APFloat &",
2352 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2353 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2354 Comment: "PatFrag predicates.");
2355}
2356
2357void GlobalISelEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) {
2358 std::vector<const Record *> MatchedRecords;
2359 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2360 P: [](const Record *R) {
2361 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2362 R->getValueAsBit(FieldName: "IsAPInt");
2363 });
2364 emitImmPredicateFnsImpl<const Record *>(
2365 OS, TypeIdentifier: "APInt", ArgType: "const APInt &", Predicates: ArrayRef<const Record *>(MatchedRecords),
2366 GetPredEnumName: &getPatFragPredicateEnumName,
2367 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2368 Comment: "PatFrag predicates.");
2369}
2370
2371void GlobalISelEmitter::emitTestSimplePredicate(raw_ostream &OS) {
2372 OS << "bool " << getClassName() << "::testSimplePredicate(unsigned) const {\n"
2373 << " llvm_unreachable(\"" + getClassName() +
2374 " does not support simple predicates!\");\n"
2375 << " return false;\n"
2376 << "}\n";
2377}
2378
2379void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) {
2380 OS << "bool " << getClassName()
2381 << "::runCustomAction(unsigned, const MatcherState&, NewMIVector &) const "
2382 "{\n"
2383 << " llvm_unreachable(\"" + getClassName() +
2384 " does not support custom C++ actions!\");\n"
2385 << "}\n";
2386}
2387
2388bool hasBFloatType(const TreePatternNode &Node) {
2389 for (unsigned I = 0, E = Node.getNumTypes(); I < E; I++) {
2390 auto Ty = Node.getType(ResNo: I);
2391 for (auto T : Ty)
2392 if (T.second == MVT::bf16 ||
2393 (T.second.isVector() && T.second.getScalarType() == MVT::bf16))
2394 return true;
2395 }
2396 for (const TreePatternNode &C : Node.children())
2397 if (hasBFloatType(Node: C))
2398 return true;
2399 return false;
2400}
2401
2402void GlobalISelEmitter::run(raw_ostream &OS) {
2403 if (!UseCoverageFile.empty()) {
2404 RuleCoverage = CodeGenCoverage();
2405 auto RuleCoverageBufOrErr = MemoryBuffer::getFile(Filename: UseCoverageFile);
2406 if (!RuleCoverageBufOrErr) {
2407 PrintWarning(WarningLoc: SMLoc(), Msg: "Missing rule coverage data");
2408 RuleCoverage = std::nullopt;
2409 } else {
2410 if (!RuleCoverage->parse(Buffer&: *RuleCoverageBufOrErr.get(), BackendName: Target.getName())) {
2411 PrintWarning(WarningLoc: SMLoc(), Msg: "Ignoring invalid or missing rule coverage data");
2412 RuleCoverage = std::nullopt;
2413 }
2414 }
2415 }
2416
2417 // Track the run-time opcode values
2418 gatherOpcodeValues();
2419 // Track the run-time LLT ID values
2420 gatherTypeIDValues();
2421
2422 // Track the GINodeEquiv definitions.
2423 gatherNodeEquivs();
2424
2425 AllPatFrags = RK.getAllDerivedDefinitions(ClassName: "PatFrags");
2426
2427 emitSourceFileHeader(
2428 Desc: ("Global Instruction Selector for the " + Target.getName() + " target")
2429 .str(),
2430 OS);
2431 std::vector<RuleMatcher> Rules;
2432 // Look through the SelectionDAG patterns we found, possibly emitting some.
2433 for (const PatternToMatch &Pat : CGP.ptms()) {
2434 ++NumPatternTotal;
2435
2436 if (Pat.getGISelShouldIgnore())
2437 continue; // skip without warning
2438
2439 // Skip any patterns containing BF16 types, as GISel cannot currently tell
2440 // the difference between fp16 and bf16. FIXME: This can be removed once
2441 // BF16 is supported properly.
2442 if (hasBFloatType(Node: Pat.getSrcPattern()))
2443 continue;
2444
2445 auto MatcherOrErr = runOnPattern(P: Pat);
2446
2447 // The pattern analysis can fail, indicating an unsupported pattern.
2448 // Report that if we've been asked to do so.
2449 if (auto Err = MatcherOrErr.takeError()) {
2450 if (WarnOnSkippedPatterns) {
2451 PrintWarning(WarningLoc: Pat.getSrcRecord()->getLoc(),
2452 Msg: "Skipped pattern: " + toString(E: std::move(Err)));
2453 } else {
2454 consumeError(Err: std::move(Err));
2455 }
2456 ++NumPatternImportsSkipped;
2457 continue;
2458 }
2459
2460 if (RuleCoverage) {
2461 if (RuleCoverage->isCovered(RuleID: MatcherOrErr->getRuleID()))
2462 ++NumPatternsTested;
2463 else
2464 PrintWarning(WarningLoc: Pat.getSrcRecord()->getLoc(),
2465 Msg: "Pattern is not covered by a test");
2466 }
2467 Rules.push_back(x: std::move(MatcherOrErr.get()));
2468 }
2469
2470 // Comparison function to order records by name.
2471 auto OrderByName = [](const Record *A, const Record *B) {
2472 return A->getName() < B->getName();
2473 };
2474
2475 std::vector<const Record *> ComplexPredicates =
2476 RK.getAllDerivedDefinitions(ClassName: "GIComplexOperandMatcher");
2477 llvm::sort(C&: ComplexPredicates, Comp: OrderByName);
2478
2479 std::vector<StringRef> CustomRendererFns;
2480 transform(Range: RK.getAllDerivedDefinitions(ClassName: "GICustomOperandRenderer"),
2481 d_first: std::back_inserter(x&: CustomRendererFns), F: [](const auto &Record) {
2482 return Record->getValueAsString("RendererFn");
2483 });
2484 // Sort and remove duplicates to get a list of unique renderer functions, in
2485 // case some were mentioned more than once.
2486 llvm::sort(C&: CustomRendererFns);
2487 CustomRendererFns.erase(first: llvm::unique(R&: CustomRendererFns),
2488 last: CustomRendererFns.end());
2489
2490 // Create a table containing the LLT objects needed by the matcher and an enum
2491 // for the matcher to reference them with.
2492 std::vector<LLTCodeGen> TypeObjects;
2493 append_range(C&: TypeObjects, R&: KnownTypes);
2494 llvm::sort(C&: TypeObjects);
2495
2496 // Sort rules.
2497 llvm::stable_sort(Range&: Rules, C: [&](const RuleMatcher &A, const RuleMatcher &B) {
2498 int ScoreA = RuleMatcherScores[A.getRuleID()];
2499 int ScoreB = RuleMatcherScores[B.getRuleID()];
2500 if (ScoreA > ScoreB)
2501 return true;
2502 if (ScoreB > ScoreA)
2503 return false;
2504 if (A.isHigherPriorityThan(B)) {
2505 assert(!B.isHigherPriorityThan(A) && "Cannot be more important "
2506 "and less important at "
2507 "the same time");
2508 return true;
2509 }
2510 return false;
2511 });
2512
2513 unsigned MaxTemporaries = 0;
2514 for (const auto &Rule : Rules)
2515 MaxTemporaries = std::max(a: MaxTemporaries, b: Rule.countRendererFns());
2516
2517 // Build match table
2518 const MatchTable Table =
2519 buildMatchTable(Rules, Optimize: OptimizeMatchTable, WithCoverage: GenerateCoverage);
2520
2521 emitPredicateBitset(OS, IfDefName: "GET_GLOBALISEL_PREDICATE_BITSET");
2522 emitTemporariesDecl(OS, IfDefName: "GET_GLOBALISEL_TEMPORARIES_DECL");
2523 emitTemporariesInit(OS, MaxTemporaries, IfDefName: "GET_GLOBALISEL_TEMPORARIES_INIT");
2524 emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexOperandMatchers: ComplexPredicates,
2525 CustomOperandRenderers: CustomRendererFns, IfDefName: "GET_GLOBALISEL_IMPL");
2526 emitPredicatesDecl(OS, IfDefName: "GET_GLOBALISEL_PREDICATES_DECL");
2527 emitPredicatesInit(OS, IfDefName: "GET_GLOBALISEL_PREDICATES_INIT");
2528}
2529
2530void GlobalISelEmitter::declareSubtargetFeature(const Record *Predicate) {
2531 SubtargetFeatures.try_emplace(k: Predicate, args&: Predicate, args: SubtargetFeatures.size());
2532}
2533
2534unsigned GlobalISelEmitter::declareHwModeCheck(StringRef HwModeFeatures) {
2535 return HwModes.emplace(args: HwModeFeatures.str(), args: HwModes.size()).first->second;
2536}
2537
2538} // end anonymous namespace
2539
2540//===----------------------------------------------------------------------===//
2541
2542static TableGen::Emitter::OptClass<GlobalISelEmitter>
2543 X("gen-global-isel", "Generate GlobalISel selector");
2544