1
2//===- GlobalISelEmitter.cpp - Generate an instruction selector -----------===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// This tablegen backend emits code for use by the GlobalISel instruction
12/// selector. See include/llvm/Target/GlobalISel/Target.td.
13///
14/// This file analyzes the patterns recognized by the SelectionDAGISel tablegen
15/// backend, filters out the ones that are unsupported, maps
16/// SelectionDAG-specific constructs to their GlobalISel counterpart
17/// (when applicable: MVT to LLT; SDNode to generic Instruction).
18///
19/// Not all patterns are supported: pass the tablegen invocation
20/// "-warn-on-skipped-patterns" to emit a warning when a pattern is skipped,
21/// as well as why.
22///
23/// The generated file defines a single method:
24/// bool <Target>InstructionSelector::selectImpl(MachineInstr &I) const;
25/// intended to be used in InstructionSelector::select as the first-step
26/// selector for the patterns that don't require complex C++.
27///
28/// FIXME: We'll probably want to eventually define a base
29/// "TargetGenInstructionSelector" class.
30///
31//===----------------------------------------------------------------------===//
32
33#include "Basic/CodeGenIntrinsics.h"
34#include "Common/CodeGenDAGPatterns.h"
35#include "Common/CodeGenInstruction.h"
36#include "Common/CodeGenRegisters.h"
37#include "Common/CodeGenTarget.h"
38#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
39#include "Common/GlobalISel/MatchTable/Matchers.h"
40#include "Common/InfoByHwMode.h"
41#include "llvm/ADT/Statistic.h"
42#include "llvm/CodeGenTypes/LowLevelType.h"
43#include "llvm/CodeGenTypes/MachineValueType.h"
44#include "llvm/Support/CodeGenCoverage.h"
45#include "llvm/Support/CommandLine.h"
46#include "llvm/Support/Error.h"
47#include "llvm/Support/ScopedPrinter.h"
48#include "llvm/TableGen/Error.h"
49#include "llvm/TableGen/Record.h"
50#include "llvm/TableGen/TableGenBackend.h"
51#include <string>
52
53using namespace llvm;
54using namespace llvm::gi;
55
56using action_iterator = RuleMatcher::action_iterator;
57
58#define DEBUG_TYPE "gisel-emitter"
59
60STATISTIC(NumPatternTotal, "Total number of patterns");
61STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG");
62STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped");
63STATISTIC(NumPatternsTested,
64 "Number of patterns executed according to coverage information");
65
66static cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel");
67
68static cl::opt<bool> WarnOnSkippedPatterns(
69 "warn-on-skipped-patterns",
70 cl::desc("Explain why a pattern was skipped for inclusion "
71 "in the GlobalISel selector"),
72 cl::init(Val: false), cl::cat(GlobalISelEmitterCat));
73
74static cl::opt<bool> GenerateCoverage(
75 "instrument-gisel-coverage",
76 cl::desc("Generate coverage instrumentation for GlobalISel"),
77 cl::init(Val: false), cl::cat(GlobalISelEmitterCat));
78
79static cl::opt<std::string> UseCoverageFile(
80 "gisel-coverage-file", cl::init(Val: ""),
81 cl::desc("Specify file to retrieve coverage information from"),
82 cl::cat(GlobalISelEmitterCat));
83
84static cl::opt<bool> OptimizeMatchTable(
85 "optimize-match-table",
86 cl::desc("Generate an optimized version of the match table"),
87 cl::init(Val: true), cl::cat(GlobalISelEmitterCat));
88
89static std::string explainPredicates(const TreePatternNode &N) {
90 std::string Explanation;
91 StringRef Separator = "";
92 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
93 const TreePredicateFn &P = Call.Fn;
94 Explanation +=
95 (Separator + P.getOrigPatFragRecord()->getRecord()->getName()).str();
96 Separator = ", ";
97
98 if (P.isAlwaysTrue())
99 Explanation += " always-true";
100 if (P.isImmediatePattern())
101 Explanation += " immediate";
102
103 if (P.isUnindexed())
104 Explanation += " unindexed";
105
106 if (P.isNonExtLoad())
107 Explanation += " non-extload";
108 if (P.isAnyExtLoad())
109 Explanation += " extload";
110 if (P.isSignExtLoad())
111 Explanation += " sextload";
112 if (P.isZeroExtLoad())
113 Explanation += " zextload";
114
115 if (P.isNonTruncStore())
116 Explanation += " non-truncstore";
117 if (P.isTruncStore())
118 Explanation += " truncstore";
119
120 if (const Record *VT = P.getMemoryVT())
121 Explanation += (" MemVT=" + VT->getName()).str();
122 if (const Record *VT = P.getScalarMemoryVT())
123 Explanation += (" ScalarVT(MemVT)=" + VT->getName()).str();
124
125 if (const ListInit *AddrSpaces = P.getAddressSpaces()) {
126 raw_string_ostream OS(Explanation);
127 OS << " AddressSpaces=[";
128
129 StringRef AddrSpaceSeparator;
130 for (const Init *Val : AddrSpaces->getElements()) {
131 const IntInit *IntVal = dyn_cast<IntInit>(Val);
132 if (!IntVal)
133 continue;
134
135 OS << AddrSpaceSeparator << IntVal->getValue();
136 AddrSpaceSeparator = ", ";
137 }
138
139 OS << ']';
140 }
141
142 int64_t MinAlign = P.getMinAlignment();
143 if (MinAlign > 0)
144 Explanation += " MinAlign=" + utostr(X: MinAlign);
145
146 if (P.isAtomicOrderingMonotonic())
147 Explanation += " monotonic";
148 if (P.isAtomicOrderingAcquire())
149 Explanation += " acquire";
150 if (P.isAtomicOrderingRelease())
151 Explanation += " release";
152 if (P.isAtomicOrderingAcquireRelease())
153 Explanation += " acq_rel";
154 if (P.isAtomicOrderingSequentiallyConsistent())
155 Explanation += " seq_cst";
156 if (P.isAtomicOrderingAcquireOrStronger())
157 Explanation += " >=acquire";
158 if (P.isAtomicOrderingWeakerThanAcquire())
159 Explanation += " <acquire";
160 if (P.isAtomicOrderingReleaseOrStronger())
161 Explanation += " >=release";
162 if (P.isAtomicOrderingWeakerThanRelease())
163 Explanation += " <release";
164 }
165 return Explanation;
166}
167
168static std::string explainOperator(const Record *Operator) {
169 if (Operator->isSubClassOf(Name: "SDNode"))
170 return (" (" + Operator->getValueAsString(FieldName: "Opcode") + ")").str();
171
172 if (Operator->isSubClassOf(Name: "Intrinsic"))
173 return (" (Operator is an Intrinsic, " + Operator->getName() + ")").str();
174
175 if (Operator->isSubClassOf(Name: "ComplexPattern"))
176 return (" (Operator is an unmapped ComplexPattern, " + Operator->getName() +
177 ")")
178 .str();
179
180 if (Operator->isSubClassOf(Name: "SDNodeXForm"))
181 return (" (Operator is an unmapped SDNodeXForm, " + Operator->getName() +
182 ")")
183 .str();
184
185 return (" (Operator " + Operator->getName() + " not understood)").str();
186}
187
188/// Helper function to let the emitter report skip reason error messages.
189static Error failedImport(const Twine &Reason) {
190 return make_error<StringError>(Args: Reason, Args: inconvertibleErrorCode());
191}
192
193static Error isTrivialOperatorNode(const TreePatternNode &N) {
194 std::string Explanation;
195 std::string Separator;
196
197 bool HasUnsupportedPredicate = false;
198 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
199 const TreePredicateFn &Predicate = Call.Fn;
200
201 if (Predicate.isAlwaysTrue())
202 continue;
203
204 if (Predicate.isImmediatePattern())
205 continue;
206
207 if (Predicate.hasNoUse() || Predicate.hasOneUse())
208 continue;
209
210 if (Predicate.isNonExtLoad() || Predicate.isAnyExtLoad() ||
211 Predicate.isSignExtLoad() || Predicate.isZeroExtLoad())
212 continue;
213
214 if (Predicate.isNonTruncStore() || Predicate.isTruncStore())
215 continue;
216
217 if (Predicate.isLoad() && Predicate.getMemoryVT())
218 continue;
219
220 if (Predicate.isStore() && Predicate.getMemoryVT())
221 continue;
222
223 if (Predicate.isLoad() || Predicate.isStore()) {
224 if (Predicate.isUnindexed())
225 continue;
226 }
227
228 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
229 const ListInit *AddrSpaces = Predicate.getAddressSpaces();
230 if (AddrSpaces && !AddrSpaces->empty())
231 continue;
232
233 if (Predicate.getMinAlignment() > 0)
234 continue;
235 }
236
237 if (Predicate.isAtomic() && Predicate.getMemoryVT())
238 continue;
239
240 if (Predicate.isAtomic() &&
241 (Predicate.isAtomicOrderingMonotonic() ||
242 Predicate.isAtomicOrderingAcquire() ||
243 Predicate.isAtomicOrderingRelease() ||
244 Predicate.isAtomicOrderingAcquireRelease() ||
245 Predicate.isAtomicOrderingSequentiallyConsistent() ||
246 Predicate.isAtomicOrderingAcquireOrStronger() ||
247 Predicate.isAtomicOrderingWeakerThanAcquire() ||
248 Predicate.isAtomicOrderingReleaseOrStronger() ||
249 Predicate.isAtomicOrderingWeakerThanRelease()))
250 continue;
251
252 if (Predicate.hasGISelPredicateCode())
253 continue;
254
255 HasUnsupportedPredicate = true;
256 Explanation = Separator + "Has a predicate (" + explainPredicates(N) + ")";
257 Separator = ", ";
258 Explanation += (Separator + "first-failing:" +
259 Predicate.getOrigPatFragRecord()->getRecord()->getName())
260 .str();
261 break;
262 }
263
264 if (!HasUnsupportedPredicate)
265 return Error::success();
266
267 return failedImport(Reason: Explanation);
268}
269
270static std::string getScopedName(unsigned Scope, const std::string &Name) {
271 return ("pred:" + Twine(Scope) + ":" + Name).str();
272}
273
274static std::string getMangledRootDefName(StringRef DefOperandName) {
275 return ("DstI[" + DefOperandName + "]").str();
276}
277
278static bool shouldUseGenericTypeForInstruction(const CodeGenInstruction *GI) {
279 if (!GI)
280 return false;
281
282 bool Unset = false;
283 bool ShouldMatchGeneric =
284 GI->TheDef->getValueAsBitOrUnset(FieldName: "GISelMatchGenericTypes", Unset);
285 return Unset ? false : ShouldMatchGeneric;
286}
287
288//===- GlobalISelEmitter class --------------------------------------------===//
289
290static Expected<LLTCodeGen> getInstResultType(const TreePatternNode &Dst,
291 const CodeGenTarget &Target) {
292 // While we allow more than one output (both implicit and explicit defs)
293 // below, we only expect one explicit def here.
294 assert(Dst.getOperator()->isSubClassOf("Instruction"));
295 const CodeGenInstruction &InstInfo = Target.getInstruction(InstRec: Dst.getOperator());
296 if (!InstInfo.Operands.NumDefs)
297 return failedImport(Reason: "Dst pattern child needs a def");
298
299 ArrayRef<TypeSetByHwMode> ChildTypes = Dst.getExtTypes();
300 if (ChildTypes.size() < 1)
301 return failedImport(Reason: "Dst pattern child has no result");
302
303 // If there are multiple results, just take the first one (this is how
304 // SelectionDAG does it).
305 std::optional<LLTCodeGen> MaybeOpTy;
306 if (ChildTypes.front().isMachineValueType()) {
307 MaybeOpTy = MVTToLLT(VT: ChildTypes.front().getMachineValueType().SimpleTy);
308 }
309
310 if (!MaybeOpTy)
311 return failedImport(Reason: "Dst operand has an unsupported type");
312 return *MaybeOpTy;
313}
314
315namespace {
316class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter {
317public:
318 explicit GlobalISelEmitter(const RecordKeeper &RK);
319
320 void emitAdditionalImpl(raw_ostream &OS) override;
321
322 void emitMIPredicateFns(raw_ostream &OS) override;
323 void emitLeafPredicateFns(raw_ostream &OS) override;
324 void emitI64ImmPredicateFns(raw_ostream &OS) override;
325 void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
326 void emitAPIntImmPredicateFns(raw_ostream &OS) override;
327 void emitTestSimplePredicate(raw_ostream &OS) override;
328 void emitRunCustomAction(raw_ostream &OS) override;
329
330 const CodeGenTarget &getTarget() const override { return Target; }
331 StringRef getClassName() const override { return ClassName; }
332
333 void run(raw_ostream &OS);
334
335private:
336 std::string ClassName;
337
338 const RecordKeeper &RK;
339 const CodeGenDAGPatterns CGP;
340 const CodeGenTarget &Target;
341 CodeGenRegBank &CGRegs;
342
343 ArrayRef<const Record *> AllPatFrags;
344
345 /// Keep track of the equivalence between SDNodes and Instruction by mapping
346 /// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to
347 /// check for attributes on the relation such as CheckMMOIsNonAtomic.
348 /// This is defined using 'GINodeEquiv' in the target description.
349 DenseMap<const Record *, const Record *> NodeEquivs;
350
351 /// Keep track of the equivalence between ComplexPattern's and
352 /// GIComplexOperandMatcher. Map entries are specified by subclassing
353 /// GIComplexPatternEquiv.
354 DenseMap<const Record *, const Record *> ComplexPatternEquivs;
355
356 /// Keep track of the equivalence between SDNodeXForm's and
357 /// GICustomOperandRenderer. Map entries are specified by subclassing
358 /// GISDNodeXFormEquiv.
359 DenseMap<const Record *, const Record *> SDNodeXFormEquivs;
360
361 /// Keep track of Scores of PatternsToMatch similar to how the DAG does.
362 /// This adds compatibility for RuleMatchers to use this for ordering rules.
363 DenseMap<uint64_t, int> RuleMatcherScores;
364
365 // Rule coverage information.
366 std::optional<CodeGenCoverage> RuleCoverage;
367
368 /// Variables used to help with collecting of named operands for predicates
369 /// with 'let PredicateCodeUsesOperands = 1'. WaitingForNamedOperands is set
370 /// to the number of named operands that predicate expects. Store locations in
371 /// StoreIdxForName correspond to the order in which operand names appear in
372 /// predicate's argument list.
373 /// When we visit named operand and WaitingForNamedOperands is not zero, add
374 /// matcher that will record operand and decrease counter.
375 unsigned WaitingForNamedOperands = 0;
376 StringMap<unsigned> StoreIdxForName;
377
378 void gatherOpcodeValues();
379 void gatherTypeIDValues();
380 void gatherNodeEquivs();
381
382 const Record *findNodeEquiv(const Record *N) const;
383 const CodeGenInstruction *getEquivNode(const Record &Equiv,
384 const TreePatternNode &N) const;
385
386 Error importRulePredicates(RuleMatcher &M,
387 ArrayRef<const Record *> Predicates);
388 Expected<InstructionMatcher &>
389 createAndImportSelDAGMatcher(RuleMatcher &Rule,
390 InstructionMatcher &InsnMatcher,
391 const TreePatternNode &Src, unsigned &TempOpIdx);
392 Error addTypeCheckPredicateForOpcode(OperandMatcher &OM,
393 const TypeSetByHwMode &VTy,
394 bool OperandIsAPointer,
395 const CodeGenInstruction *GI) const;
396 Error importComplexPatternOperandMatcher(OperandMatcher &OM, const Record *R,
397 unsigned &TempOpIdx) const;
398 Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
399 const TreePatternNode &SrcChild,
400 bool OperandIsAPointer, bool OperandIsImmArg,
401 const CodeGenInstruction *GI, unsigned OpIdx,
402 unsigned &TempOpIdx);
403
404 Expected<BuildMIAction &>
405 createAndImportInstructionRenderer(RuleMatcher &M,
406 InstructionMatcher &InsnMatcher,
407 const TreePatternNode &Dst) const;
408 Expected<action_iterator> createAndImportSubInstructionRenderer(
409 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
410 unsigned TempReg) const;
411 Expected<action_iterator>
412 createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M,
413 const TreePatternNode &Dst) const;
414
415 Expected<action_iterator>
416 importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M,
417 BuildMIAction &DstMIBuilder,
418 const TreePatternNode &Dst, bool IsRoot) const;
419
420 Expected<action_iterator>
421 importExplicitUseRenderers(action_iterator InsertPt, RuleMatcher &M,
422 BuildMIAction &DstMIBuilder,
423 const TreePatternNode &Dst) const;
424
425 Error importNamedNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
426 const TreePatternNode &N) const;
427
428 Error importLeafNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
429 const TreePatternNode &N,
430 action_iterator InsertPt) const;
431
432 Error importXFormNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
433 const TreePatternNode &N) const;
434
435 Error importInstructionNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
436 const TreePatternNode &N,
437 action_iterator &InsertPt) const;
438
439 Error importNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
440 const TreePatternNode &N,
441 action_iterator &InsertPt) const;
442
443 Error importImplicitDefRenderers(BuildMIAction &DstMIBuilder,
444 ArrayRef<const Record *> ImplicitDefs) const;
445
446 /// Analyze pattern \p P, returning a matcher for it if possible.
447 /// Otherwise, return an Error explaining why we don't support it.
448 Expected<RuleMatcher> runOnPattern(const PatternToMatch &P);
449
450 void declareSubtargetFeature(const Record *Predicate);
451
452 unsigned declareHwModeCheck(StringRef HwModeFeatures);
453
454 MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize,
455 bool WithCoverage);
456
457 /// Infer a CodeGenRegisterClass for the type of \p SuperRegNode. The returned
458 /// CodeGenRegisterClass will support the CodeGenRegisterClass of
459 /// \p SubRegNode, and the subregister index defined by \p SubRegIdxNode.
460 /// If no register class is found, return nullptr.
461 const CodeGenRegisterClass *
462 inferSuperRegisterClassForNode(const TypeSetByHwMode &Ty,
463 const TreePatternNode &SuperRegNode,
464 const TreePatternNode &SubRegIdxNode) const;
465 const CodeGenSubRegIndex *
466 inferSubRegIndexForNode(const TreePatternNode &SubRegIdxNode) const;
467
468 /// Infer a CodeGenRegisterClass which suppoorts \p Ty and \p SubRegIdxNode.
469 /// Return nullptr if no such class exists.
470 const CodeGenRegisterClass *
471 inferSuperRegisterClass(const TypeSetByHwMode &Ty,
472 const TreePatternNode &SubRegIdxNode) const;
473
474 /// Return the CodeGenRegisterClass associated with \p Leaf if it has one.
475 const CodeGenRegisterClass *
476 getRegClassFromLeaf(const TreePatternNode &Leaf) const;
477
478 /// Return a CodeGenRegisterClass for \p N if one can be found. Return
479 /// nullptr otherwise.
480 const CodeGenRegisterClass *
481 inferRegClassFromPattern(const TreePatternNode &N) const;
482
483 const CodeGenRegisterClass *
484 inferRegClassFromRegisterClassLike(const Record *RecClassLike) const;
485
486 const CodeGenRegisterClass *
487 inferRegClassFromInstructionPattern(const TreePatternNode &N,
488 unsigned ResIdx) const;
489
490 Error constrainOperands(action_iterator InsertPt, RuleMatcher &M,
491 unsigned InsnID, const TreePatternNode &Dst) const;
492
493 /// Return the size of the MemoryVT in this predicate, if possible.
494 std::optional<unsigned>
495 getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate);
496
497 // Add builtin predicates.
498 Expected<InstructionMatcher &>
499 addBuiltinPredicates(const Record *SrcGIEquivOrNull,
500 const TreePredicateFn &Predicate,
501 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher);
502};
503} // namespace
504
505static StringRef getPatFragPredicateEnumName(const Record *R) {
506 return R->getName();
507}
508
509void GlobalISelEmitter::gatherOpcodeValues() {
510 InstructionOpcodeMatcher::initOpcodeValuesMap(Target);
511}
512
513void GlobalISelEmitter::gatherTypeIDValues() {
514 LLTOperandMatcher::initTypeIDValuesMap();
515}
516
517void GlobalISelEmitter::gatherNodeEquivs() {
518 assert(NodeEquivs.empty());
519 for (const Record *Equiv : RK.getAllDerivedDefinitions(ClassName: "GINodeEquiv"))
520 NodeEquivs[Equiv->getValueAsDef(FieldName: "Node")] = Equiv;
521
522 assert(ComplexPatternEquivs.empty());
523 for (const Record *Equiv :
524 RK.getAllDerivedDefinitions(ClassName: "GIComplexPatternEquiv")) {
525 const Record *SelDAGEquiv = Equiv->getValueAsDef(FieldName: "SelDAGEquivalent");
526 if (!SelDAGEquiv)
527 continue;
528 ComplexPatternEquivs[SelDAGEquiv] = Equiv;
529 }
530
531 assert(SDNodeXFormEquivs.empty());
532 for (const Record *Equiv :
533 RK.getAllDerivedDefinitions(ClassName: "GISDNodeXFormEquiv")) {
534 const Record *SelDAGEquiv = Equiv->getValueAsDef(FieldName: "SelDAGEquivalent");
535 if (!SelDAGEquiv)
536 continue;
537 SDNodeXFormEquivs[SelDAGEquiv] = Equiv;
538 }
539}
540
541const Record *GlobalISelEmitter::findNodeEquiv(const Record *N) const {
542 return NodeEquivs.lookup(Val: N);
543}
544
545const CodeGenInstruction *
546GlobalISelEmitter::getEquivNode(const Record &Equiv,
547 const TreePatternNode &N) const {
548 if (N.getNumChildren() >= 1) {
549 // setcc operation maps to two different G_* instructions based on the type.
550 if (!Equiv.isValueUnset(FieldName: "IfFloatingPoint") &&
551 MVT(N.getChild(N: 0).getSimpleType(ResNo: 0)).isFloatingPoint())
552 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfFloatingPoint"));
553 }
554
555 if (!Equiv.isValueUnset(FieldName: "IfConvergent") &&
556 N.getIntrinsicInfo(CDP: CGP)->isConvergent)
557 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfConvergent"));
558
559 bool IsAnyExtLoad = false;
560 bool IsTruncStore = false;
561 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
562 const TreePredicateFn &Predicate = Call.Fn;
563 IsAnyExtLoad |= Predicate.isAnyExtLoad();
564 IsTruncStore |= Predicate.isTruncStore();
565 if (!Equiv.isValueUnset(FieldName: "IfSignExtend") &&
566 (Predicate.isLoad() || Predicate.isAtomic()) &&
567 Predicate.isSignExtLoad())
568 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfSignExtend"));
569
570 if (!Equiv.isValueUnset(FieldName: "IfZeroExtend") &&
571 (Predicate.isLoad() || Predicate.isAtomic()) &&
572 Predicate.isZeroExtLoad())
573 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfZeroExtend"));
574
575 if (!Equiv.isValueUnset(FieldName: "IfFPExtend") &&
576 (Predicate.isLoad() || Predicate.isAtomic()) && IsAnyExtLoad &&
577 Predicate.getMemoryVT() != nullptr &&
578 getValueType(Rec: Predicate.getMemoryVT()).isFloatingPoint())
579 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfFPExtend"));
580
581 if (!Equiv.isValueUnset(FieldName: "IfFPTrunc") &&
582 (Predicate.isStore() || Predicate.isAtomic()) && IsTruncStore &&
583 Predicate.getMemoryVT() != nullptr &&
584 getValueType(Rec: Predicate.getMemoryVT()).isFloatingPoint())
585 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfFPTrunc"));
586 }
587
588 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "I"));
589}
590
591GlobalISelEmitter::GlobalISelEmitter(const RecordKeeper &RK)
592 : GlobalISelMatchTableExecutorEmitter(), RK(RK), CGP(RK),
593 Target(CGP.getTargetInfo()), CGRegs(Target.getRegBank()) {
594 ClassName = Target.getName().str() + "InstructionSelector";
595}
596
597//===- Emitter ------------------------------------------------------------===//
598
599Error GlobalISelEmitter::importRulePredicates(
600 RuleMatcher &M, ArrayRef<const Record *> Predicates) {
601 for (const Record *Pred : Predicates) {
602 if (Pred->getValueAsString(FieldName: "CondString").empty())
603 continue;
604 declareSubtargetFeature(Predicate: Pred);
605 M.addRequiredFeature(Feature: Pred);
606 }
607
608 return Error::success();
609}
610
611std::optional<unsigned> GlobalISelEmitter::getMemSizeBitsFromPredicate(
612 const TreePredicateFn &Predicate) {
613 std::optional<LLTCodeGen> MemTyOrNone =
614 MVTToLLT(VT: getValueType(Rec: Predicate.getMemoryVT()));
615
616 if (!MemTyOrNone)
617 return std::nullopt;
618
619 // Align so unusual types like i1 don't get rounded down.
620 return llvm::alignTo(
621 Value: static_cast<unsigned>(MemTyOrNone->get().getSizeInBits()), Align: 8);
622}
623
624Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
625 const Record *SrcGIEquivOrNull, const TreePredicateFn &Predicate,
626 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher) {
627 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
628 if (const ListInit *AddrSpaces = Predicate.getAddressSpaces()) {
629 SmallVector<unsigned, 4> ParsedAddrSpaces;
630
631 for (const Init *Val : AddrSpaces->getElements()) {
632 const IntInit *IntVal = dyn_cast<IntInit>(Val);
633 if (!IntVal)
634 return failedImport(Reason: "Address space is not an integer");
635 ParsedAddrSpaces.push_back(Elt: IntVal->getValue());
636 }
637
638 if (!ParsedAddrSpaces.empty()) {
639 InsnMatcher.addPredicate<MemoryAddressSpacePredicateMatcher>(
640 args: 0, args&: ParsedAddrSpaces);
641 return InsnMatcher;
642 }
643 }
644
645 int64_t MinAlign = Predicate.getMinAlignment();
646 if (MinAlign > 0) {
647 InsnMatcher.addPredicate<MemoryAlignmentPredicateMatcher>(args: 0, args&: MinAlign);
648 return InsnMatcher;
649 }
650 }
651
652 // G_LOAD is used for both non-extending and any-extending loads.
653 if (Predicate.isLoad() || Predicate.isAtomic()) {
654 if (Predicate.isNonExtLoad()) {
655 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
656 args: 0, args: MemoryVsLLTSizePredicateMatcher::EqualTo, args: 0);
657 return InsnMatcher;
658 }
659 if (Predicate.isAnyExtLoad()) {
660 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
661 args: 0, args: MemoryVsLLTSizePredicateMatcher::LessThan, args: 0);
662 return InsnMatcher;
663 }
664 }
665
666 if (Predicate.isStore()) {
667 if (Predicate.isTruncStore()) {
668 if (Predicate.getMemoryVT() != nullptr) {
669 // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size.
670 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
671 if (!MemSizeInBits)
672 return failedImport(Reason: "MemVT could not be converted to LLT");
673
674 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(args: 0, args: *MemSizeInBits /
675 8);
676 } else {
677 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
678 args: 0, args: MemoryVsLLTSizePredicateMatcher::LessThan, args: 0);
679 }
680 return InsnMatcher;
681 }
682 if (Predicate.isNonTruncStore()) {
683 // We need to check the sizes match here otherwise we could incorrectly
684 // match truncating stores with non-truncating ones.
685 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
686 args: 0, args: MemoryVsLLTSizePredicateMatcher::EqualTo, args: 0);
687 }
688 }
689
690 assert(SrcGIEquivOrNull != nullptr && "Invalid SrcGIEquivOrNull value");
691 // No check required. We already did it by swapping the opcode.
692 if (!SrcGIEquivOrNull->isValueUnset(FieldName: "IfSignExtend") &&
693 Predicate.isSignExtLoad())
694 return InsnMatcher;
695
696 // No check required. We already did it by swapping the opcode.
697 if (!SrcGIEquivOrNull->isValueUnset(FieldName: "IfZeroExtend") &&
698 Predicate.isZeroExtLoad())
699 return InsnMatcher;
700
701 // No check required. G_STORE by itself is a non-extending store.
702 if (Predicate.isNonTruncStore())
703 return InsnMatcher;
704
705 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
706 if (Predicate.getMemoryVT() != nullptr) {
707 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
708 if (!MemSizeInBits)
709 return failedImport(Reason: "MemVT could not be converted to LLT");
710
711 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(args: 0,
712 args: *MemSizeInBits / 8);
713 return InsnMatcher;
714 }
715 }
716
717 if (Predicate.isLoad() || Predicate.isStore()) {
718 // No check required. A G_LOAD/G_STORE is an unindexed load.
719 if (Predicate.isUnindexed())
720 return InsnMatcher;
721 }
722
723 if (Predicate.isAtomic()) {
724 if (Predicate.isAtomicOrderingMonotonic()) {
725 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Monotonic");
726 return InsnMatcher;
727 }
728 if (Predicate.isAtomicOrderingAcquire()) {
729 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Acquire");
730 return InsnMatcher;
731 }
732 if (Predicate.isAtomicOrderingRelease()) {
733 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Release");
734 return InsnMatcher;
735 }
736 if (Predicate.isAtomicOrderingAcquireRelease()) {
737 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
738 args: "AcquireRelease");
739 return InsnMatcher;
740 }
741 if (Predicate.isAtomicOrderingSequentiallyConsistent()) {
742 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
743 args: "SequentiallyConsistent");
744 return InsnMatcher;
745 }
746 }
747
748 if (Predicate.isAtomicOrderingAcquireOrStronger()) {
749 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
750 args: "Acquire", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
751 return InsnMatcher;
752 }
753 if (Predicate.isAtomicOrderingWeakerThanAcquire()) {
754 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
755 args: "Acquire", args: AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
756 return InsnMatcher;
757 }
758
759 if (Predicate.isAtomicOrderingReleaseOrStronger()) {
760 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
761 args: "Release", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
762 return InsnMatcher;
763 }
764 if (Predicate.isAtomicOrderingWeakerThanRelease()) {
765 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
766 args: "Release", args: AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
767 return InsnMatcher;
768 }
769 HasAddedMatcher = false;
770 return InsnMatcher;
771}
772
773Error GlobalISelEmitter::addTypeCheckPredicateForOpcode(
774 OperandMatcher &OM, const TypeSetByHwMode &VTy, bool OperandIsAPointer,
775 const CodeGenInstruction *GI) const {
776 // Pointer information is separate from the underlying MVT. Avoid losing it by
777 // forcing the regular path.
778 if (OperandIsAPointer || VTy.isPointer())
779 return OM.addTypeCheckPredicate(VTy, OperandIsAPointer);
780
781 if (LLT::getUseExtended() && shouldUseGenericTypeForInstruction(GI)) {
782 if (!VTy.isMachineValueType())
783 return failedImport(Reason: "unsupported typeset");
784
785 auto GenericLLT = MVTToGenericLLT(VT: VTy.getMachineValueType().SimpleTy);
786 if (GenericLLT) {
787 OM.addPredicate<LLTOperandShapeMatcher>(args&: *GenericLLT);
788 return Error::success();
789 }
790 }
791
792 return OM.addTypeCheckPredicate(VTy, OperandIsAPointer);
793}
794
795Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
796 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
797 const TreePatternNode &Src, unsigned &TempOpIdx) {
798 const auto SavedFlags = Rule.setGISelFlags(Src.getGISelFlagsRecord());
799
800 const Record *SrcGIEquivOrNull = nullptr;
801 const CodeGenInstruction *SrcGIOrNull = nullptr;
802
803 // Start with the defined operands (i.e., the results of the root operator).
804 if (Src.isLeaf()) {
805 const Init *SrcInit = Src.getLeafValue();
806 if (isa<IntInit>(Val: SrcInit)) {
807 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(
808 args: &Target.getInstruction(InstRec: RK.getDef(Name: "G_CONSTANT")));
809 } else {
810 return failedImport(
811 Reason: "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
812 }
813 } else {
814 SrcGIEquivOrNull = findNodeEquiv(N: Src.getOperator());
815 if (!SrcGIEquivOrNull)
816 return failedImport(Reason: "Pattern operator lacks an equivalent Instruction" +
817 explainOperator(Operator: Src.getOperator()));
818 SrcGIOrNull = getEquivNode(Equiv: *SrcGIEquivOrNull, N: Src);
819
820 // The operators look good: match the opcode
821 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(args&: SrcGIOrNull);
822 }
823
824 // Since there are no opcodes for atomic loads and stores comparing to
825 // SelectionDAG, we add CheckMMOIsNonAtomic predicate immediately after the
826 // opcode predicate to make a logical combination of them.
827 if (SrcGIEquivOrNull &&
828 SrcGIEquivOrNull->getValueAsBit(FieldName: "CheckMMOIsNonAtomic"))
829 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "NotAtomic");
830 else if (SrcGIEquivOrNull &&
831 SrcGIEquivOrNull->getValueAsBit(FieldName: "CheckMMOIsAtomic")) {
832 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
833 args: "Unordered", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
834 }
835
836 unsigned OpIdx = 0;
837 for (const TypeSetByHwMode &VTy : Src.getExtTypes()) {
838 // Results don't have a name unless they are the root node. The caller will
839 // set the name if appropriate.
840 const bool OperandIsAPointer =
841 SrcGIOrNull && SrcGIOrNull->isOutOperandAPointer(i: OpIdx);
842 OperandMatcher &OM = InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
843 if (auto Error = addTypeCheckPredicateForOpcode(OM, VTy, OperandIsAPointer,
844 GI: SrcGIOrNull))
845 return failedImport(Reason: toString(E: std::move(Error)) +
846 " for result of Src pattern operator");
847 }
848
849 for (const TreePredicateCall &Call : Src.getPredicateCalls()) {
850 const TreePredicateFn &Predicate = Call.Fn;
851 bool HasAddedBuiltinMatcher = true;
852 if (Predicate.isAlwaysTrue())
853 continue;
854
855 if (Predicate.isImmediatePattern()) {
856 InsnMatcher.addPredicate<InstructionImmPredicateMatcher>(args: Predicate);
857 continue;
858 }
859
860 auto InsnMatcherOrError = addBuiltinPredicates(
861 SrcGIEquivOrNull, Predicate, InsnMatcher, HasAddedMatcher&: HasAddedBuiltinMatcher);
862 if (auto Error = InsnMatcherOrError.takeError())
863 return std::move(Error);
864
865 // FIXME: This should be part of addBuiltinPredicates(). If we add this at
866 // the start of addBuiltinPredicates() without returning, then there might
867 // be cases where we hit the last return before which the
868 // HasAddedBuiltinMatcher will be set to false. The predicate could be
869 // missed if we add it in the middle or at the end due to return statements
870 // after the addPredicate<>() calls.
871 if (Predicate.hasNoUse()) {
872 InsnMatcher.addPredicate<NoUsePredicateMatcher>();
873 HasAddedBuiltinMatcher = true;
874 }
875 if (Predicate.hasOneUse()) {
876 InsnMatcher.addPredicate<OneUsePredicateMatcher>();
877 HasAddedBuiltinMatcher = true;
878 }
879
880 if (Predicate.hasGISelPredicateCode()) {
881 if (Predicate.usesOperands()) {
882 assert(WaitingForNamedOperands == 0 &&
883 "previous predicate didn't find all operands or "
884 "nested predicate that uses operands");
885 TreePattern *TP = Predicate.getOrigPatFragRecord();
886 WaitingForNamedOperands = TP->getNumArgs();
887 for (unsigned I = 0; I < WaitingForNamedOperands; ++I)
888 StoreIdxForName[getScopedName(Scope: Call.Scope, Name: TP->getArgName(i: I))] = I;
889 }
890 InsnMatcher.addPredicate<GenericInstructionPredicateMatcher>(args: Predicate);
891 continue;
892 }
893 if (!HasAddedBuiltinMatcher) {
894 return failedImport(Reason: "Src pattern child has predicate (" +
895 explainPredicates(N: Src) + ")");
896 }
897 }
898
899 if (Src.isLeaf()) {
900 const Init *SrcInit = Src.getLeafValue();
901 if (const IntInit *SrcIntInit = dyn_cast<IntInit>(Val: SrcInit)) {
902 OperandMatcher &OM =
903 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: Src.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
904 OM.addPredicate<LiteralIntOperandMatcher>(args: SrcIntInit->getValue());
905 } else {
906 return failedImport(
907 Reason: "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
908 }
909 } else {
910 assert(SrcGIOrNull &&
911 "Expected to have already found an equivalent Instruction");
912 if (SrcGIOrNull->getName() == "G_CONSTANT" ||
913 SrcGIOrNull->getName() == "G_FCONSTANT" ||
914 SrcGIOrNull->getName() == "G_FRAME_INDEX") {
915 // imm/fpimm still have operands but we don't need to do anything with it
916 // here since we don't support ImmLeaf predicates yet. However, we still
917 // need to note the hidden operand to get GIM_CheckNumOperands correct.
918 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
919 return InsnMatcher;
920 }
921
922 if (SrcGIOrNull->getName() == "G_BLOCK_ADDR") {
923 // Name operand of G_BLOCK_ADDR so it can be referenced in the destination
924 // pattern.
925 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: Src.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
926 return InsnMatcher;
927 }
928
929 // Special case because the operand order is changed from setcc. The
930 // predicate operand needs to be swapped from the last operand to the first
931 // source.
932
933 unsigned NumChildren = Src.getNumChildren();
934 bool IsFCmp = SrcGIOrNull->getName() == "G_FCMP";
935
936 if (IsFCmp || SrcGIOrNull->getName() == "G_ICMP") {
937 const TreePatternNode &SrcChild = Src.getChild(N: NumChildren - 1);
938 if (SrcChild.isLeaf()) {
939 const DefInit *DI = dyn_cast<DefInit>(Val: SrcChild.getLeafValue());
940 const Record *CCDef = DI ? DI->getDef() : nullptr;
941 if (!CCDef || !CCDef->isSubClassOf(Name: "CondCode"))
942 return failedImport(Reason: "Unable to handle CondCode");
943
944 OperandMatcher &OM = InsnMatcher.addOperand(
945 OpIdx: OpIdx++, SymbolicName: SrcChild.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
946 StringRef PredType = IsFCmp ? CCDef->getValueAsString(FieldName: "FCmpPredicate")
947 : CCDef->getValueAsString(FieldName: "ICmpPredicate");
948
949 if (!PredType.empty()) {
950 OM.addPredicate<CmpPredicateOperandMatcher>(args: PredType.str());
951 // Process the other 2 operands normally.
952 --NumChildren;
953 }
954 }
955 }
956
957 // The SDAG fp_to_[su]int_sat nodes carry a trailing ValueType operand
958 // encoding the saturation width. G_FPTOSI_SAT/G_FPTOUI_SAT instead saturate
959 // to their result type and have no such operand, so drop the trailing child
960 // before matching the operands.
961 if (SrcGIOrNull->getName() == "G_FPTOSI_SAT" ||
962 SrcGIOrNull->getName() == "G_FPTOUI_SAT") {
963 const TreePatternNode &SatChild = Src.getChild(N: NumChildren - 1);
964 const DefInit *DI = SatChild.isLeaf()
965 ? dyn_cast<DefInit>(Val: SatChild.getLeafValue())
966 : nullptr;
967 if (!DI || !DI->getDef()->isSubClassOf(Name: "ValueType"))
968 return failedImport(Reason: "Expected ValueType saturation-width operand");
969 MVT SatVT = llvm::getValueType(Rec: DI->getDef());
970 if (SatVT.getScalarSizeInBits() !=
971 Src.getSimpleType(ResNo: 0).getScalarSizeInBits())
972 return failedImport(Reason: "Saturation width must match the result width");
973 --NumChildren;
974 }
975
976 // Match the used operands (i.e. the children of the operator).
977 bool IsIntrinsic =
978 SrcGIOrNull->getName() == "G_INTRINSIC" ||
979 SrcGIOrNull->getName() == "G_INTRINSIC_W_SIDE_EFFECTS" ||
980 SrcGIOrNull->getName() == "G_INTRINSIC_CONVERGENT" ||
981 SrcGIOrNull->getName() == "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS";
982 const CodeGenIntrinsic *II = Src.getIntrinsicInfo(CDP: CGP);
983 if (IsIntrinsic && !II)
984 return failedImport(Reason: "Expected IntInit containing intrinsic ID)");
985
986 for (unsigned I = 0; I != NumChildren; ++I) {
987 const TreePatternNode &SrcChild = Src.getChild(N: I);
988
989 // We need to determine the meaning of a literal integer based on the
990 // context. If this is a field required to be an immediate (such as an
991 // immarg intrinsic argument), the required predicates are different than
992 // a constant which may be materialized in a register. If we have an
993 // argument that is required to be an immediate, we should not emit an LLT
994 // type check, and should not be looking for a G_CONSTANT defined
995 // register.
996 bool OperandIsImmArg = SrcGIOrNull->isInOperandImmArg(i: I);
997
998 // SelectionDAG allows pointers to be represented with iN since it doesn't
999 // distinguish between pointers and integers but they are different types
1000 // in GlobalISel. Coerce integers to pointers to address space 0 if the
1001 // context indicates a pointer.
1002 //
1003 bool OperandIsAPointer = SrcGIOrNull->isInOperandAPointer(i: I);
1004
1005 if (IsIntrinsic) {
1006 // For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately
1007 // following the defs is an intrinsic ID.
1008 if (I == 0) {
1009 OperandMatcher &OM = InsnMatcher.addOperand(
1010 OpIdx: OpIdx++, SymbolicName: SrcChild.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
1011 OM.addPredicate<IntrinsicIDOperandMatcher>(args&: II);
1012 continue;
1013 }
1014
1015 // We have to check intrinsics for llvm_anyptr_ty and immarg parameters.
1016 //
1017 // Note that we have to look at the i-1th parameter, because we don't
1018 // have the intrinsic ID in the intrinsic's parameter list.
1019 OperandIsAPointer |= II->isParamAPointer(ParamIdx: I - 1);
1020 OperandIsImmArg |= II->isParamImmArg(ParamIdx: I - 1);
1021 }
1022
1023 if (auto Error = importChildMatcher(Rule, InsnMatcher, SrcChild,
1024 OperandIsAPointer, OperandIsImmArg,
1025 GI: SrcGIOrNull, OpIdx: OpIdx++, TempOpIdx))
1026 return std::move(Error);
1027 }
1028 }
1029
1030 return InsnMatcher;
1031}
1032
1033Error GlobalISelEmitter::importComplexPatternOperandMatcher(
1034 OperandMatcher &OM, const Record *R, unsigned &TempOpIdx) const {
1035 const auto &ComplexPattern = ComplexPatternEquivs.find(Val: R);
1036 if (ComplexPattern == ComplexPatternEquivs.end())
1037 return failedImport(Reason: "SelectionDAG ComplexPattern (" + R->getName() +
1038 ") not mapped to GlobalISel");
1039
1040 OM.addPredicate<ComplexPatternOperandMatcher>(args&: OM, args: *ComplexPattern->second);
1041 TempOpIdx++;
1042 return Error::success();
1043}
1044
1045// Get the name to use for a pattern operand. For an anonymous physical register
1046// input, this should use the register name.
1047static StringRef getSrcChildName(const TreePatternNode &SrcChild,
1048 const Record *&PhysReg) {
1049 StringRef SrcChildName = SrcChild.getName();
1050 if (SrcChildName.empty() && SrcChild.isLeaf()) {
1051 if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
1052 auto *ChildRec = ChildDefInit->getDef();
1053 if (ChildRec->isSubClassOf(Name: "Register")) {
1054 SrcChildName = ChildRec->getName();
1055 PhysReg = ChildRec;
1056 }
1057 }
1058 }
1059
1060 return SrcChildName;
1061}
1062
1063Error GlobalISelEmitter::importChildMatcher(
1064 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
1065 const TreePatternNode &SrcChild, bool OperandIsAPointer,
1066 bool OperandIsImmArg, const CodeGenInstruction *GI, unsigned OpIdx,
1067 unsigned &TempOpIdx) {
1068
1069 const Record *PhysReg = nullptr;
1070 std::string SrcChildName = getSrcChildName(SrcChild, PhysReg).str();
1071 if (!SrcChild.isLeaf() &&
1072 SrcChild.getOperator()->isSubClassOf(Name: "ComplexPattern")) {
1073 // The "name" of a non-leaf complex pattern (MY_PAT $op1, $op2) is
1074 // "MY_PAT:op1:op2" and the ones with same "name" represent same operand.
1075 std::string PatternName = SrcChild.getOperator()->getName().str();
1076 for (const TreePatternNode &Child : SrcChild.children()) {
1077 PatternName += ":";
1078 PatternName += Child.getName();
1079 }
1080 SrcChildName = PatternName;
1081 }
1082
1083 OperandMatcher &OM =
1084 PhysReg ? InsnMatcher.addPhysRegInput(Reg: PhysReg, OpIdx, TempOpIdx)
1085 : InsnMatcher.addOperand(OpIdx, SymbolicName: SrcChildName, AllocatedTemporariesBaseID: TempOpIdx);
1086 if (OM.isSameAsAnotherOperand())
1087 return Error::success();
1088
1089 ArrayRef<TypeSetByHwMode> ChildTypes = SrcChild.getExtTypes();
1090 if (ChildTypes.size() != 1)
1091 return failedImport(Reason: "Src pattern child has multiple results");
1092
1093 // Check MBB's before the type check since they are not a known type.
1094 if (!SrcChild.isLeaf()) {
1095 if (SrcChild.getOperator()->getName() == "bb") {
1096 OM.addPredicate<MBBOperandMatcher>();
1097 return Error::success();
1098 }
1099 if (SrcChild.getOperator()->getName() == "timm") {
1100 OM.addPredicate<ImmOperandMatcher>();
1101
1102 // Add predicates, if any
1103 for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {
1104 const TreePredicateFn &Predicate = Call.Fn;
1105
1106 // Only handle immediate patterns for now
1107 if (Predicate.isImmediatePattern()) {
1108 OM.addPredicate<OperandImmPredicateMatcher>(args: Predicate);
1109 }
1110 }
1111
1112 return Error::success();
1113 }
1114 } else if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
1115 auto *ChildRec = ChildDefInit->getDef();
1116 if (ChildRec->isSubClassOf(Name: "ValueType") && !SrcChild.hasName()) {
1117 // An unnamed ValueType as in (sext_inreg GPR:$foo, i8). GISel represents
1118 // this as a literal constant with the scalar size.
1119 MVT VT = llvm::getValueType(Rec: ChildRec);
1120 OM.addPredicate<LiteralIntOperandMatcher>(args: VT.getScalarSizeInBits());
1121 return Error::success();
1122 }
1123 }
1124
1125 // Immediate arguments have no meaningful type to check as they don't have
1126 // registers.
1127 if (!OperandIsImmArg) {
1128 if (auto Error = addTypeCheckPredicateForOpcode(OM, VTy: ChildTypes.front(),
1129 OperandIsAPointer, GI))
1130 return failedImport(Reason: toString(E: std::move(Error)) + " for Src operand (" +
1131 to_string(Value: SrcChild) + ")");
1132 }
1133
1134 // Try look up SrcChild for a (named) predicate operand if there is any.
1135 if (WaitingForNamedOperands) {
1136 auto &ScopedNames = SrcChild.getNamesAsPredicateArg();
1137 if (!ScopedNames.empty()) {
1138 auto PA = ScopedNames.begin();
1139 std::string Name = getScopedName(Scope: PA->getScope(), Name: PA->getIdentifier());
1140 OM.addPredicate<RecordNamedOperandMatcher>(args&: StoreIdxForName[Name], args&: Name);
1141 --WaitingForNamedOperands;
1142 }
1143 }
1144
1145 // Check for nested instructions.
1146 if (!SrcChild.isLeaf()) {
1147 if (SrcChild.getOperator()->isSubClassOf(Name: "ComplexPattern")) {
1148 // When a ComplexPattern is used as an operator, it should do the same
1149 // thing as when used as a leaf. However, the children of the operator
1150 // name the sub-operands that make up the complex operand and we must
1151 // prepare to reference them in the renderer too.
1152 unsigned RendererID = TempOpIdx;
1153 if (auto Error = importComplexPatternOperandMatcher(
1154 OM, R: SrcChild.getOperator(), TempOpIdx))
1155 return Error;
1156
1157 for (unsigned I = 0, E = SrcChild.getNumChildren(); I != E; ++I) {
1158 auto &SubOperand = SrcChild.getChild(N: I);
1159 if (!SubOperand.getName().empty()) {
1160 if (auto Error = Rule.defineComplexSubOperand(
1161 SymbolicName: SubOperand.getName(), ComplexPattern: SrcChild.getOperator(), RendererID, SubOperandID: I,
1162 ParentSymbolicName: SrcChildName))
1163 return Error;
1164 }
1165 }
1166
1167 return Error::success();
1168 }
1169
1170 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
1171 args&: InsnMatcher.getRuleMatcher(), args: SrcChild.getName());
1172 if (!MaybeInsnOperand) {
1173 // This isn't strictly true. If the user were to provide exactly the same
1174 // matchers as the original operand then we could allow it. However, it's
1175 // simpler to not permit the redundant specification.
1176 return failedImport(
1177 Reason: "Nested instruction cannot be the same as another operand");
1178 }
1179
1180 // Map the node to a gMIR instruction.
1181 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
1182 auto InsnMatcherOrError = createAndImportSelDAGMatcher(
1183 Rule, InsnMatcher&: InsnOperand.getInsnMatcher(), Src: SrcChild, TempOpIdx);
1184 if (auto Error = InsnMatcherOrError.takeError())
1185 return Error;
1186
1187 return Error::success();
1188 }
1189
1190 if (SrcChild.hasAnyPredicate()) {
1191 for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {
1192 const TreePredicateFn &Predicate = Call.Fn;
1193
1194 if (!Predicate.hasGISelLeafPredicateCode())
1195 return failedImport(Reason: "Src pattern child has unsupported predicate");
1196 OM.addPredicate<OperandLeafPredicateMatcher>(args: Predicate);
1197 }
1198 return Error::success();
1199 }
1200
1201 // Check for constant immediates.
1202 if (auto *ChildInt = dyn_cast<IntInit>(Val: SrcChild.getLeafValue())) {
1203 if (OperandIsImmArg) {
1204 // Checks for argument directly in operand list
1205 OM.addPredicate<LiteralIntOperandMatcher>(args: ChildInt->getValue());
1206 } else {
1207 // Checks for materialized constant
1208 OM.addPredicate<ConstantIntOperandMatcher>(args: ChildInt->getValue());
1209 }
1210 return Error::success();
1211 }
1212
1213 // Check for def's like register classes or ComplexPattern's.
1214 if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
1215 auto *ChildRec = ChildDefInit->getDef();
1216
1217 // Check for register classes.
1218 if (ChildRec->isSubClassOf(Name: "RegisterClassLike") ||
1219 ChildRec->isSubClassOf(Name: "RegisterOperand")) {
1220 OM.addPredicate<RegisterBankOperandMatcher>(
1221 args: Target.getRegisterClass(R: Target.getInitValueAsRegClass(
1222 V: ChildDefInit,
1223 /*AssumeRegClassByHwModeIsDefault=*/true)));
1224 return Error::success();
1225 }
1226
1227 if (ChildRec->isSubClassOf(Name: "Register")) {
1228 // This just be emitted as a copy to the specific register.
1229 ValueTypeByHwMode VT = ChildTypes.front().getValueTypeByHwMode();
1230 const CodeGenRegisterClass *RC =
1231 CGRegs.getMinimalPhysRegClass(RegRecord: ChildRec, VT: &VT);
1232 if (!RC) {
1233 return failedImport(
1234 Reason: "Could not determine physical register class of pattern source");
1235 }
1236
1237 OM.addPredicate<RegisterBankOperandMatcher>(args: *RC);
1238 return Error::success();
1239 }
1240
1241 // Check for ValueType.
1242 if (ChildRec->isSubClassOf(Name: "ValueType")) {
1243 // We already added a type check as standard practice so this doesn't need
1244 // to do anything.
1245 return Error::success();
1246 }
1247
1248 // Check for ComplexPattern's.
1249 if (ChildRec->isSubClassOf(Name: "ComplexPattern"))
1250 return importComplexPatternOperandMatcher(OM, R: ChildRec, TempOpIdx);
1251
1252 if (ChildRec->isSubClassOf(Name: "ImmLeaf")) {
1253 return failedImport(
1254 Reason: "Src pattern child def is an unsupported tablegen class (ImmLeaf)");
1255 }
1256
1257 // Place holder for SRCVALUE nodes. Nothing to do here.
1258 if (ChildRec->getName() == "srcvalue")
1259 return Error::success();
1260
1261 const bool ImmAllOnesV = ChildRec->getName() == "immAllOnesV";
1262 if (ImmAllOnesV || ChildRec->getName() == "immAllZerosV") {
1263 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
1264 args&: InsnMatcher.getRuleMatcher(), args: SrcChild.getName(), args: false);
1265 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
1266
1267 ValueTypeByHwMode VTy = ChildTypes.front().getValueTypeByHwMode();
1268
1269 const CodeGenInstruction &BuildVector =
1270 Target.getInstruction(InstRec: RK.getDef(Name: "G_BUILD_VECTOR"));
1271 const CodeGenInstruction &BuildVectorTrunc =
1272 Target.getInstruction(InstRec: RK.getDef(Name: "G_BUILD_VECTOR_TRUNC"));
1273
1274 // Treat G_BUILD_VECTOR as the canonical opcode, and G_BUILD_VECTOR_TRUNC
1275 // as an alternative.
1276 InsnOperand.getInsnMatcher().addPredicate<InstructionOpcodeMatcher>(
1277 args: ArrayRef({&BuildVector, &BuildVectorTrunc}));
1278
1279 // TODO: Handle both G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC We could
1280 // theoretically not emit any opcode check, but getOpcodeMatcher currently
1281 // has to succeed.
1282 OperandMatcher &OM =
1283 InsnOperand.getInsnMatcher().addOperand(OpIdx: 0, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
1284 if (auto Error = OM.addTypeCheckPredicate(VTy: TypeSetByHwMode(VTy),
1285 /*OperandIsAPointer=*/false))
1286 return failedImport(Reason: toString(E: std::move(Error)) +
1287 " for result of Src pattern operator");
1288
1289 InsnOperand.getInsnMatcher().addPredicate<VectorSplatImmPredicateMatcher>(
1290 args: ImmAllOnesV ? VectorSplatImmPredicateMatcher::AllOnes
1291 : VectorSplatImmPredicateMatcher::AllZeros);
1292 return Error::success();
1293 }
1294
1295 return failedImport(
1296 Reason: "Src pattern child def is an unsupported tablegen class");
1297 }
1298
1299 return failedImport(Reason: "Src pattern child is an unsupported kind");
1300}
1301
1302// Equivalent of MatcherGen::EmitResultOfNamedOperand.
1303Error GlobalISelEmitter::importNamedNodeRenderer(
1304 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N) const {
1305 StringRef NodeName = N.getName();
1306
1307 if (auto SubOperand = M.getComplexSubOperand(SymbolicName: NodeName)) {
1308 auto [ComplexPatternRec, RendererID, SubOperandIdx] = *SubOperand;
1309 MIBuilder.addRenderer<RenderComplexPatternOperand>(
1310 args: *ComplexPatternRec, args&: NodeName, args&: RendererID, args&: SubOperandIdx);
1311 return Error::success();
1312 }
1313
1314 if (!N.isLeaf()) {
1315 StringRef OperatorName = N.getOperator()->getName();
1316
1317 if (OperatorName == "imm") {
1318 MIBuilder.addRenderer<CopyConstantAsImmRenderer>(args&: M, args&: NodeName);
1319 return Error::success();
1320 }
1321
1322 if (OperatorName == "fpimm") {
1323 MIBuilder.addRenderer<CopyFConstantAsFPImmRenderer>(args&: M, args&: NodeName);
1324 return Error::success();
1325 }
1326
1327 // TODO: 'imm' and 'fpimm' are the only nodes that need special treatment.
1328 // Remove this check and add CopyRenderer unconditionally for other nodes.
1329 if (OperatorName == "bb" || OperatorName == "timm" ||
1330 OperatorName == "tframeindex") {
1331 MIBuilder.addRenderer<CopyRenderer>(args&: M, args&: NodeName);
1332 return Error::success();
1333 }
1334
1335 return failedImport(Reason: "node has unsupported operator " + to_string(Value: N));
1336 }
1337
1338 if (const auto *DI = dyn_cast<DefInit>(Val: N.getLeafValue())) {
1339 const Record *R = DI->getDef();
1340
1341 if (N.getNumResults() != 1)
1342 return failedImport(Reason: "node does not have one result " + to_string(Value: N));
1343
1344 if (R->isSubClassOf(Name: "ComplexPattern")) {
1345 auto I = ComplexPatternEquivs.find(Val: R);
1346 if (I == ComplexPatternEquivs.end())
1347 return failedImport(Reason: "ComplexPattern " + R->getName() +
1348 " does not have GISel equivalent");
1349
1350 const OperandMatcher &OM = M.getOperandMatcher(Name: NodeName);
1351 MIBuilder.addRenderer<RenderComplexPatternOperand>(
1352 args: *I->second, args&: NodeName, args: OM.getAllocatedTemporariesBaseID());
1353 return Error::success();
1354 }
1355
1356 if (R->isSubClassOf(Name: "RegisterOperand") &&
1357 !R->isValueUnset(FieldName: "GIZeroRegister")) {
1358 MIBuilder.addRenderer<CopyOrAddZeroRegRenderer>(
1359 args&: M, args&: NodeName, args: R->getValueAsDef(FieldName: "GIZeroRegister"));
1360 return Error::success();
1361 }
1362
1363 // TODO: All special cases are handled above. Remove this check and add
1364 // CopyRenderer unconditionally.
1365 if (R->isSubClassOf(Name: "RegisterClassLike") ||
1366 R->isSubClassOf(Name: "RegisterOperand") || R->isSubClassOf(Name: "ValueType")) {
1367 MIBuilder.addRenderer<CopyRenderer>(args&: M, args&: NodeName);
1368 return Error::success();
1369 }
1370 }
1371
1372 // TODO: Change this to assert and move to the beginning of the function.
1373 if (!M.hasOperand(SymbolicName: NodeName))
1374 return failedImport(Reason: "could not find node $" + NodeName +
1375 " in the source DAG");
1376
1377 // TODO: Remove this check and add CopyRenderer unconditionally.
1378 // TODO: Handle nodes with multiple results (provided they can reach here).
1379 if (isa<UnsetInit>(Val: N.getLeafValue())) {
1380 MIBuilder.addRenderer<CopyRenderer>(args&: M, args&: NodeName);
1381 return Error::success();
1382 }
1383
1384 return failedImport(Reason: "unsupported node " + to_string(Value: N));
1385}
1386
1387// Equivalent of MatcherGen::EmitResultLeafAsOperand.
1388Error GlobalISelEmitter::importLeafNodeRenderer(
1389 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N,
1390 action_iterator InsertPt) const {
1391 if (const auto *II = dyn_cast<IntInit>(Val: N.getLeafValue())) {
1392 MIBuilder.addRenderer<ImmRenderer>(args: II->getValue());
1393 return Error::success();
1394 }
1395
1396 if (const auto *DI = dyn_cast<DefInit>(Val: N.getLeafValue())) {
1397 const Record *R = DI->getDef();
1398
1399 if (R->isSubClassOf(Name: "Register") || R->getName() == "zero_reg") {
1400 MIBuilder.addRenderer<AddRegisterRenderer>(args: Target, args&: R);
1401 return Error::success();
1402 }
1403
1404 if (R->getName() == "undef_tied_input") {
1405 std::optional<LLTCodeGen> OpTyOrNone = MVTToLLT(VT: N.getSimpleType(ResNo: 0));
1406 if (!OpTyOrNone)
1407 return failedImport(Reason: "unsupported type");
1408
1409 unsigned TempRegID = M.allocateTempRegID();
1410 M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTyOrNone, args&: TempRegID);
1411
1412 auto I = M.insertAction<BuildMIAction>(
1413 InsertPt, args: M.allocateOutputInsnID(), args&: M,
1414 args: &Target.getInstruction(InstRec: RK.getDef(Name: "IMPLICIT_DEF")));
1415 auto &ImpDefBuilder = static_cast<BuildMIAction &>(**I);
1416 ImpDefBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, /*IsDef=*/args: true);
1417
1418 MIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID);
1419 return Error::success();
1420 }
1421
1422 if (R->isSubClassOf(Name: "SubRegIndex")) {
1423 const CodeGenSubRegIndex *SubRegIndex = CGRegs.findSubRegIdx(Def: R);
1424 MIBuilder.addRenderer<ImmRenderer>(args: SubRegIndex->EnumValue);
1425 return Error::success();
1426 }
1427
1428 // There are also RegisterClass / RegisterOperand operands of REG_SEQUENCE /
1429 // COPY_TO_REGCLASS, but these instructions are currently handled elsewhere.
1430 }
1431
1432 return failedImport(Reason: "unrecognized node " + to_string(Value: N));
1433}
1434
1435// Equivalent of MatcherGen::EmitResultSDNodeXFormAsOperand.
1436Error GlobalISelEmitter::importXFormNodeRenderer(
1437 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N) const {
1438 const Record *XFormRec = N.getOperator();
1439 auto I = SDNodeXFormEquivs.find(Val: XFormRec);
1440 if (I == SDNodeXFormEquivs.end())
1441 return failedImport(Reason: "SDNodeXForm " + XFormRec->getName() +
1442 " does not have GISel equivalent");
1443
1444 // TODO: Fail to import if GISDNodeXForm does not have RendererFn.
1445 // This currently results in a fatal error in emitRenderOpcodes.
1446 const Record *XFormEquivRec = I->second;
1447
1448 // The node to apply the transformation function to.
1449 // FIXME: The node may not have a name and may be a leaf. It should be
1450 // rendered first, like any other nodes. This may or may not require
1451 // introducing a temporary register, and we can't tell that without
1452 // inspecting the node (possibly recursively). This is a general drawback
1453 // of appending renderers directly to BuildMIAction.
1454 const TreePatternNode &Node = N.getChild(N: 0);
1455 StringRef NodeName = Node.getName();
1456
1457 const Record *XFormOpc = CGP.getSDNodeTransform(R: XFormRec).first;
1458 if (XFormOpc->getName() == "timm") {
1459 // If this is a TargetConstant, there won't be a corresponding
1460 // instruction to transform. Instead, this will refer directly to an
1461 // operand in an instruction's operand list.
1462 MIBuilder.addRenderer<CustomOperandRenderer>(args&: M, args: *XFormEquivRec, args&: NodeName);
1463 } else {
1464 MIBuilder.addRenderer<CustomRenderer>(args&: M, args: *XFormEquivRec, args&: NodeName);
1465 }
1466
1467 return Error::success();
1468}
1469
1470// Equivalent of MatcherGen::EmitResultInstructionAsOperand.
1471Error GlobalISelEmitter::importInstructionNodeRenderer(
1472 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N,
1473 action_iterator &InsertPt) const {
1474 Expected<LLTCodeGen> OpTy = getInstResultType(Dst: N, Target);
1475 if (!OpTy)
1476 return OpTy.takeError();
1477
1478 // TODO: See the comment in importXFormNodeRenderer. We rely on the node
1479 // requiring a temporary register, which prevents us from using this
1480 // function on the root of the destination DAG.
1481 unsigned TempRegID = M.allocateTempRegID();
1482 InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTy, args&: TempRegID);
1483 MIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID);
1484
1485 auto InsertPtOrError =
1486 createAndImportSubInstructionRenderer(InsertPt: ++InsertPt, M, Dst: N, TempReg: TempRegID);
1487 if (!InsertPtOrError)
1488 return InsertPtOrError.takeError();
1489
1490 InsertPt = *InsertPtOrError;
1491 return Error::success();
1492}
1493
1494// Equivalent of MatcherGen::EmitResultOperand.
1495Error GlobalISelEmitter::importNodeRenderer(RuleMatcher &M,
1496 BuildMIAction &MIBuilder,
1497 const TreePatternNode &N,
1498 action_iterator &InsertPt) const {
1499 if (N.hasName())
1500 return importNamedNodeRenderer(M, MIBuilder, N);
1501
1502 if (N.isLeaf())
1503 return importLeafNodeRenderer(M, MIBuilder, N, InsertPt);
1504
1505 if (N.getOperator()->isSubClassOf(Name: "SDNodeXForm"))
1506 return importXFormNodeRenderer(M, MIBuilder, N);
1507
1508 if (N.getOperator()->isSubClassOf(Name: "Instruction"))
1509 return importInstructionNodeRenderer(M, MIBuilder, N, InsertPt);
1510
1511 // Should not reach here.
1512 return failedImport(Reason: "unrecognized node " + llvm::to_string(Value: N));
1513}
1514
1515/// Generates code that builds the resulting instruction(s) from the destination
1516/// DAG. Note that to do this we do not and should not need the source DAG.
1517/// We do need to know whether a generated instruction defines a result of the
1518/// source DAG; this information is available via RuleMatcher::hasOperand.
1519Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
1520 RuleMatcher &M, InstructionMatcher &InsnMatcher,
1521 const TreePatternNode &Dst) const {
1522 auto InsertPtOrError = createInstructionRenderer(InsertPt: M.actions_end(), M, Dst);
1523 if (auto Error = InsertPtOrError.takeError())
1524 return std::move(Error);
1525
1526 action_iterator InsertPt = InsertPtOrError.get();
1527 BuildMIAction &DstMIBuilder = *static_cast<BuildMIAction *>(InsertPt->get());
1528
1529 for (auto PhysOp : M.physoperands()) {
1530 InsertPt = M.insertAction<BuildMIAction>(
1531 InsertPt, args: M.allocateOutputInsnID(), args&: M,
1532 args: &Target.getInstruction(InstRec: RK.getDef(Name: "COPY")));
1533 BuildMIAction &CopyToPhysRegMIBuilder =
1534 *static_cast<BuildMIAction *>(InsertPt->get());
1535 CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>(args: Target,
1536 args&: PhysOp.first, args: true);
1537 CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(args&: M, args&: PhysOp.first);
1538 }
1539
1540 if (auto Error = importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Dst,
1541 /*IsRoot=*/true)
1542 .takeError())
1543 return std::move(Error);
1544
1545 if (auto Error = importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst)
1546 .takeError())
1547 return std::move(Error);
1548
1549 return DstMIBuilder;
1550}
1551
1552Expected<action_iterator>
1553GlobalISelEmitter::createAndImportSubInstructionRenderer(
1554 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
1555 unsigned TempRegID) const {
1556 auto InsertPtOrError = createInstructionRenderer(InsertPt, M, Dst);
1557
1558 // TODO: Assert there's exactly one result.
1559
1560 if (auto Error = InsertPtOrError.takeError())
1561 return std::move(Error);
1562
1563 BuildMIAction &DstMIBuilder =
1564 *static_cast<BuildMIAction *>(InsertPtOrError.get()->get());
1565
1566 // Assign the result to TempReg.
1567 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, args: true);
1568
1569 // Handle additional (ignored) results.
1570 InsertPtOrError = importExplicitDefRenderers(
1571 InsertPt: std::prev(x: *InsertPtOrError), M, DstMIBuilder, Dst, /*IsRoot=*/false);
1572 if (auto Error = InsertPtOrError.takeError())
1573 return std::move(Error);
1574
1575 InsertPtOrError =
1576 importExplicitUseRenderers(InsertPt: InsertPtOrError.get(), M, DstMIBuilder, Dst);
1577 if (auto Error = InsertPtOrError.takeError())
1578 return std::move(Error);
1579
1580 if (auto Error =
1581 constrainOperands(InsertPt, M, InsnID: DstMIBuilder.getInsnID(), Dst))
1582 return std::move(Error);
1583
1584 return InsertPtOrError.get();
1585}
1586
1587Expected<action_iterator>
1588GlobalISelEmitter::createInstructionRenderer(action_iterator InsertPt,
1589 RuleMatcher &M,
1590 const TreePatternNode &Dst) const {
1591 const Record *DstOp = Dst.getOperator();
1592 if (!DstOp->isSubClassOf(Name: "Instruction")) {
1593 if (DstOp->isSubClassOf(Name: "ValueType"))
1594 return failedImport(
1595 Reason: "Pattern operator isn't an instruction (it's a ValueType)");
1596 return failedImport(Reason: "Pattern operator isn't an instruction");
1597 }
1598 const CodeGenInstruction *DstI = &Target.getInstruction(InstRec: DstOp);
1599
1600 // COPY_TO_REGCLASS is just a copy with a ConstrainOperandToRegClassAction
1601 // attached. Similarly for EXTRACT_SUBREG except that's a subregister copy.
1602 StringRef Name = DstI->getName();
1603 if (Name == "COPY_TO_REGCLASS" || Name == "EXTRACT_SUBREG")
1604 DstI = &Target.getInstruction(InstRec: RK.getDef(Name: "COPY"));
1605
1606 return M.insertAction<BuildMIAction>(InsertPt, args: M.allocateOutputInsnID(), args&: M,
1607 args&: DstI);
1608}
1609
1610Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
1611 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
1612 const TreePatternNode &Dst, bool IsRoot) const {
1613 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
1614
1615 // Process explicit defs. The caller may have already handled the first def.
1616 for (unsigned I = IsRoot ? 0 : 1, E = DstI->Operands.NumDefs; I != E; ++I) {
1617 const CGIOperandList::OperandInfo &OpInfo = DstI->Operands[I];
1618 std::string OpName = getMangledRootDefName(DefOperandName: OpInfo.Name);
1619
1620 // If the def is used in the source DAG, forward it.
1621 if (IsRoot && M.hasOperand(SymbolicName: OpName)) {
1622 // CopyRenderer saves a StringRef, so cannot pass OpName itself -
1623 // let's use a string with an appropriate lifetime.
1624 StringRef PermanentRef = M.getOperandMatcher(Name: OpName).getSymbolicName();
1625 DstMIBuilder.addRenderer<CopyRenderer>(args&: M, args&: PermanentRef);
1626 continue;
1627 }
1628
1629 // A discarded explicit def may be an optional physical register.
1630 // If this is the case, add the default register and mark it as dead.
1631 if (OpInfo.Rec->isSubClassOf(Name: "OptionalDefOperand")) {
1632 for (const TreePatternNode &DefaultOp :
1633 make_pointee_range(Range: CGP.getDefaultOperand(R: OpInfo.Rec).DefaultOps)) {
1634 // TODO: Do these checks in ParseDefaultOperands.
1635 if (!DefaultOp.isLeaf())
1636 return failedImport(Reason: "optional def is not a leaf");
1637
1638 const auto *RegDI = dyn_cast<DefInit>(Val: DefaultOp.getLeafValue());
1639 if (!RegDI)
1640 return failedImport(Reason: "optional def is not a record");
1641
1642 const Record *Reg = RegDI->getDef();
1643 if (!Reg->isSubClassOf(Name: "Register") && Reg->getName() != "zero_reg")
1644 return failedImport(Reason: "optional def is not a register");
1645
1646 DstMIBuilder.addRenderer<AddRegisterRenderer>(
1647 args: Target, args&: Reg, /*IsDef=*/args: true, /*IsDead=*/args: true);
1648 }
1649 continue;
1650 }
1651
1652 // The def is discarded, create a dead virtual register for it.
1653 const TypeSetByHwMode &ExtTy = Dst.getExtType(ResNo: I);
1654 if (!ExtTy.isMachineValueType())
1655 return failedImport(Reason: "unsupported typeset");
1656
1657 auto OpTy = MVTToLLT(VT: ExtTy.getMachineValueType().SimpleTy);
1658 if (!OpTy)
1659 return failedImport(Reason: "unsupported type");
1660
1661 unsigned TempRegID = M.allocateTempRegID();
1662 InsertPt =
1663 M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *OpTy, args&: TempRegID);
1664 DstMIBuilder.addRenderer<TempRegRenderer>(
1665 args&: TempRegID, /*IsDef=*/args: true, /*SubReg=*/args: nullptr, /*IsDead=*/args: true);
1666 }
1667
1668 // Implicit defs are not currently supported, mark all of them as dead.
1669 for (const Record *Reg : DstI->ImplicitDefs) {
1670 std::string OpName = getMangledRootDefName(DefOperandName: Reg->getName());
1671 assert(!M.hasOperand(OpName) && "The pattern should've been rejected");
1672 DstMIBuilder.setDeadImplicitDef(Reg);
1673 }
1674
1675 return InsertPt;
1676}
1677
1678Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
1679 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
1680 const TreePatternNode &Dst) const {
1681 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
1682 const CodeGenInstruction *OrigDstI =
1683 &Target.getInstruction(InstRec: Dst.getOperator());
1684
1685 StringRef Name = OrigDstI->getName();
1686 unsigned ExpectedDstINumUses = Dst.getNumChildren();
1687
1688 // EXTRACT_SUBREG needs to use a subregister COPY.
1689 if (Name == "EXTRACT_SUBREG") {
1690 if (!Dst.getChild(N: 1).isLeaf())
1691 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a leaf");
1692 const DefInit *SubRegInit =
1693 dyn_cast<DefInit>(Val: Dst.getChild(N: 1).getLeafValue());
1694 if (!SubRegInit)
1695 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a subreg index");
1696
1697 const CodeGenSubRegIndex *SubIdx =
1698 CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
1699 const TreePatternNode &ValChild = Dst.getChild(N: 0);
1700 if (!ValChild.isLeaf()) {
1701 // We really have to handle the source instruction, and then insert a
1702 // copy from the subregister.
1703 auto ExtractSrcTy = getInstResultType(Dst: ValChild, Target);
1704 if (!ExtractSrcTy)
1705 return ExtractSrcTy.takeError();
1706
1707 unsigned TempRegID = M.allocateTempRegID();
1708 InsertPt = M.insertAction<MakeTempRegisterAction>(InsertPt, args&: *ExtractSrcTy,
1709 args&: TempRegID);
1710
1711 auto InsertPtOrError = createAndImportSubInstructionRenderer(
1712 InsertPt: ++InsertPt, M, Dst: ValChild, TempRegID);
1713 if (auto Error = InsertPtOrError.takeError())
1714 return std::move(Error);
1715
1716 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, args: false, args&: SubIdx);
1717 return InsertPt;
1718 }
1719
1720 // If this is a source operand, this is just a subregister copy.
1721 const Record *RCDef =
1722 Target.getInitValueAsRegClass(V: ValChild.getLeafValue());
1723 if (!RCDef)
1724 return failedImport(Reason: "EXTRACT_SUBREG child #0 could not "
1725 "be coerced to a register class");
1726
1727 CodeGenRegisterClass *RC = CGRegs.getRegClass(RCDef);
1728
1729 const auto SrcRCDstRCPair =
1730 RC->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1731 if (SrcRCDstRCPair) {
1732 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
1733 if (SrcRCDstRCPair->first != RC)
1734 return failedImport(Reason: "EXTRACT_SUBREG requires an additional COPY");
1735 }
1736
1737 StringRef RegOperandName = Dst.getChild(N: 0).getName();
1738 if (const auto &SubOperand = M.getComplexSubOperand(SymbolicName: RegOperandName)) {
1739 DstMIBuilder.addRenderer<RenderComplexPatternOperand>(
1740 args: *std::get<0>(t: *SubOperand), args&: RegOperandName, args: std::get<1>(t: *SubOperand),
1741 args: std::get<2>(t: *SubOperand), args&: SubIdx);
1742 return InsertPt;
1743 }
1744
1745 DstMIBuilder.addRenderer<CopySubRegRenderer>(args&: M, args&: RegOperandName, args&: SubIdx);
1746 return InsertPt;
1747 }
1748
1749 if (Name == "REG_SEQUENCE") {
1750 if (!Dst.getChild(N: 0).isLeaf())
1751 return failedImport(Reason: "REG_SEQUENCE child #0 is not a leaf");
1752
1753 const Record *RCDef =
1754 Target.getInitValueAsRegClass(V: Dst.getChild(N: 0).getLeafValue());
1755 if (!RCDef)
1756 return failedImport(Reason: "REG_SEQUENCE child #0 could not "
1757 "be coerced to a register class");
1758
1759 if ((ExpectedDstINumUses - 1) % 2 != 0)
1760 return failedImport(Reason: "Malformed REG_SEQUENCE");
1761
1762 for (unsigned I = 1; I != ExpectedDstINumUses; I += 2) {
1763 const TreePatternNode &ValChild = Dst.getChild(N: I);
1764 const TreePatternNode &SubRegChild = Dst.getChild(N: I + 1);
1765
1766 if (const DefInit *SubRegInit =
1767 dyn_cast<DefInit>(Val: SubRegChild.getLeafValue())) {
1768 const CodeGenSubRegIndex *SubIdx =
1769 CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
1770
1771 if (Error Err = importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: ValChild, InsertPt))
1772 return Err;
1773
1774 DstMIBuilder.addRenderer<SubRegIndexRenderer>(args&: SubIdx);
1775 }
1776 }
1777
1778 return InsertPt;
1779 }
1780
1781 // Render the explicit uses.
1782 unsigned DstINumUses = OrigDstI->Operands.size() - OrigDstI->Operands.NumDefs;
1783 if (Name == "COPY_TO_REGCLASS") {
1784 DstINumUses--; // Ignore the class constraint.
1785 ExpectedDstINumUses--;
1786 }
1787
1788 // NumResults - This is the number of results produced by the instruction in
1789 // the "outs" list.
1790 unsigned NumResults = OrigDstI->Operands.NumDefs;
1791
1792 // Number of operands we know the output instruction must have. If it is
1793 // variadic, we could have more operands.
1794 unsigned NumFixedOperands = DstI->Operands.size();
1795
1796 // Loop over all of the fixed operands of the instruction pattern, emitting
1797 // code to fill them all in. The node 'N' usually has number children equal to
1798 // the number of input operands of the instruction. However, in cases where
1799 // there are predicate operands for an instruction, we need to fill in the
1800 // 'execute always' values. Match up the node operands to the instruction
1801 // operands to do this.
1802 unsigned Child = 0;
1803
1804 // Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the
1805 // number of operands at the end of the list which have default values.
1806 // Those can come from the pattern if it provides enough arguments, or be
1807 // filled in with the default if the pattern hasn't provided them. But any
1808 // operand with a default value _before_ the last mandatory one will be
1809 // filled in with their defaults unconditionally.
1810 unsigned NonOverridableOperands = NumFixedOperands;
1811 while (NonOverridableOperands > NumResults &&
1812 CGP.operandHasDefault(Op: DstI->Operands[NonOverridableOperands - 1].Rec))
1813 --NonOverridableOperands;
1814
1815 unsigned NumDefaultOps = 0;
1816 for (unsigned I = 0; I != DstINumUses; ++I) {
1817 unsigned InstOpNo = DstI->Operands.NumDefs + I;
1818
1819 // Determine what to emit for this operand.
1820 const Record *OperandNode = DstI->Operands[InstOpNo].Rec;
1821
1822 // If the operand has default values, introduce them now.
1823 if (CGP.operandHasDefault(Op: OperandNode) &&
1824 (InstOpNo < NonOverridableOperands || Child >= Dst.getNumChildren())) {
1825 // This is a predicate or optional def operand which the pattern has not
1826 // overridden, or which we aren't letting it override; emit the 'default
1827 // ops' operands.
1828 for (const TreePatternNode &OpNode :
1829 make_pointee_range(Range: CGP.getDefaultOperand(R: OperandNode).DefaultOps)) {
1830 if (Error Err = importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: OpNode, InsertPt))
1831 return Err;
1832 }
1833
1834 ++NumDefaultOps;
1835 continue;
1836 }
1837
1838 if (Error Err =
1839 importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: Dst.getChild(N: Child), InsertPt))
1840 return Err;
1841
1842 ++Child;
1843 }
1844
1845 if (NumDefaultOps + ExpectedDstINumUses != DstINumUses)
1846 return failedImport(Reason: "Expected " + llvm::to_string(Value: DstINumUses) +
1847 " used operands but found " +
1848 llvm::to_string(Value: ExpectedDstINumUses) +
1849 " explicit ones and " + llvm::to_string(Value: NumDefaultOps) +
1850 " default ones");
1851
1852 return InsertPt;
1853}
1854
1855Error GlobalISelEmitter::importImplicitDefRenderers(
1856 BuildMIAction &DstMIBuilder, ArrayRef<const Record *> ImplicitDefs) const {
1857 if (!ImplicitDefs.empty())
1858 return failedImport(Reason: "Pattern defines a physical register");
1859 return Error::success();
1860}
1861
1862Error GlobalISelEmitter::constrainOperands(action_iterator InsertPt,
1863 RuleMatcher &M, unsigned InsnID,
1864 const TreePatternNode &Dst) const {
1865 const Record *DstOp = Dst.getOperator();
1866 const CodeGenInstruction &DstI = Target.getInstruction(InstRec: DstOp);
1867 StringRef DstIName = DstI.getName();
1868
1869 if (DstIName == "COPY_TO_REGCLASS") {
1870 // COPY_TO_REGCLASS does not provide operand constraints itself but the
1871 // result is constrained to the class given by the second child.
1872 const Record *DstIOpRec =
1873 Target.getInitValueAsRegClass(V: Dst.getChild(N: 1).getLeafValue());
1874
1875 if (DstIOpRec == nullptr)
1876 return failedImport(Reason: "COPY_TO_REGCLASS operand #1 isn't a register class");
1877
1878 M.insertAction<ConstrainOperandToRegClassAction>(
1879 InsertPt, args&: InsnID, args: 0, args: Target.getRegisterClass(R: DstIOpRec));
1880 } else if (DstIName == "EXTRACT_SUBREG") {
1881 const CodeGenRegisterClass *SuperClass =
1882 inferRegClassFromPattern(N: Dst.getChild(N: 0));
1883 if (!SuperClass)
1884 return failedImport(
1885 Reason: "Cannot infer register class from EXTRACT_SUBREG operand #0");
1886
1887 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: Dst.getChild(N: 1));
1888 if (!SubIdx)
1889 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a subreg index");
1890
1891 // It would be nice to leave this constraint implicit but we're required
1892 // to pick a register class so constrain the result to a register class
1893 // that can hold the correct MVT.
1894 //
1895 // FIXME: This may introduce an extra copy if the chosen class doesn't
1896 // actually contain the subregisters.
1897 const auto SrcRCDstRCPair =
1898 SuperClass->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1899 if (!SrcRCDstRCPair) {
1900 return failedImport(Reason: "subreg index is incompatible "
1901 "with inferred reg class");
1902 }
1903
1904 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
1905 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1906 args&: *SrcRCDstRCPair->second);
1907 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 1,
1908 args&: *SrcRCDstRCPair->first);
1909 } else if (DstIName == "INSERT_SUBREG") {
1910 // We need to constrain the destination, a super regsister source, and a
1911 // subregister source.
1912 const CodeGenRegisterClass *SubClass =
1913 inferRegClassFromPattern(N: Dst.getChild(N: 1));
1914 if (!SubClass)
1915 return failedImport(
1916 Reason: "Cannot infer register class from INSERT_SUBREG operand #1");
1917 const CodeGenRegisterClass *SuperClass = inferSuperRegisterClassForNode(
1918 Ty: Dst.getExtType(ResNo: 0), SuperRegNode: Dst.getChild(N: 0), SubRegIdxNode: Dst.getChild(N: 2));
1919 if (!SuperClass)
1920 return failedImport(
1921 Reason: "Cannot infer register class for INSERT_SUBREG operand #0");
1922 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1923 args: *SuperClass);
1924 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 1,
1925 args: *SuperClass);
1926 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 2,
1927 args: *SubClass);
1928 } else if (DstIName == "SUBREG_TO_REG") {
1929 // We need to constrain the destination and subregister source.
1930 // Attempt to infer the subregister source from the first child. If it has
1931 // an explicitly given register class, we'll use that. Otherwise, we will
1932 // fail.
1933 const CodeGenRegisterClass *SubClass =
1934 inferRegClassFromPattern(N: Dst.getChild(N: 0));
1935 if (!SubClass)
1936 return failedImport(
1937 Reason: "Cannot infer register class from SUBREG_TO_REG child #0");
1938 // We don't have a child to look at that might have a super register node.
1939 const CodeGenRegisterClass *SuperClass =
1940 inferSuperRegisterClass(Ty: Dst.getExtType(ResNo: 0), SubRegIdxNode: Dst.getChild(N: 1));
1941 if (!SuperClass)
1942 return failedImport(
1943 Reason: "Cannot infer register class for SUBREG_TO_REG operand #0");
1944 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1945 args: *SuperClass);
1946 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 1,
1947 args: *SubClass);
1948 } else if (DstIName == "REG_SEQUENCE") {
1949 const CodeGenRegisterClass *SuperClass =
1950 inferRegClassFromPattern(N: Dst.getChild(N: 0));
1951
1952 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1953 args: *SuperClass);
1954
1955 unsigned Num = Dst.getNumChildren();
1956 for (unsigned I = 1; I != Num; I += 2) {
1957 const TreePatternNode &SubRegChild = Dst.getChild(N: I + 1);
1958
1959 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: SubRegChild);
1960 if (!SubIdx)
1961 return failedImport(Reason: "REG_SEQUENCE child is not a subreg index");
1962
1963 const auto SrcRCDstRCPair =
1964 SuperClass->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1965 if (!SrcRCDstRCPair)
1966 return failedImport(Reason: "REG_SEQUENCE subreg index is incompatible "
1967 "with inferred reg class");
1968
1969 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args&: I,
1970 args&: *SrcRCDstRCPair->second);
1971 }
1972 } else {
1973 M.insertAction<ConstrainOperandsToDefinitionAction>(InsertPt, args&: InsnID);
1974 }
1975
1976 return Error::success();
1977}
1978
1979const CodeGenRegisterClass *
1980GlobalISelEmitter::getRegClassFromLeaf(const TreePatternNode &Leaf) const {
1981 assert(Leaf.isLeaf() && "Expected leaf?");
1982 const Record *RCRec = Target.getInitValueAsRegClass(V: Leaf.getLeafValue());
1983 if (!RCRec)
1984 return nullptr;
1985 return CGRegs.getRegClass(RCRec);
1986}
1987
1988const CodeGenRegisterClass *
1989GlobalISelEmitter::inferRegClassFromPattern(const TreePatternNode &N) const {
1990 if (N.isLeaf())
1991 return getRegClassFromLeaf(Leaf: N);
1992
1993 // We don't have a leaf node, so we have to try and infer something. Check
1994 // that we have an instruction that we can infer something from.
1995
1996 // Only handle things that produce at least one value (if multiple values,
1997 // just take the first one).
1998 if (N.getNumTypes() < 1)
1999 return nullptr;
2000 const Record *OpRec = N.getOperator();
2001
2002 // We only want instructions.
2003 if (!OpRec->isSubClassOf(Name: "Instruction"))
2004 return nullptr;
2005
2006 // Don't want to try and infer things when there could potentially be more
2007 // than one candidate register class.
2008 return inferRegClassFromInstructionPattern(N, /*ResIdx=*/0);
2009}
2010
2011const CodeGenRegisterClass *
2012GlobalISelEmitter::inferRegClassFromRegisterClassLike(
2013 const Record *RegClassDef) const {
2014 if (RegClassDef->isSubClassOf(Name: "RegClassByHwMode")) {
2015 // TODO: We only are trying to match the regbank, which we assume is the
2016 // same across modes, but we are just picking one class and assuming they
2017 // all have the same bank.
2018 //
2019 // We should verify there is a common regbank among the possible classes,
2020 // and return that instead of a concrete class.
2021 const HwModeSelect &ModeSelect =
2022 Target.getHwModes().getHwModeSelect(R: RegClassDef);
2023 if (ModeSelect.Items.empty())
2024 return nullptr;
2025 return Target.getRegBank().getRegClass(ModeSelect.Items.front().second);
2026 }
2027
2028 return &Target.getRegisterClass(R: RegClassDef);
2029}
2030
2031const CodeGenRegisterClass *
2032GlobalISelEmitter::inferRegClassFromInstructionPattern(const TreePatternNode &N,
2033 unsigned ResIdx) const {
2034 const CodeGenInstruction &Inst = Target.getInstruction(InstRec: N.getOperator());
2035 assert(ResIdx < Inst.Operands.NumDefs &&
2036 "Can only infer register class for explicit defs");
2037
2038 // Handle any special-case instructions which we can safely infer register
2039 // classes from.
2040 StringRef InstName = Inst.getName();
2041 if (InstName == "REG_SEQUENCE") {
2042 // (outs $super_dst), (ins $dst_regclass, variable_ops)
2043 // Destination register class is explicitly specified by the first operand.
2044 const TreePatternNode &RCChild = N.getChild(N: 0);
2045 if (!RCChild.isLeaf())
2046 return nullptr;
2047 return getRegClassFromLeaf(Leaf: RCChild);
2048 }
2049
2050 if (InstName == "COPY_TO_REGCLASS") {
2051 // (outs $dst), (ins $src, $dst_regclass)
2052 // Destination register class is explicitly specified by the second operand.
2053 const TreePatternNode &RCChild = N.getChild(N: 1);
2054 if (!RCChild.isLeaf())
2055 return nullptr;
2056 return getRegClassFromLeaf(Leaf: RCChild);
2057 }
2058
2059 if (InstName == "INSERT_SUBREG") {
2060 // (outs $super_dst), (ins $super_src, $sub_src, $sub_idx);
2061 // If we can infer the register class for the first operand, use that.
2062 // Otherwise, find a register class that supports both the specified
2063 // sub-register index and the type of the instruction's result.
2064 const TreePatternNode &Child0 = N.getChild(N: 0);
2065 assert(Child0.getNumTypes() == 1 && "Unexpected number of types!");
2066 return inferSuperRegisterClassForNode(Ty: N.getExtType(ResNo: 0), SuperRegNode: Child0,
2067 SubRegIdxNode: N.getChild(N: 2));
2068 }
2069
2070 if (InstName == "EXTRACT_SUBREG") {
2071 // (outs $sub_dst), (ins $super_src, $sub_idx)
2072 // Find a register class that can be used for a sub-register copy from
2073 // the specified source at the specified sub-register index.
2074 const CodeGenRegisterClass *SuperRC =
2075 inferRegClassFromPattern(N: N.getChild(N: 0));
2076 if (!SuperRC)
2077 return nullptr;
2078
2079 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: N.getChild(N: 1));
2080 if (!SubIdx)
2081 return nullptr;
2082
2083 const auto SubRCAndSubRegRC =
2084 SuperRC->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
2085 if (!SubRCAndSubRegRC)
2086 return nullptr;
2087
2088 return SubRCAndSubRegRC->second;
2089 }
2090
2091 if (InstName == "SUBREG_TO_REG") {
2092 // (outs $super_dst), (ins $sub_src, $sub_idx)
2093 // Find a register class that supports both the specified sub-register
2094 // index and the type of the instruction's result.
2095 return inferSuperRegisterClass(Ty: N.getExtType(ResNo: 0), SubRegIdxNode: N.getChild(N: 1));
2096 }
2097
2098 // Handle destination record types that we can safely infer a register class
2099 // from.
2100 const auto &DstIOperand = Inst.Operands[ResIdx];
2101 const Record *DstIOpRec = DstIOperand.Rec;
2102
2103 if (DstIOpRec->isSubClassOf(Name: "RegisterOperand")) {
2104 const Record *RegClassDef = DstIOpRec->getValueAsDef(FieldName: "RegClass");
2105 return inferRegClassFromRegisterClassLike(RegClassDef);
2106 }
2107
2108 if (DstIOpRec->isSubClassOf(Name: "RegClassByHwMode"))
2109 return inferRegClassFromRegisterClassLike(RegClassDef: DstIOpRec);
2110
2111 if (DstIOpRec->isSubClassOf(Name: "RegisterClass"))
2112 return &Target.getRegisterClass(R: DstIOpRec);
2113
2114 return nullptr;
2115}
2116
2117const CodeGenRegisterClass *GlobalISelEmitter::inferSuperRegisterClass(
2118 const TypeSetByHwMode &Ty, const TreePatternNode &SubRegIdxNode) const {
2119 // We need a ValueTypeByHwMode for getSuperRegForSubReg.
2120 if (!Ty.isValueTypeByHwMode(AllowEmpty: false))
2121 return nullptr;
2122 if (!SubRegIdxNode.isLeaf())
2123 return nullptr;
2124 const DefInit *SubRegInit = dyn_cast<DefInit>(Val: SubRegIdxNode.getLeafValue());
2125 if (!SubRegInit)
2126 return nullptr;
2127 const CodeGenSubRegIndex *SubIdx = CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
2128
2129 // Use the information we found above to find a minimal register class which
2130 // supports the subregister and type we want.
2131 return CGRegs.getSuperRegForSubReg(Ty: Ty.getValueTypeByHwMode(), SubIdx,
2132 /*MustBeAllocatable=*/true);
2133}
2134
2135const CodeGenRegisterClass *GlobalISelEmitter::inferSuperRegisterClassForNode(
2136 const TypeSetByHwMode &Ty, const TreePatternNode &SuperRegNode,
2137 const TreePatternNode &SubRegIdxNode) const {
2138 // Check if we already have a defined register class for the super register
2139 // node. If we do, then we should preserve that rather than inferring anything
2140 // from the subregister index node. We can assume that whoever wrote the
2141 // pattern in the first place made sure that the super register and
2142 // subregister are compatible.
2143 if (const CodeGenRegisterClass *SuperRegisterClass =
2144 inferRegClassFromPattern(N: SuperRegNode))
2145 return SuperRegisterClass;
2146 return inferSuperRegisterClass(Ty, SubRegIdxNode);
2147}
2148
2149const CodeGenSubRegIndex *GlobalISelEmitter::inferSubRegIndexForNode(
2150 const TreePatternNode &SubRegIdxNode) const {
2151 if (!SubRegIdxNode.isLeaf())
2152 return nullptr;
2153
2154 const DefInit *SubRegInit = dyn_cast<DefInit>(Val: SubRegIdxNode.getLeafValue());
2155 if (!SubRegInit)
2156 return nullptr;
2157 return CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
2158}
2159
2160Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
2161 // Keep track of the matchers and actions to emit.
2162 int Score = P.getPatternComplexity(CGP);
2163 RuleMatcher M(P.getSrcRecord()->getLoc());
2164 RuleMatcherScores[M.getRuleID()] = Score;
2165 M.addAction<DebugCommentAction>(args: llvm::to_string(Value: P.getSrcPattern()) +
2166 " => " +
2167 llvm::to_string(Value: P.getDstPattern()));
2168
2169 SmallVector<const Record *, 4> Predicates;
2170 P.getPredicateRecords(PredicateRecs&: Predicates);
2171 if (auto Error = importRulePredicates(M, Predicates))
2172 return std::move(Error);
2173
2174 if (!P.getHwModeFeatures().empty())
2175 M.addHwModeIdx(Idx: declareHwModeCheck(HwModeFeatures: P.getHwModeFeatures()));
2176
2177 // Next, analyze the pattern operators.
2178 TreePatternNode &Src = P.getSrcPattern();
2179 TreePatternNode &Dst = P.getDstPattern();
2180
2181 // If the root of either pattern isn't a simple operator, ignore it.
2182 if (auto Err = isTrivialOperatorNode(N: Dst))
2183 return failedImport(Reason: "Dst pattern root isn't a trivial operator (" +
2184 toString(E: std::move(Err)) + ")");
2185 if (auto Err = isTrivialOperatorNode(N: Src))
2186 return failedImport(Reason: "Src pattern root isn't a trivial operator (" +
2187 toString(E: std::move(Err)) + ")");
2188
2189 // The different predicates and matchers created during
2190 // addInstructionMatcher use the RuleMatcher M to set up their
2191 // instruction ID (InsnVarID) that are going to be used when
2192 // M is going to be emitted.
2193 // However, the code doing the emission still relies on the IDs
2194 // returned during that process by the RuleMatcher when issuing
2195 // the recordInsn opcodes.
2196 // Because of that:
2197 // 1. The order in which we created the predicates
2198 // and such must be the same as the order in which we emit them,
2199 // and
2200 // 2. We need to reset the generation of the IDs in M somewhere between
2201 // addInstructionMatcher and emit
2202 //
2203 // FIXME: Long term, we don't want to have to rely on this implicit
2204 // naming being the same. One possible solution would be to have
2205 // explicit operator for operation capture and reference those.
2206 // The plus side is that it would expose opportunities to share
2207 // the capture accross rules. The downside is that it would
2208 // introduce a dependency between predicates (captures must happen
2209 // before their first use.)
2210 InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(SymbolicName: Src.getName());
2211 unsigned TempOpIdx = 0;
2212
2213 const auto SavedFlags = M.setGISelFlags(P.getSrcRecord());
2214
2215 auto InsnMatcherOrError =
2216 createAndImportSelDAGMatcher(Rule&: M, InsnMatcher&: InsnMatcherTemp, Src, TempOpIdx);
2217 if (auto Error = InsnMatcherOrError.takeError())
2218 return std::move(Error);
2219 InstructionMatcher &InsnMatcher = InsnMatcherOrError.get();
2220
2221 if (Dst.isLeaf()) {
2222 if (const Record *RCDef =
2223 Target.getInitValueAsRegClass(V: Dst.getLeafValue())) {
2224 const CodeGenRegisterClass &RC = Target.getRegisterClass(R: RCDef);
2225
2226 // We need to replace the def and all its uses with the specified
2227 // operand. However, we must also insert COPY's wherever needed.
2228 // For now, emit a copy and let the register allocator clean up.
2229 const CodeGenInstruction &DstI = Target.getInstruction(InstRec: RK.getDef(Name: "COPY"));
2230 const auto &DstIOperand = DstI.Operands[0];
2231
2232 OperandMatcher &OM0 = InsnMatcher.getOperand(OpIdx: 0);
2233 OM0.setSymbolicName(DstIOperand.Name);
2234 M.defineOperand(SymbolicName: OM0.getSymbolicName(), OM&: OM0);
2235 OM0.addPredicate<RegisterBankOperandMatcher>(args: RC);
2236
2237 auto &DstMIBuilder =
2238 M.addAction<BuildMIAction>(args: M.allocateOutputInsnID(), args&: M, args: &DstI);
2239 DstMIBuilder.addRenderer<CopyRenderer>(args&: M, args: DstIOperand.Name);
2240 DstMIBuilder.addRenderer<CopyRenderer>(args&: M, args: Dst.getName());
2241 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 0, args: RC);
2242
2243 // Erase the root.
2244 unsigned RootInsnID = InsnMatcher.getInsnVarID();
2245 if (M.tryEraseInsnID(ID: RootInsnID))
2246 M.addAction<EraseInstAction>(args&: RootInsnID);
2247
2248 // We're done with this pattern! It's eligible for GISel emission; return
2249 // it.
2250 ++NumPatternImported;
2251 return std::move(M);
2252 }
2253
2254 return failedImport(Reason: "Dst pattern root isn't a known leaf");
2255 }
2256
2257 // Start with the defined operands (i.e., the results of the root operator).
2258 const Record *DstOp = Dst.getOperator();
2259 if (!DstOp->isSubClassOf(Name: "Instruction"))
2260 return failedImport(Reason: "Pattern operator isn't an instruction");
2261
2262 const CodeGenInstruction &DstI = Target.getInstruction(InstRec: DstOp);
2263
2264 // Count both implicit and explicit defs in the dst instruction.
2265 // This avoids errors importing patterns that have inherent implicit defs.
2266 unsigned DstExpDefs = DstI.Operands.NumDefs,
2267 DstNumDefs = DstI.ImplicitDefs.size() + DstExpDefs,
2268 SrcNumDefs = Src.getExtTypes().size();
2269 if (DstNumDefs < SrcNumDefs) {
2270 if (DstNumDefs != 0)
2271 return failedImport(Reason: "Src pattern result has more defs than dst MI (" +
2272 to_string(Value: SrcNumDefs) + " def(s) vs " +
2273 to_string(Value: DstNumDefs) + " def(s))");
2274
2275 bool FoundNoUsePred = false;
2276 for (const auto &Pred : InsnMatcher.predicates()) {
2277 if ((FoundNoUsePred = isa<NoUsePredicateMatcher>(Val: Pred.get())))
2278 break;
2279 }
2280 if (!FoundNoUsePred)
2281 return failedImport(Reason: "Src pattern result has " + to_string(Value: SrcNumDefs) +
2282 " def(s) without the HasNoUse predicate set to true "
2283 "but Dst MI has no def");
2284 }
2285
2286 // The root of the match also has constraints on the register bank so that it
2287 // matches the result instruction.
2288 unsigned N = std::min(a: DstExpDefs, b: SrcNumDefs);
2289 for (unsigned I = 0; I < N; ++I) {
2290 const auto &DstIOperand = DstI.Operands[I];
2291
2292 OperandMatcher &OM = InsnMatcher.getOperand(OpIdx: I);
2293 // The operand names declared in the DstI instruction are unrelated to
2294 // those used in pattern's source and destination DAGs, so mangle the
2295 // former to prevent implicitly adding unexpected
2296 // GIM_CheckIsSameOperand predicates by the defineOperand method.
2297 OM.setSymbolicName(getMangledRootDefName(DefOperandName: DstIOperand.Name));
2298 M.defineOperand(SymbolicName: OM.getSymbolicName(), OM);
2299
2300 const CodeGenRegisterClass *RC =
2301 inferRegClassFromInstructionPattern(N: Dst, ResIdx: I);
2302 if (!RC)
2303 return failedImport(Reason: "Could not infer register class for result #" +
2304 Twine(I) + " from pattern " + to_string(Value: Dst));
2305 OM.addPredicate<RegisterBankOperandMatcher>(args: *RC);
2306 }
2307
2308 auto DstMIBuilderOrError =
2309 createAndImportInstructionRenderer(M, InsnMatcher, Dst);
2310 if (auto Error = DstMIBuilderOrError.takeError())
2311 return std::move(Error);
2312 BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get();
2313
2314 // Render the implicit defs.
2315 // These are only added to the root of the result.
2316 if (auto Error = importImplicitDefRenderers(DstMIBuilder, ImplicitDefs: P.getDstRegs()))
2317 return std::move(Error);
2318
2319 DstMIBuilder.chooseInsnToMutate(Rule&: M);
2320
2321 // Constrain the registers to classes. This is normally derived from the
2322 // emitted instruction but a few instructions require special handling.
2323 if (auto Error =
2324 constrainOperands(InsertPt: M.actions_end(), M, InsnID: DstMIBuilder.getInsnID(), Dst))
2325 return std::move(Error);
2326
2327 // Erase the root.
2328 unsigned RootInsnID = InsnMatcher.getInsnVarID();
2329 if (M.tryEraseInsnID(ID: RootInsnID))
2330 M.addAction<EraseInstAction>(args&: RootInsnID);
2331
2332 // We're done with this pattern! It's eligible for GISel emission; return it.
2333 ++NumPatternImported;
2334 return std::move(M);
2335}
2336
2337MatchTable
2338GlobalISelEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules,
2339 bool Optimize, bool WithCoverage) {
2340 if (!Optimize) {
2341 SmallVector<Matcher *> InputRules(make_pointer_range(Range&: Rules));
2342 return ::buildMatchTable(Rules: InputRules, WithCoverage);
2343 }
2344
2345 std::vector<std::unique_ptr<Matcher>> MatcherStorage;
2346 std::vector<Matcher *> OptRules = optimizeRuleset(Rules, MatcherStorage);
2347 return ::buildMatchTable(Rules: OptRules, WithCoverage);
2348}
2349
2350void GlobalISelEmitter::emitAdditionalImpl(raw_ostream &OS) {
2351 OS << "bool " << getClassName()
2352 << "::selectImpl(MachineInstr &I, CodeGenCoverage "
2353 "&CoverageInfo) const {\n"
2354 << " const PredicateBitset AvailableFeatures = "
2355 "getAvailableFeatures();\n"
2356 << " MachineIRBuilder B(I);\n"
2357 << " State.MIs.clear();\n"
2358 << " State.MIs.push_back(&I);\n\n"
2359 << " if (executeMatchTable(*this, State, ExecInfo, B"
2360 << ", getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures"
2361 << ", &CoverageInfo)) {\n"
2362 << " return true;\n"
2363 << " }\n\n"
2364 << " return false;\n"
2365 << "}\n\n";
2366}
2367
2368void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) {
2369 std::vector<const Record *> MatchedRecords;
2370 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2371 P: [](const Record *R) {
2372 return !R->getValueAsString(FieldName: "GISelPredicateCode").empty();
2373 });
2374 emitMIPredicateFnsImpl<const Record *>(
2375 OS,
2376 AdditionalDecls: " const MachineFunction &MF = *MI.getParent()->getParent();\n"
2377 " const MachineRegisterInfo &MRI = MF.getRegInfo();\n"
2378 " const auto &Operands = State.RecordedOperands;\n"
2379 " (void)Operands;\n"
2380 " (void)MRI;",
2381 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2382 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "GISelPredicateCode"); },
2383 Comment: "PatFrag predicates.");
2384}
2385
2386void GlobalISelEmitter::emitLeafPredicateFns(raw_ostream &OS) {
2387 std::vector<const Record *> MatchedRecords;
2388 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2389 P: [](const Record *R) {
2390 return (!R->getValueAsOptionalString(FieldName: "GISelLeafPredicateCode")
2391 .value_or(u: std::string())
2392 .empty());
2393 });
2394 emitLeafPredicateFnsImpl<const Record *>(
2395 OS,
2396 AdditionalDecls: " const auto &Operands = State.RecordedOperands;\n"
2397 " Register Reg = MO.getReg();\n"
2398 " (void)Operands;\n"
2399 " (void)Reg;",
2400 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2401 GetPredCode: [](const Record *R) {
2402 return R->getValueAsString(FieldName: "GISelLeafPredicateCode");
2403 },
2404 Comment: "PatFrag predicates.");
2405}
2406
2407void GlobalISelEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
2408 std::vector<const Record *> MatchedRecords;
2409 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2410 P: [](const Record *R) {
2411 bool Unset;
2412 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2413 !R->getValueAsBitOrUnset(FieldName: "IsAPFloat", Unset) &&
2414 !R->getValueAsBit(FieldName: "IsAPInt");
2415 });
2416 emitImmPredicateFnsImpl<const Record *>(
2417 OS, TypeIdentifier: "I64", ArgType: "int64_t", Predicates: ArrayRef<const Record *>(MatchedRecords),
2418 GetPredEnumName: &getPatFragPredicateEnumName,
2419 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2420 Comment: "PatFrag predicates.");
2421}
2422
2423void GlobalISelEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) {
2424 std::vector<const Record *> MatchedRecords;
2425 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2426 P: [](const Record *R) {
2427 bool Unset;
2428 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2429 R->getValueAsBitOrUnset(FieldName: "IsAPFloat", Unset);
2430 });
2431 emitImmPredicateFnsImpl<const Record *>(
2432 OS, TypeIdentifier: "APFloat", ArgType: "const APFloat &",
2433 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2434 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2435 Comment: "PatFrag predicates.");
2436}
2437
2438void GlobalISelEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) {
2439 std::vector<const Record *> MatchedRecords;
2440 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2441 P: [](const Record *R) {
2442 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2443 R->getValueAsBit(FieldName: "IsAPInt");
2444 });
2445 emitImmPredicateFnsImpl<const Record *>(
2446 OS, TypeIdentifier: "APInt", ArgType: "const APInt &", Predicates: ArrayRef<const Record *>(MatchedRecords),
2447 GetPredEnumName: &getPatFragPredicateEnumName,
2448 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2449 Comment: "PatFrag predicates.");
2450}
2451
2452void GlobalISelEmitter::emitTestSimplePredicate(raw_ostream &OS) {
2453 OS << "bool " << getClassName() << "::testSimplePredicate(unsigned) const {\n"
2454 << " llvm_unreachable(\"" + getClassName() +
2455 " does not support simple predicates!\");\n"
2456 << " return false;\n"
2457 << "}\n";
2458}
2459
2460void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) {
2461 OS << "bool " << getClassName()
2462 << "::runCustomAction(unsigned, const MatcherState&, NewMIVector &) const "
2463 "{\n"
2464 << " llvm_unreachable(\"" + getClassName() +
2465 " does not support custom C++ actions!\");\n"
2466 << "}\n";
2467}
2468
2469bool hasBFloatType(const TreePatternNode &Node) {
2470 for (unsigned I = 0, E = Node.getNumTypes(); I < E; I++) {
2471 MVT VT = Node.getSimpleType(ResNo: I);
2472 if (VT.getScalarType() == MVT::bf16)
2473 return true;
2474 }
2475 for (const TreePatternNode &C : Node.children())
2476 if (hasBFloatType(Node: C))
2477 return true;
2478 return false;
2479}
2480
2481void GlobalISelEmitter::run(raw_ostream &OS) {
2482 if (!UseCoverageFile.empty()) {
2483 RuleCoverage = CodeGenCoverage();
2484 auto RuleCoverageBufOrErr = MemoryBuffer::getFile(Filename: UseCoverageFile);
2485 if (!RuleCoverageBufOrErr) {
2486 PrintWarning(WarningLoc: SMLoc(), Msg: "Missing rule coverage data");
2487 RuleCoverage = std::nullopt;
2488 } else {
2489 if (!RuleCoverage->parse(Buffer&: *RuleCoverageBufOrErr.get(), BackendName: Target.getName())) {
2490 PrintWarning(WarningLoc: SMLoc(), Msg: "Ignoring invalid or missing rule coverage data");
2491 RuleCoverage = std::nullopt;
2492 }
2493 }
2494 }
2495
2496 // Track the run-time opcode values
2497 gatherOpcodeValues();
2498 // Track the run-time LLT ID values
2499 gatherTypeIDValues();
2500
2501 // Track the GINodeEquiv definitions.
2502 gatherNodeEquivs();
2503
2504 AllPatFrags = RK.getAllDerivedDefinitions(ClassName: "PatFrags");
2505
2506 emitSourceFileHeader(
2507 Desc: ("Global Instruction Selector for the " + Target.getName() + " target")
2508 .str(),
2509 OS);
2510 std::vector<RuleMatcher> Rules;
2511 // Look through the SelectionDAG patterns we found, possibly emitting some.
2512 for (const PatternToMatch &Pat : CGP.ptms()) {
2513 ++NumPatternTotal;
2514
2515 if (Pat.getGISelShouldIgnore())
2516 continue; // skip without warning
2517
2518 // Skip any patterns containing BF16 types, as GISel cannot currently tell
2519 // the difference between fp16 and bf16. FIXME: This can be removed once
2520 // UseExtended are universally supported.
2521 bool UseExtended = LLT::getUseExtended();
2522 if (hasBFloatType(Node: Pat.getSrcPattern()) && !UseExtended)
2523 continue;
2524
2525 auto MatcherOrErr = runOnPattern(P: Pat);
2526
2527 // The pattern analysis can fail, indicating an unsupported pattern.
2528 // Report that if we've been asked to do so.
2529 if (auto Err = MatcherOrErr.takeError()) {
2530 if (WarnOnSkippedPatterns) {
2531 PrintWarning(WarningLoc: Pat.getSrcRecord()->getLoc(),
2532 Msg: "Skipped pattern: " + toString(E: std::move(Err)));
2533 } else {
2534 consumeError(Err: std::move(Err));
2535 }
2536 ++NumPatternImportsSkipped;
2537 continue;
2538 }
2539
2540 if (RuleCoverage) {
2541 if (RuleCoverage->isCovered(RuleID: MatcherOrErr->getRuleID()))
2542 ++NumPatternsTested;
2543 else
2544 PrintWarning(WarningLoc: Pat.getSrcRecord()->getLoc(),
2545 Msg: "Pattern is not covered by a test");
2546 }
2547 Rules.push_back(x: std::move(MatcherOrErr.get()));
2548 }
2549
2550 // Comparison function to order records by name.
2551 auto OrderByName = [](const Record *A, const Record *B) {
2552 return A->getName() < B->getName();
2553 };
2554
2555 std::vector<const Record *> ComplexPredicates =
2556 RK.getAllDerivedDefinitions(ClassName: "GIComplexOperandMatcher");
2557 llvm::sort(C&: ComplexPredicates, Comp: OrderByName);
2558
2559 std::vector<StringRef> CustomRendererFns;
2560 transform(Range: RK.getAllDerivedDefinitions(ClassName: "GICustomOperandRenderer"),
2561 d_first: std::back_inserter(x&: CustomRendererFns), F: [](const auto &Record) {
2562 return Record->getValueAsString("RendererFn");
2563 });
2564 // Sort and remove duplicates to get a list of unique renderer functions, in
2565 // case some were mentioned more than once.
2566 llvm::sort(C&: CustomRendererFns);
2567 CustomRendererFns.erase(first: llvm::unique(R&: CustomRendererFns),
2568 last: CustomRendererFns.end());
2569
2570 // Create a table containing the LLT objects needed by the matcher and an enum
2571 // for the matcher to reference them with.
2572 std::vector<LLTCodeGen> TypeObjects;
2573 append_range(C&: TypeObjects, R&: KnownTypes);
2574 llvm::sort(C&: TypeObjects);
2575
2576 // Sort rules.
2577 llvm::stable_sort(Range&: Rules, C: [&](const RuleMatcher &A, const RuleMatcher &B) {
2578 int ScoreA = RuleMatcherScores[A.getRuleID()];
2579 int ScoreB = RuleMatcherScores[B.getRuleID()];
2580 if (ScoreA > ScoreB)
2581 return true;
2582 if (ScoreB > ScoreA)
2583 return false;
2584 if (A.isHigherPriorityThan(B)) {
2585 assert(!B.isHigherPriorityThan(A) && "Cannot be more important "
2586 "and less important at "
2587 "the same time");
2588 return true;
2589 }
2590 return false;
2591 });
2592
2593 unsigned MaxTemporaries = 0;
2594 for (const auto &Rule : Rules)
2595 MaxTemporaries = std::max(a: MaxTemporaries, b: Rule.countRendererFns());
2596
2597 // Build match table
2598 const MatchTable Table =
2599 buildMatchTable(Rules, Optimize: OptimizeMatchTable, WithCoverage: GenerateCoverage);
2600
2601 emitPredicateBitset(OS, IfDefName: "GET_GLOBALISEL_PREDICATE_BITSET");
2602 emitTemporariesDecl(OS, IfDefName: "GET_GLOBALISEL_TEMPORARIES_DECL");
2603 emitTemporariesInit(OS, MaxTemporaries, IfDefName: "GET_GLOBALISEL_TEMPORARIES_INIT");
2604 emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexOperandMatchers: ComplexPredicates,
2605 CustomOperandRenderers: CustomRendererFns, IfDefName: "GET_GLOBALISEL_IMPL");
2606 emitPredicatesDecl(OS, IfDefName: "GET_GLOBALISEL_PREDICATES_DECL");
2607 emitPredicatesInit(OS, IfDefName: "GET_GLOBALISEL_PREDICATES_INIT");
2608}
2609
2610void GlobalISelEmitter::declareSubtargetFeature(const Record *Predicate) {
2611 SubtargetFeatures.try_emplace(k: Predicate, args&: Predicate, args: SubtargetFeatures.size());
2612}
2613
2614unsigned GlobalISelEmitter::declareHwModeCheck(StringRef HwModeFeatures) {
2615 return HwModes.emplace(args: HwModeFeatures.str(), args: HwModes.size()).first->second;
2616}
2617
2618//===----------------------------------------------------------------------===//
2619
2620static TableGen::Emitter::OptClass<GlobalISelEmitter>
2621 X("gen-global-isel", "Generate GlobalISel selector");
2622