1
2//===- GlobalISelEmitter.cpp - Generate an instruction selector -----------===//
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// This tablegen backend emits code for use by the GlobalISel instruction
12/// selector. See include/llvm/Target/GlobalISel/Target.td.
13///
14/// This file analyzes the patterns recognized by the SelectionDAGISel tablegen
15/// backend, filters out the ones that are unsupported, maps
16/// SelectionDAG-specific constructs to their GlobalISel counterpart
17/// (when applicable: MVT to LLT; SDNode to generic Instruction).
18///
19/// Not all patterns are supported: pass the tablegen invocation
20/// "-warn-on-skipped-patterns" to emit a warning when a pattern is skipped,
21/// as well as why.
22///
23/// The generated file defines a single method:
24/// bool <Target>InstructionSelector::selectImpl(MachineInstr &I) const;
25/// intended to be used in InstructionSelector::select as the first-step
26/// selector for the patterns that don't require complex C++.
27///
28/// FIXME: We'll probably want to eventually define a base
29/// "TargetGenInstructionSelector" class.
30///
31//===----------------------------------------------------------------------===//
32
33#include "Basic/CodeGenIntrinsics.h"
34#include "Common/CodeGenDAGPatterns.h"
35#include "Common/CodeGenInstruction.h"
36#include "Common/CodeGenRegisters.h"
37#include "Common/CodeGenTarget.h"
38#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
39#include "Common/GlobalISel/MatchTable/Matchers.h"
40#include "Common/InfoByHwMode.h"
41#include "llvm/ADT/Statistic.h"
42#include "llvm/CodeGenTypes/LowLevelType.h"
43#include "llvm/CodeGenTypes/MachineValueType.h"
44#include "llvm/Support/CodeGenCoverage.h"
45#include "llvm/Support/CommandLine.h"
46#include "llvm/Support/Error.h"
47#include "llvm/Support/ScopedPrinter.h"
48#include "llvm/TableGen/Error.h"
49#include "llvm/TableGen/Record.h"
50#include "llvm/TableGen/TableGenBackend.h"
51#include <string>
52
53using namespace llvm;
54using namespace llvm::gi;
55
56using action_iterator = RuleMatcher::action_iterator;
57
58#define DEBUG_TYPE "gisel-emitter"
59
60STATISTIC(NumPatternTotal, "Total number of patterns");
61STATISTIC(NumPatternImported, "Number of patterns imported from SelectionDAG");
62STATISTIC(NumPatternImportsSkipped, "Number of SelectionDAG imports skipped");
63STATISTIC(NumPatternsTested,
64 "Number of patterns executed according to coverage information");
65
66static cl::OptionCategory GlobalISelEmitterCat("Options for -gen-global-isel");
67
68static cl::opt<bool> WarnOnSkippedPatterns(
69 "warn-on-skipped-patterns",
70 cl::desc("Explain why a pattern was skipped for inclusion "
71 "in the GlobalISel selector"),
72 cl::init(Val: false), cl::cat(GlobalISelEmitterCat));
73
74static cl::opt<bool> GenerateCoverage(
75 "instrument-gisel-coverage",
76 cl::desc("Generate coverage instrumentation for GlobalISel"),
77 cl::init(Val: false), cl::cat(GlobalISelEmitterCat));
78
79static cl::opt<std::string> UseCoverageFile(
80 "gisel-coverage-file", cl::init(Val: ""),
81 cl::desc("Specify file to retrieve coverage information from"),
82 cl::cat(GlobalISelEmitterCat));
83
84static cl::opt<bool> OptimizeMatchTable(
85 "optimize-match-table",
86 cl::desc("Generate an optimized version of the match table"),
87 cl::init(Val: true), cl::cat(GlobalISelEmitterCat));
88
89static std::string explainPredicates(const TreePatternNode &N) {
90 std::string Explanation;
91 StringRef Separator = "";
92 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
93 const TreePredicateFn &P = Call.Fn;
94 Explanation +=
95 (Separator + P.getOrigPatFragRecord()->getRecord()->getName()).str();
96 Separator = ", ";
97
98 if (P.isAlwaysTrue())
99 Explanation += " always-true";
100 if (P.isImmediatePattern())
101 Explanation += " immediate";
102
103 if (P.isUnindexed())
104 Explanation += " unindexed";
105
106 if (P.isNonExtLoad())
107 Explanation += " non-extload";
108 if (P.isAnyExtLoad())
109 Explanation += " extload";
110 if (P.isSignExtLoad())
111 Explanation += " sextload";
112 if (P.isZeroExtLoad())
113 Explanation += " zextload";
114
115 if (P.isNonTruncStore())
116 Explanation += " non-truncstore";
117 if (P.isTruncStore())
118 Explanation += " truncstore";
119
120 if (const Record *VT = P.getMemoryVT())
121 Explanation += (" MemVT=" + VT->getName()).str();
122 if (const Record *VT = P.getScalarMemoryVT())
123 Explanation += (" ScalarVT(MemVT)=" + VT->getName()).str();
124
125 if (const ListInit *AddrSpaces = P.getAddressSpaces()) {
126 raw_string_ostream OS(Explanation);
127 OS << " AddressSpaces=[";
128
129 StringRef AddrSpaceSeparator;
130 for (const Init *Val : AddrSpaces->getElements()) {
131 const IntInit *IntVal = dyn_cast<IntInit>(Val);
132 if (!IntVal)
133 continue;
134
135 OS << AddrSpaceSeparator << IntVal->getValue();
136 AddrSpaceSeparator = ", ";
137 }
138
139 OS << ']';
140 }
141
142 int64_t MinAlign = P.getMinAlignment();
143 if (MinAlign > 0)
144 Explanation += " MinAlign=" + utostr(X: MinAlign);
145
146 if (P.isAtomicOrderingMonotonic())
147 Explanation += " monotonic";
148 if (P.isAtomicOrderingAcquire())
149 Explanation += " acquire";
150 if (P.isAtomicOrderingRelease())
151 Explanation += " release";
152 if (P.isAtomicOrderingAcquireRelease())
153 Explanation += " acq_rel";
154 if (P.isAtomicOrderingSequentiallyConsistent())
155 Explanation += " seq_cst";
156 if (P.isAtomicOrderingAcquireOrStronger())
157 Explanation += " >=acquire";
158 if (P.isAtomicOrderingWeakerThanAcquire())
159 Explanation += " <acquire";
160 if (P.isAtomicOrderingReleaseOrStronger())
161 Explanation += " >=release";
162 if (P.isAtomicOrderingWeakerThanRelease())
163 Explanation += " <release";
164 }
165 return Explanation;
166}
167
168static std::string explainOperator(const Record *Operator) {
169 if (Operator->isSubClassOf(Name: "SDNode"))
170 return (" (" + Operator->getValueAsString(FieldName: "Opcode") + ")").str();
171
172 if (Operator->isSubClassOf(Name: "Intrinsic"))
173 return (" (Operator is an Intrinsic, " + Operator->getName() + ")").str();
174
175 if (Operator->isSubClassOf(Name: "ComplexPattern"))
176 return (" (Operator is an unmapped ComplexPattern, " + Operator->getName() +
177 ")")
178 .str();
179
180 if (Operator->isSubClassOf(Name: "SDNodeXForm"))
181 return (" (Operator is an unmapped SDNodeXForm, " + Operator->getName() +
182 ")")
183 .str();
184
185 return (" (Operator " + Operator->getName() + " not understood)").str();
186}
187
188/// Helper function to let the emitter report skip reason error messages.
189static Error failedImport(const Twine &Reason) {
190 return make_error<StringError>(Args: Reason, Args: inconvertibleErrorCode());
191}
192
193static Error isTrivialOperatorNode(const TreePatternNode &N) {
194 std::string Explanation;
195 std::string Separator;
196
197 bool HasUnsupportedPredicate = false;
198 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
199 const TreePredicateFn &Predicate = Call.Fn;
200
201 if (Predicate.isAlwaysTrue())
202 continue;
203
204 if (Predicate.isImmediatePattern())
205 continue;
206
207 if (Predicate.hasNoUse() || Predicate.hasOneUse())
208 continue;
209
210 if (Predicate.isNonExtLoad() || Predicate.isAnyExtLoad() ||
211 Predicate.isSignExtLoad() || Predicate.isZeroExtLoad())
212 continue;
213
214 if (Predicate.isNonTruncStore() || Predicate.isTruncStore())
215 continue;
216
217 if (Predicate.isLoad() && Predicate.getMemoryVT())
218 continue;
219
220 if (Predicate.isStore() && Predicate.getMemoryVT())
221 continue;
222
223 if (Predicate.isLoad() || Predicate.isStore()) {
224 if (Predicate.isUnindexed())
225 continue;
226 }
227
228 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
229 const ListInit *AddrSpaces = Predicate.getAddressSpaces();
230 if (AddrSpaces && !AddrSpaces->empty())
231 continue;
232
233 if (Predicate.getMinAlignment() > 0)
234 continue;
235 }
236
237 if (Predicate.isAtomic() && Predicate.getMemoryVT())
238 continue;
239
240 if (Predicate.isAtomic() &&
241 (Predicate.isAtomicOrderingMonotonic() ||
242 Predicate.isAtomicOrderingAcquire() ||
243 Predicate.isAtomicOrderingRelease() ||
244 Predicate.isAtomicOrderingAcquireRelease() ||
245 Predicate.isAtomicOrderingSequentiallyConsistent() ||
246 Predicate.isAtomicOrderingAcquireOrStronger() ||
247 Predicate.isAtomicOrderingWeakerThanAcquire() ||
248 Predicate.isAtomicOrderingReleaseOrStronger() ||
249 Predicate.isAtomicOrderingWeakerThanRelease()))
250 continue;
251
252 if (Predicate.hasGISelPredicateCode())
253 continue;
254
255 HasUnsupportedPredicate = true;
256 Explanation = Separator + "Has a predicate (" + explainPredicates(N) + ")";
257 Separator = ", ";
258 Explanation += (Separator + "first-failing:" +
259 Predicate.getOrigPatFragRecord()->getRecord()->getName())
260 .str();
261 break;
262 }
263
264 if (!HasUnsupportedPredicate)
265 return Error::success();
266
267 return failedImport(Reason: Explanation);
268}
269
270static std::string getScopedName(unsigned Scope, const std::string &Name) {
271 return ("pred:" + Twine(Scope) + ":" + Name).str();
272}
273
274static std::string getMangledRootDefName(StringRef DefOperandName) {
275 return ("DstI[" + DefOperandName + "]").str();
276}
277
278static bool shouldUseGenericTypeForInstruction(const CodeGenInstruction *GI) {
279 if (!GI)
280 return false;
281
282 bool Unset = false;
283 bool ShouldMatchGeneric =
284 GI->TheDef->getValueAsBitOrUnset(FieldName: "GISelMatchGenericTypes", Unset);
285 return Unset ? false : ShouldMatchGeneric;
286}
287
288/// Return true if \p Pred is true for any type in any HwMode of \p VTy.
289static bool anyTypeByHwMode(const TypeSetByHwMode &VTy,
290 function_ref<bool(MVT)> Pred) {
291 return any_of(Range: VTy, P: [&](const auto &P) { return any_of(P.second, Pred); });
292}
293
294/// Return true if \p Pred is true for every type in every HwMode of \p VTy.
295static bool allTypesByHwMode(const TypeSetByHwMode &VTy,
296 function_ref<bool(MVT)> Pred) {
297 return all_of(Range: VTy, P: [&](const auto &P) { return all_of(P.second, Pred); });
298}
299
300//===- GlobalISelEmitter class --------------------------------------------===//
301
302static Expected<LLTOrLLTByHwMode>
303getInstResultType(const TreePatternNode &Dst, const CodeGenTarget &Target) {
304 // While we allow more than one output (both implicit and explicit defs)
305 // below, we only expect one explicit def here.
306 assert(Dst.getOperator()->isSubClassOf("Instruction"));
307 const CodeGenInstruction &InstInfo = Target.getInstruction(InstRec: Dst.getOperator());
308 if (!InstInfo.Operands.NumDefs)
309 return failedImport(Reason: "Dst pattern child needs a def");
310
311 ArrayRef<TypeSetByHwMode> ChildTypes = Dst.getExtTypes();
312 if (ChildTypes.size() < 1)
313 return failedImport(Reason: "Dst pattern child has no result");
314
315 // If there are multiple results, just take the first one (this is how
316 // SelectionDAG does it).
317 std::optional<LLTOrLLTByHwMode> MaybeOpTy;
318 if (ChildTypes.front().isMachineValueType()) {
319 if (auto OpTy = MVTToLLT(VT: ChildTypes.front().getMachineValueType().SimpleTy))
320 MaybeOpTy = *OpTy;
321 } else {
322 MaybeOpTy = getLLTOrLLTByHwMode(VTy: ChildTypes.front());
323 }
324
325 if (!MaybeOpTy)
326 return failedImport(Reason: "Dst operand has an unsupported type");
327 return *MaybeOpTy;
328}
329
330namespace {
331class GlobalISelEmitter final : public GlobalISelMatchTableExecutorEmitter {
332public:
333 explicit GlobalISelEmitter(const RecordKeeper &RK);
334
335 void emitAdditionalImpl(raw_ostream &OS) override;
336
337 void emitMIPredicateFns(raw_ostream &OS) override;
338 void emitLeafPredicateFns(raw_ostream &OS) override;
339 void emitI64ImmPredicateFns(raw_ostream &OS) override;
340 void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
341 void emitAPIntImmPredicateFns(raw_ostream &OS) override;
342 void emitTestSimplePredicate(raw_ostream &OS) override;
343 void emitRunCustomAction(raw_ostream &OS) override;
344
345 const CodeGenTarget &getTarget() const override { return Target; }
346 StringRef getClassName() const override { return ClassName; }
347
348 void run(raw_ostream &OS);
349
350private:
351 std::string ClassName;
352
353 const RecordKeeper &RK;
354 const CodeGenDAGPatterns CGP;
355 const CodeGenTarget &Target;
356 CodeGenRegBank &CGRegs;
357
358 ArrayRef<const Record *> AllPatFrags;
359
360 /// Keep track of the equivalence between SDNodes and Instruction by mapping
361 /// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to
362 /// check for attributes on the relation such as CheckMMOIsNonAtomic.
363 /// This is defined using 'GINodeEquiv' in the target description.
364 DenseMap<const Record *, const Record *> NodeEquivs;
365
366 /// Keep track of the equivalence between ComplexPattern's and
367 /// GIComplexOperandMatcher. Map entries are specified by subclassing
368 /// GIComplexPatternEquiv.
369 DenseMap<const Record *, const Record *> ComplexPatternEquivs;
370
371 /// Keep track of the equivalence between SDNodeXForm's and
372 /// GICustomOperandRenderer. Map entries are specified by subclassing
373 /// GISDNodeXFormEquiv.
374 DenseMap<const Record *, const Record *> SDNodeXFormEquivs;
375
376 /// Keep track of Scores of PatternsToMatch similar to how the DAG does.
377 /// This adds compatibility for RuleMatchers to use this for ordering rules.
378 DenseMap<uint64_t, int> RuleMatcherScores;
379
380 // Rule coverage information.
381 std::optional<CodeGenCoverage> RuleCoverage;
382
383 /// Variables used to help with collecting of named operands for predicates
384 /// with 'let PredicateCodeUsesOperands = 1'. WaitingForNamedOperands is set
385 /// to the number of named operands that predicate expects. Store locations in
386 /// StoreIdxForName correspond to the order in which operand names appear in
387 /// predicate's argument list.
388 /// When we visit named operand and WaitingForNamedOperands is not zero, add
389 /// matcher that will record operand and decrease counter.
390 unsigned WaitingForNamedOperands = 0;
391 StringMap<unsigned> StoreIdxForName;
392
393 void gatherOpcodeValues();
394 void gatherTypeIDValues();
395 void gatherNodeEquivs();
396
397 const Record *findNodeEquiv(const Record *N) const;
398 const CodeGenInstruction *getEquivNode(const Record &Equiv,
399 const TreePatternNode &N) const;
400
401 Error importRulePredicates(RuleMatcher &M,
402 ArrayRef<const Record *> Predicates);
403 Expected<InstructionMatcher &>
404 createAndImportSelDAGMatcher(RuleMatcher &Rule,
405 InstructionMatcher &InsnMatcher,
406 const TreePatternNode &Src, unsigned &TempOpIdx);
407 Error addTypeCheckPredicateForOpcode(OperandMatcher &OM,
408 const TypeSetByHwMode &VTy,
409 bool OperandIsAPointer,
410 const CodeGenInstruction *GI) const;
411 Error importComplexPatternOperandMatcher(OperandMatcher &OM, const Record *R,
412 unsigned &TempOpIdx) const;
413 Error importChildMatcher(RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
414 const TreePatternNode &SrcChild,
415 bool OperandIsAPointer, bool OperandIsImmArg,
416 const CodeGenInstruction *GI, unsigned OpIdx,
417 unsigned &TempOpIdx);
418
419 Expected<BuildMIAction &>
420 createAndImportInstructionRenderer(RuleMatcher &M,
421 InstructionMatcher &InsnMatcher,
422 const TreePatternNode &Dst) const;
423 Expected<action_iterator> createAndImportSubInstructionRenderer(
424 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
425 unsigned TempReg) const;
426 Expected<action_iterator>
427 createInstructionRenderer(action_iterator InsertPt, RuleMatcher &M,
428 const TreePatternNode &Dst) const;
429
430 Expected<action_iterator>
431 importExplicitDefRenderers(action_iterator InsertPt, RuleMatcher &M,
432 BuildMIAction &DstMIBuilder,
433 const TreePatternNode &Dst, bool IsRoot) const;
434
435 Expected<action_iterator>
436 importExplicitUseRenderers(action_iterator InsertPt, RuleMatcher &M,
437 BuildMIAction &DstMIBuilder,
438 const TreePatternNode &Dst) const;
439
440 Error importNamedNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
441 const TreePatternNode &N) const;
442
443 Error importLeafNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
444 const TreePatternNode &N,
445 action_iterator InsertPt) const;
446
447 Error importXFormNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
448 const TreePatternNode &N) const;
449
450 Error importInstructionNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
451 const TreePatternNode &N,
452 action_iterator &InsertPt) const;
453
454 Error importNodeRenderer(RuleMatcher &M, BuildMIAction &MIBuilder,
455 const TreePatternNode &N,
456 action_iterator &InsertPt) const;
457
458 Error importImplicitDefRenderers(BuildMIAction &DstMIBuilder,
459 ArrayRef<const Record *> ImplicitDefs) const;
460
461 /// Analyze pattern \p P, returning a matcher for it if possible.
462 /// Otherwise, return an Error explaining why we don't support it.
463 Expected<RuleMatcher> runOnPattern(const PatternToMatch &P);
464
465 void declareSubtargetFeature(const Record *Predicate);
466
467 unsigned declareHwModeCheck(StringRef HwModeFeatures);
468
469 MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules, bool Optimize,
470 bool WithCoverage);
471
472 /// Infer a CodeGenRegisterClass for the type of \p SuperRegNode. The returned
473 /// CodeGenRegisterClass will support the CodeGenRegisterClass of
474 /// \p SubRegNode, and the subregister index defined by \p SubRegIdxNode.
475 /// If no register class is found, return nullptr.
476 const CodeGenRegisterClass *
477 inferSuperRegisterClassForNode(const TypeSetByHwMode &Ty,
478 const TreePatternNode &SuperRegNode,
479 const TreePatternNode &SubRegIdxNode) const;
480 const CodeGenSubRegIndex *
481 inferSubRegIndexForNode(const TreePatternNode &SubRegIdxNode) const;
482
483 /// Infer a CodeGenRegisterClass which suppoorts \p Ty and \p SubRegIdxNode.
484 /// Return nullptr if no such class exists.
485 const CodeGenRegisterClass *
486 inferSuperRegisterClass(const TypeSetByHwMode &Ty,
487 const TreePatternNode &SubRegIdxNode) const;
488
489 /// Return the CodeGenRegisterClass associated with \p Leaf if it has one.
490 const CodeGenRegisterClass *
491 getRegClassFromLeaf(const TreePatternNode &Leaf) const;
492
493 /// Return a CodeGenRegisterClass for \p N if one can be found. Return
494 /// nullptr otherwise.
495 const CodeGenRegisterClass *
496 inferRegClassFromPattern(const TreePatternNode &N) const;
497
498 const CodeGenRegisterClass *
499 inferRegClassFromRegisterClassLike(const Record *RecClassLike) const;
500
501 const CodeGenRegisterClass *
502 inferRegClassFromInstructionPattern(const TreePatternNode &N,
503 unsigned ResIdx) const;
504
505 Error constrainOperands(action_iterator InsertPt, RuleMatcher &M,
506 unsigned InsnID, const TreePatternNode &Dst) const;
507
508 /// Return the size of the MemoryVT in this predicate, if possible.
509 std::optional<unsigned>
510 getMemSizeBitsFromPredicate(const TreePredicateFn &Predicate);
511
512 // Add builtin predicates.
513 Expected<InstructionMatcher &>
514 addBuiltinPredicates(const Record *SrcGIEquivOrNull,
515 const TreePredicateFn &Predicate,
516 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher);
517};
518} // namespace
519
520static StringRef getPatFragPredicateEnumName(const Record *R) {
521 return R->getName();
522}
523
524void GlobalISelEmitter::gatherOpcodeValues() {
525 InstructionOpcodeMatcher::initOpcodeValuesMap(Target);
526}
527
528void GlobalISelEmitter::gatherTypeIDValues() {
529 LLTOperandMatcher::initTypeIDValuesMap();
530}
531
532void GlobalISelEmitter::gatherNodeEquivs() {
533 assert(NodeEquivs.empty());
534 for (const Record *Equiv : RK.getAllDerivedDefinitions(ClassName: "GINodeEquiv"))
535 NodeEquivs[Equiv->getValueAsDef(FieldName: "Node")] = Equiv;
536
537 assert(ComplexPatternEquivs.empty());
538 for (const Record *Equiv :
539 RK.getAllDerivedDefinitions(ClassName: "GIComplexPatternEquiv")) {
540 const Record *SelDAGEquiv = Equiv->getValueAsDef(FieldName: "SelDAGEquivalent");
541 if (!SelDAGEquiv)
542 continue;
543 ComplexPatternEquivs[SelDAGEquiv] = Equiv;
544 }
545
546 assert(SDNodeXFormEquivs.empty());
547 for (const Record *Equiv :
548 RK.getAllDerivedDefinitions(ClassName: "GISDNodeXFormEquiv")) {
549 const Record *SelDAGEquiv = Equiv->getValueAsDef(FieldName: "SelDAGEquivalent");
550 if (!SelDAGEquiv)
551 continue;
552 SDNodeXFormEquivs[SelDAGEquiv] = Equiv;
553 }
554}
555
556const Record *GlobalISelEmitter::findNodeEquiv(const Record *N) const {
557 return NodeEquivs.lookup(Val: N);
558}
559
560const CodeGenInstruction *
561GlobalISelEmitter::getEquivNode(const Record &Equiv,
562 const TreePatternNode &N) const {
563 if (N.getNumChildren() >= 1) {
564 // setcc operation maps to two different G_* instructions based on the type.
565 if (!Equiv.isValueUnset(FieldName: "IfFloatingPoint")) {
566 auto IsFP = [](MVT VT) { return VT.isFloatingPoint(); };
567 const TypeSetByHwMode &VTy = N.getChild(N: 0).getExtType(ResNo: 0);
568 if (anyTypeByHwMode(VTy, Pred: IsFP)) {
569 // The opcode can't depend on the HwMode.
570 if (!allTypesByHwMode(VTy, Pred: IsFP))
571 return nullptr;
572 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfFloatingPoint"));
573 }
574 }
575 }
576
577 if (!Equiv.isValueUnset(FieldName: "IfConvergent") &&
578 N.getIntrinsicInfo(CDP: CGP)->isConvergent())
579 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfConvergent"));
580
581 bool IsAnyExtLoad = false;
582 bool IsTruncStore = false;
583 for (const TreePredicateCall &Call : N.getPredicateCalls()) {
584 const TreePredicateFn &Predicate = Call.Fn;
585 IsAnyExtLoad |= Predicate.isAnyExtLoad();
586 IsTruncStore |= Predicate.isTruncStore();
587 if (!Equiv.isValueUnset(FieldName: "IfSignExtend") &&
588 (Predicate.isLoad() || Predicate.isAtomic()) &&
589 Predicate.isSignExtLoad())
590 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfSignExtend"));
591
592 if (!Equiv.isValueUnset(FieldName: "IfZeroExtend") &&
593 (Predicate.isLoad() || Predicate.isAtomic()) &&
594 Predicate.isZeroExtLoad())
595 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfZeroExtend"));
596
597 if (!Equiv.isValueUnset(FieldName: "IfFPExtend") &&
598 (Predicate.isLoad() || Predicate.isAtomic()) && IsAnyExtLoad &&
599 Predicate.getMemoryVT() != nullptr &&
600 getValueType(Rec: Predicate.getMemoryVT()).isFloatingPoint())
601 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfFPExtend"));
602
603 if (!Equiv.isValueUnset(FieldName: "IfFPTrunc") &&
604 (Predicate.isStore() || Predicate.isAtomic()) && IsTruncStore &&
605 Predicate.getMemoryVT() != nullptr &&
606 getValueType(Rec: Predicate.getMemoryVT()).isFloatingPoint())
607 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "IfFPTrunc"));
608 }
609
610 return &Target.getInstruction(InstRec: Equiv.getValueAsDef(FieldName: "I"));
611}
612
613GlobalISelEmitter::GlobalISelEmitter(const RecordKeeper &RK)
614 : GlobalISelMatchTableExecutorEmitter(), RK(RK), CGP(RK),
615 Target(CGP.getTargetInfo()), CGRegs(Target.getRegBank()) {
616 ClassName = Target.getName().str() + "InstructionSelector";
617}
618
619//===- Emitter ------------------------------------------------------------===//
620
621Error GlobalISelEmitter::importRulePredicates(
622 RuleMatcher &M, ArrayRef<const Record *> Predicates) {
623 for (const Record *Pred : Predicates) {
624 if (Pred->getValueAsString(FieldName: "CondString").empty())
625 continue;
626 declareSubtargetFeature(Predicate: Pred);
627 M.addRequiredFeature(Feature: Pred);
628 }
629
630 return Error::success();
631}
632
633std::optional<unsigned> GlobalISelEmitter::getMemSizeBitsFromPredicate(
634 const TreePredicateFn &Predicate) {
635 std::optional<LLTCodeGen> MemTyOrNone =
636 MVTToLLT(VT: getValueType(Rec: Predicate.getMemoryVT()));
637
638 if (!MemTyOrNone)
639 return std::nullopt;
640
641 // Align so unusual types like i1 don't get rounded down.
642 return llvm::alignTo(
643 Value: static_cast<unsigned>(MemTyOrNone->get().getSizeInBits()), Align: 8);
644}
645
646Expected<InstructionMatcher &> GlobalISelEmitter::addBuiltinPredicates(
647 const Record *SrcGIEquivOrNull, const TreePredicateFn &Predicate,
648 InstructionMatcher &InsnMatcher, bool &HasAddedMatcher) {
649 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
650 if (const ListInit *AddrSpaces = Predicate.getAddressSpaces()) {
651 SmallVector<unsigned, 4> ParsedAddrSpaces;
652
653 for (const Init *Val : AddrSpaces->getElements()) {
654 const IntInit *IntVal = dyn_cast<IntInit>(Val);
655 if (!IntVal)
656 return failedImport(Reason: "Address space is not an integer");
657 ParsedAddrSpaces.push_back(Elt: IntVal->getValue());
658 }
659
660 if (!ParsedAddrSpaces.empty()) {
661 InsnMatcher.addPredicate<MemoryAddressSpacePredicateMatcher>(
662 args: 0, args&: ParsedAddrSpaces);
663 return InsnMatcher;
664 }
665 }
666
667 int64_t MinAlign = Predicate.getMinAlignment();
668 if (MinAlign > 0) {
669 InsnMatcher.addPredicate<MemoryAlignmentPredicateMatcher>(args: 0, args&: MinAlign);
670 return InsnMatcher;
671 }
672 }
673
674 // G_LOAD is used for both non-extending and any-extending loads.
675 if (Predicate.isLoad() || Predicate.isAtomic()) {
676 if (Predicate.isNonExtLoad()) {
677 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
678 args: 0, args: MemoryVsLLTSizePredicateMatcher::EqualTo, args: 0);
679 return InsnMatcher;
680 }
681 if (Predicate.isAnyExtLoad()) {
682 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
683 args: 0, args: MemoryVsLLTSizePredicateMatcher::LessThan, args: 0);
684 return InsnMatcher;
685 }
686 }
687
688 if (Predicate.isStore() || Predicate.isAtomic()) {
689 if (Predicate.isTruncStore()) {
690 if (Predicate.getMemoryVT() != nullptr) {
691 // FIXME: If MemoryVT is set, we end up with 2 checks for the MMO size.
692 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
693 if (!MemSizeInBits)
694 return failedImport(Reason: "MemVT could not be converted to LLT");
695
696 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(args: 0, args: *MemSizeInBits /
697 8);
698 } else {
699 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
700 args: 0, args: MemoryVsLLTSizePredicateMatcher::LessThan, args: 0);
701 }
702 return InsnMatcher;
703 }
704 if (Predicate.isNonTruncStore()) {
705 // We need to check the sizes match here otherwise we could incorrectly
706 // match truncating stores with non-truncating ones.
707 InsnMatcher.addPredicate<MemoryVsLLTSizePredicateMatcher>(
708 args: 0, args: MemoryVsLLTSizePredicateMatcher::EqualTo, args: 0);
709 }
710 }
711
712 assert(SrcGIEquivOrNull != nullptr && "Invalid SrcGIEquivOrNull value");
713 // No check required. We already did it by swapping the opcode.
714 if (!SrcGIEquivOrNull->isValueUnset(FieldName: "IfSignExtend") &&
715 Predicate.isSignExtLoad())
716 return InsnMatcher;
717
718 // No check required. We already did it by swapping the opcode.
719 if (!SrcGIEquivOrNull->isValueUnset(FieldName: "IfZeroExtend") &&
720 Predicate.isZeroExtLoad())
721 return InsnMatcher;
722
723 // No check required. G_STORE by itself is a non-extending store.
724 if (Predicate.isNonTruncStore())
725 return InsnMatcher;
726
727 if (Predicate.isLoad() || Predicate.isStore() || Predicate.isAtomic()) {
728 if (Predicate.getMemoryVT() != nullptr) {
729 auto MemSizeInBits = getMemSizeBitsFromPredicate(Predicate);
730 if (!MemSizeInBits)
731 return failedImport(Reason: "MemVT could not be converted to LLT");
732
733 InsnMatcher.addPredicate<MemorySizePredicateMatcher>(args: 0,
734 args: *MemSizeInBits / 8);
735 return InsnMatcher;
736 }
737 }
738
739 if (Predicate.isLoad() || Predicate.isStore()) {
740 // No check required. A G_LOAD/G_STORE is an unindexed load.
741 if (Predicate.isUnindexed())
742 return InsnMatcher;
743 }
744
745 if (Predicate.isAtomic()) {
746 if (Predicate.isAtomicOrderingMonotonic()) {
747 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Monotonic");
748 return InsnMatcher;
749 }
750 if (Predicate.isAtomicOrderingAcquire()) {
751 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Acquire");
752 return InsnMatcher;
753 }
754 if (Predicate.isAtomicOrderingRelease()) {
755 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "Release");
756 return InsnMatcher;
757 }
758 if (Predicate.isAtomicOrderingAcquireRelease()) {
759 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
760 args: "AcquireRelease");
761 return InsnMatcher;
762 }
763 if (Predicate.isAtomicOrderingSequentiallyConsistent()) {
764 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
765 args: "SequentiallyConsistent");
766 return InsnMatcher;
767 }
768 }
769
770 if (Predicate.isAtomicOrderingAcquireOrStronger()) {
771 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
772 args: "Acquire", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
773 return InsnMatcher;
774 }
775 if (Predicate.isAtomicOrderingWeakerThanAcquire()) {
776 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
777 args: "Acquire", args: AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
778 return InsnMatcher;
779 }
780
781 if (Predicate.isAtomicOrderingReleaseOrStronger()) {
782 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
783 args: "Release", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
784 return InsnMatcher;
785 }
786 if (Predicate.isAtomicOrderingWeakerThanRelease()) {
787 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
788 args: "Release", args: AtomicOrderingMMOPredicateMatcher::AO_WeakerThan);
789 return InsnMatcher;
790 }
791 HasAddedMatcher = false;
792 return InsnMatcher;
793}
794
795Error GlobalISelEmitter::addTypeCheckPredicateForOpcode(
796 OperandMatcher &OM, const TypeSetByHwMode &VTy, bool OperandIsAPointer,
797 const CodeGenInstruction *GI) const {
798 // Pointer information is separate from the underlying MVT. Avoid losing it by
799 // forcing the regular path.
800 if (OperandIsAPointer || VTy.isPointer())
801 return OM.addTypeCheckPredicate(VTy, OperandIsAPointer);
802
803 if (LLT::getUseExtended() && shouldUseGenericTypeForInstruction(GI)) {
804 if (!VTy.isMachineValueType()) {
805 std::optional<LLTOrLLTByHwMode> GenericTy =
806 getLLTOrLLTByHwMode(VTy, /*Generic=*/true);
807 if (!GenericTy)
808 return failedImport(Reason: "unsupported typeset");
809 if (const auto *LLT = std::get_if<LLTCodeGen>(ptr: &*GenericTy))
810 OM.addPredicate<LLTOperandShapeMatcher>(args: *LLT);
811 else
812 OM.addPredicate<LLTOperandShapeMatcher>(
813 args&: std::get<LLTByHwMode>(v&: *GenericTy));
814 return Error::success();
815 }
816
817 auto GenericLLT = MVTToGenericLLT(VT: VTy.getMachineValueType().SimpleTy);
818 if (GenericLLT) {
819 OM.addPredicate<LLTOperandShapeMatcher>(args&: *GenericLLT);
820 return Error::success();
821 }
822 }
823
824 return OM.addTypeCheckPredicate(VTy, OperandIsAPointer);
825}
826
827Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(
828 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
829 const TreePatternNode &Src, unsigned &TempOpIdx) {
830 const auto SavedFlags = Rule.setGISelFlags(Src.getGISelFlagsRecord());
831
832 const Record *SrcGIEquivOrNull = nullptr;
833 const CodeGenInstruction *SrcGIOrNull = nullptr;
834
835 // Start with the defined operands (i.e., the results of the root operator).
836 if (Src.isLeaf()) {
837 const Init *SrcInit = Src.getLeafValue();
838 if (isa<IntInit>(Val: SrcInit)) {
839 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(
840 args: &Target.getInstruction(InstRec: RK.getDef(Name: "G_CONSTANT")));
841 } else {
842 return failedImport(
843 Reason: "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
844 }
845 } else {
846 SrcGIEquivOrNull = findNodeEquiv(N: Src.getOperator());
847 if (!SrcGIEquivOrNull)
848 return failedImport(Reason: "Pattern operator lacks an equivalent Instruction" +
849 explainOperator(Operator: Src.getOperator()));
850 SrcGIOrNull = getEquivNode(Equiv: *SrcGIEquivOrNull, N: Src);
851 if (!SrcGIOrNull)
852 return failedImport(Reason: "Pattern operator equivalent depends on HwMode");
853
854 // The operators look good: match the opcode
855 InsnMatcher.addPredicate<InstructionOpcodeMatcher>(args&: SrcGIOrNull);
856 }
857
858 // Since there are no opcodes for atomic loads and stores comparing to
859 // SelectionDAG, we add CheckMMOIsNonAtomic predicate immediately after the
860 // opcode predicate to make a logical combination of them.
861 if (SrcGIEquivOrNull &&
862 SrcGIEquivOrNull->getValueAsBit(FieldName: "CheckMMOIsNonAtomic"))
863 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(args: "NotAtomic");
864 else if (SrcGIEquivOrNull &&
865 SrcGIEquivOrNull->getValueAsBit(FieldName: "CheckMMOIsAtomic")) {
866 InsnMatcher.addPredicate<AtomicOrderingMMOPredicateMatcher>(
867 args: "Unordered", args: AtomicOrderingMMOPredicateMatcher::AO_OrStronger);
868 }
869
870 unsigned OpIdx = 0;
871 for (const TypeSetByHwMode &VTy : Src.getExtTypes()) {
872 // Results don't have a name unless they are the root node. The caller will
873 // set the name if appropriate.
874 const bool OperandIsAPointer =
875 SrcGIOrNull && SrcGIOrNull->isOutOperandAPointer(i: OpIdx);
876 OperandMatcher &OM = InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
877 if (auto Error = addTypeCheckPredicateForOpcode(OM, VTy, OperandIsAPointer,
878 GI: SrcGIOrNull))
879 return failedImport(Reason: toString(E: std::move(Error)) +
880 " for result of Src pattern operator");
881 }
882
883 for (const TreePredicateCall &Call : Src.getPredicateCalls()) {
884 const TreePredicateFn &Predicate = Call.Fn;
885 bool HasAddedBuiltinMatcher = true;
886 if (Predicate.isAlwaysTrue())
887 continue;
888
889 if (Predicate.isImmediatePattern()) {
890 InsnMatcher.addPredicate<InstructionImmPredicateMatcher>(args: Predicate);
891 continue;
892 }
893
894 auto InsnMatcherOrError = addBuiltinPredicates(
895 SrcGIEquivOrNull, Predicate, InsnMatcher, HasAddedMatcher&: HasAddedBuiltinMatcher);
896 if (auto Error = InsnMatcherOrError.takeError())
897 return std::move(Error);
898
899 // FIXME: This should be part of addBuiltinPredicates(). If we add this at
900 // the start of addBuiltinPredicates() without returning, then there might
901 // be cases where we hit the last return before which the
902 // HasAddedBuiltinMatcher will be set to false. The predicate could be
903 // missed if we add it in the middle or at the end due to return statements
904 // after the addPredicate<>() calls.
905 if (Predicate.hasNoUse()) {
906 InsnMatcher.addPredicate<NoUsePredicateMatcher>();
907 HasAddedBuiltinMatcher = true;
908 }
909 if (Predicate.hasOneUse()) {
910 InsnMatcher.addPredicate<OneUsePredicateMatcher>();
911 HasAddedBuiltinMatcher = true;
912 }
913
914 if (Predicate.hasGISelPredicateCode()) {
915 if (Predicate.usesOperands()) {
916 assert(WaitingForNamedOperands == 0 &&
917 "previous predicate didn't find all operands or "
918 "nested predicate that uses operands");
919 TreePattern *TP = Predicate.getOrigPatFragRecord();
920 WaitingForNamedOperands = TP->getNumArgs();
921 for (unsigned I = 0; I < WaitingForNamedOperands; ++I)
922 StoreIdxForName[getScopedName(Scope: Call.Scope, Name: TP->getArgName(i: I))] = I;
923 }
924 InsnMatcher.addPredicate<GenericInstructionPredicateMatcher>(args: Predicate);
925 continue;
926 }
927 if (!HasAddedBuiltinMatcher) {
928 return failedImport(Reason: "Src pattern child has predicate (" +
929 explainPredicates(N: Src) + ")");
930 }
931 }
932
933 if (Src.isLeaf()) {
934 const Init *SrcInit = Src.getLeafValue();
935 if (const IntInit *SrcIntInit = dyn_cast<IntInit>(Val: SrcInit)) {
936 OperandMatcher &OM =
937 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: Src.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
938 OM.addPredicate<LiteralIntOperandMatcher>(args: SrcIntInit->getValue());
939 } else {
940 return failedImport(
941 Reason: "Unable to deduce gMIR opcode to handle Src (which is a leaf)");
942 }
943 } else {
944 assert(SrcGIOrNull &&
945 "Expected to have already found an equivalent Instruction");
946 if (SrcGIOrNull->getName() == "G_CONSTANT" ||
947 SrcGIOrNull->getName() == "G_FCONSTANT" ||
948 SrcGIOrNull->getName() == "G_FRAME_INDEX") {
949 // imm/fpimm still have operands but we don't need to do anything with it
950 // here since we don't support ImmLeaf predicates yet. However, we still
951 // need to note the hidden operand to get GIM_CheckNumOperands correct.
952 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
953 return InsnMatcher;
954 }
955
956 if (SrcGIOrNull->getName() == "G_BLOCK_ADDR") {
957 // Name operand of G_BLOCK_ADDR so it can be referenced in the destination
958 // pattern.
959 InsnMatcher.addOperand(OpIdx: OpIdx++, SymbolicName: Src.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
960 return InsnMatcher;
961 }
962
963 // Special case because the operand order is changed from setcc. The
964 // predicate operand needs to be swapped from the last operand to the first
965 // source.
966
967 unsigned NumChildren = Src.getNumChildren();
968 bool IsFCmp = SrcGIOrNull->getName() == "G_FCMP";
969
970 if (IsFCmp || SrcGIOrNull->getName() == "G_ICMP") {
971 const TreePatternNode &SrcChild = Src.getChild(N: NumChildren - 1);
972 if (SrcChild.isLeaf()) {
973 const DefInit *DI = dyn_cast<DefInit>(Val: SrcChild.getLeafValue());
974 const Record *CCDef = DI ? DI->getDef() : nullptr;
975 if (!CCDef || !CCDef->isSubClassOf(Name: "CondCode"))
976 return failedImport(Reason: "Unable to handle CondCode");
977
978 OperandMatcher &OM = InsnMatcher.addOperand(
979 OpIdx: OpIdx++, SymbolicName: SrcChild.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
980 StringRef PredType = IsFCmp ? CCDef->getValueAsString(FieldName: "FCmpPredicate")
981 : CCDef->getValueAsString(FieldName: "ICmpPredicate");
982
983 if (!PredType.empty()) {
984 OM.addPredicate<CmpPredicateOperandMatcher>(args: PredType.str());
985 // Process the other 2 operands normally.
986 --NumChildren;
987 }
988 }
989 }
990
991 // The SDAG fp_to_[su]int_sat nodes carry a trailing ValueType operand
992 // encoding the saturation width. G_FPTOSI_SAT/G_FPTOUI_SAT instead saturate
993 // to their result type and have no such operand, so drop the trailing child
994 // before matching the operands.
995 if (SrcGIOrNull->getName() == "G_FPTOSI_SAT" ||
996 SrcGIOrNull->getName() == "G_FPTOUI_SAT") {
997 const TreePatternNode &SatChild = Src.getChild(N: NumChildren - 1);
998 const DefInit *DI = SatChild.isLeaf()
999 ? dyn_cast<DefInit>(Val: SatChild.getLeafValue())
1000 : nullptr;
1001 if (!DI || !DI->getDef()->isSubClassOf(Name: "ValueType"))
1002 return failedImport(Reason: "Expected ValueType saturation-width operand");
1003 MVT SatVT = llvm::getValueType(Rec: DI->getDef());
1004 if (!allTypesByHwMode(VTy: Src.getExtType(ResNo: 0), Pred: [&](MVT VT) {
1005 return VT.getScalarSizeInBits() == SatVT.getScalarSizeInBits();
1006 }))
1007 return failedImport(Reason: "Saturation width must match the result width");
1008 --NumChildren;
1009 }
1010
1011 // Match the used operands (i.e. the children of the operator).
1012 bool IsIntrinsic =
1013 SrcGIOrNull->getName() == "G_INTRINSIC" ||
1014 SrcGIOrNull->getName() == "G_INTRINSIC_W_SIDE_EFFECTS" ||
1015 SrcGIOrNull->getName() == "G_INTRINSIC_CONVERGENT" ||
1016 SrcGIOrNull->getName() == "G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS";
1017 const CodeGenIntrinsic *II = Src.getIntrinsicInfo(CDP: CGP);
1018 if (IsIntrinsic && !II)
1019 return failedImport(Reason: "Expected IntInit containing intrinsic ID)");
1020
1021 for (unsigned I = 0; I != NumChildren; ++I) {
1022 const TreePatternNode &SrcChild = Src.getChild(N: I);
1023
1024 // We need to determine the meaning of a literal integer based on the
1025 // context. If this is a field required to be an immediate (such as an
1026 // immarg intrinsic argument), the required predicates are different than
1027 // a constant which may be materialized in a register. If we have an
1028 // argument that is required to be an immediate, we should not emit an LLT
1029 // type check, and should not be looking for a G_CONSTANT defined
1030 // register.
1031 bool OperandIsImmArg = SrcGIOrNull->isInOperandImmArg(i: I);
1032
1033 // SelectionDAG allows pointers to be represented with iN since it doesn't
1034 // distinguish between pointers and integers but they are different types
1035 // in GlobalISel. Coerce integers to pointers to address space 0 if the
1036 // context indicates a pointer.
1037 //
1038 bool OperandIsAPointer = SrcGIOrNull->isInOperandAPointer(i: I);
1039
1040 if (IsIntrinsic) {
1041 // For G_INTRINSIC/G_INTRINSIC_W_SIDE_EFFECTS, the operand immediately
1042 // following the defs is an intrinsic ID.
1043 if (I == 0) {
1044 OperandMatcher &OM = InsnMatcher.addOperand(
1045 OpIdx: OpIdx++, SymbolicName: SrcChild.getName().str(), AllocatedTemporariesBaseID: TempOpIdx);
1046 OM.addPredicate<IntrinsicIDOperandMatcher>(args&: II);
1047 continue;
1048 }
1049
1050 // We have to check intrinsics for llvm_anyptr_ty and immarg parameters.
1051 //
1052 // Note that we have to look at the i-1th parameter, because we don't
1053 // have the intrinsic ID in the intrinsic's parameter list.
1054 OperandIsAPointer |= II->isParamAPointer(ParamIdx: I - 1);
1055 OperandIsImmArg |= II->isParamImmArg(ParamIdx: I - 1);
1056 }
1057
1058 if (auto Error = importChildMatcher(Rule, InsnMatcher, SrcChild,
1059 OperandIsAPointer, OperandIsImmArg,
1060 GI: SrcGIOrNull, OpIdx: OpIdx++, TempOpIdx))
1061 return std::move(Error);
1062 }
1063 }
1064
1065 return InsnMatcher;
1066}
1067
1068Error GlobalISelEmitter::importComplexPatternOperandMatcher(
1069 OperandMatcher &OM, const Record *R, unsigned &TempOpIdx) const {
1070 const auto &ComplexPattern = ComplexPatternEquivs.find(Val: R);
1071 if (ComplexPattern == ComplexPatternEquivs.end())
1072 return failedImport(Reason: "SelectionDAG ComplexPattern (" + R->getName() +
1073 ") not mapped to GlobalISel");
1074
1075 OM.addPredicate<ComplexPatternOperandMatcher>(args&: OM, args: *ComplexPattern->second);
1076 TempOpIdx++;
1077 return Error::success();
1078}
1079
1080// Get the name to use for a pattern operand. For an anonymous physical register
1081// input, this should use the register name.
1082static StringRef getSrcChildName(const TreePatternNode &SrcChild,
1083 const Record *&PhysReg) {
1084 StringRef SrcChildName = SrcChild.getName();
1085 if (SrcChildName.empty() && SrcChild.isLeaf()) {
1086 if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
1087 auto *ChildRec = ChildDefInit->getDef();
1088 if (ChildRec->isSubClassOf(Name: "Register")) {
1089 SrcChildName = ChildRec->getName();
1090 PhysReg = ChildRec;
1091 }
1092 }
1093 }
1094
1095 return SrcChildName;
1096}
1097
1098Error GlobalISelEmitter::importChildMatcher(
1099 RuleMatcher &Rule, InstructionMatcher &InsnMatcher,
1100 const TreePatternNode &SrcChild, bool OperandIsAPointer,
1101 bool OperandIsImmArg, const CodeGenInstruction *GI, unsigned OpIdx,
1102 unsigned &TempOpIdx) {
1103
1104 const Record *PhysReg = nullptr;
1105 std::string SrcChildName = getSrcChildName(SrcChild, PhysReg).str();
1106 if (!SrcChild.isLeaf() &&
1107 SrcChild.getOperator()->isSubClassOf(Name: "ComplexPattern")) {
1108 // The "name" of a non-leaf complex pattern (MY_PAT $op1, $op2) is
1109 // "MY_PAT:op1:op2" and the ones with same "name" represent same operand.
1110 std::string PatternName = SrcChild.getOperator()->getName().str();
1111 for (const TreePatternNode &Child : SrcChild.children()) {
1112 PatternName += ":";
1113 PatternName += Child.getName();
1114 }
1115 SrcChildName = PatternName;
1116 }
1117
1118 OperandMatcher &OM =
1119 PhysReg ? InsnMatcher.addPhysRegInput(Reg: PhysReg, OpIdx, TempOpIdx)
1120 : InsnMatcher.addOperand(OpIdx, SymbolicName: SrcChildName, AllocatedTemporariesBaseID: TempOpIdx);
1121 if (OM.isSameAsAnotherOperand())
1122 return Error::success();
1123
1124 ArrayRef<TypeSetByHwMode> ChildTypes = SrcChild.getExtTypes();
1125 if (ChildTypes.size() != 1)
1126 return failedImport(Reason: "Src pattern child has multiple results");
1127
1128 // Check MBB's before the type check since they are not a known type.
1129 if (!SrcChild.isLeaf()) {
1130 if (SrcChild.getOperator()->getName() == "bb") {
1131 OM.addPredicate<MBBOperandMatcher>();
1132 return Error::success();
1133 }
1134 if (SrcChild.getOperator()->getName() == "timm") {
1135 OM.addPredicate<ImmOperandMatcher>();
1136
1137 // Add predicates, if any
1138 for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {
1139 const TreePredicateFn &Predicate = Call.Fn;
1140
1141 // Only handle immediate patterns for now
1142 if (Predicate.isImmediatePattern()) {
1143 OM.addPredicate<OperandImmPredicateMatcher>(args: Predicate);
1144 }
1145 }
1146
1147 return Error::success();
1148 }
1149 } else if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
1150 auto *ChildRec = ChildDefInit->getDef();
1151 if (ChildRec->isSubClassOf(Name: "ValueType") && !SrcChild.hasName()) {
1152 // An unnamed ValueType as in (sext_inreg GPR:$foo, i8). GISel represents
1153 // this as a literal constant with the scalar size.
1154 MVT VT = llvm::getValueType(Rec: ChildRec);
1155 OM.addPredicate<LiteralIntOperandMatcher>(args: VT.getScalarSizeInBits());
1156 return Error::success();
1157 }
1158 }
1159
1160 // Immediate arguments have no meaningful type to check as they don't have
1161 // registers.
1162 if (!OperandIsImmArg) {
1163 if (auto Error = addTypeCheckPredicateForOpcode(OM, VTy: ChildTypes.front(),
1164 OperandIsAPointer, GI))
1165 return failedImport(Reason: toString(E: std::move(Error)) + " for Src operand (" +
1166 to_string(Value: SrcChild) + ")");
1167 }
1168
1169 // Try look up SrcChild for a (named) predicate operand if there is any.
1170 if (WaitingForNamedOperands) {
1171 auto &ScopedNames = SrcChild.getNamesAsPredicateArg();
1172 if (!ScopedNames.empty()) {
1173 auto PA = ScopedNames.begin();
1174 std::string Name = getScopedName(Scope: PA->getScope(), Name: PA->getIdentifier());
1175 OM.addPredicate<RecordNamedOperandMatcher>(args&: StoreIdxForName[Name], args&: Name);
1176 --WaitingForNamedOperands;
1177 }
1178 }
1179
1180 // Check for nested instructions.
1181 if (!SrcChild.isLeaf()) {
1182 if (SrcChild.getOperator()->isSubClassOf(Name: "ComplexPattern")) {
1183 // When a ComplexPattern is used as an operator, it should do the same
1184 // thing as when used as a leaf. However, the children of the operator
1185 // name the sub-operands that make up the complex operand and we must
1186 // prepare to reference them in the renderer too.
1187 unsigned RendererID = TempOpIdx;
1188 if (auto Error = importComplexPatternOperandMatcher(
1189 OM, R: SrcChild.getOperator(), TempOpIdx))
1190 return Error;
1191
1192 for (unsigned I = 0, E = SrcChild.getNumChildren(); I != E; ++I) {
1193 auto &SubOperand = SrcChild.getChild(N: I);
1194 if (!SubOperand.getName().empty()) {
1195 if (auto Error = Rule.defineComplexSubOperand(
1196 SymbolicName: SubOperand.getName(), ComplexPattern: SrcChild.getOperator(), RendererID, SubOperandID: I,
1197 ParentSymbolicName: SrcChildName))
1198 return Error;
1199 }
1200 }
1201
1202 return Error::success();
1203 }
1204
1205 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
1206 args&: InsnMatcher.getRuleMatcher(), args: SrcChild.getName());
1207 if (!MaybeInsnOperand) {
1208 // This isn't strictly true. If the user were to provide exactly the same
1209 // matchers as the original operand then we could allow it. However, it's
1210 // simpler to not permit the redundant specification.
1211 return failedImport(
1212 Reason: "Nested instruction cannot be the same as another operand");
1213 }
1214
1215 // Map the node to a gMIR instruction.
1216 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
1217 auto InsnMatcherOrError = createAndImportSelDAGMatcher(
1218 Rule, InsnMatcher&: InsnOperand.getInsnMatcher(), Src: SrcChild, TempOpIdx);
1219 if (auto Error = InsnMatcherOrError.takeError())
1220 return Error;
1221
1222 return Error::success();
1223 }
1224
1225 if (SrcChild.hasAnyPredicate()) {
1226 for (const TreePredicateCall &Call : SrcChild.getPredicateCalls()) {
1227 const TreePredicateFn &Predicate = Call.Fn;
1228
1229 if (!Predicate.hasGISelLeafPredicateCode())
1230 return failedImport(Reason: "Src pattern child has unsupported predicate");
1231 OM.addPredicate<OperandLeafPredicateMatcher>(args: Predicate);
1232 }
1233 return Error::success();
1234 }
1235
1236 // Check for constant immediates.
1237 if (auto *ChildInt = dyn_cast<IntInit>(Val: SrcChild.getLeafValue())) {
1238 if (OperandIsImmArg) {
1239 // Checks for argument directly in operand list
1240 OM.addPredicate<LiteralIntOperandMatcher>(args: ChildInt->getValue());
1241 } else {
1242 // Checks for materialized constant
1243 OM.addPredicate<ConstantIntOperandMatcher>(args: ChildInt->getValue());
1244 }
1245 return Error::success();
1246 }
1247
1248 // Check for def's like register classes or ComplexPattern's.
1249 if (auto *ChildDefInit = dyn_cast<DefInit>(Val: SrcChild.getLeafValue())) {
1250 auto *ChildRec = ChildDefInit->getDef();
1251
1252 // Check for register classes.
1253 if (ChildRec->isSubClassOf(Name: "RegisterClassLike") ||
1254 ChildRec->isSubClassOf(Name: "RegisterOperand")) {
1255 OM.addPredicate<RegisterBankOperandMatcher>(
1256 args: Target.getRegisterClass(R: Target.getInitValueAsRegClass(
1257 V: ChildDefInit,
1258 /*AssumeRegClassByHwModeIsDefault=*/true)));
1259 return Error::success();
1260 }
1261
1262 if (ChildRec->isSubClassOf(Name: "Register")) {
1263 // This just be emitted as a copy to the specific register.
1264 if (!ChildTypes.front().isValueTypeByHwMode(/*AllowEmpty=*/true))
1265 return failedImport(Reason: "Physical register source has multiple types");
1266 // Find the register class for each HwMode. They must all be the same.
1267 const CodeGenRegisterClass *RC = nullptr;
1268 for (const auto &[Mode, VTs] : ChildTypes.front()) {
1269 if (VTs.empty())
1270 continue;
1271 ValueTypeByHwMode VT(*VTs.begin());
1272 const CodeGenRegisterClass *ModeRC =
1273 CGRegs.getMinimalPhysRegClass(RegRecord: ChildRec, VT: &VT);
1274 if (RC && ModeRC != RC)
1275 return failedImport(
1276 Reason: "Physical register class of pattern source depends on HwMode");
1277 RC = ModeRC;
1278 }
1279 if (!RC) {
1280 return failedImport(
1281 Reason: "Could not determine physical register class of pattern source");
1282 }
1283
1284 OM.addPredicate<RegisterBankOperandMatcher>(args: *RC);
1285 return Error::success();
1286 }
1287
1288 // Check for ValueType.
1289 if (ChildRec->isSubClassOf(Name: "ValueType")) {
1290 // We already added a type check as standard practice so this doesn't need
1291 // to do anything.
1292 return Error::success();
1293 }
1294
1295 // Check for ComplexPattern's.
1296 if (ChildRec->isSubClassOf(Name: "ComplexPattern"))
1297 return importComplexPatternOperandMatcher(OM, R: ChildRec, TempOpIdx);
1298
1299 if (ChildRec->isSubClassOf(Name: "ImmLeaf")) {
1300 return failedImport(
1301 Reason: "Src pattern child def is an unsupported tablegen class (ImmLeaf)");
1302 }
1303
1304 // Place holder for SRCVALUE nodes. Nothing to do here.
1305 if (ChildRec->getName() == "srcvalue")
1306 return Error::success();
1307
1308 const bool ImmAllOnesV = ChildRec->getName() == "immAllOnesV";
1309 if (ImmAllOnesV || ChildRec->getName() == "immAllZerosV") {
1310 auto MaybeInsnOperand = OM.addPredicate<InstructionOperandMatcher>(
1311 args&: InsnMatcher.getRuleMatcher(), args: SrcChild.getName(), args: false);
1312 InstructionOperandMatcher &InsnOperand = **MaybeInsnOperand;
1313
1314 const CodeGenInstruction &BuildVector =
1315 Target.getInstruction(InstRec: RK.getDef(Name: "G_BUILD_VECTOR"));
1316 const CodeGenInstruction &BuildVectorTrunc =
1317 Target.getInstruction(InstRec: RK.getDef(Name: "G_BUILD_VECTOR_TRUNC"));
1318
1319 // Treat G_BUILD_VECTOR as the canonical opcode, and G_BUILD_VECTOR_TRUNC
1320 // as an alternative.
1321 InsnOperand.getInsnMatcher().addPredicate<InstructionOpcodeMatcher>(
1322 args: ArrayRef({&BuildVector, &BuildVectorTrunc}));
1323
1324 // TODO: Handle both G_BUILD_VECTOR and G_BUILD_VECTOR_TRUNC We could
1325 // theoretically not emit any opcode check, but getOpcodeMatcher currently
1326 // has to succeed.
1327 OperandMatcher &OM =
1328 InsnOperand.getInsnMatcher().addOperand(OpIdx: 0, SymbolicName: "", AllocatedTemporariesBaseID: TempOpIdx);
1329 if (auto Error = OM.addTypeCheckPredicate(VTy: ChildTypes.front(),
1330 /*OperandIsAPointer=*/false))
1331 return failedImport(Reason: toString(E: std::move(Error)) +
1332 " for result of Src pattern operator");
1333
1334 InsnOperand.getInsnMatcher().addPredicate<VectorSplatImmPredicateMatcher>(
1335 args: ImmAllOnesV ? VectorSplatImmPredicateMatcher::AllOnes
1336 : VectorSplatImmPredicateMatcher::AllZeros);
1337 return Error::success();
1338 }
1339
1340 return failedImport(
1341 Reason: "Src pattern child def is an unsupported tablegen class");
1342 }
1343
1344 return failedImport(Reason: "Src pattern child is an unsupported kind");
1345}
1346
1347// Equivalent of MatcherGen::EmitResultOfNamedOperand.
1348Error GlobalISelEmitter::importNamedNodeRenderer(
1349 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N) const {
1350 StringRef NodeName = N.getName();
1351
1352 if (auto SubOperand = M.getComplexSubOperand(SymbolicName: NodeName)) {
1353 auto [ComplexPatternRec, RendererID, SubOperandIdx] = *SubOperand;
1354 MIBuilder.addRenderer<RenderComplexPatternOperand>(
1355 args: *ComplexPatternRec, args&: NodeName, args&: RendererID, args&: SubOperandIdx);
1356 return Error::success();
1357 }
1358
1359 if (!N.isLeaf()) {
1360 StringRef OperatorName = N.getOperator()->getName();
1361
1362 if (OperatorName == "imm") {
1363 MIBuilder.addRenderer<CopyConstantAsImmRenderer>(args&: M, args&: NodeName);
1364 return Error::success();
1365 }
1366
1367 if (OperatorName == "fpimm") {
1368 MIBuilder.addRenderer<CopyFConstantAsFPImmRenderer>(args&: M, args&: NodeName);
1369 return Error::success();
1370 }
1371
1372 // TODO: 'imm' and 'fpimm' are the only nodes that need special treatment.
1373 // Remove this check and add CopyRenderer unconditionally for other nodes.
1374 if (OperatorName == "bb" || OperatorName == "timm" ||
1375 OperatorName == "tframeindex") {
1376 MIBuilder.addRenderer<CopyRenderer>(args&: M, args&: NodeName);
1377 return Error::success();
1378 }
1379
1380 return failedImport(Reason: "node has unsupported operator " + to_string(Value: N));
1381 }
1382
1383 if (const auto *DI = dyn_cast<DefInit>(Val: N.getLeafValue())) {
1384 const Record *R = DI->getDef();
1385
1386 if (N.getNumResults() != 1)
1387 return failedImport(Reason: "node does not have one result " + to_string(Value: N));
1388
1389 if (R->isSubClassOf(Name: "ComplexPattern")) {
1390 auto I = ComplexPatternEquivs.find(Val: R);
1391 if (I == ComplexPatternEquivs.end())
1392 return failedImport(Reason: "ComplexPattern " + R->getName() +
1393 " does not have GISel equivalent");
1394
1395 const OperandMatcher &OM = M.getOperandMatcher(Name: NodeName);
1396 MIBuilder.addRenderer<RenderComplexPatternOperand>(
1397 args: *I->second, args&: NodeName, args: OM.getAllocatedTemporariesBaseID());
1398 return Error::success();
1399 }
1400
1401 if (R->isSubClassOf(Name: "RegisterOperand") &&
1402 !R->isValueUnset(FieldName: "GIZeroRegister")) {
1403 MIBuilder.addRenderer<CopyOrAddZeroRegRenderer>(
1404 args&: M, args&: NodeName, args: R->getValueAsDef(FieldName: "GIZeroRegister"));
1405 return Error::success();
1406 }
1407
1408 // TODO: All special cases are handled above. Remove this check and add
1409 // CopyRenderer unconditionally.
1410 if (R->isSubClassOf(Name: "RegisterClassLike") ||
1411 R->isSubClassOf(Name: "RegisterOperand") || R->isSubClassOf(Name: "ValueType")) {
1412 MIBuilder.addRenderer<CopyRenderer>(args&: M, args&: NodeName);
1413 return Error::success();
1414 }
1415 }
1416
1417 // TODO: Change this to assert and move to the beginning of the function.
1418 if (!M.hasOperand(SymbolicName: NodeName))
1419 return failedImport(Reason: "could not find node $" + NodeName +
1420 " in the source DAG");
1421
1422 // TODO: Remove this check and add CopyRenderer unconditionally.
1423 // TODO: Handle nodes with multiple results (provided they can reach here).
1424 if (isa<UnsetInit>(Val: N.getLeafValue())) {
1425 MIBuilder.addRenderer<CopyRenderer>(args&: M, args&: NodeName);
1426 return Error::success();
1427 }
1428
1429 return failedImport(Reason: "unsupported node " + to_string(Value: N));
1430}
1431
1432// Equivalent of MatcherGen::EmitResultLeafAsOperand.
1433Error GlobalISelEmitter::importLeafNodeRenderer(
1434 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N,
1435 action_iterator InsertPt) const {
1436 if (const auto *II = dyn_cast<IntInit>(Val: N.getLeafValue())) {
1437 MIBuilder.addRenderer<ImmRenderer>(args: II->getValue());
1438 return Error::success();
1439 }
1440
1441 if (const auto *DI = dyn_cast<DefInit>(Val: N.getLeafValue())) {
1442 const Record *R = DI->getDef();
1443
1444 if (R->isSubClassOf(Name: "Register") || R->getName() == "zero_reg") {
1445 MIBuilder.addRenderer<AddRegisterRenderer>(args: Target, args&: R);
1446 return Error::success();
1447 }
1448
1449 if (R->getName() == "undef_tied_input") {
1450 std::optional<LLTOrLLTByHwMode> OpTyOrNone =
1451 getLLTOrLLTByHwMode(VTy: N.getExtType(ResNo: 0));
1452 if (!OpTyOrNone)
1453 return failedImport(Reason: "unsupported type");
1454
1455 unsigned TempRegID = M.allocateTempRegID();
1456 M.insertAction<MakeTempRegisterAction>(
1457 InsertPt, args: LLTCodeGenOrTempType(*OpTyOrNone), args&: TempRegID);
1458
1459 auto I = M.insertAction<BuildMIAction>(
1460 InsertPt, args: M.allocateOutputInsnID(), args&: M,
1461 args: &Target.getInstruction(InstRec: RK.getDef(Name: "IMPLICIT_DEF")));
1462 auto &ImpDefBuilder = static_cast<BuildMIAction &>(**I);
1463 ImpDefBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, /*IsDef=*/args: true);
1464
1465 MIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID);
1466 return Error::success();
1467 }
1468
1469 if (R->isSubClassOf(Name: "SubRegIndex")) {
1470 const CodeGenSubRegIndex *SubRegIndex = CGRegs.findSubRegIdx(Def: R);
1471 MIBuilder.addRenderer<ImmRenderer>(args: SubRegIndex->EnumValue);
1472 return Error::success();
1473 }
1474
1475 // There are also RegisterClass / RegisterOperand operands of REG_SEQUENCE /
1476 // COPY_TO_REGCLASS, but these instructions are currently handled elsewhere.
1477 }
1478
1479 return failedImport(Reason: "unrecognized node " + to_string(Value: N));
1480}
1481
1482// Equivalent of MatcherGen::EmitResultSDNodeXFormAsOperand.
1483Error GlobalISelEmitter::importXFormNodeRenderer(
1484 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N) const {
1485 const Record *XFormRec = N.getOperator();
1486 auto I = SDNodeXFormEquivs.find(Val: XFormRec);
1487 if (I == SDNodeXFormEquivs.end())
1488 return failedImport(Reason: "SDNodeXForm " + XFormRec->getName() +
1489 " does not have GISel equivalent");
1490
1491 // TODO: Fail to import if GISDNodeXForm does not have RendererFn.
1492 // This currently results in a fatal error in emitRenderOpcodes.
1493 const Record *XFormEquivRec = I->second;
1494
1495 // The node to apply the transformation function to.
1496 // FIXME: The node may not have a name and may be a leaf. It should be
1497 // rendered first, like any other nodes. This may or may not require
1498 // introducing a temporary register, and we can't tell that without
1499 // inspecting the node (possibly recursively). This is a general drawback
1500 // of appending renderers directly to BuildMIAction.
1501 const TreePatternNode &Node = N.getChild(N: 0);
1502 StringRef NodeName = Node.getName();
1503
1504 const Record *XFormOpc = CGP.getSDNodeTransform(R: XFormRec).first;
1505 if (XFormOpc->getName() == "timm") {
1506 // If this is a TargetConstant, there won't be a corresponding
1507 // instruction to transform. Instead, this will refer directly to an
1508 // operand in an instruction's operand list.
1509 MIBuilder.addRenderer<CustomOperandRenderer>(args&: M, args: *XFormEquivRec, args&: NodeName);
1510 } else {
1511 MIBuilder.addRenderer<CustomRenderer>(args&: M, args: *XFormEquivRec, args&: NodeName);
1512 }
1513
1514 return Error::success();
1515}
1516
1517// Equivalent of MatcherGen::EmitResultInstructionAsOperand.
1518Error GlobalISelEmitter::importInstructionNodeRenderer(
1519 RuleMatcher &M, BuildMIAction &MIBuilder, const TreePatternNode &N,
1520 action_iterator &InsertPt) const {
1521 Expected<LLTOrLLTByHwMode> OpTy = getInstResultType(Dst: N, Target);
1522 if (!OpTy)
1523 return OpTy.takeError();
1524
1525 // TODO: See the comment in importXFormNodeRenderer. We rely on the node
1526 // requiring a temporary register, which prevents us from using this
1527 // function on the root of the destination DAG.
1528 unsigned TempRegID = M.allocateTempRegID();
1529 InsertPt = M.insertAction<MakeTempRegisterAction>(
1530 InsertPt, args: LLTCodeGenOrTempType(*OpTy), args&: TempRegID);
1531 MIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID);
1532
1533 auto InsertPtOrError =
1534 createAndImportSubInstructionRenderer(InsertPt: ++InsertPt, M, Dst: N, TempReg: TempRegID);
1535 if (!InsertPtOrError)
1536 return InsertPtOrError.takeError();
1537
1538 InsertPt = *InsertPtOrError;
1539 return Error::success();
1540}
1541
1542// Equivalent of MatcherGen::EmitResultOperand.
1543Error GlobalISelEmitter::importNodeRenderer(RuleMatcher &M,
1544 BuildMIAction &MIBuilder,
1545 const TreePatternNode &N,
1546 action_iterator &InsertPt) const {
1547 if (N.hasName())
1548 return importNamedNodeRenderer(M, MIBuilder, N);
1549
1550 if (N.isLeaf())
1551 return importLeafNodeRenderer(M, MIBuilder, N, InsertPt);
1552
1553 if (N.getOperator()->isSubClassOf(Name: "SDNodeXForm"))
1554 return importXFormNodeRenderer(M, MIBuilder, N);
1555
1556 if (N.getOperator()->isSubClassOf(Name: "Instruction"))
1557 return importInstructionNodeRenderer(M, MIBuilder, N, InsertPt);
1558
1559 // Should not reach here.
1560 return failedImport(Reason: "unrecognized node " + llvm::to_string(Value: N));
1561}
1562
1563/// Generates code that builds the resulting instruction(s) from the destination
1564/// DAG. Note that to do this we do not and should not need the source DAG.
1565/// We do need to know whether a generated instruction defines a result of the
1566/// source DAG; this information is available via RuleMatcher::hasOperand.
1567Expected<BuildMIAction &> GlobalISelEmitter::createAndImportInstructionRenderer(
1568 RuleMatcher &M, InstructionMatcher &InsnMatcher,
1569 const TreePatternNode &Dst) const {
1570 auto InsertPtOrError = createInstructionRenderer(InsertPt: M.actions_end(), M, Dst);
1571 if (auto Error = InsertPtOrError.takeError())
1572 return std::move(Error);
1573
1574 action_iterator InsertPt = InsertPtOrError.get();
1575 BuildMIAction &DstMIBuilder = *static_cast<BuildMIAction *>(InsertPt->get());
1576
1577 for (auto PhysOp : M.physoperands()) {
1578 InsertPt = M.insertAction<BuildMIAction>(
1579 InsertPt, args: M.allocateOutputInsnID(), args&: M,
1580 args: &Target.getInstruction(InstRec: RK.getDef(Name: "COPY")));
1581 BuildMIAction &CopyToPhysRegMIBuilder =
1582 *static_cast<BuildMIAction *>(InsertPt->get());
1583 CopyToPhysRegMIBuilder.addRenderer<AddRegisterRenderer>(args: Target,
1584 args&: PhysOp.first, args: true);
1585 CopyToPhysRegMIBuilder.addRenderer<CopyPhysRegRenderer>(args&: M, args&: PhysOp.first);
1586 }
1587
1588 if (auto Error = importExplicitDefRenderers(InsertPt, M, DstMIBuilder, Dst,
1589 /*IsRoot=*/true)
1590 .takeError())
1591 return std::move(Error);
1592
1593 if (auto Error = importExplicitUseRenderers(InsertPt, M, DstMIBuilder, Dst)
1594 .takeError())
1595 return std::move(Error);
1596
1597 return DstMIBuilder;
1598}
1599
1600Expected<action_iterator>
1601GlobalISelEmitter::createAndImportSubInstructionRenderer(
1602 action_iterator InsertPt, RuleMatcher &M, const TreePatternNode &Dst,
1603 unsigned TempRegID) const {
1604 auto InsertPtOrError = createInstructionRenderer(InsertPt, M, Dst);
1605
1606 // TODO: Assert there's exactly one result.
1607
1608 if (auto Error = InsertPtOrError.takeError())
1609 return std::move(Error);
1610
1611 BuildMIAction &DstMIBuilder =
1612 *static_cast<BuildMIAction *>(InsertPtOrError.get()->get());
1613
1614 // Assign the result to TempReg.
1615 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, args: true);
1616
1617 // Handle additional (ignored) results.
1618 InsertPtOrError = importExplicitDefRenderers(
1619 InsertPt: std::prev(x: *InsertPtOrError), M, DstMIBuilder, Dst, /*IsRoot=*/false);
1620 if (auto Error = InsertPtOrError.takeError())
1621 return std::move(Error);
1622
1623 InsertPtOrError =
1624 importExplicitUseRenderers(InsertPt: InsertPtOrError.get(), M, DstMIBuilder, Dst);
1625 if (auto Error = InsertPtOrError.takeError())
1626 return std::move(Error);
1627
1628 if (auto Error =
1629 constrainOperands(InsertPt, M, InsnID: DstMIBuilder.getInsnID(), Dst))
1630 return std::move(Error);
1631
1632 return InsertPtOrError.get();
1633}
1634
1635Expected<action_iterator>
1636GlobalISelEmitter::createInstructionRenderer(action_iterator InsertPt,
1637 RuleMatcher &M,
1638 const TreePatternNode &Dst) const {
1639 const Record *DstOp = Dst.getOperator();
1640 if (!DstOp->isSubClassOf(Name: "Instruction")) {
1641 if (DstOp->isSubClassOf(Name: "ValueType"))
1642 return failedImport(
1643 Reason: "Pattern operator isn't an instruction (it's a ValueType)");
1644 return failedImport(Reason: "Pattern operator isn't an instruction");
1645 }
1646 const CodeGenInstruction *DstI = &Target.getInstruction(InstRec: DstOp);
1647
1648 // COPY_TO_REGCLASS is just a copy with a ConstrainOperandToRegClassAction
1649 // attached. Similarly for EXTRACT_SUBREG except that's a subregister copy.
1650 StringRef Name = DstI->getName();
1651 if (Name == "COPY_TO_REGCLASS" || Name == "EXTRACT_SUBREG")
1652 DstI = &Target.getInstruction(InstRec: RK.getDef(Name: "COPY"));
1653
1654 return M.insertAction<BuildMIAction>(InsertPt, args: M.allocateOutputInsnID(), args&: M,
1655 args&: DstI);
1656}
1657
1658Expected<action_iterator> GlobalISelEmitter::importExplicitDefRenderers(
1659 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
1660 const TreePatternNode &Dst, bool IsRoot) const {
1661 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
1662
1663 // Process explicit defs. The caller may have already handled the first def.
1664 for (unsigned I = IsRoot ? 0 : 1, E = DstI->Operands.NumDefs; I != E; ++I) {
1665 const CGIOperandList::OperandInfo &OpInfo = DstI->Operands[I];
1666 std::string OpName = getMangledRootDefName(DefOperandName: OpInfo.Name);
1667
1668 // If the def is used in the source DAG, forward it.
1669 if (IsRoot && M.hasOperand(SymbolicName: OpName)) {
1670 // CopyRenderer saves a StringRef, so cannot pass OpName itself -
1671 // let's use a string with an appropriate lifetime.
1672 StringRef PermanentRef = M.getOperandMatcher(Name: OpName).getSymbolicName();
1673 DstMIBuilder.addRenderer<CopyRenderer>(args&: M, args&: PermanentRef);
1674 continue;
1675 }
1676
1677 // A discarded explicit def may be an optional physical register.
1678 // If this is the case, add the default register and mark it as dead.
1679 if (OpInfo.Rec->isSubClassOf(Name: "OptionalDefOperand")) {
1680 for (const TreePatternNode &DefaultOp :
1681 make_pointee_range(Range: CGP.getDefaultOperand(R: OpInfo.Rec).DefaultOps)) {
1682 // TODO: Do these checks in ParseDefaultOperands.
1683 if (!DefaultOp.isLeaf())
1684 return failedImport(Reason: "optional def is not a leaf");
1685
1686 const auto *RegDI = dyn_cast<DefInit>(Val: DefaultOp.getLeafValue());
1687 if (!RegDI)
1688 return failedImport(Reason: "optional def is not a record");
1689
1690 const Record *Reg = RegDI->getDef();
1691 if (!Reg->isSubClassOf(Name: "Register") && Reg->getName() != "zero_reg")
1692 return failedImport(Reason: "optional def is not a register");
1693
1694 DstMIBuilder.addRenderer<AddRegisterRenderer>(
1695 args: Target, args&: Reg, /*IsDef=*/args: true, /*IsDead=*/args: true);
1696 }
1697 continue;
1698 }
1699
1700 // The def is discarded, create a dead virtual register for it.
1701 const TypeSetByHwMode &ExtTy = Dst.getExtType(ResNo: I);
1702 std::optional<LLTOrLLTByHwMode> OpTy;
1703 if (ExtTy.isMachineValueType()) {
1704 auto SimpleTy = MVTToLLT(VT: ExtTy.getMachineValueType().SimpleTy);
1705 if (!SimpleTy)
1706 return failedImport(Reason: "unsupported type");
1707 OpTy = *SimpleTy;
1708 } else {
1709 OpTy = getLLTOrLLTByHwMode(VTy: ExtTy);
1710 if (!OpTy)
1711 return failedImport(Reason: "unsupported typeset");
1712 }
1713
1714 unsigned TempRegID = M.allocateTempRegID();
1715 InsertPt = M.insertAction<MakeTempRegisterAction>(
1716 InsertPt, args: LLTCodeGenOrTempType(*OpTy), args&: TempRegID);
1717 DstMIBuilder.addRenderer<TempRegRenderer>(
1718 args&: TempRegID, /*IsDef=*/args: true, /*SubReg=*/args: nullptr, /*IsDead=*/args: true);
1719 }
1720
1721 // Implicit defs are not currently supported, mark all of them as dead.
1722 for (const Record *Reg : DstI->ImplicitDefs) {
1723 std::string OpName = getMangledRootDefName(DefOperandName: Reg->getName());
1724 assert(!M.hasOperand(OpName) && "The pattern should've been rejected");
1725 DstMIBuilder.setDeadImplicitDef(Reg);
1726 }
1727
1728 return InsertPt;
1729}
1730
1731Expected<action_iterator> GlobalISelEmitter::importExplicitUseRenderers(
1732 action_iterator InsertPt, RuleMatcher &M, BuildMIAction &DstMIBuilder,
1733 const TreePatternNode &Dst) const {
1734 const CodeGenInstruction *DstI = DstMIBuilder.getCGI();
1735 const CodeGenInstruction *OrigDstI =
1736 &Target.getInstruction(InstRec: Dst.getOperator());
1737
1738 StringRef Name = OrigDstI->getName();
1739 unsigned ExpectedDstINumUses = Dst.getNumChildren();
1740
1741 // EXTRACT_SUBREG needs to use a subregister COPY.
1742 if (Name == "EXTRACT_SUBREG") {
1743 if (!Dst.getChild(N: 1).isLeaf())
1744 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a leaf");
1745 const DefInit *SubRegInit =
1746 dyn_cast<DefInit>(Val: Dst.getChild(N: 1).getLeafValue());
1747 if (!SubRegInit)
1748 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a subreg index");
1749
1750 const CodeGenSubRegIndex *SubIdx =
1751 CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
1752 const TreePatternNode &ValChild = Dst.getChild(N: 0);
1753 if (!ValChild.isLeaf()) {
1754 // We really have to handle the source instruction, and then insert a
1755 // copy from the subregister.
1756 auto ExtractSrcTy = getInstResultType(Dst: ValChild, Target);
1757 if (!ExtractSrcTy)
1758 return ExtractSrcTy.takeError();
1759
1760 unsigned TempRegID = M.allocateTempRegID();
1761 InsertPt = M.insertAction<MakeTempRegisterAction>(
1762 InsertPt, args: LLTCodeGenOrTempType(*ExtractSrcTy), args&: TempRegID);
1763
1764 auto InsertPtOrError = createAndImportSubInstructionRenderer(
1765 InsertPt: ++InsertPt, M, Dst: ValChild, TempRegID);
1766 if (auto Error = InsertPtOrError.takeError())
1767 return std::move(Error);
1768
1769 DstMIBuilder.addRenderer<TempRegRenderer>(args&: TempRegID, args: false, args&: SubIdx);
1770 return InsertPt;
1771 }
1772
1773 // If this is a source operand, this is just a subregister copy.
1774 const Record *RCDef =
1775 Target.getInitValueAsRegClass(V: ValChild.getLeafValue());
1776 if (!RCDef)
1777 return failedImport(Reason: "EXTRACT_SUBREG child #0 could not "
1778 "be coerced to a register class");
1779
1780 CodeGenRegisterClass *RC = CGRegs.getRegClass(RCDef);
1781
1782 const auto SrcRCDstRCPair =
1783 RC->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1784 if (SrcRCDstRCPair) {
1785 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
1786 if (SrcRCDstRCPair->first != RC)
1787 return failedImport(Reason: "EXTRACT_SUBREG requires an additional COPY");
1788 }
1789
1790 StringRef RegOperandName = Dst.getChild(N: 0).getName();
1791 if (const auto &SubOperand = M.getComplexSubOperand(SymbolicName: RegOperandName)) {
1792 DstMIBuilder.addRenderer<RenderComplexPatternOperand>(
1793 args: *std::get<0>(t: *SubOperand), args&: RegOperandName, args: std::get<1>(t: *SubOperand),
1794 args: std::get<2>(t: *SubOperand), args&: SubIdx);
1795 return InsertPt;
1796 }
1797
1798 DstMIBuilder.addRenderer<CopySubRegRenderer>(args&: M, args&: RegOperandName, args&: SubIdx);
1799 return InsertPt;
1800 }
1801
1802 if (Name == "REG_SEQUENCE") {
1803 if (!Dst.getChild(N: 0).isLeaf())
1804 return failedImport(Reason: "REG_SEQUENCE child #0 is not a leaf");
1805
1806 const Record *RCDef =
1807 Target.getInitValueAsRegClass(V: Dst.getChild(N: 0).getLeafValue());
1808 if (!RCDef)
1809 return failedImport(Reason: "REG_SEQUENCE child #0 could not "
1810 "be coerced to a register class");
1811
1812 if ((ExpectedDstINumUses - 1) % 2 != 0)
1813 return failedImport(Reason: "Malformed REG_SEQUENCE");
1814
1815 for (unsigned I = 1; I != ExpectedDstINumUses; I += 2) {
1816 const TreePatternNode &ValChild = Dst.getChild(N: I);
1817 const TreePatternNode &SubRegChild = Dst.getChild(N: I + 1);
1818
1819 if (const DefInit *SubRegInit =
1820 dyn_cast<DefInit>(Val: SubRegChild.getLeafValue())) {
1821 const CodeGenSubRegIndex *SubIdx =
1822 CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
1823
1824 if (Error Err = importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: ValChild, InsertPt))
1825 return Err;
1826
1827 DstMIBuilder.addRenderer<SubRegIndexRenderer>(args&: SubIdx);
1828 }
1829 }
1830
1831 return InsertPt;
1832 }
1833
1834 // Render the explicit uses.
1835 unsigned DstINumUses = OrigDstI->Operands.size() - OrigDstI->Operands.NumDefs;
1836 if (Name == "COPY_TO_REGCLASS") {
1837 DstINumUses--; // Ignore the class constraint.
1838 ExpectedDstINumUses--;
1839 }
1840
1841 // NumResults - This is the number of results produced by the instruction in
1842 // the "outs" list.
1843 unsigned NumResults = OrigDstI->Operands.NumDefs;
1844
1845 // Number of operands we know the output instruction must have. If it is
1846 // variadic, we could have more operands.
1847 unsigned NumFixedOperands = DstI->Operands.size();
1848
1849 // Loop over all of the fixed operands of the instruction pattern, emitting
1850 // code to fill them all in. The node 'N' usually has number children equal to
1851 // the number of input operands of the instruction. However, in cases where
1852 // there are predicate operands for an instruction, we need to fill in the
1853 // 'execute always' values. Match up the node operands to the instruction
1854 // operands to do this.
1855 unsigned Child = 0;
1856
1857 // Similarly to the code in TreePatternNode::ApplyTypeConstraints, count the
1858 // number of operands at the end of the list which have default values.
1859 // Those can come from the pattern if it provides enough arguments, or be
1860 // filled in with the default if the pattern hasn't provided them. But any
1861 // operand with a default value _before_ the last mandatory one will be
1862 // filled in with their defaults unconditionally.
1863 unsigned NonOverridableOperands = NumFixedOperands;
1864 while (NonOverridableOperands > NumResults &&
1865 CGP.operandHasDefault(Op: DstI->Operands[NonOverridableOperands - 1].Rec))
1866 --NonOverridableOperands;
1867
1868 unsigned NumDefaultOps = 0;
1869 for (unsigned I = 0; I != DstINumUses; ++I) {
1870 unsigned InstOpNo = DstI->Operands.NumDefs + I;
1871
1872 // Determine what to emit for this operand.
1873 const Record *OperandNode = DstI->Operands[InstOpNo].Rec;
1874
1875 // If the operand has default values, introduce them now.
1876 if (CGP.operandHasDefault(Op: OperandNode) &&
1877 (InstOpNo < NonOverridableOperands || Child >= Dst.getNumChildren())) {
1878 // This is a predicate or optional def operand which the pattern has not
1879 // overridden, or which we aren't letting it override; emit the 'default
1880 // ops' operands.
1881 for (const TreePatternNode &OpNode :
1882 make_pointee_range(Range: CGP.getDefaultOperand(R: OperandNode).DefaultOps)) {
1883 if (Error Err = importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: OpNode, InsertPt))
1884 return Err;
1885 }
1886
1887 ++NumDefaultOps;
1888 continue;
1889 }
1890
1891 if (Error Err =
1892 importNodeRenderer(M, MIBuilder&: DstMIBuilder, N: Dst.getChild(N: Child), InsertPt))
1893 return Err;
1894
1895 ++Child;
1896 }
1897
1898 if (NumDefaultOps + ExpectedDstINumUses != DstINumUses)
1899 return failedImport(Reason: "Expected " + llvm::to_string(Value: DstINumUses) +
1900 " used operands but found " +
1901 llvm::to_string(Value: ExpectedDstINumUses) +
1902 " explicit ones and " + llvm::to_string(Value: NumDefaultOps) +
1903 " default ones");
1904
1905 return InsertPt;
1906}
1907
1908Error GlobalISelEmitter::importImplicitDefRenderers(
1909 BuildMIAction &DstMIBuilder, ArrayRef<const Record *> ImplicitDefs) const {
1910 if (!ImplicitDefs.empty())
1911 return failedImport(Reason: "Pattern defines a physical register");
1912 return Error::success();
1913}
1914
1915Error GlobalISelEmitter::constrainOperands(action_iterator InsertPt,
1916 RuleMatcher &M, unsigned InsnID,
1917 const TreePatternNode &Dst) const {
1918 const Record *DstOp = Dst.getOperator();
1919 const CodeGenInstruction &DstI = Target.getInstruction(InstRec: DstOp);
1920 StringRef DstIName = DstI.getName();
1921
1922 if (DstIName == "COPY_TO_REGCLASS") {
1923 // COPY_TO_REGCLASS does not provide operand constraints itself but the
1924 // result is constrained to the class given by the second child.
1925 const Record *DstIOpRec =
1926 Target.getInitValueAsRegClass(V: Dst.getChild(N: 1).getLeafValue());
1927
1928 if (DstIOpRec == nullptr)
1929 return failedImport(Reason: "COPY_TO_REGCLASS operand #1 isn't a register class");
1930
1931 M.insertAction<ConstrainOperandToRegClassAction>(
1932 InsertPt, args&: InsnID, args: 0, args: Target.getRegisterClass(R: DstIOpRec));
1933 } else if (DstIName == "EXTRACT_SUBREG") {
1934 const CodeGenRegisterClass *SuperClass =
1935 inferRegClassFromPattern(N: Dst.getChild(N: 0));
1936 if (!SuperClass)
1937 return failedImport(
1938 Reason: "Cannot infer register class from EXTRACT_SUBREG operand #0");
1939
1940 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: Dst.getChild(N: 1));
1941 if (!SubIdx)
1942 return failedImport(Reason: "EXTRACT_SUBREG child #1 is not a subreg index");
1943
1944 // It would be nice to leave this constraint implicit but we're required
1945 // to pick a register class so constrain the result to a register class
1946 // that can hold the correct MVT.
1947 //
1948 // FIXME: This may introduce an extra copy if the chosen class doesn't
1949 // actually contain the subregisters.
1950 const auto SrcRCDstRCPair =
1951 SuperClass->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
1952 if (!SrcRCDstRCPair) {
1953 return failedImport(Reason: "subreg index is incompatible "
1954 "with inferred reg class");
1955 }
1956
1957 assert(SrcRCDstRCPair->second && "Couldn't find a matching subclass");
1958 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1959 args&: *SrcRCDstRCPair->second);
1960 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 1,
1961 args&: *SrcRCDstRCPair->first);
1962 } else if (DstIName == "INSERT_SUBREG") {
1963 // We need to constrain the destination, a super regsister source, and a
1964 // subregister source.
1965 const CodeGenRegisterClass *SubClass =
1966 inferRegClassFromPattern(N: Dst.getChild(N: 1));
1967 if (!SubClass)
1968 return failedImport(
1969 Reason: "Cannot infer register class from INSERT_SUBREG operand #1");
1970 const CodeGenRegisterClass *SuperClass = inferSuperRegisterClassForNode(
1971 Ty: Dst.getExtType(ResNo: 0), SuperRegNode: Dst.getChild(N: 0), SubRegIdxNode: Dst.getChild(N: 2));
1972 if (!SuperClass)
1973 return failedImport(
1974 Reason: "Cannot infer register class for INSERT_SUBREG operand #0");
1975 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1976 args: *SuperClass);
1977 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 1,
1978 args: *SuperClass);
1979 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 2,
1980 args: *SubClass);
1981 } else if (DstIName == "SUBREG_TO_REG") {
1982 // We need to constrain the destination and subregister source.
1983 // Attempt to infer the subregister source from the first child. If it has
1984 // an explicitly given register class, we'll use that. Otherwise, we will
1985 // fail.
1986 const CodeGenRegisterClass *SubClass =
1987 inferRegClassFromPattern(N: Dst.getChild(N: 0));
1988 if (!SubClass)
1989 return failedImport(
1990 Reason: "Cannot infer register class from SUBREG_TO_REG child #0");
1991 // We don't have a child to look at that might have a super register node.
1992 const CodeGenRegisterClass *SuperClass =
1993 inferSuperRegisterClass(Ty: Dst.getExtType(ResNo: 0), SubRegIdxNode: Dst.getChild(N: 1));
1994 if (!SuperClass)
1995 return failedImport(
1996 Reason: "Cannot infer register class for SUBREG_TO_REG operand #0");
1997 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
1998 args: *SuperClass);
1999 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 1,
2000 args: *SubClass);
2001 } else if (DstIName == "REG_SEQUENCE") {
2002 const CodeGenRegisterClass *SuperClass =
2003 inferRegClassFromPattern(N: Dst.getChild(N: 0));
2004
2005 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args: 0,
2006 args: *SuperClass);
2007
2008 unsigned Num = Dst.getNumChildren();
2009 for (unsigned I = 1; I != Num; I += 2) {
2010 const TreePatternNode &SubRegChild = Dst.getChild(N: I + 1);
2011
2012 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: SubRegChild);
2013 if (!SubIdx)
2014 return failedImport(Reason: "REG_SEQUENCE child is not a subreg index");
2015
2016 const auto SrcRCDstRCPair =
2017 SuperClass->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
2018 if (!SrcRCDstRCPair)
2019 return failedImport(Reason: "REG_SEQUENCE subreg index is incompatible "
2020 "with inferred reg class");
2021
2022 M.insertAction<ConstrainOperandToRegClassAction>(InsertPt, args&: InsnID, args&: I,
2023 args&: *SrcRCDstRCPair->second);
2024 }
2025 } else {
2026 M.insertAction<ConstrainOperandsToDefinitionAction>(InsertPt, args&: InsnID);
2027 }
2028
2029 return Error::success();
2030}
2031
2032const CodeGenRegisterClass *
2033GlobalISelEmitter::getRegClassFromLeaf(const TreePatternNode &Leaf) const {
2034 assert(Leaf.isLeaf() && "Expected leaf?");
2035 const Record *RCRec = Target.getInitValueAsRegClass(V: Leaf.getLeafValue());
2036 if (!RCRec)
2037 return nullptr;
2038 return CGRegs.getRegClass(RCRec);
2039}
2040
2041const CodeGenRegisterClass *
2042GlobalISelEmitter::inferRegClassFromPattern(const TreePatternNode &N) const {
2043 if (N.isLeaf())
2044 return getRegClassFromLeaf(Leaf: N);
2045
2046 // We don't have a leaf node, so we have to try and infer something. Check
2047 // that we have an instruction that we can infer something from.
2048
2049 // Only handle things that produce at least one value (if multiple values,
2050 // just take the first one).
2051 if (N.getNumTypes() < 1)
2052 return nullptr;
2053 const Record *OpRec = N.getOperator();
2054
2055 // We only want instructions.
2056 if (!OpRec->isSubClassOf(Name: "Instruction"))
2057 return nullptr;
2058
2059 // Don't want to try and infer things when there could potentially be more
2060 // than one candidate register class.
2061 return inferRegClassFromInstructionPattern(N, /*ResIdx=*/0);
2062}
2063
2064const CodeGenRegisterClass *
2065GlobalISelEmitter::inferRegClassFromRegisterClassLike(
2066 const Record *RegClassDef) const {
2067 if (RegClassDef->isSubClassOf(Name: "RegClassByHwMode")) {
2068 // TODO: We only are trying to match the regbank, which we assume is the
2069 // same across modes, but we are just picking one class and assuming they
2070 // all have the same bank.
2071 //
2072 // We should verify there is a common regbank among the possible classes,
2073 // and return that instead of a concrete class.
2074 const HwModeSelect &ModeSelect =
2075 Target.getHwModes().getHwModeSelect(R: RegClassDef);
2076 if (ModeSelect.Items.empty())
2077 return nullptr;
2078 return Target.getRegBank().getRegClass(ModeSelect.Items.front().second);
2079 }
2080
2081 return &Target.getRegisterClass(R: RegClassDef);
2082}
2083
2084const CodeGenRegisterClass *
2085GlobalISelEmitter::inferRegClassFromInstructionPattern(const TreePatternNode &N,
2086 unsigned ResIdx) const {
2087 const CodeGenInstruction &Inst = Target.getInstruction(InstRec: N.getOperator());
2088 assert(ResIdx < Inst.Operands.NumDefs &&
2089 "Can only infer register class for explicit defs");
2090
2091 // Handle any special-case instructions which we can safely infer register
2092 // classes from.
2093 StringRef InstName = Inst.getName();
2094 if (InstName == "REG_SEQUENCE") {
2095 // (outs $super_dst), (ins $dst_regclass, variable_ops)
2096 // Destination register class is explicitly specified by the first operand.
2097 const TreePatternNode &RCChild = N.getChild(N: 0);
2098 if (!RCChild.isLeaf())
2099 return nullptr;
2100 return getRegClassFromLeaf(Leaf: RCChild);
2101 }
2102
2103 if (InstName == "COPY_TO_REGCLASS") {
2104 // (outs $dst), (ins $src, $dst_regclass)
2105 // Destination register class is explicitly specified by the second operand.
2106 const TreePatternNode &RCChild = N.getChild(N: 1);
2107 if (!RCChild.isLeaf())
2108 return nullptr;
2109 return getRegClassFromLeaf(Leaf: RCChild);
2110 }
2111
2112 if (InstName == "INSERT_SUBREG") {
2113 // (outs $super_dst), (ins $super_src, $sub_src, $sub_idx);
2114 // If we can infer the register class for the first operand, use that.
2115 // Otherwise, find a register class that supports both the specified
2116 // sub-register index and the type of the instruction's result.
2117 const TreePatternNode &Child0 = N.getChild(N: 0);
2118 assert(Child0.getNumTypes() == 1 && "Unexpected number of types!");
2119 return inferSuperRegisterClassForNode(Ty: N.getExtType(ResNo: 0), SuperRegNode: Child0,
2120 SubRegIdxNode: N.getChild(N: 2));
2121 }
2122
2123 if (InstName == "EXTRACT_SUBREG") {
2124 // (outs $sub_dst), (ins $super_src, $sub_idx)
2125 // Find a register class that can be used for a sub-register copy from
2126 // the specified source at the specified sub-register index.
2127 const CodeGenRegisterClass *SuperRC =
2128 inferRegClassFromPattern(N: N.getChild(N: 0));
2129 if (!SuperRC)
2130 return nullptr;
2131
2132 const CodeGenSubRegIndex *SubIdx = inferSubRegIndexForNode(SubRegIdxNode: N.getChild(N: 1));
2133 if (!SubIdx)
2134 return nullptr;
2135
2136 const auto SubRCAndSubRegRC =
2137 SuperRC->getMatchingSubClassWithSubRegs(RegBank&: CGRegs, SubIdx);
2138 if (!SubRCAndSubRegRC)
2139 return nullptr;
2140
2141 return SubRCAndSubRegRC->second;
2142 }
2143
2144 if (InstName == "SUBREG_TO_REG") {
2145 // (outs $super_dst), (ins $sub_src, $sub_idx)
2146 // Find a register class that supports both the specified sub-register
2147 // index and the type of the instruction's result.
2148 return inferSuperRegisterClass(Ty: N.getExtType(ResNo: 0), SubRegIdxNode: N.getChild(N: 1));
2149 }
2150
2151 // Handle destination record types that we can safely infer a register class
2152 // from.
2153 const auto &DstIOperand = Inst.Operands[ResIdx];
2154 const Record *DstIOpRec = DstIOperand.Rec;
2155
2156 if (DstIOpRec->isSubClassOf(Name: "RegisterOperand")) {
2157 const Record *RegClassDef = DstIOpRec->getValueAsDef(FieldName: "RegClass");
2158 return inferRegClassFromRegisterClassLike(RegClassDef);
2159 }
2160
2161 if (DstIOpRec->isSubClassOf(Name: "RegClassByHwMode"))
2162 return inferRegClassFromRegisterClassLike(RegClassDef: DstIOpRec);
2163
2164 if (DstIOpRec->isSubClassOf(Name: "RegisterClass"))
2165 return &Target.getRegisterClass(R: DstIOpRec);
2166
2167 return nullptr;
2168}
2169
2170const CodeGenRegisterClass *GlobalISelEmitter::inferSuperRegisterClass(
2171 const TypeSetByHwMode &Ty, const TreePatternNode &SubRegIdxNode) const {
2172 // We need a ValueTypeByHwMode for getSuperRegForSubReg.
2173 if (!Ty.isValueTypeByHwMode(AllowEmpty: false))
2174 return nullptr;
2175 if (!SubRegIdxNode.isLeaf())
2176 return nullptr;
2177 const DefInit *SubRegInit = dyn_cast<DefInit>(Val: SubRegIdxNode.getLeafValue());
2178 if (!SubRegInit)
2179 return nullptr;
2180 const CodeGenSubRegIndex *SubIdx = CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
2181
2182 // Use the information we found above to find a minimal register class which
2183 // supports the subregister and type we want.
2184 return CGRegs.getSuperRegForSubReg(Ty: Ty.getValueTypeByHwMode(), SubIdx,
2185 /*MustBeAllocatable=*/true);
2186}
2187
2188const CodeGenRegisterClass *GlobalISelEmitter::inferSuperRegisterClassForNode(
2189 const TypeSetByHwMode &Ty, const TreePatternNode &SuperRegNode,
2190 const TreePatternNode &SubRegIdxNode) const {
2191 // Check if we already have a defined register class for the super register
2192 // node. If we do, then we should preserve that rather than inferring anything
2193 // from the subregister index node. We can assume that whoever wrote the
2194 // pattern in the first place made sure that the super register and
2195 // subregister are compatible.
2196 if (const CodeGenRegisterClass *SuperRegisterClass =
2197 inferRegClassFromPattern(N: SuperRegNode))
2198 return SuperRegisterClass;
2199 return inferSuperRegisterClass(Ty, SubRegIdxNode);
2200}
2201
2202const CodeGenSubRegIndex *GlobalISelEmitter::inferSubRegIndexForNode(
2203 const TreePatternNode &SubRegIdxNode) const {
2204 if (!SubRegIdxNode.isLeaf())
2205 return nullptr;
2206
2207 const DefInit *SubRegInit = dyn_cast<DefInit>(Val: SubRegIdxNode.getLeafValue());
2208 if (!SubRegInit)
2209 return nullptr;
2210 return CGRegs.findSubRegIdx(Def: SubRegInit->getDef());
2211}
2212
2213static void collectModes(std::set<unsigned> &Modes, const TreePatternNode &N) {
2214 for (const TypeSetByHwMode &VTS : N.getExtTypes())
2215 for (const auto &I : VTS)
2216 Modes.insert(x: I.first);
2217
2218 for (const TreePatternNode &Child : N.children())
2219 collectModes(Modes, N: Child);
2220}
2221
2222/// Return true if every type in \p N has a type for \p Mode.
2223static bool hasTypesForMode(const TreePatternNode &N, unsigned Mode) {
2224 for (const TypeSetByHwMode &VTS : N.getExtTypes()) {
2225 if (!VTS.hasMode(M: Mode) && !VTS.hasDefault())
2226 return false;
2227 if (VTS.get(Mode).empty())
2228 return false;
2229 }
2230 return all_of(Range: N.children(), P: [&](const TreePatternNode &C) {
2231 return hasTypesForMode(N: C, Mode);
2232 });
2233}
2234
2235/// Since HwMode based types aren't expanded into separate patterns, a pattern
2236/// may only be valid for some of the HwModes. Return a predicate that checks
2237/// for those HwModes, or an empty string if the pattern is valid for all of
2238/// them. This matches the predicates that were used when the patterns were
2239/// expanded.
2240static std::string getHwModeCheck(const PatternToMatch &P,
2241 const CodeGenHwModes &CGH) {
2242 const TreePatternNode &Src = P.getSrcPattern();
2243 const TreePatternNode &Dst = P.getDstPattern();
2244 if (!Src.hasProperTypeByHwMode() && !Dst.hasProperTypeByHwMode())
2245 return "";
2246
2247 std::set<unsigned> Modes;
2248 collectModes(Modes, N: Src);
2249 collectModes(Modes, N: Dst);
2250
2251 SmallVector<unsigned> ValidModes;
2252 for (unsigned M : Modes)
2253 if (hasTypesForMode(N: Src, Mode: M) && hasTypesForMode(N: Dst, Mode: M))
2254 ValidModes.push_back(Elt: M);
2255
2256 // If the pattern is valid for every mode it mentions and there's a default
2257 // mode, there's nothing to check.
2258 if (ValidModes.size() == Modes.size() && Modes.count(x: DefaultMode))
2259 return "";
2260
2261 std::string DefaultCheck;
2262 std::string Check;
2263 for (unsigned M : Modes) {
2264 if (M == DefaultMode)
2265 continue;
2266 std::string PredicateCheck;
2267 raw_string_ostream PS(PredicateCheck);
2268 SubtargetFeatureInfo::emitPredicateCheck(OS&: PS, Predicates: CGH.getMode(Id: M).Predicates);
2269 if (!DefaultCheck.empty())
2270 DefaultCheck += " && ";
2271 DefaultCheck += "!(" + PredicateCheck + ")";
2272 if (is_contained(Range&: ValidModes, Element: M)) {
2273 if (!Check.empty())
2274 Check += " || ";
2275 Check += "(" + PredicateCheck + ")";
2276 }
2277 }
2278 if (is_contained(Range&: ValidModes, Element: DefaultMode)) {
2279 if (!Check.empty())
2280 Check += " || ";
2281 Check += "(" + DefaultCheck + ")";
2282 }
2283 // There is no valid mode. Use a check that can never pass.
2284 if (Check.empty())
2285 return "false";
2286 return Check;
2287}
2288
2289Expected<RuleMatcher> GlobalISelEmitter::runOnPattern(const PatternToMatch &P) {
2290 // Keep track of the matchers and actions to emit.
2291 int Score = P.getPatternComplexity(CGP);
2292 RuleMatcher M(P.getSrcRecord()->getLoc());
2293 RuleMatcherScores[M.getRuleID()] = Score;
2294 M.addAction<DebugCommentAction>(args: llvm::to_string(Value: P.getSrcPattern()) +
2295 " => " +
2296 llvm::to_string(Value: P.getDstPattern()));
2297
2298 SmallVector<const Record *, 4> Predicates;
2299 P.getPredicateRecords(PredicateRecs&: Predicates);
2300 if (auto Error = importRulePredicates(M, Predicates))
2301 return std::move(Error);
2302
2303 std::string HwModeCheck = getHwModeCheck(P, CGH: Target.getHwModes());
2304 if (!HwModeCheck.empty())
2305 M.addHwModeIdx(Idx: declareHwModeCheck(HwModeFeatures: HwModeCheck));
2306
2307 // Next, analyze the pattern operators.
2308 TreePatternNode &Src = P.getSrcPattern();
2309 TreePatternNode &Dst = P.getDstPattern();
2310
2311 // If the root of either pattern isn't a simple operator, ignore it.
2312 if (auto Err = isTrivialOperatorNode(N: Dst))
2313 return failedImport(Reason: "Dst pattern root isn't a trivial operator (" +
2314 toString(E: std::move(Err)) + ")");
2315 if (auto Err = isTrivialOperatorNode(N: Src))
2316 return failedImport(Reason: "Src pattern root isn't a trivial operator (" +
2317 toString(E: std::move(Err)) + ")");
2318
2319 // The different predicates and matchers created during
2320 // addInstructionMatcher use the RuleMatcher M to set up their
2321 // instruction ID (InsnVarID) that are going to be used when
2322 // M is going to be emitted.
2323 // However, the code doing the emission still relies on the IDs
2324 // returned during that process by the RuleMatcher when issuing
2325 // the recordInsn opcodes.
2326 // Because of that:
2327 // 1. The order in which we created the predicates
2328 // and such must be the same as the order in which we emit them,
2329 // and
2330 // 2. We need to reset the generation of the IDs in M somewhere between
2331 // addInstructionMatcher and emit
2332 //
2333 // FIXME: Long term, we don't want to have to rely on this implicit
2334 // naming being the same. One possible solution would be to have
2335 // explicit operator for operation capture and reference those.
2336 // The plus side is that it would expose opportunities to share
2337 // the capture accross rules. The downside is that it would
2338 // introduce a dependency between predicates (captures must happen
2339 // before their first use.)
2340 InstructionMatcher &InsnMatcherTemp = M.addInstructionMatcher(SymbolicName: Src.getName());
2341 unsigned TempOpIdx = 0;
2342
2343 const auto SavedFlags = M.setGISelFlags(P.getSrcRecord());
2344
2345 auto InsnMatcherOrError =
2346 createAndImportSelDAGMatcher(Rule&: M, InsnMatcher&: InsnMatcherTemp, Src, TempOpIdx);
2347 if (auto Error = InsnMatcherOrError.takeError())
2348 return std::move(Error);
2349 InstructionMatcher &InsnMatcher = InsnMatcherOrError.get();
2350
2351 if (Dst.isLeaf()) {
2352 if (const Record *RCDef =
2353 Target.getInitValueAsRegClass(V: Dst.getLeafValue())) {
2354 const CodeGenRegisterClass &RC = Target.getRegisterClass(R: RCDef);
2355
2356 // We need to replace the def and all its uses with the specified
2357 // operand. However, we must also insert COPY's wherever needed.
2358 // For now, emit a copy and let the register allocator clean up.
2359 const CodeGenInstruction &DstI = Target.getInstruction(InstRec: RK.getDef(Name: "COPY"));
2360 const auto &DstIOperand = DstI.Operands[0];
2361
2362 OperandMatcher &OM0 = InsnMatcher.getOperand(OpIdx: 0);
2363 OM0.setSymbolicName(DstIOperand.Name);
2364 M.defineOperand(SymbolicName: OM0.getSymbolicName(), OM&: OM0);
2365 OM0.addPredicate<RegisterBankOperandMatcher>(args: RC);
2366
2367 auto &DstMIBuilder =
2368 M.addAction<BuildMIAction>(args: M.allocateOutputInsnID(), args&: M, args: &DstI);
2369 DstMIBuilder.addRenderer<CopyRenderer>(args&: M, args: DstIOperand.Name);
2370 DstMIBuilder.addRenderer<CopyRenderer>(args&: M, args: Dst.getName());
2371 M.addAction<ConstrainOperandToRegClassAction>(args: 0, args: 0, args: RC);
2372
2373 // Erase the root.
2374 unsigned RootInsnID = InsnMatcher.getInsnVarID();
2375 if (M.tryEraseInsnID(ID: RootInsnID))
2376 M.addAction<EraseInstAction>(args&: RootInsnID);
2377
2378 // We're done with this pattern! It's eligible for GISel emission; return
2379 // it.
2380 ++NumPatternImported;
2381 return std::move(M);
2382 }
2383
2384 return failedImport(Reason: "Dst pattern root isn't a known leaf");
2385 }
2386
2387 // Start with the defined operands (i.e., the results of the root operator).
2388 const Record *DstOp = Dst.getOperator();
2389 if (!DstOp->isSubClassOf(Name: "Instruction"))
2390 return failedImport(Reason: "Pattern operator isn't an instruction");
2391
2392 const CodeGenInstruction &DstI = Target.getInstruction(InstRec: DstOp);
2393
2394 // Count both implicit and explicit defs in the dst instruction.
2395 // This avoids errors importing patterns that have inherent implicit defs.
2396 unsigned DstExpDefs = DstI.Operands.NumDefs,
2397 DstNumDefs = DstI.ImplicitDefs.size() + DstExpDefs,
2398 SrcNumDefs = Src.getExtTypes().size();
2399 if (DstNumDefs < SrcNumDefs) {
2400 if (DstNumDefs != 0)
2401 return failedImport(Reason: "Src pattern result has more defs than dst MI (" +
2402 to_string(Value: SrcNumDefs) + " def(s) vs " +
2403 to_string(Value: DstNumDefs) + " def(s))");
2404
2405 bool FoundNoUsePred = false;
2406 for (const auto &Pred : InsnMatcher.predicates()) {
2407 if ((FoundNoUsePred = isa<NoUsePredicateMatcher>(Val: Pred.get())))
2408 break;
2409 }
2410 if (!FoundNoUsePred)
2411 return failedImport(Reason: "Src pattern result has " + to_string(Value: SrcNumDefs) +
2412 " def(s) without the HasNoUse predicate set to true "
2413 "but Dst MI has no def");
2414 }
2415
2416 // The root of the match also has constraints on the register bank so that it
2417 // matches the result instruction.
2418 unsigned N = std::min(a: DstExpDefs, b: SrcNumDefs);
2419 for (unsigned I = 0; I < N; ++I) {
2420 const auto &DstIOperand = DstI.Operands[I];
2421
2422 OperandMatcher &OM = InsnMatcher.getOperand(OpIdx: I);
2423 // The operand names declared in the DstI instruction are unrelated to
2424 // those used in pattern's source and destination DAGs, so mangle the
2425 // former to prevent implicitly adding unexpected
2426 // GIM_CheckIsSameOperand predicates by the defineOperand method.
2427 OM.setSymbolicName(getMangledRootDefName(DefOperandName: DstIOperand.Name));
2428 M.defineOperand(SymbolicName: OM.getSymbolicName(), OM);
2429
2430 const CodeGenRegisterClass *RC =
2431 inferRegClassFromInstructionPattern(N: Dst, ResIdx: I);
2432 if (!RC)
2433 return failedImport(Reason: "Could not infer register class for result #" +
2434 Twine(I) + " from pattern " + to_string(Value: Dst));
2435 OM.addPredicate<RegisterBankOperandMatcher>(args: *RC);
2436 }
2437
2438 auto DstMIBuilderOrError =
2439 createAndImportInstructionRenderer(M, InsnMatcher, Dst);
2440 if (auto Error = DstMIBuilderOrError.takeError())
2441 return std::move(Error);
2442 BuildMIAction &DstMIBuilder = DstMIBuilderOrError.get();
2443
2444 // Render the implicit defs.
2445 // These are only added to the root of the result.
2446 if (auto Error = importImplicitDefRenderers(DstMIBuilder, ImplicitDefs: P.getDstRegs()))
2447 return std::move(Error);
2448
2449 DstMIBuilder.chooseInsnToMutate(Rule&: M);
2450
2451 // Constrain the registers to classes. This is normally derived from the
2452 // emitted instruction but a few instructions require special handling.
2453 if (auto Error =
2454 constrainOperands(InsertPt: M.actions_end(), M, InsnID: DstMIBuilder.getInsnID(), Dst))
2455 return std::move(Error);
2456
2457 // Erase the root.
2458 unsigned RootInsnID = InsnMatcher.getInsnVarID();
2459 if (M.tryEraseInsnID(ID: RootInsnID))
2460 M.addAction<EraseInstAction>(args&: RootInsnID);
2461
2462 // We're done with this pattern! It's eligible for GISel emission; return it.
2463 ++NumPatternImported;
2464 return std::move(M);
2465}
2466
2467MatchTable
2468GlobalISelEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules,
2469 bool Optimize, bool WithCoverage) {
2470 if (!Optimize) {
2471 SmallVector<Matcher *> InputRules(make_pointer_range(Range&: Rules));
2472 return ::buildMatchTable(Rules: InputRules, WithCoverage);
2473 }
2474
2475 std::vector<std::unique_ptr<Matcher>> MatcherStorage;
2476 std::vector<Matcher *> OptRules = optimizeRuleset(Rules, MatcherStorage);
2477 return ::buildMatchTable(Rules: OptRules, WithCoverage);
2478}
2479
2480void GlobalISelEmitter::emitAdditionalImpl(raw_ostream &OS) {
2481 OS << "bool " << getClassName()
2482 << "::selectImpl(MachineInstr &I, CodeGenCoverage "
2483 "&CoverageInfo) const {\n"
2484 << " const PredicateBitset AvailableFeatures = "
2485 "getAvailableFeatures();\n"
2486 << " MachineIRBuilder B(I);\n"
2487 << " State.MIs.clear();\n"
2488 << " State.MIs.push_back(&I);\n\n"
2489 << " if (executeMatchTable(*this, State, ExecInfo, B"
2490 << ", getMatchTable(), TII, MF->getRegInfo(), TRI, RBI, AvailableFeatures"
2491 << ", &CoverageInfo)) {\n"
2492 << " return true;\n"
2493 << " }\n\n"
2494 << " return false;\n"
2495 << "}\n\n";
2496}
2497
2498void GlobalISelEmitter::emitMIPredicateFns(raw_ostream &OS) {
2499 std::vector<const Record *> MatchedRecords;
2500 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2501 P: [](const Record *R) {
2502 return !R->getValueAsString(FieldName: "GISelPredicateCode").empty();
2503 });
2504 emitMIPredicateFnsImpl<const Record *>(
2505 OS,
2506 AdditionalDecls: " const MachineFunction &MF = *MI.getParent()->getParent();\n"
2507 " const MachineRegisterInfo &MRI = MF.getRegInfo();\n"
2508 " const auto &Operands = State.RecordedOperands;\n"
2509 " (void)Operands;\n"
2510 " (void)MRI;",
2511 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2512 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "GISelPredicateCode"); },
2513 Comment: "PatFrag predicates.");
2514}
2515
2516void GlobalISelEmitter::emitLeafPredicateFns(raw_ostream &OS) {
2517 std::vector<const Record *> MatchedRecords;
2518 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2519 P: [](const Record *R) {
2520 return (!R->getValueAsOptionalString(FieldName: "GISelLeafPredicateCode")
2521 .value_or(u: std::string())
2522 .empty());
2523 });
2524 emitLeafPredicateFnsImpl<const Record *>(
2525 OS,
2526 AdditionalDecls: " const auto &Operands = State.RecordedOperands;\n"
2527 " Register Reg = MO.getReg();\n"
2528 " (void)Operands;\n"
2529 " (void)Reg;",
2530 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2531 GetPredCode: [](const Record *R) {
2532 return R->getValueAsString(FieldName: "GISelLeafPredicateCode");
2533 },
2534 Comment: "PatFrag predicates.");
2535}
2536
2537void GlobalISelEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
2538 std::vector<const Record *> MatchedRecords;
2539 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2540 P: [](const Record *R) {
2541 bool Unset;
2542 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2543 !R->getValueAsBitOrUnset(FieldName: "IsAPFloat", Unset) &&
2544 !R->getValueAsBit(FieldName: "IsAPInt");
2545 });
2546 emitImmPredicateFnsImpl<const Record *>(
2547 OS, TypeIdentifier: "I64", ArgType: "int64_t", Predicates: ArrayRef<const Record *>(MatchedRecords),
2548 GetPredEnumName: &getPatFragPredicateEnumName,
2549 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2550 Comment: "PatFrag predicates.");
2551}
2552
2553void GlobalISelEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) {
2554 std::vector<const Record *> MatchedRecords;
2555 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2556 P: [](const Record *R) {
2557 bool Unset;
2558 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2559 R->getValueAsBitOrUnset(FieldName: "IsAPFloat", Unset);
2560 });
2561 emitImmPredicateFnsImpl<const Record *>(
2562 OS, TypeIdentifier: "APFloat", ArgType: "const APFloat &",
2563 Predicates: ArrayRef<const Record *>(MatchedRecords), GetPredEnumName: &getPatFragPredicateEnumName,
2564 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2565 Comment: "PatFrag predicates.");
2566}
2567
2568void GlobalISelEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) {
2569 std::vector<const Record *> MatchedRecords;
2570 llvm::copy_if(Range&: AllPatFrags, Out: std::back_inserter(x&: MatchedRecords),
2571 P: [](const Record *R) {
2572 return !R->getValueAsString(FieldName: "ImmediateCode").empty() &&
2573 R->getValueAsBit(FieldName: "IsAPInt");
2574 });
2575 emitImmPredicateFnsImpl<const Record *>(
2576 OS, TypeIdentifier: "APInt", ArgType: "const APInt &", Predicates: ArrayRef<const Record *>(MatchedRecords),
2577 GetPredEnumName: &getPatFragPredicateEnumName,
2578 GetPredCode: [](const Record *R) { return R->getValueAsString(FieldName: "ImmediateCode"); },
2579 Comment: "PatFrag predicates.");
2580}
2581
2582void GlobalISelEmitter::emitTestSimplePredicate(raw_ostream &OS) {
2583 OS << "bool " << getClassName() << "::testSimplePredicate(unsigned) const {\n"
2584 << " llvm_unreachable(\"" + getClassName() +
2585 " does not support simple predicates!\");\n"
2586 << " return false;\n"
2587 << "}\n";
2588}
2589
2590void GlobalISelEmitter::emitRunCustomAction(raw_ostream &OS) {
2591 OS << "bool " << getClassName()
2592 << "::runCustomAction(unsigned, const MatcherState&, NewMIVector &) const "
2593 "{\n"
2594 << " llvm_unreachable(\"" + getClassName() +
2595 " does not support custom C++ actions!\");\n"
2596 << "}\n";
2597}
2598
2599bool hasBFloatType(const TreePatternNode &Node) {
2600 for (unsigned I = 0, E = Node.getNumTypes(); I < E; I++) {
2601 if (anyTypeByHwMode(VTy: Node.getExtType(ResNo: I),
2602 Pred: [](MVT VT) { return VT.getScalarType() == MVT::bf16; }))
2603 return true;
2604 }
2605 for (const TreePatternNode &C : Node.children())
2606 if (hasBFloatType(Node: C))
2607 return true;
2608 return false;
2609}
2610
2611void GlobalISelEmitter::run(raw_ostream &OS) {
2612 if (!UseCoverageFile.empty()) {
2613 RuleCoverage = CodeGenCoverage();
2614 auto RuleCoverageBufOrErr = MemoryBuffer::getFile(Filename: UseCoverageFile);
2615 if (!RuleCoverageBufOrErr) {
2616 PrintWarning(WarningLoc: SMLoc(), Msg: "Missing rule coverage data");
2617 RuleCoverage = std::nullopt;
2618 } else {
2619 if (!RuleCoverage->parse(Buffer&: *RuleCoverageBufOrErr.get(), BackendName: Target.getName())) {
2620 PrintWarning(WarningLoc: SMLoc(), Msg: "Ignoring invalid or missing rule coverage data");
2621 RuleCoverage = std::nullopt;
2622 }
2623 }
2624 }
2625
2626 // Track the run-time opcode values
2627 gatherOpcodeValues();
2628 // Track the run-time LLT ID values
2629 gatherTypeIDValues();
2630
2631 // Track the GINodeEquiv definitions.
2632 gatherNodeEquivs();
2633
2634 AllPatFrags = RK.getAllDerivedDefinitions(ClassName: "PatFrags");
2635
2636 emitSourceFileHeader(
2637 Desc: ("Global Instruction Selector for the " + Target.getName() + " target")
2638 .str(),
2639 OS);
2640 std::vector<RuleMatcher> Rules;
2641 // Look through the SelectionDAG patterns we found, possibly emitting some.
2642 for (const PatternToMatch &Pat : CGP.ptms()) {
2643 ++NumPatternTotal;
2644
2645 if (Pat.getGISelShouldIgnore())
2646 continue; // skip without warning
2647
2648 // Skip any patterns containing BF16 types, as GISel cannot currently tell
2649 // the difference between fp16 and bf16. FIXME: This can be removed once
2650 // UseExtended are universally supported.
2651 bool UseExtended = LLT::getUseExtended();
2652 if (hasBFloatType(Node: Pat.getSrcPattern()) && !UseExtended)
2653 continue;
2654
2655 auto MatcherOrErr = runOnPattern(P: Pat);
2656
2657 // The pattern analysis can fail, indicating an unsupported pattern.
2658 // Report that if we've been asked to do so.
2659 if (auto Err = MatcherOrErr.takeError()) {
2660 if (WarnOnSkippedPatterns) {
2661 PrintWarning(WarningLoc: Pat.getSrcRecord()->getLoc(),
2662 Msg: "Skipped pattern: " + toString(E: std::move(Err)));
2663 } else {
2664 consumeError(Err: std::move(Err));
2665 }
2666 ++NumPatternImportsSkipped;
2667 continue;
2668 }
2669
2670 if (RuleCoverage) {
2671 if (RuleCoverage->isCovered(RuleID: MatcherOrErr->getRuleID()))
2672 ++NumPatternsTested;
2673 else
2674 PrintWarning(WarningLoc: Pat.getSrcRecord()->getLoc(),
2675 Msg: "Pattern is not covered by a test");
2676 }
2677 Rules.push_back(x: std::move(MatcherOrErr.get()));
2678 }
2679
2680 // Comparison function to order records by name.
2681 auto OrderByName = [](const Record *A, const Record *B) {
2682 return A->getName() < B->getName();
2683 };
2684
2685 std::vector<const Record *> ComplexPredicates =
2686 RK.getAllDerivedDefinitions(ClassName: "GIComplexOperandMatcher");
2687 llvm::sort(C&: ComplexPredicates, Comp: OrderByName);
2688
2689 std::vector<StringRef> CustomRendererFns;
2690 transform(Range: RK.getAllDerivedDefinitions(ClassName: "GICustomOperandRenderer"),
2691 d_first: std::back_inserter(x&: CustomRendererFns), F: [](const auto &Record) {
2692 return Record->getValueAsString("RendererFn");
2693 });
2694 // Sort and remove duplicates to get a list of unique renderer functions, in
2695 // case some were mentioned more than once.
2696 llvm::sort(C&: CustomRendererFns);
2697 CustomRendererFns.erase(first: llvm::unique(R&: CustomRendererFns),
2698 last: CustomRendererFns.end());
2699
2700 // Create a table containing the LLT objects needed by the matcher and an enum
2701 // for the matcher to reference them with.
2702 std::vector<LLTCodeGen> TypeObjects;
2703 append_range(C&: TypeObjects, R&: KnownTypes);
2704 llvm::sort(C&: TypeObjects);
2705
2706 // Sort rules.
2707 llvm::stable_sort(Range&: Rules, C: [&](const RuleMatcher &A, const RuleMatcher &B) {
2708 int ScoreA = RuleMatcherScores[A.getRuleID()];
2709 int ScoreB = RuleMatcherScores[B.getRuleID()];
2710 if (ScoreA > ScoreB)
2711 return true;
2712 if (ScoreB > ScoreA)
2713 return false;
2714 if (A.isHigherPriorityThan(B)) {
2715 assert(!B.isHigherPriorityThan(A) && "Cannot be more important "
2716 "and less important at "
2717 "the same time");
2718 return true;
2719 }
2720 return false;
2721 });
2722
2723 unsigned MaxTemporaries = 0;
2724 for (const auto &Rule : Rules)
2725 MaxTemporaries = std::max(a: MaxTemporaries, b: Rule.countRendererFns());
2726
2727 // Build match table
2728 const MatchTable Table =
2729 buildMatchTable(Rules, Optimize: OptimizeMatchTable, WithCoverage: GenerateCoverage);
2730
2731 emitPredicateBitset(OS, IfDefName: "GET_GLOBALISEL_PREDICATE_BITSET");
2732 emitTemporariesDecl(OS, IfDefName: "GET_GLOBALISEL_TEMPORARIES_DECL");
2733 emitTemporariesInit(OS, MaxTemporaries, IfDefName: "GET_GLOBALISEL_TEMPORARIES_INIT");
2734 emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexOperandMatchers: ComplexPredicates,
2735 CustomOperandRenderers: CustomRendererFns, IfDefName: "GET_GLOBALISEL_IMPL");
2736 emitPredicatesDecl(OS, IfDefName: "GET_GLOBALISEL_PREDICATES_DECL");
2737 emitPredicatesInit(OS, IfDefName: "GET_GLOBALISEL_PREDICATES_INIT");
2738}
2739
2740void GlobalISelEmitter::declareSubtargetFeature(const Record *Predicate) {
2741 SubtargetFeatures.try_emplace(k: Predicate, args&: Predicate, args: SubtargetFeatures.size());
2742}
2743
2744unsigned GlobalISelEmitter::declareHwModeCheck(StringRef HwModeFeatures) {
2745 return HwModes.emplace(args: HwModeFeatures.str(), args: HwModes.size()).first->second;
2746}
2747
2748//===----------------------------------------------------------------------===//
2749
2750static TableGen::Emitter::OptClass<GlobalISelEmitter>
2751 X("gen-global-isel", "Generate GlobalISel selector");
2752