1///===- FastISelEmitter.cpp - Generate an instruction selector ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend emits code for use by the "fast" instruction
10// selection algorithm. See the comments at the top of
11// lib/CodeGen/SelectionDAG/FastISel.cpp for background.
12//
13// This file scans through the target's tablegen instruction-info files
14// and extracts instructions with obvious-looking patterns, and it emits
15// code to look up these instructions by type and operator.
16//
17//===----------------------------------------------------------------------===//
18
19#include "Common/CodeGenDAGPatterns.h"
20#include "Common/CodeGenInstruction.h"
21#include "Common/CodeGenRegisters.h"
22#include "Common/CodeGenTarget.h"
23#include "Common/InfoByHwMode.h"
24#include "llvm/ADT/StringSwitch.h"
25#include "llvm/Support/ErrorHandling.h"
26#include "llvm/TableGen/Error.h"
27#include "llvm/TableGen/Record.h"
28#include "llvm/TableGen/TableGenBackend.h"
29#include <set>
30#include <utility>
31using namespace llvm;
32
33/// InstructionMemo - This class holds additional information about an
34/// instruction needed to emit code for it.
35///
36namespace {
37struct InstructionMemo {
38 StringRef Name;
39 const CodeGenRegisterClass *RC;
40 std::string SubRegNo;
41 std::vector<std::string> PhysRegs;
42 std::string PredicateCheck;
43
44 InstructionMemo(StringRef Name, const CodeGenRegisterClass *RC,
45 std::string SubRegNo, std::vector<std::string> PhysRegs,
46 std::string PredicateCheck)
47 : Name(Name), RC(RC), SubRegNo(std::move(SubRegNo)),
48 PhysRegs(std::move(PhysRegs)),
49 PredicateCheck(std::move(PredicateCheck)) {}
50
51 // Make sure we do not copy InstructionMemo.
52 InstructionMemo(const InstructionMemo &Other) = delete;
53 InstructionMemo(InstructionMemo &&Other) = default;
54};
55} // End anonymous namespace
56
57/// ImmPredicateSet - This uniques predicates (represented as a string) and
58/// gives them unique (small) integer ID's that start at 0.
59namespace {
60class ImmPredicateSet {
61 DenseMap<TreePattern *, unsigned> ImmIDs;
62 std::vector<TreePredicateFn> PredsByName;
63
64public:
65 unsigned getIDFor(TreePredicateFn Pred) {
66 unsigned &Entry = ImmIDs[Pred.getOrigPatFragRecord()];
67 if (Entry == 0) {
68 PredsByName.push_back(x: Pred);
69 Entry = PredsByName.size();
70 }
71 return Entry - 1;
72 }
73
74 const TreePredicateFn &getPredicate(unsigned Idx) { return PredsByName[Idx]; }
75
76 typedef std::vector<TreePredicateFn>::const_iterator iterator;
77 iterator begin() const { return PredsByName.begin(); }
78 iterator end() const { return PredsByName.end(); }
79};
80} // End anonymous namespace
81
82/// OperandsSignature - This class holds a description of a list of operand
83/// types. It has utility methods for emitting text based on the operands.
84///
85namespace {
86struct OperandsSignature {
87 class OpKind {
88 enum { OK_Reg, OK_FP, OK_Imm, OK_Invalid = -1 };
89 char Repr = OK_Invalid;
90
91 public:
92 OpKind() {}
93
94 bool operator<(OpKind RHS) const { return Repr < RHS.Repr; }
95 bool operator==(OpKind RHS) const { return Repr == RHS.Repr; }
96
97 static OpKind getReg() {
98 OpKind K;
99 K.Repr = OK_Reg;
100 return K;
101 }
102 static OpKind getFP() {
103 OpKind K;
104 K.Repr = OK_FP;
105 return K;
106 }
107 static OpKind getImm(unsigned V) {
108 assert((unsigned)OK_Imm + V < 128 &&
109 "Too many integer predicates for the 'Repr' char");
110 OpKind K;
111 K.Repr = OK_Imm + V;
112 return K;
113 }
114
115 bool isReg() const { return Repr == OK_Reg; }
116 bool isFP() const { return Repr == OK_FP; }
117 bool isImm() const { return Repr >= OK_Imm; }
118
119 unsigned getImmCode() const {
120 assert(isImm());
121 return Repr - OK_Imm;
122 }
123
124 void printManglingSuffix(raw_ostream &OS, ImmPredicateSet &ImmPredicates,
125 bool StripImmCodes) const {
126 if (isReg())
127 OS << 'r';
128 else if (isFP())
129 OS << 'f';
130 else {
131 OS << 'i';
132 if (!StripImmCodes)
133 if (unsigned Code = getImmCode())
134 OS << "_" << ImmPredicates.getPredicate(Idx: Code - 1).getFnName();
135 }
136 }
137 };
138
139 SmallVector<OpKind, 3> Operands;
140
141 bool operator<(const OperandsSignature &O) const {
142 return Operands < O.Operands;
143 }
144 bool operator==(const OperandsSignature &O) const {
145 return Operands == O.Operands;
146 }
147
148 bool empty() const { return Operands.empty(); }
149
150 bool hasAnyImmediateCodes() const {
151 return llvm::any_of(Range: Operands, P: [](OpKind Kind) {
152 return Kind.isImm() && Kind.getImmCode() != 0;
153 });
154 }
155
156 /// getWithoutImmCodes - Return a copy of this with any immediate codes forced
157 /// to zero.
158 OperandsSignature getWithoutImmCodes() const {
159 OperandsSignature Result;
160 Result.Operands.resize(N: Operands.size());
161 llvm::transform(Range: Operands, d_first: Result.Operands.begin(), F: [](OpKind Kind) {
162 return Kind.isImm() ? OpKind::getImm(V: 0) : Kind;
163 });
164 return Result;
165 }
166
167 void emitImmediatePredicate(raw_ostream &OS,
168 ImmPredicateSet &ImmPredicates) const {
169 ListSeparator LS(" &&\n ");
170 for (auto [Idx, Opnd] : enumerate(First: Operands)) {
171 if (!Opnd.isImm())
172 continue;
173
174 unsigned Code = Opnd.getImmCode();
175 if (Code == 0)
176 continue;
177
178 TreePredicateFn PredFn = ImmPredicates.getPredicate(Idx: Code - 1);
179
180 // Emit the type check.
181 TreePattern *TP = PredFn.getOrigPatFragRecord();
182 ValueTypeByHwMode VVT = TP->getTree(i: 0)->getType(ResNo: 0);
183 assert(VVT.isSimple() &&
184 "Cannot use variable value types with fast isel");
185 OS << LS << "VT == " << getEnumName(T: VVT.getSimple().SimpleTy) << " && ";
186
187 OS << PredFn.getFnName() << "(imm" << Idx << ')';
188 }
189 }
190
191 /// initialize - Examine the given pattern and initialize the contents
192 /// of the Operands array accordingly. Return true if all the operands
193 /// are supported, false otherwise.
194 ///
195 bool initialize(TreePatternNode &InstPatNode, const CodeGenTarget &Target,
196 MVT::SimpleValueType VT, ImmPredicateSet &ImmediatePredicates,
197 const CodeGenRegisterClass *OrigDstRC) {
198 if (InstPatNode.isLeaf())
199 return false;
200
201 if (InstPatNode.getOperator()->getName() == "imm") {
202 Operands.push_back(Elt: OpKind::getImm(V: 0));
203 return true;
204 }
205
206 if (InstPatNode.getOperator()->getName() == "fpimm") {
207 Operands.push_back(Elt: OpKind::getFP());
208 return true;
209 }
210
211 const CodeGenRegisterClass *DstRC = nullptr;
212
213 for (const TreePatternNode &Op : InstPatNode.children()) {
214 // Handle imm operands specially.
215 if (!Op.isLeaf() && Op.getOperator()->getName() == "imm") {
216 unsigned PredNo = 0;
217 if (!Op.getPredicateCalls().empty()) {
218 TreePredicateFn PredFn = Op.getPredicateCalls()[0].Fn;
219 // If there is more than one predicate weighing in on this operand
220 // then we don't handle it. This doesn't typically happen for
221 // immediates anyway.
222 if (Op.getPredicateCalls().size() > 1 ||
223 !PredFn.isImmediatePattern() || PredFn.usesOperands())
224 return false;
225 // Ignore any instruction with 'FastIselShouldIgnore', these are
226 // not needed and just bloat the fast instruction selector. For
227 // example, X86 doesn't need to generate code to match ADD16ri8 since
228 // ADD16ri will do just fine.
229 const Record *Rec = PredFn.getOrigPatFragRecord()->getRecord();
230 if (Rec->getValueAsBit(FieldName: "FastIselShouldIgnore"))
231 return false;
232
233 PredNo = ImmediatePredicates.getIDFor(Pred: PredFn) + 1;
234 }
235
236 Operands.push_back(Elt: OpKind::getImm(V: PredNo));
237 continue;
238 }
239
240 // For now, filter out any operand with a predicate.
241 // For now, filter out any operand with multiple values.
242 if (!Op.getPredicateCalls().empty() || Op.getNumTypes() != 1)
243 return false;
244
245 if (!Op.isLeaf()) {
246 if (Op.getOperator()->getName() == "fpimm") {
247 Operands.push_back(Elt: OpKind::getFP());
248 continue;
249 }
250 // For now, ignore other non-leaf nodes.
251 return false;
252 }
253
254 assert(Op.hasConcreteType(0) && "Type infererence not done?");
255
256 // For now, all the operands must have the same type (if they aren't
257 // immediates). Note that this causes us to reject variable sized shifts
258 // on X86.
259 if (Op.getSimpleType(ResNo: 0) != VT)
260 return false;
261
262 const DefInit *OpDI = dyn_cast<DefInit>(Val: Op.getLeafValue());
263 if (!OpDI)
264 return false;
265 const Record *OpLeafRec = OpDI->getDef();
266
267 // For now, the only other thing we accept is register operands.
268 const CodeGenRegisterClass *RC = nullptr;
269 if (OpLeafRec->isSubClassOf(Name: "RegisterOperand"))
270 OpLeafRec = OpLeafRec->getValueAsDef(FieldName: "RegClass");
271 if (OpLeafRec->isSubClassOf(Name: "RegisterClass"))
272 RC = &Target.getRegisterClass(R: OpLeafRec);
273 else if (OpLeafRec->isSubClassOf(Name: "Register"))
274 RC = Target.getRegBank().getRegClassForRegister(R: OpLeafRec);
275 else if (OpLeafRec->isSubClassOf(Name: "ValueType"))
276 RC = OrigDstRC;
277 else
278 return false;
279
280 // For now, this needs to be a register class of some sort.
281 if (!RC)
282 return false;
283
284 // For now, all the operands must have the same register class or be
285 // a strict subclass of the destination.
286 if (DstRC) {
287 if (DstRC != RC && !DstRC->hasSubClass(RC))
288 return false;
289 } else {
290 DstRC = RC;
291 }
292 Operands.push_back(Elt: OpKind::getReg());
293 }
294 return true;
295 }
296
297 void PrintParameters(raw_ostream &OS) const {
298 ListSeparator LS;
299 for (auto [Idx, Opnd] : enumerate(First: Operands)) {
300 OS << LS;
301 if (Opnd.isReg())
302 OS << "Register Op" << Idx;
303 else if (Opnd.isImm())
304 OS << "uint64_t imm" << Idx;
305 else if (Opnd.isFP())
306 OS << "const ConstantFP *f" << Idx;
307 else
308 llvm_unreachable("Unknown operand kind!");
309 }
310 }
311
312 void PrintArguments(raw_ostream &OS, ArrayRef<std::string> PhyRegs) const {
313 ListSeparator LS;
314 for (auto [Idx, Opnd, PhyReg] : enumerate(First: Operands, Rest&: PhyRegs)) {
315 if (!PhyReg.empty()) {
316 // Implicit physical register operand.
317 continue;
318 }
319
320 OS << LS;
321 if (Opnd.isReg())
322 OS << "Op" << Idx;
323 else if (Opnd.isImm())
324 OS << "imm" << Idx;
325 else if (Opnd.isFP())
326 OS << "f" << Idx;
327 else
328 llvm_unreachable("Unknown operand kind!");
329 }
330 }
331
332 void PrintArguments(raw_ostream &OS) const {
333 ListSeparator LS;
334 for (auto [Idx, Opnd] : enumerate(First: Operands)) {
335 OS << LS;
336 if (Opnd.isReg())
337 OS << "Op" << Idx;
338 else if (Opnd.isImm())
339 OS << "imm" << Idx;
340 else if (Opnd.isFP())
341 OS << "f" << Idx;
342 else
343 llvm_unreachable("Unknown operand kind!");
344 }
345 }
346
347 void PrintManglingSuffix(raw_ostream &OS, ArrayRef<std::string> PhyRegs,
348 ImmPredicateSet &ImmPredicates,
349 bool StripImmCodes = false) const {
350 for (auto [PhyReg, Opnd] : zip_equal(t&: PhyRegs, u: Operands)) {
351 if (!PhyReg.empty()) {
352 // Implicit physical register operand. e.g. Instruction::Mul expect to
353 // select to a binary op. On x86, mul may take a single operand with
354 // the other operand being implicit. We must emit something that looks
355 // like a binary instruction except for the very inner fastEmitInst_*
356 // call.
357 continue;
358 }
359 Opnd.printManglingSuffix(OS, ImmPredicates, StripImmCodes);
360 }
361 }
362
363 void PrintManglingSuffix(raw_ostream &OS, ImmPredicateSet &ImmPredicates,
364 bool StripImmCodes = false) const {
365 for (OpKind Opnd : Operands)
366 Opnd.printManglingSuffix(OS, ImmPredicates, StripImmCodes);
367 }
368};
369} // End anonymous namespace
370
371namespace {
372class FastISelMap {
373 // A multimap is needed instead of a "plain" map because the key is
374 // the instruction's complexity (an int) and they are not unique.
375 typedef std::multimap<int, InstructionMemo> PredMap;
376 typedef std::map<MVT::SimpleValueType, PredMap> RetPredMap;
377 typedef std::map<MVT::SimpleValueType, RetPredMap> TypeRetPredMap;
378 typedef std::map<StringRef, TypeRetPredMap> OpcodeTypeRetPredMap;
379 typedef std::map<OperandsSignature, OpcodeTypeRetPredMap>
380 OperandsOpcodeTypeRetPredMap;
381
382 OperandsOpcodeTypeRetPredMap SimplePatterns;
383
384 // This is used to check that there are no duplicate predicates
385 std::set<std::tuple<OperandsSignature, StringRef, MVT::SimpleValueType,
386 MVT::SimpleValueType, std::string>>
387 SimplePatternsCheck;
388
389 std::map<OperandsSignature, std::vector<OperandsSignature>>
390 SignaturesWithConstantForms;
391
392 StringRef InstNS;
393 ImmPredicateSet ImmediatePredicates;
394
395public:
396 explicit FastISelMap(StringRef InstNS);
397
398 void collectPatterns(const CodeGenDAGPatterns &CGP);
399 void printImmediatePredicates(raw_ostream &OS);
400 void printFunctionDefinitions(raw_ostream &OS);
401
402private:
403 void emitInstructionCode(raw_ostream &OS, const OperandsSignature &Operands,
404 const PredMap &PM, StringRef RetVTName);
405};
406} // End anonymous namespace
407
408static std::string getLegalCName(StringRef OpName) {
409 std::string CName = OpName.str();
410 std::string::size_type Pos = CName.find(s: "::");
411 if (Pos != std::string::npos)
412 CName.replace(pos: Pos, n1: 2, s: "_");
413 return CName;
414}
415
416FastISelMap::FastISelMap(StringRef instns) : InstNS(instns) {}
417
418static std::string PhysRegForNode(const TreePatternNode &Op,
419 const CodeGenTarget &Target) {
420 std::string PhysReg;
421
422 if (!Op.isLeaf())
423 return PhysReg;
424
425 const Record *OpLeafRec = cast<DefInit>(Val: Op.getLeafValue())->getDef();
426 if (!OpLeafRec->isSubClassOf(Name: "Register"))
427 return PhysReg;
428
429 PhysReg += cast<StringInit>(Val: OpLeafRec->getValue(Name: "Namespace")->getValue())
430 ->getValue();
431 PhysReg += "::";
432 PhysReg += Target.getRegBank().getReg(OpLeafRec)->getName();
433 return PhysReg;
434}
435
436void FastISelMap::collectPatterns(const CodeGenDAGPatterns &CGP) {
437 const CodeGenTarget &Target = CGP.getTargetInfo();
438
439 // Scan through all the patterns and record the simple ones.
440 for (const PatternToMatch &Pattern : CGP.ptms()) {
441 // For now, just look at Instructions, so that we don't have to worry
442 // about emitting multiple instructions for a pattern.
443 TreePatternNode &Dst = Pattern.getDstPattern();
444 if (Dst.isLeaf())
445 continue;
446 const Record *Op = Dst.getOperator();
447 if (!Op->isSubClassOf(Name: "Instruction"))
448 continue;
449 CodeGenInstruction &Inst = CGP.getTargetInfo().getInstruction(InstRec: Op);
450 if (Inst.Operands.empty())
451 continue;
452
453 // Allow instructions to be marked as unavailable for FastISel for
454 // certain cases, i.e. an ISA has two 'and' instruction which differ
455 // by what registers they can use but are otherwise identical for
456 // codegen purposes.
457 if (Inst.FastISelShouldIgnore)
458 continue;
459
460 // For now, ignore multi-instruction patterns.
461 bool MultiInsts = false;
462 for (const TreePatternNode &ChildOp : Dst.children()) {
463 if (ChildOp.isLeaf())
464 continue;
465 if (ChildOp.getOperator()->isSubClassOf(Name: "Instruction")) {
466 MultiInsts = true;
467 break;
468 }
469 }
470 if (MultiInsts)
471 continue;
472
473 // For now, ignore instructions where the first operand is not an
474 // output register.
475 const CodeGenRegisterClass *DstRC = nullptr;
476 std::string SubRegNo;
477 if (Op->getName() != "EXTRACT_SUBREG") {
478 const Record *Op0Rec = Inst.Operands[0].Rec;
479 if (Op0Rec->isSubClassOf(Name: "RegisterOperand"))
480 Op0Rec = Op0Rec->getValueAsDef(FieldName: "RegClass");
481 if (!Op0Rec->isSubClassOf(Name: "RegisterClass"))
482 continue;
483 DstRC = &Target.getRegisterClass(R: Op0Rec);
484 if (!DstRC)
485 continue;
486 } else {
487 // If this isn't a leaf, then continue since the register classes are
488 // a bit too complicated for now.
489 if (!Dst.getChild(N: 1).isLeaf())
490 continue;
491
492 const DefInit *SR = dyn_cast<DefInit>(Val: Dst.getChild(N: 1).getLeafValue());
493 if (SR)
494 SubRegNo = getQualifiedName(R: SR->getDef());
495 else
496 SubRegNo = Dst.getChild(N: 1).getLeafValue()->getAsString();
497 }
498
499 // Inspect the pattern.
500 TreePatternNode &InstPatNode = Pattern.getSrcPattern();
501 if (InstPatNode.isLeaf())
502 continue;
503
504 // Ignore multiple result nodes for now.
505 if (InstPatNode.getNumTypes() > 1)
506 continue;
507
508 const Record *InstPatOp = InstPatNode.getOperator();
509 StringRef OpcodeName = CGP.getSDNodeInfo(R: InstPatOp).getEnumName();
510 MVT::SimpleValueType RetVT = MVT::isVoid;
511 if (InstPatNode.getNumTypes())
512 RetVT = InstPatNode.getSimpleType(ResNo: 0);
513 MVT::SimpleValueType VT = RetVT;
514 if (InstPatNode.getNumChildren()) {
515 assert(InstPatNode.getChild(0).getNumTypes() == 1);
516 VT = InstPatNode.getChild(N: 0).getSimpleType(ResNo: 0);
517 }
518
519 // For now, filter out any instructions with predicates.
520 if (!InstPatNode.getPredicateCalls().empty())
521 continue;
522
523 // Check all the operands.
524 OperandsSignature Operands;
525 if (!Operands.initialize(InstPatNode, Target, VT, ImmediatePredicates,
526 OrigDstRC: DstRC))
527 continue;
528
529 std::vector<std::string> PhysRegInputs;
530 if (InstPatNode.getOperator()->getName() == "imm" ||
531 InstPatNode.getOperator()->getName() == "fpimm")
532 PhysRegInputs.push_back(x: "");
533 else {
534 // Compute the PhysRegs used by the given pattern, and check that
535 // the mapping from the src to dst patterns is simple.
536 bool FoundNonSimplePattern = false;
537 unsigned DstIndex = 0;
538 for (const TreePatternNode &SrcChild : InstPatNode.children()) {
539 std::string PhysReg = PhysRegForNode(Op: SrcChild, Target);
540 if (PhysReg.empty()) {
541 if (DstIndex >= Dst.getNumChildren() ||
542 Dst.getChild(N: DstIndex).getName() != SrcChild.getName()) {
543 FoundNonSimplePattern = true;
544 break;
545 }
546 ++DstIndex;
547 }
548
549 PhysRegInputs.push_back(x: std::move(PhysReg));
550 }
551
552 if (Op->getName() != "EXTRACT_SUBREG" && DstIndex < Dst.getNumChildren())
553 FoundNonSimplePattern = true;
554
555 if (FoundNonSimplePattern)
556 continue;
557 }
558
559 // Check if the operands match one of the patterns handled by FastISel.
560 std::string ManglingSuffix;
561 raw_string_ostream SuffixOS(ManglingSuffix);
562 Operands.PrintManglingSuffix(OS&: SuffixOS, ImmPredicates&: ImmediatePredicates, StripImmCodes: true);
563 if (!StringSwitch<bool>(ManglingSuffix)
564 .Cases(S0: "", S1: "r", S2: "rr", S3: "ri", S4: "i", S5: "f", Value: true)
565 .Default(Value: false))
566 continue;
567
568 // Get the predicate that guards this pattern.
569 std::string PredicateCheck = Pattern.getPredicateCheck();
570
571 // Ok, we found a pattern that we can handle. Remember it.
572 InstructionMemo Memo(Pattern.getDstPattern().getOperator()->getName(),
573 DstRC, std::move(SubRegNo), std::move(PhysRegInputs),
574 PredicateCheck);
575
576 int Complexity = Pattern.getPatternComplexity(CGP);
577
578 auto inserted_simple_pattern = SimplePatternsCheck.insert(
579 x: {Operands, OpcodeName, VT, RetVT, PredicateCheck});
580 if (!inserted_simple_pattern.second) {
581 PrintFatalError(ErrorLoc: Pattern.getSrcRecord()->getLoc(),
582 Msg: "Duplicate predicate in FastISel table!");
583 }
584
585 // Note: Instructions with the same complexity will appear in the order
586 // that they are encountered.
587 SimplePatterns[Operands][OpcodeName][VT][RetVT].emplace(args&: Complexity,
588 args: std::move(Memo));
589
590 // If any of the operands were immediates with predicates on them, strip
591 // them down to a signature that doesn't have predicates so that we can
592 // associate them with the stripped predicate version.
593 if (Operands.hasAnyImmediateCodes()) {
594 SignaturesWithConstantForms[Operands.getWithoutImmCodes()].push_back(
595 x: Operands);
596 }
597 }
598}
599
600void FastISelMap::printImmediatePredicates(raw_ostream &OS) {
601 if (ImmediatePredicates.begin() == ImmediatePredicates.end())
602 return;
603
604 OS << "\n// FastEmit Immediate Predicate functions.\n";
605 for (auto ImmediatePredicate : ImmediatePredicates) {
606 OS << "static bool " << ImmediatePredicate.getFnName()
607 << "(int64_t Imm) {\n";
608 OS << ImmediatePredicate.getImmediatePredicateCode() << "\n}\n";
609 }
610
611 OS << "\n\n";
612}
613
614void FastISelMap::emitInstructionCode(raw_ostream &OS,
615 const OperandsSignature &Operands,
616 const PredMap &PM, StringRef RetVTName) {
617 // Emit code for each possible instruction. There may be
618 // multiple if there are subtarget concerns. A reverse iterator
619 // is used to produce the ones with highest complexity first.
620
621 bool OneHadNoPredicate = false;
622 for (const auto &[_, Memo] : reverse(C: PM)) {
623 std::string PredicateCheck = Memo.PredicateCheck;
624
625 if (PredicateCheck.empty()) {
626 assert(!OneHadNoPredicate &&
627 "Multiple instructions match and more than one had "
628 "no predicate!");
629 OneHadNoPredicate = true;
630 } else {
631 if (OneHadNoPredicate) {
632 PrintFatalError(Msg: "Multiple instructions match and one with no "
633 "predicate came before one with a predicate! "
634 "name:" +
635 Memo.Name + " predicate: " + PredicateCheck);
636 }
637 OS << " if (" + PredicateCheck + ") {\n";
638 OS << " ";
639 }
640
641 for (auto [Idx, PhyReg] : enumerate(First: Memo.PhysRegs)) {
642 if (!PhyReg.empty())
643 OS << " BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, "
644 << "TII.get(TargetOpcode::COPY), " << PhyReg << ").addReg(Op" << Idx
645 << ");\n";
646 }
647
648 OS << " return fastEmitInst_";
649 if (Memo.SubRegNo.empty()) {
650 Operands.PrintManglingSuffix(OS, PhyRegs: Memo.PhysRegs, ImmPredicates&: ImmediatePredicates,
651 StripImmCodes: true);
652 OS << "(" << InstNS << "::" << Memo.Name << ", ";
653 OS << "&" << InstNS << "::" << Memo.RC->getName() << "RegClass";
654 if (!Operands.empty())
655 OS << ", ";
656 Operands.PrintArguments(OS, PhyRegs: Memo.PhysRegs);
657 OS << ");\n";
658 } else {
659 OS << "extractsubreg(" << RetVTName << ", Op0, " << Memo.SubRegNo
660 << ");\n";
661 }
662
663 if (!PredicateCheck.empty())
664 OS << " }\n";
665 }
666 // Return Register() if all of the possibilities had predicates but none
667 // were satisfied.
668 if (!OneHadNoPredicate)
669 OS << " return Register();\n";
670 OS << "}\n";
671 OS << "\n";
672}
673
674void FastISelMap::printFunctionDefinitions(raw_ostream &OS) {
675 // Now emit code for all the patterns that we collected.
676 for (const auto &SimplePattern : SimplePatterns) {
677 const OperandsSignature &Operands = SimplePattern.first;
678 const OpcodeTypeRetPredMap &OTM = SimplePattern.second;
679
680 for (const auto &[Opcode, TM] : OTM) {
681 OS << "// FastEmit functions for " << Opcode << ".\n";
682 OS << "\n";
683
684 // Emit one function for each opcode,type pair.
685 for (const auto &[VT, RM] : TM) {
686 if (RM.size() != 1) {
687 for (const auto &[RetVT, PM] : RM) {
688 OS << "Register fastEmit_" << getLegalCName(OpName: Opcode) << "_"
689 << getLegalCName(OpName: getEnumName(T: VT)) << "_"
690 << getLegalCName(OpName: getEnumName(T: RetVT)) << "_";
691 Operands.PrintManglingSuffix(OS, ImmPredicates&: ImmediatePredicates);
692 OS << "(";
693 Operands.PrintParameters(OS);
694 OS << ") {\n";
695
696 emitInstructionCode(OS, Operands, PM, RetVTName: getEnumName(T: RetVT));
697 }
698
699 // Emit one function for the type that demultiplexes on return type.
700 OS << "Register fastEmit_" << getLegalCName(OpName: Opcode) << "_"
701 << getLegalCName(OpName: getEnumName(T: VT)) << "_";
702 Operands.PrintManglingSuffix(OS, ImmPredicates&: ImmediatePredicates);
703 OS << "(MVT RetVT";
704 if (!Operands.empty())
705 OS << ", ";
706 Operands.PrintParameters(OS);
707 OS << ") {\nswitch (RetVT.SimpleTy) {\n";
708 for (const auto &[RetVT, _] : RM) {
709 OS << " case " << getEnumName(T: RetVT) << ": return fastEmit_"
710 << getLegalCName(OpName: Opcode) << "_" << getLegalCName(OpName: getEnumName(T: VT))
711 << "_" << getLegalCName(OpName: getEnumName(T: RetVT)) << "_";
712 Operands.PrintManglingSuffix(OS, ImmPredicates&: ImmediatePredicates);
713 OS << "(";
714 Operands.PrintArguments(OS);
715 OS << ");\n";
716 }
717 OS << " default: return Register();\n}\n}\n\n";
718
719 } else {
720 // Non-variadic return type.
721 OS << "Register fastEmit_" << getLegalCName(OpName: Opcode) << "_"
722 << getLegalCName(OpName: getEnumName(T: VT)) << "_";
723 Operands.PrintManglingSuffix(OS, ImmPredicates&: ImmediatePredicates);
724 OS << "(MVT RetVT";
725 if (!Operands.empty())
726 OS << ", ";
727 Operands.PrintParameters(OS);
728 OS << ") {\n";
729
730 OS << " if (RetVT.SimpleTy != " << getEnumName(T: RM.begin()->first)
731 << ")\n return Register();\n";
732
733 const PredMap &PM = RM.begin()->second;
734
735 emitInstructionCode(OS, Operands, PM, RetVTName: "RetVT");
736 }
737 }
738
739 // Emit one function for the opcode that demultiplexes based on the type.
740 OS << "Register fastEmit_" << getLegalCName(OpName: Opcode) << "_";
741 Operands.PrintManglingSuffix(OS, ImmPredicates&: ImmediatePredicates);
742 OS << "(MVT VT, MVT RetVT";
743 if (!Operands.empty())
744 OS << ", ";
745 Operands.PrintParameters(OS);
746 OS << ") {\n";
747 OS << " switch (VT.SimpleTy) {\n";
748 for (const auto &[VT, _] : TM) {
749 StringRef TypeName = getEnumName(T: VT);
750 OS << " case " << TypeName << ": return fastEmit_"
751 << getLegalCName(OpName: Opcode) << "_" << getLegalCName(OpName: TypeName) << "_";
752 Operands.PrintManglingSuffix(OS, ImmPredicates&: ImmediatePredicates);
753 OS << "(RetVT";
754 if (!Operands.empty())
755 OS << ", ";
756 Operands.PrintArguments(OS);
757 OS << ");\n";
758 }
759 OS << " default: return Register();\n";
760 OS << " }\n";
761 OS << "}\n";
762 OS << "\n";
763 }
764
765 OS << "// Top-level FastEmit function.\n";
766 OS << "\n";
767
768 // Emit one function for the operand signature that demultiplexes based
769 // on opcode and type.
770 OS << "Register fastEmit_";
771 Operands.PrintManglingSuffix(OS, ImmPredicates&: ImmediatePredicates);
772 OS << "(MVT VT, MVT RetVT, unsigned Opcode";
773 if (!Operands.empty())
774 OS << ", ";
775 Operands.PrintParameters(OS);
776 OS << ") ";
777 if (!Operands.hasAnyImmediateCodes())
778 OS << "override ";
779 OS << "{\n";
780
781 // If there are any forms of this signature available that operate on
782 // constrained forms of the immediate (e.g., 32-bit sext immediate in a
783 // 64-bit operand), check them first.
784
785 std::map<OperandsSignature, std::vector<OperandsSignature>>::iterator MI =
786 SignaturesWithConstantForms.find(x: Operands);
787 if (MI != SignaturesWithConstantForms.end()) {
788 // Unique any duplicates out of the list.
789 llvm::sort(C&: MI->second);
790 MI->second.erase(first: llvm::unique(R&: MI->second), last: MI->second.end());
791
792 // Check each in order it was seen. It would be nice to have a good
793 // relative ordering between them, but we're not going for optimality
794 // here.
795 for (const OperandsSignature &Sig : MI->second) {
796 OS << " if (";
797 Sig.emitImmediatePredicate(OS, ImmPredicates&: ImmediatePredicates);
798 OS << ")\n if (Register Reg = fastEmit_";
799 Sig.PrintManglingSuffix(OS, ImmPredicates&: ImmediatePredicates);
800 OS << "(VT, RetVT, Opcode";
801 if (!Sig.empty())
802 OS << ", ";
803 Sig.PrintArguments(OS);
804 OS << "))\n return Reg;\n\n";
805 }
806
807 // Done with this, remove it.
808 SignaturesWithConstantForms.erase(position: MI);
809 }
810
811 OS << " switch (Opcode) {\n";
812 for (const auto &[Opcode, _] : OTM) {
813 OS << " case " << Opcode << ": return fastEmit_" << getLegalCName(OpName: Opcode)
814 << "_";
815 Operands.PrintManglingSuffix(OS, ImmPredicates&: ImmediatePredicates);
816 OS << "(VT, RetVT";
817 if (!Operands.empty())
818 OS << ", ";
819 Operands.PrintArguments(OS);
820 OS << ");\n";
821 }
822 OS << " default: return Register();\n";
823 OS << " }\n";
824 OS << "}\n";
825 OS << "\n";
826 }
827
828 // TODO: SignaturesWithConstantForms should be empty here.
829}
830
831static void EmitFastISel(const RecordKeeper &RK, raw_ostream &OS) {
832 const CodeGenDAGPatterns CGP(RK);
833 const CodeGenTarget &Target = CGP.getTargetInfo();
834 emitSourceFileHeader(Desc: "\"Fast\" Instruction Selector for the " +
835 Target.getName().str() + " target",
836 OS);
837
838 // Determine the target's namespace name.
839 StringRef InstNS = Target.getInstNamespace();
840 assert(!InstNS.empty() && "Can't determine target-specific namespace!");
841
842 FastISelMap F(InstNS);
843 F.collectPatterns(CGP);
844 F.printImmediatePredicates(OS);
845 F.printFunctionDefinitions(OS);
846}
847
848static TableGen::Emitter::Opt X("gen-fast-isel", EmitFastISel,
849 "Generate a \"fast\" instruction selector");
850