1//===- GlobalISelCombinerMatchTableEmitter.cpp - --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file Generate a combiner implementation for GlobalISel from a declarative
10/// syntax using GlobalISelMatchTable.
11///
12/// Usually, TableGen backends use "assert is an error" as a means to report
13/// invalid input. They try to diagnose common case but don't try very hard and
14/// crashes can be common. This backend aims to behave closer to how a language
15/// compiler frontend would behave: we try extra hard to diagnose invalid inputs
16/// early, and any crash should be considered a bug (= a feature or diagnostic
17/// is missing).
18///
19/// While this can make the backend a bit more complex than it needs to be, it
20/// pays off because MIR patterns can get complicated. Giving useful error
21/// messages to combine writers can help boost their productivity.
22///
23/// As with anything, a good balance has to be found. We also don't want to
24/// write hundreds of lines of code to detect edge cases. In practice, crashing
25/// very occasionally, or giving poor errors in some rare instances, is fine.
26///
27//===----------------------------------------------------------------------===//
28
29#include "Basic/CodeGenIntrinsics.h"
30#include "Common/CodeGenInstruction.h"
31#include "Common/CodeGenTarget.h"
32#include "Common/GlobalISel/CXXPredicates.h"
33#include "Common/GlobalISel/CodeExpander.h"
34#include "Common/GlobalISel/CodeExpansions.h"
35#include "Common/GlobalISel/CombinerUtils.h"
36#include "Common/GlobalISel/GlobalISelMatchTable.h"
37#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
38#include "Common/GlobalISel/PatternParser.h"
39#include "Common/GlobalISel/Patterns.h"
40#include "Common/SubtargetFeatureInfo.h"
41#include "llvm/ADT/APInt.h"
42#include "llvm/ADT/EquivalenceClasses.h"
43#include "llvm/ADT/MapVector.h"
44#include "llvm/ADT/Statistic.h"
45#include "llvm/ADT/StringExtras.h"
46#include "llvm/ADT/StringSet.h"
47#include "llvm/Support/CommandLine.h"
48#include "llvm/Support/Debug.h"
49#include "llvm/Support/PrettyStackTrace.h"
50#include "llvm/Support/ScopedPrinter.h"
51#include "llvm/TableGen/Error.h"
52#include "llvm/TableGen/Record.h"
53#include "llvm/TableGen/StringMatcher.h"
54#include "llvm/TableGen/TGTimer.h"
55#include "llvm/TableGen/TableGenBackend.h"
56#include <cstdint>
57
58using namespace llvm;
59using namespace llvm::gi;
60
61#define DEBUG_TYPE "gicombiner-emitter"
62
63namespace {
64cl::OptionCategory
65 GICombinerEmitterCat("Options for -gen-global-isel-combiner");
66cl::opt<bool> StopAfterParse(
67 "gicombiner-stop-after-parse",
68 cl::desc("Stop processing after parsing rules and dump state"),
69 cl::cat(GICombinerEmitterCat));
70cl::list<std::string>
71 SelectedCombiners("combiners", cl::desc("Emit the specified combiners"),
72 cl::cat(GICombinerEmitterCat), cl::CommaSeparated);
73cl::opt<bool> DebugCXXPreds(
74 "gicombiner-debug-cxxpreds",
75 cl::desc("Add Contextual/Debug comments to all C++ predicates"),
76 cl::cat(GICombinerEmitterCat));
77cl::opt<bool> DebugTypeInfer("gicombiner-debug-typeinfer",
78 cl::desc("Print type inference debug logs"),
79 cl::cat(GICombinerEmitterCat));
80
81constexpr StringLiteral CXXCustomActionPrefix = "GICXXCustomAction_";
82constexpr StringLiteral CXXPredPrefix = "GICXXPred_MI_Predicate_";
83constexpr StringLiteral MatchDataClassName = "GIDefMatchData";
84
85//===- CodeExpansions Helpers --------------------------------------------===//
86
87void declareInstExpansion(CodeExpansions &CE, const InstructionMatcher &IM,
88 StringRef Name) {
89 CE.declare(Name, Expansion: "State.MIs[" + to_string(Value: IM.getInsnVarID()) + "]");
90}
91
92void declareInstExpansion(CodeExpansions &CE, const BuildMIAction &A,
93 StringRef Name) {
94 // Note: we use redeclare here because this may overwrite a matcher inst
95 // expansion.
96 CE.redeclare(Name, Expansion: "OutMIs[" + to_string(Value: A.getInsnID()) + "]");
97}
98
99void declareOperandExpansion(CodeExpansions &CE, const OperandMatcher &OM,
100 StringRef Name) {
101 if (OM.isVariadic()) {
102 CE.declare(Name, Expansion: "getRemainingOperands(*State.MIs[" +
103 to_string(Value: OM.getInsnVarID()) + "], " +
104 to_string(Value: OM.getOpIdx()) + ")");
105 } else {
106 CE.declare(Name, Expansion: "State.MIs[" + to_string(Value: OM.getInsnVarID()) +
107 "]->getOperand(" + to_string(Value: OM.getOpIdx()) + ")");
108 }
109}
110
111void declareTempRegExpansion(CodeExpansions &CE, unsigned TempRegID,
112 StringRef Name) {
113 CE.declare(Name, Expansion: "State.TempRegisters[" + to_string(Value: TempRegID) + "]");
114}
115
116//===- Misc. Helpers -----------------------------------------------------===//
117
118template <typename Container> auto keys(Container &&C) {
119 return map_range(C, [](auto &Entry) -> auto & { return Entry.first; });
120}
121
122template <typename Container> auto values(Container &&C) {
123 return map_range(C, [](auto &Entry) -> auto & { return Entry.second; });
124}
125
126std::string getIsEnabledPredicateEnumName(unsigned CombinerRuleID) {
127 return "GICXXPred_Simple_IsRule" + to_string(Value: CombinerRuleID) + "Enabled";
128}
129
130//===- MatchTable Helpers ------------------------------------------------===//
131
132LLTCodeGen getLLTCodeGen(const PatternType &PT) {
133 return *MVTToLLT(SVT: getValueType(Rec: PT.getLLTRecord()));
134}
135
136//===- PrettyStackTrace Helpers ------------------------------------------===//
137
138class PrettyStackTraceParse : public PrettyStackTraceEntry {
139 const Record &Def;
140
141public:
142 PrettyStackTraceParse(const Record &Def) : Def(Def) {}
143
144 void print(raw_ostream &OS) const override {
145 if (Def.isSubClassOf(Name: "GICombineRule"))
146 OS << "Parsing GICombineRule '" << Def.getName() << "'";
147 else if (Def.isSubClassOf(Name: PatFrag::ClassName))
148 OS << "Parsing " << PatFrag::ClassName << " '" << Def.getName() << "'";
149 else
150 OS << "Parsing '" << Def.getName() << "'";
151 OS << '\n';
152 }
153};
154
155class PrettyStackTraceEmit : public PrettyStackTraceEntry {
156 const Record &Def;
157 const Pattern *Pat = nullptr;
158
159public:
160 PrettyStackTraceEmit(const Record &Def, const Pattern *Pat = nullptr)
161 : Def(Def), Pat(Pat) {}
162
163 void print(raw_ostream &OS) const override {
164 if (Def.isSubClassOf(Name: "GICombineRule"))
165 OS << "Emitting GICombineRule '" << Def.getName() << "'";
166 else if (Def.isSubClassOf(Name: PatFrag::ClassName))
167 OS << "Emitting " << PatFrag::ClassName << " '" << Def.getName() << "'";
168 else
169 OS << "Emitting '" << Def.getName() << "'";
170
171 if (Pat)
172 OS << " [" << Pat->getKindName() << " '" << Pat->getName() << "']";
173 OS << '\n';
174 }
175};
176
177//===- CombineRuleOperandTypeChecker --------------------------------------===//
178
179/// This is a wrapper around OperandTypeChecker specialized for Combiner Rules.
180/// On top of doing the same things as OperandTypeChecker, this also attempts to
181/// infer as many types as possible for temporary register defs & immediates in
182/// apply patterns.
183///
184/// The inference is trivial and leverages the MCOI OperandTypes encoded in
185/// CodeGenInstructions to infer types across patterns in a CombineRule. It's
186/// thus very limited and only supports CodeGenInstructions (but that's the main
187/// use case so it's fine).
188///
189/// We only try to infer untyped operands in apply patterns when they're temp
190/// reg defs, or immediates. Inference always outputs a `TypeOf<$x>` where $x is
191/// a named operand from a match pattern.
192class CombineRuleOperandTypeChecker : private OperandTypeChecker {
193public:
194 CombineRuleOperandTypeChecker(const Record &RuleDef,
195 const OperandTable &MatchOpTable)
196 : OperandTypeChecker(RuleDef.getLoc()), RuleDef(RuleDef),
197 MatchOpTable(MatchOpTable) {}
198
199 /// Records and checks a 'match' pattern.
200 bool processMatchPattern(InstructionPattern &P);
201
202 /// Records and checks an 'apply' pattern.
203 bool processApplyPattern(InstructionPattern &P);
204
205 /// Propagates types, then perform type inference and do a second round of
206 /// propagation in the apply patterns only if any types were inferred.
207 void propagateAndInferTypes();
208
209private:
210 /// TypeEquivalenceClasses are groups of operands of an instruction that share
211 /// a common type.
212 ///
213 /// e.g. [[a, b], [c, d]] means a and b have the same type, and c and
214 /// d have the same type too. b/c and a/d don't have to have the same type,
215 /// though.
216 using TypeEquivalenceClasses = EquivalenceClasses<StringRef>;
217
218 /// \returns true for `OPERAND_GENERIC_` 0 through 5.
219 /// These are the MCOI types that can be registers. The other MCOI types are
220 /// either immediates, or fancier operands used only post-ISel, so we don't
221 /// care about them for combiners.
222 static bool canMCOIOperandTypeBeARegister(StringRef MCOIType) {
223 // Assume OPERAND_GENERIC_0 through 5 can be registers. The other MCOI
224 // OperandTypes are either never used in gMIR, or not relevant (e.g.
225 // OPERAND_GENERIC_IMM, which is definitely never a register).
226 return MCOIType.drop_back(N: 1).ends_with(Suffix: "OPERAND_GENERIC_");
227 }
228
229 /// Finds the "MCOI::"" operand types for each operand of \p CGP.
230 ///
231 /// This is a bit trickier than it looks because we need to handle variadic
232 /// in/outs.
233 ///
234 /// e.g. for
235 /// (G_BUILD_VECTOR $vec, $x, $y) ->
236 /// [MCOI::OPERAND_GENERIC_0, MCOI::OPERAND_GENERIC_1,
237 /// MCOI::OPERAND_GENERIC_1]
238 ///
239 /// For unknown types (which can happen in variadics where varargs types are
240 /// inconsistent), a unique name is given, e.g. "unknown_type_0".
241 static std::vector<std::string>
242 getMCOIOperandTypes(const CodeGenInstructionPattern &CGP);
243
244 /// Adds the TypeEquivalenceClasses for \p P in \p OutTECs.
245 void getInstEqClasses(const InstructionPattern &P,
246 TypeEquivalenceClasses &OutTECs) const;
247
248 /// Calls `getInstEqClasses` on all patterns of the rule to produce the whole
249 /// rule's TypeEquivalenceClasses.
250 TypeEquivalenceClasses getRuleEqClasses() const;
251
252 /// Tries to infer the type of the \p ImmOpIdx -th operand of \p IP using \p
253 /// TECs.
254 ///
255 /// This is achieved by trying to find a named operand in \p IP that shares
256 /// the same type as \p ImmOpIdx, and using \ref inferNamedOperandType on that
257 /// operand instead.
258 ///
259 /// \returns the inferred type or an empty PatternType if inference didn't
260 /// succeed.
261 PatternType inferImmediateType(const InstructionPattern &IP,
262 unsigned ImmOpIdx,
263 const TypeEquivalenceClasses &TECs) const;
264
265 /// Looks inside \p TECs to infer \p OpName's type.
266 ///
267 /// \returns the inferred type or an empty PatternType if inference didn't
268 /// succeed.
269 PatternType inferNamedOperandType(const InstructionPattern &IP,
270 StringRef OpName,
271 const TypeEquivalenceClasses &TECs,
272 bool AllowSelf = false) const;
273
274 const Record &RuleDef;
275 SmallVector<InstructionPattern *, 8> MatchPats;
276 SmallVector<InstructionPattern *, 8> ApplyPats;
277
278 const OperandTable &MatchOpTable;
279};
280
281bool CombineRuleOperandTypeChecker::processMatchPattern(InstructionPattern &P) {
282 MatchPats.push_back(Elt: &P);
283 return check(P, /*CheckTypeOf*/ VerifyTypeOfOperand: [](const auto &) {
284 // GITypeOf in 'match' is currently always rejected by the
285 // CombineRuleBuilder after inference is done.
286 return true;
287 });
288}
289
290bool CombineRuleOperandTypeChecker::processApplyPattern(InstructionPattern &P) {
291 ApplyPats.push_back(Elt: &P);
292 return check(P, /*CheckTypeOf*/ VerifyTypeOfOperand: [&](const PatternType &Ty) {
293 // GITypeOf<"$x"> can only be used if "$x" is a matched operand.
294 const auto OpName = Ty.getTypeOfOpName();
295 if (MatchOpTable.lookup(OpName).Found)
296 return true;
297
298 PrintError(ErrorLoc: RuleDef.getLoc(), Msg: "'" + OpName + "' ('" + Ty.str() +
299 "') does not refer to a matched operand!");
300 return false;
301 });
302}
303
304void CombineRuleOperandTypeChecker::propagateAndInferTypes() {
305 /// First step here is to propagate types using the OperandTypeChecker. That
306 /// way we ensure all uses of a given register have consistent types.
307 propagateTypes();
308
309 /// Build the TypeEquivalenceClasses for the whole rule.
310 const TypeEquivalenceClasses TECs = getRuleEqClasses();
311
312 /// Look at the apply patterns and find operands that need to be
313 /// inferred. We then try to find an equivalence class that they're a part of
314 /// and select the best operand to use for the `GITypeOf` type. We prioritize
315 /// defs of matched instructions because those are guaranteed to be registers.
316 bool InferredAny = false;
317 for (auto *Pat : ApplyPats) {
318 for (unsigned K = 0; K < Pat->operands_size(); ++K) {
319 auto &Op = Pat->getOperand(K);
320
321 // We only want to take a look at untyped defs or immediates.
322 if ((!Op.isDef() && !Op.hasImmValue()) || Op.getType())
323 continue;
324
325 // Infer defs & named immediates.
326 if (Op.isDef() || Op.isNamedImmediate()) {
327 // Check it's not a redefinition of a matched operand.
328 // In such cases, inference is not necessary because we just copy
329 // operands and don't create temporary registers.
330 if (MatchOpTable.lookup(OpName: Op.getOperandName()).Found)
331 continue;
332
333 // Inference is needed here, so try to do it.
334 if (PatternType Ty =
335 inferNamedOperandType(IP: *Pat, OpName: Op.getOperandName(), TECs)) {
336 if (DebugTypeInfer)
337 errs() << "INFER: " << Op.describe() << " -> " << Ty.str() << '\n';
338 Op.setType(Ty);
339 InferredAny = true;
340 }
341
342 continue;
343 }
344
345 // Infer immediates
346 if (Op.hasImmValue()) {
347 if (PatternType Ty = inferImmediateType(IP: *Pat, ImmOpIdx: K, TECs)) {
348 if (DebugTypeInfer)
349 errs() << "INFER: " << Op.describe() << " -> " << Ty.str() << '\n';
350 Op.setType(Ty);
351 InferredAny = true;
352 }
353 continue;
354 }
355 }
356 }
357
358 // If we've inferred any types, we want to propagate them across the apply
359 // patterns. Type inference only adds GITypeOf types that point to Matched
360 // operands, so we definitely don't want to propagate types into the match
361 // patterns as well, otherwise bad things happen.
362 if (InferredAny) {
363 OperandTypeChecker OTC(RuleDef.getLoc());
364 for (auto *Pat : ApplyPats) {
365 if (!OTC.check(P&: *Pat, VerifyTypeOfOperand: [&](const auto &) { return true; }))
366 PrintFatalError(ErrorLoc: RuleDef.getLoc(),
367 Msg: "OperandTypeChecker unexpectedly failed on '" +
368 Pat->getName() + "' during Type Inference");
369 }
370 OTC.propagateTypes();
371
372 if (DebugTypeInfer) {
373 errs() << "Apply patterns for rule " << RuleDef.getName()
374 << " after inference:\n";
375 for (auto *Pat : ApplyPats) {
376 errs() << " ";
377 Pat->print(OS&: errs(), /*PrintName*/ true);
378 errs() << '\n';
379 }
380 errs() << '\n';
381 }
382 }
383}
384
385PatternType CombineRuleOperandTypeChecker::inferImmediateType(
386 const InstructionPattern &IP, unsigned ImmOpIdx,
387 const TypeEquivalenceClasses &TECs) const {
388 // We can only infer CGPs (except intrinsics).
389 const auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: &IP);
390 if (!CGP || CGP->isIntrinsic())
391 return {};
392
393 // For CGPs, we try to infer immediates by trying to infer another named
394 // operand that shares its type.
395 //
396 // e.g.
397 // Pattern: G_BUILD_VECTOR $x, $y, 0
398 // MCOIs: [MCOI::OPERAND_GENERIC_0, MCOI::OPERAND_GENERIC_1,
399 // MCOI::OPERAND_GENERIC_1]
400 // $y has the same type as 0, so we can infer $y and get the type 0 should
401 // have.
402
403 // We infer immediates by looking for a named operand that shares the same
404 // MCOI type.
405 const auto MCOITypes = getMCOIOperandTypes(CGP: *CGP);
406 StringRef ImmOpTy = MCOITypes[ImmOpIdx];
407
408 for (const auto &[Idx, Ty] : enumerate(First: MCOITypes)) {
409 if (Idx != ImmOpIdx && Ty == ImmOpTy) {
410 const auto &Op = IP.getOperand(K: Idx);
411 if (!Op.isNamedOperand())
412 continue;
413
414 // Named operand with the same name, try to infer that.
415 if (PatternType InferTy = inferNamedOperandType(IP, OpName: Op.getOperandName(),
416 TECs, /*AllowSelf=*/true))
417 return InferTy;
418 }
419 }
420
421 return {};
422}
423
424PatternType CombineRuleOperandTypeChecker::inferNamedOperandType(
425 const InstructionPattern &IP, StringRef OpName,
426 const TypeEquivalenceClasses &TECs, bool AllowSelf) const {
427 // This is the simplest possible case, we just need to find a TEC that
428 // contains OpName. Look at all operands in equivalence class and try to
429 // find a suitable one. If `AllowSelf` is true, the operand itself is also
430 // considered suitable.
431
432 // Check for a def of a matched pattern. This is guaranteed to always
433 // be a register so we can blindly use that.
434 StringRef GoodOpName;
435 for (auto It = TECs.findLeader(V: OpName); It != TECs.member_end(); ++It) {
436 if (!AllowSelf && *It == OpName)
437 continue;
438
439 const auto LookupRes = MatchOpTable.lookup(OpName: *It);
440 if (LookupRes.Def) // Favor defs
441 return PatternType::getTypeOf(OpName: *It);
442
443 // Otherwise just save this in case we don't find any def.
444 if (GoodOpName.empty() && LookupRes.Found)
445 GoodOpName = *It;
446 }
447
448 if (!GoodOpName.empty())
449 return PatternType::getTypeOf(OpName: GoodOpName);
450
451 // No good operand found, give up.
452 return {};
453}
454
455std::vector<std::string> CombineRuleOperandTypeChecker::getMCOIOperandTypes(
456 const CodeGenInstructionPattern &CGP) {
457 // FIXME?: Should we cache this? We call it twice when inferring immediates.
458
459 static unsigned UnknownTypeIdx = 0;
460
461 std::vector<std::string> OpTypes;
462 auto &CGI = CGP.getInst();
463 const Record *VarArgsTy =
464 CGI.TheDef->isSubClassOf(Name: "GenericInstruction")
465 ? CGI.TheDef->getValueAsOptionalDef(FieldName: "variadicOpsType")
466 : nullptr;
467 std::string VarArgsTyName =
468 VarArgsTy ? ("MCOI::" + VarArgsTy->getValueAsString(FieldName: "OperandType")).str()
469 : ("unknown_type_" + Twine(UnknownTypeIdx++)).str();
470
471 // First, handle defs.
472 for (unsigned K = 0; K < CGI.Operands.NumDefs; ++K)
473 OpTypes.push_back(x: CGI.Operands[K].OperandType);
474
475 // Then, handle variadic defs if there are any.
476 if (CGP.hasVariadicDefs()) {
477 for (unsigned K = CGI.Operands.NumDefs; K < CGP.getNumInstDefs(); ++K)
478 OpTypes.push_back(x: VarArgsTyName);
479 }
480
481 // If we had variadic defs, the op idx in the pattern won't match the op idx
482 // in the CGI anymore.
483 int CGIOpOffset = int(CGI.Operands.NumDefs) - CGP.getNumInstDefs();
484 assert(CGP.hasVariadicDefs() ? (CGIOpOffset <= 0) : (CGIOpOffset == 0));
485
486 // Handle all remaining use operands, including variadic ones.
487 for (unsigned K = CGP.getNumInstDefs(); K < CGP.getNumInstOperands(); ++K) {
488 unsigned CGIOpIdx = K + CGIOpOffset;
489 if (CGIOpIdx >= CGI.Operands.size()) {
490 assert(CGP.isVariadic());
491 OpTypes.push_back(x: VarArgsTyName);
492 } else {
493 OpTypes.push_back(x: CGI.Operands[CGIOpIdx].OperandType);
494 }
495 }
496
497 assert(OpTypes.size() == CGP.operands_size());
498 return OpTypes;
499}
500
501void CombineRuleOperandTypeChecker::getInstEqClasses(
502 const InstructionPattern &P, TypeEquivalenceClasses &OutTECs) const {
503 // Determine the TypeEquivalenceClasses by:
504 // - Getting the MCOI Operand Types.
505 // - Creating a Map of MCOI Type -> [Operand Indexes]
506 // - Iterating over the map, filtering types we don't like, and just adding
507 // the array of Operand Indexes to \p OutTECs.
508
509 // We can only do this on CodeGenInstructions that aren't intrinsics. Other
510 // InstructionPatterns have no type inference information associated with
511 // them.
512 // TODO: We could try to extract some info from CodeGenIntrinsic to
513 // guide inference.
514
515 // TODO: Could we add some inference information to builtins at least? e.g.
516 // ReplaceReg should always replace with a reg of the same type, for instance.
517 // Though, those patterns are often used alone so it might not be worth the
518 // trouble to infer their types.
519 auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: &P);
520 if (!CGP || CGP->isIntrinsic())
521 return;
522
523 const auto MCOITypes = getMCOIOperandTypes(CGP: *CGP);
524 assert(MCOITypes.size() == P.operands_size());
525
526 MapVector<StringRef, SmallVector<unsigned, 0>> TyToOpIdx;
527 for (const auto &[Idx, Ty] : enumerate(First: MCOITypes))
528 TyToOpIdx[Ty].push_back(Elt: Idx);
529
530 if (DebugTypeInfer)
531 errs() << "\tGroups for " << P.getName() << ":\t";
532
533 for (const auto &[Ty, Idxs] : TyToOpIdx) {
534 if (!canMCOIOperandTypeBeARegister(MCOIType: Ty))
535 continue;
536
537 if (DebugTypeInfer)
538 errs() << '[';
539 StringRef Sep = "";
540
541 // We only collect named operands.
542 StringRef Leader;
543 for (unsigned Idx : Idxs) {
544 const auto &Op = P.getOperand(K: Idx);
545 if (!Op.isNamedOperand())
546 continue;
547
548 const auto OpName = Op.getOperandName();
549 if (DebugTypeInfer) {
550 errs() << Sep << OpName;
551 Sep = ", ";
552 }
553
554 if (Leader.empty())
555 OutTECs.insert(Data: (Leader = OpName));
556 else
557 OutTECs.unionSets(V1: Leader, V2: OpName);
558 }
559
560 if (DebugTypeInfer)
561 errs() << "] ";
562 }
563
564 if (DebugTypeInfer)
565 errs() << '\n';
566}
567
568CombineRuleOperandTypeChecker::TypeEquivalenceClasses
569CombineRuleOperandTypeChecker::getRuleEqClasses() const {
570 TypeEquivalenceClasses TECs;
571
572 if (DebugTypeInfer)
573 errs() << "Rule Operand Type Equivalence Classes for " << RuleDef.getName()
574 << ":\n";
575
576 for (const auto *Pat : MatchPats)
577 getInstEqClasses(P: *Pat, OutTECs&: TECs);
578 for (const auto *Pat : ApplyPats)
579 getInstEqClasses(P: *Pat, OutTECs&: TECs);
580
581 if (DebugTypeInfer) {
582 errs() << "Final Type Equivalence Classes: ";
583 for (const auto &Class : TECs) {
584 // only print non-empty classes.
585 if (auto MembIt = TECs.member_begin(ECV: *Class);
586 MembIt != TECs.member_end()) {
587 errs() << '[';
588 StringRef Sep = "";
589 for (; MembIt != TECs.member_end(); ++MembIt) {
590 errs() << Sep << *MembIt;
591 Sep = ", ";
592 }
593 errs() << "] ";
594 }
595 }
596 errs() << '\n';
597 }
598
599 return TECs;
600}
601
602//===- MatchData Handling -------------------------------------------------===//
603struct MatchDataDef {
604 MatchDataDef(StringRef Symbol, StringRef Type) : Symbol(Symbol), Type(Type) {}
605
606 StringRef Symbol;
607 StringRef Type;
608
609 /// \returns the desired variable name for this MatchData.
610 std::string getVarName() const {
611 // Add a prefix in case the symbol name is very generic and conflicts with
612 // something else.
613 return "GIMatchData_" + Symbol.str();
614 }
615};
616
617//===- CombineRuleBuilder -------------------------------------------------===//
618
619/// Parses combine rule and builds a small intermediate representation to tie
620/// patterns together and emit RuleMatchers to match them. This may emit more
621/// than one RuleMatcher, e.g. for `wip_match_opcode`.
622///
623/// Memory management for `Pattern` objects is done through `std::unique_ptr`.
624/// In most cases, there are two stages to a pattern's lifetime:
625/// - Creation in a `parse` function
626/// - The unique_ptr is stored in a variable, and may be destroyed if the
627/// pattern is found to be semantically invalid.
628/// - Ownership transfer into a `PatternMap`
629/// - Once a pattern is moved into either the map of Match or Apply
630/// patterns, it is known to be valid and it never moves back.
631class CombineRuleBuilder {
632public:
633 using PatternMap = MapVector<StringRef, std::unique_ptr<Pattern>>;
634 using PatternAlternatives = DenseMap<const Pattern *, unsigned>;
635
636 CombineRuleBuilder(const CodeGenTarget &CGT,
637 SubtargetFeatureInfoMap &SubtargetFeatures,
638 const Record &RuleDef, unsigned ID,
639 std::vector<RuleMatcher> &OutRMs)
640 : Parser(CGT, RuleDef.getLoc()), CGT(CGT),
641 SubtargetFeatures(SubtargetFeatures), RuleDef(RuleDef), RuleID(ID),
642 OutRMs(OutRMs) {}
643
644 /// Parses all fields in the RuleDef record.
645 bool parseAll();
646
647 /// Emits all RuleMatchers into the vector of RuleMatchers passed in the
648 /// constructor.
649 bool emitRuleMatchers();
650
651 void print(raw_ostream &OS) const;
652 void dump() const { print(OS&: dbgs()); }
653
654 /// Debug-only verification of invariants.
655#ifndef NDEBUG
656 void verify() const;
657#endif
658
659private:
660 const CodeGenInstruction &getGConstant() const {
661 return CGT.getInstruction(InstRec: RuleDef.getRecords().getDef(Name: "G_CONSTANT"));
662 }
663
664 std::optional<LLTCodeGenOrTempType>
665 getLLTCodeGenOrTempType(const PatternType &PT, RuleMatcher &RM);
666
667 void PrintError(Twine Msg) const { ::PrintError(Rec: &RuleDef, Msg); }
668 void PrintWarning(Twine Msg) const { ::PrintWarning(WarningLoc: RuleDef.getLoc(), Msg); }
669 void PrintNote(Twine Msg) const { ::PrintNote(NoteLoc: RuleDef.getLoc(), Msg); }
670
671 void print(raw_ostream &OS, const PatternAlternatives &Alts) const;
672
673 bool addApplyPattern(std::unique_ptr<Pattern> Pat);
674 bool addMatchPattern(std::unique_ptr<Pattern> Pat);
675
676 /// Adds the expansions from \see MatchDatas to \p CE.
677 void declareAllMatchDatasExpansions(CodeExpansions &CE) const;
678
679 /// Adds a matcher \p P to \p IM, expanding its code using \p CE.
680 /// Note that the predicate is added on the last InstructionMatcher.
681 ///
682 /// \p Alts is only used if DebugCXXPreds is enabled.
683 void addCXXPredicate(RuleMatcher &M, const CodeExpansions &CE,
684 const CXXPattern &P, const PatternAlternatives &Alts);
685
686 bool hasOnlyCXXApplyPatterns() const;
687 bool hasEraseRoot() const;
688
689 // Infer machine operand types and check their consistency.
690 bool typecheckPatterns();
691
692 /// For all PatFragPatterns, add a new entry in PatternAlternatives for each
693 /// PatternList it contains. This is multiplicative, so if we have 2
694 /// PatFrags with 3 alternatives each, we get 2*3 permutations added to
695 /// PermutationsToEmit. The "MaxPermutations" field controls how many
696 /// permutations are allowed before an error is emitted and this function
697 /// returns false. This is a simple safeguard to prevent combination of
698 /// PatFrags from generating enormous amounts of rules.
699 bool buildPermutationsToEmit();
700
701 /// Checks additional semantics of the Patterns.
702 bool checkSemantics();
703
704 /// Creates a new RuleMatcher with some boilerplate
705 /// settings/actions/predicates, and and adds it to \p OutRMs.
706 /// \see addFeaturePredicates too.
707 ///
708 /// \param Alts Current set of alternatives, for debug comment.
709 /// \param AdditionalComment Comment string to be added to the
710 /// `DebugCommentAction`.
711 RuleMatcher &addRuleMatcher(const PatternAlternatives &Alts,
712 Twine AdditionalComment = "");
713 bool addFeaturePredicates(RuleMatcher &M);
714
715 bool findRoots();
716 bool buildRuleOperandsTable();
717
718 bool parseDefs(const DagInit &Def);
719
720 bool emitMatchPattern(CodeExpansions &CE, const PatternAlternatives &Alts,
721 const InstructionPattern &IP);
722 bool emitMatchPattern(CodeExpansions &CE, const PatternAlternatives &Alts,
723 const AnyOpcodePattern &AOP);
724
725 bool emitPatFragMatchPattern(CodeExpansions &CE,
726 const PatternAlternatives &Alts, RuleMatcher &RM,
727 InstructionMatcher *IM,
728 const PatFragPattern &PFP,
729 DenseSet<const Pattern *> &SeenPats);
730
731 bool emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M);
732 bool emitCXXMatchApply(CodeExpansions &CE, RuleMatcher &M,
733 ArrayRef<CXXPattern *> Matchers);
734
735 // Recursively visits InstructionPatterns from P to build up the
736 // RuleMatcher actions.
737 bool emitInstructionApplyPattern(CodeExpansions &CE, RuleMatcher &M,
738 const InstructionPattern &P,
739 DenseSet<const Pattern *> &SeenPats,
740 StringMap<unsigned> &OperandToTempRegID);
741
742 bool emitCodeGenInstructionApplyImmOperand(RuleMatcher &M,
743 BuildMIAction &DstMI,
744 const CodeGenInstructionPattern &P,
745 const InstructionOperand &O);
746
747 bool emitBuiltinApplyPattern(CodeExpansions &CE, RuleMatcher &M,
748 const BuiltinPattern &P,
749 StringMap<unsigned> &OperandToTempRegID);
750
751 // Recursively visits CodeGenInstructionPattern from P to build up the
752 // RuleMatcher/InstructionMatcher. May create new InstructionMatchers as
753 // needed.
754 using OperandMapperFnRef =
755 function_ref<InstructionOperand(const InstructionOperand &)>;
756 using OperandDefLookupFn =
757 function_ref<const InstructionPattern *(StringRef)>;
758 bool emitCodeGenInstructionMatchPattern(
759 CodeExpansions &CE, const PatternAlternatives &Alts, RuleMatcher &M,
760 InstructionMatcher &IM, const CodeGenInstructionPattern &P,
761 DenseSet<const Pattern *> &SeenPats, OperandDefLookupFn LookupOperandDef,
762 OperandMapperFnRef OperandMapper = [](const auto &O) { return O; });
763
764 PatternParser Parser;
765 const CodeGenTarget &CGT;
766 SubtargetFeatureInfoMap &SubtargetFeatures;
767 const Record &RuleDef;
768 const unsigned RuleID;
769 std::vector<RuleMatcher> &OutRMs;
770
771 // For InstructionMatcher::addOperand
772 unsigned AllocatedTemporariesBaseID = 0;
773
774 /// The root of the pattern.
775 StringRef RootName;
776
777 /// These maps have ownership of the actual Pattern objects.
778 /// They both map a Pattern's name to the Pattern instance.
779 PatternMap MatchPats;
780 PatternMap ApplyPats;
781
782 /// Operand tables to tie match/apply patterns together.
783 OperandTable MatchOpTable;
784 OperandTable ApplyOpTable;
785
786 /// Set by findRoots.
787 Pattern *MatchRoot = nullptr;
788 SmallDenseSet<InstructionPattern *, 2> ApplyRoots;
789
790 SmallVector<MatchDataDef, 2> MatchDatas;
791 SmallVector<PatternAlternatives, 1> PermutationsToEmit;
792};
793
794bool CombineRuleBuilder::parseAll() {
795 auto StackTrace = PrettyStackTraceParse(RuleDef);
796
797 if (!parseDefs(Def: *RuleDef.getValueAsDag(FieldName: "Defs")))
798 return false;
799
800 const DagInit &Act0 = *RuleDef.getValueAsDag(FieldName: "Action0");
801 const DagInit &Act1 = *RuleDef.getValueAsDag(FieldName: "Action1");
802
803 StringRef Act0Op = Act0.getOperatorAsDef(Loc: RuleDef.getLoc())->getName();
804 StringRef Act1Op = Act1.getOperatorAsDef(Loc: RuleDef.getLoc())->getName();
805
806 if (Act0Op == "match" && Act1Op == "apply") {
807 if (!Parser.parsePatternList(
808 List: Act0, ParseAction: [this](auto Pat) { return addMatchPattern(Pat: std::move(Pat)); },
809 Operator: "match", AnonPatNamePrefix: (RuleDef.getName() + "_match").str()))
810 return false;
811
812 if (!Parser.parsePatternList(
813 List: Act1, ParseAction: [this](auto Pat) { return addApplyPattern(Pat: std::move(Pat)); },
814 Operator: "apply", AnonPatNamePrefix: (RuleDef.getName() + "_apply").str()))
815 return false;
816
817 } else if (Act0Op == "combine" && Act1Op == "empty_action") {
818 // combine: everything is a "match" except C++ code which is an apply.
819 const auto AddCombinePat = [this](std::unique_ptr<Pattern> Pat) {
820 if (isa<CXXPattern>(Val: Pat.get()))
821 return addApplyPattern(Pat: std::move(Pat));
822 return addMatchPattern(Pat: std::move(Pat));
823 };
824
825 if (!Parser.parsePatternList(List: Act0, ParseAction: AddCombinePat, Operator: "combine",
826 AnonPatNamePrefix: (RuleDef.getName() + "_combine").str()))
827 return false;
828
829 if (MatchPats.empty() || ApplyPats.empty()) {
830 PrintError(Msg: "'combine' action needs at least one pattern to match, and "
831 "C++ code to apply");
832 return false;
833 }
834 } else {
835 PrintError(Msg: "expected both a 'match' and 'apply' action in combine rule, "
836 "or a single 'combine' action");
837 return false;
838 }
839
840 if (!buildRuleOperandsTable() || !typecheckPatterns() || !findRoots() ||
841 !checkSemantics() || !buildPermutationsToEmit())
842 return false;
843 LLVM_DEBUG(verify());
844 return true;
845}
846
847bool CombineRuleBuilder::emitRuleMatchers() {
848 auto StackTrace = PrettyStackTraceEmit(RuleDef);
849
850 assert(MatchRoot);
851 CodeExpansions CE;
852
853 assert(!PermutationsToEmit.empty());
854 for (const auto &Alts : PermutationsToEmit) {
855 switch (MatchRoot->getKind()) {
856 case Pattern::K_AnyOpcode: {
857 if (!emitMatchPattern(CE, Alts, AOP: *cast<AnyOpcodePattern>(Val: MatchRoot)))
858 return false;
859 break;
860 }
861 case Pattern::K_PatFrag:
862 case Pattern::K_Builtin:
863 case Pattern::K_CodeGenInstruction:
864 if (!emitMatchPattern(CE, Alts, IP: *cast<InstructionPattern>(Val: MatchRoot)))
865 return false;
866 break;
867 case Pattern::K_CXX:
868 PrintError(Msg: "C++ code cannot be the root of a rule!");
869 return false;
870 default:
871 llvm_unreachable("unknown pattern kind!");
872 }
873 }
874
875 return true;
876}
877
878void CombineRuleBuilder::print(raw_ostream &OS) const {
879 OS << "(CombineRule name:" << RuleDef.getName() << " id:" << RuleID
880 << " root:" << RootName << '\n';
881
882 if (!MatchDatas.empty()) {
883 OS << " (MatchDatas\n";
884 for (const auto &MD : MatchDatas) {
885 OS << " (MatchDataDef symbol:" << MD.Symbol << " type:" << MD.Type
886 << ")\n";
887 }
888 OS << " )\n";
889 }
890
891 const auto &SeenPFs = Parser.getSeenPatFrags();
892 if (!SeenPFs.empty()) {
893 OS << " (PatFrags\n";
894 for (const auto *PF : Parser.getSeenPatFrags()) {
895 PF->print(OS, /*Indent=*/" ");
896 OS << '\n';
897 }
898 OS << " )\n";
899 }
900
901 const auto DumpPats = [&](StringRef Name, const PatternMap &Pats) {
902 OS << " (" << Name << " ";
903 if (Pats.empty()) {
904 OS << "<empty>)\n";
905 return;
906 }
907
908 OS << '\n';
909 for (const auto &[Name, Pat] : Pats) {
910 OS << " ";
911 if (Pat.get() == MatchRoot)
912 OS << "<match_root>";
913 if (isa<InstructionPattern>(Val: Pat.get()) &&
914 ApplyRoots.contains(V: cast<InstructionPattern>(Val: Pat.get())))
915 OS << "<apply_root>";
916 OS << Name << ":";
917 Pat->print(OS, /*PrintName=*/false);
918 OS << '\n';
919 }
920 OS << " )\n";
921 };
922
923 DumpPats("MatchPats", MatchPats);
924 DumpPats("ApplyPats", ApplyPats);
925
926 MatchOpTable.print(OS, Name: "MatchPats", /*Indent*/ " ");
927 ApplyOpTable.print(OS, Name: "ApplyPats", /*Indent*/ " ");
928
929 if (PermutationsToEmit.size() > 1) {
930 OS << " (PermutationsToEmit\n";
931 for (const auto &Perm : PermutationsToEmit) {
932 OS << " ";
933 print(OS, Alts: Perm);
934 OS << ",\n";
935 }
936 OS << " )\n";
937 }
938
939 OS << ")\n";
940}
941
942#ifndef NDEBUG
943void CombineRuleBuilder::verify() const {
944 const auto VerifyPats = [&](const PatternMap &Pats) {
945 for (const auto &[Name, Pat] : Pats) {
946 if (!Pat)
947 PrintFatalError("null pattern in pattern map!");
948
949 if (Name != Pat->getName()) {
950 Pat->dump();
951 PrintFatalError("Pattern name mismatch! Map name: " + Name +
952 ", Pat name: " + Pat->getName());
953 }
954
955 // Sanity check: the map should point to the same data as the Pattern.
956 // Both strings are allocated in the pool using insertStrRef.
957 if (Name.data() != Pat->getName().data()) {
958 dbgs() << "Map StringRef: '" << Name << "' @ "
959 << (const void *)Name.data() << '\n';
960 dbgs() << "Pat String: '" << Pat->getName() << "' @ "
961 << (const void *)Pat->getName().data() << '\n';
962 PrintFatalError("StringRef stored in the PatternMap is not referencing "
963 "the same string as its Pattern!");
964 }
965 }
966 };
967
968 VerifyPats(MatchPats);
969 VerifyPats(ApplyPats);
970
971 // Check there are no wip_match_opcode patterns in the "apply" patterns.
972 if (any_of(ApplyPats,
973 [&](auto &E) { return isa<AnyOpcodePattern>(E.second.get()); })) {
974 dump();
975 PrintFatalError(
976 "illegal wip_match_opcode pattern in the 'apply' patterns!");
977 }
978
979 // Check there are no nullptrs in ApplyRoots.
980 if (ApplyRoots.contains(nullptr)) {
981 PrintFatalError(
982 "CombineRuleBuilder's ApplyRoots set contains a null pointer!");
983 }
984}
985#endif
986
987std::optional<LLTCodeGenOrTempType>
988CombineRuleBuilder::getLLTCodeGenOrTempType(const PatternType &PT,
989 RuleMatcher &RM) {
990 assert(!PT.isNone());
991
992 if (PT.isLLT())
993 return getLLTCodeGen(PT);
994
995 assert(PT.isTypeOf());
996 auto &OM = RM.getOperandMatcher(Name: PT.getTypeOfOpName());
997 if (OM.isVariadic()) {
998 PrintError(Msg: "type '" + PT.str() + "' is ill-formed: '" +
999 OM.getSymbolicName() + "' is a variadic pack operand");
1000 return std::nullopt;
1001 }
1002 return OM.getTempTypeIdx(Rule&: RM);
1003}
1004
1005void CombineRuleBuilder::print(raw_ostream &OS,
1006 const PatternAlternatives &Alts) const {
1007 SmallVector<std::string, 1> Strings(
1008 map_range(C: Alts, F: [](const auto &PatAndPerm) {
1009 return PatAndPerm.first->getName().str() + "[" +
1010 to_string(PatAndPerm.second) + "]";
1011 }));
1012 // Sort so output is deterministic for tests. Otherwise it's sorted by pointer
1013 // values.
1014 sort(C&: Strings);
1015 OS << "[" << join(R&: Strings, Separator: ", ") << "]";
1016}
1017
1018bool CombineRuleBuilder::addApplyPattern(std::unique_ptr<Pattern> Pat) {
1019 StringRef Name = Pat->getName();
1020 if (ApplyPats.contains(Key: Name)) {
1021 PrintError(Msg: "'" + Name + "' apply pattern defined more than once!");
1022 return false;
1023 }
1024
1025 if (isa<AnyOpcodePattern>(Val: Pat.get())) {
1026 PrintError(Msg: "'" + Name +
1027 "': wip_match_opcode is not supported in apply patterns");
1028 return false;
1029 }
1030
1031 if (isa<PatFragPattern>(Val: Pat.get())) {
1032 PrintError(Msg: "'" + Name + "': using " + PatFrag::ClassName +
1033 " is not supported in apply patterns");
1034 return false;
1035 }
1036
1037 if (auto *CXXPat = dyn_cast<CXXPattern>(Val: Pat.get()))
1038 CXXPat->setIsApply();
1039
1040 ApplyPats[Name] = std::move(Pat);
1041 return true;
1042}
1043
1044bool CombineRuleBuilder::addMatchPattern(std::unique_ptr<Pattern> Pat) {
1045 StringRef Name = Pat->getName();
1046 if (MatchPats.contains(Key: Name)) {
1047 PrintError(Msg: "'" + Name + "' match pattern defined more than once!");
1048 return false;
1049 }
1050
1051 // For now, none of the builtins can appear in 'match'.
1052 if (const auto *BP = dyn_cast<BuiltinPattern>(Val: Pat.get())) {
1053 PrintError(Msg: "'" + BP->getInstName() +
1054 "' cannot be used in a 'match' pattern");
1055 return false;
1056 }
1057
1058 MatchPats[Name] = std::move(Pat);
1059 return true;
1060}
1061
1062void CombineRuleBuilder::declareAllMatchDatasExpansions(
1063 CodeExpansions &CE) const {
1064 for (const auto &MD : MatchDatas)
1065 CE.declare(Name: MD.Symbol, Expansion: MD.getVarName());
1066}
1067
1068void CombineRuleBuilder::addCXXPredicate(RuleMatcher &M,
1069 const CodeExpansions &CE,
1070 const CXXPattern &P,
1071 const PatternAlternatives &Alts) {
1072 // FIXME: Hack so C++ code is executed last. May not work for more complex
1073 // patterns.
1074 auto &IM = *std::prev(x: M.insnmatchers().end());
1075 auto Loc = RuleDef.getLoc();
1076 const auto AddComment = [&](raw_ostream &OS) {
1077 OS << "// Pattern Alternatives: ";
1078 print(OS, Alts);
1079 OS << '\n';
1080 };
1081 const auto &ExpandedCode =
1082 DebugCXXPreds ? P.expandCode(CE, Locs: Loc, AddComment) : P.expandCode(CE, Locs: Loc);
1083 IM->addPredicate<GenericInstructionPredicateMatcher>(
1084 args: ExpandedCode.getEnumNameWithPrefix(Prefix: CXXPredPrefix));
1085}
1086
1087bool CombineRuleBuilder::hasOnlyCXXApplyPatterns() const {
1088 return all_of(Range: ApplyPats, P: [&](auto &Entry) {
1089 return isa<CXXPattern>(Entry.second.get());
1090 });
1091}
1092
1093bool CombineRuleBuilder::hasEraseRoot() const {
1094 return any_of(Range: ApplyPats, P: [&](auto &Entry) {
1095 if (const auto *BP = dyn_cast<BuiltinPattern>(Entry.second.get()))
1096 return BP->getBuiltinKind() == BI_EraseRoot;
1097 return false;
1098 });
1099}
1100
1101bool CombineRuleBuilder::typecheckPatterns() {
1102 CombineRuleOperandTypeChecker OTC(RuleDef, MatchOpTable);
1103
1104 for (auto &Pat : values(C&: MatchPats)) {
1105 if (auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get())) {
1106 if (!OTC.processMatchPattern(P&: *IP))
1107 return false;
1108 }
1109 }
1110
1111 for (auto &Pat : values(C&: ApplyPats)) {
1112 if (auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get())) {
1113 if (!OTC.processApplyPattern(P&: *IP))
1114 return false;
1115 }
1116 }
1117
1118 OTC.propagateAndInferTypes();
1119
1120 // Always check this after in case inference adds some special types to the
1121 // match patterns.
1122 for (auto &Pat : values(C&: MatchPats)) {
1123 if (auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get())) {
1124 bool HasDiag = false;
1125 for (const auto &[Idx, Op] : enumerate(First&: IP->operands())) {
1126 if (Op.getType().isTypeOf()) {
1127 PrintError(Msg: PatternType::TypeOfClassName +
1128 " is not supported in 'match' patterns");
1129 PrintNote(Msg: "operand " + Twine(Idx) + " of '" + IP->getName() +
1130 "' has type '" + Op.getType().str() + "'");
1131 HasDiag = true;
1132 }
1133 }
1134 if (HasDiag)
1135 return false;
1136 }
1137 }
1138 return true;
1139}
1140
1141bool CombineRuleBuilder::buildPermutationsToEmit() {
1142 PermutationsToEmit.clear();
1143
1144 // Start with one empty set of alternatives.
1145 PermutationsToEmit.emplace_back();
1146 for (const auto &Pat : values(C&: MatchPats)) {
1147 unsigned NumAlts = 0;
1148 // Note: technically, AnyOpcodePattern also needs permutations, but:
1149 // - We only allow a single one of them in the root.
1150 // - They cannot be mixed with any other pattern other than C++ code.
1151 // So we don't really need to take them into account here. We could, but
1152 // that pattern is a hack anyway and the less it's involved, the better.
1153 if (const auto *PFP = dyn_cast<PatFragPattern>(Val: Pat.get()))
1154 NumAlts = PFP->getPatFrag().num_alternatives();
1155 else
1156 continue;
1157
1158 // For each pattern that needs permutations, multiply the current set of
1159 // alternatives.
1160 auto CurPerms = PermutationsToEmit;
1161 PermutationsToEmit.clear();
1162
1163 for (const auto &Perm : CurPerms) {
1164 assert(!Perm.contains(Pat.get()) && "Pattern already emitted?");
1165 for (unsigned K = 0; K < NumAlts; ++K) {
1166 PatternAlternatives NewPerm = Perm;
1167 NewPerm[Pat.get()] = K;
1168 PermutationsToEmit.emplace_back(Args: std::move(NewPerm));
1169 }
1170 }
1171 }
1172
1173 if (int64_t MaxPerms = RuleDef.getValueAsInt(FieldName: "MaxPermutations");
1174 MaxPerms > 0) {
1175 if ((int64_t)PermutationsToEmit.size() > MaxPerms) {
1176 PrintError(Msg: "cannot emit rule '" + RuleDef.getName() + "'; " +
1177 Twine(PermutationsToEmit.size()) +
1178 " permutations would be emitted, but the max is " +
1179 Twine(MaxPerms));
1180 return false;
1181 }
1182 }
1183
1184 // Ensure we always have a single empty entry, it simplifies the emission
1185 // logic so it doesn't need to handle the case where there are no perms.
1186 if (PermutationsToEmit.empty()) {
1187 PermutationsToEmit.emplace_back();
1188 return true;
1189 }
1190
1191 return true;
1192}
1193
1194bool CombineRuleBuilder::checkSemantics() {
1195 assert(MatchRoot && "Cannot call this before findRoots()");
1196
1197 const auto CheckVariadicOperands = [&](const InstructionPattern &IP,
1198 bool IsMatch) {
1199 bool HasVariadic = false;
1200 for (auto &Op : IP.operands()) {
1201 if (!Op.getType().isVariadicPack())
1202 continue;
1203
1204 HasVariadic = true;
1205
1206 if (IsMatch && &Op != &IP.operands_back()) {
1207 PrintError(Msg: "'" + IP.getInstName() +
1208 "': " + PatternType::VariadicClassName +
1209 " can only be used on the last operand");
1210 return false;
1211 }
1212
1213 if (Op.isDef()) {
1214 PrintError(Msg: "'" + IP.getInstName() + "': " +
1215 PatternType::VariadicClassName + " cannot be used on defs");
1216 return false;
1217 }
1218 }
1219
1220 if (HasVariadic && !IP.isVariadic()) {
1221 PrintError(Msg: "cannot use a " + PatternType::VariadicClassName +
1222 " operand on non-variadic instruction '" + IP.getInstName() +
1223 "'");
1224 return false;
1225 }
1226
1227 return true;
1228 };
1229
1230 bool UsesWipMatchOpcode = false;
1231 for (const auto &Match : MatchPats) {
1232 const auto *Pat = Match.second.get();
1233
1234 if (const auto *CXXPat = dyn_cast<CXXPattern>(Val: Pat)) {
1235 if (!CXXPat->getRawCode().contains(Other: "return "))
1236 PrintWarning(Msg: "'match' C++ code does not seem to return!");
1237 continue;
1238 }
1239
1240 if (const auto IP = dyn_cast<InstructionPattern>(Val: Pat)) {
1241 if (!CheckVariadicOperands(*IP, /*IsMatch=*/true))
1242 return false;
1243
1244 // MIFlags in match cannot use the following syntax: (MIFlags $mi)
1245 if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: Pat)) {
1246 if (auto *FI = CGP->getMIFlagsInfo()) {
1247 if (!FI->copy_flags().empty()) {
1248 PrintError(Msg: "'match' patterns cannot refer to flags from other "
1249 "instructions");
1250 PrintNote(Msg: "MIFlags in '" + CGP->getName() +
1251 "' refer to: " + join(R: FI->copy_flags(), Separator: ", "));
1252 return false;
1253 }
1254 }
1255 }
1256 continue;
1257 }
1258
1259 const auto *AOP = dyn_cast<AnyOpcodePattern>(Val: Pat);
1260 if (!AOP)
1261 continue;
1262
1263 if (UsesWipMatchOpcode) {
1264 PrintError(Msg: "wip_opcode_match can only be present once");
1265 return false;
1266 }
1267
1268 UsesWipMatchOpcode = true;
1269 }
1270
1271 std::optional<bool> IsUsingCXXPatterns;
1272 for (const auto &Apply : ApplyPats) {
1273 Pattern *Pat = Apply.second.get();
1274 if (IsUsingCXXPatterns) {
1275 if (*IsUsingCXXPatterns != isa<CXXPattern>(Val: Pat)) {
1276 PrintError(Msg: "'apply' patterns cannot mix C++ code with other types of "
1277 "patterns");
1278 return false;
1279 }
1280 } else {
1281 IsUsingCXXPatterns = isa<CXXPattern>(Val: Pat);
1282 }
1283
1284 assert(Pat);
1285 const auto *IP = dyn_cast<InstructionPattern>(Val: Pat);
1286 if (!IP)
1287 continue;
1288
1289 if (!CheckVariadicOperands(*IP, /*IsMatch=*/false))
1290 return false;
1291
1292 if (UsesWipMatchOpcode) {
1293 PrintError(Msg: "cannot use wip_match_opcode in combination with apply "
1294 "instruction patterns!");
1295 return false;
1296 }
1297
1298 // Check that the insts mentioned in copy_flags exist.
1299 if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: IP)) {
1300 if (auto *FI = CGP->getMIFlagsInfo()) {
1301 for (auto InstName : FI->copy_flags()) {
1302 auto It = MatchPats.find(Key: InstName);
1303 if (It == MatchPats.end()) {
1304 PrintError(Msg: "unknown instruction '$" + InstName +
1305 "' referenced in MIFlags of '" + CGP->getName() + "'");
1306 return false;
1307 }
1308
1309 if (!isa<CodeGenInstructionPattern>(Val: It->second.get())) {
1310 PrintError(
1311 Msg: "'$" + InstName +
1312 "' does not refer to a CodeGenInstruction in MIFlags of '" +
1313 CGP->getName() + "'");
1314 return false;
1315 }
1316 }
1317 }
1318 }
1319
1320 const auto *BIP = dyn_cast<BuiltinPattern>(Val: IP);
1321 if (!BIP)
1322 continue;
1323 StringRef Name = BIP->getInstName();
1324
1325 // (GIEraseInst) has to be the only apply pattern, or it can not be used at
1326 // all. The root cannot have any defs either.
1327 switch (BIP->getBuiltinKind()) {
1328 case BI_EraseRoot: {
1329 if (ApplyPats.size() > 1) {
1330 PrintError(Msg: Name + " must be the only 'apply' pattern");
1331 return false;
1332 }
1333
1334 const auto *IRoot = dyn_cast<CodeGenInstructionPattern>(Val: MatchRoot);
1335 if (!IRoot) {
1336 PrintError(Msg: Name + " can only be used if the root is a "
1337 "CodeGenInstruction or Intrinsic");
1338 return false;
1339 }
1340
1341 if (IRoot->getNumInstDefs() != 0) {
1342 PrintError(Msg: Name + " can only be used if on roots that do "
1343 "not have any output operand");
1344 PrintNote(Msg: "'" + IRoot->getInstName() + "' has " +
1345 Twine(IRoot->getNumInstDefs()) + " output operands");
1346 return false;
1347 }
1348 break;
1349 }
1350 case BI_ReplaceReg: {
1351 // (GIReplaceReg can only be used on the root instruction)
1352 // TODO: When we allow rewriting non-root instructions, also allow this.
1353 StringRef OldRegName = BIP->getOperand(K: 0).getOperandName();
1354 auto *Def = MatchOpTable.getDef(OpName: OldRegName);
1355 if (!Def) {
1356 PrintError(Msg: Name + " cannot find a matched pattern that defines '" +
1357 OldRegName + "'");
1358 return false;
1359 }
1360 if (MatchOpTable.getDef(OpName: OldRegName) != MatchRoot) {
1361 PrintError(Msg: Name + " cannot replace '" + OldRegName +
1362 "': this builtin can only replace a register defined by the "
1363 "match root");
1364 return false;
1365 }
1366 break;
1367 }
1368 }
1369 }
1370
1371 // TODO: Diagnose uses of MatchDatas if the Rule doesn't have C++ on both the
1372 // match and apply. It's useless in such cases.
1373 if (!hasOnlyCXXApplyPatterns() && !MatchDatas.empty()) {
1374 PrintError(Msg: MatchDataClassName +
1375 " can only be used if 'apply' in entirely written in C++");
1376 return false;
1377 }
1378
1379 return true;
1380}
1381
1382RuleMatcher &CombineRuleBuilder::addRuleMatcher(const PatternAlternatives &Alts,
1383 Twine AdditionalComment) {
1384 auto &RM = OutRMs.emplace_back(args: RuleDef.getLoc());
1385 addFeaturePredicates(M&: RM);
1386 RM.setPermanentGISelFlags(GISF_IgnoreCopies);
1387 RM.addRequiredSimplePredicate(PredName: getIsEnabledPredicateEnumName(CombinerRuleID: RuleID));
1388
1389 std::string Comment;
1390 raw_string_ostream CommentOS(Comment);
1391 CommentOS << "Combiner Rule #" << RuleID << ": " << RuleDef.getName();
1392 if (!Alts.empty()) {
1393 CommentOS << " @ ";
1394 print(OS&: CommentOS, Alts);
1395 }
1396 if (!AdditionalComment.isTriviallyEmpty())
1397 CommentOS << "; " << AdditionalComment;
1398 RM.addAction<DebugCommentAction>(args&: Comment);
1399 return RM;
1400}
1401
1402bool CombineRuleBuilder::addFeaturePredicates(RuleMatcher &M) {
1403 if (!RuleDef.getValue(Name: "Predicates"))
1404 return true;
1405
1406 const ListInit *Preds = RuleDef.getValueAsListInit(FieldName: "Predicates");
1407 for (const Init *PI : Preds->getElements()) {
1408 const DefInit *Pred = dyn_cast<DefInit>(Val: PI);
1409 if (!Pred)
1410 continue;
1411
1412 const Record *Def = Pred->getDef();
1413 if (!Def->isSubClassOf(Name: "Predicate")) {
1414 ::PrintError(Rec: Def, Msg: "Unknown 'Predicate' Type");
1415 return false;
1416 }
1417
1418 if (Def->getValueAsString(FieldName: "CondString").empty())
1419 continue;
1420
1421 if (SubtargetFeatures.count(x: Def) == 0) {
1422 SubtargetFeatures.emplace(
1423 args&: Def, args: SubtargetFeatureInfo(Def, SubtargetFeatures.size()));
1424 }
1425
1426 M.addRequiredFeature(Feature: Def);
1427 }
1428
1429 return true;
1430}
1431
1432bool CombineRuleBuilder::findRoots() {
1433 const auto Finish = [&]() {
1434 assert(MatchRoot);
1435
1436 if (hasOnlyCXXApplyPatterns() || hasEraseRoot())
1437 return true;
1438
1439 auto *IPRoot = dyn_cast<InstructionPattern>(Val: MatchRoot);
1440 if (!IPRoot)
1441 return true;
1442
1443 if (IPRoot->getNumInstDefs() == 0) {
1444 // No defs to work with -> find the root using the pattern name.
1445 auto It = ApplyPats.find(Key: RootName);
1446 if (It == ApplyPats.end()) {
1447 PrintError(Msg: "Cannot find root '" + RootName + "' in apply patterns!");
1448 return false;
1449 }
1450
1451 auto *ApplyRoot = dyn_cast<InstructionPattern>(Val: It->second.get());
1452 if (!ApplyRoot) {
1453 PrintError(Msg: "apply pattern root '" + RootName +
1454 "' must be an instruction pattern");
1455 return false;
1456 }
1457
1458 ApplyRoots.insert(V: ApplyRoot);
1459 return true;
1460 }
1461
1462 // Collect all redefinitions of the MatchRoot's defs and put them in
1463 // ApplyRoots.
1464 const auto DefsNeeded = IPRoot->getApplyDefsNeeded();
1465 for (auto &Op : DefsNeeded) {
1466 assert(Op.isDef() && Op.isNamedOperand());
1467 StringRef Name = Op.getOperandName();
1468
1469 auto *ApplyRedef = ApplyOpTable.getDef(OpName: Name);
1470 if (!ApplyRedef) {
1471 PrintError(Msg: "'" + Name + "' must be redefined in the 'apply' pattern");
1472 return false;
1473 }
1474
1475 ApplyRoots.insert(V: (InstructionPattern *)ApplyRedef);
1476 }
1477
1478 if (auto It = ApplyPats.find(Key: RootName); It != ApplyPats.end()) {
1479 if (find(Range&: ApplyRoots, Val: It->second.get()) == ApplyRoots.end()) {
1480 PrintError(Msg: "apply pattern '" + RootName +
1481 "' is supposed to be a root but it does not redefine any of "
1482 "the defs of the match root");
1483 return false;
1484 }
1485 }
1486
1487 return true;
1488 };
1489
1490 // Look by pattern name, e.g.
1491 // (G_FNEG $x, $y):$root
1492 if (auto MatchPatIt = MatchPats.find(Key: RootName);
1493 MatchPatIt != MatchPats.end()) {
1494 MatchRoot = MatchPatIt->second.get();
1495 return Finish();
1496 }
1497
1498 // Look by def:
1499 // (G_FNEG $root, $y)
1500 auto LookupRes = MatchOpTable.lookup(OpName: RootName);
1501 if (!LookupRes.Found) {
1502 PrintError(Msg: "Cannot find root '" + RootName + "' in match patterns!");
1503 return false;
1504 }
1505
1506 MatchRoot = LookupRes.Def;
1507 if (!MatchRoot) {
1508 PrintError(Msg: "Cannot use live-in operand '" + RootName +
1509 "' as match pattern root!");
1510 return false;
1511 }
1512
1513 return Finish();
1514}
1515
1516bool CombineRuleBuilder::buildRuleOperandsTable() {
1517 const auto DiagnoseRedefMatch = [&](StringRef OpName) {
1518 PrintError(Msg: "Operand '" + OpName +
1519 "' is defined multiple times in the 'match' patterns");
1520 };
1521
1522 const auto DiagnoseRedefApply = [&](StringRef OpName) {
1523 PrintError(Msg: "Operand '" + OpName +
1524 "' is defined multiple times in the 'apply' patterns");
1525 };
1526
1527 for (auto &Pat : values(C&: MatchPats)) {
1528 auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get());
1529 if (IP && !MatchOpTable.addPattern(P: IP, DiagnoseRedef: DiagnoseRedefMatch))
1530 return false;
1531 }
1532
1533 for (auto &Pat : values(C&: ApplyPats)) {
1534 auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get());
1535 if (IP && !ApplyOpTable.addPattern(P: IP, DiagnoseRedef: DiagnoseRedefApply))
1536 return false;
1537 }
1538
1539 return true;
1540}
1541
1542bool CombineRuleBuilder::parseDefs(const DagInit &Def) {
1543 if (Def.getOperatorAsDef(Loc: RuleDef.getLoc())->getName() != "defs") {
1544 PrintError(Msg: "Expected defs operator");
1545 return false;
1546 }
1547
1548 SmallVector<StringRef> Roots;
1549 for (unsigned I = 0, E = Def.getNumArgs(); I < E; ++I) {
1550 if (isSpecificDef(N: *Def.getArg(Num: I), Def: "root")) {
1551 Roots.emplace_back(Args: Def.getArgNameStr(Num: I));
1552 continue;
1553 }
1554
1555 // Subclasses of GIDefMatchData should declare that this rule needs to pass
1556 // data from the match stage to the apply stage, and ensure that the
1557 // generated matcher has a suitable variable for it to do so.
1558 if (const Record *MatchDataRec =
1559 getDefOfSubClass(N: *Def.getArg(Num: I), Cls: MatchDataClassName)) {
1560 MatchDatas.emplace_back(Args: Def.getArgNameStr(Num: I),
1561 Args: MatchDataRec->getValueAsString(FieldName: "Type"));
1562 continue;
1563 }
1564
1565 // Otherwise emit an appropriate error message.
1566 if (getDefOfSubClass(N: *Def.getArg(Num: I), Cls: "GIDefKind"))
1567 PrintError(Msg: "This GIDefKind not implemented in tablegen");
1568 else if (getDefOfSubClass(N: *Def.getArg(Num: I), Cls: "GIDefKindWithArgs"))
1569 PrintError(Msg: "This GIDefKindWithArgs not implemented in tablegen");
1570 else
1571 PrintError(Msg: "Expected a subclass of GIDefKind or a sub-dag whose "
1572 "operator is of type GIDefKindWithArgs");
1573 return false;
1574 }
1575
1576 if (Roots.size() != 1) {
1577 PrintError(Msg: "Combine rules must have exactly one root");
1578 return false;
1579 }
1580
1581 RootName = Roots.front();
1582 return true;
1583}
1584
1585bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
1586 const PatternAlternatives &Alts,
1587 const InstructionPattern &IP) {
1588 auto StackTrace = PrettyStackTraceEmit(RuleDef, &IP);
1589
1590 auto &M = addRuleMatcher(Alts);
1591 InstructionMatcher &IM = M.addInstructionMatcher(SymbolicName: IP.getName());
1592 declareInstExpansion(CE, IM, Name: IP.getName());
1593
1594 DenseSet<const Pattern *> SeenPats;
1595
1596 const auto FindOperandDef = [&](StringRef Op) -> InstructionPattern * {
1597 return MatchOpTable.getDef(OpName: Op);
1598 };
1599
1600 if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: &IP)) {
1601 if (!emitCodeGenInstructionMatchPattern(CE, Alts, M, IM, P: *CGP, SeenPats,
1602 LookupOperandDef: FindOperandDef))
1603 return false;
1604 } else if (const auto *PFP = dyn_cast<PatFragPattern>(Val: &IP)) {
1605 if (!PFP->getPatFrag().canBeMatchRoot()) {
1606 PrintError(Msg: "cannot use '" + PFP->getInstName() + " as match root");
1607 return false;
1608 }
1609
1610 if (!emitPatFragMatchPattern(CE, Alts, RM&: M, IM: &IM, PFP: *PFP, SeenPats))
1611 return false;
1612 } else if (isa<BuiltinPattern>(Val: &IP)) {
1613 llvm_unreachable("No match builtins known!");
1614 } else {
1615 llvm_unreachable("Unknown kind of InstructionPattern!");
1616 }
1617
1618 // Emit remaining patterns
1619 const bool IsUsingCustomCXXAction = hasOnlyCXXApplyPatterns();
1620 SmallVector<CXXPattern *, 2> CXXMatchers;
1621 for (auto &Pat : values(C&: MatchPats)) {
1622 if (SeenPats.contains(V: Pat.get()))
1623 continue;
1624
1625 switch (Pat->getKind()) {
1626 case Pattern::K_AnyOpcode:
1627 PrintError(Msg: "wip_match_opcode can not be used with instruction patterns!");
1628 return false;
1629 case Pattern::K_PatFrag: {
1630 if (!emitPatFragMatchPattern(CE, Alts, RM&: M, /*IM*/ nullptr,
1631 PFP: *cast<PatFragPattern>(Val: Pat.get()), SeenPats))
1632 return false;
1633 continue;
1634 }
1635 case Pattern::K_Builtin:
1636 PrintError(Msg: "No known match builtins");
1637 return false;
1638 case Pattern::K_CodeGenInstruction:
1639 cast<InstructionPattern>(Val: Pat.get())->reportUnreachable(Locs: RuleDef.getLoc());
1640 return false;
1641 case Pattern::K_CXX: {
1642 // Delay emission for top-level C++ matchers (which can use MatchDatas).
1643 if (IsUsingCustomCXXAction)
1644 CXXMatchers.push_back(Elt: cast<CXXPattern>(Val: Pat.get()));
1645 else
1646 addCXXPredicate(M, CE, P: *cast<CXXPattern>(Val: Pat.get()), Alts);
1647 continue;
1648 }
1649 default:
1650 llvm_unreachable("unknown pattern kind!");
1651 }
1652 }
1653
1654 return IsUsingCustomCXXAction ? emitCXXMatchApply(CE, M, Matchers: CXXMatchers)
1655 : emitApplyPatterns(CE, M);
1656}
1657
1658bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
1659 const PatternAlternatives &Alts,
1660 const AnyOpcodePattern &AOP) {
1661 auto StackTrace = PrettyStackTraceEmit(RuleDef, &AOP);
1662
1663 const bool IsUsingCustomCXXAction = hasOnlyCXXApplyPatterns();
1664 for (const CodeGenInstruction *CGI : AOP.insts()) {
1665 auto &M = addRuleMatcher(Alts, AdditionalComment: "wip_match_opcode '" +
1666 CGI->TheDef->getName() + "'");
1667
1668 InstructionMatcher &IM = M.addInstructionMatcher(SymbolicName: AOP.getName());
1669 declareInstExpansion(CE, IM, Name: AOP.getName());
1670 // declareInstExpansion needs to be identical, otherwise we need to create a
1671 // CodeExpansions object here instead.
1672 assert(IM.getInsnVarID() == 0);
1673
1674 IM.addPredicate<InstructionOpcodeMatcher>(args&: CGI);
1675
1676 // Emit remaining patterns.
1677 SmallVector<CXXPattern *, 2> CXXMatchers;
1678 for (auto &Pat : values(C&: MatchPats)) {
1679 if (Pat.get() == &AOP)
1680 continue;
1681
1682 switch (Pat->getKind()) {
1683 case Pattern::K_AnyOpcode:
1684 PrintError(Msg: "wip_match_opcode can only be present once!");
1685 return false;
1686 case Pattern::K_PatFrag: {
1687 DenseSet<const Pattern *> SeenPats;
1688 if (!emitPatFragMatchPattern(CE, Alts, RM&: M, /*IM*/ nullptr,
1689 PFP: *cast<PatFragPattern>(Val: Pat.get()),
1690 SeenPats))
1691 return false;
1692 continue;
1693 }
1694 case Pattern::K_Builtin:
1695 PrintError(Msg: "No known match builtins");
1696 return false;
1697 case Pattern::K_CodeGenInstruction:
1698 cast<InstructionPattern>(Val: Pat.get())->reportUnreachable(
1699 Locs: RuleDef.getLoc());
1700 return false;
1701 case Pattern::K_CXX: {
1702 // Delay emission for top-level C++ matchers (which can use MatchDatas).
1703 if (IsUsingCustomCXXAction)
1704 CXXMatchers.push_back(Elt: cast<CXXPattern>(Val: Pat.get()));
1705 else
1706 addCXXPredicate(M, CE, P: *cast<CXXPattern>(Val: Pat.get()), Alts);
1707 break;
1708 }
1709 default:
1710 llvm_unreachable("unknown pattern kind!");
1711 }
1712 }
1713
1714 const bool Res = IsUsingCustomCXXAction
1715 ? emitCXXMatchApply(CE, M, Matchers: CXXMatchers)
1716 : emitApplyPatterns(CE, M);
1717 if (!Res)
1718 return false;
1719 }
1720
1721 return true;
1722}
1723
1724bool CombineRuleBuilder::emitPatFragMatchPattern(
1725 CodeExpansions &CE, const PatternAlternatives &Alts, RuleMatcher &RM,
1726 InstructionMatcher *IM, const PatFragPattern &PFP,
1727 DenseSet<const Pattern *> &SeenPats) {
1728 auto StackTrace = PrettyStackTraceEmit(RuleDef, &PFP);
1729
1730 if (!SeenPats.insert(V: &PFP).second)
1731 return true;
1732
1733 const auto &PF = PFP.getPatFrag();
1734
1735 if (!IM) {
1736 // When we don't have an IM, this means this PatFrag isn't reachable from
1737 // the root. This is only acceptable if it doesn't define anything (e.g. a
1738 // pure C++ PatFrag).
1739 if (PF.num_out_params() != 0) {
1740 PFP.reportUnreachable(Locs: RuleDef.getLoc());
1741 return false;
1742 }
1743 } else {
1744 // When an IM is provided, this is reachable from the root, and we're
1745 // expecting to have output operands.
1746 // TODO: If we want to allow for multiple roots we'll need a map of IMs
1747 // then, and emission becomes a bit more complicated.
1748 assert(PF.num_roots() == 1);
1749 }
1750
1751 CodeExpansions PatFragCEs;
1752 if (!PFP.mapInputCodeExpansions(ParentCEs: CE, PatFragCEs, DiagLoc: RuleDef.getLoc()))
1753 return false;
1754
1755 // List of {ParamName, ArgName}.
1756 // When all patterns have been emitted, find expansions in PatFragCEs named
1757 // ArgName and add their expansion to CE using ParamName as the key.
1758 SmallVector<std::pair<std::string, std::string>, 4> CEsToImport;
1759
1760 // Map parameter names to the actual argument.
1761 const auto OperandMapper =
1762 [&](const InstructionOperand &O) -> InstructionOperand {
1763 if (!O.isNamedOperand())
1764 return O;
1765
1766 StringRef ParamName = O.getOperandName();
1767
1768 // Not sure what to do with those tbh. They should probably never be here.
1769 assert(!O.isNamedImmediate() && "TODO: handle named imms");
1770 unsigned PIdx = PF.getParamIdx(Name: ParamName);
1771
1772 // Map parameters to the argument values.
1773 if (PIdx == (unsigned)-1) {
1774 // This is a temp of the PatFragPattern, prefix the name to avoid
1775 // conflicts.
1776 return O.withNewName(
1777 NewName: insertStrRef(S: (PFP.getName() + "." + ParamName).str()));
1778 }
1779
1780 // The operand will be added to PatFragCEs's code expansions using the
1781 // parameter's name. If it's bound to some operand during emission of the
1782 // patterns, we'll want to add it to CE.
1783 auto ArgOp = PFP.getOperand(K: PIdx);
1784 if (ArgOp.isNamedOperand())
1785 CEsToImport.emplace_back(Args: ArgOp.getOperandName().str(), Args&: ParamName);
1786
1787 if (ArgOp.getType() && O.getType() && ArgOp.getType() != O.getType()) {
1788 StringRef PFName = PF.getName();
1789 PrintWarning(Msg: "impossible type constraints: operand " + Twine(PIdx) +
1790 " of '" + PFP.getName() + "' has type '" +
1791 ArgOp.getType().str() + "', but '" + PFName +
1792 "' constrains it to '" + O.getType().str() + "'");
1793 if (ArgOp.isNamedOperand())
1794 PrintNote(Msg: "operand " + Twine(PIdx) + " of '" + PFP.getName() +
1795 "' is '" + ArgOp.getOperandName() + "'");
1796 if (O.isNamedOperand())
1797 PrintNote(Msg: "argument " + Twine(PIdx) + " of '" + PFName + "' is '" +
1798 ParamName + "'");
1799 }
1800
1801 return ArgOp;
1802 };
1803
1804 // PatFragPatterns are only made of InstructionPatterns or CXXPatterns.
1805 // Emit instructions from the root.
1806 const auto &FragAlt = PF.getAlternative(K: Alts.lookup(Val: &PFP));
1807 const auto &FragAltOT = FragAlt.OpTable;
1808 const auto LookupOperandDef =
1809 [&](StringRef Op) -> const InstructionPattern * {
1810 return FragAltOT.getDef(OpName: Op);
1811 };
1812
1813 DenseSet<const Pattern *> PatFragSeenPats;
1814 for (const auto &[Idx, InOp] : enumerate(First: PF.out_params())) {
1815 if (InOp.Kind != PatFrag::PK_Root)
1816 continue;
1817
1818 StringRef ParamName = InOp.Name;
1819 const auto *Def = FragAltOT.getDef(OpName: ParamName);
1820 assert(Def && "PatFrag::checkSemantics should have emitted an error if "
1821 "an out operand isn't defined!");
1822 assert(isa<CodeGenInstructionPattern>(Def) &&
1823 "Nested PatFrags not supported yet");
1824
1825 if (!emitCodeGenInstructionMatchPattern(
1826 CE&: PatFragCEs, Alts, M&: RM, IM&: *IM, P: *cast<CodeGenInstructionPattern>(Val: Def),
1827 SeenPats&: PatFragSeenPats, LookupOperandDef, OperandMapper))
1828 return false;
1829 }
1830
1831 // Emit leftovers.
1832 for (const auto &Pat : FragAlt.Pats) {
1833 if (PatFragSeenPats.contains(V: Pat.get()))
1834 continue;
1835
1836 if (const auto *CXXPat = dyn_cast<CXXPattern>(Val: Pat.get())) {
1837 addCXXPredicate(M&: RM, CE: PatFragCEs, P: *CXXPat, Alts);
1838 continue;
1839 }
1840
1841 if (const auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get())) {
1842 IP->reportUnreachable(Locs: PF.getLoc());
1843 return false;
1844 }
1845
1846 llvm_unreachable("Unexpected pattern kind in PatFrag");
1847 }
1848
1849 for (const auto &[ParamName, ArgName] : CEsToImport) {
1850 // Note: we're find if ParamName already exists. It just means it's been
1851 // bound before, so we prefer to keep the first binding.
1852 CE.declare(Name: ParamName, Expansion: PatFragCEs.lookup(Variable: ArgName));
1853 }
1854
1855 return true;
1856}
1857
1858bool CombineRuleBuilder::emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M) {
1859 assert(MatchDatas.empty());
1860
1861 DenseSet<const Pattern *> SeenPats;
1862 StringMap<unsigned> OperandToTempRegID;
1863
1864 for (auto *ApplyRoot : ApplyRoots) {
1865 assert(isa<InstructionPattern>(ApplyRoot) &&
1866 "Root can only be a InstructionPattern!");
1867 if (!emitInstructionApplyPattern(CE, M,
1868 P: cast<InstructionPattern>(Val&: *ApplyRoot),
1869 SeenPats, OperandToTempRegID))
1870 return false;
1871 }
1872
1873 for (auto &Pat : values(C&: ApplyPats)) {
1874 if (SeenPats.contains(V: Pat.get()))
1875 continue;
1876
1877 switch (Pat->getKind()) {
1878 case Pattern::K_AnyOpcode:
1879 llvm_unreachable("Unexpected pattern in apply!");
1880 case Pattern::K_PatFrag:
1881 // TODO: We could support pure C++ PatFrags as a temporary thing.
1882 llvm_unreachable("Unexpected pattern in apply!");
1883 case Pattern::K_Builtin:
1884 if (!emitInstructionApplyPattern(CE, M, P: cast<BuiltinPattern>(Val&: *Pat),
1885 SeenPats, OperandToTempRegID))
1886 return false;
1887 break;
1888 case Pattern::K_CodeGenInstruction:
1889 cast<CodeGenInstructionPattern>(Val&: *Pat).reportUnreachable(Locs: RuleDef.getLoc());
1890 return false;
1891 case Pattern::K_CXX: {
1892 llvm_unreachable(
1893 "CXX Pattern Emission should have been handled earlier!");
1894 }
1895 default:
1896 llvm_unreachable("unknown pattern kind!");
1897 }
1898 }
1899
1900 // Erase the root.
1901 unsigned RootInsnID =
1902 M.getInsnVarID(InsnMatcher&: M.getInstructionMatcher(SymbolicName: MatchRoot->getName()));
1903 M.addAction<EraseInstAction>(args&: RootInsnID);
1904
1905 return true;
1906}
1907
1908bool CombineRuleBuilder::emitCXXMatchApply(CodeExpansions &CE, RuleMatcher &M,
1909 ArrayRef<CXXPattern *> Matchers) {
1910 assert(hasOnlyCXXApplyPatterns());
1911 declareAllMatchDatasExpansions(CE);
1912
1913 std::string CodeStr;
1914 raw_string_ostream OS(CodeStr);
1915
1916 for (auto &MD : MatchDatas)
1917 OS << MD.Type << " " << MD.getVarName() << ";\n";
1918
1919 if (!Matchers.empty()) {
1920 OS << "// Match Patterns\n";
1921 for (auto *M : Matchers) {
1922 OS << "if(![&](){";
1923 CodeExpander Expander(M->getRawCode(), CE, RuleDef.getLoc(),
1924 /*ShowExpansions=*/false);
1925 Expander.emit(OS);
1926 OS << "}()) {\n"
1927 << " return false;\n}\n";
1928 }
1929 }
1930
1931 OS << "// Apply Patterns\n";
1932 ListSeparator LS("\n");
1933 for (auto &Pat : ApplyPats) {
1934 auto *CXXPat = cast<CXXPattern>(Val: Pat.second.get());
1935 CodeExpander Expander(CXXPat->getRawCode(), CE, RuleDef.getLoc(),
1936 /*ShowExpansions=*/false);
1937 OS << LS;
1938 Expander.emit(OS);
1939 }
1940
1941 const auto &Code = CXXPredicateCode::getCustomActionCode(Code: CodeStr);
1942 M.setCustomCXXAction(Code.getEnumNameWithPrefix(Prefix: CXXCustomActionPrefix));
1943 return true;
1944}
1945
1946bool CombineRuleBuilder::emitInstructionApplyPattern(
1947 CodeExpansions &CE, RuleMatcher &M, const InstructionPattern &P,
1948 DenseSet<const Pattern *> &SeenPats,
1949 StringMap<unsigned> &OperandToTempRegID) {
1950 auto StackTrace = PrettyStackTraceEmit(RuleDef, &P);
1951
1952 if (!SeenPats.insert(V: &P).second)
1953 return true;
1954
1955 // First, render the uses.
1956 for (auto &Op : P.named_operands()) {
1957 if (Op.isDef())
1958 continue;
1959
1960 StringRef OpName = Op.getOperandName();
1961 if (const auto *DefPat = ApplyOpTable.getDef(OpName)) {
1962 if (!emitInstructionApplyPattern(CE, M, P: *DefPat, SeenPats,
1963 OperandToTempRegID))
1964 return false;
1965 } else {
1966 // If we have no def, check this exists in the MatchRoot.
1967 if (!Op.isNamedImmediate() && !MatchOpTable.lookup(OpName).Found) {
1968 PrintError(Msg: "invalid output operand '" + OpName +
1969 "': operand is not a live-in of the match pattern, and it "
1970 "has no definition");
1971 return false;
1972 }
1973 }
1974 }
1975
1976 if (const auto *BP = dyn_cast<BuiltinPattern>(Val: &P))
1977 return emitBuiltinApplyPattern(CE, M, P: *BP, OperandToTempRegID);
1978
1979 if (isa<PatFragPattern>(Val: &P))
1980 llvm_unreachable("PatFragPatterns is not supported in 'apply'!");
1981
1982 auto &CGIP = cast<CodeGenInstructionPattern>(Val: P);
1983
1984 // Now render this inst.
1985 auto &DstMI =
1986 M.addAction<BuildMIAction>(args: M.allocateOutputInsnID(), args: &CGIP.getInst());
1987
1988 bool HasEmittedIntrinsicID = false;
1989 const auto EmitIntrinsicID = [&]() {
1990 assert(CGIP.isIntrinsic());
1991 DstMI.addRenderer<IntrinsicIDRenderer>(args: CGIP.getIntrinsic());
1992 HasEmittedIntrinsicID = true;
1993 };
1994
1995 for (auto &Op : P.operands()) {
1996 // Emit the intrinsic ID after the last def.
1997 if (CGIP.isIntrinsic() && !Op.isDef() && !HasEmittedIntrinsicID)
1998 EmitIntrinsicID();
1999
2000 if (Op.isNamedImmediate()) {
2001 PrintError(Msg: "invalid output operand '" + Op.getOperandName() +
2002 "': output immediates cannot be named");
2003 PrintNote(Msg: "while emitting pattern '" + P.getName() + "' (" +
2004 P.getInstName() + ")");
2005 return false;
2006 }
2007
2008 if (Op.hasImmValue()) {
2009 if (!emitCodeGenInstructionApplyImmOperand(M, DstMI, P: CGIP, O: Op))
2010 return false;
2011 continue;
2012 }
2013
2014 StringRef OpName = Op.getOperandName();
2015
2016 // Uses of operand.
2017 if (!Op.isDef()) {
2018 if (auto It = OperandToTempRegID.find(Key: OpName);
2019 It != OperandToTempRegID.end()) {
2020 assert(!MatchOpTable.lookup(OpName).Found &&
2021 "Temp reg is also from match pattern?");
2022 DstMI.addRenderer<TempRegRenderer>(args&: It->second);
2023 } else {
2024 // This should be a match live in or a redef of a matched instr.
2025 // If it's a use of a temporary register, then we messed up somewhere -
2026 // the previous condition should have passed.
2027 assert(MatchOpTable.lookup(OpName).Found &&
2028 !ApplyOpTable.getDef(OpName) && "Temp reg not emitted yet!");
2029 DstMI.addRenderer<CopyRenderer>(args&: OpName);
2030 }
2031 continue;
2032 }
2033
2034 // Determine what we're dealing with. Are we replacing a matched
2035 // instruction? Creating a new one?
2036 auto OpLookupRes = MatchOpTable.lookup(OpName);
2037 if (OpLookupRes.Found) {
2038 if (OpLookupRes.isLiveIn()) {
2039 // live-in of the match pattern.
2040 PrintError(Msg: "Cannot define live-in operand '" + OpName +
2041 "' in the 'apply' pattern");
2042 return false;
2043 }
2044 assert(OpLookupRes.Def);
2045
2046 // TODO: Handle this. We need to mutate the instr, or delete the old
2047 // one.
2048 // Likewise, we also need to ensure we redef everything, if the
2049 // instr has more than one def, we need to redef all or nothing.
2050 if (OpLookupRes.Def != MatchRoot) {
2051 PrintError(Msg: "redefining an instruction other than the root is not "
2052 "supported (operand '" +
2053 OpName + "')");
2054 return false;
2055 }
2056 // redef of a match
2057 DstMI.addRenderer<CopyRenderer>(args&: OpName);
2058 continue;
2059 }
2060
2061 // Define a new register unique to the apply patterns (AKA a "temp"
2062 // register).
2063 unsigned TempRegID;
2064 if (auto It = OperandToTempRegID.find(Key: OpName);
2065 It != OperandToTempRegID.end()) {
2066 TempRegID = It->second;
2067 } else {
2068 // This is a brand new register.
2069 TempRegID = M.allocateTempRegID();
2070 OperandToTempRegID[OpName] = TempRegID;
2071 const auto Ty = Op.getType();
2072 if (!Ty) {
2073 PrintError(Msg: "def of a new register '" + OpName +
2074 "' in the apply patterns must have a type");
2075 return false;
2076 }
2077
2078 declareTempRegExpansion(CE, TempRegID, Name: OpName);
2079 // Always insert the action at the beginning, otherwise we may end up
2080 // using the temp reg before it's available.
2081 auto Result = getLLTCodeGenOrTempType(PT: Ty, RM&: M);
2082 if (!Result)
2083 return false;
2084 M.insertAction<MakeTempRegisterAction>(InsertPt: M.actions_begin(), args&: *Result,
2085 args&: TempRegID);
2086 }
2087
2088 DstMI.addRenderer<TempRegRenderer>(args&: TempRegID, /*IsDef=*/args: true);
2089 }
2090
2091 // Some intrinsics have no in operands, ensure the ID is still emitted in such
2092 // cases.
2093 if (CGIP.isIntrinsic() && !HasEmittedIntrinsicID)
2094 EmitIntrinsicID();
2095
2096 // Render MIFlags
2097 if (const auto *FI = CGIP.getMIFlagsInfo()) {
2098 for (StringRef InstName : FI->copy_flags())
2099 DstMI.addCopiedMIFlags(IM: M.getInstructionMatcher(SymbolicName: InstName));
2100 for (StringRef F : FI->set_flags())
2101 DstMI.addSetMIFlags(Flag: F);
2102 for (StringRef F : FI->unset_flags())
2103 DstMI.addUnsetMIFlags(Flag: F);
2104 }
2105
2106 // Don't allow mutating opcodes for GISel combiners. We want a more precise
2107 // handling of MIFlags so we require them to be explicitly preserved.
2108 //
2109 // TODO: We don't mutate very often, if at all in combiners, but it'd be nice
2110 // to re-enable this. We'd then need to always clear MIFlags when mutating
2111 // opcodes, and never mutate an inst that we copy flags from.
2112 // DstMI.chooseInsnToMutate(M);
2113 declareInstExpansion(CE, A: DstMI, Name: P.getName());
2114
2115 return true;
2116}
2117
2118bool CombineRuleBuilder::emitCodeGenInstructionApplyImmOperand(
2119 RuleMatcher &M, BuildMIAction &DstMI, const CodeGenInstructionPattern &P,
2120 const InstructionOperand &O) {
2121 // If we have a type, we implicitly emit a G_CONSTANT, except for G_CONSTANT
2122 // itself where we emit a CImm.
2123 //
2124 // No type means we emit a simple imm.
2125 // G_CONSTANT is a special case and needs a CImm though so this is likely a
2126 // mistake.
2127 const bool isGConstant = P.is(OpcodeName: "G_CONSTANT");
2128 const auto Ty = O.getType();
2129 if (!Ty) {
2130 if (isGConstant) {
2131 PrintError(Msg: "'G_CONSTANT' immediate must be typed!");
2132 PrintNote(Msg: "while emitting pattern '" + P.getName() + "' (" +
2133 P.getInstName() + ")");
2134 return false;
2135 }
2136
2137 DstMI.addRenderer<ImmRenderer>(args: O.getImmValue());
2138 return true;
2139 }
2140
2141 auto ImmTy = getLLTCodeGenOrTempType(PT: Ty, RM&: M);
2142 if (!ImmTy)
2143 return false;
2144
2145 if (isGConstant) {
2146 DstMI.addRenderer<ImmRenderer>(args: O.getImmValue(), args&: *ImmTy);
2147 return true;
2148 }
2149
2150 unsigned TempRegID = M.allocateTempRegID();
2151 // Ensure MakeTempReg & the BuildConstantAction occur at the beginning.
2152 auto InsertIt = M.insertAction<MakeTempRegisterAction>(InsertPt: M.actions_begin(),
2153 args&: *ImmTy, args&: TempRegID);
2154 M.insertAction<BuildConstantAction>(InsertPt: ++InsertIt, args&: TempRegID, args: O.getImmValue());
2155 DstMI.addRenderer<TempRegRenderer>(args&: TempRegID);
2156 return true;
2157}
2158
2159bool CombineRuleBuilder::emitBuiltinApplyPattern(
2160 CodeExpansions &CE, RuleMatcher &M, const BuiltinPattern &P,
2161 StringMap<unsigned> &OperandToTempRegID) {
2162 const auto Error = [&](Twine Reason) {
2163 PrintError(Msg: "cannot emit '" + P.getInstName() + "' builtin: " + Reason);
2164 return false;
2165 };
2166
2167 switch (P.getBuiltinKind()) {
2168 case BI_EraseRoot: {
2169 // Root is always inst 0.
2170 M.addAction<EraseInstAction>(/*InsnID*/ args: 0);
2171 return true;
2172 }
2173 case BI_ReplaceReg: {
2174 StringRef Old = P.getOperand(K: 0).getOperandName();
2175 StringRef New = P.getOperand(K: 1).getOperandName();
2176
2177 if (!ApplyOpTable.lookup(OpName: New).Found && !MatchOpTable.lookup(OpName: New).Found)
2178 return Error("unknown operand '" + Old + "'");
2179
2180 auto &OldOM = M.getOperandMatcher(Name: Old);
2181 if (auto It = OperandToTempRegID.find(Key: New);
2182 It != OperandToTempRegID.end()) {
2183 // Replace with temp reg.
2184 M.addAction<ReplaceRegAction>(args: OldOM.getInsnVarID(), args: OldOM.getOpIdx(),
2185 args&: It->second);
2186 } else {
2187 // Replace with matched reg.
2188 auto &NewOM = M.getOperandMatcher(Name: New);
2189 M.addAction<ReplaceRegAction>(args: OldOM.getInsnVarID(), args: OldOM.getOpIdx(),
2190 args: NewOM.getInsnVarID(), args: NewOM.getOpIdx());
2191 }
2192 // checkSemantics should have ensured that we can only rewrite the root.
2193 // Ensure we're deleting it.
2194 assert(MatchOpTable.getDef(Old) == MatchRoot);
2195 return true;
2196 }
2197 }
2198
2199 llvm_unreachable("Unknown BuiltinKind!");
2200}
2201
2202bool isLiteralImm(const InstructionPattern &P, unsigned OpIdx) {
2203 if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: &P)) {
2204 StringRef InstName = CGP->getInst().TheDef->getName();
2205 return (InstName == "G_CONSTANT" || InstName == "G_FCONSTANT") &&
2206 OpIdx == 1;
2207 }
2208
2209 llvm_unreachable("TODO");
2210}
2211
2212bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern(
2213 CodeExpansions &CE, const PatternAlternatives &Alts, RuleMatcher &M,
2214 InstructionMatcher &IM, const CodeGenInstructionPattern &P,
2215 DenseSet<const Pattern *> &SeenPats, OperandDefLookupFn LookupOperandDef,
2216 OperandMapperFnRef OperandMapper) {
2217 auto StackTrace = PrettyStackTraceEmit(RuleDef, &P);
2218
2219 if (!SeenPats.insert(V: &P).second)
2220 return true;
2221
2222 IM.addPredicate<InstructionOpcodeMatcher>(args: &P.getInst());
2223 declareInstExpansion(CE, IM, Name: P.getName());
2224
2225 // If this is an intrinsic, check the intrinsic ID.
2226 if (P.isIntrinsic()) {
2227 // The IntrinsicID's operand is the first operand after the defs.
2228 OperandMatcher &OM = IM.addOperand(OpIdx: P.getNumInstDefs(), SymbolicName: "$intrinsic_id",
2229 AllocatedTemporariesBaseID: AllocatedTemporariesBaseID++);
2230 OM.addPredicate<IntrinsicIDOperandMatcher>(args: P.getIntrinsic());
2231 }
2232
2233 // Check flags if needed.
2234 if (const auto *FI = P.getMIFlagsInfo()) {
2235 assert(FI->copy_flags().empty());
2236
2237 if (const auto &SetF = FI->set_flags(); !SetF.empty())
2238 IM.addPredicate<MIFlagsInstructionPredicateMatcher>(args: SetF.getArrayRef());
2239 if (const auto &UnsetF = FI->unset_flags(); !UnsetF.empty())
2240 IM.addPredicate<MIFlagsInstructionPredicateMatcher>(args: UnsetF.getArrayRef(),
2241 /*CheckNot=*/args: true);
2242 }
2243
2244 for (auto [Idx, OriginalO] : enumerate(First: P.operands())) {
2245 // Remap the operand. This is used when emitting InstructionPatterns inside
2246 // PatFrags, so it can remap them to the arguments passed to the pattern.
2247 //
2248 // We use the remapped operand to emit immediates, and for the symbolic
2249 // operand names (in IM.addOperand). CodeExpansions and OperandTable lookups
2250 // still use the original name.
2251 //
2252 // The "def" flag on the remapped operand is always ignored.
2253 auto RemappedO = OperandMapper(OriginalO);
2254 assert(RemappedO.isNamedOperand() == OriginalO.isNamedOperand() &&
2255 "Cannot remap an unnamed operand to a named one!");
2256
2257 const auto Ty = RemappedO.getType();
2258
2259 const auto OpName =
2260 RemappedO.isNamedOperand() ? RemappedO.getOperandName().str() : "";
2261
2262 // For intrinsics, the first use operand is the intrinsic id, so the true
2263 // operand index is shifted by 1.
2264 //
2265 // From now on:
2266 // Idx = index in the pattern operand list.
2267 // RealIdx = expected index in the MachineInstr.
2268 const unsigned RealIdx =
2269 (P.isIntrinsic() && !OriginalO.isDef()) ? (Idx + 1) : Idx;
2270
2271 if (Ty.isVariadicPack() && M.hasOperand(SymbolicName: OpName)) {
2272 // TODO: We could add some CheckIsSameOperand opcode variant that checks
2273 // all operands. We could also just emit a C++ code snippet lazily to do
2274 // the check since it's probably fairly rare that we need to do it.
2275 //
2276 // I'm just not sure it's worth the effort at this stage.
2277 PrintError(Msg: "each instance of a " + PatternType::VariadicClassName +
2278 " operand must have a unique name within the match patterns");
2279 PrintNote(Msg: "'" + OpName + "' is used multiple times");
2280 return false;
2281 }
2282
2283 OperandMatcher &OM =
2284 IM.addOperand(OpIdx: RealIdx, SymbolicName: OpName, AllocatedTemporariesBaseID: AllocatedTemporariesBaseID++,
2285 /*IsVariadic=*/Ty.isVariadicPack());
2286 if (!OpName.empty())
2287 declareOperandExpansion(CE, OM, Name: OriginalO.getOperandName());
2288
2289 if (Ty.isVariadicPack()) {
2290 // In the presence of variadics, the InstructionMatcher won't insert a
2291 // InstructionNumOperandsMatcher implicitly, so we have to emit our own.
2292 assert((Idx + 1) == P.operands_size() &&
2293 "VariadicPack isn't last operand!");
2294 auto VPTI = Ty.getVariadicPackTypeInfo();
2295 assert(VPTI.Min > 0 && (VPTI.Max == 0 || VPTI.Max > VPTI.Min));
2296 IM.addPredicate<InstructionNumOperandsMatcher>(
2297 args: RealIdx + VPTI.Min, args: InstructionNumOperandsMatcher::CheckKind::GE);
2298 if (VPTI.Max) {
2299 IM.addPredicate<InstructionNumOperandsMatcher>(
2300 args: RealIdx + VPTI.Max, args: InstructionNumOperandsMatcher::CheckKind::LE);
2301 }
2302 break;
2303 }
2304
2305 // Handle immediates.
2306 if (RemappedO.hasImmValue()) {
2307 if (isLiteralImm(P, OpIdx: Idx))
2308 OM.addPredicate<LiteralIntOperandMatcher>(args: RemappedO.getImmValue());
2309 else
2310 OM.addPredicate<ConstantIntOperandMatcher>(args: RemappedO.getImmValue());
2311 }
2312
2313 // Handle typed operands, but only bother to check if it hasn't been done
2314 // before.
2315 //
2316 // getOperandMatcher will always return the first OM to have been created
2317 // for that Operand. "OM" here is always a new OperandMatcher.
2318 //
2319 // Always emit a check for unnamed operands.
2320 if (Ty && (OpName.empty() ||
2321 !M.getOperandMatcher(Name: OpName).contains<LLTOperandMatcher>())) {
2322 // TODO: We could support GITypeOf here on the condition that the
2323 // OperandMatcher exists already. Though it's clunky to make this work
2324 // and isn't all that useful so it's just rejected in typecheckPatterns
2325 // at this time.
2326 assert(Ty.isLLT());
2327 OM.addPredicate<LLTOperandMatcher>(args: getLLTCodeGen(PT: Ty));
2328 }
2329
2330 // Stop here if the operand is a def, or if it had no name.
2331 if (OriginalO.isDef() || !OriginalO.isNamedOperand())
2332 continue;
2333
2334 const auto *DefPat = LookupOperandDef(OriginalO.getOperandName());
2335 if (!DefPat)
2336 continue;
2337
2338 if (OriginalO.hasImmValue()) {
2339 assert(!OpName.empty());
2340 // This is a named immediate that also has a def, that's not okay.
2341 // e.g.
2342 // (G_SEXT $y, (i32 0))
2343 // (COPY $x, 42:$y)
2344 PrintError(Msg: "'" + OpName +
2345 "' is a named immediate, it cannot be defined by another "
2346 "instruction");
2347 PrintNote(Msg: "'" + OpName + "' is defined by '" + DefPat->getName() + "'");
2348 return false;
2349 }
2350
2351 // From here we know that the operand defines an instruction, and we need to
2352 // emit it.
2353 auto InstOpM =
2354 OM.addPredicate<InstructionOperandMatcher>(args&: M, args: DefPat->getName());
2355 if (!InstOpM) {
2356 // TODO: copy-pasted from GlobalISelEmitter.cpp. Is it still relevant
2357 // here?
2358 PrintError(Msg: "Nested instruction '" + DefPat->getName() +
2359 "' cannot be the same as another operand '" +
2360 OriginalO.getOperandName() + "'");
2361 return false;
2362 }
2363
2364 auto &IM = (*InstOpM)->getInsnMatcher();
2365 if (const auto *CGIDef = dyn_cast<CodeGenInstructionPattern>(Val: DefPat)) {
2366 if (!emitCodeGenInstructionMatchPattern(CE, Alts, M, IM, P: *CGIDef,
2367 SeenPats, LookupOperandDef,
2368 OperandMapper))
2369 return false;
2370 continue;
2371 }
2372
2373 if (const auto *PFPDef = dyn_cast<PatFragPattern>(Val: DefPat)) {
2374 if (!emitPatFragMatchPattern(CE, Alts, RM&: M, IM: &IM, PFP: *PFPDef, SeenPats))
2375 return false;
2376 continue;
2377 }
2378
2379 llvm_unreachable("unknown type of InstructionPattern");
2380 }
2381
2382 return true;
2383}
2384
2385//===- GICombinerEmitter --------------------------------------------------===//
2386
2387/// Main implementation class. This emits the tablegenerated output.
2388///
2389/// It collects rules, uses `CombineRuleBuilder` to parse them and accumulate
2390/// RuleMatchers, then takes all the necessary state/data from the various
2391/// static storage pools and wires them together to emit the match table &
2392/// associated function/data structures.
2393class GICombinerEmitter final : public GlobalISelMatchTableExecutorEmitter {
2394 const RecordKeeper &Records;
2395 StringRef Name;
2396 const CodeGenTarget &Target;
2397 const Record *Combiner;
2398 unsigned NextRuleID = 0;
2399
2400 // List all combine rules (ID, name) imported.
2401 // Note that the combiner rule ID is different from the RuleMatcher ID. The
2402 // latter is internal to the MatchTable, the former is the canonical ID of the
2403 // combine rule used to disable/enable it.
2404 std::vector<std::pair<unsigned, std::string>> AllCombineRules;
2405
2406 // Keep track of all rules we've seen so far to ensure we don't process
2407 // the same rule twice.
2408 StringSet<> RulesSeen;
2409
2410 MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules);
2411
2412 void emitRuleConfigImpl(raw_ostream &OS);
2413
2414 void emitAdditionalImpl(raw_ostream &OS) override;
2415
2416 void emitMIPredicateFns(raw_ostream &OS) override;
2417 void emitLeafPredicateFns(raw_ostream &OS) override;
2418 void emitI64ImmPredicateFns(raw_ostream &OS) override;
2419 void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
2420 void emitAPIntImmPredicateFns(raw_ostream &OS) override;
2421 void emitTestSimplePredicate(raw_ostream &OS) override;
2422 void emitRunCustomAction(raw_ostream &OS) override;
2423
2424 const CodeGenTarget &getTarget() const override { return Target; }
2425 StringRef getClassName() const override {
2426 return Combiner->getValueAsString(FieldName: "Classname");
2427 }
2428
2429 StringRef getCombineAllMethodName() const {
2430 return Combiner->getValueAsString(FieldName: "CombineAllMethodName");
2431 }
2432
2433 std::string getRuleConfigClassName() const {
2434 return getClassName().str() + "RuleConfig";
2435 }
2436
2437 void gatherRules(std::vector<RuleMatcher> &Rules,
2438 ArrayRef<const Record *> RulesAndGroups);
2439
2440public:
2441 explicit GICombinerEmitter(const RecordKeeper &RK,
2442 const CodeGenTarget &Target, StringRef Name,
2443 const Record *Combiner);
2444 ~GICombinerEmitter() {}
2445
2446 void run(raw_ostream &OS);
2447};
2448
2449void GICombinerEmitter::emitRuleConfigImpl(raw_ostream &OS) {
2450 OS << "struct " << getRuleConfigClassName() << " {\n"
2451 << " SparseBitVector<> DisabledRules;\n\n"
2452 << " bool isRuleEnabled(unsigned RuleID) const;\n"
2453 << " bool parseCommandLineOption();\n"
2454 << " bool setRuleEnabled(StringRef RuleIdentifier);\n"
2455 << " bool setRuleDisabled(StringRef RuleIdentifier);\n"
2456 << "};\n\n";
2457
2458 std::vector<std::pair<std::string, std::string>> Cases;
2459 Cases.reserve(n: AllCombineRules.size());
2460
2461 for (const auto &[ID, Name] : AllCombineRules)
2462 Cases.emplace_back(args: Name, args: "return " + to_string(Value: ID) + ";\n");
2463
2464 OS << "static std::optional<uint64_t> getRuleIdxForIdentifier(StringRef "
2465 "RuleIdentifier) {\n"
2466 << " uint64_t I;\n"
2467 << " // getAtInteger(...) returns false on success\n"
2468 << " bool Parsed = !RuleIdentifier.getAsInteger(0, I);\n"
2469 << " if (Parsed)\n"
2470 << " return I;\n\n"
2471 << "#ifndef NDEBUG\n";
2472 StringMatcher Matcher("RuleIdentifier", Cases, OS);
2473 Matcher.Emit();
2474 OS << "#endif // ifndef NDEBUG\n\n"
2475 << " return std::nullopt;\n"
2476 << "}\n";
2477
2478 OS << "static std::optional<std::pair<uint64_t, uint64_t>> "
2479 "getRuleRangeForIdentifier(StringRef RuleIdentifier) {\n"
2480 << " std::pair<StringRef, StringRef> RangePair = "
2481 "RuleIdentifier.split('-');\n"
2482 << " if (!RangePair.second.empty()) {\n"
2483 << " const auto First = "
2484 "getRuleIdxForIdentifier(RangePair.first);\n"
2485 << " const auto Last = "
2486 "getRuleIdxForIdentifier(RangePair.second);\n"
2487 << " if (!First || !Last)\n"
2488 << " return std::nullopt;\n"
2489 << " if (First >= Last)\n"
2490 << " report_fatal_error(\"Beginning of range should be before "
2491 "end of range\");\n"
2492 << " return {{*First, *Last + 1}};\n"
2493 << " }\n"
2494 << " if (RangePair.first == \"*\") {\n"
2495 << " return {{0, " << AllCombineRules.size() << "}};\n"
2496 << " }\n"
2497 << " const auto I = getRuleIdxForIdentifier(RangePair.first);\n"
2498 << " if (!I)\n"
2499 << " return std::nullopt;\n"
2500 << " return {{*I, *I + 1}};\n"
2501 << "}\n\n";
2502
2503 for (bool Enabled : {true, false}) {
2504 OS << "bool " << getRuleConfigClassName() << "::setRule"
2505 << (Enabled ? "Enabled" : "Disabled") << "(StringRef RuleIdentifier) {\n"
2506 << " auto MaybeRange = getRuleRangeForIdentifier(RuleIdentifier);\n"
2507 << " if (!MaybeRange)\n"
2508 << " return false;\n"
2509 << " for (auto I = MaybeRange->first; I < MaybeRange->second; ++I)\n"
2510 << " DisabledRules." << (Enabled ? "reset" : "set") << "(I);\n"
2511 << " return true;\n"
2512 << "}\n\n";
2513 }
2514
2515 OS << "static std::vector<std::string> " << Name << "Option;\n"
2516 << "static cl::list<std::string> " << Name << "DisableOption(\n"
2517 << " \"" << Name.lower() << "-disable-rule\",\n"
2518 << " cl::desc(\"Disable one or more combiner rules temporarily in "
2519 << "the " << Name << " pass\"),\n"
2520 << " cl::CommaSeparated,\n"
2521 << " cl::Hidden,\n"
2522 << " cl::cat(GICombinerOptionCategory),\n"
2523 << " cl::callback([](const std::string &Str) {\n"
2524 << " " << Name << "Option.push_back(Str);\n"
2525 << " }));\n"
2526 << "static cl::list<std::string> " << Name << "OnlyEnableOption(\n"
2527 << " \"" << Name.lower() << "-only-enable-rule\",\n"
2528 << " cl::desc(\"Disable all rules in the " << Name
2529 << " pass then re-enable the specified ones\"),\n"
2530 << " cl::Hidden,\n"
2531 << " cl::cat(GICombinerOptionCategory),\n"
2532 << " cl::callback([](const std::string &CommaSeparatedArg) {\n"
2533 << " StringRef Str = CommaSeparatedArg;\n"
2534 << " " << Name << "Option.push_back(\"*\");\n"
2535 << " do {\n"
2536 << " auto X = Str.split(\",\");\n"
2537 << " " << Name << "Option.push_back((\"!\" + X.first).str());\n"
2538 << " Str = X.second;\n"
2539 << " } while (!Str.empty());\n"
2540 << " }));\n"
2541 << "\n\n"
2542 << "bool " << getRuleConfigClassName()
2543 << "::isRuleEnabled(unsigned RuleID) const {\n"
2544 << " return !DisabledRules.test(RuleID);\n"
2545 << "}\n"
2546 << "bool " << getRuleConfigClassName() << "::parseCommandLineOption() {\n"
2547 << " for (StringRef Identifier : " << Name << "Option) {\n"
2548 << " bool Enabled = Identifier.consume_front(\"!\");\n"
2549 << " if (Enabled && !setRuleEnabled(Identifier))\n"
2550 << " return false;\n"
2551 << " if (!Enabled && !setRuleDisabled(Identifier))\n"
2552 << " return false;\n"
2553 << " }\n"
2554 << " return true;\n"
2555 << "}\n\n";
2556}
2557
2558void GICombinerEmitter::emitAdditionalImpl(raw_ostream &OS) {
2559 OS << "bool " << getClassName() << "::" << getCombineAllMethodName()
2560 << "(MachineInstr &I) const {\n"
2561 << " const TargetSubtargetInfo &ST = MF.getSubtarget();\n"
2562 << " const PredicateBitset AvailableFeatures = "
2563 "getAvailableFeatures();\n"
2564 << " B.setInstrAndDebugLoc(I);\n"
2565 << " State.MIs.clear();\n"
2566 << " State.MIs.push_back(&I);\n"
2567 << " if (executeMatchTable(*this, State, ExecInfo, B"
2568 << ", getMatchTable(), *ST.getInstrInfo(), MRI, "
2569 "*MRI.getTargetRegisterInfo(), *ST.getRegBankInfo(), AvailableFeatures"
2570 << ", /*CoverageInfo*/ nullptr)) {\n"
2571 << " return true;\n"
2572 << " }\n\n"
2573 << " return false;\n"
2574 << "}\n\n";
2575}
2576
2577void GICombinerEmitter::emitMIPredicateFns(raw_ostream &OS) {
2578 auto MatchCode = CXXPredicateCode::getAllMatchCode();
2579 emitMIPredicateFnsImpl<const CXXPredicateCode *>(
2580 OS, AdditionalDecls: "", Predicates: ArrayRef<const CXXPredicateCode *>(MatchCode),
2581 GetPredEnumName: [](const CXXPredicateCode *C) -> StringRef { return C->BaseEnumName; },
2582 GetPredCode: [](const CXXPredicateCode *C) -> StringRef { return C->Code; });
2583}
2584
2585void GICombinerEmitter::emitLeafPredicateFns(raw_ostream &OS) {
2586 // Unused, but still needs to be called.
2587 emitLeafPredicateFnsImpl<unsigned>(
2588 OS, AdditionalDecls: "", Predicates: {}, GetPredEnumName: [](unsigned) { return ""; }, GetPredCode: [](unsigned) { return ""; });
2589}
2590
2591void GICombinerEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
2592 // Unused, but still needs to be called.
2593 emitImmPredicateFnsImpl<unsigned>(
2594 OS, TypeIdentifier: "I64", ArgType: "int64_t", Predicates: {}, GetPredEnumName: [](unsigned) { return ""; },
2595 GetPredCode: [](unsigned) { return ""; });
2596}
2597
2598void GICombinerEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) {
2599 // Unused, but still needs to be called.
2600 emitImmPredicateFnsImpl<unsigned>(
2601 OS, TypeIdentifier: "APFloat", ArgType: "const APFloat &", Predicates: {}, GetPredEnumName: [](unsigned) { return ""; },
2602 GetPredCode: [](unsigned) { return ""; });
2603}
2604
2605void GICombinerEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) {
2606 // Unused, but still needs to be called.
2607 emitImmPredicateFnsImpl<unsigned>(
2608 OS, TypeIdentifier: "APInt", ArgType: "const APInt &", Predicates: {}, GetPredEnumName: [](unsigned) { return ""; },
2609 GetPredCode: [](unsigned) { return ""; });
2610}
2611
2612void GICombinerEmitter::emitTestSimplePredicate(raw_ostream &OS) {
2613 if (!AllCombineRules.empty()) {
2614 OS << "enum {\n";
2615 std::string EnumeratorSeparator = " = GICXXPred_Invalid + 1,\n";
2616 // To avoid emitting a switch, we expect that all those rules are in order.
2617 // That way we can just get the RuleID from the enum by subtracting
2618 // (GICXXPred_Invalid + 1).
2619 [[maybe_unused]] unsigned ExpectedID = 0;
2620 for (const auto &ID : keys(C&: AllCombineRules)) {
2621 assert(ExpectedID == ID && "combine rules are not ordered!");
2622 ++ExpectedID;
2623 OS << " " << getIsEnabledPredicateEnumName(CombinerRuleID: ID) << EnumeratorSeparator;
2624 EnumeratorSeparator = ",\n";
2625 }
2626 OS << "};\n\n";
2627 }
2628
2629 OS << "bool " << getClassName()
2630 << "::testSimplePredicate(unsigned Predicate) const {\n"
2631 << " return RuleConfig.isRuleEnabled(Predicate - "
2632 "GICXXPred_Invalid - "
2633 "1);\n"
2634 << "}\n";
2635}
2636
2637void GICombinerEmitter::emitRunCustomAction(raw_ostream &OS) {
2638 const auto CustomActionsCode = CXXPredicateCode::getAllCustomActionsCode();
2639
2640 if (!CustomActionsCode.empty()) {
2641 OS << "enum {\n";
2642 std::string EnumeratorSeparator = " = GICXXCustomAction_Invalid + 1,\n";
2643 for (const auto &CA : CustomActionsCode) {
2644 OS << " " << CA->getEnumNameWithPrefix(Prefix: CXXCustomActionPrefix)
2645 << EnumeratorSeparator;
2646 EnumeratorSeparator = ",\n";
2647 }
2648 OS << "};\n";
2649 }
2650
2651 OS << "bool " << getClassName()
2652 << "::runCustomAction(unsigned ApplyID, const MatcherState &State, "
2653 "NewMIVector &OutMIs) const "
2654 "{\n Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]);\n";
2655 if (!CustomActionsCode.empty()) {
2656 OS << " switch(ApplyID) {\n";
2657 for (const auto &CA : CustomActionsCode) {
2658 OS << " case " << CA->getEnumNameWithPrefix(Prefix: CXXCustomActionPrefix)
2659 << ":{\n"
2660 << " " << join(R: split(Str: CA->Code, Separator: '\n'), Separator: "\n ") << '\n'
2661 << " return true;\n";
2662 OS << " }\n";
2663 }
2664 OS << " }\n";
2665 }
2666 OS << " llvm_unreachable(\"Unknown Apply Action\");\n"
2667 << "}\n";
2668}
2669
2670GICombinerEmitter::GICombinerEmitter(const RecordKeeper &RK,
2671 const CodeGenTarget &Target,
2672 StringRef Name, const Record *Combiner)
2673 : Records(RK), Name(Name), Target(Target), Combiner(Combiner) {}
2674
2675MatchTable
2676GICombinerEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules) {
2677 std::vector<Matcher *> InputRules;
2678 for (Matcher &Rule : Rules)
2679 InputRules.push_back(x: &Rule);
2680
2681 unsigned CurrentOrdering = 0;
2682 StringMap<unsigned> OpcodeOrder;
2683 for (RuleMatcher &Rule : Rules) {
2684 const StringRef Opcode = Rule.getOpcode();
2685 assert(!Opcode.empty() && "Didn't expect an undefined opcode");
2686 if (OpcodeOrder.try_emplace(Key: Opcode, Args&: CurrentOrdering).second)
2687 ++CurrentOrdering;
2688 }
2689
2690 llvm::stable_sort(Range&: InputRules, C: [&OpcodeOrder](const Matcher *A,
2691 const Matcher *B) {
2692 auto *L = static_cast<const RuleMatcher *>(A);
2693 auto *R = static_cast<const RuleMatcher *>(B);
2694 return std::tuple(OpcodeOrder[L->getOpcode()],
2695 L->insnmatchers_front().getNumOperandMatchers()) <
2696 std::tuple(OpcodeOrder[R->getOpcode()],
2697 R->insnmatchers_front().getNumOperandMatchers());
2698 });
2699
2700 for (Matcher *Rule : InputRules)
2701 Rule->optimize();
2702
2703 std::vector<std::unique_ptr<Matcher>> MatcherStorage;
2704 std::vector<Matcher *> OptRules =
2705 optimizeRules<GroupMatcher>(Rules: InputRules, MatcherStorage);
2706
2707 for (Matcher *Rule : OptRules)
2708 Rule->optimize();
2709
2710 OptRules = optimizeRules<SwitchMatcher>(Rules: OptRules, MatcherStorage);
2711
2712 return MatchTable::buildTable(Rules: OptRules, /*WithCoverage*/ false,
2713 /*IsCombiner*/ true);
2714}
2715
2716/// Recurse into GICombineGroup's and flatten the ruleset into a simple list.
2717void GICombinerEmitter::gatherRules(std::vector<RuleMatcher> &ActiveRules,
2718 ArrayRef<const Record *> RulesAndGroups) {
2719 for (const Record *Rec : RulesAndGroups) {
2720 if (!Rec->isValueUnset(FieldName: "Rules")) {
2721 gatherRules(ActiveRules, RulesAndGroups: Rec->getValueAsListOfDefs(FieldName: "Rules"));
2722 continue;
2723 }
2724
2725 StringRef RuleName = Rec->getName();
2726 if (!RulesSeen.insert(key: RuleName).second) {
2727 PrintWarning(WarningLoc: Rec->getLoc(),
2728 Msg: "skipping rule '" + Rec->getName() +
2729 "' because it has already been processed");
2730 continue;
2731 }
2732
2733 AllCombineRules.emplace_back(args&: NextRuleID, args: Rec->getName().str());
2734 CombineRuleBuilder CRB(Target, SubtargetFeatures, *Rec, NextRuleID++,
2735 ActiveRules);
2736
2737 if (!CRB.parseAll()) {
2738 assert(ErrorsPrinted && "Parsing failed without errors!");
2739 continue;
2740 }
2741
2742 if (StopAfterParse) {
2743 CRB.print(OS&: outs());
2744 continue;
2745 }
2746
2747 if (!CRB.emitRuleMatchers()) {
2748 assert(ErrorsPrinted && "Emission failed without errors!");
2749 continue;
2750 }
2751 }
2752}
2753
2754void GICombinerEmitter::run(raw_ostream &OS) {
2755 InstructionOpcodeMatcher::initOpcodeValuesMap(Target);
2756 LLTOperandMatcher::initTypeIDValuesMap();
2757
2758 TGTimer &Timer = Records.getTimer();
2759 Timer.startTimer(Name: "Gather rules");
2760 std::vector<RuleMatcher> Rules;
2761 gatherRules(ActiveRules&: Rules, RulesAndGroups: Combiner->getValueAsListOfDefs(FieldName: "Rules"));
2762 if (ErrorsPrinted)
2763 PrintFatalError(ErrorLoc: Combiner->getLoc(), Msg: "Failed to parse one or more rules");
2764
2765 if (StopAfterParse)
2766 return;
2767
2768 Timer.startTimer(Name: "Creating Match Table");
2769 unsigned MaxTemporaries = 0;
2770 for (const auto &Rule : Rules)
2771 MaxTemporaries = std::max(a: MaxTemporaries, b: Rule.countRendererFns());
2772
2773 llvm::stable_sort(Range&: Rules, C: [&](const RuleMatcher &A, const RuleMatcher &B) {
2774 if (A.isHigherPriorityThan(B)) {
2775 assert(!B.isHigherPriorityThan(A) && "Cannot be more important "
2776 "and less important at "
2777 "the same time");
2778 return true;
2779 }
2780 return false;
2781 });
2782
2783 const MatchTable Table = buildMatchTable(Rules);
2784
2785 Timer.startTimer(Name: "Emit combiner");
2786
2787 emitSourceFileHeader(Desc: getClassName().str() + " Combiner Match Table", OS);
2788
2789 SmallVector<LLTCodeGen, 16> TypeObjects;
2790 append_range(C&: TypeObjects, R&: KnownTypes);
2791 llvm::sort(C&: TypeObjects);
2792
2793 // Hack: Avoid empty declarator.
2794 if (TypeObjects.empty())
2795 TypeObjects.push_back(Elt: LLT::scalar(SizeInBits: 1));
2796
2797 // GET_GICOMBINER_DEPS, which pulls in extra dependencies.
2798 OS << "#ifdef GET_GICOMBINER_DEPS\n"
2799 << "#include \"llvm/ADT/SparseBitVector.h\"\n"
2800 << "namespace llvm {\n"
2801 << "extern cl::OptionCategory GICombinerOptionCategory;\n"
2802 << "} // end namespace llvm\n"
2803 << "#endif // ifdef GET_GICOMBINER_DEPS\n\n";
2804
2805 // GET_GICOMBINER_TYPES, which needs to be included before the declaration of
2806 // the class.
2807 OS << "#ifdef GET_GICOMBINER_TYPES\n";
2808 emitRuleConfigImpl(OS);
2809 OS << "#endif // ifdef GET_GICOMBINER_TYPES\n\n";
2810 emitPredicateBitset(OS, IfDefName: "GET_GICOMBINER_TYPES");
2811
2812 // GET_GICOMBINER_CLASS_MEMBERS, which need to be included inside the class.
2813 emitPredicatesDecl(OS, IfDefName: "GET_GICOMBINER_CLASS_MEMBERS");
2814 emitTemporariesDecl(OS, IfDefName: "GET_GICOMBINER_CLASS_MEMBERS");
2815
2816 // GET_GICOMBINER_IMPL, which needs to be included outside the class.
2817 emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexOperandMatchers: {}, CustomOperandRenderers: {},
2818 IfDefName: "GET_GICOMBINER_IMPL");
2819
2820 // GET_GICOMBINER_CONSTRUCTOR_INITS, which are in the constructor's
2821 // initializer list.
2822 emitPredicatesInit(OS, IfDefName: "GET_GICOMBINER_CONSTRUCTOR_INITS");
2823 emitTemporariesInit(OS, MaxTemporaries, IfDefName: "GET_GICOMBINER_CONSTRUCTOR_INITS");
2824}
2825
2826} // end anonymous namespace
2827
2828//===----------------------------------------------------------------------===//
2829
2830static void EmitGICombiner(const RecordKeeper &RK, raw_ostream &OS) {
2831 EnablePrettyStackTrace();
2832 const CodeGenTarget Target(RK);
2833
2834 if (SelectedCombiners.empty())
2835 PrintFatalError(Msg: "No combiners selected with -combiners");
2836 for (const auto &Combiner : SelectedCombiners) {
2837 const Record *CombinerDef = RK.getDef(Name: Combiner);
2838 if (!CombinerDef)
2839 PrintFatalError(Msg: "Could not find " + Combiner);
2840 GICombinerEmitter(RK, Target, Combiner, CombinerDef).run(OS);
2841 }
2842}
2843
2844static TableGen::Emitter::Opt X("gen-global-isel-combiner", EmitGICombiner,
2845 "Generate GlobalISel Combiner");
2846