1//===- GlobalISelCombinerMatchTableEmitter.cpp - --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file Generate a combiner implementation for GlobalISel from a declarative
10/// syntax using GlobalISelMatchTable.
11///
12/// Usually, TableGen backends use "assert is an error" as a means to report
13/// invalid input. They try to diagnose common case but don't try very hard and
14/// crashes can be common. This backend aims to behave closer to how a language
15/// compiler frontend would behave: we try extra hard to diagnose invalid inputs
16/// early, and any crash should be considered a bug (= a feature or diagnostic
17/// is missing).
18///
19/// While this can make the backend a bit more complex than it needs to be, it
20/// pays off because MIR patterns can get complicated. Giving useful error
21/// messages to combine writers can help boost their productivity.
22///
23/// As with anything, a good balance has to be found. We also don't want to
24/// write hundreds of lines of code to detect edge cases. In practice, crashing
25/// very occasionally, or giving poor errors in some rare instances, is fine.
26///
27//===----------------------------------------------------------------------===//
28
29#include "Basic/CodeGenIntrinsics.h"
30#include "Common/CodeGenInstruction.h"
31#include "Common/CodeGenTarget.h"
32#include "Common/GlobalISel/CXXPredicates.h"
33#include "Common/GlobalISel/CodeExpander.h"
34#include "Common/GlobalISel/CodeExpansions.h"
35#include "Common/GlobalISel/CombinerUtils.h"
36#include "Common/GlobalISel/GlobalISelMatchTable.h"
37#include "Common/GlobalISel/GlobalISelMatchTableExecutorEmitter.h"
38#include "Common/GlobalISel/PatternParser.h"
39#include "Common/GlobalISel/Patterns.h"
40#include "Common/SubtargetFeatureInfo.h"
41#include "llvm/ADT/APInt.h"
42#include "llvm/ADT/EquivalenceClasses.h"
43#include "llvm/ADT/MapVector.h"
44#include "llvm/ADT/Statistic.h"
45#include "llvm/ADT/StringExtras.h"
46#include "llvm/ADT/StringSet.h"
47#include "llvm/Support/CommandLine.h"
48#include "llvm/Support/Debug.h"
49#include "llvm/Support/PrettyStackTrace.h"
50#include "llvm/Support/ScopedPrinter.h"
51#include "llvm/TableGen/CodeGenHelpers.h"
52#include "llvm/TableGen/Error.h"
53#include "llvm/TableGen/Record.h"
54#include "llvm/TableGen/StringMatcher.h"
55#include "llvm/TableGen/TGTimer.h"
56#include "llvm/TableGen/TableGenBackend.h"
57#include <cstdint>
58
59using namespace llvm;
60using namespace llvm::gi;
61
62#define DEBUG_TYPE "gicombiner-emitter"
63
64static cl::OptionCategory
65 GICombinerEmitterCat("Options for -gen-global-isel-combiner");
66static cl::opt<bool> StopAfterParse(
67 "gicombiner-stop-after-parse",
68 cl::desc("Stop processing after parsing rules and dump state"),
69 cl::cat(GICombinerEmitterCat));
70static cl::list<std::string>
71 SelectedCombiners("combiners", cl::desc("Emit the specified combiners"),
72 cl::cat(GICombinerEmitterCat), cl::CommaSeparated);
73static cl::opt<bool> DebugCXXPreds(
74 "gicombiner-debug-cxxpreds",
75 cl::desc("Add Contextual/Debug comments to all C++ predicates"),
76 cl::cat(GICombinerEmitterCat));
77static cl::opt<bool> DebugTypeInfer("gicombiner-debug-typeinfer",
78 cl::desc("Print type inference debug logs"),
79 cl::cat(GICombinerEmitterCat));
80
81constexpr StringLiteral CXXCustomActionPrefix = "GICXXCustomAction_";
82constexpr StringLiteral CXXPredPrefix = "GICXXPred_MI_Predicate_";
83constexpr StringLiteral MatchDataClassName = "GIDefMatchData";
84
85//===- CodeExpansions Helpers --------------------------------------------===//
86
87static void declareInstExpansion(CodeExpansions &CE,
88 const InstructionMatcher &IM, StringRef Name) {
89 CE.declare(Name, Expansion: "State.MIs[" + to_string(Value: IM.getInsnVarID()) + "]");
90}
91
92static void declareInstExpansion(CodeExpansions &CE, const BuildMIAction &A,
93 StringRef Name) {
94 // Note: we use redeclare here because this may overwrite a matcher inst
95 // expansion.
96 CE.redeclare(Name, Expansion: "OutMIs[" + to_string(Value: A.getInsnID()) + "]");
97}
98
99static void declareOperandExpansion(CodeExpansions &CE,
100 const OperandMatcher &OM, StringRef Name) {
101 if (OM.isVariadic()) {
102 CE.declare(Name, Expansion: "getRemainingOperands(*State.MIs[" +
103 to_string(Value: OM.getInsnVarID()) + "], " +
104 to_string(Value: OM.getOpIdx()) + ")");
105 } else {
106 CE.declare(Name, Expansion: "State.MIs[" + to_string(Value: OM.getInsnVarID()) +
107 "]->getOperand(" + to_string(Value: OM.getOpIdx()) + ")");
108 }
109}
110
111static void declareTempRegExpansion(CodeExpansions &CE, unsigned TempRegID,
112 StringRef Name) {
113 CE.declare(Name, Expansion: "State.TempRegisters[" + to_string(Value: TempRegID) + "]");
114}
115
116//===- Misc. Helpers -----------------------------------------------------===//
117
118template <typename Container> static auto keys(Container &&C) {
119 return map_range(C, [](auto &Entry) -> auto & { return Entry.first; });
120}
121
122template <typename Container> static auto values(Container &&C) {
123 return map_range(C, [](auto &Entry) -> auto & { return Entry.second; });
124}
125
126static std::string getIsEnabledPredicateEnumName(unsigned CombinerRuleID) {
127 return "GICXXPred_Simple_IsRule" + to_string(Value: CombinerRuleID) + "Enabled";
128}
129
130//===- MatchTable Helpers ------------------------------------------------===//
131
132static LLTCodeGen getLLTCodeGen(const PatternType &PT) {
133 return *MVTToLLT(VT: getValueType(Rec: PT.getLLTRecord()));
134}
135
136//===- PrettyStackTrace Helpers ------------------------------------------===//
137
138namespace {
139class PrettyStackTraceParse : public PrettyStackTraceEntry {
140 const Record &Def;
141
142public:
143 PrettyStackTraceParse(const Record &Def) : Def(Def) {}
144
145 void print(raw_ostream &OS) const override {
146 if (Def.isSubClassOf(Name: "GICombineRule"))
147 OS << "Parsing GICombineRule '" << Def.getName() << "'";
148 else if (Def.isSubClassOf(Name: PatFrag::ClassName))
149 OS << "Parsing " << PatFrag::ClassName << " '" << Def.getName() << "'";
150 else
151 OS << "Parsing '" << Def.getName() << "'";
152 OS << '\n';
153 }
154};
155
156class PrettyStackTraceEmit : public PrettyStackTraceEntry {
157 const Record &Def;
158 const Pattern *Pat = nullptr;
159
160public:
161 PrettyStackTraceEmit(const Record &Def, const Pattern *Pat = nullptr)
162 : Def(Def), Pat(Pat) {}
163
164 void print(raw_ostream &OS) const override {
165 if (Def.isSubClassOf(Name: "GICombineRule"))
166 OS << "Emitting GICombineRule '" << Def.getName() << "'";
167 else if (Def.isSubClassOf(Name: PatFrag::ClassName))
168 OS << "Emitting " << PatFrag::ClassName << " '" << Def.getName() << "'";
169 else
170 OS << "Emitting '" << Def.getName() << "'";
171
172 if (Pat)
173 OS << " [" << Pat->getKindName() << " '" << Pat->getName() << "']";
174 OS << '\n';
175 }
176};
177
178//===- CombineRuleOperandTypeChecker --------------------------------------===//
179
180/// This is a wrapper around OperandTypeChecker specialized for Combiner Rules.
181/// On top of doing the same things as OperandTypeChecker, this also attempts to
182/// infer as many types as possible for temporary register defs & immediates in
183/// apply patterns.
184///
185/// The inference is trivial and leverages the MCOI OperandTypes encoded in
186/// CodeGenInstructions to infer types across patterns in a CombineRule. It's
187/// thus very limited and only supports CodeGenInstructions (but that's the main
188/// use case so it's fine).
189///
190/// We only try to infer untyped operands in apply patterns when they're temp
191/// reg defs, or immediates. Inference always outputs a `TypeOf<$x>` where $x is
192/// a named operand from a match pattern.
193class CombineRuleOperandTypeChecker : private OperandTypeChecker {
194public:
195 CombineRuleOperandTypeChecker(const Record &RuleDef,
196 const OperandTable &MatchOpTable)
197 : OperandTypeChecker(RuleDef.getLoc()), RuleDef(RuleDef),
198 MatchOpTable(MatchOpTable) {}
199
200 /// Records and checks a 'match' pattern.
201 bool processMatchPattern(InstructionPattern &P);
202
203 /// Records and checks an 'apply' pattern.
204 bool processApplyPattern(InstructionPattern &P);
205
206 /// Propagates types, then perform type inference and do a second round of
207 /// propagation in the apply patterns only if any types were inferred.
208 void propagateAndInferTypes();
209
210private:
211 /// TypeEquivalenceClasses are groups of operands of an instruction that share
212 /// a common type.
213 ///
214 /// e.g. [[a, b], [c, d]] means a and b have the same type, and c and
215 /// d have the same type too. b/c and a/d don't have to have the same type,
216 /// though.
217 using TypeEquivalenceClasses = EquivalenceClasses<StringRef>;
218
219 /// \returns true for `OPERAND_GENERIC_` 0 through 5.
220 /// These are the MCOI types that can be registers. The other MCOI types are
221 /// either immediates, or fancier operands used only post-ISel, so we don't
222 /// care about them for combiners.
223 static bool canMCOIOperandTypeBeARegister(StringRef MCOIType) {
224 // Assume OPERAND_GENERIC_0 through 5 can be registers. The other MCOI
225 // OperandTypes are either never used in gMIR, or not relevant (e.g.
226 // OPERAND_GENERIC_IMM, which is definitely never a register).
227 return MCOIType.drop_back(N: 1).ends_with(Suffix: "OPERAND_GENERIC_");
228 }
229
230 /// Finds the "MCOI::"" operand types for each operand of \p CGP.
231 ///
232 /// This is a bit trickier than it looks because we need to handle variadic
233 /// in/outs.
234 ///
235 /// e.g. for
236 /// (G_BUILD_VECTOR $vec, $x, $y) ->
237 /// [MCOI::OPERAND_GENERIC_0, MCOI::OPERAND_GENERIC_1,
238 /// MCOI::OPERAND_GENERIC_1]
239 ///
240 /// For unknown types (which can happen in variadics where varargs types are
241 /// inconsistent), a unique name is given, e.g. "unknown_type_0".
242 static std::vector<std::string>
243 getMCOIOperandTypes(const CodeGenInstructionPattern &CGP);
244
245 /// Adds the TypeEquivalenceClasses for \p P in \p OutTECs.
246 void getInstEqClasses(const InstructionPattern &P,
247 TypeEquivalenceClasses &OutTECs) const;
248
249 /// Calls `getInstEqClasses` on all patterns of the rule to produce the whole
250 /// rule's TypeEquivalenceClasses.
251 TypeEquivalenceClasses getRuleEqClasses() const;
252
253 /// Tries to infer the type of the \p ImmOpIdx -th operand of \p IP using \p
254 /// TECs.
255 ///
256 /// This is achieved by trying to find a named operand in \p IP that shares
257 /// the same type as \p ImmOpIdx, and using \ref inferNamedOperandType on that
258 /// operand instead.
259 ///
260 /// \returns the inferred type or an empty PatternType if inference didn't
261 /// succeed.
262 PatternType inferImmediateType(const InstructionPattern &IP,
263 unsigned ImmOpIdx,
264 const TypeEquivalenceClasses &TECs) const;
265
266 /// Looks inside \p TECs to infer \p OpName's type.
267 ///
268 /// \returns the inferred type or an empty PatternType if inference didn't
269 /// succeed.
270 PatternType inferNamedOperandType(const InstructionPattern &IP,
271 StringRef OpName,
272 const TypeEquivalenceClasses &TECs,
273 bool AllowSelf = false) const;
274
275 const Record &RuleDef;
276 SmallVector<InstructionPattern *, 8> MatchPats;
277 SmallVector<InstructionPattern *, 8> ApplyPats;
278
279 const OperandTable &MatchOpTable;
280};
281} // namespace
282
283bool CombineRuleOperandTypeChecker::processMatchPattern(InstructionPattern &P) {
284 MatchPats.push_back(Elt: &P);
285 return check(P, /*CheckTypeOf*/ VerifyTypeOfOperand: [](const auto &) {
286 // GITypeOf in 'match' is currently always rejected by the
287 // CombineRuleBuilder after inference is done.
288 return true;
289 });
290}
291
292bool CombineRuleOperandTypeChecker::processApplyPattern(InstructionPattern &P) {
293 ApplyPats.push_back(Elt: &P);
294 return check(P, /*CheckTypeOf*/ VerifyTypeOfOperand: [&](const PatternType &Ty) {
295 // GITypeOf<"$x"> can only be used if "$x" is a matched operand.
296 const auto OpName = Ty.getTypeOfOpName();
297 if (MatchOpTable.lookup(OpName).Found)
298 return true;
299
300 PrintError(ErrorLoc: RuleDef.getLoc(), Msg: "'" + OpName + "' ('" + Ty.str() +
301 "') does not refer to a matched operand!");
302 return false;
303 });
304}
305
306void CombineRuleOperandTypeChecker::propagateAndInferTypes() {
307 /// First step here is to propagate types using the OperandTypeChecker. That
308 /// way we ensure all uses of a given register have consistent types.
309 propagateTypes();
310
311 /// Build the TypeEquivalenceClasses for the whole rule.
312 const TypeEquivalenceClasses TECs = getRuleEqClasses();
313
314 /// Look at the apply patterns and find operands that need to be
315 /// inferred. We then try to find an equivalence class that they're a part of
316 /// and select the best operand to use for the `GITypeOf` type. We prioritize
317 /// defs of matched instructions because those are guaranteed to be registers.
318 bool InferredAny = false;
319 for (auto *Pat : ApplyPats) {
320 for (unsigned K = 0; K < Pat->operands_size(); ++K) {
321 auto &Op = Pat->getOperand(K);
322
323 // We only want to take a look at untyped defs or immediates.
324 if ((!Op.isDef() && !Op.hasImmValue()) || Op.getType())
325 continue;
326
327 // Infer defs & named immediates.
328 if (Op.isDef() || Op.isNamedImmediate()) {
329 // Check it's not a redefinition of a matched operand.
330 // In such cases, inference is not necessary because we just copy
331 // operands and don't create temporary registers.
332 if (MatchOpTable.lookup(OpName: Op.getOperandName()).Found)
333 continue;
334
335 // Inference is needed here, so try to do it.
336 if (PatternType Ty =
337 inferNamedOperandType(IP: *Pat, OpName: Op.getOperandName(), TECs)) {
338 if (DebugTypeInfer)
339 errs() << "INFER: " << Op.describe() << " -> " << Ty.str() << '\n';
340 Op.setType(Ty);
341 InferredAny = true;
342 }
343
344 continue;
345 }
346
347 // Infer immediates
348 if (Op.hasImmValue()) {
349 if (PatternType Ty = inferImmediateType(IP: *Pat, ImmOpIdx: K, TECs)) {
350 if (DebugTypeInfer)
351 errs() << "INFER: " << Op.describe() << " -> " << Ty.str() << '\n';
352 Op.setType(Ty);
353 InferredAny = true;
354 }
355 continue;
356 }
357 }
358 }
359
360 // If we've inferred any types, we want to propagate them across the apply
361 // patterns. Type inference only adds GITypeOf types that point to Matched
362 // operands, so we definitely don't want to propagate types into the match
363 // patterns as well, otherwise bad things happen.
364 if (InferredAny) {
365 OperandTypeChecker OTC(RuleDef.getLoc());
366 for (auto *Pat : ApplyPats) {
367 if (!OTC.check(P&: *Pat, VerifyTypeOfOperand: [&](const auto &) { return true; }))
368 PrintFatalError(ErrorLoc: RuleDef.getLoc(),
369 Msg: "OperandTypeChecker unexpectedly failed on '" +
370 Pat->getName() + "' during Type Inference");
371 }
372 OTC.propagateTypes();
373
374 if (DebugTypeInfer) {
375 errs() << "Apply patterns for rule " << RuleDef.getName()
376 << " after inference:\n";
377 for (auto *Pat : ApplyPats) {
378 errs() << " ";
379 Pat->print(OS&: errs(), /*PrintName*/ true);
380 errs() << '\n';
381 }
382 errs() << '\n';
383 }
384 }
385}
386
387PatternType CombineRuleOperandTypeChecker::inferImmediateType(
388 const InstructionPattern &IP, unsigned ImmOpIdx,
389 const TypeEquivalenceClasses &TECs) const {
390 // We can only infer CGPs (except intrinsics).
391 const auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: &IP);
392 if (!CGP || CGP->isIntrinsic())
393 return {};
394
395 // For CGPs, we try to infer immediates by trying to infer another named
396 // operand that shares its type.
397 //
398 // e.g.
399 // Pattern: G_BUILD_VECTOR $x, $y, 0
400 // MCOIs: [MCOI::OPERAND_GENERIC_0, MCOI::OPERAND_GENERIC_1,
401 // MCOI::OPERAND_GENERIC_1]
402 // $y has the same type as 0, so we can infer $y and get the type 0 should
403 // have.
404
405 // We infer immediates by looking for a named operand that shares the same
406 // MCOI type.
407 const auto MCOITypes = getMCOIOperandTypes(CGP: *CGP);
408 StringRef ImmOpTy = MCOITypes[ImmOpIdx];
409
410 for (const auto &[Idx, Ty] : enumerate(First: MCOITypes)) {
411 if (Idx != ImmOpIdx && Ty == ImmOpTy) {
412 const auto &Op = IP.getOperand(K: Idx);
413 if (!Op.isNamedOperand())
414 continue;
415
416 // Named operand with the same name, try to infer that.
417 if (PatternType InferTy = inferNamedOperandType(IP, OpName: Op.getOperandName(),
418 TECs, /*AllowSelf=*/true))
419 return InferTy;
420 }
421 }
422
423 return {};
424}
425
426PatternType CombineRuleOperandTypeChecker::inferNamedOperandType(
427 const InstructionPattern &IP, StringRef OpName,
428 const TypeEquivalenceClasses &TECs, bool AllowSelf) const {
429 // This is the simplest possible case, we just need to find a TEC that
430 // contains OpName. Look at all operands in equivalence class and try to
431 // find a suitable one. If `AllowSelf` is true, the operand itself is also
432 // considered suitable.
433
434 // Check for a def of a matched pattern. This is guaranteed to always
435 // be a register so we can blindly use that.
436 StringRef GoodOpName;
437 for (auto It = TECs.findLeader(V: OpName); It != TECs.member_end(); ++It) {
438 if (!AllowSelf && *It == OpName)
439 continue;
440
441 const auto LookupRes = MatchOpTable.lookup(OpName: *It);
442 if (LookupRes.Def) // Favor defs
443 return PatternType::getTypeOf(OpName: *It);
444
445 // Otherwise just save this in case we don't find any def.
446 if (GoodOpName.empty() && LookupRes.Found)
447 GoodOpName = *It;
448 }
449
450 if (!GoodOpName.empty())
451 return PatternType::getTypeOf(OpName: GoodOpName);
452
453 // No good operand found, give up.
454 return {};
455}
456
457std::vector<std::string> CombineRuleOperandTypeChecker::getMCOIOperandTypes(
458 const CodeGenInstructionPattern &CGP) {
459 // FIXME?: Should we cache this? We call it twice when inferring immediates.
460
461 static unsigned UnknownTypeIdx = 0;
462
463 std::vector<std::string> OpTypes;
464 auto &CGI = CGP.getInst();
465 const Record *VarArgsTy =
466 CGI.TheDef->isSubClassOf(Name: "GenericInstruction")
467 ? CGI.TheDef->getValueAsOptionalDef(FieldName: "variadicOpsType")
468 : nullptr;
469 std::string VarArgsTyName =
470 VarArgsTy ? ("MCOI::" + VarArgsTy->getValueAsString(FieldName: "OperandType")).str()
471 : ("unknown_type_" + Twine(UnknownTypeIdx++)).str();
472
473 // First, handle defs.
474 for (unsigned K = 0; K < CGI.Operands.NumDefs; ++K)
475 OpTypes.push_back(x: CGI.Operands[K].OperandType);
476
477 // Then, handle variadic defs if there are any.
478 if (CGP.hasVariadicDefs()) {
479 for (unsigned K = CGI.Operands.NumDefs; K < CGP.getNumInstDefs(); ++K)
480 OpTypes.push_back(x: VarArgsTyName);
481 }
482
483 // If we had variadic defs, the op idx in the pattern won't match the op idx
484 // in the CGI anymore.
485 int CGIOpOffset = int(CGI.Operands.NumDefs) - CGP.getNumInstDefs();
486 assert(CGP.hasVariadicDefs() ? (CGIOpOffset <= 0) : (CGIOpOffset == 0));
487
488 // Handle all remaining use operands, including variadic ones.
489 for (unsigned K = CGP.getNumInstDefs(); K < CGP.getNumInstOperands(); ++K) {
490 unsigned CGIOpIdx = K + CGIOpOffset;
491 if (CGIOpIdx >= CGI.Operands.size()) {
492 assert(CGP.isVariadic());
493 OpTypes.push_back(x: VarArgsTyName);
494 } else {
495 OpTypes.push_back(x: CGI.Operands[CGIOpIdx].OperandType);
496 }
497 }
498
499 assert(OpTypes.size() == CGP.operands_size());
500 return OpTypes;
501}
502
503void CombineRuleOperandTypeChecker::getInstEqClasses(
504 const InstructionPattern &P, TypeEquivalenceClasses &OutTECs) const {
505 // Determine the TypeEquivalenceClasses by:
506 // - Getting the MCOI Operand Types.
507 // - Creating a Map of MCOI Type -> [Operand Indexes]
508 // - Iterating over the map, filtering types we don't like, and just adding
509 // the array of Operand Indexes to \p OutTECs.
510
511 // We can only do this on CodeGenInstructions that aren't intrinsics. Other
512 // InstructionPatterns have no type inference information associated with
513 // them.
514 // TODO: We could try to extract some info from CodeGenIntrinsic to
515 // guide inference.
516
517 // TODO: Could we add some inference information to builtins at least? e.g.
518 // ReplaceReg should always replace with a reg of the same type, for instance.
519 // Though, those patterns are often used alone so it might not be worth the
520 // trouble to infer their types.
521 auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: &P);
522 if (!CGP || CGP->isIntrinsic())
523 return;
524
525 const auto MCOITypes = getMCOIOperandTypes(CGP: *CGP);
526 assert(MCOITypes.size() == P.operands_size());
527
528 MapVector<StringRef, SmallVector<unsigned, 0>> TyToOpIdx;
529 for (const auto &[Idx, Ty] : enumerate(First: MCOITypes))
530 TyToOpIdx[Ty].push_back(Elt: Idx);
531
532 if (DebugTypeInfer)
533 errs() << "\tGroups for " << P.getName() << ":\t";
534
535 for (const auto &[Ty, Idxs] : TyToOpIdx) {
536 if (!canMCOIOperandTypeBeARegister(MCOIType: Ty))
537 continue;
538
539 if (DebugTypeInfer)
540 errs() << '[';
541 StringRef Sep = "";
542
543 // We only collect named operands.
544 StringRef Leader;
545 for (unsigned Idx : Idxs) {
546 const auto &Op = P.getOperand(K: Idx);
547 if (!Op.isNamedOperand())
548 continue;
549
550 const auto OpName = Op.getOperandName();
551 if (DebugTypeInfer) {
552 errs() << Sep << OpName;
553 Sep = ", ";
554 }
555
556 if (Leader.empty())
557 OutTECs.insert(Data: (Leader = OpName));
558 else
559 OutTECs.unionSets(V1: Leader, V2: OpName);
560 }
561
562 if (DebugTypeInfer)
563 errs() << "] ";
564 }
565
566 if (DebugTypeInfer)
567 errs() << '\n';
568}
569
570CombineRuleOperandTypeChecker::TypeEquivalenceClasses
571CombineRuleOperandTypeChecker::getRuleEqClasses() const {
572 TypeEquivalenceClasses TECs;
573
574 if (DebugTypeInfer)
575 errs() << "Rule Operand Type Equivalence Classes for " << RuleDef.getName()
576 << ":\n";
577
578 for (const auto *Pat : MatchPats)
579 getInstEqClasses(P: *Pat, OutTECs&: TECs);
580 for (const auto *Pat : ApplyPats)
581 getInstEqClasses(P: *Pat, OutTECs&: TECs);
582
583 if (DebugTypeInfer) {
584 errs() << "Final Type Equivalence Classes: ";
585 for (const auto &Class : TECs) {
586 // only print non-empty classes.
587 if (auto MembIt = TECs.member_begin(ECV: *Class);
588 MembIt != TECs.member_end()) {
589 errs() << '[';
590 StringRef Sep = "";
591 for (; MembIt != TECs.member_end(); ++MembIt) {
592 errs() << Sep << *MembIt;
593 Sep = ", ";
594 }
595 errs() << "] ";
596 }
597 }
598 errs() << '\n';
599 }
600
601 return TECs;
602}
603
604//===- MatchData Handling -------------------------------------------------===//
605struct MatchDataDef {
606 MatchDataDef(StringRef Symbol, StringRef Type) : Symbol(Symbol), Type(Type) {}
607
608 StringRef Symbol;
609 StringRef Type;
610
611 /// \returns the desired variable name for this MatchData.
612 std::string getVarName() const {
613 // Add a prefix in case the symbol name is very generic and conflicts with
614 // something else.
615 return "GIMatchData_" + Symbol.str();
616 }
617};
618
619//===- CombineRuleBuilder -------------------------------------------------===//
620
621/// Parses combine rule and builds a small intermediate representation to tie
622/// patterns together and emit RuleMatchers to match them. This may emit more
623/// than one RuleMatcher, e.g. for `wip_match_opcode`.
624///
625/// Memory management for `Pattern` objects is done through `std::unique_ptr`.
626/// In most cases, there are two stages to a pattern's lifetime:
627/// - Creation in a `parse` function
628/// - The unique_ptr is stored in a variable, and may be destroyed if the
629/// pattern is found to be semantically invalid.
630/// - Ownership transfer into a `PatternMap`
631/// - Once a pattern is moved into either the map of Match or Apply
632/// patterns, it is known to be valid and it never moves back.
633class CombineRuleBuilder {
634public:
635 using PatternMap = MapVector<StringRef, std::unique_ptr<Pattern>>;
636 using PatternAlternatives = DenseMap<const Pattern *, unsigned>;
637
638 CombineRuleBuilder(const CodeGenTarget &CGT,
639 SubtargetFeatureInfoMap &SubtargetFeatures,
640 const Record &RuleDef, unsigned ID,
641 std::vector<RuleMatcher> &OutRMs)
642 : Parser(CGT, RuleDef.getLoc()), CGT(CGT),
643 SubtargetFeatures(SubtargetFeatures), RuleDef(RuleDef), RuleID(ID),
644 OutRMs(OutRMs) {}
645
646 /// Parses all fields in the RuleDef record.
647 bool parseAll();
648
649 /// Emits all RuleMatchers into the vector of RuleMatchers passed in the
650 /// constructor.
651 bool emitRuleMatchers();
652
653 void print(raw_ostream &OS) const;
654 void dump() const { print(OS&: dbgs()); }
655
656 /// Debug-only verification of invariants.
657#ifndef NDEBUG
658 void verify() const;
659#endif
660
661private:
662 const CodeGenInstruction &getGConstant() const {
663 return CGT.getInstruction(InstRec: RuleDef.getRecords().getDef(Name: "G_CONSTANT"));
664 }
665
666 std::optional<LLTCodeGenOrTempType>
667 getLLTCodeGenOrTempType(const PatternType &PT, RuleMatcher &RM);
668
669 void PrintError(Twine Msg) const { ::PrintError(Rec: &RuleDef, Msg); }
670 void PrintWarning(Twine Msg) const { ::PrintWarning(WarningLoc: RuleDef.getLoc(), Msg); }
671 void PrintNote(Twine Msg) const { ::PrintNote(NoteLoc: RuleDef.getLoc(), Msg); }
672
673 void print(raw_ostream &OS, const PatternAlternatives &Alts) const;
674
675 bool addApplyPattern(std::unique_ptr<Pattern> Pat);
676 bool addMatchPattern(std::unique_ptr<Pattern> Pat);
677
678 /// Adds the expansions from \see MatchDatas to \p CE.
679 void declareAllMatchDatasExpansions(CodeExpansions &CE) const;
680
681 /// Adds a matcher \p P to \p IM, expanding its code using \p CE.
682 /// Note that the predicate is added on the last InstructionMatcher.
683 ///
684 /// \p Alts is only used if DebugCXXPreds is enabled.
685 void addCXXPredicate(RuleMatcher &M, const CodeExpansions &CE,
686 const CXXPattern &P, const PatternAlternatives &Alts);
687
688 bool hasOnlyCXXApplyPatterns() const;
689 bool hasEraseRoot() const;
690
691 // Infer machine operand types and check their consistency.
692 bool typecheckPatterns();
693
694 /// For all PatFragPatterns, add a new entry in PatternAlternatives for each
695 /// PatternList it contains. This is multiplicative, so if we have 2
696 /// PatFrags with 3 alternatives each, we get 2*3 permutations added to
697 /// PermutationsToEmit. The "MaxPermutations" field controls how many
698 /// permutations are allowed before an error is emitted and this function
699 /// returns false. This is a simple safeguard to prevent combination of
700 /// PatFrags from generating enormous amounts of rules.
701 bool buildPermutationsToEmit();
702
703 /// Checks additional semantics of the Patterns.
704 bool checkSemantics();
705
706 /// Creates a new RuleMatcher with some boilerplate
707 /// settings/actions/predicates, and and adds it to \p OutRMs.
708 /// \see addFeaturePredicates too.
709 ///
710 /// \param Alts Current set of alternatives, for debug comment.
711 /// \param AdditionalComment Comment string to be added to the
712 /// `DebugCommentAction`.
713 RuleMatcher &addRuleMatcher(const PatternAlternatives &Alts,
714 Twine AdditionalComment = "");
715 bool addFeaturePredicates(RuleMatcher &M);
716
717 bool findRoots();
718 bool buildRuleOperandsTable();
719
720 bool parseDefs(const DagInit &Def);
721
722 bool emitMatchPattern(CodeExpansions &CE, const PatternAlternatives &Alts,
723 const InstructionPattern &IP);
724 bool emitMatchPattern(CodeExpansions &CE, const PatternAlternatives &Alts,
725 const AnyOpcodePattern &AOP);
726
727 bool emitPatFragMatchPattern(CodeExpansions &CE,
728 const PatternAlternatives &Alts, RuleMatcher &RM,
729 InstructionMatcher *IM,
730 const PatFragPattern &PFP,
731 DenseSet<const Pattern *> &SeenPats);
732
733 bool emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M);
734 bool emitCXXMatchApply(CodeExpansions &CE, RuleMatcher &M,
735 ArrayRef<CXXPattern *> Matchers);
736
737 // Recursively visits InstructionPatterns from P to build up the
738 // RuleMatcher actions.
739 bool emitInstructionApplyPattern(CodeExpansions &CE, RuleMatcher &M,
740 const InstructionPattern &P,
741 DenseSet<const Pattern *> &SeenPats,
742 StringMap<unsigned> &OperandToTempRegID);
743
744 bool emitCodeGenInstructionApplyImmOperand(RuleMatcher &M,
745 BuildMIAction &DstMI,
746 const CodeGenInstructionPattern &P,
747 const InstructionOperand &O);
748
749 bool emitBuiltinApplyPattern(CodeExpansions &CE, RuleMatcher &M,
750 const BuiltinPattern &P,
751 StringMap<unsigned> &OperandToTempRegID);
752
753 // Recursively visits CodeGenInstructionPattern from P to build up the
754 // RuleMatcher/InstructionMatcher. May create new InstructionMatchers as
755 // needed.
756 using OperandMapperFnRef =
757 function_ref<InstructionOperand(const InstructionOperand &)>;
758 using OperandDefLookupFn =
759 function_ref<const InstructionPattern *(StringRef)>;
760 bool emitCodeGenInstructionMatchPattern(
761 CodeExpansions &CE, const PatternAlternatives &Alts, RuleMatcher &M,
762 InstructionMatcher &IM, const CodeGenInstructionPattern &P,
763 DenseSet<const Pattern *> &SeenPats, OperandDefLookupFn LookupOperandDef,
764 OperandMapperFnRef OperandMapper = [](const auto &O) { return O; });
765
766 PatternParser Parser;
767 const CodeGenTarget &CGT;
768 SubtargetFeatureInfoMap &SubtargetFeatures;
769 const Record &RuleDef;
770 const unsigned RuleID;
771 std::vector<RuleMatcher> &OutRMs;
772
773 // For InstructionMatcher::addOperand
774 unsigned AllocatedTemporariesBaseID = 0;
775
776 /// The root of the pattern.
777 StringRef RootName;
778
779 /// These maps have ownership of the actual Pattern objects.
780 /// They both map a Pattern's name to the Pattern instance.
781 PatternMap MatchPats;
782 PatternMap ApplyPats;
783
784 /// Operand tables to tie match/apply patterns together.
785 OperandTable MatchOpTable;
786 OperandTable ApplyOpTable;
787
788 /// Set by findRoots.
789 Pattern *MatchRoot = nullptr;
790 SmallDenseSet<InstructionPattern *, 2> ApplyRoots;
791
792 SmallVector<MatchDataDef, 2> MatchDatas;
793 SmallVector<PatternAlternatives, 1> PermutationsToEmit;
794};
795
796bool CombineRuleBuilder::parseAll() {
797 auto StackTrace = PrettyStackTraceParse(RuleDef);
798
799 if (!parseDefs(Def: *RuleDef.getValueAsDag(FieldName: "Defs")))
800 return false;
801
802 const DagInit &Act0 = *RuleDef.getValueAsDag(FieldName: "Action0");
803 const DagInit &Act1 = *RuleDef.getValueAsDag(FieldName: "Action1");
804
805 StringRef Act0Op = Act0.getOperatorAsDef(Loc: RuleDef.getLoc())->getName();
806 StringRef Act1Op = Act1.getOperatorAsDef(Loc: RuleDef.getLoc())->getName();
807
808 if (Act0Op == "match" && Act1Op == "apply") {
809 if (!Parser.parsePatternList(
810 List: Act0, ParseAction: [this](auto Pat) { return addMatchPattern(Pat: std::move(Pat)); },
811 Operator: "match", AnonPatNamePrefix: (RuleDef.getName() + "_match").str()))
812 return false;
813
814 if (!Parser.parsePatternList(
815 List: Act1, ParseAction: [this](auto Pat) { return addApplyPattern(Pat: std::move(Pat)); },
816 Operator: "apply", AnonPatNamePrefix: (RuleDef.getName() + "_apply").str()))
817 return false;
818
819 } else if (Act0Op == "combine" && Act1Op == "empty_action") {
820 // combine: everything is a "match" except C++ code which is an apply.
821 const auto AddCombinePat = [this](std::unique_ptr<Pattern> Pat) {
822 if (isa<CXXPattern>(Val: Pat.get()))
823 return addApplyPattern(Pat: std::move(Pat));
824 return addMatchPattern(Pat: std::move(Pat));
825 };
826
827 if (!Parser.parsePatternList(List: Act0, ParseAction: AddCombinePat, Operator: "combine",
828 AnonPatNamePrefix: (RuleDef.getName() + "_combine").str()))
829 return false;
830
831 if (MatchPats.empty() || ApplyPats.empty()) {
832 PrintError(Msg: "'combine' action needs at least one pattern to match, and "
833 "C++ code to apply");
834 return false;
835 }
836 } else {
837 PrintError(Msg: "expected both a 'match' and 'apply' action in combine rule, "
838 "or a single 'combine' action");
839 return false;
840 }
841
842 if (!buildRuleOperandsTable() || !typecheckPatterns() || !findRoots() ||
843 !checkSemantics() || !buildPermutationsToEmit())
844 return false;
845 LLVM_DEBUG(verify());
846 return true;
847}
848
849bool CombineRuleBuilder::emitRuleMatchers() {
850 auto StackTrace = PrettyStackTraceEmit(RuleDef);
851
852 assert(MatchRoot);
853 CodeExpansions CE;
854
855 assert(!PermutationsToEmit.empty());
856 for (const auto &Alts : PermutationsToEmit) {
857 switch (MatchRoot->getKind()) {
858 case Pattern::K_AnyOpcode: {
859 if (!emitMatchPattern(CE, Alts, AOP: *cast<AnyOpcodePattern>(Val: MatchRoot)))
860 return false;
861 break;
862 }
863 case Pattern::K_PatFrag:
864 case Pattern::K_Builtin:
865 case Pattern::K_CodeGenInstruction:
866 if (!emitMatchPattern(CE, Alts, IP: *cast<InstructionPattern>(Val: MatchRoot)))
867 return false;
868 break;
869 case Pattern::K_CXX:
870 PrintError(Msg: "C++ code cannot be the root of a rule!");
871 return false;
872 default:
873 llvm_unreachable("unknown pattern kind!");
874 }
875 }
876
877 return true;
878}
879
880void CombineRuleBuilder::print(raw_ostream &OS) const {
881 OS << "(CombineRule name:" << RuleDef.getName() << " id:" << RuleID
882 << " root:" << RootName << '\n';
883
884 if (!MatchDatas.empty()) {
885 OS << " (MatchDatas\n";
886 for (const auto &MD : MatchDatas) {
887 OS << " (MatchDataDef symbol:" << MD.Symbol << " type:" << MD.Type
888 << ")\n";
889 }
890 OS << " )\n";
891 }
892
893 const auto &SeenPFs = Parser.getSeenPatFrags();
894 if (!SeenPFs.empty()) {
895 OS << " (PatFrags\n";
896 for (const auto *PF : Parser.getSeenPatFrags()) {
897 PF->print(OS, /*Indent=*/" ");
898 OS << '\n';
899 }
900 OS << " )\n";
901 }
902
903 const auto DumpPats = [&](StringRef Name, const PatternMap &Pats) {
904 OS << " (" << Name << " ";
905 if (Pats.empty()) {
906 OS << "<empty>)\n";
907 return;
908 }
909
910 OS << '\n';
911 for (const auto &[Name, Pat] : Pats) {
912 OS << " ";
913 if (Pat.get() == MatchRoot)
914 OS << "<match_root>";
915 if (isa<InstructionPattern>(Val: Pat.get()) &&
916 ApplyRoots.contains(V: cast<InstructionPattern>(Val: Pat.get())))
917 OS << "<apply_root>";
918 OS << Name << ":";
919 Pat->print(OS, /*PrintName=*/false);
920 OS << '\n';
921 }
922 OS << " )\n";
923 };
924
925 DumpPats("MatchPats", MatchPats);
926 DumpPats("ApplyPats", ApplyPats);
927
928 MatchOpTable.print(OS, Name: "MatchPats", /*Indent*/ " ");
929 ApplyOpTable.print(OS, Name: "ApplyPats", /*Indent*/ " ");
930
931 if (PermutationsToEmit.size() > 1) {
932 OS << " (PermutationsToEmit\n";
933 for (const auto &Perm : PermutationsToEmit) {
934 OS << " ";
935 print(OS, Alts: Perm);
936 OS << ",\n";
937 }
938 OS << " )\n";
939 }
940
941 OS << ")\n";
942}
943
944#ifndef NDEBUG
945void CombineRuleBuilder::verify() const {
946 const auto VerifyPats = [&](const PatternMap &Pats) {
947 for (const auto &[Name, Pat] : Pats) {
948 if (!Pat)
949 PrintFatalError("null pattern in pattern map!");
950
951 if (Name != Pat->getName()) {
952 Pat->dump();
953 PrintFatalError("Pattern name mismatch! Map name: " + Name +
954 ", Pat name: " + Pat->getName());
955 }
956
957 // Sanity check: the map should point to the same data as the Pattern.
958 // Both strings are allocated in the pool using insertStrRef.
959 if (Name.data() != Pat->getName().data()) {
960 dbgs() << "Map StringRef: '" << Name << "' @ "
961 << (const void *)Name.data() << '\n';
962 dbgs() << "Pat String: '" << Pat->getName() << "' @ "
963 << (const void *)Pat->getName().data() << '\n';
964 PrintFatalError("StringRef stored in the PatternMap is not referencing "
965 "the same string as its Pattern!");
966 }
967 }
968 };
969
970 VerifyPats(MatchPats);
971 VerifyPats(ApplyPats);
972
973 // Check there are no wip_match_opcode patterns in the "apply" patterns.
974 if (any_of(ApplyPats,
975 [&](auto &E) { return isa<AnyOpcodePattern>(E.second.get()); })) {
976 dump();
977 PrintFatalError(
978 "illegal wip_match_opcode pattern in the 'apply' patterns!");
979 }
980
981 // Check there are no nullptrs in ApplyRoots.
982 if (ApplyRoots.contains(nullptr)) {
983 PrintFatalError(
984 "CombineRuleBuilder's ApplyRoots set contains a null pointer!");
985 }
986}
987#endif
988
989std::optional<LLTCodeGenOrTempType>
990CombineRuleBuilder::getLLTCodeGenOrTempType(const PatternType &PT,
991 RuleMatcher &RM) {
992 assert(!PT.isNone());
993
994 if (PT.isLLT())
995 return getLLTCodeGen(PT);
996
997 assert(PT.isTypeOf());
998 auto &OM = RM.getOperandMatcher(Name: PT.getTypeOfOpName());
999 if (OM.isVariadic()) {
1000 PrintError(Msg: "type '" + PT.str() + "' is ill-formed: '" +
1001 OM.getSymbolicName() + "' is a variadic pack operand");
1002 return std::nullopt;
1003 }
1004 return OM.getTempTypeIdx(Rule&: RM);
1005}
1006
1007void CombineRuleBuilder::print(raw_ostream &OS,
1008 const PatternAlternatives &Alts) const {
1009 SmallVector<std::string, 1> Strings(
1010 map_range(C: Alts, F: [](const auto &PatAndPerm) {
1011 return PatAndPerm.first->getName().str() + "[" +
1012 to_string(PatAndPerm.second) + "]";
1013 }));
1014 // Sort so output is deterministic for tests. Otherwise it's sorted by pointer
1015 // values.
1016 sort(C&: Strings);
1017 OS << "[" << join(R&: Strings, Separator: ", ") << "]";
1018}
1019
1020bool CombineRuleBuilder::addApplyPattern(std::unique_ptr<Pattern> Pat) {
1021 StringRef Name = Pat->getName();
1022 if (ApplyPats.contains(Key: Name)) {
1023 PrintError(Msg: "'" + Name + "' apply pattern defined more than once!");
1024 return false;
1025 }
1026
1027 if (isa<AnyOpcodePattern>(Val: Pat.get())) {
1028 PrintError(Msg: "'" + Name +
1029 "': wip_match_opcode is not supported in apply patterns");
1030 return false;
1031 }
1032
1033 if (isa<PatFragPattern>(Val: Pat.get())) {
1034 PrintError(Msg: "'" + Name + "': using " + PatFrag::ClassName +
1035 " is not supported in apply patterns");
1036 return false;
1037 }
1038
1039 if (auto *CXXPat = dyn_cast<CXXPattern>(Val: Pat.get()))
1040 CXXPat->setIsApply();
1041
1042 ApplyPats[Name] = std::move(Pat);
1043 return true;
1044}
1045
1046bool CombineRuleBuilder::addMatchPattern(std::unique_ptr<Pattern> Pat) {
1047 StringRef Name = Pat->getName();
1048 if (MatchPats.contains(Key: Name)) {
1049 PrintError(Msg: "'" + Name + "' match pattern defined more than once!");
1050 return false;
1051 }
1052
1053 // For now, none of the builtins can appear in 'match'.
1054 if (const auto *BP = dyn_cast<BuiltinPattern>(Val: Pat.get())) {
1055 PrintError(Msg: "'" + BP->getInstName() +
1056 "' cannot be used in a 'match' pattern");
1057 return false;
1058 }
1059
1060 MatchPats[Name] = std::move(Pat);
1061 return true;
1062}
1063
1064void CombineRuleBuilder::declareAllMatchDatasExpansions(
1065 CodeExpansions &CE) const {
1066 for (const auto &MD : MatchDatas)
1067 CE.declare(Name: MD.Symbol, Expansion: MD.getVarName());
1068}
1069
1070void CombineRuleBuilder::addCXXPredicate(RuleMatcher &M,
1071 const CodeExpansions &CE,
1072 const CXXPattern &P,
1073 const PatternAlternatives &Alts) {
1074 // FIXME: Hack so C++ code is executed last. May not work for more complex
1075 // patterns.
1076 auto &IM = *std::prev(x: M.insnmatchers().end());
1077 auto Loc = RuleDef.getLoc();
1078 const auto AddComment = [&](raw_ostream &OS) {
1079 OS << "// Pattern Alternatives: ";
1080 print(OS, Alts);
1081 OS << '\n';
1082 };
1083 const auto &ExpandedCode =
1084 DebugCXXPreds ? P.expandCode(CE, Locs: Loc, AddComment) : P.expandCode(CE, Locs: Loc);
1085 IM->addPredicate<GenericInstructionPredicateMatcher>(
1086 args: ExpandedCode.getEnumNameWithPrefix(Prefix: CXXPredPrefix));
1087}
1088
1089bool CombineRuleBuilder::hasOnlyCXXApplyPatterns() const {
1090 return all_of(Range: ApplyPats, P: [&](auto &Entry) {
1091 return isa<CXXPattern>(Entry.second.get());
1092 });
1093}
1094
1095bool CombineRuleBuilder::hasEraseRoot() const {
1096 return any_of(Range: ApplyPats, P: [&](auto &Entry) {
1097 if (const auto *BP = dyn_cast<BuiltinPattern>(Entry.second.get()))
1098 return BP->getBuiltinKind() == BI_EraseRoot;
1099 return false;
1100 });
1101}
1102
1103bool CombineRuleBuilder::typecheckPatterns() {
1104 CombineRuleOperandTypeChecker OTC(RuleDef, MatchOpTable);
1105
1106 for (auto &Pat : values(C&: MatchPats)) {
1107 if (auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get())) {
1108 if (!OTC.processMatchPattern(P&: *IP))
1109 return false;
1110 }
1111 }
1112
1113 for (auto &Pat : values(C&: ApplyPats)) {
1114 if (auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get())) {
1115 if (!OTC.processApplyPattern(P&: *IP))
1116 return false;
1117 }
1118 }
1119
1120 OTC.propagateAndInferTypes();
1121
1122 // Always check this after in case inference adds some special types to the
1123 // match patterns.
1124 for (auto &Pat : values(C&: MatchPats)) {
1125 if (auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get())) {
1126 bool HasDiag = false;
1127 for (const auto &[Idx, Op] : enumerate(First&: IP->operands())) {
1128 if (Op.getType().isTypeOf()) {
1129 PrintError(Msg: PatternType::TypeOfClassName +
1130 " is not supported in 'match' patterns");
1131 PrintNote(Msg: "operand " + Twine(Idx) + " of '" + IP->getName() +
1132 "' has type '" + Op.getType().str() + "'");
1133 HasDiag = true;
1134 }
1135 }
1136 if (HasDiag)
1137 return false;
1138 }
1139 }
1140 return true;
1141}
1142
1143bool CombineRuleBuilder::buildPermutationsToEmit() {
1144 PermutationsToEmit.clear();
1145
1146 // Start with one empty set of alternatives.
1147 PermutationsToEmit.emplace_back();
1148 for (const auto &Pat : values(C&: MatchPats)) {
1149 unsigned NumAlts = 0;
1150 // Note: technically, AnyOpcodePattern also needs permutations, but:
1151 // - We only allow a single one of them in the root.
1152 // - They cannot be mixed with any other pattern other than C++ code.
1153 // So we don't really need to take them into account here. We could, but
1154 // that pattern is a hack anyway and the less it's involved, the better.
1155 if (const auto *PFP = dyn_cast<PatFragPattern>(Val: Pat.get()))
1156 NumAlts = PFP->getPatFrag().num_alternatives();
1157 else
1158 continue;
1159
1160 // For each pattern that needs permutations, multiply the current set of
1161 // alternatives.
1162 auto CurPerms = PermutationsToEmit;
1163 PermutationsToEmit.clear();
1164
1165 for (const auto &Perm : CurPerms) {
1166 assert(!Perm.contains(Pat.get()) && "Pattern already emitted?");
1167 for (unsigned K = 0; K < NumAlts; ++K) {
1168 PatternAlternatives NewPerm = Perm;
1169 NewPerm[Pat.get()] = K;
1170 PermutationsToEmit.emplace_back(Args: std::move(NewPerm));
1171 }
1172 }
1173 }
1174
1175 if (int64_t MaxPerms = RuleDef.getValueAsInt(FieldName: "MaxPermutations");
1176 MaxPerms > 0) {
1177 if ((int64_t)PermutationsToEmit.size() > MaxPerms) {
1178 PrintError(Msg: "cannot emit rule '" + RuleDef.getName() + "'; " +
1179 Twine(PermutationsToEmit.size()) +
1180 " permutations would be emitted, but the max is " +
1181 Twine(MaxPerms));
1182 return false;
1183 }
1184 }
1185
1186 // Ensure we always have a single empty entry, it simplifies the emission
1187 // logic so it doesn't need to handle the case where there are no perms.
1188 if (PermutationsToEmit.empty()) {
1189 PermutationsToEmit.emplace_back();
1190 return true;
1191 }
1192
1193 return true;
1194}
1195
1196bool CombineRuleBuilder::checkSemantics() {
1197 assert(MatchRoot && "Cannot call this before findRoots()");
1198
1199 const auto CheckVariadicOperands = [&](const InstructionPattern &IP,
1200 bool IsMatch) {
1201 bool HasVariadic = false;
1202 for (auto &Op : IP.operands()) {
1203 if (!Op.getType().isVariadicPack())
1204 continue;
1205
1206 HasVariadic = true;
1207
1208 if (IsMatch && &Op != &IP.operands_back()) {
1209 PrintError(Msg: "'" + IP.getInstName() +
1210 "': " + PatternType::VariadicClassName +
1211 " can only be used on the last operand");
1212 return false;
1213 }
1214
1215 if (Op.isDef()) {
1216 PrintError(Msg: "'" + IP.getInstName() + "': " +
1217 PatternType::VariadicClassName + " cannot be used on defs");
1218 return false;
1219 }
1220 }
1221
1222 if (HasVariadic && !IP.isVariadic()) {
1223 PrintError(Msg: "cannot use a " + PatternType::VariadicClassName +
1224 " operand on non-variadic instruction '" + IP.getInstName() +
1225 "'");
1226 return false;
1227 }
1228
1229 return true;
1230 };
1231
1232 bool UsesWipMatchOpcode = false;
1233 for (const auto &Match : MatchPats) {
1234 const auto *Pat = Match.second.get();
1235
1236 if (const auto *CXXPat = dyn_cast<CXXPattern>(Val: Pat)) {
1237 if (!CXXPat->getRawCode().contains(Other: "return "))
1238 PrintWarning(Msg: "'match' C++ code does not seem to return!");
1239 continue;
1240 }
1241
1242 if (const auto IP = dyn_cast<InstructionPattern>(Val: Pat)) {
1243 if (!CheckVariadicOperands(*IP, /*IsMatch=*/true))
1244 return false;
1245
1246 // MIFlags in match cannot use the following syntax: (MIFlags $mi)
1247 if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: Pat)) {
1248 if (auto *FI = CGP->getMIFlagsInfo()) {
1249 if (!FI->copy_flags().empty()) {
1250 PrintError(Msg: "'match' patterns cannot refer to flags from other "
1251 "instructions");
1252 PrintNote(Msg: "MIFlags in '" + CGP->getName() +
1253 "' refer to: " + join(R: FI->copy_flags(), Separator: ", "));
1254 return false;
1255 }
1256 }
1257 }
1258 continue;
1259 }
1260
1261 const auto *AOP = dyn_cast<AnyOpcodePattern>(Val: Pat);
1262 if (!AOP)
1263 continue;
1264
1265 if (UsesWipMatchOpcode) {
1266 PrintError(Msg: "wip_opcode_match can only be present once");
1267 return false;
1268 }
1269
1270 UsesWipMatchOpcode = true;
1271 }
1272
1273 std::optional<bool> IsUsingCXXPatterns;
1274 for (const auto &Apply : ApplyPats) {
1275 Pattern *Pat = Apply.second.get();
1276 if (IsUsingCXXPatterns) {
1277 if (*IsUsingCXXPatterns != isa<CXXPattern>(Val: Pat)) {
1278 PrintError(Msg: "'apply' patterns cannot mix C++ code with other types of "
1279 "patterns");
1280 return false;
1281 }
1282 } else {
1283 IsUsingCXXPatterns = isa<CXXPattern>(Val: Pat);
1284 }
1285
1286 assert(Pat);
1287 const auto *IP = dyn_cast<InstructionPattern>(Val: Pat);
1288 if (!IP)
1289 continue;
1290
1291 if (!CheckVariadicOperands(*IP, /*IsMatch=*/false))
1292 return false;
1293
1294 if (UsesWipMatchOpcode) {
1295 PrintError(Msg: "cannot use wip_match_opcode in combination with apply "
1296 "instruction patterns!");
1297 return false;
1298 }
1299
1300 // Check that the insts mentioned in copy_flags exist.
1301 if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: IP)) {
1302 if (auto *FI = CGP->getMIFlagsInfo()) {
1303 for (auto InstName : FI->copy_flags()) {
1304 auto It = MatchPats.find(Key: InstName);
1305 if (It == MatchPats.end()) {
1306 PrintError(Msg: "unknown instruction '$" + InstName +
1307 "' referenced in MIFlags of '" + CGP->getName() + "'");
1308 return false;
1309 }
1310
1311 if (!isa<CodeGenInstructionPattern>(Val: It->second.get())) {
1312 PrintError(
1313 Msg: "'$" + InstName +
1314 "' does not refer to a CodeGenInstruction in MIFlags of '" +
1315 CGP->getName() + "'");
1316 return false;
1317 }
1318 }
1319 }
1320 }
1321
1322 const auto *BIP = dyn_cast<BuiltinPattern>(Val: IP);
1323 if (!BIP)
1324 continue;
1325 StringRef Name = BIP->getInstName();
1326
1327 // (GIEraseInst) has to be the only apply pattern, or it can not be used at
1328 // all. The root cannot have any defs either.
1329 switch (BIP->getBuiltinKind()) {
1330 case BI_EraseRoot: {
1331 if (ApplyPats.size() > 1) {
1332 PrintError(Msg: Name + " must be the only 'apply' pattern");
1333 return false;
1334 }
1335
1336 const auto *IRoot = dyn_cast<CodeGenInstructionPattern>(Val: MatchRoot);
1337 if (!IRoot) {
1338 PrintError(Msg: Name + " can only be used if the root is a "
1339 "CodeGenInstruction or Intrinsic");
1340 return false;
1341 }
1342
1343 if (IRoot->getNumInstDefs() != 0) {
1344 PrintError(Msg: Name + " can only be used if on roots that do "
1345 "not have any output operand");
1346 PrintNote(Msg: "'" + IRoot->getInstName() + "' has " +
1347 Twine(IRoot->getNumInstDefs()) + " output operands");
1348 return false;
1349 }
1350 break;
1351 }
1352 case BI_ReplaceReg: {
1353 // (GIReplaceReg can only be used on the root instruction)
1354 // TODO: When we allow rewriting non-root instructions, also allow this.
1355 StringRef OldRegName = BIP->getOperand(K: 0).getOperandName();
1356 auto *Def = MatchOpTable.getDef(OpName: OldRegName);
1357 if (!Def) {
1358 PrintError(Msg: Name + " cannot find a matched pattern that defines '" +
1359 OldRegName + "'");
1360 return false;
1361 }
1362 if (MatchOpTable.getDef(OpName: OldRegName) != MatchRoot) {
1363 PrintError(Msg: Name + " cannot replace '" + OldRegName +
1364 "': this builtin can only replace a register defined by the "
1365 "match root");
1366 return false;
1367 }
1368 break;
1369 }
1370 }
1371 }
1372
1373 // TODO: Diagnose uses of MatchDatas if the Rule doesn't have C++ on both the
1374 // match and apply. It's useless in such cases.
1375 if (!hasOnlyCXXApplyPatterns() && !MatchDatas.empty()) {
1376 PrintError(Msg: MatchDataClassName +
1377 " can only be used if 'apply' in entirely written in C++");
1378 return false;
1379 }
1380
1381 return true;
1382}
1383
1384RuleMatcher &CombineRuleBuilder::addRuleMatcher(const PatternAlternatives &Alts,
1385 Twine AdditionalComment) {
1386 auto &RM = OutRMs.emplace_back(args: RuleDef.getLoc());
1387 addFeaturePredicates(M&: RM);
1388 RM.setPermanentGISelFlags(GISF_IgnoreCopies);
1389 RM.addRequiredSimplePredicate(PredName: getIsEnabledPredicateEnumName(CombinerRuleID: RuleID));
1390
1391 std::string Comment;
1392 raw_string_ostream CommentOS(Comment);
1393 CommentOS << "Combiner Rule #" << RuleID << ": " << RuleDef.getName();
1394 if (!Alts.empty()) {
1395 CommentOS << " @ ";
1396 print(OS&: CommentOS, Alts);
1397 }
1398 if (!AdditionalComment.isTriviallyEmpty())
1399 CommentOS << "; " << AdditionalComment;
1400 RM.addAction<DebugCommentAction>(args&: Comment);
1401 return RM;
1402}
1403
1404bool CombineRuleBuilder::addFeaturePredicates(RuleMatcher &M) {
1405 if (!RuleDef.getValue(Name: "Predicates"))
1406 return true;
1407
1408 const ListInit *Preds = RuleDef.getValueAsListInit(FieldName: "Predicates");
1409 for (const Init *PI : Preds->getElements()) {
1410 const DefInit *Pred = dyn_cast<DefInit>(Val: PI);
1411 if (!Pred)
1412 continue;
1413
1414 const Record *Def = Pred->getDef();
1415 if (!Def->isSubClassOf(Name: "Predicate")) {
1416 ::PrintError(Rec: Def, Msg: "Unknown 'Predicate' Type");
1417 return false;
1418 }
1419
1420 if (Def->getValueAsString(FieldName: "CondString").empty())
1421 continue;
1422
1423 if (SubtargetFeatures.count(x: Def) == 0) {
1424 SubtargetFeatures.emplace(
1425 args&: Def, args: SubtargetFeatureInfo(Def, SubtargetFeatures.size()));
1426 }
1427
1428 M.addRequiredFeature(Feature: Def);
1429 }
1430
1431 return true;
1432}
1433
1434bool CombineRuleBuilder::findRoots() {
1435 const auto Finish = [&]() {
1436 assert(MatchRoot);
1437
1438 if (hasOnlyCXXApplyPatterns() || hasEraseRoot())
1439 return true;
1440
1441 auto *IPRoot = dyn_cast<InstructionPattern>(Val: MatchRoot);
1442 if (!IPRoot)
1443 return true;
1444
1445 if (IPRoot->getNumInstDefs() == 0) {
1446 // No defs to work with -> find the root using the pattern name.
1447 auto It = ApplyPats.find(Key: RootName);
1448 if (It == ApplyPats.end()) {
1449 PrintError(Msg: "Cannot find root '" + RootName + "' in apply patterns!");
1450 return false;
1451 }
1452
1453 auto *ApplyRoot = dyn_cast<InstructionPattern>(Val: It->second.get());
1454 if (!ApplyRoot) {
1455 PrintError(Msg: "apply pattern root '" + RootName +
1456 "' must be an instruction pattern");
1457 return false;
1458 }
1459
1460 ApplyRoots.insert(V: ApplyRoot);
1461 return true;
1462 }
1463
1464 // Collect all redefinitions of the MatchRoot's defs and put them in
1465 // ApplyRoots.
1466 const auto DefsNeeded = IPRoot->getApplyDefsNeeded();
1467 for (auto &Op : DefsNeeded) {
1468 assert(Op.isDef() && Op.isNamedOperand());
1469 StringRef Name = Op.getOperandName();
1470
1471 auto *ApplyRedef = ApplyOpTable.getDef(OpName: Name);
1472 if (!ApplyRedef) {
1473 PrintError(Msg: "'" + Name + "' must be redefined in the 'apply' pattern");
1474 return false;
1475 }
1476
1477 ApplyRoots.insert(V: (InstructionPattern *)ApplyRedef);
1478 }
1479
1480 if (auto It = ApplyPats.find(Key: RootName); It != ApplyPats.end()) {
1481 if (find(Range&: ApplyRoots, Val: It->second.get()) == ApplyRoots.end()) {
1482 PrintError(Msg: "apply pattern '" + RootName +
1483 "' is supposed to be a root but it does not redefine any of "
1484 "the defs of the match root");
1485 return false;
1486 }
1487 }
1488
1489 return true;
1490 };
1491
1492 // Look by pattern name, e.g.
1493 // (G_FNEG $x, $y):$root
1494 if (auto MatchPatIt = MatchPats.find(Key: RootName);
1495 MatchPatIt != MatchPats.end()) {
1496 MatchRoot = MatchPatIt->second.get();
1497 return Finish();
1498 }
1499
1500 // Look by def:
1501 // (G_FNEG $root, $y)
1502 auto LookupRes = MatchOpTable.lookup(OpName: RootName);
1503 if (!LookupRes.Found) {
1504 PrintError(Msg: "Cannot find root '" + RootName + "' in match patterns!");
1505 return false;
1506 }
1507
1508 MatchRoot = LookupRes.Def;
1509 if (!MatchRoot) {
1510 PrintError(Msg: "Cannot use live-in operand '" + RootName +
1511 "' as match pattern root!");
1512 return false;
1513 }
1514
1515 return Finish();
1516}
1517
1518bool CombineRuleBuilder::buildRuleOperandsTable() {
1519 const auto DiagnoseRedefMatch = [&](StringRef OpName) {
1520 PrintError(Msg: "Operand '" + OpName +
1521 "' is defined multiple times in the 'match' patterns");
1522 };
1523
1524 const auto DiagnoseRedefApply = [&](StringRef OpName) {
1525 PrintError(Msg: "Operand '" + OpName +
1526 "' is defined multiple times in the 'apply' patterns");
1527 };
1528
1529 for (auto &Pat : values(C&: MatchPats)) {
1530 auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get());
1531 if (IP && !MatchOpTable.addPattern(P: IP, DiagnoseRedef: DiagnoseRedefMatch))
1532 return false;
1533 }
1534
1535 for (auto &Pat : values(C&: ApplyPats)) {
1536 auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get());
1537 if (IP && !ApplyOpTable.addPattern(P: IP, DiagnoseRedef: DiagnoseRedefApply))
1538 return false;
1539 }
1540
1541 return true;
1542}
1543
1544bool CombineRuleBuilder::parseDefs(const DagInit &Def) {
1545 if (Def.getOperatorAsDef(Loc: RuleDef.getLoc())->getName() != "defs") {
1546 PrintError(Msg: "Expected defs operator");
1547 return false;
1548 }
1549
1550 SmallVector<StringRef> Roots;
1551 for (unsigned I = 0, E = Def.getNumArgs(); I < E; ++I) {
1552 if (isSpecificDef(N: *Def.getArg(Num: I), Def: "root")) {
1553 Roots.emplace_back(Args: Def.getArgNameStr(Num: I));
1554 continue;
1555 }
1556
1557 // Subclasses of GIDefMatchData should declare that this rule needs to pass
1558 // data from the match stage to the apply stage, and ensure that the
1559 // generated matcher has a suitable variable for it to do so.
1560 if (const Record *MatchDataRec =
1561 getDefOfSubClass(N: *Def.getArg(Num: I), Cls: MatchDataClassName)) {
1562 MatchDatas.emplace_back(Args: Def.getArgNameStr(Num: I),
1563 Args: MatchDataRec->getValueAsString(FieldName: "Type"));
1564 continue;
1565 }
1566
1567 // Otherwise emit an appropriate error message.
1568 if (getDefOfSubClass(N: *Def.getArg(Num: I), Cls: "GIDefKind"))
1569 PrintError(Msg: "This GIDefKind not implemented in tablegen");
1570 else if (getDefOfSubClass(N: *Def.getArg(Num: I), Cls: "GIDefKindWithArgs"))
1571 PrintError(Msg: "This GIDefKindWithArgs not implemented in tablegen");
1572 else
1573 PrintError(Msg: "Expected a subclass of GIDefKind or a sub-dag whose "
1574 "operator is of type GIDefKindWithArgs");
1575 return false;
1576 }
1577
1578 if (Roots.size() != 1) {
1579 PrintError(Msg: "Combine rules must have exactly one root");
1580 return false;
1581 }
1582
1583 RootName = Roots.front();
1584 return true;
1585}
1586
1587bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
1588 const PatternAlternatives &Alts,
1589 const InstructionPattern &IP) {
1590 auto StackTrace = PrettyStackTraceEmit(RuleDef, &IP);
1591
1592 auto &M = addRuleMatcher(Alts);
1593 InstructionMatcher &IM = M.addInstructionMatcher(SymbolicName: IP.getName());
1594 declareInstExpansion(CE, IM, Name: IP.getName());
1595
1596 DenseSet<const Pattern *> SeenPats;
1597
1598 const auto FindOperandDef = [&](StringRef Op) -> InstructionPattern * {
1599 return MatchOpTable.getDef(OpName: Op);
1600 };
1601
1602 if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: &IP)) {
1603 if (!emitCodeGenInstructionMatchPattern(CE, Alts, M, IM, P: *CGP, SeenPats,
1604 LookupOperandDef: FindOperandDef))
1605 return false;
1606 } else if (const auto *PFP = dyn_cast<PatFragPattern>(Val: &IP)) {
1607 if (!PFP->getPatFrag().canBeMatchRoot()) {
1608 PrintError(Msg: "cannot use '" + PFP->getInstName() + " as match root");
1609 return false;
1610 }
1611
1612 if (!emitPatFragMatchPattern(CE, Alts, RM&: M, IM: &IM, PFP: *PFP, SeenPats))
1613 return false;
1614 } else if (isa<BuiltinPattern>(Val: &IP)) {
1615 llvm_unreachable("No match builtins known!");
1616 } else {
1617 llvm_unreachable("Unknown kind of InstructionPattern!");
1618 }
1619
1620 // Emit remaining patterns
1621 const bool IsUsingCustomCXXAction = hasOnlyCXXApplyPatterns();
1622 SmallVector<CXXPattern *, 2> CXXMatchers;
1623 for (auto &Pat : values(C&: MatchPats)) {
1624 if (SeenPats.contains(V: Pat.get()))
1625 continue;
1626
1627 switch (Pat->getKind()) {
1628 case Pattern::K_AnyOpcode:
1629 PrintError(Msg: "wip_match_opcode can not be used with instruction patterns!");
1630 return false;
1631 case Pattern::K_PatFrag: {
1632 if (!emitPatFragMatchPattern(CE, Alts, RM&: M, /*IM*/ nullptr,
1633 PFP: *cast<PatFragPattern>(Val: Pat.get()), SeenPats))
1634 return false;
1635 continue;
1636 }
1637 case Pattern::K_Builtin:
1638 PrintError(Msg: "No known match builtins");
1639 return false;
1640 case Pattern::K_CodeGenInstruction:
1641 cast<InstructionPattern>(Val: Pat.get())->reportUnreachable(Locs: RuleDef.getLoc());
1642 return false;
1643 case Pattern::K_CXX: {
1644 // Delay emission for top-level C++ matchers (which can use MatchDatas).
1645 if (IsUsingCustomCXXAction)
1646 CXXMatchers.push_back(Elt: cast<CXXPattern>(Val: Pat.get()));
1647 else
1648 addCXXPredicate(M, CE, P: *cast<CXXPattern>(Val: Pat.get()), Alts);
1649 continue;
1650 }
1651 default:
1652 llvm_unreachable("unknown pattern kind!");
1653 }
1654 }
1655
1656 return IsUsingCustomCXXAction ? emitCXXMatchApply(CE, M, Matchers: CXXMatchers)
1657 : emitApplyPatterns(CE, M);
1658}
1659
1660bool CombineRuleBuilder::emitMatchPattern(CodeExpansions &CE,
1661 const PatternAlternatives &Alts,
1662 const AnyOpcodePattern &AOP) {
1663 auto StackTrace = PrettyStackTraceEmit(RuleDef, &AOP);
1664
1665 const bool IsUsingCustomCXXAction = hasOnlyCXXApplyPatterns();
1666 for (const CodeGenInstruction *CGI : AOP.insts()) {
1667 auto &M = addRuleMatcher(Alts, AdditionalComment: "wip_match_opcode '" + CGI->getName() + "'");
1668
1669 InstructionMatcher &IM = M.addInstructionMatcher(SymbolicName: AOP.getName());
1670 declareInstExpansion(CE, IM, Name: AOP.getName());
1671 // declareInstExpansion needs to be identical, otherwise we need to create a
1672 // CodeExpansions object here instead.
1673 assert(IM.getInsnVarID() == 0);
1674
1675 IM.addPredicate<InstructionOpcodeMatcher>(args&: CGI);
1676
1677 // Emit remaining patterns.
1678 SmallVector<CXXPattern *, 2> CXXMatchers;
1679 for (auto &Pat : values(C&: MatchPats)) {
1680 if (Pat.get() == &AOP)
1681 continue;
1682
1683 switch (Pat->getKind()) {
1684 case Pattern::K_AnyOpcode:
1685 PrintError(Msg: "wip_match_opcode can only be present once!");
1686 return false;
1687 case Pattern::K_PatFrag: {
1688 DenseSet<const Pattern *> SeenPats;
1689 if (!emitPatFragMatchPattern(CE, Alts, RM&: M, /*IM*/ nullptr,
1690 PFP: *cast<PatFragPattern>(Val: Pat.get()),
1691 SeenPats))
1692 return false;
1693 continue;
1694 }
1695 case Pattern::K_Builtin:
1696 PrintError(Msg: "No known match builtins");
1697 return false;
1698 case Pattern::K_CodeGenInstruction:
1699 cast<InstructionPattern>(Val: Pat.get())->reportUnreachable(
1700 Locs: RuleDef.getLoc());
1701 return false;
1702 case Pattern::K_CXX: {
1703 // Delay emission for top-level C++ matchers (which can use MatchDatas).
1704 if (IsUsingCustomCXXAction)
1705 CXXMatchers.push_back(Elt: cast<CXXPattern>(Val: Pat.get()));
1706 else
1707 addCXXPredicate(M, CE, P: *cast<CXXPattern>(Val: Pat.get()), Alts);
1708 break;
1709 }
1710 default:
1711 llvm_unreachable("unknown pattern kind!");
1712 }
1713 }
1714
1715 const bool Res = IsUsingCustomCXXAction
1716 ? emitCXXMatchApply(CE, M, Matchers: CXXMatchers)
1717 : emitApplyPatterns(CE, M);
1718 if (!Res)
1719 return false;
1720 }
1721
1722 return true;
1723}
1724
1725bool CombineRuleBuilder::emitPatFragMatchPattern(
1726 CodeExpansions &CE, const PatternAlternatives &Alts, RuleMatcher &RM,
1727 InstructionMatcher *IM, const PatFragPattern &PFP,
1728 DenseSet<const Pattern *> &SeenPats) {
1729 auto StackTrace = PrettyStackTraceEmit(RuleDef, &PFP);
1730
1731 if (!SeenPats.insert(V: &PFP).second)
1732 return true;
1733
1734 const auto &PF = PFP.getPatFrag();
1735
1736 if (!IM) {
1737 // When we don't have an IM, this means this PatFrag isn't reachable from
1738 // the root. This is only acceptable if it doesn't define anything (e.g. a
1739 // pure C++ PatFrag).
1740 if (PF.num_out_params() != 0) {
1741 PFP.reportUnreachable(Locs: RuleDef.getLoc());
1742 return false;
1743 }
1744 } else {
1745 // When an IM is provided, this is reachable from the root, and we're
1746 // expecting to have output operands.
1747 // TODO: If we want to allow for multiple roots we'll need a map of IMs
1748 // then, and emission becomes a bit more complicated.
1749 assert(PF.num_roots() == 1);
1750 }
1751
1752 CodeExpansions PatFragCEs;
1753 if (!PFP.mapInputCodeExpansions(ParentCEs: CE, PatFragCEs, DiagLoc: RuleDef.getLoc()))
1754 return false;
1755
1756 // List of {ParamName, ArgName}.
1757 // When all patterns have been emitted, find expansions in PatFragCEs named
1758 // ArgName and add their expansion to CE using ParamName as the key.
1759 SmallVector<std::pair<std::string, std::string>, 4> CEsToImport;
1760
1761 // Map parameter names to the actual argument.
1762 const auto OperandMapper =
1763 [&](const InstructionOperand &O) -> InstructionOperand {
1764 if (!O.isNamedOperand())
1765 return O;
1766
1767 StringRef ParamName = O.getOperandName();
1768
1769 // Not sure what to do with those tbh. They should probably never be here.
1770 assert(!O.isNamedImmediate() && "TODO: handle named imms");
1771 unsigned PIdx = PF.getParamIdx(Name: ParamName);
1772
1773 // Map parameters to the argument values.
1774 if (PIdx == (unsigned)-1) {
1775 // This is a temp of the PatFragPattern, prefix the name to avoid
1776 // conflicts.
1777 return O.withNewName(
1778 NewName: insertStrRef(S: (PFP.getName() + "." + ParamName).str()));
1779 }
1780
1781 // The operand will be added to PatFragCEs's code expansions using the
1782 // parameter's name. If it's bound to some operand during emission of the
1783 // patterns, we'll want to add it to CE.
1784 auto ArgOp = PFP.getOperand(K: PIdx);
1785 if (ArgOp.isNamedOperand())
1786 CEsToImport.emplace_back(Args: ArgOp.getOperandName().str(), Args&: ParamName);
1787
1788 if (ArgOp.getType() && O.getType() && ArgOp.getType() != O.getType()) {
1789 StringRef PFName = PF.getName();
1790 PrintWarning(Msg: "impossible type constraints: operand " + Twine(PIdx) +
1791 " of '" + PFP.getName() + "' has type '" +
1792 ArgOp.getType().str() + "', but '" + PFName +
1793 "' constrains it to '" + O.getType().str() + "'");
1794 if (ArgOp.isNamedOperand())
1795 PrintNote(Msg: "operand " + Twine(PIdx) + " of '" + PFP.getName() +
1796 "' is '" + ArgOp.getOperandName() + "'");
1797 if (O.isNamedOperand())
1798 PrintNote(Msg: "argument " + Twine(PIdx) + " of '" + PFName + "' is '" +
1799 ParamName + "'");
1800 }
1801
1802 return ArgOp;
1803 };
1804
1805 // PatFragPatterns are only made of InstructionPatterns or CXXPatterns.
1806 // Emit instructions from the root.
1807 const auto &FragAlt = PF.getAlternative(K: Alts.lookup(Val: &PFP));
1808 const auto &FragAltOT = FragAlt.OpTable;
1809 const auto LookupOperandDef =
1810 [&](StringRef Op) -> const InstructionPattern * {
1811 return FragAltOT.getDef(OpName: Op);
1812 };
1813
1814 DenseSet<const Pattern *> PatFragSeenPats;
1815 for (const auto &[Idx, InOp] : enumerate(First: PF.out_params())) {
1816 if (InOp.Kind != PatFrag::PK_Root)
1817 continue;
1818
1819 StringRef ParamName = InOp.Name;
1820 const auto *Def = FragAltOT.getDef(OpName: ParamName);
1821 assert(Def && "PatFrag::checkSemantics should have emitted an error if "
1822 "an out operand isn't defined!");
1823 assert(isa<CodeGenInstructionPattern>(Def) &&
1824 "Nested PatFrags not supported yet");
1825
1826 if (!emitCodeGenInstructionMatchPattern(
1827 CE&: PatFragCEs, Alts, M&: RM, IM&: *IM, P: *cast<CodeGenInstructionPattern>(Val: Def),
1828 SeenPats&: PatFragSeenPats, LookupOperandDef, OperandMapper))
1829 return false;
1830 }
1831
1832 // Emit leftovers.
1833 for (const auto &Pat : FragAlt.Pats) {
1834 if (PatFragSeenPats.contains(V: Pat.get()))
1835 continue;
1836
1837 if (const auto *CXXPat = dyn_cast<CXXPattern>(Val: Pat.get())) {
1838 addCXXPredicate(M&: RM, CE: PatFragCEs, P: *CXXPat, Alts);
1839 continue;
1840 }
1841
1842 if (const auto *IP = dyn_cast<InstructionPattern>(Val: Pat.get())) {
1843 IP->reportUnreachable(Locs: PF.getLoc());
1844 return false;
1845 }
1846
1847 llvm_unreachable("Unexpected pattern kind in PatFrag");
1848 }
1849
1850 for (const auto &[ParamName, ArgName] : CEsToImport) {
1851 // Note: we're find if ParamName already exists. It just means it's been
1852 // bound before, so we prefer to keep the first binding.
1853 CE.declare(Name: ParamName, Expansion: PatFragCEs.lookup(Variable: ArgName));
1854 }
1855
1856 return true;
1857}
1858
1859bool CombineRuleBuilder::emitApplyPatterns(CodeExpansions &CE, RuleMatcher &M) {
1860 assert(MatchDatas.empty());
1861
1862 DenseSet<const Pattern *> SeenPats;
1863 StringMap<unsigned> OperandToTempRegID;
1864
1865 for (auto *ApplyRoot : ApplyRoots) {
1866 assert(isa<InstructionPattern>(ApplyRoot) &&
1867 "Root can only be a InstructionPattern!");
1868 if (!emitInstructionApplyPattern(CE, M,
1869 P: cast<InstructionPattern>(Val&: *ApplyRoot),
1870 SeenPats, OperandToTempRegID))
1871 return false;
1872 }
1873
1874 for (auto &Pat : values(C&: ApplyPats)) {
1875 if (SeenPats.contains(V: Pat.get()))
1876 continue;
1877
1878 switch (Pat->getKind()) {
1879 case Pattern::K_AnyOpcode:
1880 llvm_unreachable("Unexpected pattern in apply!");
1881 case Pattern::K_PatFrag:
1882 // TODO: We could support pure C++ PatFrags as a temporary thing.
1883 llvm_unreachable("Unexpected pattern in apply!");
1884 case Pattern::K_Builtin:
1885 if (!emitInstructionApplyPattern(CE, M, P: cast<BuiltinPattern>(Val&: *Pat),
1886 SeenPats, OperandToTempRegID))
1887 return false;
1888 break;
1889 case Pattern::K_CodeGenInstruction:
1890 cast<CodeGenInstructionPattern>(Val&: *Pat).reportUnreachable(Locs: RuleDef.getLoc());
1891 return false;
1892 case Pattern::K_CXX: {
1893 llvm_unreachable(
1894 "CXX Pattern Emission should have been handled earlier!");
1895 }
1896 default:
1897 llvm_unreachable("unknown pattern kind!");
1898 }
1899 }
1900
1901 // Erase the root.
1902 unsigned RootInsnID =
1903 M.getInsnVarID(InsnMatcher&: M.getInstructionMatcher(SymbolicName: MatchRoot->getName()));
1904 M.addAction<EraseInstAction>(args&: RootInsnID);
1905
1906 return true;
1907}
1908
1909bool CombineRuleBuilder::emitCXXMatchApply(CodeExpansions &CE, RuleMatcher &M,
1910 ArrayRef<CXXPattern *> Matchers) {
1911 assert(hasOnlyCXXApplyPatterns());
1912 declareAllMatchDatasExpansions(CE);
1913
1914 std::string CodeStr;
1915 raw_string_ostream OS(CodeStr);
1916
1917 for (auto &MD : MatchDatas)
1918 OS << MD.Type << " " << MD.getVarName() << ";\n";
1919
1920 if (!Matchers.empty()) {
1921 OS << "// Match Patterns\n";
1922 for (auto *M : Matchers) {
1923 OS << "if(![&](){";
1924 CodeExpander Expander(M->getRawCode(), CE, RuleDef.getLoc(),
1925 /*ShowExpansions=*/false);
1926 Expander.emit(OS);
1927 OS << "}()) {\n"
1928 << " return false;\n}\n";
1929 }
1930 }
1931
1932 OS << "// Apply Patterns\n";
1933 ListSeparator LS("\n");
1934 for (auto &Pat : ApplyPats) {
1935 auto *CXXPat = cast<CXXPattern>(Val: Pat.second.get());
1936 CodeExpander Expander(CXXPat->getRawCode(), CE, RuleDef.getLoc(),
1937 /*ShowExpansions=*/false);
1938 OS << LS;
1939 Expander.emit(OS);
1940 }
1941
1942 const auto &Code = CXXPredicateCode::getCustomActionCode(Code: CodeStr);
1943 M.setCustomCXXAction(Code.getEnumNameWithPrefix(Prefix: CXXCustomActionPrefix));
1944 return true;
1945}
1946
1947bool CombineRuleBuilder::emitInstructionApplyPattern(
1948 CodeExpansions &CE, RuleMatcher &M, const InstructionPattern &P,
1949 DenseSet<const Pattern *> &SeenPats,
1950 StringMap<unsigned> &OperandToTempRegID) {
1951 auto StackTrace = PrettyStackTraceEmit(RuleDef, &P);
1952
1953 if (!SeenPats.insert(V: &P).second)
1954 return true;
1955
1956 // First, render the uses.
1957 for (auto &Op : P.named_operands()) {
1958 if (Op.isDef())
1959 continue;
1960
1961 StringRef OpName = Op.getOperandName();
1962 if (const auto *DefPat = ApplyOpTable.getDef(OpName)) {
1963 if (!emitInstructionApplyPattern(CE, M, P: *DefPat, SeenPats,
1964 OperandToTempRegID))
1965 return false;
1966 } else {
1967 // If we have no def, check this exists in the MatchRoot.
1968 if (!Op.isNamedImmediate() && !MatchOpTable.lookup(OpName).Found) {
1969 PrintError(Msg: "invalid output operand '" + OpName +
1970 "': operand is not a live-in of the match pattern, and it "
1971 "has no definition");
1972 return false;
1973 }
1974 }
1975 }
1976
1977 if (const auto *BP = dyn_cast<BuiltinPattern>(Val: &P))
1978 return emitBuiltinApplyPattern(CE, M, P: *BP, OperandToTempRegID);
1979
1980 if (isa<PatFragPattern>(Val: &P))
1981 llvm_unreachable("PatFragPatterns is not supported in 'apply'!");
1982
1983 auto &CGIP = cast<CodeGenInstructionPattern>(Val: P);
1984
1985 // Now render this inst.
1986 auto &DstMI =
1987 M.addAction<BuildMIAction>(args: M.allocateOutputInsnID(), args: &CGIP.getInst());
1988
1989 bool HasEmittedIntrinsicID = false;
1990 const auto EmitIntrinsicID = [&]() {
1991 assert(CGIP.isIntrinsic());
1992 DstMI.addRenderer<IntrinsicIDRenderer>(args: CGIP.getIntrinsic());
1993 HasEmittedIntrinsicID = true;
1994 };
1995
1996 for (auto &Op : P.operands()) {
1997 // Emit the intrinsic ID after the last def.
1998 if (CGIP.isIntrinsic() && !Op.isDef() && !HasEmittedIntrinsicID)
1999 EmitIntrinsicID();
2000
2001 if (Op.isNamedImmediate()) {
2002 PrintError(Msg: "invalid output operand '" + Op.getOperandName() +
2003 "': output immediates cannot be named");
2004 PrintNote(Msg: "while emitting pattern '" + P.getName() + "' (" +
2005 P.getInstName() + ")");
2006 return false;
2007 }
2008
2009 if (Op.hasImmValue()) {
2010 if (!emitCodeGenInstructionApplyImmOperand(M, DstMI, P: CGIP, O: Op))
2011 return false;
2012 continue;
2013 }
2014
2015 StringRef OpName = Op.getOperandName();
2016
2017 // Uses of operand.
2018 if (!Op.isDef()) {
2019 if (auto It = OperandToTempRegID.find(Key: OpName);
2020 It != OperandToTempRegID.end()) {
2021 assert(!MatchOpTable.lookup(OpName).Found &&
2022 "Temp reg is also from match pattern?");
2023 DstMI.addRenderer<TempRegRenderer>(args&: It->second);
2024 } else {
2025 // This should be a match live in or a redef of a matched instr.
2026 // If it's a use of a temporary register, then we messed up somewhere -
2027 // the previous condition should have passed.
2028 assert(MatchOpTable.lookup(OpName).Found &&
2029 !ApplyOpTable.getDef(OpName) && "Temp reg not emitted yet!");
2030 DstMI.addRenderer<CopyRenderer>(args&: OpName);
2031 }
2032 continue;
2033 }
2034
2035 // Determine what we're dealing with. Are we replacing a matched
2036 // instruction? Creating a new one?
2037 auto OpLookupRes = MatchOpTable.lookup(OpName);
2038 if (OpLookupRes.Found) {
2039 if (OpLookupRes.isLiveIn()) {
2040 // live-in of the match pattern.
2041 PrintError(Msg: "Cannot define live-in operand '" + OpName +
2042 "' in the 'apply' pattern");
2043 return false;
2044 }
2045 assert(OpLookupRes.Def);
2046
2047 // TODO: Handle this. We need to mutate the instr, or delete the old
2048 // one.
2049 // Likewise, we also need to ensure we redef everything, if the
2050 // instr has more than one def, we need to redef all or nothing.
2051 if (OpLookupRes.Def != MatchRoot) {
2052 PrintError(Msg: "redefining an instruction other than the root is not "
2053 "supported (operand '" +
2054 OpName + "')");
2055 return false;
2056 }
2057 // redef of a match
2058 DstMI.addRenderer<CopyRenderer>(args&: OpName);
2059 continue;
2060 }
2061
2062 // Define a new register unique to the apply patterns (AKA a "temp"
2063 // register).
2064 unsigned TempRegID;
2065 if (auto It = OperandToTempRegID.find(Key: OpName);
2066 It != OperandToTempRegID.end()) {
2067 TempRegID = It->second;
2068 } else {
2069 // This is a brand new register.
2070 TempRegID = M.allocateTempRegID();
2071 OperandToTempRegID[OpName] = TempRegID;
2072 const auto Ty = Op.getType();
2073 if (!Ty) {
2074 PrintError(Msg: "def of a new register '" + OpName +
2075 "' in the apply patterns must have a type");
2076 return false;
2077 }
2078
2079 declareTempRegExpansion(CE, TempRegID, Name: OpName);
2080 // Always insert the action at the beginning, otherwise we may end up
2081 // using the temp reg before it's available.
2082 auto Result = getLLTCodeGenOrTempType(PT: Ty, RM&: M);
2083 if (!Result)
2084 return false;
2085 M.insertAction<MakeTempRegisterAction>(InsertPt: M.actions_begin(), args&: *Result,
2086 args&: TempRegID);
2087 }
2088
2089 DstMI.addRenderer<TempRegRenderer>(args&: TempRegID, /*IsDef=*/args: true);
2090 }
2091
2092 // Some intrinsics have no in operands, ensure the ID is still emitted in such
2093 // cases.
2094 if (CGIP.isIntrinsic() && !HasEmittedIntrinsicID)
2095 EmitIntrinsicID();
2096
2097 // Render MIFlags
2098 if (const auto *FI = CGIP.getMIFlagsInfo()) {
2099 for (StringRef InstName : FI->copy_flags())
2100 DstMI.addCopiedMIFlags(IM: M.getInstructionMatcher(SymbolicName: InstName));
2101 for (StringRef F : FI->set_flags())
2102 DstMI.addSetMIFlags(Flag: F);
2103 for (StringRef F : FI->unset_flags())
2104 DstMI.addUnsetMIFlags(Flag: F);
2105 }
2106
2107 // Don't allow mutating opcodes for GISel combiners. We want a more precise
2108 // handling of MIFlags so we require them to be explicitly preserved.
2109 //
2110 // TODO: We don't mutate very often, if at all in combiners, but it'd be nice
2111 // to re-enable this. We'd then need to always clear MIFlags when mutating
2112 // opcodes, and never mutate an inst that we copy flags from.
2113 // DstMI.chooseInsnToMutate(M);
2114 declareInstExpansion(CE, A: DstMI, Name: P.getName());
2115
2116 return true;
2117}
2118
2119bool CombineRuleBuilder::emitCodeGenInstructionApplyImmOperand(
2120 RuleMatcher &M, BuildMIAction &DstMI, const CodeGenInstructionPattern &P,
2121 const InstructionOperand &O) {
2122 // If we have a type, we implicitly emit a G_CONSTANT, except for G_CONSTANT
2123 // itself where we emit a CImm.
2124 //
2125 // No type means we emit a simple imm.
2126 // G_CONSTANT is a special case and needs a CImm though so this is likely a
2127 // mistake.
2128 const bool isGConstant = P.is(OpcodeName: "G_CONSTANT");
2129 const auto Ty = O.getType();
2130 if (!Ty) {
2131 if (isGConstant) {
2132 PrintError(Msg: "'G_CONSTANT' immediate must be typed!");
2133 PrintNote(Msg: "while emitting pattern '" + P.getName() + "' (" +
2134 P.getInstName() + ")");
2135 return false;
2136 }
2137
2138 DstMI.addRenderer<ImmRenderer>(args: O.getImmValue());
2139 return true;
2140 }
2141
2142 auto ImmTy = getLLTCodeGenOrTempType(PT: Ty, RM&: M);
2143 if (!ImmTy)
2144 return false;
2145
2146 if (isGConstant) {
2147 DstMI.addRenderer<ImmRenderer>(args: O.getImmValue(), args&: *ImmTy);
2148 return true;
2149 }
2150
2151 unsigned TempRegID = M.allocateTempRegID();
2152 // Ensure MakeTempReg & the BuildConstantAction occur at the beginning.
2153 auto InsertIt = M.insertAction<MakeTempRegisterAction>(InsertPt: M.actions_begin(),
2154 args&: *ImmTy, args&: TempRegID);
2155 M.insertAction<BuildConstantAction>(InsertPt: ++InsertIt, args&: TempRegID, args: O.getImmValue());
2156 DstMI.addRenderer<TempRegRenderer>(args&: TempRegID);
2157 return true;
2158}
2159
2160bool CombineRuleBuilder::emitBuiltinApplyPattern(
2161 CodeExpansions &CE, RuleMatcher &M, const BuiltinPattern &P,
2162 StringMap<unsigned> &OperandToTempRegID) {
2163 const auto Error = [&](Twine Reason) {
2164 PrintError(Msg: "cannot emit '" + P.getInstName() + "' builtin: " + Reason);
2165 return false;
2166 };
2167
2168 switch (P.getBuiltinKind()) {
2169 case BI_EraseRoot: {
2170 // Root is always inst 0.
2171 M.addAction<EraseInstAction>(/*InsnID*/ args: 0);
2172 return true;
2173 }
2174 case BI_ReplaceReg: {
2175 StringRef Old = P.getOperand(K: 0).getOperandName();
2176 StringRef New = P.getOperand(K: 1).getOperandName();
2177
2178 if (!ApplyOpTable.lookup(OpName: New).Found && !MatchOpTable.lookup(OpName: New).Found)
2179 return Error("unknown operand '" + Old + "'");
2180
2181 auto &OldOM = M.getOperandMatcher(Name: Old);
2182 if (auto It = OperandToTempRegID.find(Key: New);
2183 It != OperandToTempRegID.end()) {
2184 // Replace with temp reg.
2185 M.addAction<ReplaceRegAction>(args: OldOM.getInsnVarID(), args: OldOM.getOpIdx(),
2186 args&: It->second);
2187 } else {
2188 // Replace with matched reg.
2189 auto &NewOM = M.getOperandMatcher(Name: New);
2190 M.addAction<ReplaceRegAction>(args: OldOM.getInsnVarID(), args: OldOM.getOpIdx(),
2191 args: NewOM.getInsnVarID(), args: NewOM.getOpIdx());
2192 }
2193 // checkSemantics should have ensured that we can only rewrite the root.
2194 // Ensure we're deleting it.
2195 assert(MatchOpTable.getDef(Old) == MatchRoot);
2196 return true;
2197 }
2198 }
2199
2200 llvm_unreachable("Unknown BuiltinKind!");
2201}
2202
2203bool isLiteralImm(const InstructionPattern &P, unsigned OpIdx) {
2204 if (const auto *CGP = dyn_cast<CodeGenInstructionPattern>(Val: &P)) {
2205 StringRef InstName = CGP->getInst().getName();
2206 return (InstName == "G_CONSTANT" || InstName == "G_FCONSTANT") &&
2207 OpIdx == 1;
2208 }
2209
2210 llvm_unreachable("TODO");
2211}
2212
2213bool CombineRuleBuilder::emitCodeGenInstructionMatchPattern(
2214 CodeExpansions &CE, const PatternAlternatives &Alts, RuleMatcher &M,
2215 InstructionMatcher &IM, const CodeGenInstructionPattern &P,
2216 DenseSet<const Pattern *> &SeenPats, OperandDefLookupFn LookupOperandDef,
2217 OperandMapperFnRef OperandMapper) {
2218 auto StackTrace = PrettyStackTraceEmit(RuleDef, &P);
2219
2220 if (!SeenPats.insert(V: &P).second)
2221 return true;
2222
2223 IM.addPredicate<InstructionOpcodeMatcher>(args: &P.getInst());
2224 declareInstExpansion(CE, IM, Name: P.getName());
2225
2226 // If this is an intrinsic, check the intrinsic ID.
2227 if (P.isIntrinsic()) {
2228 // The IntrinsicID's operand is the first operand after the defs.
2229 OperandMatcher &OM = IM.addOperand(OpIdx: P.getNumInstDefs(), SymbolicName: "$intrinsic_id",
2230 AllocatedTemporariesBaseID: AllocatedTemporariesBaseID++);
2231 OM.addPredicate<IntrinsicIDOperandMatcher>(args: P.getIntrinsic());
2232 }
2233
2234 // Check flags if needed.
2235 if (const auto *FI = P.getMIFlagsInfo()) {
2236 assert(FI->copy_flags().empty());
2237
2238 if (const auto &SetF = FI->set_flags(); !SetF.empty())
2239 IM.addPredicate<MIFlagsInstructionPredicateMatcher>(args: SetF.getArrayRef());
2240 if (const auto &UnsetF = FI->unset_flags(); !UnsetF.empty())
2241 IM.addPredicate<MIFlagsInstructionPredicateMatcher>(args: UnsetF.getArrayRef(),
2242 /*CheckNot=*/args: true);
2243 }
2244
2245 for (auto [Idx, OriginalO] : enumerate(First: P.operands())) {
2246 // Remap the operand. This is used when emitting InstructionPatterns inside
2247 // PatFrags, so it can remap them to the arguments passed to the pattern.
2248 //
2249 // We use the remapped operand to emit immediates, and for the symbolic
2250 // operand names (in IM.addOperand). CodeExpansions and OperandTable lookups
2251 // still use the original name.
2252 //
2253 // The "def" flag on the remapped operand is always ignored.
2254 auto RemappedO = OperandMapper(OriginalO);
2255 assert(RemappedO.isNamedOperand() == OriginalO.isNamedOperand() &&
2256 "Cannot remap an unnamed operand to a named one!");
2257
2258 const auto Ty = RemappedO.getType();
2259
2260 const auto OpName =
2261 RemappedO.isNamedOperand() ? RemappedO.getOperandName().str() : "";
2262
2263 // For intrinsics, the first use operand is the intrinsic id, so the true
2264 // operand index is shifted by 1.
2265 //
2266 // From now on:
2267 // Idx = index in the pattern operand list.
2268 // RealIdx = expected index in the MachineInstr.
2269 const unsigned RealIdx =
2270 (P.isIntrinsic() && !OriginalO.isDef()) ? (Idx + 1) : Idx;
2271
2272 if (Ty.isVariadicPack() && M.hasOperand(SymbolicName: OpName)) {
2273 // TODO: We could add some CheckIsSameOperand opcode variant that checks
2274 // all operands. We could also just emit a C++ code snippet lazily to do
2275 // the check since it's probably fairly rare that we need to do it.
2276 //
2277 // I'm just not sure it's worth the effort at this stage.
2278 PrintError(Msg: "each instance of a " + PatternType::VariadicClassName +
2279 " operand must have a unique name within the match patterns");
2280 PrintNote(Msg: "'" + OpName + "' is used multiple times");
2281 return false;
2282 }
2283
2284 OperandMatcher &OM =
2285 IM.addOperand(OpIdx: RealIdx, SymbolicName: OpName, AllocatedTemporariesBaseID: AllocatedTemporariesBaseID++,
2286 /*IsVariadic=*/Ty.isVariadicPack());
2287 if (!OpName.empty())
2288 declareOperandExpansion(CE, OM, Name: OriginalO.getOperandName());
2289
2290 if (Ty.isVariadicPack()) {
2291 // In the presence of variadics, the InstructionMatcher won't insert a
2292 // InstructionNumOperandsMatcher implicitly, so we have to emit our own.
2293 assert((Idx + 1) == P.operands_size() &&
2294 "VariadicPack isn't last operand!");
2295 auto VPTI = Ty.getVariadicPackTypeInfo();
2296 assert(VPTI.Min > 0 && (VPTI.Max == 0 || VPTI.Max > VPTI.Min));
2297 IM.addPredicate<InstructionNumOperandsMatcher>(
2298 args: RealIdx + VPTI.Min, args: InstructionNumOperandsMatcher::CheckKind::GE);
2299 if (VPTI.Max) {
2300 IM.addPredicate<InstructionNumOperandsMatcher>(
2301 args: RealIdx + VPTI.Max, args: InstructionNumOperandsMatcher::CheckKind::LE);
2302 }
2303 break;
2304 }
2305
2306 // Handle immediates.
2307 if (RemappedO.hasImmValue()) {
2308 if (isLiteralImm(P, OpIdx: Idx))
2309 OM.addPredicate<LiteralIntOperandMatcher>(args: RemappedO.getImmValue());
2310 else
2311 OM.addPredicate<ConstantIntOperandMatcher>(args: RemappedO.getImmValue());
2312 }
2313
2314 // Handle typed operands, but only bother to check if it hasn't been done
2315 // before.
2316 //
2317 // getOperandMatcher will always return the first OM to have been created
2318 // for that Operand. "OM" here is always a new OperandMatcher.
2319 //
2320 // Always emit a check for unnamed operands.
2321 if (Ty && (OpName.empty() ||
2322 !M.getOperandMatcher(Name: OpName).contains<LLTOperandMatcher>())) {
2323 // TODO: We could support GITypeOf here on the condition that the
2324 // OperandMatcher exists already. Though it's clunky to make this work
2325 // and isn't all that useful so it's just rejected in typecheckPatterns
2326 // at this time.
2327 assert(Ty.isLLT());
2328 OM.addPredicate<LLTOperandMatcher>(args: getLLTCodeGen(PT: Ty));
2329 }
2330
2331 // Stop here if the operand is a def, or if it had no name.
2332 if (OriginalO.isDef() || !OriginalO.isNamedOperand())
2333 continue;
2334
2335 const auto *DefPat = LookupOperandDef(OriginalO.getOperandName());
2336 if (!DefPat)
2337 continue;
2338
2339 if (OriginalO.hasImmValue()) {
2340 assert(!OpName.empty());
2341 // This is a named immediate that also has a def, that's not okay.
2342 // e.g.
2343 // (G_SEXT $y, (i32 0))
2344 // (COPY $x, 42:$y)
2345 PrintError(Msg: "'" + OpName +
2346 "' is a named immediate, it cannot be defined by another "
2347 "instruction");
2348 PrintNote(Msg: "'" + OpName + "' is defined by '" + DefPat->getName() + "'");
2349 return false;
2350 }
2351
2352 // From here we know that the operand defines an instruction, and we need to
2353 // emit it.
2354 auto InstOpM =
2355 OM.addPredicate<InstructionOperandMatcher>(args&: M, args: DefPat->getName());
2356 if (!InstOpM) {
2357 // TODO: copy-pasted from GlobalISelEmitter.cpp. Is it still relevant
2358 // here?
2359 PrintError(Msg: "Nested instruction '" + DefPat->getName() +
2360 "' cannot be the same as another operand '" +
2361 OriginalO.getOperandName() + "'");
2362 return false;
2363 }
2364
2365 auto &IM = (*InstOpM)->getInsnMatcher();
2366 if (const auto *CGIDef = dyn_cast<CodeGenInstructionPattern>(Val: DefPat)) {
2367 if (!emitCodeGenInstructionMatchPattern(CE, Alts, M, IM, P: *CGIDef,
2368 SeenPats, LookupOperandDef,
2369 OperandMapper))
2370 return false;
2371 continue;
2372 }
2373
2374 if (const auto *PFPDef = dyn_cast<PatFragPattern>(Val: DefPat)) {
2375 if (!emitPatFragMatchPattern(CE, Alts, RM&: M, IM: &IM, PFP: *PFPDef, SeenPats))
2376 return false;
2377 continue;
2378 }
2379
2380 llvm_unreachable("unknown type of InstructionPattern");
2381 }
2382
2383 return true;
2384}
2385
2386//===- GICombinerEmitter --------------------------------------------------===//
2387
2388/// Main implementation class. This emits the tablegenerated output.
2389///
2390/// It collects rules, uses `CombineRuleBuilder` to parse them and accumulate
2391/// RuleMatchers, then takes all the necessary state/data from the various
2392/// static storage pools and wires them together to emit the match table &
2393/// associated function/data structures.
2394class GICombinerEmitter final : public GlobalISelMatchTableExecutorEmitter {
2395 const RecordKeeper &Records;
2396 StringRef Name;
2397 const CodeGenTarget &Target;
2398 const Record *Combiner;
2399 unsigned NextRuleID = 0;
2400
2401 // List all combine rules (ID, name) imported.
2402 // Note that the combiner rule ID is different from the RuleMatcher ID. The
2403 // latter is internal to the MatchTable, the former is the canonical ID of the
2404 // combine rule used to disable/enable it.
2405 std::vector<std::pair<unsigned, std::string>> AllCombineRules;
2406
2407 // Keep track of all rules we've seen so far to ensure we don't process
2408 // the same rule twice.
2409 StringSet<> RulesSeen;
2410
2411 MatchTable buildMatchTable(MutableArrayRef<RuleMatcher> Rules);
2412
2413 void emitRuleConfigImpl(raw_ostream &OS);
2414
2415 void emitAdditionalImpl(raw_ostream &OS) override;
2416
2417 void emitMIPredicateFns(raw_ostream &OS) override;
2418 void emitLeafPredicateFns(raw_ostream &OS) override;
2419 void emitI64ImmPredicateFns(raw_ostream &OS) override;
2420 void emitAPFloatImmPredicateFns(raw_ostream &OS) override;
2421 void emitAPIntImmPredicateFns(raw_ostream &OS) override;
2422 void emitTestSimplePredicate(raw_ostream &OS) override;
2423 void emitRunCustomAction(raw_ostream &OS) override;
2424
2425 const CodeGenTarget &getTarget() const override { return Target; }
2426 StringRef getClassName() const override {
2427 return Combiner->getValueAsString(FieldName: "Classname");
2428 }
2429
2430 StringRef getCombineAllMethodName() const {
2431 return Combiner->getValueAsString(FieldName: "CombineAllMethodName");
2432 }
2433
2434 std::string getRuleConfigClassName() const {
2435 return getClassName().str() + "RuleConfig";
2436 }
2437
2438 void gatherRules(std::vector<RuleMatcher> &Rules,
2439 ArrayRef<const Record *> RulesAndGroups);
2440
2441public:
2442 explicit GICombinerEmitter(const RecordKeeper &RK,
2443 const CodeGenTarget &Target, StringRef Name,
2444 const Record *Combiner);
2445 ~GICombinerEmitter() override = default;
2446
2447 void run(raw_ostream &OS);
2448};
2449
2450void GICombinerEmitter::emitRuleConfigImpl(raw_ostream &OS) {
2451 IfDefGuardEmitter If(OS, "GET_GICOMBINER_TYPES");
2452 OS << "struct " << getRuleConfigClassName() << " {\n"
2453 << " SparseBitVector<> DisabledRules;\n\n"
2454 << " bool isRuleEnabled(unsigned RuleID) const;\n"
2455 << " bool parseCommandLineOption();\n"
2456 << " bool setRuleEnabled(StringRef RuleIdentifier);\n"
2457 << " bool setRuleDisabled(StringRef RuleIdentifier);\n"
2458 << "};\n\n";
2459
2460 std::vector<std::pair<std::string, std::string>> Cases;
2461 Cases.reserve(n: AllCombineRules.size());
2462
2463 for (const auto &[ID, Name] : AllCombineRules)
2464 Cases.emplace_back(args: Name, args: "return " + to_string(Value: ID) + ";\n");
2465
2466 OS << "static std::optional<uint64_t> getRuleIdxForIdentifier(StringRef "
2467 "RuleIdentifier) {\n"
2468 << " uint64_t I;\n"
2469 << " // getAtInteger(...) returns false on success\n"
2470 << " bool Parsed = !RuleIdentifier.getAsInteger(0, I);\n"
2471 << " if (Parsed)\n"
2472 << " return I;\n\n"
2473 << "#ifndef NDEBUG\n";
2474 StringMatcher Matcher("RuleIdentifier", Cases, OS);
2475 Matcher.Emit();
2476 OS << "#endif // ifndef NDEBUG\n\n"
2477 << " return std::nullopt;\n"
2478 << "}\n";
2479
2480 OS << "static std::optional<std::pair<uint64_t, uint64_t>> "
2481 "getRuleRangeForIdentifier(StringRef RuleIdentifier) {\n"
2482 << " std::pair<StringRef, StringRef> RangePair = "
2483 "RuleIdentifier.split('-');\n"
2484 << " if (!RangePair.second.empty()) {\n"
2485 << " const auto First = "
2486 "getRuleIdxForIdentifier(RangePair.first);\n"
2487 << " const auto Last = "
2488 "getRuleIdxForIdentifier(RangePair.second);\n"
2489 << " if (!First || !Last)\n"
2490 << " return std::nullopt;\n"
2491 << " if (First >= Last)\n"
2492 << " report_fatal_error(\"Beginning of range should be before "
2493 "end of range\");\n"
2494 << " return {{*First, *Last + 1}};\n"
2495 << " }\n"
2496 << " if (RangePair.first == \"*\") {\n"
2497 << " return {{0, " << AllCombineRules.size() << "}};\n"
2498 << " }\n"
2499 << " const auto I = getRuleIdxForIdentifier(RangePair.first);\n"
2500 << " if (!I)\n"
2501 << " return std::nullopt;\n"
2502 << " return {{*I, *I + 1}};\n"
2503 << "}\n\n";
2504
2505 for (bool Enabled : {true, false}) {
2506 OS << "bool " << getRuleConfigClassName() << "::setRule"
2507 << (Enabled ? "Enabled" : "Disabled") << "(StringRef RuleIdentifier) {\n"
2508 << " auto MaybeRange = getRuleRangeForIdentifier(RuleIdentifier);\n"
2509 << " if (!MaybeRange)\n"
2510 << " return false;\n"
2511 << " for (auto I = MaybeRange->first; I < MaybeRange->second; ++I)\n"
2512 << " DisabledRules." << (Enabled ? "reset" : "set") << "(I);\n"
2513 << " return true;\n"
2514 << "}\n\n";
2515 }
2516
2517 OS << "static std::vector<std::string> " << Name << "Option;\n"
2518 << "static cl::list<std::string> " << Name << "DisableOption(\n"
2519 << " \"" << Name.lower() << "-disable-rule\",\n"
2520 << " cl::desc(\"Disable one or more combiner rules temporarily in "
2521 << "the " << Name << " pass\"),\n"
2522 << " cl::CommaSeparated,\n"
2523 << " cl::Hidden,\n"
2524 << " cl::cat(GICombinerOptionCategory),\n"
2525 << " cl::callback([](const std::string &Str) {\n"
2526 << " " << Name << "Option.push_back(Str);\n"
2527 << " }));\n"
2528 << "static cl::list<std::string> " << Name << "OnlyEnableOption(\n"
2529 << " \"" << Name.lower() << "-only-enable-rule\",\n"
2530 << " cl::desc(\"Disable all rules in the " << Name
2531 << " pass then re-enable the specified ones\"),\n"
2532 << " cl::Hidden,\n"
2533 << " cl::cat(GICombinerOptionCategory),\n"
2534 << " cl::callback([](const std::string &CommaSeparatedArg) {\n"
2535 << " StringRef Str = CommaSeparatedArg;\n"
2536 << " " << Name << "Option.push_back(\"*\");\n"
2537 << " do {\n"
2538 << " auto X = Str.split(\",\");\n"
2539 << " " << Name << "Option.push_back((\"!\" + X.first).str());\n"
2540 << " Str = X.second;\n"
2541 << " } while (!Str.empty());\n"
2542 << " }));\n"
2543 << "\n\n"
2544 << "bool " << getRuleConfigClassName()
2545 << "::isRuleEnabled(unsigned RuleID) const {\n"
2546 << " return !DisabledRules.test(RuleID);\n"
2547 << "}\n"
2548 << "bool " << getRuleConfigClassName() << "::parseCommandLineOption() {\n"
2549 << " for (StringRef Identifier : " << Name << "Option) {\n"
2550 << " bool Enabled = Identifier.consume_front(\"!\");\n"
2551 << " if (Enabled && !setRuleEnabled(Identifier))\n"
2552 << " return false;\n"
2553 << " if (!Enabled && !setRuleDisabled(Identifier))\n"
2554 << " return false;\n"
2555 << " }\n"
2556 << " return true;\n"
2557 << "}\n\n";
2558}
2559
2560void GICombinerEmitter::emitAdditionalImpl(raw_ostream &OS) {
2561 OS << "bool " << getClassName() << "::" << getCombineAllMethodName()
2562 << "(MachineInstr &I) const {\n"
2563 << " const TargetSubtargetInfo &ST = MF.getSubtarget();\n"
2564 << " const PredicateBitset AvailableFeatures = "
2565 "getAvailableFeatures();\n"
2566 << " B.setInstrAndDebugLoc(I);\n"
2567 << " State.MIs.clear();\n"
2568 << " State.MIs.push_back(&I);\n"
2569 << " if (executeMatchTable(*this, State, ExecInfo, B"
2570 << ", getMatchTable(), *ST.getInstrInfo(), MRI, "
2571 "*MRI.getTargetRegisterInfo(), *ST.getRegBankInfo(), AvailableFeatures"
2572 << ", /*CoverageInfo*/ nullptr)) {\n"
2573 << " return true;\n"
2574 << " }\n\n"
2575 << " return false;\n"
2576 << "}\n\n";
2577}
2578
2579void GICombinerEmitter::emitMIPredicateFns(raw_ostream &OS) {
2580 auto MatchCode = CXXPredicateCode::getAllMatchCode();
2581 emitMIPredicateFnsImpl<const CXXPredicateCode *>(
2582 OS, AdditionalDecls: "", Predicates: ArrayRef<const CXXPredicateCode *>(MatchCode),
2583 GetPredEnumName: [](const CXXPredicateCode *C) -> StringRef { return C->BaseEnumName; },
2584 GetPredCode: [](const CXXPredicateCode *C) -> StringRef { return C->Code; });
2585}
2586
2587void GICombinerEmitter::emitLeafPredicateFns(raw_ostream &OS) {
2588 // Unused, but still needs to be called.
2589 emitLeafPredicateFnsImpl<unsigned>(
2590 OS, AdditionalDecls: "", Predicates: {}, GetPredEnumName: [](unsigned) { return ""; }, GetPredCode: [](unsigned) { return ""; });
2591}
2592
2593void GICombinerEmitter::emitI64ImmPredicateFns(raw_ostream &OS) {
2594 // Unused, but still needs to be called.
2595 emitImmPredicateFnsImpl<unsigned>(
2596 OS, TypeIdentifier: "I64", ArgType: "int64_t", Predicates: {}, GetPredEnumName: [](unsigned) { return ""; },
2597 GetPredCode: [](unsigned) { return ""; });
2598}
2599
2600void GICombinerEmitter::emitAPFloatImmPredicateFns(raw_ostream &OS) {
2601 // Unused, but still needs to be called.
2602 emitImmPredicateFnsImpl<unsigned>(
2603 OS, TypeIdentifier: "APFloat", ArgType: "const APFloat &", Predicates: {}, GetPredEnumName: [](unsigned) { return ""; },
2604 GetPredCode: [](unsigned) { return ""; });
2605}
2606
2607void GICombinerEmitter::emitAPIntImmPredicateFns(raw_ostream &OS) {
2608 // Unused, but still needs to be called.
2609 emitImmPredicateFnsImpl<unsigned>(
2610 OS, TypeIdentifier: "APInt", ArgType: "const APInt &", Predicates: {}, GetPredEnumName: [](unsigned) { return ""; },
2611 GetPredCode: [](unsigned) { return ""; });
2612}
2613
2614void GICombinerEmitter::emitTestSimplePredicate(raw_ostream &OS) {
2615 if (!AllCombineRules.empty()) {
2616 OS << "enum {\n";
2617 std::string EnumeratorSeparator = " = GICXXPred_Invalid + 1,\n";
2618 // To avoid emitting a switch, we expect that all those rules are in order.
2619 // That way we can just get the RuleID from the enum by subtracting
2620 // (GICXXPred_Invalid + 1).
2621 [[maybe_unused]] unsigned ExpectedID = 0;
2622 for (const auto &ID : keys(C&: AllCombineRules)) {
2623 assert(ExpectedID == ID && "combine rules are not ordered!");
2624 ++ExpectedID;
2625 OS << " " << getIsEnabledPredicateEnumName(CombinerRuleID: ID) << EnumeratorSeparator;
2626 EnumeratorSeparator = ",\n";
2627 }
2628 OS << "};\n\n";
2629 }
2630
2631 OS << "bool " << getClassName()
2632 << "::testSimplePredicate(unsigned Predicate) const {\n"
2633 << " return RuleConfig.isRuleEnabled(Predicate - "
2634 "GICXXPred_Invalid - "
2635 "1);\n"
2636 << "}\n";
2637}
2638
2639void GICombinerEmitter::emitRunCustomAction(raw_ostream &OS) {
2640 const auto CustomActionsCode = CXXPredicateCode::getAllCustomActionsCode();
2641
2642 if (!CustomActionsCode.empty()) {
2643 OS << "enum {\n";
2644 std::string EnumeratorSeparator = " = GICXXCustomAction_Invalid + 1,\n";
2645 for (const auto &CA : CustomActionsCode) {
2646 OS << " " << CA->getEnumNameWithPrefix(Prefix: CXXCustomActionPrefix)
2647 << EnumeratorSeparator;
2648 EnumeratorSeparator = ",\n";
2649 }
2650 OS << "};\n";
2651 }
2652
2653 OS << "bool " << getClassName()
2654 << "::runCustomAction(unsigned ApplyID, const MatcherState &State, "
2655 "NewMIVector &OutMIs) const "
2656 "{\n Helper.getBuilder().setInstrAndDebugLoc(*State.MIs[0]);\n";
2657 if (!CustomActionsCode.empty()) {
2658 OS << " switch(ApplyID) {\n";
2659 for (const auto &CA : CustomActionsCode) {
2660 OS << " case " << CA->getEnumNameWithPrefix(Prefix: CXXCustomActionPrefix)
2661 << ":{\n"
2662 << " " << join(R: split(Str: CA->Code, Separator: '\n'), Separator: "\n ") << '\n'
2663 << " return true;\n";
2664 OS << " }\n";
2665 }
2666 OS << " }\n";
2667 }
2668 OS << " llvm_unreachable(\"Unknown Apply Action\");\n"
2669 << "}\n";
2670}
2671
2672GICombinerEmitter::GICombinerEmitter(const RecordKeeper &RK,
2673 const CodeGenTarget &Target,
2674 StringRef Name, const Record *Combiner)
2675 : Records(RK), Name(Name), Target(Target), Combiner(Combiner) {}
2676
2677MatchTable
2678GICombinerEmitter::buildMatchTable(MutableArrayRef<RuleMatcher> Rules) {
2679 std::vector<Matcher *> InputRules;
2680 for (Matcher &Rule : Rules)
2681 InputRules.push_back(x: &Rule);
2682
2683 unsigned CurrentOrdering = 0;
2684 StringMap<unsigned> OpcodeOrder;
2685 for (RuleMatcher &Rule : Rules) {
2686 const StringRef Opcode = Rule.getOpcode();
2687 assert(!Opcode.empty() && "Didn't expect an undefined opcode");
2688 if (OpcodeOrder.try_emplace(Key: Opcode, Args&: CurrentOrdering).second)
2689 ++CurrentOrdering;
2690 }
2691
2692 llvm::stable_sort(Range&: InputRules, C: [&OpcodeOrder](const Matcher *A,
2693 const Matcher *B) {
2694 auto *L = static_cast<const RuleMatcher *>(A);
2695 auto *R = static_cast<const RuleMatcher *>(B);
2696 return std::tuple(OpcodeOrder[L->getOpcode()],
2697 L->insnmatchers_front().getNumOperandMatchers()) <
2698 std::tuple(OpcodeOrder[R->getOpcode()],
2699 R->insnmatchers_front().getNumOperandMatchers());
2700 });
2701
2702 for (Matcher *Rule : InputRules)
2703 Rule->optimize();
2704
2705 std::vector<std::unique_ptr<Matcher>> MatcherStorage;
2706 std::vector<Matcher *> OptRules =
2707 optimizeRules<GroupMatcher>(Rules: InputRules, MatcherStorage);
2708
2709 for (Matcher *Rule : OptRules)
2710 Rule->optimize();
2711
2712 OptRules = optimizeRules<SwitchMatcher>(Rules: OptRules, MatcherStorage);
2713
2714 return MatchTable::buildTable(Rules: OptRules, /*WithCoverage*/ false,
2715 /*IsCombiner*/ true);
2716}
2717
2718/// Recurse into GICombineGroup's and flatten the ruleset into a simple list.
2719void GICombinerEmitter::gatherRules(std::vector<RuleMatcher> &ActiveRules,
2720 ArrayRef<const Record *> RulesAndGroups) {
2721 for (const Record *Rec : RulesAndGroups) {
2722 if (!Rec->isValueUnset(FieldName: "Rules")) {
2723 gatherRules(ActiveRules, RulesAndGroups: Rec->getValueAsListOfDefs(FieldName: "Rules"));
2724 continue;
2725 }
2726
2727 StringRef RuleName = Rec->getName();
2728 if (!RulesSeen.insert(key: RuleName).second) {
2729 PrintWarning(WarningLoc: Rec->getLoc(),
2730 Msg: "skipping rule '" + Rec->getName() +
2731 "' because it has already been processed");
2732 continue;
2733 }
2734
2735 AllCombineRules.emplace_back(args&: NextRuleID, args: Rec->getName().str());
2736 CombineRuleBuilder CRB(Target, SubtargetFeatures, *Rec, NextRuleID++,
2737 ActiveRules);
2738
2739 if (!CRB.parseAll()) {
2740 assert(ErrorsPrinted && "Parsing failed without errors!");
2741 continue;
2742 }
2743
2744 if (StopAfterParse) {
2745 CRB.print(OS&: outs());
2746 continue;
2747 }
2748
2749 if (!CRB.emitRuleMatchers()) {
2750 assert(ErrorsPrinted && "Emission failed without errors!");
2751 continue;
2752 }
2753 }
2754}
2755
2756void GICombinerEmitter::run(raw_ostream &OS) {
2757 InstructionOpcodeMatcher::initOpcodeValuesMap(Target);
2758 LLTOperandMatcher::initTypeIDValuesMap();
2759
2760 TGTimer &Timer = Records.getTimer();
2761 Timer.startTimer(Name: "Gather rules");
2762 std::vector<RuleMatcher> Rules;
2763 gatherRules(ActiveRules&: Rules, RulesAndGroups: Combiner->getValueAsListOfDefs(FieldName: "Rules"));
2764 if (ErrorsPrinted)
2765 PrintFatalError(ErrorLoc: Combiner->getLoc(), Msg: "Failed to parse one or more rules");
2766
2767 if (StopAfterParse)
2768 return;
2769
2770 Timer.startTimer(Name: "Creating Match Table");
2771 unsigned MaxTemporaries = 0;
2772 for (const auto &Rule : Rules)
2773 MaxTemporaries = std::max(a: MaxTemporaries, b: Rule.countRendererFns());
2774
2775 llvm::stable_sort(Range&: Rules, C: [&](const RuleMatcher &A, const RuleMatcher &B) {
2776 if (A.isHigherPriorityThan(B)) {
2777 assert(!B.isHigherPriorityThan(A) && "Cannot be more important "
2778 "and less important at "
2779 "the same time");
2780 return true;
2781 }
2782 return false;
2783 });
2784
2785 const MatchTable Table = buildMatchTable(Rules);
2786
2787 Timer.startTimer(Name: "Emit combiner");
2788
2789 emitSourceFileHeader(Desc: getClassName().str() + " Combiner Match Table", OS);
2790
2791 SmallVector<LLTCodeGen, 16> TypeObjects;
2792 append_range(C&: TypeObjects, R&: KnownTypes);
2793 llvm::sort(C&: TypeObjects);
2794
2795 // Hack: Avoid empty declarator.
2796 if (TypeObjects.empty())
2797 TypeObjects.push_back(Elt: LLT::scalar(SizeInBits: 1));
2798
2799 // GET_GICOMBINER_DEPS, which pulls in extra dependencies.
2800 {
2801 IfDefGuardEmitter If(OS, "GET_GICOMBINER_DEPS");
2802 OS << "#include \"llvm/ADT/SparseBitVector.h\"\n";
2803 NamespaceEmitter LlvmNS(OS, "llvm");
2804 OS << "extern cl::OptionCategory GICombinerOptionCategory;\n";
2805 }
2806
2807 // GET_GICOMBINER_TYPES, which needs to be included before the declaration of
2808 // the class.
2809 emitRuleConfigImpl(OS);
2810 emitPredicateBitset(OS, IfDefName: "GET_GICOMBINER_TYPES");
2811
2812 // GET_GICOMBINER_CLASS_MEMBERS, which need to be included inside the class.
2813 emitPredicatesDecl(OS, IfDefName: "GET_GICOMBINER_CLASS_MEMBERS");
2814 emitTemporariesDecl(OS, IfDefName: "GET_GICOMBINER_CLASS_MEMBERS");
2815
2816 // GET_GICOMBINER_IMPL, which needs to be included outside the class.
2817 emitExecutorImpl(OS, Table, TypeObjects, Rules, ComplexOperandMatchers: {}, CustomOperandRenderers: {},
2818 IfDefName: "GET_GICOMBINER_IMPL");
2819
2820 // GET_GICOMBINER_CONSTRUCTOR_INITS, which are in the constructor's
2821 // initializer list.
2822 emitPredicatesInit(OS, IfDefName: "GET_GICOMBINER_CONSTRUCTOR_INITS");
2823 emitTemporariesInit(OS, MaxTemporaries, IfDefName: "GET_GICOMBINER_CONSTRUCTOR_INITS");
2824}
2825
2826//===----------------------------------------------------------------------===//
2827
2828static void EmitGICombiner(const RecordKeeper &RK, raw_ostream &OS) {
2829 EnablePrettyStackTrace();
2830 const CodeGenTarget Target(RK);
2831
2832 if (SelectedCombiners.empty())
2833 PrintFatalError(Msg: "No combiners selected with -combiners");
2834 for (const auto &Combiner : SelectedCombiners) {
2835 const Record *CombinerDef = RK.getDef(Name: Combiner);
2836 if (!CombinerDef)
2837 PrintFatalError(Msg: "Could not find " + Combiner);
2838 GICombinerEmitter(RK, Target, Combiner, CombinerDef).run(OS);
2839 }
2840}
2841
2842static TableGen::Emitter::Opt X("gen-global-isel-combiner", EmitGICombiner,
2843 "Generate GlobalISel Combiner");
2844