1//===------ MacroFusionPredicatorEmitter.cpp - Generator for Fusion ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===---------------------------------------------------------------------===//
8//
9// MacroFusionPredicatorEmitter implements a TableGen-driven predicators
10// generator for macro-op fusions.
11//
12// This TableGen backend processes `Fusion` definitions and generates
13// predicators for checking if input instructions can be fused. These
14// predicators can used in `MacroFusion` DAG mutation.
15//
16// The generated header file contains two parts: one for predicator
17// declarations and one for predicator implementations. The user can get them
18// by defining macro `GET_<TargetName>_MACRO_FUSION_PRED_DECL` or
19// `GET_<TargetName>_MACRO_FUSION_PRED_IMPL` and then including the generated
20// header file.
21//
22// Each predicator also maintains `Statistic`s that count how often the fusion
23// is matched. A fusion that runs in both scheduling stages keeps separate
24// pre-RA and post-RA counters, because both schedulers run MacroFusion.
25//
26// A fusion can opt out of a scheduling stage via the `RunPreRA`/`RunPostRA`
27// fields. When a stage is disabled, the generated predicator returns `false`
28// early during that stage (detected via the `NoVRegs` machine function
29// property) and only keeps the counter for the stage it actually runs in.
30//
31// A fusion running in both stages will be like:
32//
33// ```
34// STATISTIC(NumNAMEPreRA, "Times NAME Triggered (pre-ra)");
35// STATISTIC(NumNAMEPostRA, "Times NAME Triggered (post-ra)");
36// bool isNAME(const TargetInstrInfo &TII,
37// const TargetSubtargetInfo &STI,
38// const MachineInstr *FirstMI,
39// const MachineInstr &SecondMI,
40// const SDep *Dep) {
41// if (isNonDataDep(Dep))
42// return false;
43// auto &MRI = SecondMI.getMF()->getRegInfo();
44// /* Predicates */
45// if (SecondMI.getMF()->getProperties().hasNoVRegs())
46// ++NumNAMEPostRA;
47// else
48// ++NumNAMEPreRA;
49// return true;
50// }
51// ```
52//
53// A fusion restricted to a single stage (e.g. pre-RA only) will be like:
54//
55// ```
56// STATISTIC(NumNAMEPreRA, "Times NAME Triggered (pre-ra)");
57// bool isNAME(const TargetInstrInfo &TII,
58// const TargetSubtargetInfo &STI,
59// const MachineInstr *FirstMI,
60// const MachineInstr &SecondMI,
61// const SDep *Dep) {
62// if (isNonDataDep(Dep))
63// return false;
64// auto &MRI = SecondMI.getMF()->getRegInfo();
65// if (SecondMI.getMF()->getProperties().hasNoVRegs())
66// return false;
67// /* Predicates */
68// ++NumNAMEPreRA;
69// return true;
70// }
71// ```
72//
73// The `Predicates` part is generated from a list of `FusionPredicate`, which
74// can be predefined predicates, a raw code string or `MCInstPredicate` defined
75// in TargetInstrPredicate.td.
76//
77//===---------------------------------------------------------------------===//
78
79#include "Common/CodeGenTarget.h"
80#include "Common/PredicateExpander.h"
81#include "llvm/Support/Debug.h"
82#include "llvm/TableGen/CodeGenHelpers.h"
83#include "llvm/TableGen/Error.h"
84#include "llvm/TableGen/Record.h"
85#include "llvm/TableGen/TableGenBackend.h"
86#include <vector>
87
88using namespace llvm;
89
90#define DEBUG_TYPE "macro-fusion-predicator"
91
92namespace {
93class MacroFusionPredicatorEmitter {
94 const RecordKeeper &Records;
95 const CodeGenTarget Target;
96
97 void emitMacroFusionDecl(ArrayRef<const Record *> Fusions,
98 PredicateExpander &PE, raw_ostream &OS);
99 void emitMacroFusionImpl(ArrayRef<const Record *> Fusions,
100 PredicateExpander &PE, raw_ostream &OS);
101 void emitPredicates(ArrayRef<const Record *> FirstPredicate,
102 bool IsCommutable, PredicateExpander &PE,
103 raw_ostream &OS);
104 void emitFirstPredicate(const Record *SecondPredicate, bool IsCommutable,
105 PredicateExpander &PE, raw_ostream &OS);
106 void emitSecondPredicate(const Record *SecondPredicate, bool IsCommutable,
107 PredicateExpander &PE, raw_ostream &OS);
108 void emitBothPredicate(const Record *Predicates, bool IsCommutable,
109 PredicateExpander &PE, raw_ostream &OS);
110
111public:
112 MacroFusionPredicatorEmitter(const RecordKeeper &R) : Records(R), Target(R) {}
113
114 void run(raw_ostream &OS);
115};
116} // End anonymous namespace.
117
118void MacroFusionPredicatorEmitter::emitMacroFusionDecl(
119 ArrayRef<const Record *> Fusions, PredicateExpander &PE, raw_ostream &OS) {
120 IfDefEmitter IfDef(
121 OS, ("GET_" + Target.getName() + "_MACRO_FUSION_PRED_DECL").str());
122 NamespaceEmitter LlvmNS(OS, "llvm");
123
124 for (const Record *Fusion : Fusions)
125 OS << "bool is" << Fusion->getName() << "(const TargetInstrInfo &, "
126 << "const TargetSubtargetInfo &, const MachineInstr *, "
127 << "const MachineInstr &, const SDep *);\n";
128}
129
130void MacroFusionPredicatorEmitter::emitMacroFusionImpl(
131 ArrayRef<const Record *> Fusions, PredicateExpander &PE, raw_ostream &OS) {
132 IfDefEmitter IfDef(
133 OS, ("GET_" + Target.getName() + "_MACRO_FUSION_PRED_IMPL").str());
134 NamespaceEmitter LlvmNS(OS, "llvm");
135
136 for (const Record *Fusion : Fusions) {
137 std::vector<const Record *> Predicates =
138 Fusion->getValueAsListOfDefs(FieldName: "Predicates");
139 bool IsCommutable = Fusion->getValueAsBit(FieldName: "IsCommutable");
140 bool RunPreRA = Fusion->getValueAsBit(FieldName: "RunPreRA");
141 bool RunPostRA = Fusion->getValueAsBit(FieldName: "RunPostRA");
142
143 if (!RunPreRA && !RunPostRA)
144 PrintFatalError(ErrorLoc: Fusion->getLoc(),
145 Msg: "Fusion '" + Fusion->getName() +
146 "' must run in at least one of the pre-RA and "
147 "post-RA scheduling stages");
148
149 // Emit the statistics that count how often this fusion is matched. The
150 // pre-RA and post-RA schedulers both run MacroFusion, so a fusion that
151 // runs in both stages keeps separate counters (distinguished below via the
152 // `NoVRegs` property) to avoid conflating the two. A fusion that opts out
153 // of a stage only needs the counter for the stage it actually runs in.
154 if (RunPreRA)
155 OS << "STATISTIC(Num" << Fusion->getName() << "PreRA, \"Times "
156 << Fusion->getName() << " Triggered (pre-ra)\");\n";
157 if (RunPostRA)
158 OS << "STATISTIC(Num" << Fusion->getName() << "PostRA, \"Times "
159 << Fusion->getName() << " Triggered (post-ra)\");\n";
160
161 OS << "bool is" << Fusion->getName() << "(\n";
162 OS.indent(NumSpaces: 4) << "const TargetInstrInfo &TII,\n";
163 OS.indent(NumSpaces: 4) << "const TargetSubtargetInfo &STI,\n";
164 OS.indent(NumSpaces: 4) << "const MachineInstr *FirstMI,\n";
165 OS.indent(NumSpaces: 4) << "const MachineInstr &SecondMI, const SDep *Dep) {\n";
166 OS.indent(NumSpaces: 2) << "if (isNonDataDep(Dep))\n";
167 OS.indent(NumSpaces: 4) << "return false;\n";
168 OS.indent(NumSpaces: 2)
169 << "[[maybe_unused]] auto &MRI = SecondMI.getMF()->getRegInfo();\n";
170
171 // If the fusion opts out of a scheduling stage, bail out early when we are
172 // running in that stage. The pre-RA scheduler still has virtual registers,
173 // while the post-RA scheduler runs after they have been allocated.
174 if (!RunPreRA) {
175 OS.indent(NumSpaces: 2) << "if (!SecondMI.getMF()->getProperties().hasNoVRegs())\n";
176 OS.indent(NumSpaces: 4) << "return false;\n";
177 }
178 if (!RunPostRA) {
179 OS.indent(NumSpaces: 2) << "if (SecondMI.getMF()->getProperties().hasNoVRegs())\n";
180 OS.indent(NumSpaces: 4) << "return false;\n";
181 }
182
183 emitPredicates(FirstPredicate: Predicates, IsCommutable, PE, OS);
184
185 // Bump the statistic for the matched stage. When the fusion runs in both
186 // stages we still have to tell them apart at runtime; otherwise the guard
187 // above already established the stage, so a single counter suffices.
188 if (RunPreRA && RunPostRA) {
189 OS.indent(NumSpaces: 2) << "if (SecondMI.getMF()->getProperties().hasNoVRegs())\n";
190 OS.indent(NumSpaces: 4) << "++Num" << Fusion->getName() << "PostRA;\n";
191 OS.indent(NumSpaces: 2) << "else\n";
192 OS.indent(NumSpaces: 4) << "++Num" << Fusion->getName() << "PreRA;\n";
193 } else if (RunPreRA) {
194 OS.indent(NumSpaces: 2) << "++Num" << Fusion->getName() << "PreRA;\n";
195 } else {
196 OS.indent(NumSpaces: 2) << "++Num" << Fusion->getName() << "PostRA;\n";
197 }
198
199 OS.indent(NumSpaces: 2) << "return true;\n";
200 OS << "}\n";
201 }
202}
203
204void MacroFusionPredicatorEmitter::emitPredicates(
205 ArrayRef<const Record *> Predicates, bool IsCommutable,
206 PredicateExpander &PE, raw_ostream &OS) {
207 for (const Record *Predicate : Predicates) {
208 const Record *Target = Predicate->getValueAsDef(FieldName: "Target");
209 if (Target->getName() == "first_fusion_target")
210 emitFirstPredicate(SecondPredicate: Predicate, IsCommutable, PE, OS);
211 else if (Target->getName() == "second_fusion_target")
212 emitSecondPredicate(SecondPredicate: Predicate, IsCommutable, PE, OS);
213 else if (Target->getName() == "both_fusion_target")
214 emitBothPredicate(Predicates: Predicate, IsCommutable, PE, OS);
215 else
216 PrintFatalError(ErrorLoc: Target->getLoc(),
217 Msg: "Unsupported 'FusionTarget': " + Target->getName());
218 }
219}
220
221void MacroFusionPredicatorEmitter::emitFirstPredicate(const Record *Predicate,
222 bool IsCommutable,
223 PredicateExpander &PE,
224 raw_ostream &OS) {
225 if (Predicate->isSubClassOf(Name: "WildcardPred")) {
226 OS.indent(NumSpaces: 2) << "if (!FirstMI)\n";
227 OS.indent(NumSpaces: 2) << " return "
228 << (Predicate->getValueAsBit(FieldName: "ReturnValue") ? "true" : "false")
229 << ";\n";
230 } else if (Predicate->isSubClassOf(Name: "OneUsePred")) {
231 OS.indent(NumSpaces: 2) << "{\n";
232 OS.indent(NumSpaces: 4) << "Register FirstDest = FirstMI->getOperand(0).getReg();\n";
233 OS.indent(NumSpaces: 4)
234 << "if (FirstDest.isVirtual() && !MRI.hasOneNonDBGUse(FirstDest))\n";
235 OS.indent(NumSpaces: 4) << " return false;\n";
236 OS.indent(NumSpaces: 2) << "}\n";
237 } else if (Predicate->isSubClassOf(Name: "FirstInstHasSameReg")) {
238 int FirstOpIdx = Predicate->getValueAsInt(FieldName: "FirstOpIdx");
239 int SecondOpIdx = Predicate->getValueAsInt(FieldName: "SecondOpIdx");
240
241 OS.indent(NumSpaces: 2) << "if (!FirstMI->getOperand(" << FirstOpIdx
242 << ").getReg().isVirtual()) {\n";
243 OS.indent(NumSpaces: 4) << "if (FirstMI->getOperand(" << FirstOpIdx
244 << ").getReg() != FirstMI->getOperand(" << SecondOpIdx
245 << ").getReg())";
246
247 if (IsCommutable) {
248 OS << " {\n";
249 OS.indent(NumSpaces: 6) << "if (!FirstMI->getDesc().isCommutable())\n";
250 OS.indent(NumSpaces: 6) << " return false;\n";
251
252 OS.indent(NumSpaces: 6)
253 << "unsigned SrcOpIdx1 = " << SecondOpIdx
254 << ", SrcOpIdx2 = TargetInstrInfo::CommuteAnyOperandIndex;\n";
255 OS.indent(NumSpaces: 6)
256 << "if (TII.findCommutedOpIndices(FirstMI, SrcOpIdx1, SrcOpIdx2))\n";
257 OS.indent(NumSpaces: 6)
258 << " if (FirstMI->getOperand(" << FirstOpIdx
259 << ").getReg() != FirstMI->getOperand(SrcOpIdx2).getReg())\n";
260 OS.indent(NumSpaces: 6) << " return false;\n";
261 OS.indent(NumSpaces: 4) << "}\n";
262 } else {
263 OS << "\n";
264 OS.indent(NumSpaces: 4) << " return false;\n";
265 }
266 OS.indent(NumSpaces: 2) << "}\n";
267 } else if (Predicate->isSubClassOf(Name: "FusionPredicateWithMCInstPredicate")) {
268 OS.indent(NumSpaces: 2) << "{\n";
269 OS.indent(NumSpaces: 4) << "[[maybe_unused]] const MachineInstr *MI = FirstMI;\n";
270 OS.indent(NumSpaces: 4) << "if (";
271 PE.setNegatePredicate(true);
272 PE.getIndent() = 3;
273 PE.expandPredicate(OS, Rec: Predicate->getValueAsDef(FieldName: "Predicate"));
274 OS << ")\n";
275 OS.indent(NumSpaces: 4) << " return false;\n";
276 OS.indent(NumSpaces: 2) << "}\n";
277 } else {
278 PrintFatalError(ErrorLoc: Predicate->getLoc(),
279 Msg: "Unsupported predicate for first instruction: " +
280 Predicate->getType()->getAsString());
281 }
282}
283
284void MacroFusionPredicatorEmitter::emitSecondPredicate(const Record *Predicate,
285 bool IsCommutable,
286 PredicateExpander &PE,
287 raw_ostream &OS) {
288 if (Predicate->isSubClassOf(Name: "FusionPredicateWithMCInstPredicate")) {
289 OS.indent(NumSpaces: 2) << "{\n";
290 OS.indent(NumSpaces: 4) << "[[maybe_unused]] const MachineInstr *MI = &SecondMI;\n";
291 OS.indent(NumSpaces: 4) << "if (";
292 PE.setNegatePredicate(true);
293 PE.getIndent() = 3;
294 PE.expandPredicate(OS, Rec: Predicate->getValueAsDef(FieldName: "Predicate"));
295 OS << ")\n";
296 OS.indent(NumSpaces: 4) << " return false;\n";
297 OS.indent(NumSpaces: 2) << "}\n";
298 } else if (Predicate->isSubClassOf(Name: "SecondInstHasSameReg")) {
299 int FirstOpIdx = Predicate->getValueAsInt(FieldName: "FirstOpIdx");
300 int SecondOpIdx = Predicate->getValueAsInt(FieldName: "SecondOpIdx");
301
302 OS.indent(NumSpaces: 2) << "if (!SecondMI.getOperand(" << FirstOpIdx
303 << ").getReg().isVirtual()) {\n";
304 OS.indent(NumSpaces: 4) << "if (SecondMI.getOperand(" << FirstOpIdx
305 << ").getReg() != SecondMI.getOperand(" << SecondOpIdx
306 << ").getReg())";
307
308 if (IsCommutable) {
309 OS << " {\n";
310 OS.indent(NumSpaces: 6) << "if (!SecondMI.getDesc().isCommutable())\n";
311 OS.indent(NumSpaces: 6) << " return false;\n";
312
313 OS.indent(NumSpaces: 6)
314 << "unsigned SrcOpIdx1 = " << SecondOpIdx
315 << ", SrcOpIdx2 = TargetInstrInfo::CommuteAnyOperandIndex;\n";
316 OS.indent(NumSpaces: 6)
317 << "if (TII.findCommutedOpIndices(SecondMI, SrcOpIdx1, SrcOpIdx2))\n";
318 OS.indent(NumSpaces: 6)
319 << " if (SecondMI.getOperand(" << FirstOpIdx
320 << ").getReg() != SecondMI.getOperand(SrcOpIdx2).getReg())\n";
321 OS.indent(NumSpaces: 6) << " return false;\n";
322 OS.indent(NumSpaces: 4) << "}\n";
323 } else {
324 OS << "\n";
325 OS.indent(NumSpaces: 4) << " return false;\n";
326 }
327 OS.indent(NumSpaces: 2) << "}\n";
328 } else {
329 PrintFatalError(ErrorLoc: Predicate->getLoc(),
330 Msg: "Unsupported predicate for second instruction: " +
331 Predicate->getType()->getAsString());
332 }
333}
334
335void MacroFusionPredicatorEmitter::emitBothPredicate(const Record *Predicate,
336 bool IsCommutable,
337 PredicateExpander &PE,
338 raw_ostream &OS) {
339 if (Predicate->isSubClassOf(Name: "FusionPredicateWithCode"))
340 OS << Predicate->getValueAsString(FieldName: "Predicate");
341 else if (Predicate->isSubClassOf(Name: "BothFusionPredicateWithMCInstPredicate")) {
342 emitFirstPredicate(Predicate, IsCommutable, PE, OS);
343 emitSecondPredicate(Predicate, IsCommutable, PE, OS);
344 } else if (Predicate->isSubClassOf(Name: "TieReg")) {
345 int FirstOpIdx = Predicate->getValueAsInt(FieldName: "FirstOpIdx");
346 int SecondOpIdx = Predicate->getValueAsInt(FieldName: "SecondOpIdx");
347 OS.indent(NumSpaces: 2) << "if (!(FirstMI->getOperand(" << FirstOpIdx
348 << ").isReg() &&\n";
349 OS.indent(NumSpaces: 2) << " SecondMI.getOperand(" << SecondOpIdx
350 << ").isReg() &&\n";
351 OS.indent(NumSpaces: 2) << " FirstMI->getOperand(" << FirstOpIdx
352 << ").getReg() == SecondMI.getOperand(" << SecondOpIdx
353 << ").getReg()))";
354
355 if (IsCommutable) {
356 OS << " {\n";
357 OS.indent(NumSpaces: 4) << "if (!SecondMI.getDesc().isCommutable())\n";
358 OS.indent(NumSpaces: 4) << " return false;\n";
359
360 OS.indent(NumSpaces: 4)
361 << "unsigned SrcOpIdx1 = " << SecondOpIdx
362 << ", SrcOpIdx2 = TargetInstrInfo::CommuteAnyOperandIndex;\n";
363 OS.indent(NumSpaces: 4)
364 << "if (TII.findCommutedOpIndices(SecondMI, SrcOpIdx1, SrcOpIdx2))\n";
365 OS.indent(NumSpaces: 4)
366 << " if (FirstMI->getOperand(" << FirstOpIdx
367 << ").getReg() != SecondMI.getOperand(SrcOpIdx2).getReg())\n";
368 OS.indent(NumSpaces: 4) << " return false;\n";
369 OS.indent(NumSpaces: 2) << "}";
370 } else {
371 OS << "\n";
372 OS.indent(NumSpaces: 2) << " return false;";
373 }
374 OS << "\n";
375 } else {
376 PrintFatalError(ErrorLoc: Predicate->getLoc(),
377 Msg: "Unsupported predicate for both instruction: " +
378 Predicate->getType()->getAsString());
379 }
380}
381
382void MacroFusionPredicatorEmitter::run(raw_ostream &OS) {
383 // Emit file header.
384 emitSourceFileHeader(Desc: "Macro Fusion Predicators", OS);
385
386 PredicateExpander PE(Target.getName());
387 PE.setByRef(false);
388 PE.setExpandForMC(false);
389
390 ArrayRef<const Record *> Fusions = Records.getAllDerivedDefinitions(ClassName: "Fusion");
391 emitMacroFusionDecl(Fusions, PE, OS);
392 OS << "\n";
393 emitMacroFusionImpl(Fusions, PE, OS);
394}
395
396static TableGen::Emitter::OptClass<MacroFusionPredicatorEmitter>
397 X("gen-macro-fusion-pred", "Generate macro fusion predicators.");
398