1//===------ MacroFusionPredicatorEmitter.cpp - Generator for Fusion ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===---------------------------------------------------------------------===//
8//
9// MacroFusionPredicatorEmitter implements a TableGen-driven predicators
10// generator for macro-op fusions.
11//
12// This TableGen backend processes `Fusion` definitions and generates
13// predicators for checking if input instructions can be fused. These
14// predicators can used in `MacroFusion` DAG mutation.
15//
16// The generated header file contains two parts: one for predicator
17// declarations and one for predicator implementations. The user can get them
18// by defining macro `GET_<TargetName>_MACRO_FUSION_PRED_DECL` or
19// `GET_<TargetName>_MACRO_FUSION_PRED_IMPL` and then including the generated
20// header file.
21//
22// Each predicator also maintains `Statistic`s that count how often the fusion
23// is matched. A fusion that runs in both scheduling stages keeps separate
24// pre-RA and post-RA counters, because both schedulers run MacroFusion.
25//
26// A fusion can opt out of a scheduling stage via the `RunPreRA`/`RunPostRA`
27// fields. When a stage is disabled, the generated predicator returns `false`
28// early during that stage (detected via the `NoVRegs` machine function
29// property) and only keeps the counter for the stage it actually runs in.
30//
31// A fusion running in both stages will be like:
32//
33// ```
34// STATISTIC(NumNAMEPreRA, "Times NAME Triggered (pre-ra)");
35// STATISTIC(NumNAMEPostRA, "Times NAME Triggered (post-ra)");
36// bool isNAME(const TargetInstrInfo &TII,
37// const TargetSubtargetInfo &STI,
38// const MachineInstr *FirstMI,
39// const MachineInstr &SecondMI,
40// const SDep *Dep) {
41// if (isNonDataDep(Dep))
42// return false;
43// auto &MRI = SecondMI.getMF()->getRegInfo();
44// /* Predicates */
45// if (SecondMI.getMF()->getProperties().hasNoVRegs())
46// ++NumNAMEPostRA;
47// else
48// ++NumNAMEPreRA;
49// return true;
50// }
51// ```
52//
53// A fusion restricted to a single stage (e.g. pre-RA only) will be like:
54//
55// ```
56// STATISTIC(NumNAMEPreRA, "Times NAME Triggered (pre-ra)");
57// bool isNAME(const TargetInstrInfo &TII,
58// const TargetSubtargetInfo &STI,
59// const MachineInstr *FirstMI,
60// const MachineInstr &SecondMI,
61// const SDep *Dep) {
62// if (isNonDataDep(Dep))
63// return false;
64// auto &MRI = SecondMI.getMF()->getRegInfo();
65// if (SecondMI.getMF()->getProperties().hasNoVRegs())
66// return false;
67// /* Predicates */
68// ++NumNAMEPreRA;
69// return true;
70// }
71// ```
72//
73// The `Predicates` part is generated from a list of `FusionPredicate`, which
74// can be predefined predicates, a raw code string or `MCInstPredicate` defined
75// in TargetInstrPredicate.td.
76//
77//===---------------------------------------------------------------------===//
78
79#include "Common/CodeGenTarget.h"
80#include "Common/PredicateExpander.h"
81#include "llvm/Support/Debug.h"
82#include "llvm/TableGen/CodeGenHelpers.h"
83#include "llvm/TableGen/Error.h"
84#include "llvm/TableGen/Record.h"
85#include "llvm/TableGen/TableGenBackend.h"
86#include <vector>
87
88using namespace llvm;
89
90#define DEBUG_TYPE "macro-fusion-predicator"
91
92namespace {
93class MacroFusionPredicatorEmitter {
94 const CodeGenTarget Target;
95
96 void emitMacroFusionDecl(ArrayRef<const Record *> Fusions,
97 PredicateExpander &PE, raw_ostream &OS);
98 void emitMacroFusionImpl(ArrayRef<const Record *> Fusions,
99 PredicateExpander &PE, raw_ostream &OS);
100 void emitPredicates(ArrayRef<const Record *> FirstPredicate,
101 bool IsCommutable, PredicateExpander &PE,
102 raw_ostream &OS);
103 void emitFirstPredicate(const Record *SecondPredicate, bool IsCommutable,
104 PredicateExpander &PE, raw_ostream &OS);
105 void emitSecondPredicate(const Record *SecondPredicate, bool IsCommutable,
106 PredicateExpander &PE, raw_ostream &OS);
107 void emitBothPredicate(const Record *Predicates, bool IsCommutable,
108 PredicateExpander &PE, raw_ostream &OS);
109
110public:
111 MacroFusionPredicatorEmitter(const RecordKeeper &R) : Target(R) {}
112
113 void run(raw_ostream &OS);
114};
115} // End anonymous namespace.
116
117void MacroFusionPredicatorEmitter::emitMacroFusionDecl(
118 ArrayRef<const Record *> Fusions, PredicateExpander &PE, raw_ostream &OS) {
119 IfDefEmitter IfDef(
120 OS, ("GET_" + Target.getName() + "_MACRO_FUSION_PRED_DECL").str());
121 NamespaceEmitter LlvmNS(OS, "llvm");
122
123 for (const Record *Fusion : Fusions)
124 OS << "bool is" << CodeGenTarget::getMacroFusionName(Fusion)
125 << "(const TargetInstrInfo &, "
126 << "const TargetSubtargetInfo &, const MachineInstr *, "
127 << "const MachineInstr &, const SDep *);\n";
128}
129
130void MacroFusionPredicatorEmitter::emitMacroFusionImpl(
131 ArrayRef<const Record *> Fusions, PredicateExpander &PE, raw_ostream &OS) {
132 IfDefEmitter IfDef(
133 OS, ("GET_" + Target.getName() + "_MACRO_FUSION_PRED_IMPL").str());
134 NamespaceEmitter LlvmNS(OS, "llvm");
135
136 for (const Record *Fusion : Fusions) {
137 StringRef Name = CodeGenTarget::getMacroFusionName(Fusion);
138 std::vector<const Record *> Predicates =
139 Fusion->getValueAsListOfDefs(FieldName: "Predicates");
140 bool IsCommutable = Fusion->getValueAsBit(FieldName: "IsCommutable");
141 bool RunPreRA = Fusion->getValueAsBit(FieldName: "RunPreRA");
142 bool RunPostRA = Fusion->getValueAsBit(FieldName: "RunPostRA");
143
144 if (!RunPreRA && !RunPostRA)
145 PrintFatalError(ErrorLoc: Fusion->getLoc(),
146 Msg: "Fusion '" + Name +
147 "' must run in at least one of the pre-RA and "
148 "post-RA scheduling stages");
149
150 // Emit the statistics that count how often this fusion is matched. The
151 // pre-RA and post-RA schedulers both run MacroFusion, so a fusion that
152 // runs in both stages keeps separate counters (distinguished below via the
153 // `NoVRegs` property) to avoid conflating the two. A fusion that opts out
154 // of a stage only needs the counter for the stage it actually runs in.
155 if (RunPreRA)
156 OS << "STATISTIC(Num" << Name << "PreRA, \"Times " << Name
157 << " Triggered (pre-ra)\");\n";
158 if (RunPostRA)
159 OS << "STATISTIC(Num" << Name << "PostRA, \"Times " << Name
160 << " Triggered (post-ra)\");\n";
161
162 OS << "bool is" << Name << "(\n";
163 OS.indent(NumSpaces: 4) << "const TargetInstrInfo &TII,\n";
164 OS.indent(NumSpaces: 4) << "const TargetSubtargetInfo &STI,\n";
165 OS.indent(NumSpaces: 4) << "const MachineInstr *FirstMI,\n";
166 OS.indent(NumSpaces: 4) << "const MachineInstr &SecondMI, const SDep *Dep) {\n";
167 OS.indent(NumSpaces: 2) << "if (isNonDataDep(Dep))\n";
168 OS.indent(NumSpaces: 4) << "return false;\n";
169 OS.indent(NumSpaces: 2)
170 << "[[maybe_unused]] auto &MRI = SecondMI.getMF()->getRegInfo();\n";
171
172 // If the fusion opts out of a scheduling stage, bail out early when we are
173 // running in that stage. The pre-RA scheduler still has virtual registers,
174 // while the post-RA scheduler runs after they have been allocated.
175 if (!RunPreRA) {
176 OS.indent(NumSpaces: 2) << "if (!SecondMI.getMF()->getProperties().hasNoVRegs())\n";
177 OS.indent(NumSpaces: 4) << "return false;\n";
178 }
179 if (!RunPostRA) {
180 OS.indent(NumSpaces: 2) << "if (SecondMI.getMF()->getProperties().hasNoVRegs())\n";
181 OS.indent(NumSpaces: 4) << "return false;\n";
182 }
183
184 emitPredicates(FirstPredicate: Predicates, IsCommutable, PE, OS);
185
186 // Bump the statistic for the matched stage. When the fusion runs in both
187 // stages we still have to tell them apart at runtime; otherwise the guard
188 // above already established the stage, so a single counter suffices.
189 if (RunPreRA && RunPostRA) {
190 OS.indent(NumSpaces: 2) << "if (SecondMI.getMF()->getProperties().hasNoVRegs())\n";
191 OS.indent(NumSpaces: 4) << "++Num" << Name << "PostRA;\n";
192 OS.indent(NumSpaces: 2) << "else\n";
193 OS.indent(NumSpaces: 4) << "++Num" << Name << "PreRA;\n";
194 } else if (RunPreRA) {
195 OS.indent(NumSpaces: 2) << "++Num" << Name << "PreRA;\n";
196 } else {
197 OS.indent(NumSpaces: 2) << "++Num" << Name << "PostRA;\n";
198 }
199
200 OS.indent(NumSpaces: 2) << "return true;\n";
201 OS << "}\n";
202 }
203}
204
205void MacroFusionPredicatorEmitter::emitPredicates(
206 ArrayRef<const Record *> Predicates, bool IsCommutable,
207 PredicateExpander &PE, raw_ostream &OS) {
208 for (const Record *Predicate : Predicates) {
209 const Record *Target = Predicate->getValueAsDef(FieldName: "Target");
210 if (Target->getName() == "first_fusion_target")
211 emitFirstPredicate(SecondPredicate: Predicate, IsCommutable, PE, OS);
212 else if (Target->getName() == "second_fusion_target")
213 emitSecondPredicate(SecondPredicate: Predicate, IsCommutable, PE, OS);
214 else if (Target->getName() == "both_fusion_target")
215 emitBothPredicate(Predicates: Predicate, IsCommutable, PE, OS);
216 else
217 PrintFatalError(ErrorLoc: Target->getLoc(),
218 Msg: "Unsupported 'FusionTarget': " + Target->getName());
219 }
220}
221
222void MacroFusionPredicatorEmitter::emitFirstPredicate(const Record *Predicate,
223 bool IsCommutable,
224 PredicateExpander &PE,
225 raw_ostream &OS) {
226 if (Predicate->isSubClassOf(Name: "WildcardPred")) {
227 OS.indent(NumSpaces: 2) << "if (!FirstMI)\n";
228 OS.indent(NumSpaces: 2) << " return "
229 << (Predicate->getValueAsBit(FieldName: "ReturnValue") ? "true" : "false")
230 << ";\n";
231 } else if (Predicate->isSubClassOf(Name: "OneUsePred")) {
232 OS.indent(NumSpaces: 2) << "{\n";
233 OS.indent(NumSpaces: 4) << "Register FirstDest = FirstMI->getOperand(0).getReg();\n";
234 OS.indent(NumSpaces: 4)
235 << "if (FirstDest.isVirtual() && !MRI.hasOneNonDBGUse(FirstDest))\n";
236 OS.indent(NumSpaces: 4) << " return false;\n";
237 OS.indent(NumSpaces: 2) << "}\n";
238 } else if (Predicate->isSubClassOf(Name: "FirstInstHasSameReg")) {
239 int FirstOpIdx = Predicate->getValueAsInt(FieldName: "FirstOpIdx");
240 int SecondOpIdx = Predicate->getValueAsInt(FieldName: "SecondOpIdx");
241
242 OS.indent(NumSpaces: 2) << "if (!FirstMI->getOperand(" << FirstOpIdx
243 << ").getReg().isVirtual()) {\n";
244 OS.indent(NumSpaces: 4) << "if (FirstMI->getOperand(" << FirstOpIdx
245 << ").getReg() != FirstMI->getOperand(" << SecondOpIdx
246 << ").getReg())";
247
248 if (IsCommutable) {
249 OS << " {\n";
250 OS.indent(NumSpaces: 6) << "if (!FirstMI->getDesc().isCommutable())\n";
251 OS.indent(NumSpaces: 6) << " return false;\n";
252
253 OS.indent(NumSpaces: 6)
254 << "unsigned SrcOpIdx1 = " << SecondOpIdx
255 << ", SrcOpIdx2 = TargetInstrInfo::CommuteAnyOperandIndex;\n";
256 OS.indent(NumSpaces: 6)
257 << "if (TII.findCommutedOpIndices(FirstMI, SrcOpIdx1, SrcOpIdx2))\n";
258 OS.indent(NumSpaces: 6)
259 << " if (FirstMI->getOperand(" << FirstOpIdx
260 << ").getReg() != FirstMI->getOperand(SrcOpIdx2).getReg())\n";
261 OS.indent(NumSpaces: 6) << " return false;\n";
262 OS.indent(NumSpaces: 4) << "}\n";
263 } else {
264 OS << "\n";
265 OS.indent(NumSpaces: 4) << " return false;\n";
266 }
267 OS.indent(NumSpaces: 2) << "}\n";
268 } else if (Predicate->isSubClassOf(Name: "FusionPredicateWithMCInstPredicate")) {
269 OS.indent(NumSpaces: 2) << "{\n";
270 OS.indent(NumSpaces: 4) << "[[maybe_unused]] const MachineInstr *MI = FirstMI;\n";
271 OS.indent(NumSpaces: 4) << "if (";
272 PE.setNegatePredicate(true);
273 PE.getIndent() = 3;
274 PE.expandPredicate(OS, Rec: Predicate->getValueAsDef(FieldName: "Predicate"));
275 OS << ")\n";
276 OS.indent(NumSpaces: 4) << " return false;\n";
277 OS.indent(NumSpaces: 2) << "}\n";
278 } else {
279 PrintFatalError(ErrorLoc: Predicate->getLoc(),
280 Msg: "Unsupported predicate for first instruction: " +
281 Predicate->getType()->getAsString());
282 }
283}
284
285void MacroFusionPredicatorEmitter::emitSecondPredicate(const Record *Predicate,
286 bool IsCommutable,
287 PredicateExpander &PE,
288 raw_ostream &OS) {
289 if (Predicate->isSubClassOf(Name: "FusionPredicateWithMCInstPredicate")) {
290 OS.indent(NumSpaces: 2) << "{\n";
291 OS.indent(NumSpaces: 4) << "[[maybe_unused]] const MachineInstr *MI = &SecondMI;\n";
292 OS.indent(NumSpaces: 4) << "if (";
293 PE.setNegatePredicate(true);
294 PE.getIndent() = 3;
295 PE.expandPredicate(OS, Rec: Predicate->getValueAsDef(FieldName: "Predicate"));
296 OS << ")\n";
297 OS.indent(NumSpaces: 4) << " return false;\n";
298 OS.indent(NumSpaces: 2) << "}\n";
299 } else if (Predicate->isSubClassOf(Name: "SecondInstHasSameReg")) {
300 int FirstOpIdx = Predicate->getValueAsInt(FieldName: "FirstOpIdx");
301 int SecondOpIdx = Predicate->getValueAsInt(FieldName: "SecondOpIdx");
302
303 OS.indent(NumSpaces: 2) << "if (!SecondMI.getOperand(" << FirstOpIdx
304 << ").getReg().isVirtual()) {\n";
305 OS.indent(NumSpaces: 4) << "if (SecondMI.getOperand(" << FirstOpIdx
306 << ").getReg() != SecondMI.getOperand(" << SecondOpIdx
307 << ").getReg())";
308
309 if (IsCommutable) {
310 OS << " {\n";
311 OS.indent(NumSpaces: 6) << "if (!SecondMI.getDesc().isCommutable())\n";
312 OS.indent(NumSpaces: 6) << " return false;\n";
313
314 OS.indent(NumSpaces: 6)
315 << "unsigned SrcOpIdx1 = " << SecondOpIdx
316 << ", SrcOpIdx2 = TargetInstrInfo::CommuteAnyOperandIndex;\n";
317 OS.indent(NumSpaces: 6)
318 << "if (TII.findCommutedOpIndices(SecondMI, SrcOpIdx1, SrcOpIdx2))\n";
319 OS.indent(NumSpaces: 6)
320 << " if (SecondMI.getOperand(" << FirstOpIdx
321 << ").getReg() != SecondMI.getOperand(SrcOpIdx2).getReg())\n";
322 OS.indent(NumSpaces: 6) << " return false;\n";
323 OS.indent(NumSpaces: 4) << "}\n";
324 } else {
325 OS << "\n";
326 OS.indent(NumSpaces: 4) << " return false;\n";
327 }
328 OS.indent(NumSpaces: 2) << "}\n";
329 } else {
330 PrintFatalError(ErrorLoc: Predicate->getLoc(),
331 Msg: "Unsupported predicate for second instruction: " +
332 Predicate->getType()->getAsString());
333 }
334}
335
336void MacroFusionPredicatorEmitter::emitBothPredicate(const Record *Predicate,
337 bool IsCommutable,
338 PredicateExpander &PE,
339 raw_ostream &OS) {
340 if (Predicate->isSubClassOf(Name: "FusionPredicateWithCode"))
341 OS << Predicate->getValueAsString(FieldName: "Predicate");
342 else if (Predicate->isSubClassOf(Name: "BothFusionPredicateWithMCInstPredicate")) {
343 emitFirstPredicate(Predicate, IsCommutable, PE, OS);
344 emitSecondPredicate(Predicate, IsCommutable, PE, OS);
345 } else if (Predicate->isSubClassOf(Name: "TieReg")) {
346 int FirstOpIdx = Predicate->getValueAsInt(FieldName: "FirstOpIdx");
347 int SecondOpIdx = Predicate->getValueAsInt(FieldName: "SecondOpIdx");
348 OS.indent(NumSpaces: 2) << "if (!(FirstMI->getOperand(" << FirstOpIdx
349 << ").isReg() &&\n";
350 OS.indent(NumSpaces: 2) << " SecondMI.getOperand(" << SecondOpIdx
351 << ").isReg() &&\n";
352 OS.indent(NumSpaces: 2) << " FirstMI->getOperand(" << FirstOpIdx
353 << ").getReg() == SecondMI.getOperand(" << SecondOpIdx
354 << ").getReg()))";
355
356 if (IsCommutable) {
357 OS << " {\n";
358 OS.indent(NumSpaces: 4) << "if (!SecondMI.getDesc().isCommutable())\n";
359 OS.indent(NumSpaces: 4) << " return false;\n";
360
361 OS.indent(NumSpaces: 4)
362 << "unsigned SrcOpIdx1 = " << SecondOpIdx
363 << ", SrcOpIdx2 = TargetInstrInfo::CommuteAnyOperandIndex;\n";
364 OS.indent(NumSpaces: 4)
365 << "if (TII.findCommutedOpIndices(SecondMI, SrcOpIdx1, SrcOpIdx2))\n";
366 OS.indent(NumSpaces: 4)
367 << " if (FirstMI->getOperand(" << FirstOpIdx
368 << ").getReg() != SecondMI.getOperand(SrcOpIdx2).getReg())\n";
369 OS.indent(NumSpaces: 4) << " return false;\n";
370 OS.indent(NumSpaces: 2) << "}";
371 } else {
372 OS << "\n";
373 OS.indent(NumSpaces: 2) << " return false;";
374 }
375 OS << "\n";
376 } else {
377 PrintFatalError(ErrorLoc: Predicate->getLoc(),
378 Msg: "Unsupported predicate for both instruction: " +
379 Predicate->getType()->getAsString());
380 }
381}
382
383void MacroFusionPredicatorEmitter::run(raw_ostream &OS) {
384 // Emit file header.
385 emitSourceFileHeader(Desc: "Macro Fusion Predicators", OS);
386
387 PredicateExpander PE(Target.getName());
388 PE.setByRef(false);
389 PE.setExpandForMC(false);
390
391 ArrayRef<const Record *> Fusions = Target.getMacroFusions();
392 emitMacroFusionDecl(Fusions, PE, OS);
393 OS << "\n";
394 emitMacroFusionImpl(Fusions, PE, OS);
395}
396
397static TableGen::Emitter::OptClass<MacroFusionPredicatorEmitter>
398 X("gen-macro-fusion-pred", "Generate macro fusion predicators.");
399