1//===- SubtargetEmitter.cpp - Generate subtarget enumerations -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend emits subtarget enumerations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "Basic/TargetFeaturesEmitter.h"
14#include "Common/CodeGenHwModes.h"
15#include "Common/CodeGenSchedule.h"
16#include "Common/CodeGenTarget.h"
17#include "Common/PredicateExpander.h"
18#include "Common/SubtargetFeatureInfo.h"
19#include "Common/Utils.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/ADT/StringMap.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/MC/MCInstrItineraries.h"
26#include "llvm/MC/MCSchedule.h"
27#include "llvm/Support/Debug.h"
28#include "llvm/Support/Format.h"
29#include "llvm/Support/raw_ostream.h"
30#include "llvm/TableGen/CodeGenHelpers.h"
31#include "llvm/TableGen/Error.h"
32#include "llvm/TableGen/Record.h"
33#include "llvm/TableGen/StringToOffsetTable.h"
34#include "llvm/TableGen/TableGenBackend.h"
35#include <algorithm>
36#include <cassert>
37#include <cstdint>
38#include <iterator>
39#include <string>
40#include <vector>
41
42using namespace llvm;
43
44#define DEBUG_TYPE "subtarget-emitter"
45
46namespace {
47
48class SubtargetEmitter : TargetFeaturesEmitter {
49 // Each processor has a SchedClassDesc table with an entry for each
50 // SchedClass. The SchedClassDesc table indexes into a global write resource
51 // table, write latency table, and read advance table.
52 struct SchedClassTables {
53 std::vector<std::vector<MCSchedClassDesc>> ProcSchedClasses;
54 std::vector<MCWriteProcResEntry> WriteProcResources;
55 std::vector<MCWriteLatencyEntry> WriteLatencies;
56 std::vector<std::string> WriterNames;
57 std::vector<MCReadAdvanceEntry> ReadAdvanceEntries;
58 size_t MaxWriteProcResEntries = 0;
59 size_t MaxWriteLatencyEntries = 0;
60
61 // Reserve an invalid entry at index 0
62 SchedClassTables() {
63 ProcSchedClasses.resize(new_size: 1);
64 WriteProcResources.resize(new_size: 1);
65 WriteLatencies.resize(new_size: 1);
66 WriterNames.push_back(x: "InvalidWrite");
67 ReadAdvanceEntries.resize(new_size: 1);
68 }
69 };
70
71 struct LessWriteProcResources {
72 bool operator()(const MCWriteProcResEntry &LHS,
73 const MCWriteProcResEntry &RHS) {
74 return LHS.ProcResourceIdx < RHS.ProcResourceIdx;
75 }
76 };
77
78 CodeGenTarget TGT;
79 CodeGenSchedModels &SchedModels;
80
81 FeatureMapTy emitEnums(raw_ostream &OS);
82 void emitSubtargetInfoMacroCalls(raw_ostream &OS);
83
84 struct MCDescInfo {
85 unsigned NumFeatures;
86 unsigned FeatureStrTabSize;
87 unsigned NumProcs;
88 unsigned SubTypeStrTabSize;
89 };
90 MCDescInfo emitMCDesc(raw_ostream &OS, const FeatureMapTy &FeatureMap);
91 void emitTargetDesc(raw_ostream &OS);
92 void emitHeader(raw_ostream &OS);
93 void emitCtor(raw_ostream &OS, MCDescInfo DescInfo);
94
95 std::pair<unsigned, unsigned>
96 featureKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
97 std::pair<unsigned, unsigned> cpuKeyValues(raw_ostream &OS,
98 const FeatureMapTy &FeatureMap);
99 void formItineraryStageString(const std::string &Names,
100 const Record *ItinData, std::string &ItinString,
101 unsigned &NStages);
102 void formItineraryOperandCycleString(const Record *ItinData,
103 std::string &ItinString,
104 unsigned &NOperandCycles);
105 void formItineraryBypassString(const std::string &Names,
106 const Record *ItinData,
107 std::string &ItinString,
108 unsigned NOperandCycles);
109 void emitStageAndOperandCycleData(
110 raw_ostream &OS, std::vector<std::vector<InstrItinerary>> &ProcItinLists);
111 void emitItineraries(raw_ostream &OS,
112 ArrayRef<std::vector<InstrItinerary>> ProcItinLists);
113 unsigned emitRegisterFileTables(const CodeGenProcModel &ProcModel,
114 raw_ostream &OS);
115 void emitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel,
116 raw_ostream &OS);
117 void emitExtraProcessorInfo(const CodeGenProcModel &ProcModel,
118 raw_ostream &OS);
119 void emitProcessorProp(raw_ostream &OS, const Record *R, StringRef Name,
120 char Separator);
121 void emitProcessorResourceSubUnits(const CodeGenProcModel &ProcModel,
122 raw_ostream &OS);
123 void emitProcessorResources(const CodeGenProcModel &ProcModel,
124 raw_ostream &OS);
125 const Record *findWriteResources(const CodeGenSchedRW &SchedWrite,
126 const CodeGenProcModel &ProcModel);
127 const Record *findReadAdvance(const CodeGenSchedRW &SchedRead,
128 const CodeGenProcModel &ProcModel);
129 void expandProcResources(ConstRecVec &PRVec,
130 std::vector<int64_t> &ReleaseAtCycles,
131 std::vector<int64_t> &AcquireAtCycles,
132 const CodeGenProcModel &ProcModel);
133 void genSchedClassTables(const CodeGenProcModel &ProcModel,
134 SchedClassTables &SchedTables);
135 void emitSchedClassTables(SchedClassTables &SchedTables, raw_ostream &OS);
136 void emitProcessorModels(raw_ostream &OS);
137 void emitSchedModelHelpers(const std::string &ClassName, raw_ostream &OS);
138 void emitSchedModelHelpersImpl(raw_ostream &OS,
139 bool OnlyExpandMCInstPredicates = false);
140 void emitGenMCSubtargetInfo(raw_ostream &OS);
141 void emitMcInstrAnalysisPredicateFunctions(raw_ostream &OS);
142
143 void emitSchedModel(raw_ostream &OS);
144 void emitGetMacroFusions(const std::string &ClassName, raw_ostream &OS);
145 void emitHwModeCheck(const std::string &ClassName, raw_ostream &OS,
146 bool IsMC);
147 void emitInlineFeatures(const std::string &ClassName, raw_ostream &OS,
148 StringRef Behavior);
149 void parseFeaturesFunction(raw_ostream &OS);
150
151public:
152 SubtargetEmitter(const RecordKeeper &R)
153 : TargetFeaturesEmitter(R), TGT(R), SchedModels(TGT.getSchedModels()) {}
154
155 void run(raw_ostream &O) override;
156};
157
158} // end anonymous namespace
159
160/// Emit some information about the SubtargetFeature as calls to a macro so
161/// that they can be used from C++.
162void SubtargetEmitter::emitSubtargetInfoMacroCalls(raw_ostream &OS) {
163 // Undef the GET_SUBTARGETINFO_MACRO macro at the end of the scope since it's
164 // used within the scope.
165 IfDefEmitter IfDefMacro(OS, "GET_SUBTARGETINFO_MACRO", /*LateUndef=*/true);
166
167 std::vector<const Record *> FeatureList =
168 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
169 llvm::sort(C&: FeatureList, Comp: LessRecordFieldFieldName());
170
171 for (const Record *Feature : FeatureList) {
172 const StringRef FieldName = Feature->getValueAsString(FieldName: "FieldName");
173 const StringRef Value = Feature->getValueAsString(FieldName: "Value");
174
175 // Only handle boolean features for now, excluding BitVectors and enums.
176 const bool IsBool = (Value == "false" || Value == "true") &&
177 !StringRef(FieldName).contains(C: '[');
178 if (!IsBool)
179 continue;
180
181 // Some features default to true, with values set to false if enabled.
182 const char *Default = Value == "false" ? "true" : "false";
183
184 // Define the getter with lowercased first char: xxxYyy() { return XxxYyy; }
185 const std::string Getter =
186 FieldName.substr(Start: 0, N: 1).lower() + FieldName.substr(Start: 1).str();
187
188 OS << "GET_SUBTARGETINFO_MACRO(" << FieldName << ", " << Default << ", "
189 << Getter << ")\n";
190 }
191}
192
193//
194// FeatureKeyValues - Emit data of all the subtarget features. Used by the
195// command line.
196//
197std::pair<unsigned, unsigned>
198SubtargetEmitter::featureKeyValues(raw_ostream &OS,
199 const FeatureMapTy &FeatureMap) {
200 std::vector<const Record *> FeatureList =
201 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
202
203 // Remove features with empty name.
204 llvm::erase_if(C&: FeatureList, P: [](const Record *Rec) {
205 return Rec->getValueAsString(FieldName: "Name").empty();
206 });
207 if (FeatureList.empty())
208 return {0, 0};
209
210 // Sort and check duplicate Feature name.
211 sortAndReportDuplicates(Records: FeatureList, ObjectName: "Feature");
212
213 StringToOffsetTable StrTab;
214 // Offsets of CommandLineName and Desc in StrTab.
215 SmallVector<std::pair<unsigned, unsigned>> StrOffs;
216 for (const Record *Feature : FeatureList) {
217 unsigned NameOff =
218 StrTab.GetOrAddStringOffset(Str: Feature->getValueAsString(FieldName: "Name"));
219 unsigned DescOff =
220 StrTab.GetOrAddStringOffset(Str: Feature->getValueAsString(FieldName: "Desc"));
221 StrOffs.emplace_back(Args&: NameOff, Args&: DescOff);
222 }
223
224 // Begin feature table.
225 OS << "// Sorted (by key) array of values for CPU features.\n"
226 << "extern const llvm::SubtargetFeatureKVStorage< " << FeatureList.size()
227 << ", " << (StrTab.size() + 1) << "> " << Target
228 << "FeatureKVStorage = {\n {\n";
229
230 for (auto [Idx, Feature] : enumerate(First&: FeatureList)) {
231 // Next feature
232 StringRef Name = Feature->getName();
233
234 // Emit as { "feature", "description", { featureEnum }, { i1 , i2 , ... , in
235 // } }
236 auto StrOff =
237 "sizeof(SubtargetFeatureKV) * " + Twine(FeatureList.size() - Idx);
238 OS << " { " << StrOff << " + " << StrOffs[Idx].first << ", " << StrOff
239 << " + " << StrOffs[Idx].second << ", " << Target << "::" << Name
240 << ", ";
241
242 ConstRecVec ImpliesList = Feature->getValueAsListOfDefs(FieldName: "Implies");
243
244 printFeatureMask(OS, FeatureList: ImpliesList, FeatureMap);
245
246 OS << " },\n";
247 }
248
249 OS << " },\n";
250 StrTab.EmitString(O&: OS);
251
252 // End feature table.
253 OS << "};\n";
254
255 return {FeatureList.size(), StrTab.size() + 1};
256}
257
258static void checkDuplicateCPUFeatures(StringRef CPUName,
259 ArrayRef<const Record *> Features,
260 ArrayRef<const Record *> TuneFeatures) {
261 // We have made sure each SubtargetFeature Record has a unique name, so we can
262 // simply use pointer sets here.
263 SmallPtrSet<const Record *, 8> FeatureSet, TuneFeatureSet;
264 for (const auto *FeatureRec : Features) {
265 if (!FeatureSet.insert(Ptr: FeatureRec).second)
266 PrintWarning(Msg: "Processor " + CPUName + " contains duplicate feature '" +
267 FeatureRec->getValueAsString(FieldName: "Name") + "'");
268 }
269
270 for (const auto *TuneFeatureRec : TuneFeatures) {
271 if (!TuneFeatureSet.insert(Ptr: TuneFeatureRec).second)
272 PrintWarning(Msg: "Processor " + CPUName +
273 " contains duplicate tune feature '" +
274 TuneFeatureRec->getValueAsString(FieldName: "Name") + "'");
275 if (FeatureSet.contains(Ptr: TuneFeatureRec))
276 PrintWarning(Msg: "Processor " + CPUName + " has '" +
277 TuneFeatureRec->getValueAsString(FieldName: "Name") +
278 "' in both feature and tune feature sets");
279 }
280}
281
282//
283// CPUKeyValues - Emit data of all the subtarget processors. Used by command
284// line.
285//
286std::pair<unsigned, unsigned>
287SubtargetEmitter::cpuKeyValues(raw_ostream &OS,
288 const FeatureMapTy &FeatureMap) {
289 // Gather and sort processor information
290 std::vector<const Record *> ProcessorList =
291 Records.getAllDerivedDefinitions(ClassName: "Processor");
292 llvm::sort(C&: ProcessorList, Comp: LessRecordFieldName());
293
294 // In the string table, include the aliases as well.
295 std::vector<const Record *> ProcessorAliasList =
296 Records.getAllDerivedDefinitionsIfDefined(ClassName: "ProcessorAlias");
297 SmallVector<StringRef> Names;
298 Names.reserve(N: ProcessorList.size() + ProcessorAliasList.size());
299 for (const Record *Processor : ProcessorList)
300 Names.push_back(Elt: Processor->getValueAsString(FieldName: "Name"));
301 for (const Record *Rec : ProcessorAliasList)
302 Names.push_back(Elt: Rec->getValueAsString(FieldName: "Name"));
303 llvm::sort(C&: Names);
304
305 StringToOffsetTable StrTab;
306 for (StringRef Name : Names)
307 StrTab.GetOrAddStringOffset(Str: Name);
308
309 // Note that unlike `FeatureKeyValues`, here we do not need to check for
310 // duplicate processors, since that is already done when the SubtargetEmitter
311 // constructor calls `getSchedModels` to build a `CodeGenSchedModels` object,
312 // which does the duplicate processor check.
313
314 // Begin processor table.
315 OS << "// Sorted (by key) array of values for CPU subtype.\n"
316 << "extern const llvm::SubtargetSubTypeKVStorage< " << ProcessorList.size()
317 << ", " << (StrTab.size() + 1) << "> " << Target
318 << "SubTypeKVStorage = {\n {\n";
319
320 for (const auto &[Idx, Processor] : enumerate(First&: ProcessorList)) {
321 StringRef Name = Processor->getValueAsString(FieldName: "Name");
322 ConstRecVec FeatureList = Processor->getValueAsListOfDefs(FieldName: "Features");
323 ConstRecVec TuneFeatureList =
324 Processor->getValueAsListOfDefs(FieldName: "TuneFeatures");
325
326 // Warn the user if there are duplicate processor features or tune
327 // features.
328 checkDuplicateCPUFeatures(CPUName: Name, Features: FeatureList, TuneFeatures: TuneFeatureList);
329
330 OS << " { sizeof(SubtargetSubTypeKV) * " << (ProcessorList.size() - Idx)
331 << " + " << StrTab.GetOrAddStringOffset(Str: Name) << ", ";
332
333 printFeatureMask(OS, FeatureList, FeatureMap);
334 OS << ", ";
335 printFeatureMask(OS, FeatureList: TuneFeatureList, FeatureMap);
336
337 // Emit the scheduler model index.
338 OS << ", " << SchedModels.getModelIndexForProc(ProcDef: Processor) << " },\n";
339 }
340
341 OS << " },\n";
342 StrTab.EmitString(O&: OS);
343
344 // End processor table.
345 OS << "};\n";
346
347 return {ProcessorList.size(), StrTab.size() + 1};
348}
349
350//
351// FormItineraryStageString - Compose a string containing the stage
352// data initialization for the specified itinerary. N is the number
353// of stages.
354//
355void SubtargetEmitter::formItineraryStageString(const std::string &Name,
356 const Record *ItinData,
357 std::string &ItinString,
358 unsigned &NStages) {
359 // Get states list
360 ConstRecVec StageList = ItinData->getValueAsListOfDefs(FieldName: "Stages");
361
362 // For each stage
363 unsigned N = NStages = StageList.size();
364 for (unsigned I = 0; I < N;) {
365 // Next stage
366 const Record *Stage = StageList[I];
367
368 // Form string as ,{ cycles, u1 | u2 | ... | un, timeinc, kind }
369 int Cycles = Stage->getValueAsInt(FieldName: "Cycles");
370 ItinString += " { " + itostr(X: Cycles) + ", ";
371
372 // Get unit list
373 ConstRecVec UnitList = Stage->getValueAsListOfDefs(FieldName: "Units");
374
375 // For each unit
376 for (unsigned J = 0, M = UnitList.size(); J < M;) {
377 // Add name and bitwise or
378 ItinString += Name + "FU::" + UnitList[J]->getName().str();
379 if (++J < M)
380 ItinString += " | ";
381 }
382
383 int TimeInc = Stage->getValueAsInt(FieldName: "TimeInc");
384 ItinString += ", " + itostr(X: TimeInc);
385
386 int Kind = Stage->getValueAsInt(FieldName: "Kind");
387 ItinString += ", (llvm::InstrStage::ReservationKinds)" + itostr(X: Kind);
388
389 // Close off stage
390 ItinString += " }";
391 if (++I < N)
392 ItinString += ", ";
393 }
394}
395
396//
397// FormItineraryOperandCycleString - Compose a string containing the
398// operand cycle initialization for the specified itinerary. N is the
399// number of operands that has cycles specified.
400//
401void SubtargetEmitter::formItineraryOperandCycleString(
402 const Record *ItinData, std::string &ItinString, unsigned &NOperandCycles) {
403 // Get operand cycle list
404 std::vector<int64_t> OperandCycleList =
405 ItinData->getValueAsListOfInts(FieldName: "OperandCycles");
406
407 // For each operand cycle
408 NOperandCycles = OperandCycleList.size();
409 ListSeparator LS;
410 for (int OCycle : OperandCycleList) {
411 // Next operand cycle
412 ItinString += LS;
413 ItinString += " " + itostr(X: OCycle);
414 }
415}
416
417void SubtargetEmitter::formItineraryBypassString(const std::string &Name,
418 const Record *ItinData,
419 std::string &ItinString,
420 unsigned NOperandCycles) {
421 ConstRecVec BypassList = ItinData->getValueAsListOfDefs(FieldName: "Bypasses");
422 unsigned N = BypassList.size();
423 unsigned I = 0;
424 ListSeparator LS;
425 for (; I < N; ++I) {
426 ItinString += LS;
427 ItinString += Name + "Bypass::" + BypassList[I]->getName().str();
428 }
429 for (; I < NOperandCycles; ++I) {
430 ItinString += LS;
431 ItinString += " 0";
432 }
433}
434
435//
436// EmitStageAndOperandCycleData - Generate unique itinerary stages and operand
437// cycle tables. Create a list of InstrItinerary objects (ProcItinLists) indexed
438// by CodeGenSchedClass::Index.
439//
440void SubtargetEmitter::emitStageAndOperandCycleData(
441 raw_ostream &OS, std::vector<std::vector<InstrItinerary>> &ProcItinLists) {
442 // Multiple processor models may share an itinerary record. Emit it once.
443 SmallPtrSet<const Record *, 8> ItinsDefSet;
444
445 // Emit functional units for all the itineraries.
446 for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) {
447 if (!ItinsDefSet.insert(Ptr: ProcModel.ItinsDef).second)
448 continue;
449
450 ConstRecVec FUs = ProcModel.ItinsDef->getValueAsListOfDefs(FieldName: "FU");
451 if (FUs.empty())
452 continue;
453
454 StringRef Name = ProcModel.ItinsDef->getName();
455 {
456 OS << "\n// Functional units for \"" << Name << "\"\n";
457 NamespaceEmitter FUNamespace(OS, (Name + Twine("FU")).str());
458
459 for (const auto &[Idx, FU] : enumerate(First&: FUs))
460 OS << " const InstrStage::FuncUnits " << FU->getName() << " = 1ULL << "
461 << Idx << ";\n";
462 }
463
464 ConstRecVec BPs = ProcModel.ItinsDef->getValueAsListOfDefs(FieldName: "BP");
465 if (BPs.empty())
466 continue;
467 OS << "\n// Pipeline forwarding paths for itineraries \"" << Name << "\"\n";
468 NamespaceEmitter BypassNamespace(OS, (Name + Twine("Bypass")).str());
469
470 OS << " const unsigned NoBypass = 0;\n";
471 for (const auto &[Idx, BP] : enumerate(First&: BPs))
472 OS << " const unsigned " << BP->getName() << " = 1 << " << Idx << ";\n";
473 }
474
475 // Begin stages table
476 std::string StageTable =
477 "\nextern const llvm::InstrStage " + Target + "Stages[] = {\n";
478 StageTable += " { 0, 0, 0, llvm::InstrStage::Required }, // No itinerary\n";
479
480 // Begin operand cycle table
481 std::string OperandCycleTable =
482 "extern const unsigned " + Target + "OperandCycles[] = {\n";
483 OperandCycleTable += " 0, // No itinerary\n";
484
485 // Begin pipeline bypass table
486 std::string BypassTable =
487 "extern const unsigned " + Target + "ForwardingPaths[] = {\n";
488 BypassTable += " 0, // No itinerary\n";
489
490 // For each Itinerary across all processors, add a unique entry to the stages,
491 // operand cycles, and pipeline bypass tables. Then add the new Itinerary
492 // object with computed offsets to the ProcItinLists result.
493 unsigned StageCount = 1, OperandCycleCount = 1;
494 StringMap<unsigned> ItinStageMap, ItinOperandMap;
495 for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) {
496 // Add process itinerary to the list.
497 std::vector<InstrItinerary> &ItinList = ProcItinLists.emplace_back();
498
499 // If this processor defines no itineraries, then leave the itinerary list
500 // empty.
501 if (!ProcModel.hasItineraries())
502 continue;
503
504 StringRef Name = ProcModel.ItinsDef->getName();
505
506 ItinList.resize(new_size: SchedModels.numInstrSchedClasses());
507 assert(ProcModel.ItinDefList.size() == ItinList.size() && "bad Itins");
508
509 for (unsigned SchedClassIdx = 0, SchedClassEnd = ItinList.size();
510 SchedClassIdx < SchedClassEnd; ++SchedClassIdx) {
511
512 // Next itinerary data
513 const Record *ItinData = ProcModel.ItinDefList[SchedClassIdx];
514
515 // Get string and stage count
516 std::string ItinStageString;
517 unsigned NStages = 0;
518 if (ItinData)
519 formItineraryStageString(Name: Name.str(), ItinData, ItinString&: ItinStageString,
520 NStages);
521
522 // Get string and operand cycle count
523 std::string ItinOperandCycleString;
524 unsigned NOperandCycles = 0;
525 std::string ItinBypassString;
526 if (ItinData) {
527 formItineraryOperandCycleString(ItinData, ItinString&: ItinOperandCycleString,
528 NOperandCycles);
529
530 formItineraryBypassString(Name: Name.str(), ItinData, ItinString&: ItinBypassString,
531 NOperandCycles);
532 }
533
534 // Check to see if stage already exists and create if it doesn't
535 uint16_t FindStage = 0;
536 if (NStages > 0) {
537 FindStage = ItinStageMap[ItinStageString];
538 if (FindStage == 0) {
539 // Emit as { cycles, u1 | u2 | ... | un, timeinc }, // indices
540 StageTable += ItinStageString + ", // " + itostr(X: StageCount);
541 if (NStages > 1)
542 StageTable += "-" + itostr(X: StageCount + NStages - 1);
543 StageTable += "\n";
544 // Record Itin class number.
545 ItinStageMap[ItinStageString] = FindStage = StageCount;
546 StageCount += NStages;
547 }
548 }
549
550 // Check to see if operand cycle already exists and create if it doesn't
551 uint16_t FindOperandCycle = 0;
552 if (NOperandCycles > 0) {
553 std::string ItinOperandString =
554 ItinOperandCycleString + ItinBypassString;
555 FindOperandCycle = ItinOperandMap[ItinOperandString];
556 if (FindOperandCycle == 0) {
557 // Emit as cycle, // index
558 OperandCycleTable += ItinOperandCycleString + ", // ";
559 std::string OperandIdxComment = itostr(X: OperandCycleCount);
560 if (NOperandCycles > 1)
561 OperandIdxComment +=
562 "-" + itostr(X: OperandCycleCount + NOperandCycles - 1);
563 OperandCycleTable += OperandIdxComment + "\n";
564 // Record Itin class number.
565 ItinOperandMap[ItinOperandCycleString] = FindOperandCycle =
566 OperandCycleCount;
567 // Emit as bypass, // index
568 BypassTable += ItinBypassString + ", // " + OperandIdxComment + "\n";
569 OperandCycleCount += NOperandCycles;
570 }
571 }
572
573 // Set up itinerary as location and location + stage count
574 int16_t NumUOps = ItinData ? ItinData->getValueAsInt(FieldName: "NumMicroOps") : 0;
575 InstrItinerary Intinerary = {
576 .NumMicroOps: NumUOps,
577 .FirstStage: FindStage,
578 .LastStage: uint16_t(FindStage + NStages),
579 .FirstOperandCycle: FindOperandCycle,
580 .LastOperandCycle: uint16_t(FindOperandCycle + NOperandCycles),
581 };
582
583 // Inject - empty slots will be 0, 0
584 ItinList[SchedClassIdx] = Intinerary;
585 }
586 }
587
588 // Closing stage
589 StageTable += " { 0, 0, 0, llvm::InstrStage::Required } // End stages\n";
590 StageTable += "};\n";
591
592 // Closing operand cycles
593 OperandCycleTable += " 0 // End operand cycles\n";
594 OperandCycleTable += "};\n";
595
596 BypassTable += " 0 // End bypass tables\n";
597 BypassTable += "};\n";
598
599 // Emit tables.
600 OS << StageTable;
601 OS << OperandCycleTable;
602 OS << BypassTable;
603}
604
605//
606// EmitProcessorData - Generate data for processor itineraries that were
607// computed during EmitStageAndOperandCycleData(). ProcItinLists lists all
608// Itineraries for each processor. The Itinerary lists are indexed on
609// CodeGenSchedClass::Index.
610//
611void SubtargetEmitter::emitItineraries(
612 raw_ostream &OS, ArrayRef<std::vector<InstrItinerary>> ProcItinLists) {
613 // Multiple processor models may share an itinerary record. Emit it once.
614 SmallPtrSet<const Record *, 8> ItinsDefSet;
615
616 for (const auto &[Proc, ItinList] :
617 zip_equal(t: SchedModels.procModels(), u&: ProcItinLists)) {
618 const Record *ItinsDef = Proc.ItinsDef;
619 if (!ItinsDefSet.insert(Ptr: ItinsDef).second)
620 continue;
621
622 // Empty itineraries aren't referenced anywhere in the tablegen output
623 // so don't emit them.
624 if (ItinList.empty())
625 continue;
626
627 // Begin processor itinerary table
628 OS << "\n";
629 OS << "static constexpr llvm::InstrItinerary " << ItinsDef->getName()
630 << "[] = {\n";
631
632 ArrayRef<CodeGenSchedClass> ItinSchedClasses =
633 SchedModels.schedClasses().take_front(N: ItinList.size());
634
635 // For each itinerary class in CodeGenSchedClass::Index order.
636 for (const auto &[Idx, Intinerary, SchedClass] :
637 enumerate(First: ItinList, Rest&: ItinSchedClasses)) {
638 // Emit Itinerary in the form of
639 // { NumMicroOps, FirstStage, LastStage, FirstOperandCycle,
640 // LastOperandCycle } // index class name
641 OS << " { " << Intinerary.NumMicroOps << ", " << Intinerary.FirstStage
642 << ", " << Intinerary.LastStage << ", " << Intinerary.FirstOperandCycle
643 << ", " << Intinerary.LastOperandCycle << " }" << ", // " << Idx << " "
644 << SchedClass.Name << "\n";
645 }
646 // End processor itinerary table
647 OS << " { 0, uint16_t(~0U), uint16_t(~0U), uint16_t(~0U), uint16_t(~0U) }"
648 "// end marker\n";
649 OS << "};\n";
650 }
651}
652
653// Emit either the value defined in the TableGen Record, or the default
654// value defined in the C++ header. The Record is null if the processor does not
655// define a model.
656void SubtargetEmitter::emitProcessorProp(raw_ostream &OS, const Record *R,
657 StringRef Name, char Separator) {
658 OS << " ";
659 int V = R ? R->getValueAsInt(FieldName: Name) : -1;
660 if (V >= 0)
661 OS << V << Separator << " // " << Name;
662 else
663 OS << "MCSchedModel::Default" << Name << Separator;
664 OS << '\n';
665}
666
667void SubtargetEmitter::emitProcessorResourceSubUnits(
668 const CodeGenProcModel &ProcModel, raw_ostream &OS) {
669 OS << "\nstatic const unsigned " << ProcModel.ModelName
670 << "ProcResourceSubUnits[] = {\n"
671 << " 0, // Invalid\n";
672
673 for (unsigned I = 0, E = ProcModel.ProcResourceDefs.size(); I < E; ++I) {
674 const Record *PRDef = ProcModel.ProcResourceDefs[I];
675 if (!PRDef->isSubClassOf(Name: "ProcResGroup"))
676 continue;
677 for (const Record *RUDef : PRDef->getValueAsListOfDefs(FieldName: "Resources")) {
678 const Record *RU =
679 SchedModels.findProcResUnits(ProcResKind: RUDef, PM: ProcModel, Loc: PRDef->getLoc());
680 for (unsigned J = 0; J < RU->getValueAsInt(FieldName: "NumUnits"); ++J) {
681 OS << " " << ProcModel.getProcResourceIdx(PRDef: RU) << ", ";
682 }
683 }
684 OS << " // " << PRDef->getName() << "\n";
685 }
686 OS << "};\n";
687}
688
689static void emitRetireControlUnitInfo(const CodeGenProcModel &ProcModel,
690 raw_ostream &OS) {
691 int64_t ReorderBufferSize = 0, MaxRetirePerCycle = 0;
692 if (const Record *RCU = ProcModel.RetireControlUnit) {
693 ReorderBufferSize =
694 std::max(a: ReorderBufferSize, b: RCU->getValueAsInt(FieldName: "ReorderBufferSize"));
695 MaxRetirePerCycle =
696 std::max(a: MaxRetirePerCycle, b: RCU->getValueAsInt(FieldName: "MaxRetirePerCycle"));
697 }
698
699 OS << ReorderBufferSize << ", // ReorderBufferSize\n ";
700 OS << MaxRetirePerCycle << ", // MaxRetirePerCycle\n ";
701}
702
703static void emitRegisterFileInfo(const CodeGenProcModel &ProcModel,
704 unsigned NumRegisterFiles,
705 unsigned NumCostEntries, raw_ostream &OS) {
706 if (NumRegisterFiles)
707 OS << ProcModel.ModelName << "RegisterFiles,\n " << (1 + NumRegisterFiles);
708 else
709 OS << "nullptr,\n 0";
710
711 OS << ", // Number of register files.\n ";
712 if (NumCostEntries)
713 OS << ProcModel.ModelName << "RegisterCosts,\n ";
714 else
715 OS << "nullptr,\n ";
716 OS << NumCostEntries << ", // Number of register cost entries.\n";
717}
718
719unsigned
720SubtargetEmitter::emitRegisterFileTables(const CodeGenProcModel &ProcModel,
721 raw_ostream &OS) {
722 if (llvm::all_of(Range: ProcModel.RegisterFiles, P: [](const CodeGenRegisterFile &RF) {
723 return RF.hasDefaultCosts();
724 }))
725 return 0;
726
727 // Print the RegisterCost table first.
728 OS << "\n// {RegisterClassID, Register Cost, AllowMoveElimination }\n";
729 OS << "static const llvm::MCRegisterCostEntry " << ProcModel.ModelName
730 << "RegisterCosts"
731 << "[] = {\n";
732
733 for (const CodeGenRegisterFile &RF : ProcModel.RegisterFiles) {
734 // Skip register files with a default cost table.
735 if (RF.hasDefaultCosts())
736 continue;
737 // Add entries to the cost table.
738 for (const CodeGenRegisterCost &RC : RF.Costs) {
739 OS << " { ";
740 const Record *Rec = RC.RCDef;
741 if (Rec->getValue(Name: "Namespace"))
742 OS << Rec->getValueAsString(FieldName: "Namespace") << "::";
743 OS << Rec->getName() << "RegClassID, " << RC.Cost << ", "
744 << RC.AllowMoveElimination << "},\n";
745 }
746 }
747 OS << "};\n";
748
749 // Now generate a table with register file info.
750 OS << "\n // {Name, #PhysRegs, #CostEntries, IndexToCostTbl, "
751 << "MaxMovesEliminatedPerCycle, AllowZeroMoveEliminationOnly }\n";
752 OS << "static const llvm::MCRegisterFileDesc " << ProcModel.ModelName
753 << "RegisterFiles"
754 << "[] = {\n"
755 << " { \"InvalidRegisterFile\", 0, 0, 0, 0, 0 },\n";
756 unsigned CostTblIndex = 0;
757
758 for (const CodeGenRegisterFile &RD : ProcModel.RegisterFiles) {
759 OS << " { ";
760 OS << '"' << RD.Name << '"' << ", " << RD.NumPhysRegs << ", ";
761 unsigned NumCostEntries = RD.Costs.size();
762 OS << NumCostEntries << ", " << CostTblIndex << ", "
763 << RD.MaxMovesEliminatedPerCycle << ", "
764 << RD.AllowZeroMoveEliminationOnly << "},\n";
765 CostTblIndex += NumCostEntries;
766 }
767 OS << "};\n";
768
769 return CostTblIndex;
770}
771
772void SubtargetEmitter::emitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel,
773 raw_ostream &OS) {
774 unsigned QueueID = 0;
775 if (ProcModel.LoadQueue) {
776 const Record *Queue = ProcModel.LoadQueue->getValueAsDef(FieldName: "QueueDescriptor");
777 QueueID = 1 + std::distance(first: ProcModel.ProcResourceDefs.begin(),
778 last: find(Range: ProcModel.ProcResourceDefs, Val: Queue));
779 }
780 OS << " " << QueueID << ", // Resource Descriptor for the Load Queue\n";
781
782 QueueID = 0;
783 if (ProcModel.StoreQueue) {
784 const Record *Queue =
785 ProcModel.StoreQueue->getValueAsDef(FieldName: "QueueDescriptor");
786 QueueID = 1 + std::distance(first: ProcModel.ProcResourceDefs.begin(),
787 last: find(Range: ProcModel.ProcResourceDefs, Val: Queue));
788 }
789 OS << " " << QueueID << ", // Resource Descriptor for the Store Queue\n";
790}
791
792void SubtargetEmitter::emitExtraProcessorInfo(const CodeGenProcModel &ProcModel,
793 raw_ostream &OS) {
794 // Generate a table of register file descriptors (one entry per each user
795 // defined register file), and a table of register costs.
796 unsigned NumCostEntries = emitRegisterFileTables(ProcModel, OS);
797
798 // Now generate a table for the extra processor info.
799 OS << "\nstatic const llvm::MCExtraProcessorInfo " << ProcModel.ModelName
800 << "ExtraInfo = {\n ";
801
802 // Add information related to the retire control unit.
803 emitRetireControlUnitInfo(ProcModel, OS);
804
805 // Add information related to the register files (i.e. where to find register
806 // file descriptors and register costs).
807 emitRegisterFileInfo(ProcModel, NumRegisterFiles: ProcModel.RegisterFiles.size(),
808 NumCostEntries, OS);
809
810 // Add information about load/store queues.
811 emitLoadStoreQueueInfo(ProcModel, OS);
812
813 OS << "};\n";
814}
815
816void SubtargetEmitter::emitProcessorResources(const CodeGenProcModel &ProcModel,
817 raw_ostream &OS) {
818 emitProcessorResourceSubUnits(ProcModel, OS);
819
820 OS << "\n// {Name, NumUnits, SuperIdx, BufferSize, SubUnitsIdxBegin}\n";
821 OS << "static const llvm::MCProcResourceDesc " << ProcModel.ModelName
822 << "ProcResources"
823 << "[] = {\n"
824 << " {\"InvalidUnit\", 0, 0, 0, 0},\n";
825
826 unsigned SubUnitsOffset = 1;
827 for (unsigned I = 0, E = ProcModel.ProcResourceDefs.size(); I < E; ++I) {
828 const Record *PRDef = ProcModel.ProcResourceDefs[I];
829
830 const Record *SuperDef = nullptr;
831 unsigned SuperIdx = 0;
832 unsigned NumUnits = 0;
833 const unsigned SubUnitsBeginOffset = SubUnitsOffset;
834 int BufferSize = PRDef->getValueAsInt(FieldName: "BufferSize");
835 if (PRDef->isSubClassOf(Name: "ProcResGroup")) {
836 for (const Record *RU : PRDef->getValueAsListOfDefs(FieldName: "Resources")) {
837 NumUnits += RU->getValueAsInt(FieldName: "NumUnits");
838 SubUnitsOffset += RU->getValueAsInt(FieldName: "NumUnits");
839 }
840 } else {
841 // Find the SuperIdx
842 if (PRDef->getValueInit(FieldName: "Super")->isComplete()) {
843 SuperDef = SchedModels.findProcResUnits(ProcResKind: PRDef->getValueAsDef(FieldName: "Super"),
844 PM: ProcModel, Loc: PRDef->getLoc());
845 SuperIdx = ProcModel.getProcResourceIdx(PRDef: SuperDef);
846 }
847 NumUnits = PRDef->getValueAsInt(FieldName: "NumUnits");
848 }
849 // Emit the ProcResourceDesc
850 OS << " {\"" << PRDef->getName() << "\", ";
851 if (PRDef->getName().size() < 15)
852 OS.indent(NumSpaces: 15 - PRDef->getName().size());
853 OS << NumUnits << ", " << SuperIdx << ", " << BufferSize << ", ";
854 if (SubUnitsBeginOffset != SubUnitsOffset) {
855 OS << ProcModel.ModelName << "ProcResourceSubUnits + "
856 << SubUnitsBeginOffset;
857 } else {
858 OS << "nullptr";
859 }
860 OS << "}, // #" << I + 1;
861 if (SuperDef)
862 OS << ", Super=" << SuperDef->getName();
863 OS << "\n";
864 }
865 OS << "};\n";
866}
867
868// Find the WriteRes Record that defines processor resources for this
869// SchedWrite.
870const Record *
871SubtargetEmitter::findWriteResources(const CodeGenSchedRW &SchedWrite,
872 const CodeGenProcModel &ProcModel) {
873
874 // Check if the SchedWrite is already subtarget-specific and directly
875 // specifies a set of processor resources.
876 if (SchedWrite.TheDef->isSubClassOf(Name: "SchedWriteRes"))
877 return SchedWrite.TheDef;
878
879 const Record *AliasDef = nullptr;
880 for (const Record *A : SchedWrite.Aliases) {
881 const CodeGenSchedRW &AliasRW =
882 SchedModels.getSchedRW(Def: A->getValueAsDef(FieldName: "AliasRW"));
883 if (AliasRW.TheDef->getValueInit(FieldName: "SchedModel")->isComplete()) {
884 const Record *ModelDef = AliasRW.TheDef->getValueAsDef(FieldName: "SchedModel");
885 if (&SchedModels.getProcModel(ModelDef) != &ProcModel)
886 continue;
887 }
888 if (AliasDef)
889 PrintFatalError(ErrorLoc: AliasRW.TheDef->getLoc(),
890 Msg: "Multiple aliases "
891 "defined for processor " +
892 ProcModel.ModelName +
893 " Ensure only one SchedAlias exists per RW.");
894 AliasDef = AliasRW.TheDef;
895 }
896 if (AliasDef && AliasDef->isSubClassOf(Name: "SchedWriteRes"))
897 return AliasDef;
898
899 // Check this processor's list of write resources.
900 const Record *ResDef = nullptr;
901
902 auto I = ProcModel.WriteResMap.find(Val: SchedWrite.TheDef);
903 if (I != ProcModel.WriteResMap.end())
904 ResDef = I->second;
905
906 if (AliasDef) {
907 I = ProcModel.WriteResMap.find(Val: AliasDef);
908 if (I != ProcModel.WriteResMap.end()) {
909 if (ResDef)
910 PrintFatalError(ErrorLoc: I->second->getLoc(),
911 Msg: "Resources are defined for both SchedWrite and its "
912 "alias on processor " +
913 ProcModel.ModelName);
914 ResDef = I->second;
915 }
916 }
917
918 // TODO: If ProcModel has a base model (previous generation processor),
919 // then call FindWriteResources recursively with that model here.
920 if (!ResDef) {
921 PrintFatalError(ErrorLoc: ProcModel.ModelDef->getLoc(),
922 Msg: Twine("Processor does not define resources for ") +
923 SchedWrite.TheDef->getName());
924 }
925 return ResDef;
926}
927
928/// Find the ReadAdvance record for the given SchedRead on this processor or
929/// return NULL.
930const Record *
931SubtargetEmitter::findReadAdvance(const CodeGenSchedRW &SchedRead,
932 const CodeGenProcModel &ProcModel) {
933 // Check for SchedReads that directly specify a ReadAdvance.
934 if (SchedRead.TheDef->isSubClassOf(Name: "SchedReadAdvance"))
935 return SchedRead.TheDef;
936
937 // Check this processor's list of aliases for SchedRead.
938 const Record *AliasDef = nullptr;
939 for (const Record *A : SchedRead.Aliases) {
940 const CodeGenSchedRW &AliasRW =
941 SchedModels.getSchedRW(Def: A->getValueAsDef(FieldName: "AliasRW"));
942 if (AliasRW.TheDef->getValueInit(FieldName: "SchedModel")->isComplete()) {
943 const Record *ModelDef = AliasRW.TheDef->getValueAsDef(FieldName: "SchedModel");
944 if (&SchedModels.getProcModel(ModelDef) != &ProcModel)
945 continue;
946 }
947 if (AliasDef)
948 PrintFatalError(ErrorLoc: AliasRW.TheDef->getLoc(),
949 Msg: "Multiple aliases "
950 "defined for processor " +
951 ProcModel.ModelName +
952 " Ensure only one SchedAlias exists per RW.");
953 AliasDef = AliasRW.TheDef;
954 }
955 if (AliasDef && AliasDef->isSubClassOf(Name: "SchedReadAdvance"))
956 return AliasDef;
957
958 // Check this processor's ReadAdvanceList.
959 const Record *ResDef = nullptr;
960
961 auto I = ProcModel.ReadAdvanceMap.find(Val: SchedRead.TheDef);
962 if (I != ProcModel.ReadAdvanceMap.end())
963 ResDef = I->second;
964
965 if (AliasDef) {
966 I = ProcModel.ReadAdvanceMap.find(Val: AliasDef);
967 if (I != ProcModel.ReadAdvanceMap.end()) {
968 if (ResDef)
969 PrintFatalError(
970 ErrorLoc: I->second->getLoc(),
971 Msg: "Resources are defined for both SchedRead and its alias on "
972 "processor " +
973 ProcModel.ModelName);
974 ResDef = I->second;
975 }
976 }
977
978 // TODO: If ProcModel has a base model (previous generation processor),
979 // then call FindReadAdvance recursively with that model here.
980 if (!ResDef && SchedRead.TheDef->getName() != "ReadDefault") {
981 PrintFatalError(ErrorLoc: ProcModel.ModelDef->getLoc(),
982 Msg: Twine("Processor does not define resources for ") +
983 SchedRead.TheDef->getName());
984 }
985 return ResDef;
986}
987
988// Expand an explicit list of processor resources into a full list of implied
989// resource groups and super resources that cover them.
990void SubtargetEmitter::expandProcResources(
991 ConstRecVec &PRVec, std::vector<int64_t> &ReleaseAtCycles,
992 std::vector<int64_t> &AcquireAtCycles, const CodeGenProcModel &PM) {
993 assert(PRVec.size() == ReleaseAtCycles.size() && "failed precondition");
994 for (unsigned I = 0, E = PRVec.size(); I != E; ++I) {
995 const Record *PRDef = PRVec[I];
996 ConstRecVec SubResources;
997 if (PRDef->isSubClassOf(Name: "ProcResGroup")) {
998 SubResources = PRDef->getValueAsListOfDefs(FieldName: "Resources");
999 } else {
1000 SubResources.push_back(x: PRDef);
1001 PRDef = SchedModels.findProcResUnits(ProcResKind: PRDef, PM, Loc: PRDef->getLoc());
1002 for (const Record *SubDef = PRDef;
1003 SubDef->getValueInit(FieldName: "Super")->isComplete();) {
1004 if (SubDef->isSubClassOf(Name: "ProcResGroup")) {
1005 // Disallow this for simplicitly.
1006 PrintFatalError(ErrorLoc: SubDef->getLoc(), Msg: "Processor resource group "
1007 " cannot be a super resources.");
1008 }
1009 const Record *SuperDef = SchedModels.findProcResUnits(
1010 ProcResKind: SubDef->getValueAsDef(FieldName: "Super"), PM, Loc: SubDef->getLoc());
1011 PRVec.push_back(x: SuperDef);
1012 ReleaseAtCycles.push_back(x: ReleaseAtCycles[I]);
1013 AcquireAtCycles.push_back(x: AcquireAtCycles[I]);
1014 SubDef = SuperDef;
1015 }
1016 }
1017 for (const Record *PR : PM.ProcResourceDefs) {
1018 if (PR == PRDef || !PR->isSubClassOf(Name: "ProcResGroup"))
1019 continue;
1020 ConstRecVec SuperResources = PR->getValueAsListOfDefs(FieldName: "Resources");
1021 bool AllContained =
1022 all_of(Range&: SubResources, P: [SuperResources](const Record *SubResource) {
1023 return is_contained(Range: SuperResources, Element: SubResource);
1024 });
1025 if (AllContained) {
1026 PRVec.push_back(x: PR);
1027 ReleaseAtCycles.push_back(x: ReleaseAtCycles[I]);
1028 AcquireAtCycles.push_back(x: AcquireAtCycles[I]);
1029 }
1030 }
1031 }
1032}
1033
1034// Generate the SchedClass table for this processor and update global
1035// tables. Must be called for each processor in order.
1036void SubtargetEmitter::genSchedClassTables(const CodeGenProcModel &ProcModel,
1037 SchedClassTables &SchedTables) {
1038 std::vector<MCSchedClassDesc> &SCTab =
1039 SchedTables.ProcSchedClasses.emplace_back();
1040 if (!ProcModel.hasInstrSchedModel())
1041 return;
1042
1043 LLVM_DEBUG(dbgs() << "\n+++ SCHED CLASSES (GenSchedClassTables) +++\n");
1044 for (const CodeGenSchedClass &SC : SchedModels.schedClasses()) {
1045 LLVM_DEBUG(SC.dump(&SchedModels));
1046
1047 MCSchedClassDesc &SCDesc = SCTab.emplace_back();
1048 // SCDesc.Name is guarded by NDEBUG
1049 SCDesc.NumMicroOps = 0;
1050 SCDesc.BeginGroup = false;
1051 SCDesc.EndGroup = false;
1052 SCDesc.RetireOOO = false;
1053 SCDesc.WriteProcResIdx = 0;
1054 SCDesc.WriteLatencyIdx = 0;
1055 SCDesc.ReadAdvanceIdx = 0;
1056
1057 // A Variant SchedClass has no resources of its own.
1058 bool HasVariants = false;
1059 for (const CodeGenSchedTransition &CGT : SC.Transitions) {
1060 if (CGT.ProcIndex == ProcModel.Index) {
1061 HasVariants = true;
1062 break;
1063 }
1064 }
1065 if (HasVariants) {
1066 SCDesc.NumMicroOps = MCSchedClassDesc::VariantNumMicroOps;
1067 continue;
1068 }
1069
1070 // Determine if the SchedClass is actually reachable on this processor. If
1071 // not don't try to locate the processor resources, it will fail.
1072 // If ProcIndices contains 0, this class applies to all processors.
1073 assert(!SC.ProcIndices.empty() && "expect at least one procidx");
1074 if (SC.ProcIndices[0] != 0) {
1075 if (!is_contained(Range: SC.ProcIndices, Element: ProcModel.Index))
1076 continue;
1077 }
1078 IdxVec Writes = SC.Writes;
1079 IdxVec Reads = SC.Reads;
1080 if (!SC.InstRWs.empty()) {
1081 // This class has a default ReadWrite list which can be overridden by
1082 // InstRW definitions.
1083 const Record *RWDef = nullptr;
1084 for (const Record *RW : SC.InstRWs) {
1085 const Record *RWModelDef = RW->getValueAsDef(FieldName: "SchedModel");
1086 if (&ProcModel == &SchedModels.getProcModel(ModelDef: RWModelDef)) {
1087 RWDef = RW;
1088 break;
1089 }
1090 }
1091 if (RWDef) {
1092 Writes.clear();
1093 Reads.clear();
1094 SchedModels.findRWs(RWDefs: RWDef->getValueAsListOfDefs(FieldName: "OperandReadWrites"),
1095 Writes, Reads);
1096 }
1097 }
1098 if (Writes.empty()) {
1099 // Check this processor's itinerary class resources.
1100 for (const Record *I : ProcModel.ItinRWDefs) {
1101 ConstRecVec Matched = I->getValueAsListOfDefs(FieldName: "MatchedItinClasses");
1102 if (is_contained(Range&: Matched, Element: SC.ItinClassDef)) {
1103 SchedModels.findRWs(RWDefs: I->getValueAsListOfDefs(FieldName: "OperandReadWrites"),
1104 Writes, Reads);
1105 break;
1106 }
1107 }
1108 if (Writes.empty()) {
1109 LLVM_DEBUG(dbgs() << ProcModel.ModelName
1110 << " does not have resources for class " << SC.Name
1111 << '\n');
1112 SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps;
1113 }
1114 }
1115 // Sum resources across all operand writes.
1116 std::vector<MCWriteProcResEntry> WriteProcResources;
1117 std::vector<MCWriteLatencyEntry> WriteLatencies;
1118 std::vector<std::string> WriterNames;
1119 std::vector<MCReadAdvanceEntry> ReadAdvanceEntries;
1120 for (unsigned W : Writes) {
1121 IdxVec WriteSeq;
1122 SchedModels.expandRWSeqForProc(RWIdx: W, RWSeq&: WriteSeq, /*IsRead=*/false, ProcModel);
1123
1124 // For each operand, create a latency entry.
1125 MCWriteLatencyEntry WLEntry;
1126 WLEntry.Cycles = 0;
1127 unsigned WriteID = WriteSeq.back();
1128 WriterNames.push_back(x: SchedModels.getSchedWrite(Idx: WriteID).Name);
1129 // If this Write is not referenced by a ReadAdvance, don't distinguish it
1130 // from other WriteLatency entries.
1131 if (!ProcModel.hasReadOfWrite(WriteDef: SchedModels.getSchedWrite(Idx: WriteID).TheDef))
1132 WriteID = 0;
1133 WLEntry.WriteResourceID = WriteID;
1134
1135 for (unsigned WS : WriteSeq) {
1136 const Record *WriteRes =
1137 findWriteResources(SchedWrite: SchedModels.getSchedWrite(Idx: WS), ProcModel);
1138
1139 // Mark the parent class as invalid for unsupported write types.
1140 if (WriteRes->getValueAsBit(FieldName: "Unsupported")) {
1141 SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps;
1142 break;
1143 }
1144 WLEntry.Cycles += WriteRes->getValueAsInt(FieldName: "Latency");
1145 SCDesc.NumMicroOps += WriteRes->getValueAsInt(FieldName: "NumMicroOps");
1146 SCDesc.BeginGroup |= WriteRes->getValueAsBit(FieldName: "BeginGroup");
1147 SCDesc.EndGroup |= WriteRes->getValueAsBit(FieldName: "EndGroup");
1148 SCDesc.BeginGroup |= WriteRes->getValueAsBit(FieldName: "SingleIssue");
1149 SCDesc.EndGroup |= WriteRes->getValueAsBit(FieldName: "SingleIssue");
1150 SCDesc.RetireOOO |= WriteRes->getValueAsBit(FieldName: "RetireOOO");
1151
1152 // Create an entry for each ProcResource listed in WriteRes.
1153 ConstRecVec PRVec = WriteRes->getValueAsListOfDefs(FieldName: "ProcResources");
1154 std::vector<int64_t> ReleaseAtCycles =
1155 WriteRes->getValueAsListOfInts(FieldName: "ReleaseAtCycles");
1156
1157 std::vector<int64_t> AcquireAtCycles =
1158 WriteRes->getValueAsListOfInts(FieldName: "AcquireAtCycles");
1159
1160 // Check consistency of the two vectors carrying the start and
1161 // stop cycles of the resources.
1162 if (!ReleaseAtCycles.empty() &&
1163 ReleaseAtCycles.size() != PRVec.size()) {
1164 // If ReleaseAtCycles is provided, check consistency.
1165 PrintFatalError(
1166 ErrorLoc: WriteRes->getLoc(),
1167 Msg: Twine("Inconsistent release at cycles: size(ReleaseAtCycles) != "
1168 "size(ProcResources): ")
1169 .concat(Suffix: Twine(PRVec.size()))
1170 .concat(Suffix: " vs ")
1171 .concat(Suffix: Twine(ReleaseAtCycles.size())));
1172 }
1173
1174 if (!AcquireAtCycles.empty() &&
1175 AcquireAtCycles.size() != PRVec.size()) {
1176 PrintFatalError(
1177 ErrorLoc: WriteRes->getLoc(),
1178 Msg: Twine("Inconsistent resource cycles: size(AcquireAtCycles) != "
1179 "size(ProcResources): ")
1180 .concat(Suffix: Twine(AcquireAtCycles.size()))
1181 .concat(Suffix: " vs ")
1182 .concat(Suffix: Twine(PRVec.size())));
1183 }
1184
1185 if (ReleaseAtCycles.empty()) {
1186 // If ReleaseAtCycles is not provided, default to one cycle
1187 // per resource.
1188 ReleaseAtCycles.resize(new_size: PRVec.size(), x: 1);
1189 }
1190
1191 if (AcquireAtCycles.empty()) {
1192 // If AcquireAtCycles is not provided, reserve the resource
1193 // starting from cycle 0.
1194 AcquireAtCycles.resize(new_size: PRVec.size(), x: 0);
1195 }
1196
1197 assert(AcquireAtCycles.size() == ReleaseAtCycles.size());
1198
1199 expandProcResources(PRVec, ReleaseAtCycles, AcquireAtCycles, PM: ProcModel);
1200 assert(AcquireAtCycles.size() == ReleaseAtCycles.size());
1201
1202 for (unsigned PRIdx = 0, PREnd = PRVec.size(); PRIdx != PREnd;
1203 ++PRIdx) {
1204 MCWriteProcResEntry WPREntry;
1205 WPREntry.ProcResourceIdx = ProcModel.getProcResourceIdx(PRDef: PRVec[PRIdx]);
1206 assert(WPREntry.ProcResourceIdx && "Bad ProcResourceIdx");
1207 WPREntry.ReleaseAtCycle = ReleaseAtCycles[PRIdx];
1208 WPREntry.AcquireAtCycle = AcquireAtCycles[PRIdx];
1209 if (AcquireAtCycles[PRIdx] > ReleaseAtCycles[PRIdx]) {
1210 PrintFatalError(
1211 ErrorLoc: WriteRes->getLoc(),
1212 Msg: Twine("Inconsistent resource cycles: AcquireAtCycles "
1213 "<= ReleaseAtCycles must hold."));
1214 }
1215 if (AcquireAtCycles[PRIdx] < 0) {
1216 PrintFatalError(ErrorLoc: WriteRes->getLoc(),
1217 Msg: Twine("Invalid value: AcquireAtCycle "
1218 "must be a non-negative value."));
1219 }
1220 // If this resource is already used in this sequence, add the current
1221 // entry's cycles so that the same resource appears to be used
1222 // serially, rather than multiple parallel uses. This is important for
1223 // in-order machine where the resource consumption is a hazard.
1224 unsigned WPRIdx = 0, WPREnd = WriteProcResources.size();
1225 for (; WPRIdx != WPREnd; ++WPRIdx) {
1226 if (WriteProcResources[WPRIdx].ProcResourceIdx ==
1227 WPREntry.ProcResourceIdx) {
1228 // TODO: multiple use of the same resources would
1229 // require either 1. thinking of how to handle multiple
1230 // intervals for the same resource in
1231 // `<Target>WriteProcResTable` (see
1232 // `SubtargetEmitter::EmitSchedClassTables`), or
1233 // 2. thinking how to merge multiple intervals into a
1234 // single interval.
1235 assert(WPREntry.AcquireAtCycle == 0 &&
1236 "multiple use ofthe same resource is not yet handled");
1237 WriteProcResources[WPRIdx].ReleaseAtCycle +=
1238 WPREntry.ReleaseAtCycle;
1239 break;
1240 }
1241 }
1242 if (WPRIdx == WPREnd)
1243 WriteProcResources.push_back(x: WPREntry);
1244 }
1245 }
1246 WriteLatencies.push_back(x: WLEntry);
1247 }
1248 // Create an entry for each operand Read in this SchedClass.
1249 // Entries must be sorted first by UseIdx then by WriteResourceID.
1250 for (unsigned UseIdx = 0, EndIdx = Reads.size(); UseIdx != EndIdx;
1251 ++UseIdx) {
1252 const Record *ReadAdvance =
1253 findReadAdvance(SchedRead: SchedModels.getSchedRead(Idx: Reads[UseIdx]), ProcModel);
1254 if (!ReadAdvance)
1255 continue;
1256
1257 // Mark the parent class as invalid for unsupported write types.
1258 if (ReadAdvance->getValueAsBit(FieldName: "Unsupported")) {
1259 SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps;
1260 break;
1261 }
1262 ConstRecVec ValidWrites =
1263 ReadAdvance->getValueAsListOfDefs(FieldName: "ValidWrites");
1264 std::vector<int64_t> CycleTunables =
1265 ReadAdvance->getValueAsListOfInts(FieldName: "CycleTunables");
1266 std::vector<std::pair<unsigned, int>> WriteIDs;
1267 assert(CycleTunables.size() <= ValidWrites.size() && "Bad ReadAdvance");
1268 CycleTunables.resize(new_size: ValidWrites.size(), x: 0);
1269 if (ValidWrites.empty())
1270 WriteIDs.emplace_back(args: 0, args: 0);
1271 else {
1272 for (const auto [VW, CT] : zip_equal(t&: ValidWrites, u&: CycleTunables)) {
1273 unsigned WriteID = SchedModels.getSchedRWIdx(Def: VW, /*IsRead=*/false);
1274 assert(WriteID != 0 &&
1275 "Expected a valid SchedRW in the list of ValidWrites");
1276 WriteIDs.emplace_back(args&: WriteID, args&: CT);
1277 }
1278 }
1279 llvm::sort(C&: WriteIDs);
1280 for (const auto &[W, T] : WriteIDs) {
1281 MCReadAdvanceEntry &RAEntry = ReadAdvanceEntries.emplace_back();
1282 RAEntry.UseIdx = UseIdx;
1283 RAEntry.WriteResourceID = W;
1284 RAEntry.Cycles = ReadAdvance->getValueAsInt(FieldName: "Cycles") + T;
1285 }
1286 }
1287 if (SCDesc.NumMicroOps == MCSchedClassDesc::InvalidNumMicroOps) {
1288 WriteProcResources.clear();
1289 WriteLatencies.clear();
1290 ReadAdvanceEntries.clear();
1291 }
1292 // Add the information for this SchedClass to the global tables using basic
1293 // compression.
1294 //
1295 // WritePrecRes entries are sorted by ProcResIdx.
1296 llvm::sort(C&: WriteProcResources, Comp: LessWriteProcResources());
1297
1298 SchedTables.MaxWriteProcResEntries =
1299 std::max(a: SchedTables.MaxWriteProcResEntries, b: WriteProcResources.size());
1300 SCDesc.NumWriteProcResEntries = WriteProcResources.size();
1301 std::vector<MCWriteProcResEntry>::iterator WPRPos =
1302 std::search(first1: SchedTables.WriteProcResources.begin(),
1303 last1: SchedTables.WriteProcResources.end(),
1304 first2: WriteProcResources.begin(), last2: WriteProcResources.end());
1305 if (WPRPos != SchedTables.WriteProcResources.end())
1306 SCDesc.WriteProcResIdx = WPRPos - SchedTables.WriteProcResources.begin();
1307 else {
1308 SCDesc.WriteProcResIdx = SchedTables.WriteProcResources.size();
1309 SchedTables.WriteProcResources.insert(position: WPRPos, first: WriteProcResources.begin(),
1310 last: WriteProcResources.end());
1311 }
1312 // Latency entries must remain in operand order.
1313 SchedTables.MaxWriteLatencyEntries =
1314 std::max(a: SchedTables.MaxWriteLatencyEntries, b: WriteLatencies.size());
1315 SCDesc.NumWriteLatencyEntries = WriteLatencies.size();
1316 std::vector<MCWriteLatencyEntry>::iterator WLPos = std::search(
1317 first1: SchedTables.WriteLatencies.begin(), last1: SchedTables.WriteLatencies.end(),
1318 first2: WriteLatencies.begin(), last2: WriteLatencies.end());
1319 if (WLPos != SchedTables.WriteLatencies.end()) {
1320 unsigned Idx = WLPos - SchedTables.WriteLatencies.begin();
1321 SCDesc.WriteLatencyIdx = Idx;
1322 for (unsigned I = 0, E = WriteLatencies.size(); I < E; ++I)
1323 if (SchedTables.WriterNames[Idx + I].find(str: WriterNames[I]) ==
1324 std::string::npos) {
1325 SchedTables.WriterNames[Idx + I] += "_" + WriterNames[I];
1326 }
1327 } else {
1328 SCDesc.WriteLatencyIdx = SchedTables.WriteLatencies.size();
1329 llvm::append_range(C&: SchedTables.WriteLatencies, R&: WriteLatencies);
1330 llvm::append_range(C&: SchedTables.WriterNames, R&: WriterNames);
1331 }
1332 // ReadAdvanceEntries must remain in operand order.
1333 SCDesc.NumReadAdvanceEntries = ReadAdvanceEntries.size();
1334 std::vector<MCReadAdvanceEntry>::iterator RAPos =
1335 std::search(first1: SchedTables.ReadAdvanceEntries.begin(),
1336 last1: SchedTables.ReadAdvanceEntries.end(),
1337 first2: ReadAdvanceEntries.begin(), last2: ReadAdvanceEntries.end());
1338 if (RAPos != SchedTables.ReadAdvanceEntries.end())
1339 SCDesc.ReadAdvanceIdx = RAPos - SchedTables.ReadAdvanceEntries.begin();
1340 else {
1341 SCDesc.ReadAdvanceIdx = SchedTables.ReadAdvanceEntries.size();
1342 llvm::append_range(C&: SchedTables.ReadAdvanceEntries, R&: ReadAdvanceEntries);
1343 }
1344 }
1345}
1346
1347// Emit SchedClass tables for all processors and associated global tables.
1348void SubtargetEmitter::emitSchedClassTables(SchedClassTables &SchedTables,
1349 raw_ostream &OS) {
1350 OS << "\nstatic_assert(" << SchedTables.MaxWriteProcResEntries
1351 << " <= UINT8_MAX, \"NumWriteProcResEntries does not fit in uint8_t\");\n"
1352 << "static_assert(" << SchedTables.MaxWriteLatencyEntries
1353 << " <= UINT8_MAX, \"NumWriteLatencyEntries does not fit in uint8_t\");\n";
1354
1355 // Emit global WriteProcResTable.
1356 OS << "\n// {ProcResourceIdx, ReleaseAtCycle, AcquireAtCycle}\n"
1357 << "extern const llvm::MCWriteProcResEntry " << Target
1358 << "WriteProcResTable[] = {\n"
1359 << " { 0, 0, 0 }, // Invalid\n";
1360 for (unsigned WPRIdx = 1, WPREnd = SchedTables.WriteProcResources.size();
1361 WPRIdx != WPREnd; ++WPRIdx) {
1362 MCWriteProcResEntry &WPREntry = SchedTables.WriteProcResources[WPRIdx];
1363 OS << " {" << format(Fmt: "%2d", Vals: WPREntry.ProcResourceIdx) << ", "
1364 << format(Fmt: "%2d", Vals: WPREntry.ReleaseAtCycle) << ", "
1365 << format(Fmt: "%2d", Vals: WPREntry.AcquireAtCycle) << "}";
1366 if (WPRIdx + 1 < WPREnd)
1367 OS << ',';
1368 OS << " // #" << WPRIdx << '\n';
1369 }
1370 OS << "}; // " << Target << "WriteProcResTable\n";
1371
1372 // Emit global WriteLatencyTable.
1373 OS << "\n// {Cycles, WriteResourceID}\n"
1374 << "extern const llvm::MCWriteLatencyEntry " << Target
1375 << "WriteLatencyTable[] = {\n"
1376 << " { 0, 0}, // Invalid\n";
1377 for (unsigned WLIdx = 1, WLEnd = SchedTables.WriteLatencies.size();
1378 WLIdx != WLEnd; ++WLIdx) {
1379 MCWriteLatencyEntry &WLEntry = SchedTables.WriteLatencies[WLIdx];
1380 OS << " {" << format(Fmt: "%2d", Vals: WLEntry.Cycles) << ", "
1381 << format(Fmt: "%2d", Vals: WLEntry.WriteResourceID) << "}";
1382 if (WLIdx + 1 < WLEnd)
1383 OS << ',';
1384 OS << " // #" << WLIdx << " " << SchedTables.WriterNames[WLIdx] << '\n';
1385 }
1386 OS << "}; // " << Target << "WriteLatencyTable\n";
1387
1388 // Emit global ReadAdvanceTable.
1389 OS << "\n// {UseIdx, WriteResourceID, Cycles}\n"
1390 << "extern const llvm::MCReadAdvanceEntry " << Target
1391 << "ReadAdvanceTable[] = {\n"
1392 << " {0, 0, 0}, // Invalid\n";
1393 for (unsigned RAIdx = 1, RAEnd = SchedTables.ReadAdvanceEntries.size();
1394 RAIdx != RAEnd; ++RAIdx) {
1395 MCReadAdvanceEntry &RAEntry = SchedTables.ReadAdvanceEntries[RAIdx];
1396 OS << " {" << RAEntry.UseIdx << ", "
1397 << format(Fmt: "%2d", Vals: RAEntry.WriteResourceID) << ", "
1398 << format(Fmt: "%2d", Vals: RAEntry.Cycles) << "}";
1399 if (RAIdx + 1 < RAEnd)
1400 OS << ',';
1401 OS << " // #" << RAIdx << '\n';
1402 }
1403 OS << "}; // " << Target << "ReadAdvanceTable\n";
1404
1405 // Pool all SchedClass names in a string table.
1406 StringToOffsetTable StrTab;
1407 unsigned InvalidNameOff = StrTab.GetOrAddStringOffset(Str: "InvalidSchedClass");
1408
1409 // Emit a SchedClass table for each processor.
1410 for (const auto &[Idx, Proc] : enumerate(First: SchedModels.procModels())) {
1411 if (!Proc.hasInstrSchedModel())
1412 continue;
1413
1414 std::vector<MCSchedClassDesc> &SCTab =
1415 SchedTables.ProcSchedClasses[1 + Idx];
1416
1417 OS << "\n// {Name, NumMicroOps, BeginGroup, EndGroup, RetireOOO,"
1418 << " ReadAdvanceIdx, WriteProcResIdx, WriteLatencyIdx,"
1419 << " NumReadAdvanceEntries, NumWriteProcResEntries,"
1420 << " NumWriteLatencyEntries}\n";
1421 OS << "static const llvm::MCSchedClassDesc " << Proc.ModelName
1422 << "SchedClasses[] = {\n";
1423
1424 // The first class is always invalid. We no way to distinguish it except by
1425 // name and position.
1426 assert(SchedModels.getSchedClass(0).Name == "NoInstrModel" &&
1427 "invalid class not first");
1428 OS << " {DBGFIELD(" << InvalidNameOff << ") "
1429 << MCSchedClassDesc::InvalidNumMicroOps
1430 << ", false, false, false, 0, 0, 0, 0, 0, 0},\n";
1431
1432 for (unsigned SCIdx = 1, SCEnd = SCTab.size(); SCIdx != SCEnd; ++SCIdx) {
1433 MCSchedClassDesc &MCDesc = SCTab[SCIdx];
1434 const CodeGenSchedClass &SchedClass = SchedModels.getSchedClass(Idx: SCIdx);
1435 unsigned NameOff = StrTab.GetOrAddStringOffset(Str: SchedClass.Name);
1436 OS << " {DBGFIELD(/*" << SchedClass.Name << "*/ " << NameOff << ") ";
1437 if (SchedClass.Name.size() < 18)
1438 OS.indent(NumSpaces: 18 - SchedClass.Name.size());
1439 OS << MCDesc.NumMicroOps << ", " << (MCDesc.BeginGroup ? "true" : "false")
1440 << ", " << (MCDesc.EndGroup ? "true" : "false") << ", "
1441 << (MCDesc.RetireOOO ? "true" : "false") << ", "
1442 << format(Fmt: "%2d", Vals: MCDesc.ReadAdvanceIdx) << ", "
1443 << format(Fmt: "%2d", Vals: MCDesc.WriteProcResIdx) << ", "
1444 << format(Fmt: "%2d", Vals: MCDesc.WriteLatencyIdx) << ", "
1445 << MCDesc.NumReadAdvanceEntries << ", "
1446 << static_cast<unsigned>(MCDesc.NumWriteProcResEntries) << ", "
1447 << static_cast<unsigned>(MCDesc.NumWriteLatencyEntries) << "}, // #"
1448 << SCIdx << '\n';
1449 }
1450 OS << "}; // " << Proc.ModelName << "SchedClasses\n";
1451 }
1452
1453 StrTab.EmitStringTableDef(OS, Name: Target + "SchedClassNames");
1454}
1455
1456void SubtargetEmitter::emitProcessorModels(raw_ostream &OS) {
1457 // For each processor model.
1458 for (const CodeGenProcModel &PM : SchedModels.procModels()) {
1459 // Emit extra processor info if available.
1460 if (PM.hasExtraProcessorInfo())
1461 emitExtraProcessorInfo(ProcModel: PM, OS);
1462 // Emit processor resource table.
1463 if (PM.hasInstrSchedModel())
1464 emitProcessorResources(ProcModel: PM, OS);
1465 else if (!PM.ProcResourceDefs.empty())
1466 PrintFatalError(ErrorLoc: PM.ModelDef->getLoc(),
1467 Msg: "SchedMachineModel defines "
1468 "ProcResources without defining WriteRes SchedWriteRes");
1469 }
1470
1471 OS << "\n";
1472 OS << "extern const llvm::MCSchedModel " << Target << "SchedModels[] = {\n";
1473 for (const CodeGenProcModel &PM : SchedModels.procModels()) {
1474 // Begin processor itinerary properties
1475 OS << "{ // " << PM.ModelName << "\n";
1476 emitProcessorProp(OS, R: PM.ModelDef, Name: "IssueWidth", Separator: ',');
1477 emitProcessorProp(OS, R: PM.ModelDef, Name: "MicroOpBufferSize", Separator: ',');
1478 emitProcessorProp(OS, R: PM.ModelDef, Name: "LoopMicroOpBufferSize", Separator: ',');
1479 emitProcessorProp(OS, R: PM.ModelDef, Name: "LoadLatency", Separator: ',');
1480 emitProcessorProp(OS, R: PM.ModelDef, Name: "HighLatency", Separator: ',');
1481 emitProcessorProp(OS, R: PM.ModelDef, Name: "MispredictPenalty", Separator: ',');
1482
1483 bool PostRAScheduler =
1484 (PM.ModelDef ? PM.ModelDef->getValueAsBit(FieldName: "PostRAScheduler") : false);
1485
1486 OS << " " << (PostRAScheduler ? "true" : "false") << ", // "
1487 << "PostRAScheduler\n";
1488
1489 bool CompleteModel =
1490 (PM.ModelDef ? PM.ModelDef->getValueAsBit(FieldName: "CompleteModel") : false);
1491
1492 OS << " " << (CompleteModel ? "true" : "false") << ", // "
1493 << "CompleteModel\n";
1494
1495 bool EnableIntervals =
1496 (PM.ModelDef ? PM.ModelDef->getValueAsBit(FieldName: "EnableIntervals") : false);
1497
1498 OS << " " << (EnableIntervals ? "true" : "false") << ", // "
1499 << "EnableIntervals\n";
1500
1501 OS << " " << PM.Index << ", // Processor ID\n";
1502 if (PM.hasInstrSchedModel())
1503 OS << " " << PM.ModelName << "ProcResources" << ",\n"
1504 << " " << PM.ModelName << "SchedClasses" << ",\n"
1505 << " " << PM.ProcResourceDefs.size() + 1 << ",\n"
1506 << " " << SchedModels.schedClasses().size() << ",\n";
1507 else
1508 OS << " nullptr, nullptr, 0, 0,"
1509 << " // No instruction-level machine model.\n";
1510 OS << " DBGVAL_OR_NULLPTR(&" << Target
1511 << "SchedClassNames), // SchedClassNames\n";
1512 if (PM.hasItineraries())
1513 OS << " " << PM.ItinsDef->getName() << ",\n";
1514 else
1515 OS << " nullptr, // No Itinerary\n";
1516 if (PM.hasExtraProcessorInfo())
1517 OS << " &" << PM.ModelName << "ExtraInfo,\n";
1518 else
1519 OS << " nullptr // No extra processor descriptor\n";
1520 OS << " },\n";
1521 }
1522 OS << "};\n";
1523}
1524
1525//
1526// EmitSchedModel - Emits all scheduling model tables, folding common patterns.
1527//
1528void SubtargetEmitter::emitSchedModel(raw_ostream &OS) {
1529 OS << "#ifdef DBGFIELD\n"
1530 << "#error \"<target>GenSubtargetInfo.inc requires a DBGFIELD macro\"\n"
1531 << "#endif\n"
1532 << "#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)\n"
1533 << "#define DBGFIELD(x) x,\n"
1534 << "#define DBGVAL_OR_NULLPTR(x) x\n"
1535 << "#else\n"
1536 << "#define DBGFIELD(x)\n"
1537 << "#define DBGVAL_OR_NULLPTR(x) nullptr\n"
1538 << "#endif\n";
1539
1540 if (SchedModels.hasItineraries()) {
1541 std::vector<std::vector<InstrItinerary>> ProcItinLists;
1542 // Emit the stage data
1543 emitStageAndOperandCycleData(OS, ProcItinLists);
1544 emitItineraries(OS, ProcItinLists);
1545 }
1546 OS << "\n// ===============================================================\n"
1547 << "// Data tables for the new per-operand machine model.\n";
1548
1549 SchedClassTables SchedTables;
1550 for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) {
1551 genSchedClassTables(ProcModel, SchedTables);
1552 }
1553 emitSchedClassTables(SchedTables, OS);
1554
1555 // Emit the processor machine model
1556 emitProcessorModels(OS);
1557
1558 OS << "\n#undef DBGFIELD\n";
1559 OS << "\n#undef DBGVAL_OR_NULLPTR\n";
1560}
1561
1562static void emitPredicateProlog(const RecordKeeper &Records, raw_ostream &OS) {
1563 std::string Buffer;
1564 raw_string_ostream Stream(Buffer);
1565
1566 // Print all PredicateProlog records to the output stream.
1567 for (const Record *P : Records.getAllDerivedDefinitions(ClassName: "PredicateProlog"))
1568 Stream << P->getValueAsString(FieldName: "Code") << '\n';
1569
1570 OS << Buffer;
1571}
1572
1573static bool isTruePredicate(const Record *Rec) {
1574 return Rec->isSubClassOf(Name: "MCSchedPredicate") &&
1575 Rec->getValueAsDef(FieldName: "Pred")->isSubClassOf(Name: "MCTrue");
1576}
1577
1578static void expandSchedPredicates(const Record *Rec, PredicateExpander &PE,
1579 bool WrapPredicate, raw_ostream &OS) {
1580 if (Rec->isSubClassOf(Name: "MCSchedPredicate")) {
1581 PE.expandPredicate(OS, Rec: Rec->getValueAsDef(FieldName: "Pred"));
1582 } else if (Rec->isSubClassOf(Name: "FeatureSchedPredicate")) {
1583 const Record *FR = Rec->getValueAsDef(FieldName: "Feature");
1584 if (PE.shouldExpandForMC()) {
1585 // MC version of this predicate will be emitted into
1586 // resolveVariantSchedClassImpl, which accesses MCSubtargetInfo
1587 // through argument STI.
1588 OS << "STI.";
1589 } else {
1590 // Otherwise, this predicate will be emitted directly into
1591 // TargetGenSubtargetInfo::resolveSchedClass, which can just access
1592 // TargetSubtargetInfo / MCSubtargetInfo through `this`.
1593 OS << "this->";
1594 }
1595 OS << "hasFeature(" << PE.getTargetName() << "::" << FR->getName() << ")";
1596 } else if (Rec->isSubClassOf(Name: "SchedPredicateCombiner")) {
1597 std::vector<const Record *> SubPreds =
1598 Rec->getValueAsListOfDefs(FieldName: "Predicates");
1599 if (SubPreds.empty())
1600 PrintFatalError(Rec, Msg: "Empty SchedPredicateCombiner is not allowed");
1601
1602 StringRef Sep;
1603 if (Rec->isSubClassOf(Name: "AllOfSchedPreds")) {
1604 Sep = " && ";
1605 } else if (Rec->isSubClassOf(Name: "AnyOfSchedPreds")) {
1606 Sep = " || ";
1607 } else if (Rec->isSubClassOf(Name: "NotSchedPred")) {
1608 if (SubPreds.size() != 1)
1609 PrintFatalError(Rec,
1610 Msg: "NotSchedPred can only have a single sub-predicate.");
1611 OS << "!";
1612 // We don't have to eagerly wrap this term right now: telling its (only)
1613 // sub-predicate to wrap itself should be sufficient.
1614 WrapPredicate = false;
1615 } else {
1616 PrintFatalError(Rec, Msg: "Unrecognized SchedPredicateCombiner");
1617 }
1618
1619 if (WrapPredicate)
1620 OS << "(";
1621
1622 ListSeparator LS(Sep);
1623 bool WrapSubPreds =
1624 SubPreds.size() > 1 || Rec->isSubClassOf(Name: "NotSchedPred");
1625 for (const Record *SubP : SubPreds)
1626 expandSchedPredicates(Rec: SubP, PE, WrapPredicate: WrapSubPreds, OS&: OS << LS);
1627
1628 if (WrapPredicate)
1629 OS << ")";
1630 } else {
1631 // Expand this legacy predicate and wrap it around braces if there is more
1632 // than one predicate to expand.
1633 OS << (WrapPredicate ? "(" : "") << Rec->getValueAsString(FieldName: "Predicate")
1634 << (WrapPredicate ? ")" : "");
1635 }
1636}
1637
1638static void emitPredicates(const CodeGenSchedTransition &T,
1639 const CodeGenSchedClass &SC, PredicateExpander &PE,
1640 raw_ostream &OS) {
1641 std::string Buffer;
1642 raw_string_ostream SS(Buffer);
1643
1644 // If not all predicates are MCTrue, then we need an if-stmt.
1645 unsigned NumNonTruePreds =
1646 T.PredTerm.size() - count_if(Range: T.PredTerm, P: isTruePredicate);
1647
1648 SS << PE.getIndent();
1649
1650 if (NumNonTruePreds) {
1651 bool FirstNonTruePredicate = true;
1652 SS << "if (";
1653
1654 PE.getIndent() += 2;
1655
1656 for (const Record *Rec : T.PredTerm) {
1657 // Skip predicates that evaluate to "true".
1658 if (isTruePredicate(Rec))
1659 continue;
1660
1661 if (FirstNonTruePredicate) {
1662 FirstNonTruePredicate = false;
1663 } else {
1664 SS << "\n";
1665 SS << PE.getIndent();
1666 SS << "&& ";
1667 }
1668
1669 expandSchedPredicates(Rec, PE, /*WrapPredicate=*/NumNonTruePreds > 1, OS&: SS);
1670 }
1671
1672 SS << ")\n"; // end of if-stmt
1673 --PE.getIndent();
1674 SS << PE.getIndent();
1675 --PE.getIndent();
1676 }
1677
1678 SS << "return " << T.ToClassIdx << "; // " << SC.Name << '\n';
1679 OS << Buffer;
1680}
1681
1682// Used by method `SubtargetEmitter::emitSchedModelHelpersImpl()` to generate
1683// epilogue code for the auto-generated helper.
1684static void emitSchedModelHelperEpilogue(raw_ostream &OS,
1685 bool ShouldReturnZero) {
1686 if (ShouldReturnZero) {
1687 OS << " // Don't know how to resolve this scheduling class.\n"
1688 << " return 0;\n";
1689 return;
1690 }
1691
1692 OS << " report_fatal_error(\"Expected a variant SchedClass\");\n";
1693}
1694
1695static bool hasMCSchedPredicate(const Record *Rec) {
1696 if (Rec->isSubClassOf(Name: "MCSchedPredicate") ||
1697 Rec->isSubClassOf(Name: "FeatureSchedPredicate"))
1698 return true;
1699
1700 if (Rec->isSubClassOf(Name: "SchedPredicateCombiner")) {
1701 // Check its sub-predicates recursively.
1702 std::vector<const Record *> SubPreds =
1703 Rec->getValueAsListOfDefs(FieldName: "Predicates");
1704 return all_of(Range&: SubPreds, P: hasMCSchedPredicate);
1705 }
1706
1707 return false;
1708}
1709static bool hasMCSchedPredicates(const CodeGenSchedTransition &T) {
1710 return all_of(Range: T.PredTerm, P: hasMCSchedPredicate);
1711}
1712
1713static void collectVariantClasses(const CodeGenSchedModels &SchedModels,
1714 IdxVec &VariantClasses,
1715 bool OnlyExpandMCInstPredicates) {
1716 for (const CodeGenSchedClass &SC : SchedModels.schedClasses()) {
1717 // Ignore non-variant scheduling classes.
1718 if (SC.Transitions.empty())
1719 continue;
1720
1721 if (OnlyExpandMCInstPredicates) {
1722 // Ignore this variant scheduling class no transitions use any meaningful
1723 // MCSchedPredicate definitions.
1724 if (llvm::none_of(Range: SC.Transitions, P: hasMCSchedPredicates))
1725 continue;
1726 }
1727
1728 VariantClasses.push_back(x: SC.Index);
1729 }
1730}
1731
1732static void collectProcessorIndices(const CodeGenSchedClass &SC,
1733 IdxVec &ProcIndices) {
1734 // A variant scheduling class may define transitions for multiple
1735 // processors. This function identifies wich processors are associated with
1736 // transition rules specified by variant class `SC`.
1737 for (const CodeGenSchedTransition &T : SC.Transitions) {
1738 IdxVec PI;
1739 std::set_union(first1: &T.ProcIndex, last1: &T.ProcIndex + 1, first2: ProcIndices.begin(),
1740 last2: ProcIndices.end(), result: std::back_inserter(x&: PI));
1741 ProcIndices = std::move(PI);
1742 }
1743}
1744
1745static bool isAlwaysTrue(const CodeGenSchedTransition &T) {
1746 return llvm::all_of(Range: T.PredTerm, P: isTruePredicate);
1747}
1748
1749void SubtargetEmitter::emitSchedModelHelpersImpl(
1750 raw_ostream &OS, bool OnlyExpandMCInstPredicates) {
1751 IdxVec VariantClasses;
1752 collectVariantClasses(SchedModels, VariantClasses,
1753 OnlyExpandMCInstPredicates);
1754
1755 if (VariantClasses.empty()) {
1756 emitSchedModelHelperEpilogue(OS, ShouldReturnZero: OnlyExpandMCInstPredicates);
1757 return;
1758 }
1759
1760 // Construct a switch statement where the condition is a check on the
1761 // scheduling class identifier. There is a `case` for every variant class
1762 // defined by the processor models of this target.
1763 // Each `case` implements a number of rules to resolve (i.e. to transition
1764 // from) a variant scheduling class to another scheduling class. Rules are
1765 // described by instances of CodeGenSchedTransition. Note that transitions may
1766 // not be valid for all processors.
1767 OS << " switch (SchedClass) {\n";
1768 for (unsigned VC : VariantClasses) {
1769 IdxVec ProcIndices;
1770 const CodeGenSchedClass &SC = SchedModels.getSchedClass(Idx: VC);
1771 collectProcessorIndices(SC, ProcIndices);
1772
1773 OS << " case " << VC << ": // " << SC.Name << '\n';
1774
1775 PredicateExpander PE(Target);
1776 PE.setByRef(false);
1777 PE.setExpandForMC(OnlyExpandMCInstPredicates);
1778 for (unsigned PI : ProcIndices) {
1779 OS << " ";
1780
1781 // Emit a guard on the processor ID.
1782 if (PI != 0) {
1783 OS << (OnlyExpandMCInstPredicates
1784 ? "if (CPUID == "
1785 : "if (SchedModel->getProcessorID() == ");
1786 OS << PI << ") ";
1787 OS << "{ // " << SchedModels.procModels()[PI].ModelName << '\n';
1788 }
1789
1790 // Now emit transitions associated with processor PI.
1791 const CodeGenSchedTransition *FinalT = nullptr;
1792 for (const CodeGenSchedTransition &T : SC.Transitions) {
1793 if (PI != 0 && T.ProcIndex != PI)
1794 continue;
1795
1796 // Emit only transitions based on MCSchedPredicate, if it's the case.
1797 // At least the transition specified by NoSchedPred is emitted,
1798 // which becomes the default transition for those variants otherwise
1799 // not based on MCSchedPredicate.
1800 // FIXME: preferably, llvm-mca should instead assume a reasonable
1801 // default when a variant transition is not based on MCSchedPredicate
1802 // for a given processor.
1803 if (OnlyExpandMCInstPredicates && !hasMCSchedPredicates(T))
1804 continue;
1805
1806 // If transition is folded to 'return X' it should be the last one.
1807 if (isAlwaysTrue(T)) {
1808 FinalT = &T;
1809 continue;
1810 }
1811 PE.getIndent() = 3;
1812 emitPredicates(T, SC: SchedModels.getSchedClass(Idx: T.ToClassIdx), PE, OS);
1813 }
1814 if (FinalT)
1815 emitPredicates(T: *FinalT, SC: SchedModels.getSchedClass(Idx: FinalT->ToClassIdx),
1816 PE, OS);
1817
1818 OS << " }\n";
1819
1820 if (PI == 0)
1821 break;
1822 }
1823
1824 if (SC.isInferred())
1825 OS << " return " << SC.Index << ";\n";
1826 OS << " break;\n";
1827 }
1828
1829 OS << " };\n";
1830
1831 emitSchedModelHelperEpilogue(OS, ShouldReturnZero: OnlyExpandMCInstPredicates);
1832}
1833
1834void SubtargetEmitter::emitSchedModelHelpers(const std::string &ClassName,
1835 raw_ostream &OS) {
1836 OS << "unsigned " << ClassName
1837 << "\n::resolveSchedClass(unsigned SchedClass, const MachineInstr *MI,"
1838 << " const TargetSchedModel *SchedModel) const {\n";
1839
1840 // Emit the predicate prolog code.
1841 emitPredicateProlog(Records, OS);
1842
1843 // Emit target predicates.
1844 emitSchedModelHelpersImpl(OS);
1845
1846 OS << "} // " << ClassName << "::resolveSchedClass\n\n";
1847
1848 OS << "unsigned " << ClassName
1849 << "\n::resolveVariantSchedClass(unsigned SchedClass, const MCInst *MI,"
1850 << " const MCInstrInfo *MCII, unsigned CPUID) const {\n"
1851 << " return " << Target << "_MC"
1852 << "::resolveVariantSchedClassImpl(SchedClass, MI, MCII, *this, CPUID);\n"
1853 << "} // " << ClassName << "::resolveVariantSchedClass\n\n";
1854
1855 STIPredicateExpander PE(Target, /*Indent=*/0);
1856 PE.setClassPrefix(ClassName);
1857 PE.setExpandDefinition(true);
1858 PE.setByRef(false);
1859
1860 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
1861 PE.expandSTIPredicate(OS, Fn);
1862}
1863
1864void SubtargetEmitter::emitHwModeCheck(const std::string &ClassName,
1865 raw_ostream &OS, bool IsMC) {
1866 const CodeGenHwModes &CGH = TGT.getHwModes();
1867 assert(CGH.getNumModeIds() > 0);
1868 if (CGH.getNumModeIds() == 1)
1869 return;
1870
1871 // Collect all HwModes and related features defined in the TD files,
1872 // and store them as a bit set.
1873 unsigned ValueTypeModes = 0;
1874 unsigned RegInfoModes = 0;
1875 unsigned EncodingInfoModes = 0;
1876 for (const auto &MS : CGH.getHwModeSelects()) {
1877 for (const HwModeSelect::PairType &P : MS.second.Items) {
1878 if (P.first == DefaultMode)
1879 continue;
1880 if (P.second->isSubClassOf(Name: "ValueType")) {
1881 ValueTypeModes |= (1 << (P.first - 1));
1882 } else if (P.second->isSubClassOf(Name: "RegInfo") ||
1883 P.second->isSubClassOf(Name: "Register") ||
1884 P.second->isSubClassOf(Name: "SubRegRange") ||
1885 P.second->isSubClassOf(Name: "RegisterClassLike")) {
1886 RegInfoModes |= (1 << (P.first - 1));
1887 } else if (P.second->isSubClassOf(Name: "InstructionEncoding")) {
1888 EncodingInfoModes |= (1 << (P.first - 1));
1889 }
1890 }
1891 }
1892
1893 // Start emitting for getHwModeSet().
1894 OS << "unsigned " << ClassName << "::getHwModeSet() const {\n";
1895 if (IsMC) {
1896 OS << " [[maybe_unused]] const FeatureBitset &FB = getFeatureBits();\n";
1897 } else {
1898 const ArrayRef<const Record *> &Prologs =
1899 Records.getAllDerivedDefinitions(ClassName: "HwModePredicateProlog");
1900 if (!Prologs.empty()) {
1901 for (const Record *P : Prologs)
1902 OS << P->getValueAsString(FieldName: "Code") << '\n';
1903 } else {
1904 // Works for most targets.
1905 OS << " [[maybe_unused]] const auto *Subtarget =\n"
1906 << " static_cast<const " << Target << "Subtarget *>(this);\n";
1907 }
1908 }
1909 OS << " // Collect HwModes and store them as a bit set.\n";
1910 OS << " unsigned Modes = 0;\n";
1911 for (unsigned M = 1, NumModes = CGH.getNumModeIds(); M != NumModes; ++M) {
1912 const HwMode &HM = CGH.getMode(Id: M);
1913 OS << " if (";
1914 if (IsMC)
1915 SubtargetFeatureInfo::emitMCPredicateCheck(OS, TargetName: Target, Predicates: HM.Predicates);
1916 else
1917 SubtargetFeatureInfo::emitPredicateCheck(OS, Predicates: HM.Predicates);
1918 OS << ") Modes |= (1 << " << (M - 1) << ");\n";
1919 }
1920 OS << " return Modes;\n}\n";
1921 // End emitting for getHwModeSet().
1922
1923 auto HandlePerMode = [&](std::string ModeType, unsigned ModeInBitSet) {
1924 OS << " case HwMode_" << ModeType << ":\n";
1925 if (ModeInBitSet == 0) {
1926 OS << " // No HwMode for " << ModeType << ".\n"
1927 << " return 0;\n";
1928 } else {
1929 OS << " Modes &= " << ModeInBitSet << ";\n"
1930 << " if (!Modes)\n return Modes;\n"
1931 << " if (!llvm::has_single_bit<unsigned>(Modes))\n"
1932 << " llvm_unreachable(\"Two or more HwModes for " << ModeType
1933 << " were found!\");\n"
1934 << " return llvm::countr_zero(Modes) + 1;\n";
1935 }
1936 };
1937
1938 // Start emitting for getHwMode().
1939 OS << "unsigned " << ClassName
1940 << "::getHwMode(enum HwModeType type) const {\n";
1941 OS << " unsigned Modes = getHwModeSet();\n\n";
1942 OS << " if (!Modes)\n return Modes;\n\n";
1943 OS << " switch (type) {\n";
1944 OS << " case HwMode_Default:\n return llvm::countr_zero(Modes) + 1;\n";
1945 HandlePerMode("ValueType", ValueTypeModes);
1946 HandlePerMode("RegInfo", RegInfoModes);
1947 HandlePerMode("EncodingInfo", EncodingInfoModes);
1948 OS << " }\n";
1949 OS << " llvm_unreachable(\"unexpected HwModeType\");\n"
1950 << " return 0; // should not get here\n}\n";
1951 // End emitting for getHwMode().
1952}
1953
1954void SubtargetEmitter::emitGetMacroFusions(const std::string &ClassName,
1955 raw_ostream &OS) {
1956 if (!TGT.hasMacroFusion())
1957 return;
1958
1959 OS << "std::vector<MacroFusionPredTy> " << ClassName
1960 << "::getMacroFusions() const {\n";
1961 OS.indent(NumSpaces: 2) << "std::vector<MacroFusionPredTy> Fusions;\n";
1962 for (auto *Fusion : TGT.getMacroFusions()) {
1963 std::string Name = Fusion->getNameInitAsString();
1964 OS.indent(NumSpaces: 2) << "if (hasFeature(" << Target << "::" << Name
1965 << ")) Fusions.push_back(llvm::is" << Name << ");\n";
1966 }
1967
1968 OS.indent(NumSpaces: 2) << "return Fusions;\n";
1969 OS << "}\n";
1970}
1971
1972// Produces a subtarget specific function for parsing
1973// the subtarget features string.
1974void SubtargetEmitter::parseFeaturesFunction(raw_ostream &OS) {
1975 ArrayRef<const Record *> Features =
1976 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
1977
1978 OS << "// ParseSubtargetFeatures - Parses features string setting specified\n"
1979 << "// subtarget options.\n"
1980 << "void llvm::";
1981 OS << Target;
1982 OS << "Subtarget::ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, "
1983 << "StringRef FS) {\n"
1984 << " LLVM_DEBUG(dbgs() << \"\\nFeatures:\" << FS);\n"
1985 << " LLVM_DEBUG(dbgs() << \"\\nCPU:\" << CPU);\n"
1986 << " LLVM_DEBUG(dbgs() << \"\\nTuneCPU:\" << TuneCPU << \"\\n\\n\");\n";
1987
1988 if (Features.empty()) {
1989 OS << "}\n";
1990 return;
1991 }
1992
1993 if (Target == "AArch64")
1994 OS << " CPU = AArch64::resolveCPUAlias(CPU);\n"
1995 << " TuneCPU = AArch64::resolveCPUAlias(TuneCPU);\n";
1996
1997 OS << " InitMCProcessorInfo(CPU, TuneCPU, FS);\n"
1998 << " const FeatureBitset &Bits = getFeatureBits();\n";
1999
2000 for (const Record *R : Features) {
2001 // Next record
2002 StringRef Instance = R->getName();
2003 StringRef Value = R->getValueAsString(FieldName: "Value");
2004 StringRef FieldName = R->getValueAsString(FieldName: "FieldName");
2005
2006 if (Value == "true" || Value == "false")
2007 OS << " if (Bits[" << Target << "::" << Instance << "]) " << FieldName
2008 << " = " << Value << ";\n";
2009 else
2010 OS << " if (Bits[" << Target << "::" << Instance << "] && " << FieldName
2011 << " < " << Value << ") " << FieldName << " = " << Value << ";\n";
2012 }
2013
2014 OS << "}\n";
2015}
2016
2017void SubtargetEmitter::emitInlineFeatures(const std::string &ClassName,
2018 raw_ostream &OS, StringRef Behavior) {
2019 std::vector<const Record *> FeatureList =
2020 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
2021 llvm::sort(C&: FeatureList, Comp: LessRecordFieldFieldName());
2022
2023 OS << "const FeatureBitset &" << ClassName << "::get" << Behavior
2024 << "Features() const {\n"
2025 << " static constexpr FeatureBitset Features = {\n";
2026
2027 for (const Record *Feature : FeatureList)
2028 if (Behavior == Feature->getValueAsDef(FieldName: "InlineBehavior")->getName())
2029 OS << Target << "::" << Feature->getName() << ",\n";
2030
2031 OS << " };\n"
2032 << " return Features;\n"
2033 << "}\n\n";
2034}
2035
2036void SubtargetEmitter::emitGenMCSubtargetInfo(raw_ostream &OS) {
2037 {
2038 NamespaceEmitter NS(OS, (Target + Twine("_MC")).str());
2039 OS << "unsigned resolveVariantSchedClassImpl(unsigned SchedClass,\n"
2040 << " const MCInst *MI, const MCInstrInfo *MCII, "
2041 << "const MCSubtargetInfo &STI, unsigned CPUID) {\n";
2042 emitSchedModelHelpersImpl(OS, /* OnlyExpandMCPredicates */ OnlyExpandMCInstPredicates: true);
2043 OS << "}\n";
2044 }
2045
2046 OS << "struct " << Target
2047 << "GenMCSubtargetInfo : public MCSubtargetInfo {\n";
2048 OS << " " << Target << "GenMCSubtargetInfo(const Triple &TT,\n"
2049 << " StringRef CPU, StringRef TuneCPU, StringRef FS,\n"
2050 << " StringTable PN,\n"
2051 << " ArrayRef<SubtargetFeatureKV> PF,\n"
2052 << " ArrayRef<SubtargetSubTypeKV> PD,\n"
2053 << " const MCSchedModel *PSM,\n"
2054 << " const MCWriteProcResEntry *WPR,\n"
2055 << " const MCWriteLatencyEntry *WL,\n"
2056 << " const MCReadAdvanceEntry *RA, const InstrStage *IS,\n"
2057 << " const unsigned *OC, const unsigned *FP) :\n"
2058 << " MCSubtargetInfo(TT, CPU, TuneCPU, FS, PN, PF, PD, PSM,\n"
2059 << " WPR, WL, RA, IS, OC, FP) { }\n\n"
2060 << " unsigned resolveVariantSchedClass(unsigned SchedClass,\n"
2061 << " const MCInst *MI, const MCInstrInfo *MCII,\n"
2062 << " unsigned CPUID) const final {\n"
2063 << " return " << Target << "_MC"
2064 << "::resolveVariantSchedClassImpl(SchedClass, MI, MCII, *this, CPUID);\n";
2065 OS << " }\n";
2066 if (TGT.getHwModes().getNumModeIds() > 1) {
2067 OS << " unsigned getHwModeSet() const final;\n";
2068 OS << " unsigned getHwMode(enum HwModeType type = HwMode_Default) const "
2069 "final;\n";
2070 }
2071 if (Target == "AArch64")
2072 OS << " bool isCPUStringValid(StringRef CPU) const final {\n"
2073 << " CPU = AArch64::resolveCPUAlias(CPU);\n"
2074 << " return MCSubtargetInfo::isCPUStringValid(CPU);\n"
2075 << " }\n";
2076 OS << "};\n";
2077 emitHwModeCheck(ClassName: Target + "GenMCSubtargetInfo", OS, /*IsMC=*/true);
2078}
2079
2080void SubtargetEmitter::emitMcInstrAnalysisPredicateFunctions(raw_ostream &OS) {
2081 STIPredicateExpander PE(Target, /*Indent=*/0);
2082
2083 {
2084 IfDefEmitter IfDefDecls(OS, "GET_STIPREDICATE_DECLS_FOR_MC_ANALYSIS");
2085 PE.setExpandForMC(true);
2086 PE.setByRef(true);
2087 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
2088 PE.expandSTIPredicate(OS, Fn);
2089 }
2090
2091 IfDefEmitter IfDefDefs(OS, "GET_STIPREDICATE_DEFS_FOR_MC_ANALYSIS");
2092 std::string ClassPrefix = Target + "MCInstrAnalysis";
2093 PE.setExpandDefinition(true);
2094 PE.setClassPrefix(ClassPrefix);
2095 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
2096 PE.expandSTIPredicate(OS, Fn);
2097}
2098
2099FeatureMapTy SubtargetEmitter::emitEnums(raw_ostream &OS) {
2100 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_ENUM");
2101 NamespaceEmitter NS(OS, "llvm");
2102 return enumeration(OS);
2103}
2104
2105SubtargetEmitter::MCDescInfo
2106SubtargetEmitter::emitMCDesc(raw_ostream &OS, const FeatureMapTy &FeatureMap) {
2107 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_MC_DESC");
2108 if (Target == "AArch64")
2109 OS << "#include \"llvm/TargetParser/AArch64TargetParser.h\"\n\n";
2110 NamespaceEmitter LlvmNS(OS, "llvm");
2111
2112 MCDescInfo Res;
2113 auto [NumFeatures, FeatureStrTabSize] = featureKeyValues(OS, FeatureMap);
2114 Res.NumFeatures = NumFeatures;
2115 Res.FeatureStrTabSize = FeatureStrTabSize;
2116 OS << "\n";
2117 emitSchedModel(OS);
2118 OS << "\n";
2119 auto [NumProcs, SubTypeStrTabSize] = cpuKeyValues(OS, FeatureMap);
2120 Res.NumProcs = NumProcs;
2121 Res.SubTypeStrTabSize = SubTypeStrTabSize;
2122 OS << "\n";
2123
2124 // MCInstrInfo initialization routine.
2125 emitGenMCSubtargetInfo(OS);
2126
2127 OS << "\nstatic inline MCSubtargetInfo *create" << Target
2128 << "MCSubtargetInfoImpl("
2129 << "const Triple &TT, StringRef CPU, StringRef TuneCPU, StringRef FS) {\n";
2130 if (Target == "AArch64")
2131 OS << " CPU = AArch64::resolveCPUAlias(CPU);\n"
2132 << " TuneCPU = AArch64::resolveCPUAlias(TuneCPU);\n";
2133 OS << " return new " << Target
2134 << "GenMCSubtargetInfo(TT, CPU, TuneCPU, FS, ";
2135 OS << "StringTable(" << Target << "SubTypeKVStorage.Strings), ";
2136 if (Res.NumFeatures)
2137 OS << Target << "FeatureKVStorage.Features, ";
2138 else
2139 OS << "{}, ";
2140 if (Res.NumProcs)
2141 OS << Target << "SubTypeKVStorage.SubTypes, " << Target << "SchedModels, ";
2142 else
2143 OS << "{}, nullptr, ";
2144 OS << '\n';
2145 OS.indent(NumSpaces: 22);
2146 OS << Target << "WriteProcResTable, " << Target << "WriteLatencyTable, "
2147 << Target << "ReadAdvanceTable, ";
2148 OS << '\n';
2149 OS.indent(NumSpaces: 22);
2150 if (SchedModels.hasItineraries()) {
2151 OS << Target << "Stages, " << Target << "OperandCycles, " << Target
2152 << "ForwardingPaths";
2153 } else {
2154 OS << "nullptr, nullptr, nullptr";
2155 }
2156 OS << ");\n}\n\n";
2157 return Res;
2158}
2159
2160void SubtargetEmitter::emitTargetDesc(raw_ostream &OS) {
2161 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_TARGET_DESC");
2162
2163 OS << "#include \"llvm/ADT/BitmaskEnum.h\"\n";
2164 OS << "#include \"llvm/Support/Debug.h\"\n";
2165 OS << "#include \"llvm/Support/raw_ostream.h\"\n\n";
2166 if (Target == "AArch64")
2167 OS << "#include \"llvm/TargetParser/AArch64TargetParser.h\"\n\n";
2168 parseFeaturesFunction(OS);
2169}
2170
2171void SubtargetEmitter::emitHeader(raw_ostream &OS) {
2172 // Create a TargetSubtargetInfo subclass to hide the MC layer initialization.
2173 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_HEADER");
2174 NamespaceEmitter LLVMNS(OS, "llvm");
2175
2176 std::string ClassName = Target + "GenSubtargetInfo";
2177 OS << "class DFAPacketizer;\n";
2178 {
2179 NamespaceEmitter MCNS(OS, (Target + Twine("_MC")).str());
2180 OS << "unsigned resolveVariantSchedClassImpl(unsigned SchedClass,"
2181 << " const MCInst *MI, const MCInstrInfo *MCII, "
2182 << "const MCSubtargetInfo &STI, unsigned CPUID);\n";
2183 }
2184 OS << "struct " << ClassName << " : public TargetSubtargetInfo {\n"
2185 << " explicit " << ClassName << "(const Triple &TT, StringRef CPU, "
2186 << "StringRef TuneCPU, StringRef FS);\n"
2187 << "public:\n"
2188 << " unsigned resolveSchedClass(unsigned SchedClass, "
2189 << " const MachineInstr *DefMI,"
2190 << " const TargetSchedModel *SchedModel) const final;\n"
2191 << " unsigned resolveVariantSchedClass(unsigned SchedClass,"
2192 << " const MCInst *MI, const MCInstrInfo *MCII,"
2193 << " unsigned CPUID) const final;\n"
2194 << " DFAPacketizer *createDFAPacketizer(const InstrItineraryData *IID)"
2195 << " const;\n";
2196
2197 const CodeGenHwModes &CGH = TGT.getHwModes();
2198 if (CGH.getNumModeIds() > 1) {
2199 OS << " enum class " << Target << "HwModeBits : unsigned {\n";
2200 for (unsigned M = 0, NumModes = CGH.getNumModeIds(); M != NumModes; ++M) {
2201 StringRef ModeName = CGH.getModeName(Id: M, /*IncludeDefault=*/true);
2202 OS << " " << ModeName << " = ";
2203 if (M == 0)
2204 OS << "0";
2205 else
2206 OS << "(1 << " << (M - 1) << ")";
2207 OS << ",\n";
2208 if (M == NumModes - 1) {
2209 OS << "\n";
2210 OS << " LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/" << ModeName
2211 << "),\n";
2212 }
2213 }
2214 OS << " };\n";
2215
2216 OS << " unsigned getHwModeSet() const final;\n";
2217 OS << " unsigned getHwMode(enum HwModeType type = HwMode_Default) const "
2218 "final;\n";
2219 }
2220 if (TGT.hasMacroFusion())
2221 OS << " std::vector<MacroFusionPredTy> getMacroFusions() const "
2222 "final;\n";
2223
2224 OS << " const FeatureBitset &getInlineIgnoreFeatures() const override;\n";
2225 OS << " const FeatureBitset &getInlineInverseFeatures() const override;\n";
2226 OS << " const FeatureBitset &getInlineMustMatchFeatures() const override;\n";
2227
2228 STIPredicateExpander PE(Target);
2229 PE.setByRef(false);
2230 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
2231 PE.expandSTIPredicate(OS, Fn);
2232 OS << "};\n";
2233}
2234
2235void SubtargetEmitter::emitCtor(raw_ostream &OS, MCDescInfo DescInfo) {
2236 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_CTOR");
2237 OS << "#include \"llvm/CodeGen/TargetSchedule.h\"\n\n";
2238
2239 NamespaceEmitter LLVMNS(OS, "llvm");
2240 OS << "extern const llvm::StringRef " << Target << "Names[];\n";
2241 if (DescInfo.NumFeatures) {
2242 OS << "extern const llvm::SubtargetFeatureKVStorage<"
2243 << DescInfo.NumFeatures << ", " << DescInfo.FeatureStrTabSize << "> "
2244 << Target << "FeatureKVStorage;\n";
2245 }
2246 OS << "extern const llvm::SubtargetSubTypeKVStorage<" << DescInfo.NumProcs
2247 << ", " << DescInfo.SubTypeStrTabSize << "> " << Target
2248 << "SubTypeKVStorage;\n";
2249 OS << "extern const llvm::MCSchedModel " << Target << "SchedModels[];\n";
2250 OS << "extern const llvm::MCWriteProcResEntry " << Target
2251 << "WriteProcResTable[];\n";
2252 OS << "extern const llvm::MCWriteLatencyEntry " << Target
2253 << "WriteLatencyTable[];\n";
2254 OS << "extern const llvm::MCReadAdvanceEntry " << Target
2255 << "ReadAdvanceTable[];\n";
2256
2257 if (SchedModels.hasItineraries()) {
2258 OS << "extern const llvm::InstrStage " << Target << "Stages[];\n";
2259 OS << "extern const unsigned " << Target << "OperandCycles[];\n";
2260 OS << "extern const unsigned " << Target << "ForwardingPaths[];\n";
2261 }
2262
2263 std::string ClassName = Target + "GenSubtargetInfo";
2264 OS << ClassName << "::" << ClassName << "(const Triple &TT, StringRef CPU, "
2265 << "StringRef TuneCPU, StringRef FS)\n";
2266
2267 if (Target == "AArch64")
2268 OS << " : TargetSubtargetInfo(TT, AArch64::resolveCPUAlias(CPU),\n"
2269 << " AArch64::resolveCPUAlias(TuneCPU), FS, ";
2270 else
2271 OS << " : TargetSubtargetInfo(TT, CPU, TuneCPU, FS, ";
2272 OS << "StringTable(" << Target << "SubTypeKVStorage.Strings), ";
2273 if (DescInfo.NumFeatures)
2274 OS << "ArrayRef(" << Target << "FeatureKVStorage.Features), ";
2275 else
2276 OS << "{}, ";
2277 if (DescInfo.NumProcs) {
2278 OS << "ArrayRef(" << Target << "SubTypeKVStorage.SubTypes), " << Target
2279 << "SchedModels, ";
2280 } else {
2281 OS << "{}, nullptr, ";
2282 }
2283 OS << '\n';
2284 OS.indent(NumSpaces: 24);
2285 OS << Target << "WriteProcResTable, " << Target << "WriteLatencyTable, "
2286 << Target << "ReadAdvanceTable, ";
2287 OS << '\n';
2288 OS.indent(NumSpaces: 24);
2289 if (SchedModels.hasItineraries()) {
2290 OS << Target << "Stages, " << Target << "OperandCycles, " << Target
2291 << "ForwardingPaths";
2292 } else {
2293 OS << "nullptr, nullptr, nullptr";
2294 }
2295 OS << ") {}\n\n";
2296
2297 emitSchedModelHelpers(ClassName, OS);
2298 emitHwModeCheck(ClassName, OS, /*IsMC=*/false);
2299 emitGetMacroFusions(ClassName, OS);
2300 emitInlineFeatures(ClassName, OS, Behavior: "InlineIgnore");
2301 emitInlineFeatures(ClassName, OS, Behavior: "InlineInverse");
2302 emitInlineFeatures(ClassName, OS, Behavior: "InlineMustMatch");
2303}
2304
2305//
2306// SubtargetEmitter::run - Main subtarget enumeration emitter.
2307//
2308void SubtargetEmitter::run(raw_ostream &OS) {
2309 emitSourceFileHeader(Desc: "Subtarget Enumeration Source Fragment", OS);
2310
2311 auto FeatureMap = emitEnums(OS);
2312 emitSubtargetInfoMacroCalls(OS);
2313 MCDescInfo DescInfo = emitMCDesc(OS, FeatureMap);
2314 emitTargetDesc(OS);
2315 emitHeader(OS);
2316 emitCtor(OS, DescInfo);
2317 emitMcInstrAnalysisPredicateFunctions(OS);
2318}
2319
2320static TableGen::Emitter::OptClass<SubtargetEmitter>
2321 X("gen-subtarget", "Generate subtarget enumerations");
2322