1//===- SubtargetEmitter.cpp - Generate subtarget enumerations -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend emits subtarget enumerations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "Basic/TargetFeaturesEmitter.h"
14#include "Common/CodeGenHwModes.h"
15#include "Common/CodeGenSchedule.h"
16#include "Common/CodeGenTarget.h"
17#include "Common/PredicateExpander.h"
18#include "Common/SubtargetFeatureInfo.h"
19#include "Common/Utils.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SmallPtrSet.h"
22#include "llvm/ADT/StringExtras.h"
23#include "llvm/ADT/StringMap.h"
24#include "llvm/ADT/StringRef.h"
25#include "llvm/ADT/StringSet.h"
26#include "llvm/MC/MCInstrItineraries.h"
27#include "llvm/MC/MCSchedule.h"
28#include "llvm/Support/Debug.h"
29#include "llvm/Support/Format.h"
30#include "llvm/Support/raw_ostream.h"
31#include "llvm/TableGen/CodeGenHelpers.h"
32#include "llvm/TableGen/Error.h"
33#include "llvm/TableGen/Record.h"
34#include "llvm/TableGen/StringToOffsetTable.h"
35#include "llvm/TableGen/TableGenBackend.h"
36#include <algorithm>
37#include <cassert>
38#include <cstdint>
39#include <iterator>
40#include <string>
41#include <vector>
42
43using namespace llvm;
44
45#define DEBUG_TYPE "subtarget-emitter"
46
47namespace {
48
49class SubtargetEmitter : TargetFeaturesEmitter {
50 // Each processor has a SchedClassDesc table with an entry for each
51 // SchedClass. The SchedClassDesc table indexes into a global write resource
52 // table, write latency table, and read advance table.
53 struct SchedClassTables {
54 std::vector<std::vector<MCSchedClassDesc>> ProcSchedClasses;
55 std::vector<MCWriteProcResEntry> WriteProcResources;
56 std::vector<MCWriteLatencyEntry> WriteLatencies;
57 std::vector<std::string> WriterNames;
58 std::vector<MCReadAdvanceEntry> ReadAdvanceEntries;
59 size_t MaxWriteProcResEntries = 0;
60 size_t MaxWriteLatencyEntries = 0;
61
62 // Reserve an invalid entry at index 0
63 SchedClassTables() {
64 ProcSchedClasses.resize(new_size: 1);
65 WriteProcResources.resize(new_size: 1);
66 WriteLatencies.resize(new_size: 1);
67 WriterNames.push_back(x: "InvalidWrite");
68 ReadAdvanceEntries.resize(new_size: 1);
69 }
70 };
71
72 struct LessWriteProcResources {
73 bool operator()(const MCWriteProcResEntry &LHS,
74 const MCWriteProcResEntry &RHS) {
75 return LHS.ProcResourceIdx < RHS.ProcResourceIdx;
76 }
77 };
78
79 CodeGenTarget TGT;
80 CodeGenSchedModels &SchedModels;
81
82 FeatureMapTy emitEnums(raw_ostream &OS);
83 void emitSubtargetInfoMacroCalls(raw_ostream &OS);
84
85 struct MCDescInfo {
86 unsigned NumFeatures;
87 unsigned FeatureStrTabSize;
88 unsigned NumProcs;
89 unsigned NumAliases;
90 unsigned SubTypeStrTabSize;
91 };
92 MCDescInfo emitMCDesc(raw_ostream &OS, const FeatureMapTy &FeatureMap);
93 void emitTargetDesc(raw_ostream &OS);
94 void emitHeader(raw_ostream &OS);
95 void emitCtor(raw_ostream &OS, MCDescInfo DescInfo);
96
97 struct CPUKeyValuesInfo {
98 unsigned NumProcs;
99 unsigned NumAliases;
100 unsigned SubTypeStrTabSize;
101 };
102 std::pair<unsigned, unsigned>
103 featureKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
104 CPUKeyValuesInfo cpuKeyValues(raw_ostream &OS,
105 const FeatureMapTy &FeatureMap);
106 void formItineraryStageString(const std::string &Names,
107 const Record *ItinData, std::string &ItinString,
108 unsigned &NStages);
109 void formItineraryOperandCycleString(const Record *ItinData,
110 std::string &ItinString,
111 unsigned &NOperandCycles);
112 void formItineraryBypassString(const std::string &Names,
113 const Record *ItinData,
114 std::string &ItinString,
115 unsigned NOperandCycles);
116 void emitStageAndOperandCycleData(
117 raw_ostream &OS, std::vector<std::vector<InstrItinerary>> &ProcItinLists);
118 void emitItineraries(raw_ostream &OS,
119 ArrayRef<std::vector<InstrItinerary>> ProcItinLists);
120 unsigned emitRegisterFileTables(const CodeGenProcModel &ProcModel,
121 raw_ostream &OS);
122 void emitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel,
123 raw_ostream &OS);
124 void emitExtraProcessorInfo(const CodeGenProcModel &ProcModel,
125 raw_ostream &OS);
126 void emitProcessorProp(raw_ostream &OS, const Record *R, StringRef Name,
127 char Separator);
128 void emitProcessorResourceSubUnits(const CodeGenProcModel &ProcModel,
129 raw_ostream &OS);
130 void emitProcessorResources(const CodeGenProcModel &ProcModel,
131 raw_ostream &OS);
132 const Record *findWriteResources(const CodeGenSchedRW &SchedWrite,
133 const CodeGenProcModel &ProcModel);
134 const Record *findReadAdvance(const CodeGenSchedRW &SchedRead,
135 const CodeGenProcModel &ProcModel);
136 void expandProcResources(ConstRecVec &PRVec,
137 std::vector<int64_t> &ReleaseAtCycles,
138 std::vector<int64_t> &AcquireAtCycles,
139 const CodeGenProcModel &ProcModel);
140 void genSchedClassTables(const CodeGenProcModel &ProcModel,
141 SchedClassTables &SchedTables);
142 void emitSchedClassTables(SchedClassTables &SchedTables, raw_ostream &OS);
143 void emitProcessorModels(raw_ostream &OS);
144 void emitSchedModelHelpers(const std::string &ClassName, raw_ostream &OS);
145 void emitSchedModelHelpersImpl(raw_ostream &OS,
146 bool OnlyExpandMCInstPredicates = false);
147 void emitGenMCSubtargetInfo(raw_ostream &OS);
148 void emitMcInstrAnalysisPredicateFunctions(raw_ostream &OS);
149
150 void emitSchedModel(raw_ostream &OS);
151 void emitGetMacroFusions(const std::string &ClassName, raw_ostream &OS);
152 void emitHwModeCheck(const std::string &ClassName, raw_ostream &OS,
153 bool IsMC);
154 void emitInlineFeatures(const std::string &ClassName, raw_ostream &OS,
155 StringRef Behavior);
156 void parseFeaturesFunction(raw_ostream &OS);
157
158public:
159 SubtargetEmitter(const RecordKeeper &R)
160 : TargetFeaturesEmitter(R), TGT(R), SchedModels(TGT.getSchedModels()) {}
161
162 void run(raw_ostream &O) override;
163};
164
165} // end anonymous namespace
166
167/// Emit some information about the SubtargetFeature as calls to a macro so
168/// that they can be used from C++.
169void SubtargetEmitter::emitSubtargetInfoMacroCalls(raw_ostream &OS) {
170 // Undef the GET_SUBTARGETINFO_MACRO macro at the end of the scope since it's
171 // used within the scope.
172 IfDefEmitter IfDefMacro(OS, "GET_SUBTARGETINFO_MACRO", /*LateUndef=*/true);
173
174 std::vector<const Record *> FeatureList =
175 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
176 llvm::sort(C&: FeatureList, Comp: LessRecordFieldFieldName());
177
178 for (const Record *Feature : FeatureList) {
179 const StringRef FieldName = Feature->getValueAsString(FieldName: "FieldName");
180 const StringRef Value = Feature->getValueAsString(FieldName: "Value");
181
182 // Only handle boolean features for now, excluding BitVectors and enums.
183 const bool IsBool = (Value == "false" || Value == "true") &&
184 !StringRef(FieldName).contains(C: '[');
185 if (!IsBool)
186 continue;
187
188 // Some features default to true, with values set to false if enabled.
189 const char *Default = Value == "false" ? "true" : "false";
190
191 // Define the getter with lowercased first char: xxxYyy() { return XxxYyy; }
192 const std::string Getter =
193 FieldName.substr(Start: 0, N: 1).lower() + FieldName.substr(Start: 1).str();
194
195 OS << "GET_SUBTARGETINFO_MACRO(" << FieldName << ", " << Default << ", "
196 << Getter << ")\n";
197 }
198}
199
200//
201// FeatureKeyValues - Emit data of all the subtarget features. Used by the
202// command line.
203//
204std::pair<unsigned, unsigned>
205SubtargetEmitter::featureKeyValues(raw_ostream &OS,
206 const FeatureMapTy &FeatureMap) {
207 std::vector<const Record *> FeatureList =
208 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
209
210 // Remove features with empty name.
211 llvm::erase_if(C&: FeatureList, P: [](const Record *Rec) {
212 return Rec->getValueAsString(FieldName: "Name").empty();
213 });
214 if (FeatureList.empty())
215 return {0, 0};
216
217 // Sort and check duplicate Feature name.
218 sortAndReportDuplicates(Records: FeatureList, ObjectName: "Feature");
219
220 StringToOffsetTable StrTab;
221 // Offsets of CommandLineName and Desc in StrTab.
222 SmallVector<std::pair<unsigned, unsigned>> StrOffs;
223 for (const Record *Feature : FeatureList) {
224 unsigned NameOff =
225 StrTab.GetOrAddStringOffset(Str: Feature->getValueAsString(FieldName: "Name"));
226 unsigned DescOff =
227 StrTab.GetOrAddStringOffset(Str: Feature->getValueAsString(FieldName: "Desc"));
228 StrOffs.emplace_back(Args&: NameOff, Args&: DescOff);
229 }
230
231 // Begin feature table.
232 OS << "// Sorted (by key) array of values for CPU features.\n"
233 << "extern const llvm::SubtargetFeatureKVStorage< " << FeatureList.size()
234 << ", " << (StrTab.size() + 1) << "> " << Target
235 << "FeatureKVStorage = {\n {\n";
236
237 for (auto [Idx, Feature] : enumerate(First&: FeatureList)) {
238 // Next feature
239 StringRef Name = Feature->getName();
240
241 // Emit as { "feature", "description", { featureEnum }, { i1 , i2 , ... , in
242 // } }
243 auto StrOff =
244 "sizeof(SubtargetFeatureKV) * " + Twine(FeatureList.size() - Idx);
245 OS << " { " << StrOff << " + " << StrOffs[Idx].first << ", " << StrOff
246 << " + " << StrOffs[Idx].second << ", " << Target << "::" << Name
247 << ", ";
248
249 ConstRecVec ImpliesList = Feature->getValueAsListOfDefs(FieldName: "Implies");
250
251 printFeatureMask(OS, FeatureList: ImpliesList, FeatureMap);
252
253 OS << " },\n";
254 }
255
256 OS << " },\n";
257 StrTab.EmitString(O&: OS);
258
259 // End feature table.
260 OS << "};\n";
261
262 return {FeatureList.size(), StrTab.size() + 1};
263}
264
265static void checkDuplicateCPUFeatures(StringRef CPUName,
266 ArrayRef<const Record *> Features,
267 ArrayRef<const Record *> TuneFeatures) {
268 // We have made sure each SubtargetFeature Record has a unique name, so we can
269 // simply use pointer sets here.
270 SmallPtrSet<const Record *, 8> FeatureSet, TuneFeatureSet;
271 for (const auto *FeatureRec : Features) {
272 if (!FeatureSet.insert(Ptr: FeatureRec).second)
273 PrintWarning(Msg: "Processor " + CPUName + " contains duplicate feature '" +
274 FeatureRec->getValueAsString(FieldName: "Name") + "'");
275 }
276
277 for (const auto *TuneFeatureRec : TuneFeatures) {
278 if (!TuneFeatureSet.insert(Ptr: TuneFeatureRec).second)
279 PrintWarning(Msg: "Processor " + CPUName +
280 " contains duplicate tune feature '" +
281 TuneFeatureRec->getValueAsString(FieldName: "Name") + "'");
282 if (FeatureSet.contains(Ptr: TuneFeatureRec))
283 PrintWarning(Msg: "Processor " + CPUName + " has '" +
284 TuneFeatureRec->getValueAsString(FieldName: "Name") +
285 "' in both feature and tune feature sets");
286 }
287}
288
289//
290// CPUKeyValues - Emit data of all the subtarget processors. Used by command
291// line.
292//
293SubtargetEmitter::CPUKeyValuesInfo
294SubtargetEmitter::cpuKeyValues(raw_ostream &OS,
295 const FeatureMapTy &FeatureMap) {
296 // Gather and sort the processors. Only real processors go in the subtype
297 // table; aliases are stored in a separate, more compact table.
298 std::vector<const Record *> ProcessorList =
299 Records.getAllDerivedDefinitions(ClassName: "Processor");
300 llvm::sort(C&: ProcessorList, Comp: LessRecordFieldName());
301
302 // Map from processor name to its index in the sorted subtype table, so that
303 // aliases can point at the canonical processor entry.
304 StringMap<unsigned> ProcessorIndex;
305 for (const auto &[Idx, Processor] : enumerate(First&: ProcessorList))
306 ProcessorIndex[Processor->getValueAsString(FieldName: "Name")] = Idx;
307
308 // Validate and resolve each alias to the index of its canonical processor.
309 struct AliasEntry {
310 StringRef Name;
311 unsigned SubTypeIdx;
312 };
313 std::vector<AliasEntry> AliasEntries;
314 std::vector<const Record *> ProcessorAliasList =
315 Records.getAllDerivedDefinitionsIfDefined(ClassName: "ProcessorAlias");
316 AliasEntries.reserve(n: ProcessorAliasList.size());
317
318 StringSet<> AliasNames;
319 for (const Record *Rec : ProcessorAliasList) {
320 StringRef Name = Rec->getValueAsString(FieldName: "Name");
321 StringRef Alias = Rec->getValueAsString(FieldName: "Alias");
322 auto It = ProcessorIndex.find(Key: Alias);
323 if (It == ProcessorIndex.end())
324 PrintFatalError(Rec, Msg: "Alias '" + Name +
325 "' references a non-existent Processor '" +
326 Alias + "'");
327 if (ProcessorIndex.contains(Key: Name))
328 PrintFatalError(Rec,
329 Msg: "Alias '" + Name + "' duplicates an existing Processor");
330 if (!AliasNames.insert(key: Name).second)
331 PrintFatalError(Rec, Msg: "Alias '" + Name + "' duplicates an existing alias");
332 AliasEntries.push_back(x: {.Name: Name, .SubTypeIdx: It->second});
333 }
334
335 // The alias table must be sorted by key for the binary search in the lookups.
336 llvm::sort(C&: AliasEntries, Comp: [](const AliasEntry &LHS, const AliasEntry &RHS) {
337 return LHS.Name < RHS.Name;
338 });
339
340 // Sort all names together so the emitted string blob is sorted.
341 SmallVector<StringRef> Names;
342 Names.reserve(N: ProcessorList.size() + AliasEntries.size());
343 for (const Record *Processor : ProcessorList)
344 Names.push_back(Elt: Processor->getValueAsString(FieldName: "Name"));
345 for (const AliasEntry &Entry : AliasEntries)
346 Names.push_back(Elt: Entry.Name);
347 llvm::sort(C&: Names);
348
349 StringToOffsetTable StrTab;
350 for (StringRef Name : Names)
351 StrTab.GetOrAddStringOffset(Str: Name);
352
353 // Note that unlike `FeatureKeyValues`, here we do not need to check for
354 // duplicate processors, since that is already done when the SubtargetEmitter
355 // constructor calls `getSchedModels` to build a `CodeGenSchedModels` object,
356 // which does the duplicate processor check.
357
358 unsigned NumProcs = ProcessorList.size();
359 unsigned NumAliases = AliasEntries.size();
360
361 // The alias array's byte size, used to reach the string blob from a subtype
362 // entry.
363 OS << "static constexpr size_t " << Target
364 << "CPUAliasArraySize = sizeof(std::array<SubtargetSubTypeAliasKV, "
365 << NumAliases << ">);\n";
366
367 // Begin processor table.
368 OS << "// Sorted (by key) array of values for CPU subtype.\n"
369 << "extern const llvm::SubtargetSubTypeKVStorage< " << NumProcs << ", "
370 << NumAliases << ", " << (StrTab.size() + 1) << "> " << Target
371 << "SubTypeKVStorage = {\n {\n";
372
373 for (const auto &[Idx, Processor] : enumerate(First&: ProcessorList)) {
374 StringRef Name = Processor->getValueAsString(FieldName: "Name");
375 ConstRecVec FeatureList = Processor->getValueAsListOfDefs(FieldName: "Features");
376 ConstRecVec TuneFeatureList =
377 Processor->getValueAsListOfDefs(FieldName: "TuneFeatures");
378
379 // Warn the user if there are duplicate processor features or tune features.
380 checkDuplicateCPUFeatures(CPUName: Name, Features: FeatureList, TuneFeatures: TuneFeatureList);
381
382 // The string blob follows the subtype and alias arrays, so skip both.
383 OS << " { sizeof(SubtargetSubTypeKV) * " << (NumProcs - Idx) << " + "
384 << Target << "CPUAliasArraySize + " << StrTab.GetOrAddStringOffset(Str: Name)
385 << ", ";
386
387 printFeatureMask(OS, FeatureList, FeatureMap);
388 OS << ", ";
389 printFeatureMask(OS, FeatureList: TuneFeatureList, FeatureMap);
390
391 // Emit the scheduler model index.
392 OS << ", " << SchedModels.getModelIndexForProc(ProcDef: Processor) << " },\n";
393 }
394
395 OS << " },\n";
396
397 // Begin alias table. The extra brace layer is std::array's wrapped C array.
398 OS << " { {\n";
399 for (const auto &[Idx, Entry] : enumerate(First&: AliasEntries)) {
400 // The string blob immediately follows the alias array.
401 OS << " { sizeof(SubtargetSubTypeAliasKV) * " << (NumAliases - Idx)
402 << " + " << StrTab.GetOrAddStringOffset(Str: Entry.Name) << ", "
403 << Entry.SubTypeIdx << " },\n";
404 }
405 OS << " } },\n";
406
407 StrTab.EmitString(O&: OS);
408
409 // End processor table.
410 OS << "};\n";
411
412 return {.NumProcs: NumProcs, .NumAliases: NumAliases, .SubTypeStrTabSize: unsigned(StrTab.size() + 1)};
413}
414
415//
416// FormItineraryStageString - Compose a string containing the stage
417// data initialization for the specified itinerary. N is the number
418// of stages.
419//
420void SubtargetEmitter::formItineraryStageString(const std::string &Name,
421 const Record *ItinData,
422 std::string &ItinString,
423 unsigned &NStages) {
424 // Get states list
425 ConstRecVec StageList = ItinData->getValueAsListOfDefs(FieldName: "Stages");
426
427 // For each stage
428 unsigned N = NStages = StageList.size();
429 for (unsigned I = 0; I < N;) {
430 // Next stage
431 const Record *Stage = StageList[I];
432
433 // Form string as ,{ cycles, u1 | u2 | ... | un, timeinc, kind }
434 int Cycles = Stage->getValueAsInt(FieldName: "Cycles");
435 ItinString += " { " + itostr(X: Cycles) + ", ";
436
437 // Get unit list
438 ConstRecVec UnitList = Stage->getValueAsListOfDefs(FieldName: "Units");
439
440 // For each unit
441 for (unsigned J = 0, M = UnitList.size(); J < M;) {
442 // Add name and bitwise or
443 ItinString += Name + "FU::" + UnitList[J]->getName().str();
444 if (++J < M)
445 ItinString += " | ";
446 }
447
448 int TimeInc = Stage->getValueAsInt(FieldName: "TimeInc");
449 ItinString += ", " + itostr(X: TimeInc);
450
451 int Kind = Stage->getValueAsInt(FieldName: "Kind");
452 ItinString += ", (llvm::InstrStage::ReservationKinds)" + itostr(X: Kind);
453
454 // Close off stage
455 ItinString += " }";
456 if (++I < N)
457 ItinString += ", ";
458 }
459}
460
461//
462// FormItineraryOperandCycleString - Compose a string containing the
463// operand cycle initialization for the specified itinerary. N is the
464// number of operands that has cycles specified.
465//
466void SubtargetEmitter::formItineraryOperandCycleString(
467 const Record *ItinData, std::string &ItinString, unsigned &NOperandCycles) {
468 // Get operand cycle list
469 std::vector<int64_t> OperandCycleList =
470 ItinData->getValueAsListOfInts(FieldName: "OperandCycles");
471
472 // For each operand cycle
473 NOperandCycles = OperandCycleList.size();
474 ListSeparator LS;
475 for (int OCycle : OperandCycleList) {
476 // Next operand cycle
477 ItinString += LS;
478 ItinString += " " + itostr(X: OCycle);
479 }
480}
481
482void SubtargetEmitter::formItineraryBypassString(const std::string &Name,
483 const Record *ItinData,
484 std::string &ItinString,
485 unsigned NOperandCycles) {
486 ConstRecVec BypassList = ItinData->getValueAsListOfDefs(FieldName: "Bypasses");
487 unsigned N = BypassList.size();
488 unsigned I = 0;
489 ListSeparator LS;
490 for (; I < N; ++I) {
491 ItinString += LS;
492 ItinString += Name + "Bypass::" + BypassList[I]->getName().str();
493 }
494 for (; I < NOperandCycles; ++I) {
495 ItinString += LS;
496 ItinString += " 0";
497 }
498}
499
500//
501// EmitStageAndOperandCycleData - Generate unique itinerary stages and operand
502// cycle tables. Create a list of InstrItinerary objects (ProcItinLists) indexed
503// by CodeGenSchedClass::Index.
504//
505void SubtargetEmitter::emitStageAndOperandCycleData(
506 raw_ostream &OS, std::vector<std::vector<InstrItinerary>> &ProcItinLists) {
507 // Multiple processor models may share an itinerary record. Emit it once.
508 SmallPtrSet<const Record *, 8> ItinsDefSet;
509
510 // Emit functional units for all the itineraries.
511 for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) {
512 if (!ItinsDefSet.insert(Ptr: ProcModel.ItinsDef).second)
513 continue;
514
515 ConstRecVec FUs = ProcModel.ItinsDef->getValueAsListOfDefs(FieldName: "FU");
516 if (FUs.empty())
517 continue;
518
519 StringRef Name = ProcModel.ItinsDef->getName();
520 {
521 OS << "\n// Functional units for \"" << Name << "\"\n";
522 NamespaceEmitter FUNamespace(OS, (Name + Twine("FU")).str());
523
524 for (const auto &[Idx, FU] : enumerate(First&: FUs))
525 OS << " const InstrStage::FuncUnits " << FU->getName() << " = 1ULL << "
526 << Idx << ";\n";
527 }
528
529 ConstRecVec BPs = ProcModel.ItinsDef->getValueAsListOfDefs(FieldName: "BP");
530 if (BPs.empty())
531 continue;
532 OS << "\n// Pipeline forwarding paths for itineraries \"" << Name << "\"\n";
533 NamespaceEmitter BypassNamespace(OS, (Name + Twine("Bypass")).str());
534
535 OS << " const unsigned NoBypass = 0;\n";
536 for (const auto &[Idx, BP] : enumerate(First&: BPs))
537 OS << " const unsigned " << BP->getName() << " = 1 << " << Idx << ";\n";
538 }
539
540 // Begin stages table
541 std::string StageTable =
542 "\nextern const llvm::InstrStage " + Target + "Stages[] = {\n";
543 StageTable += " { 0, 0, 0, llvm::InstrStage::Required }, // No itinerary\n";
544
545 // Begin operand cycle table
546 std::string OperandCycleTable =
547 "extern const unsigned " + Target + "OperandCycles[] = {\n";
548 OperandCycleTable += " 0, // No itinerary\n";
549
550 // Begin pipeline bypass table
551 std::string BypassTable =
552 "extern const unsigned " + Target + "ForwardingPaths[] = {\n";
553 BypassTable += " 0, // No itinerary\n";
554
555 // For each Itinerary across all processors, add a unique entry to the stages,
556 // operand cycles, and pipeline bypass tables. Then add the new Itinerary
557 // object with computed offsets to the ProcItinLists result.
558 unsigned StageCount = 1, OperandCycleCount = 1;
559 StringMap<unsigned> ItinStageMap, ItinOperandMap;
560 for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) {
561 // Add process itinerary to the list.
562 std::vector<InstrItinerary> &ItinList = ProcItinLists.emplace_back();
563
564 // If this processor defines no itineraries, then leave the itinerary list
565 // empty.
566 if (!ProcModel.hasItineraries())
567 continue;
568
569 StringRef Name = ProcModel.ItinsDef->getName();
570
571 ItinList.resize(new_size: SchedModels.numInstrSchedClasses());
572 assert(ProcModel.ItinDefList.size() == ItinList.size() && "bad Itins");
573
574 for (unsigned SchedClassIdx = 0, SchedClassEnd = ItinList.size();
575 SchedClassIdx < SchedClassEnd; ++SchedClassIdx) {
576
577 // Next itinerary data
578 const Record *ItinData = ProcModel.ItinDefList[SchedClassIdx];
579
580 // Get string and stage count
581 std::string ItinStageString;
582 unsigned NStages = 0;
583 if (ItinData)
584 formItineraryStageString(Name: Name.str(), ItinData, ItinString&: ItinStageString,
585 NStages);
586
587 // Get string and operand cycle count
588 std::string ItinOperandCycleString;
589 unsigned NOperandCycles = 0;
590 std::string ItinBypassString;
591 if (ItinData) {
592 formItineraryOperandCycleString(ItinData, ItinString&: ItinOperandCycleString,
593 NOperandCycles);
594
595 formItineraryBypassString(Name: Name.str(), ItinData, ItinString&: ItinBypassString,
596 NOperandCycles);
597 }
598
599 // Check to see if stage already exists and create if it doesn't
600 uint16_t FindStage = 0;
601 if (NStages > 0) {
602 FindStage = ItinStageMap[ItinStageString];
603 if (FindStage == 0) {
604 // Emit as { cycles, u1 | u2 | ... | un, timeinc }, // indices
605 StageTable += ItinStageString + ", // " + itostr(X: StageCount);
606 if (NStages > 1)
607 StageTable += "-" + itostr(X: StageCount + NStages - 1);
608 StageTable += "\n";
609 // Record Itin class number.
610 ItinStageMap[ItinStageString] = FindStage = StageCount;
611 StageCount += NStages;
612 }
613 }
614
615 // Check to see if operand cycle already exists and create if it doesn't
616 uint16_t FindOperandCycle = 0;
617 if (NOperandCycles > 0) {
618 std::string ItinOperandString =
619 ItinOperandCycleString + ItinBypassString;
620 FindOperandCycle = ItinOperandMap[ItinOperandString];
621 if (FindOperandCycle == 0) {
622 // Emit as cycle, // index
623 OperandCycleTable += ItinOperandCycleString + ", // ";
624 std::string OperandIdxComment = itostr(X: OperandCycleCount);
625 if (NOperandCycles > 1)
626 OperandIdxComment +=
627 "-" + itostr(X: OperandCycleCount + NOperandCycles - 1);
628 OperandCycleTable += OperandIdxComment + "\n";
629 // Record Itin class number.
630 ItinOperandMap[ItinOperandCycleString] = FindOperandCycle =
631 OperandCycleCount;
632 // Emit as bypass, // index
633 BypassTable += ItinBypassString + ", // " + OperandIdxComment + "\n";
634 OperandCycleCount += NOperandCycles;
635 }
636 }
637
638 // Set up itinerary as location and location + stage count
639 int16_t NumUOps = ItinData ? ItinData->getValueAsInt(FieldName: "NumMicroOps") : 0;
640 InstrItinerary Intinerary = {
641 .NumMicroOps: NumUOps,
642 .FirstStage: FindStage,
643 .LastStage: uint16_t(FindStage + NStages),
644 .FirstOperandCycle: FindOperandCycle,
645 .LastOperandCycle: uint16_t(FindOperandCycle + NOperandCycles),
646 };
647
648 // Inject - empty slots will be 0, 0
649 ItinList[SchedClassIdx] = Intinerary;
650 }
651 }
652
653 // Closing stage
654 StageTable += " { 0, 0, 0, llvm::InstrStage::Required } // End stages\n";
655 StageTable += "};\n";
656
657 // Closing operand cycles
658 OperandCycleTable += " 0 // End operand cycles\n";
659 OperandCycleTable += "};\n";
660
661 BypassTable += " 0 // End bypass tables\n";
662 BypassTable += "};\n";
663
664 // Emit tables.
665 OS << StageTable;
666 OS << OperandCycleTable;
667 OS << BypassTable;
668}
669
670//
671// EmitProcessorData - Generate data for processor itineraries that were
672// computed during EmitStageAndOperandCycleData(). ProcItinLists lists all
673// Itineraries for each processor. The Itinerary lists are indexed on
674// CodeGenSchedClass::Index.
675//
676void SubtargetEmitter::emitItineraries(
677 raw_ostream &OS, ArrayRef<std::vector<InstrItinerary>> ProcItinLists) {
678 // Multiple processor models may share an itinerary record. Emit it once.
679 SmallPtrSet<const Record *, 8> ItinsDefSet;
680
681 for (const auto &[Proc, ItinList] :
682 zip_equal(t: SchedModels.procModels(), u&: ProcItinLists)) {
683 const Record *ItinsDef = Proc.ItinsDef;
684 if (!ItinsDefSet.insert(Ptr: ItinsDef).second)
685 continue;
686
687 // Empty itineraries aren't referenced anywhere in the tablegen output
688 // so don't emit them.
689 if (ItinList.empty())
690 continue;
691
692 // Begin processor itinerary table
693 OS << "\n";
694 OS << "static constexpr llvm::InstrItinerary " << ItinsDef->getName()
695 << "[] = {\n";
696
697 ArrayRef<CodeGenSchedClass> ItinSchedClasses =
698 SchedModels.schedClasses().take_front(N: ItinList.size());
699
700 // For each itinerary class in CodeGenSchedClass::Index order.
701 for (const auto &[Idx, Intinerary, SchedClass] :
702 enumerate(First: ItinList, Rest&: ItinSchedClasses)) {
703 // Emit Itinerary in the form of
704 // { NumMicroOps, FirstStage, LastStage, FirstOperandCycle,
705 // LastOperandCycle } // index class name
706 OS << " { " << Intinerary.NumMicroOps << ", " << Intinerary.FirstStage
707 << ", " << Intinerary.LastStage << ", " << Intinerary.FirstOperandCycle
708 << ", " << Intinerary.LastOperandCycle << " }" << ", // " << Idx << " "
709 << SchedClass.Name << "\n";
710 }
711 // End processor itinerary table
712 OS << " { 0, uint16_t(~0U), uint16_t(~0U), uint16_t(~0U), uint16_t(~0U) }"
713 "// end marker\n";
714 OS << "};\n";
715 }
716}
717
718// Emit either the value defined in the TableGen Record, or the default
719// value defined in the C++ header. The Record is null if the processor does not
720// define a model.
721void SubtargetEmitter::emitProcessorProp(raw_ostream &OS, const Record *R,
722 StringRef Name, char Separator) {
723 OS << " ";
724 int V = R ? R->getValueAsInt(FieldName: Name) : -1;
725 if (V >= 0)
726 OS << V << Separator << " // " << Name;
727 else
728 OS << "MCSchedModel::Default" << Name << Separator;
729 OS << '\n';
730}
731
732void SubtargetEmitter::emitProcessorResourceSubUnits(
733 const CodeGenProcModel &ProcModel, raw_ostream &OS) {
734 OS << "\nstatic const unsigned " << ProcModel.ModelName
735 << "ProcResourceSubUnits[] = {\n"
736 << " 0, // Invalid\n";
737
738 for (unsigned I = 0, E = ProcModel.ProcResourceDefs.size(); I < E; ++I) {
739 const Record *PRDef = ProcModel.ProcResourceDefs[I];
740 if (!PRDef->isSubClassOf(Name: "ProcResGroup"))
741 continue;
742 for (const Record *RUDef : PRDef->getValueAsListOfDefs(FieldName: "Resources")) {
743 const Record *RU =
744 SchedModels.findProcResUnits(ProcResKind: RUDef, PM: ProcModel, Loc: PRDef->getLoc());
745 for (unsigned J = 0; J < RU->getValueAsInt(FieldName: "NumUnits"); ++J) {
746 OS << " " << ProcModel.getProcResourceIdx(PRDef: RU) << ", ";
747 }
748 }
749 OS << " // " << PRDef->getName() << "\n";
750 }
751 OS << "};\n";
752}
753
754static void emitRetireControlUnitInfo(const CodeGenProcModel &ProcModel,
755 raw_ostream &OS) {
756 int64_t ReorderBufferSize = 0, MaxRetirePerCycle = 0;
757 if (const Record *RCU = ProcModel.RetireControlUnit) {
758 ReorderBufferSize =
759 std::max(a: ReorderBufferSize, b: RCU->getValueAsInt(FieldName: "ReorderBufferSize"));
760 MaxRetirePerCycle =
761 std::max(a: MaxRetirePerCycle, b: RCU->getValueAsInt(FieldName: "MaxRetirePerCycle"));
762 }
763
764 OS << ReorderBufferSize << ", // ReorderBufferSize\n ";
765 OS << MaxRetirePerCycle << ", // MaxRetirePerCycle\n ";
766}
767
768static void emitRegisterFileInfo(const CodeGenProcModel &ProcModel,
769 unsigned NumRegisterFiles,
770 unsigned NumCostEntries, raw_ostream &OS) {
771 if (NumRegisterFiles)
772 OS << ProcModel.ModelName << "RegisterFiles,\n " << (1 + NumRegisterFiles);
773 else
774 OS << "nullptr,\n 0";
775
776 OS << ", // Number of register files.\n ";
777 if (NumCostEntries)
778 OS << ProcModel.ModelName << "RegisterCosts,\n ";
779 else
780 OS << "nullptr,\n ";
781 OS << NumCostEntries << ", // Number of register cost entries.\n";
782}
783
784unsigned
785SubtargetEmitter::emitRegisterFileTables(const CodeGenProcModel &ProcModel,
786 raw_ostream &OS) {
787 if (llvm::all_of(Range: ProcModel.RegisterFiles, P: [](const CodeGenRegisterFile &RF) {
788 return RF.hasDefaultCosts();
789 }))
790 return 0;
791
792 // Print the RegisterCost table first.
793 OS << "\n// {RegisterClassID, Register Cost, AllowMoveElimination }\n";
794 OS << "static const llvm::MCRegisterCostEntry " << ProcModel.ModelName
795 << "RegisterCosts"
796 << "[] = {\n";
797
798 for (const CodeGenRegisterFile &RF : ProcModel.RegisterFiles) {
799 // Skip register files with a default cost table.
800 if (RF.hasDefaultCosts())
801 continue;
802 // Add entries to the cost table.
803 for (const CodeGenRegisterCost &RC : RF.Costs) {
804 OS << " { ";
805 const Record *Rec = RC.RCDef;
806 if (Rec->getValue(Name: "Namespace"))
807 OS << Rec->getValueAsString(FieldName: "Namespace") << "::";
808 OS << Rec->getName() << "RegClassID, " << RC.Cost << ", "
809 << RC.AllowMoveElimination << "},\n";
810 }
811 }
812 OS << "};\n";
813
814 // Now generate a table with register file info.
815 OS << "\n // {Name, #PhysRegs, #CostEntries, IndexToCostTbl, "
816 << "MaxMovesEliminatedPerCycle, AllowZeroMoveEliminationOnly }\n";
817 OS << "static const llvm::MCRegisterFileDesc " << ProcModel.ModelName
818 << "RegisterFiles"
819 << "[] = {\n"
820 << " { \"InvalidRegisterFile\", 0, 0, 0, 0, 0 },\n";
821 unsigned CostTblIndex = 0;
822
823 for (const CodeGenRegisterFile &RD : ProcModel.RegisterFiles) {
824 OS << " { ";
825 OS << '"' << RD.Name << '"' << ", " << RD.NumPhysRegs << ", ";
826 unsigned NumCostEntries = RD.Costs.size();
827 OS << NumCostEntries << ", " << CostTblIndex << ", "
828 << RD.MaxMovesEliminatedPerCycle << ", "
829 << RD.AllowZeroMoveEliminationOnly << "},\n";
830 CostTblIndex += NumCostEntries;
831 }
832 OS << "};\n";
833
834 return CostTblIndex;
835}
836
837void SubtargetEmitter::emitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel,
838 raw_ostream &OS) {
839 unsigned QueueID = 0;
840 if (ProcModel.LoadQueue) {
841 const Record *Queue = ProcModel.LoadQueue->getValueAsDef(FieldName: "QueueDescriptor");
842 QueueID = 1 + std::distance(first: ProcModel.ProcResourceDefs.begin(),
843 last: find(Range: ProcModel.ProcResourceDefs, Val: Queue));
844 }
845 OS << " " << QueueID << ", // Resource Descriptor for the Load Queue\n";
846
847 QueueID = 0;
848 if (ProcModel.StoreQueue) {
849 const Record *Queue =
850 ProcModel.StoreQueue->getValueAsDef(FieldName: "QueueDescriptor");
851 QueueID = 1 + std::distance(first: ProcModel.ProcResourceDefs.begin(),
852 last: find(Range: ProcModel.ProcResourceDefs, Val: Queue));
853 }
854 OS << " " << QueueID << ", // Resource Descriptor for the Store Queue\n";
855}
856
857void SubtargetEmitter::emitExtraProcessorInfo(const CodeGenProcModel &ProcModel,
858 raw_ostream &OS) {
859 // Generate a table of register file descriptors (one entry per each user
860 // defined register file), and a table of register costs.
861 unsigned NumCostEntries = emitRegisterFileTables(ProcModel, OS);
862
863 // Now generate a table for the extra processor info.
864 OS << "\nstatic const llvm::MCExtraProcessorInfo " << ProcModel.ModelName
865 << "ExtraInfo = {\n ";
866
867 // Add information related to the retire control unit.
868 emitRetireControlUnitInfo(ProcModel, OS);
869
870 // Add information related to the register files (i.e. where to find register
871 // file descriptors and register costs).
872 emitRegisterFileInfo(ProcModel, NumRegisterFiles: ProcModel.RegisterFiles.size(),
873 NumCostEntries, OS);
874
875 // Add information about load/store queues.
876 emitLoadStoreQueueInfo(ProcModel, OS);
877
878 OS << "};\n";
879}
880
881void SubtargetEmitter::emitProcessorResources(const CodeGenProcModel &ProcModel,
882 raw_ostream &OS) {
883 emitProcessorResourceSubUnits(ProcModel, OS);
884
885 OS << "\n// {Name, NumUnits, SuperIdx, BufferSize, SubUnitsIdxBegin}\n";
886 OS << "static const llvm::MCProcResourceDesc " << ProcModel.ModelName
887 << "ProcResources"
888 << "[] = {\n"
889 << " {\"InvalidUnit\", 0, 0, 0, 0},\n";
890
891 unsigned SubUnitsOffset = 1;
892 for (unsigned I = 0, E = ProcModel.ProcResourceDefs.size(); I < E; ++I) {
893 const Record *PRDef = ProcModel.ProcResourceDefs[I];
894
895 const Record *SuperDef = nullptr;
896 unsigned SuperIdx = 0;
897 unsigned NumUnits = 0;
898 const unsigned SubUnitsBeginOffset = SubUnitsOffset;
899 int BufferSize = PRDef->getValueAsInt(FieldName: "BufferSize");
900 if (PRDef->isSubClassOf(Name: "ProcResGroup")) {
901 for (const Record *RU : PRDef->getValueAsListOfDefs(FieldName: "Resources")) {
902 NumUnits += RU->getValueAsInt(FieldName: "NumUnits");
903 SubUnitsOffset += RU->getValueAsInt(FieldName: "NumUnits");
904 }
905 } else {
906 // Find the SuperIdx
907 if (PRDef->getValueInit(FieldName: "Super")->isComplete()) {
908 SuperDef = SchedModels.findProcResUnits(ProcResKind: PRDef->getValueAsDef(FieldName: "Super"),
909 PM: ProcModel, Loc: PRDef->getLoc());
910 SuperIdx = ProcModel.getProcResourceIdx(PRDef: SuperDef);
911 }
912 NumUnits = PRDef->getValueAsInt(FieldName: "NumUnits");
913 }
914 // Emit the ProcResourceDesc
915 OS << " {\"" << PRDef->getName() << "\", ";
916 if (PRDef->getName().size() < 15)
917 OS.indent(NumSpaces: 15 - PRDef->getName().size());
918 OS << NumUnits << ", " << SuperIdx << ", " << BufferSize << ", ";
919 if (SubUnitsBeginOffset != SubUnitsOffset) {
920 OS << ProcModel.ModelName << "ProcResourceSubUnits + "
921 << SubUnitsBeginOffset;
922 } else {
923 OS << "nullptr";
924 }
925 OS << "}, // #" << I + 1;
926 if (SuperDef)
927 OS << ", Super=" << SuperDef->getName();
928 OS << "\n";
929 }
930 OS << "};\n";
931}
932
933// Find the WriteRes Record that defines processor resources for this
934// SchedWrite.
935const Record *
936SubtargetEmitter::findWriteResources(const CodeGenSchedRW &SchedWrite,
937 const CodeGenProcModel &ProcModel) {
938
939 // Check if the SchedWrite is already subtarget-specific and directly
940 // specifies a set of processor resources.
941 if (SchedWrite.TheDef->isSubClassOf(Name: "SchedWriteRes"))
942 return SchedWrite.TheDef;
943
944 const Record *AliasDef = nullptr;
945 for (const Record *A : SchedWrite.Aliases) {
946 const CodeGenSchedRW &AliasRW =
947 SchedModels.getSchedRW(Def: A->getValueAsDef(FieldName: "AliasRW"));
948 if (AliasRW.TheDef->getValueInit(FieldName: "SchedModel")->isComplete()) {
949 const Record *ModelDef = AliasRW.TheDef->getValueAsDef(FieldName: "SchedModel");
950 if (&SchedModels.getProcModel(ModelDef) != &ProcModel)
951 continue;
952 }
953 if (AliasDef)
954 PrintFatalError(ErrorLoc: AliasRW.TheDef->getLoc(),
955 Msg: "Multiple aliases "
956 "defined for processor " +
957 ProcModel.ModelName +
958 " Ensure only one SchedAlias exists per RW.");
959 AliasDef = AliasRW.TheDef;
960 }
961 if (AliasDef && AliasDef->isSubClassOf(Name: "SchedWriteRes"))
962 return AliasDef;
963
964 // Check this processor's list of write resources.
965 const Record *ResDef = nullptr;
966
967 auto I = ProcModel.WriteResMap.find(Val: SchedWrite.TheDef);
968 if (I != ProcModel.WriteResMap.end())
969 ResDef = I->second;
970
971 if (AliasDef) {
972 I = ProcModel.WriteResMap.find(Val: AliasDef);
973 if (I != ProcModel.WriteResMap.end()) {
974 if (ResDef)
975 PrintFatalError(ErrorLoc: I->second->getLoc(),
976 Msg: "Resources are defined for both SchedWrite and its "
977 "alias on processor " +
978 ProcModel.ModelName);
979 ResDef = I->second;
980 }
981 }
982
983 // TODO: If ProcModel has a base model (previous generation processor),
984 // then call FindWriteResources recursively with that model here.
985 if (!ResDef) {
986 PrintFatalError(ErrorLoc: ProcModel.ModelDef->getLoc(),
987 Msg: Twine("Processor does not define resources for ") +
988 SchedWrite.TheDef->getName());
989 }
990 return ResDef;
991}
992
993/// Find the ReadAdvance record for the given SchedRead on this processor or
994/// return NULL.
995const Record *
996SubtargetEmitter::findReadAdvance(const CodeGenSchedRW &SchedRead,
997 const CodeGenProcModel &ProcModel) {
998 // Check for SchedReads that directly specify a ReadAdvance.
999 if (SchedRead.TheDef->isSubClassOf(Name: "SchedReadAdvance"))
1000 return SchedRead.TheDef;
1001
1002 // Check this processor's list of aliases for SchedRead.
1003 const Record *AliasDef = nullptr;
1004 for (const Record *A : SchedRead.Aliases) {
1005 const CodeGenSchedRW &AliasRW =
1006 SchedModels.getSchedRW(Def: A->getValueAsDef(FieldName: "AliasRW"));
1007 if (AliasRW.TheDef->getValueInit(FieldName: "SchedModel")->isComplete()) {
1008 const Record *ModelDef = AliasRW.TheDef->getValueAsDef(FieldName: "SchedModel");
1009 if (&SchedModels.getProcModel(ModelDef) != &ProcModel)
1010 continue;
1011 }
1012 if (AliasDef)
1013 PrintFatalError(ErrorLoc: AliasRW.TheDef->getLoc(),
1014 Msg: "Multiple aliases "
1015 "defined for processor " +
1016 ProcModel.ModelName +
1017 " Ensure only one SchedAlias exists per RW.");
1018 AliasDef = AliasRW.TheDef;
1019 }
1020 if (AliasDef && AliasDef->isSubClassOf(Name: "SchedReadAdvance"))
1021 return AliasDef;
1022
1023 // Check this processor's ReadAdvanceList.
1024 const Record *ResDef = nullptr;
1025
1026 auto I = ProcModel.ReadAdvanceMap.find(Val: SchedRead.TheDef);
1027 if (I != ProcModel.ReadAdvanceMap.end())
1028 ResDef = I->second;
1029
1030 if (AliasDef) {
1031 I = ProcModel.ReadAdvanceMap.find(Val: AliasDef);
1032 if (I != ProcModel.ReadAdvanceMap.end()) {
1033 if (ResDef)
1034 PrintFatalError(
1035 ErrorLoc: I->second->getLoc(),
1036 Msg: "Resources are defined for both SchedRead and its alias on "
1037 "processor " +
1038 ProcModel.ModelName);
1039 ResDef = I->second;
1040 }
1041 }
1042
1043 // TODO: If ProcModel has a base model (previous generation processor),
1044 // then call FindReadAdvance recursively with that model here.
1045 if (!ResDef && SchedRead.TheDef->getName() != "ReadDefault") {
1046 PrintFatalError(ErrorLoc: ProcModel.ModelDef->getLoc(),
1047 Msg: Twine("Processor does not define resources for ") +
1048 SchedRead.TheDef->getName());
1049 }
1050 return ResDef;
1051}
1052
1053// Expand an explicit list of processor resources into a full list of implied
1054// resource groups and super resources that cover them.
1055void SubtargetEmitter::expandProcResources(
1056 ConstRecVec &PRVec, std::vector<int64_t> &ReleaseAtCycles,
1057 std::vector<int64_t> &AcquireAtCycles, const CodeGenProcModel &PM) {
1058 assert(PRVec.size() == ReleaseAtCycles.size() && "failed precondition");
1059 for (unsigned I = 0, E = PRVec.size(); I != E; ++I) {
1060 const Record *PRDef = PRVec[I];
1061 ConstRecVec SubResources;
1062 if (PRDef->isSubClassOf(Name: "ProcResGroup")) {
1063 SubResources = PRDef->getValueAsListOfDefs(FieldName: "Resources");
1064 } else {
1065 SubResources.push_back(x: PRDef);
1066 PRDef = SchedModels.findProcResUnits(ProcResKind: PRDef, PM, Loc: PRDef->getLoc());
1067 for (const Record *SubDef = PRDef;
1068 SubDef->getValueInit(FieldName: "Super")->isComplete();) {
1069 if (SubDef->isSubClassOf(Name: "ProcResGroup")) {
1070 // Disallow this for simplicitly.
1071 PrintFatalError(ErrorLoc: SubDef->getLoc(), Msg: "Processor resource group "
1072 " cannot be a super resources.");
1073 }
1074 const Record *SuperDef = SchedModels.findProcResUnits(
1075 ProcResKind: SubDef->getValueAsDef(FieldName: "Super"), PM, Loc: SubDef->getLoc());
1076 PRVec.push_back(x: SuperDef);
1077 ReleaseAtCycles.push_back(x: ReleaseAtCycles[I]);
1078 AcquireAtCycles.push_back(x: AcquireAtCycles[I]);
1079 SubDef = SuperDef;
1080 }
1081 }
1082 for (const Record *PR : PM.ProcResourceDefs) {
1083 if (PR == PRDef || !PR->isSubClassOf(Name: "ProcResGroup"))
1084 continue;
1085 ConstRecVec SuperResources = PR->getValueAsListOfDefs(FieldName: "Resources");
1086 bool AllContained =
1087 all_of(Range&: SubResources, P: [SuperResources](const Record *SubResource) {
1088 return is_contained(Range: SuperResources, Element: SubResource);
1089 });
1090 if (AllContained) {
1091 PRVec.push_back(x: PR);
1092 ReleaseAtCycles.push_back(x: ReleaseAtCycles[I]);
1093 AcquireAtCycles.push_back(x: AcquireAtCycles[I]);
1094 }
1095 }
1096 }
1097}
1098
1099// Generate the SchedClass table for this processor and update global
1100// tables. Must be called for each processor in order.
1101void SubtargetEmitter::genSchedClassTables(const CodeGenProcModel &ProcModel,
1102 SchedClassTables &SchedTables) {
1103 std::vector<MCSchedClassDesc> &SCTab =
1104 SchedTables.ProcSchedClasses.emplace_back();
1105 if (!ProcModel.hasInstrSchedModel())
1106 return;
1107
1108 LLVM_DEBUG(dbgs() << "\n+++ SCHED CLASSES (GenSchedClassTables) +++\n");
1109 for (const CodeGenSchedClass &SC : SchedModels.schedClasses()) {
1110 LLVM_DEBUG(SC.dump(&SchedModels));
1111
1112 MCSchedClassDesc &SCDesc = SCTab.emplace_back();
1113 // SCDesc.Name is guarded by NDEBUG
1114 SCDesc.NumMicroOps = 0;
1115 SCDesc.BeginGroup = false;
1116 SCDesc.EndGroup = false;
1117 SCDesc.RetireOOO = false;
1118 SCDesc.WriteProcResIdx = 0;
1119 SCDesc.WriteLatencyIdx = 0;
1120 SCDesc.ReadAdvanceIdx = 0;
1121
1122 // A Variant SchedClass has no resources of its own.
1123 bool HasVariants = false;
1124 for (const CodeGenSchedTransition &CGT : SC.Transitions) {
1125 if (CGT.ProcIndex == ProcModel.Index) {
1126 HasVariants = true;
1127 break;
1128 }
1129 }
1130 if (HasVariants) {
1131 SCDesc.NumMicroOps = MCSchedClassDesc::VariantNumMicroOps;
1132 continue;
1133 }
1134
1135 // Determine if the SchedClass is actually reachable on this processor. If
1136 // not don't try to locate the processor resources, it will fail.
1137 // If ProcIndices contains 0, this class applies to all processors.
1138 assert(!SC.ProcIndices.empty() && "expect at least one procidx");
1139 if (SC.ProcIndices[0] != 0) {
1140 if (!is_contained(Range: SC.ProcIndices, Element: ProcModel.Index))
1141 continue;
1142 }
1143 IdxVec Writes = SC.Writes;
1144 IdxVec Reads = SC.Reads;
1145 if (!SC.InstRWs.empty()) {
1146 // This class has a default ReadWrite list which can be overridden by
1147 // InstRW definitions.
1148 const Record *RWDef = nullptr;
1149 for (const Record *RW : SC.InstRWs) {
1150 const Record *RWModelDef = RW->getValueAsDef(FieldName: "SchedModel");
1151 if (&ProcModel == &SchedModels.getProcModel(ModelDef: RWModelDef)) {
1152 RWDef = RW;
1153 break;
1154 }
1155 }
1156 if (RWDef) {
1157 Writes.clear();
1158 Reads.clear();
1159 SchedModels.findRWs(RWDefs: RWDef->getValueAsListOfDefs(FieldName: "OperandReadWrites"),
1160 Writes, Reads);
1161 }
1162 }
1163 if (Writes.empty()) {
1164 // Check this processor's itinerary class resources.
1165 for (const Record *I : ProcModel.ItinRWDefs) {
1166 ConstRecVec Matched = I->getValueAsListOfDefs(FieldName: "MatchedItinClasses");
1167 if (is_contained(Range&: Matched, Element: SC.ItinClassDef)) {
1168 SchedModels.findRWs(RWDefs: I->getValueAsListOfDefs(FieldName: "OperandReadWrites"),
1169 Writes, Reads);
1170 break;
1171 }
1172 }
1173 if (Writes.empty()) {
1174 LLVM_DEBUG(dbgs() << ProcModel.ModelName
1175 << " does not have resources for class " << SC.Name
1176 << '\n');
1177 SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps;
1178 }
1179 }
1180 // Sum resources across all operand writes.
1181 std::vector<MCWriteProcResEntry> WriteProcResources;
1182 std::vector<MCWriteLatencyEntry> WriteLatencies;
1183 std::vector<std::string> WriterNames;
1184 std::vector<MCReadAdvanceEntry> ReadAdvanceEntries;
1185 for (unsigned W : Writes) {
1186 IdxVec WriteSeq;
1187 SchedModels.expandRWSeqForProc(RWIdx: W, RWSeq&: WriteSeq, /*IsRead=*/false, ProcModel);
1188
1189 // For each operand, create a latency entry.
1190 MCWriteLatencyEntry WLEntry;
1191 WLEntry.Cycles = 0;
1192 unsigned WriteID = WriteSeq.back();
1193 WriterNames.push_back(x: SchedModels.getSchedWrite(Idx: WriteID).Name);
1194 // If this Write is not referenced by a ReadAdvance, don't distinguish it
1195 // from other WriteLatency entries.
1196 if (!ProcModel.hasReadOfWrite(WriteDef: SchedModels.getSchedWrite(Idx: WriteID).TheDef))
1197 WriteID = 0;
1198 WLEntry.WriteResourceID = WriteID;
1199
1200 for (unsigned WS : WriteSeq) {
1201 const Record *WriteRes =
1202 findWriteResources(SchedWrite: SchedModels.getSchedWrite(Idx: WS), ProcModel);
1203
1204 // Mark the parent class as invalid for unsupported write types.
1205 if (WriteRes->getValueAsBit(FieldName: "Unsupported")) {
1206 SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps;
1207 break;
1208 }
1209 WLEntry.Cycles += WriteRes->getValueAsInt(FieldName: "Latency");
1210 SCDesc.NumMicroOps += WriteRes->getValueAsInt(FieldName: "NumMicroOps");
1211 SCDesc.BeginGroup |= WriteRes->getValueAsBit(FieldName: "BeginGroup");
1212 SCDesc.EndGroup |= WriteRes->getValueAsBit(FieldName: "EndGroup");
1213 SCDesc.BeginGroup |= WriteRes->getValueAsBit(FieldName: "SingleIssue");
1214 SCDesc.EndGroup |= WriteRes->getValueAsBit(FieldName: "SingleIssue");
1215 SCDesc.RetireOOO |= WriteRes->getValueAsBit(FieldName: "RetireOOO");
1216
1217 // Create an entry for each ProcResource listed in WriteRes.
1218 ConstRecVec PRVec = WriteRes->getValueAsListOfDefs(FieldName: "ProcResources");
1219 std::vector<int64_t> ReleaseAtCycles =
1220 WriteRes->getValueAsListOfInts(FieldName: "ReleaseAtCycles");
1221
1222 std::vector<int64_t> AcquireAtCycles =
1223 WriteRes->getValueAsListOfInts(FieldName: "AcquireAtCycles");
1224
1225 // Check consistency of the two vectors carrying the start and
1226 // stop cycles of the resources.
1227 if (!ReleaseAtCycles.empty() &&
1228 ReleaseAtCycles.size() != PRVec.size()) {
1229 // If ReleaseAtCycles is provided, check consistency.
1230 PrintFatalError(
1231 ErrorLoc: WriteRes->getLoc(),
1232 Msg: Twine("Inconsistent release at cycles: size(ReleaseAtCycles) != "
1233 "size(ProcResources): ")
1234 .concat(Suffix: Twine(PRVec.size()))
1235 .concat(Suffix: " vs ")
1236 .concat(Suffix: Twine(ReleaseAtCycles.size())));
1237 }
1238
1239 if (!AcquireAtCycles.empty() &&
1240 AcquireAtCycles.size() != PRVec.size()) {
1241 PrintFatalError(
1242 ErrorLoc: WriteRes->getLoc(),
1243 Msg: Twine("Inconsistent resource cycles: size(AcquireAtCycles) != "
1244 "size(ProcResources): ")
1245 .concat(Suffix: Twine(AcquireAtCycles.size()))
1246 .concat(Suffix: " vs ")
1247 .concat(Suffix: Twine(PRVec.size())));
1248 }
1249
1250 if (ReleaseAtCycles.empty()) {
1251 // If ReleaseAtCycles is not provided, default to one cycle
1252 // per resource.
1253 ReleaseAtCycles.resize(new_size: PRVec.size(), x: 1);
1254 }
1255
1256 if (AcquireAtCycles.empty()) {
1257 // If AcquireAtCycles is not provided, reserve the resource
1258 // starting from cycle 0.
1259 AcquireAtCycles.resize(new_size: PRVec.size(), x: 0);
1260 }
1261
1262 assert(AcquireAtCycles.size() == ReleaseAtCycles.size());
1263
1264 expandProcResources(PRVec, ReleaseAtCycles, AcquireAtCycles, PM: ProcModel);
1265 assert(AcquireAtCycles.size() == ReleaseAtCycles.size());
1266
1267 for (unsigned PRIdx = 0, PREnd = PRVec.size(); PRIdx != PREnd;
1268 ++PRIdx) {
1269 MCWriteProcResEntry WPREntry;
1270 WPREntry.ProcResourceIdx = ProcModel.getProcResourceIdx(PRDef: PRVec[PRIdx]);
1271 assert(WPREntry.ProcResourceIdx && "Bad ProcResourceIdx");
1272 WPREntry.ReleaseAtCycle = ReleaseAtCycles[PRIdx];
1273 WPREntry.AcquireAtCycle = AcquireAtCycles[PRIdx];
1274 if (AcquireAtCycles[PRIdx] > ReleaseAtCycles[PRIdx]) {
1275 PrintFatalError(
1276 ErrorLoc: WriteRes->getLoc(),
1277 Msg: Twine("Inconsistent resource cycles: AcquireAtCycles "
1278 "<= ReleaseAtCycles must hold."));
1279 }
1280 if (AcquireAtCycles[PRIdx] < 0) {
1281 PrintFatalError(ErrorLoc: WriteRes->getLoc(),
1282 Msg: Twine("Invalid value: AcquireAtCycle "
1283 "must be a non-negative value."));
1284 }
1285 // If this resource is already used in this sequence, add the current
1286 // entry's cycles so that the same resource appears to be used
1287 // serially, rather than multiple parallel uses. This is important for
1288 // in-order machine where the resource consumption is a hazard.
1289 unsigned WPRIdx = 0, WPREnd = WriteProcResources.size();
1290 for (; WPRIdx != WPREnd; ++WPRIdx) {
1291 if (WriteProcResources[WPRIdx].ProcResourceIdx ==
1292 WPREntry.ProcResourceIdx) {
1293 // TODO: multiple use of the same resources would
1294 // require either 1. thinking of how to handle multiple
1295 // intervals for the same resource in
1296 // `<Target>WriteProcResTable` (see
1297 // `SubtargetEmitter::EmitSchedClassTables`), or
1298 // 2. thinking how to merge multiple intervals into a
1299 // single interval.
1300 assert(WPREntry.AcquireAtCycle == 0 &&
1301 "multiple use ofthe same resource is not yet handled");
1302 WriteProcResources[WPRIdx].ReleaseAtCycle +=
1303 WPREntry.ReleaseAtCycle;
1304 break;
1305 }
1306 }
1307 if (WPRIdx == WPREnd)
1308 WriteProcResources.push_back(x: WPREntry);
1309 }
1310 }
1311 WriteLatencies.push_back(x: WLEntry);
1312 }
1313 // Create an entry for each operand Read in this SchedClass.
1314 // Entries must be sorted first by UseIdx then by WriteResourceID.
1315 for (unsigned UseIdx = 0, EndIdx = Reads.size(); UseIdx != EndIdx;
1316 ++UseIdx) {
1317 const Record *ReadAdvance =
1318 findReadAdvance(SchedRead: SchedModels.getSchedRead(Idx: Reads[UseIdx]), ProcModel);
1319 if (!ReadAdvance)
1320 continue;
1321
1322 // Mark the parent class as invalid for unsupported write types.
1323 if (ReadAdvance->getValueAsBit(FieldName: "Unsupported")) {
1324 SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps;
1325 break;
1326 }
1327 ConstRecVec ValidWrites =
1328 ReadAdvance->getValueAsListOfDefs(FieldName: "ValidWrites");
1329 std::vector<int64_t> CycleTunables =
1330 ReadAdvance->getValueAsListOfInts(FieldName: "CycleTunables");
1331 std::vector<std::pair<unsigned, int>> WriteIDs;
1332 assert(CycleTunables.size() <= ValidWrites.size() && "Bad ReadAdvance");
1333 CycleTunables.resize(new_size: ValidWrites.size(), x: 0);
1334 if (ValidWrites.empty())
1335 WriteIDs.emplace_back(args: 0, args: 0);
1336 else {
1337 for (const auto [VW, CT] : zip_equal(t&: ValidWrites, u&: CycleTunables)) {
1338 unsigned WriteID = SchedModels.getSchedRWIdx(Def: VW, /*IsRead=*/false);
1339 assert(WriteID != 0 &&
1340 "Expected a valid SchedRW in the list of ValidWrites");
1341 WriteIDs.emplace_back(args&: WriteID, args&: CT);
1342 }
1343 }
1344 llvm::sort(C&: WriteIDs);
1345 for (const auto &[W, T] : WriteIDs) {
1346 MCReadAdvanceEntry &RAEntry = ReadAdvanceEntries.emplace_back();
1347 RAEntry.UseIdx = UseIdx;
1348 RAEntry.WriteResourceID = W;
1349 RAEntry.Cycles = ReadAdvance->getValueAsInt(FieldName: "Cycles") + T;
1350 }
1351 }
1352 if (SCDesc.NumMicroOps == MCSchedClassDesc::InvalidNumMicroOps) {
1353 WriteProcResources.clear();
1354 WriteLatencies.clear();
1355 ReadAdvanceEntries.clear();
1356 }
1357 // Add the information for this SchedClass to the global tables using basic
1358 // compression.
1359 //
1360 // WritePrecRes entries are sorted by ProcResIdx.
1361 llvm::sort(C&: WriteProcResources, Comp: LessWriteProcResources());
1362
1363 SchedTables.MaxWriteProcResEntries =
1364 std::max(a: SchedTables.MaxWriteProcResEntries, b: WriteProcResources.size());
1365 SCDesc.NumWriteProcResEntries = WriteProcResources.size();
1366 std::vector<MCWriteProcResEntry>::iterator WPRPos =
1367 std::search(first1: SchedTables.WriteProcResources.begin(),
1368 last1: SchedTables.WriteProcResources.end(),
1369 first2: WriteProcResources.begin(), last2: WriteProcResources.end());
1370 if (WPRPos != SchedTables.WriteProcResources.end())
1371 SCDesc.WriteProcResIdx = WPRPos - SchedTables.WriteProcResources.begin();
1372 else {
1373 SCDesc.WriteProcResIdx = SchedTables.WriteProcResources.size();
1374 SchedTables.WriteProcResources.insert(position: WPRPos, first: WriteProcResources.begin(),
1375 last: WriteProcResources.end());
1376 }
1377 // Latency entries must remain in operand order.
1378 SchedTables.MaxWriteLatencyEntries =
1379 std::max(a: SchedTables.MaxWriteLatencyEntries, b: WriteLatencies.size());
1380 SCDesc.NumWriteLatencyEntries = WriteLatencies.size();
1381 std::vector<MCWriteLatencyEntry>::iterator WLPos = std::search(
1382 first1: SchedTables.WriteLatencies.begin(), last1: SchedTables.WriteLatencies.end(),
1383 first2: WriteLatencies.begin(), last2: WriteLatencies.end());
1384 if (WLPos != SchedTables.WriteLatencies.end()) {
1385 unsigned Idx = WLPos - SchedTables.WriteLatencies.begin();
1386 SCDesc.WriteLatencyIdx = Idx;
1387 for (unsigned I = 0, E = WriteLatencies.size(); I < E; ++I)
1388 if (SchedTables.WriterNames[Idx + I].find(str: WriterNames[I]) ==
1389 std::string::npos) {
1390 SchedTables.WriterNames[Idx + I] += "_" + WriterNames[I];
1391 }
1392 } else {
1393 SCDesc.WriteLatencyIdx = SchedTables.WriteLatencies.size();
1394 llvm::append_range(C&: SchedTables.WriteLatencies, R&: WriteLatencies);
1395 llvm::append_range(C&: SchedTables.WriterNames, R&: WriterNames);
1396 }
1397 // ReadAdvanceEntries must remain in operand order.
1398 SCDesc.NumReadAdvanceEntries = ReadAdvanceEntries.size();
1399 std::vector<MCReadAdvanceEntry>::iterator RAPos =
1400 std::search(first1: SchedTables.ReadAdvanceEntries.begin(),
1401 last1: SchedTables.ReadAdvanceEntries.end(),
1402 first2: ReadAdvanceEntries.begin(), last2: ReadAdvanceEntries.end());
1403 if (RAPos != SchedTables.ReadAdvanceEntries.end())
1404 SCDesc.ReadAdvanceIdx = RAPos - SchedTables.ReadAdvanceEntries.begin();
1405 else {
1406 SCDesc.ReadAdvanceIdx = SchedTables.ReadAdvanceEntries.size();
1407 llvm::append_range(C&: SchedTables.ReadAdvanceEntries, R&: ReadAdvanceEntries);
1408 }
1409 }
1410}
1411
1412// Emit SchedClass tables for all processors and associated global tables.
1413void SubtargetEmitter::emitSchedClassTables(SchedClassTables &SchedTables,
1414 raw_ostream &OS) {
1415 OS << "\nstatic_assert(" << SchedTables.MaxWriteProcResEntries
1416 << " <= UINT8_MAX, \"NumWriteProcResEntries does not fit in uint8_t\");\n"
1417 << "static_assert(" << SchedTables.MaxWriteLatencyEntries
1418 << " <= UINT8_MAX, \"NumWriteLatencyEntries does not fit in uint8_t\");\n";
1419
1420 // Emit global WriteProcResTable.
1421 OS << "\n// {ProcResourceIdx, ReleaseAtCycle, AcquireAtCycle}\n"
1422 << "extern const llvm::MCWriteProcResEntry " << Target
1423 << "WriteProcResTable[] = {\n"
1424 << " { 0, 0, 0 }, // Invalid\n";
1425 for (unsigned WPRIdx = 1, WPREnd = SchedTables.WriteProcResources.size();
1426 WPRIdx != WPREnd; ++WPRIdx) {
1427 MCWriteProcResEntry &WPREntry = SchedTables.WriteProcResources[WPRIdx];
1428 OS << " {" << format(Fmt: "%2d", Vals: WPREntry.ProcResourceIdx) << ", "
1429 << format(Fmt: "%2d", Vals: WPREntry.ReleaseAtCycle) << ", "
1430 << format(Fmt: "%2d", Vals: WPREntry.AcquireAtCycle) << "}";
1431 if (WPRIdx + 1 < WPREnd)
1432 OS << ',';
1433 OS << " // #" << WPRIdx << '\n';
1434 }
1435 OS << "}; // " << Target << "WriteProcResTable\n";
1436
1437 // Emit global WriteLatencyTable.
1438 OS << "\n// {Cycles, WriteResourceID}\n"
1439 << "extern const llvm::MCWriteLatencyEntry " << Target
1440 << "WriteLatencyTable[] = {\n"
1441 << " { 0, 0}, // Invalid\n";
1442 for (unsigned WLIdx = 1, WLEnd = SchedTables.WriteLatencies.size();
1443 WLIdx != WLEnd; ++WLIdx) {
1444 MCWriteLatencyEntry &WLEntry = SchedTables.WriteLatencies[WLIdx];
1445 OS << " {" << format(Fmt: "%2d", Vals: WLEntry.Cycles) << ", "
1446 << format(Fmt: "%2d", Vals: WLEntry.WriteResourceID) << "}";
1447 if (WLIdx + 1 < WLEnd)
1448 OS << ',';
1449 OS << " // #" << WLIdx << " " << SchedTables.WriterNames[WLIdx] << '\n';
1450 }
1451 OS << "}; // " << Target << "WriteLatencyTable\n";
1452
1453 // Emit global ReadAdvanceTable.
1454 OS << "\n// {UseIdx, WriteResourceID, Cycles}\n"
1455 << "extern const llvm::MCReadAdvanceEntry " << Target
1456 << "ReadAdvanceTable[] = {\n"
1457 << " {0, 0, 0}, // Invalid\n";
1458 for (unsigned RAIdx = 1, RAEnd = SchedTables.ReadAdvanceEntries.size();
1459 RAIdx != RAEnd; ++RAIdx) {
1460 MCReadAdvanceEntry &RAEntry = SchedTables.ReadAdvanceEntries[RAIdx];
1461 OS << " {" << RAEntry.UseIdx << ", "
1462 << format(Fmt: "%2d", Vals: RAEntry.WriteResourceID) << ", "
1463 << format(Fmt: "%2d", Vals: RAEntry.Cycles) << "}";
1464 if (RAIdx + 1 < RAEnd)
1465 OS << ',';
1466 OS << " // #" << RAIdx << '\n';
1467 }
1468 OS << "}; // " << Target << "ReadAdvanceTable\n";
1469
1470 // Pool all SchedClass names in a string table.
1471 StringToOffsetTable StrTab;
1472 unsigned InvalidNameOff = StrTab.GetOrAddStringOffset(Str: "InvalidSchedClass");
1473
1474 // Emit a SchedClass table for each processor.
1475 for (const auto &[Idx, Proc] : enumerate(First: SchedModels.procModels())) {
1476 if (!Proc.hasInstrSchedModel())
1477 continue;
1478
1479 std::vector<MCSchedClassDesc> &SCTab =
1480 SchedTables.ProcSchedClasses[1 + Idx];
1481
1482 OS << "\n// {Name, NumMicroOps, BeginGroup, EndGroup, RetireOOO,"
1483 << " ReadAdvanceIdx, WriteProcResIdx, WriteLatencyIdx,"
1484 << " NumReadAdvanceEntries, NumWriteProcResEntries,"
1485 << " NumWriteLatencyEntries}\n";
1486 OS << "static const llvm::MCSchedClassDesc " << Proc.ModelName
1487 << "SchedClasses[] = {\n";
1488
1489 // The first class is always invalid. We no way to distinguish it except by
1490 // name and position.
1491 assert(SchedModels.getSchedClass(0).Name == "NoInstrModel" &&
1492 "invalid class not first");
1493 OS << " {DBGFIELD(" << InvalidNameOff << ") "
1494 << MCSchedClassDesc::InvalidNumMicroOps
1495 << ", false, false, false, 0, 0, 0, 0, 0, 0},\n";
1496
1497 for (unsigned SCIdx = 1, SCEnd = SCTab.size(); SCIdx != SCEnd; ++SCIdx) {
1498 MCSchedClassDesc &MCDesc = SCTab[SCIdx];
1499 const CodeGenSchedClass &SchedClass = SchedModels.getSchedClass(Idx: SCIdx);
1500 unsigned NameOff = StrTab.GetOrAddStringOffset(Str: SchedClass.Name);
1501 OS << " {DBGFIELD(/*" << SchedClass.Name << "*/ " << NameOff << ") ";
1502 if (SchedClass.Name.size() < 18)
1503 OS.indent(NumSpaces: 18 - SchedClass.Name.size());
1504 OS << MCDesc.NumMicroOps << ", " << (MCDesc.BeginGroup ? "true" : "false")
1505 << ", " << (MCDesc.EndGroup ? "true" : "false") << ", "
1506 << (MCDesc.RetireOOO ? "true" : "false") << ", "
1507 << format(Fmt: "%2d", Vals: MCDesc.ReadAdvanceIdx) << ", "
1508 << format(Fmt: "%2d", Vals: MCDesc.WriteProcResIdx) << ", "
1509 << format(Fmt: "%2d", Vals: MCDesc.WriteLatencyIdx) << ", "
1510 << MCDesc.NumReadAdvanceEntries << ", "
1511 << static_cast<unsigned>(MCDesc.NumWriteProcResEntries) << ", "
1512 << static_cast<unsigned>(MCDesc.NumWriteLatencyEntries) << "}, // #"
1513 << SCIdx << '\n';
1514 }
1515 OS << "}; // " << Proc.ModelName << "SchedClasses\n";
1516 }
1517
1518 StrTab.EmitStringTableDef(OS, Name: Target + "SchedClassNames");
1519}
1520
1521void SubtargetEmitter::emitProcessorModels(raw_ostream &OS) {
1522 // For each processor model.
1523 for (const CodeGenProcModel &PM : SchedModels.procModels()) {
1524 // Emit extra processor info if available.
1525 if (PM.hasExtraProcessorInfo())
1526 emitExtraProcessorInfo(ProcModel: PM, OS);
1527 // Emit processor resource table.
1528 if (PM.hasInstrSchedModel())
1529 emitProcessorResources(ProcModel: PM, OS);
1530 else if (!PM.ProcResourceDefs.empty())
1531 PrintFatalError(ErrorLoc: PM.ModelDef->getLoc(),
1532 Msg: "SchedMachineModel defines "
1533 "ProcResources without defining WriteRes SchedWriteRes");
1534 }
1535
1536 OS << "\n";
1537 OS << "extern const llvm::MCSchedModel " << Target << "SchedModels[] = {\n";
1538 for (const CodeGenProcModel &PM : SchedModels.procModels()) {
1539 // Begin processor itinerary properties
1540 OS << "{ // " << PM.ModelName << "\n";
1541 emitProcessorProp(OS, R: PM.ModelDef, Name: "IssueWidth", Separator: ',');
1542 emitProcessorProp(OS, R: PM.ModelDef, Name: "MicroOpBufferSize", Separator: ',');
1543 emitProcessorProp(OS, R: PM.ModelDef, Name: "LoopMicroOpBufferSize", Separator: ',');
1544 emitProcessorProp(OS, R: PM.ModelDef, Name: "LoadLatency", Separator: ',');
1545 emitProcessorProp(OS, R: PM.ModelDef, Name: "HighLatency", Separator: ',');
1546 emitProcessorProp(OS, R: PM.ModelDef, Name: "MispredictPenalty", Separator: ',');
1547
1548 bool PostRAScheduler =
1549 (PM.ModelDef ? PM.ModelDef->getValueAsBit(FieldName: "PostRAScheduler") : false);
1550
1551 OS << " " << (PostRAScheduler ? "true" : "false") << ", // "
1552 << "PostRAScheduler\n";
1553
1554 bool CompleteModel =
1555 (PM.ModelDef ? PM.ModelDef->getValueAsBit(FieldName: "CompleteModel") : false);
1556
1557 OS << " " << (CompleteModel ? "true" : "false") << ", // "
1558 << "CompleteModel\n";
1559
1560 bool EnableIntervals =
1561 (PM.ModelDef ? PM.ModelDef->getValueAsBit(FieldName: "EnableIntervals") : false);
1562
1563 OS << " " << (EnableIntervals ? "true" : "false") << ", // "
1564 << "EnableIntervals\n";
1565
1566 OS << " " << PM.Index << ", // Processor ID\n";
1567 if (PM.hasInstrSchedModel())
1568 OS << " " << PM.ModelName << "ProcResources" << ",\n"
1569 << " " << PM.ModelName << "SchedClasses" << ",\n"
1570 << " " << PM.ProcResourceDefs.size() + 1 << ",\n"
1571 << " " << SchedModels.schedClasses().size() << ",\n";
1572 else
1573 OS << " nullptr, nullptr, 0, 0,"
1574 << " // No instruction-level machine model.\n";
1575 OS << " DBGVAL_OR_NULLPTR(&" << Target
1576 << "SchedClassNames), // SchedClassNames\n";
1577 if (PM.hasItineraries())
1578 OS << " " << PM.ItinsDef->getName() << ",\n";
1579 else
1580 OS << " nullptr, // No Itinerary\n";
1581 if (PM.hasExtraProcessorInfo())
1582 OS << " &" << PM.ModelName << "ExtraInfo,\n";
1583 else
1584 OS << " nullptr // No extra processor descriptor\n";
1585 OS << " },\n";
1586 }
1587 OS << "};\n";
1588}
1589
1590//
1591// EmitSchedModel - Emits all scheduling model tables, folding common patterns.
1592//
1593void SubtargetEmitter::emitSchedModel(raw_ostream &OS) {
1594 OS << "#ifdef DBGFIELD\n"
1595 << "#error \"<target>GenSubtargetInfo.inc requires a DBGFIELD macro\"\n"
1596 << "#endif\n"
1597 << "#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)\n"
1598 << "#define DBGFIELD(x) x,\n"
1599 << "#define DBGVAL_OR_NULLPTR(x) x\n"
1600 << "#else\n"
1601 << "#define DBGFIELD(x)\n"
1602 << "#define DBGVAL_OR_NULLPTR(x) nullptr\n"
1603 << "#endif\n";
1604
1605 if (SchedModels.hasItineraries()) {
1606 std::vector<std::vector<InstrItinerary>> ProcItinLists;
1607 // Emit the stage data
1608 emitStageAndOperandCycleData(OS, ProcItinLists);
1609 emitItineraries(OS, ProcItinLists);
1610 }
1611 OS << "\n// ===============================================================\n"
1612 << "// Data tables for the new per-operand machine model.\n";
1613
1614 SchedClassTables SchedTables;
1615 for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) {
1616 genSchedClassTables(ProcModel, SchedTables);
1617 }
1618 emitSchedClassTables(SchedTables, OS);
1619
1620 // Emit the processor machine model
1621 emitProcessorModels(OS);
1622
1623 OS << "\n#undef DBGFIELD\n";
1624 OS << "\n#undef DBGVAL_OR_NULLPTR\n";
1625}
1626
1627static void emitPredicateProlog(const RecordKeeper &Records, raw_ostream &OS) {
1628 std::string Buffer;
1629 raw_string_ostream Stream(Buffer);
1630
1631 // Print all PredicateProlog records to the output stream.
1632 for (const Record *P : Records.getAllDerivedDefinitions(ClassName: "PredicateProlog"))
1633 Stream << P->getValueAsString(FieldName: "Code") << '\n';
1634
1635 OS << Buffer;
1636}
1637
1638static bool isTruePredicate(const Record *Rec) {
1639 return Rec->isSubClassOf(Name: "MCSchedPredicate") &&
1640 Rec->getValueAsDef(FieldName: "Pred")->isSubClassOf(Name: "MCTrue");
1641}
1642
1643static void expandSchedPredicates(const Record *Rec, PredicateExpander &PE,
1644 bool WrapPredicate, raw_ostream &OS) {
1645 if (Rec->isSubClassOf(Name: "MCSchedPredicate")) {
1646 PE.expandPredicate(OS, Rec: Rec->getValueAsDef(FieldName: "Pred"));
1647 } else if (Rec->isSubClassOf(Name: "FeatureSchedPredicate")) {
1648 const Record *FR = Rec->getValueAsDef(FieldName: "Feature");
1649 if (PE.shouldExpandForMC()) {
1650 // MC version of this predicate will be emitted into
1651 // resolveVariantSchedClassImpl, which accesses MCSubtargetInfo
1652 // through argument STI.
1653 OS << "STI.";
1654 } else {
1655 // Otherwise, this predicate will be emitted directly into
1656 // TargetGenSubtargetInfo::resolveSchedClass, which can just access
1657 // TargetSubtargetInfo / MCSubtargetInfo through `this`.
1658 OS << "this->";
1659 }
1660 OS << "hasFeature(" << PE.getTargetName() << "::" << FR->getName() << ")";
1661 } else if (Rec->isSubClassOf(Name: "SchedPredicateCombiner")) {
1662 std::vector<const Record *> SubPreds =
1663 Rec->getValueAsListOfDefs(FieldName: "Predicates");
1664 if (SubPreds.empty())
1665 PrintFatalError(Rec, Msg: "Empty SchedPredicateCombiner is not allowed");
1666
1667 StringRef Sep;
1668 if (Rec->isSubClassOf(Name: "AllOfSchedPreds")) {
1669 Sep = " && ";
1670 } else if (Rec->isSubClassOf(Name: "AnyOfSchedPreds")) {
1671 Sep = " || ";
1672 } else if (Rec->isSubClassOf(Name: "NotSchedPred")) {
1673 if (SubPreds.size() != 1)
1674 PrintFatalError(Rec,
1675 Msg: "NotSchedPred can only have a single sub-predicate.");
1676 OS << "!";
1677 // We don't have to eagerly wrap this term right now: telling its (only)
1678 // sub-predicate to wrap itself should be sufficient.
1679 WrapPredicate = false;
1680 } else {
1681 PrintFatalError(Rec, Msg: "Unrecognized SchedPredicateCombiner");
1682 }
1683
1684 if (WrapPredicate)
1685 OS << "(";
1686
1687 ListSeparator LS(Sep);
1688 bool WrapSubPreds =
1689 SubPreds.size() > 1 || Rec->isSubClassOf(Name: "NotSchedPred");
1690 for (const Record *SubP : SubPreds)
1691 expandSchedPredicates(Rec: SubP, PE, WrapPredicate: WrapSubPreds, OS&: OS << LS);
1692
1693 if (WrapPredicate)
1694 OS << ")";
1695 } else {
1696 // Expand this legacy predicate and wrap it around braces if there is more
1697 // than one predicate to expand.
1698 OS << (WrapPredicate ? "(" : "") << Rec->getValueAsString(FieldName: "Predicate")
1699 << (WrapPredicate ? ")" : "");
1700 }
1701}
1702
1703static void emitPredicates(const CodeGenSchedTransition &T,
1704 const CodeGenSchedClass &SC, PredicateExpander &PE,
1705 raw_ostream &OS) {
1706 std::string Buffer;
1707 raw_string_ostream SS(Buffer);
1708
1709 // If not all predicates are MCTrue, then we need an if-stmt.
1710 unsigned NumNonTruePreds =
1711 T.PredTerm.size() - count_if(Range: T.PredTerm, P: isTruePredicate);
1712
1713 SS << PE.getIndent();
1714
1715 if (NumNonTruePreds) {
1716 bool FirstNonTruePredicate = true;
1717 SS << "if (";
1718
1719 PE.getIndent() += 2;
1720
1721 for (const Record *Rec : T.PredTerm) {
1722 // Skip predicates that evaluate to "true".
1723 if (isTruePredicate(Rec))
1724 continue;
1725
1726 if (FirstNonTruePredicate) {
1727 FirstNonTruePredicate = false;
1728 } else {
1729 SS << "\n";
1730 SS << PE.getIndent();
1731 SS << "&& ";
1732 }
1733
1734 expandSchedPredicates(Rec, PE, /*WrapPredicate=*/NumNonTruePreds > 1, OS&: SS);
1735 }
1736
1737 SS << ")\n"; // end of if-stmt
1738 --PE.getIndent();
1739 SS << PE.getIndent();
1740 --PE.getIndent();
1741 }
1742
1743 SS << "return " << T.ToClassIdx << "; // " << SC.Name << '\n';
1744 OS << Buffer;
1745}
1746
1747// Used by method `SubtargetEmitter::emitSchedModelHelpersImpl()` to generate
1748// epilogue code for the auto-generated helper.
1749static void emitSchedModelHelperEpilogue(raw_ostream &OS,
1750 bool ShouldReturnZero) {
1751 if (ShouldReturnZero) {
1752 OS << " // Don't know how to resolve this scheduling class.\n"
1753 << " return 0;\n";
1754 return;
1755 }
1756
1757 OS << " report_fatal_error(\"Expected a variant SchedClass\");\n";
1758}
1759
1760static bool hasMCSchedPredicate(const Record *Rec) {
1761 if (Rec->isSubClassOf(Name: "MCSchedPredicate") ||
1762 Rec->isSubClassOf(Name: "FeatureSchedPredicate"))
1763 return true;
1764
1765 if (Rec->isSubClassOf(Name: "SchedPredicateCombiner")) {
1766 // Check its sub-predicates recursively.
1767 std::vector<const Record *> SubPreds =
1768 Rec->getValueAsListOfDefs(FieldName: "Predicates");
1769 return all_of(Range&: SubPreds, P: hasMCSchedPredicate);
1770 }
1771
1772 return false;
1773}
1774static bool hasMCSchedPredicates(const CodeGenSchedTransition &T) {
1775 return all_of(Range: T.PredTerm, P: hasMCSchedPredicate);
1776}
1777
1778static void collectVariantClasses(const CodeGenSchedModels &SchedModels,
1779 IdxVec &VariantClasses,
1780 bool OnlyExpandMCInstPredicates) {
1781 for (const CodeGenSchedClass &SC : SchedModels.schedClasses()) {
1782 // Ignore non-variant scheduling classes.
1783 if (SC.Transitions.empty())
1784 continue;
1785
1786 if (OnlyExpandMCInstPredicates) {
1787 // Ignore this variant scheduling class no transitions use any meaningful
1788 // MCSchedPredicate definitions.
1789 if (llvm::none_of(Range: SC.Transitions, P: hasMCSchedPredicates))
1790 continue;
1791 }
1792
1793 VariantClasses.push_back(x: SC.Index);
1794 }
1795}
1796
1797static void collectProcessorIndices(const CodeGenSchedClass &SC,
1798 IdxVec &ProcIndices) {
1799 // A variant scheduling class may define transitions for multiple
1800 // processors. This function identifies wich processors are associated with
1801 // transition rules specified by variant class `SC`.
1802 for (const CodeGenSchedTransition &T : SC.Transitions) {
1803 IdxVec PI;
1804 std::set_union(first1: &T.ProcIndex, last1: &T.ProcIndex + 1, first2: ProcIndices.begin(),
1805 last2: ProcIndices.end(), result: std::back_inserter(x&: PI));
1806 ProcIndices = std::move(PI);
1807 }
1808}
1809
1810static bool isAlwaysTrue(const CodeGenSchedTransition &T) {
1811 return llvm::all_of(Range: T.PredTerm, P: isTruePredicate);
1812}
1813
1814void SubtargetEmitter::emitSchedModelHelpersImpl(
1815 raw_ostream &OS, bool OnlyExpandMCInstPredicates) {
1816 IdxVec VariantClasses;
1817 collectVariantClasses(SchedModels, VariantClasses,
1818 OnlyExpandMCInstPredicates);
1819
1820 if (VariantClasses.empty()) {
1821 emitSchedModelHelperEpilogue(OS, ShouldReturnZero: OnlyExpandMCInstPredicates);
1822 return;
1823 }
1824
1825 // Construct a switch statement where the condition is a check on the
1826 // scheduling class identifier. There is a `case` for every variant class
1827 // defined by the processor models of this target.
1828 // Each `case` implements a number of rules to resolve (i.e. to transition
1829 // from) a variant scheduling class to another scheduling class. Rules are
1830 // described by instances of CodeGenSchedTransition. Note that transitions may
1831 // not be valid for all processors.
1832 OS << " switch (SchedClass) {\n";
1833 for (unsigned VC : VariantClasses) {
1834 IdxVec ProcIndices;
1835 const CodeGenSchedClass &SC = SchedModels.getSchedClass(Idx: VC);
1836 collectProcessorIndices(SC, ProcIndices);
1837
1838 OS << " case " << VC << ": // " << SC.Name << '\n';
1839
1840 PredicateExpander PE(Target);
1841 PE.setByRef(false);
1842 PE.setExpandForMC(OnlyExpandMCInstPredicates);
1843 for (unsigned PI : ProcIndices) {
1844 OS << " ";
1845
1846 // Emit a guard on the processor ID.
1847 if (PI != 0) {
1848 OS << (OnlyExpandMCInstPredicates
1849 ? "if (CPUID == "
1850 : "if (SchedModel->getProcessorID() == ");
1851 OS << PI << ") ";
1852 OS << "{ // " << SchedModels.procModels()[PI].ModelName << '\n';
1853 }
1854
1855 // Now emit transitions associated with processor PI.
1856 const CodeGenSchedTransition *FinalT = nullptr;
1857 for (const CodeGenSchedTransition &T : SC.Transitions) {
1858 if (PI != 0 && T.ProcIndex != PI)
1859 continue;
1860
1861 // Emit only transitions based on MCSchedPredicate, if it's the case.
1862 // At least the transition specified by NoSchedPred is emitted,
1863 // which becomes the default transition for those variants otherwise
1864 // not based on MCSchedPredicate.
1865 // FIXME: preferably, llvm-mca should instead assume a reasonable
1866 // default when a variant transition is not based on MCSchedPredicate
1867 // for a given processor.
1868 if (OnlyExpandMCInstPredicates && !hasMCSchedPredicates(T))
1869 continue;
1870
1871 // If transition is folded to 'return X' it should be the last one.
1872 if (isAlwaysTrue(T)) {
1873 FinalT = &T;
1874 continue;
1875 }
1876 PE.getIndent() = 3;
1877 emitPredicates(T, SC: SchedModels.getSchedClass(Idx: T.ToClassIdx), PE, OS);
1878 }
1879 if (FinalT)
1880 emitPredicates(T: *FinalT, SC: SchedModels.getSchedClass(Idx: FinalT->ToClassIdx),
1881 PE, OS);
1882
1883 OS << " }\n";
1884
1885 if (PI == 0)
1886 break;
1887 }
1888
1889 if (SC.isInferred())
1890 OS << " return " << SC.Index << ";\n";
1891 OS << " break;\n";
1892 }
1893
1894 OS << " };\n";
1895
1896 emitSchedModelHelperEpilogue(OS, ShouldReturnZero: OnlyExpandMCInstPredicates);
1897}
1898
1899void SubtargetEmitter::emitSchedModelHelpers(const std::string &ClassName,
1900 raw_ostream &OS) {
1901 OS << "unsigned " << ClassName
1902 << "\n::resolveSchedClass(unsigned SchedClass, const MachineInstr *MI,"
1903 << " const TargetSchedModel *SchedModel) const {\n";
1904
1905 // Emit the predicate prolog code.
1906 emitPredicateProlog(Records, OS);
1907
1908 // Emit target predicates.
1909 emitSchedModelHelpersImpl(OS);
1910
1911 OS << "} // " << ClassName << "::resolveSchedClass\n\n";
1912
1913 OS << "unsigned " << ClassName
1914 << "\n::resolveVariantSchedClass(unsigned SchedClass, const MCInst *MI,"
1915 << " const MCInstrInfo *MCII, unsigned CPUID) const {\n"
1916 << " return " << Target << "_MC"
1917 << "::resolveVariantSchedClassImpl(SchedClass, MI, MCII, *this, CPUID);\n"
1918 << "} // " << ClassName << "::resolveVariantSchedClass\n\n";
1919
1920 STIPredicateExpander PE(Target, /*Indent=*/0);
1921 PE.setClassPrefix(ClassName);
1922 PE.setExpandDefinition(true);
1923 PE.setByRef(false);
1924
1925 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
1926 PE.expandSTIPredicate(OS, Fn);
1927}
1928
1929void SubtargetEmitter::emitHwModeCheck(const std::string &ClassName,
1930 raw_ostream &OS, bool IsMC) {
1931 const CodeGenHwModes &CGH = TGT.getHwModes();
1932 assert(CGH.getNumModeIds() > 0);
1933 if (CGH.getNumModeIds() == 1)
1934 return;
1935
1936 // Collect all HwModes and related features defined in the TD files,
1937 // and store them as a bit set.
1938 unsigned ValueTypeModes = 0;
1939 unsigned RegInfoModes = 0;
1940 unsigned EncodingInfoModes = 0;
1941 for (const auto &MS : CGH.getHwModeSelects()) {
1942 for (const HwModeSelect::PairType &P : MS.second.Items) {
1943 if (P.first == DefaultMode)
1944 continue;
1945 if (P.second->isSubClassOf(Name: "ValueType")) {
1946 ValueTypeModes |= (1 << (P.first - 1));
1947 } else if (P.second->isSubClassOf(Name: "RegInfo") ||
1948 P.second->isSubClassOf(Name: "Register") ||
1949 P.second->isSubClassOf(Name: "SubRegRange") ||
1950 P.second->isSubClassOf(Name: "RegisterClassLike")) {
1951 RegInfoModes |= (1 << (P.first - 1));
1952 } else if (P.second->isSubClassOf(Name: "InstructionEncoding")) {
1953 EncodingInfoModes |= (1 << (P.first - 1));
1954 }
1955 }
1956 }
1957
1958 // Start emitting for getHwModeSet().
1959 OS << "unsigned " << ClassName << "::getHwModeSet() const {\n";
1960 if (IsMC) {
1961 OS << " [[maybe_unused]] const FeatureBitset &FB = getFeatureBits();\n";
1962 } else {
1963 const ArrayRef<const Record *> &Prologs =
1964 Records.getAllDerivedDefinitions(ClassName: "HwModePredicateProlog");
1965 if (!Prologs.empty()) {
1966 for (const Record *P : Prologs)
1967 OS << P->getValueAsString(FieldName: "Code") << '\n';
1968 } else {
1969 // Works for most targets.
1970 OS << " [[maybe_unused]] const auto *Subtarget =\n"
1971 << " static_cast<const " << Target << "Subtarget *>(this);\n";
1972 }
1973 }
1974 OS << " // Collect HwModes and store them as a bit set.\n";
1975 OS << " unsigned Modes = 0;\n";
1976 for (unsigned M = 1, NumModes = CGH.getNumModeIds(); M != NumModes; ++M) {
1977 const HwMode &HM = CGH.getMode(Id: M);
1978 OS << " if (";
1979 if (IsMC)
1980 SubtargetFeatureInfo::emitMCPredicateCheck(OS, TargetName: Target, Predicates: HM.Predicates);
1981 else
1982 SubtargetFeatureInfo::emitPredicateCheck(OS, Predicates: HM.Predicates);
1983 OS << ") Modes |= (1 << " << (M - 1) << ");\n";
1984 }
1985 OS << " return Modes;\n}\n";
1986 // End emitting for getHwModeSet().
1987
1988 auto HandlePerMode = [&](std::string ModeType, unsigned ModeInBitSet) {
1989 OS << " case HwMode_" << ModeType << ":\n";
1990 if (ModeInBitSet == 0) {
1991 OS << " // No HwMode for " << ModeType << ".\n"
1992 << " return 0;\n";
1993 } else {
1994 OS << " Modes &= " << ModeInBitSet << ";\n"
1995 << " if (!Modes)\n return Modes;\n"
1996 << " if (!llvm::has_single_bit<unsigned>(Modes))\n"
1997 << " llvm_unreachable(\"Two or more HwModes for " << ModeType
1998 << " were found!\");\n"
1999 << " return llvm::countr_zero(Modes) + 1;\n";
2000 }
2001 };
2002
2003 // Start emitting for getHwMode().
2004 OS << "unsigned " << ClassName
2005 << "::getHwMode(enum HwModeType type) const {\n";
2006 OS << " unsigned Modes = getHwModeSet();\n\n";
2007 OS << " if (!Modes)\n return Modes;\n\n";
2008 OS << " switch (type) {\n";
2009 OS << " case HwMode_Default:\n return llvm::countr_zero(Modes) + 1;\n";
2010 HandlePerMode("ValueType", ValueTypeModes);
2011 HandlePerMode("RegInfo", RegInfoModes);
2012 HandlePerMode("EncodingInfo", EncodingInfoModes);
2013 OS << " }\n";
2014 OS << " llvm_unreachable(\"unexpected HwModeType\");\n"
2015 << " return 0; // should not get here\n}\n";
2016 // End emitting for getHwMode().
2017}
2018
2019void SubtargetEmitter::emitGetMacroFusions(const std::string &ClassName,
2020 raw_ostream &OS) {
2021 if (!TGT.hasMacroFusion())
2022 return;
2023
2024 OS << "std::vector<MacroFusionPredTy> " << ClassName
2025 << "::getMacroFusions() const {\n";
2026 OS.indent(NumSpaces: 2) << "std::vector<MacroFusionPredTy> Fusions;\n";
2027 for (auto *Fusion : TGT.getMacroFusions()) {
2028 std::string Name = Fusion->getNameInitAsString();
2029 OS.indent(NumSpaces: 2) << "if (hasFeature(" << Target << "::" << Name
2030 << ")) Fusions.push_back(llvm::is" << Name << ");\n";
2031 }
2032
2033 OS.indent(NumSpaces: 2) << "return Fusions;\n";
2034 OS << "}\n";
2035}
2036
2037// Produces a subtarget specific function for parsing
2038// the subtarget features string.
2039void SubtargetEmitter::parseFeaturesFunction(raw_ostream &OS) {
2040 ArrayRef<const Record *> Features =
2041 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
2042
2043 OS << "// ParseSubtargetFeatures - Parses features string setting specified\n"
2044 << "// subtarget options.\n"
2045 << "void llvm::";
2046 OS << Target;
2047 OS << "Subtarget::ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, "
2048 << "StringRef FS) {\n"
2049 << " LLVM_DEBUG(dbgs() << \"\\nFeatures:\" << FS);\n"
2050 << " LLVM_DEBUG(dbgs() << \"\\nCPU:\" << CPU);\n"
2051 << " LLVM_DEBUG(dbgs() << \"\\nTuneCPU:\" << TuneCPU << \"\\n\\n\");\n";
2052
2053 if (Features.empty()) {
2054 OS << "}\n";
2055 return;
2056 }
2057
2058 OS << " InitMCProcessorInfo(CPU, TuneCPU, FS);\n"
2059 << " const FeatureBitset &Bits = getFeatureBits();\n";
2060
2061 for (const Record *R : Features) {
2062 // Next record
2063 StringRef Instance = R->getName();
2064 StringRef Value = R->getValueAsString(FieldName: "Value");
2065 StringRef FieldName = R->getValueAsString(FieldName: "FieldName");
2066
2067 if (Value == "true" || Value == "false")
2068 OS << " if (Bits[" << Target << "::" << Instance << "]) " << FieldName
2069 << " = " << Value << ";\n";
2070 else
2071 OS << " if (Bits[" << Target << "::" << Instance << "] && " << FieldName
2072 << " < " << Value << ") " << FieldName << " = " << Value << ";\n";
2073 }
2074
2075 OS << "}\n";
2076}
2077
2078void SubtargetEmitter::emitInlineFeatures(const std::string &ClassName,
2079 raw_ostream &OS, StringRef Behavior) {
2080 std::vector<const Record *> FeatureList =
2081 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
2082 llvm::sort(C&: FeatureList, Comp: LessRecordFieldFieldName());
2083
2084 OS << "const FeatureBitset &" << ClassName << "::get" << Behavior
2085 << "Features() const {\n"
2086 << " static constexpr FeatureBitset Features = {\n";
2087
2088 for (const Record *Feature : FeatureList)
2089 if (Behavior == Feature->getValueAsDef(FieldName: "InlineBehavior")->getName())
2090 OS << Target << "::" << Feature->getName() << ",\n";
2091
2092 OS << " };\n"
2093 << " return Features;\n"
2094 << "}\n\n";
2095}
2096
2097void SubtargetEmitter::emitGenMCSubtargetInfo(raw_ostream &OS) {
2098 {
2099 NamespaceEmitter NS(OS, (Target + Twine("_MC")).str());
2100 OS << "unsigned resolveVariantSchedClassImpl(unsigned SchedClass,\n"
2101 << " const MCInst *MI, const MCInstrInfo *MCII, "
2102 << "const MCSubtargetInfo &STI, unsigned CPUID) {\n";
2103 emitSchedModelHelpersImpl(OS, /* OnlyExpandMCPredicates */ OnlyExpandMCInstPredicates: true);
2104 OS << "}\n";
2105 }
2106
2107 OS << "struct " << Target
2108 << "GenMCSubtargetInfo : public MCSubtargetInfo {\n";
2109 OS << " " << Target << "GenMCSubtargetInfo(const Triple &TT,\n"
2110 << " StringRef CPU, StringRef TuneCPU, StringRef FS,\n"
2111 << " StringTable PN,\n"
2112 << " ArrayRef<SubtargetFeatureKV> PF,\n"
2113 << " ArrayRef<SubtargetSubTypeKV> PD,\n"
2114 << " ArrayRef<SubtargetSubTypeAliasKV> PA,\n"
2115 << " const MCSchedModel *PSM,\n"
2116 << " const MCWriteProcResEntry *WPR,\n"
2117 << " const MCWriteLatencyEntry *WL,\n"
2118 << " const MCReadAdvanceEntry *RA, const InstrStage *IS,\n"
2119 << " const unsigned *OC, const unsigned *FP) :\n"
2120 << " MCSubtargetInfo(TT, CPU, TuneCPU, FS, PN, PF, PD, PA, PSM,\n"
2121 << " WPR, WL, RA, IS, OC, FP) { }\n\n"
2122 << " unsigned resolveVariantSchedClass(unsigned SchedClass,\n"
2123 << " const MCInst *MI, const MCInstrInfo *MCII,\n"
2124 << " unsigned CPUID) const final {\n"
2125 << " return " << Target << "_MC"
2126 << "::resolveVariantSchedClassImpl(SchedClass, MI, MCII, *this, CPUID);\n";
2127 OS << " }\n";
2128 if (TGT.getHwModes().getNumModeIds() > 1) {
2129 OS << " unsigned getHwModeSet() const final;\n";
2130 OS << " unsigned getHwMode(enum HwModeType type = HwMode_Default) const "
2131 "final;\n";
2132 }
2133 OS << "};\n";
2134 emitHwModeCheck(ClassName: Target + "GenMCSubtargetInfo", OS, /*IsMC=*/true);
2135}
2136
2137void SubtargetEmitter::emitMcInstrAnalysisPredicateFunctions(raw_ostream &OS) {
2138 STIPredicateExpander PE(Target, /*Indent=*/0);
2139
2140 {
2141 IfDefEmitter IfDefDecls(OS, "GET_STIPREDICATE_DECLS_FOR_MC_ANALYSIS");
2142 PE.setExpandForMC(true);
2143 PE.setByRef(true);
2144 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
2145 PE.expandSTIPredicate(OS, Fn);
2146 }
2147
2148 IfDefEmitter IfDefDefs(OS, "GET_STIPREDICATE_DEFS_FOR_MC_ANALYSIS");
2149 std::string ClassPrefix = Target + "MCInstrAnalysis";
2150 PE.setExpandDefinition(true);
2151 PE.setClassPrefix(ClassPrefix);
2152 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
2153 PE.expandSTIPredicate(OS, Fn);
2154}
2155
2156FeatureMapTy SubtargetEmitter::emitEnums(raw_ostream &OS) {
2157 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_ENUM");
2158 NamespaceEmitter NS(OS, "llvm");
2159 return enumeration(OS);
2160}
2161
2162SubtargetEmitter::MCDescInfo
2163SubtargetEmitter::emitMCDesc(raw_ostream &OS, const FeatureMapTy &FeatureMap) {
2164 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_MC_DESC");
2165 NamespaceEmitter LlvmNS(OS, "llvm");
2166
2167 MCDescInfo Res;
2168 auto [NumFeatures, FeatureStrTabSize] = featureKeyValues(OS, FeatureMap);
2169 Res.NumFeatures = NumFeatures;
2170 Res.FeatureStrTabSize = FeatureStrTabSize;
2171 OS << "\n";
2172 emitSchedModel(OS);
2173 OS << "\n";
2174 auto [NumProcs, NumAliases, SubTypeStrTabSize] = cpuKeyValues(OS, FeatureMap);
2175 Res.NumProcs = NumProcs;
2176 Res.NumAliases = NumAliases;
2177 Res.SubTypeStrTabSize = SubTypeStrTabSize;
2178 OS << "\n";
2179
2180 // MCInstrInfo initialization routine.
2181 emitGenMCSubtargetInfo(OS);
2182
2183 OS << "\nstatic inline MCSubtargetInfo *create" << Target
2184 << "MCSubtargetInfoImpl("
2185 << "const Triple &TT, StringRef CPU, StringRef TuneCPU, StringRef FS) {\n";
2186 OS << " return new " << Target
2187 << "GenMCSubtargetInfo(TT, CPU, TuneCPU, FS, ";
2188 OS << "StringTable(" << Target << "SubTypeKVStorage.Strings), ";
2189 if (Res.NumFeatures)
2190 OS << Target << "FeatureKVStorage.Features, ";
2191 else
2192 OS << "{}, ";
2193 if (Res.NumProcs)
2194 OS << Target << "SubTypeKVStorage.SubTypes, "
2195 << "ArrayRef(" << Target << "SubTypeKVStorage.Aliases).take_front("
2196 << Res.NumAliases << "), " << Target << "SchedModels, ";
2197 else
2198 OS << "{}, {}, nullptr, ";
2199 OS << '\n';
2200 OS.indent(NumSpaces: 22);
2201 OS << Target << "WriteProcResTable, " << Target << "WriteLatencyTable, "
2202 << Target << "ReadAdvanceTable, ";
2203 OS << '\n';
2204 OS.indent(NumSpaces: 22);
2205 if (SchedModels.hasItineraries()) {
2206 OS << Target << "Stages, " << Target << "OperandCycles, " << Target
2207 << "ForwardingPaths";
2208 } else {
2209 OS << "nullptr, nullptr, nullptr";
2210 }
2211 OS << ");\n}\n\n";
2212 return Res;
2213}
2214
2215void SubtargetEmitter::emitTargetDesc(raw_ostream &OS) {
2216 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_TARGET_DESC");
2217
2218 OS << "#include \"llvm/ADT/BitmaskEnum.h\"\n";
2219 OS << "#include \"llvm/Support/Debug.h\"\n";
2220 OS << "#include \"llvm/Support/raw_ostream.h\"\n\n";
2221 parseFeaturesFunction(OS);
2222}
2223
2224void SubtargetEmitter::emitHeader(raw_ostream &OS) {
2225 // Create a TargetSubtargetInfo subclass to hide the MC layer initialization.
2226 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_HEADER");
2227 NamespaceEmitter LLVMNS(OS, "llvm");
2228
2229 std::string ClassName = Target + "GenSubtargetInfo";
2230 OS << "class DFAPacketizer;\n";
2231 {
2232 NamespaceEmitter MCNS(OS, (Target + Twine("_MC")).str());
2233 OS << "unsigned resolveVariantSchedClassImpl(unsigned SchedClass,"
2234 << " const MCInst *MI, const MCInstrInfo *MCII, "
2235 << "const MCSubtargetInfo &STI, unsigned CPUID);\n";
2236 }
2237 OS << "struct " << ClassName << " : public TargetSubtargetInfo {\n"
2238 << " explicit " << ClassName << "(const Triple &TT, StringRef CPU, "
2239 << "StringRef TuneCPU, StringRef FS);\n"
2240 << "public:\n"
2241 << " unsigned resolveSchedClass(unsigned SchedClass, "
2242 << " const MachineInstr *DefMI,"
2243 << " const TargetSchedModel *SchedModel) const final;\n"
2244 << " unsigned resolveVariantSchedClass(unsigned SchedClass,"
2245 << " const MCInst *MI, const MCInstrInfo *MCII,"
2246 << " unsigned CPUID) const final;\n"
2247 << " DFAPacketizer *createDFAPacketizer(const InstrItineraryData *IID)"
2248 << " const;\n";
2249
2250 const CodeGenHwModes &CGH = TGT.getHwModes();
2251 if (CGH.getNumModeIds() > 1) {
2252 OS << " enum class " << Target << "HwModeBits : unsigned {\n";
2253 for (unsigned M = 0, NumModes = CGH.getNumModeIds(); M != NumModes; ++M) {
2254 StringRef ModeName = CGH.getModeName(Id: M, /*IncludeDefault=*/true);
2255 OS << " " << ModeName << " = ";
2256 if (M == 0)
2257 OS << "0";
2258 else
2259 OS << "(1 << " << (M - 1) << ")";
2260 OS << ",\n";
2261 if (M == NumModes - 1) {
2262 OS << "\n";
2263 OS << " LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/" << ModeName
2264 << "),\n";
2265 }
2266 }
2267 OS << " };\n";
2268
2269 OS << " unsigned getHwModeSet() const final;\n";
2270 OS << " unsigned getHwMode(enum HwModeType type = HwMode_Default) const "
2271 "final;\n";
2272 }
2273 if (TGT.hasMacroFusion())
2274 OS << " std::vector<MacroFusionPredTy> getMacroFusions() const "
2275 "final;\n";
2276
2277 OS << " const FeatureBitset &getInlineIgnoreFeatures() const override;\n";
2278 OS << " const FeatureBitset &getInlineInverseFeatures() const override;\n";
2279 OS << " const FeatureBitset &getInlineMustMatchFeatures() const override;\n";
2280
2281 STIPredicateExpander PE(Target);
2282 PE.setByRef(false);
2283 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
2284 PE.expandSTIPredicate(OS, Fn);
2285 OS << "};\n";
2286}
2287
2288void SubtargetEmitter::emitCtor(raw_ostream &OS, MCDescInfo DescInfo) {
2289 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_CTOR");
2290 OS << "#include \"llvm/CodeGen/TargetSchedule.h\"\n\n";
2291
2292 NamespaceEmitter LLVMNS(OS, "llvm");
2293 OS << "extern const llvm::StringRef " << Target << "Names[];\n";
2294 if (DescInfo.NumFeatures) {
2295 OS << "extern const llvm::SubtargetFeatureKVStorage<"
2296 << DescInfo.NumFeatures << ", " << DescInfo.FeatureStrTabSize << "> "
2297 << Target << "FeatureKVStorage;\n";
2298 }
2299 OS << "extern const llvm::SubtargetSubTypeKVStorage<" << DescInfo.NumProcs
2300 << ", " << DescInfo.NumAliases << ", " << DescInfo.SubTypeStrTabSize
2301 << "> " << Target << "SubTypeKVStorage;\n";
2302 OS << "extern const llvm::MCSchedModel " << Target << "SchedModels[];\n";
2303 OS << "extern const llvm::MCWriteProcResEntry " << Target
2304 << "WriteProcResTable[];\n";
2305 OS << "extern const llvm::MCWriteLatencyEntry " << Target
2306 << "WriteLatencyTable[];\n";
2307 OS << "extern const llvm::MCReadAdvanceEntry " << Target
2308 << "ReadAdvanceTable[];\n";
2309
2310 if (SchedModels.hasItineraries()) {
2311 OS << "extern const llvm::InstrStage " << Target << "Stages[];\n";
2312 OS << "extern const unsigned " << Target << "OperandCycles[];\n";
2313 OS << "extern const unsigned " << Target << "ForwardingPaths[];\n";
2314 }
2315
2316 std::string ClassName = Target + "GenSubtargetInfo";
2317 OS << ClassName << "::" << ClassName << "(const Triple &TT, StringRef CPU, "
2318 << "StringRef TuneCPU, StringRef FS)\n";
2319
2320 OS << " : TargetSubtargetInfo(TT, CPU, TuneCPU, FS, ";
2321 OS << "StringTable(" << Target << "SubTypeKVStorage.Strings), ";
2322 if (DescInfo.NumFeatures)
2323 OS << "ArrayRef(" << Target << "FeatureKVStorage.Features), ";
2324 else
2325 OS << "{}, ";
2326 if (DescInfo.NumProcs) {
2327 OS << "ArrayRef(" << Target << "SubTypeKVStorage.SubTypes), "
2328 << "ArrayRef(" << Target << "SubTypeKVStorage.Aliases).take_front("
2329 << DescInfo.NumAliases << "), " << Target << "SchedModels, ";
2330 } else {
2331 OS << "{}, {}, nullptr, ";
2332 }
2333 OS << '\n';
2334 OS.indent(NumSpaces: 24);
2335 OS << Target << "WriteProcResTable, " << Target << "WriteLatencyTable, "
2336 << Target << "ReadAdvanceTable, ";
2337 OS << '\n';
2338 OS.indent(NumSpaces: 24);
2339 if (SchedModels.hasItineraries()) {
2340 OS << Target << "Stages, " << Target << "OperandCycles, " << Target
2341 << "ForwardingPaths";
2342 } else {
2343 OS << "nullptr, nullptr, nullptr";
2344 }
2345 OS << ") {}\n\n";
2346
2347 emitSchedModelHelpers(ClassName, OS);
2348 emitHwModeCheck(ClassName, OS, /*IsMC=*/false);
2349 emitGetMacroFusions(ClassName, OS);
2350 emitInlineFeatures(ClassName, OS, Behavior: "InlineIgnore");
2351 emitInlineFeatures(ClassName, OS, Behavior: "InlineInverse");
2352 emitInlineFeatures(ClassName, OS, Behavior: "InlineMustMatch");
2353}
2354
2355//
2356// SubtargetEmitter::run - Main subtarget enumeration emitter.
2357//
2358void SubtargetEmitter::run(raw_ostream &OS) {
2359 emitSourceFileHeader(Desc: "Subtarget Enumeration Source Fragment", OS);
2360
2361 auto FeatureMap = emitEnums(OS);
2362 emitSubtargetInfoMacroCalls(OS);
2363 MCDescInfo DescInfo = emitMCDesc(OS, FeatureMap);
2364 emitTargetDesc(OS);
2365 emitHeader(OS);
2366 emitCtor(OS, DescInfo);
2367 emitMcInstrAnalysisPredicateFunctions(OS);
2368}
2369
2370static TableGen::Emitter::OptClass<SubtargetEmitter>
2371 X("gen-subtarget", "Generate subtarget enumerations");
2372