1//===- SubtargetEmitter.cpp - Generate subtarget enumerations -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This tablegen backend emits subtarget enumerations.
10//
11//===----------------------------------------------------------------------===//
12
13#include "Basic/TargetFeaturesEmitter.h"
14#include "Common/CodeGenHwModes.h"
15#include "Common/CodeGenSchedule.h"
16#include "Common/CodeGenTarget.h"
17#include "Common/PredicateExpander.h"
18#include "Common/SubtargetFeatureInfo.h"
19#include "Common/Utils.h"
20#include "llvm/ADT/MapVector.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/StringExtras.h"
24#include "llvm/ADT/StringMap.h"
25#include "llvm/ADT/StringRef.h"
26#include "llvm/ADT/StringSet.h"
27#include "llvm/MC/MCInstrItineraries.h"
28#include "llvm/MC/MCSchedule.h"
29#include "llvm/Support/Debug.h"
30#include "llvm/Support/Format.h"
31#include "llvm/Support/raw_ostream.h"
32#include "llvm/TableGen/CodeGenHelpers.h"
33#include "llvm/TableGen/Error.h"
34#include "llvm/TableGen/Record.h"
35#include "llvm/TableGen/StringToOffsetTable.h"
36#include "llvm/TableGen/TableGenBackend.h"
37#include <algorithm>
38#include <cassert>
39#include <cstdint>
40#include <iterator>
41#include <string>
42#include <vector>
43
44using namespace llvm;
45
46#define DEBUG_TYPE "subtarget-emitter"
47
48namespace {
49
50class SubtargetEmitter : TargetFeaturesEmitter {
51 // Each processor has a SchedClassDesc table with an entry for each
52 // SchedClass. The SchedClassDesc table indexes into a global write resource
53 // table, write latency table, and read advance table.
54 struct SchedClassEntries {
55 std::vector<MCWriteProcResEntry> WriteProcResources;
56 std::vector<MCWriteLatencyEntry> WriteLatencies;
57 std::vector<std::string> WriterNames;
58 std::vector<MCReadAdvanceEntry> ReadAdvanceEntries;
59 };
60
61 struct SchedClassTables {
62 std::vector<std::vector<MCSchedClassDesc>> ProcSchedClasses;
63 std::vector<MCWriteProcResEntry> WriteProcResources;
64 std::vector<MCWriteLatencyEntry> WriteLatencies;
65 std::vector<std::string> WriterNames;
66 std::vector<MCReadAdvanceEntry> ReadAdvanceEntries;
67 size_t MaxWriteProcResEntries = 0;
68 size_t MaxWriteLatencyEntries = 0;
69
70 // Reserve an invalid entry at index 0
71 SchedClassTables() {
72 ProcSchedClasses.resize(new_size: 1);
73 WriteProcResources.resize(new_size: 1);
74 WriteLatencies.resize(new_size: 1);
75 WriterNames.push_back(x: "InvalidWrite");
76 ReadAdvanceEntries.resize(new_size: 1);
77 }
78 };
79
80 struct LessWriteProcResources {
81 bool operator()(const MCWriteProcResEntry &LHS,
82 const MCWriteProcResEntry &RHS) {
83 return LHS.ProcResourceIdx < RHS.ProcResourceIdx;
84 }
85 };
86
87 CodeGenTarget TGT;
88 CodeGenSchedModels &SchedModels;
89
90 FeatureMapTy emitEnums(raw_ostream &OS);
91 void emitSubtargetInfoMacroCalls(raw_ostream &OS);
92
93 struct MCDescInfo {
94 unsigned NumFeatures;
95 unsigned FeatureStrTabSize;
96 unsigned NumProcs;
97 unsigned NumAliases;
98 unsigned SubTypeStrTabSize;
99 };
100 MCDescInfo emitMCDesc(raw_ostream &OS, const FeatureMapTy &FeatureMap);
101 void emitTargetDesc(raw_ostream &OS);
102 void emitHeader(raw_ostream &OS);
103 void emitCtor(raw_ostream &OS, MCDescInfo DescInfo);
104
105 struct CPUKeyValuesInfo {
106 unsigned NumProcs;
107 unsigned NumAliases;
108 unsigned SubTypeStrTabSize;
109 };
110 std::pair<unsigned, unsigned>
111 featureKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
112 CPUKeyValuesInfo cpuKeyValues(raw_ostream &OS,
113 const FeatureMapTy &FeatureMap);
114 void formItineraryStageString(const std::string &Names,
115 const Record *ItinData, std::string &ItinString,
116 unsigned &NStages);
117 void formItineraryOperandCycleString(const Record *ItinData,
118 std::string &ItinString,
119 unsigned &NOperandCycles);
120 void formItineraryBypassString(const std::string &Names,
121 const Record *ItinData,
122 std::string &ItinString,
123 unsigned NOperandCycles);
124 void emitStageAndOperandCycleData(
125 raw_ostream &OS, std::vector<std::vector<InstrItinerary>> &ProcItinLists);
126 void emitItineraries(raw_ostream &OS,
127 ArrayRef<std::vector<InstrItinerary>> ProcItinLists);
128 unsigned emitRegisterFileTables(const CodeGenProcModel &ProcModel,
129 raw_ostream &OS);
130 void emitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel,
131 raw_ostream &OS);
132 void emitExtraProcessorInfo(const CodeGenProcModel &ProcModel,
133 raw_ostream &OS);
134 void emitProcessorProp(raw_ostream &OS, const Record *R, StringRef Name,
135 char Separator);
136 void emitProcessorResourceSubUnits(const CodeGenProcModel &ProcModel,
137 raw_ostream &OS);
138 void emitProcessorResources(const CodeGenProcModel &ProcModel,
139 raw_ostream &OS);
140 const Record *findWriteResources(const CodeGenSchedRW &SchedWrite,
141 const CodeGenProcModel &ProcModel);
142 const Record *findReadAdvance(const CodeGenSchedRW &SchedRead,
143 const CodeGenProcModel &ProcModel);
144 void expandProcResources(ConstRecVec &PRVec,
145 std::vector<int64_t> &ReleaseAtCycles,
146 std::vector<int64_t> &AcquireAtCycles,
147 const CodeGenProcModel &ProcModel);
148 const CodeGenSchedRW *genSchedClassEntries(ArrayRef<unsigned> Writes,
149 ArrayRef<unsigned> Reads,
150 const CodeGenProcModel &ProcModel,
151 MCSchedClassDesc &SCDesc,
152 SchedClassEntries &Entries);
153 void genSchedClassTables(const CodeGenProcModel &ProcModel,
154 SchedClassTables &SchedTables);
155 void emitSchedClassTables(SchedClassTables &SchedTables, raw_ostream &OS);
156 void emitProcessorModels(raw_ostream &OS);
157 void emitSchedModelHelpers(const std::string &ClassName, raw_ostream &OS);
158 void emitSchedModelHelpersImpl(raw_ostream &OS,
159 bool OnlyExpandMCInstPredicates = false);
160 void emitVariantSchedClassTargets(raw_ostream &OS);
161 void emitGenMCSubtargetInfo(raw_ostream &OS);
162 void emitMcInstrAnalysisPredicateFunctions(raw_ostream &OS);
163
164 void emitSchedModel(raw_ostream &OS);
165 void emitGetMacroFusions(const std::string &ClassName, raw_ostream &OS);
166 void emitHwModeCheck(const std::string &ClassName, raw_ostream &OS,
167 bool IsMC);
168 void emitInlineFeatures(const std::string &ClassName, raw_ostream &OS,
169 StringRef Behavior);
170 void parseFeaturesFunction(raw_ostream &OS);
171
172public:
173 SubtargetEmitter(const RecordKeeper &R)
174 : TargetFeaturesEmitter(R), TGT(R), SchedModels(TGT.getSchedModels()) {}
175
176 void run(raw_ostream &O) override;
177};
178
179} // end anonymous namespace
180
181/// Emit some information about the SubtargetFeature as calls to a macro so
182/// that they can be used from C++.
183void SubtargetEmitter::emitSubtargetInfoMacroCalls(raw_ostream &OS) {
184 // Undef the GET_SUBTARGETINFO_MACRO macro at the end of the scope since it's
185 // used within the scope.
186 IfDefEmitter IfDefMacro(OS, "GET_SUBTARGETINFO_MACRO", /*LateUndef=*/true);
187
188 std::vector<const Record *> FeatureList =
189 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
190 llvm::sort(C&: FeatureList, Comp: LessRecordFieldFieldName());
191
192 for (const Record *Feature : FeatureList) {
193 const StringRef FieldName = Feature->getValueAsString(FieldName: "FieldName");
194 const StringRef Value = Feature->getValueAsString(FieldName: "Value");
195
196 // Only handle boolean features for now, excluding BitVectors and enums.
197 const bool IsBool = (Value == "false" || Value == "true") &&
198 !StringRef(FieldName).contains(C: '[');
199 if (!IsBool)
200 continue;
201
202 // Some features default to true, with values set to false if enabled.
203 const char *Default = Value == "false" ? "true" : "false";
204
205 // Define the getter with lowercased first char: xxxYyy() { return XxxYyy; }
206 const std::string Getter =
207 FieldName.substr(Start: 0, N: 1).lower() + FieldName.substr(Start: 1).str();
208
209 OS << "GET_SUBTARGETINFO_MACRO(" << FieldName << ", " << Default << ", "
210 << Getter << ")\n";
211 }
212}
213
214//
215// FeatureKeyValues - Emit data of all the subtarget features. Used by the
216// command line.
217//
218std::pair<unsigned, unsigned>
219SubtargetEmitter::featureKeyValues(raw_ostream &OS,
220 const FeatureMapTy &FeatureMap) {
221 std::vector<const Record *> FeatureList =
222 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
223
224 // Remove features with empty name.
225 llvm::erase_if(C&: FeatureList, P: [](const Record *Rec) {
226 return Rec->getValueAsString(FieldName: "Name").empty();
227 });
228 if (FeatureList.empty())
229 return {0, 0};
230
231 // Sort and check duplicate Feature name.
232 sortAndReportDuplicates(Records: FeatureList, ObjectName: "Feature");
233
234 StringToOffsetTable StrTab;
235 // Offsets of CommandLineName and Desc in StrTab.
236 SmallVector<std::pair<unsigned, unsigned>> StrOffs;
237 for (const Record *Feature : FeatureList) {
238 unsigned NameOff =
239 StrTab.GetOrAddStringOffset(Str: Feature->getValueAsString(FieldName: "Name"));
240 unsigned DescOff =
241 StrTab.GetOrAddStringOffset(Str: Feature->getValueAsString(FieldName: "Desc"));
242 StrOffs.emplace_back(Args&: NameOff, Args&: DescOff);
243 }
244
245 // Begin feature table.
246 OS << "// Sorted (by key) array of values for CPU features.\n"
247 << "extern const llvm::SubtargetFeatureKVStorage< " << FeatureList.size()
248 << ", " << (StrTab.size() + 1) << "> " << Target
249 << "FeatureKVStorage = {\n {\n";
250
251 for (auto [Idx, Feature] : enumerate(First&: FeatureList)) {
252 // Next feature
253 StringRef Name = Feature->getName();
254
255 // Emit as { "feature", "description", { featureEnum }, { i1 , i2 , ... , in
256 // } }
257 auto StrOff =
258 "sizeof(SubtargetFeatureKV) * " + Twine(FeatureList.size() - Idx);
259 OS << " { " << StrOff << " + " << StrOffs[Idx].first << ", " << StrOff
260 << " + " << StrOffs[Idx].second << ", " << Target << "::" << Name
261 << ", ";
262
263 ConstRecVec ImpliesList = Feature->getValueAsListOfDefs(FieldName: "Implies");
264
265 printFeatureMask(OS, FeatureList: ImpliesList, FeatureMap);
266
267 OS << " },\n";
268 }
269
270 OS << " },\n";
271 StrTab.EmitString(O&: OS);
272
273 // End feature table.
274 OS << "};\n";
275
276 return {FeatureList.size(), StrTab.size() + 1};
277}
278
279static void checkDuplicateCPUFeatures(StringRef CPUName,
280 ArrayRef<const Record *> Features,
281 ArrayRef<const Record *> TuneFeatures) {
282 // We have made sure each SubtargetFeature Record has a unique name, so we can
283 // simply use pointer sets here.
284 SmallPtrSet<const Record *, 8> FeatureSet, TuneFeatureSet;
285 for (const auto *FeatureRec : Features) {
286 if (!FeatureSet.insert(Ptr: FeatureRec).second)
287 PrintWarning(Msg: "Processor " + CPUName + " contains duplicate feature '" +
288 FeatureRec->getValueAsString(FieldName: "Name") + "'");
289 }
290
291 for (const auto *TuneFeatureRec : TuneFeatures) {
292 if (!TuneFeatureSet.insert(Ptr: TuneFeatureRec).second)
293 PrintWarning(Msg: "Processor " + CPUName +
294 " contains duplicate tune feature '" +
295 TuneFeatureRec->getValueAsString(FieldName: "Name") + "'");
296 if (FeatureSet.contains(Ptr: TuneFeatureRec))
297 PrintWarning(Msg: "Processor " + CPUName + " has '" +
298 TuneFeatureRec->getValueAsString(FieldName: "Name") +
299 "' in both feature and tune feature sets");
300 }
301}
302
303//
304// CPUKeyValues - Emit data of all the subtarget processors. Used by command
305// line.
306//
307SubtargetEmitter::CPUKeyValuesInfo
308SubtargetEmitter::cpuKeyValues(raw_ostream &OS,
309 const FeatureMapTy &FeatureMap) {
310 // Gather and sort the processors. Only real processors go in the subtype
311 // table; aliases are stored in a separate, more compact table.
312 std::vector<const Record *> ProcessorList =
313 Records.getAllDerivedDefinitions(ClassName: "Processor");
314 llvm::sort(C&: ProcessorList, Comp: LessRecordFieldName());
315
316 // Map from processor name to its index in the sorted subtype table, so that
317 // aliases can point at the canonical processor entry.
318 StringMap<unsigned> ProcessorIndex;
319 for (const auto &[Idx, Processor] : enumerate(First&: ProcessorList))
320 ProcessorIndex[Processor->getValueAsString(FieldName: "Name")] = Idx;
321
322 // Validate and resolve each alias to the index of its canonical processor.
323 struct AliasEntry {
324 StringRef Name;
325 unsigned SubTypeIdx;
326 };
327 std::vector<AliasEntry> AliasEntries;
328 std::vector<const Record *> ProcessorAliasList =
329 Records.getAllDerivedDefinitionsIfDefined(ClassName: "ProcessorAlias");
330 AliasEntries.reserve(n: ProcessorAliasList.size());
331
332 StringSet<> AliasNames;
333 for (const Record *Rec : ProcessorAliasList) {
334 StringRef Name = Rec->getValueAsString(FieldName: "Name");
335 StringRef Alias = Rec->getValueAsString(FieldName: "Alias");
336 auto It = ProcessorIndex.find(Key: Alias);
337 if (It == ProcessorIndex.end())
338 PrintFatalError(Rec, Msg: "Alias '" + Name +
339 "' references a non-existent Processor '" +
340 Alias + "'");
341 if (ProcessorIndex.contains(Key: Name))
342 PrintFatalError(Rec,
343 Msg: "Alias '" + Name + "' duplicates an existing Processor");
344 if (!AliasNames.insert(key: Name).second)
345 PrintFatalError(Rec, Msg: "Alias '" + Name + "' duplicates an existing alias");
346 AliasEntries.push_back(x: {.Name: Name, .SubTypeIdx: It->second});
347 }
348
349 // The alias table must be sorted by key for the binary search in the lookups.
350 llvm::sort(C&: AliasEntries, Comp: [](const AliasEntry &LHS, const AliasEntry &RHS) {
351 return LHS.Name < RHS.Name;
352 });
353
354 // Sort all names together so the emitted string blob is sorted.
355 SmallVector<StringRef> Names;
356 Names.reserve(N: ProcessorList.size() + AliasEntries.size());
357 for (const Record *Processor : ProcessorList)
358 Names.push_back(Elt: Processor->getValueAsString(FieldName: "Name"));
359 for (const AliasEntry &Entry : AliasEntries)
360 Names.push_back(Elt: Entry.Name);
361 llvm::sort(C&: Names);
362
363 StringToOffsetTable StrTab;
364 for (StringRef Name : Names)
365 StrTab.GetOrAddStringOffset(Str: Name);
366
367 // Note that unlike `FeatureKeyValues`, here we do not need to check for
368 // duplicate processors, since that is already done when the SubtargetEmitter
369 // constructor calls `getSchedModels` to build a `CodeGenSchedModels` object,
370 // which does the duplicate processor check.
371
372 unsigned NumProcs = ProcessorList.size();
373 unsigned NumAliases = AliasEntries.size();
374
375 // The alias array's byte size, used to reach the string blob from a subtype
376 // entry.
377 OS << "static constexpr size_t " << Target
378 << "CPUAliasArraySize = sizeof(std::array<SubtargetSubTypeAliasKV, "
379 << NumAliases << ">);\n";
380
381 // Begin processor table.
382 OS << "// Sorted (by key) array of values for CPU subtype.\n"
383 << "extern const llvm::SubtargetSubTypeKVStorage< " << NumProcs << ", "
384 << NumAliases << ", " << (StrTab.size() + 1) << "> " << Target
385 << "SubTypeKVStorage = {\n {\n";
386
387 for (const auto &[Idx, Processor] : enumerate(First&: ProcessorList)) {
388 StringRef Name = Processor->getValueAsString(FieldName: "Name");
389 ConstRecVec FeatureList = Processor->getValueAsListOfDefs(FieldName: "Features");
390 ConstRecVec TuneFeatureList =
391 Processor->getValueAsListOfDefs(FieldName: "TuneFeatures");
392
393 // Warn the user if there are duplicate processor features or tune features.
394 checkDuplicateCPUFeatures(CPUName: Name, Features: FeatureList, TuneFeatures: TuneFeatureList);
395
396 // The string blob follows the subtype and alias arrays, so skip both.
397 OS << " { sizeof(SubtargetSubTypeKV) * " << (NumProcs - Idx) << " + "
398 << Target << "CPUAliasArraySize + " << StrTab.GetOrAddStringOffset(Str: Name)
399 << ", ";
400
401 printFeatureMask(OS, FeatureList, FeatureMap);
402 OS << ", ";
403 printFeatureMask(OS, FeatureList: TuneFeatureList, FeatureMap);
404
405 // Emit the scheduler model index.
406 OS << ", " << SchedModels.getModelIndexForProc(ProcDef: Processor) << " },\n";
407 }
408
409 OS << " },\n";
410
411 // Begin alias table. The extra brace layer is std::array's wrapped C array.
412 OS << " { {\n";
413 for (const auto &[Idx, Entry] : enumerate(First&: AliasEntries)) {
414 // The string blob immediately follows the alias array.
415 OS << " { sizeof(SubtargetSubTypeAliasKV) * " << (NumAliases - Idx)
416 << " + " << StrTab.GetOrAddStringOffset(Str: Entry.Name) << ", "
417 << Entry.SubTypeIdx << " },\n";
418 }
419 OS << " } },\n";
420
421 StrTab.EmitString(O&: OS);
422
423 // End processor table.
424 OS << "};\n";
425
426 return {.NumProcs: NumProcs, .NumAliases: NumAliases, .SubTypeStrTabSize: unsigned(StrTab.size() + 1)};
427}
428
429//
430// FormItineraryStageString - Compose a string containing the stage
431// data initialization for the specified itinerary. N is the number
432// of stages.
433//
434void SubtargetEmitter::formItineraryStageString(const std::string &Name,
435 const Record *ItinData,
436 std::string &ItinString,
437 unsigned &NStages) {
438 // Get states list
439 ConstRecVec StageList = ItinData->getValueAsListOfDefs(FieldName: "Stages");
440
441 // For each stage
442 unsigned N = NStages = StageList.size();
443 for (unsigned I = 0; I < N;) {
444 // Next stage
445 const Record *Stage = StageList[I];
446
447 // Form string as ,{ cycles, u1 | u2 | ... | un, timeinc, kind }
448 int Cycles = Stage->getValueAsInt(FieldName: "Cycles");
449 ItinString += " { " + itostr(X: Cycles) + ", ";
450
451 // Get unit list
452 ConstRecVec UnitList = Stage->getValueAsListOfDefs(FieldName: "Units");
453
454 // For each unit
455 for (unsigned J = 0, M = UnitList.size(); J < M;) {
456 // Add name and bitwise or
457 ItinString += Name + "FU::" + UnitList[J]->getName().str();
458 if (++J < M)
459 ItinString += " | ";
460 }
461
462 int TimeInc = Stage->getValueAsInt(FieldName: "TimeInc");
463 ItinString += ", " + itostr(X: TimeInc);
464
465 int Kind = Stage->getValueAsInt(FieldName: "Kind");
466 ItinString += ", (llvm::InstrStage::ReservationKinds)" + itostr(X: Kind);
467
468 // Close off stage
469 ItinString += " }";
470 if (++I < N)
471 ItinString += ", ";
472 }
473}
474
475//
476// FormItineraryOperandCycleString - Compose a string containing the
477// operand cycle initialization for the specified itinerary. N is the
478// number of operands that has cycles specified.
479//
480void SubtargetEmitter::formItineraryOperandCycleString(
481 const Record *ItinData, std::string &ItinString, unsigned &NOperandCycles) {
482 // Get operand cycle list
483 std::vector<int64_t> OperandCycleList =
484 ItinData->getValueAsListOfInts(FieldName: "OperandCycles");
485
486 // For each operand cycle
487 NOperandCycles = OperandCycleList.size();
488 ListSeparator LS;
489 for (int OCycle : OperandCycleList) {
490 // Next operand cycle
491 ItinString += LS;
492 ItinString += " " + itostr(X: OCycle);
493 }
494}
495
496void SubtargetEmitter::formItineraryBypassString(const std::string &Name,
497 const Record *ItinData,
498 std::string &ItinString,
499 unsigned NOperandCycles) {
500 ConstRecVec BypassList = ItinData->getValueAsListOfDefs(FieldName: "Bypasses");
501 unsigned N = BypassList.size();
502 unsigned I = 0;
503 ListSeparator LS;
504 for (; I < N; ++I) {
505 ItinString += LS;
506 ItinString += Name + "Bypass::" + BypassList[I]->getName().str();
507 }
508 for (; I < NOperandCycles; ++I) {
509 ItinString += LS;
510 ItinString += " 0";
511 }
512}
513
514//
515// EmitStageAndOperandCycleData - Generate unique itinerary stages and operand
516// cycle tables. Create a list of InstrItinerary objects (ProcItinLists) indexed
517// by CodeGenSchedClass::Index.
518//
519void SubtargetEmitter::emitStageAndOperandCycleData(
520 raw_ostream &OS, std::vector<std::vector<InstrItinerary>> &ProcItinLists) {
521 // Multiple processor models may share an itinerary record. Emit it once.
522 SmallPtrSet<const Record *, 8> ItinsDefSet;
523
524 // Emit functional units for all the itineraries.
525 for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) {
526 if (!ItinsDefSet.insert(Ptr: ProcModel.ItinsDef).second)
527 continue;
528
529 ConstRecVec FUs = ProcModel.ItinsDef->getValueAsListOfDefs(FieldName: "FU");
530 if (FUs.empty())
531 continue;
532
533 StringRef Name = ProcModel.ItinsDef->getName();
534 {
535 OS << "\n// Functional units for \"" << Name << "\"\n";
536 NamespaceEmitter FUNamespace(OS, (Name + Twine("FU")).str());
537
538 for (const auto &[Idx, FU] : enumerate(First&: FUs))
539 OS << " const InstrStage::FuncUnits " << FU->getName() << " = 1ULL << "
540 << Idx << ";\n";
541 }
542
543 ConstRecVec BPs = ProcModel.ItinsDef->getValueAsListOfDefs(FieldName: "BP");
544 if (BPs.empty())
545 continue;
546 OS << "\n// Pipeline forwarding paths for itineraries \"" << Name << "\"\n";
547 NamespaceEmitter BypassNamespace(OS, (Name + Twine("Bypass")).str());
548
549 OS << " const unsigned NoBypass = 0;\n";
550 for (const auto &[Idx, BP] : enumerate(First&: BPs))
551 OS << " const unsigned " << BP->getName() << " = 1 << " << Idx << ";\n";
552 }
553
554 // Begin stages table
555 std::string StageTable =
556 "\nextern const llvm::InstrStage " + Target + "Stages[] = {\n";
557 StageTable += " { 0, 0, 0, llvm::InstrStage::Required }, // No itinerary\n";
558
559 // Begin operand cycle table
560 std::string OperandCycleTable =
561 "extern const unsigned " + Target + "OperandCycles[] = {\n";
562 OperandCycleTable += " 0, // No itinerary\n";
563
564 // Begin pipeline bypass table
565 std::string BypassTable =
566 "extern const unsigned " + Target + "ForwardingPaths[] = {\n";
567 BypassTable += " 0, // No itinerary\n";
568
569 // For each Itinerary across all processors, add a unique entry to the stages,
570 // operand cycles, and pipeline bypass tables. Then add the new Itinerary
571 // object with computed offsets to the ProcItinLists result.
572 unsigned StageCount = 1, OperandCycleCount = 1;
573 StringMap<unsigned> ItinStageMap, ItinOperandMap;
574 for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) {
575 // Add process itinerary to the list.
576 std::vector<InstrItinerary> &ItinList = ProcItinLists.emplace_back();
577
578 // If this processor defines no itineraries, then leave the itinerary list
579 // empty.
580 if (!ProcModel.hasItineraries())
581 continue;
582
583 StringRef Name = ProcModel.ItinsDef->getName();
584
585 ItinList.resize(new_size: SchedModels.numInstrSchedClasses());
586 assert(ProcModel.ItinDefList.size() == ItinList.size() && "bad Itins");
587
588 for (unsigned SchedClassIdx = 0, SchedClassEnd = ItinList.size();
589 SchedClassIdx < SchedClassEnd; ++SchedClassIdx) {
590
591 // Next itinerary data
592 const Record *ItinData = ProcModel.ItinDefList[SchedClassIdx];
593
594 // Get string and stage count
595 std::string ItinStageString;
596 unsigned NStages = 0;
597 if (ItinData)
598 formItineraryStageString(Name: Name.str(), ItinData, ItinString&: ItinStageString,
599 NStages);
600
601 // Get string and operand cycle count
602 std::string ItinOperandCycleString;
603 unsigned NOperandCycles = 0;
604 std::string ItinBypassString;
605 if (ItinData) {
606 formItineraryOperandCycleString(ItinData, ItinString&: ItinOperandCycleString,
607 NOperandCycles);
608
609 formItineraryBypassString(Name: Name.str(), ItinData, ItinString&: ItinBypassString,
610 NOperandCycles);
611 }
612
613 // Check to see if stage already exists and create if it doesn't
614 uint16_t FindStage = 0;
615 if (NStages > 0) {
616 FindStage = ItinStageMap[ItinStageString];
617 if (FindStage == 0) {
618 // Emit as { cycles, u1 | u2 | ... | un, timeinc }, // indices
619 StageTable += ItinStageString + ", // " + itostr(X: StageCount);
620 if (NStages > 1)
621 StageTable += "-" + itostr(X: StageCount + NStages - 1);
622 StageTable += "\n";
623 // Record Itin class number.
624 ItinStageMap[ItinStageString] = FindStage = StageCount;
625 StageCount += NStages;
626 }
627 }
628
629 // Check to see if operand cycle already exists and create if it doesn't
630 uint16_t FindOperandCycle = 0;
631 if (NOperandCycles > 0) {
632 std::string ItinOperandString =
633 ItinOperandCycleString + ItinBypassString;
634 FindOperandCycle = ItinOperandMap[ItinOperandString];
635 if (FindOperandCycle == 0) {
636 // Emit as cycle, // index
637 OperandCycleTable += ItinOperandCycleString + ", // ";
638 std::string OperandIdxComment = itostr(X: OperandCycleCount);
639 if (NOperandCycles > 1)
640 OperandIdxComment +=
641 "-" + itostr(X: OperandCycleCount + NOperandCycles - 1);
642 OperandCycleTable += OperandIdxComment + "\n";
643 // Record Itin class number.
644 ItinOperandMap[ItinOperandCycleString] = FindOperandCycle =
645 OperandCycleCount;
646 // Emit as bypass, // index
647 BypassTable += ItinBypassString + ", // " + OperandIdxComment + "\n";
648 OperandCycleCount += NOperandCycles;
649 }
650 }
651
652 // Set up itinerary as location and location + stage count
653 int16_t NumUOps = ItinData ? ItinData->getValueAsInt(FieldName: "NumMicroOps") : 0;
654 InstrItinerary Intinerary = {
655 .NumMicroOps: NumUOps,
656 .FirstStage: FindStage,
657 .LastStage: uint16_t(FindStage + NStages),
658 .FirstOperandCycle: FindOperandCycle,
659 .LastOperandCycle: uint16_t(FindOperandCycle + NOperandCycles),
660 };
661
662 // Inject - empty slots will be 0, 0
663 ItinList[SchedClassIdx] = Intinerary;
664 }
665 }
666
667 // Closing stage
668 StageTable += " { 0, 0, 0, llvm::InstrStage::Required } // End stages\n";
669 StageTable += "};\n";
670
671 // Closing operand cycles
672 OperandCycleTable += " 0 // End operand cycles\n";
673 OperandCycleTable += "};\n";
674
675 BypassTable += " 0 // End bypass tables\n";
676 BypassTable += "};\n";
677
678 // Emit tables.
679 OS << StageTable;
680 OS << OperandCycleTable;
681 OS << BypassTable;
682}
683
684//
685// EmitProcessorData - Generate data for processor itineraries that were
686// computed during EmitStageAndOperandCycleData(). ProcItinLists lists all
687// Itineraries for each processor. The Itinerary lists are indexed on
688// CodeGenSchedClass::Index.
689//
690void SubtargetEmitter::emitItineraries(
691 raw_ostream &OS, ArrayRef<std::vector<InstrItinerary>> ProcItinLists) {
692 // Multiple processor models may share an itinerary record. Emit it once.
693 SmallPtrSet<const Record *, 8> ItinsDefSet;
694
695 for (const auto &[Proc, ItinList] :
696 zip_equal(t: SchedModels.procModels(), u&: ProcItinLists)) {
697 const Record *ItinsDef = Proc.ItinsDef;
698 if (!ItinsDefSet.insert(Ptr: ItinsDef).second)
699 continue;
700
701 // Empty itineraries aren't referenced anywhere in the tablegen output
702 // so don't emit them.
703 if (ItinList.empty())
704 continue;
705
706 // Begin processor itinerary table
707 OS << "\n";
708 OS << "static constexpr llvm::InstrItinerary " << ItinsDef->getName()
709 << "[] = {\n";
710
711 ArrayRef<CodeGenSchedClass> ItinSchedClasses =
712 SchedModels.schedClasses().take_front(N: ItinList.size());
713
714 // For each itinerary class in CodeGenSchedClass::Index order.
715 for (const auto &[Idx, Intinerary, SchedClass] :
716 enumerate(First: ItinList, Rest&: ItinSchedClasses)) {
717 // Emit Itinerary in the form of
718 // { NumMicroOps, FirstStage, LastStage, FirstOperandCycle,
719 // LastOperandCycle } // index class name
720 OS << " { " << Intinerary.NumMicroOps << ", " << Intinerary.FirstStage
721 << ", " << Intinerary.LastStage << ", " << Intinerary.FirstOperandCycle
722 << ", " << Intinerary.LastOperandCycle << " }" << ", // " << Idx << " "
723 << SchedClass.Name << "\n";
724 }
725 // End processor itinerary table
726 OS << " { 0, uint16_t(~0U), uint16_t(~0U), uint16_t(~0U), uint16_t(~0U) }"
727 "// end marker\n";
728 OS << "};\n";
729 }
730}
731
732// Emit either the value defined in the TableGen Record, or the default
733// value defined in the C++ header. The Record is null if the processor does not
734// define a model.
735void SubtargetEmitter::emitProcessorProp(raw_ostream &OS, const Record *R,
736 StringRef Name, char Separator) {
737 OS << " ";
738 int V = R ? R->getValueAsInt(FieldName: Name) : -1;
739 if (V >= 0)
740 OS << V << Separator << " // " << Name;
741 else
742 OS << "MCSchedModel::Default" << Name << Separator;
743 OS << '\n';
744}
745
746void SubtargetEmitter::emitProcessorResourceSubUnits(
747 const CodeGenProcModel &ProcModel, raw_ostream &OS) {
748 OS << "\nstatic const unsigned " << ProcModel.ModelName
749 << "ProcResourceSubUnits[] = {\n"
750 << " 0, // Invalid\n";
751
752 for (unsigned I = 0, E = ProcModel.ProcResourceDefs.size(); I < E; ++I) {
753 const Record *PRDef = ProcModel.ProcResourceDefs[I];
754 if (!PRDef->isSubClassOf(Name: "ProcResGroup"))
755 continue;
756 for (const Record *RUDef : PRDef->getValueAsListOfDefs(FieldName: "Resources")) {
757 const Record *RU =
758 SchedModels.findProcResUnits(ProcResKind: RUDef, PM: ProcModel, Loc: PRDef->getLoc());
759 for (unsigned J = 0; J < RU->getValueAsInt(FieldName: "NumUnits"); ++J) {
760 OS << " " << ProcModel.getProcResourceIdx(PRDef: RU) << ", ";
761 }
762 }
763 OS << " // " << PRDef->getName() << "\n";
764 }
765 OS << "};\n";
766}
767
768static void emitRetireControlUnitInfo(const CodeGenProcModel &ProcModel,
769 raw_ostream &OS) {
770 int64_t ReorderBufferSize = 0, MaxRetirePerCycle = 0;
771 if (const Record *RCU = ProcModel.RetireControlUnit) {
772 ReorderBufferSize =
773 std::max(a: ReorderBufferSize, b: RCU->getValueAsInt(FieldName: "ReorderBufferSize"));
774 MaxRetirePerCycle =
775 std::max(a: MaxRetirePerCycle, b: RCU->getValueAsInt(FieldName: "MaxRetirePerCycle"));
776 }
777
778 OS << ReorderBufferSize << ", // ReorderBufferSize\n ";
779 OS << MaxRetirePerCycle << ", // MaxRetirePerCycle\n ";
780}
781
782static void emitRegisterFileInfo(const CodeGenProcModel &ProcModel,
783 unsigned NumRegisterFiles,
784 unsigned NumCostEntries, raw_ostream &OS) {
785 if (NumRegisterFiles)
786 OS << ProcModel.ModelName << "RegisterFiles,\n " << (1 + NumRegisterFiles);
787 else
788 OS << "nullptr,\n 0";
789
790 OS << ", // Number of register files.\n ";
791 if (NumCostEntries)
792 OS << ProcModel.ModelName << "RegisterCosts,\n ";
793 else
794 OS << "nullptr,\n ";
795 OS << NumCostEntries << ", // Number of register cost entries.\n";
796}
797
798unsigned
799SubtargetEmitter::emitRegisterFileTables(const CodeGenProcModel &ProcModel,
800 raw_ostream &OS) {
801 if (llvm::all_of(Range: ProcModel.RegisterFiles, P: [](const CodeGenRegisterFile &RF) {
802 return RF.hasDefaultCosts();
803 }))
804 return 0;
805
806 // Print the RegisterCost table first.
807 OS << "\n// {RegisterClassID, Register Cost, AllowMoveElimination }\n";
808 OS << "static const llvm::MCRegisterCostEntry " << ProcModel.ModelName
809 << "RegisterCosts"
810 << "[] = {\n";
811
812 for (const CodeGenRegisterFile &RF : ProcModel.RegisterFiles) {
813 // Skip register files with a default cost table.
814 if (RF.hasDefaultCosts())
815 continue;
816 // Add entries to the cost table.
817 for (const CodeGenRegisterCost &RC : RF.Costs) {
818 OS << " { ";
819 const Record *Rec = RC.RCDef;
820 if (Rec->getValue(Name: "Namespace"))
821 OS << Rec->getValueAsString(FieldName: "Namespace") << "::";
822 OS << Rec->getName() << "RegClassID, " << RC.Cost << ", "
823 << RC.AllowMoveElimination << "},\n";
824 }
825 }
826 OS << "};\n";
827
828 // Now generate a table with register file info.
829 OS << "\n // {Name, #PhysRegs, #CostEntries, IndexToCostTbl, "
830 << "MaxMovesEliminatedPerCycle, AllowZeroMoveEliminationOnly }\n";
831 OS << "static const llvm::MCRegisterFileDesc " << ProcModel.ModelName
832 << "RegisterFiles"
833 << "[] = {\n"
834 << " { \"InvalidRegisterFile\", 0, 0, 0, 0, 0 },\n";
835 unsigned CostTblIndex = 0;
836
837 for (const CodeGenRegisterFile &RD : ProcModel.RegisterFiles) {
838 OS << " { ";
839 OS << '"' << RD.Name << '"' << ", " << RD.NumPhysRegs << ", ";
840 unsigned NumCostEntries = RD.Costs.size();
841 OS << NumCostEntries << ", " << CostTblIndex << ", "
842 << RD.MaxMovesEliminatedPerCycle << ", "
843 << RD.AllowZeroMoveEliminationOnly << "},\n";
844 CostTblIndex += NumCostEntries;
845 }
846 OS << "};\n";
847
848 return CostTblIndex;
849}
850
851void SubtargetEmitter::emitLoadStoreQueueInfo(const CodeGenProcModel &ProcModel,
852 raw_ostream &OS) {
853 unsigned QueueID = 0;
854 if (ProcModel.LoadQueue) {
855 const Record *Queue = ProcModel.LoadQueue->getValueAsDef(FieldName: "QueueDescriptor");
856 QueueID = 1 + std::distance(first: ProcModel.ProcResourceDefs.begin(),
857 last: find(Range: ProcModel.ProcResourceDefs, Val: Queue));
858 }
859 OS << " " << QueueID << ", // Resource Descriptor for the Load Queue\n";
860
861 QueueID = 0;
862 if (ProcModel.StoreQueue) {
863 const Record *Queue =
864 ProcModel.StoreQueue->getValueAsDef(FieldName: "QueueDescriptor");
865 QueueID = 1 + std::distance(first: ProcModel.ProcResourceDefs.begin(),
866 last: find(Range: ProcModel.ProcResourceDefs, Val: Queue));
867 }
868 OS << " " << QueueID << ", // Resource Descriptor for the Store Queue\n";
869}
870
871void SubtargetEmitter::emitExtraProcessorInfo(const CodeGenProcModel &ProcModel,
872 raw_ostream &OS) {
873 // Generate a table of register file descriptors (one entry per each user
874 // defined register file), and a table of register costs.
875 unsigned NumCostEntries = emitRegisterFileTables(ProcModel, OS);
876
877 // Now generate a table for the extra processor info.
878 OS << "\nstatic const llvm::MCExtraProcessorInfo " << ProcModel.ModelName
879 << "ExtraInfo = {\n ";
880
881 // Add information related to the retire control unit.
882 emitRetireControlUnitInfo(ProcModel, OS);
883
884 // Add information related to the register files (i.e. where to find register
885 // file descriptors and register costs).
886 emitRegisterFileInfo(ProcModel, NumRegisterFiles: ProcModel.RegisterFiles.size(),
887 NumCostEntries, OS);
888
889 // Add information about load/store queues.
890 emitLoadStoreQueueInfo(ProcModel, OS);
891
892 OS << "};\n";
893}
894
895void SubtargetEmitter::emitProcessorResources(const CodeGenProcModel &ProcModel,
896 raw_ostream &OS) {
897 emitProcessorResourceSubUnits(ProcModel, OS);
898
899 OS << "\n// {Name, NumUnits, SuperIdx, BufferSize, SubUnitsIdxBegin}\n";
900 OS << "static const llvm::MCProcResourceDesc " << ProcModel.ModelName
901 << "ProcResources"
902 << "[] = {\n"
903 << " {\"InvalidUnit\", 0, 0, 0, 0},\n";
904
905 unsigned SubUnitsOffset = 1;
906 for (unsigned I = 0, E = ProcModel.ProcResourceDefs.size(); I < E; ++I) {
907 const Record *PRDef = ProcModel.ProcResourceDefs[I];
908
909 const Record *SuperDef = nullptr;
910 unsigned SuperIdx = 0;
911 unsigned NumUnits = 0;
912 const unsigned SubUnitsBeginOffset = SubUnitsOffset;
913 int BufferSize = PRDef->getValueAsInt(FieldName: "BufferSize");
914 if (PRDef->isSubClassOf(Name: "ProcResGroup")) {
915 for (const Record *RU : PRDef->getValueAsListOfDefs(FieldName: "Resources")) {
916 NumUnits += RU->getValueAsInt(FieldName: "NumUnits");
917 SubUnitsOffset += RU->getValueAsInt(FieldName: "NumUnits");
918 }
919 } else {
920 // Find the SuperIdx
921 if (PRDef->getValueInit(FieldName: "Super")->isComplete()) {
922 SuperDef = SchedModels.findProcResUnits(ProcResKind: PRDef->getValueAsDef(FieldName: "Super"),
923 PM: ProcModel, Loc: PRDef->getLoc());
924 SuperIdx = ProcModel.getProcResourceIdx(PRDef: SuperDef);
925 }
926 NumUnits = PRDef->getValueAsInt(FieldName: "NumUnits");
927 }
928 // Emit the ProcResourceDesc
929 OS << " {\"" << PRDef->getName() << "\", ";
930 if (PRDef->getName().size() < 15)
931 OS.indent(NumSpaces: 15 - PRDef->getName().size());
932 OS << NumUnits << ", " << SuperIdx << ", " << BufferSize << ", ";
933 if (SubUnitsBeginOffset != SubUnitsOffset) {
934 OS << ProcModel.ModelName << "ProcResourceSubUnits + "
935 << SubUnitsBeginOffset;
936 } else {
937 OS << "nullptr";
938 }
939 OS << "}, // #" << I + 1;
940 if (SuperDef)
941 OS << ", Super=" << SuperDef->getName();
942 OS << "\n";
943 }
944 OS << "};\n";
945}
946
947// Find the WriteRes Record that defines processor resources for this
948// SchedWrite.
949const Record *
950SubtargetEmitter::findWriteResources(const CodeGenSchedRW &SchedWrite,
951 const CodeGenProcModel &ProcModel) {
952
953 // Check if the SchedWrite is already subtarget-specific and directly
954 // specifies a set of processor resources.
955 if (SchedWrite.TheDef->isSubClassOf(Name: "SchedWriteRes"))
956 return SchedWrite.TheDef;
957
958 const Record *AliasDef = nullptr;
959 for (const Record *A : SchedWrite.Aliases) {
960 const CodeGenSchedRW &AliasRW =
961 SchedModels.getSchedRW(Def: A->getValueAsDef(FieldName: "AliasRW"));
962 if (AliasRW.TheDef->getValueInit(FieldName: "SchedModel")->isComplete()) {
963 const Record *ModelDef = AliasRW.TheDef->getValueAsDef(FieldName: "SchedModel");
964 if (&SchedModels.getProcModel(ModelDef) != &ProcModel)
965 continue;
966 }
967 if (AliasDef)
968 PrintFatalError(ErrorLoc: AliasRW.TheDef->getLoc(),
969 Msg: "Multiple aliases "
970 "defined for processor " +
971 ProcModel.ModelName +
972 " Ensure only one SchedAlias exists per RW.");
973 AliasDef = AliasRW.TheDef;
974 }
975 if (AliasDef && AliasDef->isSubClassOf(Name: "SchedWriteRes"))
976 return AliasDef;
977
978 // Check this processor's list of write resources.
979 const Record *ResDef = nullptr;
980
981 auto I = ProcModel.WriteResMap.find(Val: SchedWrite.TheDef);
982 if (I != ProcModel.WriteResMap.end())
983 ResDef = I->second;
984
985 if (AliasDef) {
986 I = ProcModel.WriteResMap.find(Val: AliasDef);
987 if (I != ProcModel.WriteResMap.end()) {
988 if (ResDef)
989 PrintFatalError(ErrorLoc: I->second->getLoc(),
990 Msg: "Resources are defined for both SchedWrite and its "
991 "alias on processor " +
992 ProcModel.ModelName);
993 ResDef = I->second;
994 }
995 }
996
997 // TODO: If ProcModel has a base model (previous generation processor),
998 // then call FindWriteResources recursively with that model here.
999 return ResDef;
1000}
1001
1002/// Find the ReadAdvance record for the given SchedRead on this processor or
1003/// return NULL.
1004const Record *
1005SubtargetEmitter::findReadAdvance(const CodeGenSchedRW &SchedRead,
1006 const CodeGenProcModel &ProcModel) {
1007 // Check for SchedReads that directly specify a ReadAdvance.
1008 if (SchedRead.TheDef->isSubClassOf(Name: "SchedReadAdvance"))
1009 return SchedRead.TheDef;
1010
1011 // Check this processor's list of aliases for SchedRead.
1012 const Record *AliasDef = nullptr;
1013 for (const Record *A : SchedRead.Aliases) {
1014 const CodeGenSchedRW &AliasRW =
1015 SchedModels.getSchedRW(Def: A->getValueAsDef(FieldName: "AliasRW"));
1016 if (AliasRW.TheDef->getValueInit(FieldName: "SchedModel")->isComplete()) {
1017 const Record *ModelDef = AliasRW.TheDef->getValueAsDef(FieldName: "SchedModel");
1018 if (&SchedModels.getProcModel(ModelDef) != &ProcModel)
1019 continue;
1020 }
1021 if (AliasDef)
1022 PrintFatalError(ErrorLoc: AliasRW.TheDef->getLoc(),
1023 Msg: "Multiple aliases "
1024 "defined for processor " +
1025 ProcModel.ModelName +
1026 " Ensure only one SchedAlias exists per RW.");
1027 AliasDef = AliasRW.TheDef;
1028 }
1029 if (AliasDef && AliasDef->isSubClassOf(Name: "SchedReadAdvance"))
1030 return AliasDef;
1031
1032 // Check this processor's ReadAdvanceList.
1033 const Record *ResDef = nullptr;
1034
1035 auto I = ProcModel.ReadAdvanceMap.find(Val: SchedRead.TheDef);
1036 if (I != ProcModel.ReadAdvanceMap.end())
1037 ResDef = I->second;
1038
1039 if (AliasDef) {
1040 I = ProcModel.ReadAdvanceMap.find(Val: AliasDef);
1041 if (I != ProcModel.ReadAdvanceMap.end()) {
1042 if (ResDef)
1043 PrintFatalError(
1044 ErrorLoc: I->second->getLoc(),
1045 Msg: "Resources are defined for both SchedRead and its alias on "
1046 "processor " +
1047 ProcModel.ModelName);
1048 ResDef = I->second;
1049 }
1050 }
1051
1052 // TODO: If ProcModel has a base model (previous generation processor),
1053 // then call FindReadAdvance recursively with that model here.
1054 return ResDef;
1055}
1056
1057// Expand an explicit list of processor resources into a full list of implied
1058// resource groups and super resources that cover them.
1059void SubtargetEmitter::expandProcResources(
1060 ConstRecVec &PRVec, std::vector<int64_t> &ReleaseAtCycles,
1061 std::vector<int64_t> &AcquireAtCycles, const CodeGenProcModel &PM) {
1062 assert(PRVec.size() == ReleaseAtCycles.size() && "failed precondition");
1063 for (unsigned I = 0, E = PRVec.size(); I != E; ++I) {
1064 const Record *PRDef = PRVec[I];
1065 ConstRecVec SubResources;
1066 if (PRDef->isSubClassOf(Name: "ProcResGroup")) {
1067 SubResources = PRDef->getValueAsListOfDefs(FieldName: "Resources");
1068 } else {
1069 SubResources.push_back(x: PRDef);
1070 PRDef = SchedModels.findProcResUnits(ProcResKind: PRDef, PM, Loc: PRDef->getLoc());
1071 for (const Record *SubDef = PRDef;
1072 SubDef->getValueInit(FieldName: "Super")->isComplete();) {
1073 if (SubDef->isSubClassOf(Name: "ProcResGroup")) {
1074 // Disallow this for simplicitly.
1075 PrintFatalError(ErrorLoc: SubDef->getLoc(), Msg: "Processor resource group "
1076 " cannot be a super resources.");
1077 }
1078 const Record *SuperDef = SchedModels.findProcResUnits(
1079 ProcResKind: SubDef->getValueAsDef(FieldName: "Super"), PM, Loc: SubDef->getLoc());
1080 PRVec.push_back(x: SuperDef);
1081 ReleaseAtCycles.push_back(x: ReleaseAtCycles[I]);
1082 AcquireAtCycles.push_back(x: AcquireAtCycles[I]);
1083 SubDef = SuperDef;
1084 }
1085 }
1086 for (const Record *PR : PM.ProcResourceDefs) {
1087 if (PR == PRDef || !PR->isSubClassOf(Name: "ProcResGroup"))
1088 continue;
1089 ConstRecVec SuperResources = PR->getValueAsListOfDefs(FieldName: "Resources");
1090 bool AllContained =
1091 all_of(Range&: SubResources, P: [SuperResources](const Record *SubResource) {
1092 return is_contained(Range: SuperResources, Element: SubResource);
1093 });
1094 if (AllContained) {
1095 PRVec.push_back(x: PR);
1096 ReleaseAtCycles.push_back(x: ReleaseAtCycles[I]);
1097 AcquireAtCycles.push_back(x: AcquireAtCycles[I]);
1098 }
1099 }
1100 }
1101}
1102
1103// Generate the SchedClass table for this processor and update global
1104// tables. Must be called for each processor in order.
1105const CodeGenSchedRW *SubtargetEmitter::genSchedClassEntries(
1106 ArrayRef<unsigned> Writes, ArrayRef<unsigned> Reads,
1107 const CodeGenProcModel &ProcModel, MCSchedClassDesc &SCDesc,
1108 SchedClassEntries &Entries) {
1109 // Sum resources across all operand writes.
1110 auto &[WriteProcResources, WriteLatencies, WriterNames, ReadAdvanceEntries] =
1111 Entries;
1112 for (unsigned W : Writes) {
1113 IdxVec WriteSeq;
1114 SchedModels.expandRWSeqForProc(RWIdx: W, RWSeq&: WriteSeq, /*IsRead=*/false, ProcModel);
1115
1116 // For each operand, create a latency entry.
1117 MCWriteLatencyEntry WLEntry;
1118 WLEntry.Cycles = 0;
1119 unsigned WriteID = WriteSeq.back();
1120 WriterNames.push_back(x: SchedModels.getSchedWrite(Idx: WriteID).Name);
1121 // If this Write is not referenced by a ReadAdvance, don't distinguish it
1122 // from other WriteLatency entries.
1123 if (!ProcModel.hasReadOfWrite(WriteDef: SchedModels.getSchedWrite(Idx: WriteID).TheDef))
1124 WriteID = 0;
1125 WLEntry.WriteResourceID = WriteID;
1126
1127 for (unsigned WS : WriteSeq) {
1128 const Record *WriteRes =
1129 findWriteResources(SchedWrite: SchedModels.getSchedWrite(Idx: WS), ProcModel);
1130 if (!WriteRes)
1131 return &SchedModels.getSchedWrite(Idx: WS);
1132
1133 // Mark the parent class as invalid for unsupported write types.
1134 if (WriteRes->getValueAsBit(FieldName: "Unsupported")) {
1135 SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps;
1136 break;
1137 }
1138 WLEntry.Cycles += WriteRes->getValueAsInt(FieldName: "Latency");
1139 SCDesc.NumMicroOps += WriteRes->getValueAsInt(FieldName: "NumMicroOps");
1140 SCDesc.BeginGroup |= WriteRes->getValueAsBit(FieldName: "BeginGroup");
1141 SCDesc.EndGroup |= WriteRes->getValueAsBit(FieldName: "EndGroup");
1142 SCDesc.BeginGroup |= WriteRes->getValueAsBit(FieldName: "SingleIssue");
1143 SCDesc.EndGroup |= WriteRes->getValueAsBit(FieldName: "SingleIssue");
1144 SCDesc.RetireOOO |= WriteRes->getValueAsBit(FieldName: "RetireOOO");
1145
1146 // Create an entry for each ProcResource listed in WriteRes.
1147 ConstRecVec PRVec = WriteRes->getValueAsListOfDefs(FieldName: "ProcResources");
1148 std::vector<int64_t> ReleaseAtCycles =
1149 WriteRes->getValueAsListOfInts(FieldName: "ReleaseAtCycles");
1150
1151 std::vector<int64_t> AcquireAtCycles =
1152 WriteRes->getValueAsListOfInts(FieldName: "AcquireAtCycles");
1153
1154 // Check consistency of the two vectors carrying the start and
1155 // stop cycles of the resources.
1156 if (!ReleaseAtCycles.empty() && ReleaseAtCycles.size() != PRVec.size()) {
1157 // If ReleaseAtCycles is provided, check consistency.
1158 PrintFatalError(
1159 ErrorLoc: WriteRes->getLoc(),
1160 Msg: Twine("Inconsistent release at cycles: size(ReleaseAtCycles) != "
1161 "size(ProcResources): ")
1162 .concat(Suffix: Twine(PRVec.size()))
1163 .concat(Suffix: " vs ")
1164 .concat(Suffix: Twine(ReleaseAtCycles.size())));
1165 }
1166
1167 if (!AcquireAtCycles.empty() && AcquireAtCycles.size() != PRVec.size()) {
1168 PrintFatalError(
1169 ErrorLoc: WriteRes->getLoc(),
1170 Msg: Twine("Inconsistent resource cycles: size(AcquireAtCycles) != "
1171 "size(ProcResources): ")
1172 .concat(Suffix: Twine(AcquireAtCycles.size()))
1173 .concat(Suffix: " vs ")
1174 .concat(Suffix: Twine(PRVec.size())));
1175 }
1176
1177 if (ReleaseAtCycles.empty()) {
1178 // If ReleaseAtCycles is not provided, default to one cycle
1179 // per resource.
1180 ReleaseAtCycles.resize(new_size: PRVec.size(), x: 1);
1181 }
1182
1183 if (AcquireAtCycles.empty()) {
1184 // If AcquireAtCycles is not provided, reserve the resource
1185 // starting from cycle 0.
1186 AcquireAtCycles.resize(new_size: PRVec.size(), x: 0);
1187 }
1188
1189 assert(AcquireAtCycles.size() == ReleaseAtCycles.size());
1190
1191 expandProcResources(PRVec, ReleaseAtCycles, AcquireAtCycles, PM: ProcModel);
1192 assert(AcquireAtCycles.size() == ReleaseAtCycles.size());
1193
1194 for (unsigned PRIdx = 0, PREnd = PRVec.size(); PRIdx != PREnd; ++PRIdx) {
1195 MCWriteProcResEntry WPREntry;
1196 WPREntry.ProcResourceIdx = ProcModel.getProcResourceIdx(PRDef: PRVec[PRIdx]);
1197 assert(WPREntry.ProcResourceIdx && "Bad ProcResourceIdx");
1198 WPREntry.ReleaseAtCycle = ReleaseAtCycles[PRIdx];
1199 WPREntry.AcquireAtCycle = AcquireAtCycles[PRIdx];
1200 if (AcquireAtCycles[PRIdx] > ReleaseAtCycles[PRIdx]) {
1201 PrintFatalError(ErrorLoc: WriteRes->getLoc(),
1202 Msg: Twine("Inconsistent resource cycles: AcquireAtCycles "
1203 "<= ReleaseAtCycles must hold."));
1204 }
1205 if (AcquireAtCycles[PRIdx] < 0) {
1206 PrintFatalError(ErrorLoc: WriteRes->getLoc(),
1207 Msg: Twine("Invalid value: AcquireAtCycle "
1208 "must be a non-negative value."));
1209 }
1210 // If this resource is already used in this sequence, add the current
1211 // entry's cycles so that the same resource appears to be used
1212 // serially, rather than multiple parallel uses. This is important for
1213 // in-order machine where the resource consumption is a hazard.
1214 unsigned WPRIdx = 0, WPREnd = WriteProcResources.size();
1215 for (; WPRIdx != WPREnd; ++WPRIdx) {
1216 if (WriteProcResources[WPRIdx].ProcResourceIdx ==
1217 WPREntry.ProcResourceIdx) {
1218 // TODO: multiple use of the same resources would
1219 // require either 1. thinking of how to handle multiple
1220 // intervals for the same resource in
1221 // `<Target>WriteProcResTable` (see
1222 // `SubtargetEmitter::EmitSchedClassTables`), or
1223 // 2. thinking how to merge multiple intervals into a
1224 // single interval.
1225 assert(WPREntry.AcquireAtCycle == 0 &&
1226 "multiple use ofthe same resource is not yet handled");
1227 WriteProcResources[WPRIdx].ReleaseAtCycle +=
1228 WPREntry.ReleaseAtCycle;
1229 break;
1230 }
1231 }
1232 if (WPRIdx == WPREnd)
1233 WriteProcResources.push_back(x: WPREntry);
1234 }
1235 }
1236 WriteLatencies.push_back(x: WLEntry);
1237 }
1238 // Create an entry for each operand Read in this SchedClass.
1239 // Entries must be sorted first by UseIdx then by WriteResourceID.
1240 for (unsigned UseIdx = 0, EndIdx = Reads.size(); UseIdx != EndIdx; ++UseIdx) {
1241 const CodeGenSchedRW &SchedRead = SchedModels.getSchedRead(Idx: Reads[UseIdx]);
1242 const Record *ReadAdvance = findReadAdvance(SchedRead, ProcModel);
1243 if (!ReadAdvance) {
1244 if (SchedRead.TheDef->getName() != "ReadDefault")
1245 return &SchedRead;
1246 continue;
1247 }
1248
1249 // Mark the parent class as invalid for unsupported write types.
1250 if (ReadAdvance->getValueAsBit(FieldName: "Unsupported")) {
1251 SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps;
1252 break;
1253 }
1254 ConstRecVec ValidWrites = ReadAdvance->getValueAsListOfDefs(FieldName: "ValidWrites");
1255 std::vector<int64_t> CycleTunables =
1256 ReadAdvance->getValueAsListOfInts(FieldName: "CycleTunables");
1257 std::vector<std::pair<unsigned, int>> WriteIDs;
1258 assert(CycleTunables.size() <= ValidWrites.size() && "Bad ReadAdvance");
1259 CycleTunables.resize(new_size: ValidWrites.size(), x: 0);
1260 if (ValidWrites.empty())
1261 WriteIDs.emplace_back(args: 0, args: 0);
1262 else {
1263 for (const auto [VW, CT] : zip_equal(t&: ValidWrites, u&: CycleTunables)) {
1264 unsigned WriteID = SchedModels.getSchedRWIdx(Def: VW, /*IsRead=*/false);
1265 assert(WriteID != 0 &&
1266 "Expected a valid SchedRW in the list of ValidWrites");
1267 WriteIDs.emplace_back(args&: WriteID, args&: CT);
1268 }
1269 }
1270 llvm::sort(C&: WriteIDs);
1271 for (const auto &[W, T] : WriteIDs) {
1272 MCReadAdvanceEntry &RAEntry = ReadAdvanceEntries.emplace_back();
1273 RAEntry.UseIdx = UseIdx;
1274 RAEntry.WriteResourceID = W;
1275 RAEntry.Cycles = ReadAdvance->getValueAsInt(FieldName: "Cycles") + T;
1276 }
1277 }
1278 if (SCDesc.NumMicroOps == MCSchedClassDesc::InvalidNumMicroOps) {
1279 WriteProcResources.clear();
1280 WriteLatencies.clear();
1281 ReadAdvanceEntries.clear();
1282 }
1283 // WritePrecRes entries are sorted by ProcResIdx.
1284 llvm::sort(C&: WriteProcResources, Comp: LessWriteProcResources());
1285 return nullptr;
1286}
1287
1288void SubtargetEmitter::genSchedClassTables(const CodeGenProcModel &ProcModel,
1289 SchedClassTables &SchedTables) {
1290 std::vector<MCSchedClassDesc> &SCTab =
1291 SchedTables.ProcSchedClasses.emplace_back();
1292 if (!ProcModel.hasInstrSchedModel())
1293 return;
1294
1295 LLVM_DEBUG(dbgs() << "\n+++ SCHED CLASSES (GenSchedClassTables) +++\n");
1296 MapVector<const Record *, bool> RedundantInstRWs;
1297 for (const CodeGenSchedClass &SC : SchedModels.schedClasses()) {
1298 LLVM_DEBUG(SC.dump(&SchedModels));
1299
1300 MCSchedClassDesc &SCDesc = SCTab.emplace_back();
1301 // SCDesc.Name is guarded by NDEBUG
1302 SCDesc.NumMicroOps = 0;
1303 SCDesc.BeginGroup = false;
1304 SCDesc.EndGroup = false;
1305 SCDesc.RetireOOO = false;
1306 SCDesc.WriteProcResIdx = 0;
1307 SCDesc.WriteLatencyIdx = 0;
1308 SCDesc.ReadAdvanceIdx = 0;
1309
1310 // A Variant SchedClass has no resources of its own.
1311 bool HasVariants = false;
1312 for (const CodeGenSchedTransition &CGT : SC.Transitions) {
1313 if (CGT.ProcIndex == ProcModel.Index) {
1314 HasVariants = true;
1315 break;
1316 }
1317 }
1318 if (HasVariants) {
1319 SCDesc.NumMicroOps = MCSchedClassDesc::VariantNumMicroOps;
1320 continue;
1321 }
1322
1323 // Determine if the SchedClass is actually reachable on this processor. If
1324 // not don't try to locate the processor resources, it will fail.
1325 // If ProcIndices contains 0, this class applies to all processors.
1326 assert(!SC.ProcIndices.empty() && "expect at least one procidx");
1327 if (SC.ProcIndices[0] != 0) {
1328 if (!is_contained(Range: SC.ProcIndices, Element: ProcModel.Index))
1329 continue;
1330 }
1331 IdxVec Writes = SC.Writes;
1332 IdxVec Reads = SC.Reads;
1333 const Record *RWDef = nullptr;
1334 if (!SC.InstRWs.empty()) {
1335 // This class has a default ReadWrite list which can be overridden by
1336 // InstRW definitions.
1337 for (const Record *RW : SC.InstRWs) {
1338 const Record *RWModelDef = RW->getValueAsDef(FieldName: "SchedModel");
1339 if (&ProcModel == &SchedModels.getProcModel(ModelDef: RWModelDef)) {
1340 RWDef = RW;
1341 break;
1342 }
1343 }
1344 if (RWDef) {
1345 Writes.clear();
1346 Reads.clear();
1347 SchedModels.findRWs(RWDefs: RWDef->getValueAsListOfDefs(FieldName: "OperandReadWrites"),
1348 Writes, Reads);
1349 }
1350 }
1351 if (Writes.empty()) {
1352 // Check this processor's itinerary class resources.
1353 for (const Record *I : ProcModel.ItinRWDefs) {
1354 ConstRecVec Matched = I->getValueAsListOfDefs(FieldName: "MatchedItinClasses");
1355 if (is_contained(Range&: Matched, Element: SC.ItinClassDef)) {
1356 SchedModels.findRWs(RWDefs: I->getValueAsListOfDefs(FieldName: "OperandReadWrites"),
1357 Writes, Reads);
1358 break;
1359 }
1360 }
1361 if (Writes.empty()) {
1362 LLVM_DEBUG(dbgs() << ProcModel.ModelName
1363 << " does not have resources for class " << SC.Name
1364 << '\n');
1365 SCDesc.NumMicroOps = MCSchedClassDesc::InvalidNumMicroOps;
1366 }
1367 }
1368 SchedClassEntries Entries;
1369 if (const CodeGenSchedRW *RW =
1370 genSchedClassEntries(Writes, Reads, ProcModel, SCDesc, Entries))
1371 PrintFatalError(ErrorLoc: ProcModel.ModelDef->getLoc(),
1372 Msg: Twine("Processor does not define resources for ") +
1373 RW->TheDef->getName());
1374 if (RWDef && ProcModel.ModelDef->getValueAsBit(FieldName: "WarnRedundantInstRW")) {
1375 bool Redundant = false;
1376 if (!SC.Writes.empty() &&
1377 SCDesc.NumMicroOps != MCSchedClassDesc::InvalidNumMicroOps) {
1378 MCSchedClassDesc BaseDesc{};
1379 SchedClassEntries BaseEntries;
1380 Redundant =
1381 !genSchedClassEntries(Writes: SC.Writes, Reads: SC.Reads, ProcModel, SCDesc&: BaseDesc,
1382 Entries&: BaseEntries) &&
1383 BaseDesc.NumMicroOps == SCDesc.NumMicroOps &&
1384 BaseDesc.BeginGroup == SCDesc.BeginGroup &&
1385 BaseDesc.EndGroup == SCDesc.EndGroup &&
1386 BaseDesc.RetireOOO == SCDesc.RetireOOO &&
1387 BaseEntries.WriteProcResources == Entries.WriteProcResources &&
1388 BaseEntries.WriteLatencies == Entries.WriteLatencies &&
1389 BaseEntries.ReadAdvanceEntries == Entries.ReadAdvanceEntries;
1390 }
1391 RedundantInstRWs.try_emplace(Key: RWDef, Args: true).first->second &= Redundant;
1392 }
1393 auto &[WriteProcResources, WriteLatencies, WriterNames,
1394 ReadAdvanceEntries] = Entries;
1395 // Add the information for this SchedClass to the global tables using basic
1396 // compression.
1397
1398 SchedTables.MaxWriteProcResEntries =
1399 std::max(a: SchedTables.MaxWriteProcResEntries, b: WriteProcResources.size());
1400 SCDesc.NumWriteProcResEntries = WriteProcResources.size();
1401 std::vector<MCWriteProcResEntry>::iterator WPRPos =
1402 std::search(first1: SchedTables.WriteProcResources.begin(),
1403 last1: SchedTables.WriteProcResources.end(),
1404 first2: WriteProcResources.begin(), last2: WriteProcResources.end());
1405 if (WPRPos != SchedTables.WriteProcResources.end())
1406 SCDesc.WriteProcResIdx = WPRPos - SchedTables.WriteProcResources.begin();
1407 else {
1408 SCDesc.WriteProcResIdx = SchedTables.WriteProcResources.size();
1409 SchedTables.WriteProcResources.insert(position: WPRPos, first: WriteProcResources.begin(),
1410 last: WriteProcResources.end());
1411 }
1412 // Latency entries must remain in operand order.
1413 SchedTables.MaxWriteLatencyEntries =
1414 std::max(a: SchedTables.MaxWriteLatencyEntries, b: WriteLatencies.size());
1415 SCDesc.NumWriteLatencyEntries = WriteLatencies.size();
1416 std::vector<MCWriteLatencyEntry>::iterator WLPos = std::search(
1417 first1: SchedTables.WriteLatencies.begin(), last1: SchedTables.WriteLatencies.end(),
1418 first2: WriteLatencies.begin(), last2: WriteLatencies.end());
1419 if (WLPos != SchedTables.WriteLatencies.end()) {
1420 unsigned Idx = WLPos - SchedTables.WriteLatencies.begin();
1421 SCDesc.WriteLatencyIdx = Idx;
1422 for (unsigned I = 0, E = WriteLatencies.size(); I < E; ++I)
1423 if (SchedTables.WriterNames[Idx + I].find(str: WriterNames[I]) ==
1424 std::string::npos) {
1425 SchedTables.WriterNames[Idx + I] += "_" + WriterNames[I];
1426 }
1427 } else {
1428 SCDesc.WriteLatencyIdx = SchedTables.WriteLatencies.size();
1429 llvm::append_range(C&: SchedTables.WriteLatencies, R&: WriteLatencies);
1430 llvm::append_range(C&: SchedTables.WriterNames, R&: WriterNames);
1431 }
1432 // ReadAdvanceEntries must remain in operand order.
1433 SCDesc.NumReadAdvanceEntries = ReadAdvanceEntries.size();
1434 std::vector<MCReadAdvanceEntry>::iterator RAPos =
1435 std::search(first1: SchedTables.ReadAdvanceEntries.begin(),
1436 last1: SchedTables.ReadAdvanceEntries.end(),
1437 first2: ReadAdvanceEntries.begin(), last2: ReadAdvanceEntries.end());
1438 if (RAPos != SchedTables.ReadAdvanceEntries.end())
1439 SCDesc.ReadAdvanceIdx = RAPos - SchedTables.ReadAdvanceEntries.begin();
1440 else {
1441 SCDesc.ReadAdvanceIdx = SchedTables.ReadAdvanceEntries.size();
1442 llvm::append_range(C&: SchedTables.ReadAdvanceEntries, R&: ReadAdvanceEntries);
1443 }
1444 }
1445 for (const auto &[RWDef, Redundant] : RedundantInstRWs)
1446 if (Redundant)
1447 PrintWarning(WarningLoc: RWDef->getLoc(), Msg: "InstRW on " + ProcModel.ModelName +
1448 " matches the base SchedRW of all "
1449 "its instructions");
1450}
1451
1452// Emit SchedClass tables for all processors and associated global tables.
1453void SubtargetEmitter::emitSchedClassTables(SchedClassTables &SchedTables,
1454 raw_ostream &OS) {
1455 OS << "\nstatic_assert(" << SchedTables.MaxWriteProcResEntries
1456 << " <= UINT8_MAX, \"NumWriteProcResEntries does not fit in uint8_t\");\n"
1457 << "static_assert(" << SchedTables.MaxWriteLatencyEntries
1458 << " <= UINT8_MAX, \"NumWriteLatencyEntries does not fit in uint8_t\");\n";
1459
1460 // Emit global WriteProcResTable.
1461 OS << "\n// {ProcResourceIdx, ReleaseAtCycle, AcquireAtCycle}\n"
1462 << "extern const llvm::MCWriteProcResEntry " << Target
1463 << "WriteProcResTable[] = {\n"
1464 << " { 0, 0, 0 }, // Invalid\n";
1465 for (unsigned WPRIdx = 1, WPREnd = SchedTables.WriteProcResources.size();
1466 WPRIdx != WPREnd; ++WPRIdx) {
1467 MCWriteProcResEntry &WPREntry = SchedTables.WriteProcResources[WPRIdx];
1468 OS << " {" << format(Fmt: "%2d", Vals: WPREntry.ProcResourceIdx) << ", "
1469 << format(Fmt: "%2d", Vals: WPREntry.ReleaseAtCycle) << ", "
1470 << format(Fmt: "%2d", Vals: WPREntry.AcquireAtCycle) << "}";
1471 if (WPRIdx + 1 < WPREnd)
1472 OS << ',';
1473 OS << " // #" << WPRIdx << '\n';
1474 }
1475 OS << "}; // " << Target << "WriteProcResTable\n";
1476
1477 // Emit global WriteLatencyTable.
1478 OS << "\n// {Cycles, WriteResourceID}\n"
1479 << "extern const llvm::MCWriteLatencyEntry " << Target
1480 << "WriteLatencyTable[] = {\n"
1481 << " { 0, 0}, // Invalid\n";
1482 for (unsigned WLIdx = 1, WLEnd = SchedTables.WriteLatencies.size();
1483 WLIdx != WLEnd; ++WLIdx) {
1484 MCWriteLatencyEntry &WLEntry = SchedTables.WriteLatencies[WLIdx];
1485 OS << " {" << format(Fmt: "%2d", Vals: WLEntry.Cycles) << ", "
1486 << format(Fmt: "%2d", Vals: WLEntry.WriteResourceID) << "}";
1487 if (WLIdx + 1 < WLEnd)
1488 OS << ',';
1489 OS << " // #" << WLIdx << " " << SchedTables.WriterNames[WLIdx] << '\n';
1490 }
1491 OS << "}; // " << Target << "WriteLatencyTable\n";
1492
1493 // Emit global ReadAdvanceTable.
1494 OS << "\n// {UseIdx, WriteResourceID, Cycles}\n"
1495 << "extern const llvm::MCReadAdvanceEntry " << Target
1496 << "ReadAdvanceTable[] = {\n"
1497 << " {0, 0, 0}, // Invalid\n";
1498 for (unsigned RAIdx = 1, RAEnd = SchedTables.ReadAdvanceEntries.size();
1499 RAIdx != RAEnd; ++RAIdx) {
1500 MCReadAdvanceEntry &RAEntry = SchedTables.ReadAdvanceEntries[RAIdx];
1501 OS << " {" << RAEntry.UseIdx << ", "
1502 << format(Fmt: "%2d", Vals: RAEntry.WriteResourceID) << ", "
1503 << format(Fmt: "%2d", Vals: RAEntry.Cycles) << "}";
1504 if (RAIdx + 1 < RAEnd)
1505 OS << ',';
1506 OS << " // #" << RAIdx << '\n';
1507 }
1508 OS << "}; // " << Target << "ReadAdvanceTable\n";
1509
1510 // Pool all SchedClass names in a string table.
1511 StringToOffsetTable StrTab;
1512 unsigned InvalidNameOff = StrTab.GetOrAddStringOffset(Str: "InvalidSchedClass");
1513
1514 // Emit a SchedClass table for each processor.
1515 for (const auto &[Idx, Proc] : enumerate(First: SchedModels.procModels())) {
1516 if (!Proc.hasInstrSchedModel())
1517 continue;
1518
1519 std::vector<MCSchedClassDesc> &SCTab =
1520 SchedTables.ProcSchedClasses[1 + Idx];
1521
1522 OS << "\n// {Name, NumMicroOps, BeginGroup, EndGroup, RetireOOO,"
1523 << " ReadAdvanceIdx, WriteProcResIdx, WriteLatencyIdx,"
1524 << " NumReadAdvanceEntries, NumWriteProcResEntries,"
1525 << " NumWriteLatencyEntries}\n";
1526 OS << "static const llvm::MCSchedClassDesc " << Proc.ModelName
1527 << "SchedClasses[] = {\n";
1528
1529 // The first class is always invalid. We no way to distinguish it except by
1530 // name and position.
1531 assert(SchedModels.getSchedClass(0).Name == "NoInstrModel" &&
1532 "invalid class not first");
1533 OS << " {DBGFIELD(" << InvalidNameOff << ") "
1534 << MCSchedClassDesc::InvalidNumMicroOps
1535 << ", false, false, false, 0, 0, 0, 0, 0, 0},\n";
1536
1537 for (unsigned SCIdx = 1, SCEnd = SCTab.size(); SCIdx != SCEnd; ++SCIdx) {
1538 MCSchedClassDesc &MCDesc = SCTab[SCIdx];
1539 const CodeGenSchedClass &SchedClass = SchedModels.getSchedClass(Idx: SCIdx);
1540 unsigned NameOff = StrTab.GetOrAddStringOffset(Str: SchedClass.Name);
1541 OS << " {DBGFIELD(/*" << SchedClass.Name << "*/ " << NameOff << ") ";
1542 if (SchedClass.Name.size() < 18)
1543 OS.indent(NumSpaces: 18 - SchedClass.Name.size());
1544 OS << MCDesc.NumMicroOps << ", " << (MCDesc.BeginGroup ? "true" : "false")
1545 << ", " << (MCDesc.EndGroup ? "true" : "false") << ", "
1546 << (MCDesc.RetireOOO ? "true" : "false") << ", "
1547 << format(Fmt: "%2d", Vals: MCDesc.ReadAdvanceIdx) << ", "
1548 << format(Fmt: "%2d", Vals: MCDesc.WriteProcResIdx) << ", "
1549 << format(Fmt: "%2d", Vals: MCDesc.WriteLatencyIdx) << ", "
1550 << MCDesc.NumReadAdvanceEntries << ", "
1551 << static_cast<unsigned>(MCDesc.NumWriteProcResEntries) << ", "
1552 << static_cast<unsigned>(MCDesc.NumWriteLatencyEntries) << "}, // #"
1553 << SCIdx << '\n';
1554 }
1555 OS << "}; // " << Proc.ModelName << "SchedClasses\n";
1556 }
1557
1558 StrTab.EmitStringTableDef(OS, Name: Target + "SchedClassNames");
1559}
1560
1561void SubtargetEmitter::emitProcessorModels(raw_ostream &OS) {
1562 // For each processor model.
1563 for (const CodeGenProcModel &PM : SchedModels.procModels()) {
1564 // Emit extra processor info if available.
1565 if (PM.hasExtraProcessorInfo())
1566 emitExtraProcessorInfo(ProcModel: PM, OS);
1567 // Emit processor resource table.
1568 if (PM.hasInstrSchedModel())
1569 emitProcessorResources(ProcModel: PM, OS);
1570 else if (!PM.ProcResourceDefs.empty())
1571 PrintFatalError(ErrorLoc: PM.ModelDef->getLoc(),
1572 Msg: "SchedMachineModel defines "
1573 "ProcResources without defining WriteRes SchedWriteRes");
1574 }
1575
1576 OS << "\n";
1577 OS << "extern const llvm::MCSchedModel " << Target << "SchedModels[] = {\n";
1578 for (const CodeGenProcModel &PM : SchedModels.procModels()) {
1579 // Begin processor itinerary properties
1580 OS << "{ // " << PM.ModelName << "\n";
1581 emitProcessorProp(OS, R: PM.ModelDef, Name: "IssueWidth", Separator: ',');
1582 emitProcessorProp(OS, R: PM.ModelDef, Name: "MicroOpBufferSize", Separator: ',');
1583 emitProcessorProp(OS, R: PM.ModelDef, Name: "LoopMicroOpBufferSize", Separator: ',');
1584 emitProcessorProp(OS, R: PM.ModelDef, Name: "LoadLatency", Separator: ',');
1585 emitProcessorProp(OS, R: PM.ModelDef, Name: "HighLatency", Separator: ',');
1586 emitProcessorProp(OS, R: PM.ModelDef, Name: "MispredictPenalty", Separator: ',');
1587
1588 bool PostRAScheduler =
1589 (PM.ModelDef ? PM.ModelDef->getValueAsBit(FieldName: "PostRAScheduler") : false);
1590
1591 OS << " " << (PostRAScheduler ? "true" : "false") << ", // "
1592 << "PostRAScheduler\n";
1593
1594 bool CompleteModel =
1595 (PM.ModelDef ? PM.ModelDef->getValueAsBit(FieldName: "CompleteModel") : false);
1596
1597 OS << " " << (CompleteModel ? "true" : "false") << ", // "
1598 << "CompleteModel\n";
1599
1600 bool EnableIntervals =
1601 (PM.ModelDef ? PM.ModelDef->getValueAsBit(FieldName: "EnableIntervals") : false);
1602
1603 OS << " " << (EnableIntervals ? "true" : "false") << ", // "
1604 << "EnableIntervals\n";
1605
1606 OS << " " << PM.Index << ", // Processor ID\n";
1607 if (PM.hasInstrSchedModel())
1608 OS << " " << PM.ModelName << "ProcResources" << ",\n"
1609 << " " << PM.ModelName << "SchedClasses" << ",\n"
1610 << " " << PM.ProcResourceDefs.size() + 1 << ",\n"
1611 << " " << SchedModels.schedClasses().size() << ",\n";
1612 else
1613 OS << " nullptr, nullptr, 0, 0,"
1614 << " // No instruction-level machine model.\n";
1615 OS << " DBGVAL_OR_NULLPTR(&" << Target
1616 << "SchedClassNames), // SchedClassNames\n";
1617 if (PM.hasItineraries())
1618 OS << " " << PM.ItinsDef->getName() << ",\n";
1619 else
1620 OS << " nullptr, // No Itinerary\n";
1621 if (PM.hasExtraProcessorInfo())
1622 OS << " &" << PM.ModelName << "ExtraInfo,\n";
1623 else
1624 OS << " nullptr // No extra processor descriptor\n";
1625 OS << " },\n";
1626 }
1627 OS << "};\n";
1628}
1629
1630//
1631// EmitSchedModel - Emits all scheduling model tables, folding common patterns.
1632//
1633void SubtargetEmitter::emitSchedModel(raw_ostream &OS) {
1634 OS << "#ifdef DBGFIELD\n"
1635 << "#error \"<target>GenSubtargetInfo.inc requires a DBGFIELD macro\"\n"
1636 << "#endif\n"
1637 << "#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)\n"
1638 << "#define DBGFIELD(x) x,\n"
1639 << "#define DBGVAL_OR_NULLPTR(x) x\n"
1640 << "#else\n"
1641 << "#define DBGFIELD(x)\n"
1642 << "#define DBGVAL_OR_NULLPTR(x) nullptr\n"
1643 << "#endif\n";
1644
1645 if (SchedModels.hasItineraries()) {
1646 std::vector<std::vector<InstrItinerary>> ProcItinLists;
1647 // Emit the stage data
1648 emitStageAndOperandCycleData(OS, ProcItinLists);
1649 emitItineraries(OS, ProcItinLists);
1650 }
1651 OS << "\n// ===============================================================\n"
1652 << "// Data tables for the new per-operand machine model.\n";
1653
1654 SchedClassTables SchedTables;
1655 for (const CodeGenProcModel &ProcModel : SchedModels.procModels()) {
1656 genSchedClassTables(ProcModel, SchedTables);
1657 }
1658 emitSchedClassTables(SchedTables, OS);
1659
1660 // Emit the processor machine model
1661 emitProcessorModels(OS);
1662
1663 OS << "\n#undef DBGFIELD\n";
1664 OS << "\n#undef DBGVAL_OR_NULLPTR\n";
1665}
1666
1667static void emitPredicateProlog(const RecordKeeper &Records, raw_ostream &OS) {
1668 std::string Buffer;
1669 raw_string_ostream Stream(Buffer);
1670
1671 // Print all PredicateProlog records to the output stream.
1672 for (const Record *P : Records.getAllDerivedDefinitions(ClassName: "PredicateProlog"))
1673 Stream << P->getValueAsString(FieldName: "Code") << '\n';
1674
1675 OS << Buffer;
1676}
1677
1678static bool isTruePredicate(const Record *Rec) {
1679 return Rec->isSubClassOf(Name: "MCSchedPredicate") &&
1680 Rec->getValueAsDef(FieldName: "Pred")->isSubClassOf(Name: "MCTrue");
1681}
1682
1683static void expandSchedPredicates(const Record *Rec, PredicateExpander &PE,
1684 bool WrapPredicate, raw_ostream &OS) {
1685 if (Rec->isSubClassOf(Name: "MCSchedPredicate")) {
1686 PE.expandPredicate(OS, Rec: Rec->getValueAsDef(FieldName: "Pred"));
1687 } else if (Rec->isSubClassOf(Name: "FeatureSchedPredicate")) {
1688 const Record *FR = Rec->getValueAsDef(FieldName: "Feature");
1689 if (PE.shouldExpandForMC()) {
1690 // MC version of this predicate will be emitted into
1691 // resolveVariantSchedClassImpl, which accesses MCSubtargetInfo
1692 // through argument STI.
1693 OS << "STI.";
1694 } else {
1695 // Otherwise, this predicate will be emitted directly into
1696 // TargetGenSubtargetInfo::resolveSchedClass, which can just access
1697 // TargetSubtargetInfo / MCSubtargetInfo through `this`.
1698 OS << "this->";
1699 }
1700 OS << "hasFeature(" << PE.getTargetName() << "::" << FR->getName() << ")";
1701 } else if (Rec->isSubClassOf(Name: "SchedPredicateCombiner")) {
1702 std::vector<const Record *> SubPreds =
1703 Rec->getValueAsListOfDefs(FieldName: "Predicates");
1704 if (SubPreds.empty())
1705 PrintFatalError(Rec, Msg: "Empty SchedPredicateCombiner is not allowed");
1706
1707 StringRef Sep;
1708 if (Rec->isSubClassOf(Name: "AllOfSchedPreds")) {
1709 Sep = " && ";
1710 } else if (Rec->isSubClassOf(Name: "AnyOfSchedPreds")) {
1711 Sep = " || ";
1712 } else if (Rec->isSubClassOf(Name: "NotSchedPred")) {
1713 if (SubPreds.size() != 1)
1714 PrintFatalError(Rec,
1715 Msg: "NotSchedPred can only have a single sub-predicate.");
1716 OS << "!";
1717 // We don't have to eagerly wrap this term right now: telling its (only)
1718 // sub-predicate to wrap itself should be sufficient.
1719 WrapPredicate = false;
1720 } else {
1721 PrintFatalError(Rec, Msg: "Unrecognized SchedPredicateCombiner");
1722 }
1723
1724 if (WrapPredicate)
1725 OS << "(";
1726
1727 ListSeparator LS(Sep);
1728 bool WrapSubPreds =
1729 SubPreds.size() > 1 || Rec->isSubClassOf(Name: "NotSchedPred");
1730 for (const Record *SubP : SubPreds)
1731 expandSchedPredicates(Rec: SubP, PE, WrapPredicate: WrapSubPreds, OS&: OS << LS);
1732
1733 if (WrapPredicate)
1734 OS << ")";
1735 } else {
1736 // Expand this legacy predicate and wrap it around braces if there is more
1737 // than one predicate to expand.
1738 OS << (WrapPredicate ? "(" : "") << Rec->getValueAsString(FieldName: "Predicate")
1739 << (WrapPredicate ? ")" : "");
1740 }
1741}
1742
1743static void emitPredicates(const CodeGenSchedTransition &T,
1744 const CodeGenSchedClass &SC, PredicateExpander &PE,
1745 raw_ostream &OS) {
1746 std::string Buffer;
1747 raw_string_ostream SS(Buffer);
1748
1749 // If not all predicates are MCTrue, then we need an if-stmt.
1750 unsigned NumNonTruePreds =
1751 T.PredTerm.size() - count_if(Range: T.PredTerm, P: isTruePredicate);
1752
1753 SS << PE.getIndent();
1754
1755 if (NumNonTruePreds) {
1756 bool FirstNonTruePredicate = true;
1757 SS << "if (";
1758
1759 PE.getIndent() += 2;
1760
1761 for (const Record *Rec : T.PredTerm) {
1762 // Skip predicates that evaluate to "true".
1763 if (isTruePredicate(Rec))
1764 continue;
1765
1766 if (FirstNonTruePredicate) {
1767 FirstNonTruePredicate = false;
1768 } else {
1769 SS << "\n";
1770 SS << PE.getIndent();
1771 SS << "&& ";
1772 }
1773
1774 expandSchedPredicates(Rec, PE, /*WrapPredicate=*/NumNonTruePreds > 1, OS&: SS);
1775 }
1776
1777 SS << ")\n"; // end of if-stmt
1778 --PE.getIndent();
1779 SS << PE.getIndent();
1780 --PE.getIndent();
1781 }
1782
1783 SS << "return " << T.ToClassIdx << "; // " << SC.Name << '\n';
1784 OS << Buffer;
1785}
1786
1787// Used by method `SubtargetEmitter::emitSchedModelHelpersImpl()` to generate
1788// epilogue code for the auto-generated helper.
1789static void emitSchedModelHelperEpilogue(raw_ostream &OS,
1790 bool ShouldReturnZero) {
1791 if (ShouldReturnZero) {
1792 OS << " // Don't know how to resolve this scheduling class.\n"
1793 << " return 0;\n";
1794 return;
1795 }
1796
1797 OS << " report_fatal_error(\"Expected a variant SchedClass\");\n";
1798}
1799
1800static bool hasMCSchedPredicate(const Record *Rec) {
1801 if (Rec->isSubClassOf(Name: "MCSchedPredicate") ||
1802 Rec->isSubClassOf(Name: "FeatureSchedPredicate"))
1803 return true;
1804
1805 if (Rec->isSubClassOf(Name: "SchedPredicateCombiner")) {
1806 // Check its sub-predicates recursively.
1807 std::vector<const Record *> SubPreds =
1808 Rec->getValueAsListOfDefs(FieldName: "Predicates");
1809 return all_of(Range&: SubPreds, P: hasMCSchedPredicate);
1810 }
1811
1812 return false;
1813}
1814static bool hasMCSchedPredicates(const CodeGenSchedTransition &T) {
1815 return all_of(Range: T.PredTerm, P: hasMCSchedPredicate);
1816}
1817
1818static void collectVariantClasses(const CodeGenSchedModels &SchedModels,
1819 IdxVec &VariantClasses,
1820 bool OnlyExpandMCInstPredicates) {
1821 for (const CodeGenSchedClass &SC : SchedModels.schedClasses()) {
1822 // Ignore non-variant scheduling classes.
1823 if (SC.Transitions.empty())
1824 continue;
1825
1826 if (OnlyExpandMCInstPredicates) {
1827 // Ignore this variant scheduling class no transitions use any meaningful
1828 // MCSchedPredicate definitions.
1829 if (llvm::none_of(Range: SC.Transitions, P: hasMCSchedPredicates))
1830 continue;
1831 }
1832
1833 VariantClasses.push_back(x: SC.Index);
1834 }
1835}
1836
1837static void collectProcessorIndices(const CodeGenSchedClass &SC,
1838 IdxVec &ProcIndices) {
1839 // A variant scheduling class may define transitions for multiple
1840 // processors. This function identifies wich processors are associated with
1841 // transition rules specified by variant class `SC`.
1842 for (const CodeGenSchedTransition &T : SC.Transitions) {
1843 IdxVec PI;
1844 std::set_union(first1: &T.ProcIndex, last1: &T.ProcIndex + 1, first2: ProcIndices.begin(),
1845 last2: ProcIndices.end(), result: std::back_inserter(x&: PI));
1846 ProcIndices = std::move(PI);
1847 }
1848}
1849
1850static bool isAlwaysTrue(const CodeGenSchedTransition &T) {
1851 return llvm::all_of(Range: T.PredTerm, P: isTruePredicate);
1852}
1853
1854void SubtargetEmitter::emitSchedModelHelpersImpl(
1855 raw_ostream &OS, bool OnlyExpandMCInstPredicates) {
1856 IdxVec VariantClasses;
1857 collectVariantClasses(SchedModels, VariantClasses,
1858 OnlyExpandMCInstPredicates);
1859
1860 if (VariantClasses.empty()) {
1861 emitSchedModelHelperEpilogue(OS, ShouldReturnZero: OnlyExpandMCInstPredicates);
1862 return;
1863 }
1864
1865 // Construct a switch statement where the condition is a check on the
1866 // scheduling class identifier. There is a `case` for every variant class
1867 // defined by the processor models of this target.
1868 // Each `case` implements a number of rules to resolve (i.e. to transition
1869 // from) a variant scheduling class to another scheduling class. Rules are
1870 // described by instances of CodeGenSchedTransition. Note that transitions may
1871 // not be valid for all processors.
1872 OS << " switch (SchedClass) {\n";
1873 for (unsigned VC : VariantClasses) {
1874 IdxVec ProcIndices;
1875 const CodeGenSchedClass &SC = SchedModels.getSchedClass(Idx: VC);
1876 collectProcessorIndices(SC, ProcIndices);
1877
1878 OS << " case " << VC << ": // " << SC.Name << '\n';
1879
1880 PredicateExpander PE(Target);
1881 PE.setByRef(false);
1882 PE.setExpandForMC(OnlyExpandMCInstPredicates);
1883 for (unsigned PI : ProcIndices) {
1884 OS << " ";
1885
1886 // Emit a guard on the processor ID.
1887 if (PI != 0) {
1888 OS << (OnlyExpandMCInstPredicates
1889 ? "if (CPUID == "
1890 : "if (SchedModel->getProcessorID() == ");
1891 OS << PI << ") ";
1892 OS << "{ // " << SchedModels.procModels()[PI].ModelName << '\n';
1893 }
1894
1895 // Now emit transitions associated with processor PI.
1896 const CodeGenSchedTransition *FinalT = nullptr;
1897 for (const CodeGenSchedTransition &T : SC.Transitions) {
1898 if (PI != 0 && T.ProcIndex != PI)
1899 continue;
1900
1901 // Emit only transitions based on MCSchedPredicate, if it's the case.
1902 // At least the transition specified by NoSchedPred is emitted,
1903 // which becomes the default transition for those variants otherwise
1904 // not based on MCSchedPredicate.
1905 // FIXME: preferably, llvm-mca should instead assume a reasonable
1906 // default when a variant transition is not based on MCSchedPredicate
1907 // for a given processor.
1908 if (OnlyExpandMCInstPredicates && !hasMCSchedPredicates(T))
1909 continue;
1910
1911 // If transition is folded to 'return X' it should be the last one.
1912 if (isAlwaysTrue(T)) {
1913 FinalT = &T;
1914 continue;
1915 }
1916 PE.getIndent() = 3;
1917 emitPredicates(T, SC: SchedModels.getSchedClass(Idx: T.ToClassIdx), PE, OS);
1918 }
1919 if (FinalT)
1920 emitPredicates(T: *FinalT, SC: SchedModels.getSchedClass(Idx: FinalT->ToClassIdx),
1921 PE, OS);
1922
1923 OS << " }\n";
1924
1925 if (PI == 0)
1926 break;
1927 }
1928
1929 if (SC.isInferred())
1930 OS << " return " << SC.Index << ";\n";
1931 OS << " break;\n";
1932 }
1933
1934 OS << " };\n";
1935
1936 emitSchedModelHelperEpilogue(OS, ShouldReturnZero: OnlyExpandMCInstPredicates);
1937}
1938
1939// Emits the scheduling classes each variant scheduling class may be resolved
1940// to, so the variant classes can be estimated without an instruction.
1941void SubtargetEmitter::emitVariantSchedClassTargets(raw_ostream &OS) {
1942 OS << "ArrayRef<unsigned> getVariantSchedClassTargets(unsigned SchedClass,\n"
1943 << " unsigned CPUID) {\n";
1944 IdxVec VariantClasses;
1945 collectVariantClasses(SchedModels, VariantClasses,
1946 /*OnlyExpandMCInstPredicates=*/false);
1947 if (VariantClasses.empty()) {
1948 OS << " return {};\n}\n";
1949 return;
1950 }
1951 OS << " switch (SchedClass) {\n";
1952 for (unsigned VC : VariantClasses) {
1953 const CodeGenSchedClass &SC = SchedModels.getSchedClass(Idx: VC);
1954 IdxVec ProcIndices;
1955 collectProcessorIndices(SC, ProcIndices);
1956 OS << " case " << VC << ": // " << SC.Name << '\n';
1957 for (unsigned PI : ProcIndices) {
1958 OS << " ";
1959 if (PI != 0)
1960 OS << "if (CPUID == " << PI << ") ";
1961 OS << "{ // " << SchedModels.procModels()[PI].ModelName << '\n'
1962 << " static const unsigned Targets[] = {";
1963 ListSeparator LS;
1964 for (const CodeGenSchedTransition &T : SC.Transitions)
1965 if (PI == 0 || T.ProcIndex == PI)
1966 OS << LS << T.ToClassIdx << " /* "
1967 << SchedModels.getSchedClass(Idx: T.ToClassIdx).Name << " */";
1968 OS << "};\n"
1969 << " return Targets;\n"
1970 << " }\n";
1971 if (PI == 0)
1972 break;
1973 }
1974 OS << " break;\n";
1975 }
1976 OS << " }\n"
1977 << " return {};\n"
1978 << "}\n";
1979}
1980
1981void SubtargetEmitter::emitSchedModelHelpers(const std::string &ClassName,
1982 raw_ostream &OS) {
1983 OS << "unsigned " << ClassName
1984 << "\n::resolveSchedClass(unsigned SchedClass, const MachineInstr *MI,"
1985 << " const TargetSchedModel *SchedModel) const {\n";
1986
1987 // Emit the predicate prolog code.
1988 emitPredicateProlog(Records, OS);
1989
1990 // Emit target predicates.
1991 emitSchedModelHelpersImpl(OS);
1992
1993 OS << "} // " << ClassName << "::resolveSchedClass\n\n";
1994
1995 OS << "unsigned " << ClassName
1996 << "\n::resolveVariantSchedClass(unsigned SchedClass, const MCInst *MI,"
1997 << " const MCInstrInfo *MCII, unsigned CPUID) const {\n"
1998 << " return " << Target << "_MC"
1999 << "::resolveVariantSchedClassImpl(SchedClass, MI, MCII, *this, CPUID);\n"
2000 << "} // " << ClassName << "::resolveVariantSchedClass\n\n";
2001
2002 OS << "std::unique_ptr<MCSubtargetInfo> " << ClassName
2003 << "::clone() const {\n"
2004 << " return " << Target << "_MC::cloneSubtargetInfo(*this);\n"
2005 << "} // " << ClassName << "::clone\n\n";
2006
2007 STIPredicateExpander PE(Target, /*Indent=*/0);
2008 PE.setClassPrefix(ClassName);
2009 PE.setExpandDefinition(true);
2010 PE.setByRef(false);
2011
2012 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
2013 PE.expandSTIPredicate(OS, Fn);
2014}
2015
2016void SubtargetEmitter::emitHwModeCheck(const std::string &ClassName,
2017 raw_ostream &OS, bool IsMC) {
2018 const CodeGenHwModes &CGH = TGT.getHwModes();
2019 assert(CGH.getNumModeIds() > 0);
2020 if (CGH.getNumModeIds() == 1)
2021 return;
2022
2023 // Collect all HwModes and related features defined in the TD files,
2024 // and store them as a bit set.
2025 unsigned ValueTypeModes = 0;
2026 unsigned RegInfoModes = 0;
2027 unsigned EncodingInfoModes = 0;
2028 for (const auto &MS : CGH.getHwModeSelects()) {
2029 for (const HwModeSelect::PairType &P : MS.second.Items) {
2030 if (P.first == DefaultMode)
2031 continue;
2032 if (P.second->isSubClassOf(Name: "ValueType")) {
2033 ValueTypeModes |= (1 << (P.first - 1));
2034 } else if (P.second->isSubClassOf(Name: "RegInfo") ||
2035 P.second->isSubClassOf(Name: "Register") ||
2036 P.second->isSubClassOf(Name: "SubRegRange") ||
2037 P.second->isSubClassOf(Name: "RegisterClassLike")) {
2038 RegInfoModes |= (1 << (P.first - 1));
2039 } else if (P.second->isSubClassOf(Name: "InstructionEncoding")) {
2040 EncodingInfoModes |= (1 << (P.first - 1));
2041 }
2042 }
2043 }
2044
2045 // Start emitting for getHwModeSet().
2046 OS << "unsigned " << ClassName << "::getHwModeSet() const {\n";
2047 if (IsMC) {
2048 OS << " [[maybe_unused]] const FeatureBitset &FB = getFeatureBits();\n";
2049 } else {
2050 const ArrayRef<const Record *> &Prologs =
2051 Records.getAllDerivedDefinitions(ClassName: "HwModePredicateProlog");
2052 if (!Prologs.empty()) {
2053 for (const Record *P : Prologs)
2054 OS << P->getValueAsString(FieldName: "Code") << '\n';
2055 } else {
2056 // Works for most targets.
2057 OS << " [[maybe_unused]] const auto *Subtarget =\n"
2058 << " static_cast<const " << Target << "Subtarget *>(this);\n";
2059 }
2060 }
2061 OS << " // Collect HwModes and store them as a bit set.\n";
2062 OS << " unsigned Modes = 0;\n";
2063 for (unsigned M = 1, NumModes = CGH.getNumModeIds(); M != NumModes; ++M) {
2064 const HwMode &HM = CGH.getMode(Id: M);
2065 OS << " if (";
2066 if (IsMC)
2067 SubtargetFeatureInfo::emitMCPredicateCheck(OS, TargetName: Target, Predicates: HM.Predicates);
2068 else
2069 SubtargetFeatureInfo::emitPredicateCheck(OS, Predicates: HM.Predicates);
2070 OS << ") Modes |= (1 << " << (M - 1) << ");\n";
2071 }
2072 OS << " return Modes;\n}\n";
2073 // End emitting for getHwModeSet().
2074
2075 auto HandlePerMode = [&](std::string ModeType, unsigned ModeInBitSet) {
2076 OS << " case HwMode_" << ModeType << ":\n";
2077 if (ModeInBitSet == 0) {
2078 OS << " // No HwMode for " << ModeType << ".\n"
2079 << " return 0;\n";
2080 } else {
2081 OS << " Modes &= " << ModeInBitSet << ";\n"
2082 << " if (!Modes)\n return Modes;\n"
2083 << " if (!llvm::has_single_bit<unsigned>(Modes))\n"
2084 << " llvm_unreachable(\"Two or more HwModes for " << ModeType
2085 << " were found!\");\n"
2086 << " return llvm::countr_zero(Modes) + 1;\n";
2087 }
2088 };
2089
2090 // Start emitting for getHwMode().
2091 OS << "unsigned " << ClassName
2092 << "::getHwMode(enum HwModeType type) const {\n";
2093 OS << " unsigned Modes = getHwModeSet();\n\n";
2094 OS << " if (!Modes)\n return Modes;\n\n";
2095 OS << " switch (type) {\n";
2096 OS << " case HwMode_Default:\n return llvm::countr_zero(Modes) + 1;\n";
2097 HandlePerMode("ValueType", ValueTypeModes);
2098 HandlePerMode("RegInfo", RegInfoModes);
2099 HandlePerMode("EncodingInfo", EncodingInfoModes);
2100 OS << " }\n";
2101 OS << " llvm_unreachable(\"unexpected HwModeType\");\n"
2102 << " return 0; // should not get here\n}\n";
2103 // End emitting for getHwMode().
2104}
2105
2106void SubtargetEmitter::emitGetMacroFusions(const std::string &ClassName,
2107 raw_ostream &OS) {
2108 if (!TGT.hasMacroFusion())
2109 return;
2110
2111 OS << "std::vector<MacroFusionPredTy> " << ClassName
2112 << "::getMacroFusions() const {\n";
2113 OS.indent(NumSpaces: 2) << "std::vector<MacroFusionPredTy> Fusions;\n";
2114 for (auto *Fusion : TGT.getMacroFusions()) {
2115 StringRef Name = CodeGenTarget::getMacroFusionName(Fusion);
2116 OS.indent(NumSpaces: 2) << "if (hasFeature(" << Target << "::" << Name
2117 << ")) Fusions.push_back(llvm::is" << Name << ");\n";
2118 }
2119
2120 OS.indent(NumSpaces: 2) << "return Fusions;\n";
2121 OS << "}\n";
2122}
2123
2124// Produces a subtarget specific function for parsing
2125// the subtarget features string.
2126void SubtargetEmitter::parseFeaturesFunction(raw_ostream &OS) {
2127 ArrayRef<const Record *> Features =
2128 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
2129
2130 OS << "// ParseSubtargetFeatures - Parses features string setting specified\n"
2131 << "// subtarget options.\n"
2132 << "void llvm::";
2133 OS << Target;
2134 OS << "Subtarget::ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, "
2135 << "StringRef FS) {\n"
2136 << " LLVM_DEBUG(dbgs() << \"\\nFeatures:\" << FS);\n"
2137 << " LLVM_DEBUG(dbgs() << \"\\nCPU:\" << CPU);\n"
2138 << " LLVM_DEBUG(dbgs() << \"\\nTuneCPU:\" << TuneCPU << \"\\n\\n\");\n";
2139
2140 if (Features.empty()) {
2141 OS << "}\n";
2142 return;
2143 }
2144
2145 OS << " InitMCProcessorInfo(CPU, TuneCPU, FS);\n"
2146 << " const FeatureBitset &Bits = getFeatureBits();\n";
2147
2148 for (const Record *R : Features) {
2149 // Next record
2150 StringRef Instance = R->getName();
2151 StringRef Value = R->getValueAsString(FieldName: "Value");
2152 StringRef FieldName = R->getValueAsString(FieldName: "FieldName");
2153
2154 if (Value == "true" || Value == "false")
2155 OS << " if (Bits[" << Target << "::" << Instance << "]) " << FieldName
2156 << " = " << Value << ";\n";
2157 else
2158 OS << " if (Bits[" << Target << "::" << Instance << "] && " << FieldName
2159 << " < " << Value << ") " << FieldName << " = " << Value << ";\n";
2160 }
2161
2162 OS << "}\n";
2163}
2164
2165void SubtargetEmitter::emitInlineFeatures(const std::string &ClassName,
2166 raw_ostream &OS, StringRef Behavior) {
2167 std::vector<const Record *> FeatureList =
2168 Records.getAllDerivedDefinitions(ClassName: "SubtargetFeature");
2169 llvm::sort(C&: FeatureList, Comp: LessRecordFieldFieldName());
2170
2171 OS << "const FeatureBitset &" << ClassName << "::get" << Behavior
2172 << "Features() const {\n"
2173 << " static constexpr FeatureBitset Features = {\n";
2174
2175 for (const Record *Feature : FeatureList)
2176 if (Behavior == Feature->getValueAsDef(FieldName: "InlineBehavior")->getName())
2177 OS << Target << "::" << Feature->getName() << ",\n";
2178
2179 OS << " };\n"
2180 << " return Features;\n"
2181 << "}\n\n";
2182}
2183
2184void SubtargetEmitter::emitGenMCSubtargetInfo(raw_ostream &OS) {
2185 {
2186 NamespaceEmitter NS(OS, (Target + Twine("_MC")).str());
2187 OS << "unsigned resolveVariantSchedClassImpl(unsigned SchedClass,\n"
2188 << " const MCInst *MI, const MCInstrInfo *MCII, "
2189 << "const MCSubtargetInfo &STI, unsigned CPUID) {\n";
2190 emitSchedModelHelpersImpl(OS, /* OnlyExpandMCPredicates */ OnlyExpandMCInstPredicates: true);
2191 OS << "}\n";
2192 emitVariantSchedClassTargets(OS);
2193 }
2194
2195 OS << "struct " << Target
2196 << "GenMCSubtargetInfo : public MCSubtargetInfo {\n";
2197 OS << " " << Target << "GenMCSubtargetInfo(const Triple &TT,\n"
2198 << " StringRef CPU, StringRef TuneCPU, StringRef FS,\n"
2199 << " StringTable PN,\n"
2200 << " ArrayRef<SubtargetFeatureKV> PF,\n"
2201 << " ArrayRef<SubtargetSubTypeKV> PD,\n"
2202 << " ArrayRef<SubtargetSubTypeAliasKV> PA,\n"
2203 << " const MCSchedModel *PSM,\n"
2204 << " const MCWriteProcResEntry *WPR,\n"
2205 << " const MCWriteLatencyEntry *WL,\n"
2206 << " const MCReadAdvanceEntry *RA, const InstrStage *IS,\n"
2207 << " const unsigned *OC, const unsigned *FP) :\n"
2208 << " MCSubtargetInfo(TT, CPU, TuneCPU, FS, PN, PF, PD, PA, PSM,\n"
2209 << " WPR, WL, RA, IS, OC, FP) { }\n\n"
2210 << " explicit " << Target
2211 << "GenMCSubtargetInfo(const MCSubtargetInfo &STI)\n"
2212 << " : MCSubtargetInfo(STI) {}\n\n"
2213 << " std::unique_ptr<MCSubtargetInfo> clone() const final {\n"
2214 << " return std::make_unique<" << Target
2215 << "GenMCSubtargetInfo>(*this);\n"
2216 << " }\n\n"
2217 << " unsigned resolveVariantSchedClass(unsigned SchedClass,\n"
2218 << " const MCInst *MI, const MCInstrInfo *MCII,\n"
2219 << " unsigned CPUID) const final {\n"
2220 << " return " << Target << "_MC"
2221 << "::resolveVariantSchedClassImpl(SchedClass, MI, MCII, *this, CPUID);\n";
2222 OS << " }\n";
2223 if (TGT.getHwModes().getNumModeIds() > 1) {
2224 OS << " unsigned getHwModeSet() const final;\n";
2225 OS << " unsigned getHwMode(enum HwModeType type = HwMode_Default) const "
2226 "final;\n";
2227 }
2228 OS << "};\n";
2229 emitHwModeCheck(ClassName: Target + "GenMCSubtargetInfo", OS, /*IsMC=*/true);
2230 {
2231 NamespaceEmitter NS(OS, (Target + Twine("_MC")).str());
2232 OS << "std::unique_ptr<MCSubtargetInfo> cloneSubtargetInfo("
2233 << "const MCSubtargetInfo &STI) {\n"
2234 << " return std::make_unique<" << Target
2235 << "GenMCSubtargetInfo>(STI);\n"
2236 << "}\n";
2237 }
2238}
2239
2240void SubtargetEmitter::emitMcInstrAnalysisPredicateFunctions(raw_ostream &OS) {
2241 STIPredicateExpander PE(Target, /*Indent=*/0);
2242
2243 {
2244 IfDefEmitter IfDefDecls(OS, "GET_STIPREDICATE_DECLS_FOR_MC_ANALYSIS");
2245 PE.setExpandForMC(true);
2246 PE.setByRef(true);
2247 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
2248 PE.expandSTIPredicate(OS, Fn);
2249 }
2250
2251 IfDefEmitter IfDefDefs(OS, "GET_STIPREDICATE_DEFS_FOR_MC_ANALYSIS");
2252 std::string ClassPrefix = Target + "MCInstrAnalysis";
2253 PE.setExpandDefinition(true);
2254 PE.setClassPrefix(ClassPrefix);
2255 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
2256 PE.expandSTIPredicate(OS, Fn);
2257}
2258
2259FeatureMapTy SubtargetEmitter::emitEnums(raw_ostream &OS) {
2260 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_ENUM");
2261 NamespaceEmitter NS(OS, "llvm");
2262 return enumeration(OS);
2263}
2264
2265SubtargetEmitter::MCDescInfo
2266SubtargetEmitter::emitMCDesc(raw_ostream &OS, const FeatureMapTy &FeatureMap) {
2267 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_MC_DESC");
2268 NamespaceEmitter LlvmNS(OS, "llvm");
2269
2270 MCDescInfo Res;
2271 auto [NumFeatures, FeatureStrTabSize] = featureKeyValues(OS, FeatureMap);
2272 Res.NumFeatures = NumFeatures;
2273 Res.FeatureStrTabSize = FeatureStrTabSize;
2274 OS << "\n";
2275 emitSchedModel(OS);
2276 OS << "\n";
2277 auto [NumProcs, NumAliases, SubTypeStrTabSize] = cpuKeyValues(OS, FeatureMap);
2278 Res.NumProcs = NumProcs;
2279 Res.NumAliases = NumAliases;
2280 Res.SubTypeStrTabSize = SubTypeStrTabSize;
2281 OS << "\n";
2282
2283 // MCInstrInfo initialization routine.
2284 emitGenMCSubtargetInfo(OS);
2285
2286 OS << "\nstatic inline MCSubtargetInfo *create" << Target
2287 << "MCSubtargetInfoImpl("
2288 << "const Triple &TT, StringRef CPU, StringRef TuneCPU, StringRef FS) {\n";
2289 OS << " return new " << Target
2290 << "GenMCSubtargetInfo(TT, CPU, TuneCPU, FS, ";
2291 OS << "StringTable(" << Target << "SubTypeKVStorage.Strings), ";
2292 if (Res.NumFeatures)
2293 OS << Target << "FeatureKVStorage.Features, ";
2294 else
2295 OS << "{}, ";
2296 if (Res.NumProcs)
2297 OS << Target << "SubTypeKVStorage.SubTypes, "
2298 << "ArrayRef(" << Target << "SubTypeKVStorage.Aliases).take_front("
2299 << Res.NumAliases << "), " << Target << "SchedModels, ";
2300 else
2301 OS << "{}, {}, nullptr, ";
2302 OS << '\n';
2303 OS.indent(NumSpaces: 22);
2304 OS << Target << "WriteProcResTable, " << Target << "WriteLatencyTable, "
2305 << Target << "ReadAdvanceTable, ";
2306 OS << '\n';
2307 OS.indent(NumSpaces: 22);
2308 if (SchedModels.hasItineraries()) {
2309 OS << Target << "Stages, " << Target << "OperandCycles, " << Target
2310 << "ForwardingPaths";
2311 } else {
2312 OS << "nullptr, nullptr, nullptr";
2313 }
2314 OS << ");\n}\n\n";
2315 return Res;
2316}
2317
2318void SubtargetEmitter::emitTargetDesc(raw_ostream &OS) {
2319 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_TARGET_DESC");
2320
2321 OS << "#include \"llvm/ADT/BitmaskEnum.h\"\n";
2322 OS << "#include \"llvm/Support/Debug.h\"\n";
2323 OS << "#include \"llvm/Support/raw_ostream.h\"\n\n";
2324 parseFeaturesFunction(OS);
2325}
2326
2327void SubtargetEmitter::emitHeader(raw_ostream &OS) {
2328 // Create a TargetSubtargetInfo subclass to hide the MC layer initialization.
2329 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_HEADER");
2330 NamespaceEmitter LLVMNS(OS, "llvm");
2331
2332 std::string ClassName = Target + "GenSubtargetInfo";
2333 OS << "class DFAPacketizer;\n";
2334 {
2335 NamespaceEmitter MCNS(OS, (Target + Twine("_MC")).str());
2336 OS << "unsigned resolveVariantSchedClassImpl(unsigned SchedClass,"
2337 << " const MCInst *MI, const MCInstrInfo *MCII, "
2338 << "const MCSubtargetInfo &STI, unsigned CPUID);\n"
2339 << "std::unique_ptr<MCSubtargetInfo> cloneSubtargetInfo("
2340 << "const MCSubtargetInfo &STI);\n";
2341 OS << "ArrayRef<unsigned> getVariantSchedClassTargets(unsigned SchedClass,"
2342 << " unsigned CPUID);\n";
2343 }
2344 OS << "struct " << ClassName << " : public TargetSubtargetInfo {\n"
2345 << " explicit " << ClassName << "(const Triple &TT, StringRef CPU, "
2346 << "StringRef TuneCPU, StringRef FS);\n"
2347 << "public:\n"
2348 << " std::unique_ptr<MCSubtargetInfo> clone() const final;\n"
2349 << " unsigned resolveSchedClass(unsigned SchedClass, "
2350 << " const MachineInstr *DefMI,"
2351 << " const TargetSchedModel *SchedModel) const final;\n"
2352 << " unsigned resolveVariantSchedClass(unsigned SchedClass,"
2353 << " const MCInst *MI, const MCInstrInfo *MCII,"
2354 << " unsigned CPUID) const final;\n"
2355 << " DFAPacketizer *createDFAPacketizer(const InstrItineraryData *IID)"
2356 << " const;\n";
2357
2358 const CodeGenHwModes &CGH = TGT.getHwModes();
2359 if (CGH.getNumModeIds() > 1) {
2360 OS << " enum class " << Target << "HwModeBits : unsigned {\n";
2361 for (unsigned M = 0, NumModes = CGH.getNumModeIds(); M != NumModes; ++M) {
2362 StringRef ModeName = CGH.getModeName(Id: M, /*IncludeDefault=*/true);
2363 OS << " " << ModeName << " = ";
2364 if (M == 0)
2365 OS << "0";
2366 else
2367 OS << "(1 << " << (M - 1) << ")";
2368 OS << ",\n";
2369 if (M == NumModes - 1) {
2370 OS << "\n";
2371 OS << " LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/" << ModeName
2372 << "),\n";
2373 }
2374 }
2375 OS << " };\n";
2376
2377 OS << " unsigned getHwModeSet() const final;\n";
2378 OS << " unsigned getHwMode(enum HwModeType type = HwMode_Default) const "
2379 "final;\n";
2380 }
2381 if (TGT.hasMacroFusion())
2382 OS << " std::vector<MacroFusionPredTy> getMacroFusions() const "
2383 "final;\n";
2384
2385 OS << " const FeatureBitset &getInlineIgnoreFeatures() const override;\n";
2386 OS << " const FeatureBitset &getInlineInverseFeatures() const override;\n";
2387 OS << " const FeatureBitset &getInlineMustMatchFeatures() const override;\n";
2388
2389 STIPredicateExpander PE(Target);
2390 PE.setByRef(false);
2391 for (const STIPredicateFunction &Fn : SchedModels.getSTIPredicates())
2392 PE.expandSTIPredicate(OS, Fn);
2393 OS << "};\n";
2394}
2395
2396void SubtargetEmitter::emitCtor(raw_ostream &OS, MCDescInfo DescInfo) {
2397 IfDefEmitter IfDef(OS, "GET_SUBTARGETINFO_CTOR");
2398 OS << "#include \"llvm/CodeGen/TargetSchedule.h\"\n\n";
2399
2400 NamespaceEmitter LLVMNS(OS, "llvm");
2401 OS << "extern const llvm::StringRef " << Target << "Names[];\n";
2402 if (DescInfo.NumFeatures) {
2403 OS << "extern const llvm::SubtargetFeatureKVStorage<"
2404 << DescInfo.NumFeatures << ", " << DescInfo.FeatureStrTabSize << "> "
2405 << Target << "FeatureKVStorage;\n";
2406 }
2407 OS << "extern const llvm::SubtargetSubTypeKVStorage<" << DescInfo.NumProcs
2408 << ", " << DescInfo.NumAliases << ", " << DescInfo.SubTypeStrTabSize
2409 << "> " << Target << "SubTypeKVStorage;\n";
2410 OS << "extern const llvm::MCSchedModel " << Target << "SchedModels[];\n";
2411 OS << "extern const llvm::MCWriteProcResEntry " << Target
2412 << "WriteProcResTable[];\n";
2413 OS << "extern const llvm::MCWriteLatencyEntry " << Target
2414 << "WriteLatencyTable[];\n";
2415 OS << "extern const llvm::MCReadAdvanceEntry " << Target
2416 << "ReadAdvanceTable[];\n";
2417
2418 if (SchedModels.hasItineraries()) {
2419 OS << "extern const llvm::InstrStage " << Target << "Stages[];\n";
2420 OS << "extern const unsigned " << Target << "OperandCycles[];\n";
2421 OS << "extern const unsigned " << Target << "ForwardingPaths[];\n";
2422 }
2423
2424 std::string ClassName = Target + "GenSubtargetInfo";
2425 OS << ClassName << "::" << ClassName << "(const Triple &TT, StringRef CPU, "
2426 << "StringRef TuneCPU, StringRef FS)\n";
2427
2428 OS << " : TargetSubtargetInfo(TT, CPU, TuneCPU, FS, ";
2429 OS << "StringTable(" << Target << "SubTypeKVStorage.Strings), ";
2430 if (DescInfo.NumFeatures)
2431 OS << "ArrayRef(" << Target << "FeatureKVStorage.Features), ";
2432 else
2433 OS << "{}, ";
2434 if (DescInfo.NumProcs) {
2435 OS << "ArrayRef(" << Target << "SubTypeKVStorage.SubTypes), "
2436 << "ArrayRef(" << Target << "SubTypeKVStorage.Aliases).take_front("
2437 << DescInfo.NumAliases << "), " << Target << "SchedModels, ";
2438 } else {
2439 OS << "{}, {}, nullptr, ";
2440 }
2441 OS << '\n';
2442 OS.indent(NumSpaces: 24);
2443 OS << Target << "WriteProcResTable, " << Target << "WriteLatencyTable, "
2444 << Target << "ReadAdvanceTable, ";
2445 OS << '\n';
2446 OS.indent(NumSpaces: 24);
2447 if (SchedModels.hasItineraries()) {
2448 OS << Target << "Stages, " << Target << "OperandCycles, " << Target
2449 << "ForwardingPaths";
2450 } else {
2451 OS << "nullptr, nullptr, nullptr";
2452 }
2453 OS << ") {}\n\n";
2454
2455 emitSchedModelHelpers(ClassName, OS);
2456 emitHwModeCheck(ClassName, OS, /*IsMC=*/false);
2457 emitGetMacroFusions(ClassName, OS);
2458 emitInlineFeatures(ClassName, OS, Behavior: "InlineIgnore");
2459 emitInlineFeatures(ClassName, OS, Behavior: "InlineInverse");
2460 emitInlineFeatures(ClassName, OS, Behavior: "InlineMustMatch");
2461}
2462
2463//
2464// SubtargetEmitter::run - Main subtarget enumeration emitter.
2465//
2466void SubtargetEmitter::run(raw_ostream &OS) {
2467 emitSourceFileHeader(Desc: "Subtarget Enumeration Source Fragment", OS);
2468
2469 auto FeatureMap = emitEnums(OS);
2470 emitSubtargetInfoMacroCalls(OS);
2471 MCDescInfo DescInfo = emitMCDesc(OS, FeatureMap);
2472 emitTargetDesc(OS);
2473 emitHeader(OS);
2474 emitCtor(OS, DescInfo);
2475 emitMcInstrAnalysisPredicateFunctions(OS);
2476}
2477
2478static TableGen::Emitter::OptClass<SubtargetEmitter>
2479 X("gen-subtarget", "Generate subtarget enumerations");
2480