1//===- RuntimeLibcallEmitter.cpp - Properties from RuntimeLibcalls.td -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#define DEBUG_TYPE "runtime-libcall-emitter"
10
11#include "RuntimeLibcalls.h"
12
13#include "SequenceToOffsetTable.h"
14#include "llvm/ADT/DenseSet.h"
15#include "llvm/ADT/MapVector.h"
16#include "llvm/ADT/StringExtras.h"
17#include "llvm/ADT/StringRef.h"
18#include "llvm/Support/Debug.h"
19#include "llvm/Support/Format.h"
20#include "llvm/Support/FormatVariadic.h"
21#include "llvm/Support/raw_ostream.h"
22#include "llvm/Support/xxhash.h"
23#include "llvm/TableGen/CodeGenHelpers.h"
24#include "llvm/TableGen/Error.h"
25#include "llvm/TableGen/Record.h"
26#include "llvm/TableGen/SetTheory.h"
27#include "llvm/TableGen/StringToOffsetTable.h"
28#include "llvm/TableGen/TableGenBackend.h"
29
30using namespace llvm;
31
32namespace {
33// Pair of a RuntimeLibcallAvailability and LibcallCallingConv to use as a map
34// key.
35struct PredicateWithCC {
36 const Record *Availability = nullptr;
37 const Record *CallingConv = nullptr;
38
39 PredicateWithCC() = default;
40 PredicateWithCC(std::pair<const Record *, const Record *> P)
41 : Availability(P.first), CallingConv(P.second) {}
42
43 PredicateWithCC(const Record *P, const Record *C)
44 : Availability(P), CallingConv(C) {}
45};
46
47inline bool operator==(PredicateWithCC LHS, PredicateWithCC RHS) {
48 return LHS.Availability == RHS.Availability &&
49 LHS.CallingConv == RHS.CallingConv;
50}
51} // namespace
52
53namespace {
54/// A floating-point libcall family parsed from a RuntimeLibcallFamily record.
55struct FPLibcallFamily {
56 StringRef Base;
57 std::vector<StringRef> Intrinsics;
58 std::vector<StringRef> VectorSuffixes;
59
60 explicit FPLibcallFamily(const Record *R)
61 : Base(R->getValueAsString(FieldName: "LibcallBase")),
62 Intrinsics(R->getValueAsListOfStrings(FieldName: "Intrinsics")),
63 VectorSuffixes(R->getValueAsListOfStrings(FieldName: "VectorSuffixes")) {}
64};
65} // namespace
66
67namespace llvm {
68template <> struct DenseMapInfo<PredicateWithCC, void> {
69 static unsigned getHashValue(const PredicateWithCC Val) {
70 auto Pair = std::make_pair(x: Val.Availability, y: Val.CallingConv);
71 return DenseMapInfo<
72 std::pair<const Record *, const Record *>>::getHashValue(PairVal: Pair);
73 }
74
75 static bool isEqual(PredicateWithCC LHS, PredicateWithCC RHS) {
76 return LHS == RHS;
77 }
78};
79
80class RuntimeLibcallEmitter {
81private:
82 const RecordKeeper &Records;
83 RuntimeLibcalls Libcalls;
84
85 void emitGetRuntimeLibcallEnum(raw_ostream &OS) const;
86
87 void emitNameMatchHashTable(raw_ostream &OS,
88 StringToOffsetTable &OffsetTable) const;
89
90 void emitGetInitRuntimeLibcallNames(raw_ostream &OS) const;
91 void emitGetInitRuntimeLibcallSignatures(raw_ostream &OS) const;
92 void emitMaskedVectorLibcallMapping(raw_ostream &OS) const;
93
94 // Emit the sorted per-predicate `setAvailable` tables/loops. The
95 // always-available bucket emits at \p BaseIndent; each predicated bucket is
96 // wrapped in `if (pred)`. All calls are emitted in member context (no
97 // receiver prefix).
98 void
99 emitPredicateGroups(raw_ostream &OS, const Record *R,
100 DenseMap<PredicateWithCC, LibcallsWithCC> &Pred2Funcs,
101 SetVector<PredicateWithCC> &PredicateSorter,
102 unsigned BaseIndent) const;
103
104 // A LibraryRef opt-out: the impls a consumer drops from a shared library,
105 // plus the consumer's triple predicate.
106 struct LibraryExclusion {
107 const Record *TriplePred;
108 std::vector<const RuntimeLibcallImpl *> Impls;
109 };
110
111 // A SystemRuntimeLibrary's default calling convention and the libraries it
112 // dispatches, as (LibraryName, FuncSuffix) pairs.
113 struct SystemLibraryDispatch {
114 const Record *System;
115 StringRef DefaultCC;
116 SetVector<std::pair<StringRef, StringRef>> Libs;
117 };
118
119 // A single LibcallLibrary variant, expanded into its per-predicate impl
120 // groups. Unconditional impls are tracked separately for cross-variant
121 // deduplication. A variant is Deferred when it re-adds an impl its own
122 // consumer excludes; deferred variants are emitted after the LibraryRef
123 // exclusions so the re-add wins over the opt-out.
124 struct ExpandedLibrary {
125 const Record *Lib;
126 DenseMap<PredicateWithCC, LibcallsWithCC> Pred2Funcs;
127 SetVector<PredicateWithCC> PredicateSorter;
128 SetVector<const RuntimeLibcallImpl *> Unconditional;
129 bool Deferred = false;
130 };
131
132 // Emit one variant's guarded `setAvailable` block into the enclosing
133 // `setAvailableLibFuncs_<name>` function.
134 void emitLibraryVariant(raw_ostream &OS, ExpandedLibrary &EL) const;
135
136 // Emit a `setAvailableLibFuncs_<name>` member function for all LibcallLibrary
137 // defs sharing \p Name, each gated by its own availability predicate. \p
138 // Exclusions are emitted as guarded setUnavailable calls at the end.
139 void emitLibraryFunction(raw_ostream &OS, StringRef Name,
140 ArrayRef<const Record *> Libs,
141 ArrayRef<LibraryExclusion> Exclusions) const;
142
143 // Group all LibcallLibrary defs by their shared LibraryName, preserving
144 // definition order. Both the member-declaration fragment and the definitions
145 // iterate this to stay in lockstep.
146 MapVector<StringRef, std::vector<const Record *>>
147 collectLibrariesByName() const;
148
149 // Get the sorted, unique list of every defined LibraryName.
150 std::vector<StringRef> collectLibraryNames() const;
151
152 void emitRuntimeLibcallsInfoMemberDecls(raw_ostream &OS) const;
153
154 std::vector<SystemLibraryDispatch> collectSystemLibraryDispatch() const;
155
156 void emitSystemRuntimeLibrarySetCalls(
157 raw_ostream &OS, ArrayRef<SystemLibraryDispatch> Systems) const;
158
159 // Emit setAvailableLibraryLibcalls, which enables a library by name.
160 void emitLibraryByNameSetCalls(raw_ostream &OS,
161 ArrayRef<SystemLibraryDispatch> Systems) const;
162
163 DenseSet<StringRef> collectLibcallNames() const;
164
165 void checkFPLibcallFamilies(ArrayRef<FPLibcallFamily> Families,
166 const DenseSet<StringRef> &LibcallNames) const;
167
168 void emitFPLibcallSelectorDecls(raw_ostream &OS,
169 ArrayRef<FPLibcallFamily> Families) const;
170
171 void emitFPLibcallSelectors(raw_ostream &OS,
172 ArrayRef<FPLibcallFamily> Families,
173 const DenseSet<StringRef> &LibcallNames) const;
174
175 void
176 emitGetLibcallForIntrinsic(raw_ostream &OS,
177 ArrayRef<FPLibcallFamily> Families,
178 const DenseSet<StringRef> &LibcallNames) const;
179
180public:
181 RuntimeLibcallEmitter(const RecordKeeper &R) : Records(R), Libcalls(R) {}
182
183 void run(raw_ostream &OS);
184};
185
186} // End anonymous namespace.
187
188// Emit the linker name \p Name as a C++ identifier suffix, replacing characters
189// invalid in an identifier (e.g. the '-' in "compiler-rt") with '_'.
190static void emitLibFuncSuffix(raw_ostream &OS, StringRef Name) {
191 for (char C : Name)
192 OS << (isAlnum(C) || C == '_' ? C : '_');
193}
194
195static void emitRuntimeLibraryEnumerator(raw_ostream &OS, StringRef Name) {
196 OS << "RTLIB::RuntimeLibrary::";
197 emitLibFuncSuffix(OS, Name);
198}
199
200void RuntimeLibcallEmitter::emitGetRuntimeLibcallEnum(raw_ostream &OS) const {
201 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_ENUM");
202
203 OS << "namespace llvm {\n"
204 "namespace RTLIB {\n"
205 "enum Libcall : unsigned short {\n";
206
207 for (const RuntimeLibcall &LibCall : Libcalls.getRuntimeLibcallDefList()) {
208 StringRef Name = LibCall.getName();
209 OS << " " << Name << " = " << LibCall.getEnumVal() << ",\n";
210 }
211
212 OS << " UNKNOWN_LIBCALL = " << Libcalls.getRuntimeLibcallDefList().size()
213 << "\n};\n\n"
214 "enum LibcallImpl : unsigned short {\n"
215 " Unsupported = 0,\n";
216
217 for (const RuntimeLibcallImpl &LibCall :
218 Libcalls.getRuntimeLibcallImplDefList()) {
219 OS << " impl_" << LibCall.getName() << " = " << LibCall.getEnumVal()
220 << ", // " << LibCall.getLibcallFuncName() << '\n';
221 }
222
223 OS << "};\n"
224 << "constexpr size_t NumLibcallImpls = "
225 << Libcalls.getRuntimeLibcallImplDefList().size() + 1
226 << ";\n\n"
227 "enum FuncArgTypeID : char {\n"
228 " NoFuncArgType = 0,\n";
229
230 for (const auto *R : Libcalls.getFuncArgTypeList()) {
231 if (R->getName() == "NoneType")
232 continue;
233 OS << " " << R->getName() << ",\n";
234 }
235
236 OS << "};\n\n"
237 "enum class RuntimeLibrary : unsigned char {\n"
238 " UnknownLibrary = 0,\n";
239 for (StringRef Name : collectLibraryNames()) {
240 OS << " ";
241 emitLibFuncSuffix(OS, Name);
242 OS << ",\n";
243 }
244
245 OS << "};\n"
246 "} // End namespace RTLIB\n"
247 "} // End namespace llvm\n";
248}
249
250// StringMap uses xxh3_64bits, truncated to uint32_t.
251static uint64_t hash(StringRef Str) {
252 return static_cast<uint32_t>(xxh3_64bits(data: Str));
253}
254
255static void emitHashFunction(raw_ostream &OS) {
256 OS << "static inline uint64_t hash(StringRef Str) {\n"
257 " return static_cast<uint32_t>(xxh3_64bits(Str));\n"
258 "}\n\n";
259}
260
261/// Return the table size, maximum number of collisions for the set of hashes
262static std::pair<int, int>
263computePerfectHashParameters(ArrayRef<uint64_t> Hashes) {
264 // Chosen based on experimentation with llvm/benchmarks/RuntimeLibcalls.cpp
265 const int SizeOverhead = 4;
266
267 // Index derived from hash -> number of collisions.
268 DenseMap<uint64_t, int> Table;
269
270 unsigned NumHashes = Hashes.size();
271
272 for (int MaxCollisions = 1;; ++MaxCollisions) {
273 for (unsigned N = NextPowerOf2(A: NumHashes - 1); N < SizeOverhead * NumHashes;
274 N <<= 1) {
275 Table.clear();
276
277 bool NeedResize = false;
278 for (uint64_t H : Hashes) {
279 uint64_t Idx = H % static_cast<uint64_t>(N);
280 if (++Table[Idx] > MaxCollisions) {
281 // Need to resize the final table if we increased the collision count.
282 NeedResize = true;
283 break;
284 }
285 }
286
287 if (!NeedResize)
288 return {N, MaxCollisions};
289 }
290 }
291}
292
293static std::vector<unsigned>
294constructPerfectHashTable(ArrayRef<RuntimeLibcallImpl> Keywords,
295 ArrayRef<uint64_t> Hashes,
296 ArrayRef<unsigned> TableValues, int Size,
297 int Collisions, StringToOffsetTable &OffsetTable) {
298 std::vector<unsigned> Lookup(Size * Collisions);
299
300 for (auto [HashValue, TableValue] : zip(t&: Hashes, u&: TableValues)) {
301 uint64_t Idx = (HashValue % static_cast<uint64_t>(Size)) *
302 static_cast<uint64_t>(Collisions);
303
304 bool Found = false;
305 for (int J = 0; J < Collisions; ++J) {
306 unsigned &Entry = Lookup[Idx + J];
307 if (Entry == 0) {
308 Entry = TableValue;
309 Found = true;
310 break;
311 }
312 }
313
314 if (!Found)
315 reportFatalInternalError(reason: "failure to hash");
316 }
317
318 return Lookup;
319}
320
321/// Generate hash table based lookup by name.
322void RuntimeLibcallEmitter::emitNameMatchHashTable(
323 raw_ostream &OS, StringToOffsetTable &OffsetTable) const {
324 ArrayRef<RuntimeLibcallImpl> RuntimeLibcallImplDefList =
325 Libcalls.getRuntimeLibcallImplDefList();
326 std::vector<uint64_t> Hashes(RuntimeLibcallImplDefList.size());
327 std::vector<unsigned> TableValues(RuntimeLibcallImplDefList.size());
328 DenseSet<StringRef> SeenFuncNames;
329
330 size_t MaxFuncNameSize = 0;
331 size_t Index = 0;
332
333 for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
334 StringRef ImplName = LibCallImpl.getLibcallFuncName();
335 if (SeenFuncNames.insert(V: ImplName).second) {
336 MaxFuncNameSize = std::max(a: MaxFuncNameSize, b: ImplName.size());
337 TableValues[Index] = LibCallImpl.getEnumVal();
338 Hashes[Index++] = hash(Str: ImplName);
339 }
340 }
341
342 // Trim excess elements from non-unique entries.
343 Hashes.resize(new_size: SeenFuncNames.size());
344 TableValues.resize(new_size: SeenFuncNames.size());
345
346 LLVM_DEBUG({
347 for (const RuntimeLibcallImpl &LibCallImpl : RuntimeLibcallImplDefList) {
348 StringRef ImplName = LibCallImpl.getLibcallFuncName();
349 if (ImplName.size() == MaxFuncNameSize) {
350 dbgs() << "Maximum runtime libcall name size: " << ImplName << '('
351 << MaxFuncNameSize << ")\n";
352 }
353 }
354 });
355
356 // Early exiting on the symbol name provides a significant speedup in the miss
357 // case on the set of symbols in a clang binary. Emit this as an inlinable
358 // precondition in the header.
359 //
360 // The empty check is also used to get sensible behavior on anonymous
361 // functions.
362 //
363 // TODO: It may make more sense to split the search by string size more. There
364 // are a few outliers, most call names are small.
365 {
366 IfDefEmitter IfDef(OS, "GET_LOOKUP_LIBCALL_IMPL_NAME_BODY");
367
368 OS << " size_t Size = Name.size();\n"
369 " if (Size == 0 || Size > "
370 << MaxFuncNameSize
371 << ")\n"
372 " return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);\n"
373 " return lookupLibcallImplNameImpl(Name);\n";
374 }
375
376 auto [Size, Collisions] = computePerfectHashParameters(Hashes);
377 std::vector<unsigned> Lookup =
378 constructPerfectHashTable(Keywords: RuntimeLibcallImplDefList, Hashes, TableValues,
379 Size, Collisions, OffsetTable);
380
381 LLVM_DEBUG(dbgs() << "Runtime libcall perfect hashing parameters: Size = "
382 << Size << ", maximum collisions = " << Collisions << '\n');
383
384 IfDefEmitter IfDef(OS, "DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME");
385 emitHashFunction(OS);
386
387 OS << "iota_range<RTLIB::LibcallImpl> RTLIB::RuntimeLibcallsInfo::"
388 "lookupLibcallImplNameImpl(StringRef Name) {\n";
389
390 // Emit RTLIB::LibcallImpl values
391 OS << " static constexpr uint16_t HashTableNameToEnum[" << Lookup.size()
392 << "] = {\n";
393
394 for (unsigned TableVal : Lookup)
395 OS << " " << TableVal << ",\n";
396
397 OS << " };\n\n";
398
399 OS << " unsigned Idx = (hash(Name) % " << Size << ") * " << Collisions
400 << ";\n\n"
401 " for (int I = 0; I != "
402 << Collisions << R"(; ++I) {
403 const uint16_t Entry = HashTableNameToEnum[Idx + I];
404 const uint16_t StrOffset = RuntimeLibcallNameOffsetTable[Entry];
405 const uint8_t StrSize = RuntimeLibcallNameSizeTable[Entry];
406 StringRef Str(
407 &RTLIB::RuntimeLibcallsInfo::RuntimeLibcallImplNameTableStorage[StrOffset],
408 StrSize);
409 if (Str == Name)
410 return libcallImplNameHit(Entry, StrOffset);
411 }
412
413 return enum_seq(RTLIB::Unsupported, RTLIB::Unsupported);
414}
415)";
416}
417
418void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallNames(
419 raw_ostream &OS) const {
420 // Emit the implementation names
421 StringToOffsetTable Table(/*AppendZero=*/true,
422 "RTLIB::RuntimeLibcallsInfo::");
423
424 {
425 IfDefEmitter IfDef(OS, "GET_INIT_RUNTIME_LIBCALL_NAMES");
426
427 for (const RuntimeLibcallImpl &LibCallImpl :
428 Libcalls.getRuntimeLibcallImplDefList())
429 Table.GetOrAddStringOffset(Str: LibCallImpl.getLibcallFuncName());
430
431 Table.EmitStringTableDef(OS, Name: "RuntimeLibcallImplNameTable");
432 OS << R"(
433const uint16_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameOffsetTable[] = {
434)";
435
436 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: ""),
437 Vals: ""); // Unsupported entry
438 for (const RuntimeLibcallImpl &LibCallImpl :
439 Libcalls.getRuntimeLibcallImplDefList()) {
440 StringRef ImplName = LibCallImpl.getLibcallFuncName();
441 OS << formatv(Fmt: " {}, // {}\n", Vals: Table.GetStringOffset(Str: ImplName), Vals&: ImplName);
442 }
443 OS << "};\n";
444
445 OS << R"(
446const uint8_t RTLIB::RuntimeLibcallsInfo::RuntimeLibcallNameSizeTable[] = {
447)";
448
449 OS << " 0,\n";
450 for (const RuntimeLibcallImpl &LibCallImpl :
451 Libcalls.getRuntimeLibcallImplDefList())
452 OS << " " << LibCallImpl.getLibcallFuncName().size() << ",\n";
453 OS << "};\n\n";
454
455 // Emit the reverse mapping from implementation libraries to RTLIB::Libcall
456 OS << "const RTLIB::Libcall llvm::RTLIB::RuntimeLibcallsInfo::"
457 "ImplToLibcall[RTLIB::NumLibcallImpls] = {\n"
458 " RTLIB::UNKNOWN_LIBCALL, // RTLIB::Unsupported\n";
459
460 for (const RuntimeLibcallImpl &LibCallImpl :
461 Libcalls.getRuntimeLibcallImplDefList()) {
462 const RuntimeLibcall *Provides = LibCallImpl.getProvides();
463 OS << " ";
464 Provides->emitEnumEntry(OS);
465 OS << ", // ";
466 LibCallImpl.emitEnumEntry(OS);
467 OS << '\n';
468 }
469
470 OS << "};\n\n";
471 }
472
473 emitNameMatchHashTable(OS, OffsetTable&: Table);
474}
475
476using Signature = std::vector<StringRef>;
477
478static Signature getSignature(const Record *R) {
479 const auto *Tys = R->getValueAsListInit(FieldName: "ArgumentTypes");
480 Signature Sig;
481 Sig.reserve(n: Tys->size() + 1);
482 const Record *RetType = R->getValueAsOptionalDef(FieldName: "ReturnType");
483 if (RetType && RetType->getName() != "NoneType")
484 Sig.push_back(x: RetType->getName());
485 for (unsigned I = 0, E = Tys->size(); I < E; ++I)
486 Sig.push_back(x: Tys->getElementAsRecord(Idx: I)->getName());
487 return Sig;
488}
489
490void RuntimeLibcallEmitter::emitGetInitRuntimeLibcallSignatures(
491 raw_ostream &OS) const {
492 SequenceToOffsetTable<Signature> SignatureTable("NoFuncArgType");
493
494 for (const RuntimeLibcall &LC : Libcalls.getRuntimeLibcallDefList())
495 SignatureTable.add(Seq: getSignature(R: LC.getDef()));
496 SignatureTable.layout();
497
498 IfDefEmitter IfDef(OS, "GET_INIT_RUNTIME_LIBCALL_SIGNATURES");
499
500 OS << R"(
501const FuncArgTypeID RTLIB::RuntimeLibcallsInfo::SignatureTable[] = {
502)";
503 SignatureTable.emit(OS, Print: [](raw_ostream &OS, StringRef E) { OS << E; });
504 OS << "};\n";
505
506 OS << R"(
507const uint16_t RTLIB::RuntimeLibcallsInfo::SignatureOffset[] = {
508)";
509 for (const RuntimeLibcall &LC : Libcalls.getRuntimeLibcallDefList()) {
510 const Record *LibcallDef = LC.getDef();
511 OS << formatv(Fmt: " {}, // {}\n", Vals: SignatureTable.get(Seq: getSignature(R: LibcallDef)),
512 Vals: LibcallDef->getName());
513 }
514 OS << "};\n";
515}
516
517static void emitMaskedVectorLibcallSwitch(
518 raw_ostream &OS, StringRef FuncName, bool ToUnmasked,
519 ArrayRef<std::pair<const RuntimeLibcall *, const RuntimeLibcall *>>
520 MaskedToUnmasked) {
521 OS << "RTLIB::Libcall RTLIB::RuntimeLibcallsInfo::" << FuncName
522 << "(RTLIB::Libcall LC) {\n"
523 " switch (LC) {\n";
524 for (auto [Masked, Unmasked] : MaskedToUnmasked) {
525 OS << " case ";
526 if (ToUnmasked) {
527 Masked->emitEnumEntry(OS);
528 OS << ":\n return ";
529 Unmasked->emitEnumEntry(OS);
530 } else {
531 Unmasked->emitEnumEntry(OS);
532 OS << ":\n return ";
533 Masked->emitEnumEntry(OS);
534 }
535
536 OS << ";\n";
537 }
538
539 OS << " default:\n"
540 " return RTLIB::UNKNOWN_LIBCALL;\n"
541 " }\n"
542 "}\n";
543}
544
545void RuntimeLibcallEmitter::emitMaskedVectorLibcallMapping(
546 raw_ostream &OS) const {
547 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_MASKED_VECTOR_LIBCALL_MAPPING");
548
549 SmallVector<std::pair<const RuntimeLibcall *, const RuntimeLibcall *>, 0>
550 MaskedToUnmasked;
551 for (const RuntimeLibcall &LC : Libcalls.getRuntimeLibcallDefList()) {
552 const Record *Def = LC.getDef();
553 if (Def->isSubClassOf(Name: "MaskedRuntimeLibcall")) {
554 MaskedToUnmasked.emplace_back(
555 Args: &LC, Args: Libcalls.getRuntimeLibcall(Def: Def->getValueAsDef(FieldName: "Unmasked")));
556 }
557 }
558
559 emitMaskedVectorLibcallSwitch(OS, FuncName: "getUnmaskedVectorLibcall",
560 /*ToUnmasked=*/true, MaskedToUnmasked);
561 OS << '\n';
562 emitMaskedVectorLibcallSwitch(OS, FuncName: "getMaskedVectorLibcall",
563 /*ToUnmasked=*/false, MaskedToUnmasked);
564}
565
566void RuntimeLibcallEmitter::emitPredicateGroups(
567 raw_ostream &OS, const Record *R,
568 DenseMap<PredicateWithCC, LibcallsWithCC> &Pred2Funcs,
569 SetVector<PredicateWithCC> &PredicateSorter, unsigned BaseIndent) const {
570 SmallVector<PredicateWithCC, 0> SortedPredicates =
571 PredicateSorter.takeVector();
572
573 llvm::sort(C&: SortedPredicates, Comp: [](PredicateWithCC A, PredicateWithCC B) {
574 StringRef AName = A.Availability ? A.Availability->getName() : "";
575 StringRef BName = B.Availability ? B.Availability->getName() : "";
576 if (AName != BName)
577 return AName < BName;
578 // Break name ties on the calling convention for a deterministic order.
579 StringRef ACC = A.CallingConv ? A.CallingConv->getName() : "";
580 StringRef BCC = B.CallingConv ? B.CallingConv->getName() : "";
581 return ACC < BCC;
582 });
583
584 for (PredicateWithCC Entry : SortedPredicates) {
585 AvailabilityPredicate SubsetPredicate(Entry.Availability);
586 unsigned IndentDepth = BaseIndent;
587
588 auto It = Pred2Funcs.find(Val: Entry);
589 if (It == Pred2Funcs.end())
590 continue;
591
592 // Shared-core deduplication can empty a bucket.
593 if (It->second.LibcallImpls.empty())
594 continue;
595
596 if (!SubsetPredicate.isAlwaysAvailable()) {
597 IndentDepth = BaseIndent + 2;
598
599 OS << indent(IndentDepth);
600 SubsetPredicate.emitIf(OS);
601 }
602
603 LibcallsWithCC &FuncsWithCC = It->second;
604
605 std::vector<const RuntimeLibcallImpl *> &Funcs = FuncsWithCC.LibcallImpls;
606
607 // Records which impls are available, not which is selected, so a libcall
608 // may have more than one. Order is irrelevant (each entry is a setAvailable
609 // call); sort by the provided libcall, breaking ties on the impl enum for a
610 // deterministic total order.
611 llvm::sort(C&: Funcs, Comp: [](const RuntimeLibcallImpl *A,
612 const RuntimeLibcallImpl *B) {
613 return std::make_pair(x: A->getProvides()->getEnumVal(), y: A->getEnumVal()) <
614 std::make_pair(x: B->getProvides()->getEnumVal(), y: B->getEnumVal());
615 });
616
617 OS << indent(IndentDepth + 2)
618 << "static const RTLIB::LibcallImpl LibraryCalls";
619 SubsetPredicate.emitTableVariableNameSuffix(OS);
620 if (FuncsWithCC.CallingConv)
621 OS << '_' << FuncsWithCC.CallingConv->getName();
622
623 OS << "[] = {\n";
624 for (const RuntimeLibcallImpl *LibCallImpl : Funcs) {
625 OS << indent(IndentDepth + 6);
626 LibCallImpl->emitEnumEntry(OS);
627 OS << ", // " << LibCallImpl->getLibcallFuncName() << '\n';
628 }
629
630 OS << indent(IndentDepth + 2) << "};\n\n"
631 << indent(IndentDepth + 2)
632 << "for (const RTLIB::LibcallImpl Impl : LibraryCalls";
633 SubsetPredicate.emitTableVariableNameSuffix(OS);
634 if (FuncsWithCC.CallingConv)
635 OS << '_' << FuncsWithCC.CallingConv->getName();
636
637 OS << ") {\n" << indent(IndentDepth + 4) << "setAvailable(Impl);\n";
638
639 if (FuncsWithCC.CallingConv) {
640 StringRef CCEnum =
641 FuncsWithCC.CallingConv->getValueAsString(FieldName: "CallingConv");
642 OS << indent(IndentDepth + 4) << "setLibcallImplCallingConv(Impl, "
643 << CCEnum << ");\n";
644 }
645
646 OS << indent(IndentDepth + 2) << "}\n";
647 OS << '\n';
648
649 if (!SubsetPredicate.isAlwaysAvailable()) {
650 OS << indent(IndentDepth);
651 SubsetPredicate.emitEndIf(OS);
652 OS << '\n';
653 }
654 }
655}
656
657void RuntimeLibcallEmitter::emitLibraryVariant(raw_ostream &OS,
658 ExpandedLibrary &EL) const {
659 AvailabilityPredicate LibPred(EL.Lib->getValueAsDef(FieldName: "Pred"));
660
661 if (!LibPred.isAlwaysAvailable()) {
662 OS << indent(2);
663 LibPred.emitIf(OS);
664 } else {
665 // Own block scope so per-variant `LibraryCalls` tables do not collide.
666 OS << indent(2) << "{\n";
667 }
668
669 emitPredicateGroups(OS, R: EL.Lib, Pred2Funcs&: EL.Pred2Funcs, PredicateSorter&: EL.PredicateSorter,
670 /*BaseIndent=*/2);
671
672 if (!LibPred.isAlwaysAvailable()) {
673 OS << indent(2);
674 LibPred.emitEndIf(OS);
675 } else {
676 OS << indent(2) << "}\n";
677 }
678}
679
680void RuntimeLibcallEmitter::emitLibraryFunction(
681 raw_ostream &OS, StringRef Name, ArrayRef<const Record *> Libs,
682 ArrayRef<LibraryExclusion> Exclusions) const {
683 OS << "void llvm::RTLIB::RuntimeLibcallsInfo::setAvailableLibFuncs_";
684 emitLibFuncSuffix(OS, Name);
685 OS << "(const llvm::Triple &TT, "
686 "ExceptionHandling ExceptionModel, FloatABI::ABIType FloatABI, "
687 "StringRef ABIName, "
688 "LongDoubleFormat LongDoubleFormat, CallingConv::ID DefaultCC) {\n";
689
690 SmallVector<ExpandedLibrary, 2> Expanded;
691 for (const Record *Lib : Libs) {
692 ExpandedLibrary EL;
693 EL.Lib = Lib;
694
695 // Expand this library's members with a library-local Func2Preds.
696 SetTheory Sets;
697 DenseMap<const RuntimeLibcallImpl *,
698 std::pair<std::vector<const Record *>, const Record *>>
699 Func2Preds;
700 Sets.addExpander(ClassName: "LibcallImpls", std::make_unique<LibcallPredicateExpander>(
701 args: Libcalls, args&: Func2Preds));
702
703 SetTheory::RecSet Elements;
704 Sets.evaluate(Expr: Lib->getValueInit(FieldName: "Impls"), Elts&: Elements, Loc: Lib->getLoc());
705
706 EL.PredicateSorter.insert(
707 X: PredicateWithCC()); // No predicate or CC override first.
708
709 for (const Record *Elt : Elements) {
710 const RuntimeLibcallImpl *LibCallImpl =
711 Libcalls.getRuntimeLibcallImpl(Def: Elt);
712 if (!LibCallImpl) {
713 PrintError(Rec: Lib, Msg: "entry for LibcallLibrary is not a RuntimeLibcallImpl");
714 PrintNote(NoteLoc: Elt->getLoc(), Msg: "invalid entry `" + Elt->getName() + "`");
715 continue;
716 }
717
718 auto It = Func2Preds.find(Val: LibCallImpl);
719 if (It == Func2Preds.end()) {
720 EL.Pred2Funcs[PredicateWithCC()].LibcallImpls.push_back(x: LibCallImpl);
721 EL.Unconditional.insert(X: LibCallImpl);
722 continue;
723 }
724
725 for (const Record *Pred : It->second.first) {
726 const Record *CC = It->second.second;
727 PredicateWithCC Key(Pred, CC);
728 auto &Entry = EL.Pred2Funcs[Key];
729 Entry.LibcallImpls.push_back(x: LibCallImpl);
730 Entry.CallingConv = CC;
731 EL.PredicateSorter.insert(X: Key);
732 }
733 }
734
735 Expanded.push_back(Elt: std::move(EL));
736 }
737
738 // Impls unconditional in every variant are emitted once and stripped from
739 // each variant, so the shared core is not repeated.
740 SetVector<const RuntimeLibcallImpl *> SharedCore;
741 if (Expanded.size() > 1) {
742 for (const RuntimeLibcallImpl *Impl : Expanded.front().Unconditional) {
743 if (all_of(Range: drop_begin(RangeOrContainer&: Expanded), P: [&](const ExpandedLibrary &EL) {
744 return EL.Unconditional.contains(key: Impl);
745 }))
746 SharedCore.insert(X: Impl);
747 }
748 }
749
750 if (!SharedCore.empty()) {
751 // Emit the shared core once, then strip it from every variant.
752 DenseMap<PredicateWithCC, LibcallsWithCC> CorePred2Funcs;
753 SetVector<PredicateWithCC> CoreSorter;
754 CoreSorter.insert(X: PredicateWithCC());
755 for (const RuntimeLibcallImpl *Impl : SharedCore)
756 CorePred2Funcs[PredicateWithCC()].LibcallImpls.push_back(x: Impl);
757 emitPredicateGroups(OS, R: Libs.front(), Pred2Funcs&: CorePred2Funcs, PredicateSorter&: CoreSorter,
758 /*BaseIndent=*/0);
759
760 for (ExpandedLibrary &EL : Expanded) {
761 auto &Funcs = EL.Pred2Funcs[PredicateWithCC()].LibcallImpls;
762 llvm::erase_if(C&: Funcs, P: [&](const RuntimeLibcallImpl *Impl) {
763 return SharedCore.contains(key: Impl);
764 });
765 }
766 }
767
768 // Mark a variant deferred when it re-adds an impl its own consumer excludes
769 // (same triple, via LibraryRef). Such a variant must be emitted after the
770 // exclusion so the re-add wins while the exclusion still suppresses every
771 // other variant's contribution.
772 for (ExpandedLibrary &EL : Expanded) {
773 const Record *ELPred = EL.Lib->getValueAsDef(FieldName: "Pred");
774 SetVector<const RuntimeLibcallImpl *> Impls;
775 for (const auto &[Key, Funcs] : EL.Pred2Funcs)
776 Impls.insert(Start: Funcs.LibcallImpls.begin(), End: Funcs.LibcallImpls.end());
777
778 for (const LibraryExclusion &Excl : Exclusions) {
779 if (Excl.TriplePred != ELPred)
780 continue;
781 if (any_of(Range: Excl.Impls, P: [&](const RuntimeLibcallImpl *Impl) {
782 return Impls.contains(key: Impl);
783 })) {
784 EL.Deferred = true;
785 break;
786 }
787 }
788 }
789
790 // Emit each non-deferred variant under its own Pred.
791 for (ExpandedLibrary &EL : Expanded) {
792 if (!EL.Deferred)
793 emitLibraryVariant(OS, EL);
794 }
795
796 // Emit each consumer's LibraryRef opt-outs.
797 for (const LibraryExclusion &Excl : Exclusions) {
798 OS << '\n' << indent(2);
799 AvailabilityPredicate ExcludePred(Excl.TriplePred);
800 ExcludePred.emitIf(OS);
801 for (const RuntimeLibcallImpl *Impl : Excl.Impls) {
802 OS << indent(4) << "setUnavailable(";
803 Impl->emitEnumEntry(OS);
804 OS << "); // " << Impl->getLibcallFuncName() << '\n';
805 }
806
807 OS << indent(2);
808 ExcludePred.emitEndIf(OS);
809 }
810
811 // Deferred variants: emitted after exclusions so a target's own re-adds
812 // override its own LibraryRef opt-outs (the exclusion still applied above
813 // to every other variant's contributions).
814 for (ExpandedLibrary &EL : Expanded) {
815 if (EL.Deferred)
816 emitLibraryVariant(OS, EL);
817 }
818
819 OS << "}\n\n";
820}
821
822// The setAvailableLibFuncs_ suffix (and merge key) for a library: its shared
823// LibraryName normally, or its own def name if isolated (so it does not merge).
824static StringRef libFuncKey(const Record *Lib) {
825 return Lib->getValueAsBit(FieldName: "Isolated") ? Lib->getName()
826 : Lib->getValueAsString(FieldName: "LibraryName");
827}
828
829MapVector<StringRef, std::vector<const Record *>>
830RuntimeLibcallEmitter::collectLibrariesByName() const {
831 MapVector<StringRef, std::vector<const Record *>> LibsByName;
832 for (const Record *Lib : Records.getAllDerivedDefinitions(ClassName: "LibcallLibrary"))
833 LibsByName[libFuncKey(Lib)].push_back(x: Lib);
834 return LibsByName;
835}
836
837std::vector<StringRef> RuntimeLibcallEmitter::collectLibraryNames() const {
838 std::vector<StringRef> Names;
839 for (const Record *Lib : Records.getAllDerivedDefinitions(ClassName: "LibcallLibrary"))
840 Names.push_back(x: Lib->getValueAsString(FieldName: "LibraryName"));
841 llvm::sort(C&: Names);
842 Names.erase(first: llvm::unique(R&: Names), last: Names.end());
843 return Names;
844}
845
846void RuntimeLibcallEmitter::emitRuntimeLibcallsInfoMemberDecls(
847 raw_ostream &OS) const {
848 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALLS_INFO_MEMBER_DECLS");
849 for (const auto &[Name, Libs] : collectLibrariesByName()) {
850 OS << "void setAvailableLibFuncs_";
851 emitLibFuncSuffix(OS, Name);
852 OS << "(const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
853 "FloatABI::ABIType FloatABI, StringRef ABIName, "
854 "LongDoubleFormat LongDoubleFormat, CallingConv::ID DefaultCC);\n";
855 }
856
857 OS << "bool setAvailableLibraryLibcalls(RTLIB::RuntimeLibrary Library, "
858 "const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
859 "FloatABI::ABIType FloatABI, StringRef ABIName, "
860 "LongDoubleFormat LongDoubleFormat);\n";
861}
862
863std::vector<RuntimeLibcallEmitter::SystemLibraryDispatch>
864RuntimeLibcallEmitter::collectSystemLibraryDispatch() const {
865 std::vector<SystemLibraryDispatch> Systems;
866 for (const Record *R :
867 Records.getAllDerivedDefinitions(ClassName: "SystemRuntimeLibrary")) {
868 SystemLibraryDispatch &S = Systems.emplace_back();
869 S.System = R;
870 if (const Record *DefaultCCClass =
871 R->getValueAsDef(FieldName: "DefaultLibcallCallingConv"))
872 S.DefaultCC = DefaultCCClass->getValueAsString(FieldName: "CallingConv").trim();
873
874 // A library referenced more than once is dispatched once.
875 const DagInit *MemberDag =
876 R->getValueAsDef(FieldName: "MemberList")->getValueAsDag(FieldName: "MemberList");
877 for (const Init *Arg : MemberDag->getArgs()) {
878 const auto *DI = dyn_cast<DefInit>(Val: Arg);
879 const Record *Lib = DI ? DI->getDef() : nullptr;
880 if (Lib && Lib->isSubClassOf(Name: "LibraryRef"))
881 Lib = Lib->getValueAsDef(FieldName: "Library");
882
883 if (!Lib || !Lib->isSubClassOf(Name: "LibcallLibrary")) {
884 PrintError(Rec: R, Msg: "SystemRuntimeLibrary member `" + Arg->getAsString() +
885 "` is not a LibcallLibrary or LibraryRef");
886 continue;
887 }
888
889 S.Libs.insert(X: {Lib->getValueAsString(FieldName: "LibraryName"), libFuncKey(Lib)});
890 }
891 }
892
893 return Systems;
894}
895
896void RuntimeLibcallEmitter::emitSystemRuntimeLibrarySetCalls(
897 raw_ostream &OS, ArrayRef<SystemLibraryDispatch> Systems) const {
898 // Collect each shared library's LibraryRef opt-outs, keyed by library name,
899 // so its library function can emit them.
900 MapVector<StringRef, std::vector<LibraryExclusion>> ExclusionsByLibName;
901 for (const SystemLibraryDispatch &S : Systems) {
902 const Record *R = S.System;
903 const DagInit *MemberDag =
904 R->getValueAsDef(FieldName: "MemberList")->getValueAsDag(FieldName: "MemberList");
905 for (const Init *Arg : MemberDag->getArgs()) {
906 const auto *DI = dyn_cast<DefInit>(Val: Arg);
907 if (!DI || !DI->getDef()->isSubClassOf(Name: "LibraryRef"))
908 continue;
909 const Record *Def = DI->getDef();
910 LibraryExclusion Excl{.TriplePred: R->getValueAsDef(FieldName: "TriplePred"), .Impls: {}};
911 for (const Record *ExcludeRec : Def->getValueAsListOfDefs(FieldName: "Exclude")) {
912 if (const RuntimeLibcallImpl *Impl =
913 Libcalls.getRuntimeLibcallImpl(Def: ExcludeRec))
914 Excl.Impls.push_back(x: Impl);
915 }
916
917 if (!Excl.Impls.empty()) {
918 StringRef LibName =
919 Def->getValueAsDef(FieldName: "Library")->getValueAsString(FieldName: "LibraryName");
920 ExclusionsByLibName[LibName].push_back(x: std::move(Excl));
921 }
922 }
923 }
924
925 for (const auto &[Name, Libs] : collectLibrariesByName())
926 emitLibraryFunction(OS, Name, Libs, Exclusions: ExclusionsByLibName.lookup(Key: Name));
927
928 OS << "void llvm::RTLIB::RuntimeLibcallsInfo::setTargetRuntimeLibcallSets("
929 "const llvm::Triple &TT, ExceptionHandling ExceptionModel, "
930 "FloatABI::ABIType FloatABI, "
931 "StringRef ABIName, LongDoubleFormat LongDoubleFormat) {\n";
932
933 for (const SystemLibraryDispatch &S : Systems) {
934 OS << '\n';
935
936 AvailabilityPredicate TopLevelPredicate(
937 S.System->getValueAsDef(FieldName: "TriplePred"));
938
939 OS << indent(2);
940 TopLevelPredicate.emitIf(OS);
941
942 // The default is a property of the target, and is passed to the libraries
943 // as an argument.
944 StringRef DefaultCCArg = "CallingConv::C";
945 if (!S.DefaultCC.empty()) {
946 OS << " const CallingConv::ID DefaultCC = " << S.DefaultCC << ";\n"
947 << " for (CallingConv::ID &Entry : LibcallImplCallingConvs) {\n"
948 " Entry = DefaultCC;\n"
949 " }\n\n";
950 DefaultCCArg = "DefaultCC";
951 }
952
953 // Dispatch each library to setAvailableLibFuncs_<FuncSuffix> under an
954 // isLibraryAvailable(Name) guard. LibraryRef exclusions are applied inside
955 // the library function.
956 for (auto [Name, FuncSuffix] : S.Libs) {
957 OS << indent(4) << "if (isLibraryAvailable(";
958 emitRuntimeLibraryEnumerator(OS, Name);
959 OS << "))\n" << indent(6) << "setAvailableLibFuncs_";
960 emitLibFuncSuffix(OS, Name: FuncSuffix);
961 OS << "(TT, ExceptionModel, FloatABI, ABIName, LongDoubleFormat, "
962 << DefaultCCArg << ");\n";
963 }
964 if (!S.Libs.empty())
965 OS << '\n';
966
967 OS << indent(4) << "return;\n" << indent(2);
968 TopLevelPredicate.emitEndIf(OS);
969 }
970
971 // FIXME: This should be a fatal error. A few contexts are improperly relying
972 // on RuntimeLibcalls constructed with fully unknown triples.
973 OS << " LLVM_DEBUG(dbgs() << \"no system runtime library applied to target "
974 "\\'\" << TT.str() << \"\\'\\n\");\n"
975 "}\n\n";
976}
977
978void RuntimeLibcallEmitter::emitLibraryByNameSetCalls(
979 raw_ostream &OS, ArrayRef<SystemLibraryDispatch> Systems) const {
980 // Libraries a SystemRuntimeLibrary uses are enabled by
981 // setTargetRuntimeLibcallSets instead.
982 DenseSet<StringRef> SystemLibNames;
983 for (const SystemLibraryDispatch &S : Systems) {
984 for (auto [LibName, FuncSuffix] : S.Libs)
985 SystemLibNames.insert(V: LibName);
986 }
987
988 OS << "bool llvm::RTLIB::RuntimeLibcallsInfo::setAvailableLibraryLibcalls("
989 "RTLIB::RuntimeLibrary Library, const llvm::Triple &TT, "
990 "ExceptionHandling ExceptionModel, FloatABI::ABIType FloatABI, "
991 "StringRef ABIName, LongDoubleFormat LongDoubleFormat) {\n"
992 " switch (Library) {\n";
993
994 for (const auto &[Name, Libs] : collectLibrariesByName()) {
995 if (Libs.front()->getValueAsBit(FieldName: "Isolated") ||
996 SystemLibNames.contains(V: Name))
997 continue;
998
999 OS << " case ";
1000 emitRuntimeLibraryEnumerator(OS, Name);
1001 OS << ":\n" << indent(4) << "setAvailableLibFuncs_";
1002 emitLibFuncSuffix(OS, Name);
1003 OS << "(TT, ExceptionModel, FloatABI, ABIName, LongDoubleFormat, "
1004 "CallingConv::C);\n"
1005 << indent(4) << "return true;\n";
1006 }
1007
1008 OS << " default:\n"
1009 " return false;\n"
1010 " }\n"
1011 "}\n\n";
1012}
1013
1014// Scalar FP type suffixes in the argument order of RTLIB::getFPLibCall, paired
1015// with the llvm::Type predicate used by the IR-level mapping.
1016static constexpr std::pair<StringRef, StringRef> ScalarFPSuffixes[] = {
1017 {"F32", "isFloatTy()"}, {"F64", "isDoubleTy()"},
1018 {"F80", "isX86_FP80Ty()"}, {"F128", "isFP128Ty()"},
1019 {"PPCF128", "isPPC_FP128Ty()"},
1020};
1021
1022static std::vector<FPLibcallFamily>
1023collectFPLibcallFamilies(const RecordKeeper &Records) {
1024 std::vector<FPLibcallFamily> Families;
1025 for (const Record *R :
1026 Records.getAllDerivedDefinitions(ClassName: "RuntimeLibcallFamily"))
1027 Families.emplace_back(args&: R);
1028 llvm::sort(C&: Families, Comp: [](const FPLibcallFamily &A, const FPLibcallFamily &B) {
1029 return A.Base < B.Base;
1030 });
1031 return Families;
1032}
1033
1034DenseSet<StringRef> RuntimeLibcallEmitter::collectLibcallNames() const {
1035 DenseSet<StringRef> LibcallNames;
1036 for (const RuntimeLibcall &LC : Libcalls.getRuntimeLibcallDefList())
1037 LibcallNames.insert(V: LC.getName());
1038 return LibcallNames;
1039}
1040
1041void RuntimeLibcallEmitter::checkFPLibcallFamilies(
1042 ArrayRef<FPLibcallFamily> Families,
1043 const DenseSet<StringRef> &LibcallNames) const {
1044 std::vector<std::pair<StringRef, StringRef>> IntrinsicToBase;
1045 for (const FPLibcallFamily &Family : Families)
1046 for (StringRef Intrinsic : Family.Intrinsics)
1047 IntrinsicToBase.emplace_back(args&: Intrinsic, args: Family.Base);
1048 llvm::sort(C&: IntrinsicToBase);
1049
1050 for (size_t I = 1, E = IntrinsicToBase.size(); I < E; ++I) {
1051 if (IntrinsicToBase[I].first == IntrinsicToBase[I - 1].first)
1052 PrintFatalError(
1053 Msg: "intrinsic '" + IntrinsicToBase[I].first +
1054 "' is mapped by multiple RuntimeLibcallFamily records ('" +
1055 IntrinsicToBase[I - 1].second + "' and '" +
1056 IntrinsicToBase[I].second + "')");
1057 }
1058
1059 for (const FPLibcallFamily &Family : Families) {
1060 bool AnyScalarLibcall = any_of(
1061 Range: ScalarFPSuffixes, P: [&](const std::pair<StringRef, StringRef> &Entry) {
1062 return LibcallNames.contains(V: (Family.Base + "_" + Entry.first).str());
1063 });
1064 if (!AnyScalarLibcall)
1065 PrintFatalError(Msg: "no runtime libcall found for base name '" + Family.Base +
1066 "'");
1067 }
1068}
1069
1070/// Generate the declarations for the RTLIB::get<base>(EVT) selectors.
1071void RuntimeLibcallEmitter::emitFPLibcallSelectorDecls(
1072 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families) const {
1073 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_FP_SELECTOR_DECLS");
1074 for (const FPLibcallFamily &Family : Families)
1075 OS << "LLVM_ABI Libcall get" << Family.Base << "(EVT VT);\n";
1076}
1077
1078/// Generate the backend RTLIB::get<base>(EVT) selectors from the floating-point
1079/// libcall families.
1080void RuntimeLibcallEmitter::emitFPLibcallSelectors(
1081 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families,
1082 const DenseSet<StringRef> &LibcallNames) const {
1083 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_FP_SELECTORS");
1084
1085 // Only emit a libcall enumerator if it actually exists in the declared set.
1086 auto scalarEnum = [&](StringRef Base, StringRef Suffix) -> std::string {
1087 if (LibcallNames.contains(V: (Base + "_" + Suffix).str()))
1088 return ("RTLIB::" + Base + "_" + Suffix).str();
1089 return "RTLIB::UNKNOWN_LIBCALL";
1090 };
1091
1092 for (const FPLibcallFamily &Family : Families) {
1093 StringRef Base = Family.Base;
1094 OS << "RTLIB::Libcall llvm::RTLIB::get" << Base << "(EVT VT) {\n";
1095
1096 if (!Family.VectorSuffixes.empty()) {
1097 OS << " if (VT.isVector()) {\n"
1098 " if (!VT.isSimple())\n"
1099 " return RTLIB::UNKNOWN_LIBCALL;\n"
1100 " switch (VT.getSimpleVT().SimpleTy) {\n";
1101 for (StringRef Suffix : Family.VectorSuffixes) {
1102 OS << " case MVT::" << Suffix.lower()
1103 << ":\n return RTLIB::" << Base << "_" << Suffix << ";\n";
1104 }
1105 OS << " default:\n"
1106 " return RTLIB::UNKNOWN_LIBCALL;\n"
1107 " }\n"
1108 " }\n";
1109 }
1110
1111 OS << " return getFPLibCall(VT";
1112 for (auto [Suffix, Pred] : ScalarFPSuffixes)
1113 OS << ", " << scalarEnum(Base, Suffix);
1114 OS << ");\n}\n\n";
1115 }
1116}
1117
1118/// Emit the mapping from floating-point math intrinsics to the runtime libcall
1119/// they may lower to, keyed by intrinsic ID and floating-point type. This is
1120/// the IR-level counterpart to the backend's RTLIB::getXXX(EVT) selectors.
1121void RuntimeLibcallEmitter::emitGetLibcallForIntrinsic(
1122 raw_ostream &OS, ArrayRef<FPLibcallFamily> Families,
1123 const DenseSet<StringRef> &LibcallNames) const {
1124 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALL_INTRINSIC_TO_LIBCALL");
1125
1126 std::vector<std::pair<StringRef, StringRef>> IntrinsicToBase;
1127 for (const FPLibcallFamily &Family : Families)
1128 for (StringRef Intrinsic : Family.Intrinsics)
1129 IntrinsicToBase.emplace_back(args&: Intrinsic, args: Family.Base);
1130 llvm::sort(C&: IntrinsicToBase);
1131
1132 MapVector<StringRef, SmallVector<StringRef, 2>> BaseToIntrinsics;
1133 for (auto [Intrinsic, Base] : IntrinsicToBase)
1134 BaseToIntrinsics[Base].push_back(Elt: Intrinsic);
1135
1136 OS << "RTLIB::Libcall "
1137 "llvm::RTLIB::RuntimeLibcallsInfo::getLibcallForIntrinsic("
1138 "Intrinsic::ID ID, FunctionType *FTy) {\n"
1139 " Type *Ty = FTy->getReturnType();\n"
1140 " if (!Ty->isFloatingPointTy()) {\n"
1141 " for (Type *ParamTy : FTy->params()) {\n"
1142 " if (ParamTy->isFloatingPointTy()) {\n"
1143 " Ty = ParamTy;\n"
1144 " break;\n"
1145 " }\n"
1146 " }\n"
1147 " }\n"
1148 " if (!Ty->isFloatingPointTy())\n"
1149 " return RTLIB::UNKNOWN_LIBCALL;\n"
1150 " switch (ID) {\n";
1151
1152 for (const auto &[Base, Intrinsics] : BaseToIntrinsics) {
1153 SmallVector<std::pair<StringRef, StringRef>, 5> Arms;
1154 for (auto [Suffix, Pred] : ScalarFPSuffixes)
1155 if (LibcallNames.contains(V: (Base + "_" + Suffix).str()))
1156 Arms.emplace_back(Args&: Suffix, Args&: Pred);
1157
1158 for (StringRef Intrinsic : Intrinsics)
1159 OS << " case Intrinsic::" << Intrinsic << ":\n";
1160 for (auto [Suffix, Pred] : Arms)
1161 OS << " if (Ty->" << Pred << ")\n return RTLIB::" << Base << "_"
1162 << Suffix << ";\n";
1163 OS << " return RTLIB::UNKNOWN_LIBCALL;\n";
1164 }
1165
1166 OS << " default:\n"
1167 " return RTLIB::UNKNOWN_LIBCALL;\n"
1168 " }\n"
1169 "}\n";
1170}
1171
1172void RuntimeLibcallEmitter::run(raw_ostream &OS) {
1173 emitSourceFileHeader(Desc: "Runtime LibCalls Source Fragment", OS, Record: Records);
1174 emitGetRuntimeLibcallEnum(OS);
1175
1176 emitGetInitRuntimeLibcallNames(OS);
1177 emitGetInitRuntimeLibcallSignatures(OS);
1178 emitMaskedVectorLibcallMapping(OS);
1179
1180 emitRuntimeLibcallsInfoMemberDecls(OS);
1181
1182 {
1183 IfDefEmitter IfDef(OS, "GET_RUNTIME_LIBCALLS_INFO");
1184 std::vector<SystemLibraryDispatch> Systems = collectSystemLibraryDispatch();
1185 emitSystemRuntimeLibrarySetCalls(OS, Systems);
1186 emitLibraryByNameSetCalls(OS, Systems);
1187 }
1188
1189 std::vector<FPLibcallFamily> FPFamilies = collectFPLibcallFamilies(Records);
1190 DenseSet<StringRef> LibcallNames = collectLibcallNames();
1191 checkFPLibcallFamilies(Families: FPFamilies, LibcallNames);
1192 emitFPLibcallSelectorDecls(OS, Families: FPFamilies);
1193 emitFPLibcallSelectors(OS, Families: FPFamilies, LibcallNames);
1194 emitGetLibcallForIntrinsic(OS, Families: FPFamilies, LibcallNames);
1195}
1196
1197static TableGen::Emitter::OptClass<RuntimeLibcallEmitter>
1198 X("gen-runtime-libcalls", "Generate RuntimeLibcalls");
1199